From b3e865ac8462f2bcc13c2a1c990c7acd391e2122 Mon Sep 17 00:00:00 2001 From: Geoffrey Booth Date: Wed, 20 Mar 2024 08:48:05 -0700 Subject: [PATCH 01/33] module: eliminate performance cost of detection for cjs entry PR-URL: https://github.com/nodejs/node/pull/52093 Reviewed-By: Matteo Collina Reviewed-By: Yagiz Nizipli Reviewed-By: Joyee Cheung Reviewed-By: Jacob Smith Reviewed-By: Richard Lau --- benchmark/misc/startup-core.js | 1 + lib/internal/modules/cjs/loader.js | 12 +- lib/internal/modules/helpers.js | 35 ++++++ lib/internal/modules/run_main.js | 42 +++++-- src/node_contextify.cc | 196 ++++++++++++++++------------- src/node_contextify.h | 3 + 6 files changed, 192 insertions(+), 97 deletions(-) diff --git a/benchmark/misc/startup-core.js b/benchmark/misc/startup-core.js index 62ead40742f61d..053a1ec0cbff8f 100644 --- a/benchmark/misc/startup-core.js +++ b/benchmark/misc/startup-core.js @@ -9,6 +9,7 @@ const bench = common.createBenchmark(main, { script: [ 'benchmark/fixtures/require-builtins', 'test/fixtures/semicolon', + 'test/fixtures/snapshot/typescript', ], mode: ['process', 'worker'], n: [30], diff --git a/lib/internal/modules/cjs/loader.js b/lib/internal/modules/cjs/loader.js index c284b39b1ac13e..62dd51ed76e612 100644 --- a/lib/internal/modules/cjs/loader.js +++ b/lib/internal/modules/cjs/loader.js @@ -106,6 +106,7 @@ module.exports = { kModuleExportNames, kModuleCircularVisited, initializeCJS, + entryPointSource: undefined, // Set below. Module, wrapSafe, kIsMainSymbol, @@ -1392,8 +1393,15 @@ function wrapSafe(filename, content, cjsModuleInstance, codeCache) { return result; } catch (err) { if (process.mainModule === cjsModuleInstance) { - const { enrichCJSError } = require('internal/modules/esm/translators'); - enrichCJSError(err, content, filename); + if (getOptionValue('--experimental-detect-module')) { + // For the main entry point, cache the source to potentially retry as ESM. + module.exports.entryPointSource = content; + } else { + // We only enrich the error (print a warning) if we're sure we're going to for-sure throw it; so if we're + // retrying as ESM, wait until we know whether we're going to retry before calling `enrichCJSError`. + const { enrichCJSError } = require('internal/modules/esm/translators'); + enrichCJSError(err, content, filename); + } } throw err; } diff --git a/lib/internal/modules/helpers.js b/lib/internal/modules/helpers.js index 7cdde181e97a10..dd4c3924a47037 100644 --- a/lib/internal/modules/helpers.js +++ b/lib/internal/modules/helpers.js @@ -19,6 +19,15 @@ const { } = require('internal/errors').codes; const { BuiltinModule } = require('internal/bootstrap/realm'); +const { + shouldRetryAsESM: contextifyShouldRetryAsESM, + constants: { + syntaxDetectionErrors: { + esmSyntaxErrorMessages, + throwsOnlyInCommonJSErrorMessages, + }, + }, +} = internalBinding('contextify'); const { validateString } = require('internal/validators'); const fs = require('fs'); // Import all of `fs` so that it can be monkey-patched. const internalFS = require('internal/fs/utils'); @@ -329,6 +338,31 @@ function urlToFilename(url) { return url; } +let esmSyntaxErrorMessagesSet; // Declared lazily in shouldRetryAsESM +let throwsOnlyInCommonJSErrorMessagesSet; // Declared lazily in shouldRetryAsESM +/** + * After an attempt to parse a module as CommonJS throws an error, should we try again as ESM? + * We only want to try again as ESM if the error is due to syntax that is only valid in ESM; and if the CommonJS parse + * throws on an error that would not have been a syntax error in ESM (like via top-level `await` or a lexical + * redeclaration of one of the CommonJS variables) then we need to parse again to see if it would have thrown in ESM. + * @param {string} errorMessage The string message thrown by V8 when attempting to parse as CommonJS + * @param {string} source Module contents + */ +function shouldRetryAsESM(errorMessage, source) { + esmSyntaxErrorMessagesSet ??= new SafeSet(esmSyntaxErrorMessages); + if (esmSyntaxErrorMessagesSet.has(errorMessage)) { + return true; + } + + throwsOnlyInCommonJSErrorMessagesSet ??= new SafeSet(throwsOnlyInCommonJSErrorMessages); + if (throwsOnlyInCommonJSErrorMessagesSet.has(errorMessage)) { + return /** @type {boolean} */(contextifyShouldRetryAsESM(source)); + } + + return false; +} + + // Whether we have started executing any user-provided CJS code. // This is set right before we call the wrapped CJS code (not after, // in case we are half-way in the execution when internals check this). @@ -362,6 +396,7 @@ module.exports = { loadBuiltinModule, makeRequireFunction, normalizeReferrerURL, + shouldRetryAsESM, stripBOM, toRealPath, hasStartedUserCJSExecution() { diff --git a/lib/internal/modules/run_main.js b/lib/internal/modules/run_main.js index 4ad694debfc72f..cfe00865d1f22a 100644 --- a/lib/internal/modules/run_main.js +++ b/lib/internal/modules/run_main.js @@ -4,7 +4,6 @@ const { StringPrototypeEndsWith, } = primordials; -const { containsModuleSyntax } = internalBinding('contextify'); const { getOptionValue } = require('internal/options'); const path = require('path'); const { pathToFileURL } = require('internal/url'); @@ -85,10 +84,6 @@ function shouldUseESMLoader(mainPath) { case 'commonjs': return false; default: { // No package.json or no `type` field. - if (getOptionValue('--experimental-detect-module')) { - // If the first argument of `containsModuleSyntax` is undefined, it will read `mainPath` from the file system. - return containsModuleSyntax(undefined, mainPath); - } return false; } } @@ -153,12 +148,43 @@ function runEntryPointWithESMLoader(callback) { * by `require('module')`) even when the entry point is ESM. * This monkey-patchable code is bypassed under `--experimental-default-type=module`. * Because of backwards compatibility, this function is exposed publicly via `import { runMain } from 'node:module'`. + * When `--experimental-detect-module` is passed, this function will attempt to run ambiguous (no explicit extension, no + * `package.json` type field) entry points as CommonJS first; under certain conditions, it will retry running as ESM. * @param {string} main - First positional CLI argument, such as `'entry.js'` from `node entry.js` */ function executeUserEntryPoint(main = process.argv[1]) { const resolvedMain = resolveMainPath(main); const useESMLoader = shouldUseESMLoader(resolvedMain); - if (useESMLoader) { + + // Unless we know we should use the ESM loader to handle the entry point per the checks in `shouldUseESMLoader`, first + // try to run the entry point via the CommonJS loader; and if that fails under certain conditions, retry as ESM. + let retryAsESM = false; + if (!useESMLoader) { + const cjsLoader = require('internal/modules/cjs/loader'); + const { Module } = cjsLoader; + if (getOptionValue('--experimental-detect-module')) { + try { + // Module._load is the monkey-patchable CJS module loader. + Module._load(main, null, true); + } catch (error) { + const source = cjsLoader.entryPointSource; + const { shouldRetryAsESM } = require('internal/modules/helpers'); + retryAsESM = shouldRetryAsESM(error.message, source); + // In case the entry point is a large file, such as a bundle, + // ensure no further references can prevent it being garbage-collected. + cjsLoader.entryPointSource = undefined; + if (!retryAsESM) { + const { enrichCJSError } = require('internal/modules/esm/translators'); + enrichCJSError(error, source, resolvedMain); + throw error; + } + } + } else { // `--experimental-detect-module` is not passed + Module._load(main, null, true); + } + } + + if (useESMLoader || retryAsESM) { const mainPath = resolvedMain || main; const mainURL = pathToFileURL(mainPath).href; @@ -167,10 +193,6 @@ function executeUserEntryPoint(main = process.argv[1]) { // even after the event loop stops running. return cascadedLoader.import(mainURL, undefined, { __proto__: null }, true); }); - } else { - // Module._load is the monkey-patchable CJS module loader. - const { Module } = require('internal/modules/cjs/loader'); - Module._load(main, null, true); } } diff --git a/src/node_contextify.cc b/src/node_contextify.cc index 8951cd378a9025..0a409c5086f713 100644 --- a/src/node_contextify.cc +++ b/src/node_contextify.cc @@ -364,6 +364,7 @@ void ContextifyContext::CreatePerIsolateProperties( SetMethod(isolate, target, "makeContext", MakeContext); SetMethod(isolate, target, "compileFunction", CompileFunction); SetMethod(isolate, target, "containsModuleSyntax", ContainsModuleSyntax); + SetMethod(isolate, target, "shouldRetryAsESM", ShouldRetryAsESM); } void ContextifyContext::RegisterExternalReferences( @@ -371,6 +372,7 @@ void ContextifyContext::RegisterExternalReferences( registry->Register(MakeContext); registry->Register(CompileFunction); registry->Register(ContainsModuleSyntax); + registry->Register(ShouldRetryAsESM); registry->Register(PropertyGetterCallback); registry->Register(PropertySetterCallback); registry->Register(PropertyDescriptorCallback); @@ -1447,7 +1449,7 @@ Local ContextifyContext::CompileFunctionAndCacheResult( // While top-level `await` is not permitted in CommonJS, it returns the same // error message as when `await` is used in a sync function, so we don't use it // as a disambiguation. -constexpr std::array esm_syntax_error_messages = { +static std::vector esm_syntax_error_messages = { "Cannot use import statement outside a module", // `import` statements "Unexpected token 'export'", // `export` statements "Cannot use 'import.meta' outside a module"}; // `import.meta` references @@ -1462,7 +1464,7 @@ constexpr std::array esm_syntax_error_messages = { // - Top-level `await`: if the user writes `await` at the top level of a // CommonJS module, it will throw a syntax error; but the same code is valid // in ESM. -constexpr std::array throws_only_in_cjs_error_messages = { +static std::vector throws_only_in_cjs_error_messages = { "Identifier 'module' has already been declared", "Identifier 'exports' has already been declared", "Identifier 'require' has already been declared", @@ -1482,6 +1484,10 @@ void ContextifyContext::ContainsModuleSyntax( env, "containsModuleSyntax needs at least 1 argument"); } + // Argument 1: source code + CHECK(args[0]->IsString()); + auto code = args[0].As(); + // Argument 2: filename; if undefined, use empty string Local filename = String::Empty(isolate); if (!args[1]->IsUndefined()) { @@ -1489,28 +1495,6 @@ void ContextifyContext::ContainsModuleSyntax( filename = args[1].As(); } - // Argument 1: source code; if undefined, read from filename in argument 2 - Local code; - if (args[0]->IsUndefined()) { - CHECK(!filename.IsEmpty()); - const char* filename_str = Utf8Value(isolate, filename).out(); - std::string contents; - int result = ReadFileSync(&contents, filename_str); - if (result != 0) { - isolate->ThrowException( - ERR_MODULE_NOT_FOUND(isolate, "Cannot read file %s", filename_str)); - return; - } - code = String::NewFromUtf8(isolate, - contents.c_str(), - v8::NewStringType::kNormal, - contents.length()) - .ToLocalChecked(); - } else { - CHECK(args[0]->IsString()); - code = args[0].As(); - } - // TODO(geoffreybooth): Centralize this rather than matching the logic in // cjs/loader.js and translators.js Local script_id = String::Concat( @@ -1540,73 +1524,95 @@ void ContextifyContext::ContainsModuleSyntax( bool should_retry_as_esm = false; if (try_catch.HasCaught() && !try_catch.HasTerminated()) { - Utf8Value message_value(env->isolate(), try_catch.Message()->Get()); - auto message = message_value.ToStringView(); + should_retry_as_esm = + ContextifyContext::ShouldRetryAsESMInternal(env, code); + } + args.GetReturnValue().Set(should_retry_as_esm); +} - for (const auto& error_message : esm_syntax_error_messages) { - if (message.find(error_message) != std::string_view::npos) { - should_retry_as_esm = true; - break; - } - } +void ContextifyContext::ShouldRetryAsESM( + const FunctionCallbackInfo& args) { + Environment* env = Environment::GetCurrent(args); + + CHECK_EQ(args.Length(), 1); // code + + // Argument 1: source code + Local code; + CHECK(args[0]->IsString()); + code = args[0].As(); - if (!should_retry_as_esm) { - for (const auto& error_message : throws_only_in_cjs_error_messages) { - if (message.find(error_message) != std::string_view::npos) { - // Try parsing again where the CommonJS wrapper is replaced by an - // async function wrapper. If the new parse succeeds, then the error - // was caused by either a top-level declaration of one of the CommonJS - // module variables, or a top-level `await`. - TryCatchScope second_parse_try_catch(env); - code = - String::Concat(isolate, - String::NewFromUtf8(isolate, "(async function() {") - .ToLocalChecked(), - code); - code = String::Concat( - isolate, - code, - String::NewFromUtf8(isolate, "})();").ToLocalChecked()); - ScriptCompiler::Source wrapped_source = GetCommonJSSourceInstance( - isolate, code, filename, 0, 0, host_defined_options, nullptr); - std::ignore = ScriptCompiler::CompileFunction( - context, - &wrapped_source, - params.size(), - params.data(), - 0, - nullptr, - options, - v8::ScriptCompiler::NoCacheReason::kNoCacheNoReason); - if (!second_parse_try_catch.HasTerminated()) { - if (second_parse_try_catch.HasCaught()) { - // If on the second parse an error is thrown by ESM syntax, then - // what happened was that the user had top-level `await` or a - // top-level declaration of one of the CommonJS module variables - // above the first `import` or `export`. - Utf8Value second_message_value( - env->isolate(), second_parse_try_catch.Message()->Get()); - auto second_message = second_message_value.ToStringView(); - for (const auto& error_message : esm_syntax_error_messages) { - if (second_message.find(error_message) != - std::string_view::npos) { - should_retry_as_esm = true; - break; - } - } - } else { - // No errors thrown in the second parse, so most likely the error - // was caused by a top-level `await` or a top-level declaration of - // one of the CommonJS module variables. - should_retry_as_esm = true; - } - } - break; + bool should_retry_as_esm = + ContextifyContext::ShouldRetryAsESMInternal(env, code); + + args.GetReturnValue().Set(should_retry_as_esm); +} + +bool ContextifyContext::ShouldRetryAsESMInternal(Environment* env, + Local code) { + Isolate* isolate = env->isolate(); + + Local script_id = + FIXED_ONE_BYTE_STRING(isolate, "[retry_as_esm_check]"); + Local id_symbol = Symbol::New(isolate, script_id); + + Local host_defined_options = + GetHostDefinedOptions(isolate, id_symbol); + ScriptCompiler::Source source = + GetCommonJSSourceInstance(isolate, + code, + script_id, // filename + 0, // line offset + 0, // column offset + host_defined_options, + nullptr); // cached_data + + TryCatchScope try_catch(env); + ShouldNotAbortOnUncaughtScope no_abort_scope(env); + + // Try parsing where instead of the CommonJS wrapper we use an async function + // wrapper. If the parse succeeds, then any CommonJS parse error for this + // module was caused by either a top-level declaration of one of the CommonJS + // module variables, or a top-level `await`. + code = String::Concat( + isolate, FIXED_ONE_BYTE_STRING(isolate, "(async function() {"), code); + code = String::Concat(isolate, code, FIXED_ONE_BYTE_STRING(isolate, "})();")); + + ScriptCompiler::Source wrapped_source = GetCommonJSSourceInstance( + isolate, code, script_id, 0, 0, host_defined_options, nullptr); + + Local context = env->context(); + std::vector> params = GetCJSParameters(env->isolate_data()); + USE(ScriptCompiler::CompileFunction( + context, + &wrapped_source, + params.size(), + params.data(), + 0, + nullptr, + ScriptCompiler::kNoCompileOptions, + v8::ScriptCompiler::NoCacheReason::kNoCacheNoReason)); + + if (!try_catch.HasTerminated()) { + if (try_catch.HasCaught()) { + // If on the second parse an error is thrown by ESM syntax, then + // what happened was that the user had top-level `await` or a + // top-level declaration of one of the CommonJS module variables + // above the first `import` or `export`. + Utf8Value message_value(env->isolate(), try_catch.Message()->Get()); + auto message_view = message_value.ToStringView(); + for (const auto& error_message : esm_syntax_error_messages) { + if (message_view.find(error_message) != std::string_view::npos) { + return true; } } + } else { + // No errors thrown in the second parse, so most likely the error + // was caused by a top-level `await` or a top-level declaration of + // one of the CommonJS module variables. + return true; } } - args.GetReturnValue().Set(should_retry_as_esm); + return false; } static void CompileFunctionForCJSLoader( @@ -1767,6 +1773,7 @@ static void CreatePerContextProperties(Local target, Local constants = Object::New(env->isolate()); Local measure_memory = Object::New(env->isolate()); Local memory_execution = Object::New(env->isolate()); + Local syntax_detection_errors = Object::New(env->isolate()); { Local memory_mode = Object::New(env->isolate()); @@ -1787,6 +1794,25 @@ static void CreatePerContextProperties(Local target, READONLY_PROPERTY(constants, "measureMemory", measure_memory); + { + Local esm_syntax_error_messages_array = + ToV8Value(context, esm_syntax_error_messages).ToLocalChecked(); + READONLY_PROPERTY(syntax_detection_errors, + "esmSyntaxErrorMessages", + esm_syntax_error_messages_array); + } + + { + Local throws_only_in_cjs_error_messages_array = + ToV8Value(context, throws_only_in_cjs_error_messages).ToLocalChecked(); + READONLY_PROPERTY(syntax_detection_errors, + "throwsOnlyInCommonJSErrorMessages", + throws_only_in_cjs_error_messages_array); + } + + READONLY_PROPERTY( + constants, "syntaxDetectionErrors", syntax_detection_errors); + target->Set(context, env->constants_string(), constants).Check(); } diff --git a/src/node_contextify.h b/src/node_contextify.h index 517e3f44d32490..ff975c8cf135e4 100644 --- a/src/node_contextify.h +++ b/src/node_contextify.h @@ -110,6 +110,9 @@ class ContextifyContext : public BaseObject { const v8::ScriptCompiler::Source& source); static void ContainsModuleSyntax( const v8::FunctionCallbackInfo& args); + static void ShouldRetryAsESM(const v8::FunctionCallbackInfo& args); + static bool ShouldRetryAsESMInternal(Environment* env, + v8::Local code); static void WeakCallback( const v8::WeakCallbackInfo& data); static void PropertyGetterCallback( From cf363a47beb61ac42efb95353dbf3c1cdf135f4d Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Fri, 19 Apr 2024 16:29:08 +0200 Subject: [PATCH 02/33] module: detect ESM syntax by trying to recompile as SourceTextModule Instead of using an async function wrapper, just try compiling code with unknown module format as SourceTextModule when it cannot be compiled as CJS and the error message indicates that it's worth a retry. If it can be parsed as SourceTextModule then it's considered ESM. Also, move shouldRetryAsESM() to C++ completely so that we can reuse it in the CJS module loader for require(esm). Drive-by: move methods that don't belong to ContextifyContext out as static methods and move GetHostDefinedOptions to ModuleWrap. PR-URL: https://github.com/nodejs/node/pull/52413 Reviewed-By: Geoffrey Booth Reviewed-By: Jacob Smith --- lib/internal/modules/esm/get_format.js | 4 +- lib/internal/modules/helpers.js | 34 -- lib/internal/modules/run_main.js | 26 +- src/module_wrap.cc | 122 +++++-- src/module_wrap.h | 21 +- src/node_contextify.cc | 420 ++++++++++--------------- src/node_contextify.h | 15 - 7 files changed, 303 insertions(+), 339 deletions(-) diff --git a/lib/internal/modules/esm/get_format.js b/lib/internal/modules/esm/get_format.js index 1fe5564545dbc8..f07f2a42b260a3 100644 --- a/lib/internal/modules/esm/get_format.js +++ b/lib/internal/modules/esm/get_format.js @@ -114,7 +114,7 @@ function getFileProtocolModuleFormat(url, context = { __proto__: null }, ignoreE // but this gets called again from `defaultLoad`/`defaultLoadSync`. if (getOptionValue('--experimental-detect-module')) { const format = source ? - (containsModuleSyntax(`${source}`, fileURLToPath(url)) ? 'module' : 'commonjs') : + (containsModuleSyntax(`${source}`, fileURLToPath(url), url) ? 'module' : 'commonjs') : null; if (format === 'module') { // This module has a .js extension, a package.json with no `type` field, and ESM syntax. @@ -158,7 +158,7 @@ function getFileProtocolModuleFormat(url, context = { __proto__: null }, ignoreE if (!source) { return null; } const format = getFormatOfExtensionlessFile(url); if (format === 'module') { - return containsModuleSyntax(`${source}`, fileURLToPath(url)) ? 'module' : 'commonjs'; + return containsModuleSyntax(`${source}`, fileURLToPath(url), url) ? 'module' : 'commonjs'; } return format; } diff --git a/lib/internal/modules/helpers.js b/lib/internal/modules/helpers.js index dd4c3924a47037..d560c0d8089e19 100644 --- a/lib/internal/modules/helpers.js +++ b/lib/internal/modules/helpers.js @@ -19,15 +19,6 @@ const { } = require('internal/errors').codes; const { BuiltinModule } = require('internal/bootstrap/realm'); -const { - shouldRetryAsESM: contextifyShouldRetryAsESM, - constants: { - syntaxDetectionErrors: { - esmSyntaxErrorMessages, - throwsOnlyInCommonJSErrorMessages, - }, - }, -} = internalBinding('contextify'); const { validateString } = require('internal/validators'); const fs = require('fs'); // Import all of `fs` so that it can be monkey-patched. const internalFS = require('internal/fs/utils'); @@ -338,30 +329,6 @@ function urlToFilename(url) { return url; } -let esmSyntaxErrorMessagesSet; // Declared lazily in shouldRetryAsESM -let throwsOnlyInCommonJSErrorMessagesSet; // Declared lazily in shouldRetryAsESM -/** - * After an attempt to parse a module as CommonJS throws an error, should we try again as ESM? - * We only want to try again as ESM if the error is due to syntax that is only valid in ESM; and if the CommonJS parse - * throws on an error that would not have been a syntax error in ESM (like via top-level `await` or a lexical - * redeclaration of one of the CommonJS variables) then we need to parse again to see if it would have thrown in ESM. - * @param {string} errorMessage The string message thrown by V8 when attempting to parse as CommonJS - * @param {string} source Module contents - */ -function shouldRetryAsESM(errorMessage, source) { - esmSyntaxErrorMessagesSet ??= new SafeSet(esmSyntaxErrorMessages); - if (esmSyntaxErrorMessagesSet.has(errorMessage)) { - return true; - } - - throwsOnlyInCommonJSErrorMessagesSet ??= new SafeSet(throwsOnlyInCommonJSErrorMessages); - if (throwsOnlyInCommonJSErrorMessagesSet.has(errorMessage)) { - return /** @type {boolean} */(contextifyShouldRetryAsESM(source)); - } - - return false; -} - // Whether we have started executing any user-provided CJS code. // This is set right before we call the wrapped CJS code (not after, @@ -396,7 +363,6 @@ module.exports = { loadBuiltinModule, makeRequireFunction, normalizeReferrerURL, - shouldRetryAsESM, stripBOM, toRealPath, hasStartedUserCJSExecution() { diff --git a/lib/internal/modules/run_main.js b/lib/internal/modules/run_main.js index cfe00865d1f22a..7a99c386fe6ded 100644 --- a/lib/internal/modules/run_main.js +++ b/lib/internal/modules/run_main.js @@ -1,7 +1,9 @@ 'use strict'; const { + ObjectGetPrototypeOf, StringPrototypeEndsWith, + SyntaxErrorPrototype, } = primordials; const { getOptionValue } = require('internal/options'); @@ -155,7 +157,7 @@ function runEntryPointWithESMLoader(callback) { function executeUserEntryPoint(main = process.argv[1]) { const resolvedMain = resolveMainPath(main); const useESMLoader = shouldUseESMLoader(resolvedMain); - + let mainURL; // Unless we know we should use the ESM loader to handle the entry point per the checks in `shouldUseESMLoader`, first // try to run the entry point via the CommonJS loader; and if that fails under certain conditions, retry as ESM. let retryAsESM = false; @@ -163,19 +165,21 @@ function executeUserEntryPoint(main = process.argv[1]) { const cjsLoader = require('internal/modules/cjs/loader'); const { Module } = cjsLoader; if (getOptionValue('--experimental-detect-module')) { + // TODO(joyeecheung): handle this in the CJS loader. Don't try-catch here. try { // Module._load is the monkey-patchable CJS module loader. Module._load(main, null, true); } catch (error) { - const source = cjsLoader.entryPointSource; - const { shouldRetryAsESM } = require('internal/modules/helpers'); - retryAsESM = shouldRetryAsESM(error.message, source); - // In case the entry point is a large file, such as a bundle, - // ensure no further references can prevent it being garbage-collected. - cjsLoader.entryPointSource = undefined; + if (error != null && ObjectGetPrototypeOf(error) === SyntaxErrorPrototype) { + const { shouldRetryAsESM } = internalBinding('contextify'); + const mainPath = resolvedMain || main; + mainURL = pathToFileURL(mainPath).href; + retryAsESM = shouldRetryAsESM(error.message, cjsLoader.entryPointSource, mainURL); + // In case the entry point is a large file, such as a bundle, + // ensure no further references can prevent it being garbage-collected. + cjsLoader.entryPointSource = undefined; + } if (!retryAsESM) { - const { enrichCJSError } = require('internal/modules/esm/translators'); - enrichCJSError(error, source, resolvedMain); throw error; } } @@ -186,7 +190,9 @@ function executeUserEntryPoint(main = process.argv[1]) { if (useESMLoader || retryAsESM) { const mainPath = resolvedMain || main; - const mainURL = pathToFileURL(mainPath).href; + if (mainURL === undefined) { + mainURL = pathToFileURL(mainPath).href; + } runEntryPointWithESMLoader((cascadedLoader) => { // Note that if the graph contains unfinished TLA, this may never resolve diff --git a/src/module_wrap.cc b/src/module_wrap.cc index eea74bed4bb8a9..4cd75447b77526 100644 --- a/src/module_wrap.cc +++ b/src/module_wrap.cc @@ -102,9 +102,17 @@ ModuleWrap* ModuleWrap::GetFromModule(Environment* env, return nullptr; } -// new ModuleWrap(url, context, source, lineOffset, columnOffset, cachedData) +Local ModuleWrap::GetHostDefinedOptions( + Isolate* isolate, Local id_symbol) { + Local host_defined_options = + PrimitiveArray::New(isolate, HostDefinedOptions::kLength); + host_defined_options->Set(isolate, HostDefinedOptions::kID, id_symbol); + return host_defined_options; +} + +// new ModuleWrap(url, context, source, lineOffset, columnOffset[, cachedData]); // new ModuleWrap(url, context, source, lineOffset, columOffset, -// hostDefinedOption) +// idSymbol); // new ModuleWrap(url, context, exportNames, evaluationCallback[, cjsModule]) void ModuleWrap::New(const FunctionCallbackInfo& args) { CHECK(args.IsConstructCall()); @@ -134,7 +142,7 @@ void ModuleWrap::New(const FunctionCallbackInfo& args) { int column_offset = 0; bool synthetic = args[2]->IsArray(); - + bool can_use_builtin_cache = false; Local host_defined_options = PrimitiveArray::New(isolate, HostDefinedOptions::kLength); Local id_symbol; @@ -143,9 +151,10 @@ void ModuleWrap::New(const FunctionCallbackInfo& args) { // cjsModule]) CHECK(args[3]->IsFunction()); } else { - // new ModuleWrap(url, context, source, lineOffset, columOffset, cachedData) + // new ModuleWrap(url, context, source, lineOffset, columOffset[, + // cachedData]); // new ModuleWrap(url, context, source, lineOffset, columOffset, - // hostDefinedOption) + // idSymbol); CHECK(args[2]->IsString()); CHECK(args[3]->IsNumber()); line_offset = args[3].As()->Value(); @@ -153,10 +162,13 @@ void ModuleWrap::New(const FunctionCallbackInfo& args) { column_offset = args[4].As()->Value(); if (args[5]->IsSymbol()) { id_symbol = args[5].As(); + can_use_builtin_cache = + (id_symbol == + realm->isolate_data()->source_text_module_default_hdo()); } else { id_symbol = Symbol::New(isolate, url); } - host_defined_options->Set(isolate, HostDefinedOptions::kID, id_symbol); + host_defined_options = GetHostDefinedOptions(isolate, id_symbol); if (that->SetPrivate(context, realm->isolate_data()->host_defined_option_symbol(), @@ -189,36 +201,34 @@ void ModuleWrap::New(const FunctionCallbackInfo& args) { module = Module::CreateSyntheticModule(isolate, url, export_names, SyntheticModuleEvaluationStepsCallback); } else { - ScriptCompiler::CachedData* cached_data = nullptr; + // When we are compiling for the default loader, this will be + // std::nullopt, and CompileSourceTextModule() should use + // on-disk cache (not present on v20.x). + std::optional user_cached_data; + if (id_symbol != + realm->isolate_data()->source_text_module_default_hdo()) { + user_cached_data = nullptr; + } if (args[5]->IsArrayBufferView()) { + CHECK(!can_use_builtin_cache); // We don't use this option internally. Local cached_data_buf = args[5].As(); uint8_t* data = static_cast(cached_data_buf->Buffer()->Data()); - cached_data = + user_cached_data = new ScriptCompiler::CachedData(data + cached_data_buf->ByteOffset(), cached_data_buf->ByteLength()); } - Local source_text = args[2].As(); - ScriptOrigin origin(isolate, - url, - line_offset, - column_offset, - true, // is cross origin - -1, // script id - Local(), // source map URL - false, // is opaque (?) - false, // is WASM - true, // is ES Module - host_defined_options); - ScriptCompiler::Source source(source_text, origin, cached_data); - ScriptCompiler::CompileOptions options; - if (source.GetCachedData() == nullptr) { - options = ScriptCompiler::kNoCompileOptions; - } else { - options = ScriptCompiler::kConsumeCodeCache; - } - if (!ScriptCompiler::CompileModule(isolate, &source, options) + + bool cache_rejected = false; + if (!CompileSourceTextModule(realm, + source_text, + url, + line_offset, + column_offset, + host_defined_options, + user_cached_data, + &cache_rejected) .ToLocal(&module)) { if (try_catch.HasCaught() && !try_catch.HasTerminated()) { CHECK(!try_catch.Message().IsEmpty()); @@ -231,8 +241,9 @@ void ModuleWrap::New(const FunctionCallbackInfo& args) { } return; } - if (options == ScriptCompiler::kConsumeCodeCache && - source.GetCachedData()->rejected) { + + if (user_cached_data.has_value() && user_cached_data.value() != nullptr && + cache_rejected) { THROW_ERR_VM_MODULE_CACHED_DATA_REJECTED( realm, "cachedData buffer was rejected"); try_catch.ReThrow(); @@ -275,6 +286,57 @@ void ModuleWrap::New(const FunctionCallbackInfo& args) { args.GetReturnValue().Set(that); } +MaybeLocal ModuleWrap::CompileSourceTextModule( + Realm* realm, + Local source_text, + Local url, + int line_offset, + int column_offset, + Local host_defined_options, + std::optional user_cached_data, + bool* cache_rejected) { + Isolate* isolate = realm->isolate(); + EscapableHandleScope scope(isolate); + ScriptOrigin origin(isolate, + url, + line_offset, + column_offset, + true, // is cross origin + -1, // script id + Local(), // source map URL + false, // is opaque (?) + false, // is WASM + true, // is ES Module + host_defined_options); + ScriptCompiler::CachedData* cached_data = nullptr; + // When compiling for the default loader, user_cached_data is std::nullptr. + // When compiling for vm.Module, it's either nullptr or a pointer to the + // cached data. + if (user_cached_data.has_value()) { + cached_data = user_cached_data.value(); + } + + ScriptCompiler::Source source(source_text, origin, cached_data); + ScriptCompiler::CompileOptions options; + if (cached_data == nullptr) { + options = ScriptCompiler::kNoCompileOptions; + } else { + options = ScriptCompiler::kConsumeCodeCache; + } + + Local module; + if (!ScriptCompiler::CompileModule(isolate, &source, options) + .ToLocal(&module)) { + return scope.EscapeMaybe(MaybeLocal()); + } + + if (options == ScriptCompiler::kConsumeCodeCache) { + *cache_rejected = source.GetCachedData()->rejected; + } + + return scope.Escape(module); +} + static Local createImportAttributesContainer( Realm* realm, Isolate* isolate, diff --git a/src/module_wrap.h b/src/module_wrap.h index 45a338b38e01c8..09da8ee43fa8b4 100644 --- a/src/module_wrap.h +++ b/src/module_wrap.h @@ -3,10 +3,12 @@ #if defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS -#include +#include #include +#include #include #include "base_object.h" +#include "v8-script.h" namespace node { @@ -68,6 +70,23 @@ class ModuleWrap : public BaseObject { return true; } + static v8::Local GetHostDefinedOptions( + v8::Isolate* isolate, v8::Local symbol); + + // When user_cached_data is not std::nullopt, use the code cache if it's not + // nullptr, otherwise don't use code cache. + // TODO(joyeecheung): when it is std::nullopt, use on-disk cache + // See: https://github.com/nodejs/node/issues/47472 + static v8::MaybeLocal CompileSourceTextModule( + Realm* realm, + v8::Local source_text, + v8::Local url, + int line_offset, + int column_offset, + v8::Local host_defined_options, + std::optional user_cached_data, + bool* cache_rejected); + private: ModuleWrap(Realm* realm, v8::Local object, diff --git a/src/node_contextify.cc b/src/node_contextify.cc index 0a409c5086f713..7eb7335975e219 100644 --- a/src/node_contextify.cc +++ b/src/node_contextify.cc @@ -363,16 +363,12 @@ void ContextifyContext::CreatePerIsolateProperties( Isolate* isolate = isolate_data->isolate(); SetMethod(isolate, target, "makeContext", MakeContext); SetMethod(isolate, target, "compileFunction", CompileFunction); - SetMethod(isolate, target, "containsModuleSyntax", ContainsModuleSyntax); - SetMethod(isolate, target, "shouldRetryAsESM", ShouldRetryAsESM); } void ContextifyContext::RegisterExternalReferences( ExternalReferenceRegistry* registry) { registry->Register(MakeContext); registry->Register(CompileFunction); - registry->Register(ContainsModuleSyntax); - registry->Register(ShouldRetryAsESM); registry->Register(PropertyGetterCallback); registry->Register(PropertySetterCallback); registry->Register(PropertyDescriptorCallback); @@ -1196,15 +1192,6 @@ ContextifyScript::ContextifyScript(Environment* env, Local object) ContextifyScript::~ContextifyScript() {} -static Local GetHostDefinedOptions(Isolate* isolate, - Local id_symbol) { - Local host_defined_options = - PrimitiveArray::New(isolate, loader::HostDefinedOptions::kLength); - host_defined_options->Set( - isolate, loader::HostDefinedOptions::kID, id_symbol); - return host_defined_options; -} - void ContextifyContext::CompileFunction( const FunctionCallbackInfo& args) { Environment* env = Environment::GetCurrent(args); @@ -1278,16 +1265,27 @@ void ContextifyContext::CompileFunction( } Local host_defined_options = - GetHostDefinedOptions(isolate, id_symbol); - ScriptCompiler::Source source = - GetCommonJSSourceInstance(isolate, - code, - filename, - line_offset, - column_offset, - host_defined_options, - cached_data); - ScriptCompiler::CompileOptions options = GetCompileOptions(source); + loader::ModuleWrap::GetHostDefinedOptions(isolate, id_symbol); + + ScriptOrigin origin(isolate, + filename, + line_offset, // line offset + column_offset, // column offset + true, // is cross origin + -1, // script id + Local(), // source map URL + false, // is opaque (?) + false, // is WASM + false, // is ES Module + host_defined_options); + ScriptCompiler::Source source(code, origin, cached_data); + + ScriptCompiler::CompileOptions options; + if (source.GetCachedData() != nullptr) { + options = ScriptCompiler::kConsumeCodeCache; + } else { + options = ScriptCompiler::kNoCompileOptions; + } Context::Scope scope(parsing_context); @@ -1335,39 +1333,6 @@ void ContextifyContext::CompileFunction( args.GetReturnValue().Set(result); } -ScriptCompiler::Source ContextifyContext::GetCommonJSSourceInstance( - Isolate* isolate, - Local code, - Local filename, - int line_offset, - int column_offset, - Local host_defined_options, - ScriptCompiler::CachedData* cached_data) { - ScriptOrigin origin(isolate, - filename, - line_offset, // line offset - column_offset, // column offset - true, // is cross origin - -1, // script id - Local(), // source map URL - false, // is opaque (?) - false, // is WASM - false, // is ES Module - host_defined_options); - return ScriptCompiler::Source(code, origin, cached_data); -} - -ScriptCompiler::CompileOptions ContextifyContext::GetCompileOptions( - const ScriptCompiler::Source& source) { - ScriptCompiler::CompileOptions options; - if (source.GetCachedData() != nullptr) { - options = ScriptCompiler::kConsumeCodeCache; - } else { - options = ScriptCompiler::kNoCompileOptions; - } - return options; -} - static std::vector> GetCJSParameters(IsolateData* data) { return { data->exports_string(), @@ -1473,160 +1438,17 @@ static std::vector throws_only_in_cjs_error_messages = { "await is only valid in async functions and " "the top level bodies of modules"}; -void ContextifyContext::ContainsModuleSyntax( - const FunctionCallbackInfo& args) { - Environment* env = Environment::GetCurrent(args); - Isolate* isolate = env->isolate(); - Local context = env->context(); - - if (args.Length() == 0) { - return THROW_ERR_MISSING_ARGS( - env, "containsModuleSyntax needs at least 1 argument"); - } - - // Argument 1: source code - CHECK(args[0]->IsString()); - auto code = args[0].As(); - - // Argument 2: filename; if undefined, use empty string - Local filename = String::Empty(isolate); - if (!args[1]->IsUndefined()) { - CHECK(args[1]->IsString()); - filename = args[1].As(); - } - - // TODO(geoffreybooth): Centralize this rather than matching the logic in - // cjs/loader.js and translators.js - Local script_id = String::Concat( - isolate, String::NewFromUtf8(isolate, "cjs:").ToLocalChecked(), filename); - Local id_symbol = Symbol::New(isolate, script_id); - - Local host_defined_options = - GetHostDefinedOptions(isolate, id_symbol); - ScriptCompiler::Source source = GetCommonJSSourceInstance( - isolate, code, filename, 0, 0, host_defined_options, nullptr); - ScriptCompiler::CompileOptions options = GetCompileOptions(source); - - std::vector> params = GetCJSParameters(env->isolate_data()); - - TryCatchScope try_catch(env); - ShouldNotAbortOnUncaughtScope no_abort_scope(env); - - ContextifyContext::CompileFunctionAndCacheResult(env, - context, - &source, - params, - std::vector>(), - options, - true, - id_symbol, - try_catch); - - bool should_retry_as_esm = false; - if (try_catch.HasCaught() && !try_catch.HasTerminated()) { - should_retry_as_esm = - ContextifyContext::ShouldRetryAsESMInternal(env, code); - } - args.GetReturnValue().Set(should_retry_as_esm); -} - -void ContextifyContext::ShouldRetryAsESM( - const FunctionCallbackInfo& args) { - Environment* env = Environment::GetCurrent(args); - - CHECK_EQ(args.Length(), 1); // code - - // Argument 1: source code - Local code; - CHECK(args[0]->IsString()); - code = args[0].As(); - - bool should_retry_as_esm = - ContextifyContext::ShouldRetryAsESMInternal(env, code); - - args.GetReturnValue().Set(should_retry_as_esm); -} - -bool ContextifyContext::ShouldRetryAsESMInternal(Environment* env, - Local code) { - Isolate* isolate = env->isolate(); - - Local script_id = - FIXED_ONE_BYTE_STRING(isolate, "[retry_as_esm_check]"); - Local id_symbol = Symbol::New(isolate, script_id); - - Local host_defined_options = - GetHostDefinedOptions(isolate, id_symbol); - ScriptCompiler::Source source = - GetCommonJSSourceInstance(isolate, - code, - script_id, // filename - 0, // line offset - 0, // column offset - host_defined_options, - nullptr); // cached_data - - TryCatchScope try_catch(env); - ShouldNotAbortOnUncaughtScope no_abort_scope(env); - - // Try parsing where instead of the CommonJS wrapper we use an async function - // wrapper. If the parse succeeds, then any CommonJS parse error for this - // module was caused by either a top-level declaration of one of the CommonJS - // module variables, or a top-level `await`. - code = String::Concat( - isolate, FIXED_ONE_BYTE_STRING(isolate, "(async function() {"), code); - code = String::Concat(isolate, code, FIXED_ONE_BYTE_STRING(isolate, "})();")); - - ScriptCompiler::Source wrapped_source = GetCommonJSSourceInstance( - isolate, code, script_id, 0, 0, host_defined_options, nullptr); - - Local context = env->context(); - std::vector> params = GetCJSParameters(env->isolate_data()); - USE(ScriptCompiler::CompileFunction( - context, - &wrapped_source, - params.size(), - params.data(), - 0, - nullptr, - ScriptCompiler::kNoCompileOptions, - v8::ScriptCompiler::NoCacheReason::kNoCacheNoReason)); - - if (!try_catch.HasTerminated()) { - if (try_catch.HasCaught()) { - // If on the second parse an error is thrown by ESM syntax, then - // what happened was that the user had top-level `await` or a - // top-level declaration of one of the CommonJS module variables - // above the first `import` or `export`. - Utf8Value message_value(env->isolate(), try_catch.Message()->Get()); - auto message_view = message_value.ToStringView(); - for (const auto& error_message : esm_syntax_error_messages) { - if (message_view.find(error_message) != std::string_view::npos) { - return true; - } - } - } else { - // No errors thrown in the second parse, so most likely the error - // was caused by a top-level `await` or a top-level declaration of - // one of the CommonJS module variables. - return true; - } - } - return false; -} - -static void CompileFunctionForCJSLoader( - const FunctionCallbackInfo& args) { - CHECK(args[0]->IsString()); - CHECK(args[1]->IsString()); - Local code = args[0].As(); - Local filename = args[1].As(); - Isolate* isolate = args.GetIsolate(); - Local context = isolate->GetCurrentContext(); - Environment* env = Environment::GetCurrent(context); +static MaybeLocal CompileFunctionForCJSLoader(Environment* env, + Local context, + Local code, + Local filename, + bool* cache_rejected) { + Isolate* isolate = context->GetIsolate(); + EscapableHandleScope scope(isolate); Local symbol = env->vm_dynamic_import_default_internal(); - Local hdo = GetHostDefinedOptions(isolate, symbol); + Local hdo = + loader::ModuleWrap::GetHostDefinedOptions(isolate, symbol); ScriptOrigin origin(isolate, filename, 0, // line offset @@ -1641,7 +1463,6 @@ static void CompileFunctionForCJSLoader( ScriptCompiler::CachedData* cached_data = nullptr; #ifndef DISABLE_SINGLE_EXECUTABLE_APPLICATION - bool used_cache_from_sea = false; if (sea::IsSingleExecutable()) { sea::SeaResource sea = sea::FindSingleExecutableResource(); if (sea.use_code_cache()) { @@ -1650,16 +1471,18 @@ static void CompileFunctionForCJSLoader( reinterpret_cast(data.data()), static_cast(data.size()), v8::ScriptCompiler::CachedData::BufferNotOwned); - used_cache_from_sea = true; } } #endif ScriptCompiler::Source source(code, origin, cached_data); - - TryCatchScope try_catch(env); + ScriptCompiler::CompileOptions options; + if (cached_data == nullptr) { + options = ScriptCompiler::kNoCompileOptions; + } else { + options = ScriptCompiler::kConsumeCodeCache; + } std::vector> params = GetCJSParameters(env->isolate_data()); - MaybeLocal maybe_fn = ScriptCompiler::CompileFunction( context, &source, @@ -1668,27 +1491,43 @@ static void CompileFunctionForCJSLoader( 0, /* context extensions size */ nullptr, /* context extensions data */ // TODO(joyeecheung): allow optional eager compilation. - cached_data == nullptr ? ScriptCompiler::kNoCompileOptions - : ScriptCompiler::kConsumeCodeCache, - v8::ScriptCompiler::NoCacheReason::kNoCacheNoReason); + options); Local fn; if (!maybe_fn.ToLocal(&fn)) { - if (try_catch.HasCaught() && !try_catch.HasTerminated()) { - errors::DecorateErrorStack(env, try_catch); - if (!try_catch.HasTerminated()) { - try_catch.ReThrow(); - } - return; - } + return scope.EscapeMaybe(MaybeLocal()); } + if (options == ScriptCompiler::kConsumeCodeCache) { + *cache_rejected = source.GetCachedData()->rejected; + } + return scope.Escape(fn); +} + +static void CompileFunctionForCJSLoader( + const FunctionCallbackInfo& args) { + CHECK(args[0]->IsString()); + CHECK(args[1]->IsString()); + Local code = args[0].As(); + Local filename = args[1].As(); + Isolate* isolate = args.GetIsolate(); + Local context = isolate->GetCurrentContext(); + Environment* env = Environment::GetCurrent(context); + bool cache_rejected = false; -#ifndef DISABLE_SINGLE_EXECUTABLE_APPLICATION - if (used_cache_from_sea) { - cache_rejected = source.GetCachedData()->rejected; + Local fn; + { + TryCatchScope try_catch(env); + if (!CompileFunctionForCJSLoader( + env, context, code, filename, &cache_rejected) + .ToLocal(&fn)) { + CHECK(try_catch.HasCaught()); + CHECK(!try_catch.HasTerminated()); + errors::DecorateErrorStack(env, try_catch); + try_catch.ReThrow(); + return; + } } -#endif std::vector> names = { env->cached_data_rejected_string(), @@ -1705,6 +1544,108 @@ static void CompileFunctionForCJSLoader( args.GetReturnValue().Set(result); } +static bool ShouldRetryAsESM(Realm* realm, + Local message, + Local code, + Local resource_name) { + Isolate* isolate = realm->isolate(); + + Utf8Value message_value(isolate, message); + auto message_view = message_value.ToStringView(); + + // These indicates that the file contains syntaxes that are only valid in + // ESM. So it must be true. + for (const auto& error_message : esm_syntax_error_messages) { + if (message_view.find(error_message) != std::string_view::npos) { + return true; + } + } + + // Check if the error message is allowed in ESM but not in CommonJS. If it + // is the case, let's check if file can be compiled as ESM. + bool maybe_valid_in_esm = false; + for (const auto& error_message : throws_only_in_cjs_error_messages) { + if (message_view.find(error_message) != std::string_view::npos) { + maybe_valid_in_esm = true; + break; + } + } + if (!maybe_valid_in_esm) { + return false; + } + + bool cache_rejected = false; + TryCatchScope try_catch(realm->env()); + ShouldNotAbortOnUncaughtScope no_abort_scope(realm->env()); + Local module; + Local hdo = loader::ModuleWrap::GetHostDefinedOptions( + isolate, realm->isolate_data()->source_text_module_default_hdo()); + if (loader::ModuleWrap::CompileSourceTextModule( + realm, code, resource_name, 0, 0, hdo, nullptr, &cache_rejected) + .ToLocal(&module)) { + return true; + } + + return false; +} + +static void ShouldRetryAsESM(const FunctionCallbackInfo& args) { + Realm* realm = Realm::GetCurrent(args); + + CHECK_EQ(args.Length(), 3); // message, code, resource_name + CHECK(args[0]->IsString()); + Local message = args[0].As(); + CHECK(args[1]->IsString()); + Local code = args[1].As(); + CHECK(args[2]->IsString()); + Local resource_name = args[2].As(); + + args.GetReturnValue().Set( + ShouldRetryAsESM(realm, message, code, resource_name)); +} + +static void ContainsModuleSyntax(const FunctionCallbackInfo& args) { + Isolate* isolate = args.GetIsolate(); + Local context = isolate->GetCurrentContext(); + Realm* realm = Realm::GetCurrent(context); + Environment* env = realm->env(); + + CHECK_GE(args.Length(), 2); + + // Argument 1: source code + CHECK(args[0]->IsString()); + Local code = args[0].As(); + + // Argument 2: filename + CHECK(args[1]->IsString()); + Local filename = args[1].As(); + + // Argument 2: resource name (URL for ES module). + Local resource_name = filename; + if (args[2]->IsString()) { + resource_name = args[2].As(); + } + + bool cache_rejected = false; + Local message; + { + Local fn; + TryCatchScope try_catch(env); + ShouldNotAbortOnUncaughtScope no_abort_scope(env); + if (CompileFunctionForCJSLoader( + env, context, code, filename, &cache_rejected) + .ToLocal(&fn)) { + args.GetReturnValue().Set(false); + return; + } + CHECK(try_catch.HasCaught()); + message = try_catch.Message()->Get(); + } + + bool result = ShouldRetryAsESM(realm, message, code, resource_name); + args.GetReturnValue().Set(result); +} + static void StartSigintWatchdog(const FunctionCallbackInfo& args) { int ret = SigintWatchdogHelper::GetInstance()->Start(); args.GetReturnValue().Set(ret == 0); @@ -1761,6 +1702,9 @@ void CreatePerIsolateProperties(IsolateData* isolate_data, target, "compileFunctionForCJSLoader", CompileFunctionForCJSLoader); + + SetMethod(isolate, target, "containsModuleSyntax", ContainsModuleSyntax); + SetMethod(isolate, target, "shouldRetryAsESM", ShouldRetryAsESM); } static void CreatePerContextProperties(Local target, @@ -1773,7 +1717,6 @@ static void CreatePerContextProperties(Local target, Local constants = Object::New(env->isolate()); Local measure_memory = Object::New(env->isolate()); Local memory_execution = Object::New(env->isolate()); - Local syntax_detection_errors = Object::New(env->isolate()); { Local memory_mode = Object::New(env->isolate()); @@ -1794,25 +1737,6 @@ static void CreatePerContextProperties(Local target, READONLY_PROPERTY(constants, "measureMemory", measure_memory); - { - Local esm_syntax_error_messages_array = - ToV8Value(context, esm_syntax_error_messages).ToLocalChecked(); - READONLY_PROPERTY(syntax_detection_errors, - "esmSyntaxErrorMessages", - esm_syntax_error_messages_array); - } - - { - Local throws_only_in_cjs_error_messages_array = - ToV8Value(context, throws_only_in_cjs_error_messages).ToLocalChecked(); - READONLY_PROPERTY(syntax_detection_errors, - "throwsOnlyInCommonJSErrorMessages", - throws_only_in_cjs_error_messages_array); - } - - READONLY_PROPERTY( - constants, "syntaxDetectionErrors", syntax_detection_errors); - target->Set(context, env->constants_string(), constants).Check(); } @@ -1825,6 +1749,8 @@ void RegisterExternalReferences(ExternalReferenceRegistry* registry) { registry->Register(StopSigintWatchdog); registry->Register(WatchdogHasPendingSigint); registry->Register(MeasureMemory); + registry->Register(ContainsModuleSyntax); + registry->Register(ShouldRetryAsESM); } } // namespace contextify } // namespace node diff --git a/src/node_contextify.h b/src/node_contextify.h index ff975c8cf135e4..88b5684844b915 100644 --- a/src/node_contextify.h +++ b/src/node_contextify.h @@ -98,21 +98,6 @@ class ContextifyContext : public BaseObject { bool produce_cached_data, v8::Local id_symbol, const errors::TryCatchScope& try_catch); - static v8::ScriptCompiler::Source GetCommonJSSourceInstance( - v8::Isolate* isolate, - v8::Local code, - v8::Local filename, - int line_offset, - int column_offset, - v8::Local host_defined_options, - v8::ScriptCompiler::CachedData* cached_data); - static v8::ScriptCompiler::CompileOptions GetCompileOptions( - const v8::ScriptCompiler::Source& source); - static void ContainsModuleSyntax( - const v8::FunctionCallbackInfo& args); - static void ShouldRetryAsESM(const v8::FunctionCallbackInfo& args); - static bool ShouldRetryAsESMInternal(Environment* env, - v8::Local code); static void WeakCallback( const v8::WeakCallbackInfo& data); static void PropertyGetterCallback( From 54773824d3c5e1535a908e6337b68e9601c692b0 Mon Sep 17 00:00:00 2001 From: Chengzhong Wu Date: Thu, 25 Apr 2024 11:33:15 +0100 Subject: [PATCH 03/33] lib,src: iterate module requests of a module wrap in JS Avoid repetitively calling into JS callback from C++ in `ModuleWrap::Link`. This removes the convoluted callback style of the internal `ModuleWrap` link step. PR-URL: https://github.com/nodejs/node/pull/52058 Reviewed-By: Joyee Cheung --- lib/internal/modules/esm/loader.js | 2 +- lib/internal/modules/esm/module_job.js | 92 +++++---- lib/internal/vm/module.js | 91 +++++---- src/env_properties.h | 3 + src/module_wrap.cc | 214 +++++++-------------- src/module_wrap.h | 9 +- src/util-inl.h | 16 ++ src/util.h | 5 + test/parallel/test-internal-module-wrap.js | 16 +- 9 files changed, 219 insertions(+), 229 deletions(-) diff --git a/lib/internal/modules/esm/loader.js b/lib/internal/modules/esm/loader.js index 9c7a52019ced60..0878e257ade491 100644 --- a/lib/internal/modules/esm/loader.js +++ b/lib/internal/modules/esm/loader.js @@ -318,7 +318,7 @@ class ModuleLoader { * @param {object} importAttributes import attributes from the import statement. * @returns {ModuleJobBase} */ - getModuleWrapForRequire(specifier, parentURL, importAttributes) { + getModuleJobForRequire(specifier, parentURL, importAttributes) { assert(getOptionValue('--experimental-require-module')); if (canParse(specifier)) { diff --git a/lib/internal/modules/esm/module_job.js b/lib/internal/modules/esm/module_job.js index 4bb6a72c72aa06..f752e9ed1d35f4 100644 --- a/lib/internal/modules/esm/module_job.js +++ b/lib/internal/modules/esm/module_job.js @@ -1,8 +1,8 @@ 'use strict'; const { + Array, ArrayPrototypeJoin, - ArrayPrototypePush, ArrayPrototypeSome, FunctionPrototype, ObjectSetPrototypeOf, @@ -82,31 +82,8 @@ class ModuleJob extends ModuleJobBase { this.modulePromise = PromiseResolve(this.modulePromise); } - // Wait for the ModuleWrap instance being linked with all dependencies. - const link = async () => { - this.module = await this.modulePromise; - assert(this.module instanceof ModuleWrap); - - // Explicitly keeping track of dependency jobs is needed in order - // to flatten out the dependency graph below in `_instantiate()`, - // so that circular dependencies can't cause a deadlock by two of - // these `link` callbacks depending on each other. - const dependencyJobs = []; - const promises = this.module.link(async (specifier, attributes) => { - const job = await this.#loader.getModuleJob(specifier, url, attributes); - debug(`async link() ${this.url} -> ${specifier}`, job); - ArrayPrototypePush(dependencyJobs, job); - return job.modulePromise; - }); - - if (promises !== undefined) { - await SafePromiseAllReturnVoid(promises); - } - - return SafePromiseAllReturnArrayLike(dependencyJobs); - }; // Promise for the list of all dependencyJobs. - this.linked = link(); + this.linked = this._link(); // This promise is awaited later anyway, so silence // 'unhandled rejection' warnings. PromisePrototypeThen(this.linked, undefined, noop); @@ -116,6 +93,49 @@ class ModuleJob extends ModuleJobBase { this.instantiated = undefined; } + /** + * Iterates the module requests and links with the loader. + * @returns {Promise} Dependency module jobs. + */ + async _link() { + this.module = await this.modulePromise; + assert(this.module instanceof ModuleWrap); + + const moduleRequests = this.module.getModuleRequests(); + // Explicitly keeping track of dependency jobs is needed in order + // to flatten out the dependency graph below in `_instantiate()`, + // so that circular dependencies can't cause a deadlock by two of + // these `link` callbacks depending on each other. + // Create an ArrayLike to avoid calling into userspace with `.then` + // when returned from the async function. + const dependencyJobs = Array(moduleRequests.length); + ObjectSetPrototypeOf(dependencyJobs, null); + + // Specifiers should be aligned with the moduleRequests array in order. + const specifiers = Array(moduleRequests.length); + const modulePromises = Array(moduleRequests.length); + // Iterate with index to avoid calling into userspace with `Symbol.iterator`. + for (let idx = 0; idx < moduleRequests.length; idx++) { + const { specifier, attributes } = moduleRequests[idx]; + + const dependencyJobPromise = this.#loader.getModuleJob( + specifier, this.url, attributes, + ); + const modulePromise = PromisePrototypeThen(dependencyJobPromise, (job) => { + debug(`async link() ${this.url} -> ${specifier}`, job); + dependencyJobs[idx] = job; + return job.modulePromise; + }); + modulePromises[idx] = modulePromise; + specifiers[idx] = specifier; + } + + const modules = await SafePromiseAllReturnArrayLike(modulePromises); + this.module.link(specifiers, modules); + + return dependencyJobs; + } + instantiate() { if (this.instantiated === undefined) { this.instantiated = this._instantiate(); @@ -269,18 +289,20 @@ class ModuleJobSync extends ModuleJobBase { super(url, importAttributes, moduleWrap, isMain, inspectBrk, true); assert(this.module instanceof ModuleWrap); this.#loader = loader; - const moduleRequests = this.module.getModuleRequestsSync(); - const linked = []; + const moduleRequests = this.module.getModuleRequests(); + // Specifiers should be aligned with the moduleRequests array in order. + const specifiers = Array(moduleRequests.length); + const modules = Array(moduleRequests.length); + const jobs = Array(moduleRequests.length); for (let i = 0; i < moduleRequests.length; ++i) { - const { 0: specifier, 1: attributes } = moduleRequests[i]; - const job = this.#loader.getModuleWrapForRequire(specifier, url, attributes); - const isLast = (i === moduleRequests.length - 1); - // TODO(joyeecheung): make the resolution callback deal with both promisified - // an raw module wraps, then we don't need to wrap it with a promise here. - this.module.cacheResolvedWrapsSync(specifier, PromiseResolve(job.module), isLast); - ArrayPrototypePush(linked, job); + const { specifier, attributes } = moduleRequests[i]; + const job = this.#loader.getModuleJobForRequire(specifier, url, attributes); + specifiers[i] = specifier; + modules[i] = job.module; + jobs[i] = job; } - this.linked = linked; + this.module.link(specifiers, modules); + this.linked = jobs; } get modulePromise() { diff --git a/lib/internal/vm/module.js b/lib/internal/vm/module.js index 0a8419f8ca4454..9e500cde86105b 100644 --- a/lib/internal/vm/module.js +++ b/lib/internal/vm/module.js @@ -2,16 +2,20 @@ const assert = require('internal/assert'); const { + Array, ArrayIsArray, ArrayPrototypeForEach, ArrayPrototypeIndexOf, + ArrayPrototypeMap, ArrayPrototypeSome, ObjectDefineProperty, ObjectGetPrototypeOf, ObjectPrototypeHasOwnProperty, ObjectSetPrototypeOf, + PromiseResolve, + PromisePrototypeThen, ReflectApply, - SafePromiseAllReturnVoid, + SafePromiseAllReturnArrayLike, Symbol, SymbolToStringTag, TypeError, @@ -293,44 +297,61 @@ class SourceTextModule extends Module { importModuleDynamically, }); - this[kLink] = async (linker) => { - this.#statusOverride = 'linking'; + this[kDependencySpecifiers] = undefined; + } - const promises = this[kWrap].link(async (identifier, attributes) => { - const module = await linker(identifier, this, { attributes, assert: attributes }); - if (!isModule(module)) { - throw new ERR_VM_MODULE_NOT_MODULE(); - } - if (module.context !== this.context) { - throw new ERR_VM_MODULE_DIFFERENT_CONTEXT(); - } - if (module.status === 'errored') { - throw new ERR_VM_MODULE_LINK_FAILURE(`request for '${identifier}' resolved to an errored module`, module.error); - } - if (module.status === 'unlinked') { - await module[kLink](linker); - } - return module[kWrap]; + async [kLink](linker) { + this.#statusOverride = 'linking'; + + const moduleRequests = this[kWrap].getModuleRequests(); + // Iterates the module requests and links with the linker. + // Specifiers should be aligned with the moduleRequests array in order. + const specifiers = Array(moduleRequests.length); + const modulePromises = Array(moduleRequests.length); + // Iterates with index to avoid calling into userspace with `Symbol.iterator`. + for (let idx = 0; idx < moduleRequests.length; idx++) { + const { specifier, attributes } = moduleRequests[idx]; + + const linkerResult = linker(specifier, this, { + attributes, + assert: attributes, }); + const modulePromise = PromisePrototypeThen( + PromiseResolve(linkerResult), async (module) => { + if (!isModule(module)) { + throw new ERR_VM_MODULE_NOT_MODULE(); + } + if (module.context !== this.context) { + throw new ERR_VM_MODULE_DIFFERENT_CONTEXT(); + } + if (module.status === 'errored') { + throw new ERR_VM_MODULE_LINK_FAILURE(`request for '${specifier}' resolved to an errored module`, module.error); + } + if (module.status === 'unlinked') { + await module[kLink](linker); + } + return module[kWrap]; + }); + modulePromises[idx] = modulePromise; + specifiers[idx] = specifier; + } - try { - if (promises !== undefined) { - await SafePromiseAllReturnVoid(promises); - } - } catch (e) { - this.#error = e; - throw e; - } finally { - this.#statusOverride = undefined; - } - }; - - this[kDependencySpecifiers] = undefined; + try { + const modules = await SafePromiseAllReturnArrayLike(modulePromises); + this[kWrap].link(specifiers, modules); + } catch (e) { + this.#error = e; + throw e; + } finally { + this.#statusOverride = undefined; + } } get dependencySpecifiers() { validateInternalField(this, kDependencySpecifiers, 'SourceTextModule'); - this[kDependencySpecifiers] ??= this[kWrap].getStaticDependencySpecifiers(); + // TODO(legendecas): add a new getter to expose the import attributes as the value type + // of [[RequestedModules]] is changed in https://tc39.es/proposal-import-attributes/#table-cyclic-module-fields. + this[kDependencySpecifiers] ??= ArrayPrototypeMap(this[kWrap].getModuleRequests(), (request) => request.specifier); return this[kDependencySpecifiers]; } @@ -392,10 +413,10 @@ class SyntheticModule extends Module { context, identifier, }); + } - this[kLink] = () => this[kWrap].link(() => { - assert.fail('link callback should not be called'); - }); + [kLink]() { + /** nothing to do for synthetic modules */ } setExport(name, value) { diff --git a/src/env_properties.h b/src/env_properties.h index 8243a71c035f9b..f0813da6708762 100644 --- a/src/env_properties.h +++ b/src/env_properties.h @@ -71,6 +71,7 @@ V(args_string, "args") \ V(asn1curve_string, "asn1Curve") \ V(async_ids_stack_string, "async_ids_stack") \ + V(attributes_string, "attributes") \ V(base_string, "base") \ V(bits_string, "bits") \ V(block_list_string, "blockList") \ @@ -303,6 +304,7 @@ V(sni_context_string, "sni_context") \ V(source_string, "source") \ V(source_map_url_string, "sourceMapURL") \ + V(specifier_string, "specifier") \ V(stack_string, "stack") \ V(standard_name_string, "standardName") \ V(start_time_string, "startTime") \ @@ -377,6 +379,7 @@ V(intervalhistogram_constructor_template, v8::FunctionTemplate) \ V(libuv_stream_wrap_ctor_template, v8::FunctionTemplate) \ V(message_port_constructor_template, v8::FunctionTemplate) \ + V(module_wrap_constructor_template, v8::FunctionTemplate) \ V(microtask_queue_ctor_template, v8::FunctionTemplate) \ V(pipe_constructor_template, v8::FunctionTemplate) \ V(promise_wrap_template, v8::ObjectTemplate) \ diff --git a/src/module_wrap.cc b/src/module_wrap.cc index 4cd75447b77526..13f7416ff2a9d5 100644 --- a/src/module_wrap.cc +++ b/src/module_wrap.cc @@ -343,136 +343,105 @@ static Local createImportAttributesContainer( Local raw_attributes, const int elements_per_attribute) { CHECK_EQ(raw_attributes->Length() % elements_per_attribute, 0); - Local attributes = - Object::New(isolate, v8::Null(isolate), nullptr, nullptr, 0); + size_t num_attributes = raw_attributes->Length() / elements_per_attribute; + std::vector> names(num_attributes); + std::vector> values(num_attributes); + for (int i = 0; i < raw_attributes->Length(); i += elements_per_attribute) { - attributes - ->Set(realm->context(), - raw_attributes->Get(realm->context(), i).As(), - raw_attributes->Get(realm->context(), i + 1).As()) - .ToChecked(); + int idx = i / elements_per_attribute; + names[idx] = raw_attributes->Get(realm->context(), i).As(); + values[idx] = raw_attributes->Get(realm->context(), i + 1).As(); } - return attributes; + return Object::New( + isolate, v8::Null(isolate), names.data(), values.data(), num_attributes); } -void ModuleWrap::GetModuleRequestsSync( - const FunctionCallbackInfo& args) { - Realm* realm = Realm::GetCurrent(args); - Isolate* isolate = args.GetIsolate(); - - Local that = args.This(); - - ModuleWrap* obj; - ASSIGN_OR_RETURN_UNWRAP(&obj, that); - - CHECK(!obj->linked_); +static Local createModuleRequestsContainer( + Realm* realm, Isolate* isolate, Local raw_requests) { + std::vector> requests(raw_requests->Length()); - Local module = obj->module_.Get(isolate); - Local module_requests = module->GetModuleRequests(); - const int module_requests_length = module_requests->Length(); - - std::vector> requests; - requests.reserve(module_requests_length); - // call the dependency resolve callbacks - for (int i = 0; i < module_requests_length; i++) { + for (int i = 0; i < raw_requests->Length(); i++) { Local module_request = - module_requests->Get(realm->context(), i).As(); - Local raw_attributes = module_request->GetImportAssertions(); - std::vector> request = { - module_request->GetSpecifier(), - createImportAttributesContainer(realm, isolate, raw_attributes, 3), - }; - requests.push_back(Array::New(isolate, request.data(), request.size())); - } + raw_requests->Get(realm->context(), i).As(); - args.GetReturnValue().Set( - Array::New(isolate, requests.data(), requests.size())); -} - -void ModuleWrap::CacheResolvedWrapsSync( - const FunctionCallbackInfo& args) { - Isolate* isolate = args.GetIsolate(); + Local specifier = module_request->GetSpecifier(); - CHECK_EQ(args.Length(), 3); - CHECK(args[0]->IsString()); - CHECK(args[1]->IsPromise()); - CHECK(args[2]->IsBoolean()); + // Contains the import assertions for this request in the form: + // [key1, value1, source_offset1, key2, value2, source_offset2, ...]. + Local raw_attributes = module_request->GetImportAssertions(); + Local attributes = + createImportAttributesContainer(realm, isolate, raw_attributes, 3); - ModuleWrap* dependent; - ASSIGN_OR_RETURN_UNWRAP(&dependent, args.This()); + Local names[] = { + realm->isolate_data()->specifier_string(), + realm->isolate_data()->attributes_string(), + }; + Local values[] = { + specifier, + attributes, + }; + DCHECK_EQ(arraysize(names), arraysize(values)); - Utf8Value specifier(isolate, args[0]); - dependent->resolve_cache_[specifier.ToString()].Reset(isolate, - args[1].As()); + Local request = Object::New( + isolate, v8::Null(isolate), names, values, arraysize(names)); - if (args[2].As()->Value()) { - dependent->linked_ = true; + requests[i] = request; } + + return Array::New(isolate, requests.data(), requests.size()); } -void ModuleWrap::Link(const FunctionCallbackInfo& args) { +void ModuleWrap::GetModuleRequests(const FunctionCallbackInfo& args) { Realm* realm = Realm::GetCurrent(args); Isolate* isolate = args.GetIsolate(); - - CHECK_EQ(args.Length(), 1); - CHECK(args[0]->IsFunction()); - Local that = args.This(); ModuleWrap* obj; ASSIGN_OR_RETURN_UNWRAP(&obj, that); - if (obj->linked_) - return; - obj->linked_ = true; + Local module = obj->module_.Get(isolate); + args.GetReturnValue().Set(createModuleRequestsContainer( + realm, isolate, module->GetModuleRequests())); +} - Local resolver_arg = args[0].As(); +// moduleWrap.link(specifiers, moduleWraps) +void ModuleWrap::Link(const FunctionCallbackInfo& args) { + Realm* realm = Realm::GetCurrent(args); + Isolate* isolate = args.GetIsolate(); + Local context = realm->context(); - Local mod_context = obj->context(); - Local module = obj->module_.Get(isolate); + ModuleWrap* dependent; + ASSIGN_OR_RETURN_UNWRAP(&dependent, args.This()); - Local module_requests = module->GetModuleRequests(); - const int module_requests_length = module_requests->Length(); - MaybeStackBuffer, 16> promises(module_requests_length); + CHECK_EQ(args.Length(), 2); - // call the dependency resolve callbacks - for (int i = 0; i < module_requests_length; i++) { - Local module_request = - module_requests->Get(realm->context(), i).As(); - Local specifier = module_request->GetSpecifier(); - Utf8Value specifier_utf8(realm->isolate(), specifier); - std::string specifier_std(*specifier_utf8, specifier_utf8.length()); + Local specifiers = args[0].As(); + Local modules = args[1].As(); + CHECK_EQ(specifiers->Length(), modules->Length()); - Local raw_attributes = module_request->GetImportAssertions(); - Local attributes = - createImportAttributesContainer(realm, isolate, raw_attributes, 3); + std::vector> specifiers_buffer; + if (FromV8Array(context, specifiers, &specifiers_buffer).IsNothing()) { + return; + } + std::vector> modules_buffer; + if (FromV8Array(context, modules, &modules_buffer).IsNothing()) { + return; + } - Local argv[] = { - specifier, - attributes, - }; + for (uint32_t i = 0; i < specifiers->Length(); i++) { + Local specifier_str = + specifiers_buffer[i].Get(isolate).As(); + Local module_object = modules_buffer[i].Get(isolate).As(); - MaybeLocal maybe_resolve_return_value = - resolver_arg->Call(mod_context, that, arraysize(argv), argv); - if (maybe_resolve_return_value.IsEmpty()) { - return; - } - Local resolve_return_value = - maybe_resolve_return_value.ToLocalChecked(); - if (!resolve_return_value->IsPromise()) { - THROW_ERR_VM_MODULE_LINK_FAILURE( - realm, "request for '%s' did not return promise", specifier_std); - return; - } - Local resolve_promise = resolve_return_value.As(); - obj->resolve_cache_[specifier_std].Reset(isolate, resolve_promise); + CHECK( + realm->isolate_data()->module_wrap_constructor_template()->HasInstance( + module_object)); - promises[i] = resolve_promise; + Utf8Value specifier(isolate, specifier_str); + dependent->resolve_cache_[specifier.ToString()].Reset(isolate, + module_object); } - - args.GetReturnValue().Set( - Array::New(isolate, promises.out(), promises.length())); } void ModuleWrap::Instantiate(const FunctionCallbackInfo& args) { @@ -733,29 +702,6 @@ void ModuleWrap::GetStatus(const FunctionCallbackInfo& args) { args.GetReturnValue().Set(module->GetStatus()); } -void ModuleWrap::GetStaticDependencySpecifiers( - const FunctionCallbackInfo& args) { - Realm* realm = Realm::GetCurrent(args); - ModuleWrap* obj; - ASSIGN_OR_RETURN_UNWRAP(&obj, args.This()); - - Local module = obj->module_.Get(realm->isolate()); - - Local module_requests = module->GetModuleRequests(); - int count = module_requests->Length(); - - MaybeStackBuffer, 16> specifiers(count); - - for (int i = 0; i < count; i++) { - Local module_request = - module_requests->Get(realm->context(), i).As(); - specifiers[i] = module_request->GetSpecifier(); - } - - args.GetReturnValue().Set( - Array::New(realm->isolate(), specifiers.out(), count)); -} - void ModuleWrap::GetError(const FunctionCallbackInfo& args) { Isolate* isolate = args.GetIsolate(); ModuleWrap* obj; @@ -793,16 +739,8 @@ MaybeLocal ModuleWrap::ResolveModuleCallback( return MaybeLocal(); } - Local resolve_promise = + Local module_object = dependent->resolve_cache_[specifier_std].Get(isolate); - - if (resolve_promise->State() != Promise::kFulfilled) { - THROW_ERR_VM_MODULE_LINK_FAILURE( - env, "request for '%s' is not yet fulfilled", specifier_std); - return MaybeLocal(); - } - - Local module_object = resolve_promise->Result().As(); if (module_object.IsEmpty() || !module_object->IsObject()) { THROW_ERR_VM_MODULE_LINK_FAILURE( env, "request for '%s' did not return an object", specifier_std); @@ -1029,9 +967,7 @@ void ModuleWrap::CreatePerIsolateProperties(IsolateData* isolate_data, ModuleWrap::kInternalFieldCount); SetProtoMethod(isolate, tpl, "link", Link); - SetProtoMethod(isolate, tpl, "getModuleRequestsSync", GetModuleRequestsSync); - SetProtoMethod( - isolate, tpl, "cacheResolvedWrapsSync", CacheResolvedWrapsSync); + SetProtoMethod(isolate, tpl, "getModuleRequests", GetModuleRequests); SetProtoMethod(isolate, tpl, "instantiateSync", InstantiateSync); SetProtoMethod(isolate, tpl, "evaluateSync", EvaluateSync); SetProtoMethod(isolate, tpl, "getNamespaceSync", GetNamespaceSync); @@ -1043,12 +979,8 @@ void ModuleWrap::CreatePerIsolateProperties(IsolateData* isolate_data, SetProtoMethodNoSideEffect(isolate, tpl, "getNamespace", GetNamespace); SetProtoMethodNoSideEffect(isolate, tpl, "getStatus", GetStatus); SetProtoMethodNoSideEffect(isolate, tpl, "getError", GetError); - SetProtoMethodNoSideEffect(isolate, - tpl, - "getStaticDependencySpecifiers", - GetStaticDependencySpecifiers); - SetConstructorFunction(isolate, target, "ModuleWrap", tpl); + isolate_data->set_module_wrap_constructor_template(tpl); SetMethod(isolate, target, @@ -1086,8 +1018,7 @@ void ModuleWrap::RegisterExternalReferences( registry->Register(New); registry->Register(Link); - registry->Register(GetModuleRequestsSync); - registry->Register(CacheResolvedWrapsSync); + registry->Register(GetModuleRequests); registry->Register(InstantiateSync); registry->Register(EvaluateSync); registry->Register(GetNamespaceSync); @@ -1098,7 +1029,6 @@ void ModuleWrap::RegisterExternalReferences( registry->Register(GetNamespace); registry->Register(GetStatus); registry->Register(GetError); - registry->Register(GetStaticDependencySpecifiers); registry->Register(SetImportModuleDynamicallyCallback); registry->Register(SetInitializeImportMetaObjectCallback); diff --git a/src/module_wrap.h b/src/module_wrap.h index 09da8ee43fa8b4..b5d0c48997680d 100644 --- a/src/module_wrap.h +++ b/src/module_wrap.h @@ -97,9 +97,7 @@ class ModuleWrap : public BaseObject { ~ModuleWrap() override; static void New(const v8::FunctionCallbackInfo& args); - static void GetModuleRequestsSync( - const v8::FunctionCallbackInfo& args); - static void CacheResolvedWrapsSync( + static void GetModuleRequests( const v8::FunctionCallbackInfo& args); static void InstantiateSync(const v8::FunctionCallbackInfo& args); static void EvaluateSync(const v8::FunctionCallbackInfo& args); @@ -111,8 +109,6 @@ class ModuleWrap : public BaseObject { static void GetNamespace(const v8::FunctionCallbackInfo& args); static void GetStatus(const v8::FunctionCallbackInfo& args); static void GetError(const v8::FunctionCallbackInfo& args); - static void GetStaticDependencySpecifiers( - const v8::FunctionCallbackInfo& args); static void SetImportModuleDynamicallyCallback( const v8::FunctionCallbackInfo& args); @@ -132,10 +128,9 @@ class ModuleWrap : public BaseObject { static ModuleWrap* GetFromModule(node::Environment*, v8::Local); v8::Global module_; - std::unordered_map> resolve_cache_; + std::unordered_map> resolve_cache_; contextify::ContextifyContext* contextify_context_ = nullptr; bool synthetic_ = false; - bool linked_ = false; int module_hash_; }; diff --git a/src/util-inl.h b/src/util-inl.h index 31ffa4934cf005..03db47a8bfaab3 100644 --- a/src/util-inl.h +++ b/src/util-inl.h @@ -403,6 +403,22 @@ inline char* UncheckedCalloc(size_t n) { return UncheckedCalloc(n); } // headers than we really need to. void ThrowErrStringTooLong(v8::Isolate* isolate); +v8::Maybe FromV8Array(v8::Local context, + v8::Local js_array, + std::vector>* out) { + uint32_t count = js_array->Length(); + out->reserve(count); + v8::Isolate* isolate = context->GetIsolate(); + for (size_t i = 0; i < count; ++i) { + v8::Local element; + if (!js_array->Get(context, i).ToLocal(&element)) { + return v8::Nothing(); + } + out->push_back(v8::Global(isolate, element)); + } + return v8::JustVoid(); +} + v8::MaybeLocal ToV8Value(v8::Local context, std::string_view str, v8::Isolate* isolate) { diff --git a/src/util.h b/src/util.h index b2c152a865ee76..07873fd1d605f4 100644 --- a/src/util.h +++ b/src/util.h @@ -702,6 +702,11 @@ struct FunctionDeleter { template using DeleteFnPtr = typename FunctionDeleter::Pointer; +// Mocking the FromV8Array from newer version of Node.js which relies +// on V8 API that does not exist on v20.x to smooth out backports. +inline v8::Maybe FromV8Array(v8::Local context, + v8::Local js_array, + std::vector>* out); std::vector SplitString(const std::string_view in, const std::string_view delim); diff --git a/test/parallel/test-internal-module-wrap.js b/test/parallel/test-internal-module-wrap.js index 520a83a3a47c0e..3839338bc2da98 100644 --- a/test/parallel/test-internal-module-wrap.js +++ b/test/parallel/test-internal-module-wrap.js @@ -5,23 +5,21 @@ const assert = require('assert'); const { internalBinding } = require('internal/test/binding'); const { ModuleWrap } = internalBinding('module_wrap'); -const { getPromiseDetails, isPromise } = internalBinding('util'); -const setTimeoutAsync = require('util').promisify(setTimeout); const foo = new ModuleWrap('foo', undefined, 'export * from "bar";', 0, 0); const bar = new ModuleWrap('bar', undefined, 'export const five = 5', 0, 0); (async () => { - const promises = foo.link(() => setTimeoutAsync(1000).then(() => bar)); - assert.strictEqual(promises.length, 1); - assert(isPromise(promises[0])); - - await Promise.all(promises); - - assert.strictEqual(getPromiseDetails(promises[0])[1], bar); + const moduleRequests = foo.getModuleRequests(); + assert.strictEqual(moduleRequests.length, 1); + assert.strictEqual(moduleRequests[0].specifier, 'bar'); + foo.link(['bar'], [bar]); foo.instantiate(); assert.strictEqual(await foo.evaluate(-1, false), undefined); assert.strictEqual(foo.getNamespace().five, 5); + + // Check that the module requests are the same after linking, instantiate, and evaluation. + assert.deepStrictEqual(moduleRequests, foo.getModuleRequests()); })().then(common.mustCall()); From cb4d3a68d6ac950e3f046feb194819f106ddefe7 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Mon, 29 Apr 2024 22:21:53 +0200 Subject: [PATCH 04/33] module: support ESM detection in the CJS loader This patch: 1. Adds ESM syntax detection to compileFunctionForCJSLoader() for --experimental-detect-module and allow it to emit the warning for how to load ESM when it's used to parse ESM as CJS but detection is not enabled. 2. Moves the ESM detection of --experimental-detect-module for the entrypoint from executeUserEntryPoint() into Module.prototype._compile() and handle it directly in the CJS loader so that the errors thrown during compilation *and execution* during the loading of the entrypoint does not need to be bubbled all the way up. If the entrypoint doesn't parse as CJS, and detection is enabled, the CJS loader will re-load the entrypoint as ESM on the spot asynchronously using runEntryPointWithESMLoader() and cascadedLoader.import(). This is fine for the entrypoint because unlike require(ESM) we don't the namespace of the entrypoint synchronously, and can just ignore the returned value. In this case process.mainModule is reset to undefined as they are not available for ESM entrypoints. 3. Supports --experimental-detect-module for require(esm). PR-URL: https://github.com/nodejs/node/pull/52047 Reviewed-By: Geoffrey Booth Reviewed-By: Antoine du Hamel --- .eslintrc.js | 2 + doc/api/modules.md | 26 ++++-- lib/internal/modules/cjs/loader.js | 87 ++++++++++--------- lib/internal/modules/esm/translators.js | 54 +++--------- lib/internal/modules/run_main.js | 30 +------ src/node_contextify.cc | 82 ++++++++++++++--- src/node_errors.cc | 8 +- src/node_errors.h | 3 + ...t-require-module-detect-entry-point-aou.js | 7 ++ .../test-require-module-detect-entry-point.js | 7 ++ .../test-require-module-dont-detect-cjs.js | 11 +++ .../test-require-module-with-detection.js | 18 ++++ 12 files changed, 206 insertions(+), 129 deletions(-) create mode 100644 test/es-module/test-require-module-detect-entry-point-aou.js create mode 100644 test/es-module/test-require-module-detect-entry-point.js create mode 100644 test/es-module/test-require-module-dont-detect-cjs.js create mode 100644 test/es-module/test-require-module-with-detection.js diff --git a/.eslintrc.js b/.eslintrc.js index 9eaa7e3f47b0e2..7b075ab83463aa 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -58,6 +58,8 @@ module.exports = { 'test/es-module/test-esm-example-loader.js', 'test/es-module/test-esm-type-flag.js', 'test/es-module/test-esm-type-flag-alias.js', + 'test/es-module/test-require-module-detect-entry-point.js', + 'test/es-module/test-require-module-detect-entry-point-aou.js', ], parserOptions: { sourceType: 'module' }, }, diff --git a/doc/api/modules.md b/doc/api/modules.md index 8b5840d88778e4..f2e6a0042b4e1b 100644 --- a/doc/api/modules.md +++ b/doc/api/modules.md @@ -187,9 +187,12 @@ regarding which files are parsed as ECMAScript modules. If `--experimental-require-module` is enabled, and the ECMAScript module being loaded by `require()` meets the following requirements: -* Explicitly marked as an ES module with a `"type": "module"` field in - the closest package.json or a `.mjs` extension. -* Fully synchronous (contains no top-level `await`). +* The module is fully synchronous (contains no top-level `await`); and +* One of these conditions are met: + 1. The file has a `.mjs` extension. + 2. The file has a `.js` extension, and the closest `package.json` contains `"type": "module"` + 3. The file has a `.js` extension, the closest `package.json` does not contain + `"type": "commonjs"`, and `--experimental-detect-module` is enabled. `require()` will load the requested module as an ES Module, and return the module name space object. In this case it is similar to dynamic @@ -256,18 +259,27 @@ require(X) from module at path Y 6. LOAD_NODE_MODULES(X, dirname(Y)) 7. THROW "not found" +MAYBE_DETECT_AND_LOAD(X) +1. If X parses as a CommonJS module, load X as a CommonJS module. STOP. +2. Else, if `--experimental-require-module` and `--experimental-detect-module` are + enabled, and the source code of X can be parsed as ECMAScript module using + DETECT_MODULE_SYNTAX defined in + the ESM resolver, + a. Load X as an ECMAScript module. STOP. +3. THROW the SyntaxError from attempting to parse X as CommonJS in 1. STOP. + LOAD_AS_FILE(X) 1. If X is a file, load X as its file extension format. STOP 2. If X.js is a file, a. Find the closest package scope SCOPE to X. - b. If no scope was found, load X.js as a CommonJS module. STOP. + b. If no scope was found + 1. MAYBE_DETECT_AND_LOAD(X.js) c. If the SCOPE/package.json contains "type" field, 1. If the "type" field is "module", load X.js as an ECMAScript module. STOP. - 2. Else, load X.js as an CommonJS module. STOP. + 2. If the "type" field is "commonjs", load X.js as an CommonJS module. STOP. + d. MAYBE_DETECT_AND_LOAD(X.js) 3. If X.json is a file, load X.json to a JavaScript Object. STOP 4. If X.node is a file, load X.node as binary addon. STOP -5. If X.mjs is a file, and `--experimental-require-module` is enabled, - load X.mjs as an ECMAScript module. STOP LOAD_INDEX(X) 1. If X/index.js is a file diff --git a/lib/internal/modules/cjs/loader.js b/lib/internal/modules/cjs/loader.js index 62dd51ed76e612..2346b39e5abe87 100644 --- a/lib/internal/modules/cjs/loader.js +++ b/lib/internal/modules/cjs/loader.js @@ -106,7 +106,6 @@ module.exports = { kModuleExportNames, kModuleCircularVisited, initializeCJS, - entryPointSource: undefined, // Set below. Module, wrapSafe, kIsMainSymbol, @@ -1333,9 +1332,18 @@ function loadESMFromCJS(mod, filename) { const source = getMaybeCachedSource(mod, filename); const cascadedLoader = require('internal/modules/esm/loader').getOrInitializeCascadedLoader(); const isMain = mod[kIsMainSymbol]; - // TODO(joyeecheung): we may want to invent optional special handling for default exports here. - // For now, it's good enough to be identical to what `import()` returns. - mod.exports = cascadedLoader.importSyncForRequire(mod, filename, source, isMain, mod[kModuleParent]); + if (isMain) { + require('internal/modules/run_main').runEntryPointWithESMLoader((cascadedLoader) => { + const mainURL = pathToFileURL(filename).href; + cascadedLoader.import(mainURL, undefined, { __proto__: null }, true); + }); + // ESM won't be accessible via process.mainModule. + setOwnProperty(process, 'mainModule', undefined); + } else { + // TODO(joyeecheung): we may want to invent optional special handling for default exports here. + // For now, it's good enough to be identical to what `import()` returns. + mod.exports = cascadedLoader.importSyncForRequire(mod, filename, source, isMain, mod[kModuleParent]); + } } /** @@ -1344,8 +1352,10 @@ function loadESMFromCJS(mod, filename) { * @param {string} content The content of the file being loaded * @param {Module} cjsModuleInstance The CommonJS loader instance * @param {object} codeCache The SEA code cache + * @param {'commonjs'|undefined} format Intended format of the module. */ -function wrapSafe(filename, content, cjsModuleInstance, codeCache) { +function wrapSafe(filename, content, cjsModuleInstance, codeCache, format) { + assert(format !== 'module'); // ESM should be handled in loadESMFromCJS(). const hostDefinedOptionId = vm_dynamic_import_default_internal; const importModuleDynamically = vm_dynamic_import_default_internal; if (patched) { @@ -1375,36 +1385,23 @@ function wrapSafe(filename, content, cjsModuleInstance, codeCache) { }; } - try { - const result = compileFunctionForCJSLoader(content, filename); - - // cachedDataRejected is only set for cache coming from SEA. - if (codeCache && - result.cachedDataRejected !== false && - internalBinding('sea').isSea()) { - process.emitWarning('Code cache data rejected.'); - } + const isMain = !!(cjsModuleInstance && cjsModuleInstance[kIsMainSymbol]); + const shouldDetectModule = (format !== 'commonjs' && getOptionValue('--experimental-detect-module')); + const result = compileFunctionForCJSLoader(content, filename, isMain, shouldDetectModule); - // Cache the source map for the module if present. - if (result.sourceMapURL) { - maybeCacheSourceMap(filename, content, this, false, undefined, result.sourceMapURL); - } + // cachedDataRejected is only set for cache coming from SEA. + if (codeCache && + result.cachedDataRejected !== false && + internalBinding('sea').isSea()) { + process.emitWarning('Code cache data rejected.'); + } - return result; - } catch (err) { - if (process.mainModule === cjsModuleInstance) { - if (getOptionValue('--experimental-detect-module')) { - // For the main entry point, cache the source to potentially retry as ESM. - module.exports.entryPointSource = content; - } else { - // We only enrich the error (print a warning) if we're sure we're going to for-sure throw it; so if we're - // retrying as ESM, wait until we know whether we're going to retry before calling `enrichCJSError`. - const { enrichCJSError } = require('internal/modules/esm/translators'); - enrichCJSError(err, content, filename); - } - } - throw err; + // Cache the source map for the module if present. + if (result.sourceMapURL) { + maybeCacheSourceMap(filename, content, this, false, undefined, result.sourceMapURL); } + + return result; } /** @@ -1412,9 +1409,9 @@ function wrapSafe(filename, content, cjsModuleInstance, codeCache) { * `exports`) to the file. Returns exception, if any. * @param {string} content The source code of the module * @param {string} filename The file path of the module - * @param {boolean} loadAsESM Whether it's known to be ESM via .mjs or "type" in package.json. + * @param {'module'|'commonjs'|undefined} format Intended format of the module. */ -Module.prototype._compile = function(content, filename, loadAsESM = false) { +Module.prototype._compile = function(content, filename, format) { let moduleURL; let redirects; const manifest = policy()?.manifest; @@ -1424,17 +1421,24 @@ Module.prototype._compile = function(content, filename, loadAsESM = false) { manifest.assertIntegrity(moduleURL, content); } + let compiledWrapper; + if (format !== 'module') { + const result = wrapSafe(filename, content, this, undefined, format); + compiledWrapper = result.function; + if (result.canParseAsESM) { + format = 'module'; + } + } + // TODO(joyeecheung): when the module is the entry point, consider allowing TLA. // Only modules being require()'d really need to avoid TLA. - if (loadAsESM) { + if (format === 'module') { // Pass the source into the .mjs extension handler indirectly through the cache. this[kModuleSource] = content; loadESMFromCJS(this, filename); return; } - const { function: compiledWrapper } = wrapSafe(filename, content, this); - // TODO(joyeecheung): the detection below is unnecessarily complex. Using the // kIsMainSymbol, or a kBreakOnStartSymbol that gets passed from // higher level instead of doing hacky detection here. @@ -1511,12 +1515,13 @@ Module._extensions['.js'] = function(module, filename) { // If already analyzed the source, then it will be cached. const content = getMaybeCachedSource(module, filename); + let format; if (StringPrototypeEndsWith(filename, '.js')) { const pkg = packageJsonReader.readPackageScope(filename) || { __proto__: null }; // Function require shouldn't be used in ES modules. if (pkg.data?.type === 'module') { if (getOptionValue('--experimental-require-module')) { - module._compile(content, filename, true); + module._compile(content, filename, 'module'); return; } @@ -1550,10 +1555,14 @@ Module._extensions['.js'] = function(module, filename) { } } throw err; + } else if (pkg.data?.type === 'commonjs') { + format = 'commonjs'; } + } else if (StringPrototypeEndsWith(filename, '.cjs')) { + format = 'commonjs'; } - module._compile(content, filename, false); + module._compile(content, filename, format); }; /** diff --git a/lib/internal/modules/esm/translators.js b/lib/internal/modules/esm/translators.js index 044d820161a5f9..52f73368724df4 100644 --- a/lib/internal/modules/esm/translators.js +++ b/lib/internal/modules/esm/translators.js @@ -4,7 +4,6 @@ const { ArrayPrototypeMap, Boolean, JSONParse, - ObjectGetPrototypeOf, ObjectPrototypeHasOwnProperty, ObjectKeys, ReflectApply, @@ -15,7 +14,6 @@ const { StringPrototypeReplaceAll, StringPrototypeSlice, StringPrototypeStartsWith, - SyntaxErrorPrototype, globalThis: { WebAssembly }, } = primordials; @@ -30,7 +28,6 @@ function lazyTypes() { } const { - containsModuleSyntax, compileFunctionForCJSLoader, } = internalBinding('contextify'); @@ -62,7 +59,6 @@ const { const { maybeCacheSourceMap } = require('internal/source_map/source_map_cache'); const moduleWrap = internalBinding('module_wrap'); const { ModuleWrap } = moduleWrap; -const { emitWarningSync } = require('internal/process/warning'); // Lazy-loading to avoid circular dependencies. let getSourceSync; @@ -107,7 +103,6 @@ function initCJSParseSync() { const translators = new SafeMap(); exports.translators = translators; -exports.enrichCJSError = enrichCJSError; let DECODER = null; /** @@ -169,25 +164,6 @@ translators.set('module', function moduleStrategy(url, source, isMain) { return module; }); -/** - * Provide a more informative error for CommonJS imports. - * @param {Error | any} err - * @param {string} [content] Content of the file, if known. - * @param {string} [filename] The filename of the erroring module. - */ -function enrichCJSError(err, content, filename) { - if (err != null && ObjectGetPrototypeOf(err) === SyntaxErrorPrototype && - containsModuleSyntax(content, filename)) { - // Emit the warning synchronously because we are in the middle of handling - // a SyntaxError that will throw and likely terminate the process before an - // asynchronous warning would be emitted. - emitWarningSync( - 'To load an ES module, set "type": "module" in the package.json or use ' + - 'the .mjs extension.', - ); - } -} - /** * Loads a CommonJS module via the ESM Loader sync CommonJS translator. * This translator creates its own version of the `require` function passed into CommonJS modules. @@ -197,15 +173,11 @@ function enrichCJSError(err, content, filename) { * @param {string} source - The source code of the module. * @param {string} url - The URL of the module. * @param {string} filename - The filename of the module. + * @param {boolean} isMain - Whether the module is the entrypoint */ -function loadCJSModule(module, source, url, filename) { - let compileResult; - try { - compileResult = compileFunctionForCJSLoader(source, filename); - } catch (err) { - enrichCJSError(err, source, filename); - throw err; - } +function loadCJSModule(module, source, url, filename, isMain) { + const compileResult = compileFunctionForCJSLoader(source, filename, isMain, false); + // Cache the source map for the cjs module if present. if (compileResult.sourceMapURL) { maybeCacheSourceMap(url, source, null, false, undefined, compileResult.sourceMapURL); @@ -283,7 +255,7 @@ function createCJSModuleWrap(url, source, isMain, loadCJS = loadCJSModule) { debug(`Loading CJSModule ${url}`); if (!module.loaded) { - loadCJS(module, source, url, filename); + loadCJS(module, source, url, filename, !!isMain); } let exports; @@ -315,9 +287,10 @@ translators.set('commonjs-sync', function requireCommonJS(url, source, isMain) { initCJSParseSync(); assert(!isMain); // This is only used by imported CJS modules. - return createCJSModuleWrap(url, source, isMain, (module, source, url, filename) => { + return createCJSModuleWrap(url, source, isMain, (module, source, url, filename, isMain) => { assert(module === CJSModule._cache[filename]); - CJSModule._load(filename); + assert(!isMain); + CJSModule._load(filename, null, isMain); }); }); @@ -340,14 +313,9 @@ translators.set('commonjs', async function commonjsStrategy(url, source, // For backward-compatibility, it's possible to return a nullish value for // CJS source associated with a file: URL. In this case, the source is // obtained by calling the monkey-patchable CJS loader. - const cjsLoader = source == null ? (module, source, url, filename) => { - try { - assert(module === CJSModule._cache[filename]); - CJSModule._load(filename); - } catch (err) { - enrichCJSError(err, source, filename); - throw err; - } + const cjsLoader = source == null ? (module, source, url, filename, isMain) => { + assert(module === CJSModule._cache[filename]); + CJSModule._load(filename, undefined, isMain); } : loadCJSModule; try { diff --git a/lib/internal/modules/run_main.js b/lib/internal/modules/run_main.js index 7a99c386fe6ded..7f6a1e9536d7d4 100644 --- a/lib/internal/modules/run_main.js +++ b/lib/internal/modules/run_main.js @@ -1,9 +1,7 @@ 'use strict'; const { - ObjectGetPrototypeOf, StringPrototypeEndsWith, - SyntaxErrorPrototype, } = primordials; const { getOptionValue } = require('internal/options'); @@ -160,35 +158,11 @@ function executeUserEntryPoint(main = process.argv[1]) { let mainURL; // Unless we know we should use the ESM loader to handle the entry point per the checks in `shouldUseESMLoader`, first // try to run the entry point via the CommonJS loader; and if that fails under certain conditions, retry as ESM. - let retryAsESM = false; if (!useESMLoader) { const cjsLoader = require('internal/modules/cjs/loader'); const { Module } = cjsLoader; - if (getOptionValue('--experimental-detect-module')) { - // TODO(joyeecheung): handle this in the CJS loader. Don't try-catch here. - try { - // Module._load is the monkey-patchable CJS module loader. - Module._load(main, null, true); - } catch (error) { - if (error != null && ObjectGetPrototypeOf(error) === SyntaxErrorPrototype) { - const { shouldRetryAsESM } = internalBinding('contextify'); - const mainPath = resolvedMain || main; - mainURL = pathToFileURL(mainPath).href; - retryAsESM = shouldRetryAsESM(error.message, cjsLoader.entryPointSource, mainURL); - // In case the entry point is a large file, such as a bundle, - // ensure no further references can prevent it being garbage-collected. - cjsLoader.entryPointSource = undefined; - } - if (!retryAsESM) { - throw error; - } - } - } else { // `--experimental-detect-module` is not passed - Module._load(main, null, true); - } - } - - if (useESMLoader || retryAsESM) { + Module._load(main, null, true); + } else { const mainPath = resolvedMain || main; if (mainURL === undefined) { mainURL = pathToFileURL(mainPath).href; diff --git a/src/node_contextify.cc b/src/node_contextify.cc index 7eb7335975e219..2d954c7570a6f5 100644 --- a/src/node_contextify.cc +++ b/src/node_contextify.cc @@ -28,8 +28,10 @@ #include "node_errors.h" #include "node_external_reference.h" #include "node_internals.h" +#include "node_process.h" #include "node_sea.h" #include "node_snapshot_builder.h" +#include "node_url.h" #include "node_watchdog.h" #include "util-inl.h" @@ -56,6 +58,7 @@ using v8::Maybe; using v8::MaybeLocal; using v8::MeasureMemoryExecution; using v8::MeasureMemoryMode; +using v8::Message; using v8::MicrotaskQueue; using v8::MicrotasksPolicy; using v8::Name; @@ -1504,50 +1507,109 @@ static MaybeLocal CompileFunctionForCJSLoader(Environment* env, return scope.Escape(fn); } +static bool warned_about_require_esm = false; +// TODO(joyeecheung): this was copied from the warning previously emitted in the +// JS land, but it's not very helpful. There should be specific information +// about which file or which package.json to update. +const char* require_esm_warning = + "To load an ES module, set \"type\": \"module\" in the package.json or use " + "the .mjs extension."; + +static bool ShouldRetryAsESM(Realm* realm, + Local message, + Local code, + Local resource_name); + static void CompileFunctionForCJSLoader( const FunctionCallbackInfo& args) { CHECK(args[0]->IsString()); CHECK(args[1]->IsString()); + CHECK(args[2]->IsBoolean()); + CHECK(args[3]->IsBoolean()); Local code = args[0].As(); Local filename = args[1].As(); + bool should_detect_module = args[3].As()->Value(); + Isolate* isolate = args.GetIsolate(); Local context = isolate->GetCurrentContext(); - Environment* env = Environment::GetCurrent(context); + Realm* realm = Realm::GetCurrent(context); + Environment* env = realm->env(); bool cache_rejected = false; Local fn; + Local cjs_exception; + Local cjs_message; + { + ShouldNotAbortOnUncaughtScope no_abort_scope(realm->env()); TryCatchScope try_catch(env); if (!CompileFunctionForCJSLoader( env, context, code, filename, &cache_rejected) .ToLocal(&fn)) { CHECK(try_catch.HasCaught()); CHECK(!try_catch.HasTerminated()); - errors::DecorateErrorStack(env, try_catch); - try_catch.ReThrow(); + cjs_exception = try_catch.Exception(); + cjs_message = try_catch.Message(); + errors::DecorateErrorStack(env, cjs_exception, cjs_message); + } + } + + bool can_parse_as_esm = false; + if (!cjs_exception.IsEmpty()) { + // Use the URL to match what would be used in the origin if it's going to + // be reparsed as ESM. + Utf8Value filename_utf8(isolate, filename); + std::string url = url::FromFilePath(filename_utf8.ToStringView()); + Local url_value; + if (!String::NewFromUtf8(isolate, url.c_str()).ToLocal(&url_value)) { + return; + } + can_parse_as_esm = + ShouldRetryAsESM(realm, cjs_message->Get(), code, url_value); + if (!can_parse_as_esm) { + // The syntax error is not related to ESM, throw the original error. + isolate->ThrowException(cjs_exception); + return; + } + + if (!should_detect_module) { + bool should_throw = true; + if (!warned_about_require_esm) { + // This needs to call process.emit('warning') in JS which can throw if + // the user listener throws. In that case, don't try to throw the syntax + // error. + should_throw = + ProcessEmitWarningSync(env, require_esm_warning).IsJust(); + } + if (should_throw) { + isolate->ThrowException(cjs_exception); + } return; } } + Local undefined = v8::Undefined(isolate); std::vector> names = { env->cached_data_rejected_string(), env->source_map_url_string(), env->function_string(), + FIXED_ONE_BYTE_STRING(isolate, "canParseAsESM"), }; std::vector> values = { Boolean::New(isolate, cache_rejected), - fn->GetScriptOrigin().SourceMapUrl(), - fn, + fn.IsEmpty() ? undefined : fn->GetScriptOrigin().SourceMapUrl(), + fn.IsEmpty() ? undefined : fn.As(), + Boolean::New(isolate, can_parse_as_esm), }; Local result = Object::New( isolate, v8::Null(isolate), names.data(), values.data(), names.size()); args.GetReturnValue().Set(result); } -static bool ShouldRetryAsESM(Realm* realm, - Local message, - Local code, - Local resource_name) { +bool ShouldRetryAsESM(Realm* realm, + Local message, + Local code, + Local resource_name) { Isolate* isolate = realm->isolate(); Utf8Value message_value(isolate, message); @@ -1581,7 +1643,7 @@ static bool ShouldRetryAsESM(Realm* realm, Local hdo = loader::ModuleWrap::GetHostDefinedOptions( isolate, realm->isolate_data()->source_text_module_default_hdo()); if (loader::ModuleWrap::CompileSourceTextModule( - realm, code, resource_name, 0, 0, hdo, nullptr, &cache_rejected) + realm, code, resource_name, 0, 0, hdo, std::nullopt, &cache_rejected) .ToLocal(&module)) { return true; } diff --git a/src/node_errors.cc b/src/node_errors.cc index 69e474257b0427..78d67504b40a25 100644 --- a/src/node_errors.cc +++ b/src/node_errors.cc @@ -1141,15 +1141,19 @@ void Initialize(Local target, void DecorateErrorStack(Environment* env, const errors::TryCatchScope& try_catch) { - Local exception = try_catch.Exception(); + DecorateErrorStack(env, try_catch.Exception(), try_catch.Message()); +} +void DecorateErrorStack(Environment* env, + Local exception, + Local message) { if (!exception->IsObject()) return; Local err_obj = exception.As(); if (IsExceptionDecorated(env, err_obj)) return; - AppendExceptionLine(env, exception, try_catch.Message(), CONTEXTIFY_ERROR); + AppendExceptionLine(env, exception, message, CONTEXTIFY_ERROR); TryCatchScope try_catch_scope(env); // Ignore exceptions below. MaybeLocal stack = err_obj->Get(env->context(), env->stack_string()); MaybeLocal maybe_value = diff --git a/src/node_errors.h b/src/node_errors.h index ac07b96b5cad0f..d5e2f86f516bbb 100644 --- a/src/node_errors.h +++ b/src/node_errors.h @@ -301,6 +301,9 @@ void PerIsolateMessageListener(v8::Local message, void DecorateErrorStack(Environment* env, const errors::TryCatchScope& try_catch); +void DecorateErrorStack(Environment* env, + v8::Local error, + v8::Local message); class PrinterTryCatch : public v8::TryCatch { public: diff --git a/test/es-module/test-require-module-detect-entry-point-aou.js b/test/es-module/test-require-module-detect-entry-point-aou.js new file mode 100644 index 00000000000000..e92d4d8273d708 --- /dev/null +++ b/test/es-module/test-require-module-detect-entry-point-aou.js @@ -0,0 +1,7 @@ +// Flags: --experimental-require-module --experimental-detect-module --abort-on-uncaught-exception + +import { mustCall } from '../common/index.mjs'; +const fn = mustCall(() => { + console.log('hello'); +}); +fn(); diff --git a/test/es-module/test-require-module-detect-entry-point.js b/test/es-module/test-require-module-detect-entry-point.js new file mode 100644 index 00000000000000..d7b479383fbeb8 --- /dev/null +++ b/test/es-module/test-require-module-detect-entry-point.js @@ -0,0 +1,7 @@ +// Flags: --experimental-require-module --experimental-detect-module + +import { mustCall } from '../common/index.mjs'; +const fn = mustCall(() => { + console.log('hello'); +}); +fn(); diff --git a/test/es-module/test-require-module-dont-detect-cjs.js b/test/es-module/test-require-module-dont-detect-cjs.js new file mode 100644 index 00000000000000..b4b5b7387d6663 --- /dev/null +++ b/test/es-module/test-require-module-dont-detect-cjs.js @@ -0,0 +1,11 @@ +// Flags: --experimental-require-module --experimental-detect-module +'use strict'; + +require('../common'); +const assert = require('assert'); + +assert.throws(() => { + require('../fixtures/es-modules/es-note-unexpected-export-1.cjs'); +}, { + message: /Unexpected token 'export'/ +}); diff --git a/test/es-module/test-require-module-with-detection.js b/test/es-module/test-require-module-with-detection.js new file mode 100644 index 00000000000000..36da19f3b96270 --- /dev/null +++ b/test/es-module/test-require-module-with-detection.js @@ -0,0 +1,18 @@ +// Flags: --experimental-require-module --experimental-detect-module +'use strict'; + +require('../common'); +const assert = require('assert'); +const { isModuleNamespaceObject } = require('util/types'); + +{ + const mod = require('../fixtures/es-modules/loose.js'); + assert.deepStrictEqual({ ...mod }, { default: 'module' }); + assert(isModuleNamespaceObject(mod)); +} + +{ + const mod = require('../fixtures/es-modules/package-without-type/noext-esm'); + assert.deepStrictEqual({ ...mod }, { default: 'module' }); + assert(isModuleNamespaceObject(mod)); +} From b6268a24ef20a9c49a6da6ea18567f41f06405e9 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Tue, 7 May 2024 08:12:16 +0800 Subject: [PATCH 05/33] module: cache synchronous module jobs before linking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit So that if there are circular dependencies in the synchronous module graph, they could be resolved using the cached jobs. In case linking fails and the error gets caught, reset the cache right after linking. If it succeeds, the caller will cache it again. Otherwise the error bubbles up to users, and since we unset the cache for the unlinkable module the next attempt would still fail. PR-URL: https://github.com/nodejs/node/pull/52868 Fixes: https://github.com/nodejs/node/issues/52864 Reviewed-By: Moshe Atlow Reviewed-By: Vinícius Lourenço Claro Cardoso Reviewed-By: Antoine du Hamel Reviewed-By: Chengzhong Wu --- lib/internal/modules/esm/module_job.js | 39 ++++++++++++------- lib/internal/modules/esm/module_map.js | 6 +++ .../test-require-module-cycle-cjs-esm-esm.js | 8 ++++ .../es-modules/cjs-esm-esm-cycle/a.mjs | 1 + .../es-modules/cjs-esm-esm-cycle/b.mjs | 2 + .../es-modules/cjs-esm-esm-cycle/c.cjs | 1 + 6 files changed, 43 insertions(+), 14 deletions(-) create mode 100644 test/es-module/test-require-module-cycle-cjs-esm-esm.js create mode 100644 test/fixtures/es-modules/cjs-esm-esm-cycle/a.mjs create mode 100644 test/fixtures/es-modules/cjs-esm-esm-cycle/b.mjs create mode 100644 test/fixtures/es-modules/cjs-esm-esm-cycle/c.cjs diff --git a/lib/internal/modules/esm/module_job.js b/lib/internal/modules/esm/module_job.js index f752e9ed1d35f4..abdd96673f72b3 100644 --- a/lib/internal/modules/esm/module_job.js +++ b/lib/internal/modules/esm/module_job.js @@ -287,22 +287,33 @@ class ModuleJobSync extends ModuleJobBase { #loader = null; constructor(loader, url, importAttributes, moduleWrap, isMain, inspectBrk) { super(url, importAttributes, moduleWrap, isMain, inspectBrk, true); - assert(this.module instanceof ModuleWrap); this.#loader = loader; - const moduleRequests = this.module.getModuleRequests(); - // Specifiers should be aligned with the moduleRequests array in order. - const specifiers = Array(moduleRequests.length); - const modules = Array(moduleRequests.length); - const jobs = Array(moduleRequests.length); - for (let i = 0; i < moduleRequests.length; ++i) { - const { specifier, attributes } = moduleRequests[i]; - const job = this.#loader.getModuleJobForRequire(specifier, url, attributes); - specifiers[i] = specifier; - modules[i] = job.module; - jobs[i] = job; + + assert(this.module instanceof ModuleWrap); + // Store itself into the cache first before linking in case there are circular + // references in the linking. + loader.loadCache.set(url, importAttributes.type, this); + + try { + const moduleRequests = this.module.getModuleRequests(); + // Specifiers should be aligned with the moduleRequests array in order. + const specifiers = Array(moduleRequests.length); + const modules = Array(moduleRequests.length); + const jobs = Array(moduleRequests.length); + for (let i = 0; i < moduleRequests.length; ++i) { + const { specifier, attributes } = moduleRequests[i]; + const job = this.#loader.getModuleJobForRequire(specifier, url, attributes); + specifiers[i] = specifier; + modules[i] = job.module; + jobs[i] = job; + } + this.module.link(specifiers, modules); + this.linked = jobs; + } finally { + // Restore it - if it succeeds, we'll reset in the caller; Otherwise it's + // not cached and if the error is caught, subsequent attempt would still fail. + loader.loadCache.delete(url, importAttributes.type); } - this.module.link(specifiers, modules); - this.linked = jobs; } get modulePromise() { diff --git a/lib/internal/modules/esm/module_map.js b/lib/internal/modules/esm/module_map.js index ab1171eaa47b02..247bde93cabd70 100644 --- a/lib/internal/modules/esm/module_map.js +++ b/lib/internal/modules/esm/module_map.js @@ -114,6 +114,12 @@ class LoadCache extends SafeMap { validateString(type, 'type'); return super.get(url)?.[type] !== undefined; } + delete(url, type = kImplicitAssertType) { + const cached = super.get(url); + if (cached) { + cached[type] = undefined; + } + } } module.exports = { diff --git a/test/es-module/test-require-module-cycle-cjs-esm-esm.js b/test/es-module/test-require-module-cycle-cjs-esm-esm.js new file mode 100644 index 00000000000000..a83d7ee7a71bb2 --- /dev/null +++ b/test/es-module/test-require-module-cycle-cjs-esm-esm.js @@ -0,0 +1,8 @@ +// Flags: --experimental-require-module +'use strict'; + +// This tests that ESM <-> ESM cycle is allowed in a require()-d graph. +const common = require('../common'); +const cycle = require('../fixtures/es-modules/cjs-esm-esm-cycle/c.cjs'); + +common.expectRequiredModule(cycle, { b: 5 }); diff --git a/test/fixtures/es-modules/cjs-esm-esm-cycle/a.mjs b/test/fixtures/es-modules/cjs-esm-esm-cycle/a.mjs new file mode 100644 index 00000000000000..798e86506da2a9 --- /dev/null +++ b/test/fixtures/es-modules/cjs-esm-esm-cycle/a.mjs @@ -0,0 +1 @@ +export { b } from './b.mjs'; diff --git a/test/fixtures/es-modules/cjs-esm-esm-cycle/b.mjs b/test/fixtures/es-modules/cjs-esm-esm-cycle/b.mjs new file mode 100644 index 00000000000000..9e909cd6856455 --- /dev/null +++ b/test/fixtures/es-modules/cjs-esm-esm-cycle/b.mjs @@ -0,0 +1,2 @@ +import './a.mjs' +export const b = 5; diff --git a/test/fixtures/es-modules/cjs-esm-esm-cycle/c.cjs b/test/fixtures/es-modules/cjs-esm-esm-cycle/c.cjs new file mode 100644 index 00000000000000..f9361ecd59d11e --- /dev/null +++ b/test/fixtures/es-modules/cjs-esm-esm-cycle/c.cjs @@ -0,0 +1 @@ +module.exports = require('./a.mjs'); From 1af45259dbd7260a94387642748819c120ef78c1 Mon Sep 17 00:00:00 2001 From: Antoine du Hamel Date: Sat, 25 May 2024 05:51:47 +0300 Subject: [PATCH 06/33] module: do not set CJS variables for Worker eval PR-URL: https://github.com/nodejs/node/pull/53050 Reviewed-By: Geoffrey Booth Reviewed-By: James M Snell --- lib/internal/process/execution.js | 2 +- src/node_contextify.cc | 18 +++++++++---- test/es-module/test-esm-detect-ambiguous.mjs | 28 ++++++++++++++++++++ 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/lib/internal/process/execution.js b/lib/internal/process/execution.js index 4f72aa41251836..09b18b8f2c37db 100644 --- a/lib/internal/process/execution.js +++ b/lib/internal/process/execution.js @@ -83,7 +83,7 @@ function evalScript(name, body, breakFirstLine, print, shouldLoadESM = false) { if (getOptionValue('--experimental-detect-module') && getOptionValue('--input-type') === '' && getOptionValue('--experimental-default-type') === '' && - containsModuleSyntax(body, name)) { + containsModuleSyntax(body, name, null, 'no CJS variables')) { return evalModuleEntryPoint(body, print); } diff --git a/src/node_contextify.cc b/src/node_contextify.cc index 2d954c7570a6f5..471cd017567fd9 100644 --- a/src/node_contextify.cc +++ b/src/node_contextify.cc @@ -1445,7 +1445,8 @@ static MaybeLocal CompileFunctionForCJSLoader(Environment* env, Local context, Local code, Local filename, - bool* cache_rejected) { + bool* cache_rejected, + bool is_cjs_scope) { Isolate* isolate = context->GetIsolate(); EscapableHandleScope scope(isolate); @@ -1485,7 +1486,10 @@ static MaybeLocal CompileFunctionForCJSLoader(Environment* env, options = ScriptCompiler::kConsumeCodeCache; } - std::vector> params = GetCJSParameters(env->isolate_data()); + std::vector> params; + if (is_cjs_scope) { + params = GetCJSParameters(env->isolate_data()); + } MaybeLocal maybe_fn = ScriptCompiler::CompileFunction( context, &source, @@ -1544,7 +1548,7 @@ static void CompileFunctionForCJSLoader( ShouldNotAbortOnUncaughtScope no_abort_scope(realm->env()); TryCatchScope try_catch(env); if (!CompileFunctionForCJSLoader( - env, context, code, filename, &cache_rejected) + env, context, code, filename, &cache_rejected, true) .ToLocal(&fn)) { CHECK(try_catch.HasCaught()); CHECK(!try_catch.HasTerminated()); @@ -1682,11 +1686,15 @@ static void ContainsModuleSyntax(const FunctionCallbackInfo& args) { CHECK(args[1]->IsString()); Local filename = args[1].As(); - // Argument 2: resource name (URL for ES module). + // Argument 3: resource name (URL for ES module). Local resource_name = filename; if (args[2]->IsString()) { resource_name = args[2].As(); } + // Argument 4: flag to indicate if CJS variables should not be in scope + // (they should be for normal CommonJS modules, but not for the + // CommonJS eval scope). + bool cjs_var = !args[3]->IsString(); bool cache_rejected = false; Local message; @@ -1695,7 +1703,7 @@ static void ContainsModuleSyntax(const FunctionCallbackInfo& args) { TryCatchScope try_catch(env); ShouldNotAbortOnUncaughtScope no_abort_scope(env); if (CompileFunctionForCJSLoader( - env, context, code, filename, &cache_rejected) + env, context, code, filename, &cache_rejected, cjs_var) .ToLocal(&fn)) { args.GetReturnValue().Set(false); return; diff --git a/test/es-module/test-esm-detect-ambiguous.mjs b/test/es-module/test-esm-detect-ambiguous.mjs index c027f46328acee..8c70542b2ead72 100644 --- a/test/es-module/test-esm-detect-ambiguous.mjs +++ b/test/es-module/test-esm-detect-ambiguous.mjs @@ -44,6 +44,19 @@ describe('--experimental-detect-module', { concurrency: true }, () => { strictEqual(signal, null); }); + it('should not switch to module if code is parsable as script', async () => { + const { code, signal, stdout, stderr } = await spawnPromisified(process.execPath, [ + '--experimental-detect-module', + '--eval', + 'let __filename,__dirname,require,module,exports;this.a', + ]); + + strictEqual(stderr, ''); + strictEqual(stdout, ''); + strictEqual(code, 0); + strictEqual(signal, null); + }); + it('should be overridden by --experimental-default-type', async () => { const { code, signal, stdout, stderr } = await spawnPromisified(process.execPath, [ '--experimental-detect-module', @@ -393,3 +406,18 @@ describe('Wrapping a `require` of an ES module while using `--abort-on-uncaught- strictEqual(signal, null); }); }); + +describe('when working with Worker threads', () => { + it('should support sloppy scripts that declare CJS "global-like" variables', async () => { + const { code, signal, stdout, stderr } = await spawnPromisified(process.execPath, [ + '--experimental-detect-module', + '--eval', + 'new worker_threads.Worker("let __filename,__dirname,require,module,exports;this.a",{eval:true})', + ]); + + strictEqual(stderr, ''); + strictEqual(stdout, ''); + strictEqual(code, 0); + strictEqual(signal, null); + }); +}); From 9bc8eabd3b06be83c54f8f733467d2d4b742bc42 Mon Sep 17 00:00:00 2001 From: Chengzhong Wu Date: Tue, 28 May 2024 23:56:56 +0100 Subject: [PATCH 07/33] lib: allow CJS source map cache to be reclaimed Unifies the CJS and ESM source map cache map with SourceMapCacheMap and allows the CJS cache entries to be queried more efficiently with a source url without iteration on an IterableWeakMap. Add a test to verify that the CJS source map cache entry can be reclaimed. PR-URL: https://github.com/nodejs/node/pull/51711 Reviewed-By: Joyee Cheung Reviewed-By: Antoine du Hamel --- lib/internal/modules/cjs/loader.js | 4 +- lib/internal/modules/esm/translators.js | 7 +- lib/internal/modules/esm/utils.js | 2 +- .../source_map/prepare_stack_trace.js | 25 ++- lib/internal/source_map/source_map_cache.js | 164 ++++++++++-------- .../source_map/source_map_cache_map.js | 115 ++++++++++++ lib/internal/util/iterable_weak_map.js | 84 --------- src/env_properties.h | 3 +- src/module_wrap.cc | 8 + test/fixtures/source-map/no-throw.js | 34 ++++ test/fixtures/source-map/no-throw.ts | 42 +++++ test/fixtures/source-map/no-throw2.js | 35 ++++ .../output/source_map_disabled_by_api.js | 4 +- .../output/source_map_prepare_stack_trace.js | 6 +- .../test-internal-iterable-weak-map.js | 101 ----------- .../test-source-map-cjs-require-cache.js | 37 ++++ 16 files changed, 394 insertions(+), 277 deletions(-) create mode 100644 lib/internal/source_map/source_map_cache_map.js delete mode 100644 lib/internal/util/iterable_weak_map.js create mode 100644 test/fixtures/source-map/no-throw.js create mode 100644 test/fixtures/source-map/no-throw.ts create mode 100644 test/fixtures/source-map/no-throw2.js delete mode 100644 test/parallel/test-internal-iterable-weak-map.js create mode 100644 test/parallel/test-source-map-cjs-require-cache.js diff --git a/lib/internal/modules/cjs/loader.js b/lib/internal/modules/cjs/loader.js index 2346b39e5abe87..32cc78112a786d 100644 --- a/lib/internal/modules/cjs/loader.js +++ b/lib/internal/modules/cjs/loader.js @@ -1375,7 +1375,7 @@ function wrapSafe(filename, content, cjsModuleInstance, codeCache, format) { // Cache the source map for the module if present. const { sourceMapURL } = script; if (sourceMapURL) { - maybeCacheSourceMap(filename, content, this, false, undefined, sourceMapURL); + maybeCacheSourceMap(filename, content, cjsModuleInstance, false, undefined, sourceMapURL); } return { @@ -1398,7 +1398,7 @@ function wrapSafe(filename, content, cjsModuleInstance, codeCache, format) { // Cache the source map for the module if present. if (result.sourceMapURL) { - maybeCacheSourceMap(filename, content, this, false, undefined, result.sourceMapURL); + maybeCacheSourceMap(filename, content, cjsModuleInstance, false, undefined, result.sourceMapURL); } return result; diff --git a/lib/internal/modules/esm/translators.js b/lib/internal/modules/esm/translators.js index 52f73368724df4..8e33d71b17fe5e 100644 --- a/lib/internal/modules/esm/translators.js +++ b/lib/internal/modules/esm/translators.js @@ -178,13 +178,12 @@ translators.set('module', function moduleStrategy(url, source, isMain) { function loadCJSModule(module, source, url, filename, isMain) { const compileResult = compileFunctionForCJSLoader(source, filename, isMain, false); + const { function: compiledWrapper, sourceMapURL } = compileResult; // Cache the source map for the cjs module if present. - if (compileResult.sourceMapURL) { - maybeCacheSourceMap(url, source, null, false, undefined, compileResult.sourceMapURL); + if (sourceMapURL) { + maybeCacheSourceMap(url, source, module, false, undefined, sourceMapURL); } - const compiledWrapper = compileResult.function; - const cascadedLoader = require('internal/modules/esm/loader').getOrInitializeCascadedLoader(); const __dirname = dirname(filename); // eslint-disable-next-line func-name-matching,func-style diff --git a/lib/internal/modules/esm/utils.js b/lib/internal/modules/esm/utils.js index 150816057129c1..dfa32f493b262e 100644 --- a/lib/internal/modules/esm/utils.js +++ b/lib/internal/modules/esm/utils.js @@ -343,7 +343,7 @@ function compileSourceTextModule(url, source, cascadedLoader) { } // Cache the source map for the module if present. if (wrap.sourceMapURL) { - maybeCacheSourceMap(url, source, null, false, undefined, wrap.sourceMapURL); + maybeCacheSourceMap(url, source, wrap, false, undefined, wrap.sourceMapURL); } return wrap; } diff --git a/lib/internal/source_map/prepare_stack_trace.js b/lib/internal/source_map/prepare_stack_trace.js index 7f99303b034e94..f06ea6380ceee4 100644 --- a/lib/internal/source_map/prepare_stack_trace.js +++ b/lib/internal/source_map/prepare_stack_trace.js @@ -139,9 +139,15 @@ function getOriginalSymbolName(sourceMap, callSite, callerCallSite) { } } -// Places a snippet of code from where the exception was originally thrown -// above the stack trace. This logic is modeled after GetErrorSource in -// node_errors.cc. +/** + * Return a snippet of code from where the exception was originally thrown + * above the stack trace. This called from GetErrorSource in node_errors.cc. + * @param {import('internal/source_map/source_map').SourceMap} sourceMap - the source map to be used + * @param {string} originalSourcePath - path or url of the original source + * @param {number} originalLine - line number in the original source + * @param {number} originalColumn - column number in the original source + * @returns {string | undefined} - the exact line in the source content or undefined if file not found + */ function getErrorSource( sourceMap, originalSourcePath, @@ -179,6 +185,12 @@ function getErrorSource( return exceptionLine; } +/** + * Retrieve the original source code from the source map's `sources` list or disk. + * @param {import('internal/source_map/source_map').SourceMap.payload} payload + * @param {string} originalSourcePath - path or url of the original source + * @returns {string | undefined} - the source content or undefined if file not found + */ function getOriginalSource(payload, originalSourcePath) { let source; // payload.sources has been normalized to be an array of absolute urls. @@ -202,6 +214,13 @@ function getOriginalSource(payload, originalSourcePath) { return source; } +/** + * Retrieve exact line in the original source code from the source map's `sources` list or disk. + * @param {string} fileName - actual file name + * @param {number} lineNumber - actual line number + * @param {number} columnNumber - actual column number + * @returns {string | undefined} - the source content or undefined if file not found + */ function getSourceMapErrorSource(fileName, lineNumber, columnNumber) { const sm = findSourceMap(fileName); if (sm === undefined) { diff --git a/lib/internal/source_map/source_map_cache.js b/lib/internal/source_map/source_map_cache.js index 53c3374fc09176..f21b3719ad806a 100644 --- a/lib/internal/source_map/source_map_cache.js +++ b/lib/internal/source_map/source_map_cache.js @@ -3,7 +3,6 @@ const { ArrayPrototypePush, JSONParse, - ObjectKeys, RegExpPrototypeExec, SafeMap, StringPrototypeCodePointAt, @@ -25,17 +24,15 @@ const { } = require('internal/errors'); const { getLazy } = require('internal/util'); -// Since the CJS module cache is mutable, which leads to memory leaks when -// modules are deleted, we use a WeakMap so that the source map cache will -// be purged automatically: -const getCjsSourceMapCache = getLazy(() => { - const { IterableWeakMap } = require('internal/util/iterable_weak_map'); - return new IterableWeakMap(); +const getModuleSourceMapCache = getLazy(() => { + const { SourceMapCacheMap } = require('internal/source_map/source_map_cache_map'); + return new SourceMapCacheMap(); }); -// The esm cache is not mutable, so we can use a Map without memory concerns: -const esmSourceMapCache = new SafeMap(); -// The generated sources is not mutable, so we can use a Map without memory concerns: +// The generated source module/script instance is not accessible, so we can use +// a Map without memory concerns. Separate generated source entries with the module +// source entries to avoid overriding the module source entries with arbitrary +// source url magic comments. const generatedSourceMapCache = new SafeMap(); const kLeadingProtocol = /^\w+:\/\//; const kSourceMappingURLMagicComment = /\/[*/]#\s+sourceMappingURL=(?[^\s]+)/g; @@ -52,6 +49,10 @@ function getSourceMapsEnabled() { return sourceMapsEnabled; } +/** + * Enables or disables source maps programmatically. + * @param {boolean} val + */ function setSourceMapsEnabled(val) { validateBoolean(val, 'val'); @@ -72,6 +73,14 @@ function setSourceMapsEnabled(val) { sourceMapsEnabled = val; } +/** + * Extracts the source url from the content if present. For example + * //# sourceURL=file:///path/to/file + * + * Read more at: https://tc39.es/source-map-spec/#linking-evald-code-to-named-generated-code + * @param {string} content - source content + * @returns {string | null} source url or null if not present + */ function extractSourceURLMagicComment(content) { let match; let matchSourceURL; @@ -90,6 +99,14 @@ function extractSourceURLMagicComment(content) { return sourceURL; } +/** + * Extracts the source map url from the content if present. For example + * //# sourceMappingURL=file:///path/to/file + * + * Read more at: https://tc39.es/source-map-spec/#linking-generated-code + * @param {string} content - source content + * @returns {string | null} source map url or null if not present + */ function extractSourceMapURLMagicComment(content) { let match; let lastMatch; @@ -104,7 +121,17 @@ function extractSourceMapURLMagicComment(content) { return lastMatch.groups.sourceMappingURL; } -function maybeCacheSourceMap(filename, content, cjsModuleInstance, isGeneratedSource, sourceURL, sourceMapURL) { +/** + * Caches the source map if it is present in the content, with the given filename, moduleInstance, and sourceURL. + * @param {string} filename - the actual filename + * @param {string} content - the actual source content + * @param {import('internal/modules/cjs/loader').Module | ModuleWrap} moduleInstance - a module instance that + * associated with the source, once this is reclaimed, the source map entry will be removed from the cache + * @param {boolean} isGeneratedSource - if the source was generated and evaluated with the global eval + * @param {string | undefined} sourceURL - the source url + * @param {string | undefined} sourceMapURL - the source map url + */ +function maybeCacheSourceMap(filename, content, moduleInstance, isGeneratedSource, sourceURL, sourceMapURL) { const sourceMapsEnabled = getSourceMapsEnabled(); if (!(process.env.NODE_V8_COVERAGE || sourceMapsEnabled)) return; const { normalizeReferrerURL } = require('internal/modules/helpers'); @@ -130,45 +157,32 @@ function maybeCacheSourceMap(filename, content, cjsModuleInstance, isGeneratedSo } const data = dataFromUrl(filename, sourceMapURL); - const url = data ? null : sourceMapURL; - if (cjsModuleInstance) { - getCjsSourceMapCache().set(cjsModuleInstance, { - __proto__: null, - filename, - lineLengths: lineLengths(content), - data, - url, - sourceURL, - }); - } else if (isGeneratedSource) { - const entry = { - __proto__: null, - lineLengths: lineLengths(content), - data, - url, - sourceURL, - }; + const entry = { + __proto__: null, + lineLengths: lineLengths(content), + data, + // Save the source map url if it is not a data url. + sourceMapURL: data ? null : sourceMapURL, + sourceURL, + }; + + if (isGeneratedSource) { generatedSourceMapCache.set(filename, entry); if (sourceURL) { generatedSourceMapCache.set(sourceURL, entry); } - } else { - // If there is no cjsModuleInstance and is not generated source assume we are in a - // "modules/esm" context. - const entry = { - __proto__: null, - lineLengths: lineLengths(content), - data, - url, - sourceURL, - }; - esmSourceMapCache.set(filename, entry); - if (sourceURL) { - esmSourceMapCache.set(sourceURL, entry); - } + return; } + // If it is not a generated source, we assume we are in a "cjs/esm" + // context. + const keys = sourceURL ? [filename, sourceURL] : [filename]; + getModuleSourceMapCache().set(keys, entry, moduleInstance); } +/** + * Caches the source map if it is present in the eval'd source. + * @param {string} content - the eval'd source code + */ function maybeCacheGeneratedSourceMap(content) { const sourceMapsEnabled = getSourceMapsEnabled(); if (!(process.env.NODE_V8_COVERAGE || sourceMapsEnabled)) return; @@ -186,6 +200,14 @@ function maybeCacheGeneratedSourceMap(content) { } } +/** + * Resolves source map payload data from the source url and source map url. + * If the source map url is a data url, the data is returned. + * Otherwise the source map url is resolved to a file path and the file is read. + * @param {string} sourceURL - url of the source file + * @param {string} sourceMappingURL - url of the source map + * @returns {object} deserialized source map JSON object + */ function dataFromUrl(sourceURL, sourceMappingURL) { try { const url = new URL(sourceMappingURL); @@ -227,7 +249,11 @@ function lineLengths(content) { return output; } - +/** + * Read source map from file. + * @param {string} mapURL - file url of the source map + * @returns {object} deserialized source map JSON object + */ function sourceMapFromFile(mapURL) { try { const fs = require('fs'); @@ -281,39 +307,36 @@ function sourcesToAbsolute(baseURL, data) { return data; } -// WARNING: The `sourceMapCacheToObject` and `appendCJSCache` run during -// shutdown. In particular, they also run when Workers are terminated, making -// it important that they do not call out to any user-provided code, including -// built-in prototypes that might have been tampered with. +// WARNING: The `sourceMapCacheToObject` runs during shutdown. In particular, +// it also runs when Workers are terminated, making it important that it does +// not call out to any user-provided code, including built-in prototypes that +// might have been tampered with. // Get serialized representation of source-map cache, this is used // to persist a cache of source-maps to disk when NODE_V8_COVERAGE is enabled. function sourceMapCacheToObject() { - const obj = { __proto__: null }; - - for (const { 0: k, 1: v } of esmSourceMapCache) { - obj[k] = v; - } - - appendCJSCache(obj); - - if (ObjectKeys(obj).length === 0) { + const moduleSourceMapCache = getModuleSourceMapCache(); + if (moduleSourceMapCache.size === 0) { return undefined; } - return obj; -} -function appendCJSCache(obj) { - for (const value of getCjsSourceMapCache()) { - obj[value.filename] = { + const obj = { __proto__: null }; + for (const { 0: k, 1: v } of moduleSourceMapCache) { + obj[k] = { __proto__: null, - lineLengths: value.lineLengths, - data: value.data, - url: value.url, + lineLengths: v.lineLengths, + data: v.data, + url: v.sourceMapURL, }; } + return obj; } +/** + * Find a source map for a given actual source URL or path. + * @param {string} sourceURL - actual source URL or path + * @returns {import('internal/source_map/source_map').SourceMap | undefined} a source map or undefined if not found + */ function findSourceMap(sourceURL) { if (RegExpPrototypeExec(kLeadingProtocol, sourceURL) === null) { sourceURL = pathToFileURL(sourceURL).href; @@ -321,16 +344,7 @@ function findSourceMap(sourceURL) { if (!SourceMap) { SourceMap = require('internal/source_map/source_map').SourceMap; } - let entry = esmSourceMapCache.get(sourceURL) ?? generatedSourceMapCache.get(sourceURL); - if (entry === undefined) { - for (const value of getCjsSourceMapCache()) { - const filename = value.filename; - const cachedSourceURL = value.sourceURL; - if (sourceURL === filename || sourceURL === cachedSourceURL) { - entry = value; - } - } - } + const entry = getModuleSourceMapCache().get(sourceURL) ?? generatedSourceMapCache.get(sourceURL); if (entry === undefined) { return undefined; } diff --git a/lib/internal/source_map/source_map_cache_map.js b/lib/internal/source_map/source_map_cache_map.js new file mode 100644 index 00000000000000..e8adfe83708316 --- /dev/null +++ b/lib/internal/source_map/source_map_cache_map.js @@ -0,0 +1,115 @@ +'use strict'; + +const { + ArrayPrototypeForEach, + ObjectFreeze, + SafeFinalizationRegistry, + SafeMap, + SafeWeakRef, + SymbolIterator, +} = primordials; +const { + privateSymbols: { + source_map_data_private_symbol, + }, +} = internalBinding('util'); + +/** + * Specialized map of WeakRefs to module instances that caches source map + * entries by `filename` and `sourceURL`. Cached entries can be iterated with + * `for..of` syntax. + * + * The cache map maintains the cache entries by: + * - `weakModuleMap`(Map): a strong sourceURL -> WeakRef(Module), + * - WeakRef(Module[source_map_data_private_symbol]): source map data. + * + * Obsolete `weakModuleMap` entries are removed by the `finalizationRegistry` + * callback. This pattern decouples the strong url reference to the source map + * data and allow the cache to be reclaimed eagerly, without depending on an + * undeterministic callback of a finalization registry. + */ +class SourceMapCacheMap { + /** + * @type {Map>} + * The cached module instance can be removed from the global module registry + * with approaches like mutating `require.cache`. + * The `weakModuleMap` exposes entries by `filename` and `sourceURL`. + * In the case of mutated module registry, obsolete entries are removed from + * the cache by the `finalizationRegistry`. + */ + #weakModuleMap = new SafeMap(); + + #cleanup = ({ keys }) => { + // Delete the entry if the weak target has been reclaimed. + // If the weak target is not reclaimed, the entry was overridden by a new + // weak target. + ArrayPrototypeForEach(keys, (key) => { + const ref = this.#weakModuleMap.get(key); + if (ref && ref.deref() === undefined) { + this.#weakModuleMap.delete(key); + } + }); + }; + #finalizationRegistry = new SafeFinalizationRegistry(this.#cleanup); + + /** + * Sets the value for the given key, associated with the given module + * instance. + * @param {string[]} keys array of urls to index the value entry. + * @param {*} sourceMapData the value entry. + * @param {object} moduleInstance an object that can be weakly referenced and + * invalidate the [key, value] entry after this object is reclaimed. + */ + set(keys, sourceMapData, moduleInstance) { + const weakRef = new SafeWeakRef(moduleInstance); + ArrayPrototypeForEach(keys, (key) => this.#weakModuleMap.set(key, weakRef)); + moduleInstance[source_map_data_private_symbol] = sourceMapData; + this.#finalizationRegistry.register(moduleInstance, { keys }); + } + + /** + * Get an entry by the given key. + * @param {string} key a file url or source url + */ + get(key) { + const weakRef = this.#weakModuleMap.get(key); + const moduleInstance = weakRef?.deref(); + if (moduleInstance === undefined) { + return; + } + return moduleInstance[source_map_data_private_symbol]; + } + + /** + * Estimate the size of the cache. The actual size may be smaller because + * some entries may be reclaimed with the module instance. + */ + get size() { + return this.#weakModuleMap.size; + } + + [SymbolIterator]() { + const iterator = this.#weakModuleMap.entries(); + + const next = () => { + const result = iterator.next(); + if (result.done) return result; + const { 0: key, 1: weakRef } = result.value; + const moduleInstance = weakRef.deref(); + if (moduleInstance == null) return next(); + const value = moduleInstance[source_map_data_private_symbol]; + return { done: false, value: [key, value] }; + }; + + return { + [SymbolIterator]() { return this; }, + next, + }; + } +} + +ObjectFreeze(SourceMapCacheMap.prototype); + +module.exports = { + SourceMapCacheMap, +}; diff --git a/lib/internal/util/iterable_weak_map.js b/lib/internal/util/iterable_weak_map.js deleted file mode 100644 index 16694ffdb11de8..00000000000000 --- a/lib/internal/util/iterable_weak_map.js +++ /dev/null @@ -1,84 +0,0 @@ -'use strict'; - -const { - ObjectFreeze, - SafeFinalizationRegistry, - SafeSet, - SafeWeakMap, - SafeWeakRef, - SymbolIterator, -} = primordials; - -// This class is modified from the example code in the WeakRefs specification: -// https://github.com/tc39/proposal-weakrefs -// Licensed under ECMA's MIT-style license, see: -// https://github.com/tc39/ecma262/blob/HEAD/LICENSE.md -class IterableWeakMap { - #weakMap = new SafeWeakMap(); - #refSet = new SafeSet(); - #finalizationGroup = new SafeFinalizationRegistry(cleanup); - - set(key, value) { - const entry = this.#weakMap.get(key); - if (entry) { - // If there's already an entry for the object represented by "key", - // the value can be updated without creating a new WeakRef: - this.#weakMap.set(key, { value, ref: entry.ref }); - } else { - const ref = new SafeWeakRef(key); - this.#weakMap.set(key, { value, ref }); - this.#refSet.add(ref); - this.#finalizationGroup.register(key, { - set: this.#refSet, - ref, - }, ref); - } - } - - get(key) { - return this.#weakMap.get(key)?.value; - } - - has(key) { - return this.#weakMap.has(key); - } - - delete(key) { - const entry = this.#weakMap.get(key); - if (!entry) { - return false; - } - this.#weakMap.delete(key); - this.#refSet.delete(entry.ref); - this.#finalizationGroup.unregister(entry.ref); - return true; - } - - [SymbolIterator]() { - const iterator = this.#refSet[SymbolIterator](); - - const next = () => { - const result = iterator.next(); - if (result.done) return result; - const key = result.value.deref(); - if (key == null) return next(); - const { value } = this.#weakMap.get(key); - return { done: false, value }; - }; - - return { - [SymbolIterator]() { return this; }, - next, - }; - } -} - -function cleanup({ set, ref }) { - set.delete(ref); -} - -ObjectFreeze(IterableWeakMap.prototype); - -module.exports = { - IterableWeakMap, -}; diff --git a/src/env_properties.h b/src/env_properties.h index f0813da6708762..63fea9e479953d 100644 --- a/src/env_properties.h +++ b/src/env_properties.h @@ -32,7 +32,8 @@ V(untransferable_object_private_symbol, "node:untransferableObject") \ V(exit_info_private_symbol, "node:exit_info_private_symbol") \ V(promise_trace_id, "node:promise_trace_id") \ - V(require_private_symbol, "node:require_private_symbol") + V(require_private_symbol, "node:require_private_symbol") \ + V(source_map_data_private_symbol, "node:source_map_data_private_symbol") // Symbols are per-isolate primitives but Environment proxies them // for the sake of convenience. diff --git a/src/module_wrap.cc b/src/module_wrap.cc index 13f7416ff2a9d5..d7ec99736473b6 100644 --- a/src/module_wrap.cc +++ b/src/module_wrap.cc @@ -270,6 +270,14 @@ void ModuleWrap::New(const FunctionCallbackInfo& args) { return; } + // Initialize an empty slot for source map cache before the object is frozen. + if (that->SetPrivate(context, + realm->isolate_data()->source_map_data_private_symbol(), + Undefined(isolate)) + .IsNothing()) { + return; + } + // Use the extras object as an object whose GetCreationContext() will be the // original `context`, since the `Context` itself strictly speaking cannot // be stored in an internal field. diff --git a/test/fixtures/source-map/no-throw.js b/test/fixtures/source-map/no-throw.js new file mode 100644 index 00000000000000..4c4e5fa47158da --- /dev/null +++ b/test/fixtures/source-map/no-throw.js @@ -0,0 +1,34 @@ +var Foo = /** @class */ (function () { + function Foo(x) { + if (x === void 0) { x = 33; } + this.x = x ? x : 99; + if (this.x) { + this.methodA(); + } + else { + this.methodB(); + } + this.methodC(); + } + Foo.prototype.methodA = function () { + }; + Foo.prototype.methodB = function () { + }; + Foo.prototype.methodC = function () { + }; + Foo.prototype.methodD = function () { + }; + return Foo; +}()); +var a = new Foo(0); +var b = new Foo(33); +a.methodD(); +module.exports = { + a: a, + b: b, + Foo: Foo, +}; +// To recreate: +// +// npx tsc --outDir test/fixtures/source-map --sourceMap --inlineSources test/fixtures/source-map/no-throw.ts +//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibm8tdGhyb3cuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyJuby10aHJvdy50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQTtJQUVFLGFBQWEsQ0FBTTtRQUFOLGtCQUFBLEVBQUEsTUFBTTtRQUNqQixJQUFJLENBQUMsQ0FBQyxHQUFHLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUE7UUFDbkIsSUFBSSxJQUFJLENBQUMsQ0FBQyxFQUFFLENBQUM7WUFDWCxJQUFJLENBQUMsT0FBTyxFQUFFLENBQUE7UUFDaEIsQ0FBQzthQUFNLENBQUM7WUFDTixJQUFJLENBQUMsT0FBTyxFQUFFLENBQUE7UUFDaEIsQ0FBQztRQUNELElBQUksQ0FBQyxPQUFPLEVBQUUsQ0FBQTtJQUNoQixDQUFDO0lBQ0QscUJBQU8sR0FBUDtJQUVBLENBQUM7SUFDRCxxQkFBTyxHQUFQO0lBRUEsQ0FBQztJQUNELHFCQUFPLEdBQVA7SUFFQSxDQUFDO0lBQ0QscUJBQU8sR0FBUDtJQUVBLENBQUM7SUFDSCxVQUFDO0FBQUQsQ0FBQyxBQXZCRCxJQXVCQztBQUVELElBQU0sQ0FBQyxHQUFHLElBQUksR0FBRyxDQUFDLENBQUMsQ0FBQyxDQUFBO0FBQ3BCLElBQU0sQ0FBQyxHQUFHLElBQUksR0FBRyxDQUFDLEVBQUUsQ0FBQyxDQUFBO0FBQ3JCLENBQUMsQ0FBQyxPQUFPLEVBQUUsQ0FBQTtBQU1YLE1BQU0sQ0FBQyxPQUFPLEdBQUc7SUFDZixDQUFDLEdBQUE7SUFDRCxDQUFDLEdBQUE7SUFDRCxHQUFHLEtBQUE7Q0FDSixDQUFBO0FBRUQsZUFBZTtBQUNmLEVBQUU7QUFDRiw2R0FBNkciLCJzb3VyY2VzQ29udGVudCI6WyJjbGFzcyBGb28ge1xuICB4O1xuICBjb25zdHJ1Y3RvciAoeCA9IDMzKSB7XG4gICAgdGhpcy54ID0geCA/IHggOiA5OVxuICAgIGlmICh0aGlzLngpIHtcbiAgICAgIHRoaXMubWV0aG9kQSgpXG4gICAgfSBlbHNlIHtcbiAgICAgIHRoaXMubWV0aG9kQigpXG4gICAgfVxuICAgIHRoaXMubWV0aG9kQygpXG4gIH1cbiAgbWV0aG9kQSAoKSB7XG5cbiAgfVxuICBtZXRob2RCICgpIHtcblxuICB9XG4gIG1ldGhvZEMgKCkge1xuXG4gIH1cbiAgbWV0aG9kRCAoKSB7XG5cbiAgfVxufVxuXG5jb25zdCBhID0gbmV3IEZvbygwKVxuY29uc3QgYiA9IG5ldyBGb28oMzMpXG5hLm1ldGhvZEQoKVxuXG5kZWNsYXJlIGNvbnN0IG1vZHVsZToge1xuICBleHBvcnRzOiBhbnlcbn1cblxubW9kdWxlLmV4cG9ydHMgPSB7XG4gIGEsXG4gIGIsXG4gIEZvbyxcbn1cblxuLy8gVG8gcmVjcmVhdGU6XG4vL1xuLy8gbnB4IHRzYyAtLW91dERpciB0ZXN0L2ZpeHR1cmVzL3NvdXJjZS1tYXAgLS1zb3VyY2VNYXAgLS1pbmxpbmVTb3VyY2VzIHRlc3QvZml4dHVyZXMvc291cmNlLW1hcC9uby10aHJvdy50c1xuIl19 \ No newline at end of file diff --git a/test/fixtures/source-map/no-throw.ts b/test/fixtures/source-map/no-throw.ts new file mode 100644 index 00000000000000..71d065bca933d2 --- /dev/null +++ b/test/fixtures/source-map/no-throw.ts @@ -0,0 +1,42 @@ +class Foo { + x; + constructor (x = 33) { + this.x = x ? x : 99 + if (this.x) { + this.methodA() + } else { + this.methodB() + } + this.methodC() + } + methodA () { + + } + methodB () { + + } + methodC () { + + } + methodD () { + + } +} + +const a = new Foo(0) +const b = new Foo(33) +a.methodD() + +declare const module: { + exports: any +} + +module.exports = { + a, + b, + Foo, +} + +// To recreate: +// +// npx tsc --outDir test/fixtures/source-map --inlineSourceMap --inlineSources test/fixtures/source-map/no-throw.ts diff --git a/test/fixtures/source-map/no-throw2.js b/test/fixtures/source-map/no-throw2.js new file mode 100644 index 00000000000000..57a294ff3a2a63 --- /dev/null +++ b/test/fixtures/source-map/no-throw2.js @@ -0,0 +1,35 @@ +var Foo = /** @class */ (function () { + function Foo(x) { + if (x === void 0) { x = 33; } + this.x = x ? x : 99; + if (this.x) { + this.methodA(); + } + else { + this.methodB(); + } + this.methodC(); + } + Foo.prototype.methodA = function () { + }; + Foo.prototype.methodB = function () { + }; + Foo.prototype.methodC = function () { + }; + Foo.prototype.methodD = function () { + }; + return Foo; +}()); +var a = new Foo(0); +var b = new Foo(33); +a.methodD(); +module.exports = { + a: a, + b: b, + Foo: Foo, +}; +// To recreate: +// +// npx tsc --outDir test/fixtures/source-map --sourceMap --inlineSources test/fixtures/source-map/no-throw.ts +// cp test/fixtures/source-map/no-throw.ts test/fixtures/source-map/no-throw2.ts +//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibm8tdGhyb3cuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyJuby10aHJvdy50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQTtJQUVFLGFBQWEsQ0FBTTtRQUFOLGtCQUFBLEVBQUEsTUFBTTtRQUNqQixJQUFJLENBQUMsQ0FBQyxHQUFHLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUE7UUFDbkIsSUFBSSxJQUFJLENBQUMsQ0FBQyxFQUFFLENBQUM7WUFDWCxJQUFJLENBQUMsT0FBTyxFQUFFLENBQUE7UUFDaEIsQ0FBQzthQUFNLENBQUM7WUFDTixJQUFJLENBQUMsT0FBTyxFQUFFLENBQUE7UUFDaEIsQ0FBQztRQUNELElBQUksQ0FBQyxPQUFPLEVBQUUsQ0FBQTtJQUNoQixDQUFDO0lBQ0QscUJBQU8sR0FBUDtJQUVBLENBQUM7SUFDRCxxQkFBTyxHQUFQO0lBRUEsQ0FBQztJQUNELHFCQUFPLEdBQVA7SUFFQSxDQUFDO0lBQ0QscUJBQU8sR0FBUDtJQUVBLENBQUM7SUFDSCxVQUFDO0FBQUQsQ0FBQyxBQXZCRCxJQXVCQztBQUVELElBQU0sQ0FBQyxHQUFHLElBQUksR0FBRyxDQUFDLENBQUMsQ0FBQyxDQUFBO0FBQ3BCLElBQU0sQ0FBQyxHQUFHLElBQUksR0FBRyxDQUFDLEVBQUUsQ0FBQyxDQUFBO0FBQ3JCLENBQUMsQ0FBQyxPQUFPLEVBQUUsQ0FBQTtBQU1YLE1BQU0sQ0FBQyxPQUFPLEdBQUc7SUFDZixDQUFDLEdBQUE7SUFDRCxDQUFDLEdBQUE7SUFDRCxHQUFHLEtBQUE7Q0FDSixDQUFBO0FBRUQsZUFBZTtBQUNmLEVBQUU7QUFDRiw2R0FBNkciLCJzb3VyY2VzQ29udGVudCI6WyJjbGFzcyBGb28ge1xuICB4O1xuICBjb25zdHJ1Y3RvciAoeCA9IDMzKSB7XG4gICAgdGhpcy54ID0geCA/IHggOiA5OVxuICAgIGlmICh0aGlzLngpIHtcbiAgICAgIHRoaXMubWV0aG9kQSgpXG4gICAgfSBlbHNlIHtcbiAgICAgIHRoaXMubWV0aG9kQigpXG4gICAgfVxuICAgIHRoaXMubWV0aG9kQygpXG4gIH1cbiAgbWV0aG9kQSAoKSB7XG5cbiAgfVxuICBtZXRob2RCICgpIHtcblxuICB9XG4gIG1ldGhvZEMgKCkge1xuXG4gIH1cbiAgbWV0aG9kRCAoKSB7XG5cbiAgfVxufVxuXG5jb25zdCBhID0gbmV3IEZvbygwKVxuY29uc3QgYiA9IG5ldyBGb28oMzMpXG5hLm1ldGhvZEQoKVxuXG5kZWNsYXJlIGNvbnN0IG1vZHVsZToge1xuICBleHBvcnRzOiBhbnlcbn1cblxubW9kdWxlLmV4cG9ydHMgPSB7XG4gIGEsXG4gIGIsXG4gIEZvbyxcbn1cblxuLy8gVG8gcmVjcmVhdGU6XG4vL1xuLy8gbnB4IHRzYyAtLW91dERpciB0ZXN0L2ZpeHR1cmVzL3NvdXJjZS1tYXAgLS1zb3VyY2VNYXAgLS1pbmxpbmVTb3VyY2VzIHRlc3QvZml4dHVyZXMvc291cmNlLW1hcC9uby10aHJvdy50c1xuIl19 \ No newline at end of file diff --git a/test/fixtures/source-map/output/source_map_disabled_by_api.js b/test/fixtures/source-map/output/source_map_disabled_by_api.js index d94a6310cff7ae..8f455f26b6c9c4 100644 --- a/test/fixtures/source-map/output/source_map_disabled_by_api.js +++ b/test/fixtures/source-map/output/source_map_disabled_by_api.js @@ -15,10 +15,10 @@ try { console.log(e); } +// Delete the CJS module cache and loading the module again with source maps +// support enabled programmatically. delete require.cache[require .resolve('../enclosing-call-site-min.js')]; - -// Re-enable. process.setSourceMapsEnabled(true); assert.strictEqual(process.sourceMapsEnabled, true); diff --git a/test/fixtures/source-map/output/source_map_prepare_stack_trace.js b/test/fixtures/source-map/output/source_map_prepare_stack_trace.js index d49bad116b479f..1b04e0a3ac221b 100644 --- a/test/fixtures/source-map/output/source_map_prepare_stack_trace.js +++ b/test/fixtures/source-map/output/source_map_prepare_stack_trace.js @@ -20,10 +20,8 @@ try { console.log(e); } -delete require.cache[require - .resolve('../enclosing-call-site-min.js')]; - -// Disable +// Source maps support is disabled programmatically even without deleting the +// CJS module cache. process.setSourceMapsEnabled(false); assert.strictEqual(process.sourceMapsEnabled, false); diff --git a/test/parallel/test-internal-iterable-weak-map.js b/test/parallel/test-internal-iterable-weak-map.js deleted file mode 100644 index f2befe13da87f3..00000000000000 --- a/test/parallel/test-internal-iterable-weak-map.js +++ /dev/null @@ -1,101 +0,0 @@ -// Flags: --expose-gc --expose-internals -'use strict'; - -const common = require('../common'); -const { deepStrictEqual, strictEqual } = require('assert'); -const { IterableWeakMap } = require('internal/util/iterable_weak_map'); - -// Ensures iterating over the map does not rely on methods which can be -// mutated by users. -Reflect.getPrototypeOf(function*() {}).prototype.next = common.mustNotCall(); -Reflect.getPrototypeOf(new Set()[Symbol.iterator]()).next = - common.mustNotCall(); - -// It drops entry if a reference is no longer held. -{ - const wm = new IterableWeakMap(); - const _cache = { - moduleA: {}, - moduleB: {}, - moduleC: {}, - }; - wm.set(_cache.moduleA, 'hello'); - wm.set(_cache.moduleB, 'discard'); - wm.set(_cache.moduleC, 'goodbye'); - delete _cache.moduleB; - setImmediate(() => { - _cache; // eslint-disable-line no-unused-expressions - globalThis.gc(); - const values = [...wm]; - deepStrictEqual(values, ['hello', 'goodbye']); - }); -} - -// It updates an existing entry, if the same key is provided twice. -{ - const wm = new IterableWeakMap(); - const _cache = { - moduleA: {}, - moduleB: {}, - }; - wm.set(_cache.moduleA, 'hello'); - wm.set(_cache.moduleB, 'goodbye'); - wm.set(_cache.moduleB, 'goodnight'); - const values = [...wm]; - deepStrictEqual(values, ['hello', 'goodnight']); -} - -// It allows entry to be deleted by key. -{ - const wm = new IterableWeakMap(); - const _cache = { - moduleA: {}, - moduleB: {}, - moduleC: {}, - }; - wm.set(_cache.moduleA, 'hello'); - wm.set(_cache.moduleB, 'discard'); - wm.set(_cache.moduleC, 'goodbye'); - wm.delete(_cache.moduleB); - const values = [...wm]; - deepStrictEqual(values, ['hello', 'goodbye']); -} - -// It handles delete for key that does not exist. -{ - const wm = new IterableWeakMap(); - const _cache = { - moduleA: {}, - moduleB: {}, - moduleC: {}, - }; - wm.set(_cache.moduleA, 'hello'); - wm.set(_cache.moduleC, 'goodbye'); - wm.delete(_cache.moduleB); - const values = [...wm]; - deepStrictEqual(values, ['hello', 'goodbye']); -} - -// It allows an entry to be fetched by key. -{ - const wm = new IterableWeakMap(); - const _cache = { - moduleA: {}, - moduleB: {}, - moduleC: {}, - }; - wm.set(_cache.moduleA, 'hello'); - wm.set(_cache.moduleB, 'discard'); - wm.set(_cache.moduleC, 'goodbye'); - strictEqual(wm.get(_cache.moduleB), 'discard'); -} - -// It returns true for has() if key exists. -{ - const wm = new IterableWeakMap(); - const _cache = { - moduleA: {}, - }; - wm.set(_cache.moduleA, 'hello'); - strictEqual(wm.has(_cache.moduleA), true); -} diff --git a/test/parallel/test-source-map-cjs-require-cache.js b/test/parallel/test-source-map-cjs-require-cache.js new file mode 100644 index 00000000000000..155b7a66efab1b --- /dev/null +++ b/test/parallel/test-source-map-cjs-require-cache.js @@ -0,0 +1,37 @@ +// Flags: --enable-source-maps --max-old-space-size=10 --expose-gc + +/** + * This test verifies that the source map of a CJS module is cleared after the + * CJS module is reclaimed by GC. + */ + +'use strict'; +require('../common'); +const { gcUntil } = require('../common/gc'); +const assert = require('node:assert'); +const { findSourceMap } = require('node:module'); + +const moduleId = require.resolve('../fixtures/source-map/no-throw.js'); +const moduleIdRepeat = require.resolve('../fixtures/source-map/no-throw2.js'); + +function run(moduleId) { + require(moduleId); + delete require.cache[moduleId]; + const idx = module.children.findIndex((child) => child.id === moduleId); + assert.ok(idx >= 0); + module.children.splice(idx, 1); + + // Verify that the source map is still available + assert.notStrictEqual(findSourceMap(moduleId), undefined); +} + +// Run the test in a function scope so that every variable can be reclaimed by GC. +run(moduleId); + +// Run until the source map is cleared by GC, or fail the test after determined iterations. +gcUntil('SourceMap of deleted CJS module is cleared', () => { + // Repetitively load a second module with --max-old-space-size=10 to make GC more aggressive. + run(moduleIdRepeat); + // Verify that the source map is cleared. + return findSourceMap(moduleId) == null; +}); From e337526a52309aab35a59e92ec4ecad777e59d18 Mon Sep 17 00:00:00 2001 From: Yagiz Nizipli Date: Thu, 13 Jun 2024 14:32:59 -0400 Subject: [PATCH 08/33] lib: reduce amount of caught URL errors PR-URL: https://github.com/nodejs/node/pull/52658 Reviewed-By: Antoine du Hamel Reviewed-By: James M Snell Reviewed-By: Daniel Lemire --- lib/internal/modules/esm/hooks.js | 8 ++------ lib/internal/modules/esm/loader.js | 8 ++++---- lib/internal/source_map/source_map_cache.js | 15 +++++++-------- lib/internal/url.js | 1 + 4 files changed, 14 insertions(+), 18 deletions(-) diff --git a/lib/internal/modules/esm/hooks.js b/lib/internal/modules/esm/hooks.js index 88c66f89a83c66..c13d7c8ad3321c 100644 --- a/lib/internal/modules/esm/hooks.js +++ b/lib/internal/modules/esm/hooks.js @@ -38,7 +38,7 @@ const { ERR_WORKER_UNSERIALIZABLE_ERROR, } = require('internal/errors').codes; const { exitCodes: { kUnfinishedTopLevelAwait } } = internalBinding('errors'); -const { URL } = require('internal/url'); +const { URLParse } = require('internal/url'); const { canParse: URLCanParse } = internalBinding('url'); const { receiveMessageOnPort } = require('worker_threads'); const { @@ -471,11 +471,7 @@ class Hooks { let responseURLObj; if (typeof responseURL === 'string') { - try { - responseURLObj = new URL(responseURL); - } catch { - // responseURLObj not defined will throw in next branch. - } + responseURLObj = URLParse(responseURL); } if (responseURLObj?.href !== responseURL) { diff --git a/lib/internal/modules/esm/loader.js b/lib/internal/modules/esm/loader.js index 0878e257ade491..f94696fa804759 100644 --- a/lib/internal/modules/esm/loader.js +++ b/lib/internal/modules/esm/loader.js @@ -28,14 +28,13 @@ const { ERR_UNKNOWN_MODULE_FORMAT, } = require('internal/errors').codes; const { getOptionValue } = require('internal/options'); -const { isURL, pathToFileURL, URL } = require('internal/url'); +const { isURL, pathToFileURL, URLParse } = require('internal/url'); const { emitExperimentalWarning, kEmptyObject } = require('internal/util'); const { compileSourceTextModule, getDefaultConditions, } = require('internal/modules/esm/utils'); const { kImplicitAssertType } = require('internal/modules/esm/assert'); -const { canParse } = internalBinding('url'); const { ModuleWrap, kEvaluating, kEvaluated } = internalBinding('module_wrap'); const { urlToFilename, @@ -321,8 +320,9 @@ class ModuleLoader { getModuleJobForRequire(specifier, parentURL, importAttributes) { assert(getOptionValue('--experimental-require-module')); - if (canParse(specifier)) { - const protocol = new URL(specifier).protocol; + const parsed = URLParse(specifier); + if (parsed != null) { + const protocol = parsed.protocol; if (protocol === 'https:' || protocol === 'http:') { throw new ERR_NETWORK_IMPORT_DISALLOWED(specifier, parentURL, 'ES modules cannot be loaded by require() from the network'); diff --git a/lib/internal/source_map/source_map_cache.js b/lib/internal/source_map/source_map_cache.js index f21b3719ad806a..9defc32da8e1e6 100644 --- a/lib/internal/source_map/source_map_cache.js +++ b/lib/internal/source_map/source_map_cache.js @@ -39,7 +39,7 @@ const kSourceMappingURLMagicComment = /\/[*/]#\s+sourceMappingURL=(?[^\s]+)/g; const { isAbsolute } = require('path'); -const { fileURLToPath, pathToFileURL, URL } = require('internal/url'); +const { fileURLToPath, pathToFileURL, URL, URLParse } = require('internal/url'); let SourceMap; @@ -209,8 +209,9 @@ function maybeCacheGeneratedSourceMap(content) { * @returns {object} deserialized source map JSON object */ function dataFromUrl(sourceURL, sourceMappingURL) { - try { - const url = new URL(sourceMappingURL); + const url = URLParse(sourceMappingURL); + + if (url != null) { switch (url.protocol) { case 'data:': return sourceMapFromDataUrl(sourceURL, url.pathname); @@ -218,12 +219,10 @@ function dataFromUrl(sourceURL, sourceMappingURL) { debug(`unknown protocol ${url.protocol}`); return null; } - } catch (err) { - debug(err); - // If no scheme is present, we assume we are dealing with a file path. - const mapURL = new URL(sourceMappingURL, sourceURL).href; - return sourceMapFromFile(mapURL); } + + const mapURL = new URL(sourceMappingURL, sourceURL).href; + return sourceMapFromFile(mapURL); } // Cache the length of each line in the file that a source map was extracted diff --git a/lib/internal/url.js b/lib/internal/url.js index 42debfc20005b0..91976bceb0db19 100644 --- a/lib/internal/url.js +++ b/lib/internal/url.js @@ -1624,6 +1624,7 @@ module.exports = { installObjectURLMethods, URL, URLSearchParams, + URLParse: URL.parse, domainToASCII, domainToUnicode, urlToHttpOptions, From 07b24a3d241d80a109b19f966fae469363ecda16 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Fri, 5 Jul 2024 21:58:35 +0200 Subject: [PATCH 09/33] src: refactor embedded entrypoint loading This patch: 1. Refactor the routines used to compile and run an embedder entrypoint. In JS land special handling for SEA is done directly in main/embedding.js instead of clobbering the CJS loader. Add warnings to remind users that currently the require() in SEA bundled scripts only supports loading builtins. 2. Don't use the bundled SEA code cache when compiling CJS loaded from disk, since in that case we are certainly not compiling the code bundled into the SEA. Use a is_sea_main flag in CompileFunctionForCJSLoader() (which replaces an unused argument) to pass this into the C++ land - the code cache is still read directly from C++ to avoid the overhead of ArrayBuffer creation. 3. Move SEA loading code into MaybeLoadSingleExecutableApplication() which calls LoadEnvironment() with its own StartExecutionCallback(). This avoids more hidden switches in StartExecution() and make them explicit. Also add some TODOs about how to support ESM in embedded applications. 4. Add more comments PR-URL: https://github.com/nodejs/node/pull/53573 Reviewed-By: Geoffrey Booth Reviewed-By: Chengzhong Wu Reviewed-By: Matteo Collina Reviewed-By: James M Snell --- lib/internal/main/check_syntax.js | 2 +- lib/internal/main/embedding.js | 105 +++++++++++++++++++++++- lib/internal/modules/cjs/loader.js | 17 +--- lib/internal/modules/esm/translators.js | 2 +- lib/internal/util/embedding.js | 51 ------------ src/node.cc | 8 ++ src/node_contextify.cc | 55 ++++++++----- src/node_main_instance.cc | 15 +--- src/node_sea.cc | 62 +++++++++----- src/node_sea.h | 7 ++ test/fixtures/sea.js | 12 ++- 11 files changed, 208 insertions(+), 128 deletions(-) delete mode 100644 lib/internal/util/embedding.js diff --git a/lib/internal/main/check_syntax.js b/lib/internal/main/check_syntax.js index 5a7ab5dc19e4e7..aa14dca8999e89 100644 --- a/lib/internal/main/check_syntax.js +++ b/lib/internal/main/check_syntax.js @@ -75,5 +75,5 @@ async function checkSyntax(source, filename) { return; } - wrapSafe(filename, source); + wrapSafe(filename, source, undefined, 'commonjs'); } diff --git a/lib/internal/main/embedding.js b/lib/internal/main/embedding.js index cc7cb0eee9d837..e547e77e9090df 100644 --- a/lib/internal/main/embedding.js +++ b/lib/internal/main/embedding.js @@ -1,15 +1,116 @@ 'use strict'; + +// This main script is currently only run when LoadEnvironment() +// is run with a non-null StartExecutionCallback or a UTF8 +// main script. Effectively there are two cases where this happens: +// 1. It's a single-executable application *loading* a main script +// bundled into the executable. This is currently done from +// NodeMainInstance::Run(). +// 2. It's an embedder application and LoadEnvironment() is invoked +// as described above. + const { prepareMainThreadExecution, } = require('internal/process/pre_execution'); -const { isExperimentalSeaWarningNeeded } = internalBinding('sea'); +const { isExperimentalSeaWarningNeeded, isSea } = internalBinding('sea'); const { emitExperimentalWarning } = require('internal/util'); -const { embedderRequire, embedderRunCjs } = require('internal/util/embedding'); +const { emitWarningSync } = require('internal/process/warning'); +const { BuiltinModule: { normalizeRequirableId } } = require('internal/bootstrap/realm'); +const { Module } = require('internal/modules/cjs/loader'); +const { compileFunctionForCJSLoader } = internalBinding('contextify'); +const { maybeCacheSourceMap } = require('internal/source_map/source_map_cache'); + +const { codes: { + ERR_UNKNOWN_BUILTIN_MODULE, +} } = require('internal/errors'); +// Don't expand process.argv[1] because in a single-executable application or an +// embedder application, the user main script isn't necessarily provided via the +// command line (e.g. it could be provided via an API or bundled into the executable). prepareMainThreadExecution(false, true); +const isLoadingSea = isSea(); if (isExperimentalSeaWarningNeeded()) { emitExperimentalWarning('Single executable application'); } +// This is roughly the same as: +// +// const mod = new Module(filename); +// mod._compile(content, filename); +// +// but the code has been duplicated because currently there is no way to set the +// value of require.main to module. +// +// TODO(RaisinTen): Find a way to deduplicate this. +function embedderRunCjs(content) { + // The filename of the module (used for CJS module lookup) + // is always the same as the location of the executable itself + // at the time of the loading (which means it changes depending + // on where the executable is in the file system). + const filename = process.execPath; + const customModule = new Module(filename, null); + + const { + function: compiledWrapper, + cachedDataRejected, + sourceMapURL, + } = compileFunctionForCJSLoader( + content, + filename, + isLoadingSea, // is_sea_main + false, // should_detect_module, ESM should be supported differently for embedded code + ); + // Cache the source map for the module if present. + if (sourceMapURL) { + maybeCacheSourceMap( + filename, + content, + customModule, + false, // isGeneratedSource + undefined, // sourceURL, TODO(joyeecheung): should be extracted by V8 + sourceMapURL, + ); + } + + // cachedDataRejected is only set if cache from SEA is used. + if (cachedDataRejected !== false && isLoadingSea) { + emitWarningSync('Code cache data rejected.'); + } + + // Patch the module to make it look almost like a regular CJS module + // instance. + customModule.filename = process.execPath; + customModule.paths = Module._nodeModulePaths(process.execPath); + embedderRequire.main = customModule; + + return compiledWrapper( + customModule.exports, // exports + embedderRequire, // require + customModule, // module + process.execPath, // __filename + customModule.path, // __dirname + ); +} + +let warnedAboutBuiltins = false; + +function embedderRequire(id) { + const normalizedId = normalizeRequirableId(id); + if (!normalizedId) { + if (isLoadingSea && !warnedAboutBuiltins) { + emitWarningSync( + 'Currently the require() provided to the main script embedded into ' + + 'single-executable applications only supports loading built-in modules.\n' + + 'To load a module from disk after the single executable application is ' + + 'launched, use require("module").createRequire().\n' + + 'Support for bundled module loading or virtual file systems are under ' + + 'discussions in https://github.com/nodejs/single-executable'); + warnedAboutBuiltins = true; + } + throw new ERR_UNKNOWN_BUILTIN_MODULE(id); + } + return require(normalizedId); +} + return [process, embedderRequire, embedderRunCjs]; diff --git a/lib/internal/modules/cjs/loader.js b/lib/internal/modules/cjs/loader.js index 32cc78112a786d..c3bbeef5f0a5f9 100644 --- a/lib/internal/modules/cjs/loader.js +++ b/lib/internal/modules/cjs/loader.js @@ -1350,11 +1350,10 @@ function loadESMFromCJS(mod, filename) { * Wraps the given content in a script and runs it in a new context. * @param {string} filename The name of the file being loaded * @param {string} content The content of the file being loaded - * @param {Module} cjsModuleInstance The CommonJS loader instance - * @param {object} codeCache The SEA code cache + * @param {Module|undefined} cjsModuleInstance The CommonJS loader instance * @param {'commonjs'|undefined} format Intended format of the module. */ -function wrapSafe(filename, content, cjsModuleInstance, codeCache, format) { +function wrapSafe(filename, content, cjsModuleInstance, format) { assert(format !== 'module'); // ESM should be handled in loadESMFromCJS(). const hostDefinedOptionId = vm_dynamic_import_default_internal; const importModuleDynamically = vm_dynamic_import_default_internal; @@ -1385,16 +1384,8 @@ function wrapSafe(filename, content, cjsModuleInstance, codeCache, format) { }; } - const isMain = !!(cjsModuleInstance && cjsModuleInstance[kIsMainSymbol]); const shouldDetectModule = (format !== 'commonjs' && getOptionValue('--experimental-detect-module')); - const result = compileFunctionForCJSLoader(content, filename, isMain, shouldDetectModule); - - // cachedDataRejected is only set for cache coming from SEA. - if (codeCache && - result.cachedDataRejected !== false && - internalBinding('sea').isSea()) { - process.emitWarning('Code cache data rejected.'); - } + const result = compileFunctionForCJSLoader(content, filename, false /* is_sea_main */, shouldDetectModule); // Cache the source map for the module if present. if (result.sourceMapURL) { @@ -1423,7 +1414,7 @@ Module.prototype._compile = function(content, filename, format) { let compiledWrapper; if (format !== 'module') { - const result = wrapSafe(filename, content, this, undefined, format); + const result = wrapSafe(filename, content, this, format); compiledWrapper = result.function; if (result.canParseAsESM) { format = 'module'; diff --git a/lib/internal/modules/esm/translators.js b/lib/internal/modules/esm/translators.js index 8e33d71b17fe5e..8007427fe2337d 100644 --- a/lib/internal/modules/esm/translators.js +++ b/lib/internal/modules/esm/translators.js @@ -176,7 +176,7 @@ translators.set('module', function moduleStrategy(url, source, isMain) { * @param {boolean} isMain - Whether the module is the entrypoint */ function loadCJSModule(module, source, url, filename, isMain) { - const compileResult = compileFunctionForCJSLoader(source, filename, isMain, false); + const compileResult = compileFunctionForCJSLoader(source, filename, false /* is_sea_main */, false); const { function: compiledWrapper, sourceMapURL } = compileResult; // Cache the source map for the cjs module if present. diff --git a/lib/internal/util/embedding.js b/lib/internal/util/embedding.js deleted file mode 100644 index 7e4cd565492843..00000000000000 --- a/lib/internal/util/embedding.js +++ /dev/null @@ -1,51 +0,0 @@ -'use strict'; -const { BuiltinModule: { normalizeRequirableId } } = require('internal/bootstrap/realm'); -const { Module, wrapSafe } = require('internal/modules/cjs/loader'); -const { codes: { ERR_UNKNOWN_BUILTIN_MODULE } } = require('internal/errors'); -const { getCodePath, isSea } = internalBinding('sea'); - -// This is roughly the same as: -// -// const mod = new Module(filename); -// mod._compile(contents, filename); -// -// but the code has been duplicated because currently there is no way to set the -// value of require.main to module. -// -// TODO(RaisinTen): Find a way to deduplicate this. - -function embedderRunCjs(contents) { - const filename = process.execPath; - const { function: compiledWrapper } = wrapSafe( - isSea() ? getCodePath() : filename, - contents); - - const customModule = new Module(filename, null); - customModule.filename = filename; - customModule.paths = Module._nodeModulePaths(customModule.path); - - const customExports = customModule.exports; - - embedderRequire.main = customModule; - - const customFilename = customModule.filename; - - const customDirname = customModule.path; - - return compiledWrapper( - customExports, - embedderRequire, - customModule, - customFilename, - customDirname); -} - -function embedderRequire(id) { - const normalizedId = normalizeRequirableId(id); - if (!normalizedId) { - throw new ERR_UNKNOWN_BUILTIN_MODULE(id); - } - return require(normalizedId); -} - -module.exports = { embedderRequire, embedderRunCjs }; diff --git a/src/node.cc b/src/node.cc index 1fc236d88e53eb..35b455242be78e 100644 --- a/src/node.cc +++ b/src/node.cc @@ -308,6 +308,14 @@ std::optional CallbackInfoFromArray( CHECK(process_obj->IsObject()); CHECK(require_fn->IsFunction()); CHECK(runcjs_fn->IsFunction()); + // TODO(joyeecheung): some support for running ESM as an entrypoint + // is needed. The simplest API would be to add a run_esm to + // StartExecutionCallbackInfo which compiles, links (to builtins) + // and evaluates a SourceTextModule. + // TODO(joyeecheung): the env pointer should be part of + // StartExecutionCallbackInfo, otherwise embedders are forced to use + // lambdas to pass it into the callback, which can make the code + // difficult to read. node::StartExecutionCallbackInfo info{process_obj.As(), require_fn.As(), runcjs_fn.As()}; diff --git a/src/node_contextify.cc b/src/node_contextify.cc index 471cd017567fd9..7f82bca746408b 100644 --- a/src/node_contextify.cc +++ b/src/node_contextify.cc @@ -1441,12 +1441,17 @@ static std::vector throws_only_in_cjs_error_messages = { "await is only valid in async functions and " "the top level bodies of modules"}; -static MaybeLocal CompileFunctionForCJSLoader(Environment* env, - Local context, - Local code, - Local filename, - bool* cache_rejected, - bool is_cjs_scope) { +// If cached_data is provided, it would be used for the compilation and +// the on-disk compilation cache from NODE_COMPILE_CACHE (if configured) +// would be ignored. +static MaybeLocal CompileFunctionForCJSLoader( + Environment* env, + Local context, + Local code, + Local filename, + bool* cache_rejected, + bool is_cjs_scope, + ScriptCompiler::CachedData* cached_data) { Isolate* isolate = context->GetIsolate(); EscapableHandleScope scope(isolate); @@ -1464,20 +1469,7 @@ static MaybeLocal CompileFunctionForCJSLoader(Environment* env, false, // is WASM false, // is ES Module hdo); - ScriptCompiler::CachedData* cached_data = nullptr; -#ifndef DISABLE_SINGLE_EXECUTABLE_APPLICATION - if (sea::IsSingleExecutable()) { - sea::SeaResource sea = sea::FindSingleExecutableResource(); - if (sea.use_code_cache()) { - std::string_view data = sea.code_cache.value(); - cached_data = new ScriptCompiler::CachedData( - reinterpret_cast(data.data()), - static_cast(data.size()), - v8::ScriptCompiler::CachedData::BufferNotOwned); - } - } -#endif ScriptCompiler::Source source(code, origin, cached_data); ScriptCompiler::CompileOptions options; if (cached_data == nullptr) { @@ -1532,6 +1524,7 @@ static void CompileFunctionForCJSLoader( CHECK(args[3]->IsBoolean()); Local code = args[0].As(); Local filename = args[1].As(); + bool is_sea_main = args[2].As()->Value(); bool should_detect_module = args[3].As()->Value(); Isolate* isolate = args.GetIsolate(); @@ -1544,11 +1537,31 @@ static void CompileFunctionForCJSLoader( Local cjs_exception; Local cjs_message; + ScriptCompiler::CachedData* cached_data = nullptr; +#ifndef DISABLE_SINGLE_EXECUTABLE_APPLICATION + if (is_sea_main) { + sea::SeaResource sea = sea::FindSingleExecutableResource(); + // Use the "main" field in SEA config for the filename. + Local filename_from_sea; + if (!ToV8Value(context, sea.code_path).ToLocal(&filename_from_sea)) { + return; + } + filename = filename_from_sea.As(); + if (sea.use_code_cache()) { + std::string_view data = sea.code_cache.value(); + cached_data = new ScriptCompiler::CachedData( + reinterpret_cast(data.data()), + static_cast(data.size()), + v8::ScriptCompiler::CachedData::BufferNotOwned); + } + } +#endif + { ShouldNotAbortOnUncaughtScope no_abort_scope(realm->env()); TryCatchScope try_catch(env); if (!CompileFunctionForCJSLoader( - env, context, code, filename, &cache_rejected, true) + env, context, code, filename, &cache_rejected, true, cached_data) .ToLocal(&fn)) { CHECK(try_catch.HasCaught()); CHECK(!try_catch.HasTerminated()); @@ -1703,7 +1716,7 @@ static void ContainsModuleSyntax(const FunctionCallbackInfo& args) { TryCatchScope try_catch(env); ShouldNotAbortOnUncaughtScope no_abort_scope(env); if (CompileFunctionForCJSLoader( - env, context, code, filename, &cache_rejected, cjs_var) + env, context, code, filename, &cache_rejected, cjs_var, nullptr) .ToLocal(&fn)) { args.GetReturnValue().Set(false); return; diff --git a/src/node_main_instance.cc b/src/node_main_instance.cc index 22b35e33e8fa50..4119ac1b002681 100644 --- a/src/node_main_instance.cc +++ b/src/node_main_instance.cc @@ -103,20 +103,7 @@ ExitCode NodeMainInstance::Run() { void NodeMainInstance::Run(ExitCode* exit_code, Environment* env) { if (*exit_code == ExitCode::kNoFailure) { - bool runs_sea_code = false; -#ifndef DISABLE_SINGLE_EXECUTABLE_APPLICATION - if (sea::IsSingleExecutable()) { - sea::SeaResource sea = sea::FindSingleExecutableResource(); - if (!sea.use_snapshot()) { - runs_sea_code = true; - std::string_view code = sea.main_code_or_snapshot; - LoadEnvironment(env, code); - } - } -#endif - // Either there is already a snapshot main function from SEA, or it's not - // a SEA at all. - if (!runs_sea_code) { + if (!sea::MaybeLoadSingleExecutableApplication(env)) { LoadEnvironment(env, StartExecutionCallback{}); } diff --git a/src/node_sea.cc b/src/node_sea.cc index 1cb09365dfca54..fb9f933a19fa70 100644 --- a/src/node_sea.cc +++ b/src/node_sea.cc @@ -36,6 +36,7 @@ using v8::FunctionCallbackInfo; using v8::HandleScope; using v8::Isolate; using v8::Local; +using v8::MaybeLocal; using v8::NewStringType; using v8::Object; using v8::ScriptCompiler; @@ -261,25 +262,6 @@ void IsExperimentalSeaWarningNeeded(const FunctionCallbackInfo& args) { sea_resource.flags & SeaFlags::kDisableExperimentalSeaWarning)); } -void GetCodePath(const FunctionCallbackInfo& args) { - DCHECK(IsSingleExecutable()); - - Isolate* isolate = args.GetIsolate(); - - SeaResource sea_resource = FindSingleExecutableResource(); - - Local code_path; - if (!String::NewFromUtf8(isolate, - sea_resource.code_path.data(), - NewStringType::kNormal, - sea_resource.code_path.length()) - .ToLocal(&code_path)) { - return; - } - - args.GetReturnValue().Set(code_path); -} - std::tuple FixupArgsForSEA(int argc, char** argv) { // Repeats argv[0] at position 1 on argv as a replacement for the missing // entry point file path. @@ -619,6 +601,46 @@ void GetAsset(const FunctionCallbackInfo& args) { args.GetReturnValue().Set(ab); } +MaybeLocal LoadSingleExecutableApplication( + const StartExecutionCallbackInfo& info) { + // Here we are currently relying on the fact that in NodeMainInstance::Run(), + // env->context() is entered. + Local context = Isolate::GetCurrent()->GetCurrentContext(); + Environment* env = Environment::GetCurrent(context); + SeaResource sea = FindSingleExecutableResource(); + + CHECK(!sea.use_snapshot()); + // TODO(joyeecheung): this should be an external string. Refactor UnionBytes + // and make it easy to create one based on static content on the fly. + Local main_script = + ToV8Value(env->context(), sea.main_code_or_snapshot).ToLocalChecked(); + return info.run_cjs->Call( + env->context(), Null(env->isolate()), 1, &main_script); +} + +bool MaybeLoadSingleExecutableApplication(Environment* env) { +#ifndef DISABLE_SINGLE_EXECUTABLE_APPLICATION + if (!IsSingleExecutable()) { + return false; + } + + SeaResource sea = FindSingleExecutableResource(); + + if (sea.use_snapshot()) { + // The SEA preparation blob building process should already enforce this, + // this check is just here to guard against the unlikely case where + // the SEA preparation blob has been manually modified by someone. + CHECK(!env->snapshot_deserialize_main().IsEmpty()); + LoadEnvironment(env, StartExecutionCallback{}); + return true; + } + + LoadEnvironment(env, LoadSingleExecutableApplication); + return true; +#endif + return false; +} + void Initialize(Local target, Local unused, Local context, @@ -628,14 +650,12 @@ void Initialize(Local target, target, "isExperimentalSeaWarningNeeded", IsExperimentalSeaWarningNeeded); - SetMethod(context, target, "getCodePath", GetCodePath); SetMethod(context, target, "getAsset", GetAsset); } void RegisterExternalReferences(ExternalReferenceRegistry* registry) { registry->Register(IsSea); registry->Register(IsExperimentalSeaWarningNeeded); - registry->Register(GetCodePath); registry->Register(GetAsset); } diff --git a/src/node_sea.h b/src/node_sea.h index 6f2f51d997dc73..f3b3c34d26a969 100644 --- a/src/node_sea.h +++ b/src/node_sea.h @@ -14,6 +14,7 @@ #include "node_exit_code.h" namespace node { +class Environment; namespace sea { // A special number that will appear at the beginning of the single executable // preparation blobs ready to be injected into the binary. We use this to check @@ -49,6 +50,12 @@ node::ExitCode BuildSingleExecutableBlob( const std::string& config_path, const std::vector& args, const std::vector& exec_args); + +// Try loading the Environment as a single-executable application. +// Returns true if it is loaded as a single-executable application. +// Otherwise returns false and the caller is expected to call LoadEnvironment() +// differently. +bool MaybeLoadSingleExecutableApplication(Environment* env); } // namespace sea } // namespace node diff --git a/test/fixtures/sea.js b/test/fixtures/sea.js index e7b7f46ff00a86..6dea6960997feb 100644 --- a/test/fixtures/sea.js +++ b/test/fixtures/sea.js @@ -5,10 +5,14 @@ const createdRequire = createRequire(__filename); // because we set NODE_TEST_DIR=/Users/iojs/node-tmp on Jenkins CI. const { expectWarning, mustNotCall } = createdRequire(process.env.COMMON_DIRECTORY); +const builtinWarning = +`Currently the require() provided to the main script embedded into single-executable applications only supports loading built-in modules. +To load a module from disk after the single executable application is launched, use require("module").createRequire(). +Support for bundled module loading or virtual file systems are under discussions in https://github.com/nodejs/single-executable`; + +expectWarning('Warning', builtinWarning); // Triggered by require() calls below. // This additionally makes sure that no unexpected warnings are emitted. -if (createdRequire('./sea-config.json').disableExperimentalSEAWarning) { - process.on('warning', mustNotCall()); -} else { +if (!createdRequire('./sea-config.json').disableExperimentalSEAWarning) { expectWarning('ExperimentalWarning', 'Single executable application is an experimental feature and ' + 'might change at any time'); @@ -22,7 +26,7 @@ const { deepStrictEqual, strictEqual, throws } = require('assert'); const { dirname } = require('node:path'); // Checks that the source filename is used in the error stack trace. -strictEqual(new Error('lol').stack.split('\n')[1], ' at sea.js:25:13'); +strictEqual(new Error('lol').stack.split('\n')[1], ' at sea.js:29:13'); // Should be possible to require a core module that requires using the "node:" // scheme. From ad274c7017866ed27965ffef3d429a1dcddf292e Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Wed, 20 Mar 2024 21:41:22 +0100 Subject: [PATCH 10/33] module: add __esModule to require()'d ESM Tooling in the ecosystem have been using the __esModule property to recognize transpiled ESM in consuming code. For example, a 'log' package written in ESM: export function log(val) { console.log(val); } Can be transpiled as: exports.__esModule = true; exports.default = function log(val) { console.log(val); } The consuming code may be written like this in ESM: import log from 'log' Which gets transpiled to: const _mod = require('log'); const log = _mod.__esModule ? _mod.default : _mod; So to allow transpiled consuming code to recognize require()'d real ESM as ESM and pick up the default exports, we add a __esModule property by building a source text module facade for any module that has a default export and add .__esModule = true to the exports. We don't do this to modules that don't have default exports to avoid the unnecessary overhead. This maintains the enumerability of the re-exported names and the live binding of the exports. The source of the facade is defined as a constant per-isolate property required_module_facade_source_string, which looks like this export * from 'original'; export { default } from 'original'; export const __esModule = true; And the 'original' module request is always resolved by createRequiredModuleFacade() to wrap which is a ModuleWrap wrapping over the original module. PR-URL: https://github.com/nodejs/node/pull/52166 Refs: https://github.com/nodejs/node/issues/52134 Reviewed-By: Matteo Collina Reviewed-By: James M Snell Reviewed-By: Filip Skokan Reviewed-By: Chengzhong Wu Reviewed-By: Guy Bedford Reviewed-By: Geoffrey Booth --- doc/api/modules.md | 36 +++++++--- lib/internal/modules/cjs/loader.js | 55 +++++++++++++-- lib/internal/modules/esm/loader.js | 6 +- src/env.h | 2 + src/env_properties.h | 6 ++ src/module_wrap.cc | 70 +++++++++++++++++++ src/module_wrap.h | 3 + test/common/index.js | 9 ++- .../test-require-module-default-extension.js | 6 +- .../test-require-module-defined-esmodule.js | 23 ++++++ .../test-require-module-dynamic-import-1.js | 3 +- .../test-require-module-dynamic-import-2.js | 3 +- .../test-require-module-dynamic-import-3.js | 3 +- .../test-require-module-dynamic-import-4.js | 3 +- .../es-module/test-require-module-implicit.js | 6 +- .../test-require-module-transpiled.js | 30 ++++++++ .../test-require-module-with-detection.js | 10 +-- test/es-module/test-require-module.js | 18 ++--- .../es-modules/export-es-module-2.mjs | 2 + test/fixtures/es-modules/export-es-module.mjs | 2 + .../dist/import-both.cjs | 27 +++++++ .../dist/import-default.cjs | 7 ++ .../dist/import-named.cjs | 4 ++ .../node_modules/logger/logger.mjs | 2 + .../node_modules/logger/package.json | 4 ++ .../src/import-both.mjs | 2 + .../src/import-default.mjs | 2 + .../src/import-named.mjs | 2 + .../transpile.cjs | 23 ++++++ test/parallel/test-runner-module-mocking.js | 2 +- 30 files changed, 316 insertions(+), 55 deletions(-) create mode 100644 test/es-module/test-require-module-defined-esmodule.js create mode 100644 test/es-module/test-require-module-transpiled.js create mode 100644 test/fixtures/es-modules/export-es-module-2.mjs create mode 100644 test/fixtures/es-modules/export-es-module.mjs create mode 100644 test/fixtures/es-modules/transpiled-cjs-require-module/dist/import-both.cjs create mode 100644 test/fixtures/es-modules/transpiled-cjs-require-module/dist/import-default.cjs create mode 100644 test/fixtures/es-modules/transpiled-cjs-require-module/dist/import-named.cjs create mode 100644 test/fixtures/es-modules/transpiled-cjs-require-module/node_modules/logger/logger.mjs create mode 100644 test/fixtures/es-modules/transpiled-cjs-require-module/node_modules/logger/package.json create mode 100644 test/fixtures/es-modules/transpiled-cjs-require-module/src/import-both.mjs create mode 100644 test/fixtures/es-modules/transpiled-cjs-require-module/src/import-default.mjs create mode 100644 test/fixtures/es-modules/transpiled-cjs-require-module/src/import-named.mjs create mode 100644 test/fixtures/es-modules/transpiled-cjs-require-module/transpile.cjs diff --git a/doc/api/modules.md b/doc/api/modules.md index f2e6a0042b4e1b..a6ac4a27761eaf 100644 --- a/doc/api/modules.md +++ b/doc/api/modules.md @@ -195,33 +195,51 @@ loaded by `require()` meets the following requirements: `"type": "commonjs"`, and `--experimental-detect-module` is enabled. `require()` will load the requested module as an ES Module, and return -the module name space object. In this case it is similar to dynamic +the module namespace object. In this case it is similar to dynamic `import()` but is run synchronously and returns the name space object directly. +With the following ES Modules: + ```mjs -// point.mjs +// distance.mjs export function distance(a, b) { return (b.x - a.x) ** 2 + (b.y - a.y) ** 2; } +``` + +```mjs +// point.mjs class Point { constructor(x, y) { this.x = x; this.y = y; } } export default Point; ``` +A CommonJS module can load them with `require()` under `--experimental-detect-module`: + ```cjs -const required = require('./point.mjs'); +const distance = require('./distance.mjs'); +console.log(distance); // [Module: null prototype] { -// default: [class Point], // distance: [Function: distance] // } -console.log(required); -(async () => { - const imported = await import('./point.mjs'); - console.log(imported === required); // true -})(); +const point = require('./point.mjs'); +console.log(point); +// [Module: null prototype] { +// default: [class Point], +// __esModule: true, +// } ``` +For interoperability with existing tools that convert ES Modules into CommonJS, +which could then load real ES Modules through `require()`, the returned namespace +would contain a `__esModule: true` property if it has a `default` export so that +consuming code generated by tools can recognize the default exports in real +ES Modules. If the namespace already defines `__esModule`, this would not be added. +This property is experimental and can change in the future. It should only be used +by tools converting ES modules into CommonJS modules, following existing ecosystem +conventions. Code authored directly in CommonJS should avoid depending on it. + If the module being `require()`'d contains top-level `await`, or the module graph it `import`s contains top-level `await`, [`ERR_REQUIRE_ASYNC_MODULE`][] will be thrown. In this case, users should diff --git a/lib/internal/modules/cjs/loader.js b/lib/internal/modules/cjs/loader.js index c3bbeef5f0a5f9..a861c68dd7e33c 100644 --- a/lib/internal/modules/cjs/loader.js +++ b/lib/internal/modules/cjs/loader.js @@ -41,6 +41,7 @@ const { ObjectFreeze, ObjectGetOwnPropertyDescriptor, ObjectGetPrototypeOf, + ObjectHasOwn, ObjectKeys, ObjectPrototype, ObjectPrototypeHasOwnProperty, @@ -71,7 +72,7 @@ const { }, } = internalBinding('util'); -const { kEvaluated } = internalBinding('module_wrap'); +const { kEvaluated, createRequiredModuleFacade } = internalBinding('module_wrap'); // Internal properties for Module instances. /** @@ -1340,9 +1341,55 @@ function loadESMFromCJS(mod, filename) { // ESM won't be accessible via process.mainModule. setOwnProperty(process, 'mainModule', undefined); } else { - // TODO(joyeecheung): we may want to invent optional special handling for default exports here. - // For now, it's good enough to be identical to what `import()` returns. - mod.exports = cascadedLoader.importSyncForRequire(mod, filename, source, isMain, mod[kModuleParent]); + const { + wrap, + namespace, + } = cascadedLoader.importSyncForRequire(mod, filename, source, isMain, mod[kModuleParent]); + // Tooling in the ecosystem have been using the __esModule property to recognize + // transpiled ESM in consuming code. For example, a 'log' package written in ESM: + // + // export default function log(val) { console.log(val); } + // + // Can be transpiled as: + // + // exports.__esModule = true; + // exports.default = function log(val) { console.log(val); } + // + // The consuming code may be written like this in ESM: + // + // import log from 'log' + // + // Which gets transpiled to: + // + // const _mod = require('log'); + // const log = _mod.__esModule ? _mod.default : _mod; + // + // So to allow transpiled consuming code to recognize require()'d real ESM + // as ESM and pick up the default exports, we add a __esModule property by + // building a source text module facade for any module that has a default + // export and add .__esModule = true to the exports. This maintains the + // enumerability of the re-exported names and the live binding of the exports, + // without incurring a non-trivial per-access overhead on the exports. + // + // The source of the facade is defined as a constant per-isolate property + // required_module_default_facade_source_string, which looks like this + // + // export * from 'original'; + // export { default } from 'original'; + // export const __esModule = true; + // + // And the 'original' module request is always resolved by + // createRequiredModuleFacade() to `wrap` which is a ModuleWrap wrapping + // over the original module. + + // We don't do this to modules that don't have default exports to avoid + // the unnecessary overhead. If __esModule is already defined, we will + // also skip the extension to allow users to override it. + if (!ObjectHasOwn(namespace, 'default') || ObjectHasOwn(namespace, '__esModule')) { + mod.exports = namespace; + } else { + mod.exports = createRequiredModuleFacade(wrap); + } } } diff --git a/lib/internal/modules/esm/loader.js b/lib/internal/modules/esm/loader.js index f94696fa804759..570454819bb260 100644 --- a/lib/internal/modules/esm/loader.js +++ b/lib/internal/modules/esm/loader.js @@ -269,7 +269,7 @@ class ModuleLoader { * @param {string} source Source code. TODO(joyeecheung): pass the raw buffer. * @param {string} isMain Whether this module is a main module. * @param {CJSModule|undefined} parent Parent module, if any. - * @returns {{ModuleWrap}} + * @returns {{wrap: ModuleWrap, namespace: ModuleNamespaceObject}} */ importSyncForRequire(mod, filename, source, isMain, parent) { const url = pathToFileURL(filename).href; @@ -294,7 +294,7 @@ class ModuleLoader { } throw new ERR_REQUIRE_CYCLE_MODULE(message); } - return job.module.getNamespaceSync(); + return { wrap: job.module, namespace: job.module.getNamespaceSync() }; } // TODO(joyeecheung): refactor this so that we pre-parse in C++ and hit the // cache here, or use a carrier object to carry the compiled module script @@ -306,7 +306,7 @@ class ModuleLoader { job = new ModuleJobSync(this, url, kEmptyObject, wrap, isMain, inspectBrk); this.loadCache.set(url, kImplicitAssertType, job); mod[kRequiredModuleSymbol] = job.module; - return job.runSync().namespace; + return { wrap: job.module, namespace: job.runSync().namespace }; } /** diff --git a/src/env.h b/src/env.h index 2ec0a56e05ff87..598082f68aefe2 100644 --- a/src/env.h +++ b/src/env.h @@ -1061,6 +1061,8 @@ class Environment : public MemoryRetainer { std::vector supported_hash_algorithms; #endif // HAVE_OPENSSL + v8::Global temporary_required_module_facade_original; + private: // V8 has changed the constructor of exceptions, support both APIs before Node // updates to V8 12.1. diff --git a/src/env_properties.h b/src/env_properties.h index 63fea9e479953d..555c8fd09111ea 100644 --- a/src/env_properties.h +++ b/src/env_properties.h @@ -252,6 +252,7 @@ V(openssl_error_stack, "opensslErrorStack") \ V(options_string, "options") \ V(order_string, "order") \ + V(original_string, "original") \ V(output_string, "output") \ V(overlapped_string, "overlapped") \ V(parse_error_string, "Parse Error") \ @@ -285,6 +286,11 @@ V(regexp_string, "regexp") \ V(rename_string, "rename") \ V(replacement_string, "replacement") \ + V(required_module_facade_url_string, \ + "node:internal/require_module_default_facade") \ + V(required_module_facade_source_string, \ + "export * from 'original'; export { default } from 'original'; export " \ + "const __esModule = true;") \ V(require_string, "require") \ V(resource_string, "resource") \ V(retry_string, "retry") \ diff --git a/src/module_wrap.cc b/src/module_wrap.cc index d7ec99736473b6..64bb1ec3db9b14 100644 --- a/src/module_wrap.cc +++ b/src/module_wrap.cc @@ -966,6 +966,70 @@ void ModuleWrap::CreateCachedData(const FunctionCallbackInfo& args) { } } +// This v8::Module::ResolveModuleCallback simply links `import 'original'` +// to the env->temporary_required_module_facade_original() which is stashed +// right before this callback is called and will be restored as soon as +// v8::Module::Instantiate() returns. +MaybeLocal LinkRequireFacadeWithOriginal( + Local context, + Local specifier, + Local import_attributes, + Local referrer) { + Environment* env = Environment::GetCurrent(context); + Isolate* isolate = context->GetIsolate(); + CHECK(specifier->Equals(context, env->original_string()).ToChecked()); + CHECK(!env->temporary_required_module_facade_original.IsEmpty()); + return env->temporary_required_module_facade_original.Get(isolate); +} + +// Wraps an existing source text module with a facade that adds +// .__esModule = true to the exports. +// See env->required_module_facade_source_string() for the source. +void ModuleWrap::CreateRequiredModuleFacade( + const FunctionCallbackInfo& args) { + Isolate* isolate = args.GetIsolate(); + Local context = isolate->GetCurrentContext(); + Environment* env = Environment::GetCurrent(context); + CHECK(args[0]->IsObject()); // original module + Local wrap = args[0].As(); + ModuleWrap* original; + ASSIGN_OR_RETURN_UNWRAP(&original, wrap); + + // Use the same facade source and URL to hit the compilation cache. + ScriptOrigin origin(isolate, + env->required_module_facade_url_string(), + 0, // line offset + 0, // column offset + true, // is cross origin + -1, // script id + Local(), // source map URL + false, // is opaque (?) + false, // is WASM + true); // is ES Module + ScriptCompiler::Source source(env->required_module_facade_source_string(), + origin); + + // The module facade instantiation simply links `import 'original'` in the + // facade with the original module and should never fail. + Local facade = + ScriptCompiler::CompileModule(isolate, &source).ToLocalChecked(); + // Stash the original module in temporary_required_module_facade_original + // for the LinkRequireFacadeWithOriginal() callback to pick it up. + CHECK(env->temporary_required_module_facade_original.IsEmpty()); + env->temporary_required_module_facade_original.Reset( + isolate, original->module_.Get(isolate)); + CHECK(facade->InstantiateModule(context, LinkRequireFacadeWithOriginal) + .IsJust()); + env->temporary_required_module_facade_original.Reset(); + + // The evaluation of the facade is synchronous. + Local evaluated = facade->Evaluate(context).ToLocalChecked(); + CHECK(evaluated->IsPromise()); + CHECK_EQ(evaluated.As()->State(), Promise::PromiseState::kFulfilled); + + args.GetReturnValue().Set(facade->GetModuleNamespace()); +} + void ModuleWrap::CreatePerIsolateProperties(IsolateData* isolate_data, Local target) { Isolate* isolate = isolate_data->isolate(); @@ -998,6 +1062,10 @@ void ModuleWrap::CreatePerIsolateProperties(IsolateData* isolate_data, target, "setInitializeImportMetaObjectCallback", SetInitializeImportMetaObjectCallback); + SetMethod(isolate, + target, + "createRequiredModuleFacade", + CreateRequiredModuleFacade); } void ModuleWrap::CreatePerContextProperties(Local target, @@ -1038,6 +1106,8 @@ void ModuleWrap::RegisterExternalReferences( registry->Register(GetStatus); registry->Register(GetError); + registry->Register(CreateRequiredModuleFacade); + registry->Register(SetImportModuleDynamicallyCallback); registry->Register(SetInitializeImportMetaObjectCallback); } diff --git a/src/module_wrap.h b/src/module_wrap.h index b5d0c48997680d..51b127209af695 100644 --- a/src/module_wrap.h +++ b/src/module_wrap.h @@ -87,6 +87,9 @@ class ModuleWrap : public BaseObject { std::optional user_cached_data, bool* cache_rejected); + static void CreateRequiredModuleFacade( + const v8::FunctionCallbackInfo& args); + private: ModuleWrap(Realm* realm, v8::Local object, diff --git a/test/common/index.js b/test/common/index.js index c39488dd0b9819..9733f8746baa0e 100644 --- a/test/common/index.js +++ b/test/common/index.js @@ -927,9 +927,14 @@ function getPrintedStackTrace(stderr) { * @param {object} mod result returned by require() * @param {object} expectation shape of expected namespace. */ -function expectRequiredModule(mod, expectation) { +function expectRequiredModule(mod, expectation, checkESModule = true) { + const clone = { ...mod }; + if (Object.hasOwn(mod, 'default') && checkESModule) { + assert.strictEqual(mod.__esModule, true); + delete clone.__esModule; + } assert(isModuleNamespaceObject(mod)); - assert.deepStrictEqual({ ...mod }, { ...expectation }); + assert.deepStrictEqual(clone, { ...expectation }); } const common = { diff --git a/test/es-module/test-require-module-default-extension.js b/test/es-module/test-require-module-default-extension.js index 7c49e21aba9a15..c126affe983264 100644 --- a/test/es-module/test-require-module-default-extension.js +++ b/test/es-module/test-require-module-default-extension.js @@ -1,13 +1,11 @@ // Flags: --experimental-require-module 'use strict'; -require('../common'); +const { expectRequiredModule } = require('../common'); const assert = require('assert'); -const { isModuleNamespaceObject } = require('util/types'); const mod = require('../fixtures/es-modules/package-default-extension/index.mjs'); -assert.deepStrictEqual({ ...mod }, { entry: 'mjs' }); -assert(isModuleNamespaceObject(mod)); +expectRequiredModule(mod, { entry: 'mjs' }); assert.throws(() => { const mod = require('../fixtures/es-modules/package-default-extension'); diff --git a/test/es-module/test-require-module-defined-esmodule.js b/test/es-module/test-require-module-defined-esmodule.js new file mode 100644 index 00000000000000..68225ebdbd93cd --- /dev/null +++ b/test/es-module/test-require-module-defined-esmodule.js @@ -0,0 +1,23 @@ +// Flags: --experimental-require-module +'use strict'; +const common = require('../common'); + +// If an ESM already defines __esModule to be something else, +// require(esm) should allow the user override. +{ + const mod = require('../fixtures/es-modules/export-es-module.mjs'); + common.expectRequiredModule( + mod, + { default: { hello: 'world' }, __esModule: 'test' }, + false, + ); +} + +{ + const mod = require('../fixtures/es-modules/export-es-module-2.mjs'); + common.expectRequiredModule( + mod, + { default: { hello: 'world' }, __esModule: false }, + false, + ); +} diff --git a/test/es-module/test-require-module-dynamic-import-1.js b/test/es-module/test-require-module-dynamic-import-1.js index 000e31485f559e..939a2cdbcc93bf 100644 --- a/test/es-module/test-require-module-dynamic-import-1.js +++ b/test/es-module/test-require-module-dynamic-import-1.js @@ -21,8 +21,7 @@ const { pathToFileURL } = require('url'); const url = pathToFileURL(path.resolve(__dirname, id)); const imported = await import(url); const required = require(id); - assert.strictEqual(imported, required, - `import()'ed and require()'ed result of ${id} was not reference equal`); + common.expectRequiredModule(required, imported); } const id = '../fixtures/es-modules/data-import.mjs'; diff --git a/test/es-module/test-require-module-dynamic-import-2.js b/test/es-module/test-require-module-dynamic-import-2.js index 6c31c04f0b2e77..a3e24a800c1803 100644 --- a/test/es-module/test-require-module-dynamic-import-2.js +++ b/test/es-module/test-require-module-dynamic-import-2.js @@ -21,8 +21,7 @@ const path = require('path'); const url = pathToFileURL(path.resolve(__dirname, id)); const required = require(id); const imported = await import(url); - assert.strictEqual(imported, required, - `import()'ed and require()'ed result of ${id} was not reference equal`); + common.expectRequiredModule(required, imported); } const id = '../fixtures/es-modules/data-import.mjs'; diff --git a/test/es-module/test-require-module-dynamic-import-3.js b/test/es-module/test-require-module-dynamic-import-3.js index 7a5fbf1a137f96..53fcdc48c8c552 100644 --- a/test/es-module/test-require-module-dynamic-import-3.js +++ b/test/es-module/test-require-module-dynamic-import-3.js @@ -5,10 +5,9 @@ // be loaded by dynamic import(). const common = require('../common'); -const assert = require('assert'); (async () => { const required = require('../fixtures/es-modules/require-and-import/load.cjs'); const imported = await import('../fixtures/es-modules/require-and-import/load.mjs'); - assert.deepStrictEqual({ ...required }, { ...imported }); + common.expectRequiredModule(required, imported); })().then(common.mustCall()); diff --git a/test/es-module/test-require-module-dynamic-import-4.js b/test/es-module/test-require-module-dynamic-import-4.js index 414cd70d82d33a..88c565d2ba2e47 100644 --- a/test/es-module/test-require-module-dynamic-import-4.js +++ b/test/es-module/test-require-module-dynamic-import-4.js @@ -5,10 +5,9 @@ // be loaded by require(). const common = require('../common'); -const assert = require('assert'); (async () => { const imported = await import('../fixtures/es-modules/require-and-import/load.mjs'); const required = require('../fixtures/es-modules/require-and-import/load.cjs'); - assert.deepStrictEqual({ ...required }, { ...imported }); + common.expectRequiredModule(required, imported); })().then(common.mustCall()); diff --git a/test/es-module/test-require-module-implicit.js b/test/es-module/test-require-module-implicit.js index 5b5a4a4bbb47b0..2e3a5d94352dbb 100644 --- a/test/es-module/test-require-module-implicit.js +++ b/test/es-module/test-require-module-implicit.js @@ -3,9 +3,8 @@ // Tests that require()ing modules without explicit module type information // warns and errors. -require('../common'); +const common = require('../common'); const assert = require('assert'); -const { isModuleNamespaceObject } = require('util/types'); assert.throws(() => { require('../fixtures/es-modules/package-without-type/noext-esm'); @@ -28,6 +27,5 @@ assert.throws(() => { code: 'MODULE_NOT_FOUND' }); const mod = require(`${id}.mjs`); - assert.deepStrictEqual({ ...mod }, { hello: 'world' }); - assert(isModuleNamespaceObject(mod)); + common.expectRequiredModule(mod, { hello: 'world' }); } diff --git a/test/es-module/test-require-module-transpiled.js b/test/es-module/test-require-module-transpiled.js new file mode 100644 index 00000000000000..b927507b876370 --- /dev/null +++ b/test/es-module/test-require-module-transpiled.js @@ -0,0 +1,30 @@ +'use strict'; +require('../common'); +const { spawnSyncAndAssert } = require('../common/child_process'); +const fixtures = require('../common/fixtures'); + +// This is a minimum integration test for CJS transpiled from ESM that tries to load real ESM. + +spawnSyncAndAssert(process.execPath, [ + '--experimental-require-module', + fixtures.path('es-modules', 'transpiled-cjs-require-module', 'dist', 'import-both.cjs'), +], { + trim: true, + stdout: 'import both', +}); + +spawnSyncAndAssert(process.execPath, [ + '--experimental-require-module', + fixtures.path('es-modules', 'transpiled-cjs-require-module', 'dist', 'import-named.cjs'), +], { + trim: true, + stdout: 'import named', +}); + +spawnSyncAndAssert(process.execPath, [ + '--experimental-require-module', + fixtures.path('es-modules', 'transpiled-cjs-require-module', 'dist', 'import-default.cjs'), +], { + trim: true, + stdout: 'import default', +}); diff --git a/test/es-module/test-require-module-with-detection.js b/test/es-module/test-require-module-with-detection.js index 36da19f3b96270..baf993d3253ec7 100644 --- a/test/es-module/test-require-module-with-detection.js +++ b/test/es-module/test-require-module-with-detection.js @@ -1,18 +1,14 @@ // Flags: --experimental-require-module --experimental-detect-module 'use strict'; -require('../common'); -const assert = require('assert'); -const { isModuleNamespaceObject } = require('util/types'); +const common = require('../common'); { const mod = require('../fixtures/es-modules/loose.js'); - assert.deepStrictEqual({ ...mod }, { default: 'module' }); - assert(isModuleNamespaceObject(mod)); + common.expectRequiredModule(mod, { default: 'module' }); } { const mod = require('../fixtures/es-modules/package-without-type/noext-esm'); - assert.deepStrictEqual({ ...mod }, { default: 'module' }); - assert(isModuleNamespaceObject(mod)); + common.expectRequiredModule(mod, { default: 'module' }); } diff --git a/test/es-module/test-require-module.js b/test/es-module/test-require-module.js index 631f5d731a5c86..2cc7d0198837df 100644 --- a/test/es-module/test-require-module.js +++ b/test/es-module/test-require-module.js @@ -3,7 +3,6 @@ const common = require('../common'); const assert = require('assert'); -const { isModuleNamespaceObject } = require('util/types'); common.expectWarning( 'ExperimentalWarning', @@ -14,22 +13,19 @@ common.expectWarning( // Test named exports. { const mod = require('../fixtures/es-module-loaders/module-named-exports.mjs'); - assert.deepStrictEqual({ ...mod }, { foo: 'foo', bar: 'bar' }); - assert(isModuleNamespaceObject(mod)); + common.expectRequiredModule(mod, { foo: 'foo', bar: 'bar' }); } // Test ESM that import ESM. { const mod = require('../fixtures/es-modules/import-esm.mjs'); - assert.deepStrictEqual({ ...mod }, { hello: 'world' }); - assert(isModuleNamespaceObject(mod)); + common.expectRequiredModule(mod, { hello: 'world' }); } // Test ESM that import CJS. { const mod = require('../fixtures/es-modules/cjs-exports.mjs'); - assert.deepStrictEqual({ ...mod }, {}); - assert(isModuleNamespaceObject(mod)); + common.expectRequiredModule(mod, { }); } // Test ESM that require() CJS. @@ -39,24 +35,20 @@ common.expectWarning( // re-export everything from the CJS version. assert.strictEqual(common.mustCall, mjs.mustCall); assert.strictEqual(common.localIPv6Hosts, mjs.localIPv6Hosts); - assert(!isModuleNamespaceObject(common)); - assert(isModuleNamespaceObject(mjs)); } // Test "type": "module" and "main" field in package.json. // Also, test default export. { const mod = require('../fixtures/es-modules/package-type-module'); - assert.deepStrictEqual({ ...mod }, { default: 'package-type-module' }); - assert(isModuleNamespaceObject(mod)); + common.expectRequiredModule(mod, { default: 'package-type-module' }); } // Test data: import. { const mod = require('../fixtures/es-modules/data-import.mjs'); - assert.deepStrictEqual({ ...mod }, { + common.expectRequiredModule(mod, { data: { hello: 'world' }, id: 'data:text/javascript,export default %7B%20hello%3A%20%22world%22%20%7D' }); - assert(isModuleNamespaceObject(mod)); } diff --git a/test/fixtures/es-modules/export-es-module-2.mjs b/test/fixtures/es-modules/export-es-module-2.mjs new file mode 100644 index 00000000000000..81f61095ce75af --- /dev/null +++ b/test/fixtures/es-modules/export-es-module-2.mjs @@ -0,0 +1,2 @@ +export const __esModule = false; +export default { hello: 'world' }; diff --git a/test/fixtures/es-modules/export-es-module.mjs b/test/fixtures/es-modules/export-es-module.mjs new file mode 100644 index 00000000000000..a85dea6c5e3d00 --- /dev/null +++ b/test/fixtures/es-modules/export-es-module.mjs @@ -0,0 +1,2 @@ +export const __esModule = 'test'; +export default { hello: 'world' }; diff --git a/test/fixtures/es-modules/transpiled-cjs-require-module/dist/import-both.cjs b/test/fixtures/es-modules/transpiled-cjs-require-module/dist/import-both.cjs new file mode 100644 index 00000000000000..62c00a7f046304 --- /dev/null +++ b/test/fixtures/es-modules/transpiled-cjs-require-module/dist/import-both.cjs @@ -0,0 +1,27 @@ +"use strict"; +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + var desc = Object.getOwnPropertyDescriptor(m, k); + if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) { + desc = { enumerable: true, get: function() { return m[k]; } }; + } + Object.defineProperty(o, k2, desc); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; +})); +var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { + Object.defineProperty(o, "default", { enumerable: true, value: v }); +}) : function(o, v) { + o["default"] = v; +}); +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); + __setModuleDefault(result, mod); + return result; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +var logger_1 = __importStar(require("logger")); +(0, logger_1.log)(new logger_1.default(), 'import both'); diff --git a/test/fixtures/es-modules/transpiled-cjs-require-module/dist/import-default.cjs b/test/fixtures/es-modules/transpiled-cjs-require-module/dist/import-default.cjs new file mode 100644 index 00000000000000..9880e20de5c91a --- /dev/null +++ b/test/fixtures/es-modules/transpiled-cjs-require-module/dist/import-default.cjs @@ -0,0 +1,7 @@ +"use strict"; +var __importDefault = (this && this.__importDefault) || function (mod) { + return (mod && mod.__esModule) ? mod : { "default": mod }; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +var logger_1 = __importDefault(require("logger")); +new logger_1.default().log('import default'); diff --git a/test/fixtures/es-modules/transpiled-cjs-require-module/dist/import-named.cjs b/test/fixtures/es-modules/transpiled-cjs-require-module/dist/import-named.cjs new file mode 100644 index 00000000000000..9daf6c92ff566d --- /dev/null +++ b/test/fixtures/es-modules/transpiled-cjs-require-module/dist/import-named.cjs @@ -0,0 +1,4 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +var logger_1 = require("logger"); +(0, logger_1.log)(console, 'import named'); diff --git a/test/fixtures/es-modules/transpiled-cjs-require-module/node_modules/logger/logger.mjs b/test/fixtures/es-modules/transpiled-cjs-require-module/node_modules/logger/logger.mjs new file mode 100644 index 00000000000000..abcac43fff428f --- /dev/null +++ b/test/fixtures/es-modules/transpiled-cjs-require-module/node_modules/logger/logger.mjs @@ -0,0 +1,2 @@ +export default class Logger { log(val) { console.log(val); } } +export function log(logger, val) { logger.log(val) }; diff --git a/test/fixtures/es-modules/transpiled-cjs-require-module/node_modules/logger/package.json b/test/fixtures/es-modules/transpiled-cjs-require-module/node_modules/logger/package.json new file mode 100644 index 00000000000000..56c237df953d7a --- /dev/null +++ b/test/fixtures/es-modules/transpiled-cjs-require-module/node_modules/logger/package.json @@ -0,0 +1,4 @@ +{ + "name": "logger", + "main": "logger.mjs" +} \ No newline at end of file diff --git a/test/fixtures/es-modules/transpiled-cjs-require-module/src/import-both.mjs b/test/fixtures/es-modules/transpiled-cjs-require-module/src/import-both.mjs new file mode 100644 index 00000000000000..7773ccb2bc1d20 --- /dev/null +++ b/test/fixtures/es-modules/transpiled-cjs-require-module/src/import-both.mjs @@ -0,0 +1,2 @@ +import Logger, { log } from 'logger'; +log(new Logger(), 'import both'); diff --git a/test/fixtures/es-modules/transpiled-cjs-require-module/src/import-default.mjs b/test/fixtures/es-modules/transpiled-cjs-require-module/src/import-default.mjs new file mode 100644 index 00000000000000..16c123bbccb0f9 --- /dev/null +++ b/test/fixtures/es-modules/transpiled-cjs-require-module/src/import-default.mjs @@ -0,0 +1,2 @@ +import Logger from 'logger'; +new Logger().log('import default'); diff --git a/test/fixtures/es-modules/transpiled-cjs-require-module/src/import-named.mjs b/test/fixtures/es-modules/transpiled-cjs-require-module/src/import-named.mjs new file mode 100644 index 00000000000000..489d0886e542f7 --- /dev/null +++ b/test/fixtures/es-modules/transpiled-cjs-require-module/src/import-named.mjs @@ -0,0 +1,2 @@ +import { log } from 'logger'; +log(console, 'import named'); diff --git a/test/fixtures/es-modules/transpiled-cjs-require-module/transpile.cjs b/test/fixtures/es-modules/transpiled-cjs-require-module/transpile.cjs new file mode 100644 index 00000000000000..da9978164b7c6b --- /dev/null +++ b/test/fixtures/es-modules/transpiled-cjs-require-module/transpile.cjs @@ -0,0 +1,23 @@ +'use strict'; + +// This script is used to transpile ESM fixtures from the src/ directory +// to CJS modules in dist/. The transpiled CJS files are used to test +// integration of transpiled CJS modules loading real ESM. + +const { readFileSync, writeFileSync, readdirSync } = require('node:fs'); + +// We use typescript.js because it's already in the code base as a fixture. +// Most ecosystem tools follow a similar pattern, and this produces a bare +// minimum integration test for existing patterns. +const ts = require('../../snapshot/typescript'); +const { join } = require('node:path'); +const sourceDir = join(__dirname, 'src'); +const files = readdirSync(sourceDir); +for (const filename of files) { + const filePath = join(sourceDir, filename); + const source = readFileSync(filePath, 'utf8'); + const { outputText } = ts.transpileModule(source, { + compilerOptions: { module: ts.ModuleKind.NodeNext } + }); + writeFileSync(join(__dirname, 'dist', filename.replace('.mjs', '.cjs')), outputText, 'utf8'); +} diff --git a/test/parallel/test-runner-module-mocking.js b/test/parallel/test-runner-module-mocking.js index 1906903dfac868..1b19ce49657ad6 100644 --- a/test/parallel/test-runner-module-mocking.js +++ b/test/parallel/test-runner-module-mocking.js @@ -338,7 +338,7 @@ test('ESM mocking with namedExports option', async (t) => { assert.strictEqual(mocked.default, 'mock default'); assert.strictEqual(mocked.val1, 'mock value'); t.mock.reset(); - assert.strictEqual(original, require(fixture)); + common.expectRequiredModule(require(fixture), original); }); await t.test('throws if named exports cannot be applied to defaultExport as CJS', async (t) => { From f6b0e4e20299bb94db79f68daccfa5d1a42142a8 Mon Sep 17 00:00:00 2001 From: Antoine du Hamel Date: Thu, 18 Jul 2024 19:57:40 +0200 Subject: [PATCH 11/33] esm: refactor `get_format` PR-URL: https://github.com/nodejs/node/pull/53872 Reviewed-By: Geoffrey Booth Reviewed-By: Paolo Insogna Reviewed-By: Matteo Collina --- lib/internal/modules/esm/get_format.js | 58 +++++++++++++++----------- 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/lib/internal/modules/esm/get_format.js b/lib/internal/modules/esm/get_format.js index f07f2a42b260a3..5c551d4af2df22 100644 --- a/lib/internal/modules/esm/get_format.js +++ b/lib/internal/modules/esm/get_format.js @@ -17,6 +17,7 @@ const { mimeToFormat, } = require('internal/modules/esm/formats'); +const detectModule = getOptionValue('--experimental-detect-module'); const experimentalNetworkImports = getOptionValue('--experimental-network-imports'); const { containsModuleSyntax } = internalBinding('contextify'); @@ -33,6 +34,17 @@ const protocolHandlers = { 'node:'() { return 'builtin'; }, }; +/** + * Determine whether the given ambiguous source contains CommonJS or ES module syntax. + * @param {string | Buffer | undefined} source + * @param {URL} url + */ +function detectModuleFormat(source, url) { + if (!source) { return detectModule ? null : 'commonjs'; } + if (!detectModule) { return 'commonjs'; } + return containsModuleSyntax(`${source}`, fileURLToPath(url), url) ? 'module' : 'commonjs'; +} + /** * @param {URL} parsed * @returns {string | null} @@ -112,26 +124,23 @@ function getFileProtocolModuleFormat(url, context = { __proto__: null }, ignoreE default: { // The user did not pass `--experimental-default-type`. // `source` is undefined when this is called from `defaultResolve`; // but this gets called again from `defaultLoad`/`defaultLoadSync`. - if (getOptionValue('--experimental-detect-module')) { - const format = source ? - (containsModuleSyntax(`${source}`, fileURLToPath(url), url) ? 'module' : 'commonjs') : - null; - if (format === 'module') { - // This module has a .js extension, a package.json with no `type` field, and ESM syntax. - // Warn about the missing `type` field so that the user can avoid the performance penalty of detection. - typelessPackageJsonFilesWarnedAbout ??= new SafeSet(); - if (!typelessPackageJsonFilesWarnedAbout.has(pjsonPath)) { - const warning = `${url} parsed as an ES module because module syntax was detected;` + - ` to avoid the performance penalty of syntax detection, add "type": "module" to ${pjsonPath}`; - process.emitWarning(warning, { - code: 'MODULE_TYPELESS_PACKAGE_JSON', - }); - typelessPackageJsonFilesWarnedAbout.add(pjsonPath); - } + // For ambiguous files (no type field, .js extension) we return + // undefined from `resolve` and re-run the check in `load`. + const format = detectModuleFormat(source, url); + if (format === 'module') { + // This module has a .js extension, a package.json with no `type` field, and ESM syntax. + // Warn about the missing `type` field so that the user can avoid the performance penalty of detection. + typelessPackageJsonFilesWarnedAbout ??= new SafeSet(); + if (!typelessPackageJsonFilesWarnedAbout.has(pjsonPath)) { + const warning = `${url} parsed as an ES module because module syntax was detected;` + + ` to avoid the performance penalty of syntax detection, add "type": "module" to ${pjsonPath}`; + process.emitWarning(warning, { + code: 'MODULE_TYPELESS_PACKAGE_JSON', + }); + typelessPackageJsonFilesWarnedAbout.add(pjsonPath); } - return format; } - return 'commonjs'; + return format; } } } @@ -154,15 +163,14 @@ function getFileProtocolModuleFormat(url, context = { __proto__: null }, ignoreE return 'commonjs'; } default: { // The user did not pass `--experimental-default-type`. - if (getOptionValue('--experimental-detect-module')) { - if (!source) { return null; } - const format = getFormatOfExtensionlessFile(url); - if (format === 'module') { - return containsModuleSyntax(`${source}`, fileURLToPath(url), url) ? 'module' : 'commonjs'; - } + if (!source) { + return null; + } + const format = getFormatOfExtensionlessFile(url); + if (format === 'wasm') { return format; } - return 'commonjs'; + return detectModuleFormat(source, url); } } } From bea73c8fa6eb72531af8a157390f747f9f00c68b Mon Sep 17 00:00:00 2001 From: Geoffrey Booth Date: Sat, 20 Jul 2024 11:30:46 -0700 Subject: [PATCH 12/33] module: unflag detect-module PR-URL: https://github.com/nodejs/node/pull/53619 Reviewed-By: Benjamin Gruenbaum Reviewed-By: Matteo Collina Reviewed-By: James M Snell Reviewed-By: Antoine du Hamel --- doc/api/cli.md | 48 ++++++---------- doc/api/esm.md | 6 +- doc/api/modules.md | 4 +- doc/api/packages.md | 56 ++++++++++++++++--- lib/internal/main/check_syntax.js | 14 ++--- lib/internal/modules/run_main.js | 2 +- src/node_options.cc | 3 +- src/node_options.h | 2 +- test/es-module/test-esm-cjs-exports.js | 3 +- test/es-module/test-esm-detect-ambiguous.mjs | 40 +++++-------- .../test-esm-extensionless-esm-and-wasm.mjs | 21 +++---- test/es-module/test-esm-import-flag.mjs | 4 +- test/es-module/test-esm-loader-hooks.mjs | 16 +++--- test/es-module/test-esm-resolve-type.mjs | 12 ++-- ...t-require-module-detect-entry-point-aou.js | 2 +- .../test-require-module-detect-entry-point.js | 2 +- .../test-require-module-dont-detect-cjs.js | 2 +- .../es-module/test-require-module-implicit.js | 12 ---- .../test-require-module-with-detection.js | 2 +- .../builtin-named-exports-loader.mjs | 2 +- 20 files changed, 122 insertions(+), 131 deletions(-) diff --git a/doc/api/cli.md b/doc/api/cli.md index df7031f65d44f9..9d771266035cac 100644 --- a/doc/api/cli.md +++ b/doc/api/cli.md @@ -887,38 +887,6 @@ files with no extension will be treated as WebAssembly if they begin with the WebAssembly magic number (`\0asm`); otherwise they will be treated as ES module JavaScript. -### `--experimental-detect-module` - - - -> Stability: 1.1 - Active development - -Node.js will inspect the source code of ambiguous input to determine whether it -contains ES module syntax; if such syntax is detected, the input will be treated -as an ES module. - -Ambiguous input is defined as: - -* Files with a `.js` extension or no extension; and either no controlling - `package.json` file or one that lacks a `type` field; and - `--experimental-default-type` is not specified. -* String input (`--eval` or STDIN) when neither `--input-type` nor - `--experimental-default-type` are specified. - -ES module syntax is defined as syntax that would throw when evaluated as -CommonJS. This includes the following: - -* `import` statements (but _not_ `import()` expressions, which are valid in - CommonJS). -* `export` statements. -* `import.meta` references. -* `await` at the top level of a module. -* Lexical redeclarations of the CommonJS wrapper variables (`require`, `module`, - `exports`, `__dirname`, `__filename`). - ### `--experimental-eventsource` + +Disable using [syntax detection][] to determine module type. + ### `--no-experimental-fetch` + +> Stability: 1.2 - Release candidate + +Node.js will inspect the source code of ambiguous input to determine whether it +contains ES module syntax; if such syntax is detected, the input will be treated +as an ES module. + +Ambiguous input is defined as: + +* Files with a `.js` extension or no extension; and either no controlling + `package.json` file or one that lacks a `type` field; and + `--experimental-default-type` is not specified. +* String input (`--eval` or STDIN) when neither `--input-type` nor + `--experimental-default-type` are specified. + +ES module syntax is defined as syntax that would throw when evaluated as +CommonJS. This includes the following: + +* `import` statements (but _not_ `import()` expressions, which are valid in + CommonJS). +* `export` statements. +* `import.meta` references. +* `await` at the top level of a module. +* Lexical redeclarations of the CommonJS wrapper variables (`require`, `module`, + `exports`, `__dirname`, `__filename`). + ### Modules loaders Node.js has two systems for resolving a specifier and loading modules. @@ -1369,6 +1407,7 @@ This field defines [subpath imports][] for the current package. [ES modules]: esm.md [Node.js documentation for this section]: https://github.com/nodejs/node/blob/HEAD/doc/api/packages.md#conditions-definitions [Runtime Keys]: https://runtime-keys.proposal.wintercg.org/ +[Syntax detection]: #syntax-detection [WinterCG]: https://wintercg.org/ [`"exports"`]: #exports [`"imports"`]: #imports @@ -1378,7 +1417,6 @@ This field defines [subpath imports][] for the current package. [`"type"`]: #type [`--conditions` / `-C` flag]: #resolving-user-conditions [`--experimental-default-type`]: cli.md#--experimental-default-typetype -[`--experimental-detect-module`]: cli.md#--experimental-detect-module [`--no-addons` flag]: cli.md#--no-addons [`ERR_PACKAGE_PATH_NOT_EXPORTED`]: errors.md#err_package_path_not_exported [`esm`]: https://github.com/standard-things/esm#readme diff --git a/lib/internal/main/check_syntax.js b/lib/internal/main/check_syntax.js index aa14dca8999e89..aa521dea92e314 100644 --- a/lib/internal/main/check_syntax.js +++ b/lib/internal/main/check_syntax.js @@ -57,23 +57,23 @@ function loadESMIfNeeded(cb) { } async function checkSyntax(source, filename) { - let isModule = true; + let format; if (filename === '[stdin]' || filename === '[eval]') { - isModule = getOptionValue('--input-type') === 'module' || - (getOptionValue('--experimental-default-type') === 'module' && getOptionValue('--input-type') !== 'commonjs'); + format = (getOptionValue('--input-type') === 'module' || + (getOptionValue('--experimental-default-type') === 'module' && getOptionValue('--input-type') !== 'commonjs')) ? + 'module' : 'commonjs'; } else { const { defaultResolve } = require('internal/modules/esm/resolve'); const { defaultGetFormat } = require('internal/modules/esm/get_format'); const { url } = await defaultResolve(pathToFileURL(filename).toString()); - const format = await defaultGetFormat(new URL(url)); - isModule = format === 'module'; + format = await defaultGetFormat(new URL(url)); } - if (isModule) { + if (format === 'module') { const { ModuleWrap } = internalBinding('module_wrap'); new ModuleWrap(filename, undefined, source, 0, 0); return; } - wrapSafe(filename, source, undefined, 'commonjs'); + wrapSafe(filename, source, undefined, format); } diff --git a/lib/internal/modules/run_main.js b/lib/internal/modules/run_main.js index 7f6a1e9536d7d4..a22bddf635f5e6 100644 --- a/lib/internal/modules/run_main.js +++ b/lib/internal/modules/run_main.js @@ -148,7 +148,7 @@ function runEntryPointWithESMLoader(callback) { * by `require('module')`) even when the entry point is ESM. * This monkey-patchable code is bypassed under `--experimental-default-type=module`. * Because of backwards compatibility, this function is exposed publicly via `import { runMain } from 'node:module'`. - * When `--experimental-detect-module` is passed, this function will attempt to run ambiguous (no explicit extension, no + * Because of module detection, this function will attempt to run ambiguous (no explicit extension, no * `package.json` type field) entry points as CommonJS first; under certain conditions, it will retry running as ESM. * @param {string} main - First positional CLI argument, such as `'entry.js'` from `node entry.js` */ diff --git a/src/node_options.cc b/src/node_options.cc index f6ff810953b224..2491524b89fc27 100644 --- a/src/node_options.cc +++ b/src/node_options.cc @@ -364,7 +364,8 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() { "when ambiguous modules fail to evaluate because they contain " "ES module syntax, try again to evaluate them as ES modules", &EnvironmentOptions::detect_module, - kAllowedInEnvvar); + kAllowedInEnvvar, + true); AddOption("--experimental-print-required-tla", "Print pending top-level await. If --experimental-require-module " "is true, evaluate asynchronous graphs loaded by `require()` but " diff --git a/src/node_options.h b/src/node_options.h index 10c220f6612233..70e8998284386d 100644 --- a/src/node_options.h +++ b/src/node_options.h @@ -115,7 +115,7 @@ class EnvironmentOptions : public Options { public: bool abort_on_uncaught_exception = false; std::vector conditions; - bool detect_module = false; + bool detect_module = true; bool print_required_tla = false; bool require_module = false; std::string dns_result_order; diff --git a/test/es-module/test-esm-cjs-exports.js b/test/es-module/test-esm-cjs-exports.js index 4f79d2ce4bcb8c..6d79517073ab83 100644 --- a/test/es-module/test-esm-cjs-exports.js +++ b/test/es-module/test-esm-cjs-exports.js @@ -21,9 +21,8 @@ describe('ESM: importing CJS', { concurrency: true }, () => { const invalidEntry = fixtures.path('/es-modules/cjs-exports-invalid.mjs'); const { code, signal, stderr } = await spawnPromisified(execPath, [invalidEntry]); + assert.match(stderr, /SyntaxError: The requested module '\.\/invalid-cjs\.js' does not provide an export named 'default'/); assert.strictEqual(code, 1); assert.strictEqual(signal, null); - assert.ok(stderr.includes('Warning: To load an ES module')); - assert.ok(stderr.includes('Unexpected token \'export\'')); }); }); diff --git a/test/es-module/test-esm-detect-ambiguous.mjs b/test/es-module/test-esm-detect-ambiguous.mjs index 8c70542b2ead72..91c8489d30df09 100644 --- a/test/es-module/test-esm-detect-ambiguous.mjs +++ b/test/es-module/test-esm-detect-ambiguous.mjs @@ -4,11 +4,10 @@ import { spawn } from 'node:child_process'; import { describe, it } from 'node:test'; import { strictEqual, match } from 'node:assert'; -describe('--experimental-detect-module', { concurrency: true }, () => { - describe('string input', { concurrency: true }, () => { +describe('Module syntax detection', { concurrency: !process.env.TEST_PARALLEL }, () => { + describe('string input', { concurrency: !process.env.TEST_PARALLEL }, () => { it('permits ESM syntax in --eval input without requiring --input-type=module', async () => { const { stdout, stderr, code, signal } = await spawnPromisified(process.execPath, [ - '--experimental-detect-module', '--eval', 'import { version } from "node:process"; console.log(version);', ]); @@ -22,9 +21,7 @@ describe('--experimental-detect-module', { concurrency: true }, () => { // ESM is unsupported for --print via --input-type=module it('permits ESM syntax in STDIN input without requiring --input-type=module', async () => { - const child = spawn(process.execPath, [ - '--experimental-detect-module', - ]); + const child = spawn(process.execPath, []); child.stdin.end('console.log(typeof import.meta.resolve)'); match((await child.stdout.toArray()).toString(), /^function\r?\n$/); @@ -32,7 +29,6 @@ describe('--experimental-detect-module', { concurrency: true }, () => { it('should be overridden by --input-type', async () => { const { code, signal, stdout, stderr } = await spawnPromisified(process.execPath, [ - '--experimental-detect-module', '--input-type=commonjs', '--eval', 'import.meta.url', @@ -46,7 +42,6 @@ describe('--experimental-detect-module', { concurrency: true }, () => { it('should not switch to module if code is parsable as script', async () => { const { code, signal, stdout, stderr } = await spawnPromisified(process.execPath, [ - '--experimental-detect-module', '--eval', 'let __filename,__dirname,require,module,exports;this.a', ]); @@ -59,7 +54,6 @@ describe('--experimental-detect-module', { concurrency: true }, () => { it('should be overridden by --experimental-default-type', async () => { const { code, signal, stdout, stderr } = await spawnPromisified(process.execPath, [ - '--experimental-detect-module', '--experimental-default-type=commonjs', '--eval', 'import.meta.url', @@ -73,7 +67,6 @@ describe('--experimental-detect-module', { concurrency: true }, () => { it('does not trigger detection via source code `eval()`', async () => { const { code, signal, stdout, stderr } = await spawnPromisified(process.execPath, [ - '--experimental-detect-module', '--eval', 'eval("import \'nonexistent\';");', ]); @@ -115,7 +108,6 @@ describe('--experimental-detect-module', { concurrency: true }, () => { it(testName, async () => { const { stdout, stderr, code, signal } = await spawnPromisified(process.execPath, [ '--no-warnings', - '--experimental-detect-module', entryPath, ]); @@ -157,7 +149,6 @@ describe('--experimental-detect-module', { concurrency: true }, () => { it(testName, async () => { const { stdout, stderr, code, signal } = await spawnPromisified(process.execPath, [ '--no-warnings', - '--experimental-detect-module', entryPath, ]); @@ -171,7 +162,6 @@ describe('--experimental-detect-module', { concurrency: true }, () => { it('should not hint wrong format in resolve hook', async () => { let writeSync; const { stdout, stderr, code, signal } = await spawnPromisified(process.execPath, [ - '--experimental-detect-module', '--no-warnings', '--loader', `data:text/javascript,import { writeSync } from "node:fs"; export ${encodeURIComponent( @@ -209,7 +199,6 @@ describe('--experimental-detect-module', { concurrency: true }, () => { ]) { it(testName, async () => { const { stdout, stderr, code, signal } = await spawnPromisified(process.execPath, [ - '--experimental-detect-module', entryPath, ]); @@ -238,7 +227,6 @@ describe('--experimental-detect-module', { concurrency: true }, () => { ]) { it(testName, async () => { const { stdout, stderr, code, signal } = await spawnPromisified(process.execPath, [ - '--experimental-detect-module', entryPath, ]); @@ -254,7 +242,6 @@ describe('--experimental-detect-module', { concurrency: true }, () => { describe('syntax that errors in CommonJS but works in ESM', { concurrency: true }, () => { it('permits top-level `await`', async () => { const { stdout, stderr, code, signal } = await spawnPromisified(process.execPath, [ - '--experimental-detect-module', '--eval', 'await Promise.resolve(); console.log("executed");', ]); @@ -267,7 +254,6 @@ describe('--experimental-detect-module', { concurrency: true }, () => { it('permits top-level `await` above import/export syntax', async () => { const { stdout, stderr, code, signal } = await spawnPromisified(process.execPath, [ - '--experimental-detect-module', '--eval', 'await Promise.resolve(); import "node:os"; console.log("executed");', ]); @@ -280,7 +266,6 @@ describe('--experimental-detect-module', { concurrency: true }, () => { it('still throws on `await` in an ordinary sync function', async () => { const { stdout, stderr, code, signal } = await spawnPromisified(process.execPath, [ - '--experimental-detect-module', '--eval', 'function fn() { await Promise.resolve(); } fn();', ]); @@ -293,7 +278,6 @@ describe('--experimental-detect-module', { concurrency: true }, () => { it('throws on undefined `require` when top-level `await` triggers ESM parsing', async () => { const { stdout, stderr, code, signal } = await spawnPromisified(process.execPath, [ - '--experimental-detect-module', '--eval', 'const fs = require("node:fs"); await Promise.resolve();', ]); @@ -307,7 +291,6 @@ describe('--experimental-detect-module', { concurrency: true }, () => { it('permits declaration of CommonJS module variables', async () => { const { stdout, stderr, code, signal } = await spawnPromisified(process.execPath, [ '--no-warnings', - '--experimental-detect-module', fixtures.path('es-modules/package-without-type/commonjs-wrapper-variables.js'), ]); @@ -319,7 +302,6 @@ describe('--experimental-detect-module', { concurrency: true }, () => { it('permits declaration of CommonJS module variables above import/export', async () => { const { stdout, stderr, code, signal } = await spawnPromisified(process.execPath, [ - '--experimental-detect-module', '--eval', 'const module = 3; import "node:os"; console.log("executed");', ]); @@ -332,7 +314,6 @@ describe('--experimental-detect-module', { concurrency: true }, () => { it('still throws on double `const` declaration not at the top level', async () => { const { stdout, stderr, code, signal } = await spawnPromisified(process.execPath, [ - '--experimental-detect-module', '--eval', 'function fn() { const require = 1; const require = 2; } fn();', ]); @@ -361,7 +342,6 @@ describe('--experimental-detect-module', { concurrency: true }, () => { ]) { it(testName, async () => { const { stdout, stderr, code, signal } = await spawnPromisified(process.execPath, [ - '--experimental-detect-module', entryPath, ]); @@ -374,7 +354,6 @@ describe('--experimental-detect-module', { concurrency: true }, () => { it('warns only once for a package.json that affects multiple files', async () => { const { stdout, stderr, code, signal } = await spawnPromisified(process.execPath, [ - '--experimental-detect-module', fixtures.path('es-modules/package-without-type/detected-as-esm.js'), ]); @@ -384,6 +363,18 @@ describe('--experimental-detect-module', { concurrency: true }, () => { strictEqual(code, 0); strictEqual(signal, null); }); + + it('can be disabled via --no-experimental-detect-module', async () => { + const { stdout, stderr, code, signal } = await spawnPromisified(process.execPath, [ + '--no-experimental-detect-module', + fixtures.path('es-modules/package-without-type/module.js'), + ]); + + match(stderr, /SyntaxError: Unexpected token 'export'/); + strictEqual(stdout, ''); + strictEqual(code, 1); + strictEqual(signal, null); + }); }); }); @@ -410,7 +401,6 @@ describe('Wrapping a `require` of an ES module while using `--abort-on-uncaught- describe('when working with Worker threads', () => { it('should support sloppy scripts that declare CJS "global-like" variables', async () => { const { code, signal, stdout, stderr } = await spawnPromisified(process.execPath, [ - '--experimental-detect-module', '--eval', 'new worker_threads.Worker("let __filename,__dirname,require,module,exports;this.a",{eval:true})', ]); diff --git a/test/es-module/test-esm-extensionless-esm-and-wasm.mjs b/test/es-module/test-esm-extensionless-esm-and-wasm.mjs index db20bc047feec1..5ae6c7dc6ae2ff 100644 --- a/test/es-module/test-esm-extensionless-esm-and-wasm.mjs +++ b/test/es-module/test-esm-extensionless-esm-and-wasm.mjs @@ -55,27 +55,20 @@ describe('extensionless Wasm modules within a "type": "module" package scope', { }); }); -describe('extensionless ES modules within no package scope', { concurrency: true }, () => { - // This succeeds with `--experimental-default-type=module` - it('should error as the entry point', async () => { +describe('extensionless ES modules within no package scope', { concurrency: !process.env.TEST_PARALLEL }, () => { + it('should run as the entry point', async () => { const { code, signal, stdout, stderr } = await spawnPromisified(process.execPath, [ fixtures.path('es-modules/noext-esm'), ]); - match(stderr, /SyntaxError/); - strictEqual(stdout, ''); - strictEqual(code, 1); + strictEqual(stdout, 'executed\n'); + strictEqual(stderr, ''); + strictEqual(code, 0); strictEqual(signal, null); }); - // This succeeds with `--experimental-default-type=module` - it('should error on import', async () => { - try { - await import(fixtures.fileURL('es-modules/noext-esm')); - mustNotCall(); - } catch (err) { - ok(err instanceof SyntaxError); - } + it('should run on import', async () => { + await import(fixtures.fileURL('es-modules/noext-esm')); }); }); diff --git a/test/es-module/test-esm-import-flag.mjs b/test/es-module/test-esm-import-flag.mjs index ede317b1d585de..63f5db1fa23d7b 100644 --- a/test/es-module/test-esm-import-flag.mjs +++ b/test/es-module/test-esm-import-flag.mjs @@ -146,9 +146,9 @@ describe('import modules using --import', { concurrency: true }, () => { ] ); - assert.match(stderr, /SyntaxError: Unexpected token 'export'/); + assert.strictEqual(stderr, ''); assert.match(stdout, /^\.mjs file\r?\n$/); - assert.strictEqual(code, 1); + assert.strictEqual(code, 0); assert.strictEqual(signal, null); }); diff --git a/test/es-module/test-esm-loader-hooks.mjs b/test/es-module/test-esm-loader-hooks.mjs index 5b367c7519b16e..50c414ff50829c 100644 --- a/test/es-module/test-esm-loader-hooks.mjs +++ b/test/es-module/test-esm-loader-hooks.mjs @@ -716,15 +716,15 @@ describe('Loader hooks', { concurrency: true }, () => { '--no-warnings', '--experimental-loader', `data:text/javascript,import{readFile}from"node:fs/promises";import{fileURLToPath}from"node:url";export ${ - async function load(u, c, n) { - const r = await n(u, c); - if (u.endsWith('/common/index.js')) { - r.source = '"use strict";module.exports=require("node:module").createRequire(' + - `${JSON.stringify(u)})(${JSON.stringify(fileURLToPath(u))});\n`; - } else if (c.format === 'commonjs') { - r.source = await readFile(new URL(u)); + async function load(url, context, nextLoad) { + const result = await nextLoad(url, context); + if (url.endsWith('/common/index.js')) { + result.source = '"use strict";module.exports=require("node:module").createRequire(' + + `${JSON.stringify(url)})(${JSON.stringify(fileURLToPath(url))});\n`; + } else if (url.startsWith('file:') && (context.format == null || context.format === 'commonjs')) { + result.source = await readFile(new URL(url)); } - return r; + return result; }}`, '--experimental-loader', fixtures.fileURL('es-module-loaders/loader-resolve-passthru.mjs'), diff --git a/test/es-module/test-esm-resolve-type.mjs b/test/es-module/test-esm-resolve-type.mjs index 0f442ed569f848..9288ce95c55879 100644 --- a/test/es-module/test-esm-resolve-type.mjs +++ b/test/es-module/test-esm-resolve-type.mjs @@ -40,8 +40,8 @@ try { [ [ '/es-modules/package-type-module/index.js', 'module' ], [ '/es-modules/package-type-commonjs/index.js', 'commonjs' ], - [ '/es-modules/package-without-type/index.js', 'commonjs' ], - [ '/es-modules/package-without-pjson/index.js', 'commonjs' ], + [ '/es-modules/package-without-type/index.js', null ], + [ '/es-modules/package-without-pjson/index.js', null ], ].forEach(([ testScript, expectedType ]) => { const resolvedPath = path.resolve(fixtures.path(testScript)); const resolveResult = resolve(url.pathToFileURL(resolvedPath)); @@ -54,11 +54,11 @@ try { * * for test-module-ne: everything .js that is not 'module' is 'commonjs' */ - for (const [ moduleName, moduleExtenstion, moduleType, expectedResolvedType ] of + for (const [ moduleName, moduleExtension, moduleType, expectedResolvedType ] of [ [ 'test-module-mainjs', 'js', 'module', 'module'], [ 'test-module-mainmjs', 'mjs', 'module', 'module'], [ 'test-module-cjs', 'js', 'commonjs', 'commonjs'], - [ 'test-module-ne', 'js', undefined, 'commonjs'], + [ 'test-module-ne', 'js', undefined, null], ]) { process.chdir(previousCwd); tmpdir.refresh(); @@ -72,14 +72,14 @@ try { const mDir = rel(`node_modules/${moduleName}`); const subDir = rel(`node_modules/${moduleName}/subdir`); const pkg = rel(`node_modules/${moduleName}/package.json`); - const script = rel(`node_modules/${moduleName}/subdir/mainfile.${moduleExtenstion}`); + const script = rel(`node_modules/${moduleName}/subdir/mainfile.${moduleExtension}`); createDir(nmDir); createDir(mDir); createDir(subDir); const pkgJsonContent = { ...(moduleType !== undefined) && { type: moduleType }, - main: `subdir/mainfile.${moduleExtenstion}` + main: `subdir/mainfile.${moduleExtension}` }; fs.writeFileSync(pkg, JSON.stringify(pkgJsonContent)); fs.writeFileSync(script, diff --git a/test/es-module/test-require-module-detect-entry-point-aou.js b/test/es-module/test-require-module-detect-entry-point-aou.js index e92d4d8273d708..128615761f10ea 100644 --- a/test/es-module/test-require-module-detect-entry-point-aou.js +++ b/test/es-module/test-require-module-detect-entry-point-aou.js @@ -1,4 +1,4 @@ -// Flags: --experimental-require-module --experimental-detect-module --abort-on-uncaught-exception +// Flags: --experimental-require-module --abort-on-uncaught-exception import { mustCall } from '../common/index.mjs'; const fn = mustCall(() => { diff --git a/test/es-module/test-require-module-detect-entry-point.js b/test/es-module/test-require-module-detect-entry-point.js index d7b479383fbeb8..253fe06fdb7a3d 100644 --- a/test/es-module/test-require-module-detect-entry-point.js +++ b/test/es-module/test-require-module-detect-entry-point.js @@ -1,4 +1,4 @@ -// Flags: --experimental-require-module --experimental-detect-module +// Flags: --experimental-require-module import { mustCall } from '../common/index.mjs'; const fn = mustCall(() => { diff --git a/test/es-module/test-require-module-dont-detect-cjs.js b/test/es-module/test-require-module-dont-detect-cjs.js index b4b5b7387d6663..99f49bc59d17c4 100644 --- a/test/es-module/test-require-module-dont-detect-cjs.js +++ b/test/es-module/test-require-module-dont-detect-cjs.js @@ -1,4 +1,4 @@ -// Flags: --experimental-require-module --experimental-detect-module +// Flags: --experimental-require-module 'use strict'; require('../common'); diff --git a/test/es-module/test-require-module-implicit.js b/test/es-module/test-require-module-implicit.js index 2e3a5d94352dbb..e9483ba4da1192 100644 --- a/test/es-module/test-require-module-implicit.js +++ b/test/es-module/test-require-module-implicit.js @@ -6,18 +6,6 @@ const common = require('../common'); const assert = require('assert'); -assert.throws(() => { - require('../fixtures/es-modules/package-without-type/noext-esm'); -}, { - message: /Unexpected token 'export'/ -}); - -assert.throws(() => { - require('../fixtures/es-modules/loose.js'); -}, { - message: /Unexpected token 'export'/ -}); - { // .mjs should not be matched as default extensions. const id = '../fixtures/es-modules/should-not-be-resolved'; diff --git a/test/es-module/test-require-module-with-detection.js b/test/es-module/test-require-module-with-detection.js index baf993d3253ec7..cd94d5fc88e907 100644 --- a/test/es-module/test-require-module-with-detection.js +++ b/test/es-module/test-require-module-with-detection.js @@ -1,4 +1,4 @@ -// Flags: --experimental-require-module --experimental-detect-module +// Flags: --experimental-require-module 'use strict'; const common = require('../common'); diff --git a/test/fixtures/es-module-loaders/builtin-named-exports-loader.mjs b/test/fixtures/es-module-loaders/builtin-named-exports-loader.mjs index fca31c585a6ea9..f1b770d04f4f61 100644 --- a/test/fixtures/es-module-loaders/builtin-named-exports-loader.mjs +++ b/test/fixtures/es-module-loaders/builtin-named-exports-loader.mjs @@ -28,7 +28,7 @@ export function load(url, context, next) { source: generateBuiltinModule(urlObj.pathname), format: 'commonjs', }; - } else if (context.format === 'commonjs') { + } else if (context.format === undefined || context.format === null || context.format === 'commonjs') { return { shortCircuit: true, source: readFileSync(new URL(url)), From 4e3963ff33c7f44d4dce540b56e17987fb5dedba Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Mon, 29 Jul 2024 19:46:41 +0200 Subject: [PATCH 13/33] module: do not warn for typeless package.json when there isn't one It was intended that warnings should only be emitted for an existing package.json without a type. This fixes a confusing warning telling users to update /package.json when there are no package.json on the lookup path at all, like this: [MODULE_TYPELESS_PACKAGE_JSON] Warning: ... parsed as an ES module because module syntax was detected; to avoid the performance penalty of syntax detection, add "type": "module" to /package.json Drive-by: update the warning message to be clear about reparsing and make it clear what's actionable. PR-URL: https://github.com/nodejs/node/pull/54045 Reviewed-By: Geoffrey Booth Reviewed-By: Antoine du Hamel Reviewed-By: Benjamin Gruenbaum Reviewed-By: Matteo Collina Reviewed-By: James M Snell --- lib/internal/modules/esm/get_format.js | 27 ++++++++++++-------- test/es-module/test-esm-detect-ambiguous.mjs | 12 +++++++++ 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/lib/internal/modules/esm/get_format.js b/lib/internal/modules/esm/get_format.js index 5c551d4af2df22..ae7b73008f3d06 100644 --- a/lib/internal/modules/esm/get_format.js +++ b/lib/internal/modules/esm/get_format.js @@ -95,6 +95,19 @@ function underNodeModules(url) { } let typelessPackageJsonFilesWarnedAbout; +function warnTypelessPackageJsonFile(pjsonPath, url) { + typelessPackageJsonFilesWarnedAbout ??= new SafeSet(); + if (!typelessPackageJsonFilesWarnedAbout.has(pjsonPath)) { + const warning = `Module type of ${url} is not specified and it doesn't parse as CommonJS.\n` + + 'Reparsing as ES module because module syntax was detected. This incurs a performance overhead.\n' + + `To eliminate this warning, add "type": "module" to ${pjsonPath}.`; + process.emitWarning(warning, { + code: 'MODULE_TYPELESS_PACKAGE_JSON', + }); + typelessPackageJsonFilesWarnedAbout.add(pjsonPath); + } +} + /** * @param {URL} url * @param {{parentURL: string; source?: Buffer}} context @@ -106,7 +119,7 @@ function getFileProtocolModuleFormat(url, context = { __proto__: null }, ignoreE const ext = extname(url); if (ext === '.js') { - const { type: packageType, pjsonPath } = getPackageScopeConfig(url); + const { type: packageType, pjsonPath, exists: foundPackageJson } = getPackageScopeConfig(url); if (packageType !== 'none') { return packageType; } @@ -127,18 +140,10 @@ function getFileProtocolModuleFormat(url, context = { __proto__: null }, ignoreE // For ambiguous files (no type field, .js extension) we return // undefined from `resolve` and re-run the check in `load`. const format = detectModuleFormat(source, url); - if (format === 'module') { + if (format === 'module' && foundPackageJson) { // This module has a .js extension, a package.json with no `type` field, and ESM syntax. // Warn about the missing `type` field so that the user can avoid the performance penalty of detection. - typelessPackageJsonFilesWarnedAbout ??= new SafeSet(); - if (!typelessPackageJsonFilesWarnedAbout.has(pjsonPath)) { - const warning = `${url} parsed as an ES module because module syntax was detected;` + - ` to avoid the performance penalty of syntax detection, add "type": "module" to ${pjsonPath}`; - process.emitWarning(warning, { - code: 'MODULE_TYPELESS_PACKAGE_JSON', - }); - typelessPackageJsonFilesWarnedAbout.add(pjsonPath); - } + warnTypelessPackageJsonFile(pjsonPath, url); } return format; } diff --git a/test/es-module/test-esm-detect-ambiguous.mjs b/test/es-module/test-esm-detect-ambiguous.mjs index 91c8489d30df09..2d7df8d308082f 100644 --- a/test/es-module/test-esm-detect-ambiguous.mjs +++ b/test/es-module/test-esm-detect-ambiguous.mjs @@ -352,6 +352,18 @@ describe('Module syntax detection', { concurrency: !process.env.TEST_PARALLEL }, }); } + it('does not warn when there are no package.json', async () => { + const { stdout, stderr, code, signal } = await spawnPromisified(process.execPath, [ + fixtures.path('es-modules/loose.js'), + ]); + + strictEqual(stderr, ''); + strictEqual(stdout, 'executed\n'); + strictEqual(code, 0); + strictEqual(signal, null); + }); + + it('warns only once for a package.json that affects multiple files', async () => { const { stdout, stderr, code, signal } = await spawnPromisified(process.execPath, [ fixtures.path('es-modules/package-without-type/detected-as-esm.js'), From 11f03faec671dc87c54a1d5ea6cdcb9134227518 Mon Sep 17 00:00:00 2001 From: Antoine du Hamel Date: Wed, 11 Sep 2024 23:48:02 +0200 Subject: [PATCH 14/33] esm: throw `ERR_REQUIRE_ESM` instead of `ERR_INTERNAL_ASSERTION` PR-URL: https://github.com/nodejs/node/pull/54868 Fixes: https://github.com/nodejs/node/issues/54773 Reviewed-By: James M Snell Reviewed-By: Luigi Pinca --- lib/internal/modules/esm/loader.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/internal/modules/esm/loader.js b/lib/internal/modules/esm/loader.js index 570454819bb260..d5ffd5f32f599e 100644 --- a/lib/internal/modules/esm/loader.js +++ b/lib/internal/modules/esm/loader.js @@ -318,8 +318,6 @@ class ModuleLoader { * @returns {ModuleJobBase} */ getModuleJobForRequire(specifier, parentURL, importAttributes) { - assert(getOptionValue('--experimental-require-module')); - const parsed = URLParse(specifier); if (parsed != null) { const protocol = parsed.protocol; @@ -338,6 +336,9 @@ class ModuleLoader { } const { url, format } = resolveResult; + if (!getOptionValue('--experimental-require-module')) { + throw new ERR_REQUIRE_ESM(url, true); + } const resolvedImportAttributes = resolveResult.importAttributes ?? importAttributes; let job = this.loadCache.get(url, resolvedImportAttributes.type); if (job !== undefined) { From 3348ece6cf676fd746ec910842d1965940a21c74 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Tue, 27 Aug 2024 17:49:25 +0200 Subject: [PATCH 15/33] module: remove bogus assertion in CJS entrypoint handling with --import The synchronous CJS translator can handle entrypoints now, this can be hit when --import is used, so lift the bogus assertions and added tests. PR-URL: https://github.com/nodejs/node/pull/54592 Fixes: https://github.com/nodejs/node/issues/54577 Reviewed-By: James M Snell Reviewed-By: Matteo Collina Reviewed-By: Antoine du Hamel --- lib/internal/modules/esm/translators.js | 2 - test/es-module/test-require-module-preload.js | 155 ++++++++++++------ 2 files changed, 101 insertions(+), 56 deletions(-) diff --git a/lib/internal/modules/esm/translators.js b/lib/internal/modules/esm/translators.js index 8007427fe2337d..a4100d9ca63b64 100644 --- a/lib/internal/modules/esm/translators.js +++ b/lib/internal/modules/esm/translators.js @@ -284,11 +284,9 @@ function createCJSModuleWrap(url, source, isMain, loadCJS = loadCJSModule) { translators.set('commonjs-sync', function requireCommonJS(url, source, isMain) { initCJSParseSync(); - assert(!isMain); // This is only used by imported CJS modules. return createCJSModuleWrap(url, source, isMain, (module, source, url, filename, isMain) => { assert(module === CJSModule._cache[filename]); - assert(!isMain); CJSModule._load(filename, null, isMain); }); }); diff --git a/test/es-module/test-require-module-preload.js b/test/es-module/test-require-module-preload.js index cd51e201b63df8..f2e572969a050c 100644 --- a/test/es-module/test-require-module-preload.js +++ b/test/es-module/test-require-module-preload.js @@ -1,70 +1,117 @@ 'use strict'; require('../common'); -const { spawnSyncAndExitWithoutError } = require('../common/child_process'); -const fixtures = require('../common/fixtures'); - +const { spawnSyncAndAssert } = require('../common/child_process'); +const { fixturesDir } = require('../common/fixtures'); const stderr = /ExperimentalWarning: Support for loading ES Module in require/; -// Test named exports. -{ - spawnSyncAndExitWithoutError( - process.execPath, - [ '--experimental-require-module', '-r', fixtures.path('../fixtures/es-module-loaders/module-named-exports.mjs') ], - { - stderr, - } - ); -} +function testPreload(preloadFlag) { + // Test named exports. + { + spawnSyncAndAssert( + process.execPath, + [ + '--experimental-require-module', + preloadFlag, + './es-module-loaders/module-named-exports.mjs', + './printA.js', + ], + { + cwd: fixturesDir + }, + { + stdout: 'A', + stderr, + trim: true, + } + ); + } -// Test ESM that import ESM. -{ - spawnSyncAndExitWithoutError( - process.execPath, - [ '--experimental-require-module', '-r', fixtures.path('../fixtures/es-modules/import-esm.mjs') ], - { - stderr, - stdout: 'world', - trim: true, - } - ); -} + // Test ESM that import ESM. + { + spawnSyncAndAssert( + process.execPath, + [ + '--experimental-require-module', + preloadFlag, + './es-modules/import-esm.mjs', + './printA.js', + ], + { + cwd: fixturesDir + }, + { + stderr, + stdout: /^world\s+A$/, + trim: true, + } + ); + } -// Test ESM that import CJS. -{ - spawnSyncAndExitWithoutError( - process.execPath, - [ '--experimental-require-module', '-r', fixtures.path('../fixtures/es-modules/cjs-exports.mjs') ], - { - stdout: 'ok', - stderr, - trim: true, - } - ); -} + // Test ESM that import CJS. + { + spawnSyncAndAssert( + process.execPath, + [ + '--experimental-require-module', + preloadFlag, + './es-modules/cjs-exports.mjs', + './printA.js', + ], + { + cwd: fixturesDir + }, + { + stdout: /^ok\s+A$/, + stderr, + trim: true, + } + ); + } -// Test ESM that require() CJS. -// Can't use the common/index.mjs here because that checks the globals, and -// -r injects a bunch of globals. -{ - spawnSyncAndExitWithoutError( - process.execPath, - [ '--experimental-require-module', '-r', fixtures.path('../fixtures/es-modules/require-cjs.mjs') ], - { - stdout: 'world', - stderr, - trim: true, - } - ); + // Test ESM that require() CJS. + // Can't use the common/index.mjs here because that checks the globals, and + // -r injects a bunch of globals. + { + spawnSyncAndAssert( + process.execPath, + [ + '--experimental-require-module', + preloadFlag, + './es-modules/require-cjs.mjs', + './printA.js', + ], + { + cwd: fixturesDir + }, + { + stdout: /^world\s+A$/, + stderr, + trim: true, + } + ); + } } -// Test "type": "module" and "main" field in package.json. +testPreload('--require'); +testPreload('--import'); + +// Test "type": "module" and "main" field in package.json, this is only for --require because +// --import does not support extension-less preloads. { - spawnSyncAndExitWithoutError( + spawnSyncAndAssert( process.execPath, - [ '--experimental-require-module', '-r', fixtures.path('../fixtures/es-modules/package-type-module') ], + [ + '--experimental-require-module', + '--require', + './es-modules/package-type-module', + './printA.js', + ], + { + cwd: fixturesDir + }, { - stdout: 'package-type-module', + stdout: /^package-type-module\s+A$/, stderr, trim: true, } From 03c58010119dc03abbf61205c13862d2a9ae900c Mon Sep 17 00:00:00 2001 From: Antoine du Hamel Date: Thu, 19 Sep 2024 12:34:13 +0200 Subject: [PATCH 16/33] module: report unfinished TLA in ambiguous modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR-URL: https://github.com/nodejs/node/pull/54980 Fixes: https://github.com/nodejs/node/issues/54931 Reviewed-By: Michaël Zasso Reviewed-By: Marco Ippolito Reviewed-By: Joyee Cheung --- lib/internal/modules/cjs/loader.js | 2 +- test/es-module/test-esm-detect-ambiguous.mjs | 12 ++++++++++++ test/fixtures/es-modules/tla/unresolved.js | 1 + 3 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 test/fixtures/es-modules/tla/unresolved.js diff --git a/lib/internal/modules/cjs/loader.js b/lib/internal/modules/cjs/loader.js index a861c68dd7e33c..6dba8d926316ce 100644 --- a/lib/internal/modules/cjs/loader.js +++ b/lib/internal/modules/cjs/loader.js @@ -1336,7 +1336,7 @@ function loadESMFromCJS(mod, filename) { if (isMain) { require('internal/modules/run_main').runEntryPointWithESMLoader((cascadedLoader) => { const mainURL = pathToFileURL(filename).href; - cascadedLoader.import(mainURL, undefined, { __proto__: null }, true); + return cascadedLoader.import(mainURL, undefined, { __proto__: null }, true); }); // ESM won't be accessible via process.mainModule. setOwnProperty(process, 'mainModule', undefined); diff --git a/test/es-module/test-esm-detect-ambiguous.mjs b/test/es-module/test-esm-detect-ambiguous.mjs index 2d7df8d308082f..b0a62e6dcfb290 100644 --- a/test/es-module/test-esm-detect-ambiguous.mjs +++ b/test/es-module/test-esm-detect-ambiguous.mjs @@ -252,6 +252,18 @@ describe('Module syntax detection', { concurrency: !process.env.TEST_PARALLEL }, strictEqual(signal, null); }); + it('reports unfinished top-level `await`', async () => { + const { stdout, stderr, code, signal } = await spawnPromisified(process.execPath, [ + '--no-warnings', + fixtures.path('es-modules/tla/unresolved.js'), + ]); + + strictEqual(stderr, ''); + strictEqual(stdout, ''); + strictEqual(code, 13); + strictEqual(signal, null); + }); + it('permits top-level `await` above import/export syntax', async () => { const { stdout, stderr, code, signal } = await spawnPromisified(process.execPath, [ '--eval', diff --git a/test/fixtures/es-modules/tla/unresolved.js b/test/fixtures/es-modules/tla/unresolved.js new file mode 100644 index 00000000000000..231a8cd634825c --- /dev/null +++ b/test/fixtures/es-modules/tla/unresolved.js @@ -0,0 +1 @@ +await new Promise(() => {}); From 58856918a00bea150a80c1b044bf206b2d7ac4e2 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Wed, 25 Sep 2024 08:35:26 +0200 Subject: [PATCH 17/33] module: implement the "module-sync" exports condition This patch implements a "module-sync" exports condition for packages to supply a sycnrhonous ES module to the Node.js module loader, no matter it's being required or imported. This is similar to the "module" condition that bundlers have been using to support `require(esm)` in Node.js, and allows dual-package authors to opt into ESM-first only newer versions of Node.js that supports require(esm) while avoiding the dual-package hazard. ```json { "type": "module", "exports": { "node": { // On new version of Node.js, both require() and import get // the ESM version "module-sync": "./index.js", // On older version of Node.js, where "module" and // require(esm) are not supported, use the transpiled CJS version // to avoid dual-package hazard. Library authors can decide // to drop support for older versions of Node.js when they think // it's time. "default": "./dist/index.cjs" }, // On any other environment, use the ESM version. "default": "./index.js" } } ``` We end up implementing a condition with a different name instead of reusing "module", because existing code in the ecosystem using the "module" condition sometimes also expect the module resolution for these ESM files to work in CJS style, which is supported by bundlers, but the native Node.js loader has intentionally made ESM resolution different from CJS resolution (e.g. forbidding `import './noext'` or `import './directory'`), so it would be semver-major to implement a `"module"` condition without implementing the forbidden ESM resolution rules. For now, this just implments a new condition as semver-minor so it can be backported to older LTS. Refs: https://webpack.js.org/guides/package-exports/#target-environment-independent-packages PR-URL: https://github.com/nodejs/node/pull/54648 Fixes: https://github.com/nodejs/node/issues/52173 Refs: https://github.com/joyeecheung/test-module-condition Refs: https://github.com/nodejs/node/issues/52697 Reviewed-By: Jacob Smith Reviewed-By: Jan Krems Reviewed-By: Chengzhong Wu --- doc/api/modules.md | 18 ++++++++---- doc/api/packages.md | 19 ++++++++++-- lib/internal/modules/esm/get_format.js | 2 +- lib/internal/modules/esm/utils.js | 3 ++ lib/internal/modules/helpers.js | 3 ++ ...port-module-conditional-exports-module.mjs | 29 +++++++++++++++++++ ...quire-module-conditional-exports-module.js | 15 ++++++++++ .../module-condition/dynamic_import.js | 5 ++++ .../es-modules/module-condition/import.mjs | 7 +++++ .../import-module-require/import.js | 1 + .../import-module-require/module.js | 1 + .../import-module-require/package.json | 11 +++++++ .../import-module-require/require.cjs | 1 + .../node_modules/module-and-import/import.js | 1 + .../node_modules/module-and-import/module.js | 1 + .../module-and-import/package.json | 10 +++++++ .../node_modules/module-and-require/module.js | 1 + .../module-and-require/package.json | 10 +++++++ .../module-and-require/require.cjs | 1 + .../module-import-require/import.js | 1 + .../module-import-require/module.js | 1 + .../module-import-require/package.json | 11 +++++++ .../module-import-require/require.cjs | 1 + .../node_modules/module-only/module.js | 1 + .../node_modules/module-only/package.json | 6 ++++ .../module-require-import/import.js | 1 + .../module-require-import/module.js | 1 + .../module-require-import/package.json | 11 +++++++ .../module-require-import/require.cjs | 1 + .../require-module-import/import.js | 1 + .../require-module-import/module.js | 1 + .../require-module-import/package.json | 11 +++++++ .../require-module-import/require.cjs | 1 + .../es-modules/module-condition/require.cjs | 1 + 34 files changed, 180 insertions(+), 9 deletions(-) create mode 100644 test/es-module/test-import-module-conditional-exports-module.mjs create mode 100644 test/es-module/test-require-module-conditional-exports-module.js create mode 100644 test/fixtures/es-modules/module-condition/dynamic_import.js create mode 100644 test/fixtures/es-modules/module-condition/import.mjs create mode 100644 test/fixtures/es-modules/module-condition/node_modules/import-module-require/import.js create mode 100644 test/fixtures/es-modules/module-condition/node_modules/import-module-require/module.js create mode 100644 test/fixtures/es-modules/module-condition/node_modules/import-module-require/package.json create mode 100644 test/fixtures/es-modules/module-condition/node_modules/import-module-require/require.cjs create mode 100644 test/fixtures/es-modules/module-condition/node_modules/module-and-import/import.js create mode 100644 test/fixtures/es-modules/module-condition/node_modules/module-and-import/module.js create mode 100644 test/fixtures/es-modules/module-condition/node_modules/module-and-import/package.json create mode 100644 test/fixtures/es-modules/module-condition/node_modules/module-and-require/module.js create mode 100644 test/fixtures/es-modules/module-condition/node_modules/module-and-require/package.json create mode 100644 test/fixtures/es-modules/module-condition/node_modules/module-and-require/require.cjs create mode 100644 test/fixtures/es-modules/module-condition/node_modules/module-import-require/import.js create mode 100644 test/fixtures/es-modules/module-condition/node_modules/module-import-require/module.js create mode 100644 test/fixtures/es-modules/module-condition/node_modules/module-import-require/package.json create mode 100644 test/fixtures/es-modules/module-condition/node_modules/module-import-require/require.cjs create mode 100644 test/fixtures/es-modules/module-condition/node_modules/module-only/module.js create mode 100644 test/fixtures/es-modules/module-condition/node_modules/module-only/package.json create mode 100644 test/fixtures/es-modules/module-condition/node_modules/module-require-import/import.js create mode 100644 test/fixtures/es-modules/module-condition/node_modules/module-require-import/module.js create mode 100644 test/fixtures/es-modules/module-condition/node_modules/module-require-import/package.json create mode 100644 test/fixtures/es-modules/module-condition/node_modules/module-require-import/require.cjs create mode 100644 test/fixtures/es-modules/module-condition/node_modules/require-module-import/import.js create mode 100644 test/fixtures/es-modules/module-condition/node_modules/require-module-import/module.js create mode 100644 test/fixtures/es-modules/module-condition/node_modules/require-module-import/package.json create mode 100644 test/fixtures/es-modules/module-condition/node_modules/require-module-import/require.cjs create mode 100644 test/fixtures/es-modules/module-condition/require.cjs diff --git a/doc/api/modules.md b/doc/api/modules.md index ffd5855cf5e80a..1ead13fa32f218 100644 --- a/doc/api/modules.md +++ b/doc/api/modules.md @@ -342,9 +342,12 @@ LOAD_PACKAGE_IMPORTS(X, DIR) 1. Find the closest package scope SCOPE to DIR. 2. If no scope was found, return. 3. If the SCOPE/package.json "imports" is null or undefined, return. -4. let MATCH = PACKAGE_IMPORTS_RESOLVE(X, pathToFileURL(SCOPE), - ["node", "require"]) defined in the ESM resolver. -5. RESOLVE_ESM_MATCH(MATCH). +4. If `--experimental-require-module` is enabled + a. let CONDITIONS = ["node", "require", "module-sync"] + b. Else, let CONDITIONS = ["node", "require"] +5. let MATCH = PACKAGE_IMPORTS_RESOLVE(X, pathToFileURL(SCOPE), + CONDITIONS) defined in the ESM resolver. +6. RESOLVE_ESM_MATCH(MATCH). LOAD_PACKAGE_EXPORTS(X, DIR) 1. Try to interpret X as a combination of NAME and SUBPATH where the name @@ -353,9 +356,12 @@ LOAD_PACKAGE_EXPORTS(X, DIR) return. 3. Parse DIR/NAME/package.json, and look for "exports" field. 4. If "exports" is null or undefined, return. -5. let MATCH = PACKAGE_EXPORTS_RESOLVE(pathToFileURL(DIR/NAME), "." + SUBPATH, - `package.json` "exports", ["node", "require"]) defined in the ESM resolver. -6. RESOLVE_ESM_MATCH(MATCH) +5. If `--experimental-require-module` is enabled + a. let CONDITIONS = ["node", "require", "module-sync"] + b. Else, let CONDITIONS = ["node", "require"] +6. let MATCH = PACKAGE_EXPORTS_RESOLVE(pathToFileURL(DIR/NAME), "." + SUBPATH, + `package.json` "exports", CONDITIONS) defined in the ESM resolver. +7. RESOLVE_ESM_MATCH(MATCH) LOAD_PACKAGE_SELF(X, DIR) 1. Find the closest package scope SCOPE to DIR. diff --git a/doc/api/packages.md b/doc/api/packages.md index 18862dc193cd74..5dfc9e7b1da2c3 100644 --- a/doc/api/packages.md +++ b/doc/api/packages.md @@ -665,6 +665,10 @@ specific to least specific as conditions should be defined: formats include CommonJS, JSON, native addons, and ES modules if `--experimental-require-module` is enabled. _Always mutually exclusive with `"import"`._ +* `"module-sync"` - matches no matter the package is loaded via `import`, + `import()` or `require()`. The format is expected to be ES modules that does + not contain top-level await in its module graph - if it does, + `ERR_REQUIRE_ASYNC_MODULE` will be thrown when the module is `require()`-ed. * `"default"` - the generic fallback that always matches. Can be a CommonJS or ES module file. _This condition should always come last._ @@ -769,7 +773,7 @@ In node, conditions have very few restrictions, but specifically these include: ### Community Conditions Definitions -Condition strings other than the `"import"`, `"require"`, `"node"`, +Condition strings other than the `"import"`, `"require"`, `"node"`, `"module-sync"`, `"node-addons"` and `"default"` conditions [implemented in Node.js core](#conditional-exports) are ignored by default. @@ -900,6 +904,17 @@ $ node other.js ## Dual CommonJS/ES module packages + + Prior to the introduction of support for ES modules in Node.js, it was a common pattern for package authors to include both CommonJS and ES module JavaScript sources in their package, with `package.json` [`"main"`][] specifying the @@ -912,7 +927,7 @@ ignores) the top-level `"module"` field. Node.js can now run ES module entry points, and a package can contain both CommonJS and ES module entry points (either via separate specifiers such as `'pkg'` and `'pkg/es-module'`, or both at the same specifier via [Conditional -exports][]). Unlike in the scenario where `"module"` is only used by bundlers, +exports][]). Unlike in the scenario where top-level `"module"` field is only used by bundlers, or ES module files are transpiled into CommonJS on the fly before evaluation by Node.js, the files referenced by the ES module entry point are evaluated as ES modules. diff --git a/lib/internal/modules/esm/get_format.js b/lib/internal/modules/esm/get_format.js index ae7b73008f3d06..fe7528e32a497c 100644 --- a/lib/internal/modules/esm/get_format.js +++ b/lib/internal/modules/esm/get_format.js @@ -97,7 +97,7 @@ function underNodeModules(url) { let typelessPackageJsonFilesWarnedAbout; function warnTypelessPackageJsonFile(pjsonPath, url) { typelessPackageJsonFilesWarnedAbout ??= new SafeSet(); - if (!typelessPackageJsonFilesWarnedAbout.has(pjsonPath)) { + if (!underNodeModules(url) && !typelessPackageJsonFilesWarnedAbout.has(pjsonPath)) { const warning = `Module type of ${url} is not specified and it doesn't parse as CommonJS.\n` + 'Reparsing as ES module because module syntax was detected. This incurs a performance overhead.\n' + `To eliminate this warning, add "type": "module" to ${pjsonPath}.`; diff --git a/lib/internal/modules/esm/utils.js b/lib/internal/modules/esm/utils.js index dfa32f493b262e..f94349630e9220 100644 --- a/lib/internal/modules/esm/utils.js +++ b/lib/internal/modules/esm/utils.js @@ -83,6 +83,9 @@ function initializeDefaultConditions() { ...userConditions, ]); defaultConditionsSet = new SafeSet(defaultConditions); + if (getOptionValue('--experimental-require-module')) { + defaultConditionsSet.add('module-sync'); + } } /** diff --git a/lib/internal/modules/helpers.js b/lib/internal/modules/helpers.js index d560c0d8089e19..39815fc9b3a999 100644 --- a/lib/internal/modules/helpers.js +++ b/lib/internal/modules/helpers.js @@ -76,6 +76,9 @@ function initializeCjsConditions() { ...addonConditions, ...userConditions, ]); + if (getOptionValue('--experimental-require-module')) { + cjsConditions.add('module-sync'); + } } /** diff --git a/test/es-module/test-import-module-conditional-exports-module.mjs b/test/es-module/test-import-module-conditional-exports-module.mjs new file mode 100644 index 00000000000000..c751996c804529 --- /dev/null +++ b/test/es-module/test-import-module-conditional-exports-module.mjs @@ -0,0 +1,29 @@ +// Flags: --experimental-require-module + +import '../common/index.mjs'; +import assert from 'node:assert'; +import * as staticImport from '../fixtures/es-modules/module-condition/import.mjs'; +import { import as _import } from '../fixtures/es-modules/module-condition/dynamic_import.js'; + +async function dynamicImport(id) { + const result = await _import(id); + return result.resolved; +} + +assert.deepStrictEqual({ ...staticImport }, { + import_module_require: 'import', + module_and_import: 'module', + module_and_require: 'module', + module_import_require: 'module', + module_only: 'module', + module_require_import: 'module', + require_module_import: 'module', +}); + +assert.strictEqual(await dynamicImport('import-module-require'), 'import'); +assert.strictEqual(await dynamicImport('module-and-import'), 'module'); +assert.strictEqual(await dynamicImport('module-and-require'), 'module'); +assert.strictEqual(await dynamicImport('module-import-require'), 'module'); +assert.strictEqual(await dynamicImport('module-only'), 'module'); +assert.strictEqual(await dynamicImport('module-require-import'), 'module'); +assert.strictEqual(await dynamicImport('require-module-import'), 'module'); diff --git a/test/es-module/test-require-module-conditional-exports-module.js b/test/es-module/test-require-module-conditional-exports-module.js new file mode 100644 index 00000000000000..2a9e83869053d3 --- /dev/null +++ b/test/es-module/test-require-module-conditional-exports-module.js @@ -0,0 +1,15 @@ +// Flags: --experimental-require-module +'use strict'; + +require('../common'); +const assert = require('assert'); + +const loader = require('../fixtures/es-modules/module-condition/require.cjs'); + +assert.strictEqual(loader.require('import-module-require').resolved, 'module'); +assert.strictEqual(loader.require('module-and-import').resolved, 'module'); +assert.strictEqual(loader.require('module-and-require').resolved, 'module'); +assert.strictEqual(loader.require('module-import-require').resolved, 'module'); +assert.strictEqual(loader.require('module-only').resolved, 'module'); +assert.strictEqual(loader.require('module-require-import').resolved, 'module'); +assert.strictEqual(loader.require('require-module-import').resolved, 'require'); diff --git a/test/fixtures/es-modules/module-condition/dynamic_import.js b/test/fixtures/es-modules/module-condition/dynamic_import.js new file mode 100644 index 00000000000000..7c4cd42d7037f4 --- /dev/null +++ b/test/fixtures/es-modules/module-condition/dynamic_import.js @@ -0,0 +1,5 @@ +function load(id) { + return import(id); +} + +export { load as import }; diff --git a/test/fixtures/es-modules/module-condition/import.mjs b/test/fixtures/es-modules/module-condition/import.mjs new file mode 100644 index 00000000000000..ae12fbf1429424 --- /dev/null +++ b/test/fixtures/es-modules/module-condition/import.mjs @@ -0,0 +1,7 @@ +export { resolved as import_module_require } from 'import-module-require'; +export { resolved as module_and_import } from 'module-and-import'; +export { resolved as module_and_require } from 'module-and-require'; +export { resolved as module_import_require } from 'module-import-require'; +export { resolved as module_only } from 'module-only'; +export { resolved as module_require_import } from 'module-require-import'; +export { resolved as require_module_import } from 'require-module-import'; diff --git a/test/fixtures/es-modules/module-condition/node_modules/import-module-require/import.js b/test/fixtures/es-modules/module-condition/node_modules/import-module-require/import.js new file mode 100644 index 00000000000000..58f10e20bf3041 --- /dev/null +++ b/test/fixtures/es-modules/module-condition/node_modules/import-module-require/import.js @@ -0,0 +1 @@ +export const resolved = 'import'; \ No newline at end of file diff --git a/test/fixtures/es-modules/module-condition/node_modules/import-module-require/module.js b/test/fixtures/es-modules/module-condition/node_modules/import-module-require/module.js new file mode 100644 index 00000000000000..136ec680a7e3a8 --- /dev/null +++ b/test/fixtures/es-modules/module-condition/node_modules/import-module-require/module.js @@ -0,0 +1 @@ +export const resolved = 'module'; \ No newline at end of file diff --git a/test/fixtures/es-modules/module-condition/node_modules/import-module-require/package.json b/test/fixtures/es-modules/module-condition/node_modules/import-module-require/package.json new file mode 100644 index 00000000000000..46a096c2e7396d --- /dev/null +++ b/test/fixtures/es-modules/module-condition/node_modules/import-module-require/package.json @@ -0,0 +1,11 @@ +{ + "type": "module", + "exports": { + "node": { + "import": "./import.js", + "module-sync": "./module.js", + "require": "./require.cjs" + }, + "default": "./module.js" + } +} \ No newline at end of file diff --git a/test/fixtures/es-modules/module-condition/node_modules/import-module-require/require.cjs b/test/fixtures/es-modules/module-condition/node_modules/import-module-require/require.cjs new file mode 100644 index 00000000000000..6dd2c2ec97abd6 --- /dev/null +++ b/test/fixtures/es-modules/module-condition/node_modules/import-module-require/require.cjs @@ -0,0 +1 @@ +exports.resolved = 'require'; diff --git a/test/fixtures/es-modules/module-condition/node_modules/module-and-import/import.js b/test/fixtures/es-modules/module-condition/node_modules/module-and-import/import.js new file mode 100644 index 00000000000000..58f10e20bf3041 --- /dev/null +++ b/test/fixtures/es-modules/module-condition/node_modules/module-and-import/import.js @@ -0,0 +1 @@ +export const resolved = 'import'; \ No newline at end of file diff --git a/test/fixtures/es-modules/module-condition/node_modules/module-and-import/module.js b/test/fixtures/es-modules/module-condition/node_modules/module-and-import/module.js new file mode 100644 index 00000000000000..136ec680a7e3a8 --- /dev/null +++ b/test/fixtures/es-modules/module-condition/node_modules/module-and-import/module.js @@ -0,0 +1 @@ +export const resolved = 'module'; \ No newline at end of file diff --git a/test/fixtures/es-modules/module-condition/node_modules/module-and-import/package.json b/test/fixtures/es-modules/module-condition/node_modules/module-and-import/package.json new file mode 100644 index 00000000000000..4425147f7dab2c --- /dev/null +++ b/test/fixtures/es-modules/module-condition/node_modules/module-and-import/package.json @@ -0,0 +1,10 @@ +{ + "type": "module", + "exports": { + "node": { + "module-sync": "./module.js", + "import": "./import.js" + }, + "default": "./module.js" + } +} diff --git a/test/fixtures/es-modules/module-condition/node_modules/module-and-require/module.js b/test/fixtures/es-modules/module-condition/node_modules/module-and-require/module.js new file mode 100644 index 00000000000000..136ec680a7e3a8 --- /dev/null +++ b/test/fixtures/es-modules/module-condition/node_modules/module-and-require/module.js @@ -0,0 +1 @@ +export const resolved = 'module'; \ No newline at end of file diff --git a/test/fixtures/es-modules/module-condition/node_modules/module-and-require/package.json b/test/fixtures/es-modules/module-condition/node_modules/module-and-require/package.json new file mode 100644 index 00000000000000..57598fb5014bb0 --- /dev/null +++ b/test/fixtures/es-modules/module-condition/node_modules/module-and-require/package.json @@ -0,0 +1,10 @@ +{ + "type": "module", + "exports": { + "node": { + "module-sync": "./module.js", + "require": "./require.cjs" + }, + "default": "./module.js" + } +} \ No newline at end of file diff --git a/test/fixtures/es-modules/module-condition/node_modules/module-and-require/require.cjs b/test/fixtures/es-modules/module-condition/node_modules/module-and-require/require.cjs new file mode 100644 index 00000000000000..6dd2c2ec97abd6 --- /dev/null +++ b/test/fixtures/es-modules/module-condition/node_modules/module-and-require/require.cjs @@ -0,0 +1 @@ +exports.resolved = 'require'; diff --git a/test/fixtures/es-modules/module-condition/node_modules/module-import-require/import.js b/test/fixtures/es-modules/module-condition/node_modules/module-import-require/import.js new file mode 100644 index 00000000000000..58f10e20bf3041 --- /dev/null +++ b/test/fixtures/es-modules/module-condition/node_modules/module-import-require/import.js @@ -0,0 +1 @@ +export const resolved = 'import'; \ No newline at end of file diff --git a/test/fixtures/es-modules/module-condition/node_modules/module-import-require/module.js b/test/fixtures/es-modules/module-condition/node_modules/module-import-require/module.js new file mode 100644 index 00000000000000..136ec680a7e3a8 --- /dev/null +++ b/test/fixtures/es-modules/module-condition/node_modules/module-import-require/module.js @@ -0,0 +1 @@ +export const resolved = 'module'; \ No newline at end of file diff --git a/test/fixtures/es-modules/module-condition/node_modules/module-import-require/package.json b/test/fixtures/es-modules/module-condition/node_modules/module-import-require/package.json new file mode 100644 index 00000000000000..bbc8c0d286e656 --- /dev/null +++ b/test/fixtures/es-modules/module-condition/node_modules/module-import-require/package.json @@ -0,0 +1,11 @@ +{ + "type": "module", + "exports": { + "node": { + "module-sync": "./module.js", + "import": "./import.js", + "require": "./require.cjs" + }, + "default": "./module.js" + } +} \ No newline at end of file diff --git a/test/fixtures/es-modules/module-condition/node_modules/module-import-require/require.cjs b/test/fixtures/es-modules/module-condition/node_modules/module-import-require/require.cjs new file mode 100644 index 00000000000000..6dd2c2ec97abd6 --- /dev/null +++ b/test/fixtures/es-modules/module-condition/node_modules/module-import-require/require.cjs @@ -0,0 +1 @@ +exports.resolved = 'require'; diff --git a/test/fixtures/es-modules/module-condition/node_modules/module-only/module.js b/test/fixtures/es-modules/module-condition/node_modules/module-only/module.js new file mode 100644 index 00000000000000..136ec680a7e3a8 --- /dev/null +++ b/test/fixtures/es-modules/module-condition/node_modules/module-only/module.js @@ -0,0 +1 @@ +export const resolved = 'module'; \ No newline at end of file diff --git a/test/fixtures/es-modules/module-condition/node_modules/module-only/package.json b/test/fixtures/es-modules/module-condition/node_modules/module-only/package.json new file mode 100644 index 00000000000000..2d29bbdf325e1a --- /dev/null +++ b/test/fixtures/es-modules/module-condition/node_modules/module-only/package.json @@ -0,0 +1,6 @@ +{ + "type": "module", + "exports": { + "module-sync": "./module.js" + } +} diff --git a/test/fixtures/es-modules/module-condition/node_modules/module-require-import/import.js b/test/fixtures/es-modules/module-condition/node_modules/module-require-import/import.js new file mode 100644 index 00000000000000..58f10e20bf3041 --- /dev/null +++ b/test/fixtures/es-modules/module-condition/node_modules/module-require-import/import.js @@ -0,0 +1 @@ +export const resolved = 'import'; \ No newline at end of file diff --git a/test/fixtures/es-modules/module-condition/node_modules/module-require-import/module.js b/test/fixtures/es-modules/module-condition/node_modules/module-require-import/module.js new file mode 100644 index 00000000000000..136ec680a7e3a8 --- /dev/null +++ b/test/fixtures/es-modules/module-condition/node_modules/module-require-import/module.js @@ -0,0 +1 @@ +export const resolved = 'module'; \ No newline at end of file diff --git a/test/fixtures/es-modules/module-condition/node_modules/module-require-import/package.json b/test/fixtures/es-modules/module-condition/node_modules/module-require-import/package.json new file mode 100644 index 00000000000000..1490f04b27f9ad --- /dev/null +++ b/test/fixtures/es-modules/module-condition/node_modules/module-require-import/package.json @@ -0,0 +1,11 @@ +{ + "type": "module", + "exports": { + "node": { + "module-sync": "./module.js", + "require": "./require.cjs", + "import": "./import.js" + }, + "default": "./module.js" + } +} diff --git a/test/fixtures/es-modules/module-condition/node_modules/module-require-import/require.cjs b/test/fixtures/es-modules/module-condition/node_modules/module-require-import/require.cjs new file mode 100644 index 00000000000000..f5bf7ed32992e1 --- /dev/null +++ b/test/fixtures/es-modules/module-condition/node_modules/module-require-import/require.cjs @@ -0,0 +1 @@ +export const resolved = 'require'; diff --git a/test/fixtures/es-modules/module-condition/node_modules/require-module-import/import.js b/test/fixtures/es-modules/module-condition/node_modules/require-module-import/import.js new file mode 100644 index 00000000000000..58f10e20bf3041 --- /dev/null +++ b/test/fixtures/es-modules/module-condition/node_modules/require-module-import/import.js @@ -0,0 +1 @@ +export const resolved = 'import'; \ No newline at end of file diff --git a/test/fixtures/es-modules/module-condition/node_modules/require-module-import/module.js b/test/fixtures/es-modules/module-condition/node_modules/require-module-import/module.js new file mode 100644 index 00000000000000..136ec680a7e3a8 --- /dev/null +++ b/test/fixtures/es-modules/module-condition/node_modules/require-module-import/module.js @@ -0,0 +1 @@ +export const resolved = 'module'; \ No newline at end of file diff --git a/test/fixtures/es-modules/module-condition/node_modules/require-module-import/package.json b/test/fixtures/es-modules/module-condition/node_modules/require-module-import/package.json new file mode 100644 index 00000000000000..b62f96c997ecb7 --- /dev/null +++ b/test/fixtures/es-modules/module-condition/node_modules/require-module-import/package.json @@ -0,0 +1,11 @@ +{ + "type": "module", + "exports": { + "node": { + "require": "./require.cjs", + "module-sync": "./module.js", + "import": "./module.js" + }, + "default": "./module.js" + } +} diff --git a/test/fixtures/es-modules/module-condition/node_modules/require-module-import/require.cjs b/test/fixtures/es-modules/module-condition/node_modules/require-module-import/require.cjs new file mode 100644 index 00000000000000..6dd2c2ec97abd6 --- /dev/null +++ b/test/fixtures/es-modules/module-condition/node_modules/require-module-import/require.cjs @@ -0,0 +1 @@ +exports.resolved = 'require'; diff --git a/test/fixtures/es-modules/module-condition/require.cjs b/test/fixtures/es-modules/module-condition/require.cjs new file mode 100644 index 00000000000000..2457758a98f32b --- /dev/null +++ b/test/fixtures/es-modules/module-condition/require.cjs @@ -0,0 +1 @@ +exports.require = require; From 8e684e38830321678e94fc930c792d95d8669c22 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Tue, 17 Sep 2024 20:38:33 +0200 Subject: [PATCH 18/33] module: refator ESM loader for adding future synchronous hooks This lays the foundation for supporting synchronous hooks proposed in https://github.com/nodejs/loaders/pull/198 for ESM. - Corrects and adds several JSDoc comments for internal functions of the ESM loader, as well as explaining how require() for import CJS work in the special resolve/load paths. This doesn't consolidate it with import in require(esm) yet due to caching differences, which is left as a TODO. - The moduleProvider passed into ModuleJob is replaced as moduleOrModulePromise, we call the translators directly in the ESM loader and verify it right after loading for clarity. - Reuse a few refactored out helpers for require(esm) in getModuleJobForRequire(). PR-URL: https://github.com/nodejs/node/pull/54769 Reviewed-By: Matteo Collina Reviewed-By: Stephen Belanger Reviewed-By: James M Snell --- lib/internal/modules/esm/loader.js | 317 ++++++++++++++++-------- lib/internal/modules/esm/module_job.js | 61 +++-- lib/internal/modules/esm/translators.js | 37 +-- 3 files changed, 264 insertions(+), 151 deletions(-) diff --git a/lib/internal/modules/esm/loader.js b/lib/internal/modules/esm/loader.js index d5ffd5f32f599e..820399a577ce9a 100644 --- a/lib/internal/modules/esm/loader.js +++ b/lib/internal/modules/esm/loader.js @@ -205,12 +205,10 @@ class ModuleLoader { } async eval(source, url) { - const evalInstance = (url) => { - return compileSourceTextModule(url, source, this); - }; const { ModuleJob } = require('internal/modules/esm/module_job'); + const wrap = compileSourceTextModule(url, source, this); const job = new ModuleJob( - this, url, undefined, evalInstance, false, false); + this, url, undefined, wrap, false, false); this.loadCache.set(url, undefined, job); const { module } = await job.run(); @@ -222,40 +220,49 @@ class ModuleLoader { } /** - * Get a (possibly still pending) module job from the cache, - * or create one and return its Promise. - * @param {string} specifier The string after `from` in an `import` statement, - * or the first parameter of an `import()` - * expression - * @param {string | undefined} parentURL The URL of the module importing this - * one, unless this is the Node.js entry - * point. - * @param {Record} importAttributes Validations for the - * module import. - * @returns {Promise} The (possibly pending) module job + * Get a (possibly not yet fully linked) module job from the cache, or create one and return its Promise. + * @param {string} specifier The module request of the module to be resolved. Typically, what's + * requested by `import ''` or `import('')`. + * @param {string} [parentURL] The URL of the module where the module request is initiated. + * It's undefined if it's from the root module. + * @param {ImportAttributes} importAttributes Attributes from the import statement or expression. + * @returns {Promise} importAttributes Validations for the - * module import. - * @param {string} [parentURL] The absolute URL of the module importing this - * one, unless this is the Node.js entry point - * @param {string} [format] The format hint possibly returned by the - * `resolve` hook - * @returns {Promise} The (possibly pending) module job + * Translate a loaded module source into a ModuleWrap. This is run synchronously, + * but the translator may return the ModuleWrap in a Promise. + * @param {stirng} url URL of the module to be translated. + * @param {string} format Format of the module to be translated. This is used to find + * matching translators. + * @param {ModuleSource} source Source of the module to be translated. + * @param {boolean} isMain Whether the module to be translated is the entry point. + * @returns {ModuleWrap | Promise} */ - #createModuleJob(url, importAttributes, parentURL, format, sync) { - const callTranslator = ({ format: finalFormat, responseURL, source }, isMain) => { - const translator = getTranslators().get(finalFormat); + #translate(url, format, source, isMain) { + this.validateLoadResult(url, format); + const translator = getTranslators().get(format); + + if (!translator) { + throw new ERR_UNKNOWN_MODULE_FORMAT(format, url); + } - if (!translator) { - throw new ERR_UNKNOWN_MODULE_FORMAT(finalFormat, responseURL); + return FunctionPrototypeCall(translator, this, url, source, isMain); + } + + /** + * Load a module and translate it into a ModuleWrap for require() in imported CJS. + * This is run synchronously, and the translator always return a ModuleWrap synchronously. + * @param {string} url URL of the module to be translated. + * @param {object} loadContext See {@link load} + * @param {boolean} isMain Whether the module to be translated is the entry point. + * @returns {ModuleWrap} + */ + loadAndTranslateForRequireInImportedCJS(url, loadContext, isMain) { + const { format: formatFromLoad, source } = this.#loadSync(url, loadContext); + + if (formatFromLoad === 'wasm') { // require(wasm) is not supported. + throw new ERR_UNKNOWN_MODULE_FORMAT(formatFromLoad, url); + } + + if (formatFromLoad === 'module' || formatFromLoad === 'module-typescript') { + if (!getOptionValue('--experimental-require-module')) { + throw new ERR_REQUIRE_ESM(url, true); } + } - return FunctionPrototypeCall(translator, this, responseURL, source, isMain); - }; - const context = { format, importAttributes }; + let finalFormat = formatFromLoad; + if (formatFromLoad === 'commonjs') { + finalFormat = 'require-commonjs'; + } + if (formatFromLoad === 'commonjs-typescript') { + finalFormat = 'require-commonjs-typescript'; + } + + const wrap = this.#translate(url, finalFormat, source, isMain); + assert(wrap instanceof ModuleWrap, `Translator used for require(${url}) should not be async`); + return wrap; + } - const moduleProvider = sync ? - (url, isMain) => callTranslator(this.loadSync(url, context), isMain) : - async (url, isMain) => callTranslator(await this.load(url, context), isMain); + /** + * Load a module and translate it into a ModuleWrap for ordinary imported ESM. + * This is run asynchronously. + * @param {string} url URL of the module to be translated. + * @param {object} loadContext See {@link load} + * @param {boolean} isMain Whether the module to be translated is the entry point. + * @returns {Promise} + */ + async loadAndTranslate(url, loadContext, isMain) { + const { format, source, responseURL } = await this.load(url, loadContext); + return this.#translate(responseURL, format, source, isMain); + } + + /** + * Load a module and translate it into a ModuleWrap, and create a ModuleJob from it. + * This runs synchronously. If isForRequireInImportedCJS is true, the module should be linked + * by the time this returns. Otherwise it may still have pending module requests. + * @param {string} url The URL that was resolved for this module. + * @param {ImportAttributes} importAttributes See {@link getModuleJobForImport} + * @param {string} [parentURL] See {@link getModuleJobForImport} + * @param {string} [format] The format hint possibly returned by the `resolve` hook + * @param {boolean} isForRequireInImportedCJS Whether this module job is created for require() + * in imported CJS. + * @returns {ModuleJobBase} The (possibly pending) module job + */ + #createModuleJob(url, importAttributes, parentURL, format, isForRequireInImportedCJS) { + const context = { format, importAttributes }; const isMain = parentURL === undefined; + let moduleOrModulePromise; + if (isForRequireInImportedCJS) { + moduleOrModulePromise = this.loadAndTranslateForRequireInImportedCJS(url, context, isMain); + } else { + moduleOrModulePromise = this.loadAndTranslate(url, context, isMain); + } + const inspectBrk = ( isMain && getOptionValue('--inspect-brk') @@ -449,10 +508,10 @@ class ModuleLoader { this, url, importAttributes, - moduleProvider, + moduleOrModulePromise, isMain, inspectBrk, - sync, + isForRequireInImportedCJS, ); this.loadCache.set(url, importAttributes.type, job); @@ -470,7 +529,7 @@ class ModuleLoader { * @returns {Promise} */ async import(specifier, parentURL, importAttributes) { - const moduleJob = await this.getModuleJob(specifier, parentURL, importAttributes); + const moduleJob = await this.getModuleJobForImport(specifier, parentURL, importAttributes); const { module } = await moduleJob.run(); return module.getNamespace(); } @@ -490,39 +549,72 @@ class ModuleLoader { } /** - * Resolve the location of the module. - * @param {string} originalSpecifier The specified URL path of the module to - * be resolved. - * @param {string} [parentURL] The URL path of the module's parent. - * @param {ImportAttributes} importAttributes Attributes from the import - * statement or expression. - * @returns {{ format: string, url: URL['href'] }} + * Resolve a module request to a URL identifying the location of the module. Handles customization hooks, + * if any. + * @param {string|URL} specifier The module request of the module to be resolved. Typically, what's + * requested by `import specifier`, `import(specifier)` or + * `import.meta.resolve(specifier)`. + * @param {string} [parentURL] The URL of the module where the module request is initiated. + * It's undefined if it's from the root module. + * @param {ImportAttributes} importAttributes Attributes from the import statement or expression. + * @returns {Promise<{format: string, url: string}>} */ - resolve(originalSpecifier, parentURL, importAttributes) { - originalSpecifier = `${originalSpecifier}`; - if (this.#customizations) { - return this.#customizations.resolve(originalSpecifier, parentURL, importAttributes); + resolve(specifier, parentURL, importAttributes) { + specifier = `${specifier}`; + if (this.#customizations) { // Only has module.register hooks. + return this.#customizations.resolve(specifier, parentURL, importAttributes); } - const requestKey = this.#resolveCache.serializeKey(originalSpecifier, importAttributes); + return this.#cachedDefaultResolve(specifier, parentURL, importAttributes); + } + + /** + * Either return a cached resolution, or perform the default resolution which is synchronous, and + * cache the result. + * @param {string} specifier See {@link resolve}. + * @param {string} [parentURL] See {@link resolve}. + * @param {ImportAttributes} importAttributes See {@link resolve}. + * @returns {{ format: string, url: string }} + */ + #cachedDefaultResolve(specifier, parentURL, importAttributes) { + const requestKey = this.#resolveCache.serializeKey(specifier, importAttributes); const cachedResult = this.#resolveCache.get(requestKey, parentURL); if (cachedResult != null) { return cachedResult; } - const result = this.defaultResolve(originalSpecifier, parentURL, importAttributes); + const result = this.defaultResolve(specifier, parentURL, importAttributes); this.#resolveCache.set(requestKey, parentURL, result); return result; } /** - * Just like `resolve` except synchronous. This is here specifically to support - * `import.meta.resolve` which must happen synchronously. + * This is the default resolve step for future synchronous hooks, which incorporates asynchronous hooks + * from module.register() which are run in a blocking fashion for it to be synchronous. + * @param {string|URL} specifier See {@link resolveSync}. + * @param {{ parentURL?: string, importAttributes: ImportAttributes}} context See {@link resolveSync}. + * @returns {{ format: string, url: string }} */ - resolveSync(originalSpecifier, parentURL, importAttributes) { - originalSpecifier = `${originalSpecifier}`; + #resolveAndMaybeBlockOnLoaderThread(specifier, context) { if (this.#customizations) { - return this.#customizations.resolveSync(originalSpecifier, parentURL, importAttributes); + return this.#customizations.resolveSync(specifier, context.parentURL, context.importAttributes); } - return this.defaultResolve(originalSpecifier, parentURL, importAttributes); + return this.#cachedDefaultResolve(specifier, context.parentURL, context.importAttributes); + } + + /** + * Similar to {@link resolve}, but the results are always synchronously returned. If there are any + * asynchronous resolve hooks from module.register(), it will block until the results are returned + * from the loader thread for this to be synchornous. + * This is here to support `import.meta.resolve()`, `require()` in imported CJS, and + * future synchronous hooks. + * + * TODO(joyeecheung): consolidate the cache behavior and use this in require(esm). + * @param {string|URL} specifier See {@link resolve}. + * @param {string} [parentURL] See {@link resolve}. + * @param {ImportAttributes} [importAttributes] See {@link resolve}. + * @returns {{ format: string, url: string }} + */ + resolveSync(specifier, parentURL, importAttributes = { __proto__: null }) { + return this.#resolveAndMaybeBlockOnLoaderThread(`${specifier}`, { parentURL, importAttributes }); } /** @@ -544,36 +636,49 @@ class ModuleLoader { } /** - * Provide source that is understood by one of Node's translators. - * @param {URL['href']} url The URL/path of the module to be loaded - * @param {object} [context] Metadata about the module + * Provide source that is understood by one of Node's translators. Handles customization hooks, + * if any. + * @param {string} url The URL of the module to be loaded. + * @param {object} context Metadata about the module * @returns {Promise<{ format: ModuleFormat, source: ModuleSource }>} */ async load(url, context) { + if (this.#customizations) { + return this.#customizations.load(url, context); + } + defaultLoad ??= require('internal/modules/esm/load').defaultLoad; - const result = this.#customizations ? - await this.#customizations.load(url, context) : - await defaultLoad(url, context); - this.validateLoadResult(url, result?.format); - return result; + return defaultLoad(url, context); } - loadSync(url, context) { + /** + * This is the default load step for future synchronous hooks, which incorporates asynchronous hooks + * from module.register() which are run in a blocking fashion for it to be synchronous. + * @param {string} url See {@link load} + * @param {object} context See {@link load} + * @returns {{ format: ModuleFormat, source: ModuleSource }} + */ + #loadAndMaybeBlockOnLoaderThread(url, context) { + if (this.#customizations) { + return this.#customizations.loadSync(url, context); + } defaultLoadSync ??= require('internal/modules/esm/load').defaultLoadSync; + return defaultLoadSync(url, context); + } - let result = this.#customizations ? - this.#customizations.loadSync(url, context) : - defaultLoadSync(url, context); - let format = result?.format; - if (format === 'module') { - throw new ERR_REQUIRE_ESM(url, true); - } - if (format === 'commonjs') { - format = 'require-commonjs'; - result = { __proto__: result, format }; - } - this.validateLoadResult(url, format); - return result; + /** + * Similar to {@link load} but this is always run synchronously. If there are asynchronous hooks + * from module.register(), this blocks on the loader thread for it to return synchronously. + * + * This is here to support `require()` in imported CJS and future synchronous hooks. + * + * TODO(joyeecheung): consolidate the cache behavior and use this in require(esm). + * @param {string} url See {@link load} + * @param {object} [context] See {@link load} + * @returns {{ format: ModuleFormat, source: ModuleSource }} + */ + #loadSync(url, context) { + return this.#loadAndMaybeBlockOnLoaderThread(url, context); } validateLoadResult(url, format) { diff --git a/lib/internal/modules/esm/module_job.js b/lib/internal/modules/esm/module_job.js index abdd96673f72b3..2f0f6e3aca56e8 100644 --- a/lib/internal/modules/esm/module_job.js +++ b/lib/internal/modules/esm/module_job.js @@ -10,7 +10,6 @@ const { PromisePrototypeThen, RegExpPrototypeExec, RegExpPrototypeSymbolReplace, - ReflectApply, SafePromiseAllReturnArrayLike, SafePromiseAllReturnVoid, SafeSet, @@ -51,13 +50,12 @@ const isCommonJSGlobalLikeNotDefinedError = (errorMessage) => ); class ModuleJobBase { - constructor(url, importAttributes, moduleWrapMaybePromise, isMain, inspectBrk) { + constructor(url, importAttributes, isMain, inspectBrk) { this.importAttributes = importAttributes; this.isMain = isMain; this.inspectBrk = inspectBrk; this.url = url; - this.module = moduleWrapMaybePromise; } } @@ -65,21 +63,29 @@ class ModuleJobBase { * its dependencies, over time. */ class ModuleJob extends ModuleJobBase { #loader = null; - // `loader` is the Loader instance used for loading dependencies. + + /** + * @param {ModuleLoader} loader The ESM loader. + * @param {string} url URL of the module to be wrapped in ModuleJob. + * @param {ImportAttributes} importAttributes Import attributes from the import statement. + * @param {ModuleWrap|Promise} moduleOrModulePromise Translated ModuleWrap for the module. + * @param {boolean} isMain Whether the module is the entry point. + * @param {boolean} inspectBrk Whether this module should be evaluated with the + * first line paused in the debugger (because --inspect-brk is passed). + * @param {boolean} isForRequireInImportedCJS Whether this is created for require() in imported CJS. + */ constructor(loader, url, importAttributes = { __proto__: null }, - moduleProvider, isMain, inspectBrk, sync = false) { - const modulePromise = ReflectApply(moduleProvider, loader, [url, isMain]); - super(url, importAttributes, modulePromise, isMain, inspectBrk); + moduleOrModulePromise, isMain, inspectBrk, isForRequireInImportedCJS = false) { + super(url, importAttributes, isMain, inspectBrk); this.#loader = loader; - // Expose the promise to the ModuleWrap directly for linking below. - // `this.module` is also filled in below. - this.modulePromise = modulePromise; - if (sync) { - this.module = this.modulePromise; + // Expose the promise to the ModuleWrap directly for linking below. + if (isForRequireInImportedCJS) { + this.module = moduleOrModulePromise; + assert(this.module instanceof ModuleWrap); this.modulePromise = PromiseResolve(this.module); } else { - this.modulePromise = PromiseResolve(this.modulePromise); + this.modulePromise = moduleOrModulePromise; } // Promise for the list of all dependencyJobs. @@ -118,7 +124,7 @@ class ModuleJob extends ModuleJobBase { for (let idx = 0; idx < moduleRequests.length; idx++) { const { specifier, attributes } = moduleRequests[idx]; - const dependencyJobPromise = this.#loader.getModuleJob( + const dependencyJobPromise = this.#loader.getModuleJobForImport( specifier, this.url, attributes, ); const modulePromise = PromisePrototypeThen(dependencyJobPromise, (job) => { @@ -280,14 +286,33 @@ class ModuleJob extends ModuleJobBase { } } -// This is a fully synchronous job and does not spawn additional threads in any way. -// All the steps are ensured to be synchronous and it throws on instantiating -// an asynchronous graph. +/** + * This is a fully synchronous job and does not spawn additional threads in any way. + * All the steps are ensured to be synchronous and it throws on instantiating + * an asynchronous graph. It also disallows CJS <-> ESM cycles. + * + * This is used for ES modules loaded via require(esm). Modules loaded by require() in + * imported CJS are handled by ModuleJob with the isForRequireInImportedCJS set to true instead. + * The two currently have different caching behaviors. + * TODO(joyeecheung): consolidate this with the isForRequireInImportedCJS variant of ModuleJob. + */ class ModuleJobSync extends ModuleJobBase { #loader = null; + + /** + * @param {ModuleLoader} loader The ESM loader. + * @param {string} url URL of the module to be wrapped in ModuleJob. + * @param {ImportAttributes} importAttributes Import attributes from the import statement. + * @param {ModuleWrap} moduleWrap Translated ModuleWrap for the module. + * @param {boolean} isMain Whether the module is the entry point. + * @param {boolean} inspectBrk Whether this module should be evaluated with the + * first line paused in the debugger (because --inspect-brk is passed). + */ constructor(loader, url, importAttributes, moduleWrap, isMain, inspectBrk) { - super(url, importAttributes, moduleWrap, isMain, inspectBrk, true); + super(url, importAttributes, isMain, inspectBrk, true); + this.#loader = loader; + this.module = moduleWrap; assert(this.module instanceof ModuleWrap); // Store itself into the cache first before linking in case there are circular diff --git a/lib/internal/modules/esm/translators.js b/lib/internal/modules/esm/translators.js index a4100d9ca63b64..23c14e2ec6c85e 100644 --- a/lib/internal/modules/esm/translators.js +++ b/lib/internal/modules/esm/translators.js @@ -74,28 +74,11 @@ function getSource(url) { /** @type {import('deps/cjs-module-lexer/lexer.js').parse} */ let cjsParse; /** - * Initializes the CommonJS module lexer parser. - * If WebAssembly is available, it uses the optimized version from the dist folder. - * Otherwise, it falls back to the JavaScript version from the lexer folder. + * Initializes the CommonJS module lexer parser using the JavaScript version. + * TODO(joyeecheung): Use `require('internal/deps/cjs-module-lexer/dist/lexer').initSync()` + * when cjs-module-lexer 1.4.0 is rolled in. */ -async function initCJSParse() { - if (typeof WebAssembly === 'undefined') { - initCJSParseSync(); - } else { - const { parse, init } = - require('internal/deps/cjs-module-lexer/dist/lexer'); - try { - await init(); - cjsParse = parse; - } catch { - initCJSParseSync(); - } - } -} - function initCJSParseSync() { - // TODO(joyeecheung): implement a binding that directly compiles using - // v8::WasmModuleObject::Compile() synchronously. if (cjsParse === undefined) { cjsParse = require('internal/deps/cjs-module-lexer/lexer').parse; } @@ -203,7 +186,7 @@ function loadCJSModule(module, source, url, filename, isMain) { } specifier = `${pathToFileURL(path)}`; } - const job = cascadedLoader.getModuleJobSync(specifier, url, importAttributes); + const job = cascadedLoader.getModuleJobForRequireInImportedCJS(specifier, url, importAttributes); job.runSync(); return cjsCache.get(job.url).exports; }; @@ -294,6 +277,7 @@ translators.set('commonjs-sync', function requireCommonJS(url, source, isMain) { // Handle CommonJS modules referenced by `require` calls. // This translator function must be sync, as `require` is sync. translators.set('require-commonjs', (url, source, isMain) => { + initCJSParseSync(); assert(cjsParse); return createCJSModuleWrap(url, source); @@ -301,10 +285,9 @@ translators.set('require-commonjs', (url, source, isMain) => { // Handle CommonJS modules referenced by `import` statements or expressions, // or as the initial entry point when the ESM loader handles a CommonJS entry. -translators.set('commonjs', async function commonjsStrategy(url, source, - isMain) { +translators.set('commonjs', function commonjsStrategy(url, source, isMain) { if (!cjsParse) { - await initCJSParse(); + initCJSParseSync(); } // For backward-compatibility, it's possible to return a nullish value for @@ -322,7 +305,6 @@ translators.set('commonjs', async function commonjsStrategy(url, source, // Continue regardless of error. } return createCJSModuleWrap(url, source, isMain, cjsLoader); - }); /** @@ -483,8 +465,9 @@ translators.set('wasm', async function(url, source) { let compiled; try { - // TODO(joyeecheung): implement a binding that directly compiles using - // v8::WasmModuleObject::Compile() synchronously. + // TODO(joyeecheung): implement a translator that just uses + // compiled = new WebAssembly.Module(source) to compile it + // synchronously. compiled = await WebAssembly.compile(source); } catch (err) { err.message = errPath(url) + ': ' + err.message; From 4ab3aff17a68d13ffe93b98a29bb907b413bc54b Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Thu, 26 Sep 2024 16:21:37 +0200 Subject: [PATCH 19/33] module: unflag --experimental-require-module This unflags --experimental-require-module so require(esm) can be used without the flag. For now, when require() actually encounters an ESM, it will still emit an experimental warning. To opt out of the feature, --no-experimental-require-module can be used. There are some tests specifically testing ERR_REQUIRE_ESM. Some of them are repurposed to test --no-experimental-require-module. Some of them are modified to just expect loading require(esm) to work, when it's appropriate. PR-URL: https://github.com/nodejs/node/pull/55085 Refs: https://github.com/nodejs/node/issues/52697 Reviewed-By: Matteo Collina Reviewed-By: Marco Ippolito Reviewed-By: Rafael Gonzaga Reviewed-By: Yagiz Nizipli Reviewed-By: LiviaMedeiros Reviewed-By: Antoine du Hamel Reviewed-By: Filip Skokan Reviewed-By: Michael Dawson Reviewed-By: Richard Lau --- doc/api/cli.md | 30 +++++++++++++--- doc/api/errors.md | 23 +++++++++---- doc/api/esm.md | 2 +- doc/api/modules.md | 34 +++++++++---------- doc/api/packages.md | 6 ++-- lib/internal/modules/cjs/loader.js | 2 +- lib/internal/modules/esm/load.js | 10 ------ lib/internal/modules/esm/loader.js | 8 ++--- lib/internal/modules/esm/module_job.js | 1 + lib/internal/modules/esm/utils.js | 6 ++-- src/node_options.cc | 5 +-- src/node_options.h | 2 +- test/es-module/test-cjs-esm-warn.js | 11 ++++-- test/es-module/test-esm-detect-ambiguous.mjs | 7 ++-- test/es-module/test-esm-loader-hooks.mjs | 1 + .../es-module/test-esm-type-field-errors-2.js | 17 ++++++++++ test/es-module/test-esm-type-field-errors.js | 6 ---- test/es-module/test-require-module-preload.js | 7 ++-- .../loader-with-custom-condition.mjs | 1 + test/parallel/test-require-mjs.js | 5 +++ 20 files changed, 116 insertions(+), 68 deletions(-) create mode 100644 test/es-module/test-esm-type-field-errors-2.js diff --git a/doc/api/cli.md b/doc/api/cli.md index 9d771266035cac..88c961b99a0a42 100644 --- a/doc/api/cli.md +++ b/doc/api/cli.md @@ -992,7 +992,13 @@ Use the specified file as a security policy. ### `--experimental-require-module` > Stability: 1.1 - Active Development @@ -1555,6 +1561,24 @@ added: v16.6.0 Use this flag to disable top-level await in REPL. +### `--no-experimental-require-module` + + + +> Stability: 1.1 - Active Development + +Disable support for loading a synchronous ES module graph in `require()`. + +See [Loading ECMAScript modules using `require()`][]. + ### `--no-extra-info-on-fatal-exception` -This flag is only useful when `--experimental-require-module` is enabled. - -If the ES module being `require()`'d contains top-level await, this flag +If the ES module being `require()`'d contains top-level `await`, this flag allows Node.js to evaluate the module, try to locate the top-level awaits, and print their location to help users find them. diff --git a/doc/api/errors.md b/doc/api/errors.md index b07ed9a740a228..e93d0dcb102bd1 100644 --- a/doc/api/errors.md +++ b/doc/api/errors.md @@ -2535,8 +2535,8 @@ object. > Stability: 1 - Experimental -When trying to `require()` a [ES Module][] under `--experimental-require-module`, -a CommonJS to ESM or ESM to CommonJS edge participates in an immediate cycle. +When trying to `require()` a [ES Module][], a CommonJS to ESM or ESM to CommonJS edge +participates in an immediate cycle. This is not allowed because ES Modules cannot be evaluated while they are already being evaluated. @@ -2550,8 +2550,8 @@ module, and should be done lazily in an inner function. > Stability: 1 - Experimental -When trying to `require()` a [ES Module][] under `--experimental-require-module`, -the module turns out to be asynchronous. That is, it contains top-level await. +When trying to `require()` a [ES Module][], the module turns out to be asynchronous. +That is, it contains top-level await. To see where the top-level await is, use `--experimental-print-required-tla` (this would execute the modules @@ -2561,12 +2561,20 @@ before looking for the top-level awaits). ### `ERR_REQUIRE_ESM` -> Stability: 1 - Experimental + + +> Stability: 0 - Deprecated An attempt was made to `require()` an [ES Module][]. -To enable `require()` for synchronous module graphs (without -top-level `await`), use `--experimental-require-module`. +This error has been deprecated since `require()` now supports loading synchronous +ES modules. When `require()` encounters an ES module that contains top-level +`await`, it will throw [`ERR_REQUIRE_ASYNC_MODULE`][] instead. @@ -3908,6 +3916,7 @@ An error occurred trying to allocate memory. This should never happen. [`ERR_INVALID_ARG_TYPE`]: #err_invalid_arg_type [`ERR_MISSING_MESSAGE_PORT_IN_TRANSFER_LIST`]: #err_missing_message_port_in_transfer_list [`ERR_MISSING_TRANSFERABLE_IN_TRANSFER_LIST`]: #err_missing_transferable_in_transfer_list +[`ERR_REQUIRE_ASYNC_MODULE`]: #err_require_async_module [`EventEmitter`]: events.md#class-eventemitter [`MessagePort`]: worker_threads.md#class-messageport [`Object.getPrototypeOf`]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/getPrototypeOf diff --git a/doc/api/esm.md b/doc/api/esm.md index 6db69ccd27ff43..3100d5756cd0c2 100644 --- a/doc/api/esm.md +++ b/doc/api/esm.md @@ -450,7 +450,7 @@ compatibility. ### `require` The CommonJS module `require` currently only supports loading synchronous ES -modules when `--experimental-require-module` is enabled. +modules (that is, ES modules that do not use top-level `await`). See [Loading ECMAScript modules using `require()`][] for details. diff --git a/doc/api/modules.md b/doc/api/modules.md index 1ead13fa32f218..1564ea2899ea7d 100644 --- a/doc/api/modules.md +++ b/doc/api/modules.md @@ -172,20 +172,17 @@ relative, and based on the real path of the files making the calls to -> Stability: 1.1 - Active Development. Enable this API with the -> [`--experimental-require-module`][] CLI flag. - The `.mjs` extension is reserved for [ECMAScript Modules][]. -Currently, if the flag `--experimental-require-module` is not used, loading -an ECMAScript module using `require()` will throw a [`ERR_REQUIRE_ESM`][] -error, and users need to use [`import()`][] instead. See -[Determining module system][] section for more info +See [Determining module system][] section for more info regarding which files are parsed as ECMAScript modules. -If `--experimental-require-module` is enabled, and the ECMAScript module being -loaded by `require()` meets the following requirements: +`require()` only supports loading ECMAScript modules that meet the following requirements: * The module is fully synchronous (contains no top-level `await`); and * One of these conditions are met: @@ -194,8 +191,8 @@ loaded by `require()` meets the following requirements: 3. The file has a `.js` extension, the closest `package.json` does not contain `"type": "commonjs"`, and the module contains ES module syntax. -`require()` will load the requested module as an ES Module, and return -the module namespace object. In this case it is similar to dynamic +If the ES Module being loaded meet the requirements, `require()` can load it and +return the module namespace object. In this case it is similar to dynamic `import()` but is run synchronously and returns the name space object directly. @@ -214,7 +211,7 @@ class Point { export default Point; ``` -A CommonJS module can load them with `require()` under `--experimental-detect-module`: +A CommonJS module can load them with `require()`: ```cjs const distance = require('./distance.mjs'); @@ -243,13 +240,19 @@ conventions. Code authored directly in CommonJS should avoid depending on it. If the module being `require()`'d contains top-level `await`, or the module graph it `import`s contains top-level `await`, [`ERR_REQUIRE_ASYNC_MODULE`][] will be thrown. In this case, users should -load the asynchronous module using `import()`. +load the asynchronous module using [`import()`][]. If `--experimental-print-required-tla` is enabled, instead of throwing `ERR_REQUIRE_ASYNC_MODULE` before evaluation, Node.js will evaluate the module, try to locate the top-level awaits, and print their location to help users fix them. +Support for loading ES modules using `require()` is currently +experimental and can be disabled using `--no-experimental-require-module`. +When `require()` actually encounters an ES module for the +first time in the process, it will emit an experimental warning. The +warning is expected to be removed when this feature stablizes. + ## All together @@ -279,8 +282,7 @@ require(X) from module at path Y MAYBE_DETECT_AND_LOAD(X) 1. If X parses as a CommonJS module, load X as a CommonJS module. STOP. -2. Else, if `--experimental-require-module` is - enabled, and the source code of X can be parsed as ECMAScript module using +2. Else, if the source code of X can be parsed as ECMAScript module using DETECT_MODULE_SYNTAX defined in the ESM resolver, a. Load X as an ECMAScript module. STOP. @@ -1196,9 +1198,7 @@ This section was moved to [GLOBAL_FOLDERS]: #loading-from-the-global-folders [`"main"`]: packages.md#main [`"type"`]: packages.md#type -[`--experimental-require-module`]: cli.md#--experimental-require-module [`ERR_REQUIRE_ASYNC_MODULE`]: errors.md#err_require_async_module -[`ERR_REQUIRE_ESM`]: errors.md#err_require_esm [`ERR_UNSUPPORTED_DIR_IMPORT`]: errors.md#err_unsupported_dir_import [`MODULE_NOT_FOUND`]: errors.md#module_not_found [`__dirname`]: #__dirname diff --git a/doc/api/packages.md b/doc/api/packages.md index 5dfc9e7b1da2c3..05f676e7f4ee3d 100644 --- a/doc/api/packages.md +++ b/doc/api/packages.md @@ -172,8 +172,7 @@ There is the CommonJS module loader: * It treats all files that lack `.json` or `.node` extensions as JavaScript text files. * It can only be used to [load ECMASCript modules from CommonJS modules][] if - the module graph is synchronous (that contains no top-level `await`) when - `--experimental-require-module` is enabled. + the module graph is synchronous (that contains no top-level `await`). When used to load a JavaScript text file that is not an ECMAScript module, the file will be loaded as a CommonJS module. @@ -662,8 +661,7 @@ specific to least specific as conditions should be defined: * `"require"` - matches when the package is loaded via `require()`. The referenced file should be loadable with `require()` although the condition matches regardless of the module format of the target file. Expected - formats include CommonJS, JSON, native addons, and ES modules - if `--experimental-require-module` is enabled. _Always mutually + formats include CommonJS, JSON, native addons, and ES modules. _Always mutually exclusive with `"import"`._ * `"module-sync"` - matches no matter the package is loaded via `import`, `import()` or `require()`. The format is expected to be ES modules that does diff --git a/lib/internal/modules/cjs/loader.js b/lib/internal/modules/cjs/loader.js index 6dba8d926316ce..ef6562b82b5193 100644 --- a/lib/internal/modules/cjs/loader.js +++ b/lib/internal/modules/cjs/loader.js @@ -435,7 +435,6 @@ function initializeCJS() { require('internal/modules/run_main').executeUserEntryPoint; if (getOptionValue('--experimental-require-module')) { - emitExperimentalWarning('Support for loading ES Module in require()'); Module._extensions['.mjs'] = loadESMFromCJS; } } @@ -1341,6 +1340,7 @@ function loadESMFromCJS(mod, filename) { // ESM won't be accessible via process.mainModule. setOwnProperty(process, 'mainModule', undefined); } else { + emitExperimentalWarning('Support for loading ES Module in require()'); const { wrap, namespace, diff --git a/lib/internal/modules/esm/load.js b/lib/internal/modules/esm/load.js index 7b77af35a1dfeb..5239bc8ed883a5 100644 --- a/lib/internal/modules/esm/load.js +++ b/lib/internal/modules/esm/load.js @@ -152,11 +152,6 @@ async function defaultLoad(url, context = kEmptyObject) { validateAttributes(url, format, importAttributes); - // Use the synchronous commonjs translator which can deal with cycles. - if (format === 'commonjs' && getOptionValue('--experimental-require-module')) { - format = 'commonjs-sync'; - } - return { __proto__: null, format, @@ -206,11 +201,6 @@ function defaultLoadSync(url, context = kEmptyObject) { validateAttributes(url, format, importAttributes); - // Use the synchronous commonjs translator which can deal with cycles. - if (format === 'commonjs' && getOptionValue('--experimental-require-module')) { - format = 'commonjs-sync'; - } - return { __proto__: null, format, diff --git a/lib/internal/modules/esm/loader.js b/lib/internal/modules/esm/loader.js index 820399a577ce9a..7047925f2d02fb 100644 --- a/lib/internal/modules/esm/loader.js +++ b/lib/internal/modules/esm/loader.js @@ -364,15 +364,15 @@ class ModuleLoader { defaultLoadSync ??= require('internal/modules/esm/load').defaultLoadSync; const loadResult = defaultLoadSync(url, { format, importAttributes }); - const { - format: finalFormat, - source, - } = loadResult; + + // Use the synchronous commonjs translator which can deal with cycles. + const finalFormat = loadResult.format === 'commonjs' ? 'commonjs-sync' : loadResult.format; if (finalFormat === 'wasm') { assert.fail('WASM is currently unsupported by require(esm)'); } + const { source } = loadResult; const isMain = (parentURL === undefined); const wrap = this.#translate(url, finalFormat, source, isMain); assert(wrap instanceof ModuleWrap, `Translator used for require(${url}) should not be async`); diff --git a/lib/internal/modules/esm/module_job.js b/lib/internal/modules/esm/module_job.js index 2f0f6e3aca56e8..ea5354189a5bab 100644 --- a/lib/internal/modules/esm/module_job.js +++ b/lib/internal/modules/esm/module_job.js @@ -353,6 +353,7 @@ class ModuleJobSync extends ModuleJobBase { } runSync() { + // TODO(joyeecheung): add the error decoration logic from the async instantiate. this.module.instantiateSync(); setHasStartedUserESMExecution(); const namespace = this.module.evaluateSync(); diff --git a/lib/internal/modules/esm/utils.js b/lib/internal/modules/esm/utils.js index f94349630e9220..38b5eaf1d7e0f2 100644 --- a/lib/internal/modules/esm/utils.js +++ b/lib/internal/modules/esm/utils.js @@ -75,17 +75,15 @@ function initializeDefaultConditions() { const userConditions = getOptionValue('--conditions'); const noAddons = getOptionValue('--no-addons'); const addonConditions = noAddons ? [] : ['node-addons']; - + const moduleConditions = getOptionValue('--experimental-require-module') ? ['module-sync'] : []; defaultConditions = ObjectFreeze([ 'node', 'import', + ...moduleConditions, ...addonConditions, ...userConditions, ]); defaultConditionsSet = new SafeSet(defaultConditions); - if (getOptionValue('--experimental-require-module')) { - defaultConditionsSet.add('module-sync'); - } } /** diff --git a/src/node_options.cc b/src/node_options.cc index 2491524b89fc27..e6924b29d0b49e 100644 --- a/src/node_options.cc +++ b/src/node_options.cc @@ -374,9 +374,10 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() { &EnvironmentOptions::print_required_tla, kAllowedInEnvvar); AddOption("--experimental-require-module", - "Allow loading explicit ES Modules in require().", + "Allow loading synchronous ES Modules in require().", &EnvironmentOptions::require_module, - kAllowedInEnvvar); + kAllowedInEnvvar, + true); AddOption("--diagnostic-dir", "set dir for all output files" " (default: current working directory)", diff --git a/src/node_options.h b/src/node_options.h index 70e8998284386d..6f95470c7f513d 100644 --- a/src/node_options.h +++ b/src/node_options.h @@ -117,7 +117,7 @@ class EnvironmentOptions : public Options { std::vector conditions; bool detect_module = true; bool print_required_tla = false; - bool require_module = false; + bool require_module = true; std::string dns_result_order; bool enable_source_maps = false; bool experimental_eventsource = false; diff --git a/test/es-module/test-cjs-esm-warn.js b/test/es-module/test-cjs-esm-warn.js index 7ac85fd58c5f18..44f8e21da9d686 100644 --- a/test/es-module/test-cjs-esm-warn.js +++ b/test/es-module/test-cjs-esm-warn.js @@ -1,3 +1,6 @@ +// Previously, this tested that require(esm) throws ERR_REQUIRE_ESM, which is no longer applicable +// since require(esm) is now supported. The test has been repurposed to ensure that the old behavior +// is preserved when the --no-experimental-require-module flag is used. 'use strict'; const { spawnPromisified } = require('../common'); @@ -22,7 +25,9 @@ describe('CJS ↔︎ ESM interop warnings', { concurrency: true }, () => { fixtures.path('/es-modules/package-type-module/cjs.js') ); const basename = 'cjs.js'; - const { code, signal, stderr } = await spawnPromisified(execPath, [requiringCjsAsEsm]); + const { code, signal, stderr } = await spawnPromisified(execPath, [ + '--no-experimental-require-module', requiringCjsAsEsm, + ]); assert.ok( stderr.replaceAll('\r', '').includes( @@ -48,7 +53,9 @@ describe('CJS ↔︎ ESM interop warnings', { concurrency: true }, () => { fixtures.path('/es-modules/package-type-module/esm.js') ); const basename = 'esm.js'; - const { code, signal, stderr } = await spawnPromisified(execPath, [requiringEsm]); + const { code, signal, stderr } = await spawnPromisified(execPath, [ + '--no-experimental-require-module', requiringEsm, + ]); assert.ok( stderr.replace(/\r/g, '').includes( diff --git a/test/es-module/test-esm-detect-ambiguous.mjs b/test/es-module/test-esm-detect-ambiguous.mjs index b0a62e6dcfb290..8c26c696fa2e0a 100644 --- a/test/es-module/test-esm-detect-ambiguous.mjs +++ b/test/es-module/test-esm-detect-ambiguous.mjs @@ -402,15 +402,16 @@ describe('Module syntax detection', { concurrency: !process.env.TEST_PARALLEL }, }); }); -// Validate temporarily disabling `--abort-on-uncaught-exception` -// while running `containsModuleSyntax`. +// Checks the error caught during module detection does not trigger abort when +// `--abort-on-uncaught-exception` is passed in (as that's a caught internal error). // Ref: https://github.com/nodejs/node/issues/50878 describe('Wrapping a `require` of an ES module while using `--abort-on-uncaught-exception`', () => { it('should work', async () => { const { code, signal, stdout, stderr } = await spawnPromisified(process.execPath, [ '--abort-on-uncaught-exception', + '--no-warnings', '--eval', - 'assert.throws(() => require("./package-type-module/esm.js"), { code: "ERR_REQUIRE_ESM" })', + 'require("./package-type-module/esm.js")', ], { cwd: fixtures.path('es-modules'), }); diff --git a/test/es-module/test-esm-loader-hooks.mjs b/test/es-module/test-esm-loader-hooks.mjs index 50c414ff50829c..1476bc44cb7520 100644 --- a/test/es-module/test-esm-loader-hooks.mjs +++ b/test/es-module/test-esm-loader-hooks.mjs @@ -739,6 +739,7 @@ describe('Loader hooks', { concurrency: true }, () => { describe('should use hooks', async () => { const { code, signal, stdout, stderr } = await spawnPromisified(process.execPath, [ + '--no-experimental-require-module', '--import', fixtures.fileURL('es-module-loaders/builtin-named-exports.mjs'), fixtures.path('es-modules/require-esm-throws-with-loaders.js'), diff --git a/test/es-module/test-esm-type-field-errors-2.js b/test/es-module/test-esm-type-field-errors-2.js new file mode 100644 index 00000000000000..3ea259446c7fb2 --- /dev/null +++ b/test/es-module/test-esm-type-field-errors-2.js @@ -0,0 +1,17 @@ +// Flags: --no-experimental-require-module +// Previously, this tested that require(esm) throws ERR_REQUIRE_ESM, which is no longer applicable +// since require(esm) is now supported. The test has been repurposed to ensure that the old behavior +// is preserved when the --no-experimental-require-module flag is used. + +'use strict'; +require('../common'); +const assert = require('assert'); +const { describe, it } = require('node:test'); + +describe('Errors related to ESM type field', () => { + it('Should throw an error when loading CJS from a `type: "module"` package.', () => { + assert.throws(() => require('../fixtures/es-modules/package-type-module/index.js'), { + code: 'ERR_REQUIRE_ESM' + }); + }); +}); diff --git a/test/es-module/test-esm-type-field-errors.js b/test/es-module/test-esm-type-field-errors.js index 9ec9aa64e18c07..4bf52f3ad6e7d3 100644 --- a/test/es-module/test-esm-type-field-errors.js +++ b/test/es-module/test-esm-type-field-errors.js @@ -50,12 +50,6 @@ describe('ESM type field errors', { concurrency: true }, () => { true, ); }); - - it('--input-type=module disallowed for directories', () => { - assert.throws(() => require('../fixtures/es-modules/package-type-module/index.js'), { - code: 'ERR_REQUIRE_ESM' - }); - }); }); function expect(opt = '', inputFile, want, wantsError = false) { diff --git a/test/es-module/test-require-module-preload.js b/test/es-module/test-require-module-preload.js index f2e572969a050c..65ec1a93bc9d4b 100644 --- a/test/es-module/test-require-module-preload.js +++ b/test/es-module/test-require-module-preload.js @@ -3,9 +3,12 @@ require('../common'); const { spawnSyncAndAssert } = require('../common/child_process'); const { fixturesDir } = require('../common/fixtures'); -const stderr = /ExperimentalWarning: Support for loading ES Module in require/; +const warningRE = /ExperimentalWarning: Support for loading ES Module in require/; function testPreload(preloadFlag) { + // The warning is only emitted when ESM is loaded by --require. + const stderr = preloadFlag !== '--import' ? warningRE : undefined; + // Test named exports. { spawnSyncAndAssert( @@ -112,7 +115,7 @@ testPreload('--import'); }, { stdout: /^package-type-module\s+A$/, - stderr, + stderr: warningRE, trim: true, } ); diff --git a/test/fixtures/es-module-loaders/loader-with-custom-condition.mjs b/test/fixtures/es-module-loaders/loader-with-custom-condition.mjs index 3aefed51d57d3e..90c55fa576bb3c 100644 --- a/test/fixtures/es-module-loaders/loader-with-custom-condition.mjs +++ b/test/fixtures/es-module-loaders/loader-with-custom-condition.mjs @@ -5,6 +5,7 @@ export async function resolve(specifier, context, defaultResolve) { deepStrictEqual([...context.conditions].sort(), [ 'import', + 'module-sync', 'node', 'node-addons', ]); diff --git a/test/parallel/test-require-mjs.js b/test/parallel/test-require-mjs.js index 112f08879d4290..c169ec07ab6bd8 100644 --- a/test/parallel/test-require-mjs.js +++ b/test/parallel/test-require-mjs.js @@ -1,3 +1,8 @@ +// Flags: --no-experimental-require-module +// Previously, this tested that require(esm) throws ERR_REQUIRE_ESM, which is no longer applicable +// since require(esm) is now supported. The test has been repurposed to ensure that the old behavior +// is preserved when the --no-experimental-require-module flag is used. + 'use strict'; require('../common'); const assert = require('assert'); From 824d4179d77125ac180160328eed77a259d44d86 Mon Sep 17 00:00:00 2001 From: Guy Bedford Date: Sun, 14 Jul 2024 13:00:49 -0700 Subject: [PATCH 20/33] module: support 'module.exports' interop export in require(esm) PR-URL: https://github.com/nodejs/node/pull/54563 Reviewed-By: Matteo Collina Reviewed-By: Antoine du Hamel Reviewed-By: James M Snell Reviewed-By: Joyee Cheung --- doc/api/modules.md | 66 ++++++++++++++++++- lib/internal/modules/cjs/loader.js | 11 ++-- .../es-module/test-require-as-esm-interop.mjs | 51 ++++++++++++++ .../node_modules/interop-cjs-esm/false-esm.js | 3 + .../node_modules/interop-cjs-esm/false.js | 3 + .../interop-cjs-esm/fauxesmdefault-esm.js | 3 + .../interop-cjs-esm/fauxesmdefault.js | 3 + .../interop-cjs-esm/fauxesmmixed-esm.js | 4 ++ .../interop-cjs-esm/fauxesmmixed.js | 3 + .../interop-cjs-esm/fauxesmnamed-esm.js | 3 + .../interop-cjs-esm/fauxesmnamed.js | 3 + .../interop-cjs-esm/object-esm.js | 3 + .../node_modules/interop-cjs-esm/object.js | 3 + .../node_modules/interop-cjs-esm/package.json | 17 +++++ .../interop-cjs-esm/string-esm.js | 3 + .../node_modules/interop-cjs-esm/string.js | 3 + .../node_modules/interop-cjs/false-esm.js | 1 + .../node_modules/interop-cjs/false.js | 1 + .../interop-cjs/fauxesmdefault-esm.js | 1 + .../interop-cjs/fauxesmdefault.js | 1 + .../interop-cjs/fauxesmmixed-esm.js | 1 + .../node_modules/interop-cjs/fauxesmmixed.js | 1 + .../interop-cjs/fauxesmnamed-esm.js | 1 + .../node_modules/interop-cjs/fauxesmnamed.js | 1 + .../node_modules/interop-cjs/object-esm.js | 1 + .../node_modules/interop-cjs/object.js | 1 + .../node_modules/interop-cjs/package.json | 17 +++++ .../node_modules/interop-cjs/string-esm.js | 1 + .../node_modules/interop-cjs/string.js | 1 + .../interop-cjsdep-false-esm/dep.js | 3 + .../interop-cjsdep-false-esm/package.json | 4 ++ .../node_modules/interop-cjsdep-false/dep.js | 1 + .../interop-cjsdep-false/package.json | 4 ++ .../interop-cjsdep-fauxesmdefault-esm/dep.js | 7 ++ .../package.json | 4 ++ .../interop-cjsdep-fauxesmdefault/dep.js | 2 + .../package.json | 4 ++ .../interop-cjsdep-fauxesmmixed-esm/dep.js | 9 +++ .../package.json | 4 ++ .../interop-cjsdep-fauxesmmixed/dep.js | 4 ++ .../interop-cjsdep-fauxesmmixed/package.json | 4 ++ .../interop-cjsdep-fauxesmnamed-esm/dep.js | 8 +++ .../package.json | 4 ++ .../interop-cjsdep-fauxesmnamed/dep.js | 3 + .../interop-cjsdep-fauxesmnamed/package.json | 4 ++ .../interop-cjsdep-object-esm/dep.js | 8 +++ .../interop-cjsdep-object-esm/package.json | 4 ++ .../node_modules/interop-cjsdep-object/dep.js | 2 + .../interop-cjsdep-object/package.json | 4 ++ .../interop-cjsdep-string-esm/dep.js | 3 + .../interop-cjsdep-string-esm/package.json | 4 ++ .../node_modules/interop-cjsdep-string/dep.js | 1 + .../interop-cjsdep-string/package.json | 4 ++ test/fixtures/pkgexports.mjs | 4 ++ 54 files changed, 308 insertions(+), 6 deletions(-) create mode 100644 test/es-module/test-require-as-esm-interop.mjs create mode 100644 test/fixtures/node_modules/interop-cjs-esm/false-esm.js create mode 100644 test/fixtures/node_modules/interop-cjs-esm/false.js create mode 100644 test/fixtures/node_modules/interop-cjs-esm/fauxesmdefault-esm.js create mode 100644 test/fixtures/node_modules/interop-cjs-esm/fauxesmdefault.js create mode 100644 test/fixtures/node_modules/interop-cjs-esm/fauxesmmixed-esm.js create mode 100644 test/fixtures/node_modules/interop-cjs-esm/fauxesmmixed.js create mode 100644 test/fixtures/node_modules/interop-cjs-esm/fauxesmnamed-esm.js create mode 100644 test/fixtures/node_modules/interop-cjs-esm/fauxesmnamed.js create mode 100644 test/fixtures/node_modules/interop-cjs-esm/object-esm.js create mode 100644 test/fixtures/node_modules/interop-cjs-esm/object.js create mode 100644 test/fixtures/node_modules/interop-cjs-esm/package.json create mode 100644 test/fixtures/node_modules/interop-cjs-esm/string-esm.js create mode 100644 test/fixtures/node_modules/interop-cjs-esm/string.js create mode 100644 test/fixtures/node_modules/interop-cjs/false-esm.js create mode 100644 test/fixtures/node_modules/interop-cjs/false.js create mode 100644 test/fixtures/node_modules/interop-cjs/fauxesmdefault-esm.js create mode 100644 test/fixtures/node_modules/interop-cjs/fauxesmdefault.js create mode 100644 test/fixtures/node_modules/interop-cjs/fauxesmmixed-esm.js create mode 100644 test/fixtures/node_modules/interop-cjs/fauxesmmixed.js create mode 100644 test/fixtures/node_modules/interop-cjs/fauxesmnamed-esm.js create mode 100644 test/fixtures/node_modules/interop-cjs/fauxesmnamed.js create mode 100644 test/fixtures/node_modules/interop-cjs/object-esm.js create mode 100644 test/fixtures/node_modules/interop-cjs/object.js create mode 100644 test/fixtures/node_modules/interop-cjs/package.json create mode 100644 test/fixtures/node_modules/interop-cjs/string-esm.js create mode 100644 test/fixtures/node_modules/interop-cjs/string.js create mode 100644 test/fixtures/node_modules/interop-cjsdep-false-esm/dep.js create mode 100644 test/fixtures/node_modules/interop-cjsdep-false-esm/package.json create mode 100644 test/fixtures/node_modules/interop-cjsdep-false/dep.js create mode 100644 test/fixtures/node_modules/interop-cjsdep-false/package.json create mode 100644 test/fixtures/node_modules/interop-cjsdep-fauxesmdefault-esm/dep.js create mode 100644 test/fixtures/node_modules/interop-cjsdep-fauxesmdefault-esm/package.json create mode 100644 test/fixtures/node_modules/interop-cjsdep-fauxesmdefault/dep.js create mode 100644 test/fixtures/node_modules/interop-cjsdep-fauxesmdefault/package.json create mode 100644 test/fixtures/node_modules/interop-cjsdep-fauxesmmixed-esm/dep.js create mode 100644 test/fixtures/node_modules/interop-cjsdep-fauxesmmixed-esm/package.json create mode 100644 test/fixtures/node_modules/interop-cjsdep-fauxesmmixed/dep.js create mode 100644 test/fixtures/node_modules/interop-cjsdep-fauxesmmixed/package.json create mode 100644 test/fixtures/node_modules/interop-cjsdep-fauxesmnamed-esm/dep.js create mode 100644 test/fixtures/node_modules/interop-cjsdep-fauxesmnamed-esm/package.json create mode 100644 test/fixtures/node_modules/interop-cjsdep-fauxesmnamed/dep.js create mode 100644 test/fixtures/node_modules/interop-cjsdep-fauxesmnamed/package.json create mode 100644 test/fixtures/node_modules/interop-cjsdep-object-esm/dep.js create mode 100644 test/fixtures/node_modules/interop-cjsdep-object-esm/package.json create mode 100644 test/fixtures/node_modules/interop-cjsdep-object/dep.js create mode 100644 test/fixtures/node_modules/interop-cjsdep-object/package.json create mode 100644 test/fixtures/node_modules/interop-cjsdep-string-esm/dep.js create mode 100644 test/fixtures/node_modules/interop-cjsdep-string-esm/package.json create mode 100644 test/fixtures/node_modules/interop-cjsdep-string/dep.js create mode 100644 test/fixtures/node_modules/interop-cjsdep-string/package.json diff --git a/doc/api/modules.md b/doc/api/modules.md index 1564ea2899ea7d..fa5f27398300dd 100644 --- a/doc/api/modules.md +++ b/doc/api/modules.md @@ -176,6 +176,9 @@ changes: - version: REPLACEME pr-url: https://github.com/nodejs/node/pull/55085 description: require() now supports loading synchronous ES modules by default. + - version: REPLACEME + pr-url: https://github.com/nodejs/node/pull/54563 + description: Support `'module.exports'` interop export in `require(esm)`. --> The `.mjs` extension is reserved for [ECMAScript Modules][]. @@ -205,10 +208,9 @@ export function distance(a, b) { return (b.x - a.x) ** 2 + (b.y - a.y) ** 2; } ```mjs // point.mjs -class Point { +export default class Point { constructor(x, y) { this.x = x; this.y = y; } } -export default Point; ``` A CommonJS module can load them with `require()`: @@ -237,6 +239,66 @@ This property is experimental and can change in the future. It should only be us by tools converting ES modules into CommonJS modules, following existing ecosystem conventions. Code authored directly in CommonJS should avoid depending on it. +When a ES Module contains both named exports and a default export, the result returned by `require()` +is the module namespace object, which places the default export in the `.default` property, similar to +the results returned by `import()`. +To customize what should be returned by `require(esm)` directly, the ES Module can export the +desired value using the string name `"module.exports"`. + + + +```mjs +// point.mjs +export default class Point { + constructor(x, y) { this.x = x; this.y = y; } +} + +// `distance` is lost to CommonJS consumers of this module, unless it's +// added to `Point` as a static property. +export function distance(a, b) { return (b.x - a.x) ** 2 + (b.y - a.y) ** 2; } +export { Point as 'module.exports' } +``` + + + +```cjs +const Point = require('./point.mjs'); +console.log(Point); // [class Point] + +// Named exports are lost when 'module.exports' is used +const { distance } = require('./point.mjs'); +console.log(distance); // undefined +``` + +Notice in the example above, when the `module.exports` export name is used, named exports +will be lost to CommonJS consumers. To allow CommonJS consumers to continue accessing +named exports, the module can make sure that the default export is an object with the +named exports attached to it as properties. For example with the example above, +`distance` can be attached to the default export, the `Point` class, as a static method. + + + +```mjs +export function distance(a, b) { return (b.x - a.x) ** 2 + (b.y - a.y) ** 2; } + +export default class Point { + constructor(x, y) { this.x = x; this.y = y; } + static distance = distance; +} + +export { Point as 'module.exports' } +``` + + + +```cjs +const Point = require('./point.mjs'); +console.log(Point); // [class Point] + +const { distance } = require('./point.mjs'); +console.log(distance); // [Function: distance] +``` + If the module being `require()`'d contains top-level `await`, or the module graph it `import`s contains top-level `await`, [`ERR_REQUIRE_ASYNC_MODULE`][] will be thrown. In this case, users should diff --git a/lib/internal/modules/cjs/loader.js b/lib/internal/modules/cjs/loader.js index ef6562b82b5193..ca78705e0140a4 100644 --- a/lib/internal/modules/cjs/loader.js +++ b/lib/internal/modules/cjs/loader.js @@ -1382,10 +1382,13 @@ function loadESMFromCJS(mod, filename) { // createRequiredModuleFacade() to `wrap` which is a ModuleWrap wrapping // over the original module. - // We don't do this to modules that don't have default exports to avoid - // the unnecessary overhead. If __esModule is already defined, we will - // also skip the extension to allow users to override it. - if (!ObjectHasOwn(namespace, 'default') || ObjectHasOwn(namespace, '__esModule')) { + // We don't do this to modules that are marked as CJS ESM or that + // don't have default exports to avoid the unnecessary overhead. + // If __esModule is already defined, we will also skip the extension + // to allow users to override it. + if (ObjectHasOwn(namespace, 'module.exports')) { + mod.exports = namespace['module.exports']; + } else if (!ObjectHasOwn(namespace, 'default') || ObjectHasOwn(namespace, '__esModule')) { mod.exports = namespace; } else { mod.exports = createRequiredModuleFacade(wrap); diff --git a/test/es-module/test-require-as-esm-interop.mjs b/test/es-module/test-require-as-esm-interop.mjs new file mode 100644 index 00000000000000..8364fd6c2fceca --- /dev/null +++ b/test/es-module/test-require-as-esm-interop.mjs @@ -0,0 +1,51 @@ +// Flags: --experimental-require-module +import '../common/index.mjs'; +import assert from 'assert'; +import { directRequireFixture, importFixture } from '../fixtures/pkgexports.mjs'; + +const tests = { + 'false': false, + 'string': 'cjs', + 'object': { a: 'cjs a', b: 'cjs b' }, + 'fauxesmdefault': { default: 'faux esm default' }, + 'fauxesmmixed': { default: 'faux esm default', a: 'faux esm a', b: 'faux esm b' }, + 'fauxesmnamed': { a: 'faux esm a', b: 'faux esm b' } +}; + +// This test demonstrates interop between CJS and CJS represented as ESM +// under the new `export { ... as 'module.exports'}` pattern, for the above cases. +for (const [test, exactShape] of Object.entries(tests)) { + // Each case represents a CJS dependency, which has the expected shape in CJS: + assert.deepStrictEqual(directRequireFixture(`interop-cjsdep-${test}`), exactShape); + + // Each dependency is reexported through CJS as if it is a library being consumed, + // which in CJS is fully shape-preserving: + assert.deepStrictEqual(directRequireFixture(`interop-cjs/${test}`), exactShape); + + // Now we have ESM conversions of these dependencies, using `export ... as "module.exports"` + // staring with the conversion of those dependencies into ESM under require(esm): + assert.deepStrictEqual(directRequireFixture(`interop-cjsdep-${test}-esm`), exactShape); + + // When importing these ESM conversions, from require(esm), we should preserve the shape: + assert.deepStrictEqual(directRequireFixture(`interop-cjs/${test}-esm`), exactShape); + + // Now if the importer itself is converted into ESM, it should still be able to load the original + // CJS and reexport it, preserving the shape: + assert.deepStrictEqual(directRequireFixture(`interop-cjs-esm/${test}`), exactShape); + + // And then if we have the converted CJS to ESM importing from converted CJS to ESM, + // that should also work: + assert.deepStrictEqual(directRequireFixture(`interop-cjs-esm/${test}-esm`), exactShape); + + // Finally, the CJS ESM representation under `import()` should match all these cases equivalently, + // where the CJS module is exported as the default export: + const esmCjsImport = await importFixture(`interop-cjsdep-${test}`); + assert.deepStrictEqual(esmCjsImport.default, exactShape); + + assert.deepStrictEqual((await importFixture(`interop-cjsdep-${test}`)).default, exactShape); + assert.deepStrictEqual((await importFixture(`interop-cjs/${test}`)).default, exactShape); + assert.deepStrictEqual((await importFixture(`interop-cjsdep-${test}-esm`)).default, exactShape); + assert.deepStrictEqual((await importFixture(`interop-cjs/${test}-esm`)).default, exactShape); + assert.deepStrictEqual((await importFixture(`interop-cjs-esm/${test}`)).default, exactShape); + assert.deepStrictEqual((await importFixture(`interop-cjs-esm/${test}-esm`)).default, exactShape); +} diff --git a/test/fixtures/node_modules/interop-cjs-esm/false-esm.js b/test/fixtures/node_modules/interop-cjs-esm/false-esm.js new file mode 100644 index 00000000000000..44dd289dd080f4 --- /dev/null +++ b/test/fixtures/node_modules/interop-cjs-esm/false-esm.js @@ -0,0 +1,3 @@ +import dep from 'interop-cjsdep-false-esm'; +export default dep; +export { dep as 'module.exports' } diff --git a/test/fixtures/node_modules/interop-cjs-esm/false.js b/test/fixtures/node_modules/interop-cjs-esm/false.js new file mode 100644 index 00000000000000..63d896a4736bd6 --- /dev/null +++ b/test/fixtures/node_modules/interop-cjs-esm/false.js @@ -0,0 +1,3 @@ +import dep from 'interop-cjsdep-false'; +export default dep; +export { dep as 'module.exports' } diff --git a/test/fixtures/node_modules/interop-cjs-esm/fauxesmdefault-esm.js b/test/fixtures/node_modules/interop-cjs-esm/fauxesmdefault-esm.js new file mode 100644 index 00000000000000..3b87be4bd0c0bc --- /dev/null +++ b/test/fixtures/node_modules/interop-cjs-esm/fauxesmdefault-esm.js @@ -0,0 +1,3 @@ +import dep from 'interop-cjsdep-fauxesmdefault-esm'; +export default dep; +export { dep as 'module.exports' } diff --git a/test/fixtures/node_modules/interop-cjs-esm/fauxesmdefault.js b/test/fixtures/node_modules/interop-cjs-esm/fauxesmdefault.js new file mode 100644 index 00000000000000..325fa352e16a81 --- /dev/null +++ b/test/fixtures/node_modules/interop-cjs-esm/fauxesmdefault.js @@ -0,0 +1,3 @@ +import dep from 'interop-cjsdep-fauxesmdefault'; +export default dep; +export { dep as 'module.exports' } diff --git a/test/fixtures/node_modules/interop-cjs-esm/fauxesmmixed-esm.js b/test/fixtures/node_modules/interop-cjs-esm/fauxesmmixed-esm.js new file mode 100644 index 00000000000000..9390b24f8d8bf4 --- /dev/null +++ b/test/fixtures/node_modules/interop-cjs-esm/fauxesmmixed-esm.js @@ -0,0 +1,4 @@ +import dep from 'interop-cjsdep-fauxesmmixed-esm'; +export default dep; +export { dep as 'module.exports' } + diff --git a/test/fixtures/node_modules/interop-cjs-esm/fauxesmmixed.js b/test/fixtures/node_modules/interop-cjs-esm/fauxesmmixed.js new file mode 100644 index 00000000000000..d649233736c31c --- /dev/null +++ b/test/fixtures/node_modules/interop-cjs-esm/fauxesmmixed.js @@ -0,0 +1,3 @@ +import dep from 'interop-cjsdep-fauxesmmixed'; +export default dep; +export { dep as 'module.exports' } diff --git a/test/fixtures/node_modules/interop-cjs-esm/fauxesmnamed-esm.js b/test/fixtures/node_modules/interop-cjs-esm/fauxesmnamed-esm.js new file mode 100644 index 00000000000000..6ba7e266bc57bd --- /dev/null +++ b/test/fixtures/node_modules/interop-cjs-esm/fauxesmnamed-esm.js @@ -0,0 +1,3 @@ +import dep from 'interop-cjsdep-fauxesmnamed-esm'; +export default dep; +export { dep as 'module.exports' } diff --git a/test/fixtures/node_modules/interop-cjs-esm/fauxesmnamed.js b/test/fixtures/node_modules/interop-cjs-esm/fauxesmnamed.js new file mode 100644 index 00000000000000..7c6c6bfc14adcd --- /dev/null +++ b/test/fixtures/node_modules/interop-cjs-esm/fauxesmnamed.js @@ -0,0 +1,3 @@ +import dep from 'interop-cjsdep-fauxesmnamed'; +export default dep; +export { dep as 'module.exports' } diff --git a/test/fixtures/node_modules/interop-cjs-esm/object-esm.js b/test/fixtures/node_modules/interop-cjs-esm/object-esm.js new file mode 100644 index 00000000000000..0a4e4283625e5b --- /dev/null +++ b/test/fixtures/node_modules/interop-cjs-esm/object-esm.js @@ -0,0 +1,3 @@ +import dep from 'interop-cjsdep-object-esm'; +export default dep; +export { dep as 'module.exports' } diff --git a/test/fixtures/node_modules/interop-cjs-esm/object.js b/test/fixtures/node_modules/interop-cjs-esm/object.js new file mode 100644 index 00000000000000..08ef3ba4f96f3d --- /dev/null +++ b/test/fixtures/node_modules/interop-cjs-esm/object.js @@ -0,0 +1,3 @@ +import dep from 'interop-cjsdep-object'; +export default dep; +export { dep as 'module.exports' } diff --git a/test/fixtures/node_modules/interop-cjs-esm/package.json b/test/fixtures/node_modules/interop-cjs-esm/package.json new file mode 100644 index 00000000000000..95315e635e7c30 --- /dev/null +++ b/test/fixtures/node_modules/interop-cjs-esm/package.json @@ -0,0 +1,17 @@ +{ + "type": "module", + "exports": { + "./false-esm": "./false-esm.js", + "./false": "./false.js", + "./fauxesmdefault-esm": "./fauxesmdefault-esm.js", + "./fauxesmdefault": "./fauxesmdefault.js", + "./fauxesmmixed-esm": "./fauxesmmixed-esm.js", + "./fauxesmmixed": "./fauxesmmixed.js", + "./fauxesmnamed-esm": "./fauxesmnamed-esm.js", + "./fauxesmnamed": "./fauxesmnamed.js", + "./object-esm": "./object-esm.js", + "./object": "./object.js", + "./string-esm": "./string-esm.js", + "./string": "./string.js" + } +} diff --git a/test/fixtures/node_modules/interop-cjs-esm/string-esm.js b/test/fixtures/node_modules/interop-cjs-esm/string-esm.js new file mode 100644 index 00000000000000..6072f01bc95eed --- /dev/null +++ b/test/fixtures/node_modules/interop-cjs-esm/string-esm.js @@ -0,0 +1,3 @@ +import dep from 'interop-cjsdep-string-esm'; +export default dep; +export { dep as 'module.exports' } diff --git a/test/fixtures/node_modules/interop-cjs-esm/string.js b/test/fixtures/node_modules/interop-cjs-esm/string.js new file mode 100644 index 00000000000000..74297e8eee8d39 --- /dev/null +++ b/test/fixtures/node_modules/interop-cjs-esm/string.js @@ -0,0 +1,3 @@ +import dep from 'interop-cjsdep-string'; +export default dep; +export { dep as 'module.exports' } diff --git a/test/fixtures/node_modules/interop-cjs/false-esm.js b/test/fixtures/node_modules/interop-cjs/false-esm.js new file mode 100644 index 00000000000000..eeb934952b51de --- /dev/null +++ b/test/fixtures/node_modules/interop-cjs/false-esm.js @@ -0,0 +1 @@ +module.exports = require('interop-cjsdep-false-esm'); diff --git a/test/fixtures/node_modules/interop-cjs/false.js b/test/fixtures/node_modules/interop-cjs/false.js new file mode 100644 index 00000000000000..459ccbeee45944 --- /dev/null +++ b/test/fixtures/node_modules/interop-cjs/false.js @@ -0,0 +1 @@ +module.exports = require('interop-cjsdep-false'); diff --git a/test/fixtures/node_modules/interop-cjs/fauxesmdefault-esm.js b/test/fixtures/node_modules/interop-cjs/fauxesmdefault-esm.js new file mode 100644 index 00000000000000..45e91e3c3c80da --- /dev/null +++ b/test/fixtures/node_modules/interop-cjs/fauxesmdefault-esm.js @@ -0,0 +1 @@ +module.exports = require('interop-cjsdep-fauxesmdefault-esm'); diff --git a/test/fixtures/node_modules/interop-cjs/fauxesmdefault.js b/test/fixtures/node_modules/interop-cjs/fauxesmdefault.js new file mode 100644 index 00000000000000..c6e5176f528416 --- /dev/null +++ b/test/fixtures/node_modules/interop-cjs/fauxesmdefault.js @@ -0,0 +1 @@ +module.exports = require('interop-cjsdep-fauxesmdefault'); diff --git a/test/fixtures/node_modules/interop-cjs/fauxesmmixed-esm.js b/test/fixtures/node_modules/interop-cjs/fauxesmmixed-esm.js new file mode 100644 index 00000000000000..6b16d657b0e42d --- /dev/null +++ b/test/fixtures/node_modules/interop-cjs/fauxesmmixed-esm.js @@ -0,0 +1 @@ +module.exports = require('interop-cjsdep-fauxesmmixed-esm'); diff --git a/test/fixtures/node_modules/interop-cjs/fauxesmmixed.js b/test/fixtures/node_modules/interop-cjs/fauxesmmixed.js new file mode 100644 index 00000000000000..8897d00fb3e563 --- /dev/null +++ b/test/fixtures/node_modules/interop-cjs/fauxesmmixed.js @@ -0,0 +1 @@ +module.exports = require('interop-cjsdep-fauxesmmixed'); diff --git a/test/fixtures/node_modules/interop-cjs/fauxesmnamed-esm.js b/test/fixtures/node_modules/interop-cjs/fauxesmnamed-esm.js new file mode 100644 index 00000000000000..67ed05bfa7f6d4 --- /dev/null +++ b/test/fixtures/node_modules/interop-cjs/fauxesmnamed-esm.js @@ -0,0 +1 @@ +module.exports = require('interop-cjsdep-fauxesmnamed-esm'); diff --git a/test/fixtures/node_modules/interop-cjs/fauxesmnamed.js b/test/fixtures/node_modules/interop-cjs/fauxesmnamed.js new file mode 100644 index 00000000000000..092ec333260157 --- /dev/null +++ b/test/fixtures/node_modules/interop-cjs/fauxesmnamed.js @@ -0,0 +1 @@ +module.exports = require('interop-cjsdep-fauxesmnamed'); diff --git a/test/fixtures/node_modules/interop-cjs/object-esm.js b/test/fixtures/node_modules/interop-cjs/object-esm.js new file mode 100644 index 00000000000000..f0e0d3fa3dad07 --- /dev/null +++ b/test/fixtures/node_modules/interop-cjs/object-esm.js @@ -0,0 +1 @@ +module.exports = require('interop-cjsdep-object-esm'); diff --git a/test/fixtures/node_modules/interop-cjs/object.js b/test/fixtures/node_modules/interop-cjs/object.js new file mode 100644 index 00000000000000..ff2d57e11b824f --- /dev/null +++ b/test/fixtures/node_modules/interop-cjs/object.js @@ -0,0 +1 @@ +module.exports = require('interop-cjsdep-object'); diff --git a/test/fixtures/node_modules/interop-cjs/package.json b/test/fixtures/node_modules/interop-cjs/package.json new file mode 100644 index 00000000000000..9e8c0b3f59712c --- /dev/null +++ b/test/fixtures/node_modules/interop-cjs/package.json @@ -0,0 +1,17 @@ +{ + "type": "commonjs", + "exports": { + "./false-esm": "./false-esm.js", + "./false": "./false.js", + "./fauxesmdefault-esm": "./fauxesmdefault-esm.js", + "./fauxesmdefault": "./fauxesmdefault.js", + "./fauxesmmixed-esm": "./fauxesmmixed-esm.js", + "./fauxesmmixed": "./fauxesmmixed.js", + "./fauxesmnamed-esm": "./fauxesmnamed-esm.js", + "./fauxesmnamed": "./fauxesmnamed.js", + "./object-esm": "./object-esm.js", + "./object": "./object.js", + "./string-esm": "./string-esm.js", + "./string": "./string.js" + } +} diff --git a/test/fixtures/node_modules/interop-cjs/string-esm.js b/test/fixtures/node_modules/interop-cjs/string-esm.js new file mode 100644 index 00000000000000..bdd23a1b1db4da --- /dev/null +++ b/test/fixtures/node_modules/interop-cjs/string-esm.js @@ -0,0 +1 @@ +module.exports = require('interop-cjsdep-string-esm'); diff --git a/test/fixtures/node_modules/interop-cjs/string.js b/test/fixtures/node_modules/interop-cjs/string.js new file mode 100644 index 00000000000000..65010183f2f76b --- /dev/null +++ b/test/fixtures/node_modules/interop-cjs/string.js @@ -0,0 +1 @@ +module.exports = require('interop-cjsdep-string'); diff --git a/test/fixtures/node_modules/interop-cjsdep-false-esm/dep.js b/test/fixtures/node_modules/interop-cjsdep-false-esm/dep.js new file mode 100644 index 00000000000000..7ef62281a771cb --- /dev/null +++ b/test/fixtures/node_modules/interop-cjsdep-false-esm/dep.js @@ -0,0 +1,3 @@ +const output = false; +export default output; +export { output as 'module.exports' } diff --git a/test/fixtures/node_modules/interop-cjsdep-false-esm/package.json b/test/fixtures/node_modules/interop-cjsdep-false-esm/package.json new file mode 100644 index 00000000000000..f5b94a85497063 --- /dev/null +++ b/test/fixtures/node_modules/interop-cjsdep-false-esm/package.json @@ -0,0 +1,4 @@ +{ + "type": "module", + "exports": "./dep.js" +} diff --git a/test/fixtures/node_modules/interop-cjsdep-false/dep.js b/test/fixtures/node_modules/interop-cjsdep-false/dep.js new file mode 100644 index 00000000000000..a5d30209bd0bbc --- /dev/null +++ b/test/fixtures/node_modules/interop-cjsdep-false/dep.js @@ -0,0 +1 @@ +module.exports = false; diff --git a/test/fixtures/node_modules/interop-cjsdep-false/package.json b/test/fixtures/node_modules/interop-cjsdep-false/package.json new file mode 100644 index 00000000000000..e6fc9863573e3d --- /dev/null +++ b/test/fixtures/node_modules/interop-cjsdep-false/package.json @@ -0,0 +1,4 @@ +{ + "type": "commonjs", + "exports": "./dep.js" +} diff --git a/test/fixtures/node_modules/interop-cjsdep-fauxesmdefault-esm/dep.js b/test/fixtures/node_modules/interop-cjsdep-fauxesmdefault-esm/dep.js new file mode 100644 index 00000000000000..731bf61547a427 --- /dev/null +++ b/test/fixtures/node_modules/interop-cjsdep-fauxesmdefault-esm/dep.js @@ -0,0 +1,7 @@ +const exports = {}; +Object.defineProperty(exports, '__esModule', { value: true }); +exports.default = 'faux esm default'; + +export default exports; + +export { exports as 'module.exports' } \ No newline at end of file diff --git a/test/fixtures/node_modules/interop-cjsdep-fauxesmdefault-esm/package.json b/test/fixtures/node_modules/interop-cjsdep-fauxesmdefault-esm/package.json new file mode 100644 index 00000000000000..f5b94a85497063 --- /dev/null +++ b/test/fixtures/node_modules/interop-cjsdep-fauxesmdefault-esm/package.json @@ -0,0 +1,4 @@ +{ + "type": "module", + "exports": "./dep.js" +} diff --git a/test/fixtures/node_modules/interop-cjsdep-fauxesmdefault/dep.js b/test/fixtures/node_modules/interop-cjsdep-fauxesmdefault/dep.js new file mode 100644 index 00000000000000..5cd0b37b27d054 --- /dev/null +++ b/test/fixtures/node_modules/interop-cjsdep-fauxesmdefault/dep.js @@ -0,0 +1,2 @@ +Object.defineProperty(exports, '__esModule', { value: true }); +exports.default = 'faux esm default'; diff --git a/test/fixtures/node_modules/interop-cjsdep-fauxesmdefault/package.json b/test/fixtures/node_modules/interop-cjsdep-fauxesmdefault/package.json new file mode 100644 index 00000000000000..e6fc9863573e3d --- /dev/null +++ b/test/fixtures/node_modules/interop-cjsdep-fauxesmdefault/package.json @@ -0,0 +1,4 @@ +{ + "type": "commonjs", + "exports": "./dep.js" +} diff --git a/test/fixtures/node_modules/interop-cjsdep-fauxesmmixed-esm/dep.js b/test/fixtures/node_modules/interop-cjsdep-fauxesmmixed-esm/dep.js new file mode 100644 index 00000000000000..0eef418de77ea7 --- /dev/null +++ b/test/fixtures/node_modules/interop-cjsdep-fauxesmmixed-esm/dep.js @@ -0,0 +1,9 @@ +const exports = {}; +Object.defineProperty(exports, '__esModule', { value: true }); +exports.default = 'faux esm default'; +exports.a = 'faux esm a'; +exports.b = 'faux esm b'; + +export default exports; + +export { exports as 'module.exports' } diff --git a/test/fixtures/node_modules/interop-cjsdep-fauxesmmixed-esm/package.json b/test/fixtures/node_modules/interop-cjsdep-fauxesmmixed-esm/package.json new file mode 100644 index 00000000000000..f5b94a85497063 --- /dev/null +++ b/test/fixtures/node_modules/interop-cjsdep-fauxesmmixed-esm/package.json @@ -0,0 +1,4 @@ +{ + "type": "module", + "exports": "./dep.js" +} diff --git a/test/fixtures/node_modules/interop-cjsdep-fauxesmmixed/dep.js b/test/fixtures/node_modules/interop-cjsdep-fauxesmmixed/dep.js new file mode 100644 index 00000000000000..926ce033f829a0 --- /dev/null +++ b/test/fixtures/node_modules/interop-cjsdep-fauxesmmixed/dep.js @@ -0,0 +1,4 @@ +Object.defineProperty(exports, '__esModule', { value: true }); +exports.default = 'faux esm default'; +exports.a = 'faux esm a'; +exports.b = 'faux esm b'; diff --git a/test/fixtures/node_modules/interop-cjsdep-fauxesmmixed/package.json b/test/fixtures/node_modules/interop-cjsdep-fauxesmmixed/package.json new file mode 100644 index 00000000000000..e6fc9863573e3d --- /dev/null +++ b/test/fixtures/node_modules/interop-cjsdep-fauxesmmixed/package.json @@ -0,0 +1,4 @@ +{ + "type": "commonjs", + "exports": "./dep.js" +} diff --git a/test/fixtures/node_modules/interop-cjsdep-fauxesmnamed-esm/dep.js b/test/fixtures/node_modules/interop-cjsdep-fauxesmnamed-esm/dep.js new file mode 100644 index 00000000000000..ac048c7b1634a7 --- /dev/null +++ b/test/fixtures/node_modules/interop-cjsdep-fauxesmnamed-esm/dep.js @@ -0,0 +1,8 @@ +const exports = {}; +Object.defineProperty(exports, '__esModule', { value: true }); +exports.a = 'faux esm a'; +exports.b = 'faux esm b'; + +export default exports; + +export { exports as 'module.exports' } diff --git a/test/fixtures/node_modules/interop-cjsdep-fauxesmnamed-esm/package.json b/test/fixtures/node_modules/interop-cjsdep-fauxesmnamed-esm/package.json new file mode 100644 index 00000000000000..f5b94a85497063 --- /dev/null +++ b/test/fixtures/node_modules/interop-cjsdep-fauxesmnamed-esm/package.json @@ -0,0 +1,4 @@ +{ + "type": "module", + "exports": "./dep.js" +} diff --git a/test/fixtures/node_modules/interop-cjsdep-fauxesmnamed/dep.js b/test/fixtures/node_modules/interop-cjsdep-fauxesmnamed/dep.js new file mode 100644 index 00000000000000..e3218c09b35752 --- /dev/null +++ b/test/fixtures/node_modules/interop-cjsdep-fauxesmnamed/dep.js @@ -0,0 +1,3 @@ +Object.defineProperty(exports, '__esModule', { value: true }); +exports.a = 'faux esm a'; +exports.b = 'faux esm b'; diff --git a/test/fixtures/node_modules/interop-cjsdep-fauxesmnamed/package.json b/test/fixtures/node_modules/interop-cjsdep-fauxesmnamed/package.json new file mode 100644 index 00000000000000..e6fc9863573e3d --- /dev/null +++ b/test/fixtures/node_modules/interop-cjsdep-fauxesmnamed/package.json @@ -0,0 +1,4 @@ +{ + "type": "commonjs", + "exports": "./dep.js" +} diff --git a/test/fixtures/node_modules/interop-cjsdep-object-esm/dep.js b/test/fixtures/node_modules/interop-cjsdep-object-esm/dep.js new file mode 100644 index 00000000000000..f53cd1c055bd0a --- /dev/null +++ b/test/fixtures/node_modules/interop-cjsdep-object-esm/dep.js @@ -0,0 +1,8 @@ +const output = { + a: 'cjs a', + b: 'cjs b' +}; + +export default output; + +export { output as 'module.exports' } diff --git a/test/fixtures/node_modules/interop-cjsdep-object-esm/package.json b/test/fixtures/node_modules/interop-cjsdep-object-esm/package.json new file mode 100644 index 00000000000000..f5b94a85497063 --- /dev/null +++ b/test/fixtures/node_modules/interop-cjsdep-object-esm/package.json @@ -0,0 +1,4 @@ +{ + "type": "module", + "exports": "./dep.js" +} diff --git a/test/fixtures/node_modules/interop-cjsdep-object/dep.js b/test/fixtures/node_modules/interop-cjsdep-object/dep.js new file mode 100644 index 00000000000000..c37089bccc741e --- /dev/null +++ b/test/fixtures/node_modules/interop-cjsdep-object/dep.js @@ -0,0 +1,2 @@ +exports.a = 'cjs a'; +exports.b = 'cjs b'; diff --git a/test/fixtures/node_modules/interop-cjsdep-object/package.json b/test/fixtures/node_modules/interop-cjsdep-object/package.json new file mode 100644 index 00000000000000..e6fc9863573e3d --- /dev/null +++ b/test/fixtures/node_modules/interop-cjsdep-object/package.json @@ -0,0 +1,4 @@ +{ + "type": "commonjs", + "exports": "./dep.js" +} diff --git a/test/fixtures/node_modules/interop-cjsdep-string-esm/dep.js b/test/fixtures/node_modules/interop-cjsdep-string-esm/dep.js new file mode 100644 index 00000000000000..8b718c32a1fab5 --- /dev/null +++ b/test/fixtures/node_modules/interop-cjsdep-string-esm/dep.js @@ -0,0 +1,3 @@ +const output = 'cjs'; +export default output; +export { output as 'module.exports' } diff --git a/test/fixtures/node_modules/interop-cjsdep-string-esm/package.json b/test/fixtures/node_modules/interop-cjsdep-string-esm/package.json new file mode 100644 index 00000000000000..f5b94a85497063 --- /dev/null +++ b/test/fixtures/node_modules/interop-cjsdep-string-esm/package.json @@ -0,0 +1,4 @@ +{ + "type": "module", + "exports": "./dep.js" +} diff --git a/test/fixtures/node_modules/interop-cjsdep-string/dep.js b/test/fixtures/node_modules/interop-cjsdep-string/dep.js new file mode 100644 index 00000000000000..b2825bd3c9949b --- /dev/null +++ b/test/fixtures/node_modules/interop-cjsdep-string/dep.js @@ -0,0 +1 @@ +module.exports = 'cjs'; diff --git a/test/fixtures/node_modules/interop-cjsdep-string/package.json b/test/fixtures/node_modules/interop-cjsdep-string/package.json new file mode 100644 index 00000000000000..e6fc9863573e3d --- /dev/null +++ b/test/fixtures/node_modules/interop-cjsdep-string/package.json @@ -0,0 +1,4 @@ +{ + "type": "commonjs", + "exports": "./dep.js" +} diff --git a/test/fixtures/pkgexports.mjs b/test/fixtures/pkgexports.mjs index 7d642c443e6b71..b71566b95c79ac 100644 --- a/test/fixtures/pkgexports.mjs +++ b/test/fixtures/pkgexports.mjs @@ -3,6 +3,10 @@ import { createRequire } from 'module'; const rawRequire = createRequire(fileURLToPath(import.meta.url)); +export function directRequireFixture(specifier) { + return rawRequire(specifier); +} + export async function requireFixture(specifier) { return { default: rawRequire(specifier ) }; } From 6ab6126794f1500a26853df72e8e671b8bcd3acf Mon Sep 17 00:00:00 2001 From: Antoine du Hamel Date: Wed, 2 Oct 2024 23:02:31 +0200 Subject: [PATCH 21/33] doc: update `require(ESM)` history and stability status PR-URL: https://github.com/nodejs/node/pull/55199 Reviewed-By: Moshe Atlow Reviewed-By: Guy Bedford Reviewed-By: Joyee Cheung --- doc/api/modules.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/api/modules.md b/doc/api/modules.md index fa5f27398300dd..86f5df057e9a2f 100644 --- a/doc/api/modules.md +++ b/doc/api/modules.md @@ -171,16 +171,20 @@ relative, and based on the real path of the files making the calls to ## Loading ECMAScript modules using `require()` +> Stability: 1.2 - Release candidate + The `.mjs` extension is reserved for [ECMAScript Modules][]. See [Determining module system][] section for more info regarding which files are parsed as ECMAScript modules. From b641ee2abb2eac7d38970d4240ea46946b3e80f6 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Fri, 4 Oct 2024 19:29:34 +0200 Subject: [PATCH 22/33] module: use kNodeModulesRE to detect node_modules This is faster and more consistent with other places using the regular expression to detect node_modules. PR-URL: https://github.com/nodejs/node/pull/55243 Reviewed-By: Antoine du Hamel Reviewed-By: Jacob Smith Reviewed-By: Richard Lau Reviewed-By: Marco Ippolito --- lib/internal/modules/esm/load.js | 4 +++- lib/internal/util.js | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/internal/modules/esm/load.js b/lib/internal/modules/esm/load.js index 5239bc8ed883a5..d5004f4495bacc 100644 --- a/lib/internal/modules/esm/load.js +++ b/lib/internal/modules/esm/load.js @@ -5,7 +5,9 @@ const { RegExpPrototypeExec, decodeURIComponent, } = primordials; -const { kEmptyObject } = require('internal/util'); +const { + kEmptyObject, +} = require('internal/util'); const { defaultGetFormat } = require('internal/modules/esm/get_format'); const { validateAttributes, emitImportAssertionWarning } = require('internal/modules/esm/assert'); diff --git a/lib/internal/util.js b/lib/internal/util.js index 40e6f5e5090295..ea1ef8ca90c626 100644 --- a/lib/internal/util.js +++ b/lib/internal/util.js @@ -477,6 +477,10 @@ function spliceOne(list, index) { const kNodeModulesRE = /^(?:.*)[\\/]node_modules[\\/]/; +function isUnderNodeModules(filename) { + return filename && (RegExpPrototypeExec(kNodeModulesRE, filename) !== null); +} + let getStructuredStackImpl; function lazyGetStructuredStack() { @@ -524,7 +528,7 @@ function isInsideNodeModules() { ) { continue; } - return RegExpPrototypeExec(kNodeModulesRE, filename) !== null; + return isUnderNodeModules(filename); } } return false; @@ -913,6 +917,7 @@ module.exports = { isArrayBufferDetached, isError, isInsideNodeModules, + isUnderNodeModules, join, lazyDOMException, lazyDOMExceptionClass, From f801d347afb8631d785858dd1880b7860e7e3a31 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Mon, 7 Oct 2024 17:26:10 +0200 Subject: [PATCH 23/33] process: add process.features.require_module For detecting whether `require(esm)` is supported without triggering the experimental warning. PR-URL: https://github.com/nodejs/node/pull/55241 Reviewed-By: Richard Lau Reviewed-By: Matteo Collina --- doc/api/modules.md | 3 ++ doc/api/process.md | 12 ++++++ lib/internal/bootstrap/node.js | 6 +++ .../test-require-module-feature-detect.js | 37 +++++++++++++++++++ test/parallel/test-process-features.js | 1 + 5 files changed, 59 insertions(+) create mode 100644 test/es-module/test-require-module-feature-detect.js diff --git a/doc/api/modules.md b/doc/api/modules.md index 86f5df057e9a2f..9e4d3a223b8df6 100644 --- a/doc/api/modules.md +++ b/doc/api/modules.md @@ -318,6 +318,8 @@ experimental and can be disabled using `--no-experimental-require-module`. When `require()` actually encounters an ES module for the first time in the process, it will emit an experimental warning. The warning is expected to be removed when this feature stablizes. +This feature can be detected by checking if +[`process.features.require_module`][] is `true`. ## All together @@ -1280,6 +1282,7 @@ This section was moved to [`node:test`]: test.md [`package.json`]: packages.md#nodejs-packagejson-field-definitions [`path.dirname()`]: path.md#pathdirnamepath +[`process.features.require_module`]: process.md#processfeaturesrequire_module [`require.main`]: #requiremain [exports shortcut]: #exports-shortcut [module resolution]: #all-together diff --git a/doc/api/process.md b/doc/api/process.md index 68727c2bb61a0c..58df034fc46c52 100644 --- a/doc/api/process.md +++ b/doc/api/process.md @@ -1917,6 +1917,17 @@ added: v0.5.3 A boolean value that is `true` if the current Node.js build includes support for IPv6. +## `process.features.require_module` + + + +* {boolean} + +A boolean value that is `true` if the current Node.js build supports +[loading ECMAScript modules using `require()`][]. + ## `process.features.tls` + +Prints information about usage of [Loading ECMAScript modules using `require()`][]. + +When `mode` is `all`, all usage is printed. When `mode` is `no-node-modules`, usage +from the `node_modules` folder is excluded. + ### `--trace-sigint`