esm: pjson "mode": "esm" flag indicating ".js" es modules

This allows ".js" files to be loaded as es modules whenever the parent
package.json file contains "mode": "esm". Original proposal and discussion
at nodejs/node-eps/pull/60

Author: guybedford

lib/internal/loader/DefaultResolve.js

@@ -18,13 +18,13 @@ module.exports.search = search;
 
 const realpathCache = new Map();
 
-function search(target, base) {
+function search(target, base, checkPjsonMode) {
   if (base === undefined) {
     // We cannot search without a base.
     throw new errors.Error('ERR_MISSING_MODULE', target);
   }
   try {
-    return moduleWrapResolve(target, base);
+    return moduleWrapResolve(target, base, checkPjsonMode);
   } catch (e) {
     e.stack; // cause V8 to generate stack before rethrow
     let error = e;
@@ -65,10 +65,12 @@ function resolve(specifier, parentURL) {
     mainFormat = process.binding('config').mainFormat;
   }
 
-  let url;
+  let url, esm;
   try {
-    url = search(specifier, parentURL ||
-                 getURLFromFilePath(`${process.cwd()}/`).href);
+    ({ url, esm } =
+      search(specifier,
+             parentURL || getURLFromFilePath(`${process.cwd()}/`).href,
+             mainFormat ? false : true));
   } catch (e) {
     if (typeof e.message === 'string' &&
         StringStartsWith(e.message, 'Cannot find module'))
@@ -93,9 +95,10 @@ function resolve(specifier, parentURL) {
   if (parentURL === undefined)
     return {
       url: `${url}`,
-      format: mainFormat || extensionFormatMap[ext] || 'commonjs'
+      format: mainFormat ||
+        (esm ? 'esm' : extensionFormatMap[ext] || 'commonjs')
     };
 
   // all other imports are extension-format-based
-  return { url: `${url}`, format: extensionFormatMap[ext] };
+  return { url: `${url}`, format: esm ? 'esm' : extensionFormatMap[ext] };
 }

src/env.h

@@ -137,6 +137,7 @@ class ModuleWrap;
   V(env_pairs_string, "envPairs")                                             \
   V(errno_string, "errno")                                                    \
   V(error_string, "error")                                                    \
+  V(esm_string, "esm")                                                        \
   V(exiting_string, "_exiting")                                               \
   V(exit_code_string, "exitCode")                                             \
   V(exit_string, "exit")                                                      \
@@ -173,6 +174,7 @@ class ModuleWrap;
   V(max_buffer_string, "maxBuffer")                                           \
   V(message_string, "message")                                                \
   V(minttl_string, "minttl")                                                  \
+  V(mode_string, "mode")                                                      \
   V(modulus_string, "modulus")                                                \
   V(name_string, "name")                                                      \
   V(netmask_string, "netmask")                                                \
@@ -252,6 +254,7 @@ class ModuleWrap;
   V(type_string, "type")                                                      \
   V(uid_string, "uid")                                                        \
   V(unknown_string, "<unknown>")                                              \
+  V(url_string, "url")                                                        \
   V(user_string, "user")                                                      \
   V(username_string, "username")                                              \
   V(valid_from_string, "valid_from")                                          \

src/module_wrap.cc

@@ -350,10 +350,9 @@ enum CheckFileOptions {
   CLOSE_AFTER_CHECK
 };
 
-Maybe<uv_file> CheckFile(const URL& search,
+Maybe<uv_file> CheckFile(const std::string& path,
                          CheckFileOptions opt = CLOSE_AFTER_CHECK) {
   uv_fs_t fs_req;
-  std::string path = search.ToFilePath();
   if (path.empty()) {
     return Nothing<uv_file>();
   }
@@ -383,23 +382,103 @@ Maybe<uv_file> CheckFile(const URL& search,
   return Just(fd);
 }
 
+PackageJson emptyPackage = { false, false, "", false };
+std::unordered_map<std::string, PackageJson> pjson_cache_;
+PackageJson GetPackageJson(Environment* env, const std::string path) {
+  auto existing = pjson_cache_.find(path);
+  if (existing != pjson_cache_.end()) {
+    return existing->second;
+  }
+  Maybe<uv_file> check = CheckFile(path, LEAVE_OPEN_AFTER_CHECK);
+  if (check.IsNothing()) {
+    return (pjson_cache_[path] = emptyPackage);
+  }
+
+  Isolate* isolate = env->isolate();
+  Local<Context> context = isolate->GetCurrentContext();
+  std::string pkg_src = ReadFile(check.FromJust());
+  uv_fs_t fs_req;
+  uv_fs_close(nullptr, &fs_req, check.FromJust(), nullptr);
+  uv_fs_req_cleanup(&fs_req);
+
+  // It's not okay for the called of this method to not be able to tell
+  // whether an exception is pending or not.
+  TryCatch try_catch(isolate);
+
+  Local<String> src;
+  if (!String::NewFromUtf8(isolate,
+                           pkg_src.c_str(),
+                           v8::NewStringType::kNormal,
+                           pkg_src.length()).ToLocal(&src)) {
+    return (pjson_cache_[path] = emptyPackage);
+  }
+
+  Local<Value> pkg_json;
+  if (!JSON::Parse(context, src).ToLocal(&pkg_json) || !pkg_json->IsObject())
+    return (pjson_cache_[path] = emptyPackage);
+  Local<Value> pkg_main;
+  bool has_main = false;
+  std::string main_std;
+  if (pkg_json.As<Object>()->Get(context, env->main_string())
+                              .ToLocal(&pkg_main) && pkg_main->IsString()) {
+    has_main = true;
+    Utf8Value main_utf8(isolate, pkg_main.As<String>());
+    main_std = std::string(*main_utf8, main_utf8.length());
+  }
+
+  bool esm = false;
+  Local<Value> pkg_mode;
+  std::string pkg_mode_std;
+  if (pkg_json.As<Object>()->Get(context, env->mode_string())
+                              .ToLocal(&pkg_mode) && pkg_mode->IsString()) {
+    Utf8Value pkg_mode_utf8(isolate, pkg_mode.As<String>());
+    pkg_mode_std = std::string(*pkg_mode_utf8, pkg_mode_utf8.length());
+    if (pkg_mode_std == "esm") {
+      esm = true;
+    }
+  }
+
+  PackageJson pjson = { true, has_main, main_std, esm };
+  pjson_cache_[path] = pjson;
+  return pjson;
+}
+
+bool CheckPjsonEsmMode(Environment* env, const URL& search) {
+  URL pjsonPath("package.json", &search);
+  PackageJson pjson;
+  do {
+    pjson = GetPackageJson(env, pjsonPath.ToFilePath());
+    if (pjson.exists) {
+      break;
+    }
+    URL lastPjsonPath = pjsonPath;
+    pjsonPath = URL("../package.json", pjsonPath);
+    if (pjsonPath.path() == lastPjsonPath.path()) {
+      break;
+    }
+  } while (true);
+  return pjson.exists && pjson.esm;
+}
+
+
 enum ResolveExtensionsOptions {
   TRY_EXACT_NAME,
   ONLY_VIA_EXTENSIONS
 };
 
 template<ResolveExtensionsOptions options>
-Maybe<URL> ResolveExtensions(const URL& search) {
+Maybe<URL> ResolveExtensions(Environment* env, const URL& search) {
   if (options == TRY_EXACT_NAME) {
-    Maybe<uv_file> check = CheckFile(search);
+    std::string filePath = search.ToFilePath();
+    Maybe<uv_file> check = CheckFile(filePath);
     if (!check.IsNothing()) {
       return Just(search);
     }
   }
 
   for (const char* extension : EXTENSIONS) {
     URL guess(search.path() + extension, &search);
-    Maybe<uv_file> check = CheckFile(guess);
+    Maybe<uv_file> check = CheckFile(guess.ToFilePath());
     if (!check.IsNothing()) {
       return Just(guess);
     }
@@ -408,50 +487,21 @@ Maybe<URL> ResolveExtensions(const URL& search) {
   return Nothing<URL>();
 }
 
-inline Maybe<URL> ResolveIndex(const URL& search) {
-  return ResolveExtensions<ONLY_VIA_EXTENSIONS>(URL("index", search));
+inline Maybe<URL> ResolveIndex(Environment* env, const URL& search) {
+  return ResolveExtensions<ONLY_VIA_EXTENSIONS>(env, URL("index", search));
 }
 
 Maybe<URL> ResolveMain(Environment* env, const URL& search) {
   URL pkg("package.json", &search);
-  Maybe<uv_file> check = CheckFile(pkg, LEAVE_OPEN_AFTER_CHECK);
-  if (check.IsNothing()) {
-    return Nothing<URL>();
-  }
-
-  Isolate* isolate = env->isolate();
-  Local<Context> context = isolate->GetCurrentContext();
-  std::string pkg_src = ReadFile(check.FromJust());
-  uv_fs_t fs_req;
-  uv_fs_close(nullptr, &fs_req, check.FromJust(), nullptr);
-  uv_fs_req_cleanup(&fs_req);
-
-  // It's not okay for the called of this method to not be able to tell
-  // whether an exception is pending or not.
-  TryCatch try_catch(isolate);
-
-  Local<String> src;
-  if (!String::NewFromUtf8(isolate,
-                           pkg_src.c_str(),
-                           v8::NewStringType::kNormal,
-                           pkg_src.length()).ToLocal(&src)) {
-    return Nothing<URL>();
-  }
 
-  Local<Value> pkg_json;
-  if (!JSON::Parse(context, src).ToLocal(&pkg_json) || !pkg_json->IsObject())
-    return Nothing<URL>();
-  Local<Value> pkg_main;
-  if (!pkg_json.As<Object>()->Get(context, env->main_string())
-                              .ToLocal(&pkg_main) || !pkg_main->IsString()) {
+  PackageJson pjson = GetPackageJson(env, pkg.ToFilePath());
+  if (!pjson.exists || !pjson.has_main) {
     return Nothing<URL>();
   }
-  Utf8Value main_utf8(isolate, pkg_main.As<String>());
-  std::string main_std(*main_utf8, main_utf8.length());
-  if (!ShouldBeTreatedAsRelativeOrAbsolutePath(main_std)) {
-    main_std.insert(0, "./");
+  if (!ShouldBeTreatedAsRelativeOrAbsolutePath(pjson.main)) {
+    return Resolve(env, "./" + pjson.main, search);
   }
-  return Resolve(env, main_std, search);
+  return Resolve(env, pjson.main, search);
 }
 
 Maybe<URL> ResolveModule(Environment* env,
@@ -461,7 +511,8 @@ Maybe<URL> ResolveModule(Environment* env,
   URL dir("");
   do {
     dir = parent;
-    Maybe<URL> check = Resolve(env, "./node_modules/" + specifier, dir, true);
+    Maybe<URL> check =
+        Resolve(env, "./node_modules/" + specifier, dir);
     if (!check.IsNothing()) {
       const size_t limit = specifier.find('/');
       const size_t spec_len =
@@ -481,28 +532,22 @@ Maybe<URL> ResolveModule(Environment* env,
   return Nothing<URL>();
 }
 
-Maybe<URL> ResolveDirectory(Environment* env,
-                            const URL& search,
-                            bool read_pkg_json) {
-  if (read_pkg_json) {
-    Maybe<URL> main = ResolveMain(env, search);
-    if (!main.IsNothing())
-      return main;
-  }
-  return ResolveIndex(search);
+Maybe<URL> ResolveDirectory(Environment* env, const URL& search) {
+  Maybe<URL> main = ResolveMain(env, search);
+  if (!main.IsNothing())
+    return main;
+  return ResolveIndex(env, search);
 }
 
 }  // anonymous namespace
 
-
 Maybe<URL> Resolve(Environment* env,
                    const std::string& specifier,
-                   const URL& base,
-                   bool read_pkg_json) {
+                   const URL& base) {
   URL pure_url(specifier);
   if (!(pure_url.flags() & URL_FLAGS_FAILED)) {
     // just check existence, without altering
-    Maybe<uv_file> check = CheckFile(pure_url);
+    Maybe<uv_file> check = CheckFile(pure_url.ToFilePath());
     if (check.IsNothing()) {
       return Nothing<URL>();
     }
@@ -513,13 +558,14 @@ Maybe<URL> Resolve(Environment* env,
   }
   if (ShouldBeTreatedAsRelativeOrAbsolutePath(specifier)) {
     URL resolved(specifier, base);
-    Maybe<URL> file = ResolveExtensions<TRY_EXACT_NAME>(resolved);
+    Maybe<URL> file =
+        ResolveExtensions<TRY_EXACT_NAME>(env, resolved);
     if (!file.IsNothing())
       return file;
     if (specifier.back() != '/') {
       resolved = URL(specifier + "/", base);
     }
-    return ResolveDirectory(env, resolved, read_pkg_json);
+    return ResolveDirectory(env, resolved);
   } else {
     return ResolveModule(env, specifier, base);
   }
@@ -532,8 +578,9 @@ void ModuleWrap::Resolve(const FunctionCallbackInfo<Value>& args) {
     env->ThrowError("resolve() must not be called as a constructor");
     return;
   }
-  if (args.Length() != 2) {
-    env->ThrowError("resolve must have exactly 2 arguments (string, string)");
+  if (args.Length() != 3) {
+    env->ThrowError(
+      "resolve must have exactly 3 arguments (string, string, boolean)");
     return;
   }
 
@@ -556,14 +603,43 @@ void ModuleWrap::Resolve(const FunctionCallbackInfo<Value>& args) {
     return;
   }
 
-  Maybe<URL> result = node::loader::Resolve(env, specifier_std, url, true);
-  if (result.IsNothing() || (result.FromJust().flags() & URL_FLAGS_FAILED)) {
+  if (!args[2]->IsBoolean()) {
+    env->ThrowError("third argument is not a boolean");
+    return;
+  }
+  bool check_pjson_mode = args[2]->ToBoolean().As<v8::Boolean>()->Value();
+
+  Maybe<URL> result = node::loader::Resolve(env, specifier_std, url);
+  if (result.IsNothing() ||
+      (result.FromJust().flags() & URL_FLAGS_FAILED)) {
     std::string msg = "Cannot find module " + specifier_std;
     env->ThrowError(msg.c_str());
     return;
   }
 
-  args.GetReturnValue().Set(result.FromJust().ToObject(env));
+  bool esm = false;
+  if (check_pjson_mode) {
+    std::string path = result.FromJust().ToFilePath();
+    if (path.substr(path.length() - 3, 3) == ".js") {
+      esm = CheckPjsonEsmMode(env, result.FromJust());
+    }
+  }
+
+  Local<Object> resolved = Object::New(env->isolate());
+
+  resolved->DefineOwnProperty(
+    env->context(),
+    env->esm_string(),
+    v8::Boolean::New(env->isolate(), esm),
+    v8::ReadOnly);
+
+  resolved->DefineOwnProperty(
+    env->context(),
+    env->url_string(),
+    result.FromJust().ToObject(env),
+    v8::ReadOnly);
+
+  args.GetReturnValue().Set(resolved);
 }
 
 static MaybeLocal<Promise> ImportModuleDynamically(

src/module_wrap.h

@@ -12,10 +12,16 @@
 namespace node {
 namespace loader {
 
+struct PackageJson {
+  bool exists;
+  bool has_main;
+  std::string main;
+  bool esm;
+};
+
 v8::Maybe<url::URL> Resolve(Environment* env,
                             const std::string& specifier,
-                            const url::URL& base,
-                            bool read_pkg_json = false);
+                            const url::URL& base);
 
 class ModuleWrap : public BaseObject {
  public:

test/es-module/test-pjson-cjs-esm-nested.mjs

@@ -0,0 +1,6 @@
+// Flags: --experimental-modules
+/* eslint-disable required-modules */
+import m from '../fixtures/es-modules/esm-cjs-nested/module';
+import assert from 'assert';
+
+assert.strictEqual(m, 'cjs');