feat(builtin): add a toolchain to new core that exposes the node for any platform

Author: alexeagle

e2e/core/BUILD.bazel

@@ -1,35 +1,80 @@
 load("@bazel_skylib//rules:write_file.bzl", "write_file")
 load("@bazel_skylib//rules:diff_test.bzl", "diff_test")
+load(":defs.bzl", "my_nodejs")
 
+# Trivial test fixture: a nodejs program that writes to a file
 write_file(
     name = "js",
     out = "some.js",
     content = ["require('fs').writeFileSync(process.argv[2], 'stuff')"],
 )
 
-# Temporary fixture until we have toolchains hooked up in the core package
-alias(
-    name = "node_bin",
-    actual = select({
-        "@bazel_tools//src/conditions:darwin_arm64": "@node16_darwin_arm64//:node_bin",
-        "@bazel_tools//src/conditions:darwin_x86_64": "@node16_darwin_amd64//:node_bin",
-        "@bazel_tools//src/conditions:linux_aarch64": "@node16_linux_arm64//:node_bin",
-        "@bazel_tools//src/conditions:linux_s390x": "@node16_linux_s390x//:node_bin",
-        "@bazel_tools//src/conditions:linux_x86_64": "@node16_linux_amd64//:node_bin",
-        "@bazel_tools//src/conditions:linux_ppc64le": "@node16_linux_ppc64le//:node_bin",
-        "@bazel_tools//src/conditions:windows": "@node16_windows_amd64//:node_bin",
-        "//conditions:default": "@node16_linux_amd64//:node_bin",
-    }),
-)
+# This technique can be used to directly grab a node binary as a label, however it has the
+# downside that analysis phase on this select() statement will cause an eager fetch of all
+# the platforms and therefore download a bunch of node binaries.
+# This is what toolchains solves, so we don't recommend doing this.
+# alias(
+#     name = "node_bin",
+#     actual = select({
+#         "@bazel_tools//src/conditions:darwin_arm64": "@node16_darwin_arm64//:node_bin",
+#         "@bazel_tools//src/conditions:darwin_x86_64": "@node16_darwin_amd64//:node_bin",
+#         "@bazel_tools//src/conditions:linux_aarch64": "@node16_linux_arm64//:node_bin",
+#         "@bazel_tools//src/conditions:linux_s390x": "@node16_linux_s390x//:node_bin",
+#         "@bazel_tools//src/conditions:linux_x86_64": "@node16_linux_amd64//:node_bin",
+#         "@bazel_tools//src/conditions:linux_ppc64le": "@node16_linux_ppc64le//:node_bin",
+#         "@bazel_tools//src/conditions:windows": "@node16_windows_amd64//:node_bin",
+#         "//conditions:default": "@node16_linux_amd64//:node_bin",
+#     }),
+# )
+# genrule(
+#     name = "use_node_bin",
+#     srcs = ["some.js"],
+#     outs = ["thing1"],
+#     cmd = "$(execpath :node_bin) $(execpath some.js) $@",
+#     tools = [":node_bin"],
+# )
+
+# In theory, you can use the node toolchain together with a genrule().
+# However the genrule implementation doesn't perform toolchain resolution.
+# See this comment from Jay Conrod about a similar question for rules_go
+# https://github.com/bazelbuild/rules_go/issues/2255#issuecomment-545478712
+# That means you must resolve the toolchain yourself, with a select()
+# and that falls down the same deoptimization described above:
+# it will eager-fetch node for all platforms.
+# So instead we recommend always writing a custom rule to access the node binary.
+# alias(
+#     name = "node_toolchain",
+#     actual = select({
+#         "@bazel_tools//src/conditions:darwin_arm64": "@node16_darwin_arm64//:node_toolchain",
+#         "@bazel_tools//src/conditions:darwin_x86_64": "@node16_darwin_amd64//:node_toolchain",
+#         "@bazel_tools//src/conditions:linux_aarch64": "@node16_linux_arm64//:node_toolchain",
+#         "@bazel_tools//src/conditions:linux_s390x": "@node16_linux_s390x//:node_toolchain",
+#         "@bazel_tools//src/conditions:linux_x86_64": "@node16_linux_amd64//:node_toolchain",
+#         "@bazel_tools//src/conditions:linux_ppc64le": "@node16_linux_ppc64le//:node_toolchain",
+#         "@bazel_tools//src/conditions:windows": "@node16_windows_amd64//:node_toolchain",
+#         "//conditions:default": "@node16_linux_amd64//:node_toolchain",
+#     }),
+# )
+# genrule(
+#     name = "use_node_toolchain",
+#     srcs = ["some.js"],
+#     outs = ["thing2"],
+#     cmd = "$(NODE_PATH) $(execpath some.js) $@",
+#     toolchains = [":node_toolchain"],
+#     # It will also fail to include the files from the node_toolchain, so you're
+#     # forced to repeat the label of the node binary as an explicit input.
+#     tools = ["@node16_host//:node_bin"],
+# )
 
-genrule(
-    name = "try",
-    srcs = ["some.js"],
-    outs = ["thing"],
-    cmd = "$(execpath :node_bin) $(execpath some.js) $@",
-    tools = [":node_bin"],
+# Here, my_nodejs is a fake for something like nodejs_binary or
+# some other custom rule that runs node.
+my_nodejs(
+    name = "run",
+    out = "thing",
+    entry_point = "some.js",
 )
 
+# Assert that the node program wrote the file we expect
 write_file(
     name = "write_expected",
     out = "expected",
@@ -41,3 +86,4 @@ diff_test(
     file1 = "expected",
     file2 = "thing",
 )
+# end Assert

e2e/core/defs.bzl

@@ -0,0 +1,20 @@
+"Simple rule to test nodejs toolchain"
+
+def _my_nodejs_impl(ctx):
+    toolchain = ctx.toolchains["@rules_nodejs//nodejs:toolchain_type"].nodeinfo
+    ctx.actions.run(
+        inputs = toolchain.tool_files + [ctx.file.entry_point],
+        executable = toolchain.target_tool_path,
+        arguments = [ctx.file.entry_point.path, ctx.outputs.out.path],
+        outputs = [ctx.outputs.out],
+    )
+    return []
+
+my_nodejs = rule(
+    implementation = _my_nodejs_impl,
+    attrs = {
+        "entry_point": attr.label(allow_single_file = True),
+        "out": attr.output(),
+    },
+    toolchains = ["@rules_nodejs//nodejs:toolchain_type"],
+)

internal/node/node_repositories.bzl

@@ -19,45 +19,10 @@ See https://docs.bazel.build/versions/main/skylark/repository_rules.html
 """
 
 load("//internal/common:check_bazel_version.bzl", "check_bazel_version")
-load("//nodejs/private:os_name.bzl", "OS_ARCH_NAMES", "assert_node_exists_for_host", "node_exists_for_os", "os_name")
-load("//toolchains/node:node_toolchain_configure.bzl", "node_toolchain_configure")
+load("//nodejs/private:nodejs_repo_host_os_alias.bzl", "nodejs_repo_host_os_alias")
+load("//nodejs/private:os_name.bzl", "OS_ARCH_NAMES", "node_exists_for_os", "os_name")
 load("//nodejs:repositories.bzl", "DEFAULT_NODE_VERSION", node_repositories_rule = "node_repositories")
-
-def _nodejs_host_os_alias_impl(repository_ctx):
-    assert_node_exists_for_host(repository_ctx)
-
-    # Base BUILD file for this repository
-    repository_ctx.file("BUILD.bazel", content = """# Generated by node_repositories.bzl
-package(default_visibility = ["//visibility:public"])
-# aliases for exports_files
-alias(name = "run_npm.sh.template", actual = "{node_repository}//:run_npm.sh.template")
-alias(name = "run_npm.bat.template", actual = "{node_repository}//:run_npm.bat.template")
-alias(name = "bin/node_repo_args.sh", actual = "{node_repository}//:bin/node_repo_args.sh")
-# aliases for other aliases
-alias(name = "node_bin", actual = "{node_repository}//:node_bin")
-alias(name = "npm_bin", actual = "{node_repository}//:npm_bin")
-alias(name = "npx_bin", actual = "{node_repository}//:npx_bin")
-alias(name = "yarn_bin", actual = "{node_repository}//:yarn_bin")
-alias(name = "node", actual = "{node_repository}//:node")
-alias(name = "npm", actual = "{node_repository}//:npm")
-alias(name = "yarn", actual = "{node_repository}//:yarn")
-alias(name = "npm_node_repositories", actual = "{node_repository}//:npm_node_repositories")
-alias(name = "yarn_node_repositories", actual = "{node_repository}//:yarn_node_repositories")
-alias(name = "node_files", actual = "{node_repository}//:node_files")
-alias(name = "yarn_files", actual = "{node_repository}//:yarn_files")
-alias(name = "npm_files", actual = "{node_repository}//:npm_files")
-exports_files(["index.bzl"])
-""".format(node_repository = "@nodejs_%s" % os_name(repository_ctx)))
-
-    # index.bzl file for this repository
-    repository_ctx.file("index.bzl", content = """# Generated by node_repositories.bzl
-host_platform="{host_platform}"
-""".format(host_platform = os_name(repository_ctx)))
-
-_nodejs_repo_host_os_alias = repository_rule(
-    _nodejs_host_os_alias_impl,
-    attrs = {"node_version": attr.string()},
-)
+load("//toolchains/node:node_toolchain_configure.bzl", "node_toolchain_configure")
 
 def node_repositories(**kwargs):
     """
@@ -101,7 +66,7 @@ def node_repositories(**kwargs):
     # This "nodejs" repo is just for convenience so one does not have to target @nodejs_<os_name>//...
     # All it does is create aliases to the @nodejs_<host_os>_<host_arch> repository
     _maybe(
-        _nodejs_repo_host_os_alias,
+        nodejs_repo_host_os_alias,
         name = "nodejs",
         node_version = node_version,
     )

nodejs/BUILD.bazel

@@ -15,3 +15,12 @@ bzl_library(
         "//third_party/github.com/bazelbuild/bazel-skylib:bzl",
     ],
 )
+
+# This is the target rule authors should put in their "toolchains"
+# attribute in order to get a node interpreter for the correct
+# platform.
+# See https://docs.bazel.build/versions/main/toolchains.html#writing-rules-that-use-toolchains
+toolchain_type(
+    name = "toolchain_type",
+    visibility = ["//visibility:public"],
+)

nodejs/private/nodejs_repo_host_os_alias.bzl

@@ -0,0 +1,53 @@
+"Provide convenience repository for the host platform like @nodejs"
+
+load("//nodejs/private:os_name.bzl", "os_name")
+
+def _nodejs_host_os_alias_impl(repository_ctx):
+    # Base BUILD file for this repository
+    repository_ctx.file("BUILD.bazel", """# Generated by nodejs_repo_host_os_alias.bzl
+package(default_visibility = ["//visibility:public"])
+# aliases for exports_files
+alias(name = "run_npm.sh.template",     actual = "@{node_repository}_{os_name}//:run_npm.sh.template")
+alias(name = "run_npm.bat.template",    actual = "@{node_repository}_{os_name}//:run_npm.bat.template")
+alias(name = "bin/node_repo_args.sh",   actual = "@{node_repository}_{os_name}//:bin/node_repo_args.sh")
+
+# aliases for other aliases
+alias(name = "node_bin",                actual = "@{node_repository}_{os_name}//:node_bin")
+alias(name = "npm_bin",                 actual = "@{node_repository}_{os_name}//:npm_bin")
+alias(name = "npx_bin",                 actual = "@{node_repository}_{os_name}//:npx_bin")
+alias(name = "yarn_bin",                actual = "@{node_repository}_{os_name}//:yarn_bin")
+alias(name = "node",                    actual = "@{node_repository}_{os_name}//:node")
+alias(name = "npm",                     actual = "@{node_repository}_{os_name}//:npm")
+alias(name = "yarn",                    actual = "@{node_repository}_{os_name}//:yarn")
+alias(name = "npm_node_repositories",   actual = "@{node_repository}_{os_name}//:npm_node_repositories")
+alias(name = "yarn_node_repositories",  actual = "@{node_repository}_{os_name}//:yarn_node_repositories")
+alias(name = "node_files",              actual = "@{node_repository}_{os_name}//:node_files")
+alias(name = "yarn_files",              actual = "@{node_repository}_{os_name}//:yarn_files")
+alias(name = "npm_files",               actual = "@{node_repository}_{os_name}//:npm_files")
+exports_files(["index.bzl"])
+""".format(
+        node_repository = repository_ctx.attr.user_node_repository_name,
+        os_name = os_name(repository_ctx),
+    ))
+
+    # index.bzl file for this repository
+    repository_ctx.file("index.bzl", content = """# Generated by nodejs_repo_host_os_alias.bzl
+host_platform="{host_platform}"
+""".format(host_platform = os_name(repository_ctx)))
+
+nodejs_repo_host_os_alias = repository_rule(
+    _nodejs_host_os_alias_impl,
+    doc = """Creates a repository with a shorter name meant for the host platform, which contains
+
+    - A BUILD.bazel file declaring aliases to the host platform's node binaries
+    - index.bzl containing some constants
+    """,
+    attrs = {
+        "user_node_repository_name": attr.string(
+            default = "nodejs",
+            doc = "User-provided name from the workspace file, eg. node16",
+        ),
+        # FIXME: this seems unused, but not the time to make that edit right now
+        "node_version": attr.string(),
+    },
+)

nodejs/private/toolchains_repo.bzl

@@ -0,0 +1,99 @@
+"""Create a repository to hold the toolchains
+
+This follows guidance here:
+https://docs.bazel.build/versions/main/skylark/deploying.html#registering-toolchains
+"
+Note that in order to resolve toolchains in the analysis phase
+Bazel needs to analyze all toolchain targets that are registered.
+Bazel will not need to analyze all targets referenced by toolchain.toolchain attribute.
+If in order to register toolchains you need to perform complex computation in the repository,
+consider splitting the repository with toolchain targets
+from the repository with <LANG>_toolchain targets.
+Former will be always fetched,
+and the latter will only be fetched when user actually needs to build <LANG> code.
+"
+The "complex computation" in our case is simply downloading the node binaries from nodejs.org.
+This guidance tells us how to avoid that: we put the toolchain targets in the alias repository
+with only the toolchain attribute pointing into the platform-specific repositories.
+"""
+
+PLATFORMS = {
+    "darwin_amd64": struct(
+        compatible_with = [
+            "@platforms//os:macos",
+            "@platforms//cpu:x86_64",
+        ],
+    ),
+    "darwin_arm64": struct(
+        compatible_with = [
+            "@platforms//os:macos",
+            "@platforms//cpu:aarch64",
+        ],
+    ),
+    "linux_amd64": struct(
+        compatible_with = [
+            "@platforms//os:linux",
+            "@platforms//cpu:x86_64",
+        ],
+    ),
+    "linux_arm64": struct(
+        compatible_with = [
+            "@platforms//os:linux",
+            "@platforms//cpu:aarch64",
+        ],
+    ),
+    "windows_amd64": struct(
+        compatible_with = [
+            "@platforms//os:windows",
+            "@platforms//cpu:x86_64",
+        ],
+    ),
+    "linux_s390x": struct(
+        compatible_with = [
+            "@platforms//os:linux",
+            "@platforms//cpu:s390x",
+        ],
+    ),
+    "linux_ppc64le": struct(
+        compatible_with = [
+            "@platforms//os:linux",
+            "@platforms//cpu:ppc",
+        ],
+    ),
+}
+
+def _impl(repository_ctx):
+    build_content = """# Generated by toolchains_repo.bzl
+#
+# These can be registered in the workspace file or passed to --extra_toolchains flag.
+# By default all these toolchains are registered by the nodejs_register_toolchains macro
+# so you don't normally need to interact with these targets.
+"""
+
+    for [platform, meta] in PLATFORMS.items():
+        build_content += """
+toolchain(
+    name = "{platform}_toolchain",
+    exec_compatible_with = {compatible_with},
+    target_compatible_with = {compatible_with},
+    toolchain = "@{user_node_repository_name}_{platform}//:node_toolchain",
+    toolchain_type = "@rules_nodejs//nodejs:toolchain_type",
+)
+""".format(
+            platform = platform,
+            name = repository_ctx.attr.name,
+            user_node_repository_name = repository_ctx.attr.user_node_repository_name,
+            compatible_with = meta.compatible_with,
+        )
+
+    # Base BUILD file for this repository
+    repository_ctx.file("BUILD.bazel", build_content)
+
+toolchains_repo = repository_rule(
+    _impl,
+    doc = """Creates a repository with toolchain definitions for all known platforms
+     which can be registered or selected.""",
+    attrs = {
+        "user_node_repository_name": attr.string(doc = "what the user chose for the base name, eg. node16"),
+    },
+)