Upgrade to Bazel 8 (#479)

* Build everything with Bazel 7 and 8

* Simplified `publish_binary` runfiles handling. This might be a breaking change for some users if they are e.g. not using `include_runfiles` in `pkg_tar`

Author: purkhusid

.bazelci/presubmit.yml

@@ -3,28 +3,67 @@ buildifier:
   version: 7.1.0
 
 matrix:
-  platform: ["windows", "ubuntu2004", "rbe_ubuntu1604", "macos", "macos_arm64", "ubuntu2004_arm64", "windows_arm64"]
-  working_directory: ["", "examples"]
-  strict_deps: ["--@rules_dotnet//dotnet/settings:strict_deps=true", "--@rules_dotnet//dotnet/settings:strict_deps=false"]
-  flag_combo: [["--enable_bzlmod", "--@rules_dotnet//dotnet/settings:strict_deps=true"], ["--enable_bzlmod", "--@rules_dotnet//dotnet/settings:strict_deps=false"]]
+  bazel: ["7.x", "8.x"]
+  platform:
+    [
+      "windows",
+      "ubuntu2004",
+      "rbe_ubuntu1604",
+      "macos",
+      "macos_arm64",
+      "ubuntu2004_arm64",
+      "windows_arm64",
+    ]
+  flag_combo:
+    [
+      ["--@rules_dotnet//dotnet/settings:strict_deps=true"],
+      ["--@rules_dotnet//dotnet/settings:strict_deps=false"],
+    ]
   e2e_working_directory: ["e2e/net8.0", "e2e/net9.0"]
 
 tasks:
   test_all:
     name: Build and test
+    bazel: ${{ bazel }}
     platform: ${{ platform }}
-    working_directory: ${{ working_directory }}
+    working_directory: ""
     include_json_profile:
       - build
       - test
     build_targets:
       - "//..."
+      - "-//docs/..."
     test_targets:
       - "//..."
+      - "-//docs/..."
     build_flags: ${{ flag_combo }}
     test_flags: ${{ flag_combo }}
+  examples:
+    name: Build and test
+    bazel: ${{ bazel }}
+    platform: ${{ platform }}
+    working_directory: "examples"
+    include_json_profile:
+      - build
+      - test
+    build_targets:
+      - "//..."
+    test_targets:
+      - "//..."
+    build_flags: ${{ flag_combo }}
+    test_flags: ${{ flag_combo }}
+  docs:
+    name: Docs
+    bazel: 8.x
+    platform: ubuntu2004
+    working_directory: ""
+    build_targets:
+      - "//docs/..."
+    test_targets:
+      - "//docs/..."
   e2e:
     name: ${{ e2e_working_directory }}
+    bazel: ${{ bazel }}
     platform: ${{ platform }}
     working_directory: ${{ e2e_working_directory }}
     include_json_profile:

.bazelversion

@@ -1 +1 @@
-7.1.1
+8.1.1

.bcr/presubmit.yml

@@ -1,8 +1,16 @@
 bcr_test_module:
   module_path: "e2e/smoke"
   matrix:
-    platform: ["windows", "ubuntu1804", "macos", "macos_arm64", "ubuntu2004_arm64", "windows_arm64"]
-    bazel: ["7.x"]
+    platform:
+      [
+        "windows",
+        "ubuntu1804",
+        "macos",
+        "macos_arm64",
+        "ubuntu2004_arm64",
+        "windows_arm64",
+      ]
+    bazel: ["7.x", "8.x"]
   tasks:
     run_tests:
       name: "Run smoke test"

MODULE.bazel

@@ -33,12 +33,13 @@ bazel_dep(name = "platforms", version = "0.0.10")
 bazel_dep(name = "aspect_bazel_lib", version = "2.9.1")
 
 # Dev dependencies
-bazel_dep(name = "stardoc", version = "0.7.1", dev_dependency = True)
-bazel_dep(name = "rules_pkg", version = "1.0.1", dev_dependency = True)
-bazel_dep(name = "gazelle", version = "0.39.1", dev_dependency = True, repo_name = "bazel_gazelle")
+bazel_dep(name = "stardoc", version = "0.8.0", dev_dependency = True)
+bazel_dep(name = "rules_pkg", version = "1.1.0", dev_dependency = True)
+bazel_dep(name = "gazelle", version = "0.42.0", dev_dependency = True, repo_name = "bazel_gazelle")
 bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True)
-bazel_dep(name = "rules_cc", version = "0.0.12", dev_dependency = True)
-bazel_dep(name = "rules_testing", version = "0.6.0", dev_dependency = True)
+bazel_dep(name = "rules_cc", version = "0.1.1", dev_dependency = True)
+bazel_dep(name = "rules_testing", version = "0.7.0", dev_dependency = True)
+bazel_dep(name = "bazel_ci_rules", version = "1.0.0", dev_dependency = True)
 bazel_dep(name = "dotnet_test_resources_other_repo", version = "", dev_dependency = True)
 local_path_override(
     module_name = "dotnet_test_resources_other_repo",
@@ -47,3 +48,6 @@ local_path_override(
 
 rules_dotnet_dev_nuget_packages_extension = use_extension("//dotnet:paket.rules_dotnet_dev_nuget_packages_extension.bzl", "rules_dotnet_dev_nuget_packages_extension", dev_dependency = True)
 use_repo(rules_dotnet_dev_nuget_packages_extension, "paket.rules_dotnet_dev_nuget_packages")
+
+internal_dev_deps = use_extension("//dotnet:internal_dev_deps.bzl", "internal_dev_deps", dev_dependency = True)
+use_repo(internal_dev_deps, "buildkite_config")

WORKSPACE

@@ -0,0 +1,16 @@
+"""Module extension for internal dev_dependency=True setup."""
+
+load("@bazel_ci_rules//:rbe_repo.bzl", "rbe_preconfig")
+
+def _internal_dev_deps_impl(mctx):
+    _ = mctx  # @unused
+
+    rbe_preconfig(
+        name = "buildkite_config",
+        toolchain = "ubuntu1804-bazel-java11",
+    )
+
+internal_dev_deps = module_extension(
+    implementation = _internal_dev_deps_impl,
+    doc = "This extension creates internal rules_dotnet dev dependencies.",
+)

WORKSPACE.bzlmod

@@ -4,7 +4,7 @@ Rules for compiling F# binaries.
 
 load("@bazel_skylib//lib:paths.bzl", "paths")
 load("@bazel_skylib//lib:shell.bzl", "shell")
-load("//dotnet/private:common.bzl", "generate_depsjson", "generate_runtimeconfig", "to_rlocation_path")
+load("//dotnet/private:common.bzl", "generate_depsjson", "generate_runtimeconfig")
 load("//dotnet/private:providers.bzl", "DotnetAssemblyCompileInfo", "DotnetAssemblyRuntimeInfo", "DotnetBinaryInfo")
 load("//dotnet/private/transitions:default_transition.bzl", "default_transition")
 load("//dotnet/private/transitions:tfm_transition.bzl", "tfm_transition")
@@ -27,7 +27,7 @@ def _get_assembly_files(assembly_info, transitive_runtime_deps):
         data += dep.data
     return (libs, native, data, appsetting_files)
 
-def _copy_to_publish(ctx, runtime_identifier, runtime_pack_info, binary_info, assembly_info, transitive_runtime_deps, repo_mapping_manifest):
+def _copy_to_publish(ctx, runtime_identifier, runtime_pack_info, binary_info, assembly_info, transitive_runtime_deps):
     is_windows = ctx.target_platform_has_constraint(ctx.attr._windows_constraint[platform_common.ConstraintValueInfo])
     inputs = [binary_info.dll]
     main_dll_copy = ctx.actions.declare_file(
@@ -63,23 +63,15 @@ def _copy_to_publish(ctx, runtime_identifier, runtime_pack_info, binary_info, as
     # the runfiles lib. End users should not expect files in the `data` attribute
     # to be resolvable by relative paths. They need to use the runfiles lib.
     #
-    # Since we want the published binary and all it's files to be easily extracted
-    # into e.g. a tar/zip/docker we manually create the runfiles structure because
-    # there are many sharp edges with extracting runfiles from Bazel. By manually
-    # creating the runfiles structure the runfiles are just normal files in the
-    # DefaultInfo provider and can thus be easily forwarded to filegroups/tars/containers.
+    # The end-user will have to use rules that also pull the runfiles. For examples if
+    # they use rules_pkg they have to use `include_runfiles` on the pkt_tar rule.
     #
     # The runfiles library follows the spec and tries to find a `<DLL>.runfiles` directory
     # next to the the DLL based on argv0 of the running process if
     # RUNFILES_DIR/RUNFILES_MANIFEST_FILE/RUNFILES_MANIFEST_ONLY is not set).
+    runfiles = []
     for file in data:
-        inputs.append(file)
-        manifest_path = to_rlocation_path(ctx, file)
-        output = ctx.actions.declare_file(
-            "{}/publish/{}/{}.runfiles/{}".format(ctx.label.name, runtime_identifier, paths.replace_extension(binary_info.dll.basename, ""), manifest_path),
-        )
-        outputs.append(output)
-        _copy_file(script_body, file, output, is_windows = is_windows)
+        runfiles.append(file)
 
     for file in appsetting_files:
         inputs.append(file)
@@ -89,16 +81,6 @@ def _copy_to_publish(ctx, runtime_identifier, runtime_pack_info, binary_info, as
         outputs.append(output)
         _copy_file(script_body, file, output, is_windows = is_windows)
 
-    # The repo mapping manifest is not part of the runfiles by default so we
-    # copy it to the runfiles directory manually.
-    if repo_mapping_manifest:
-        inputs.append(repo_mapping_manifest)
-        output = ctx.actions.declare_file(
-            "{}/publish/{}/{}.runfiles/_repo_mapping".format(ctx.label.name, runtime_identifier, paths.replace_extension(binary_info.dll.basename, "")),
-        )
-        outputs.append(output)
-        _copy_file(script_body, repo_mapping_manifest, output, is_windows = is_windows)
-
     # In case the publish is self-contained there needs to be a runtime pack available
     # with the runtime dependencies that are required for the targeted runtime.
     # The runtime pack contents should always be copied to the root of the publish folder
@@ -129,7 +111,7 @@ def _copy_to_publish(ctx, runtime_identifier, runtime_pack_info, binary_info, as
         tools = [copy_script],
     )
 
-    return (main_dll_copy, outputs)
+    return (main_dll_copy, outputs, runfiles)
 
 def _create_shim_exe(ctx, apphost_pack_info, dll):
     windows_constraint = ctx.attr._windows_constraint[platform_common.ConstraintValueInfo]
@@ -171,8 +153,6 @@ def _generate_depsjson(
     )
 
 def _publish_binary_impl(ctx):
-    repo_mapping_manifest = ctx.attr.binary[0][DefaultInfo].files_to_run.repo_mapping_manifest
-
     assembly_compile_info = ctx.attr.binary[0][DotnetAssemblyCompileInfo]
     assembly_runtime_info = ctx.attr.binary[0][DotnetAssemblyRuntimeInfo]
     binary_info = ctx.attr.binary[0][DotnetBinaryInfo]
@@ -184,14 +164,13 @@ def _publish_binary_impl(ctx):
     runtime_identifier = ctx.attr.runtime_identifier if ctx.attr.runtime_identifier else binary_info.runtime_pack_info.runtime_identifier
     roll_forward_behavior = ctx.attr.roll_forward_behavior
 
-    (main_dll, runfiles) = _copy_to_publish(
+    (main_dll, outputs, runfiles) = _copy_to_publish(
         ctx,
         runtime_identifier,
         runtime_pack_info,
         binary_info,
         assembly_runtime_info,
         transitive_runtime_deps,
-        repo_mapping_manifest,
     )
 
     apphost_shim = _create_shim_exe(ctx, binary_info.apphost_pack_info, main_dll)
@@ -225,8 +204,8 @@ def _publish_binary_impl(ctx):
     return [
         DefaultInfo(
             executable = apphost_shim,
-            files = depset([apphost_shim, main_dll, runtimeconfig, depsjson] + runfiles),
-            runfiles = ctx.runfiles(files = [apphost_shim, main_dll, runtimeconfig, depsjson] + runfiles),
+            files = depset([apphost_shim, main_dll, runtimeconfig, depsjson] + outputs),
+            runfiles = ctx.runfiles(files = [apphost_shim, main_dll, runtimeconfig, depsjson] + outputs + runfiles),
         ),
     ]
 

docs/csharp_nunit_test.md

@@ -2,18 +2,26 @@
 
 set -euo pipefail
 
-if [[ $# -ne 3 ]]; then
+if [[ $# -lt 3 ]]; then
     echo >&2 "Usage: $0 /path/to/binary expected_file_type expected_arch"
     exit 1
 fi
 
 binary="$1"
 expected_file_type="$2"
-expected_arch="$3"
+expected_arches="${@:3}"
 
 out="$(file "$(readlink "${binary}")")"
 
-if [[ "${out}" != *"${expected_file_type}"* || "${out}" != *"${expected_arch}"* ]]; then
-    echo >&2 "Wrong file type: ${out}"
-    exit 1
-fi
+# Loop over all expected arches and check if the output contains any of them
+for expected_arch in ${expected_arches}; do
+    if [[ "${out}" == *"${expected_arch}"* ]]; then
+        if [[ "${out}" == *"${expected_file_type}"* ]]; then
+            exit 0
+        fi
+    fi
+done
+
+echo >&2 "Wrong file type: ${out}"
+exit 1
+

docs/defs.md

@@ -6,22 +6,23 @@ load(
 )
 
 permutations = [
-    ("linux-x64", "'ELF 64-bit'", "'x86-64'"),
-    ("linux-arm64", "'ELF 64-bit'", "'aarch64'"),
-    ("osx-x64", "'Mach-O'", "'x86_64'"),
-    ("osx-arm64", "'Mach-O'", "'arm64'"),
-    ("win-x64", "'PE32+'", "'x86-64'"),
-    ("win-arm64", "'PE32+'", "'Aarch64'"),
-    ("linux-musl-x64", "'ELF 64-bit'", "'x86-64'"),
-    ("linux-musl-arm64", "'ELF 64-bit'", "'aarch64'"),
+    ("linux-x64", "'ELF 64-bit'", ["'x86-64'"]),
+    ("linux-arm64", "'ELF 64-bit'", ["'aarch64'"]),
+    ("osx-x64", "'Mach-O'", ["'x86_64'"]),
+    ("osx-arm64", "'Mach-O'", ["'arm64'"]),
+    ("win-x64", "'PE32+'", ["'x86-64'"]),
+    # The `file` command returns different output when run on Linux/Windows
+    ("win-arm64", "'PE32+'", ["'Aarch64'", "'ARM64'"]),
+    ("linux-musl-x64", "'ELF 64-bit'", ["'x86-64'"]),
+    ("linux-musl-arm64", "'ELF 64-bit'", ["'aarch64'"]),
     # The permutations below are to verify that if we use a higher level rid like `fedora-x64` or `alpine-x64`
     # then we will find the closes rid that matches and has the runtime/apphost packs
-    ("fedora-x64", "'ELF 64-bit'", "'x86-64'"),
-    ("ubuntu.20.04-x64", "'ELF 64-bit'", "'x86-64'"),
-    ("alpine-x64", "'ELF 64-bit'", "'x86-64'"),
-    ("fedora-arm64", "'ELF 64-bit'", "'aarch64'"),
-    ("ubuntu.20.04-arm64", "'ELF 64-bit'", "'aarch64'"),
-    ("alpine-arm64", "'ELF 64-bit'", "'aarch64'"),
+    ("fedora-x64", "'ELF 64-bit'", ["'x86-64'"]),
+    ("ubuntu.20.04-x64", "'ELF 64-bit'", ["'x86-64'"]),
+    ("alpine-x64", "'ELF 64-bit'", ["'x86-64'"]),
+    ("fedora-arm64", "'ELF 64-bit'", ["'aarch64'"]),
+    ("ubuntu.20.04-arm64", "'ELF 64-bit'", ["'aarch64'"]),
+    ("alpine-arm64", "'ELF 64-bit'", ["'aarch64'"]),
 ]
 
 # buildifier: disable=unnamed-macro
@@ -41,8 +42,7 @@ def tests():
             args = [
                 "$(rootpath :cross_publish_{})".format(rid),
                 expected_file_type,
-                expected_arch,
-            ],
+            ] + expected_arch,
             data = [
                 ":cross_publish_{}".format(rid),
             ],