working standalone openmp build

Author: ZuseZ4

src/bootstrap/src/core/build_steps/compile.rs

@@ -1437,6 +1437,11 @@ fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelect
         cargo.env("LLVM_ENZYME", "1");
     }
     let llvm::LlvmResult { host_llvm_config, .. } = builder.ensure(llvm::Llvm { target });
+    if builder.config.llvm_enzyme {
+        builder.ensure(llvm::Offload { target });
+        cargo.env("LLVM_OFFLOAD", "1");
+    }
+
     cargo.env("LLVM_CONFIG", &host_llvm_config);
 
     // Some LLVM linker flags (-L and -l) may be needed to link `rustc_llvm`. Its build script

src/bootstrap/src/core/build_steps/llvm.rs

@@ -457,7 +457,7 @@ impl Step for Llvm {
         // We will optimize it when we get closer to releasing it on nightly.
         if builder.config.llvm_offload {
             //enabled_llvm_projects.push("lld");
-            enabled_llvm_runtimes.push("offload");
+            //enabled_llvm_runtimes.push("offload");
             //FIXME(ZuseZ4): LLVM intends to drop the offload dependency on openmp.
             //Remove this line once they achieved it.
             //enabled_llvm_runtimes.push("openmp");
@@ -470,8 +470,8 @@ impl Step for Llvm {
             //let runtime_targets = vec!["default", "amdgcn-amd-amdhsa", "nvptx64-nvidia-cuda"];
             //cfg.define("LLVM_RUNTIME_TARGETS", runtime_targets.join(";"));
 
-            cfg.define("LLVM_INCLUDE_TESTS", "OFF");
-            cfg.define("LLVM_BUILD_TESTS", "OFF");
+            //cfg.define("LLVM_INCLUDE_TESTS", "OFF");
+            //cfg.define("LLVM_BUILD_TESTS", "OFF");
             //cfg.define("RUNTIMES_nvptx64-nvidia-cuda_LLVM_ENABLE_RUNTIMES", "openmp");
             //cfg.define("RUNTIMES_amdgcn-amd-amdhsa_LLVM_ENABLE_RUNTIMES", "openmp");
         }
@@ -908,6 +908,129 @@ fn get_var(var_base: &str, host: &str, target: &str) -> Option<OsString> {
         .or_else(|| env::var_os(var_base))
 }
 
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Offload {
+    pub target: TargetSelection,
+}
+
+impl Step for Offload {
+    type Output = PathBuf;
+    const IS_HOST: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/llvm-project/openmp")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(Offload { target: run.target });
+    }
+
+    /// Compile OpenMP offload runtimes for `target`.
+    fn run(self, builder: &Builder<'_>) -> PathBuf {
+        if builder.config.dry_run() {
+            return PathBuf::from("/");
+        }
+        let target = self.target;
+
+        let LlvmResult { host_llvm_config, .. } = builder.ensure(Llvm { target: self.target });
+
+        let out_dir = builder.llvm_out(target);
+        static STAMP_HASH_MEMO: OnceLock<String> = OnceLock::new();
+        //let smart_stamp_hash = STAMP_HASH_MEMO.get_or_init(|| {
+        //    generate_smart_stamp_hash(
+        //        builder,
+        //        &builder.config.src.join("src/llvm-project/openmp"),
+        //        builder.offload_info.sha().unwrap_or_default(),
+        //    )
+        //});
+        //let stamp = BuildStamp::new(&out_dir).with_prefix("enzyme").add_stamp(smart_stamp_hash);
+
+        trace!("checking build stamp to see if we need to rebuild offload/openmp artifacts");
+        //if stamp.is_up_to_date() {
+        //    trace!(?out_dir, "offload/openmp build artifacts are up to date");
+        //    if stamp.stamp().is_empty() {
+        //        builder.info(
+        //            "Could not determine the Offload submodule commit hash. \
+        //             Assuming that an Offload rebuild is not necessary.",
+        //        );
+        //        builder.info(&format!(
+        //            "To force Offload to rebuild, remove the file `{}`",
+        //            stamp.path().display()
+        //        ));
+        //    }
+        //    return out_dir;
+        //}
+
+        trace!(?target, "(re)building openmp artifacts");
+        builder.info(&format!("Building OpenMP for {target}"));
+        //t!(stamp.remove());
+        let _time = helpers::timeit(builder);
+        t!(fs::create_dir_all(&out_dir));
+
+        builder.config.update_submodule(Path::new("src").join("llvm-project").to_str().unwrap());
+        let mut cfg = cmake::Config::new(builder.src.join("src/llvm-project/runtimes/"));
+        configure_cmake(builder, target, &mut cfg, true, LdFlags::default(), &[]);
+
+        // Re-use the same flags as llvm to control the level of debug information
+        // generated by Enzyme.
+        // FIXME(ZuseZ4): Find a nicer way to use Enzyme Debug builds.
+        let profile = match (builder.config.llvm_optimize, builder.config.llvm_release_debuginfo) {
+            (false, _) => "Debug",
+            (true, false) => "Release",
+            (true, true) => "RelWithDebInfo",
+        };
+        trace!(?profile);
+
+        cfg.out_dir(&out_dir)
+            .profile(profile)
+            .env("LLVM_CONFIG_REAL", &host_llvm_config)
+            .define("LLVM_ENABLE_ASSERTIONS", "ON")
+            .define("TARGET_TRIPLE", &target.triple)
+            .define("LLVM_ENABLE_RUNTIMES", "openmp")
+            .define("CMAKE_C_COMPILER", builder.cc(target))
+            .define("CMAKE_CXX_COMPILER", builder.cxx(target).unwrap())
+            .define("LLVM_DEFAULT_TARGET_TRIPLE", &target.triple)
+            .define("OPENMP_STANDALONE_BUILD", "ON")
+            .define(
+                "LLVM_ROOT",
+                "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/build/",
+            )
+            .define("LLVM_DIR", builder.llvm_out(target).join("lib").join("cmake").join("llvm"));
+        ///tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/lib/cmake/llvm
+        //$> cmake ../runtimes \ # Point to the runtimes build
+        //   -G Ninja                                  \
+        //   -DLLVM_ENABLE_RUNTIMES=openmp               \
+        //   -DCMAKE_C_COMPILER=$TARGET_C_COMPILER     \
+        //   -DCMAKE_CXX_COMPILER=$TARGET_CXX_COMPILER \
+        //   -DLLVM_DEFAULT_TARGET_TRIPLE=$TARGET_TRIPLE     \
+        //   -DCMAKE_BUILD_TYPE=Release
+        cfg.build();
+
+        //t!(stamp.write());
+        out_dir
+
+        //$> cd llvm-project  # The llvm-project checkout
+        //$> mkdir build # A different build directory for the build tools
+        //$> cd build
+        //$> TARGET_TRIPLE=<amdgcn-amd-amdhsa or nvptx64-nvidia-cuda>
+        //$> TARGET_C_COMPILER=</path/to/clang>
+        //$> TARGET_CXX_COMPILER=</path/to/clang++>
+        //$> ninja install
+        //
+        //You can do the same thing for the offload project.
+        //
+        //$> TARGET_C_COMPILER=</path/to/clang>
+        //$> TARGET_CXX_COMPILER=</path/to/clang++>
+        //$> cmake ../runtimes \ # Point to the runtimes build
+        //   -G Ninja                                  \
+        //   -DLLVM_ENABLE_RUNTIMES=openmp               \
+        //   -DCMAKE_C_COMPILER=$TARGET_C_COMPILER     \
+        //   -DCMAKE_CXX_COMPILER=$TARGET_CXX_COMPILER \
+        //   -DCMAKE_BUILD_TYPE=Release
+        //$> ninja install
+    }
+}
+
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 pub struct Enzyme {
     pub target: TargetSelection,