crashdump: enable crashdump feature for release builds

- Improve documentation and describe how an user can create separate files for debug information
- Change the output file directory to be configurable using HYPERLIGHT_CORE_DUMP_DIR environment variable
- Change output file name to include a timestamp

Signed-off-by: Doru Blânzeanu <[email protected]>

Author: dblnz

Cargo.lock

@@ -274,3 +274,149 @@ To do this in vscode, the following configuration can be used to add debug confi
 press the `pause` button. This is a known issue with the `CodeLldb` extension [#1245](https://github.com/vadimcn/codelldb/issues/1245).
 The `cppdbg` extension works as expected and stops at the entry point of the program.**
 
+## Compiling guests with debug information for release builds
+
+This section explains how to compile a guest with debugging information but still have optimized code, and how to separate the debug information from the binary.
+
+### Creating a release build with debug information
+
+To create a release build with debug information, you can add a custom profile to your `Cargo.toml` file:
+
+```toml
+[profile.release-with-debug]
+inherits = "release"
+debug = true
+```
+
+This creates a new profile called `release-with-debug` that inherits all settings from the release profile but adds debug information.
+
+### Splitting debug information from the binary
+
+To reduce the binary size while still having debug information available, you can split the debug information into a separate file.
+This is useful for production environments where you want smaller binaries but still want to be able to debug crashes.
+
+Here's a step-by-step guide:
+
+1. Build your guest with the release-with-debug profile:
+   ```bash
+   cargo build --profile release-with-debug
+   ```
+
+2. Locate your binary in the target directory:
+   ```bash
+   TARGET_DIR="target"
+   PROFILE="release-with-debug"
+   ARCH="x86_64-unknown-none" # Your target architecture
+   BUILD_DIR="${TARGET_DIR}/${ARCH}/${PROFILE}"
+   BINARY=$(find "${BUILD_DIR}" -type f -executable -name "guest-binary" | head -1)
+   ```
+
+3. Extract debug information into a full debug file:
+   ```bash
+   DEBUG_FILE_FULL="${BINARY}.debug.full"
+   objcopy --only-keep-debug "${BINARY}" "${DEBUG_FILE_FULL}"
+   ```
+
+4. Create a symbols-only debug file (smaller, but still useful for stack traces):
+   ```bash
+   DEBUG_FILE="${BINARY}.debug"
+   objcopy --keep-file-symbols "${DEBUG_FILE_FULL}" "${DEBUG_FILE}"
+   ```
+
+5. Strip debug information from the original binary but keep function names:
+   ```bash
+   objcopy --strip-debug "${BINARY}"
+   ```
+
+6. Add a debug link to the stripped binary:
+   ```bash
+   objcopy --add-gnu-debuglink="${DEBUG_FILE}" "${BINARY}"
+   ```
+
+After these steps, you'll have:
+- An optimized binary with function names for basic stack traces
+- A symbols-only debug file for stack traces
+- A full debug file for complete source-level debugging
+
+### Analyzing core dumps with the debug files
+
+When you have a core dump from a crashed guest, you can analyze it with different levels of detail using either GDB or LLDB.
+
+#### Using GDB
+
+1. For basic analysis with function names (stack traces):
+   ```bash
+   gdb ${BINARY} -c /path/to/core.dump
+   ```
+
+2. For full source-level debugging:
+   ```bash
+   gdb -s ${DEBUG_FILE_FULL} ${BINARY} -c /path/to/core.dump
+   ```
+
+#### Using LLDB
+
+LLDB provides similar capabilities with slightly different commands:
+
+1. For basic analysis with function names (stack traces):
+   ```bash
+   lldb ${BINARY} -c /path/to/core.dump
+   ```
+
+2. For full source-level debugging:
+   ```bash
+   lldb -o "target create -c /path/to/core.dump ${BINARY}" -o "add-dsym ${DEBUG_FILE_FULL}"
+   ```
+
+3. If your debug symbols are in a separate file:
+   ```bash
+   lldb ${BINARY} -c /path/to/core.dump
+   (lldb) add-dsym ${DEBUG_FILE_FULL}
+   ```
+
+### VSCode Debug Configurations
+
+You can configure VSCode (in `.vscode/launch.json`) to use these files by modifying the debug configurations:
+
+#### For GDB
+
+```json
+{
+    "name": "[GDB] Load core dump with full debug symbols",
+    "type": "cppdbg",
+    "request": "launch",
+    "program": "${input:program}",
+    "coreDumpPath": "${input:core_dump}",
+    "cwd": "${workspaceFolder}",
+    "MIMode": "gdb",
+    "externalConsole": false,
+    "miDebuggerPath": "/usr/bin/gdb",
+    "setupCommands": [
+        {
+            "description": "Enable pretty-printing for gdb",
+            "text": "-enable-pretty-printing",
+            "ignoreFailures": true
+        }
+    ]
+}
+```
+
+#### For LLDB
+
+```json
+{
+    "name": "[LLDB] Load core dump with full debug symbols",
+    "type": "lldb",
+    "request": "launch",
+    "program": "${input:program}",
+    "cwd": "${workspaceFolder}",
+    "processCreateCommands": [],
+    "targetCreateCommands": [
+        "target create -c ${input:core_dump} ${input:program}"
+    ],
+    "postRunCommands": [
+        // if debug symbols are in a different file
+        "add-dsym ${input:debug_file_path}"
+    ]
+}
+```

docs/how-to-debug-a-hyperlight-guest.md

@@ -38,11 +38,11 @@ vmm-sys-util = "0.14.0"
 crossbeam = "0.8.0"
 crossbeam-channel = "0.5.15"
 thiserror = "2.0.12"
-tempfile = { version = "3.20", optional = true }
+chrono = { version = "0.4", optional = true }
 anyhow = "1.0"
 metrics = "0.24.2"
 serde_json = "1.0"
-elfcore = { git = "https://github.com/dblnz/elfcore.git", rev = "1a57f04272dd54bc06df638f4027debe6d0f694f" }
+elfcore = { git = "https://github.com/hyperlight-dev/elfcore.git", rev = "cef4c80e26bf4b2a5599e50d2d1730965f942c13" }
 
 [target.'cfg(windows)'.dependencies]
 windows = { version = "0.61", features = [
@@ -124,7 +124,8 @@ function_call_metrics = []
 executable_heap = []
 # This feature enables printing of debug information to stdout in debug builds
 print_debug = []
-crashdump = ["dep:tempfile"] # Dumps the VM state to a file on unexpected errors or crashes. The path of the file will be printed on stdout and logged. This feature can only be used in debug builds.
+# Dumps the VM state to a file on unexpected errors or crashes. The path of the file will be printed on stdout and logged.
+crashdump = ["dep:chrono"]
 kvm = ["dep:kvm-bindings", "dep:kvm-ioctls"]
 mshv2 = ["dep:mshv-bindings2", "dep:mshv-ioctls2"]
 mshv3 = ["dep:mshv-bindings3", "dep:mshv-ioctls3"]

src/hyperlight_host/Cargo.toml

@@ -93,8 +93,7 @@ fn main() -> Result<()> {
         gdb: { all(feature = "gdb", debug_assertions, any(feature = "kvm", feature = "mshv2", feature = "mshv3"), target_os = "linux") },
         kvm: { all(feature = "kvm", target_os = "linux") },
         mshv: { all(any(feature = "mshv2", feature = "mshv3"), target_os = "linux") },
-        // crashdump feature is aliased with debug_assertions to make it only available in debug-builds.
-        crashdump: { all(feature = "crashdump", debug_assertions) },
+        crashdump: { all(feature = "crashdump") },
         // print_debug feature is aliased with debug_assertions to make it only available in debug-builds.
         print_debug: { all(feature = "print_debug", debug_assertions) },
         // the following features are mutually exclusive but rather than enforcing that here we are enabling mshv3 to override mshv2 when both are enabled

src/hyperlight_host/build.rs

@@ -16,11 +16,11 @@ limitations under the License.
 
 use std::cmp::min;
 
+use chrono;
 use elfcore::{
     ArchComponentState, ArchState, CoreDumpBuilder, CoreError, Elf64_Auxv, ProcessInfoSource,
     ReadProcessMemory, ThreadView, VaProtection, VaRegion,
 };
-use tempfile::NamedTempFile;
 
 use super::Hypervisor;
 use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags};
@@ -103,12 +103,12 @@ impl GuestView {
         let filename = ctx
             .filename
             .as_ref()
-            .map_or(|| "<unknown>".to_string(), |s| s.to_string());
+            .map_or("<unknown>".to_string(), |s| s.to_string());
 
         let cmd = ctx
             .binary
             .as_ref()
-            .map_or(|| "<unknown>".to_string(), |s| s.to_string());
+            .map_or("<unknown>".to_string(), |s| s.to_string());
 
         // The xsave state is checked as it can be empty
         let mut components = vec![];
@@ -262,10 +262,6 @@ impl ReadProcessMemory for GuestMemReader {
 pub(crate) fn crashdump_to_tempfile(hv: &dyn Hypervisor) -> Result<()> {
     log::info!("Creating core dump file...");
 
-    // Create a temporary file with a recognizable prefix
-    let temp_file = NamedTempFile::with_prefix("hl_core_")
-        .map_err(|e| new_error!("Failed to create temporary file: {:?}", e))?;
-
     // Get crash context from hypervisor
     let ctx = hv
         .crashdump_context()
@@ -278,16 +274,39 @@ pub(crate) fn crashdump_to_tempfile(hv: &dyn Hypervisor) -> Result<()> {
     // Create and write core dump
     let core_builder = CoreDumpBuilder::from_source(guest_view, memory_reader);
 
+    // Generate timestamp string for the filename using chrono
+    let timestamp = chrono::Local::now()
+        .format("%Y%m%d_T%H%M%S%.3f")
+        .to_string();
+
+    // Determine the output directory based on environment variable
+    let output_dir = if let Ok(dump_dir) = std::env::var("HYPERLIGHT_CORE_DUMP_DIR") {
+        // Create the directory if it doesn't exist
+        let path = std::path::Path::new(&dump_dir);
+        if !path.exists() {
+            std::fs::create_dir_all(path)
+                .map_err(|e| new_error!("Failed to create core dump directory: {:?}", e))?;
+        }
+        std::path::PathBuf::from(dump_dir)
+    } else {
+        // Fall back to the system temp directory
+        std::env::temp_dir()
+    };
+
+    // Create the filename with timestamp
+    let filename = format!("hl_core_{}.elf", timestamp);
+    let file_path = output_dir.join(filename);
+
+    // Create the file
+    let file = std::fs::File::create(&file_path)
+        .map_err(|e| new_error!("Failed to create core dump file: {:?}", e))?;
+
+    // Write the core dump directly to the file
     core_builder
-        .write(&temp_file)
+        .write(&file)
         .map_err(|e| new_error!("Failed to write core dump: {:?}", e))?;
 
-    let persist_path = temp_file.path().with_extension("elf");
-    temp_file
-        .persist(&persist_path)
-        .map_err(|e| new_error!("Failed to persist core dump file: {:?}", e))?;
-
-    let path_string = persist_path.to_string_lossy().to_string();
+    let path_string = file_path.to_string_lossy().to_string();
 
     println!("Core dump created successfully: {}", path_string);
     log::error!("Core dump file: {}", path_string);