diff --git a/.github/workflows/dep_rust.yml b/.github/workflows/dep_rust.yml index e814d2771..7f00a777c 100644 --- a/.github/workflows/dep_rust.yml +++ b/.github/workflows/dep_rust.yml @@ -136,6 +136,12 @@ jobs: RUST_LOG: debug run: just test-rust-gdb-debugging ${{ matrix.config }} ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}} + - name: Run Rust Crashdump tests + env: + CARGO_TERM_COLOR: always + RUST_LOG: debug + run: just test-rust-crashdump ${{ matrix.config }} ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}} + ### Benchmarks ### - name: Install github-cli (Linux mariner) if: runner.os == 'Linux' && matrix.hypervisor == 'mshv' diff --git a/Cargo.lock b/Cargo.lock index eff2feaf8..e95a6a8ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -663,6 +663,20 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +[[package]] +name = "elfcore" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824386967a6a98e7f99d5c15d40cd1534b0ebfc4193a7109689dcf5322e8d744" +dependencies = [ + "libc", + "nix", + "smallvec", + "thiserror 1.0.69", + "tracing", + "zerocopy 0.7.35", +] + [[package]] name = "endian-type" version = "0.1.2" @@ -1218,10 +1232,12 @@ dependencies = [ "built", "cfg-if", "cfg_aliases", + "chrono", "criterion", "crossbeam", "crossbeam-channel", "crossbeam-queue", + "elfcore", "env_logger", "flatbuffers", "gdbstub", @@ -1897,6 +1913,17 @@ dependencies = [ "smallvec", ] +[[package]] +name = "nix" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "libc", +] + [[package]] name = "nom" version = "7.1.3" diff --git a/Justfile b/Justfile index b491d1db7..b14448166 100644 --- a/Justfile +++ b/Justfile @@ -81,6 +81,9 @@ test-like-ci config=default-target hypervisor="kvm": @# without any driver (should fail to compile) just test-compilation-fail {{config}} + @# test the crashdump feature + just test-rust-crashdump {{config}} + # runs all tests test target=default-target features="": (test-unit target features) (test-isolated target features) (test-integration "rust" target features) (test-integration "c" target features) (test-seccomp target features) @@ -123,6 +126,10 @@ test-rust-gdb-debugging target=default-target features="": cargo test --profile={{ if target == "debug" { "dev" } else { target } }} --example guest-debugging {{ if features =="" {'--features gdb'} else { "--features gdb," + features } }} cargo test --profile={{ if target == "debug" { "dev" } else { target } }} {{ if features =="" {'--features gdb'} else { "--features gdb," + features } }} -- test_gdb +# rust test for crashdump +test-rust-crashdump target=default-target features="": + cargo test --profile={{ if target == "debug" { "dev" } else { target } }} {{ if features =="" {'--features crashdump'} else { "--features crashdump," + features } }} -- test_crashdump + ################ ### LINTING #### diff --git a/docs/how-to-debug-a-hyperlight-guest.md b/docs/how-to-debug-a-hyperlight-guest.md index 8e5dd7d55..f3a565abc 100644 --- a/docs/how-to-debug-a-hyperlight-guest.md +++ b/docs/how-to-debug-a-hyperlight-guest.md @@ -201,3 +201,233 @@ involved in the gdb debugging of a Hyperlight guest running inside a **KVM** or └─┘ │ | | | │ | └───────────────────────────────────────────────────────────────────────────────────────────────┘ ``` + +## Dumping the guest state to an ELF core dump when an unhandled crash occurs + +When a guest crashes because of an unknown VmExit or unhandled exception, the vCPU state is dumped to an `ELF` core dump file. +This can be used to inspect the state of the guest at the time of the crash. + +To make Hyperlight dump the state of the vCPU (general purpose registers, registers) to an `ELF` core dump file, enable the `crashdump` +feature and run. +The feature enables the creation of core dump files for both debug and release builds of Hyperlight hosts. +By default, Hyperlight places the core dumps in the temporary directory (platform specific). +To change this, use the `HYPERLIGHT_CORE_DUMP_DIR` environment variable to specify a directory. +The name and location of the dump file will be printed to the console and logged as an error message. + +**NOTE**: If the directory provided by `HYPERLIGHT_CORE_DUMP_DIR` does not exist, Hyperlight places the file in the temporary directory. +**NOTE**: By enabling the `crashdump` feature, you instruct Hyperlight to create core dump files for all sandboxes when an unhandled crash occurs. +To selectively disable this feature for a specific sandbox, you can set the `guest_core_dump` field to `false` in the `SandboxConfiguration`. +```rust + let mut cfg = SandboxConfiguration::default(); + cfg.set_guest_core_dump(false); // Disable core dump for this sandbox +``` + +### Inspecting the core dump + +After the core dump has been created, to inspect the state of the guest, load the core dump file using `gdb` or `lldb`. +**NOTE: This feature has been tested with version `15.0` of `gdb` and version `17` of `lldb`, earlier versions may not work, it is recommended to use these versions or later.** + +To do this in vscode, the following configuration can be used to add debug configurations: + +```vscode +{ + "version": "0.2.0", + "inputs": [ + { + "id": "core_dump", + "type": "promptString", + "description": "Path to the core dump file", + }, + { + "id": "program", + "type": "promptString", + "description": "Path to the program to debug", + } + ], + "configurations": [ + { + "name": "[GDB] Load core dump file", + "type": "cppdbg", + "request": "launch", + "program": "${input:program}", + "coreDumpPath": "${input:core_dump}", + "cwd": "${workspaceFolder}", + "MIMode": "gdb", + "externalConsole": false, + "miDebuggerPath": "/usr/bin/gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + }, + { + "description": "Set Disassembly Flavor to Intel", + "text": "-gdb-set disassembly-flavor intel", + "ignoreFailures": true + } + ] + }, + { + "name": "[LLDB] Load core dump file", + "type": "lldb", + "request": "launch", + "stopOnEntry": true, + "processCreateCommands": [], + "targetCreateCommands": [ + "target create -c ${input:core_dump} ${input:program}", + ], + }, + ] +} +``` +**NOTE: The `CodeLldb` debug session does not stop after launching. To see the code, stack frames and registers you need to +press the `pause` button. This is a known issue with the `CodeLldb` extension [#1245](https://github.com/vadimcn/codelldb/issues/1245). +The `cppdbg` extension works as expected and stops at the entry point of the program.** + +## Compiling guests with debug information for release builds + +This section explains how to compile a guest with debugging information but still have optimized code, and how to separate the debug information from the binary. + +### Creating a release build with debug information + +To create a release build with debug information, you can add a custom profile to your `Cargo.toml` file: + +```toml +[profile.release-with-debug] +inherits = "release" +debug = true +``` + +This creates a new profile called `release-with-debug` that inherits all settings from the release profile but adds debug information. + +### Splitting debug information from the binary + +To reduce the binary size while still having debug information available, you can split the debug information into a separate file. +This is useful for production environments where you want smaller binaries but still want to be able to debug crashes. + +Here's a step-by-step guide: + +1. Build your guest with the release-with-debug profile: + ```bash + cargo build --profile release-with-debug + ``` + +2. Locate your binary in the target directory: + ```bash + TARGET_DIR="target" + PROFILE="release-with-debug" + ARCH="x86_64-unknown-none" # Your target architecture + BUILD_DIR="${TARGET_DIR}/${ARCH}/${PROFILE}" + BINARY=$(find "${BUILD_DIR}" -type f -executable -name "guest-binary" | head -1) + ``` + +3. Extract debug information into a full debug file: + ```bash + DEBUG_FILE_FULL="${BINARY}.debug.full" + objcopy --only-keep-debug "${BINARY}" "${DEBUG_FILE_FULL}" + ``` + +4. Create a symbols-only debug file (smaller, but still useful for stack traces): + ```bash + DEBUG_FILE="${BINARY}.debug" + objcopy --keep-file-symbols "${DEBUG_FILE_FULL}" "${DEBUG_FILE}" + ``` + +5. Strip debug information from the original binary but keep function names: + ```bash + objcopy --strip-debug "${BINARY}" + ``` + +6. Add a debug link to the stripped binary: + ```bash + objcopy --add-gnu-debuglink="${DEBUG_FILE}" "${BINARY}" + ``` + +After these steps, you'll have: +- An optimized binary with function names for basic stack traces +- A symbols-only debug file for stack traces +- A full debug file for complete source-level debugging + +### Analyzing core dumps with the debug files + +When you have a core dump from a crashed guest, you can analyze it with different levels of detail using either GDB or LLDB. + +#### Using GDB + +1. For basic analysis with function names (stack traces): + ```bash + gdb ${BINARY} -c /path/to/core.dump + ``` + +2. For full source-level debugging: + ```bash + gdb -s ${DEBUG_FILE_FULL} ${BINARY} -c /path/to/core.dump + ``` + +#### Using LLDB + +LLDB provides similar capabilities with slightly different commands: + +1. For basic analysis with function names (stack traces): + ```bash + lldb ${BINARY} -c /path/to/core.dump + ``` + +2. For full source-level debugging: + ```bash + lldb -o "target create -c /path/to/core.dump ${BINARY}" -o "add-dsym ${DEBUG_FILE_FULL}" + ``` + +3. If your debug symbols are in a separate file: + ```bash + lldb ${BINARY} -c /path/to/core.dump + (lldb) add-dsym ${DEBUG_FILE_FULL} + ``` + +### VSCode Debug Configurations + +You can configure VSCode (in `.vscode/launch.json`) to use these files by modifying the debug configurations: + +#### For GDB + +```json +{ + "name": "[GDB] Load core dump with full debug symbols", + "type": "cppdbg", + "request": "launch", + "program": "${input:program}", + "coreDumpPath": "${input:core_dump}", + "cwd": "${workspaceFolder}", + "MIMode": "gdb", + "externalConsole": false, + "miDebuggerPath": "/usr/bin/gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ] +} +``` + +#### For LLDB + +```json +{ + "name": "[LLDB] Load core dump with full debug symbols", + "type": "lldb", + "request": "launch", + "program": "${input:program}", + "cwd": "${workspaceFolder}", + "processCreateCommands": [], + "targetCreateCommands": [ + "target create -c ${input:core_dump} ${input:program}" + ], + "postRunCommands": [ + // if debug symbols are in a different file + "add-dsym ${input:debug_file_path}" + ] +} +``` diff --git a/src/hyperlight_host/Cargo.toml b/src/hyperlight_host/Cargo.toml index 1352422d7..b94aebe0e 100644 --- a/src/hyperlight_host/Cargo.toml +++ b/src/hyperlight_host/Cargo.toml @@ -38,10 +38,11 @@ vmm-sys-util = "0.14.0" crossbeam = "0.8.0" crossbeam-channel = "0.5.15" thiserror = "2.0.12" -tempfile = { version = "3.20", optional = true } +chrono = { version = "0.4", optional = true } anyhow = "1.0" metrics = "0.24.2" serde_json = "1.0" +elfcore = "2.0" [target.'cfg(windows)'.dependencies] windows = { version = "0.61", features = [ @@ -124,7 +125,8 @@ function_call_metrics = [] executable_heap = [] # This feature enables printing of debug information to stdout in debug builds print_debug = [] -crashdump = ["dep:tempfile"] # Dumps the VM state to a file on unexpected errors or crashes. The path of the file will be printed on stdout and logged. This feature can only be used in debug builds. +# Dumps the VM state to a file on unexpected errors or crashes. The path of the file will be printed on stdout and logged. +crashdump = ["dep:chrono"] kvm = ["dep:kvm-bindings", "dep:kvm-ioctls"] mshv2 = ["dep:mshv-bindings2", "dep:mshv-ioctls2"] mshv3 = ["dep:mshv-bindings3", "dep:mshv-ioctls3"] diff --git a/src/hyperlight_host/build.rs b/src/hyperlight_host/build.rs index 4484e27d5..75f9eba53 100644 --- a/src/hyperlight_host/build.rs +++ b/src/hyperlight_host/build.rs @@ -93,8 +93,7 @@ fn main() -> Result<()> { gdb: { all(feature = "gdb", debug_assertions, any(feature = "kvm", feature = "mshv2", feature = "mshv3"), target_os = "linux") }, kvm: { all(feature = "kvm", target_os = "linux") }, mshv: { all(any(feature = "mshv2", feature = "mshv3"), target_os = "linux") }, - // crashdump feature is aliased with debug_assertions to make it only available in debug-builds. - crashdump: { all(feature = "crashdump", debug_assertions) }, + crashdump: { all(feature = "crashdump") }, // print_debug feature is aliased with debug_assertions to make it only available in debug-builds. print_debug: { all(feature = "print_debug", debug_assertions) }, // the following features are mutually exclusive but rather than enforcing that here we are enabling mshv3 to override mshv2 when both are enabled diff --git a/src/hyperlight_host/src/hypervisor/crashdump.rs b/src/hyperlight_host/src/hypervisor/crashdump.rs index 3467f5a81..e6e776c37 100644 --- a/src/hyperlight_host/src/hypervisor/crashdump.rs +++ b/src/hyperlight_host/src/hypervisor/crashdump.rs @@ -14,47 +14,479 @@ See the License for the specific language governing permissions and limitations under the License. */ +use std::cmp::min; use std::io::Write; -use tempfile::NamedTempFile; +use chrono; +use elfcore::{ + ArchComponentState, ArchState, CoreDumpBuilder, CoreError, Elf64_Auxv, ProcessInfoSource, + ReadProcessMemory, ThreadView, VaProtection, VaRegion, +}; use super::Hypervisor; +use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags}; use crate::{Result, new_error}; -/// Dump registers + memory regions + raw memory to a tempfile -#[cfg(crashdump)] -pub(crate) fn crashdump_to_tempfile(hv: &dyn Hypervisor) -> Result<()> { - let mut temp_file = NamedTempFile::with_prefix("mem")?; - let hv_details = format!("{:#x?}", hv); +/// This constant is used to identify the XSAVE state in the core dump +const NT_X86_XSTATE: u32 = 0x202; +/// This constant identifies the entry point of the program in an Auxiliary Vector +/// note of ELF. This tells a debugger whether the entry point of the program changed +/// so it can load the symbols correctly. +const AT_ENTRY: u64 = 9; +/// This constant is used to mark the end of the Auxiliary Vector note +const AT_NULL: u64 = 0; +/// The PID of the core dump process - this is a placeholder value +const CORE_DUMP_PID: i32 = 1; +/// The page size of the core dump +const CORE_DUMP_PAGE_SIZE: usize = 0x1000; - // write hypervisor details such as registers, info about mapped memory regions, etc. - temp_file.write_all(hv_details.as_bytes())?; - temp_file.write_all(b"================ MEMORY DUMP =================\n")?; +/// Structure to hold the crash dump context +/// This structure contains the information needed to create a core dump +#[derive(Debug)] +pub(crate) struct CrashDumpContext<'a> { + regions: &'a [MemoryRegion], + regs: [u64; 27], + xsave: Vec, + entry: u64, + binary: Option, + filename: Option, +} - // write the raw memory dump for each memory region - for region in hv.get_memory_regions() { - if region.host_region.start == 0 || region.host_region.is_empty() { - continue; +impl<'a> CrashDumpContext<'a> { + pub(crate) fn new( + regions: &'a [MemoryRegion], + regs: [u64; 27], + xsave: Vec, + entry: u64, + binary: Option, + filename: Option, + ) -> Self { + Self { + regions, + regs, + xsave, + entry, + binary, + filename, } - // SAFETY: we got this memory region from the hypervisor so should never be invalid - let region_slice = unsafe { - std::slice::from_raw_parts( - region.host_region.start as *const u8, - region.host_region.len(), - ) + } +} + +/// Structure that contains the process information for the core dump +/// This serves as a source of information for `elfcore`'s [`CoreDumpBuilder`] +struct GuestView { + regions: Vec, + threads: Vec, + aux_vector: Vec, +} + +impl GuestView { + fn new(ctx: &CrashDumpContext) -> Self { + // Map the regions to the format `CoreDumpBuilder` expects + let regions = ctx + .regions + .iter() + .filter(|r| !r.host_region.is_empty()) + .map(|r| VaRegion { + begin: r.guest_region.start as u64, + end: r.guest_region.end as u64, + offset: r.host_region.start as u64, + protection: VaProtection { + is_private: false, + read: r.flags.contains(MemoryRegionFlags::READ), + write: r.flags.contains(MemoryRegionFlags::WRITE), + execute: r.flags.contains(MemoryRegionFlags::EXECUTE), + }, + mapped_file_name: None, + }) + .collect(); + + let filename = ctx + .filename + .as_ref() + .map_or("".to_string(), |s| s.to_string()); + + let cmd = ctx + .binary + .as_ref() + .map_or("".to_string(), |s| s.to_string()); + + // The xsave state is checked as it can be empty + let mut components = vec![]; + if !ctx.xsave.is_empty() { + components.push(ArchComponentState { + name: "XSAVE", + note_type: NT_X86_XSTATE, + note_name: b"LINUX", + data: ctx.xsave.clone(), + }); + } + + // Create the thread view + // The thread view contains the information about the thread + // NOTE: Some of these fields are not used in the current implementation + let thread = ThreadView { + flags: 0, // Kernel flags for the process + tid: 1, + uid: 0, // User ID + gid: 0, // Group ID + comm: filename, + ppid: 0, // Parent PID + pgrp: 0, // Process group ID + nice: 0, // Nice value + state: 0, // Process state + utime: 0, // User time + stime: 0, // System time + cutime: 0, // Children User time + cstime: 0, // Children User time + cursig: 0, // Current signal + session: 0, // Session ID of the process + sighold: 0, // Blocked signal + sigpend: 0, // Pending signal + cmd_line: cmd, + + arch_state: Box::new(ArchState { + gpr_state: ctx.regs.to_vec(), + components, + }), }; - temp_file.write_all(region_slice)?; + + // Create the auxv vector + // The first entry is AT_ENTRY, which is the entry point of the program + // The entry point is the address where the program starts executing + // This helps the debugger to know that the entry is changed by an offset + // so the symbols can be loaded correctly. + // The second entry is AT_NULL, which marks the end of the vector + let auxv = vec![ + Elf64_Auxv { + a_type: AT_ENTRY, + a_val: ctx.entry, + }, + Elf64_Auxv { + a_type: AT_NULL, + a_val: 0, + }, + ]; + + Self { + regions, + threads: vec![thread], + aux_vector: auxv, + } + } +} + +impl ProcessInfoSource for GuestView { + fn pid(&self) -> i32 { + CORE_DUMP_PID + } + fn threads(&self) -> &[elfcore::ThreadView] { + &self.threads + } + fn page_size(&self) -> usize { + CORE_DUMP_PAGE_SIZE + } + fn aux_vector(&self) -> Option<&[elfcore::Elf64_Auxv]> { + Some(&self.aux_vector) } - temp_file.flush()?; + fn va_regions(&self) -> &[elfcore::VaRegion] { + &self.regions + } + fn mapped_files(&self) -> Option<&[elfcore::MappedFile]> { + // We don't have mapped files + None + } +} - // persist the tempfile to disk - let persist_path = temp_file.path().with_extension("dmp"); - temp_file - .persist(&persist_path) - .map_err(|e| new_error!("Failed to persist crashdump file: {:?}", e))?; +/// Structure that reads the guest memory +/// This structure serves as a custom memory reader for `elfcore`'s +/// [`CoreDumpBuilder`] +struct GuestMemReader { + regions: Vec, +} - println!("Memory dumped to file: {:?}", persist_path); - log::error!("Memory dumped to file: {:?}", persist_path); +impl GuestMemReader { + fn new(ctx: &CrashDumpContext) -> Self { + Self { + regions: ctx.regions.to_vec(), + } + } +} + +impl ReadProcessMemory for GuestMemReader { + fn read_process_memory( + &mut self, + base: usize, + buf: &mut [u8], + ) -> std::result::Result { + for r in self.regions.iter() { + // Check if the base address is within the guest region + if base >= r.guest_region.start && base < r.guest_region.end { + let offset = base - r.guest_region.start; + let region_slice = unsafe { + std::slice::from_raw_parts( + r.host_region.start as *const u8, + r.host_region.len(), + ) + }; + + // Calculate how much we can copy + let copy_size = min(buf.len(), region_slice.len() - offset); + if copy_size == 0 { + return std::result::Result::Ok(0); + } + + // Only copy the amount that fits in both buffers + buf[..copy_size].copy_from_slice(®ion_slice[offset..offset + copy_size]); + + // Return the number of bytes copied + return std::result::Result::Ok(copy_size); + } + } + + // If we reach here, we didn't find a matching region + std::result::Result::Ok(0) + } +} + +/// Create core dump file from the hypervisor information if the sandbox is configured +/// to allow core dumps. +/// +/// This function generates an ELF core dump file capturing the hypervisor's state, +/// which can be used for debugging when crashes occur. +/// The location of the core dump file is determined by the `HYPERLIGHT_CORE_DUMP_DIR` +/// environment variable. If not set, it defaults to the system's temporary directory. +/// +/// # Arguments +/// * `hv`: Reference to the hypervisor implementation +/// +/// # Returns +/// * `Result<()>`: Success or error +pub(crate) fn generate_crashdump(hv: &dyn Hypervisor) -> Result<()> { + // Get crash context from hypervisor + let ctx = hv + .crashdump_context() + .map_err(|e| new_error!("Failed to get crashdump context: {:?}", e))?; + + // Get env variable for core dump directory + let core_dump_dir = std::env::var("HYPERLIGHT_CORE_DUMP_DIR").ok(); + + // Compute file path on the filesystem + let file_path = core_dump_file_path(core_dump_dir); + + let create_dump_file = || { + // Create the file + Ok(Box::new( + std::fs::File::create(&file_path) + .map_err(|e| new_error!("Failed to create core dump file: {:?}", e))?, + ) as Box) + }; + + if let Ok(nbytes) = checked_core_dump(ctx, create_dump_file) { + if nbytes > 0 { + println!("Core dump created successfully: {}", file_path); + log::error!("Core dump file: {}", file_path); + } + } else { + log::error!("Failed to create core dump file"); + } Ok(()) } + +/// Computes the file path for the core dump file. +/// +/// The file path is generated based on the current timestamp and an +/// output directory. +/// If the directory does not exist, it falls back to the system's temp directory. +/// If the variable is not set, it defaults to the system's temporary directory. +/// The filename is formatted as `hl_core_.elf`. +/// +/// Arguments: +/// * `dump_dir`: The environment variable value to check for the output directory. +/// +/// Returns: +/// * `String`: The file path for the core dump file. +fn core_dump_file_path(dump_dir: Option) -> String { + // Generate timestamp string for the filename using chrono + let timestamp = chrono::Local::now() + .format("%Y%m%d_T%H%M%S%.3f") + .to_string(); + + // Determine the output directory based on environment variable + let output_dir = if let Some(dump_dir) = dump_dir { + // Check if the directory exists + // If it doesn't exist, fall back to the system temp directory + // This is to ensure that the core dump can be created even if the directory is not set + if std::path::Path::new(&dump_dir).exists() { + std::path::PathBuf::from(dump_dir) + } else { + log::warn!( + "Directory \"{}\" does not exist, falling back to temp directory", + dump_dir + ); + std::env::temp_dir() + } + } else { + // Fall back to the system temp directory + std::env::temp_dir() + }; + + // Create the filename with timestamp + let filename = format!("hl_core_{}.elf", timestamp); + let file_path = output_dir.join(filename); + + file_path.to_string_lossy().to_string() +} + +/// Create core dump from Hypervisor context if the sandbox is configured to allow core dumps. +/// +/// Arguments: +/// * `ctx`: Optional crash dump context from the hypervisor. This contains the information +/// needed to create the core dump. If `None`, no core dump will be created. +/// * `get_writer`: Closure that returns a writer to the output destination. +/// +/// Returns: +/// * `Result`: The number of bytes written to the core dump file. +fn checked_core_dump( + ctx: Option, + get_writer: impl FnOnce() -> Result>, +) -> Result { + let mut nbytes = 0; + // If the HV returned a context it means we can create a core dump + // This is the case when the sandbox has been configured at runtime to allow core dumps + if let Some(ctx) = ctx { + log::info!("Creating core dump file..."); + + // Set up data sources for the core dump + let guest_view = GuestView::new(&ctx); + let memory_reader = GuestMemReader::new(&ctx); + + // Create and write core dump + let core_builder = CoreDumpBuilder::from_source(guest_view, memory_reader); + + let writer = get_writer()?; + // Write the core dump directly to the file + nbytes = core_builder + .write(writer) + .map_err(|e| new_error!("Failed to write core dump: {:?}", e))?; + } + + Ok(nbytes) +} + +/// Test module for the crash dump functionality +#[cfg(test)] +mod test { + use super::*; + + /// Test the core_dump_file_path function when the environment variable is set to an existing + /// directory + #[test] + fn test_crashdump_file_path_valid() { + // Get CWD + let valid_dir = std::env::current_dir() + .unwrap() + .to_string_lossy() + .to_string(); + + // Call the function + let path = core_dump_file_path(Some(valid_dir.clone())); + + // Check if the path is correct + assert!(path.contains(&valid_dir)); + } + + /// Test the core_dump_file_path function when the environment variable is set to an invalid + /// directory + #[test] + fn test_crashdump_file_path_invalid() { + // Call the function + let path = core_dump_file_path(Some("/tmp/not_existing_dir".to_string())); + + // Get the temp directory + let temp_dir = std::env::temp_dir().to_string_lossy().to_string(); + + // Check if the path is correct + assert!(path.contains(&temp_dir)); + } + + /// Test the core_dump_file_path function when the environment is not set + /// Check against the default temp directory by using the env::temp_dir() function + #[test] + fn test_crashdump_file_path_default() { + // Call the function + let path = core_dump_file_path(None); + + let temp_dir = std::env::temp_dir().to_string_lossy().to_string(); + + // Check if the path is correct + assert!(path.starts_with(&temp_dir)); + } + + /// Test core is not created when the context is None + #[test] + fn test_crashdump_not_created_when_context_is_none() { + // Call the function with None context + let result = checked_core_dump(None, || Ok(Box::new(std::io::empty()))); + + // Check if the result is ok and the number of bytes is 0 + assert!(result.is_ok()); + assert_eq!(result.unwrap(), 0); + } + + /// Test the core dump creation with no regions fails + #[test] + fn test_crashdump_write_fails_when_no_regions() { + // Create a dummy context + let ctx = CrashDumpContext::new( + &[], + [0; 27], + vec![], + 0, + Some("dummy_binary".to_string()), + Some("dummy_filename".to_string()), + ); + + let get_writer = || Ok(Box::new(std::io::empty()) as Box); + + // Call the function + let result = checked_core_dump(Some(ctx), get_writer); + + // Check if the result is an error + // This should fail because there are no regions + assert!(result.is_err()); + } + + /// Check core dump with a dummy region to local vec + /// This test checks if the core dump is created successfully + #[test] + fn test_crashdump_dummy_core_dump() { + let dummy_vec = vec![0; 0x1000]; + let regions = vec![MemoryRegion { + guest_region: 0x1000..0x2000, + host_region: dummy_vec.as_ptr() as usize..dummy_vec.as_ptr() as usize + dummy_vec.len(), + flags: MemoryRegionFlags::READ | MemoryRegionFlags::WRITE, + region_type: crate::mem::memory_region::MemoryRegionType::Code, + }]; + // Create a dummy context + let ctx = CrashDumpContext::new( + ®ions, + [0; 27], + vec![], + 0x1000, + Some("dummy_binary".to_string()), + Some("dummy_filename".to_string()), + ); + + let get_writer = || Ok(Box::new(std::io::empty()) as Box); + + // Call the function + let result = checked_core_dump(Some(ctx), get_writer); + + // Check if the result is ok and the number of bytes is 0 + assert!(result.is_ok()); + // Check the number of bytes written is more than 0x1000 (the size of the region) + assert_eq!(result.unwrap(), 0x2000); + } +} diff --git a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs index 21c1c9182..a35ffe703 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs @@ -50,6 +50,8 @@ use mshv_bindings::{ }; use mshv_ioctls::{Mshv, MshvError, VcpuFd, VmFd}; use tracing::{Span, instrument}; +#[cfg(crashdump)] +use {super::crashdump, std::path::Path}; use super::fpu::{FP_CONTROL_WORD_DEFAULT, FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT}; #[cfg(gdb)] @@ -68,6 +70,8 @@ use crate::hypervisor::HyperlightExit; use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags}; use crate::mem::ptr::{GuestPtr, RawPtr}; use crate::sandbox::SandboxConfiguration; +#[cfg(crashdump)] +use crate::sandbox::uninitialized::SandboxRuntimeConfig; use crate::{Result, log_then_return, new_error}; #[cfg(gdb)] @@ -302,6 +306,8 @@ pub(crate) struct HypervLinuxDriver { debug: Option, #[cfg(gdb)] gdb_conn: Option>, + #[cfg(crashdump)] + rt_cfg: SandboxRuntimeConfig, } impl HypervLinuxDriver { @@ -321,6 +327,7 @@ impl HypervLinuxDriver { pml4_ptr: GuestPtr, config: &SandboxConfiguration, #[cfg(gdb)] gdb_conn: Option>, + #[cfg(crashdump)] rt_cfg: SandboxRuntimeConfig, ) -> Result { let mshv = Mshv::new()?; let pr = Default::default(); @@ -408,6 +415,8 @@ impl HypervLinuxDriver { debug, #[cfg(gdb)] gdb_conn, + #[cfg(crashdump)] + rt_cfg, }) } @@ -750,8 +759,61 @@ impl Hypervisor for HypervLinuxDriver { } #[cfg(crashdump)] - fn get_memory_regions(&self) -> &[MemoryRegion] { - &self.mem_regions + fn crashdump_context(&self) -> Result> { + if self.rt_cfg.guest_core_dump { + let mut regs = [0; 27]; + + let vcpu_regs = self.vcpu_fd.get_regs()?; + let sregs = self.vcpu_fd.get_sregs()?; + let xsave = self.vcpu_fd.get_xsave()?; + + // Set up the registers for the crash dump + regs[0] = vcpu_regs.r15; // r15 + regs[1] = vcpu_regs.r14; // r14 + regs[2] = vcpu_regs.r13; // r13 + regs[3] = vcpu_regs.r12; // r12 + regs[4] = vcpu_regs.rbp; // rbp + regs[5] = vcpu_regs.rbx; // rbx + regs[6] = vcpu_regs.r11; // r11 + regs[7] = vcpu_regs.r10; // r10 + regs[8] = vcpu_regs.r9; // r9 + regs[9] = vcpu_regs.r8; // r8 + regs[10] = vcpu_regs.rax; // rax + regs[11] = vcpu_regs.rcx; // rcx + regs[12] = vcpu_regs.rdx; // rdx + regs[13] = vcpu_regs.rsi; // rsi + regs[14] = vcpu_regs.rdi; // rdi + regs[15] = 0; // orig rax + regs[16] = vcpu_regs.rip; // rip + regs[17] = sregs.cs.selector as u64; // cs + regs[18] = vcpu_regs.rflags; // eflags + regs[19] = vcpu_regs.rsp; // rsp + regs[20] = sregs.ss.selector as u64; // ss + regs[21] = sregs.fs.base; // fs_base + regs[22] = sregs.gs.base; // gs_base + regs[23] = sregs.ds.selector as u64; // ds + regs[24] = sregs.es.selector as u64; // es + regs[25] = sregs.fs.selector as u64; // fs + regs[26] = sregs.gs.selector as u64; // gs + + // Get the filename from the binary path + let filename = self.rt_cfg.binary_path.clone().and_then(|path| { + Path::new(&path) + .file_name() + .and_then(|name| name.to_os_string().into_string().ok()) + }); + + Ok(Some(crashdump::CrashDumpContext::new( + &self.mem_regions, + regs, + xsave.buffer.to_vec(), + self.entrypoint, + self.rt_cfg.binary_path.clone(), + filename, + ))) + } else { + Ok(None) + } } #[cfg(gdb)] @@ -875,6 +937,15 @@ mod tests { &config, #[cfg(gdb)] None, + #[cfg(crashdump)] + SandboxRuntimeConfig { + #[cfg(crashdump)] + binary_path: None, + #[cfg(gdb)] + debug_info: None, + #[cfg(crashdump)] + guest_core_dump: true, + }, ) .unwrap(); } diff --git a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs index c732058d2..0bc7886ad 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs @@ -30,6 +30,8 @@ use windows::Win32::System::Hypervisor::{ WHvCancelRunVirtualProcessor, WHvX64RegisterCr0, WHvX64RegisterCr3, WHvX64RegisterCr4, WHvX64RegisterCs, WHvX64RegisterEfer, }; +#[cfg(crashdump)] +use {super::crashdump, std::path::Path}; use super::fpu::{FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT}; #[cfg(gdb)] @@ -47,6 +49,8 @@ use crate::hypervisor::fpu::FP_CONTROL_WORD_DEFAULT; use crate::hypervisor::wrappers::WHvGeneralRegisters; use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags}; use crate::mem::ptr::{GuestPtr, RawPtr}; +#[cfg(crashdump)] +use crate::sandbox::uninitialized::SandboxRuntimeConfig; use crate::{Result, debug, new_error}; /// A Hypervisor driver for HyperV-on-Windows. @@ -59,6 +63,8 @@ pub(crate) struct HypervWindowsDriver { orig_rsp: GuestPtr, mem_regions: Vec, interrupt_handle: Arc, + #[cfg(crashdump)] + rt_cfg: SandboxRuntimeConfig, } /* This does not automatically impl Send/Sync because the host * address of the shared memory region is a raw pointer, which are @@ -69,6 +75,7 @@ unsafe impl Send for HypervWindowsDriver {} unsafe impl Sync for HypervWindowsDriver {} impl HypervWindowsDriver { + #[allow(clippy::too_many_arguments)] #[instrument(err(Debug), skip_all, parent = Span::current(), level = "Trace")] pub(crate) fn new( mem_regions: Vec, @@ -78,6 +85,7 @@ impl HypervWindowsDriver { entrypoint: u64, rsp: u64, mmap_file_handle: HandleWrapper, + #[cfg(crashdump)] rt_cfg: SandboxRuntimeConfig, ) -> Result { // create and setup hypervisor partition let mut partition = VMPartition::new(1)?; @@ -112,6 +120,8 @@ impl HypervWindowsDriver { partition_handle, dropped: AtomicBool::new(false), }), + #[cfg(crashdump)] + rt_cfg, }) } @@ -514,8 +524,61 @@ impl Hypervisor for HypervWindowsDriver { } #[cfg(crashdump)] - fn get_memory_regions(&self) -> &[MemoryRegion] { - &self.mem_regions + fn crashdump_context(&self) -> Result> { + if self.rt_cfg.guest_core_dump { + let mut regs = [0; 27]; + + let vcpu_regs = self.processor.get_regs()?; + let sregs = self.processor.get_sregs()?; + let xsave = self.processor.get_xsave()?; + + // Set the registers in the order expected by the crashdump context + regs[0] = vcpu_regs.r15; // r15 + regs[1] = vcpu_regs.r14; // r14 + regs[2] = vcpu_regs.r13; // r13 + regs[3] = vcpu_regs.r12; // r12 + regs[4] = vcpu_regs.rbp; // rbp + regs[5] = vcpu_regs.rbx; // rbx + regs[6] = vcpu_regs.r11; // r11 + regs[7] = vcpu_regs.r10; // r10 + regs[8] = vcpu_regs.r9; // r9 + regs[9] = vcpu_regs.r8; // r8 + regs[10] = vcpu_regs.rax; // rax + regs[11] = vcpu_regs.rcx; // rcx + regs[12] = vcpu_regs.rdx; // rdx + regs[13] = vcpu_regs.rsi; // rsi + regs[14] = vcpu_regs.rdi; // rdi + regs[15] = 0; // orig rax + regs[16] = vcpu_regs.rip; // rip + regs[17] = unsafe { sregs.cs.Segment.Selector } as u64; // cs + regs[18] = vcpu_regs.rflags; // eflags + regs[19] = vcpu_regs.rsp; // rsp + regs[20] = unsafe { sregs.ss.Segment.Selector } as u64; // ss + regs[21] = unsafe { sregs.fs.Segment.Base }; // fs_base + regs[22] = unsafe { sregs.gs.Segment.Base }; // gs_base + regs[23] = unsafe { sregs.ds.Segment.Selector } as u64; // ds + regs[24] = unsafe { sregs.es.Segment.Selector } as u64; // es + regs[25] = unsafe { sregs.fs.Segment.Selector } as u64; // fs + regs[26] = unsafe { sregs.gs.Segment.Selector } as u64; // gs + + // Get the filename from the config + let filename = self.rt_cfg.binary_path.clone().and_then(|path| { + Path::new(&path) + .file_name() + .and_then(|name| name.to_os_string().into_string().ok()) + }); + + Ok(Some(crashdump::CrashDumpContext::new( + &self.mem_regions, + regs, + xsave, + self.entrypoint, + self.rt_cfg.binary_path.clone(), + filename, + ))) + } else { + Ok(None) + } } } diff --git a/src/hyperlight_host/src/hypervisor/kvm.rs b/src/hyperlight_host/src/hypervisor/kvm.rs index bf537fdb6..03f03aeaa 100644 --- a/src/hyperlight_host/src/hypervisor/kvm.rs +++ b/src/hyperlight_host/src/hypervisor/kvm.rs @@ -26,6 +26,8 @@ use kvm_ioctls::Cap::UserMemory; use kvm_ioctls::{Kvm, VcpuExit, VcpuFd, VmFd}; use log::LevelFilter; use tracing::{Span, instrument}; +#[cfg(crashdump)] +use {super::crashdump, std::path::Path}; use super::fpu::{FP_CONTROL_WORD_DEFAULT, FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT}; #[cfg(gdb)] @@ -43,6 +45,8 @@ use crate::HyperlightError; use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags}; use crate::mem::ptr::{GuestPtr, RawPtr}; use crate::sandbox::SandboxConfiguration; +#[cfg(crashdump)] +use crate::sandbox::uninitialized::SandboxRuntimeConfig; use crate::{Result, log_then_return, new_error}; /// Return `true` if the KVM API is available, version 12, and has UserMemory capability, or `false` otherwise @@ -290,6 +294,8 @@ pub(crate) struct KVMDriver { debug: Option, #[cfg(gdb)] gdb_conn: Option>, + #[cfg(crashdump)] + rt_cfg: SandboxRuntimeConfig, } impl KVMDriver { @@ -304,6 +310,7 @@ impl KVMDriver { rsp: u64, config: &SandboxConfiguration, #[cfg(gdb)] gdb_conn: Option>, + #[cfg(crashdump)] rt_cfg: SandboxRuntimeConfig, ) -> Result { let kvm = Kvm::new()?; @@ -363,6 +370,8 @@ impl KVMDriver { debug, #[cfg(gdb)] gdb_conn, + #[cfg(crashdump)] + rt_cfg, }; Ok(ret) } @@ -658,8 +667,67 @@ impl Hypervisor for KVMDriver { } #[cfg(crashdump)] - fn get_memory_regions(&self) -> &[MemoryRegion] { - &self.mem_regions + fn crashdump_context(&self) -> Result> { + if self.rt_cfg.guest_core_dump { + let mut regs = [0; 27]; + + let vcpu_regs = self.vcpu_fd.get_regs()?; + let sregs = self.vcpu_fd.get_sregs()?; + let xsave = self.vcpu_fd.get_xsave()?; + + // Set the registers in the order expected by the crashdump context + regs[0] = vcpu_regs.r15; // r15 + regs[1] = vcpu_regs.r14; // r14 + regs[2] = vcpu_regs.r13; // r13 + regs[3] = vcpu_regs.r12; // r12 + regs[4] = vcpu_regs.rbp; // rbp + regs[5] = vcpu_regs.rbx; // rbx + regs[6] = vcpu_regs.r11; // r11 + regs[7] = vcpu_regs.r10; // r10 + regs[8] = vcpu_regs.r9; // r9 + regs[9] = vcpu_regs.r8; // r8 + regs[10] = vcpu_regs.rax; // rax + regs[11] = vcpu_regs.rcx; // rcx + regs[12] = vcpu_regs.rdx; // rdx + regs[13] = vcpu_regs.rsi; // rsi + regs[14] = vcpu_regs.rdi; // rdi + regs[15] = 0; // orig rax + regs[16] = vcpu_regs.rip; // rip + regs[17] = sregs.cs.selector as u64; // cs + regs[18] = vcpu_regs.rflags; // eflags + regs[19] = vcpu_regs.rsp; // rsp + regs[20] = sregs.ss.selector as u64; // ss + regs[21] = sregs.fs.base; // fs_base + regs[22] = sregs.gs.base; // gs_base + regs[23] = sregs.ds.selector as u64; // ds + regs[24] = sregs.es.selector as u64; // es + regs[25] = sregs.fs.selector as u64; // fs + regs[26] = sregs.gs.selector as u64; // gs + + // Get the filename from the runtime config + let filename = self.rt_cfg.binary_path.clone().and_then(|path| { + Path::new(&path) + .file_name() + .and_then(|name| name.to_os_string().into_string().ok()) + }); + + // The [`CrashDumpContext`] accepts xsave as a vector of u8, so we need to convert the + // xsave region to a vector of u8 + Ok(Some(crashdump::CrashDumpContext::new( + &self.mem_regions, + regs, + xsave + .region + .iter() + .flat_map(|item| item.to_le_bytes()) + .collect::>(), + self.entrypoint, + self.rt_cfg.binary_path.clone(), + filename, + ))) + } else { + Ok(None) + } } #[cfg(gdb)] diff --git a/src/hyperlight_host/src/hypervisor/mod.rs b/src/hyperlight_host/src/hypervisor/mod.rs index a7ca4761f..bc6baa7cf 100644 --- a/src/hyperlight_host/src/hypervisor/mod.rs +++ b/src/hyperlight_host/src/hypervisor/mod.rs @@ -227,7 +227,7 @@ pub(crate) trait Hypervisor: Debug + Sync + Send { fn as_mut_hypervisor(&mut self) -> &mut dyn Hypervisor; #[cfg(crashdump)] - fn get_memory_regions(&self) -> &[MemoryRegion]; + fn crashdump_context(&self) -> Result>; #[cfg(gdb)] /// handles the cases when the vCPU stops due to a Debug event @@ -269,7 +269,7 @@ impl VirtualCPU { } Ok(HyperlightExit::Mmio(addr)) => { #[cfg(crashdump)] - crashdump::crashdump_to_tempfile(hv)?; + crashdump::generate_crashdump(hv)?; mem_access_fn .clone() @@ -281,7 +281,7 @@ impl VirtualCPU { } Ok(HyperlightExit::AccessViolation(addr, tried, region_permission)) => { #[cfg(crashdump)] - crashdump::crashdump_to_tempfile(hv)?; + crashdump::generate_crashdump(hv)?; if region_permission.intersects(MemoryRegionFlags::STACK_GUARD) { return Err(HyperlightError::StackOverflow()); @@ -300,14 +300,14 @@ impl VirtualCPU { } Ok(HyperlightExit::Unknown(reason)) => { #[cfg(crashdump)] - crashdump::crashdump_to_tempfile(hv)?; + crashdump::generate_crashdump(hv)?; log_then_return!("Unexpected VM Exit {:?}", reason); } Ok(HyperlightExit::Retry()) => continue, Err(e) => { #[cfg(crashdump)] - crashdump::crashdump_to_tempfile(hv)?; + crashdump::generate_crashdump(hv)?; return Err(e); } @@ -455,6 +455,8 @@ pub(crate) mod tests { use crate::hypervisor::DbgMemAccessHandlerCaller; use crate::mem::ptr::RawPtr; use crate::sandbox::uninitialized::GuestBinary; + #[cfg(any(crashdump, gdb))] + use crate::sandbox::uninitialized::SandboxRuntimeConfig; use crate::sandbox::uninitialized_evolve::set_up_hypervisor_partition; use crate::sandbox::{SandboxConfiguration, UninitializedSandbox}; use crate::{Result, is_hypervisor_present, new_error}; @@ -498,10 +500,17 @@ pub(crate) mod tests { let filename = dummy_guest_as_string().map_err(|e| new_error!("{}", e))?; let config: SandboxConfiguration = Default::default(); + #[cfg(any(crashdump, gdb))] + let rt_cfg: SandboxRuntimeConfig = Default::default(); let sandbox = UninitializedSandbox::new(GuestBinary::FilePath(filename.clone()), Some(config))?; let (_hshm, mut gshm) = sandbox.mgr.build(); - let mut vm = set_up_hypervisor_partition(&mut gshm, &config)?; + let mut vm = set_up_hypervisor_partition( + &mut gshm, + &config, + #[cfg(any(crashdump, gdb))] + &rt_cfg, + )?; vm.initialise( RawPtr::from(0x230000), 1234567890, diff --git a/src/hyperlight_host/src/hypervisor/windows_hypervisor_platform.rs b/src/hyperlight_host/src/hypervisor/windows_hypervisor_platform.rs index bb1877b4c..3ea9d61d8 100644 --- a/src/hyperlight_host/src/hypervisor/windows_hypervisor_platform.rs +++ b/src/hyperlight_host/src/hypervisor/windows_hypervisor_platform.rs @@ -26,7 +26,7 @@ use windows_result::HRESULT; use super::wrappers::HandleWrapper; use crate::hypervisor::wrappers::{WHvFPURegisters, WHvGeneralRegisters, WHvSpecialRegisters}; use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags}; -use crate::{Result, new_error}; +use crate::{HyperlightError, Result, new_error}; /// Interop calls for Windows Hypervisor Platform APIs /// @@ -409,6 +409,59 @@ impl VMProcessor { } } + #[cfg(crashdump)] + #[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")] + pub(super) fn get_xsave(&self) -> Result> { + // Get the required buffer size by calling with NULL buffer + let mut buffer_size_needed: u32 = 0; + + unsafe { + // First call with NULL buffer to get required size + // If the buffer is not large enough, the return value is WHV_E_INSUFFICIENT_BUFFER. + // In this case, BytesWritten receives the required buffer size. + let result = WHvGetVirtualProcessorXsaveState( + self.get_partition_hdl(), + 0, + std::ptr::null_mut(), + 0, + &mut buffer_size_needed, + ); + + // If it failed for reasons other than insufficient buffer, return error + if let Err(e) = result { + if e.code() != windows::Win32::Foundation::WHV_E_INSUFFICIENT_BUFFER { + return Err(HyperlightError::WindowsAPIError(e)); + } + } + } + + // Create a buffer with the appropriate size + let mut xsave_buffer = vec![0; buffer_size_needed as usize]; + + // Get the Xsave state + let mut written_bytes = 0; + unsafe { + WHvGetVirtualProcessorXsaveState( + self.get_partition_hdl(), + 0, + xsave_buffer.as_mut_ptr() as *mut std::ffi::c_void, + buffer_size_needed, + &mut written_bytes, + ) + }?; + + // Check if the number of written bytes matches the expected size + if written_bytes != buffer_size_needed { + return Err(new_error!( + "Failed to get Xsave state: expected {} bytes, got {}", + buffer_size_needed, + written_bytes + )); + } + + Ok(xsave_buffer) + } + pub(super) fn set_fpu(&mut self, regs: &WHvFPURegisters) -> Result<()> { const LEN: usize = 26; diff --git a/src/hyperlight_host/src/sandbox/config.rs b/src/hyperlight_host/src/sandbox/config.rs index f0725be12..f535b385d 100644 --- a/src/hyperlight_host/src/sandbox/config.rs +++ b/src/hyperlight_host/src/sandbox/config.rs @@ -35,6 +35,14 @@ pub struct DebugInfo { #[derive(Copy, Clone, Debug, Eq, PartialEq)] #[repr(C)] pub struct SandboxConfiguration { + /// Guest core dump output directory + /// This field is by default set to true which means the value core dumps will be placed in: + /// - HYPERLIGHT_CORE_DUMP_DIR environment variable if it is set + /// - default value of the temporary directory + /// + /// The core dump files generation can be disabled by setting this field to false. + #[cfg(crashdump)] + guest_core_dump: bool, /// Guest gdb debug port #[cfg(gdb)] guest_debug_info: Option, @@ -100,6 +108,7 @@ impl SandboxConfiguration { interrupt_retry_delay: Duration, interrupt_vcpu_sigrtmin_offset: u8, #[cfg(gdb)] guest_debug_info: Option, + #[cfg(crashdump)] guest_core_dump: bool, ) -> Self { Self { input_data_size: max(input_data_size, Self::MIN_INPUT_SIZE), @@ -110,6 +119,8 @@ impl SandboxConfiguration { interrupt_vcpu_sigrtmin_offset, #[cfg(gdb)] guest_debug_info, + #[cfg(crashdump)] + guest_core_dump, } } @@ -174,6 +185,15 @@ impl SandboxConfiguration { Ok(()) } + /// Toggles the guest core dump generation for a sandbox + /// Setting this to false disables the core dump generation + /// This is only used when the `crashdump` feature is enabled + #[cfg(crashdump)] + #[instrument(skip_all, parent = Span::current(), level= "Trace")] + pub fn set_guest_core_dump(&mut self, enable: bool) { + self.guest_core_dump = enable; + } + /// Sets the configuration for the guest debug #[cfg(gdb)] #[instrument(skip_all, parent = Span::current(), level= "Trace")] @@ -191,6 +211,12 @@ impl SandboxConfiguration { self.output_data_size } + #[cfg(crashdump)] + #[instrument(skip_all, parent = Span::current(), level= "Trace")] + pub(crate) fn get_guest_core_dump(&self) -> bool { + self.guest_core_dump + } + #[cfg(gdb)] #[instrument(skip_all, parent = Span::current(), level= "Trace")] pub(crate) fn get_guest_debug_info(&self) -> Option { @@ -236,6 +262,8 @@ impl Default for SandboxConfiguration { Self::INTERRUPT_VCPU_SIGRTMIN_OFFSET, #[cfg(gdb)] None, + #[cfg(crashdump)] + true, ) } } @@ -260,6 +288,8 @@ mod tests { SandboxConfiguration::INTERRUPT_VCPU_SIGRTMIN_OFFSET, #[cfg(gdb)] None, + #[cfg(crashdump)] + true, ); let exe_info = simple_guest_exe_info().unwrap(); @@ -287,6 +317,8 @@ mod tests { SandboxConfiguration::INTERRUPT_VCPU_SIGRTMIN_OFFSET, #[cfg(gdb)] None, + #[cfg(crashdump)] + true, ); assert_eq!(SandboxConfiguration::MIN_INPUT_SIZE, cfg.input_data_size); assert_eq!(SandboxConfiguration::MIN_OUTPUT_SIZE, cfg.output_data_size); diff --git a/src/hyperlight_host/src/sandbox/uninitialized.rs b/src/hyperlight_host/src/sandbox/uninitialized.rs index 16df2bb05..624d894d8 100644 --- a/src/hyperlight_host/src/sandbox/uninitialized.rs +++ b/src/hyperlight_host/src/sandbox/uninitialized.rs @@ -52,6 +52,17 @@ const EXTRA_ALLOWED_SYSCALLS_FOR_WRITER_FUNC: &[super::ExtraAllowedSyscall] = &[ libc::SYS_close, ]; +#[cfg(any(crashdump, gdb))] +#[derive(Clone, Debug, Default)] +pub(crate) struct SandboxRuntimeConfig { + #[cfg(crashdump)] + pub(crate) binary_path: Option, + #[cfg(gdb)] + pub(crate) debug_info: Option, + #[cfg(crashdump)] + pub(crate) guest_core_dump: bool, +} + /// A preliminary `Sandbox`, not yet ready to execute guest code. /// /// Prior to initializing a full-fledged `Sandbox`, you must create one of @@ -66,6 +77,8 @@ pub struct UninitializedSandbox { pub(crate) mgr: MemMgrWrapper, pub(crate) max_guest_log_level: Option, pub(crate) config: SandboxConfiguration, + #[cfg(any(crashdump, gdb))] + pub(crate) rt_cfg: SandboxRuntimeConfig, } impl crate::sandbox_state::sandbox::UninitializedSandbox for UninitializedSandbox { @@ -145,18 +158,43 @@ impl UninitializedSandbox { GuestBinary::FilePath(binary_path) => { let path = Path::new(&binary_path) .canonicalize() - .map_err(|e| new_error!("GuestBinary not found: '{}': {}", binary_path, e))?; - GuestBinary::FilePath( - path.into_os_string() - .into_string() - .map_err(|e| new_error!("Error converting OsString to String: {:?}", e))?, - ) + .map_err(|e| new_error!("GuestBinary not found: '{}': {}", binary_path, e))? + .into_os_string() + .into_string() + .map_err(|e| new_error!("Error converting OsString to String: {:?}", e))?; + + GuestBinary::FilePath(path) } buffer @ GuestBinary::Buffer(_) => buffer, }; let sandbox_cfg = cfg.unwrap_or_default(); + #[cfg(any(crashdump, gdb))] + let rt_cfg = { + #[cfg(crashdump)] + let guest_core_dump = sandbox_cfg.get_guest_core_dump(); + + #[cfg(gdb)] + let debug_info = sandbox_cfg.get_guest_debug_info(); + + #[cfg(crashdump)] + let binary_path = if let GuestBinary::FilePath(ref path) = guest_binary { + Some(path.clone()) + } else { + None + }; + + SandboxRuntimeConfig { + #[cfg(crashdump)] + binary_path, + #[cfg(gdb)] + debug_info, + #[cfg(crashdump)] + guest_core_dump, + } + }; + let mut mem_mgr_wrapper = { let mut mgr = UninitializedSandbox::load_guest_binary(sandbox_cfg, &guest_binary)?; let stack_guard = Self::create_stack_guard(); @@ -173,6 +211,8 @@ impl UninitializedSandbox { mgr: mem_mgr_wrapper, max_guest_log_level: None, config: sandbox_cfg, + #[cfg(any(crashdump, gdb))] + rt_cfg, }; // If we were passed a writer for host print register it otherwise use the default. diff --git a/src/hyperlight_host/src/sandbox/uninitialized_evolve.rs b/src/hyperlight_host/src/sandbox/uninitialized_evolve.rs index bdcf1a792..9bda0cb73 100644 --- a/src/hyperlight_host/src/sandbox/uninitialized_evolve.rs +++ b/src/hyperlight_host/src/sandbox/uninitialized_evolve.rs @@ -23,6 +23,8 @@ use super::SandboxConfiguration; use super::hypervisor::{HypervisorType, get_available_hypervisor}; #[cfg(gdb)] use super::mem_access::dbg_mem_access_handler_wrapper; +#[cfg(any(crashdump, gdb))] +use super::uninitialized::SandboxRuntimeConfig; use crate::HyperlightError::NoHypervisorFound; use crate::hypervisor::Hypervisor; use crate::hypervisor::handlers::{MemAccessHandlerCaller, OutBHandlerCaller}; @@ -69,7 +71,12 @@ where ) -> Result, { let (hshm, mut gshm) = u_sbox.mgr.build(); - let mut vm = set_up_hypervisor_partition(&mut gshm, &u_sbox.config)?; + let mut vm = set_up_hypervisor_partition( + &mut gshm, + &u_sbox.config, + #[cfg(any(crashdump, gdb))] + &u_sbox.rt_cfg, + )?; let outb_hdl = outb_handler_wrapper(hshm.clone(), u_sbox.host_funcs.clone()); let seed = { @@ -141,6 +148,7 @@ pub(super) fn evolve_impl_multi_use(u_sbox: UninitializedSandbox) -> Result, #[cfg_attr(target_os = "windows", allow(unused_variables))] config: &SandboxConfiguration, + #[cfg(any(crashdump, gdb))] rt_cfg: &SandboxRuntimeConfig, ) -> Result> { let mem_size = u64::try_from(mgr.shared_mem.mem_size())?; let mut regions = mgr.layout.get_memory_regions(&mgr.shared_mem)?; @@ -176,7 +184,7 @@ pub(crate) fn set_up_hypervisor_partition( // Create gdb thread if gdb is enabled and the configuration is provided #[cfg(gdb)] - let gdb_conn = if let Some(DebugInfo { port }) = config.get_guest_debug_info() { + let gdb_conn = if let Some(DebugInfo { port }) = rt_cfg.debug_info { use crate::hypervisor::gdb::create_gdb_thread; let gdb_conn = create_gdb_thread(port, unsafe { libc::pthread_self() }); @@ -206,6 +214,8 @@ pub(crate) fn set_up_hypervisor_partition( config, #[cfg(gdb)] gdb_conn, + #[cfg(crashdump)] + rt_cfg.clone(), )?; Ok(Box::new(hv)) } @@ -220,6 +230,8 @@ pub(crate) fn set_up_hypervisor_partition( config, #[cfg(gdb)] gdb_conn, + #[cfg(crashdump)] + rt_cfg.clone(), )?; Ok(Box::new(hv)) } @@ -241,6 +253,8 @@ pub(crate) fn set_up_hypervisor_partition( entrypoint_ptr.absolute()?, rsp_ptr.absolute()?, HandleWrapper::from(mmap_file_handle), + #[cfg(crashdump)] + rt_cfg.clone(), )?; Ok(Box::new(hv)) } diff --git a/src/tests/rust_guests/witguest/Cargo.lock b/src/tests/rust_guests/witguest/Cargo.lock index 573a444ee..6a1aef84a 100644 --- a/src/tests/rust_guests/witguest/Cargo.lock +++ b/src/tests/rust_guests/witguest/Cargo.lock @@ -347,9 +347,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.94" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" dependencies = [ "unicode-ident", ] @@ -477,9 +477,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.100" +version = "2.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" +checksum = "f6397daf94fa90f058bd0fd88429dd9e5738999cca8d701813c80723add80462" dependencies = [ "proc-macro2", "quote",