Add confidential guest support: Secure Execution support

Example to start a confidential guest using two host-key documents
```
$ vng -r --confidential-guest --confidential-guest-args
host-key-document=/home/mhartmay/storage/git/hostkeys/a46/HKD-3931-02772A8.crt
--confidential-guest-args
host-key-document=/home/mhartmay/storage/git/hostkeys/b35/HKD-9175-029DE48.crt
```

Another example where always the given `pvimg create` arguments are used. To do
 so, modify the
`default_opts` sections in `~/.config/virtme-ng/virtme-ng.conf` as following:
```
{
    "default_opts": {
        "confidential_guest_args": ["host-key-document=/home/user/HKD.crt"]
    },
}
```

Now you can simply run `vng --confidential-guest` to prepare the Secure
Execution boot image using the given host-key document.

Signed-off-by: Marc Hartmayer <mhartmay@linux.ibm.com>

Author: mhartmay

README.md

@@ -525,6 +525,61 @@ Examples
    $ sudo vng --nvgpu "01:00.0" -r linux
 ```
 
+ - Run virtme-ng with confidential guest support (currently only s390x support). **Use this only for test purposes!**
+
+   For [IBM Secure
+   Execution](https://www.ibm.com/docs/en/linux-on-systems?topic=management-secure-execution)
+   the guest boot image/kernel needs to be prepared using at least one so-called
+   host-key document. This document can be passed to the `pvimg create` command
+   by means of the `--confidential-guest-args` option.
+
+   ```
+   vng --confidential-guest --confidential_guest_args 'host-key-document=/home/user/HKD.crt'
+   ```
+
+   As this host-key document does not change that frequently, it makes sense to
+   make use of [`virtme-ng.conf`](#configuration). To do so, modify the
+   `default_opts` sections in `~/.config/virtme-ng/virtme-ng.conf` as following:
+
+   ```
+   {
+       "default_opts": {
+           "confidential_guest_args": ["host-key-document=/home/user/HKD.crt"]
+       },
+   }
+   ```
+
+   Now you can simply run `vng --confidential-guest` to prepare the boot image
+   and start a confidential guest using this newly created boot image.
+
+   + Run virtme-ng with confidential dump support (currently only s390x support)
+
+     ```
+     # Specify a confidential dump encryption key, e.g. via the virtme-ng.conf
+       {
+        "default_opts": {
+           "confidential_guest_args": ["host-key-document=/home/user/HKD.crt", "cck=/home/user/cck"]
+       }
+
+     # Start the vng instance in debug mode
+     $ vng --confidential-guest --debug
+
+     # In a separate shell session trigger the memory dump to /tmp/vmcore.img
+     $ vng --dump /tmp/vmcore.img
+
+     # Decrypt the dump on-the-fly (fusermount3 is needed)
+     $ zgetdump --mount /mnt/ --key /home/user/cck /tmp/vmcore.img
+
+     # Use drgn to read 'jiffies' from the memory dump:
+     $ echo "print(prog['jiffies'])" | drgn -q -s vmlinux -c /mnt/dump.elf
+     drgn 0.0.23 (using Python 3.11.6, elfutils 0.189, with libkdumpfile)
+     For help, type help(drgn).
+     >>> import drgn
+     >>> from drgn import NULL, Object, cast, container_of, execscript, offsetof, reinterpret, sizeof
+     >>> from drgn.helpers.common import *
+     >>> from drgn.helpers.linux import *
+     ```
+
 Implementation details
 ======================
 

virtme/architectures.py

@@ -5,8 +5,15 @@
 # as a file called LICENSE with SHA-256 hash:
 # 8177f97513213526df2cf6184d8ff986c675afb514d4e68a404010521b880643
 
+import functools
 import os
-from typing import List, Optional
+import shlex
+import shutil
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+from typing import Any, List, Optional
 
 
 class Arch:
@@ -71,6 +78,25 @@ def qemu_sound_args() -> List[str]:
     def qemu_vmcoreinfo_args() -> List[str]:
         return ["-device", "vmcoreinfo"]
 
+    @staticmethod
+    def qemu_confidential_guest_args(
+        is_native: bool, use_kvm: bool, **kwargs
+    ) -> List[str]:
+        raise RuntimeError("Confidential guest is not supported by the architecture")
+
+    # pylint: disable=unused-argument
+    def qemu_confidential_guest_img_preparation(
+        self,
+        *,
+        kernel: Path,
+        cmdline: str,
+        initrd: Optional[Path] = None,
+        dry_run: bool = False,
+        verbose: bool = False,
+        **kwargs,
+    ) -> Optional[Path]:
+        return None
+
     @staticmethod
     def qemu_serial_console_args() -> List[str]:
         # We should be using the new-style -device serialdev,chardev=xyz,
@@ -387,11 +413,24 @@ def qemu_nodisplay_args():
 
 
 class Arch_s390x(Arch):
+    PV_DUMP_SUPPORT = 0x8000000000
+
     def __init__(self):
         Arch.__init__(self, "s390x")
 
         self.linuxname = "s390"
 
+    @staticmethod
+    def test_uv_support(value: int) -> bool:
+        with open("/sys/firmware/uv/query/facilities", "rb") as f:
+            data = f.read().decode()
+
+        inst_call_list = data.splitlines()
+        assert len(inst_call_list) == 4
+        inst_call_list_int = list(map(functools.partial(int, base=16), inst_call_list))
+        first_inst_call = inst_call_list_int[0]
+        return first_inst_call & value
+
     @staticmethod
     def virtiofs_support() -> bool:
         return True
@@ -451,6 +490,85 @@ def earlyconsole_args() -> List[str]:
     def qemu_vmcoreinfo_args() -> List[str]:
         return []
 
+    @staticmethod
+    def qemu_confidential_guest_args(
+        is_native: bool, use_kvm: bool, **kwargs
+    ) -> List[str]:
+        try:
+            with open("/sys/firmware/uv/prot_virt_host", "rb") as f:
+                data = f.read().decode()
+        except FileNotFoundError:
+            return RuntimeError(
+                "Secure Execution is not supported by your kernel/hardware."
+            )
+        if int(data) != 1:
+            return RuntimeError(
+                "Secure Execution is not supported on your system. 'prot_virt=1' missing?"
+            )
+        if not is_native or not use_kvm:
+            return RuntimeError("KVM must be used for Secure Execution")
+
+        return [
+            "-object",
+            "s390-pv-guest,id=pv0",
+            "-machine",
+            "confidential-guest-support=pv0",
+        ]
+
+    def qemu_confidential_guest_img_preparation(
+        self,
+        *,
+        kernel: Path,
+        cmdline: str,
+        initrd: Optional[Path] = None,
+        dry_run: bool = False,
+        verbose: bool = False,
+        **kwargs,
+    ) -> Optional[Any]:
+        if "host-key-document" not in kwargs:
+            raise RuntimeError(
+                "virtme-run: At least one host-key-document=$HKD must be specified"
+            )
+        if not shutil.which("pvimg"):
+            raise RuntimeError(
+                "'pvimg' must be installed for the Secure Execution guest image preparation"
+            )
+
+        confidential_guest_args = [
+            f"--{key}={value}" for key, values in kwargs.items() for value in values
+        ]
+
+        output = tempfile.NamedTemporaryFile(suffix="seimg")
+        if self.test_uv_support(self.PV_DUMP_SUPPORT) and "cck" in kwargs:
+            if verbose:
+                sys.stderr.write(f"virtme: using CCK in '{kwargs.get('cck')[-1]}'\n")
+            confidential_guest_args += ["--enable-dump"]
+        prepare_cmd = [
+            "pvimg",
+            "create",
+            f"--kernel={kernel}",
+            f"--output={output.name}",
+            "--no-verify",
+            "--overwrite",
+        ] + confidential_guest_args
+        if initrd is not None:
+            prepare_cmd.append(f"--ramdisk={initrd}")
+        with tempfile.NamedTemporaryFile() as f:
+            f.write(cmdline.encode("ascii"))
+            f.flush()
+            prepare_cmd.append(f"--parmfile={f.name}")
+            prepare_cmd_str = shlex.join(prepare_cmd)
+            if dry_run:
+                print(prepare_cmd_str)
+            else:
+                subprocess.check_call(
+                    prepare_cmd,
+                    stderr=subprocess.PIPE,
+                    text=True,
+                    close_fds=False,
+                )
+        return output
+
     def img_name(self) -> List[str]:
         return ["vmlinuz", "image"]
 

virtme/commands/run.py

@@ -39,6 +39,23 @@
 from ..util import SilentError, find_binary_or_raise, get_username
 
 
+class kwargsAppendAction(argparse.Action):
+    def __call__(self, parser, namespace, pairs, option_string=None):
+        assert len(pairs) > 0
+        for pair in pairs:
+            try:
+                key, value = pair.split("=")
+            except ValueError as error:
+                raise argparse.ArgumentError(
+                    self, f'could not parse argument "{pair}" as key=value format'
+                ) from error
+            d = getattr(namespace, self.dest) or {}
+            values = d.get(key, [])
+            values.append(value)
+            d[key] = values
+            setattr(namespace, self.dest, d)
+
+
 def make_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(
         description="Virtualize your system (or another) under a kernel image",
@@ -299,6 +316,18 @@ def make_parser() -> argparse.ArgumentParser:
         action="store",
         help="Save the generated initramfs to the specified path",
     )
+    g.add_argument(
+        "--confidential-guest",
+        action="store_true",
+        help="Prepare image for confidential guest",
+    )
+    g.add_argument(
+        "--confidential-guest-args",
+        nargs="+",
+        action=kwargsAppendAction,
+        metavar="KEY=VALUE",
+        help="Add key/value params. May appear multiple times.",
+    )
     g.add_argument(
         "--show-boot-console",
         action="store_true",
@@ -1888,8 +1917,28 @@ def get_net_mac(index):
     # sure that 'init=' appears directly before '--'.
     kernelargs.extend(initcmds)
 
-    # Load a normal kernel
-    qemuargs.extend(["-kernel", kernel.kimg])
+    kernel_cmdline = " ".join(quote_karg(a) for a in kernelargs)
+    if args.confidential_guest:
+        qemuargs.extend(
+            arch.qemu_confidential_guest_args(is_native=is_native, use_kvm=kvm_ok)
+        )
+        output = arch.qemu_confidential_guest_img_preparation(
+            kernel=kernel.kimg,
+            cmdline=kernel_cmdline,
+            initrd=initrdpath,
+            dry_run=args.dry_run,
+            verbose=args.verbose,
+            **args.confidential_guest_args,
+        )
+        if output is not None:
+            kimg = output.name
+        else:
+            kimg = kernel.kimg
+    else:
+        kimg = kernel.kimg
+
+    # Load the kernel
+    qemuargs.extend(["-kernel", kimg])
     if kernelargs:
         if args.systemd:
             init_environment_vars = []
@@ -1913,7 +1962,7 @@ def get_net_mac(index):
                     f.write(f"\nExecStart={guest_tools_path}/{virtme_init_cmd}")
                 else:
                     f.write(f"\nExecStart=/run/virtme/guesttools/{virtme_init_cmd}")
-        qemuargs.extend(["-append", " ".join(quote_karg(a) for a in kernelargs)])
+        qemuargs.extend(["-append", kernel_cmdline])
     if initrdpath is not None:
         qemuargs.extend(["-initrd", initrdpath])
     if kernel.dtb is not None:

virtme_ng/run.py

@@ -410,6 +410,18 @@ def make_parser():
         action="store_true",
         help="Use an initramfs even if unnecessary",
     )
+    parser.add_argument(
+        "--confidential-guest",
+        action="store_true",
+        help="Prepare image for confidential guest",
+    )
+    parser.add_argument(
+        "--confidential-guest-args",
+        action="extend",
+        nargs="+",
+        type=str,
+        help="Confidential guest keyword arguments",
+    )
 
     parser.add_argument(
         "--sound",
@@ -1271,6 +1283,19 @@ def _get_virtme_nvgpu(self, args):
         else:
             self.virtme_param["nvgpu"] = ""
 
+    def _get_virtme_confidential_guest(self, args):
+        if args.confidential_guest:
+            if args.confidential_guest_args is None:
+                arg_fail(
+                    "error: --confidential-guest can be used only with --confidential-guest-args",
+                    show_usage=False,
+                )
+            self.virtme_param["confidential_guest"] = (
+                f"--confidential-guest --confidential-guest-args {shlex.join(args.confidential_guest_args)}"
+            )
+        else:
+            self.virtme_param["confidential_guest"] = ""
+
     def _get_virtme_qemu_opts(self, args):
         qemu_args = ""
         if args.debug:
@@ -1310,6 +1335,7 @@ def run(self, args):
         self._get_virtme_disk(args)
         self._get_virtme_sound(args)
         self._get_virtme_vmcoreinfo(args)
+        self._get_virtme_confidential_guest(args)
         self._get_virtme_disable_microvm(args)
         self._get_virtme_disable_monitor(args)
         self._get_virtme_disable_kvm(args)
@@ -1364,6 +1390,7 @@ def run(self, args):
             + f"{self.virtme_param['disable_kvm']} "
             + f"{self.virtme_param['ssh_tcp']} "
             + f"{self.virtme_param['force_9p']} "
+            + f"{self.virtme_param['confidential_guest']} "
             + f"{self.virtme_param['force_initramfs']} "
             + f"{self.virtme_param['graphics']} "
             + f"{self.virtme_param['verbose']} "