Skip to content

Commit c40d67c

Browse files
JamesC1305JamesC1305
committed
Create performance tests for hotplug latency
Create performance tests that measure the latency of hotplugging using both a udev rule and an a userspace agent to online vCPUs. Signed-off-by: James Curtis <[email protected]>
1 parent 2029a08 commit c40d67c

File tree

7 files changed

+247
-0
lines changed

7 files changed

+247
-0
lines changed

‎tests/conftest.py

+3
Original file line numberDiff line numberDiff line change
@@ -377,6 +377,9 @@ def rootfs_fxt(request, record_property):
377377
guest_kernel_linux_5_10 = pytest.fixture(
378378
guest_kernel_fxt, params=kernel_params("vmlinux-5.10*")
379379
)
380+
guest_kernel_linux_acpi_only = pytest.fixture(
381+
guest_kernel_fxt, params=kernel_params("vmlinux-5.10.219")
382+
)
380383
# Use the unfiltered selector, since we don't officially support 6.1 yet.
381384
# TODO: switch to default selector once we add full 6.1 support.
382385
guest_kernel_linux_6_1 = pytest.fixture(

‎tests/host_tools/1-cpu-hotplug.rules

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
SUBSYSTEM=="cpu", ACTION=="add", ATTR{online}!="1", ATTR{online}="1"

‎tests/host_tools/hotplug.sh

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/bin/bash
2+
# Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
while :; do
6+
[[ -d /sys/devices/system/cpu/cpu$1 ]] && break
7+
done
8+
9+
for i in $(seq 1 $1); do
10+
echo 1 >/sys/devices/system/cpu/cpu$i/online
11+
done
12+
13+
while :; do
14+
[[ $(nproc) == $((1 + $1)) ]] && break
15+
done
16+
17+
/home/hotplug_time.o

‎tests/host_tools/hotplug_time.c

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
// Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
// Init wrapper for boot timing. It points at /sbin/init.
5+
6+
#include <fcntl.h>
7+
#include <sys/mman.h>
8+
#include <sys/types.h>
9+
#include <unistd.h>
10+
11+
// Base address values are defined in arch/src/lib.rs as arch::MMIO_MEM_START.
12+
// Values are computed in arch/src/<arch>/mod.rs from the architecture layouts.
13+
// Position on the bus is defined by MMIO_LEN increments, where MMIO_LEN is
14+
// defined as 0x1000 in vmm/src/device_manager/mmio.rs.
15+
#ifdef __x86_64__
16+
#define MAGIC_MMIO_SIGNAL_GUEST_BOOT_COMPLETE 0xd0000000
17+
#endif
18+
#ifdef __aarch64__
19+
#define MAGIC_MMIO_SIGNAL_GUEST_BOOT_COMPLETE 0x40000000
20+
#endif
21+
22+
#define MAGIC_VALUE_SIGNAL_GUEST_BOOT_COMPLETE 123
23+
24+
int main() {
25+
int fd = open("/dev/mem", (O_RDWR | O_SYNC | O_CLOEXEC));
26+
int mapped_size = getpagesize();
27+
28+
char *map_base = mmap(NULL, mapped_size, PROT_WRITE, MAP_SHARED, fd,
29+
MAGIC_MMIO_SIGNAL_GUEST_BOOT_COMPLETE);
30+
31+
*map_base = MAGIC_VALUE_SIGNAL_GUEST_BOOT_COMPLETE;
32+
msync(map_base, mapped_size, MS_ASYNC);
33+
}

‎tests/host_tools/hotplug_time.o

879 KB
Binary file not shown.

‎tests/host_tools/hotplug_udev.sh

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/bash
2+
# Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
while :; do
6+
[[ $(nproc) == $((1 + $1)) ]] && break
7+
done
8+
9+
/home/hotplug_time.o

‎tests/integration_tests/performance/test_vcpu_hotplug.py

+184
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
# Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
"""Testing hotplug performance"""
5+
6+
import platform
7+
import re
8+
import time
9+
from pathlib import Path
10+
11+
import pandas
12+
import pytest
13+
14+
from framework.utils_cpuid import check_guest_cpuid_output
15+
from host_tools.cargo_build import gcc_compile
16+
17+
18+
@pytest.mark.nonci
19+
@pytest.mark.skipif(
20+
platform.machine() != "x86_64", reason="Hotplug only enabled on x86_64."
21+
)
22+
@pytest.mark.parametrize(
23+
"vcpu_count", [2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]
24+
)
25+
def test_custom_udev_rule_latency(
26+
microvm_factory, guest_kernel_linux_6_1, rootfs_rw, vcpu_count, results_dir
27+
):
28+
"""Test the latency for hotplugging and booting CPUs in the guest"""
29+
gcc_compile(Path("./host_tools/hotplug_time.c"), Path("host_tools/hotplug_time.o"))
30+
data = []
31+
for _ in range(20):
32+
uvm_hotplug = microvm_factory.build(guest_kernel_linux_6_1, rootfs_rw)
33+
uvm_hotplug.jailer.extra_args.update({"boot-timer": None, "no-seccomp": None})
34+
uvm_hotplug.help.enable_console()
35+
uvm_hotplug.spawn()
36+
uvm_hotplug.basic_config(vcpu_count=1, mem_size_mib=128)
37+
uvm_hotplug.add_net_iface()
38+
uvm_hotplug.start()
39+
uvm_hotplug.ssh.scp_put(
40+
Path("./host_tools/hotplug_udev.sh"), Path("/home/hotplug_udev.sh")
41+
)
42+
uvm_hotplug.ssh.scp_put(
43+
Path("./host_tools/hotplug_time.o"), Path("/home/hotplug_time.o")
44+
)
45+
uvm_hotplug.ssh.scp_put(
46+
Path("./host_tools/1-cpu-hotplug.rules"),
47+
Path("/usr/lib/udev/rules.d/1-cpu-hotplug.rules"),
48+
)
49+
uvm_hotplug.ssh.run(
50+
f"udevadm control --reload-rules && tmux new-session -d /bin/bash /home/hotplug_udev.sh {vcpu_count}"
51+
)
52+
53+
uvm_hotplug.api.hotplug.put(Vcpu={"add": vcpu_count})
54+
time.sleep(5)
55+
56+
# Extract API call duration
57+
api_duration = (
58+
float(
59+
re.findall(
60+
r"Total previous API call duration: (\d+) us\.",
61+
uvm_hotplug.log_data,
62+
)[-1]
63+
)
64+
/ 1000
65+
)
66+
try:
67+
timestamp = (
68+
float(
69+
re.findall(
70+
r"Guest-boot-time\s+\=\s+(\d+)\s+us", uvm_hotplug.log_data
71+
)[0]
72+
)
73+
/ 1000
74+
)
75+
except IndexError:
76+
uvm_hotplug.kill()
77+
data.append({"vcpus": vcpu_count, "api": api_duration, "onlining": None})
78+
continue
79+
80+
data.append({"vcpus": vcpu_count, "api": api_duration, "onlining": timestamp})
81+
82+
check_guest_cpuid_output(
83+
uvm_hotplug,
84+
"lscpu",
85+
None,
86+
":",
87+
{
88+
"CPU(s)": str(1 + vcpu_count),
89+
"On-line CPU(s) list": f"0-{vcpu_count}",
90+
},
91+
)
92+
uvm_hotplug.kill()
93+
94+
output_file = results_dir / f"hotplug-{vcpu_count}.csv"
95+
96+
csv_data = pandas.DataFrame.from_dict(data).to_csv(
97+
index=False,
98+
float_format="%.3f",
99+
)
100+
101+
output_file.write_text(csv_data)
102+
103+
104+
@pytest.mark.nonci
105+
@pytest.mark.skipif(
106+
platform.machine() != "x86_64", reason="Hotplug only enabled on x86_64."
107+
)
108+
@pytest.mark.parametrize(
109+
"vcpu_count", [2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]
110+
)
111+
def test_manual_latency(
112+
microvm_factory, guest_kernel_linux_6_1, rootfs_rw, vcpu_count, results_dir
113+
):
114+
"""Test the latency for hotplugging and booting CPUs in the guest"""
115+
gcc_compile(Path("./host_tools/hotplug_time.c"), Path("host_tools/hotplug_time.o"))
116+
data = []
117+
for _ in range(20):
118+
uvm_hotplug = microvm_factory.build(guest_kernel_linux_6_1, rootfs_rw)
119+
uvm_hotplug.jailer.extra_args.update({"boot-timer": None, "no-seccomp": None})
120+
uvm_hotplug.help.enable_console()
121+
uvm_hotplug.spawn()
122+
uvm_hotplug.basic_config(vcpu_count=1, mem_size_mib=128)
123+
uvm_hotplug.add_net_iface()
124+
uvm_hotplug.start()
125+
uvm_hotplug.ssh.scp_put(
126+
Path("./host_tools/hotplug.sh"), Path("/home/hotplug.sh")
127+
)
128+
uvm_hotplug.ssh.scp_put(
129+
Path("./host_tools/hotplug_time.o"), Path("/home/hotplug_time.o")
130+
)
131+
uvm_hotplug.ssh.run(
132+
f"tmux new-session -d /bin/bash /home/hotplug.sh {vcpu_count}"
133+
)
134+
135+
uvm_hotplug.api.hotplug.put(Vcpu={"add": vcpu_count})
136+
137+
time.sleep(5)
138+
# Extract API call duration
139+
api_duration = (
140+
float(
141+
re.findall(
142+
r"Total previous API call duration: (\d+) us\.",
143+
uvm_hotplug.log_data,
144+
)[-1]
145+
)
146+
/ 1000
147+
)
148+
try:
149+
timestamp = (
150+
float(
151+
re.findall(
152+
r"Guest-boot-time\s+\=\s+(\d+)\s+us", uvm_hotplug.log_data
153+
)[0]
154+
)
155+
/ 1000
156+
)
157+
except IndexError:
158+
data.append({"vcpus": vcpu_count, "api": api_duration, "onlining": None})
159+
uvm_hotplug.kill()
160+
continue
161+
162+
data.append({"vcpus": vcpu_count, "api": api_duration, "onlining": timestamp})
163+
164+
check_guest_cpuid_output(
165+
uvm_hotplug,
166+
"lscpu",
167+
None,
168+
":",
169+
{
170+
"CPU(s)": str(1 + vcpu_count),
171+
"On-line CPU(s) list": f"0-{vcpu_count}",
172+
},
173+
)
174+
175+
uvm_hotplug.kill()
176+
177+
output_file = results_dir / f"hotplug-{vcpu_count}.csv"
178+
179+
csv_data = pandas.DataFrame.from_dict(data).to_csv(
180+
index=False,
181+
float_format="%.3f",
182+
)
183+
184+
output_file.write_text(csv_data)

0 commit comments

Comments
 (0)