Skip to content

Commit 43f42a6

Browse files
elliotbermanelliotberman
committed
tests: oci: Use saxpy as the test
The current oci test requires downloading and building cuda-samples. This requires a very large container containing the CUDA runtime and development tools. Instead, build a simple CUDA binary (saxpy) separately and create a container with only the CUDA runtime and saxpy.
1 parent b6a89a7 commit 43f42a6

File tree

2 files changed

+130
-56
lines changed

2 files changed

+130
-56
lines changed

‎mk-overlay.nix‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -250,7 +250,7 @@ makeScope final.newScope (self: {
250250
inherit (finalSamples) callPackage;
251251
});
252252

253-
tests = final.callPackages ./pkgs/tests { inherit l4tMajorMinorPatchVersion l4tAtLeast; };
253+
tests = final.callPackages ./pkgs/tests { };
254254

255255
kernelPackagesOverlay = final: _:
256256
if self.l4tAtLeast "36" then {

‎pkgs/tests/default.nix‎

Lines changed: 129 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -1,66 +1,140 @@
1-
{ l4tMajorMinorPatchVersion
1+
{ cudaPackages
22
, dockerTools
3-
, writeShellScriptBin
3+
, go
44
, lib
5-
, l4tAtLeast
6-
, fetchFromGitHub
7-
, buildEnv
5+
, path
6+
, writeShellScriptBin
7+
, writeText
8+
, writeTextFile
89
}:
9-
# https://docs.nvidia.com/jetson/archives/r36.4.4/DeveloperGuide/SD/TestPlanValidation.html#nvidia-containers
1010
let
11-
l4tImage = {
12-
"35" = dockerTools.buildImage {
13-
name = "l4t-jetpack-with-samples";
11+
cudaBase = dockerTools.pullImage ({
12+
imageName = "nvidia/cuda";
13+
} // lib.getAttrFromPath [ cudaPackages.cudaMajorMinorVersion go.GOARCH ] {
14+
"13.0".amd64 = {
15+
finalImageTag = "13.0.2-base-ubuntu24.04";
16+
imageDigest = "sha256:605fb0c8acf8674e164d822da8a8521f3a655056e569f0899e72ae940e1fe7dc";
17+
sha256 = "";
18+
};
19+
"13.0".arm64 = {
20+
finalImageTag = "13.0.2-base-ubuntu24.04";
21+
imageDigest = "sha256:1ca86773be1716af6cfff3d2eb8cd10d4d9cac181931d1ee9be792d3e33c7322";
22+
sha256 = "sha256-IOtmpA2xiNL1/zodFdUz5/nrDYMBssDibwYE0uwrCcI= ";
23+
};
24+
"12.6".amd64 = {
25+
finalImageTag = "12.6.3-base-ubuntu24.04";
26+
imageDigest = "sha256:6b3201183858bad08441837f5a5efc2c75290135cc25fcc87d9ff763190cfd09";
27+
sha256 = "sha256-326qF7q4RtMnye/yOeh+lgilopoCrEJyV0WuTQlN9JM=";
28+
};
29+
"12.6".arm64 = {
30+
finalImageTag = "12.6.3-base-ubuntu24.04";
31+
imageDigest = "sha256:ba25600cea339517a8e55361a27535bbfdeae1936663e9b4970f3cec2fbbd165";
32+
sha256 = "sha256-lii+GS0havrQceqqYvHa35Lwe7I2rCQ3pVN5XvE2rmw=";
33+
};
34+
"11.8".amd64 = {
35+
finalImageTag = "11.8.0-base-ubuntu22.04";
36+
imageDigest = "sha256:79e5b2cf878ee9006f5b3738caeea34fdc7708a32db53fe3e80db0b48bd286a0";
37+
sha256 = "sha256-UHNlMC/LlQ6bJLMQ+tftDsvjxSgi+gmJvKoRbbH6CNA=";
38+
};
39+
"11.4".arm64 = {
40+
finalImageTag = "11.4.3-base-ubuntu20.04";
41+
imageDigest = "sha256:ef4813458a4b40b41f21c5fc505a7c7a39e12616273b2fb8ab10a1ebee541ac7";
42+
sha256 = "sha256-Oxx5w0OXq74oUHqaOYF2Xb4q8cxz2srhVvst+mFjC04=";
43+
};
44+
});
1445

15-
fromImage = dockerTools.pullImage {
16-
imageName = "nvcr.io/nvidia/l4t-jetpack";
46+
cudaDevel = dockerTools.pullImage ({
47+
imageName = "nvidia/cuda";
48+
} // lib.getAttrFromPath [ cudaPackages.cudaMajorMinorVersion go.GOARCH ] {
49+
"13.0".amd64 = {
50+
finalImageTag = "13.0.2-devel-ubuntu24.04";
51+
imageDigest = "sha256:0eee3094c71518ad31d011a594ae6ed6de72959ee07e318cb31cffe71690e90c";
52+
sha256 = "sha256-5PcPyKbIrNfHOsNJ8MCsBMSpVizR01qWaUtVsbEyqZE=";
53+
};
54+
"13.0".arm64 = {
55+
finalImageTag = "13.0.2-devel-ubuntu24.04";
56+
imageDigest = "sha256:450d11555d20ac8ebbbc13ebf17589c2bd42869171a90179ce7098b4a5e64c6a";
57+
sha256 = "sha256-fUwPPtLwAhU9UwxM59x35mboOPuxA1D6nV8N64I0uPI=";
58+
};
59+
"12.6".amd64 = {
60+
finalImageTag = "12.6.3-devel-ubuntu24.04";
61+
imageDigest = "sha256:badf6c452e8b1efea49d0bb956bef78adcf60e7f87ac77333208205f00ac9ade";
62+
sha256 = "sha256-P1XZZFmMP76XjZtagtNDXAlYcBR3OW+h/2fYMIMHCVM=";
63+
};
64+
"12.6".arm64 = {
65+
finalImageTag = "12.6.3-devel-ubuntu24.04";
66+
imageDigest = "sha256:37191266f9cad5651a92a4b56d8a03bb072bb27d768663e0fc9b6c7ecdfa0a11";
67+
sha256 = "sha256-+21AwCDcGtT3g3xcXi7wpzj3u1Wy7zRhFPumYVnfa+A=";
68+
};
69+
"11.8".amd64 = {
70+
finalImageTag = "11.8.0-devel-ubuntu22.04";
71+
imageDigest = "sha256:60eda04ab6790aa76d73bf0df245b361eabc6d8f7b6f6cf9846c70f399b9a1eb";
72+
sha256 = "sha256-PKn6U9m4jRXsnAxCLEw+BW3lYWdflKzLex2bA28ASa0=";
73+
};
74+
"11.4".arm64 = {
75+
finalImageTag = "11.4.3-devel-ubuntu20.04";
76+
imageDigest = "sha256:c5b82aac11fbba9f6d1a2802faa6739f597ba4ac3cdbd95205bd8e817dca74e3";
77+
sha256 = "sha256-oVILXM3crP2EwH/N+KhhAH2cm8Kwuihk705R88KNfcM=";
78+
};
79+
});
80+
81+
saxpyLayer = dockerTools.mkRootLayer {
82+
name = "saxpy-builder";
83+
84+
fromImage = cudaDevel;
85+
86+
copyToRoot = [
87+
(path + "/pkgs/development/cuda-modules/packages/saxpy/src/")
88+
];
89+
90+
runAsRoot = ''
91+
/usr/local/cuda/bin/nvcc saxpy.cu
92+
'';
93+
94+
baseJson = writeText "saxpy-builder-config.json" (
95+
builtins.toJSON {
96+
created = "1970-01-01T00:00:01Z";
97+
config = null;
98+
architecture = go.GOARCH;
99+
preferLocalBuild = true;
17100
os = "linux";
18-
arch = "arm64";
19-
imageDigest = "sha256:d1c8e971ab994235840eacc31c4ef4173bf9156317b1bf8aabe7e01eb21b2a0e";
20-
finalImageTag = "r35.4.1"; # As of 2024-10-27 there is (still) no 35.6.0 image published
21-
sha256 = "sha256-IDePYGssk6yrcaocnluxBaRJb7BrXxS7tBlEo6hNtHw=";
22-
};
101+
}
102+
);
23103

24-
config.cmd = [ "bash" "-c" "cd /usr/local/cuda/samples/1_Utilities/deviceQuery && make && ./deviceQuery" ];
25-
};
26-
"36" =
27-
let
28-
cuda-samples = fetchFromGitHub {
29-
owner = "NVIDIA";
30-
repo = "cuda-samples";
31-
tag = "v12.5"; # There is no 12.6 tag
32-
hash = "sha256-LL9A6olrpSAqePumNzQbAdljnzhOehmqqOy5sJieJk8=";
33-
};
34-
35-
extraPrefix = "/share";
36-
in
37-
dockerTools.buildImage {
38-
name = "l4t-jetpack-with-samples";
39-
40-
fromImage = dockerTools.pullImage {
41-
imageName = "nvcr.io/nvidia/l4t-jetpack";
42-
os = "linux";
43-
arch = "arm64";
44-
imageDigest = "sha256:34ccf0f3b63c6da9eee45f2e79de9bf7fdf3beda9abfd72bbf285ae9d40bb673";
45-
finalImageTag = "r36.4.0";
46-
sha256 = "sha256-+5+GRmyCl2ZcdYIJHU5snuFzEx1QkZic9bhtx9ZjXeo=";
47-
};
48-
49-
copyToRoot = [
50-
(buildEnv {
51-
name = "cuda-samples-fhs";
52-
paths = [ cuda-samples ];
53-
inherit extraPrefix;
54-
})
55-
];
56-
57-
config.Cmd = [ "bash" "-c" "make -C ${extraPrefix}/Samples/1_Utilities/deviceQuery && ${extraPrefix}/Samples/1_Utilities/deviceQuery/deviceQuery" ];
58-
};
59-
}.${lib.versions.major l4tMajorMinorPatchVersion};
104+
diskSize = 1024 * 20; # 20GB
105+
buildVMMemorySize = 2048; # 2GB
106+
};
107+
108+
testContainer = dockerTools.buildLayeredImage {
109+
name = "oci-saxpy-container-test";
110+
fromImage = cudaBase;
111+
112+
contents = [
113+
saxpyLayer
114+
115+
(writeTextFile {
116+
name = "script-to-run-in-container";
117+
text = ''
118+
#!/bin/sh
119+
120+
set -ex
121+
122+
# nvidia-smi
123+
124+
tar xf /layer.tar
125+
/a.out
126+
'';
127+
executable = true;
128+
destination = "/script-in-container";
129+
})
130+
];
131+
132+
config.cmd = [ "/script-in-container" ];
133+
};
60134
in
61135
{
62136
oci = writeShellScriptBin "oci-test" ''
63-
image=${l4tImage.imageName}:${l4tImage.imageTag}
137+
image=${testContainer.imageName}:${testContainer.imageTag}
64138
65139
for runtime in docker podman; do
66140
if command -v $runtime 2>&1 >/dev/null; then
@@ -70,8 +144,8 @@ in
70144
continue
71145
fi
72146
73-
echo "loading image ${l4tImage} with tag $image..."
74-
"$runtime" load --input=${l4tImage}
147+
echo "loading image ${testContainer} with tag $image..."
148+
"$runtime" load --input=${testContainer}
75149
echo "loaded image"
76150
77151
echo "testing without NVIDIA passthru, which should fail"

0 commit comments

Comments
 (0)