Skip to content

Commit 5d16cfa

Browse files
elliotbermanelliotberman
committed
tests: oci: Use saxpy as the test
The current oci test requires downloading and building cuda-samples. This requires a very large container containing the CUDA runtime and development tools. Instead, build a simple CUDA binary (saxpy) separately and create a container with only the CUDA runtime and saxpy.
1 parent 8b74772 commit 5d16cfa

File tree

1 file changed

+132
-55
lines changed

1 file changed

+132
-55
lines changed

‎pkgs/tests/default.nix‎

Lines changed: 132 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -1,64 +1,141 @@
1-
{ l4tMajorMinorPatchVersion
1+
{ cudaPackages
22
, dockerTools
3-
, writeShellScriptBin
3+
, go
44
, lib
5-
, l4tAtLeast
6-
, fetchFromGitHub
7-
, buildEnv
5+
, path
86
, stdenv
97
, dlopenOverride
8+
, writeShellScriptBin
9+
, writeText
10+
, writeTextFile
1011
}:
11-
# https://docs.nvidia.com/jetson/archives/r36.4.4/DeveloperGuide/SD/TestPlanValidation.html#nvidia-containers
1212
let
13-
l4tImage = {
14-
"35" = dockerTools.buildImage {
15-
name = "l4t-jetpack-with-samples";
13+
# cuda "base" with only libcuda.so.1 and other essentials; used on device
14+
cudaBase = dockerTools.pullImage ({
15+
imageName = "nvidia/cuda";
16+
} // lib.getAttrFromPath [ cudaPackages.cudaMajorMinorVersion go.GOARCH ] {
17+
"13.0".amd64 = {
18+
finalImageTag = "13.0.2-base-ubuntu24.04";
19+
imageDigest = "sha256:605fb0c8acf8674e164d822da8a8521f3a655056e569f0899e72ae940e1fe7dc";
20+
sha256 = "";
21+
};
22+
"13.0".arm64 = {
23+
finalImageTag = "13.0.2-base-ubuntu24.04";
24+
imageDigest = "sha256:1ca86773be1716af6cfff3d2eb8cd10d4d9cac181931d1ee9be792d3e33c7322";
25+
sha256 = "sha256-IOtmpA2xiNL1/zodFdUz5/nrDYMBssDibwYE0uwrCcI= ";
26+
};
27+
"12.6".amd64 = {
28+
finalImageTag = "12.6.3-base-ubuntu24.04";
29+
imageDigest = "sha256:6b3201183858bad08441837f5a5efc2c75290135cc25fcc87d9ff763190cfd09";
30+
sha256 = "sha256-326qF7q4RtMnye/yOeh+lgilopoCrEJyV0WuTQlN9JM=";
31+
};
32+
"12.6".arm64 = {
33+
finalImageTag = "12.6.3-base-ubuntu24.04";
34+
imageDigest = "sha256:ba25600cea339517a8e55361a27535bbfdeae1936663e9b4970f3cec2fbbd165";
35+
sha256 = "sha256-lii+GS0havrQceqqYvHa35Lwe7I2rCQ3pVN5XvE2rmw=";
36+
};
37+
"11.8".amd64 = {
38+
finalImageTag = "11.8.0-base-ubuntu22.04";
39+
imageDigest = "sha256:79e5b2cf878ee9006f5b3738caeea34fdc7708a32db53fe3e80db0b48bd286a0";
40+
sha256 = "sha256-UHNlMC/LlQ6bJLMQ+tftDsvjxSgi+gmJvKoRbbH6CNA=";
41+
};
42+
"11.4".arm64 = {
43+
finalImageTag = "11.4.3-base-ubuntu20.04";
44+
imageDigest = "sha256:ef4813458a4b40b41f21c5fc505a7c7a39e12616273b2fb8ab10a1ebee541ac7";
45+
sha256 = "sha256-Oxx5w0OXq74oUHqaOYF2Xb4q8cxz2srhVvst+mFjC04=";
46+
};
47+
});
48+
49+
# cuda "development" with libcuda and all the development tools; used to build saxpy the not-nixos way
50+
cudaDevel = dockerTools.pullImage ({
51+
imageName = "nvidia/cuda";
52+
} // lib.getAttrFromPath [ cudaPackages.cudaMajorMinorVersion go.GOARCH ] {
53+
"13.0".amd64 = {
54+
finalImageTag = "13.0.2-devel-ubuntu24.04";
55+
imageDigest = "sha256:0eee3094c71518ad31d011a594ae6ed6de72959ee07e318cb31cffe71690e90c";
56+
sha256 = "sha256-5PcPyKbIrNfHOsNJ8MCsBMSpVizR01qWaUtVsbEyqZE=";
57+
};
58+
"13.0".arm64 = {
59+
finalImageTag = "13.0.2-devel-ubuntu24.04";
60+
imageDigest = "sha256:450d11555d20ac8ebbbc13ebf17589c2bd42869171a90179ce7098b4a5e64c6a";
61+
sha256 = "sha256-fUwPPtLwAhU9UwxM59x35mboOPuxA1D6nV8N64I0uPI=";
62+
};
63+
"12.6".amd64 = {
64+
finalImageTag = "12.6.3-devel-ubuntu24.04";
65+
imageDigest = "sha256:badf6c452e8b1efea49d0bb956bef78adcf60e7f87ac77333208205f00ac9ade";
66+
sha256 = "sha256-P1XZZFmMP76XjZtagtNDXAlYcBR3OW+h/2fYMIMHCVM=";
67+
};
68+
"12.6".arm64 = {
69+
finalImageTag = "12.6.3-devel-ubuntu24.04";
70+
imageDigest = "sha256:37191266f9cad5651a92a4b56d8a03bb072bb27d768663e0fc9b6c7ecdfa0a11";
71+
sha256 = "sha256-+21AwCDcGtT3g3xcXi7wpzj3u1Wy7zRhFPumYVnfa+A=";
72+
};
73+
"11.8".amd64 = {
74+
finalImageTag = "11.8.0-devel-ubuntu22.04";
75+
imageDigest = "sha256:60eda04ab6790aa76d73bf0df245b361eabc6d8f7b6f6cf9846c70f399b9a1eb";
76+
sha256 = "sha256-PKn6U9m4jRXsnAxCLEw+BW3lYWdflKzLex2bA28ASa0=";
77+
};
78+
"11.4".arm64 = {
79+
finalImageTag = "11.4.3-devel-ubuntu20.04";
80+
imageDigest = "sha256:c5b82aac11fbba9f6d1a2802faa6739f597ba4ac3cdbd95205bd8e817dca74e3";
81+
sha256 = "sha256-oVILXM3crP2EwH/N+KhhAH2cm8Kwuihk705R88KNfcM=";
82+
};
83+
});
84+
85+
saxpyLayer = dockerTools.mkRootLayer {
86+
name = "saxpy-builder";
1687

17-
fromImage = dockerTools.pullImage {
18-
imageName = "nvcr.io/nvidia/l4t-jetpack";
88+
fromImage = cudaDevel;
89+
90+
copyToRoot = [
91+
(path + "/pkgs/development/cuda-modules/packages/saxpy/src/")
92+
];
93+
94+
runAsRoot = ''
95+
/usr/local/cuda/bin/nvcc saxpy.cu
96+
'';
97+
98+
# See nixpkgs/pkgs/build-support/docker/default.nix buildImage for construction of baseJson
99+
baseJson = writeText "saxpy-builder-config.json" (
100+
builtins.toJSON {
101+
created = "1970-01-01T00:00:01Z";
102+
config = null;
103+
architecture = go.GOARCH;
104+
preferLocalBuild = true;
19105
os = "linux";
20-
arch = "arm64";
21-
imageDigest = "sha256:d1c8e971ab994235840eacc31c4ef4173bf9156317b1bf8aabe7e01eb21b2a0e";
22-
finalImageTag = "r35.4.1"; # As of 2024-10-27 there is (still) no 35.6.0 image published
23-
sha256 = "sha256-IDePYGssk6yrcaocnluxBaRJb7BrXxS7tBlEo6hNtHw=";
24-
};
106+
}
107+
);
25108

26-
config.cmd = [ "bash" "-c" "cd /usr/local/cuda/samples/1_Utilities/deviceQuery && make && ./deviceQuery" ];
27-
};
28-
"36" =
29-
let
30-
cuda-samples = fetchFromGitHub {
31-
owner = "NVIDIA";
32-
repo = "cuda-samples";
33-
tag = "v12.5"; # There is no 12.6 tag
34-
hash = "sha256-LL9A6olrpSAqePumNzQbAdljnzhOehmqqOy5sJieJk8=";
35-
};
36-
37-
extraPrefix = "/share";
38-
in
39-
dockerTools.buildImage {
40-
name = "l4t-jetpack-with-samples";
41-
42-
fromImage = dockerTools.pullImage {
43-
imageName = "nvcr.io/nvidia/l4t-jetpack";
44-
os = "linux";
45-
arch = "arm64";
46-
imageDigest = "sha256:34ccf0f3b63c6da9eee45f2e79de9bf7fdf3beda9abfd72bbf285ae9d40bb673";
47-
finalImageTag = "r36.4.0";
48-
sha256 = "sha256-+5+GRmyCl2ZcdYIJHU5snuFzEx1QkZic9bhtx9ZjXeo=";
49-
};
50-
51-
copyToRoot = [
52-
(buildEnv {
53-
name = "cuda-samples-fhs";
54-
paths = [ cuda-samples ];
55-
inherit extraPrefix;
56-
})
57-
];
58-
59-
config.Cmd = [ "bash" "-c" "make -C ${extraPrefix}/Samples/1_Utilities/deviceQuery && ${extraPrefix}/Samples/1_Utilities/deviceQuery/deviceQuery" ];
60-
};
61-
}.${lib.versions.major l4tMajorMinorPatchVersion};
109+
diskSize = 1024 * 20; # 20GB
110+
buildVMMemorySize = 2048; # 2GB
111+
};
112+
113+
testContainer = dockerTools.buildLayeredImage {
114+
name = "oci-saxpy-container-test";
115+
fromImage = cudaBase;
116+
117+
contents = [
118+
saxpyLayer
119+
120+
(writeTextFile {
121+
name = "script-to-run-in-container";
122+
text = ''
123+
#!/bin/sh
124+
125+
set -ex
126+
127+
# nvidia-smi
128+
129+
tar xf /layer.tar
130+
/a.out
131+
'';
132+
executable = true;
133+
destination = "/script-in-container";
134+
})
135+
];
136+
137+
config.cmd = [ "/script-in-container" ];
138+
};
62139

63140
normal = stdenv.mkDerivation {
64141
name = "test-app";
@@ -104,7 +181,7 @@ let
104181
in
105182
{
106183
oci = writeShellScriptBin "oci-test" ''
107-
image=${l4tImage.imageName}:${l4tImage.imageTag}
184+
image=${testContainer.imageName}:${testContainer.imageTag}
108185
109186
for runtime in docker podman; do
110187
if command -v $runtime 2>&1 >/dev/null; then
@@ -114,8 +191,8 @@ in
114191
continue
115192
fi
116193
117-
echo "loading image ${l4tImage} with tag $image..."
118-
"$runtime" load --input=${l4tImage}
194+
echo "loading image ${testContainer} with tag $image..."
195+
"$runtime" load --input=${testContainer}
119196
echo "loaded image"
120197
121198
echo "testing without NVIDIA passthru, which should fail"

0 commit comments

Comments
 (0)