Skip to content

Commit 5eeeb78

Browse files
guymguymguymguym
committed
rdma scripts
Signed-off-by: Guy Margalit <[email protected]>
1 parent ac805a6 commit 5eeeb78

File tree

6 files changed

+158
-0
lines changed

6 files changed

+158
-0
lines changed

‎cudaguy.c

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
// Build: gcc -o cudaguy cudaguy.c -I/usr/local/cuda/include/ -L/usr/local/cuda/lib64 -lcuda
2+
// Run: ./cudaguy
3+
4+
#include <cuda.h>
5+
#include <stdint.h>
6+
#include <stdio.h>
7+
#include <stdlib.h>
8+
9+
#define CU_TRY(fn) \
10+
do { \
11+
CUresult r = fn; \
12+
if (r != CUDA_SUCCESS) { \
13+
const char* cuda_err = ""; \
14+
cuGetErrorName(r, &cuda_err); \
15+
fprintf(stderr, "CUDA error: %s %s\n", cuda_err, #fn); \
16+
exit(1); \
17+
} \
18+
} while (0)
19+
20+
int
21+
main()
22+
{
23+
size_t size = 8 * 1024 * 1024;
24+
CUdevice cuda_device = 0;
25+
CUcontext cuda_ctx = 0;
26+
CUdeviceptr cuda_ptr = 0;
27+
CUmemorytype mem_type = CU_MEMORYTYPE_HOST;
28+
char* host_ptr = (char*)malloc(size);
29+
30+
CU_TRY(cuInit(0));
31+
CU_TRY(cuDeviceGet(&cuda_device, 0););
32+
CU_TRY(cuCtxCreate(&cuda_ctx, 0, cuda_device));
33+
fprintf(stderr, "CUDA initialized: device %d context %p\n", cuda_device, (void*)cuda_ctx);
34+
35+
CU_TRY(cuMemAlloc(&cuda_ptr, size));
36+
CU_TRY(cuMemsetD8(cuda_ptr, 'A', size));
37+
CU_TRY(cuCtxSynchronize());
38+
fprintf(stderr, "CUDA allocated %p size %zu\n", (void*)cuda_ptr, size);
39+
40+
CU_TRY(cuPointerGetAttribute(&mem_type, CU_POINTER_ATTRIBUTE_MEMORY_TYPE, cuda_ptr));
41+
fprintf(stderr, "CUDA buffer mem type: %d\n", mem_type);
42+
43+
memset(host_ptr, 'B', size);
44+
CU_TRY(cuMemcpyDtoH(host_ptr, cuda_ptr, size));
45+
46+
// skip repeating 'A' at the end, while keeping the first 10 chars,
47+
// and terminate the string for printing
48+
int i = size - 1;
49+
while (i > 10 && host_ptr[i] == 'A') --i;
50+
host_ptr[i] = '\0';
51+
fprintf(stderr, "CUDA copied to host: %s\n", host_ptr);
52+
53+
free(host_ptr);
54+
CU_TRY(cuMemFree(cuda_ptr));
55+
CU_TRY(cuCtxDestroy(cuda_ctx));
56+
fprintf(stderr, "CUDA freed\n");
57+
58+
return 0;
59+
}

‎rdma-build1.sh

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
ssh $GDS4N1 'cd /root/guym/noobaa-core && source /root/.nvm/nvm.sh && make RDMA=1' &&
2+
printf "\n\n IT'S DONE. \n\n"

‎rdma-build2.sh

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
ssh $GDS4N2 'cd /root/guym/noobaa-core && source /root/.nvm/nvm.sh && make RDMA=1 CUDA=1' &&
2+
printf "\n\n IT'S DONE. \n\n"

‎rdma-push1.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
git push -f ssh://$GDS4N1:/root/guym/noobaa-core guy-rdma &&
2+
ssh $GDS4N1 'cd /root/guym/noobaa-core && git rebase guy-rdma' &&
3+
printf "\n\n IT'S DONE. \n\n"

‎rdma-push2.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
git push -f ssh://$GDS4N2:/root/guym/noobaa-core guy-rdma &&
2+
ssh $GDS4N2 'cd /root/guym/noobaa-core && git rebase guy-rdma' &&
3+
printf "\n\n IT'S DONE. \n\n"

‎rdma-run.sh

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
## variables
2+
CUDA_PATH="$(realpath /usr/local/cuda)"
3+
CUOBJ_PATH="$(realpath ../cuObject-0.7.2-Linux_x86_64/src)"
4+
CUFILE_ENV_PATH_JSON="$(realpath ../cuobj.json)"
5+
RDMA_LIBS="$CUOBJ_PATH/lib/libcuobjserver.so $CUOBJ_PATH/lib/libcuobjclient.so $CUOBJ_PATH/lib/libcufile.so.1.13.0 $CUOBJ_PATH/lib/libcufile_rdma.so.1.13.0"
6+
7+
## git push to hosts
8+
./rdma-push.sh
9+
10+
## build script
11+
./rdma-build.sh
12+
13+
## build commands
14+
make RDMA=1
15+
make RDMA=1 CUDA=1
16+
17+
# quick rebuild:
18+
rm -rf build/Release/obj.target/{rdma,cuda}_napi/ &&
19+
rm -f build/Release/{rdma,cuda}_napi.a &&
20+
rm -f build/Release/nb_native.node
21+
22+
23+
## rdma_speed
24+
UV_THREADPOOL_SIZE=4 LD_PRELOAD="$RDMA_LIBS" node src/tools/rdma_speed.js --server
25+
UV_THREADPOOL_SIZE=4 LD_PRELOAD="$RDMA_LIBS" CUFILE_ENV_PATH_JSON="$CUFILE_ENV_PATH_JSON" node src/tools/rdma_speed.js --client --op GET
26+
UV_THREADPOOL_SIZE=4 LD_PRELOAD="$RDMA_LIBS" CUFILE_ENV_PATH_JSON="$CUFILE_ENV_PATH_JSON" node src/tools/rdma_speed.js --client --op PUT
27+
# --op PUT --forks 1 --concur 16
28+
# --pool_size $((4*32)) --size 32
29+
# --perf-basic-prof
30+
31+
## http_speed
32+
node src/tools/http_speed.js --server --buf $((8*1024*1024)) --size 8 --forks 8
33+
node src/tools/http_speed.js --client 172.16.0.61 --buf $((8*1024*1024)) --size 8 --forks 8 --concur 8 --method GET
34+
node src/tools/http_speed.js --client 172.16.0.61 --buf $((8*1024*1024)) --size 8 --forks 8 --concur 8 --method PUT
35+
36+
## noobaa server (local ips 172.16.0.61 and 172.16.0.71)
37+
LD_PRELOAD="$RDMA_LIBS" LOCAL_IP=172.16.0.61 node src/cmd/nsfs.js
38+
LD_PRELOAD="$RDMA_LIBS" LOCAL_IP=172.16.0.71 node src/cmd/nsfs.js
39+
40+
####################################################
41+
## client (local ips 172.16.0.62 and 172.16.0.72) ##
42+
####################################################
43+
44+
## s3cat
45+
DISABLE_INIT_RANDOM_SEED=true \
46+
node src/tools/s3cat.js \
47+
--endpoint http://172.16.0.61:6001 \
48+
--access_key $AWS_ACCESS_KEY_ID \
49+
--secret_key $AWS_SECRET_ACCESS_KEY \
50+
--bucket bucket1 \
51+
--ls
52+
53+
# --upload
54+
# --get upload-m6s4i12b
55+
56+
57+
UV_THREADPOOL_SIZE=4 \
58+
DISABLE_INIT_RANDOM_SEED=true \
59+
LD_PRELOAD="$RDMA_LIBS" \
60+
CUFILE_ENV_PATH_JSON="$CUFILE_ENV_PATH_JSON" \
61+
node src/tools/s3perf.js \
62+
--local_ip 172.16.0.62 \
63+
--endpoint http://172.16.0.61:6001 \
64+
--selfsigned \
65+
--access_key $AWS_ACCESS_KEY_ID \
66+
--secret_key $AWS_SECRET_ACCESS_KEY \
67+
--bucket bucket1 \
68+
--time 120 \
69+
--get fs_speed \
70+
--concur 1 \
71+
--forks 1 \
72+
--rdma \
73+
--cuda
74+
75+
76+
## warp
77+
../warp get \
78+
--host 172.16.0.61:6001 \
79+
--access-key $AWS_ACCESS_KEY_ID \
80+
--secret-key $AWS_SECRET_ACCESS_KEY \
81+
--duration 20s \
82+
--obj.size 100MiB \
83+
--objects 100 \
84+
--concurrent 20 \
85+
--disable-multipart \
86+
--disable-sha256-payload \
87+
--noclear \
88+
--list-existing
89+

0 commit comments

Comments
 (0)