diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..a00f873 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,41 @@ +FROM vanessa/slurm:18.08.6 + +LABEL org.label-schema.vcs-url="https://github.com/hpc/spindle" \ + org.label-schema.docker.cmd="docker-compose up -d" \ + org.label-schema.name="spindle" \ + org.label-schema.description="Spindle with SLURM on Centos 7" \ + maintainer="Vanessa Sochat" + +# Install ompi +RUN wget https://www.open-mpi.org/software/ompi/v1.10/downloads/openmpi-1.10.2.tar.gz && \ + tar -xzvf openmpi-1.10.2.tar.gz && cd openmpi-1.10.2/ && \ + ./configure --with-slurm --prefix="/home/$USER/.openmpi" && \ + make && make install + +ENV PATH "$PATH:/home/$USER/.openmpi/bin" +ENV LD_LIBRARY_PATH "$LD_LIBRARY_PATH:/home/$USER/.openmpi/lib/" + +ENV TMPDIR /tmp + +RUN yum install -y nc openssh-server openssh-clients iputils automake && git clone https://github.com/hpc/spindle && \ + cd spindle && \ +# --with-rsh-launch-option --with-testrm=slurm +# ./configure --with-munge-dir=/etc/munge --enable-sec-munge --with-slurm-dir=/etc/slurm --with-rsh-launch-option && \ + ./configure --with-testrm=slurm --with-rsh-launch --with-rsh-cmd=/usr/bin/ssh --with-tmpdir=/tmp && \ + make && make install + +RUN mkdir -p /var/run/sshd && \ + chmod 0755 /var/run/sshd && \ + useradd -p spindle --create-home --shell /bin/bash spindle + +RUN ssh-keygen -b 2048 -t rsa -f /root/.ssh/id_rsa -q -N "" && \ + su spindle -c "ssh-keygen -b 2048 -t rsa -f /home/spindle/.ssh/id_rsa -q -N ''" && \ + cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys && \ + cat /home/spindle/.ssh/id_rsa.pub >> /home/spindle/.ssh/authorized_keys + +# Put spindle n debug mode, can be 1,2,3 (or unset) +ENV SPINDLE_DEBUG 3 +EXPOSE 22 + +COPY docker-entrypoint.sh /docker-entrypoint.sh +ENTRYPOINT ["/docker-entrypoint.sh"] diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..2f45ccf --- /dev/null +++ b/docker/README.md @@ -0,0 +1,271 @@ +# Spindle in Docker + +This directory contains a set of container recipes and scripts to allow you +to quickly bring up your own tiny cluster with [docker-compose](https://docs.docker.com/compose/install/), install +spindle, and give it a try. You will need both [docker-compose](https://docs.docker.com/compose/install/) +and [Docker](https://docs.docker.com/get-docker/) installed for this tutorial. + +## 1. Build Containers + +The [Dockerfile](Dockerfile) here is the base for building containers. +So running the build is as easy as: + +```bash +$ docker-compose build +``` + +And then bringing them up: + +```bash +$ docker-compose up -d +``` + +And checking that they are running + +```bash +$ docker-compose ps + Name Command State Ports +------------------------------------------------------------------------ +c1 /usr/local/bin/docker-entr ... Up 6818/tcp +c2 /usr/local/bin/docker-entr ... Up 6818/tcp +mysql docker-entrypoint.sh mysqld Up 3306/tcp, 33060/tcp +slurmctld /usr/local/bin/docker-entr ... Up 6817/tcp +slurmdbd /usr/local/bin/docker-entr ... Up 6819/tcp +``` + +Each of c1 and c2 are nodes for our cluster, and then slurmctld is like the login node. + +```bash +$ docker exec -it slurmctld bash +``` +Try running a job! + +```bash +$ sbatch --wrap="sleep 20" +# squeue + JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON) + 1 normal wrap root R 0:00 1 c1 +``` + +## 2. Interact with spindle + +Spindle should already be installed, and you can see the steps if you view the +[Dockerfile](Dockerfile). + +```bash +# which spindle +/usr/local/bin/spindle +``` + +You can try running a job first without spindle: + +```bash +$ srun -N 1 cat /proc/self/maps +``` + +If you try *with* spindle, you need to allocate a session first: + +```bash +$ salloc -N 1 +$ spindle --no-mpi cat /proc/self/maps +``` + +Before we run tests, let's get the hosts c1 and c2 added (it will ask you to confirm yes for each) + +```bash +$ ssh c1 +exit + +$ ssh c2 +exit +``` + +**NOTE** stopped here - this doesn't actually work. +If you want to view the source code, go to /spindle. + +```bash +cd /spindle/testsuite +./runTests +``` + +If you need more debug output: + +```bash +# 1, 2, 3 +export SPINDLE_DEBUG=2 +``` + +Optionally, you can better configure the cluster: + +```bash +sacctmgr -i create cluster spindle-cluster +sacctmgr add account spindle --immediate +sacctmgr create user spindle defaultaccount=spindle adminlevel=[None] --immediate +``` + +## 3. Use Spindle + +The first sanity check to see if spindle is working is to look at this output: + +```bash +$ cat /proc/self/maps +[root@slurmctld /]# cat /proc/self/maps +00400000-0040b000 r-xp 00000000 00:7d 27450779 /usr/bin/cat +0060b000-0060c000 r--p 0000b000 00:7d 27450779 /usr/bin/cat +0060c000-0060d000 rw-p 0000c000 00:7d 27450779 /usr/bin/cat +0189d000-018be000 rw-p 00000000 00:00 0 [heap] +7f2204c09000-7f2204dcb000 r-xp 00000000 00:7d 27466152 /usr/lib64/libc-2.17.so +7f2204dcb000-7f2204fcb000 ---p 001c2000 00:7d 27466152 /usr/lib64/libc-2.17.so +7f2204fcb000-7f2204fcf000 r--p 001c2000 00:7d 27466152 /usr/lib64/libc-2.17.so +7f2204fcf000-7f2204fd1000 rw-p 001c6000 00:7d 27466152 /usr/lib64/libc-2.17.so +7f2204fd1000-7f2204fd6000 rw-p 00000000 00:00 0 +7f2204fd6000-7f2204ff8000 r-xp 00000000 00:7d 27466129 /usr/lib64/ld-2.17.so +7f2205064000-7f22051ed000 r--p 00000000 00:7d 27573565 /usr/lib/locale/locale-archive +7f22051ed000-7f22051f0000 rw-p 00000000 00:00 0 +7f22051f6000-7f22051f7000 rw-p 00000000 00:00 0 +7f22051f7000-7f22051f8000 r--p 00021000 00:7d 27466129 /usr/lib64/ld-2.17.so +7f22051f8000-7f22051f9000 rw-p 00022000 00:7d 27466129 /usr/lib64/ld-2.17.so +7f22051f9000-7f22051fa000 rw-p 00000000 00:00 0 +7fff660dd000-7fff660fe000 rw-p 00000000 00:00 0 [stack] +7fff661dc000-7fff661df000 r--p 00000000 00:00 0 [vvar] +7fff661df000-7fff661e0000 r-xp 00000000 00:00 0 [vdso] +ffffffffff600000-ffffffffff601000 --xp 00000000 00:00 0 [vsyscall] +``` + +and compare that with the same command, but with spindle: + +```bash +$ spindle --no-mpi cat /proc/self/maps +00400000-0040b000 r-xp 00000000 00:7d 28646903 /tmp/spindle.84/usr/bin/1-spindlens-file-cat +0060b000-0060c000 r--p 0000b000 00:7d 27450779 /usr/bin/cat +0060c000-0060d000 rw-p 0000c000 00:7d 27450779 /usr/bin/cat +01dc8000-01de9000 rw-p 00000000 00:00 0 [heap] +7f8622f69000-7f86230f2000 r--p 00000000 00:7d 27573565 /usr/lib/locale/locale-archive +7f86230f2000-7f86232b4000 r-xp 00000000 00:7d 27466152 /usr/lib64/libc-2.17.so +7f86232b4000-7f86234b4000 ---p 001c2000 00:7d 27466152 /usr/lib64/libc-2.17.so +7f86234b4000-7f86234b8000 r--p 001c2000 00:7d 27466152 /usr/lib64/libc-2.17.so +7f86234b8000-7f86234ba000 rw-p 001c6000 00:7d 27466152 /usr/lib64/libc-2.17.so +7f86234ba000-7f86234bf000 rw-p 00000000 00:00 0 +7f86234bf000-7f8623681000 r-xp 00000000 00:7d 27466152 /usr/lib64/libc-2.17.so +7f8623681000-7f8623881000 ---p 001c2000 00:7d 27466152 /usr/lib64/libc-2.17.so +7f8623881000-7f8623887000 rw-p 001c2000 00:7d 27466152 /usr/lib64/libc-2.17.so +7f8623887000-7f862388c000 rw-p 00000000 00:00 0 +7f862388c000-7f86238a9000 r-xp 00000000 00:7d 28646902 /tmp/spindle.84/usr/local/lib/spindle/0-spindlens-file-libspindle_audit_pipe.so +7f86238a9000-7f8623aa8000 ---p 0001d000 00:7d 28646902 /tmp/spindle.84/usr/local/lib/spindle/0-spindlens-file-libspindle_audit_pipe.so +7f8623aa8000-7f8623aaa000 rw-p 0001c000 00:7d 28646902 /tmp/spindle.84/usr/local/lib/spindle/0-spindlens-file-libspindle_audit_pipe.so +7f8623aaa000-7f8623aad000 rw-p 00000000 00:00 0 +7f8623aad000-7f8623acf000 r-xp 00000000 00:7d 27466129 /usr/lib64/ld-2.17.so +7f8623bc3000-7f8623cc5000 rw-p 00000000 00:00 0 +7f8623ccb000-7f8623cce000 rw-p 00000000 00:00 0 +7f8623cce000-7f8623cd0000 rw-p 00021000 00:7d 27466129 /usr/lib64/ld-2.17.so +7f8623cd0000-7f8623cd1000 rw-p 00000000 00:00 0 +7ffd73932000-7ffd73953000 rw-p 00000000 00:00 0 [stack] +7ffd739bd000-7ffd739c0000 r--p 00000000 00:00 0 [vvar] +7ffd739c0000-7ffd739c1000 r-xp 00000000 00:00 0 [vdso] +ffffffffff600000-ffffffffff601000 --xp 00000000 00:00 0 [vsyscall] +``` + +You should see several paths replaced with spindle ones. + +Next, let's try running a benchmarking tool with and without spindle. +This benchmark is called "Pynamic." Let's first clone it: + +```bash +$ git clone https://github.com/LLNL/pynamic +$ cd pynamic/pynamic-pyMPI-2.6a1 + +# Run python config_pynamic.py to see usage +``` +And then we would build shared libraries as follows. We are doing to decrease +from the default because it will take forever! + +```bash +# usage: config_pynamic.py [options] [-c ] +# example: config_pynamic.py 900 1250 -e -u 350 1250 -n 150 +# = total number of shared objects to produce +# = average number of functions per shared object +$ python config_pynamic.py 900 1250 -e -u 350 1250 -n 150 +``` + +Don't actually do that - it will never finish and control+C won't kill it! +Try this one instead, with a timer + +```bash +$ time python config_pynamic.py 30 1250 -e -u 350 1250 -n 150 + +************************************************ +summary of pynamic-sdb-pyMPI executable and 10 shared libraries +Size of aggregate total of shared libraries: 2.5MB +Size of aggregate texts of shared libraries: 6.8MB +Size of aggregate data of shared libraries: 408.4KB +Size of aggregate debug sections of shared libraries: 0B +Size of aggregate symbol tables of shared libraries: 0B +Size of aggregate string table size of shared libraries: 0B +************************************************ + +real 21m33.556s +user 14m54.538s +sys 3m31.206s +``` + +The above does take a bit (as you can see from the time) so let's try it now with +spindle: + +```bash +$ time spindle python config_pynamic.py 30 1250 -e -u 350 1250 -n 150 +``` + +**under development, not written yet, debugging things!** + +``` + 3.1 TO TEST + % python pynamic_driver.py `date +%s` + + or in a batchxterm: + + % srun pyMPI pynamic_driver.py `date +%s` + + % srun pynamic-pyMPI pynamic_driver.py `date +%s` + + % srun pynamic-sdb-pyMPI pynamic_driver.py `date +%s` + + % srun pynamic-bigexe pynamic_driver.py `date +%s` + + # note: Pynamic creates 3 executables: + # pyMPI - a vanilla pyMPI build + # pynamic-pyMPI - pyMPI with the generated .so's linked in + # pynamic-sdb-pyMPI - pyMPI with the generated libraries statically linked in + # and 2 optional executables (with the -b flag) + # pynamic-bigexe-pyMPI - a larger pyMPI with the generated .so's linked in + # pynamic-bigexe-sdb-pyMPI - a larger pyMPI with the generated libraries staically linked in + +-------------------------------------------------------- +4. CONTACTS + Greg Lee + Dong Ahn + Bronis de Supinski + John Gyllenhaal + +# run the pynamic benchmark with and without spindle +/cat/proc/self/maps + +prints out for each library loaded and bincat parts of address space takes up +run same command under spindle + +spindle --no-mpi cat /proc/self/maps + +to check if install works and is visible outside of spindle itself +/proc/pid/maps + +https://github.com/LLNL/pynamic +``` + + +## 4. Clean Up + +When you are done, exit from the container, stop and remove your images: + +```bash +$ docker-compose stop +$ docker-compose rm +``` diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..9a74b28 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,90 @@ +version: "2.2" + +services: + mysql: + image: mysql:5.7 + hostname: mysql + container_name: mysql + environment: + MYSQL_RANDOM_ROOT_PASSWORD: "yes" + MYSQL_DATABASE: slurm_acct_db + MYSQL_USER: slurm + MYSQL_PASSWORD: password + volumes: + - var_lib_mysql:/var/lib/mysql + + slurmdbd: + build: + context: . + command: "slurmdbd" + container_name: slurmdbd + hostname: slurmdbd + privileged: true + volumes: + - etc_munge:/etc/munge + - etc_slurm:/etc/slurm + - var_log_slurm:/var/log/slurm + - tmp_slurm:/tmp + - ./sshd_config:/etc/ssh/sshd_config + expose: + - "6819" + - "22" + depends_on: + - mysql + + slurmctld: + build: + context: . + privileged: true + command: "slurmctld" + container_name: slurmctld + hostname: slurmctld + volumes_from: + - slurmdbd + expose: + - "6817" + - "22" + depends_on: + - "slurmdbd" + + c1: + build: + context: . + command: "slurmd" + privileged: true + hostname: c1 + container_name: c1 + volumes_from: + - slurmctld + links: + - slurmctld + expose: + - "6818" + - "22" + depends_on: + - "slurmctld" + + c2: + build: + context: . + command: "slurmd" + privileged: true + hostname: c2 + container_name: c2 + links: + - slurmctld + volumes_from: + - slurmctld + expose: + - "6818" + - "22" + depends_on: + - "slurmctld" + +volumes: + etc_munge: + etc_slurm: + slurm_jobdir: + var_lib_mysql: + var_log_slurm: + tmp_slurm: diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh new file mode 100755 index 0000000..cccbfd5 --- /dev/null +++ b/docker/docker-entrypoint.sh @@ -0,0 +1,69 @@ +#!/bin/bash +set -e + +# Start ssh in all containers +systemctl enable sshd + +# Ensure TMPDIR envar is defined for all users, spindle needs it +echo "export TMPDIR=/tmp" >> /etc/profile.d/tmpdir.sh + +ssh-keygen -A +/usr/sbin/sshd -D & + +if [ "$1" = "slurmdbd" ] +then + echo "---> Starting the MUNGE Authentication service (munged) ..." + gosu munge /usr/sbin/munged + + echo "---> Starting the Slurm Database Daemon (slurmdbd) ..." + + { + . /etc/slurm/slurmdbd.conf + until echo "SELECT 1" | mysql -h $StorageHost -u$StorageUser -p$StoragePass 2>&1 > /dev/null + do + echo "-- Waiting for database to become active ..." + sleep 2 + done + } + echo "-- Database is now active ..." + + exec gosu slurm /usr/sbin/slurmdbd -Dvvv +fi + +if [ "$1" = "slurmctld" ] +then + echo "---> Starting the MUNGE Authentication service (munged) ..." + gosu munge /usr/sbin/munged + + echo "---> Waiting for slurmdbd to become active before starting slurmctld ..." + + until 2>/dev/null >/dev/tcp/slurmdbd/6819 + do + echo "-- slurmdbd is not available. Sleeping ..." + sleep 2 + done + echo "-- slurmdbd is now active ..." + + echo "---> Starting the Slurm Controller Daemon (slurmctld) ..." + exec gosu slurm /usr/sbin/slurmctld -Dvvv +fi + +if [ "$1" = "slurmd" ] +then + echo "---> Starting the MUNGE Authentication service (munged) ..." + gosu munge /usr/sbin/munged + + echo "---> Waiting for slurmctld to become active before starting slurmd..." + + until 2>/dev/null >/dev/tcp/slurmctld/6817 + do + echo "-- slurmctld is not available. Sleeping ..." + sleep 2 + done + echo "-- slurmctld is now active ..." + + echo "---> Starting the Slurm Node Daemon (slurmd) ..." + exec /usr/sbin/slurmd -Dvvv +fi + +exec "$@" diff --git a/docker/slurm.conf b/docker/slurm.conf new file mode 100644 index 0000000..8686e23 --- /dev/null +++ b/docker/slurm.conf @@ -0,0 +1,87 @@ +# +# Example slurm.conf file. Please run configurator.html +# (in doc/html) to build a configuration file customized +# for your environment. +# +# +# slurm.conf file generated by configurator.html. +# +# See the slurm.conf man page for more information. +# +ClusterName=linux +ControlMachine=linux0 +#ControlAddr= +#BackupController= +#BackupAddr= +# +SlurmUser=slurm +#SlurmdUser=root +SlurmctldPort=6817 +SlurmdPort=6818 +AuthType=auth/munge +#JobCredentialPrivateKey= +#JobCredentialPublicCertificate= +StateSaveLocation=/var/spool/slurm/ctld +SlurmdSpoolDir=/var/spool/slurm/d +SwitchType=switch/none +MpiDefault=none +SlurmctldPidFile=/var/run/slurmctld.pid +SlurmdPidFile=/var/run/slurmd.pid +ProctrackType=proctrack/pgid +#PluginDir= +#FirstJobId= +ReturnToService=0 +#MaxJobCount= +#PlugStackConfig= +#PropagatePrioProcess= +#PropagateResourceLimits= +#PropagateResourceLimitsExcept= +#Prolog= +#Epilog= +#SrunProlog= +#SrunEpilog= +#TaskProlog= +#TaskEpilog= +#TaskPlugin= +#TrackWCKey=no +#TreeWidth=50 +#TmpFS= +#UsePAM= +# +# TIMERS +SlurmctldTimeout=300 +SlurmdTimeout=300 +InactiveLimit=0 +MinJobAge=300 +KillWait=30 +Waittime=0 +# +# SCHEDULING +SchedulerType=sched/backfill +#SchedulerAuth= +SelectType=select/cons_tres +SelectTypeParameters=CR_Core +#PriorityType=priority/multifactor +#PriorityDecayHalfLife=14-0 +#PriorityUsageResetPeriod=14-0 +#PriorityWeightFairshare=100000 +#PriorityWeightAge=1000 +#PriorityWeightPartition=10000 +#PriorityWeightJobSize=1000 +#PriorityMaxAge=1-0 +# +# LOGGING +SlurmctldDebug=info +SlurmctldLogFile=/var/log/slurmctld.log +SlurmdDebug=info +SlurmdLogFile=/var/log/slurmd.log +JobCompType=jobcomp/none +#JobCompLoc= +# +# ACCOUNTING +#JobAcctGatherType=jobacct_gather/linux +#JobAcctGatherFrequency=30 +# +# COMPUTE NODES +NodeName=linux[1-32] Procs=1 State=UNKNOWN +PartitionName=debug Nodes=ALL Default=YES MaxTime=INFINITE State=UP diff --git a/docker/slurmdbd.conf b/docker/slurmdbd.conf new file mode 100644 index 0000000..1e36286 --- /dev/null +++ b/docker/slurmdbd.conf @@ -0,0 +1,39 @@ +# +# Example slurmdbd.conf file. +# +# See the slurmdbd.conf man page for more information. +# +# Archive info +#ArchiveJobs=yes +#ArchiveDir="/tmp" +#ArchiveSteps=yes +#ArchiveScript= +#JobPurge=12 +#StepPurge=1 +# +# Authentication info +AuthType=auth/munge +#AuthInfo=/var/run/munge/munge.socket.2 +# +# slurmDBD info +DbdAddr=localhost +DbdHost=localhost +#DbdPort=7031 +SlurmUser=slurm +#MessageTimeout=300 +DebugLevel=verbose +#DefaultQOS=normal,standby +LogFile=/var/log/slurm/slurmdbd.log +PidFile=/var/run/slurmdbd.pid +#PluginDir=/usr/lib/slurm +#PrivateData=accounts,users,usage,jobs +#TrackWCKey=yes +# +# Database info +StorageType=accounting_storage/mysql +#StorageHost=localhost +#StoragePort=1234 +StoragePass=password +StorageUser=slurm +#StorageLoc=slurm_acct_db + diff --git a/docker/sshd_config b/docker/sshd_config new file mode 100644 index 0000000..7ca03bf --- /dev/null +++ b/docker/sshd_config @@ -0,0 +1,2 @@ +ChallengeResponseAuthentication no +PasswordAuthentication no