diff --git a/docker/Dockerfile b/recipes/devito-gcc-mpich/docker/Dockerfile similarity index 96% rename from docker/Dockerfile rename to recipes/devito-gcc-mpich/docker/Dockerfile index af2ef1e..730c2a9 100644 --- a/docker/Dockerfile +++ b/recipes/devito-gcc-mpich/docker/Dockerfile @@ -1,4 +1,5 @@ FROM ubuntu:18.04 +RUN rm /bin/sh && ln -s /bin/bash /bin/sh # set up base and ssh keys COPY ssh_config /root/.ssh/config diff --git a/docker/ssh_config b/recipes/devito-gcc-mpich/docker/ssh_config similarity index 100% rename from docker/ssh_config rename to recipes/devito-gcc-mpich/docker/ssh_config diff --git a/shipyard-config/config.yaml b/recipes/devito-gcc-mpich/shipyard-config/config.yaml similarity index 100% rename from shipyard-config/config.yaml rename to recipes/devito-gcc-mpich/shipyard-config/config.yaml diff --git a/shipyard-config/jobs.yaml b/recipes/devito-gcc-mpich/shipyard-config/jobs.yaml similarity index 96% rename from shipyard-config/jobs.yaml rename to recipes/devito-gcc-mpich/shipyard-config/jobs.yaml index f957335..6fd098f 100644 --- a/shipyard-config/jobs.yaml +++ b/recipes/devito-gcc-mpich/shipyard-config/jobs.yaml @@ -19,7 +19,7 @@ job_specifications: num_instances: pool_current_dedicated mpi: runtime: mpich - processes_per_node: 2 + processes_per_node: 1 options: - --bind-to socket pre_execution_command: lscpu; diff --git a/recipes/devito-gcc-mpich/shipyard-config/pool.yaml b/recipes/devito-gcc-mpich/shipyard-config/pool.yaml new file mode 100644 index 0000000..13a8662 --- /dev/null +++ b/recipes/devito-gcc-mpich/shipyard-config/pool.yaml @@ -0,0 +1,16 @@ +pool_specification: + id: devito-gcc-mpich8 + vm_configuration: + platform_image: + offer: UbuntuServer + publisher: Canonical + sku: 18.04-LTS + native: true + vm_count: + dedicated: 4 + low_priority: 0 + resize_timeout: 00:40:00 + vm_size: Standard_H8 + inter_node_communication_enabled: true + ssh: + username: shipyard diff --git a/recipes/devito-icc-intelmpi-drivers/docker/Dockerfile b/recipes/devito-icc-intelmpi-drivers/docker/Dockerfile new file mode 100644 index 0000000..5599a8c --- /dev/null +++ b/recipes/devito-icc-intelmpi-drivers/docker/Dockerfile @@ -0,0 +1,113 @@ +FROM centos:7.6.1810 +RUN rm /bin/sh && ln -s /bin/bash /bin/sh + +RUN yum install -y gcc gcc-c++ make vim htop + +# ICC +COPY conf_mpi_test.cfg /tmp/icc-config.cfg +ADD ./parallel_studio_xe_2020_cluster_edition_online /tmp/intel + + +RUN cd /tmp/intel && \ + ./install.sh --silent=/tmp/icc-config.cfg && \ + cd ../../ && \ + rm -rf /tmp/intel && \ + rm /tmp/icc-config.cfg + +ENV MANPATH=/usr/share/man:/usr/local/man +ENV LD_LIBRARY_PATH=/usr/lib:/usr/lib64 + +RUN ls /opt/intelicc/ +RUN mkdir /opt/intelicc/compilers_and_libraries_2020.0.166/linux/licenses/ +COPY x.lic /opt/intelicc/compilers_and_libraries_2020.0.166/linux/licenses/ +# set up workdir +ENV COMPILERVARS_ARCHITECTURE='intel64' +ENV COMPILERVARS_PLATFORM='linux' +ENV ICC='source /opt/intelicc/bin/iccvars.sh' +ENV MPIICC='source /opt/intelicc/bin/compilervars.sh' +ENV CC=icc + +RUN yum clean all +RUN rm -rf /opt/intelicc/advisor_2020.0.0.604394 +RUN rm -rf /opt/intelicc/vtune_profiler_2020.0.0.605129 +RUN rm -rf /opt/intelicc/inspector_2020.0.0.603904 + +# set up base +COPY ssh_config /root/.ssh/config +RUN yum install -y ca-certificates wget openssh-server openssh-clients net-tools \ + numactl-devel gtk2 atk cairo tcsh libnl3 tcl libmnl tk \ + htop vim git-core which pciutils lsof ethtool gcc-gfortran + +# set up ssh keys +RUN mkdir -p /var/run/sshd \ + && ssh-keygen -A \ + && sed -i 's/UsePAM yes/UsePAM no/g' /etc/ssh/sshd_config \ + && sed -i 's/#PermitRootLogin yes/PermitRootLogin yes/g' /etc/ssh/sshd_config \ + && sed -i 's/#RSAAuthentication yes/RSAAuthentication yes/g' /etc/ssh/sshd_config \ + && sed -i 's/#PubkeyAuthentication yes/PubkeyAuthentication yes/g' /etc/ssh/sshd_config \ + && ssh-keygen -f /root/.ssh/id_rsa -t rsa -N '' \ + && chmod 600 /root/.ssh/config \ + && chmod 700 /root/.ssh \ + && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys + +# nompi vars +ADD varsnompi.sh /opt/intelicc/bin/varsnompi.sh +ENV ICC='source /opt/intelicc/bin/varsnompi.sh' + + +##### MPI install +# mlnx (IB required) +ADD ./mlnx /tmp/mpi/mlnx +RUN $ICC\ + && F77=ifort FC=ifort CC=icc CXX=icpc /tmp/mpi/mlnx/mlnxofedinstall --user-space-only --without-fw-update --all --force \ + && rm -rf /tmp/mpi/mlnx + +# HPC-X (IB required) +ADD ./hpcx /opt/hpcx +ENV HPCX_HOME=/opt/hpcx + +RUN yum install -y systemd-devel libgudev1-devel libgudev1 zlib-devel zlib \ + libxml2 libxml2-devel binutils binutils-devel libpsm2-devel libpsm2 +# OpenMPi from HPC-x + +# download and install MPICH +RUN tar xfp ${HPCX_HOME}/sources/openmpi-gitclone.tar.gz +RUN cd openmpi-gitclone \ + && $ICC \ + && source $HPCX_HOME/hpcx-init.sh && hpcx_load \ + && CC=icc CXX=icpc F77=ifort FC=ifort ./configure \ + --with-ucx=$HPCX_HOME/ucx \ + --with-hcoll=/$HPCX_HOME/hcoll \ + --prefix=${HPCX_HOME}/ompi-icc \ + --with-platform=optimized \ + --enable-mca-no-build=btl-uct \ + --with-device=ch4:ucx 2>&1 | tee config-icc-output.log \ + && make -j32 all 2>&1 | tee build_icc.log \ + && make -j24 install 2>&1 | tee install_icc.log \ + && cd .. \ + && rm -rf openmpi-gitclone + +ENV OMPI_ALLOW_RUN_AS_ROOT=1 +ENV OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 + +# Devito +RUN yum install -y python3 python3-pip python3-devel python3-libs +RUN pip3 install --upgrade pip +RUN git clone https://github.com/devitocodes/devito.git +# RUN cd devito && $ICC && pip3 install --force-reinstall --no-cache-dir --upgrade -e . +RUN cd devito && CC=gcc && pip3 install --force-reinstall --no-cache-dir --upgrade -e . +RUN cd devito && $MPIICC && pip3 install mpi4py +RUN $MPIICC && DEVITO_ARCH=intel python3 devito/examples/seismic/acoustic/acoustic_example.py + +# +# Install storage queue +RUN pip3 install azure-storage-blob h5py segyio +RUN pip3 install azure-storage + +# Add application +# ADD ./tti /app/tti +# COPY setup-ompi.sh /opt/setup-ompi.sh + +# set up sshd on port 23 +EXPOSE 23 +CMD ["/usr/sbin/sshd", "-D", "-p", "23"] \ No newline at end of file diff --git a/recipes/devito-icc-intelmpi-drivers/docker/ssh_config b/recipes/devito-icc-intelmpi-drivers/docker/ssh_config new file mode 100644 index 0000000..e0e128e --- /dev/null +++ b/recipes/devito-icc-intelmpi-drivers/docker/ssh_config @@ -0,0 +1,4 @@ +Host * + Port 23 + StrictHostKeyChecking no + UserKnownHostsFile /dev/null diff --git a/recipes/devito-icc-intelmpi-drivers/shipyard-config/config.yaml b/recipes/devito-icc-intelmpi-drivers/shipyard-config/config.yaml new file mode 100644 index 0000000..6c43238 --- /dev/null +++ b/recipes/devito-icc-intelmpi-drivers/shipyard-config/config.yaml @@ -0,0 +1,13 @@ +batch_shipyard: + storage_account_settings: mystorageaccount +global_resources: + docker_images: + - gbisbas/devitoicc:v0.8dhc44 + volumes: + shared_data_volumes: + azureblob_vol: + volume_driver: azureblob + storage_account_settings: mystorageaccount + azure_blob_container_name: devito-bench + container_path: $AZ_BATCH_NODE_SHARED_DIR/bench-results + bind_options: rw \ No newline at end of file diff --git a/recipes/devito-icc-intelmpi-drivers/shipyard-config/jobs.yaml b/recipes/devito-icc-intelmpi-drivers/shipyard-config/jobs.yaml new file mode 100644 index 0000000..35f3654 --- /dev/null +++ b/recipes/devito-icc-intelmpi-drivers/shipyard-config/jobs.yaml @@ -0,0 +1,30 @@ +job_specifications: +- id: bench-icc-intelmpi + shared_data_volumes: + - azureblob_vol + tasks: + - docker_image: gbisbas/devitoicc:v0.8dhc44 + environment_variables: + DEVITO_ARCH: intel + OMP_NUM_THREADS: 22 + DEVITO_MPI: 1 + DEVITO_OPENMP: 1 + KMP_AFFINITY: compact, granularity=core + DEVITO_LOGGING: DEBUG + # I_MPI_DEBUG: 2 + I_MPI_PIN: 1 + I_MPI_PIN_DOMAIN: sockets + # I_MPI_HYDRA_DEBUG: 1 + FI_LOG_LEVEL: debug + LC_ALL: 'en_US.UTF-8' + UCX_TLS: ud,sm,self + default_working_dir: container + multi_instance: + num_instances: pool_current_dedicated + mpi: + runtime: intelmpi + processes_per_node: 1 + pre_execution_command: source /opt/intelicc/compilers_and_libraries_2020.0.166/linux/bin/compilervars.sh -arch intel64 -platform linux; which icc; which mpicc; which mpirun; which mpiexec; ls; + command: python3 devito/examples/seismic/acoustic/acoustic_example.py + # IMB-MPI1 + # python3 devito/benchmarks/user/benchmark.py bench -P elastic --tn 400 -d 100 100 100 --resultsdir $AZ_BATCH_NODE_SHARED_DIR/results_bench --arch intel --dse advanced --dle advanced -x 1 \ No newline at end of file diff --git a/recipes/devito-icc-intelmpi-drivers/shipyard-config/pool.yaml b/recipes/devito-icc-intelmpi-drivers/shipyard-config/pool.yaml new file mode 100644 index 0000000..d0724b8 --- /dev/null +++ b/recipes/devito-icc-intelmpi-drivers/shipyard-config/pool.yaml @@ -0,0 +1,15 @@ +pool_specification: + id: devito-drivers-icc-intelmpi + vm_configuration: + platform_image: + publisher: OpenLogic + offer: CentOS-HPC + sku: '7.6' + vm_count: + dedicated: 4 + low_priority: 0 + resize_timeout: 00:40:00 + vm_size: Standard_HC44rs + inter_node_communication_enabled: true + ssh: + username: shipyard diff --git a/recipes/devito-icc-intelmpi/docker/Dockerfile b/recipes/devito-icc-intelmpi/docker/Dockerfile new file mode 100644 index 0000000..b8434e7 --- /dev/null +++ b/recipes/devito-icc-intelmpi/docker/Dockerfile @@ -0,0 +1,54 @@ +FROM centos:7.6.1810 +RUN rm /bin/sh && ln -s /bin/bash /bin/sh + +# set up base and ssh keys +COPY ssh_config /root/.ssh/config + +RUN yum swap -y fakesystemd systemd +RUN yum -y update \ + && yum install -y \ + build-essential ca-certificates wget openssh-client numactl openssh-server curl gcc gcc-c++ systemd-devel ucx libgudev1-devel libgudev1 zlib-devel zlib libxml2 libxml2-devel binutils binutils-devel numactl-devel install libpsm2 libpsm2-devel librdmacm librdmacm-devel\ + && yum clean all \ + && mkdir -p /var/run/sshd \ + && ssh-keygen -A \ + && sed -i 's/UsePAM yes/UsePAM no/g' /etc/ssh/sshd_config \ + && sed -i 's/#PermitRootLogin yes/PermitRootLogin yes/g' /etc/ssh/sshd_config \ + && sed -i 's/#RSAAuthentication yes/RSAAuthentication yes/g' /etc/ssh/sshd_config \ + && sed -i 's/#PubkeyAuthentication yes/PubkeyAuthentication yes/g' /etc/ssh/sshd_config \ + && ssh-keygen -f /root/.ssh/id_rsa -t rsa -N '' \ + && chmod 600 /root/.ssh/config \ + && chmod 700 /root/.ssh \ + && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys + +ENV DEBIAN_FRONTEND=noninteractive + +RUN yum install -y cpio libgtk2.0 xorg openbox +COPY conf_mpi_test.cfg /tmp/icc-config.cfg +ADD /parallel_studio_xe_2020_cluster_edition_online /tmp/intel +RUN cd /tmp/intel && \ + ./install.sh --silent=/tmp/icc-config.cfg && \ + cd ../../ && \ + rm -rf /tmp/intel && \ + rm /tmp/icc-config.cfg + +RUN mkdir /opt/intelicc/compilers_and_libraries_2020.0.166/linux/licenses/ +COPY x.lic /opt/intelicc/compilers_and_libraries_2020.0.166/linux/licenses/ +RUN ls /opt/intelicc/compilers_and_libraries_2020.0.166/linux/licenses/ + +RUN source /opt/intelicc/compilers_and_libraries/linux/bin/compilervars.sh -arch intel64 -platform linux && icc --help +RUN source /opt/intelicc/compilers_and_libraries/linux/mpi/intel64/bin/mpivars.sh && mpirun --help + +# RUN ls /opt/intel/compilers_and_libraries/linux/mpi/intel64/bin/ + +# Utils +RUN yum install -y htop vim git-core + +# Devito +RUN yum install -y python3 python3-pip python3-devel +RUN pip3 install --upgrade pip +RUN git clone https://github.com/devitocodes/devito.git +RUN source /opt/intelicc/compilers_and_libraries/linux/mpi/intel64/bin/mpivars.sh && cd devito && pip3 install -e .[extras] + +# set up sshd on port 23 +EXPOSE 23 +CMD ["/usr/sbin/sshd", "-D", "-p", "23"] diff --git a/recipes/devito-icc-intelmpi/docker/conf_mpi_test.cfg b/recipes/devito-icc-intelmpi/docker/conf_mpi_test.cfg new file mode 100644 index 0000000..4619d04 --- /dev/null +++ b/recipes/devito-icc-intelmpi/docker/conf_mpi_test.cfg @@ -0,0 +1,19 @@ +ACCEPT_EULA=accept +CONTINUE_WITH_OPTIONAL_ERROR=yes +PSET_INSTALL_DIR=/opt/intel +CONTINUE_WITH_INSTALLDIR_OVERWRITE=yes +PSET_MODE=install +ACTIVATION_SERIAL_NUMBER=xxxx-yyyyyyyy +ACTIVATION_TYPE=serial_number +AMPLIFIER_SAMPLING_DRIVER_INSTALL_TYPE=build +AMPLIFIER_DRIVER_ACCESS_GROUP=vtune +AMPLIFIER_DRIVER_PERMISSIONS=660 +AMPLIFIER_LOAD_DRIVER=yes +AMPLIFIER_C_COMPILER=/usr/bin/gcc +AMPLIFIER_KERNEL_SRC_DIR=/lib/modules/5.3.0-29-generic/build +AMPLIFIER_MAKE_COMMAND=/usr/bin/make +AMPLIFIER_INSTALL_BOOT_SCRIPT=yes +AMPLIFIER_DRIVER_PER_USER_MODE=no +INTEL_SW_IMPROVEMENT_PROGRAM_CONSENT=no +ARCH_SELECTED=ALL +COMPONENTS=;intel-conda-index-tool__x86_64;intel-comp__x86_64;intel-comp-32bit__x86_64;intel-comp-doc__noarch;intel-comp-l-all-common__noarch;intel-comp-l-all-vars__noarch;intel-comp-nomcu-vars__noarch;intel-comp-ps-32bit__x86_64;intel-comp-ps__x86_64;intel-comp-ps-ss-bec__x86_64;intel-comp-ps-ss-bec-32bit__x86_64;intel-openmp__x86_64;intel-openmp-32bit__x86_64;intel-openmp-common__noarch;intel-openmp-common-icc__noarch;intel-openmp-common-ifort__noarch;intel-openmp-ifort__x86_64;intel-openmp-ifort-32bit__x86_64;intel-tbb-libs-32bit__x86_64;intel-tbb-libs__x86_64;intel-idesupport-icc-common-ps__noarch;intel-conda-intel-openmp-linux-64-shadow-package__x86_64;intel-conda-intel-openmp-linux-32-shadow-package__x86_64;intel-conda-icc_rt-linux-64-shadow-package__x86_64;intel-icc__x86_64;intel-icc-32bit__x86_64;intel-c-comp-common__noarch;intel-icc-common__noarch;intel-icc-common-ps__noarch;intel-icc-doc__noarch;intel-icc-ps__x86_64;intel-icc-ps-ss-bec__x86_64;intel-icc-ps-ss-bec-32bit__x86_64;intel-icx__x86_64;intel-icx-common__noarch;intel-tbb-devel-32bit__x86_64;intel-tbb-devel__x86_64;intel-tbb-common__noarch;intel-tbb-doc__noarch;intel-conda-tbb-linux-64-shadow-package__x86_64;intel-conda-tbb-linux-32-shadow-package__x86_64;intel-conda-tbb-devel-linux-64-shadow-package__x86_64;intel-conda-tbb-devel-linux-32-shadow-package__x86_64;intel-imb__x86_64;intel-mpi-rt__x86_64;intel-mpi-sdk__x86_64;intel-mpi-doc__x86_64;intel-mpi-samples__x86_64;intel-conda-impi_rt-linux-64-shadow-package__x86_64;intel-conda-impi-devel-linux-64-shadow-package__x86_64;intel-gdb__x86_64;intel-gdb-source__noarch;intel-gdb-python-source__noarch;intel-gdb-common__noarch;intel-gdb-common-ps__noarch;intel-icsxe__noarch;intel-psxe-common__noarch;intel-psxe-doc__noarch;intel-psxe-common-doc__noarch;intel-icsxe-doc__noarch;intel-psxe-licensing__noarch;intel-psxe-licensing-doc__noarch;intel-icsxe-pset diff --git a/recipes/devito-icc-intelmpi/docker/ssh_config b/recipes/devito-icc-intelmpi/docker/ssh_config new file mode 100644 index 0000000..e0e128e --- /dev/null +++ b/recipes/devito-icc-intelmpi/docker/ssh_config @@ -0,0 +1,4 @@ +Host * + Port 23 + StrictHostKeyChecking no + UserKnownHostsFile /dev/null diff --git a/recipes/devito-icc-intelmpi/shipyard-config/config.yaml b/recipes/devito-icc-intelmpi/shipyard-config/config.yaml new file mode 100644 index 0000000..04fef20 --- /dev/null +++ b/recipes/devito-icc-intelmpi/shipyard-config/config.yaml @@ -0,0 +1,13 @@ +batch_shipyard: + storage_account_settings: mystorageaccount +global_resources: + docker_images: + - gbisbas/devitoicc:v0.7hc44 + volumes: + shared_data_volumes: + azureblob_vol: + volume_driver: azureblob + storage_account_settings: mystorageaccount + azure_blob_container_name: devito-bench + container_path: $AZ_BATCH_NODE_SHARED_DIR/bench-results + bind_options: rw \ No newline at end of file diff --git a/recipes/devito-icc-intelmpi/shipyard-config/jobs.yaml b/recipes/devito-icc-intelmpi/shipyard-config/jobs.yaml new file mode 100644 index 0000000..a140cd5 --- /dev/null +++ b/recipes/devito-icc-intelmpi/shipyard-config/jobs.yaml @@ -0,0 +1,30 @@ +job_specifications: +- id: bench-icc-intelmpi + shared_data_volumes: + - azureblob_vol + tasks: + - docker_image: gbisbas/devitoicc:v0.7hc44 + environment_variables: + DEVITO_ARCH: intel + OMP_NUM_THREADS: 22 + DEVITO_MPI: full + DEVITO_OPENMP: 1 + KMP_AFFINITY: compact, granularity=core + DEVITO_LOGGING: DEBUG + I_MPI_DEBUG: 2 + I_MPI_PIN: 1 + I_MPI_PIN_DOMAIN: sockets + I_MPI_HYDRA_DEBUG: 1 + I_MPI_FABRICS: ofa + FI_LOG_LEVEL: debug + FI_PROVIDER: verbs + LC_ALL: en_US.utf8 + UCX_TLS: ud,sm,self + default_working_dir: container + multi_instance: + num_instances: pool_current_dedicated + mpi: + runtime: intelmpi + processes_per_node: 1 + pre_execution_command: source /opt/intelicc/compilers_and_libraries_2020.0.166/linux/bin/compilervars.sh -arch intel64 -platform linux; which icc; which mpicc; which mpirun; which mpiexec; + command: python3 devito/examples/seismic/acoustic/acoustic_example.py \ No newline at end of file diff --git a/recipes/devito-icc-intelmpi/shipyard-config/pool.yaml b/recipes/devito-icc-intelmpi/shipyard-config/pool.yaml new file mode 100644 index 0000000..f4d2c16 --- /dev/null +++ b/recipes/devito-icc-intelmpi/shipyard-config/pool.yaml @@ -0,0 +1,15 @@ +pool_specification: + id: devito-icc-intelmpi + vm_configuration: + platform_image: + publisher: OpenLogic + offer: CentOS-HPC + sku: '7.6' + vm_count: + dedicated: 4 + low_priority: 0 + resize_timeout: 00:40:00 + vm_size: Standard_HC44rs + inter_node_communication_enabled: true + ssh: + username: shipyard diff --git a/recipes/devito-icc-mpich/docker/Dockerfile b/recipes/devito-icc-mpich/docker/Dockerfile new file mode 100644 index 0000000..6cc07c9 --- /dev/null +++ b/recipes/devito-icc-mpich/docker/Dockerfile @@ -0,0 +1,51 @@ +FROM ubuntu:18.04 +RUN rm /bin/sh && ln -s /bin/bash /bin/sh + +# set up base and ssh keys +COPY ssh_config /root/.ssh/config + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential ca-certificates wget openssh-client openssh-server \ + mpich libmpich-dev \ + && mkdir -p /var/run/sshd \ + && ssh-keygen -A \ + && sed -i 's/UsePAM yes/UsePAM no/g' /etc/ssh/sshd_config \ + && sed -i 's/#PermitRootLogin yes/PermitRootLogin yes/g' /etc/ssh/sshd_config \ + && sed -i 's/#RSAAuthentication yes/RSAAuthentication yes/g' /etc/ssh/sshd_config \ + && sed -i 's/#PubkeyAuthentication yes/PubkeyAuthentication yes/g' /etc/ssh/sshd_config \ + && ssh-keygen -f /root/.ssh/id_rsa -t rsa -N '' \ + && chmod 600 /root/.ssh/config \ + && chmod 700 /root/.ssh \ + && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get install -y cpio libgtk2.0 xorg openbox + +COPY /parallel_studio_xe_2020_cluster_edition_online/silent.cfg /tmp/icc-config.cfg +ADD /parallel_studio_xe_2020_cluster_edition_online /tmp/intel +RUN cd /tmp/intel && \ + ./install.sh --silent=/tmp/icc-config.cfg --components intel-mpi-psxe__x86_64 && \ + ./install.sh --silent=/tmp/icc-config.cfg --components intel-icc__x86_64 && \ + ./install.sh --silent=/tmp/icc-config.cfg --components intel-mpi-sdk__x86_64 && \ + ./install.sh --silent=/tmp/icc-config.cfg --components intel-mpi-rt__x86_64 && \ + cd ../../ && \ + rm -rf /tmp/intel && \ + rm /tmp/icc-config.cfg + +RUN ls opt/intel/compilers_and_libraries/linux/ +RUN source /opt/intel/compilers_and_libraries/linux/bin/compilervars.sh -arch intel64 -platform linux +RUN source /opt/intel/compilers_and_libraries/linux/mpi/intel64/bin/mpivars.sh + +# Utils +RUN apt-get install -y htop vim git-core + +# Devito +RUN apt-get install -y python3 python3-pip +RUN pip3 install --upgrade pip +RUN git clone https://github.com/devitocodes/devito.git +RUN source /opt/intel/compilers_and_libraries/linux/mpi/intel64/bin/mpivars.sh && cd devito && pip3 install -e .[extras] + +# set up sshd on port 23 +EXPOSE 23 +CMD ["/usr/sbin/sshd", "-D", "-p", "23"] diff --git a/recipes/devito-icc-mpich/docker/ssh_config b/recipes/devito-icc-mpich/docker/ssh_config new file mode 100644 index 0000000..e0e128e --- /dev/null +++ b/recipes/devito-icc-mpich/docker/ssh_config @@ -0,0 +1,4 @@ +Host * + Port 23 + StrictHostKeyChecking no + UserKnownHostsFile /dev/null diff --git a/recipes/devito-icc-mpich/shipyard-config/config.yaml b/recipes/devito-icc-mpich/shipyard-config/config.yaml new file mode 100644 index 0000000..04166f8 --- /dev/null +++ b/recipes/devito-icc-mpich/shipyard-config/config.yaml @@ -0,0 +1,13 @@ +batch_shipyard: + storage_account_settings: mystorageaccount +global_resources: + docker_images: + - gbisbas/devitoicc:v0.3 + volumes: + shared_data_volumes: + azureblob_vol: + volume_driver: azureblob + storage_account_settings: mystorageaccount + azure_blob_container_name: devito-bench + container_path: $AZ_BATCH_NODE_SHARED_DIR/bench-results + bind_options: rw \ No newline at end of file diff --git a/recipes/devito-icc-mpich/shipyard-config/jobs.yaml b/recipes/devito-icc-mpich/shipyard-config/jobs.yaml new file mode 100644 index 0000000..d197c65 --- /dev/null +++ b/recipes/devito-icc-mpich/shipyard-config/jobs.yaml @@ -0,0 +1,25 @@ +job_specifications: +- id: bench-icc-mpich + shared_data_volumes: + - azureblob_vol + tasks: + - docker_image: gbisbas/devitoicc:v0.3 + environment_variables: + DEVITO_ARCH: intel + OMP_NUM_THREADS: 8 + DEVITO_MPI: 1 + DEVITO_OPENMP: 1 + DEVITO_LOGGING: 'DEBUG' + LC_ALL: 'C.UTF-8' + LANG: 'C.UTF-8' + KMP_AFFINITY: verbose,scatter,granularity=fine + default_working_dir: container + multi_instance: + num_instances: pool_current_dedicated + mpi: + runtime: mpich + processes_per_node: 2 + options: + - --bind-to socket + pre_execution_command: source /opt/intel/compilers_and_libraries/linux/bin/compilervars.sh -arch intel64 -platform linux; + command: python3 devito/benchmarks/user/benchmark.py bench -P acoustic --tn 4000 -d 600 600 600 --resultsdir $AZ_BATCH_NODE_SHARED_DIR/results_bench --dse advanced --dle advanced --autotune off -x 1 \ No newline at end of file diff --git a/shipyard-config/pool.yaml b/recipes/devito-icc-mpich/shipyard-config/pool.yaml similarity index 93% rename from shipyard-config/pool.yaml rename to recipes/devito-icc-mpich/shipyard-config/pool.yaml index 675570f..0a12901 100644 --- a/shipyard-config/pool.yaml +++ b/recipes/devito-icc-mpich/shipyard-config/pool.yaml @@ -1,5 +1,5 @@ pool_specification: - id: devito-gcc-mpich + id: devito-icc-mpich vm_configuration: platform_image: offer: UbuntuServer diff --git a/shipyard-config/credentials.yaml b/shipyard-config/credentials.yaml deleted file mode 100644 index 517051e..0000000 --- a/shipyard-config/credentials.yaml +++ /dev/null @@ -1,9 +0,0 @@ -credentials: - batch: - account_key: - account_service_url: - storage: - mystorageaccount: - account: - account_key: - endpoint: core.windows.net diff --git a/shipyard-config/devitobench_poc.sh b/shipyard-config/devitobench_poc.sh deleted file mode 100644 index 01615a1..0000000 --- a/shipyard-config/devitobench_poc.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -# This bash scripts takes 2 arguments: -# Arg1 is installation path of shipyard -# Arg2 is the crentials file name - -shipyard pool add --credentials $1 -shipyard jobs add --credentials $1 diff --git a/shipyard-config/make-azure-jobs.py b/shipyard-config/make-azure-jobs.py deleted file mode 100644 index 0807021..0000000 --- a/shipyard-config/make-azure-jobs.py +++ /dev/null @@ -1,80 +0,0 @@ -import os - -import click - -from benchmark import option_simulation -import devito - - -@click.group() -def menu(): - pass - - -@menu.command(name='generate') -@option_simulation -@click.option('-id', default='bench', help='Job ID') -@click.option('-image', default='mloubout/devito-bench:v0.1', help='Devito docker image') -@click.option('-arch', default='gcc-8', help='Devito architecture') -@click.option('-nt', default='4', help='OpenMP number of threads') -@click.option('-runtime', default='mpich', help='MPI runtime') -@click.option('-ppn', default='1', help='Processes per node') -@click.option('-tn', default=50, help='Number of timesteps') -def generate(**kwargs): - join = lambda l: ' '.join('%d' % i for i in l) - args = dict(kwargs) - args['shape'] = join(args['shape']) - args['space_order'] = join(args['space_order']) - - args['home'] = os.path.dirname(os.path.dirname(devito.__file__)) - - template_header = """\ -job_specifications: -- id: %(id)s - shared_data_volumes: - - azureblob_vol - tasks: - - docker_image: %(image)s - environment_variables: - DEVITO_ARCH: %(arch)s - OMP_NUM_THREADS: %(nt)s - OMP_PROC_BIND: 'close' - DEVITO_MPI: 1 - DEVITO_OPENMP: 1 - OMP_PLACES: 'cores' - DEVITO_LOGGING: 'DEBUG' - LC_ALL: 'C.UTF-8' - LANG: 'C.UTF-8' - default_working_dir: container - multi_instance: - num_instances: pool_current_dedicated - mpi: - runtime: mpich - processes_per_node: %(ppn)s - options: - - --bind-to socket - command: python3 devito/benchmarks/user/benchmark.py bench -P acoustic --tn %(tn)s -d %(shape)s -so %(space_order)s --resultsdir $AZ_BATCH_NODE_SHARED_DIR/results_bench -x 3 -""" # noqa - - # Generate one PBS file for each `np` value - for nvm in kwargs['nt']: - args['nt'] = nvm - - cmds = [] - cmds = ' \n'.join(cmds) - - body = ' \n'.join([template_header % args, cmds]) - - with open('job_nn%d.gen.yaml' % int(nvm), 'w') as f: - f.write(body) - - -@menu.command(name='cleanup') -def cleanup(): - for f in os.listdir(): - if f.endswith('.gen.yaml'): - os.remove(f) - - -if __name__ == "__main__": - menu() diff --git a/shipyard-config/make-pool.py b/shipyard-config/make-pool.py deleted file mode 100644 index 0cdbb14..0000000 --- a/shipyard-config/make-pool.py +++ /dev/null @@ -1,70 +0,0 @@ -import os - -import click - -from benchmark import option_simulation -import devito - - -@click.group() -def menu(): - pass - - -@menu.command(name='generate') -@option_simulation -@click.option('-id', default='devito-bench', help='Pool ID') -@click.option('-dedvm', multiple=True, default=[1], help='Dedicated VMs') -@click.option('-lowvm', default='0', help='Low priority VMs') -@click.option('-vmsize', default='Standard_H16', help='VM size') -@click.option('-offer', default='UbuntuServer', help='Platform image offer') -@click.option('-pub', default='Canonical', help='Platform image publisher') -@click.option('-sku', default='18.04-LTS', help='Platform image SKU') -def generate(**kwargs): - join = lambda l: ' '.join('%d' % i for i in l) - args = dict(kwargs) - args['shape'] = join(args['shape']) - args['space_order'] = join(args['space_order']) - - args['home'] = os.path.dirname(os.path.dirname(devito.__file__)) - - template_header = """\ -pool_specification: - id: %(id)s - vm_configuration: - platform_image: - offer: %(offer)s - publisher: %(pub)s - sku: %(sku)s - native: true - vm_count: - dedicated: %(dedvm)s - low_priority: %(lowvm)s - vm_size: %(vmsize)s - inter_node_communication_enabled: true - ssh: - username: shipyard -""" # noqa - - # Generate one pool file for each `np` value - for nvm in kwargs['dedvm']: - args['dedvm'] = nvm - - cmds = [] - cmds = ' \n'.join(cmds) - - body = ' \n'.join([template_header % args, cmds]) - - with open('pbs_nn%d.gen.yaml' % int(nvm), 'w') as f: - f.write(body) - - -@menu.command(name='cleanup') -def cleanup(): - for f in os.listdir(): - if f.endswith('.gen.yaml'): - os.remove(f) - - -if __name__ == "__main__": - menu()