Pytorchでの分散学習

1. pytorch + horovod 環境の構築手順

[username@es1 ~]$ qrsh -g grpname -l rt_G.small=1

[username@g0001 ~]$ module load python/3.6/3.6.5 cuda/10.1/10.1.243 cudnn/7.6/7.6.5 nccl/2.5/2.5.6-1 openmpi/2.1.6 gcc/7.4.0

[username@g0001 ~]$ python3 -m venv ~/venv/pytorch+horovod

[username@g0001 ~]$ source ~/venv/pytorch+horovod/bin/activate

(pytorch+horovod) [username@g0001 ~]$ pip3 install --upgrade pip setuptools

(pytorch+horovod) [username@g0001 ~]$ pip3 install torch torchvision

(pytorch+horovod) [username@g0001 ~]$ HOROVOD_WITH_PYTORCH=1 HOROVOD_GPU_OPERATIONS=NCCL HOROVOD_NCCL_HOME=$NCCL_HOME pip3 install --no-cache-dir horovod

(pytorch+horovod) [username@g0001 ~]$ deactivate

[username@g0001 ~]$ exit

2. 実行スクリプト（例）

#!/bin/sh

#$ -l rt_F=2

#$ -l h_rt=1:23:45

#$ -j y

#$ -cwd

source /etc/profile.d/modules.sh

module load python/3.6/3.6.5 cuda/10.1/10.1.243 cudnn/7.6/7.6.5 nccl/2.5/2.5.6-1 openmpi/2.1.6 gcc/7.4.0

source ~/venv/pytorch+horovod/bin/activate

NUM_NODES=${NHOSTS}

NUM_GPUS_PER_NODE=4

NUM_GPUS_PER_SOCKET=$(expr ${NUM_GPUS_PER_NODE} / 2)

NUM_PROCS=$(expr ${NUM_NODES} \* ${NUM_GPUS_PER_NODE})

MPIOPTS="-np ${NUM_PROCS} -map-by ppr:${NUM_GPUS_PER_NODE}:node -mca pml ob1 -mca btl ^openib -mca btl_tcp_if_include bond0"

mpirun ${MPIOPTS} python3 /<path to directory>/pytorch_mnist.py

deactivate