Add code for DDP tutorial series [PR 2 / 3]

distributed/ddp-tutorial-series/slurm/config.yaml.template

@@ -0,0 +1,33 @@
+Region: us-east-1
+
+Image:
+  Os: ubuntu1804
+
+SharedStorage:
+  - MountDir: /shared
+    Name: shared-fs
+    StorageType: FsxLustre
+    FsxLustreSettings:
+      StorageCapacity: 1200
+      DeploymentType: SCRATCH_1
+      StorageType: SSD
+
+HeadNode:
+  InstanceType: c5.xlarge
+  Networking:
+    SubnetId: subnet-xxxxxxx
+  Ssh:
+    KeyName: your-keyname-file
+
+Scheduling:
+  Scheduler: slurm
+  SlurmQueues:
+  - Name: train
+    ComputeResources:
+    - Name: p32xlarge
+      InstanceType: p3.2xlarge
+      MinCount: 0
+      MaxCount: 5
+    Networking:
+      SubnetIds:
+      - subnet-xxxxxxx

distributed/ddp-tutorial-series/slurm/sbatch_run.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+#SBATCH --job-name=multinode-example
+#SBATCH --nodes=4
+#SBATCH --ntasks=4
+#SBATCH --gpus-per-task=1
+#SBATCH --cpus-per-task=4
+
+nodes=( $( scontrol show hostnames $SLURM_JOB_NODELIST ) )
+nodes_array=($nodes)
+head_node=${nodes_array[0]}
+head_node_ip=$(srun --nodes=1 --ntasks=1 -w "$head_node" hostname --ip-address)
+
+echo Node IP: $head_node_ip
+export LOGLEVEL=INFO
+
+srun torchrun \
+--nnodes 4 \
+--nproc_per_node 1 \
+--rdzv_id $RANDOM \
+--rdzv_backend c10d \
+--rdzv_endpoint $head_node_ip:29500 \
+/shared/examples/multinode_torchrun.py 50 10

distributed/ddp-tutorial-series/slurm/setup_pcluster_slurm.md

@@ -0,0 +1,51 @@
+# Setup AWS cluster with pcluster
+
+## 1. Sign in to an AWS instance
+
+## 2. Install pcluster
+```
+pip3 install awscli -U --user
+pip3 install "aws-parallelcluster" --upgrade --user
+```
+
+## 3. Create a cluster config file
+```
+pcluster configure --config config.yaml
+```
+See config.yaml.template for an example
+
+
+## 4. Create the cluster
+```
+pcluster create-cluster --cluster-name dist-ml --cluster-configuration config.yaml
+```
+
+### 4a. Track progress
+```
+pcluster list-clusters
+```
+
+## 5. Login to cluster headnode
+```
+pcluster ssh --cluster-name dist-ml -i your-keyname-file
+```
+
+## 6. Install dependencies
+```
+sudo apt-get update
+sudo apt-get install -y python3-venv
+python3 -m venv /shared/venv/
+source /shared/venv/bin/activate
+pip install wheel
+echo 'source /shared/venv/bin/activate' >> ~/.bashrc
+```
+
+## 7. Download training code and install requirements
+```
+cd /shared
+git clone --depth 1 https://github.com/pytorch/examples;
+cd /shared/examples
+git filter-branch --prune-empty --subdirectory-filter distributed/ddp-tutorial-series
+python3 -m pip install setuptools==59.5.0
+pip install -r requirements.txt
+```