OpenMPI with ROCm Support

❯ cat rocm-mpi-install.txt
# ubuntu 22.04 LTS

apt update
apt install git vim libtool flex
apt install build-essential

# make sure user ubuntu is part of render group
ubuntu@gpu4:~/scratch/osu/osu-micro-benchmarks-7.2$ id
uid=1000(ubuntu) gid=1000(ubuntu) groups=1000(ubuntu),4(adm),20(dialout),24(cdrom),25(floppy),27(sudo),29(audio),30(dip),44(video),46(plugdev),110(render),119(netdev),120(lxd)

mkdir ~/scratch
cd ~/scratch

# make ucx
git clone https://github.com/openucx/ucx.git -b v1.15.x
cd ucx
./autogen.sh
mkdir build
cd build
../configure --prefix=/opt/apps/ucx/1.15 --with-rocm=/opt/rocm
make -j 64
sudo make -j 64 install

# make openmpi
mkdir ~/scratch/ompi
cd ~/scratch/ompi
git clone --recursive https://github.com/open-mpi/ompi.git -b v5.0.x
cd ompi
./autogen.pl
mkdir build
cd build
../configure --prefix=/opt/apps/ompi/5.0.x --with-ucx=/opt/apps/ucx/1.15 --with-rocm=/opt/rocm
make -j 64
make -j 64 install

# rocm OSU benchmarks
mkdir ~/scratch/osu
cd ~/scratch/osu
wget http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-7.2.tar.gz
tar zxvf osu-micro-benchmarks-7.2.tar.gz
cd osu-micro-benchmarks-7.2/
./configure --enable-rocm --with-rocm=/opt/rocm CC=/opt/apps/ompi/5.0.x/bin/mpicc CXX=/opt/apps/ompi/5.0.x/bin/mpicxx LDFLAGS="-L/opt/apps/ompi/5.0.x/lib -lmpi -L/opt/rocm/lib  -D__HIP_PLATFORM_HCC__ -D__HIP_PLATFORM_AMD__ -I/opt/rocm/include -I/opt/rocm/lib/llvm/lib/clang/17.0.0 -lamdhip64" CXXFLAGS="-D__HIP_PLATFORM_AMD__ -std=c++11" CFLAGS="-D__HIP_PLATFORM_AMD__"
make -j 64

# to run the tests
ubuntu@gpu4:~/scratch/osu/osu-micro-benchmarks-7.2$ cat run_script.sh
#!/bin/bash

for i in {0..7}
do
	for j in {0..7}
	do
		export HIP_VISIBLE_DEVICES=${i},${j}
		echo $HIP_VISIBLE_DEVICES
	        mpirun -np 2 -x UCX_TLS=sm,self,rocm --mca pml ucx ./c/mpi/pt2pt/standard/osu_bw D D | tail -n 1
	done
done
this seems to speed up BW when gpu cards are on the same die.
export HSA_ENABLE_SDMA=0