OpenHPC: Adding Infiniband Support

Adding Mellanox OFED into the CHROOT

Before proceeding ensure the following files appear as they do below

vi /etc/warewulf/provision.conf #Ensure exclude looks like this.

exclude += /tmp/*
exclude += /var/log/*
exclude += /var/chroots/*
#exclude += /var/cache
exclude += /usr/src
#exclude += /usr/share
#exclude += /home/*

vi /etc/warewulf/bootstrap.conf #Hash out all Infiniband drivers

# Infiniband drivers and Mellanox drivers
#drivers += ib_ipath, ib_iser, ib_srpt, ib_sdp, ib_mthca, ib_qib, iw_cxgb3, cxgb3
#drivers += iw_nes, mlx4_ib, ib_srp, ib_ipoib, ib_addr, rdma_cm, ib_ucm
#drivers += ib_ucm, ib_uverbs, ib_umad, ib_cm, ib_mad, iw_cm, ib_core
#drivers += rdma_ucm, ib_sa, mlx4_en, mlx4_core
#drivers += rds, rds_rdma, rds_tcp, mlx4_vnic, mlx4_vnic_helper

#Unhash the modprobe for the Mellanox Modules
modprobe += mlx4_core log_num_mtts=20 log_mtts_per_seg=6, ib_srp

Copy the unpacked tar file of the MLNX OFED to the Chroot and Install, so you're aware, there will be many error messages about proc not being mounted, ignore these the driver install is still taking place and will prompt you as needed

wwsh file import /opt/ohpc/pub/examples/network/centos/ifcfg-ib0.ww
wwsh -y file set ifcfg-ib0.ww --path=/etc/sysconfig/network-scripts/ifcfg-ib0
wwsh -y provision set c0 --files=ifcfg-ib0.ww

wwsh node set --netdev=ib0 --ipaddr=some.ip.address.here --netmask=some.netmask.here nodename