OpenHPC: Adding Infiniband Support

Adding Mellanox OFED into the CHROOT

Before proceeding ensure the following files appear as they do below

vi /etc/warewulf/provision.conf #Ensure exclude looks like this.

exclude += /tmp/*
exclude += /var/log/*
exclude += /var/chroots/*
#exclude += /var/cache
exclude += /usr/src
#exclude += /usr/share
#exclude += /home/*

vi /etc/warewulf/bootstrap.conf #Hash out all Infiniband drivers

# Infiniband drivers and Mellanox drivers
#drivers += ib_ipath, ib_iser, ib_srpt, ib_sdp, ib_mthca, ib_qib, iw_cxgb3, cxgb3
#drivers += iw_nes, mlx4_ib, ib_srp, ib_ipoib, ib_addr, rdma_cm, ib_ucm
#drivers += ib_ucm, ib_uverbs, ib_umad, ib_cm, ib_mad, iw_cm, ib_core
#drivers += rdma_ucm, ib_sa, mlx4_en, mlx4_core
#drivers += rds, rds_rdma, rds_tcp, mlx4_vnic, mlx4_vnic_helper

#Unhash the modprobe for the Mellanox Modules
modprobe += mlx4_core log_num_mtts=20 log_mtts_per_seg=6, ib_srp

Copy the unpacked tar file of the MLNX OFED to the Chroot and Install, so you're aware, there will be many error messages about proc not being mounted, ignore these the driver install is still taking place and will prompt you as needed

wwsh file import /opt/ohpc/pub/examples/network/centos/ifcfg-ib0.ww
wwsh -y file set ifcfg-ib0.ww --path=/etc/sysconfig/network-scripts/ifcfg-ib0
wwsh -y provision set c0 --files=ifcfg-ib0.ww

wwsh node set --netdev=ib0 --ipaddr=some.ip.address.here --netmask=some.netmask.here nodename

Setting up Fast Ethernet Interface

Steps above work fine, but renaming to enp2

[root@head1 ~]# wwsh file show ifcfg-enp2s0f0.ww
DEVICE=enp2s0f0
BOOTPROTO=static
IPADDR=%{NETDEVS::ENP2S0F0::IPADDR}
NETMASK=%{NETDEVS::ENP2S0F0::NETMASK}
ONBOOT=yes
NM_CONTROLLED=no
DEVTIMEOUT=5

Script to setup

#!/bin/bash

IP_ADDR=11
for nodenum in {01..99}
do
        echo "Setting up node${nodenum}"
        echo "wwsh node set --netdev=enp2s0f0  --ipaddr=10.10.30.${IP_ADDR} --netmask=255.255.255.0 node${nodenum}"
        wwsh node set --netdev=enp2s0f0  --ipaddr=10.10.30.${IP_ADDR} --netmask=255.255.255.0 node${nodenum}
        wwsh -y provision set node${nodenum} --fileadd ifcfg-enp2s0f0.ww
        ((IP_ADDR++))
done

for nodenum in {100..119}
do 
        echo "Setting up node${nodenum}"
        echo "wwsh node set --netdev=enp2s0f0  --ipaddr=10.10.30.${IP_ADDR} --netmask=255.255.255.0 node${nodenum}"
        wwsh node set --netdev=enp2s0f0  --ipaddr=10.10.30.${IP_ADDR} --netmask=255.255.255.0 node${nodenum}
        wwsh -y provision set node${nodenum} --fileadd ifcfg-enp2s0f0.ww
        ((IP_ADDR++))
done

for nodenum in {120..128}
do 
        echo "Setting up storagenode${nodenum}"
        echo "wwsh node set --netdev=enp2s0f0  --ipaddr=10.10.30.${IP_ADDR} --netmask=255.255.255.0 storagenode${nodenum}"
        wwsh node set --netdev=enp2s0f0  --ipaddr=10.10.30.${IP_ADDR} --netmask=255.255.255.0 storageode${nodenum}
        wwsh -y provision set storagenode${nodenum} --fileadd ifcfg-enp2s0f0.ww
        ((IP_ADDR++))
done