OpenHPC: Adding Infiniband Support - Revision history

David at 12:14, 8 November 2016

2016-11-08T12:14:00Z

David at 13:44, 3 October 2016

2016-10-03T13:44:21Z

Adam at 11:55, 16 December 2015

2015-12-16T11:55:04Z

Adam: Created page with "== Adding Mellanox OFED into the CHROOT == '''Before proceeding ensure the following files appear as they do below''' vi /etc/warewulf/provision.conf #Ensure exclude looks..."

2015-12-16T11:42:07Z

Created page with "== Adding Mellanox OFED into the CHROOT == '''Before proceeding ensure the following files appear as they do below''' vi /etc/warewulf/provision.conf #Ensure exclude looks..."

New page

== Adding Mellanox OFED into the CHROOT ==

'''Before proceeding ensure the following files appear as they do below'''

vi /etc/warewulf/provision.conf #Ensure exclude looks like this.
<syntaxhighlight>
exclude += /tmp/*
exclude += /var/log/*
exclude += /var/chroots/*
#exclude += /var/cache
exclude += /usr/src
#exclude += /usr/share
#exclude += /home/*
</syntaxhighlight>
vi /etc/warewulf/bootstrap.conf #Hash out all Infiniband drivers

<syntaxhighlight>
# Infiniband drivers and Mellanox drivers
#drivers += ib_ipath, ib_iser, ib_srpt, ib_sdp, ib_mthca, ib_qib, iw_cxgb3, cxgb3
#drivers += iw_nes, mlx4_ib, ib_srp, ib_ipoib, ib_addr, rdma_cm, ib_ucm
#drivers += ib_ucm, ib_uverbs, ib_umad, ib_cm, ib_mad, iw_cm, ib_core
#drivers += rdma_ucm, ib_sa, mlx4_en, mlx4_core
#drivers += rds, rds_rdma, rds_tcp, mlx4_vnic, mlx4_vnic_helper

#Unhash the modprobe for the Mellanox Modules
modprobe += mlx4_core log_num_mtts=20 log_mtts_per_seg=6, ib_srp
</syntaxhighlight>

'''Copy the unpacked tar file of the MLNX OFED to the Chroot and Install, so you're aware, there will be many error messages about proc not being mounted, ignore these the driver install is still taking place and will prompt you as needed'''

← Older revision		Revision as of 13:44, 3 October 2016
Line 35:		Line 35:

	wwsh node set --netdev=ib0 --ipaddr=some.ip.address.here --netmask=some.netmask.here nodename		wwsh node set --netdev=ib0 --ipaddr=some.ip.address.here --netmask=some.netmask.here nodename
		+	</syntaxhighlight>
		+
		+
		+	== Setting up Fast Ethernet Interface ==
		+	Steps above work fine, but renaming to enp2
		+
		+	<syntaxhighlight>
		+	[root@head1 ~]# wwsh file show ifcfg-enp2s0f0.ww
		+	DEVICE=enp2s0f0
		+	BOOTPROTO=static
		+	IPADDR=%{NETDEVS::ENP2S0F0::IPADDR}
		+	NETMASK=%{NETDEVS::ENP2S0F0::NETMASK}
		+	ONBOOT=yes
		+	NM_CONTROLLED=no
		+	DEVTIMEOUT=5
		+	</syntaxhighlight>
		+
		+	Script to setup
		+
		+	<syntaxhighlight>
		+	#!/bin/bash
		+
		+	IP_ADDR=11
		+	for nodenum in {01..99}
		+	do
		+	echo "Setting up node${nodenum}"
		+	echo "wwsh node set --netdev=enp2s0f0 --ipaddr=10.10.30.${IP_ADDR} --netmask=255.255.255.0 node${nodenum}"
		+	wwsh node set --netdev=enp2s0f0 --ipaddr=10.10.30.${IP_ADDR} --netmask=255.255.255.0 node${nodenum}
		+	wwsh -y provision set node${nodenum} --fileadd ifcfg-enp2s0f0.ww
		+	((IP_ADDR++))
		+	done
		+
		+	for nodenum in {100..119}
		+	do
		+	echo "Setting up node${nodenum}"
		+	echo "wwsh node set --netdev=enp2s0f0 --ipaddr=10.10.30.${IP_ADDR} --netmask=255.255.255.0 node${nodenum}"
		+	wwsh node set --netdev=enp2s0f0 --ipaddr=10.10.30.${IP_ADDR} --netmask=255.255.255.0 node${nodenum}
		+	wwsh -y provision set node${nodenum} --fileadd ifcfg-enp2s0f0.ww
		+	((IP_ADDR++))
		+	done
		+
		+	for nodenum in {120..128}
		+	do
		+	echo "Setting up storagenode${nodenum}"
		+	echo "wwsh node set --netdev=enp2s0f0 --ipaddr=10.10.30.${IP_ADDR} --netmask=255.255.255.0 storagenode${nodenum}"
		+	wwsh node set --netdev=enp2s0f0 --ipaddr=10.10.30.${IP_ADDR} --netmask=255.255.255.0 storageode${nodenum}
		+	wwsh -y provision set storagenode${nodenum} --fileadd ifcfg-enp2s0f0.ww
		+	((IP_ADDR++))
		+	done
	</syntaxhighlight>		</syntaxhighlight>

← Older revision		Revision as of 11:55, 16 December 2015
Line 28:		Line 28:

	'''Copy the unpacked tar file of the MLNX OFED to the Chroot and Install, so you're aware, there will be many error messages about proc not being mounted, ignore these the driver install is still taking place and will prompt you as needed'''		'''Copy the unpacked tar file of the MLNX OFED to the Chroot and Install, so you're aware, there will be many error messages about proc not being mounted, ignore these the driver install is still taking place and will prompt you as needed'''
		+
		+	<syntaxhighlight>
		+	wwsh file import /opt/ohpc/pub/examples/network/centos/ifcfg-ib0.ww
		+	wwsh -y file set ifcfg-ib0.ww --path=/etc/sysconfig/network-scripts/ifcfg-ib0
		+	wwsh -y provision set c0 --files=ifcfg-ib0.ww
		+
		+	wwsh node set --netdev=ib0 --ipaddr=some.ip.address.here --netmask=some.netmask.here nodename
		+	</syntaxhighlight>

@@ Line 3: / Line 3: @@
 '''Before proceeding ensure the following files appear as they do below'''
-vi /etc/warewulf/provision.conf		#Ensure exclude looks like this.
+<tt>vi /etc/warewulf/provision.conf</tt>		#Ensure exclude looks like this.
 <syntaxhighlight>
 exclude += /tmp/*