Rocks: Adding an Infiniband Network on Rocks 5.4

Infiniband Script to add IPoIB to headnode and all compute nodes

• Tested on Rocks 5.4, OFED-1.5.3, CentOS 5.5
• The script below will add all of the headnode and compute nodes' ib0 interface to the database and create /etc/sysconfig/network-scripts/ifcfg-ib0.

Assumptions

• Compute nodes = compute-0-[1..20]
• Starting IP Address for compute nodes = 192.168.2.100
• Headnode = ironman
• Headnode IP Address = 192.168.2.99
• InfiniBand Network name = ibnet

Please edit the script below to reflect your network and number of nodes.

#!/bin/bash

echo "Working on: rocks add network ibnet subnet=192.168.2.0 netmask=255.255.255.0";
 rocks add network ibnet subnet=192.168.2.0 netmask=255.255.255.0;
echo "Done...";

echo "Setting up ib0 on the headnode named ironman..."
 rocks add host interface ironman ib0
 rocks set host interface ip ironman ib0 192.168.2.99         
 rocks set host interface name ironman ib0 iironman
 rocks set host interface module ironman ib0 ib_ipoib
 rocks set host interface subnet ironman ib0 ibnet       
echo "Done..."

#Set compute nodes
IP=100;
NODE=1;
NNODES=21

echo "Now working on the compute nodes..."
while [ $NODE -lt $NNODES ]; do  
        echo "Adding ib0 network to compute-0-$NODE"
        rocks add host interface compute-0-$NODE ib0 
        rocks set host interface ip compute-0-$NODE ib0 192.168.2.$IP; 
        rocks set host interface name compute-0-$NODE ib0 icompute-0-$NODE;
        rocks set host interface module compute-0-$NODE ib0 ib_ipoib;
        rocks set host interface subnet compute-0-$NODE ib0 ibnet; 
    
        let IP++; 
        let NODE++; 
done;
echo "Done..."

#synchronise
echo "Now sync config... ";
rocks sync config;
echo "Done..."

echo "Now sync host network compute..."
NODE=1;
NNODES=21
while [ $NODE -lt $NNODES ]; do
        echo "Syncing network ib0 on compute-0-$NODE...";
        rocks sync host network compute-0-$NODE;
        let NODE++; 
done
echo "Done..."

#Check if all is ok in the ibnet network
echo "Let us double check if this has been set up correctly..."
rocks report host | grep ibnet
echo "DONE..."

#Let us bring up the ib0 interfaces
echo "Now lets bring-up the ib0 interfaces UP on the compute nodes..."
IP=100
for i in {1..20}
do
	echo "compute-0-${i}"
	ssh compute-0-${i} ifconfig ib0 192.168.2.${n} netmask 255.255.255.0 up
	ssh compute-0-${i} ifconfig ib0 | grep "inet addr"
	let n++
	sleep 1
done

echo "Now lets bring-up the ib0 interfaces UP on the headnode..."
ifconfig ib0 192.168.2.99 netmask 255.255.255.0 up
ifconfig ib0 | grep "inet addr"