Lustre install for Unit.tv

From Define Wiki
Jump to navigation Jump to search

Preparation

Nodes

IML:

  1. CentOS 6.6
  2. Cobbler
    1. let kickstart file to only install system on first partition
  3. IML

MDS

  1. CentOS 6.6
  2. RAID:
    1. small partition for MGS
    2. large partition for MDT

OSS:

  1. CentOS 6.6
  2. RAID:
    1. 2 RAID arays: r6

Network

  1. management netowrk
    1. PXE network: 172.28.11.xx
    2. subnet for eth1 on iml:
      1. ip: 172.105.0.0
      2. IPMI network: 172.105.1.xx
  2. IB network (private)
    1. IP: 192.168.0.x
  3. ETH Network (private)
    1. IP: 192.168.1.x
  4. Switch
    1. enable ETH license
  5. Cards
    1. first port IB, second port eth
  • IML
    • IP: 172.28.0.114

Hardware

  • Network
    • all IB card should be set to port1 -> IB, port2 -> ETH
    • some port on IB switch should be ETH
  • for every pair of JBOD, connect network cable directly to each other

Installation Steps

1. install cobbler

1.1 follow the steps on wiki

1.2 check all services are running, httpd, dhcpd, xinetd, cobblerd

1.3 open the ports for cobbler and disable SELINUX 

[root@localhost ~]# cat /etc/sysconfig/iptables
# Generated by iptables-save v1.4.7 on Wed Jul  8 16:37:31 2015
*nat
:PREROUTING ACCEPT [0:0]
:POSTROUTING ACCEPT [44:2156]
:OUTPUT ACCEPT [48:2352]
-A POSTROUTING -o eth1 -j MASQUERADE 
COMMIT
# Completed on Wed Jul  8 16:37:31 2015
# Generated by iptables-save v1.4.7 on Wed Jul  8 16:37:31 2015
*filter
:INPUT ACCEPT [0:0]
:FORWARD ACCEPT [0:0]
:OUTPUT ACCEPT [31985:2140861]
-A INPUT -s 172.105.0.0/16 -p udp -m udp --dport 69 -j ACCEPT 
-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT 
-A INPUT -p icmp -j ACCEPT 
-A INPUT -i lo -j ACCEPT 
-A INPUT -p tcp -m state --state NEW -m tcp --dport 22 -j ACCEPT 
-A INPUT -p tcp -m tcp --dport 69 -j ACCEPT 
-A INPUT -p tcp -m tcp --dport 80 -j ACCEPT 
-A INPUT -p tcp -m tcp --dport 443 -j ACCEPT 
-A INPUT -p tcp -m tcp --dport 25151 -j ACCEPT 
-A INPUT -p udp -m state --state NEW -m udp --dport 69 -j ACCEPT 
-A FORWARD -i eth1 -o eth0 -m state --state RELATED,ESTABLISHED -j ACCEPT 
-A FORWARD -i eth0 -o eth1 -j ACCEPT 
COMMIT
# Completed on Wed Jul  8 16:37:31 2015

1.4 enable traffic forwarding 

[root@localhost ~]# cat /etc/rc.local 
#!/bin/sh
#
# This script will be executed *after* all the other init scripts.
# You can put your own initialization stuff in here if you don't
# want to do the full Sys V style init stuff.

touch /var/lock/subsys/local
iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE
iptables -A FORWARD -i eth0 -o eth1 -m state --state RELATED,ESTABLISHED -j ACCEPT
iptables -A FORWARD -i eth1 -o eth0 -j ACCEPT

reference content for kickstart file to do the partition: 

# Allow anaconda to partition the system as needed
clearpart --all

part raid.sda1  --asprimary     --fstype="raid" --ondisk=sda    --size=512
part raid.sda2  --asprimary     --fstype="raid" --ondisk=sda    --size=20000
part raid.sda3                  --fstype="raid" --ondisk=sda    --size=5000
part raid.sda4                  --fstype="raid" --ondisk=sda    --size=10000
part raid.sda5                  --fstype="raid" --ondisk=sda    --size=5000
part raid.sda6                  --fstype="raid" --ondisk=sda    --size=64000
part raid.sda7                  --fstype="raid" --ondisk=sda    --size=1        --grow

part raid.sdb1 --asprimary      --fstype="raid" --ondisk=sdb    --size=512
part raid.sdb2 --asprimary      --fstype="raid" --ondisk=sdb    --size=20000
part raid.sdb3                  --fstype="raid" --ondisk=sdb    --size=5000
part raid.sdb4                  --fstype="raid" --ondisk=sdb    --size=10000
part raid.sdb5                  --fstype="raid" --ondisk=sdb    --size=5000
part raid.sdb6                  --fstype="raid" --ondisk=sdb    --size=64000
part raid.sdb7                  --fstype="raid" --ondisk=sdb    --size=1        --grow

raid /boot              --device=md0 --fstype="ext4" --level=RAID1 raid.sda1 raid.sdb1
raid /                  --device=md1 --fstype="ext4" --level=RAID1 raid.sda2 raid.sdb2
raid /tmp               --device=md2 --fstype="ext4" --level=RAID1 raid.sda3 raid.sdb3
raid /var               --device=md3 --fstype="ext4" --level=RAID1 raid.sda4 raid.sdb4
raid /usr/vice/cache    --device=md4 --fstype="ext4" --level=RAID1 raid.sda5 raid.sdb5
raid /swap              --device=md5 --fstype="swap" --level=RAID1 raid.sda6 raid.sdb6
raid /pool              --device=md6 --fstype="ext4" --level=RAID0 raid.sda7 raid.sdb7

2 Set up IML

2.0.1 set up virtual env on CentOS

follow this page:http://wiki.bostonlabs.co.uk/w/index.php?title=KVM:_Installation_and_operation

and disable dhcp on virt-manager on this page:http://wiki.bostonlabs.co.uk/w/index.php?title=KVM:_Tips_and_Tricks#Disable_DHCP_on_KVM_Server

2.1 install and configure IML with the latest RPMs

2.1.1 change the eht0 ip to 172.28.11.1

3 deploy all other nodes using cobbler

4 deploy IEEL on all other nodes

172.105.0.103

Retrieved from "http://wiki.define-technology.com/mediawiki-1.35.0/index.php?title=Lustre_install_for_Unit.tv&oldid=8621"