Lustre Intel: Notes from Installing Plymouth IEEL 3.0 Non HA LDISKFS

From Define Wiki
Jump to navigation Jump to search

Initial Setup

  • setup the servers (all MDS and OSS nodes)
- /etc/hosts
- hostname fqdn (hostnamectl set-hostname oss2.plymouth.net)
- iptables disabled
- selinux disabled
- yum -y groupinstall 'Development tools'
- yum -y install libyaml net-snmp-libs  lm_sensors-libs net-snmp-agent-libs

Short install version (Server)

cd ee-3.0.0.0
./create_installer ldiskfs
tar zxvf lustre-ldiskfs-el7-installer.tar.gz
cd lustre-ldiskfs/
./install

Do it the hard way / Manually (Server)

[root@mds1 el7]# pwd
/root/scratch/ee-3.0.0.0/el7
tar zxvf e2fsprogs-1.42.13.wc4-bundle.tar.gz
rpm -Uvh e2fsprogs-*rpm libss-1.42.13.wc4-7.el7.x86_64.rpm  libcom_err-1.42.13.wc4-7.el7.x86_64.rpm

tar zxvf lustre-2.7.15.3-bundle.tar.gz
# should try -i not -U as its cleans up!
#rpm -Uvh kernel-* --force
rpm -ivh kernel-* --force
rpm -ivh lustre-2.7.15.3-*rpm lustre-modules-2.7.15.3-3.10.0_327.13.1.el7_lustre.x86_64.x86_64.rpm lustre-osd-ldiskfs-*
# note didnt do the ZFS ones...

Setup MDS

# MDS made partitions / not sure if aligned
(parted) mkpart                                                          
Partition name?  []? MGT                                                  
File system type?  [ext2]? ext3                                          
Start? 143G
End? 153G      

(parted) mkpart
Partition name?  []? MDT                                                  
File system type?  [ext2]? ext3                                          
Start? 153G
End? -1

Lets create the lustre FS

On the MDS first 

 mkfs.lustre --mgs --reformat /dev/sda5
 mkfs.lustre --mdt --fsname=lfs1  --reformat --index=0 --mgsnode=172.28.1.12@tcp --mkfsoptions="-J size=2048" /dev/sda6

For an IB / OPA network 

 mkfs.lustre --mdt --fsname=lfs1  --reformat --index=0 --mgsnode=192.168.0.200@o2ib --mkfsoptions="-J size=2048" /dev/sda6

 mkdir -p /lustre/mgt
 mkdir -p /lustre/lfs1-mdt
 mount -t lustre /dev/sda5 /lustre/mgt/
 mount -t lustre /dev/sda6 /lustre/lfs1-mdt/

Setup the OSS(s)

# on to the oss1
mkfs.lustre --ost --fsname=lfs1  --reformat --index=1 --mgsnode=172.28.1.12@tcp --mkfsoptions="-J size=2048" /dev/sda
mkfs.lustre --ost --fsname=lfs1  --reformat --index=2 --mgsnode=172.28.1.12@tcp --mkfsoptions="-J size=2048" /dev/sdb
mkfs.lustre --ost --fsname=lfs1  --reformat --index=3 --mgsnode=172.28.1.12@tcp --mkfsoptions="-J size=2048" /dev/sdc

# for IB
mkfs.lustre --ost --fsname=lfs1  --reformat --index=1 --mgsnode=192.168.0.200@o2ib --mkfsoptions="-J size=2048" /dev/sda
mkfs.lustre --ost --fsname=lfs1  --reformat --index=2 --mgsnode=192.168.0.200@o2ib --mkfsoptions="-J size=2048" /dev/sdb
mkfs.lustre --ost --fsname=lfs1  --reformat --index=3 --mgsnode=192.168.0.200@o2ib --mkfsoptions="-J size=2048" /dev/sdc

mkdir /lustre/lfs1-ost00 -p
mkdir /lustre/lfs1-ost01 -p
mkdir /lustre/lfs1-ost02 -p
mount -t lustre /dev/sda /lustre/lfs1-ost00
mount -t lustre /dev/sdb /lustre/lfs1-ost01
mount -t lustre /dev/sdc /lustre/lfs1-ost02

# on oss2
mkfs.lustre --ost --fsname=lfs1  --reformat --index=4 --mgsnode=172.28.1.12@tcp --mkfsoptions="-J size=2048" /dev/sda
mkfs.lustre --ost --fsname=lfs1  --reformat --index=5 --mgsnode=172.28.1.12@tcp --mkfsoptions="-J size=2048" /dev/sdb
mkfs.lustre --ost --fsname=lfs1  --reformat --index=6 --mgsnode=172.28.1.12@tcp --mkfsoptions="-J size=2048" /dev/sdc

# for IB
mkfs.lustre --ost --fsname=lfs1  --reformat --index=4 --mgsnode=192.168.0.200@o2ib --mkfsoptions="-J size=2048" /dev/sda
mkfs.lustre --ost --fsname=lfs1  --reformat --index=5 --mgsnode=192.168.0.200@o2ib --mkfsoptions="-J size=2048" /dev/sdb
mkfs.lustre --ost --fsname=lfs1  --reformat --index=6 --mgsnode=192.168.0.200@o2ib --mkfsoptions="-J size=2048" /dev/sdc


mkdir /lustre/lfs1-ost03 -p
mkdir /lustre/lfs1-ost04 -p
mkdir /lustre/lfs1-ost05 -p
mount -t lustre /dev/sda /lustre/lfs1-ost03
mount -t lustre /dev/sdb /lustre/lfs1-ost04
mount -t lustre /dev/sdc /lustre/lfs1-ost05

Verify everything is working

# verify the MDS (make sure all devices are 'UP')
[root@mds1 ~]# lctl dl
  0 UP osd-ldiskfs MGS-osd MGS-osd_UUID 5
  1 UP mgs MGS MGS 9
  2 UP mgc MGC172.28.1.12@tcp 9c31f8f0-ab99-1621-2c32-94bb77f29ea1 5
  3 UP osd-ldiskfs lfs1-MDT0000-osd lfs1-MDT0000-osd_UUID 13
  4 UP mds MDS MDS_uuid 3
  5 UP lod lfs1-MDT0000-mdtlov lfs1-MDT0000-mdtlov_UUID 4
  6 UP mdt lfs1-MDT0000 lfs1-MDT0000_UUID 17
  7 UP mdd lfs1-MDD0000 lfs1-MDD0000_UUID 4
  8 UP qmt lfs1-QMT0000 lfs1-QMT0000_UUID 4
  9 UP lwp lfs1-MDT0000-lwp-MDT0000 lfs1-MDT0000-lwp-MDT0000_UUID 5
10 UP osp lfs1-OST0001-osc-MDT0000 lfs1-MDT0000-mdtlov_UUID 5
11 UP osp lfs1-OST0002-osc-MDT0000 lfs1-MDT0000-mdtlov_UUID 5
12 UP osp lfs1-OST0003-osc-MDT0000 lfs1-MDT0000-mdtlov_UUID 5
13 UP osp lfs1-OST0004-osc-MDT0000 lfs1-MDT0000-mdtlov_UUID 5
14 UP osp lfs1-OST0005-osc-MDT0000 lfs1-MDT0000-mdtlov_UUID 5
15 UP osp lfs1-OST0006-osc-MDT0000 lfs1-MDT0000-mdtlov_UUID 5
# note; same as: cat /proc/fs/lustre/devices

Setup the Clients

# on the client
wget ftp://fr2.rpmfind.net/linux/centos/7.2.1511/updates/x86_64/Packages/kernel-3.10.0-327.13.1.el7.x86_64.rpm
# for ee-3.0.1.3
wget ftp://fr2.rpmfind.net/linux/centos/7.2.1511/updates/x86_64/Packages/kernel-3.10.0-327.36.2.el7.x86_64.rpm
rpm -ivh kernel-3.10.0-327.13.1.el7.x86_64.rpm
# check and make sure grub boots in to the right kernel
grubby --info=ALL
grubby --default-index
grubby --set-default 0
grubby --default-index

reboot

yum -y install libyaml net-snmp-libs  lm_sensors-libs net-snmp-agent-libs expect
tar zxvf ee-3.0.0.0.tar.gz
cd ee-3.0.0.0/el7
tar zxvf lustre-client-2.7.15.3-bundle.tar.gz
rpm -ivh lustre-*rpm
tar zxvf e2fsprogs-1.42.13.wc4-bundle.tar.gz
rpm -ivh e2fsprogs-*rpm libss-1.42.13.wc4-7.el7.x86_64.rpm  libcom_err-1.42.13.wc4-7.el7.x86_64.rpm --force

# note if performing this as part of the chroot on the headnode, then update the VNFS image
# wwvnfs -y --chroot=/opt/ohpc/admin/images/centos7.2-hfi/

# check the /etc/modprobe.d/lustre.conf
[root@head ~]# cat /etc/modprobe.d/lustre.conf 
options lnet networks=tcp(eno1),o2ib(ib0)


modprobe lustre
mount.lustre 172.28.1.12@tcp:/lfs1 /mnt/lfs1/
# IB version
mount.lustre 192.168.0.200@o2ib:/lfs1 /mnt/lustre

# make persistent, add to /etc/fstab
192.168.0.200@o2ib:/lfs1        /mnt/lustre     lustre  defaults 0 0

# verify mounted / space ok etc
[root@head el7]# df -h
Filesystem               Size  Used Avail Use% Mounted on
/dev/mapper/centos-root   50G   43G  7.4G  86% /
devtmpfs                  63G     0   63G   0% /dev
tmpfs                     63G     0   63G   0% /dev/shm
tmpfs                     63G  9.2M   63G   1% /run
tmpfs                     63G     0   63G   0% /sys/fs/cgroup
/dev/mapper/centos-home  184G  182G  2.7G  99% /home
/dev/sda1                497M  250M  248M  51% /boot
tmpfs                     13G     0   13G   0% /run/user/0
172.28.1.12@tcp:/lfs1    197T  7.2M  187T   1% /mnt/lfs1
Retrieved from "http://wiki.define-technology.com/mediawiki-1.35.0/index.php?title=Lustre_Intel:_Notes_from_Installing_Plymouth_IEEL_3.0_Non_HA_LDISKFS&oldid=18121"