View source for Add compute/ceph nodes to LMX2

<pre>
root@ausyd-mha1-lc01:~/scratch/ceph-ansible-lmx2# history | grep ansible-playbook
 1274  ansible-playbook --limit mhas1hci-nd005,mhas1hci-nd006 site.yml -i inventory
 1996  history | grep ansible-playbook
 
manually updated /etc/hosts on lmx2

root@ausyd-mha1-lc01:~/scratch/ceph-ansible-lmx2# scp lmx2-n2-api:/etc/hosts updated-lmx2-hosts
The authenticity of host 'lmx2-n2-api (10.14.24.2)' can't be established.
ECDSA key fingerprint is SHA256:9K29Qu3Rkvh3pBhonNQJr2STwyJfLABlKTh8Fet2qbk.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'lmx2-n2-api' (ECDSA) to the list of known hosts.
hosts                                                                              100% 2960     3.6MB/s   00:00
root@ausyd-mha1-lc01:~/scratch/ceph-ansible-lmx2# cat updated-lmx2-hosts
127.0.0.1 localhost

# The following lines are desirable for IPv6 capable hosts
::1     ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters

10.14.11.31 ausyd-mha1-lc01 c01
10.14.11.32 ausyd-mha1-lc02 c02
10.14.11.33 ausyd-mha1-lc03 c03
10.14.11.34 ausyd-mha1-lc04 c04
10.14.11.35 ausyd-mha1-lc05 c05
10.14.11.36 ausyd-mha1-lc06 c06

10.14.14.1 ausyd-mha1-lc01-api c01-api
10.14.14.2 ausyd-mha1-lc02-api c02-api
10.14.14.3 ausyd-mha1-lc03-api c03-api
10.14.14.4 ausyd-mha1-lc04-api c04-api
10.14.14.5 ausyd-mha1-lc05-api c05-api
10.14.14.6 ausyd-mha1-lc06-api c06-api

10.14.15.1 ausyd-mha1-lc01-stor c01-stor
10.14.15.2 ausyd-mha1-lc02-stor c02-stor
10.14.15.3 ausyd-mha1-lc03-stor c03-stor
10.14.15.4 ausyd-mha1-lc04-stor c04-stor
10.14.15.5 ausyd-mha1-lc05-stor c05-stor
10.14.15.6 ausyd-mha1-lc06-stor c06-stor

## LMX2
10.14.21.31 mhas1hci-nd001.dt.internal mhas1hci-nd001 lmx2-n1
10.14.21.32 mhas1hci-nd002.dt.internal mhas1hci-nd002 lmx2-n2
10.14.21.33 mhas1hci-nd003.dt.internal mhas1hci-nd003 lmx2-n3
10.14.21.34 mhas1hci-nd004.dt.internal mhas1hci-nd004 lmx2-n4
10.14.21.35 mhas1hci-nd005.dt.internal mhas1hci-nd005 lmx2-n5
10.14.21.36 mhas1hci-nd006.dt.internal mhas1hci-nd006 lmx2-n6
10.14.21.37 mhas1hci-nd007.dt.internal mhas1hci-nd007 lmx2-n7
10.14.21.38 mhas1hci-nd008.dt.internal mhas1hci-nd008 lmx2-n8
10.14.21.39 mhas1hci-nd009.dt.internal mhas1hci-nd009 lmx2-n9



10.14.24.1 mhas1hci-nd001-api.dt.internal mhas1hci-nd001-api lmx2-n1-api
10.14.24.2 mhas1hci-nd002-api.dt.internal mhas1hci-nd002-api lmx2-n2-api
10.14.24.3 mhas1hci-nd003-api.dt.internal mhas1hci-nd003-api lmx2-n3-api
10.14.24.4 mhas1hci-nd004-api.dt.internal mhas1hci-nd004-api lmx2-n4-api
10.14.24.5 mhas1hci-nd005-api.dt.internal mhas1hci-nd005-api lmx2-n5-api
10.14.24.6 mhas1hci-nd006-api.dt.internal mhas1hci-nd006-api lmx2-n6-api
10.14.24.7 mhas1hci-nd007-api.dt.internal mhas1hci-nd007-api lmx2-n7-api
10.14.24.8 mhas1hci-nd008-api.dt.internal mhas1hci-nd008-api lmx2-n8-api
10.14.24.9 mhas1hci-nd009-api.dt.internal mhas1hci-nd009-api lmx2-n9-api


10.14.25.1 mhas1hci-nd001-stor.dt.internal mhas1hci-nd001-stor lmx2-n1-stor
10.14.25.2 mhas1hci-nd002-stor.dt.internal mhas1hci-nd002-stor lmx2-n2-stor
10.14.25.3 mhas1hci-nd003-stor.dt.internal mhas1hci-nd003-stor lmx2-n3-stor
10.14.25.4 mhas1hci-nd004-stor.dt.internal mhas1hci-nd004-stor lmx2-n4-stor
10.14.25.5 mhas1hci-nd005-stor.dt.internal mhas1hci-nd005-stor lmx2-n5-stor
10.14.25.6 mhas1hci-nd006-stor.dt.internal mhas1hci-nd006-stor lmx2-n6-stor
10.14.25.7 mhas1hci-nd007-stor.dt.internal mhas1hci-nd007-stor lmx2-n7-stor
10.14.25.8 mhas1hci-nd008-stor.dt.internal mhas1hci-nd008-stor lmx2-n8-stor
10.14.25.9 mhas1hci-nd009-stor.dt.internal mhas1hci-nd009-stor lmx2-n9-stor

# BEGIN ANSIBLE GENERATED HOSTS
10.14.24.2 mhas1hci-nd002
10.14.24.3 mhas1hci-nd003
10.14.24.4 mhas1hci-nd004
10.14.24.1 mhas1hci-nd001
# END ANSIBLE GENERATED HOSTS





scp to all lmx2 nodes




root@ausyd-mha1-lc01:~/scratch/ceph-ansible-lmx2# scp updated-lmx2-hosts lmx2-n7-api:/etc/hosts
The authenticity of host 'lmx2-n7-api (10.14.24.7)' can't be established.
ECDSA key fingerprint is SHA256:LTx8pMBuSA6Vm1WxQsKhsjuXXz1L/fz+89lo4oQrJdE.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'lmx2-n7-api' (ECDSA) to the list of known hosts.
updated-lmx2-hosts                                                                 100% 2960     6.5MB/s   00:00
root@ausyd-mha1-lc01:~/scratch/ceph-ansible-lmx2# scp updated-lmx2-hosts lmx2-n8-api:/etc/hosts
The authenticity of host 'lmx2-n8-api (10.14.24.8)' can't be established.
ECDSA key fingerprint is SHA256:e8HHRX6j3m1gJsbHk6lfjW/tokDT3vaExoaryrqGyPg.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'lmx2-n8-api,10.14.24.8' (ECDSA) to the list of known hosts.
updated-lmx2-hosts                                                                 100% 2960     6.8MB/s   00:00
root@ausyd-mha1-lc01:~/scratch/ceph-ansible-lmx2# scp updated-lmx2-hosts lmx2-n9-api:/etc/hosts
The authenticity of host 'lmx2-n9-api (10.14.24.9)' can't be established.
ECDSA key fingerprint is SHA256:OEOLl3vYgMuTvbRyyrAhMIIhmtMSohWl2JXRw3sehk0.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'lmx2-n9-api,10.14.24.9' (ECDSA) to the list of known hosts.
updated-lmx2-hosts                                                                 100% 2960     6.2MB/s   00:00
root@ausyd-mha1-lc01:~/scratch/ceph-ansible-lmx2# scp updated-lmx2-hosts lmx2-n1:/etc/hosts
updated-lmx2-hosts                                                                 100% 2960     4.1MB/s   00:00
root@ausyd-mha1-lc01:~/scratch/ceph-ansible-lmx2# scp updated-lmx2-hosts lmx2-n6:/etc/hosts
updated-lmx2-hosts                                                                 100% 2960     5.1MB/s   00:00
root@ausyd-mha1-lc01:~/scratch/ceph-ansible-lmx2# scp updated-lmx2-hosts lmx2-n5:/etc/hosts
The authenticity of host 'lmx2-n5 (10.14.21.35)' can't be established.
ECDSA key fingerprint is SHA256:V4raFxGp3UJbxD297FPbQaS4VwbFiWYQajmD9RTIcUc.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'lmx2-n5' (ECDSA) to the list of known hosts.
updated-lmx2-hosts                                                                 100% 2960     4.6MB/s   00:00
root@ausyd-mha1-lc01:~/scratch/ceph-ansible-lmx2# scp updated-lmx2-hosts lmx2-n4:/etc/hosts
updated-lmx2-hosts                                                                 100% 2960     3.8MB/s   00:00
root@ausyd-mha1-lc01:~/scratch/ceph-ansible-lmx2# scp updated-lmx2-hosts lmx2-n3:/etc/hosts
updated-lmx2-hosts                                                                 100% 2960     4.7MB/s   00:00



update ceph-ansible inventory

root@ausyd-mha1-lc01:~/scratch/ceph-ansible-lmx2# cp inventory inventory.new
root@ausyd-mha1-lc01:~/scratch/ceph-ansible-lmx2# mv inventory inventory.6nodes
root@ausyd-mha1-lc01:~/scratch/ceph-ansible-lmx2# mv inventory.new inventory
root@ausyd-mha1-lc01:~/scratch/ceph-ansible-lmx2# diff inventory.6nodes inventory
root@ausyd-mha1-lc01:~/scratch/ceph-ansible-lmx2# vi inventory
root@ausyd-mha1-lc01:~/scratch/ceph-ansible-lmx2# cat  inventory
[mons]
mhas1hci-nd001
mhas1hci-nd002
mhas1hci-nd003

[osds]
mhas1hci-nd001
mhas1hci-nd002
mhas1hci-nd003
mhas1hci-nd004
mhas1hci-nd005
mhas1hci-nd006
mhas1hci-nd007
mhas1hci-nd008
mhas1hci-nd009

[mgrs]
mhas1hci-nd001
mhas1hci-nd002
mhas1hci-nd003
mhas1hci-nd004

[mdss]
mhas1hci-nd001
mhas1hci-nd002
mhas1hci-nd003
mhas1hci-nd004

[rgws]
mhas1hci-nd001
mhas1hci-nd002
mhas1hci-nd003

[grafana-server]
mhas1hci-nd001

root@ausyd-mha1-lc01:~/scratch/ceph-ansible-lmx2# diff inventory.6nodes inventory
12a13,15
> mhas1hci-nd007
> mhas1hci-nd008
> mhas1hci-nd009

check time sync

needed to install chrony on 7-9 and copy in config from 2 and run chronyc makestep

root@ausyd-mha1-lc01:~# for i in {1..9};do echo lmx2-n$i;  ssh lmx2-n$i chronyc tracking | grep System; done
lmx2-n1
System time     : 0.000135971 seconds fast of NTP time
lmx2-n2
System time     : 0.000021810 seconds fast of NTP time
lmx2-n3
System time     : 0.000026905 seconds slow of NTP time
lmx2-n4
System time     : 0.000003501 seconds slow of NTP time
lmx2-n5
System time     : 0.000018944 seconds slow of NTP time
lmx2-n6
System time     : 0.000065017 seconds slow of NTP time
lmx2-n7
System time     : 0.000039590 seconds fast of NTP time
lmx2-n8
System time     : 0.000031564 seconds slow of NTP time
lmx2-n9
System time     : 0.000001359 seconds fast of NTP time





root@ausyd-mha1-lc01:~/scratch/ceph-ansible-lmx2# ansible-playbook --limit mhas1hci-nd007,mhas1hci-nd008,mhas1hci-nd009 site.yml -i inventory


all completed fine

speed up rebalance

root@mhas1hci-nd001:~# ceph-conf --show-config | egrep "osd_recovery_max_active|osd_recovery_op_priority|osd_max_backfills"
osd_max_backfills = 1
osd_recovery_max_active = 0
osd_recovery_max_active_hdd = 3
osd_recovery_max_active_ssd = 10
osd_recovery_op_priority = 3
root@mhas1hci-nd001:~# ceph tell 'osd.*' injectargs --osd-max-backfills=3 --osd-recovery-max-active=9
.....


remember to rerun
ceph tell 'osd.*' injectargs --osd-max-backfills=1 --osd-recovery-max-active=3

to put it back after


# this is needed to refresh the updated /etc/hosts file in the lmx2 container

docker restart kolla-deploy-lmx2

apply patch from here to kolla container



root@kolla-deploy-lmx2:/kolla# vi /usr/local/share/kolla-ansible/ansible/library/kolla_docker.py
root@kolla-deploy-lmx2:/kolla# mv /usr/local/share/kolla-ansible/ansible/library/kolla_docker.py /usr/local/share/kolla-ansible/ansible/library/kolla_docker.py.old
root@kolla-deploy-lmx2:/kolla# cp /usr/local/share/kolla-ansible/ansible/library/kolla_docker.py.old /usr/local/share/kolla-ansible/ansible/library/kolla_docker.py
root@kolla-deploy-lmx2:/kolla# vi /usr/local/share/kolla-ansible/ansible/library/kolla_docker.py
root@kolla-deploy-lmx2:/kolla# diff /usr/local/share/kolla-ansible/ansible/library/kolla_docker.py.old /usr/local/share/kolla-ansible/ansible/library/kolla_docker.py
492,493c492
<             'kernel_memory': 'KernelMemory', 'blkio_weight': 'BlkioWeight',
<             'ulimits': 'Ulimits'}
---
>             'blkio_weight': 'BlkioWeight', 'ulimits': 'Ulimits'}

docker exec -it kolla-deploy-lmx2 kolla-ansible -i /etc/kolla/multinode bootstrap-servers --limit mhas1hci-nd007,mhas1hci-nd008,mhas1hci-nd009
docker exec -it kolla-deploy-lmx2 kolla-ansible -i /etc/kolla/multinode pull --limit mhas1hci-nd007,mhas1hci-nd008,mhas1hci-nd009

needed to messa around with our registry as the passord was wrong, the ssl cert out of date and it wasn't listening on :443 anymore

docker exec -it kolla-deploy-lmx2 kolla-ansible -i /etc/kolla/multinode pull --limit mhas1hci-nd007,mhas1hci-nd008,mhas1hci-nd009
docker exec -it kolla-deploy-lmx2 kolla-ansible -i /etc/kolla/multinode deploy --limit mhas1hci-nd007,mhas1hci-nd008,mhas1hci-nd009

spotted potential issue with hostnames showing up using fqdns but we have confirmed this doesn't eem to matter on train




there is a potential issue because libvirt is too old for the AMD CPUs in the new node

this means it cannot generate the cpu topology info and causes this error on nova compute start

2024-02-07 04:55:54.435 7 ERROR nova.compute.manager [req-a930f93b-849b-4dca-b562-61c2967d6ba0 - - - - -] Error updating resources for node mhas1hci-nd007.dt.internal.: AttributeError: 'NoneType' object has no attribute 'cells'
2024-02-07 04:55:54.435 7 ERROR nova.compute.manager Traceback (most recent call last):
2024-02-07 04:55:54.435 7 ERROR nova.compute.manager   File "/usr/lib/python3/dist-packages/nova/compute/manager.py", line 8778, in _update_available_resource_for_node
2024-02-07 04:55:54.435 7 ERROR nova.compute.manager     startup=startup)
2024-02-07 04:55:54.435 7 ERROR nova.compute.manager   File "/usr/lib/python3/dist-packages/nova/compute/resource_tracker.py", line 887, in update_available_resource
2024-02-07 04:55:54.435 7 ERROR nova.compute.manager     self._update_available_resource(context, resources, startup=startup)
2024-02-07 04:55:54.435 7 ERROR nova.compute.manager   File "/usr/lib/python3/dist-packages/oslo_concurrency/lockutils.py", line 328, in inner
2024-02-07 04:55:54.435 7 ERROR nova.compute.manager     return f(*args, **kwargs)
2024-02-07 04:55:54.435 7 ERROR nova.compute.manager   File "/usr/lib/python3/dist-packages/nova/compute/resource_tracker.py", line 972, in _update_available_resource
2024-02-07 04:55:54.435 7 ERROR nova.compute.manager     self._update(context, cn, startup=startup)
2024-02-07 04:55:54.435 7 ERROR nova.compute.manager   File "/usr/lib/python3/dist-packages/nova/compute/resource_tracker.py", line 1237, in _update
2024-02-07 04:55:54.435 7 ERROR nova.compute.manager     self._update_to_placement(context, compute_node, startup)
2024-02-07 04:55:54.435 7 ERROR nova.compute.manager   File "/usr/lib/python3/dist-packages/retrying.py", line 49, in wrapped_f
2024-02-07 04:55:54.435 7 ERROR nova.compute.manager     return Retrying(*dargs, **dkw).call(f, *args, **kw)
2024-02-07 04:55:54.435 7 ERROR nova.compute.manager   File "/usr/lib/python3/dist-packages/retrying.py", line 206, in call
2024-02-07 04:55:54.435 7 ERROR nova.compute.manager     return attempt.get(self._wrap_exception)
2024-02-07 04:55:54.435 7 ERROR nova.compute.manager   File "/usr/lib/python3/dist-packages/retrying.py", line 247, in get
2024-02-07 04:55:54.435 7 ERROR nova.compute.manager     six.reraise(self.value[0], self.value[1], self.value[2])
2024-02-07 04:55:54.435 7 ERROR nova.compute.manager   File "/usr/lib/python3/dist-packages/six.py", line 693, in reraise
2024-02-07 04:55:54.435 7 ERROR nova.compute.manager     raise value
2024-02-07 04:55:54.435 7 ERROR nova.compute.manager   File "/usr/lib/python3/dist-packages/retrying.py", line 200, in call
2024-02-07 04:55:54.435 7 ERROR nova.compute.manager     attempt = Attempt(fn(*args, **kwargs), attempt_number, False)
2024-02-07 04:55:54.435 7 ERROR nova.compute.manager   File "/usr/lib/python3/dist-packages/nova/compute/resource_tracker.py", line 1157, in _update_to_placement
2024-02-07 04:55:54.435 7 ERROR nova.compute.manager     self.driver.update_provider_tree(prov_tree, nodename)
2024-02-07 04:55:54.435 7 ERROR nova.compute.manager   File "/usr/lib/python3/dist-packages/nova/virt/libvirt/driver.py", line 7450, in update_provider_tree
2024-02-07 04:55:54.435 7 ERROR nova.compute.manager     traits = self._get_cpu_traits()
2024-02-07 04:55:54.435 7 ERROR nova.compute.manager   File "/usr/lib/python3/dist-packages/nova/virt/libvirt/driver.py", line 10569, in _get_cpu_traits
2024-02-07 04:55:54.435 7 ERROR nova.compute.manager     traits[ot.HW_CPU_HYPERTHREADING] = self._host.has_hyperthreading
2024-02-07 04:55:54.435 7 ERROR nova.compute.manager   File "/usr/lib/python3/dist-packages/nova/virt/libvirt/host.py", line 1217, in has_hyperthreading
2024-02-07 04:55:54.435 7 ERROR nova.compute.manager     for cell in self.get_capabilities().host.topology.cells:
2024-02-07 04:55:54.435 7 ERROR nova.compute.manager AttributeError: 'NoneType' object has no attribute 'cells'
2024-02-07 04:55:54.435 7 ERROR nova.compute.manager


THIS IS NOTHING TO DO WITH nova_cell!!!!!!!!!!!!!!!!!!!!

DO NOT WASTE A DAY LOOKING THERE IT'S LIBVIRT

this is the porblem old node:
(nova-libvirt)[root@mhas1hci-nd005 /]# virsh capabilities | grep topology
      <topology sockets='1' cores='26' threads='2'/>
    <topology>
    </topology>

new node:

(nova-libvirt)[root@mhas1hci-nd007 /]# virsh capabilities | grep topology
      <topology sockets='1' cores='64' threads='2'/>
	  
see how it's missing the whole 2nd topology section

/usr/lib/python3/dist-packages/nova/virt/libvirt/host.py
    def has_hyperthreading(self):
        """Determine if host CPU has SMT, a.k.a. HyperThreading.

        :return: True if the host has SMT enabled, else False.
        """
        if self._has_hyperthreading is not None:
            return self._has_hyperthreading

        self._has_hyperthreading = False

        # we don't use '/capabilities/host/cpu/topology' since libvirt doesn't
        # guarantee the accuracy of this information
        for cell in self.get_capabilities().host.topology.cells:                       <<<<<<<<<<<<<<<<<<<<<<<<<<<<1217
            if any(len(cpu.siblings) > 1 for cpu in cell.cpus if cpu.siblings):
                self._has_hyperthreading = True
                break

        return self._has_hyperthreading

and this is self.get_capabilities()
    def get_capabilities(self):
        """Returns the host capabilities information

        Returns an instance of config.LibvirtConfigCaps representing
        the capabilities of the host.

        Note: The result is cached in the member attribute _caps.

        :returns: a config.LibvirtConfigCaps object
        """
        if not self._caps:
            xmlstr = self.get_connection().getCapabilities()
            self._log_host_capabilities(xmlstr)
            self._caps = vconfig.LibvirtConfigCaps()
            self._caps.parse_str(xmlstr)
            # NOTE(mriedem): Don't attempt to get baseline CPU features
            # if libvirt can't determine the host cpu model.
            if (hasattr(libvirt,
                        'VIR_CONNECT_BASELINE_CPU_EXPAND_FEATURES') and
                    self._caps.host.cpu.model is not None):
                try:
                    xml_str = self._caps.host.cpu.to_xml()
                    if six.PY3 and isinstance(xml_str, six.binary_type):
                        xml_str = xml_str.decode('utf-8')
                    features = self.get_connection().baselineCPU(
                        [xml_str],
                        libvirt.VIR_CONNECT_BASELINE_CPU_EXPAND_FEATURES)
                    if features:
                        cpu = vconfig.LibvirtConfigCPU()
                        cpu.parse_str(features)
                        self._caps.host.cpu.features = cpu.features
                except libvirt.libvirtError as ex:
                    error_code = ex.get_error_code()
                    if error_code == libvirt.VIR_ERR_NO_SUPPORT:
                        LOG.warning("URI %(uri)s does not support full set"
                                    " of host capabilities: %(error)s",
                                     {'uri': self._uri, 'error': ex})
                    else:
                        raise
        return self._caps
		
if we print the workig topology in full here you will clearly see why it is failing. We can try and bodge this later if needed:
    <topology>
      <cells num='2'>
        <cell id='0'>
          <memory unit='KiB'>131655608</memory>
          <pages unit='KiB' size='4'>32913902</pages>
          <pages unit='KiB' size='2048'>0</pages>
          <pages unit='KiB' size='1048576'>0</pages>
          <distances>
            <sibling id='0' value='10'/>
            <sibling id='1' value='21'/>
          </distances>
          <cpus num='52'>
            <cpu id='0' socket_id='0' core_id='0' siblings='0,52'/>
            <cpu id='2' socket_id='0' core_id='6' siblings='2,54'/>
            <cpu id='4' socket_id='0' core_id='1' siblings='4,56'/>
            <cpu id='6' socket_id='0' core_id='5' siblings='6,58'/>
            <cpu id='8' socket_id='0' core_id='2' siblings='8,60'/>
            <cpu id='10' socket_id='0' core_id='4' siblings='10,62'/>
            <cpu id='12' socket_id='0' core_id='3' siblings='12,64'/>
            <cpu id='14' socket_id='0' core_id='13' siblings='14,66'/>
            <cpu id='16' socket_id='0' core_id='8' siblings='16,68'/>
            <cpu id='18' socket_id='0' core_id='12' siblings='18,70'/>
            <cpu id='20' socket_id='0' core_id='9' siblings='20,72'/>
            <cpu id='22' socket_id='0' core_id='11' siblings='22,74'/>
            <cpu id='24' socket_id='0' core_id='10' siblings='24,76'/>
            <cpu id='26' socket_id='0' core_id='22' siblings='26,78'/>
            <cpu id='28' socket_id='0' core_id='16' siblings='28,80'/>
            <cpu id='30' socket_id='0' core_id='21' siblings='30,82'/>
            <cpu id='32' socket_id='0' core_id='17' siblings='32,84'/>
            <cpu id='34' socket_id='0' core_id='20' siblings='34,86'/>
            <cpu id='36' socket_id='0' core_id='18' siblings='36,88'/>
            <cpu id='38' socket_id='0' core_id='19' siblings='38,90'/>
            <cpu id='40' socket_id='0' core_id='24' siblings='40,92'/>
            <cpu id='42' socket_id='0' core_id='29' siblings='42,94'/>
            <cpu id='44' socket_id='0' core_id='25' siblings='44,96'/>
            <cpu id='46' socket_id='0' core_id='28' siblings='46,98'/>
            <cpu id='48' socket_id='0' core_id='26' siblings='48,100'/>
            <cpu id='50' socket_id='0' core_id='27' siblings='50,102'/>
            <cpu id='52' socket_id='0' core_id='0' siblings='0,52'/>
            <cpu id='54' socket_id='0' core_id='6' siblings='2,54'/>
            <cpu id='56' socket_id='0' core_id='1' siblings='4,56'/>
            <cpu id='58' socket_id='0' core_id='5' siblings='6,58'/>
            <cpu id='60' socket_id='0' core_id='2' siblings='8,60'/>
            <cpu id='62' socket_id='0' core_id='4' siblings='10,62'/>
            <cpu id='64' socket_id='0' core_id='3' siblings='12,64'/>
            <cpu id='66' socket_id='0' core_id='13' siblings='14,66'/>
            <cpu id='68' socket_id='0' core_id='8' siblings='16,68'/>
            <cpu id='70' socket_id='0' core_id='12' siblings='18,70'/>
            <cpu id='72' socket_id='0' core_id='9' siblings='20,72'/>
            <cpu id='74' socket_id='0' core_id='11' siblings='22,74'/>
            <cpu id='76' socket_id='0' core_id='10' siblings='24,76'/>
            <cpu id='78' socket_id='0' core_id='22' siblings='26,78'/>
            <cpu id='80' socket_id='0' core_id='16' siblings='28,80'/>
            <cpu id='82' socket_id='0' core_id='21' siblings='30,82'/>
            <cpu id='84' socket_id='0' core_id='17' siblings='32,84'/>
            <cpu id='86' socket_id='0' core_id='20' siblings='34,86'/>
            <cpu id='88' socket_id='0' core_id='18' siblings='36,88'/>
            <cpu id='90' socket_id='0' core_id='19' siblings='38,90'/>
            <cpu id='92' socket_id='0' core_id='24' siblings='40,92'/>
            <cpu id='94' socket_id='0' core_id='29' siblings='42,94'/>
            <cpu id='96' socket_id='0' core_id='25' siblings='44,96'/>
            <cpu id='98' socket_id='0' core_id='28' siblings='46,98'/>
            <cpu id='100' socket_id='0' core_id='26' siblings='48,100'/>
            <cpu id='102' socket_id='0' core_id='27' siblings='50,102'/>
          </cpus>
        </cell>
        <cell id='1'>
          <memory unit='KiB'>132086476</memory>
          <pages unit='KiB' size='4'>33021619</pages>
          <pages unit='KiB' size='2048'>0</pages>
          <pages unit='KiB' size='1048576'>0</pages>
          <distances>
            <sibling id='0' value='21'/>
            <sibling id='1' value='10'/>
          </distances>
          <cpus num='52'>
            <cpu id='1' socket_id='1' core_id='0' siblings='1,53'/>
            <cpu id='3' socket_id='1' core_id='6' siblings='3,55'/>
            <cpu id='5' socket_id='1' core_id='1' siblings='5,57'/>
            <cpu id='7' socket_id='1' core_id='5' siblings='7,59'/>
            <cpu id='9' socket_id='1' core_id='2' siblings='9,61'/>
            <cpu id='11' socket_id='1' core_id='4' siblings='11,63'/>
            <cpu id='13' socket_id='1' core_id='3' siblings='13,65'/>
            <cpu id='15' socket_id='1' core_id='13' siblings='15,67'/>
            <cpu id='17' socket_id='1' core_id='8' siblings='17,69'/>
            <cpu id='19' socket_id='1' core_id='12' siblings='19,71'/>
            <cpu id='21' socket_id='1' core_id='9' siblings='21,73'/>
            <cpu id='23' socket_id='1' core_id='11' siblings='23,75'/>
            <cpu id='25' socket_id='1' core_id='10' siblings='25,77'/>
            <cpu id='27' socket_id='1' core_id='22' siblings='27,79'/>
            <cpu id='29' socket_id='1' core_id='16' siblings='29,81'/>
            <cpu id='31' socket_id='1' core_id='21' siblings='31,83'/>
            <cpu id='33' socket_id='1' core_id='17' siblings='33,85'/>
            <cpu id='35' socket_id='1' core_id='20' siblings='35,87'/>
            <cpu id='37' socket_id='1' core_id='18' siblings='37,89'/>
            <cpu id='39' socket_id='1' core_id='19' siblings='39,91'/>
            <cpu id='41' socket_id='1' core_id='24' siblings='41,93'/>
            <cpu id='43' socket_id='1' core_id='29' siblings='43,95'/>
            <cpu id='45' socket_id='1' core_id='25' siblings='45,97'/>
            <cpu id='47' socket_id='1' core_id='28' siblings='47,99'/>
            <cpu id='49' socket_id='1' core_id='26' siblings='49,101'/>
            <cpu id='51' socket_id='1' core_id='27' siblings='51,103'/>
            <cpu id='53' socket_id='1' core_id='0' siblings='1,53'/>
            <cpu id='55' socket_id='1' core_id='6' siblings='3,55'/>
            <cpu id='57' socket_id='1' core_id='1' siblings='5,57'/>
            <cpu id='59' socket_id='1' core_id='5' siblings='7,59'/>
            <cpu id='61' socket_id='1' core_id='2' siblings='9,61'/>
            <cpu id='63' socket_id='1' core_id='4' siblings='11,63'/>
            <cpu id='65' socket_id='1' core_id='3' siblings='13,65'/>
            <cpu id='67' socket_id='1' core_id='13' siblings='15,67'/>
            <cpu id='69' socket_id='1' core_id='8' siblings='17,69'/>
            <cpu id='71' socket_id='1' core_id='12' siblings='19,71'/>
            <cpu id='73' socket_id='1' core_id='9' siblings='21,73'/>
            <cpu id='75' socket_id='1' core_id='11' siblings='23,75'/>
            <cpu id='77' socket_id='1' core_id='10' siblings='25,77'/>
            <cpu id='79' socket_id='1' core_id='22' siblings='27,79'/>
            <cpu id='81' socket_id='1' core_id='16' siblings='29,81'/>
            <cpu id='83' socket_id='1' core_id='21' siblings='31,83'/>
            <cpu id='85' socket_id='1' core_id='17' siblings='33,85'/>
            <cpu id='87' socket_id='1' core_id='20' siblings='35,87'/>
            <cpu id='89' socket_id='1' core_id='18' siblings='37,89'/>
            <cpu id='91' socket_id='1' core_id='19' siblings='39,91'/>
            <cpu id='93' socket_id='1' core_id='24' siblings='41,93'/>
            <cpu id='95' socket_id='1' core_id='29' siblings='43,95'/>
            <cpu id='97' socket_id='1' core_id='25' siblings='45,97'/>
            <cpu id='99' socket_id='1' core_id='28' siblings='47,99'/>
            <cpu id='101' socket_id='1' core_id='26' siblings='49,101'/>
            <cpu id='103' socket_id='1' core_id='27' siblings='51,103'/>
          </cpus>
        </cell>
      </cells>
    </topology>
	
	
CONFRIMED BUG https://bugzilla.redhat.com/show_bug.cgi?id=1860231

in future we may want to move the short hostnames to the front and move the shortest hostname to the internal api net

NPS4 may fix it!
</pre>