Rebuild cinder with lightbits integration - train notes

From Define Wiki
Revision as of 16:15, 28 July 2021 by Antony (talk | contribs) (→‎UPDATE from 2.0.6 to 2.2.2 notes)
(diff) ← Older revision | Latest revision (diff) | Newer revision → (diff)
Jump to navigation Jump to search

Dockerfiles for building the containers

[root@registry-dt ~]# cat define-cloud/dockerfiles/cinder-volume/template-overrides.j2
{% extends parent_template %}

# Cinder volume
{% block cinder_volume_footer %}

# Our custom cinder_volume additions to add lightbits

RUN cd /tmp && curl -O https://www.define-technology.com/assets/lightbits_files/cinder_lightbits-0~0~feb46cdb13a4-1.noarch.rpm \
    && curl -O https://www.define-technology.com/assets/lightbits_files/nvme-cli-1.9~2.0.6-1.x86_64.rpm \ 
    && curl -O https://www.define-technology.com/assets/lightbits_files/os_brick_lightbits-0~0~a7762cb1f97c-1.noarch.rpm \
    && curl -O https://www.define-technology.com/assets/lightbits_files/discovery-client-2.0.6-1~5729656ed3c5.x86_64.rpm \
    && curl -O https://www.define-technology.com/assets/lightbits_files/kolla_start_lb_cinder.sh

RUN dnf -y localinstall /tmp/nvme-cli-1.9~2.0.6-1.x86_64.rpm 
RUN rpm -ivh /tmp/os_brick_lightbits-0~0~a7762cb1f97c-1.noarch.rpm --nodeps
RUN dnf -y localinstall /tmp/cinder_lightbits-0~0~feb46cdb13a4-1.noarch.rpm 
RUN dnf -y localinstall /tmp/discovery-client-2.0.6-1~5729656ed3c5.x86_64.rpm

RUN mkdir -p /etc/discovery-client/discovery.d/ \
    && chmod 777 /etc/discovery-client \
    && chmod 777 /etc/discovery-client/discovery.d/ \
    && touch /var/log/discovery-client.log \
    && chmod 777 /var/log/discovery-client.log

RUN echo -e "\n\ncinder  ALL=(ALL)       NOPASSWD: ALL\n" >> /etc/sudoers

RUN sed -i '522s/\ or\ len/:\ \# or len/g' /usr/lib/python3.6/site-packages/cinder/volume/drivers/lightos.py

# commented out to run the discovery service in the hypervisor
# RUN chmod 755 /tmp/kolla_start_lb_cinder.sh && mv /tmp/kolla_start_lb_cinder.sh /usr/local/bin/kolla_start

RUN yum clean all && rm -rf /var/cache/yum

{% endblock %}
[root@registry-dt ~]# cat define-cloud/dockerfiles/cinder-volume/template-overrides.j2^C
[root@registry-dt ~]# cat define-cloud/dockerfiles/nova-compute/template-overrides.j2
{% extends parent_template %}

# Nova compute 
{% block nova_compute_footer %}

# Our custom nova-compute additions to add lightbits

RUN cd /tmp && curl -O https://www.define-technology.com/assets/lightbits_files/nova_lightbits-0~0~844b9c5c7a62-1.noarch.rpm \
    && curl -O https://www.define-technology.com/assets/lightbits_files/nvme-cli-1.9~2.0.6-1.x86_64.rpm \ 
    && curl -O https://www.define-technology.com/assets/lightbits_files/os_brick_lightbits-0~0~a7762cb1f97c-1.noarch.rpm \
    && curl -O https://www.define-technology.com/assets/lightbits_files/discovery-client-2.0.6-1~5729656ed3c5.x86_64.rpm \
    && curl -O https://www.define-technology.com/assets/lightbits_files/kolla_start_lb_nova.sh


RUN yum -y localinstall /tmp/nvme-cli-1.9~2.0.6-1.x86_64.rpm 
RUN rpm -ivh /tmp/os_brick_lightbits-0~0~a7762cb1f97c-1.noarch.rpm --nodeps
RUN rpm -ivh /tmp/nova_lightbits-0~0~844b9c5c7a62-1.noarch.rpm --nodeps
RUN yum -y localinstall /tmp/discovery-client-2.0.6-1~5729656ed3c5.x86_64.rpm

RUN mkdir -p /etc/discovery-client/discovery.d/ \
    && chmod 777 /etc/discovery-client \
    && chmod 777 /etc/discovery-client/discovery.d/ \
    && touch /var/log/discovery-client.log \
    && chmod 777 /var/log/discovery-client.log

RUN echo -e "\n\nnova  ALL=(ALL)       NOPASSWD: ALL\n" >> /etc/sudoers

# commented out to run the discovery service in the hypervisor
# RUN chmod 755 /tmp/kolla_start_lb_nova.sh && mv /tmp/kolla_start_lb_nova.sh /usr/local/bin/kolla_start

RUN yum clean all && rm -rf /var/cache/yum

# hostnqn to be created at hypervisor level and passed in via docker 
# RUN nvme gen-hostnqn > /etc/nvme/hostnqn

{% endblock %}

Dump of all notes

[david@Davids-MacBook-Pro-4 notes]$ cat lightbits-notes.txt 
# config file
[root@openstack _data]# cat ~/kolla/config/cinder/cinder-volume.conf 
[DEFAULT]
enabled_backends=lightos

[lightos]
volume_driver = cinder.volume.drivers.lightos.LightOSVolumeDriver 
lightos_api_address = 10.20.60.101
lightos_api_port = 80
lightos_default_compression = False
lightos_default_num_replicas = 3

# docker reconfigure - may need to comment 

docker will be restarting - so set enabled_backends=lvm and restart so you can cp the files below

# cinder volume service 

docker cp ~/openstack_rpms/cinder_lightbits-2.0.6~0~fa6cb67938d3-1.noarch.rpm cinder_volume:/tmp
docker cp ~/openstack_rpms/os_brick_lightbits-2.0.6~0~986ba852b203-1.noarch.rpm cinder_volume:/tmp
docker cp ~/client_rpms/discovery-client-2.0.6-1~5729656ed3c5.x86_64.rpm cinder_volume:/tmp

# drop to the container 

docker exec -it cinder_volume bash

yum -y install python3 nvme-cli
yum localinstall /tmp/*rpm

# edit the (usr/local/bin)kolla_start script (im adding lines 17/18)

     17 echo "Starting Lightbits discovery client"
     18 /usr/bin/discovery-client serve &
     19 
     20 echo "Running command: '${CMD}${ARGS:+ $ARGS}'"
     21 exec ${CMD} ${ARGS}

# exit out of container and restart cinder volume service

# here's an idea of a successful log from the startup 
2020-11-24 12:28:09.759 6 INFO cinder.rpc [req-040f2d64-854e-4729-b668-fc79ef485ab4 - - - - -] Automatically selected cinder-scheduler objects version 1.38 as minimum service version.
2020-11-24 12:28:09.763 6 INFO cinder.rpc [req-040f2d64-854e-4729-b668-fc79ef485ab4 - - - - -] Automatically selected cinder-scheduler RPC version 3.11 as minimum service version.
2020-11-24 12:28:09.811 6 INFO cinder.volume.manager [req-040f2d64-854e-4729-b668-fc79ef485ab4 - - - - -] Determined volume DB was empty at startup.
2020-11-24 12:28:09.831 6 INFO cinder.volume.manager [req-040f2d64-854e-4729-b668-fc79ef485ab4 - - - - -] Image-volume cache disabled for host openstack.dt.internal@lightos.
2020-11-24 12:28:09.846 6 INFO oslo_service.service [req-040f2d64-854e-4729-b668-fc79ef485ab4 - - - - -] Starting 1 workers
2020-11-24 12:28:09.857 42 INFO cinder.service [-] Starting cinder-volume node (version 14.3.0)
2020-11-24 12:28:09.876 42 INFO cinder.volume.manager [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Starting volume driver LightOSVolumeDriver (2.0.6~0~fa6cb67938d3)
2020-11-24 12:28:09.876 42 INFO cinder.volume.drivers.lightos [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Invoking get_cluster using GET url: /api/v1/cluster request.body: {}
2020-11-24 12:28:09.913 42 INFO cinder.volume.drivers.lightos [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Connected to LightOS cluster 556661f0-163c-4e23-8324-2cdabdc0b8e3 subsysnqn nqn.2014-08.org.nvmexpress:NVMf:uuid:be009c18-3ccd-47d4-8a98-679279df7347
2020-11-24 12:28:09.914 42 INFO cinder.volume.drivers.lightos [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Invoking get_nodes using GET url: /api/v1/nodes request.body: {}
2020-11-24 12:28:09.930 42 INFO oslo.privsep.daemon [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Running privsep helper: ['sudo', 'cinder-rootwrap', '/etc/cinder/rootwrap.conf', 'privsep-helper', '--config-file', '/etc/cinder/cinder.conf', '--privsep_context', 'os_brick.privileged.default', '--privsep_sock_path', '/tmp/tmpzq0mlY/privsep.sock']
2020-11-24 12:28:10.784 42 INFO oslo.privsep.daemon [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Spawned new privsep daemon via rootwrap
2020-11-24 12:28:10.713 60 INFO oslo.privsep.daemon [-] privsep daemon starting
2020-11-24 12:28:10.720 60 INFO oslo.privsep.daemon [-] privsep process running with uid/gid: 0/0
2020-11-24 12:28:10.725 60 INFO oslo.privsep.daemon [-] privsep process running with capabilities (eff/prm/inh): CAP_SYS_ADMIN/CAP_SYS_ADMIN/none
2020-11-24 12:28:10.726 60 INFO oslo.privsep.daemon [-] privsep daemon running as pid 60
2020-11-24 12:28:10.950 42 INFO cinder.volume.drivers.lightos [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Invoking get_cluster using GET url: /api/v1/cluster request.body: {}
2020-11-24 12:28:11.032 42 INFO cinder.volume.drivers.lightos [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Invoking get_cluster using GET url: /api/v1/cluster request.body: {}
2020-11-24 12:28:11.034 42 INFO cinder.keymgr.migration [req-f5a76b7b-c132-4aa7-80b2-6c8735e5f695 - - - - -] Not migrating encryption keys because the ConfKeyManager's fixed_key is not in use.
2020-11-24 12:28:11.105 42 INFO cinder.volume.manager [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Driver initialization completed successfully.
2020-11-24 12:28:11.110 42 INFO cinder.manager [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Initiating service 3 cleanup
2020-11-24 12:28:11.114 42 INFO cinder.manager [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Service 3 cleanup completed.
2020-11-24 12:28:11.174 42 INFO cinder.volume.manager [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Initializing RPC dependent components of volume driver LightOSVolumeDriver (2.0.6~0~fa6cb67938d3)
2020-11-24 12:28:11.175 42 INFO cinder.volume.drivers.lightos [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Invoking get_cluster using GET url: /api/v1/cluster request.body: {}
2020-11-24 12:28:11.235 42 INFO cinder.volume.manager [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Driver post RPC initialization completed successfully.
2020-11-24 12:30:07.239 42 INFO cinder.volume.drivers.lightos [req-9923f295-ba0e-4837-953d-a54f3b4b6d67 - - - - -] Invoking get_cluster using GET url: /api/v1/cluster request.body: {}


openstack volume create --size 5 firstvol

(tail cinder-volume.log)

# nova service 

docker cp nova_lightbits-0~0~844b9c5c7a62-1.noarch.rpm nova_compute:/tmp
docker cp os_brick_lightbits-0~0~a7762cb1f97c-1.noarch.rpm nova_compute:/tmp

docker exec -u root -it nova_compute bash
mkdir -p /etc/discovery-client/discovery.d/
yum -y install python3 nvme-cli
yum -y localinstall /tmp/*rpm 

exit; docker restart nova_compute

# note needs nova_compute /etc/discovery_client/

# needed a docker-persist-local plugin 
#curl -fsSL https://raw.githubusercontent.com/MatchbookLab/local-persist/master/scripts/install.sh | bash

# create the volume 
#[root@openstack-c8 nova-compute]# docker volume create -d local-persist -o mountpoint=/etc/discovery-client/dicovery.d/ --name=discovery-client
#discovery-client

# drop into kolla-deploy container and edit the file (note train release) 
docker exec -it kolla-deploy bash
apt-get install vim 
Change the bind mounts in  /usr/local/share/kolla-ansible/ansible/roles/nova-cell/defaults/main.yml
		

306 nova_compute_default_volumes:
307   - "{{ node_config_directory }}/nova-compute/:{{ container_config_directory }}/:ro"
308   - "/etc/localtime:/etc/localtime:ro"
309   - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_os_family == 'Debian' else '' }}"
310   - "/lib/modules:/lib/modules:ro"
311   - "/run:/run:shared"
312   - "/dev:/dev"
313   - "/etc/discovery-client/:/etc/discovery-client/"
314   - "/etc/nvme/hostnqn:/etc/nvme/hostnqn"

and in /usr/local/share/kolla-ansible/ansible/roles/cinder/defaults/main.yml (line 150) 

146 cinder_volume_default_volumes:
147   - "{{ node_config_directory }}/cinder-volume/:{{ container_config_directory }}/:ro"
148   - "/etc/localtime:/etc/localtime:ro"
149   - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_os_family == 'Debian' else '' }}"
150   - "/dev/:/dev/"
151   - "/etc/discovery-client/:/etc/discovery-client/"
152   - "/etc/nvme/hostnqn:/etc/nvme/hostnqn"

exit; 
chmod 777 /etc/discovery-client/discovery.d/

docker exec -it kolla-deploy kolla-ansible -i  /etc/kolla/all-in-one reconfigure -t nova

# perform both updates on cinder,nova 
root@kolla-deploy:/kolla# kolla-ansible -i /etc/kolla/multinode reconfigure -t cinder,nova 

# updated with container version 

# run the discovery client on the all-in-one system
docker run -it --rm --privileged --name=discovery_client --net=host -v /dev:/dev/ -v /etc/discovery-client/discovery.d:/etc/discovery-client/discovery.d docker.lightbitslabs.com/lightos-csi/lb-nvme-discovery-client:1.2.0
very-client:1.2.0
Unable to find image 'docker.lightbitslabs.com/lightos-csi/lb-nvme-discovery-client:1.2.0' locally
1.2.0: Pulling from lightos-csi/lb-nvme-discovery-client
5ad559c5ae16: Pull complete
d9f91a9750a7: Pull complete
07193a77857f: Pull complete
0091b54d678a: Pull complete
Digest: sha256:c909bcf8a88f6641c52d69ee4f1ee2b64b9883843feee7c3f291baad82616996
Status: Downloaded newer image for docker.lightbitslabs.com/lightos-csi/lb-nvme-discovery-client:1.2.0                                                                                                   
^B^[[Cdiscovery-client configuration: model.AppConfig{Cores:[]int{0}, Logging:logging.Config{Filename:"/var/log/discovery-client.log", ReportCaller:true, Level:"info"}, ClientConfigDir:"/etc/discovery-client/discovery.d/", Debug:model.DebugInfo{Endpoint:"0.0.0.0:6060", Enablepprof:true, Metrics:true}, ReconnectInterval:5000000000, InternalDir:"/etc/discovery-client/internal/", LogPagePaginationEnabled:false}
level=info msg="******************** /discovery-client started ********************" func=cmd.serveCmdFunc file="serve.go:64"                                                                            
level=warning msg="folder \"/etc/discovery-client/internal/\" does not exists. creating it" func="application.(*App).Start" file="app.go:58"  

# note problems with nova so trying the mount one directory further up 
cd /etc/discovery-client/  
ls                                                                                                            
docker cp loving_pare:/etc/discovery-client/discovery-client.yaml .                                                    
mkdir internal                                                                                
docker cp loving_pare:/etc/discovery-client/internal/internal.json ./internal/             
docker run -it --rm --privileged --name=discovery_client --net=host -v /dev:/dev/ -v /etc/discovery-client/discovery.d:/etc/discovery-client/discovery.d docker.lightbitslabs.com/lightos-csi/lb-nvme-discovery-client:1.2.0

[root@openstack-c8 scratch]# docker cp cinder_lightbits-0~0~feb46cdb13a4-1.noarch.rpm cinder_volume:/tmp
[root@openstack-c8 scratch]# docker cp os_brick_lightbits-0~0~a7762cb1f97c-1.noarch.rpm  cinder_volume:/tmp
[root@openstack-c8 scratch]# docker exec -it cinder_volume bash 
(cinder-volume)[cinder@openstack-c8 /]$ 




# debugging 
modprobe nvme
modprobe nvme-fabrics
modprove nvme-tcp 




## Fresh start centos 8.2
modprobe nvme
modprobe nvme-fabrics
modprove nvme-tcp 

[root@openstack-c8 ~]# cat ~/kolla/config/cinder/cinder-volume.conf 
[DEFAULT]
enabled_backends=lvm

[lightos]
volume_driver = cinder.volume.drivers.lightos.LightOSVolumeDriver 
lightos_api_address = 10.20.60.101
lightos_api_port = 80
lightos_default_compression = False
lightos_default_num_replicas = 3

# note to start with set enabled_backends=lvm to allow the container start 

[root@openstack-c8 client_rpms]# docker cp discovery-client-2.0.6-1~5729656ed3c5.x86_64.rpm cinder_volume:/tmp
[root@openstack-c8 client_rpms]# docker cp nvme-cli-1.9~2.0.6-1.x86_64.rpm cinder_volume:/tmp
[root@openstack-c8 client_rpms]# cd ../openstack_rpms/
[root@openstack-c8 openstack_rpms]# docker cp ./cinder_lightbits-2.0.6~0~fa6cb67938d3-1.noarch.rpm cinder_volume:/tmp
[root@openstack-c8 openstack_rpms]# docker cp ./os_brick_lightbits-2.0.6~0~986ba852b203-1.noarch.rpm cinder_volume:/tmp
[root@openstack-c8 openstack_rpms]# docker exec -u root -it cinder_volume bash
(cinder-volume)[cinder@openstack-c8 /]$ yum -y install python3

(cinder-volume)[root@openstack-c8 /]# ls /tmp/*rpm  -l
-rw-r--r--. 1 root root   17318 Nov 23 05:57 /tmp/cinder_lightbits-2.0.6~0~fa6cb67938d3-1.noarch.rpm
-rw-r--r--. 1 root root 4944484 Nov 24 10:10 /tmp/discovery-client-2.0.6-1~5729656ed3c5.x86_64.rpm
-rw-r--r--. 1 root root  796771 Nov 24 10:10 /tmp/nvme-cli-1.9~2.0.6-1.x86_64.rpm
-rw-r--r--. 1 root root   13263 Nov 23 05:57 /tmp/os_brick_lightbits-2.0.6~0~986ba852b203-1.noarch.rpm

(cinder-volume)[root@openstack-c8 /]# yum -y localinstall /tmp/*rpm 

# add the following section to start the discovery client on the node

(cinder-volume)[root@openstack-c8 /]# cat /usr/local/bin/kolla_start 
#!/bin/bash
set -o errexit
set -o xtrace

# Processing /var/lib/kolla/config_files/config.json as root.  This is necessary
# to permit certain files to be controlled by the root user which should
# not be writable by the dropped-privileged user, especially /run_command
sudo -E kolla_set_configs
CMD=$(cat /run_command)
ARGS=""

if [[ ! "${!KOLLA_SKIP_EXTEND_START[@]}" ]]; then
    # Run additional commands if present
    . kolla_extend_start
fi

# edit the kolla_start script (im adding lines 17/18)
echo "Starting Lightbits discovery client"
/usr/bin/discovery-client serve &

echo "Running command: '${CMD}${ARGS:+ $ARGS}'"
exec ${CMD} ${ARGS}

# end of file 

# exit out of the container back to the node
exit

# set container back to lightos
[root@openstack-c8 ~]# cat ~/kolla/config/cinder/cinder-volume.conf 
[DEFAULT]
enabled_backends=lightos

[lightos]
volume_driver = cinder.volume.drivers.lightos.LightOSVolumeDriver 
lightos_api_address = 10.20.60.101
lightos_api_port = 80
lightos_default_compression = False
lightos_default_num_replicas = 3

# faster way avoid the kolla reconfigure
(cinder-volume)[root@openstack-c8 /]# exit
exit
[root@openstack-c8 ~]# vi /etc/kolla/cinder-volume/cinder.conf 
[root@openstack-c8 ~]# docker restart cinder_volume
cinder_volume

# check logs 

tail -f /var/lib/docker/volumes/kolla_logs/_data/cinder/cinder-volume.log 
...
2020-12-04 07:08:24.135 42 INFO cinder.volume.manager [req-d2a6ac32-06f2-43b8-9019-1564f93f6cdb - - - - -] Initializing RPC dependent components of volume driver LightOSVolumeDriver (2.0.6~0~fa6cb67938d3)
2020-12-04 07:08:24.136 42 INFO cinder.volume.drivers.lightos [req-d2a6ac32-06f2-43b8-9019-1564f93f6cdb - - - - -] Invoking get_cluster using GET url: /api/v1/cluster request.body: {}
2020-12-04 07:08:24.193 42 INFO cinder.volume.manager [req-d2a6ac32-06f2-43b8-9019-1564f93f6cdb - - - - -] Driver post RPC initialization completed successfully.

# ok thats cinder setup - lets turn to nova_compute now

[root@openstack-c8 ~]# docker cp openstack_rpms/os_brick_lightbits-2.0.6~0~986ba852b203-1.noarch.rpm nova_compute:/tmp
[root@openstack-c8 ~]# docker cp openstack_rpms/nova_lightbits-2.0.6~0~2778d6e7595e-1.noarch.rpm nova_compute:/tmp
[root@openstack-c8 ~]# docker cp client_rpms/nvme-cli-1.9~2.0.6-1.x86_64.rpm nova_compute:/tmp
[root@openstack-c8 ~]# docker cp client_rpms/discovery-client-2.0.6-1~5729656ed3c5.x86_64.rpm nova_compute:/tmp
[root@openstack-c8 ~]# docker exec -u root -it nova_compute bash
(nova-compute)[root@openstack-c8 /]# yum -y install python3 
yum -y localinstall /tmp/*rpm

# edit the port as its already using 6060
(nova-compute)[root@openstack-c8 /]# cat /etc/discovery-client/discovery-client.yaml 
cores: [0]
clientConfigDir: /etc/discovery-client/discovery.d/
internalDir: /etc/discovery-client/internal/
reconnectInterval: 5s
logPagePaginationEnabled: false
logging:
  filename: "/var/log/discovery-client.log"
  level: info
  reportCaller: true
debug:
  metrics: true
  enablepprof: true
  endpoint: 0.0.0.0:6061


# chmod so nova can access
(nova-compute)[root@openstack-c8 /]# chown nova:nova -R /etc/discovery-client* -R 

# tell the container to start discovery-client on startup via kolla_start
# file /usr/local/bin/kolla_start 
# edit the kolla_start script (im adding lines 17/18)
echo "Starting Lightbits discovery client"
/usr/bin/discovery-client serve &


# create a hostnqn for nova
(nova-compute)[root@openstack-c8 /]# nvme gen-hostnqn > /etc/nvme/hostnqn

exit; docker restart nova_compute 
# check docker for discovery-client running 

# back to kolla-deploy bash
openstack volume create --size 10 lbvol1
openstack server add volume demo1 lbvol1


# check things out 
[root@openstack-c8 ~]# docker exec -u root -it nova_compute bash 
(nova-compute)[root@openstack-c8 /]# nvme list
Node             SN                   Model                                    Namespace Usage                      Format           FW Rev  
---------------- -------------------- ---------------------------------------- --------- -------------------------- ---------------- --------
/dev/nvme0n1     9ac7a0891e7cad9b     Lightbits LightOS                        8          10.74  GB /  10.74  GB      4 KiB +  0 B   2.0
/dev/nvme0n2     9ac7a0891e7cad9b     Lightbits LightOS                        9           0.00   B /  10.74  GB      4 KiB +  0 B   2.0
/dev/nvme0n3     9ac7a0891e7cad9b     Lightbits LightOS                        10          5.37  GB /   5.37  GB      4 KiB +  0 B   2.0
/dev/nvme1n1     9ac7a0891e7cad9b     Lightbits LightOS                        8           0.00   B /  10.74  GB      4 KiB +  0 B   2.0
/dev/nvme1n2     9ac7a0891e7cad9b     Lightbits LightOS                        9           0.00   B /  10.74  GB      4 KiB +  0 B   2.0
/dev/nvme1n3     9ac7a0891e7cad9b     Lightbits LightOS                        10          0.00   B /   5.37  GB      4 KiB +  0 B   2.0
/dev/nvme2n1     9ac7a0891e7cad9b     Lightbits LightOS                        8           0.00   B /  10.74  GB      4 KiB +  0 B   2.0
/dev/nvme2n2     9ac7a0891e7cad9b     Lightbits LightOS                        9          10.74  GB /  10.74  GB      4 KiB +  0 B   2.0
/dev/nvme2n3     9ac7a0891e7cad9b     Lightbits LightOS                        10          0.00   B /   5.37  GB      4 KiB +  0 B   2.0

### Redo with discovery-client run as a container (with updated files from Muli) 

# recreate the discovery-client dir 
[root@openstack-c8 ~]# rm -rf /etc/discovery-client/
[root@openstack-c8 ~]# mkdir -p /etc/discovery-client/
[root@openstack-c8 ~]# mkdir -p /etc/discovery-client/discovery.d
[root@openstack-c8 ~]# chmod 777 /etc/discovery-client/

[root@openstack-c8 ~]# docker run -it --rm --privileged --name=discovery_client --net=host -v /dev:/dev/ -v /etc/discovery-client/discovery.d:/etc/discovery-client/discovery.d  docker.lightbitslabs.com/lightos-csi/lb-nvme-discovery-client:1.2.0

# drop into the kolla deploy image 
root@kolla-deploy:/kolla# grep -n discovery  /usr/local/share/kolla-ansible/ansible/roles/cinder/defaults/main.yml
151:  - "/etc/discovery-client/:/etc/discovery-client/"

root@kolla-deploy:/kolla# grep -n discovery  /usr/local/share/kolla-ansible/ansible/roles/nova-cell/defaults/main.yml 
313:  - "/etc/discovery-client:/etc/discovery-client"

# reconfigure once done
[root@openstack-c8 scratch]# docker exec -it kolla-deploy kolla-ansible -i  /etc/kolla/all-in-one reconfigure


# ok lets now install the packages in the containers 
docker cp cinder_lightbits-0~0~feb46cdb13a4-1.noarch.rpm cinder_volume:/tmp   
docker cp os_brick_lightbits-0~0~a7762cb1f97c-1.noarch.rpm cinder_volume:/tmp 
docker cp ../client_rpms/nvme-cli-1.9~2.0.6-1.x86_64.rpm cinder_volume:/tmp

docker exec -u root -it cinder_volume bash  

(cinder-volume)[root@openstack-c8 /]# yum -y install python3  
(cinder-volume)[root@openstack-c8 /]# yum -y localinstall /tmp/*rpm 
exit

# set cinder.conf back to lightos 
[root@openstack-c8 scratch]# grep -n lightos /etc/kolla/cinder-volume/cinder.conf
19:enabled_backends = lightos
66:[lightos]
67:volume_driver = cinder.volume.drivers.lightos.LightOSVolumeDriver
68:lightos_api_address = 10.20.60.101
69:lightos_api_port = 80
70:lightos_default_compression = False
71:lightos_default_num_replicas = 3

docker restart cinder_volume

# lets setup the nova compute container 

docker cp nova_lightbits-0~0~844b9c5c7a62-1.noarch.rpm nova_compute:/tmp  
docker cp os_brick_lightbits-0~0~a7762cb1f97c-1.noarch.rpm nova_compute:/tmp
docker cp ../client_rpms/nvme-cli-1.9~2.0.6-1.x86_64.rpm  nova_compute:/tmp 
docker exec -u root -it nova_compute bash   

(nova-compute)[root@openstack-c8 /]# yum -y install python3
(nova-compute)[root@openstack-c8 /]# yum -y localinstall /tmp/*rpm  
# /etc/nvme/hostnqn existed before i installed the packages!
(nova-compute)[root@openstack-c8 /]# nvme gen-hostnqn > /etc/nvme/hostnqn 

# make sure the openstack envirnoment has had the init-runonce script ran against it 


# Final take - we have created lightos containers for cinder-volume and nova-compute

# in the globals file

# for train on centos 8 
openstack_tag: "train"
 
horizon_tag: "stein-definetech"
nova_compute_tag: "stein-lightos"
cinder_volume_tag: "stein-lightos"
cinder_tag: "stein"



##### deploying the server yum 
yum install python3
pip3 install ansible 
pip3 install netaddr


## for centos8 starview

cd /tmp && curl -O https://www.define-technology.com/assets/lightbits_files/cinder_lightbits-0~0~feb46cdb13a4-1.noarch.rpm     && curl -O https://www.define-technology.com/assets/lightbits_files/nvme-cli-1.9~2.0.6-1.x86_64.rpm     && curl -O https://www.define-technology.com/assets/lightbits_files/os_brick_lightbits-0~0~a7762cb1f97c-1.noarch.rpm
curl -O https://www.define-technology.com/assets/lightbits_files/discovery-client-2.0.6-1~5729656ed3c5.x86_64.rpm

dnf -y localinstall ./nvme-cli-1.9~2.0.6-1.x86_64.rpm 
rpm -ivh /tmp/os_brick_lightbits-0~0~a7762cb1f97c-1.noarch.rpm --nodeps
dnf -y localinstall /tmp/cinder_lightbits-0~0~feb46cdb13a4-1.noarch.rpm
dnf -y localinstall /tmp/discovery-client-2.0.6-1~5729656ed3c5.x86_64.rpm 
 
vi /usr/local/bin/kolla_start 
# add 
fi

echo "Starting Lightbits discovery client"
/usr/bin/discovery-client serve &

echo "Running command: '${CMD}${ARGS:+ $ARGS}'"
exec ${CMD} ${ARGS}
## 

mkdir -p /etc/discovery-client/discovery.d/ 
chmod 777 /etc/discovery-client
chmod 777 /etc/discovery-client/discovery.d/
touch /var/log/discovery-client.log
chmod 777 /var/log/discovery-client.log 

# setup in openstack / after deploy
root@kolla-deploy:/kolla# openstack volume type create --property volume_backend_name=lightos lightos
+-------------+--------------------------------------+
| Field       | Value                                |
+-------------+--------------------------------------+
| description | None                                 |
| id          | d3b0f515-b0a5-4f64-8d68-e909ba1c4e5b |
| is_public   | True                                 |
| name        | lightos                              |
| properties  | volume_backend_name='lightos'        |
+-------------+--------------------------------------+

root@kolla-deploy:/kolla# cat /etc/kolla/config/cinder/cinder-volume.conf 
[DEFAULT]
enabled_backends=rbd-1,lightos

[rbd-1]
rbd_ceph_conf=/etc/ceph/ceph.conf
rbd_user=cinder
backend_host=rbd:volumes
rbd_pool=volumes
volume_backend_name=rbd-1
volume_driver=cinder.volume.drivers.rbd.RBDDriver
rbd_secret_uuid = 69e591d1-4901-41f9-8d49-73a8d4ff7d76

[lightos]
volume_driver = cinder.volume.drivers.lightos.LightOSVolumeDriver
lightos_api_address = 192.168.1.34
lightos_api_port = 80
lightos_default_compression = False
lightos_default_num_replicas = 1
volume_backend_name = lightos


### check out manually
    8  nvme connect -t tcp -a 192.168.12.34 -s 4420 -n
    9  nvme connect -t tcp -a 192.168.12.34 -s 4420 -n nqn.2016-01.com.lightbitslabs:uuid:5d7e3672-dd22-455d-a23e-1a5628f92bf8 -q ALLOW_NONE

()[root@stu-prr-hci1-n1-mlnx ~]# nvme disconnect-all
()[root@stu-prr-hci1-n1-mlnx ~]# ls /dev/nvm* 
/dev/nvme-fabrics


GET NQN from 
lbcli get cluster 

nqn.2016-01.com.lightbitslabs:uuid:77774c12-60b0-480c-9983-cf524a57f3b4
[david@Davids-MacBook-Pro-4 notes]$


UPDATE from 2.0.6 to 2.2.2 notes

Install discovery client on HOST OS

to document

update your cinder.conf ON THE kolla-deploy HOST

NOTE: not yet sure where we get the JWT setting yet

192.168.102.10 is the IP of the lightbits node in this config 

vi ~/kolla/config/cinder/cinder-volume.conf

[DEFAULT]
enabled_backends=lightos

[lightos]
volume_driver = cinder.volume.drivers.lightos.LightOSVolumeDriver

# Ex. lightos_api_address = 192.168.67.78,192.168.34.56,192.168.12.17
lightos_api_address = 192.168.102.10
lightos_api_port = 443
lightos_default_compression = True
lightos_default_num_replicas = 1
volume_backend_name = lightos
lightos_skip_ssl_verify=True
lightos_jwt=eyJhbGciOiJSUzI1NiIsImtpZCI6InN5c3RlbTpyb290IiwidHlwIjoiSldUIn0.eyJhdWQiOiJMaWdodE9TIiwiZXhwIjoxNjU4OTI2ODIwLCJpYXQiOjE2MjczOTA4MjAsImlzcyI6Ii9yb290L2xpZ2h0b3MtY2VydGlmaWNhdGVzL2NlcnQtbGItYWRtaW4ta2V5LnBlbSIsImp0aSI6ImZWQUVyVVQ5cHNBaFZoeWY3SVRjalEiLCJuYmYiOjE2MjczOTA4MjAsInJvbGVzIjpbInN5c3RlbTpjbHVzdGVyLWFkbWluIl0sInN1YiI6ImxpZ2h0b3MtY2xpZW50In0.q344_XJwBeq_qeTE7ZDkFUfeHbaZ-bLz8DTKquh3Y07kpuvWSTMVUNEfUnTlPhsLaB_rP6h-20lDi1BK4kOFhD7PtLYLbGpIEOMmYKpmsIrlb7ayU2zEat3jHHds0IPzkpV9u4Vp8wUKYdpr0azLGJlGD8h7pUwsGE6o_XLPXPYIBtoW1yhvgcjpGKrAvRrLOYQmi0X6XFkc1FLU_UKJDtYJSf-wEWP33J21uj7Rl-4-ojicNz_P-lG-QMyB-Nnq_w4kigBpPnKlPZ3LkysyLxxFb22AnmSUfhkXjqzjHf9ZINeDpLM5rSCDuyX_1_FXE9AF5A1ZUdM6T48B24_3aw

make sure you are mounting /etc/discovery-client and /etc/nvme/hostnqn in the cinder_volume and nova_compute container

Edit ~/kolla/globals.yml 

vi ~/kolla/globals.yml
...
# if installing lightos then set the following to pass the hostnqn and discovery dir from the host to the relevant containers 
lightos_extra_volumes: ["/etc/discovery-client/:/etc/discovery-client/","/etc/nvme/hostnqn:/etc/nvme/hostnqn"]
nova_compute_extra_volumes: "{{ nova_extra_volumes | default([]) + lightos_extra_volumes }}"
cinder_volume_extra_volumes:  "{{ cinder_extra_volumes | default([]) + lightos_extra_volumes }}"

reconfigure kolla warning this will probably re-pull nova-compute AND cinder volume so do this now or you have to repeat (go on try it)

podman exec -it kolla-deploy
kolla-ansible -i /etc/kolla/multinode reconfigure nova,cinder
.....

remove old rpms from containers

DO NOT UPGRADE

showing removal of discovery client here just in case 

[root@server01 ~]# docker exec -u root -it cinder_volume bash
(cinder-volume)[root@server01 /]# cd /etc/discovery-client/openstack_rpms/
(cinder-volume)[root@server01 openstack_rpms]# ls
cinder_lightbits-2.2.2~94478f35ae2c-1.noarch.rpm  nova_lightbits-2.2.2~076a3dcde0f4-1.noarch.rpm  os_brick_lightbits-2.2.2~64377d98e9de-1.noarch.rpm
(cinder-volume)[root@server01 openstack_rpms]# rpm -e cinder_lightbits os_brick_lightbits discovery-client
error: package discovery-client is not installed
search for cinder module
/usr/lib/python3.6/site-packages/cinder
+ case "$1" in
+ cd /opt/extra_lightbits
+ python3 -c 'from openstack_patcher import *; unpatch_os_brick()'
search for os_brick module
/usr/lib/python3.6/site-packages/os_brick

install new rpms

note --nodeps on the os-brick rpm! 

(cinder-volume)[root@server01 openstack_rpms]# rpm --nodeps -ivh os_brick_lightbits-2.2.2~64377d98e9de-1.noarch.rpm
Verifying...                          ################################# [100%]
Preparing...                          ################################# [100%]
Updating / installing...
   1:os_brick_lightbits-2.2.2~64377d98################################# [100%]
search for os_brick module
/usr/lib/python3.6/site-packages/os_brick
(cinder-volume)[root@server01 openstack_rpms]# rpm -ivh cinder_lightbits-2.2.2~94478f35ae2c-1.noarch.rpm
Verifying...                          ################################# [100%]
Preparing...                          ################################# [100%]
Updating / installing...
   1:cinder_lightbits-2.2.2~94478f35ae################################# [100%]
search for cinder module
/usr/lib/python3.6/site-packages/cinder

Comment out discovery process check

if you forget this the driver will fail to start with this error 

2021-07-28 20:30:37.446 31 ERROR cinder.volume.manager   File "/usr/lib/python3.6/site-packages/cinder/volume/drivers/lightos.py", line 723, in check_for_setup_error
2021-07-28 20:30:37.446 31 ERROR cinder.volume.manager     found_dsc = self.connector.find_dsc()
2021-07-28 20:30:37.446 31 ERROR cinder.volume.manager   File "/usr/lib/python3.6/site-packages/os_brick/initiator/connectors/lightos.py", line 77, in find_dsc
2021-07-28 20:30:37.446 31 ERROR cinder.volume.manager     out, err = self._execute(*cmd, root_helper=self._root_helper, run_as_root=True)
2021-07-28 20:30:37.446 31 ERROR cinder.volume.manager   File "/usr/lib/python3.6/site-packages/os_brick/executor.py", line 52, in _execute
2021-07-28 20:30:37.446 31 ERROR cinder.volume.manager     result = self.__execute(*args, **kwargs)
2021-07-28 20:30:37.446 31 ERROR cinder.volume.manager   File "/usr/lib/python3.6/site-packages/os_brick/privileged/rootwrap.py", line 169, in execute
2021-07-28 20:30:37.446 31 ERROR cinder.volume.manager     return execute_root(*cmd, **kwargs)
2021-07-28 20:30:37.446 31 ERROR cinder.volume.manager   File "/usr/lib/python3.6/site-packages/oslo_privsep/priv_context.py", line 245, in _wrap
2021-07-28 20:30:37.446 31 ERROR cinder.volume.manager     return self.channel.remote_call(name, args, kwargs)
2021-07-28 20:30:37.446 31 ERROR cinder.volume.manager   File "/usr/lib/python3.6/site-packages/oslo_privsep/daemon.py", line 224, in remote_call
2021-07-28 20:30:37.446 31 ERROR cinder.volume.manager     raise exc_type(*result[2])
2021-07-28 20:30:37.446 31 ERROR cinder.volume.manager oslo_concurrency.processutils.ProcessExecutionError: Unexpected error while running command.
2021-07-28 20:30:37.446 31 ERROR cinder.volume.manager Command: bash -c ps ax | grep discovery-client | grep -v grep
2021-07-28 20:30:37.446 31 ERROR cinder.volume.manager Exit code: 1

easiest way is to install vim in the container but there is a broken reop in train so use this 

(cinder-volume)[root@server01 openstack_rpms]# yum -y --disablerepo=rabbitmq_rabbitmq-erlang install vim

and comment out the following lines fro 723 onwards:

vi /usr/lib/python3.6/site-packages/cinder/volume/drivers/lightos.py 

#        found_dsc = self.connector.find_dsc()
#        if not found_dsc:
#            msg = 'LIGHTOS: Cinder driver requires a local discovery_client for image_to/from_volume operations'
#            raise exception.VolumeBackendAPIException(message=msg)

restart cinder-volume container

(cinder-volume)[root@server01 openstack_rpms]# exit
exit
[root@server01 ~]# docker restart cinder_volume
cinder_volume

don't forget to check the logs for issues! 

[root@server01 ~]# tail -n 30 /var/log/kolla/cinder/cinder-volume.log
2021-07-28 22:29:25.039 31 ERROR cinder.service [-] Manager for service cinder-volume server01.dt.api@lightos is reporting problems, not sending heartbeat. Service will appear "down".
2021-07-28 22:29:35.042 31 ERROR cinder.service [-] Manager for service cinder-volume server01.dt.api@lightos is reporting problems, not sending heartbeat. Service will appear "down".
2021-07-28 22:29:39.731 31 WARNING cinder.volume.manager [req-58c14c44-1739-48b0-a2fb-ab4302a7a8c9 - - - - -] Update driver status failed: (config name lightos) is uninitialized.
2021-07-28 22:29:45.045 31 ERROR cinder.service [-] Manager for service cinder-volume server01.dt.api@lightos is reporting problems, not sending heartbeat. Service will appear "down".
2021-07-28 22:29:55.047 31 ERROR cinder.service [-] Manager for service cinder-volume server01.dt.api@lightos is reporting problems, not sending heartbeat. Service will appear "down".
2021-07-28 22:29:58.996 7 INFO oslo_service.service [req-a5c5068f-7b18-4c74-9d63-9e9ae9e8fb62 - - - - -] Caught SIGTERM, stopping children
2021-07-28 22:29:59.003 7 INFO oslo_service.service [req-a5c5068f-7b18-4c74-9d63-9e9ae9e8fb62 - - - - -] Waiting on 1 children to exit
2021-07-28 22:30:05.049 31 ERROR cinder.service [-] Manager for service cinder-volume server01.dt.api@lightos is reporting problems, not sending heartbeat. Service will appear "down".
2021-07-28 22:30:11.038 7 INFO cinder.rpc [req-1f6010ba-4953-4b9f-b82e-38cb21d58331 - - - - -] Automatically selected cinder-scheduler objects version 1.38 as minimum service version.
2021-07-28 22:30:11.043 7 INFO cinder.rpc [req-1f6010ba-4953-4b9f-b82e-38cb21d58331 - - - - -] Automatically selected cinder-scheduler RPC version 3.11 as minimum service version.
2021-07-28 22:30:11.135 7 INFO cinder.volume.manager [req-1f6010ba-4953-4b9f-b82e-38cb21d58331 - - - - -] Determined volume DB was not empty at startup.
2021-07-28 22:30:11.157 7 INFO cinder.volume.manager [req-1f6010ba-4953-4b9f-b82e-38cb21d58331 - - - - -] Image-volume cache disabled for host server01.dt.api@lightos.
2021-07-28 22:30:11.171 7 INFO oslo_service.service [req-1f6010ba-4953-4b9f-b82e-38cb21d58331 - - - - -] Starting 1 workers
2021-07-28 22:30:11.181 31 INFO cinder.service [-] Starting cinder-volume node (version 15.5.0)
2021-07-28 22:30:11.203 31 INFO cinder.volume.manager [req-53e5cd89-7c5f-422c-8ab4-de252c7c2985 - - - - -] Starting volume driver LightOSVolumeDriver (2.2.2~94478f35ae2c)
2021-07-28 22:30:11.203 31 INFO cinder.volume.drivers.lightos [req-53e5cd89-7c5f-422c-8ab4-de252c7c2985 - - - - -] Invoking get_cluster_info using GET url: /api/v2/clusterinfo request.body: {} ssl_verify: False
2021-07-28 22:30:11.258 31 INFO cinder.volume.drivers.lightos [req-53e5cd89-7c5f-422c-8ab4-de252c7c2985 - - - - -] Connected to LightOS cluster a03d47ba-7a6f-4bb2-8d9e-9b59daa27a9f subsysnqn nqn.2016-01.com.lightbitslabs:uuid:4743d933-4058-4a28-9483-6942455f1c3c
2021-07-28 22:30:11.258 31 INFO cinder.volume.drivers.lightos [req-53e5cd89-7c5f-422c-8ab4-de252c7c2985 - - - - -] Invoking get_nodes using GET url: /api/v2/nodes request.body: {} ssl_verify: False
2021-07-28 22:30:11.290 31 INFO oslo.privsep.daemon [req-53e5cd89-7c5f-422c-8ab4-de252c7c2985 - - - - -] Running privsep helper: ['sudo', 'cinder-rootwrap', '/etc/cinder/rootwrap.conf', 'privsep-helper', '--config-file', '/etc/cinder/cinder.conf', '--privsep_context', 'os_brick.privileged.default', '--privsep_sock_path', '/tmp/tmp1s31gsr0/privsep.sock']
2021-07-28 22:30:11.953 31 INFO oslo.privsep.daemon [req-53e5cd89-7c5f-422c-8ab4-de252c7c2985 - - - - -] Spawned new privsep daemon via rootwrap
2021-07-28 22:30:11.858 38 INFO oslo.privsep.daemon [-] privsep daemon starting
2021-07-28 22:30:11.864 38 INFO oslo.privsep.daemon [-] privsep process running with uid/gid: 0/0
2021-07-28 22:30:11.867 38 INFO oslo.privsep.daemon [-] privsep process running with capabilities (eff/prm/inh): CAP_SYS_ADMIN/CAP_SYS_ADMIN/none
2021-07-28 22:30:11.867 38 INFO oslo.privsep.daemon [-] privsep daemon running as pid 38
2021-07-28 22:30:12.178 31 INFO cinder.keymgr.migration [req-c09dec64-f454-4c83-8950-92f03ba1da85 - - - - -] Not migrating encryption keys because the ConfKeyManager's fixed_key is not in use.
2021-07-28 22:30:12.191 31 INFO cinder.volume.manager [req-53e5cd89-7c5f-422c-8ab4-de252c7c2985 - - - - -] Driver initialization completed successfully.
2021-07-28 22:30:12.195 31 INFO cinder.manager [req-53e5cd89-7c5f-422c-8ab4-de252c7c2985 - - - - -] Initiating service 34 cleanup
2021-07-28 22:30:12.198 31 INFO cinder.manager [req-53e5cd89-7c5f-422c-8ab4-de252c7c2985 - - - - -] Service 34 cleanup completed.
2021-07-28 22:30:12.307 31 INFO cinder.volume.manager [req-53e5cd89-7c5f-422c-8ab4-de252c7c2985 - - - - -] Initializing RPC dependent components of volume driver LightOSVolumeDriver (2.2.2~94478f35ae2c)
2021-07-28 22:30:12.325 31 INFO cinder.volume.manager [req-53e5cd89-7c5f-422c-8ab4-de252c7c2985 - - - - -] Driver post RPC initialization completed successfully.

Cinder is ALL GOOD!

uninstall rpms from nova_compute container

TODO:

add evidence of this didn't document as went along 

# rpm -e cinder_lightbits os_brick_lightbits discovery-client

install new rpms in nova_compute container

TODO:

add evidence of this didn't document as went along 

# cd /etc/discovery-client/openstack_rpms
# rpm -i --nodeps os_brick_lightbits-2.2.2~64377d98e9de-1.noarch.rpm
# rpm -i nova_lightbits-2.2.2~076a3dcde0f4-1.noarch.rpm

stop this container starting discovery client on start

check usr/local/bin/kolla_start 

(nova-compute)[root@server01 /]# cat /usr/local/bin/kolla_start
#!/bin/bash
set -o errexit
set -o xtrace

# Processing /var/lib/kolla/config_files/config.json as root.  This is necessary
# to permit certain files to be controlled by the root user which should
# not be writable by the dropped-privileged user, especially /run_command
sudo -E kolla_set_configs
CMD=$(cat /run_command)
ARGS=""

# Install/remove custom CA certificates
sudo kolla_copy_cacerts

if [[ ! "${!KOLLA_SKIP_EXTEND_START[@]}" ]]; then
    # Run additional commands if present
    . kolla_extend_start
fi

echo "Starting Lightbits discovery client"
sudo /usr/bin/discovery-client serve &

echo "Running command: '${CMD}${ARGS:+ $ARGS}'"
exec ${CMD} ${ARGS}

See the lines: 

echo "Starting Lightbits discovery client"
sudo /usr/bin/discovery-client serve &

comment or remove them

restart container

and check it all still works of course, mine was fine

Retrieved from "http://wiki.define-technology.com/mediawiki-1.35.0/index.php?title=Rebuild_cinder_with_lightbits_integration_-_train_notes&oldid=32627"