Rebuild cinder with lightbits integration - train notes
Jump to navigation
Jump to search
Dockerfiles for building the containers
[root@registry-dt ~]# cat define-cloud/dockerfiles/cinder-volume/template-overrides.j2
{% extends parent_template %}
# Cinder volume
{% block cinder_volume_footer %}
# Our custom cinder_volume additions to add lightbits
RUN cd /tmp && curl -O https://www.define-technology.com/assets/lightbits_files/cinder_lightbits-0~0~feb46cdb13a4-1.noarch.rpm \
&& curl -O https://www.define-technology.com/assets/lightbits_files/nvme-cli-1.9~2.0.6-1.x86_64.rpm \
&& curl -O https://www.define-technology.com/assets/lightbits_files/os_brick_lightbits-0~0~a7762cb1f97c-1.noarch.rpm \
&& curl -O https://www.define-technology.com/assets/lightbits_files/discovery-client-2.0.6-1~5729656ed3c5.x86_64.rpm \
&& curl -O https://www.define-technology.com/assets/lightbits_files/kolla_start_lb_cinder.sh
RUN dnf -y localinstall /tmp/nvme-cli-1.9~2.0.6-1.x86_64.rpm
RUN rpm -ivh /tmp/os_brick_lightbits-0~0~a7762cb1f97c-1.noarch.rpm --nodeps
RUN dnf -y localinstall /tmp/cinder_lightbits-0~0~feb46cdb13a4-1.noarch.rpm
RUN dnf -y localinstall /tmp/discovery-client-2.0.6-1~5729656ed3c5.x86_64.rpm
RUN mkdir -p /etc/discovery-client/discovery.d/ \
&& chmod 777 /etc/discovery-client \
&& chmod 777 /etc/discovery-client/discovery.d/ \
&& touch /var/log/discovery-client.log \
&& chmod 777 /var/log/discovery-client.log
RUN echo -e "\n\ncinder ALL=(ALL) NOPASSWD: ALL\n" >> /etc/sudoers
RUN sed -i '522s/\ or\ len/:\ \# or len/g' /usr/lib/python3.6/site-packages/cinder/volume/drivers/lightos.py
# commented out to run the discovery service in the hypervisor
# RUN chmod 755 /tmp/kolla_start_lb_cinder.sh && mv /tmp/kolla_start_lb_cinder.sh /usr/local/bin/kolla_start
RUN yum clean all && rm -rf /var/cache/yum
{% endblock %}
[root@registry-dt ~]# cat define-cloud/dockerfiles/cinder-volume/template-overrides.j2^C
[root@registry-dt ~]# cat define-cloud/dockerfiles/nova-compute/template-overrides.j2
{% extends parent_template %}
# Nova compute
{% block nova_compute_footer %}
# Our custom nova-compute additions to add lightbits
RUN cd /tmp && curl -O https://www.define-technology.com/assets/lightbits_files/nova_lightbits-0~0~844b9c5c7a62-1.noarch.rpm \
&& curl -O https://www.define-technology.com/assets/lightbits_files/nvme-cli-1.9~2.0.6-1.x86_64.rpm \
&& curl -O https://www.define-technology.com/assets/lightbits_files/os_brick_lightbits-0~0~a7762cb1f97c-1.noarch.rpm \
&& curl -O https://www.define-technology.com/assets/lightbits_files/discovery-client-2.0.6-1~5729656ed3c5.x86_64.rpm \
&& curl -O https://www.define-technology.com/assets/lightbits_files/kolla_start_lb_nova.sh
RUN yum -y localinstall /tmp/nvme-cli-1.9~2.0.6-1.x86_64.rpm
RUN rpm -ivh /tmp/os_brick_lightbits-0~0~a7762cb1f97c-1.noarch.rpm --nodeps
RUN rpm -ivh /tmp/nova_lightbits-0~0~844b9c5c7a62-1.noarch.rpm --nodeps
RUN yum -y localinstall /tmp/discovery-client-2.0.6-1~5729656ed3c5.x86_64.rpm
RUN mkdir -p /etc/discovery-client/discovery.d/ \
&& chmod 777 /etc/discovery-client \
&& chmod 777 /etc/discovery-client/discovery.d/ \
&& touch /var/log/discovery-client.log \
&& chmod 777 /var/log/discovery-client.log
RUN echo -e "\n\nnova ALL=(ALL) NOPASSWD: ALL\n" >> /etc/sudoers
# commented out to run the discovery service in the hypervisor
# RUN chmod 755 /tmp/kolla_start_lb_nova.sh && mv /tmp/kolla_start_lb_nova.sh /usr/local/bin/kolla_start
RUN yum clean all && rm -rf /var/cache/yum
# hostnqn to be created at hypervisor level and passed in via docker
# RUN nvme gen-hostnqn > /etc/nvme/hostnqn
{% endblock %}
Dump of all notes
[david@Davids-MacBook-Pro-4 notes]$ cat lightbits-notes.txt
# config file
[root@openstack _data]# cat ~/kolla/config/cinder/cinder-volume.conf
[DEFAULT]
enabled_backends=lightos
[lightos]
volume_driver = cinder.volume.drivers.lightos.LightOSVolumeDriver
lightos_api_address = 10.20.60.101
lightos_api_port = 80
lightos_default_compression = False
lightos_default_num_replicas = 3
# docker reconfigure - may need to comment
docker will be restarting - so set enabled_backends=lvm and restart so you can cp the files below
# cinder volume service
docker cp ~/openstack_rpms/cinder_lightbits-2.0.6~0~fa6cb67938d3-1.noarch.rpm cinder_volume:/tmp
docker cp ~/openstack_rpms/os_brick_lightbits-2.0.6~0~986ba852b203-1.noarch.rpm cinder_volume:/tmp
docker cp ~/client_rpms/discovery-client-2.0.6-1~5729656ed3c5.x86_64.rpm cinder_volume:/tmp
# drop to the container
docker exec -it cinder_volume bash
yum -y install python3 nvme-cli
yum localinstall /tmp/*rpm
# edit the (usr/local/bin)kolla_start script (im adding lines 17/18)
17 echo "Starting Lightbits discovery client"
18 /usr/bin/discovery-client serve &
19
20 echo "Running command: '${CMD}${ARGS:+ $ARGS}'"
21 exec ${CMD} ${ARGS}
# exit out of container and restart cinder volume service
# here's an idea of a successful log from the startup
2020-11-24 12:28:09.759 6 INFO cinder.rpc [req-040f2d64-854e-4729-b668-fc79ef485ab4 - - - - -] Automatically selected cinder-scheduler objects version 1.38 as minimum service version.
2020-11-24 12:28:09.763 6 INFO cinder.rpc [req-040f2d64-854e-4729-b668-fc79ef485ab4 - - - - -] Automatically selected cinder-scheduler RPC version 3.11 as minimum service version.
2020-11-24 12:28:09.811 6 INFO cinder.volume.manager [req-040f2d64-854e-4729-b668-fc79ef485ab4 - - - - -] Determined volume DB was empty at startup.
2020-11-24 12:28:09.831 6 INFO cinder.volume.manager [req-040f2d64-854e-4729-b668-fc79ef485ab4 - - - - -] Image-volume cache disabled for host openstack.dt.internal@lightos.
2020-11-24 12:28:09.846 6 INFO oslo_service.service [req-040f2d64-854e-4729-b668-fc79ef485ab4 - - - - -] Starting 1 workers
2020-11-24 12:28:09.857 42 INFO cinder.service [-] Starting cinder-volume node (version 14.3.0)
2020-11-24 12:28:09.876 42 INFO cinder.volume.manager [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Starting volume driver LightOSVolumeDriver (2.0.6~0~fa6cb67938d3)
2020-11-24 12:28:09.876 42 INFO cinder.volume.drivers.lightos [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Invoking get_cluster using GET url: /api/v1/cluster request.body: {}
2020-11-24 12:28:09.913 42 INFO cinder.volume.drivers.lightos [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Connected to LightOS cluster 556661f0-163c-4e23-8324-2cdabdc0b8e3 subsysnqn nqn.2014-08.org.nvmexpress:NVMf:uuid:be009c18-3ccd-47d4-8a98-679279df7347
2020-11-24 12:28:09.914 42 INFO cinder.volume.drivers.lightos [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Invoking get_nodes using GET url: /api/v1/nodes request.body: {}
2020-11-24 12:28:09.930 42 INFO oslo.privsep.daemon [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Running privsep helper: ['sudo', 'cinder-rootwrap', '/etc/cinder/rootwrap.conf', 'privsep-helper', '--config-file', '/etc/cinder/cinder.conf', '--privsep_context', 'os_brick.privileged.default', '--privsep_sock_path', '/tmp/tmpzq0mlY/privsep.sock']
2020-11-24 12:28:10.784 42 INFO oslo.privsep.daemon [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Spawned new privsep daemon via rootwrap
2020-11-24 12:28:10.713 60 INFO oslo.privsep.daemon [-] privsep daemon starting
2020-11-24 12:28:10.720 60 INFO oslo.privsep.daemon [-] privsep process running with uid/gid: 0/0
2020-11-24 12:28:10.725 60 INFO oslo.privsep.daemon [-] privsep process running with capabilities (eff/prm/inh): CAP_SYS_ADMIN/CAP_SYS_ADMIN/none
2020-11-24 12:28:10.726 60 INFO oslo.privsep.daemon [-] privsep daemon running as pid 60
2020-11-24 12:28:10.950 42 INFO cinder.volume.drivers.lightos [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Invoking get_cluster using GET url: /api/v1/cluster request.body: {}
2020-11-24 12:28:11.032 42 INFO cinder.volume.drivers.lightos [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Invoking get_cluster using GET url: /api/v1/cluster request.body: {}
2020-11-24 12:28:11.034 42 INFO cinder.keymgr.migration [req-f5a76b7b-c132-4aa7-80b2-6c8735e5f695 - - - - -] Not migrating encryption keys because the ConfKeyManager's fixed_key is not in use.
2020-11-24 12:28:11.105 42 INFO cinder.volume.manager [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Driver initialization completed successfully.
2020-11-24 12:28:11.110 42 INFO cinder.manager [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Initiating service 3 cleanup
2020-11-24 12:28:11.114 42 INFO cinder.manager [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Service 3 cleanup completed.
2020-11-24 12:28:11.174 42 INFO cinder.volume.manager [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Initializing RPC dependent components of volume driver LightOSVolumeDriver (2.0.6~0~fa6cb67938d3)
2020-11-24 12:28:11.175 42 INFO cinder.volume.drivers.lightos [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Invoking get_cluster using GET url: /api/v1/cluster request.body: {}
2020-11-24 12:28:11.235 42 INFO cinder.volume.manager [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Driver post RPC initialization completed successfully.
2020-11-24 12:30:07.239 42 INFO cinder.volume.drivers.lightos [req-9923f295-ba0e-4837-953d-a54f3b4b6d67 - - - - -] Invoking get_cluster using GET url: /api/v1/cluster request.body: {}
openstack volume create --size 5 firstvol
(tail cinder-volume.log)
# nova service
docker cp nova_lightbits-0~0~844b9c5c7a62-1.noarch.rpm nova_compute:/tmp
docker cp os_brick_lightbits-0~0~a7762cb1f97c-1.noarch.rpm nova_compute:/tmp
docker exec -u root -it nova_compute bash
mkdir -p /etc/discovery-client/discovery.d/
yum -y install python3 nvme-cli
yum -y localinstall /tmp/*rpm
exit; docker restart nova_compute
# note needs nova_compute /etc/discovery_client/
# needed a docker-persist-local plugin
#curl -fsSL https://raw.githubusercontent.com/MatchbookLab/local-persist/master/scripts/install.sh | bash
# create the volume
#[root@openstack-c8 nova-compute]# docker volume create -d local-persist -o mountpoint=/etc/discovery-client/dicovery.d/ --name=discovery-client
#discovery-client
# drop into kolla-deploy container and edit the file (note train release)
docker exec -it kolla-deploy bash
apt-get install vim
Change the bind mounts in /usr/local/share/kolla-ansible/ansible/roles/nova-cell/defaults/main.yml
306 nova_compute_default_volumes:
307 - "{{ node_config_directory }}/nova-compute/:{{ container_config_directory }}/:ro"
308 - "/etc/localtime:/etc/localtime:ro"
309 - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_os_family == 'Debian' else '' }}"
310 - "/lib/modules:/lib/modules:ro"
311 - "/run:/run:shared"
312 - "/dev:/dev"
313 - "/etc/discovery-client/:/etc/discovery-client/"
314 - "/etc/nvme/hostnqn:/etc/nvme/hostnqn"
and in /usr/local/share/kolla-ansible/ansible/roles/cinder/defaults/main.yml (line 150)
146 cinder_volume_default_volumes:
147 - "{{ node_config_directory }}/cinder-volume/:{{ container_config_directory }}/:ro"
148 - "/etc/localtime:/etc/localtime:ro"
149 - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_os_family == 'Debian' else '' }}"
150 - "/dev/:/dev/"
151 - "/etc/discovery-client/:/etc/discovery-client/"
152 - "/etc/nvme/hostnqn:/etc/nvme/hostnqn"
exit;
chmod 777 /etc/discovery-client/discovery.d/
docker exec -it kolla-deploy kolla-ansible -i /etc/kolla/all-in-one reconfigure -t nova
# perform both updates on cinder,nova
root@kolla-deploy:/kolla# kolla-ansible -i /etc/kolla/multinode reconfigure -t cinder,nova
# updated with container version
# run the discovery client on the all-in-one system
docker run -it --rm --privileged --name=discovery_client --net=host -v /dev:/dev/ -v /etc/discovery-client/discovery.d:/etc/discovery-client/discovery.d docker.lightbitslabs.com/lightos-csi/lb-nvme-discovery-client:1.2.0
very-client:1.2.0
Unable to find image 'docker.lightbitslabs.com/lightos-csi/lb-nvme-discovery-client:1.2.0' locally
1.2.0: Pulling from lightos-csi/lb-nvme-discovery-client
5ad559c5ae16: Pull complete
d9f91a9750a7: Pull complete
07193a77857f: Pull complete
0091b54d678a: Pull complete
Digest: sha256:c909bcf8a88f6641c52d69ee4f1ee2b64b9883843feee7c3f291baad82616996
Status: Downloaded newer image for docker.lightbitslabs.com/lightos-csi/lb-nvme-discovery-client:1.2.0
^B^[[Cdiscovery-client configuration: model.AppConfig{Cores:[]int{0}, Logging:logging.Config{Filename:"/var/log/discovery-client.log", ReportCaller:true, Level:"info"}, ClientConfigDir:"/etc/discovery-client/discovery.d/", Debug:model.DebugInfo{Endpoint:"0.0.0.0:6060", Enablepprof:true, Metrics:true}, ReconnectInterval:5000000000, InternalDir:"/etc/discovery-client/internal/", LogPagePaginationEnabled:false}
level=info msg="******************** /discovery-client started ********************" func=cmd.serveCmdFunc file="serve.go:64"
level=warning msg="folder \"/etc/discovery-client/internal/\" does not exists. creating it" func="application.(*App).Start" file="app.go:58"
# note problems with nova so trying the mount one directory further up
cd /etc/discovery-client/
ls
docker cp loving_pare:/etc/discovery-client/discovery-client.yaml .
mkdir internal
docker cp loving_pare:/etc/discovery-client/internal/internal.json ./internal/
docker run -it --rm --privileged --name=discovery_client --net=host -v /dev:/dev/ -v /etc/discovery-client/discovery.d:/etc/discovery-client/discovery.d docker.lightbitslabs.com/lightos-csi/lb-nvme-discovery-client:1.2.0
[root@openstack-c8 scratch]# docker cp cinder_lightbits-0~0~feb46cdb13a4-1.noarch.rpm cinder_volume:/tmp
[root@openstack-c8 scratch]# docker cp os_brick_lightbits-0~0~a7762cb1f97c-1.noarch.rpm cinder_volume:/tmp
[root@openstack-c8 scratch]# docker exec -it cinder_volume bash
(cinder-volume)[cinder@openstack-c8 /]$
# debugging
modprobe nvme
modprobe nvme-fabrics
modprove nvme-tcp
## Fresh start centos 8.2
modprobe nvme
modprobe nvme-fabrics
modprove nvme-tcp
[root@openstack-c8 ~]# cat ~/kolla/config/cinder/cinder-volume.conf
[DEFAULT]
enabled_backends=lvm
[lightos]
volume_driver = cinder.volume.drivers.lightos.LightOSVolumeDriver
lightos_api_address = 10.20.60.101
lightos_api_port = 80
lightos_default_compression = False
lightos_default_num_replicas = 3
# note to start with set enabled_backends=lvm to allow the container start
[root@openstack-c8 client_rpms]# docker cp discovery-client-2.0.6-1~5729656ed3c5.x86_64.rpm cinder_volume:/tmp
[root@openstack-c8 client_rpms]# docker cp nvme-cli-1.9~2.0.6-1.x86_64.rpm cinder_volume:/tmp
[root@openstack-c8 client_rpms]# cd ../openstack_rpms/
[root@openstack-c8 openstack_rpms]# docker cp ./cinder_lightbits-2.0.6~0~fa6cb67938d3-1.noarch.rpm cinder_volume:/tmp
[root@openstack-c8 openstack_rpms]# docker cp ./os_brick_lightbits-2.0.6~0~986ba852b203-1.noarch.rpm cinder_volume:/tmp
[root@openstack-c8 openstack_rpms]# docker exec -u root -it cinder_volume bash
(cinder-volume)[cinder@openstack-c8 /]$ yum -y install python3
(cinder-volume)[root@openstack-c8 /]# ls /tmp/*rpm -l
-rw-r--r--. 1 root root 17318 Nov 23 05:57 /tmp/cinder_lightbits-2.0.6~0~fa6cb67938d3-1.noarch.rpm
-rw-r--r--. 1 root root 4944484 Nov 24 10:10 /tmp/discovery-client-2.0.6-1~5729656ed3c5.x86_64.rpm
-rw-r--r--. 1 root root 796771 Nov 24 10:10 /tmp/nvme-cli-1.9~2.0.6-1.x86_64.rpm
-rw-r--r--. 1 root root 13263 Nov 23 05:57 /tmp/os_brick_lightbits-2.0.6~0~986ba852b203-1.noarch.rpm
(cinder-volume)[root@openstack-c8 /]# yum -y localinstall /tmp/*rpm
# add the following section to start the discovery client on the node
(cinder-volume)[root@openstack-c8 /]# cat /usr/local/bin/kolla_start
#!/bin/bash
set -o errexit
set -o xtrace
# Processing /var/lib/kolla/config_files/config.json as root. This is necessary
# to permit certain files to be controlled by the root user which should
# not be writable by the dropped-privileged user, especially /run_command
sudo -E kolla_set_configs
CMD=$(cat /run_command)
ARGS=""
if [[ ! "${!KOLLA_SKIP_EXTEND_START[@]}" ]]; then
# Run additional commands if present
. kolla_extend_start
fi
# edit the kolla_start script (im adding lines 17/18)
echo "Starting Lightbits discovery client"
/usr/bin/discovery-client serve &
echo "Running command: '${CMD}${ARGS:+ $ARGS}'"
exec ${CMD} ${ARGS}
# end of file
# exit out of the container back to the node
exit
# set container back to lightos
[root@openstack-c8 ~]# cat ~/kolla/config/cinder/cinder-volume.conf
[DEFAULT]
enabled_backends=lightos
[lightos]
volume_driver = cinder.volume.drivers.lightos.LightOSVolumeDriver
lightos_api_address = 10.20.60.101
lightos_api_port = 80
lightos_default_compression = False
lightos_default_num_replicas = 3
# faster way avoid the kolla reconfigure
(cinder-volume)[root@openstack-c8 /]# exit
exit
[root@openstack-c8 ~]# vi /etc/kolla/cinder-volume/cinder.conf
[root@openstack-c8 ~]# docker restart cinder_volume
cinder_volume
# check logs
tail -f /var/lib/docker/volumes/kolla_logs/_data/cinder/cinder-volume.log
...
2020-12-04 07:08:24.135 42 INFO cinder.volume.manager [req-d2a6ac32-06f2-43b8-9019-1564f93f6cdb - - - - -] Initializing RPC dependent components of volume driver LightOSVolumeDriver (2.0.6~0~fa6cb67938d3)
2020-12-04 07:08:24.136 42 INFO cinder.volume.drivers.lightos [req-d2a6ac32-06f2-43b8-9019-1564f93f6cdb - - - - -] Invoking get_cluster using GET url: /api/v1/cluster request.body: {}
2020-12-04 07:08:24.193 42 INFO cinder.volume.manager [req-d2a6ac32-06f2-43b8-9019-1564f93f6cdb - - - - -] Driver post RPC initialization completed successfully.
# ok thats cinder setup - lets turn to nova_compute now
[root@openstack-c8 ~]# docker cp openstack_rpms/os_brick_lightbits-2.0.6~0~986ba852b203-1.noarch.rpm nova_compute:/tmp
[root@openstack-c8 ~]# docker cp openstack_rpms/nova_lightbits-2.0.6~0~2778d6e7595e-1.noarch.rpm nova_compute:/tmp
[root@openstack-c8 ~]# docker cp client_rpms/nvme-cli-1.9~2.0.6-1.x86_64.rpm nova_compute:/tmp
[root@openstack-c8 ~]# docker cp client_rpms/discovery-client-2.0.6-1~5729656ed3c5.x86_64.rpm nova_compute:/tmp
[root@openstack-c8 ~]# docker exec -u root -it nova_compute bash
(nova-compute)[root@openstack-c8 /]# yum -y install python3
yum -y localinstall /tmp/*rpm
# edit the port as its already using 6060
(nova-compute)[root@openstack-c8 /]# cat /etc/discovery-client/discovery-client.yaml
cores: [0]
clientConfigDir: /etc/discovery-client/discovery.d/
internalDir: /etc/discovery-client/internal/
reconnectInterval: 5s
logPagePaginationEnabled: false
logging:
filename: "/var/log/discovery-client.log"
level: info
reportCaller: true
debug:
metrics: true
enablepprof: true
endpoint: 0.0.0.0:6061
# chmod so nova can access
(nova-compute)[root@openstack-c8 /]# chown nova:nova -R /etc/discovery-client* -R
# tell the container to start discovery-client on startup via kolla_start
# file /usr/local/bin/kolla_start
# edit the kolla_start script (im adding lines 17/18)
echo "Starting Lightbits discovery client"
/usr/bin/discovery-client serve &
# create a hostnqn for nova
(nova-compute)[root@openstack-c8 /]# nvme gen-hostnqn > /etc/nvme/hostnqn
exit; docker restart nova_compute
# check docker for discovery-client running
# back to kolla-deploy bash
openstack volume create --size 10 lbvol1
openstack server add volume demo1 lbvol1
# check things out
[root@openstack-c8 ~]# docker exec -u root -it nova_compute bash
(nova-compute)[root@openstack-c8 /]# nvme list
Node SN Model Namespace Usage Format FW Rev
---------------- -------------------- ---------------------------------------- --------- -------------------------- ---------------- --------
/dev/nvme0n1 9ac7a0891e7cad9b Lightbits LightOS 8 10.74 GB / 10.74 GB 4 KiB + 0 B 2.0
/dev/nvme0n2 9ac7a0891e7cad9b Lightbits LightOS 9 0.00 B / 10.74 GB 4 KiB + 0 B 2.0
/dev/nvme0n3 9ac7a0891e7cad9b Lightbits LightOS 10 5.37 GB / 5.37 GB 4 KiB + 0 B 2.0
/dev/nvme1n1 9ac7a0891e7cad9b Lightbits LightOS 8 0.00 B / 10.74 GB 4 KiB + 0 B 2.0
/dev/nvme1n2 9ac7a0891e7cad9b Lightbits LightOS 9 0.00 B / 10.74 GB 4 KiB + 0 B 2.0
/dev/nvme1n3 9ac7a0891e7cad9b Lightbits LightOS 10 0.00 B / 5.37 GB 4 KiB + 0 B 2.0
/dev/nvme2n1 9ac7a0891e7cad9b Lightbits LightOS 8 0.00 B / 10.74 GB 4 KiB + 0 B 2.0
/dev/nvme2n2 9ac7a0891e7cad9b Lightbits LightOS 9 10.74 GB / 10.74 GB 4 KiB + 0 B 2.0
/dev/nvme2n3 9ac7a0891e7cad9b Lightbits LightOS 10 0.00 B / 5.37 GB 4 KiB + 0 B 2.0
### Redo with discovery-client run as a container (with updated files from Muli)
# recreate the discovery-client dir
[root@openstack-c8 ~]# rm -rf /etc/discovery-client/
[root@openstack-c8 ~]# mkdir -p /etc/discovery-client/
[root@openstack-c8 ~]# mkdir -p /etc/discovery-client/discovery.d
[root@openstack-c8 ~]# chmod 777 /etc/discovery-client/
[root@openstack-c8 ~]# docker run -it --rm --privileged --name=discovery_client --net=host -v /dev:/dev/ -v /etc/discovery-client/discovery.d:/etc/discovery-client/discovery.d docker.lightbitslabs.com/lightos-csi/lb-nvme-discovery-client:1.2.0
# drop into the kolla deploy image
root@kolla-deploy:/kolla# grep -n discovery /usr/local/share/kolla-ansible/ansible/roles/cinder/defaults/main.yml
151: - "/etc/discovery-client/:/etc/discovery-client/"
root@kolla-deploy:/kolla# grep -n discovery /usr/local/share/kolla-ansible/ansible/roles/nova-cell/defaults/main.yml
313: - "/etc/discovery-client:/etc/discovery-client"
# reconfigure once done
[root@openstack-c8 scratch]# docker exec -it kolla-deploy kolla-ansible -i /etc/kolla/all-in-one reconfigure
# ok lets now install the packages in the containers
docker cp cinder_lightbits-0~0~feb46cdb13a4-1.noarch.rpm cinder_volume:/tmp
docker cp os_brick_lightbits-0~0~a7762cb1f97c-1.noarch.rpm cinder_volume:/tmp
docker cp ../client_rpms/nvme-cli-1.9~2.0.6-1.x86_64.rpm cinder_volume:/tmp
docker exec -u root -it cinder_volume bash
(cinder-volume)[root@openstack-c8 /]# yum -y install python3
(cinder-volume)[root@openstack-c8 /]# yum -y localinstall /tmp/*rpm
exit
# set cinder.conf back to lightos
[root@openstack-c8 scratch]# grep -n lightos /etc/kolla/cinder-volume/cinder.conf
19:enabled_backends = lightos
66:[lightos]
67:volume_driver = cinder.volume.drivers.lightos.LightOSVolumeDriver
68:lightos_api_address = 10.20.60.101
69:lightos_api_port = 80
70:lightos_default_compression = False
71:lightos_default_num_replicas = 3
docker restart cinder_volume
# lets setup the nova compute container
docker cp nova_lightbits-0~0~844b9c5c7a62-1.noarch.rpm nova_compute:/tmp
docker cp os_brick_lightbits-0~0~a7762cb1f97c-1.noarch.rpm nova_compute:/tmp
docker cp ../client_rpms/nvme-cli-1.9~2.0.6-1.x86_64.rpm nova_compute:/tmp
docker exec -u root -it nova_compute bash
(nova-compute)[root@openstack-c8 /]# yum -y install python3
(nova-compute)[root@openstack-c8 /]# yum -y localinstall /tmp/*rpm
# /etc/nvme/hostnqn existed before i installed the packages!
(nova-compute)[root@openstack-c8 /]# nvme gen-hostnqn > /etc/nvme/hostnqn
# make sure the openstack envirnoment has had the init-runonce script ran against it
# Final take - we have created lightos containers for cinder-volume and nova-compute
# in the globals file
# for train on centos 8
openstack_tag: "train"
horizon_tag: "stein-definetech"
nova_compute_tag: "stein-lightos"
cinder_volume_tag: "stein-lightos"
cinder_tag: "stein"
##### deploying the server yum
yum install python3
pip3 install ansible
pip3 install netaddr
## for centos8 starview
cd /tmp && curl -O https://www.define-technology.com/assets/lightbits_files/cinder_lightbits-0~0~feb46cdb13a4-1.noarch.rpm && curl -O https://www.define-technology.com/assets/lightbits_files/nvme-cli-1.9~2.0.6-1.x86_64.rpm && curl -O https://www.define-technology.com/assets/lightbits_files/os_brick_lightbits-0~0~a7762cb1f97c-1.noarch.rpm
curl -O https://www.define-technology.com/assets/lightbits_files/discovery-client-2.0.6-1~5729656ed3c5.x86_64.rpm
dnf -y localinstall ./nvme-cli-1.9~2.0.6-1.x86_64.rpm
rpm -ivh /tmp/os_brick_lightbits-0~0~a7762cb1f97c-1.noarch.rpm --nodeps
dnf -y localinstall /tmp/cinder_lightbits-0~0~feb46cdb13a4-1.noarch.rpm
dnf -y localinstall /tmp/discovery-client-2.0.6-1~5729656ed3c5.x86_64.rpm
vi /usr/local/bin/kolla_start
# add
fi
echo "Starting Lightbits discovery client"
/usr/bin/discovery-client serve &
echo "Running command: '${CMD}${ARGS:+ $ARGS}'"
exec ${CMD} ${ARGS}
##
mkdir -p /etc/discovery-client/discovery.d/
chmod 777 /etc/discovery-client
chmod 777 /etc/discovery-client/discovery.d/
touch /var/log/discovery-client.log
chmod 777 /var/log/discovery-client.log
# setup in openstack / after deploy
root@kolla-deploy:/kolla# openstack volume type create --property volume_backend_name=lightos lightos
+-------------+--------------------------------------+
| Field | Value |
+-------------+--------------------------------------+
| description | None |
| id | d3b0f515-b0a5-4f64-8d68-e909ba1c4e5b |
| is_public | True |
| name | lightos |
| properties | volume_backend_name='lightos' |
+-------------+--------------------------------------+
root@kolla-deploy:/kolla# cat /etc/kolla/config/cinder/cinder-volume.conf
[DEFAULT]
enabled_backends=rbd-1,lightos
[rbd-1]
rbd_ceph_conf=/etc/ceph/ceph.conf
rbd_user=cinder
backend_host=rbd:volumes
rbd_pool=volumes
volume_backend_name=rbd-1
volume_driver=cinder.volume.drivers.rbd.RBDDriver
rbd_secret_uuid = 69e591d1-4901-41f9-8d49-73a8d4ff7d76
[lightos]
volume_driver = cinder.volume.drivers.lightos.LightOSVolumeDriver
lightos_api_address = 192.168.1.34
lightos_api_port = 80
lightos_default_compression = False
lightos_default_num_replicas = 1
volume_backend_name = lightos
### check out manually
8 nvme connect -t tcp -a 192.168.12.34 -s 4420 -n
9 nvme connect -t tcp -a 192.168.12.34 -s 4420 -n nqn.2016-01.com.lightbitslabs:uuid:5d7e3672-dd22-455d-a23e-1a5628f92bf8 -q ALLOW_NONE
()[root@stu-prr-hci1-n1-mlnx ~]# nvme disconnect-all
()[root@stu-prr-hci1-n1-mlnx ~]# ls /dev/nvm*
/dev/nvme-fabrics
GET NQN from
lbcli get cluster
nqn.2016-01.com.lightbitslabs:uuid:77774c12-60b0-480c-9983-cf524a57f3b4
[david@Davids-MacBook-Pro-4 notes]$
UPDATE from 2.0.6 to 2.2.2 notes
Install discovery client on HOST OS
to document
update your cinder.conf ON THE kolla-deploy HOST
NOTE: not yet sure where we get the JWT setting yet
192.168.102.10 is the IP of the lightbits node in this config
vi ~/kolla/config/cinder/cinder-volume.conf [DEFAULT] enabled_backends=lightos [lightos] volume_driver = cinder.volume.drivers.lightos.LightOSVolumeDriver # Ex. lightos_api_address = 192.168.67.78,192.168.34.56,192.168.12.17 lightos_api_address = 192.168.102.10 lightos_api_port = 443 lightos_default_compression = True lightos_default_num_replicas = 1 volume_backend_name = lightos lightos_skip_ssl_verify=True lightos_jwt=eyJhbGciOiJSUzI1NiIsImtpZCI6InN5c3RlbTpyb290IiwidHlwIjoiSldUIn0.eyJhdWQiOiJMaWdodE9TIiwiZXhwIjoxNjU4OTI2ODIwLCJpYXQiOjE2MjczOTA4MjAsImlzcyI6Ii9yb290L2xpZ2h0b3MtY2VydGlmaWNhdGVzL2NlcnQtbGItYWRtaW4ta2V5LnBlbSIsImp0aSI6ImZWQUVyVVQ5cHNBaFZoeWY3SVRjalEiLCJuYmYiOjE2MjczOTA4MjAsInJvbGVzIjpbInN5c3RlbTpjbHVzdGVyLWFkbWluIl0sInN1YiI6ImxpZ2h0b3MtY2xpZW50In0.q344_XJwBeq_qeTE7ZDkFUfeHbaZ-bLz8DTKquh3Y07kpuvWSTMVUNEfUnTlPhsLaB_rP6h-20lDi1BK4kOFhD7PtLYLbGpIEOMmYKpmsIrlb7ayU2zEat3jHHds0IPzkpV9u4Vp8wUKYdpr0azLGJlGD8h7pUwsGE6o_XLPXPYIBtoW1yhvgcjpGKrAvRrLOYQmi0X6XFkc1FLU_UKJDtYJSf-wEWP33J21uj7Rl-4-ojicNz_P-lG-QMyB-Nnq_w4kigBpPnKlPZ3LkysyLxxFb22AnmSUfhkXjqzjHf9ZINeDpLM5rSCDuyX_1_FXE9AF5A1ZUdM6T48B24_3aw
make sure you are mounting /etc/discovery-client and /etc/nvme/hostnqn in the cinder_volume and nova_compute container
Edit ~/kolla/globals.yml
vi ~/kolla/globals.yml
...
# if installing lightos then set the following to pass the hostnqn and discovery dir from the host to the relevant containers
lightos_extra_volumes: ["/etc/discovery-client/:/etc/discovery-client/","/etc/nvme/hostnqn:/etc/nvme/hostnqn"]
nova_compute_extra_volumes: "{{ nova_extra_volumes | default([]) + lightos_extra_volumes }}"
cinder_volume_extra_volumes: "{{ cinder_extra_volumes | default([]) + lightos_extra_volumes }}"
reconfigure kolla warning this will probably re-pull nova-compute AND cinder volume so do this now or you have to repeat (go on try it)
podman exec -it kolla-deploy kolla-ansible -i /etc/kolla/multinode reconfigure nova,cinder .....
remove old rpms from containers
DO NOT UPGRADE
showing removal of discovery client here just in case
[root@server01 ~]# docker exec -u root -it cinder_volume bash (cinder-volume)[root@server01 /]# cd /etc/discovery-client/openstack_rpms/ (cinder-volume)[root@server01 openstack_rpms]# ls cinder_lightbits-2.2.2~94478f35ae2c-1.noarch.rpm nova_lightbits-2.2.2~076a3dcde0f4-1.noarch.rpm os_brick_lightbits-2.2.2~64377d98e9de-1.noarch.rpm (cinder-volume)[root@server01 openstack_rpms]# rpm -e cinder_lightbits os_brick_lightbits discovery-client error: package discovery-client is not installed search for cinder module /usr/lib/python3.6/site-packages/cinder + case "$1" in + cd /opt/extra_lightbits + python3 -c 'from openstack_patcher import *; unpatch_os_brick()' search for os_brick module /usr/lib/python3.6/site-packages/os_brick
install new rpms
note --nodeps on the os-brick rpm!
(cinder-volume)[root@server01 openstack_rpms]# rpm --nodeps -ivh os_brick_lightbits-2.2.2~64377d98e9de-1.noarch.rpm Verifying... ################################# [100%] Preparing... ################################# [100%] Updating / installing... 1:os_brick_lightbits-2.2.2~64377d98################################# [100%] search for os_brick module /usr/lib/python3.6/site-packages/os_brick (cinder-volume)[root@server01 openstack_rpms]# rpm -ivh cinder_lightbits-2.2.2~94478f35ae2c-1.noarch.rpm Verifying... ################################# [100%] Preparing... ################################# [100%] Updating / installing... 1:cinder_lightbits-2.2.2~94478f35ae################################# [100%] search for cinder module /usr/lib/python3.6/site-packages/cinder
Comment out discovery process check
if you forget this the driver will fail to start with this error
2021-07-28 20:30:37.446 31 ERROR cinder.volume.manager File "/usr/lib/python3.6/site-packages/cinder/volume/drivers/lightos.py", line 723, in check_for_setup_error 2021-07-28 20:30:37.446 31 ERROR cinder.volume.manager found_dsc = self.connector.find_dsc() 2021-07-28 20:30:37.446 31 ERROR cinder.volume.manager File "/usr/lib/python3.6/site-packages/os_brick/initiator/connectors/lightos.py", line 77, in find_dsc 2021-07-28 20:30:37.446 31 ERROR cinder.volume.manager out, err = self._execute(*cmd, root_helper=self._root_helper, run_as_root=True) 2021-07-28 20:30:37.446 31 ERROR cinder.volume.manager File "/usr/lib/python3.6/site-packages/os_brick/executor.py", line 52, in _execute 2021-07-28 20:30:37.446 31 ERROR cinder.volume.manager result = self.__execute(*args, **kwargs) 2021-07-28 20:30:37.446 31 ERROR cinder.volume.manager File "/usr/lib/python3.6/site-packages/os_brick/privileged/rootwrap.py", line 169, in execute 2021-07-28 20:30:37.446 31 ERROR cinder.volume.manager return execute_root(*cmd, **kwargs) 2021-07-28 20:30:37.446 31 ERROR cinder.volume.manager File "/usr/lib/python3.6/site-packages/oslo_privsep/priv_context.py", line 245, in _wrap 2021-07-28 20:30:37.446 31 ERROR cinder.volume.manager return self.channel.remote_call(name, args, kwargs) 2021-07-28 20:30:37.446 31 ERROR cinder.volume.manager File "/usr/lib/python3.6/site-packages/oslo_privsep/daemon.py", line 224, in remote_call 2021-07-28 20:30:37.446 31 ERROR cinder.volume.manager raise exc_type(*result[2]) 2021-07-28 20:30:37.446 31 ERROR cinder.volume.manager oslo_concurrency.processutils.ProcessExecutionError: Unexpected error while running command. 2021-07-28 20:30:37.446 31 ERROR cinder.volume.manager Command: bash -c ps ax | grep discovery-client | grep -v grep 2021-07-28 20:30:37.446 31 ERROR cinder.volume.manager Exit code: 1
easiest way is to install vim in the container but there is a broken reop in train so use this
(cinder-volume)[root@server01 openstack_rpms]# yum -y --disablerepo=rabbitmq_rabbitmq-erlang install vim
and comment out the following lines fro 723 onwards:
vi /usr/lib/python3.6/site-packages/cinder/volume/drivers/lightos.py
# found_dsc = self.connector.find_dsc() # if not found_dsc: # msg = 'LIGHTOS: Cinder driver requires a local discovery_client for image_to/from_volume operations' # raise exception.VolumeBackendAPIException(message=msg)
restart cinder-volume container
(cinder-volume)[root@server01 openstack_rpms]# exit exit [root@server01 ~]# docker restart cinder_volume cinder_volume
don't forget to check the logs for issues!
[root@server01 ~]# tail -n 30 /var/log/kolla/cinder/cinder-volume.log
2021-07-28 22:29:25.039 31 ERROR cinder.service [-] Manager for service cinder-volume server01.dt.api@lightos is reporting problems, not sending heartbeat. Service will appear "down".
2021-07-28 22:29:35.042 31 ERROR cinder.service [-] Manager for service cinder-volume server01.dt.api@lightos is reporting problems, not sending heartbeat. Service will appear "down".
2021-07-28 22:29:39.731 31 WARNING cinder.volume.manager [req-58c14c44-1739-48b0-a2fb-ab4302a7a8c9 - - - - -] Update driver status failed: (config name lightos) is uninitialized.
2021-07-28 22:29:45.045 31 ERROR cinder.service [-] Manager for service cinder-volume server01.dt.api@lightos is reporting problems, not sending heartbeat. Service will appear "down".
2021-07-28 22:29:55.047 31 ERROR cinder.service [-] Manager for service cinder-volume server01.dt.api@lightos is reporting problems, not sending heartbeat. Service will appear "down".
2021-07-28 22:29:58.996 7 INFO oslo_service.service [req-a5c5068f-7b18-4c74-9d63-9e9ae9e8fb62 - - - - -] Caught SIGTERM, stopping children
2021-07-28 22:29:59.003 7 INFO oslo_service.service [req-a5c5068f-7b18-4c74-9d63-9e9ae9e8fb62 - - - - -] Waiting on 1 children to exit
2021-07-28 22:30:05.049 31 ERROR cinder.service [-] Manager for service cinder-volume server01.dt.api@lightos is reporting problems, not sending heartbeat. Service will appear "down".
2021-07-28 22:30:11.038 7 INFO cinder.rpc [req-1f6010ba-4953-4b9f-b82e-38cb21d58331 - - - - -] Automatically selected cinder-scheduler objects version 1.38 as minimum service version.
2021-07-28 22:30:11.043 7 INFO cinder.rpc [req-1f6010ba-4953-4b9f-b82e-38cb21d58331 - - - - -] Automatically selected cinder-scheduler RPC version 3.11 as minimum service version.
2021-07-28 22:30:11.135 7 INFO cinder.volume.manager [req-1f6010ba-4953-4b9f-b82e-38cb21d58331 - - - - -] Determined volume DB was not empty at startup.
2021-07-28 22:30:11.157 7 INFO cinder.volume.manager [req-1f6010ba-4953-4b9f-b82e-38cb21d58331 - - - - -] Image-volume cache disabled for host server01.dt.api@lightos.
2021-07-28 22:30:11.171 7 INFO oslo_service.service [req-1f6010ba-4953-4b9f-b82e-38cb21d58331 - - - - -] Starting 1 workers
2021-07-28 22:30:11.181 31 INFO cinder.service [-] Starting cinder-volume node (version 15.5.0)
2021-07-28 22:30:11.203 31 INFO cinder.volume.manager [req-53e5cd89-7c5f-422c-8ab4-de252c7c2985 - - - - -] Starting volume driver LightOSVolumeDriver (2.2.2~94478f35ae2c)
2021-07-28 22:30:11.203 31 INFO cinder.volume.drivers.lightos [req-53e5cd89-7c5f-422c-8ab4-de252c7c2985 - - - - -] Invoking get_cluster_info using GET url: /api/v2/clusterinfo request.body: {} ssl_verify: False
2021-07-28 22:30:11.258 31 INFO cinder.volume.drivers.lightos [req-53e5cd89-7c5f-422c-8ab4-de252c7c2985 - - - - -] Connected to LightOS cluster a03d47ba-7a6f-4bb2-8d9e-9b59daa27a9f subsysnqn nqn.2016-01.com.lightbitslabs:uuid:4743d933-4058-4a28-9483-6942455f1c3c
2021-07-28 22:30:11.258 31 INFO cinder.volume.drivers.lightos [req-53e5cd89-7c5f-422c-8ab4-de252c7c2985 - - - - -] Invoking get_nodes using GET url: /api/v2/nodes request.body: {} ssl_verify: False
2021-07-28 22:30:11.290 31 INFO oslo.privsep.daemon [req-53e5cd89-7c5f-422c-8ab4-de252c7c2985 - - - - -] Running privsep helper: ['sudo', 'cinder-rootwrap', '/etc/cinder/rootwrap.conf', 'privsep-helper', '--config-file', '/etc/cinder/cinder.conf', '--privsep_context', 'os_brick.privileged.default', '--privsep_sock_path', '/tmp/tmp1s31gsr0/privsep.sock']
2021-07-28 22:30:11.953 31 INFO oslo.privsep.daemon [req-53e5cd89-7c5f-422c-8ab4-de252c7c2985 - - - - -] Spawned new privsep daemon via rootwrap
2021-07-28 22:30:11.858 38 INFO oslo.privsep.daemon [-] privsep daemon starting
2021-07-28 22:30:11.864 38 INFO oslo.privsep.daemon [-] privsep process running with uid/gid: 0/0
2021-07-28 22:30:11.867 38 INFO oslo.privsep.daemon [-] privsep process running with capabilities (eff/prm/inh): CAP_SYS_ADMIN/CAP_SYS_ADMIN/none
2021-07-28 22:30:11.867 38 INFO oslo.privsep.daemon [-] privsep daemon running as pid 38
2021-07-28 22:30:12.178 31 INFO cinder.keymgr.migration [req-c09dec64-f454-4c83-8950-92f03ba1da85 - - - - -] Not migrating encryption keys because the ConfKeyManager's fixed_key is not in use.
2021-07-28 22:30:12.191 31 INFO cinder.volume.manager [req-53e5cd89-7c5f-422c-8ab4-de252c7c2985 - - - - -] Driver initialization completed successfully.
2021-07-28 22:30:12.195 31 INFO cinder.manager [req-53e5cd89-7c5f-422c-8ab4-de252c7c2985 - - - - -] Initiating service 34 cleanup
2021-07-28 22:30:12.198 31 INFO cinder.manager [req-53e5cd89-7c5f-422c-8ab4-de252c7c2985 - - - - -] Service 34 cleanup completed.
2021-07-28 22:30:12.307 31 INFO cinder.volume.manager [req-53e5cd89-7c5f-422c-8ab4-de252c7c2985 - - - - -] Initializing RPC dependent components of volume driver LightOSVolumeDriver (2.2.2~94478f35ae2c)
2021-07-28 22:30:12.325 31 INFO cinder.volume.manager [req-53e5cd89-7c5f-422c-8ab4-de252c7c2985 - - - - -] Driver post RPC initialization completed successfully.
Cinder is ALL GOOD!
uninstall rpms from nova_compute container
TODO:
add evidence of this didn't document as went along
# rpm -e cinder_lightbits os_brick_lightbits discovery-client
install new rpms in nova_compute container
TODO:
add evidence of this didn't document as went along
# cd /etc/discovery-client/openstack_rpms # rpm -i --nodeps os_brick_lightbits-2.2.2~64377d98e9de-1.noarch.rpm # rpm -i nova_lightbits-2.2.2~076a3dcde0f4-1.noarch.rpm
stop this container starting discovery client on start
check usr/local/bin/kolla_start
(nova-compute)[root@server01 /]# cat /usr/local/bin/kolla_start
#!/bin/bash
set -o errexit
set -o xtrace
# Processing /var/lib/kolla/config_files/config.json as root. This is necessary
# to permit certain files to be controlled by the root user which should
# not be writable by the dropped-privileged user, especially /run_command
sudo -E kolla_set_configs
CMD=$(cat /run_command)
ARGS=""
# Install/remove custom CA certificates
sudo kolla_copy_cacerts
if [[ ! "${!KOLLA_SKIP_EXTEND_START[@]}" ]]; then
# Run additional commands if present
. kolla_extend_start
fi
echo "Starting Lightbits discovery client"
sudo /usr/bin/discovery-client serve &
echo "Running command: '${CMD}${ARGS:+ $ARGS}'"
exec ${CMD} ${ARGS}
See the lines:
echo "Starting Lightbits discovery client" sudo /usr/bin/discovery-client serve &
comment or remove them
restart container
and check it all still works of course, mine was fine