Rebuild cinder with lightbits integration - train notes
Jump to navigation
Jump to search
Dump of all notes
[david@Davids-MacBook-Pro-4 notes]$ cat lightbits-notes.txt
# config file
[root@openstack _data]# cat ~/kolla/config/cinder/cinder-volume.conf
[DEFAULT]
enabled_backends=lightos
[lightos]
volume_driver = cinder.volume.drivers.lightos.LightOSVolumeDriver
lightos_api_address = 10.20.60.101
lightos_api_port = 80
lightos_default_compression = False
lightos_default_num_replicas = 3
# docker reconfigure - may need to comment
docker will be restarting - so set enabled_backends=lvm and restart so you can cp the files below
# cinder volume service
docker cp ~/openstack_rpms/cinder_lightbits-2.0.6~0~fa6cb67938d3-1.noarch.rpm cinder_volume:/tmp
docker cp ~/openstack_rpms/os_brick_lightbits-2.0.6~0~986ba852b203-1.noarch.rpm cinder_volume:/tmp
docker cp ~/client_rpms/discovery-client-2.0.6-1~5729656ed3c5.x86_64.rpm cinder_volume:/tmp
# drop to the container
docker exec -it cinder_volume bash
yum -y install python3 nvme-cli
yum localinstall /tmp/*rpm
# edit the (usr/local/bin)kolla_start script (im adding lines 17/18)
17 echo "Starting Lightbits discovery client"
18 /usr/bin/discovery-client serve &
19
20 echo "Running command: '${CMD}${ARGS:+ $ARGS}'"
21 exec ${CMD} ${ARGS}
# exit out of container and restart cinder volume service
# here's an idea of a successful log from the startup
2020-11-24 12:28:09.759 6 INFO cinder.rpc [req-040f2d64-854e-4729-b668-fc79ef485ab4 - - - - -] Automatically selected cinder-scheduler objects version 1.38 as minimum service version.
2020-11-24 12:28:09.763 6 INFO cinder.rpc [req-040f2d64-854e-4729-b668-fc79ef485ab4 - - - - -] Automatically selected cinder-scheduler RPC version 3.11 as minimum service version.
2020-11-24 12:28:09.811 6 INFO cinder.volume.manager [req-040f2d64-854e-4729-b668-fc79ef485ab4 - - - - -] Determined volume DB was empty at startup.
2020-11-24 12:28:09.831 6 INFO cinder.volume.manager [req-040f2d64-854e-4729-b668-fc79ef485ab4 - - - - -] Image-volume cache disabled for host openstack.dt.internal@lightos.
2020-11-24 12:28:09.846 6 INFO oslo_service.service [req-040f2d64-854e-4729-b668-fc79ef485ab4 - - - - -] Starting 1 workers
2020-11-24 12:28:09.857 42 INFO cinder.service [-] Starting cinder-volume node (version 14.3.0)
2020-11-24 12:28:09.876 42 INFO cinder.volume.manager [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Starting volume driver LightOSVolumeDriver (2.0.6~0~fa6cb67938d3)
2020-11-24 12:28:09.876 42 INFO cinder.volume.drivers.lightos [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Invoking get_cluster using GET url: /api/v1/cluster request.body: {}
2020-11-24 12:28:09.913 42 INFO cinder.volume.drivers.lightos [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Connected to LightOS cluster 556661f0-163c-4e23-8324-2cdabdc0b8e3 subsysnqn nqn.2014-08.org.nvmexpress:NVMf:uuid:be009c18-3ccd-47d4-8a98-679279df7347
2020-11-24 12:28:09.914 42 INFO cinder.volume.drivers.lightos [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Invoking get_nodes using GET url: /api/v1/nodes request.body: {}
2020-11-24 12:28:09.930 42 INFO oslo.privsep.daemon [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Running privsep helper: ['sudo', 'cinder-rootwrap', '/etc/cinder/rootwrap.conf', 'privsep-helper', '--config-file', '/etc/cinder/cinder.conf', '--privsep_context', 'os_brick.privileged.default', '--privsep_sock_path', '/tmp/tmpzq0mlY/privsep.sock']
2020-11-24 12:28:10.784 42 INFO oslo.privsep.daemon [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Spawned new privsep daemon via rootwrap
2020-11-24 12:28:10.713 60 INFO oslo.privsep.daemon [-] privsep daemon starting
2020-11-24 12:28:10.720 60 INFO oslo.privsep.daemon [-] privsep process running with uid/gid: 0/0
2020-11-24 12:28:10.725 60 INFO oslo.privsep.daemon [-] privsep process running with capabilities (eff/prm/inh): CAP_SYS_ADMIN/CAP_SYS_ADMIN/none
2020-11-24 12:28:10.726 60 INFO oslo.privsep.daemon [-] privsep daemon running as pid 60
2020-11-24 12:28:10.950 42 INFO cinder.volume.drivers.lightos [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Invoking get_cluster using GET url: /api/v1/cluster request.body: {}
2020-11-24 12:28:11.032 42 INFO cinder.volume.drivers.lightos [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Invoking get_cluster using GET url: /api/v1/cluster request.body: {}
2020-11-24 12:28:11.034 42 INFO cinder.keymgr.migration [req-f5a76b7b-c132-4aa7-80b2-6c8735e5f695 - - - - -] Not migrating encryption keys because the ConfKeyManager's fixed_key is not in use.
2020-11-24 12:28:11.105 42 INFO cinder.volume.manager [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Driver initialization completed successfully.
2020-11-24 12:28:11.110 42 INFO cinder.manager [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Initiating service 3 cleanup
2020-11-24 12:28:11.114 42 INFO cinder.manager [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Service 3 cleanup completed.
2020-11-24 12:28:11.174 42 INFO cinder.volume.manager [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Initializing RPC dependent components of volume driver LightOSVolumeDriver (2.0.6~0~fa6cb67938d3)
2020-11-24 12:28:11.175 42 INFO cinder.volume.drivers.lightos [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Invoking get_cluster using GET url: /api/v1/cluster request.body: {}
2020-11-24 12:28:11.235 42 INFO cinder.volume.manager [req-450b49da-f260-4a46-9a52-745ce3c9a1ec - - - - -] Driver post RPC initialization completed successfully.
2020-11-24 12:30:07.239 42 INFO cinder.volume.drivers.lightos [req-9923f295-ba0e-4837-953d-a54f3b4b6d67 - - - - -] Invoking get_cluster using GET url: /api/v1/cluster request.body: {}
openstack volume create --size 5 firstvol
(tail cinder-volume.log)
# nova service
docker cp nova_lightbits-0~0~844b9c5c7a62-1.noarch.rpm nova_compute:/tmp
docker cp os_brick_lightbits-0~0~a7762cb1f97c-1.noarch.rpm nova_compute:/tmp
docker exec -u root -it nova_compute bash
mkdir -p /etc/discovery-client/discovery.d/
yum -y install python3 nvme-cli
yum -y localinstall /tmp/*rpm
exit; docker restart nova_compute
# note needs nova_compute /etc/discovery_client/
# needed a docker-persist-local plugin
#curl -fsSL https://raw.githubusercontent.com/MatchbookLab/local-persist/master/scripts/install.sh | bash
# create the volume
#[root@openstack-c8 nova-compute]# docker volume create -d local-persist -o mountpoint=/etc/discovery-client/dicovery.d/ --name=discovery-client
#discovery-client
# drop into kolla-deploy container and edit the file (note train release)
docker exec -it kolla-deploy bash
apt-get install vim
Change the bind mounts in /usr/local/share/kolla-ansible/ansible/roles/nova-cell/defaults/main.yml
306 nova_compute_default_volumes:
307 - "{{ node_config_directory }}/nova-compute/:{{ container_config_directory }}/:ro"
308 - "/etc/localtime:/etc/localtime:ro"
309 - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_os_family == 'Debian' else '' }}"
310 - "/lib/modules:/lib/modules:ro"
311 - "/run:/run:shared"
312 - "/dev:/dev"
313 - "/etc/discovery-client/:/etc/discovery-client/"
314 - "/etc/nvme/hostnqn:/etc/nvme/hostnqn"
and in /usr/local/share/kolla-ansible/ansible/roles/cinder/defaults/main.yml (line 150)
146 cinder_volume_default_volumes:
147 - "{{ node_config_directory }}/cinder-volume/:{{ container_config_directory }}/:ro"
148 - "/etc/localtime:/etc/localtime:ro"
149 - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_os_family == 'Debian' else '' }}"
150 - "/dev/:/dev/"
151 - "/etc/discovery-client/:/etc/discovery-client/"
152 - "/etc/nvme/hostnqn:/etc/nvme/hostnqn"
exit;
chmod 777 /etc/discovery-client/discovery.d/
docker exec -it kolla-deploy kolla-ansible -i /etc/kolla/all-in-one reconfigure -t nova
# perform both updates on cinder,nova
root@kolla-deploy:/kolla# kolla-ansible -i /etc/kolla/multinode reconfigure -t cinder,nova
# updated with container version
# run the discovery client on the all-in-one system
docker run -it --rm --privileged --name=discovery_client --net=host -v /dev:/dev/ -v /etc/discovery-client/discovery.d:/etc/discovery-client/discovery.d docker.lightbitslabs.com/lightos-csi/lb-nvme-discovery-client:1.2.0
very-client:1.2.0
Unable to find image 'docker.lightbitslabs.com/lightos-csi/lb-nvme-discovery-client:1.2.0' locally
1.2.0: Pulling from lightos-csi/lb-nvme-discovery-client
5ad559c5ae16: Pull complete
d9f91a9750a7: Pull complete
07193a77857f: Pull complete
0091b54d678a: Pull complete
Digest: sha256:c909bcf8a88f6641c52d69ee4f1ee2b64b9883843feee7c3f291baad82616996
Status: Downloaded newer image for docker.lightbitslabs.com/lightos-csi/lb-nvme-discovery-client:1.2.0
^B^[[Cdiscovery-client configuration: model.AppConfig{Cores:[]int{0}, Logging:logging.Config{Filename:"/var/log/discovery-client.log", ReportCaller:true, Level:"info"}, ClientConfigDir:"/etc/discovery-client/discovery.d/", Debug:model.DebugInfo{Endpoint:"0.0.0.0:6060", Enablepprof:true, Metrics:true}, ReconnectInterval:5000000000, InternalDir:"/etc/discovery-client/internal/", LogPagePaginationEnabled:false}
level=info msg="******************** /discovery-client started ********************" func=cmd.serveCmdFunc file="serve.go:64"
level=warning msg="folder \"/etc/discovery-client/internal/\" does not exists. creating it" func="application.(*App).Start" file="app.go:58"
# note problems with nova so trying the mount one directory further up
cd /etc/discovery-client/
ls
docker cp loving_pare:/etc/discovery-client/discovery-client.yaml .
mkdir internal
docker cp loving_pare:/etc/discovery-client/internal/internal.json ./internal/
docker run -it --rm --privileged --name=discovery_client --net=host -v /dev:/dev/ -v /etc/discovery-client/discovery.d:/etc/discovery-client/discovery.d docker.lightbitslabs.com/lightos-csi/lb-nvme-discovery-client:1.2.0
[root@openstack-c8 scratch]# docker cp cinder_lightbits-0~0~feb46cdb13a4-1.noarch.rpm cinder_volume:/tmp
[root@openstack-c8 scratch]# docker cp os_brick_lightbits-0~0~a7762cb1f97c-1.noarch.rpm cinder_volume:/tmp
[root@openstack-c8 scratch]# docker exec -it cinder_volume bash
(cinder-volume)[cinder@openstack-c8 /]$
# debugging
modprobe nvme
modprobe nvme-fabrics
modprove nvme-tcp
## Fresh start centos 8.2
modprobe nvme
modprobe nvme-fabrics
modprove nvme-tcp
[root@openstack-c8 ~]# cat ~/kolla/config/cinder/cinder-volume.conf
[DEFAULT]
enabled_backends=lvm
[lightos]
volume_driver = cinder.volume.drivers.lightos.LightOSVolumeDriver
lightos_api_address = 10.20.60.101
lightos_api_port = 80
lightos_default_compression = False
lightos_default_num_replicas = 3
# note to start with set enabled_backends=lvm to allow the container start
[root@openstack-c8 client_rpms]# docker cp discovery-client-2.0.6-1~5729656ed3c5.x86_64.rpm cinder_volume:/tmp
[root@openstack-c8 client_rpms]# docker cp nvme-cli-1.9~2.0.6-1.x86_64.rpm cinder_volume:/tmp
[root@openstack-c8 client_rpms]# cd ../openstack_rpms/
[root@openstack-c8 openstack_rpms]# docker cp ./cinder_lightbits-2.0.6~0~fa6cb67938d3-1.noarch.rpm cinder_volume:/tmp
[root@openstack-c8 openstack_rpms]# docker cp ./os_brick_lightbits-2.0.6~0~986ba852b203-1.noarch.rpm cinder_volume:/tmp
[root@openstack-c8 openstack_rpms]# docker exec -u root -it cinder_volume bash
(cinder-volume)[cinder@openstack-c8 /]$ yum -y install python3
(cinder-volume)[root@openstack-c8 /]# ls /tmp/*rpm -l
-rw-r--r--. 1 root root 17318 Nov 23 05:57 /tmp/cinder_lightbits-2.0.6~0~fa6cb67938d3-1.noarch.rpm
-rw-r--r--. 1 root root 4944484 Nov 24 10:10 /tmp/discovery-client-2.0.6-1~5729656ed3c5.x86_64.rpm
-rw-r--r--. 1 root root 796771 Nov 24 10:10 /tmp/nvme-cli-1.9~2.0.6-1.x86_64.rpm
-rw-r--r--. 1 root root 13263 Nov 23 05:57 /tmp/os_brick_lightbits-2.0.6~0~986ba852b203-1.noarch.rpm
(cinder-volume)[root@openstack-c8 /]# yum -y localinstall /tmp/*rpm
# add the following section to start the discovery client on the node
(cinder-volume)[root@openstack-c8 /]# cat /usr/local/bin/kolla_start
#!/bin/bash
set -o errexit
set -o xtrace
# Processing /var/lib/kolla/config_files/config.json as root. This is necessary
# to permit certain files to be controlled by the root user which should
# not be writable by the dropped-privileged user, especially /run_command
sudo -E kolla_set_configs
CMD=$(cat /run_command)
ARGS=""
if [[ ! "${!KOLLA_SKIP_EXTEND_START[@]}" ]]; then
# Run additional commands if present
. kolla_extend_start
fi
# edit the kolla_start script (im adding lines 17/18)
echo "Starting Lightbits discovery client"
/usr/bin/discovery-client serve &
echo "Running command: '${CMD}${ARGS:+ $ARGS}'"
exec ${CMD} ${ARGS}
# end of file
# exit out of the container back to the node
exit
# set container back to lightos
[root@openstack-c8 ~]# cat ~/kolla/config/cinder/cinder-volume.conf
[DEFAULT]
enabled_backends=lightos
[lightos]
volume_driver = cinder.volume.drivers.lightos.LightOSVolumeDriver
lightos_api_address = 10.20.60.101
lightos_api_port = 80
lightos_default_compression = False
lightos_default_num_replicas = 3
# faster way avoid the kolla reconfigure
(cinder-volume)[root@openstack-c8 /]# exit
exit
[root@openstack-c8 ~]# vi /etc/kolla/cinder-volume/cinder.conf
[root@openstack-c8 ~]# docker restart cinder_volume
cinder_volume
# check logs
tail -f /var/lib/docker/volumes/kolla_logs/_data/cinder/cinder-volume.log
...
2020-12-04 07:08:24.135 42 INFO cinder.volume.manager [req-d2a6ac32-06f2-43b8-9019-1564f93f6cdb - - - - -] Initializing RPC dependent components of volume driver LightOSVolumeDriver (2.0.6~0~fa6cb67938d3)
2020-12-04 07:08:24.136 42 INFO cinder.volume.drivers.lightos [req-d2a6ac32-06f2-43b8-9019-1564f93f6cdb - - - - -] Invoking get_cluster using GET url: /api/v1/cluster request.body: {}
2020-12-04 07:08:24.193 42 INFO cinder.volume.manager [req-d2a6ac32-06f2-43b8-9019-1564f93f6cdb - - - - -] Driver post RPC initialization completed successfully.
# ok thats cinder setup - lets turn to nova_compute now
[root@openstack-c8 ~]# docker cp openstack_rpms/os_brick_lightbits-2.0.6~0~986ba852b203-1.noarch.rpm nova_compute:/tmp
[root@openstack-c8 ~]# docker cp openstack_rpms/nova_lightbits-2.0.6~0~2778d6e7595e-1.noarch.rpm nova_compute:/tmp
[root@openstack-c8 ~]# docker cp client_rpms/nvme-cli-1.9~2.0.6-1.x86_64.rpm nova_compute:/tmp
[root@openstack-c8 ~]# docker cp client_rpms/discovery-client-2.0.6-1~5729656ed3c5.x86_64.rpm nova_compute:/tmp
[root@openstack-c8 ~]# docker exec -u root -it nova_compute bash
(nova-compute)[root@openstack-c8 /]# yum -y install python3
yum -y localinstall /tmp/*rpm
# edit the port as its already using 6060
(nova-compute)[root@openstack-c8 /]# cat /etc/discovery-client/discovery-client.yaml
cores: [0]
clientConfigDir: /etc/discovery-client/discovery.d/
internalDir: /etc/discovery-client/internal/
reconnectInterval: 5s
logPagePaginationEnabled: false
logging:
filename: "/var/log/discovery-client.log"
level: info
reportCaller: true
debug:
metrics: true
enablepprof: true
endpoint: 0.0.0.0:6061
# chmod so nova can access
(nova-compute)[root@openstack-c8 /]# chown nova:nova -R /etc/discovery-client* -R
# tell the container to start discovery-client on startup via kolla_start
# file /usr/local/bin/kolla_start
# edit the kolla_start script (im adding lines 17/18)
echo "Starting Lightbits discovery client"
/usr/bin/discovery-client serve &
# create a hostnqn for nova
(nova-compute)[root@openstack-c8 /]# nvme gen-hostnqn > /etc/nvme/hostnqn
exit; docker restart nova_compute
# check docker for discovery-client running
# back to kolla-deploy bash
openstack volume create --size 10 lbvol1
openstack server add volume demo1 lbvol1
# check things out
[root@openstack-c8 ~]# docker exec -u root -it nova_compute bash
(nova-compute)[root@openstack-c8 /]# nvme list
Node SN Model Namespace Usage Format FW Rev
---------------- -------------------- ---------------------------------------- --------- -------------------------- ---------------- --------
/dev/nvme0n1 9ac7a0891e7cad9b Lightbits LightOS 8 10.74 GB / 10.74 GB 4 KiB + 0 B 2.0
/dev/nvme0n2 9ac7a0891e7cad9b Lightbits LightOS 9 0.00 B / 10.74 GB 4 KiB + 0 B 2.0
/dev/nvme0n3 9ac7a0891e7cad9b Lightbits LightOS 10 5.37 GB / 5.37 GB 4 KiB + 0 B 2.0
/dev/nvme1n1 9ac7a0891e7cad9b Lightbits LightOS 8 0.00 B / 10.74 GB 4 KiB + 0 B 2.0
/dev/nvme1n2 9ac7a0891e7cad9b Lightbits LightOS 9 0.00 B / 10.74 GB 4 KiB + 0 B 2.0
/dev/nvme1n3 9ac7a0891e7cad9b Lightbits LightOS 10 0.00 B / 5.37 GB 4 KiB + 0 B 2.0
/dev/nvme2n1 9ac7a0891e7cad9b Lightbits LightOS 8 0.00 B / 10.74 GB 4 KiB + 0 B 2.0
/dev/nvme2n2 9ac7a0891e7cad9b Lightbits LightOS 9 10.74 GB / 10.74 GB 4 KiB + 0 B 2.0
/dev/nvme2n3 9ac7a0891e7cad9b Lightbits LightOS 10 0.00 B / 5.37 GB 4 KiB + 0 B 2.0
### Redo with discovery-client run as a container (with updated files from Muli)
# recreate the discovery-client dir
[root@openstack-c8 ~]# rm -rf /etc/discovery-client/
[root@openstack-c8 ~]# mkdir -p /etc/discovery-client/
[root@openstack-c8 ~]# mkdir -p /etc/discovery-client/discovery.d
[root@openstack-c8 ~]# chmod 777 /etc/discovery-client/
[root@openstack-c8 ~]# docker run -it --rm --privileged --name=discovery_client --net=host -v /dev:/dev/ -v /etc/discovery-client/discovery.d:/etc/discovery-client/discovery.d docker.lightbitslabs.com/lightos-csi/lb-nvme-discovery-client:1.2.0
# drop into the kolla deploy image
root@kolla-deploy:/kolla# grep -n discovery /usr/local/share/kolla-ansible/ansible/roles/cinder/defaults/main.yml
151: - "/etc/discovery-client/:/etc/discovery-client/"
root@kolla-deploy:/kolla# grep -n discovery /usr/local/share/kolla-ansible/ansible/roles/nova-cell/defaults/main.yml
313: - "/etc/discovery-client:/etc/discovery-client"
# reconfigure once done
[root@openstack-c8 scratch]# docker exec -it kolla-deploy kolla-ansible -i /etc/kolla/all-in-one reconfigure
# ok lets now install the packages in the containers
docker cp cinder_lightbits-0~0~feb46cdb13a4-1.noarch.rpm cinder_volume:/tmp
docker cp os_brick_lightbits-0~0~a7762cb1f97c-1.noarch.rpm cinder_volume:/tmp
docker cp ../client_rpms/nvme-cli-1.9~2.0.6-1.x86_64.rpm cinder_volume:/tmp
docker exec -u root -it cinder_volume bash
(cinder-volume)[root@openstack-c8 /]# yum -y install python3
(cinder-volume)[root@openstack-c8 /]# yum -y localinstall /tmp/*rpm
exit
# set cinder.conf back to lightos
[root@openstack-c8 scratch]# grep -n lightos /etc/kolla/cinder-volume/cinder.conf
19:enabled_backends = lightos
66:[lightos]
67:volume_driver = cinder.volume.drivers.lightos.LightOSVolumeDriver
68:lightos_api_address = 10.20.60.101
69:lightos_api_port = 80
70:lightos_default_compression = False
71:lightos_default_num_replicas = 3
docker restart cinder_volume
# lets setup the nova compute container
docker cp nova_lightbits-0~0~844b9c5c7a62-1.noarch.rpm nova_compute:/tmp
docker cp os_brick_lightbits-0~0~a7762cb1f97c-1.noarch.rpm nova_compute:/tmp
docker cp ../client_rpms/nvme-cli-1.9~2.0.6-1.x86_64.rpm nova_compute:/tmp
docker exec -u root -it nova_compute bash
(nova-compute)[root@openstack-c8 /]# yum -y install python3
(nova-compute)[root@openstack-c8 /]# yum -y localinstall /tmp/*rpm
# /etc/nvme/hostnqn existed before i installed the packages!
(nova-compute)[root@openstack-c8 /]# nvme gen-hostnqn > /etc/nvme/hostnqn
# make sure the openstack envirnoment has had the init-runonce script ran against it
# Final take - we have created lightos containers for cinder-volume and nova-compute
# in the globals file
# for train on centos 8
openstack_tag: "train"
horizon_tag: "stein-definetech"
nova_compute_tag: "stein-lightos"
cinder_volume_tag: "stein-lightos"
cinder_tag: "stein"
##### deploying the server yum
yum install python3
pip3 install ansible
pip3 install netaddr
## for centos8 starview
cd /tmp && curl -O https://www.define-technology.com/assets/lightbits_files/cinder_lightbits-0~0~feb46cdb13a4-1.noarch.rpm && curl -O https://www.define-technology.com/assets/lightbits_files/nvme-cli-1.9~2.0.6-1.x86_64.rpm && curl -O https://www.define-technology.com/assets/lightbits_files/os_brick_lightbits-0~0~a7762cb1f97c-1.noarch.rpm
curl -O https://www.define-technology.com/assets/lightbits_files/discovery-client-2.0.6-1~5729656ed3c5.x86_64.rpm
dnf -y localinstall ./nvme-cli-1.9~2.0.6-1.x86_64.rpm
rpm -ivh /tmp/os_brick_lightbits-0~0~a7762cb1f97c-1.noarch.rpm --nodeps
dnf -y localinstall /tmp/cinder_lightbits-0~0~feb46cdb13a4-1.noarch.rpm
dnf -y localinstall /tmp/discovery-client-2.0.6-1~5729656ed3c5.x86_64.rpm
vi /usr/local/bin/kolla_start
# add
fi
echo "Starting Lightbits discovery client"
/usr/bin/discovery-client serve &
echo "Running command: '${CMD}${ARGS:+ $ARGS}'"
exec ${CMD} ${ARGS}
##
mkdir -p /etc/discovery-client/discovery.d/
chmod 777 /etc/discovery-client
chmod 777 /etc/discovery-client/discovery.d/
touch /var/log/discovery-client.log
chmod 777 /var/log/discovery-client.log
# setup in openstack / after deploy
root@kolla-deploy:/kolla# openstack volume type create --property volume_backend_name=lightos lightos
+-------------+--------------------------------------+
| Field | Value |
+-------------+--------------------------------------+
| description | None |
| id | d3b0f515-b0a5-4f64-8d68-e909ba1c4e5b |
| is_public | True |
| name | lightos |
| properties | volume_backend_name='lightos' |
+-------------+--------------------------------------+
root@kolla-deploy:/kolla# cat /etc/kolla/config/cinder/cinder-volume.conf
[DEFAULT]
enabled_backends=rbd-1,lightos
[rbd-1]
rbd_ceph_conf=/etc/ceph/ceph.conf
rbd_user=cinder
backend_host=rbd:volumes
rbd_pool=volumes
volume_backend_name=rbd-1
volume_driver=cinder.volume.drivers.rbd.RBDDriver
rbd_secret_uuid = 69e591d1-4901-41f9-8d49-73a8d4ff7d76
[lightos]
volume_driver = cinder.volume.drivers.lightos.LightOSVolumeDriver
lightos_api_address = 192.168.1.34
lightos_api_port = 80
lightos_default_compression = False
lightos_default_num_replicas = 1
volume_backend_name = lightos
### check out manually
8 nvme connect -t tcp -a 192.168.12.34 -s 4420 -n
9 nvme connect -t tcp -a 192.168.12.34 -s 4420 -n nqn.2016-01.com.lightbitslabs:uuid:5d7e3672-dd22-455d-a23e-1a5628f92bf8 -q ALLOW_NONE
()[root@stu-prr-hci1-n1-mlnx ~]# nvme disconnect-all
()[root@stu-prr-hci1-n1-mlnx ~]# ls /dev/nvm*
/dev/nvme-fabrics
GET NQN from
lbcli get cluster
nqn.2016-01.com.lightbitslabs:uuid:77774c12-60b0-480c-9983-cf524a57f3b4
[david@Davids-MacBook-Pro-4 notes]$