Ceph: Commands and Cheatsheet

Working with Ceph pools

Check the pools

[root@deploy-ext kolla]# ceph osd pool ls
device_health_metrics
images
volumes
backups
manila_data
manila_metadata
.rgw.root
default.rgw.log
default.rgw.control
default.rgw.meta
default.rgw.buckets.index
default.rgw.buckets.data
default.rgw.buckets.non-ec

Create a pool

ceph osd pool create dptest 128 128

[root@deploy-ext kolla]# rbd create --size 20480 --pool dptest vol01
[root@deploy-ext kolla]# rbd info dptest/vol01
rbd image 'vol01':
	size 20 GiB in 5120 objects
	order 22 (4 MiB objects)
	snapshot_count: 0
	id: 180b9ee11c2183
	block_name_prefix: rbd_data.180b9ee11c2183
	format: 2
	features: layering, exclusive-lock, object-map, fast-diff, deep-flatten
	op_features:
	flags:
	create_timestamp: Thu Sep  2 07:28:16 2021
	access_timestamp: Thu Sep  2 07:28:16 2021
	modify_timestamp: Thu Sep  2 07:28:16 2021

Resize a pool

[root@deploy-ext kolla]# rbd resize dptest/vol01 --size 51200
Resizing image: 100% complete...done.
[root@deploy-ext kolla]# rbd info dptest/vol01
rbd image 'vol01':
	size 50 GiB in 12800 objects
	order 22 (4 MiB objects)
	snapshot_count: 0
	id: 180b9ee11c2183
	block_name_prefix: rbd_data.180b9ee11c2183
	format: 2
	features: layering, exclusive-lock, object-map, fast-diff, deep-flatten
	op_features:
	flags:
	create_timestamp: Thu Sep  2 07:28:16 2021
	access_timestamp: Thu Sep  2 07:28:16 2021
	modify_timestamp: Thu Sep  2 07:28:16 2021

Working with Volumes

Map a device / mount a volume

[root@deploy-ext kolla]# rbd map dptest/vol01
/dev/rbd0

[root@deploy-ext kolla]# mkfs.xfs /dev/rbd/dptest/vol01
[root@deploy-ext kolla]# mount /dev/rbd/dptest/vol01 ./ceph-vol/
[root@deploy-ext kolla]# df -h | grep ceph-vol
/dev/rbd0             50G  390M   50G   1% /root/kolla/ceph-vol

Ceph maintenance

Stop rebalancing (useful for rebooting systems etc)

ceph osd set noout; ceph osd set norebalance

# perform reboot / maintenance ;

Ceph locks

# here we had an instance loose connectivity to ceph
# in openstack set state to active (node reset-state --active UUID)
# openstack server stop UUID
# then remove the lock as below
# then we could snapshot
ceph osd unset norebalance; ceph osd unset noout

# checking volume info (openstack env timed out, went into error
root@str-237:~# rbd info  volumes/volume-417feeef-d79d-4a31-af13-f1bee971284b
rbd image 'volume-417feeef-d79d-4a31-af13-f1bee971284b':
	size 3.9 TiB in 512000 objects
	order 23 (8 MiB objects)
	snapshot_count: 0
	id: 5ba8f5a87a6a8
	block_name_prefix: rbd_data.5ba8f5a87a6a8
	format: 2
	features: layering, exclusive-lock, object-map, fast-diff, deep-flatten
	op_features:
	flags:
	create_timestamp: Tue Feb 14 19:37:14 2023
	access_timestamp: Thu Feb 23 19:18:17 2023
	modify_timestamp: Thu Feb 23 19:18:21 2023
	parent: volumes/volume-7aedaaa5-f547-401a-b4cb-1d2271eb1c7d@snapshot-509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824
	overlap: 3.9 TiB

root@str-237:~# rbd lock ls volumes/volume-417feeef-d79d-4a31-af13-f1bee971284b
There is 1 exclusive lock on this image.
Locker          ID                    Address
client.6007059  auto 139760250032464  10.16.31.12:0/3416256037


root@str-237:~# rbd lock rm volumes/volume-417feeef-d79d-4a31-af13-f1bee971284b "auto 139760250032464" client.6007059

root@str-237:~# rbd lock ls volumes/volume-417feeef-d79d-4a31-af13-f1bee971284b

Ceph drain a host

cephadm shell
ceph orch host ls
ceph orch host drain node003
ceph orch osd rm status
ceph orch ps node003


# outputs 

[ceph: root@node003 /]# ceph orch host ls
HOST     ADDR         LABELS  STATUS
deploy   10.10.15.1   _admin
node001  10.10.15.14  _admin
node002  10.10.15.15  _admin
node003  10.10.15.16  _admin

[ceph: root@node003 /]# ceph orch host drain node003
Scheduled to remove the following daemons from host 'node003'
type                 id
-------------------- ---------------
osd                  12
osd                  13
osd                  14
crash                node003
mon                  node003
node-exporter        node003
osd                  16

[ceph: root@node003 /]# ceph orch osd rm status
OSD_ID  HOST     STATE     PG_COUNT  REPLACE  FORCE  DRAIN_STARTED_AT
12      node003  draining  71        False    False  2023-10-09 15:05:52.022780
13      node003  draining  63        False    False  2023-10-09 15:05:52.832505
14      node003  draining  63        False    False  2023-10-09 15:05:53.833922
16      node003  draining  61        False    False  2023-10-09 15:05:51.014545

[ceph: root@node003 /]# ceph orch  ps node003
NAME         HOST     PORTS  STATUS        REFRESHED  AGE  MEM USE  MEM LIM  VERSION  IMAGE ID      CONTAINER ID
mon.node003  node003         running (5M)    82s ago  22M    1570M    2048M  16.2.6   02a72919e474  e96bc70dbff9
osd.12       node003         running (5M)    82s ago  22M    20.0G    4096M  16.2.6   02a72919e474  930a4def245a
osd.13       node003         running (5M)    82s ago  22M    17.0G    4096M  16.2.6   02a72919e474  87c6bd55a2ac
osd.14       node003         running (5M)    82s ago  22M    30.7G    4096M  16.2.6   02a72919e474  5c6c858a2bd3
osd.16       node003         running (5M)    82s ago  22M    33.1G    4096M  16.2.6   02a72919e474  76708b8a5a9e

[ceph: root@node001 /]# ceph orch ps node003
NAME         HOST     PORTS  STATUS        REFRESHED  AGE  MEM USE  MEM LIM  VERSION  IMAGE ID      CONTAINER ID
mon.node003  node003         running (5M)     5m ago  22M    1642M    2048M  16.2.6   02a72919e474  e96bc70dbff9

[ceph: root@node001 /]# ceph orch rm node003
Failed to remove service. <node003> was not found. Running service names can be found with "ceph orch ls"
? someone already removed?

Re-add a drained node

if you have drainied a node and want to re-add it

re-add it to ceph

ceph orch host add node003 10.10.15.16

check to see if the disks are available for ceph

#  ceph orch device ls --wide
Hostname  Path          Type  Transport  RPM      Vendor  Model             Serial               Size   Health   Ident  Fault  Available  Reject Reasons
deploy    /dev/sdc      ssd   Unknown    Unknown  ATA     SAMSUNG MZ7LH960  S45NNC0NB45447        960G  Unknown  N/A    N/A    No         Insufficient space (<10 extents) on vgs, LVM detected, locked
deploy    /dev/sdd      ssd   Unknown    Unknown  ATA     SAMSUNG MZ7LH960  S45NNC0NB43506        960G  Unknown  N/A    N/A    No         Insufficient space (<10 extents) on vgs, LVM detected, locked
deploy    /dev/sde      ssd   Unknown    Unknown  ATA     SAMSUNG MZ7LH960  S45NNC0NB43517        960G  Unknown  N/A    N/A    No         Insufficient space (<10 extents) on vgs, LVM detected, locked
deploy    /dev/sdf      ssd   Unknown    Unknown  ATA     SAMSUNG MZ7LH960  S45NNC0NB45440        960G  Unknown  N/A    N/A    No         Insufficient space (<10 extents) on vgs, LVM detected, locked
node001   /dev/sdc      ssd   Unknown    Unknown  ATA     SAMSUNG MZ7LH960  S45NNC0NB43508        960G  Unknown  N/A    N/A    No         Insufficient space (<10 extents) on vgs, LVM detected, locked
node001   /dev/sdd      ssd   Unknown    Unknown  ATA     SAMSUNG MZ7LH960  S45NNC0NB48184        960G  Unknown  N/A    N/A    No         Insufficient space (<10 extents) on vgs, LVM detected, locked
node001   /dev/sde      ssd   Unknown    Unknown  ATA     SAMSUNG MZ7LH960  S45NNC0NB45448        960G  Unknown  N/A    N/A    No         Insufficient space (<10 extents) on vgs, LVM detected, locked
node001   /dev/sdf      ssd   Unknown    Unknown  ATA     SAMSUNG MZ7LH960  S45NNC0NB48182        960G  Unknown  N/A    N/A    No         Insufficient space (<10 extents) on vgs, LVM detected, locked
node002   /dev/nvme0n1  ssd   Unknown    Unknown  N/A     T408-AIC          TA19-09-02-C33-0030   536G  Unknown  N/A    N/A    Yes
node002   /dev/sdc      ssd   Unknown    Unknown  ATA     SAMSUNG MZ7LH960  S45NNC0NB43509        960G  Unknown  N/A    N/A    No         Insufficient space (<10 extents) on vgs, LVM detected, locked
node002   /dev/sdd      ssd   Unknown    Unknown  ATA     SAMSUNG MZ7LH960  S45NNC0NB45445        960G  Unknown  N/A    N/A    No         Insufficient space (<10 extents) on vgs, LVM detected, locked
node002   /dev/sde      ssd   Unknown    Unknown  ATA     SAMSUNG MZ7LH960  S45NNC0NB43514        960G  Unknown  N/A    N/A    No         Insufficient space (<10 extents) on vgs, LVM detected, locked
node002   /dev/sdf      ssd   Unknown    Unknown  ATA     SAMSUNG MZ7LH960  S45NNC0NB48189        960G  Unknown  N/A    N/A    No         Insufficient space (<10 extents) on vgs, LVM detected, locked
node003   /dev/nvme0n1  ssd   Unknown    Unknown  N/A     T408-AIC          TA19-09-02-C33-0016   536G  Unknown  N/A    N/A    Yes
node003   /dev/sdc      ssd   Unknown    Unknown  ATA     SAMSUNG MZ7LH960  S45NNC0NB43507        960G  Unknown  N/A    N/A    No         Insufficient space (<10 extents) on vgs, LVM detected, locked
node003   /dev/sdd      ssd   Unknown    Unknown  ATA     SAMSUNG MZ7LH960  S45NNC0NB43499        960G  Unknown  N/A    N/A    No         Insufficient space (<10 extents) on vgs, LVM detected, locked
node003   /dev/sde      ssd   Unknown    Unknown  ATA     SAMSUNG MZ7LH960  S45NNC0NB45446        960G  Unknown  N/A    N/A    No         Insufficient space (<10 extents) on vgs, LVM detected, locked
node003   /dev/sdf      ssd   Unknown    Unknown  ATA     SAMSUNG MZ7LH960  S45NNC0NB45461        960G  Unknown  N/A    N/A    No         Insufficient space (<10 extents) on vgs, LVM detected, locked

here they aren't because the old VG still exists

ssh into the node and jump into the cephadm shell and zap them! first run ceph-volume lvm list to see which drives they were then zap each one after making sure you are on the right node

[root@node003 ~]# cephadm shell
Inferring fsid 40d3a83a-4639-11ec-8388-3cecef04bf3c
Using recent ceph image quay.io/ceph/ceph@sha256:5755c3a5c197ef186b8186212e023565f15b799f1ed411207f2c3fcd4a80ab45
[ceph: root@node003 /]# ceph-volume lvm list


====== osd.12 ======

  [block]       /dev/ceph-cb7a9da8-e813-4783-87cb-5333d5b8d51c/osd-block-24b9b57d-f351-46dc-949f-8d28282e9724

      block device              /dev/ceph-cb7a9da8-e813-4783-87cb-5333d5b8d51c/osd-block-24b9b57d-f351-46dc-949f-8d28282e9724
      block uuid                kYkioH-r7BC-c3b5-dVLT-EKRp-3QNh-m2CDI5
      cephx lockbox secret
      cluster fsid              40d3a83a-4639-11ec-8388-3cecef04bf3c
      cluster name              ceph
      crush device class        None
      encrypted                 0
      osd fsid                  24b9b57d-f351-46dc-949f-8d28282e9724
      osd id                    12
      osdspec affinity          dashboard-admin-1636997944996
      type                      block
      vdo                       0
      devices                   /dev/sdc

====== osd.13 ======

  [block]       /dev/ceph-a30f427b-8300-4feb-b630-eb49e55ec34e/osd-block-d11072db-2d8b-4668-a391-f0f68816f7fd

      block device              /dev/ceph-a30f427b-8300-4feb-b630-eb49e55ec34e/osd-block-d11072db-2d8b-4668-a391-f0f68816f7fd
      block uuid                TScZzF-ej2H-cpWM-dPUH-fioK-XXHu-Zbe942
      cephx lockbox secret
      cluster fsid              40d3a83a-4639-11ec-8388-3cecef04bf3c
      cluster name              ceph
      crush device class        None
      encrypted                 0
      osd fsid                  d11072db-2d8b-4668-a391-f0f68816f7fd
      osd id                    13
      osdspec affinity          dashboard-admin-1636997944996
      type                      block
      vdo                       0
      devices                   /dev/sdd

====== osd.14 ======

  [block]       /dev/ceph-1780c956-58c3-4d3e-9975-e5712e5d50ce/osd-block-7c783e2b-3c7b-4b5c-b57c-c5b2e824959f

      block device              /dev/ceph-1780c956-58c3-4d3e-9975-e5712e5d50ce/osd-block-7c783e2b-3c7b-4b5c-b57c-c5b2e824959f
      block uuid                8PsGoM-AP1r-0FLC-BOUZ-T4Np-1BRe-KEuMwE
      cephx lockbox secret
      cluster fsid              40d3a83a-4639-11ec-8388-3cecef04bf3c
      cluster name              ceph
      crush device class        None
      encrypted                 0
      osd fsid                  7c783e2b-3c7b-4b5c-b57c-c5b2e824959f
      osd id                    14
      osdspec affinity          dashboard-admin-1636997944996
      type                      block
      vdo                       0
      devices                   /dev/sde

====== osd.16 ======

  [block]       /dev/ceph-bfd69196-153f-46e8-9398-9134fa307733/osd-block-46046b45-03d1-46e9-9cc7-a2a442bc6d02

      block device              /dev/ceph-bfd69196-153f-46e8-9398-9134fa307733/osd-block-46046b45-03d1-46e9-9cc7-a2a442bc6d02
      block uuid                ed9iDy-MVGC-o8VM-06v5-P8e2-CxnK-5OxFQx
      cephx lockbox secret
      cluster fsid              40d3a83a-4639-11ec-8388-3cecef04bf3c
      cluster name              ceph
      crush device class        None
      encrypted                 0
      osd fsid                  46046b45-03d1-46e9-9cc7-a2a442bc6d02
      osd id                    16
      osdspec affinity          dashboard-admin-1636997944996
      type                      block
      vdo                       0
      devices                   /dev/sdf

[ceph: root@node003 /]# ceph-volume lvm zap /dev/sdc --destroy
--> Zapping: /dev/sdc
--> Zapping lvm member /dev/sdc. lv_path is /dev/ceph-cb7a9da8-e813-4783-87cb-5333d5b8d51c/osd-block-24b9b57d-f351-46dc-949f-8d28282e9724
Running command: /usr/bin/dd if=/dev/zero of=/dev/ceph-cb7a9da8-e813-4783-87cb-5333d5b8d51c/osd-block-24b9b57d-f351-46dc-949f-8d28282e9724 bs=1M count=10 conv=fsync
 stderr: 10+0 records in
10+0 records out
 stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.072238 s, 145 MB/s
--> Only 1 LV left in VG, will proceed to destroy volume group ceph-cb7a9da8-e813-4783-87cb-5333d5b8d51c
Running command: /usr/sbin/vgremove -v -f ceph-cb7a9da8-e813-4783-87cb-5333d5b8d51c
 stderr: Removing ceph--cb7a9da8--e813--4783--87cb--5333d5b8d51c-osd--block--24b9b57d--f351--46dc--949f--8d28282e9724 (253:5)
 stderr: Archiving volume group "ceph-cb7a9da8-e813-4783-87cb-5333d5b8d51c" metadata (seqno 5).
 stderr: Releasing logical volume "osd-block-24b9b57d-f351-46dc-949f-8d28282e9724"
 stderr: Creating volume group backup "/etc/lvm/backup/ceph-cb7a9da8-e813-4783-87cb-5333d5b8d51c" (seqno 6).
 stdout: Logical volume "osd-block-24b9b57d-f351-46dc-949f-8d28282e9724" successfully removed
 stderr: Removing physical volume "/dev/sdc" from volume group "ceph-cb7a9da8-e813-4783-87cb-5333d5b8d51c"
 stdout: Volume group "ceph-cb7a9da8-e813-4783-87cb-5333d5b8d51c" successfully removed
Running command: /usr/bin/dd if=/dev/zero of=/dev/sdc bs=1M count=10 conv=fsync
 stderr: 10+0 records in
10+0 records out
 stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0708403 s, 148 MB/s
--> Zapping successful for: <Raw Device: /dev/sdc>
[ceph: root@node003 /]# ceph -s
  cluster:
    id:     40d3a83a-4639-11ec-8388-3cecef04bf3c
    health: HEALTH_WARN
            1/4 mons down, quorum node001,node002,node003
            1 pool(s) do not have an application enabled

  services:
    mon: 4 daemons, quorum node001,node002,node003 (age 10h), out of quorum: deploy
    mgr: node001.thgtmt(active, since 9h), standbys: node002.gsprmz
    osd: 12 osds: 12 up (since 17h), 12 in (since 5M)

  data:
    pools:   9 pools, 584 pgs
    objects: 401.05k objects, 1.8 TiB
    usage:   5.3 TiB used, 5.2 TiB / 10 TiB avail
    pgs:     584 active+clean

  io:
    client:   16 MiB/s rd, 570 KiB/s wr, 4.01k op/s rd, 86 op/s wr
[ceph: root@node003 /]# ceph-volume lvm zap /dev/sdd --destroy
<snip>
--> Zapping successful for: <Raw Device: /dev/sdd>
[ceph: root@node003 /]# ceph-volume lvm zap /dev/sde --destroy
--> Zapping: /dev/sde
<snip>
--> Zapping successful for: <Raw Device: /dev/sde>
[ceph: root@node003 /]# ceph-volume lvm zap /dev/sdf --destroy
--> Zapping: /dev/sdf

Assuming that the ceph orchestrator is still running then it should start them for you

[root@deploy ~]# ceph orch ps
NAME                   HOST     PORTS        STATUS         REFRESHED  AGE  MEM USE  MEM LIM  VERSION    IMAGE ID      CONTAINER ID
alertmanager.node001   node001  *:9093,9094  running (5M)      7m ago  22M    75.8M        -  0.20.0     0881eb8f169f  016d9674735d
crash.deploy           deploy                running (5M)      9m ago  22M    28.9M        -  16.2.6     02a72919e474  4bc3ee42dd35
crash.node001          node001               running (5M)      7m ago  22M    23.4M        -  16.2.6     02a72919e474  c1d2dde0dde4
crash.node002          node002               running (5M)      4s ago  22M    7105k        -  16.2.6     02a72919e474  0955608ff18d
crash.node003          node003               running (60m)     3m ago  60m    6413k        -  16.2.6     02a72919e474  f7f8bb1676da
grafana.node001        node001  *:3000       running (5M)      7m ago  22M     151M        -  6.7.4      557c83e11646  3ccda42d7d9f
mgr.node001.thgtmt     node001  *:8443,9283  running (5M)      7m ago  22M     643M        -  16.2.6     02a72919e474  d6a1964a7879
mgr.node002.gsprmz     node002  *:8443,9283  running (5M)      4s ago  22M     485M        -  16.2.6     02a72919e474  bdd0924ab48e
mon.deploy             deploy                error             9m ago  18M        -    2048M  <unknown>  <unknown>     <unknown>
mon.node001            node001               running (5M)      7m ago  22M    1635M    2048M  16.2.6     02a72919e474  abe841493d7a
mon.node002            node002               running (5M)      4s ago  22M    1666M    2048M  16.2.6     02a72919e474  cfbacb26d1a7
mon.node003            node003               running (12h)     3m ago  22M     977M    2048M  16.2.6     02a72919e474  edde532df630
node-exporter.deploy   deploy   *:9100       running (5M)      9m ago  22M    66.6M        -  0.18.1     e5a616e4b9cf  fb4d8a3007a7
node-exporter.node001  node001  *:9100       running (5M)      7m ago  22M    67.3M        -  0.18.1     e5a616e4b9cf  100daf55dc10
node-exporter.node002  node002  *:9100       running (5M)      4s ago  22M    67.0M        -  0.18.1     e5a616e4b9cf  bcc94fa5029b
node-exporter.node003  node003  *:9100       running (60m)     3m ago  60m    42.1M        -  0.18.1     e5a616e4b9cf  e28345452f6a
osd.0                  deploy                running (5M)      9m ago  22M    42.8G    4096M  16.2.6     02a72919e474  b3b83e70e39e
osd.1                  deploy                running (5M)      9m ago  22M    44.7G    4096M  16.2.6     02a72919e474  62875261b781
osd.10                 node002               running (5M)      4s ago  22M    16.2G    4096M  16.2.6     02a72919e474  b93e2c9f522a
osd.11                 node002               running (5M)      4s ago  22M    15.9G    4096M  16.2.6     02a72919e474  b0a6708c217e
osd.12                 node003               running (59m)     3m ago  59m    2942M    4096M  16.2.6     02a72919e474  d8a9e6b34e8e
osd.13                 node003               running (59m)     3m ago  59m    3081M    4096M  16.2.6     02a72919e474  3e5129206a99
osd.14                 node003               running (59m)     3m ago  59m    3033M    4096M  16.2.6     02a72919e474  5e7eaece5ac8
osd.15                 node003               running (59m)     3m ago  59m    3259M    4096M  16.2.6     02a72919e474  670189a8ba11
osd.2                  deploy                running (5M)      9m ago  22M    27.9G    4096M  16.2.6     02a72919e474  797fec7a5249
osd.3                  deploy                running (5M)      9m ago  13M    68.6G    4096M  16.2.6     02a72919e474  20bdf4cd7e21
osd.4                  node001               running (5M)      7m ago  22M    13.5G    4096M  16.2.6     02a72919e474  e920a032c446
osd.5                  node001               running (5M)      7m ago  22M    10.4G    4096M  16.2.6     02a72919e474  cfd25cd7297d
osd.6                  node001               running (5M)      7m ago  22M    15.4G    4096M  16.2.6     02a72919e474  6a09fd60fa9c
osd.7                  node001               running (5M)      7m ago  22M    25.4G    4096M  16.2.6     02a72919e474  4e429076d3f5
osd.8                  node002               running (5M)      4s ago  22M    14.7G    4096M  16.2.6     02a72919e474  a97212b40475
osd.9                  node002               running (5M)      4s ago  22M    9731M    4096M  16.2.6     02a72919e474  c655f1983800
prometheus.node001     node001  *:9095       running (59m)     7m ago  22M     291M        -  2.18.1     de242295e225  dfe09178132a

ceph node appears offline but has fuctioning components

this looks like this

[root@deploy ~]# ceph orch host ls
HOST     ADDR         LABELS  STATUS
deploy   10.10.15.1   _admin  Offline
node001  10.10.15.14  _admin
node002  10.10.15.15  _admin
node003  10.10.15.16  _admin

[root@deploy ~]# ceph osd tree
ID   CLASS  WEIGHT    TYPE NAME                       STATUS  REWEIGHT  PRI-AFF
 -1         13.97253  root default
-13         13.97253      room testroom
-12         13.97253          rack rack1
-11         13.97253              chassis AMD_twin_1
 -3          3.49316                  host deploy
  0    ssd   0.87329                      osd.0           up   1.00000  1.00000
  1    ssd   0.87329                      osd.1           up   1.00000  1.00000
  2    ssd   0.87329                      osd.2           up   1.00000  1.00000
  3    ssd   0.87329                      osd.3           up   1.00000  1.00000
snip

I looked in the journal for this node and noticed that there were ssh login fails from the active ceph mgmt node. Turns out the cpeh pubkey had been lost in this node and adding it back allowed it to be re-detected.

Oct 10 09:55:23 deploy sshd[3610000]: Failed password for root from 10.10.15.14 port 54210 ssh2
Oct 10 09:55:23 deploy sshd[3610000]: Connection closed by authenticating user root 10.10.15.14 port 54210 [preauth]

There was a scary moment when it briefly did something that interrupted the connection to the mons. . .

but this only affected this node and it immediately started applying the osd change from the previous section

Ceph MON failed and it won't come back

Related to the above ssh issue breaking the the orchestrator for many months. . . we were seeing this:

[root@deploy ~]# ceph -s
  cluster:
    id:     40d3a83a-4639-11ec-8388-3cecef04bf3c
    health: HEALTH_WARN
            1 failed cephadm daemon(s)
            1/4 mons down, quorum node001,node002,node003
            1 pool(s) do not have an application enabled

  services:
    mon: 4 daemons, quorum node001,node002,node003 (age 12h), out of quorum: deploy
    mgr: node001.thgtmt(active, since 11h), standbys: node002.gsprmz
    osd: 16 osds: 16 up (since 71m), 16 in (since 71m)

  data:
    pools:   9 pools, 584 pgs
    objects: 401.06k objects, 1.8 TiB
    usage:   5.3 TiB used, 8.7 TiB / 14 TiB avail
    pgs:     584 active+clean

  io:
    client:   16 MiB/s rd, 1.3 MiB/s wr, 4.01k op/s rd, 78 op/s wr

[root@deploy ~]# ceph health detail
HEALTH_WARN 1 failed cephadm daemon(s); 1/4 mons down, quorum node001,node002,node003; 1 pool(s) do not have an application enabled
[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s)
    daemon mon.deploy on deploy is in error state
[WRN] MON_DOWN: 1/4 mons down, quorum node001,node002,node003
    mon.deploy (rank 3) addr [v2:10.10.15.1:3300/0,v1:10.10.15.1:6789/0] is down (out of quorum)
[WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled
    application not enabled on pool 'backups'
    use 'ceph osd pool application enable <pool-name> <app-name>', where <app-name> is 'cephfs', 'rbd', 'rgw', or freeform for custom applications.


[root@deploy ~]# ceph orch ps
NAME                   HOST     PORTS        STATUS         REFRESHED  AGE  MEM USE  MEM LIM  VERSION    IMAGE ID      CONTAINER ID
alertmanager.node001   node001  *:9093,9094  running (5M)    106s ago  22M    75.9M        -  0.20.0     0881eb8f169f  016d9674735d
crash.deploy           deploy                running (5M)      4m ago  22M    28.9M        -  16.2.6     02a72919e474  4bc3ee42dd35
crash.node001          node001               running (5M)    106s ago  22M    23.4M        -  16.2.6     02a72919e474  c1d2dde0dde4
crash.node002          node002               running (5M)      4m ago  22M    7105k        -  16.2.6     02a72919e474  0955608ff18d
crash.node003          node003               running (85m)     7m ago  85m    6413k        -  16.2.6     02a72919e474  f7f8bb1676da
grafana.node001        node001  *:3000       running (5M)    106s ago  22M     151M        -  6.7.4      557c83e11646  3ccda42d7d9f
mgr.node001.thgtmt     node001  *:8443,9283  running (5M)    106s ago  22M     651M        -  16.2.6     02a72919e474  d6a1964a7879
mgr.node002.gsprmz     node002  *:8443,9283  running (5M)      4m ago  22M     485M        -  16.2.6     02a72919e474  bdd0924ab48e
mon.deploy             deploy                error             4m ago  18M        -    2048M  <unknown>  <unknown>     <unknown>
mon.node001            node001               running (5M)    106s ago  22M    1652M    2048M  16.2.6     02a72919e474  abe841493d7a
mon.node002            node002               running (5M)      4m ago  22M    1610M    2048M  16.2.6     02a72919e474  cfbacb26d1a7
mon.node003            node003               running (12h)     7m ago  22M    1025M    2048M  16.2.6     02a72919e474  edde532df630

it is not there at all, no container no nothing (I lost that output but trust me)

turns out the right thing to do it delete it!

[root@deploy ~]# ceph orch daemon rm mon.deploy --force
Removed mon.deploy from host 'deploy'

and then it redeploys it!

[root@deploy ~]# ceph orch ps --service_name mon
NAME         HOST     PORTS  STATUS         REFRESHED  AGE  MEM USE  MEM LIM  VERSION  IMAGE ID      CONTAINER ID
mon.deploy   deploy          running (5m)      5m ago   5m     614M    2048M  16.2.6   02a72919e474  2d8441a1e78a
mon.node001  node001         running (5M)      8m ago  22M    1652M    2048M  16.2.6   02a72919e474  abe841493d7a
mon.node002  node002         running (5M)     10m ago  22M    1610M    2048M  16.2.6   02a72919e474  cfbacb26d1a7
mon.node003  node003         running (12h)     2m ago  22M     968M    2048M  16.2.6   02a72919e474  edde532df630

Ceph Speed up Rebalance

sometimes you want ceph to get a shift on so you can get on with your life. The default rebalancing options are conservative to minimise the performance impact.

the following parameters affect rebuild speed

[root@deploy ~]# ceph config get osd osd_max_backfills
1
[root@deploy ~]# ceph config get osd osd_recovery_max_active
0
[root@deploy ~]# ceph config get osd osd_recovery_op_priority
3

to speed up all recovery use this:

[root@deploy ~]# ceph tell 'osd.*' injectargs --osd-max-backfills=3 --osd-recovery-max-active=9 --osd_recovery_op_priority=2
osd.0: {}
osd.0: osd_max_backfills = '3' osd_recovery_max_active = '9' osd_recovery_op_priority = '2' (not observed, change may require restart)
osd.1: {}
osd.1: osd_max_backfills = '3' osd_recovery_max_active = '9' osd_recovery_op_priority = '2' (not observed, change may require restart)
osd.2: {}
osd.2: osd_max_backfills = '3' osd_recovery_max_active = '9' osd_recovery_op_priority = '2' (not observed, change may require restart)
osd.3: {}
osd.3: osd_max_backfills = '3' osd_recovery_max_active = '9' osd_recovery_op_priority = '2' (not observed, change may require restart)
osd.4: {}
osd.4: osd_max_backfills = '3' osd_recovery_max_active = '9' osd_recovery_op_priority = '2' (not observed, change may require restart)
osd.5: {}
osd.5: osd_max_backfills = '3' osd_recovery_max_active = '9' osd_recovery_op_priority = '2' (not observed, change may require restart)
osd.6: {}
osd.6: osd_max_backfills = '3' osd_recovery_max_active = '9' osd_recovery_op_priority = '2' (not observed, change may require restart)
osd.7: {}
osd.7: osd_max_backfills = '3' osd_recovery_max_active = '9' osd_recovery_op_priority = '2' (not observed, change may require restart)
osd.8: {}
osd.8: osd_max_backfills = '3' osd_recovery_max_active = '9' osd_recovery_op_priority = '2' (not observed, change may require restart)
osd.9: {}
osd.9: osd_max_backfills = '3' osd_recovery_max_active = '9' osd_recovery_op_priority = '2' (not observed, change may require restart)
osd.10: {}
osd.10: osd_max_backfills = '3' osd_recovery_max_active = '9' osd_recovery_op_priority = '2' (not observed, change may require restart)
osd.11: {}
osd.11: osd_max_backfills = '3' osd_recovery_max_active = '9' osd_recovery_op_priority = '2' (not observed, change may require restart)
osd.12: {}
osd.12: osd_max_backfills = '3' osd_recovery_max_active = '9' osd_recovery_op_priority = '2' (not observed, change may require restart)
osd.13: {}
osd.13: osd_max_backfills = '3' osd_recovery_max_active = '9' osd_recovery_op_priority = '2' (not observed, change may require restart)
osd.14: {}
osd.14: osd_max_backfills = '3' osd_recovery_max_active = '9' osd_recovery_op_priority = '2' (not observed, change may require restart)
osd.15: {}
osd.15: osd_max_backfills = '3' osd_recovery_max_active = '9' osd_recovery_op_priority = '2' (not observed, change may require restart)

to go back to defaults use this

ceph tell 'osd.*' injectargs --osd-max-backfills=1 --osd-recovery-max-active=0 --osd_recovery_op_priority=3

Flatten all voluems

for vol in $(rbd  list volumes); do for kid in $(rbd chidren volumes/$vol); do echo rbd flatten $kid; done; done

example

root@str-238:~# for vol in $(rbd  list volumes); do for kid in $(rbd children volumes/$vol); do echo rbd flatten $kid; done; done
rbd flatten volumes/volume-4054433b-03cf-4cb1-8948-c6ffe6e2955c
rbd flatten volumes/volume-5408fbf0-46f6-41d9-937a-239be5055c4c
rbd flatten volumes/volume-86056c2b-a904-4228-8dd1-c4cb01648f0f
rbd flatten volumes/volume-c4d0ce91-8f23-40cf-9772-cca3c77ae672
rbd flatten volumes/volume-93f22739-5ac8-4a88-8c49-de4c2234e7e7
rbd flatten volumes/volume-ae27171c-f11e-44e7-8928-e72efe9b45c6
rbd flatten volumes/volume-0da62386-8eb4-4963-95ff-d9a2b561061a
rbd flatten volumes/volume-2a7ffe62-54e7-45e4-a61e-b066d29acc50
rbd flatten volumes/volume-321e3628-170f-42c9-9f37-66413250c441
rbd flatten volumes/volume-3f79a1dd-6cf8-4dea-b98f-b7597d3a5a6d
rbd flatten volumes/volume-467bb817-3c6c-4f44-b767-35109127cb7c
rbd flatten volumes/volume-6ef47ed4-a802-4e65-bb07-8c36b4a1fe5e
rbd flatten volumes/volume-88f473eb-d2fa-4e37-82f2-54fd85c25d5e
rbd flatten volumes/volume-d4af50d5-2182-47b3-941c-997e74c9341c
rbd flatten volumes/volume-dc48db05-56e2-448e-99cb-e07e65977b53
rbd flatten volumes/volume-f035099d-a4c1-489d-9253-edebe3413fec
rbd flatten volumes/volume-998bcdd4-3588-4464-ac8b-88c5387402e0
rbd flatten volumes/volume-19186f06-f923-4416-89e3-f52f6c54570b
rbd flatten volumes/volume-417feeef-d79d-4a31-af13-f1bee971284b
rbd flatten volumes/volume-7894f6c4-c815-4132-896e-c4d5f41efdb9
rbd flatten volumes/volume-8889e3b6-5d6c-4bf6-b17c-434ffb95154f
rbd flatten volumes/volume-b96a70f0-7326-4a7d-9c36-928bf2abd1ce
rbd flatten volumes/volume-1fc5a6f4-88ee-43bc-afdc-acb762bbec9d
rbd flatten volumes/volume-61192337-5f2f-400e-b04f-337ce502b326
rbd flatten volumes/volume-6efd9e53-4cdd-4b99-84b2-114bd7e276c3
rbd flatten volumes/volume-9ce7b127-acae-4e94-9b21-079633f008bd
rbd flatten volumes/volume-b1a50f3e-6c7c-4fa8-a1ae-1a488fc4d3b3
rbd flatten volumes/volume-d2f33e0e-d7f5-4eae-8efb-27a71f335d12
rbd flatten volumes/volume-f465b10e-5d01-4b5d-a786-51cc217220ee
rbd flatten volumes/volume-64187c05-3299-4a2e-ab1b-e527f483cb6c
rbd flatten volumes/volume-1dcf464a-7efd-4ad4-a774-367188f55448
rbd flatten volumes/volume-1cbdf9bb-2f89-47a7-86d2-16d0739a8451
rbd flatten volumes/volume-a1468374-b1c3-4c26-a2da-c2cfeee1c27d
rbd flatten volumes/volume-16b5ad27-ecf5-46e7-81ef-b79af5618dd0.deleted
rbd flatten volumes/volume-9fb3978c-050d-4bfc-b9ce-9cd213994d96

and if you remove the echo

root@str-238:~# for vol in $(rbd  list volumes); do for kid in $(rbd children volumes/$vol); do rbd flatten $kid; done; done
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-5408fbf0-46f6-41d9-937a-239be5055c4c
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-86056c2b-a904-4228-8dd1-c4cb01648f0f
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-c4d0ce91-8f23-40cf-9772-cca3c77ae672
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-93f22739-5ac8-4a88-8c49-de4c2234e7e7
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-ae27171c-f11e-44e7-8928-e72efe9b45c6
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-0da62386-8eb4-4963-95ff-d9a2b561061a
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-2a7ffe62-54e7-45e4-a61e-b066d29acc50
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-321e3628-170f-42c9-9f37-66413250c441
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-3f79a1dd-6cf8-4dea-b98f-b7597d3a5a6d
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-467bb817-3c6c-4f44-b767-35109127cb7c
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-6ef47ed4-a802-4e65-bb07-8c36b4a1fe5e
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-88f473eb-d2fa-4e37-82f2-54fd85c25d5e
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-d4af50d5-2182-47b3-941c-997e74c9341c
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-dc48db05-56e2-448e-99cb-e07e65977b53
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-f035099d-a4c1-489d-9253-edebe3413fec
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-998bcdd4-3588-4464-ac8b-88c5387402e0
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-19186f06-f923-4416-89e3-f52f6c54570b
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-417feeef-d79d-4a31-af13-f1bee971284b
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-7894f6c4-c815-4132-896e-c4d5f41efdb9
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-8889e3b6-5d6c-4bf6-b17c-434ffb95154f
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-b96a70f0-7326-4a7d-9c36-928bf2abd1ce
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-1fc5a6f4-88ee-43bc-afdc-acb762bbec9d
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-61192337-5f2f-400e-b04f-337ce502b326
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-6efd9e53-4cdd-4b99-84b2-114bd7e276c3
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-9ce7b127-acae-4e94-9b21-079633f008bd
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-b1a50f3e-6c7c-4fa8-a1ae-1a488fc4d3b3
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-d2f33e0e-d7f5-4eae-8efb-27a71f335d12
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-f465b10e-5d01-4b5d-a786-51cc217220ee
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-64187c05-3299-4a2e-ab1b-e527f483cb6c
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-1dcf464a-7efd-4ad4-a774-367188f55448
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-1cbdf9bb-2f89-47a7-86d2-16d0739a8451
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-a1468374-b1c3-4c26-a2da-c2cfeee1c27d
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-16b5ad27-ecf5-46e7-81ef-b79af5618dd0.deleted
Image flatten: 100% complete...done.
root@str-238:~# rbd flatten volumes/volume-9fb3978c-050d-4bfc-b9ce-9cd213994d96
Image flatten: 100% complete...done.

cleanly remove ALL volumes snapshots and images from Openstack and Ceph

this needs a huge cleanup but in the interests of getting this somewhere safe

(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume list --all --long -c ID -c Status -c Type -c Name -c Size | grep -v -i storpool
+--------------------------------------+--------------------------------------------+----------------+-------+-----------------+
| ID                                   | Name                                       | Status         |  Size | Type            |
+--------------------------------------+--------------------------------------------+----------------+-------+-----------------+
| 85ecb198-c2de-4ecf-90b6-7f658085d5f5 |                                            | available      |    50 | ceph-replicated |
| 27b56326-21fe-4514-9b7f-52e56e00002a |                                            | available      |     1 | ceph-replicated |
| 9b1d7f51-7486-4b01-b269-2ed143fc7428 | image-c50eb461-29c9-4f25-9ec6-a4eb657b5c2b | available      |    50 | ceph-replicated |
| 639983da-294f-45ac-883f-b7ec3d0f4ff3 |                                            | available      |    50 | ceph-replicated |
| 3d53a2d8-2fc3-4475-b5aa-a8ae4ef8adb8 | image-1b4c2247-af20-4b85-a72f-5f90883d991b | available      |    50 | ceph-replicated |
| 29c54d8a-5e6d-4523-98a7-5f30c793068f | image-6be53f5f-984f-4033-bc91-f05b68feff54 | available      |    50 | ceph-replicated |
| 9ead516b-aaf4-492e-9457-b4be23f10d7f |                                            | available      |    80 | ceph-replicated |
| 9dd70ba7-1387-4d9d-bde0-ae2c843e535f |                                            | available      |   160 | ceph-replicated |
| 0e993fe7-c7a5-467c-9d72-4532cc316741 | image-3c87407d-8aa1-48e4-9687-355ff90021d4 | available      |   160 | ceph-replicated |
| d698f7bf-fff5-450d-8ed6-2f62e4ca80cf |                                            | available      |    40 | ceph-replicated |
| 68fcd1c6-60d3-4e7d-a7e5-d13a8d7bd424 |                                            | available      |    40 | ceph-replicated |
| e5d32199-8115-43a8-a8df-fa2f98442f96 |                                            | available      |    40 | ceph-replicated |
| 0677c8e8-f4e7-42f4-8a57-7439b70e5653 |                                            | available      |    40 | ceph-replicated |
| 6efa66a3-89f9-4f44-9526-f6425c962a75 |                                            | available      |  6000 | __DEFAULT__     |
| 29dacac9-93dc-408a-bf3f-327e8590acad |                                            | available      |   500 | ceph-replicated |
| 4f9393d2-ee56-49b3-ac8c-3ba192ec8921 | test-q35-ubuntu                            | available      |    40 | ceph-replicated |
| 64187c05-3299-4a2e-ab1b-e527f483cb6c |                                            | available      |   250 | __DEFAULT__     |
| 62547f47-0a75-400b-848a-a13ba8523613 |                                            | available      |  6000 | __DEFAULT__     |
| e2c8bfb6-346b-485e-a621-15cb0b9f82c9 |                                            | available      |   100 | ceph-replicated |
| 3a887d0d-4b7d-4ee7-8b83-dbc21d199822 |                                            | available      |   200 | ceph-replicated |
| 77b5618b-91c3-428d-ba70-2e399c46878d |                                            | available      |    40 | ceph-replicated |
| 0929147c-7e6a-464d-aa43-54212b955a58 | sp-test-clone                              | available      |    40 | ceph-replicated |
| 3b794a23-c30f-44a8-a1c1-763ab4ee1635 | datadrive643873c9987ba_copy                | error_deleting |   250 | ceph-replicated |
| 34daba88-c273-4b3b-8954-6ad6832bdd21 | image-7dc4d363-721d-42be-a457-c60c07a7119c | error_deleting |   100 | ceph-replicated |
| dc275cb9-6709-4dc1-8e9d-5943a3b4964e |                                            | available      |   100 | ceph-replicated |
| 683b6a60-45df-4366-a7b7-860b2338aa33 |                                            | available      |    40 | ceph-replicated |
| c4d0ce91-8f23-40cf-9772-cca3c77ae672 |                                            | available      |  6000 | __DEFAULT__     |
| a1468374-b1c3-4c26-a2da-c2cfeee1c27d |                                            | available      |   100 | __DEFAULT__     |
| 467bb817-3c6c-4f44-b767-35109127cb7c |                                            | available      |   100 | __DEFAULT__     |
| 4054433b-03cf-4cb1-8948-c6ffe6e2955c | testing-makeairun-mongodb-root             | available      |  6000 | __DEFAULT__     |
| 1dcf464a-7efd-4ad4-a774-367188f55448 |                                            | available      |   100 | __DEFAULT__     |
| 7cc5f04a-0ecf-4ddc-a2c4-5c0d3a0b5913 | testing-volume                             | available      |    40 | __DEFAULT__     |
| 79ccdd09-102d-42e6-bc6c-01b6a2d19892 |                                            | available      |   500 | __DEFAULT__     |
| 9fb3978c-050d-4bfc-b9ce-9cd213994d96 |                                            | available      |   100 | __DEFAULT__     |
| 86056c2b-a904-4228-8dd1-c4cb01648f0f |                                            | available      |  6000 | __DEFAULT__     |
| 723b7867-c8fc-48bd-a48b-e3956bddc173 |                                            | available      |  6000 | __DEFAULT__     |
| b96a70f0-7326-4a7d-9c36-928bf2abd1ce |                                            | available      |  4000 | __DEFAULT__     |
| 8eb7069c-5c07-43c6-a91f-f1d79a071a91 |                                            | available      |   200 | __DEFAULT__     |
| e9638883-b712-4fcc-beec-b0fc7aa0ac75 |                                            | available      |  4000 | __DEFAULT__     |
| 68348ebf-fbb8-4d5a-8965-bc84f70a059a |                                            | available      |  4000 | __DEFAULT__     |
| bb55d0ba-7dec-458f-a7fc-b6d4b46a9936 |                                            | available      |  6000 | __DEFAULT__     |
| 8a8bf436-da79-49ce-979c-4f312e9015da |                                            | available      |  6000 | __DEFAULT__     |
| 5408fbf0-46f6-41d9-937a-239be5055c4c |                                            | available      |  6000 | __DEFAULT__     |
| 19186f06-f923-4416-89e3-f52f6c54570b |                                            | available      |  4000 | __DEFAULT__     |
| 21622152-bc69-4d80-83e4-216d22bfb193 |                                            | available      |   500 | __DEFAULT__     |
| d4dda353-7cb0-4a60-9c76-edfa119d86bc |                                            | available      |  4000 | __DEFAULT__     |
| 101cf309-5eab-45e4-9128-3572e107d237 |                                            | available      |  6000 | __DEFAULT__     |
| f8f47d4a-1baa-4a3e-a5e0-98d911f700b0 |                                            | available      |   200 | __DEFAULT__     |
| 93f22739-5ac8-4a88-8c49-de4c2234e7e7 |                                            | available      |   250 | __DEFAULT__     |
| 7894f6c4-c815-4132-896e-c4d5f41efdb9 |                                            | available      |  4000 | __DEFAULT__     |
| 4f2d0656-d7ff-4e43-8b93-24e74ee04b72 |                                            | available      |  4000 | __DEFAULT__     |
| 763547b6-db23-496f-b852-46f1126411d2 | depExpansion                               | available      |  4000 | __DEFAULT__     |
| d2f33e0e-d7f5-4eae-8efb-27a71f335d12 |                                            | available      |   500 | __DEFAULT__     |
| 8889e3b6-5d6c-4bf6-b17c-434ffb95154f |                                            | available      |  4000 | __DEFAULT__     |
| 998bcdd4-3588-4464-ac8b-88c5387402e0 |                                            | available      |   500 | __DEFAULT__     |
| 1fc5a6f4-88ee-43bc-afdc-acb762bbec9d |                                            | available      |   500 | __DEFAULT__     |
| 4f80c812-cc7e-4b04-8148-ffd01ff69f14 | pvc-6f36e0db-2097-421d-b9a0-5119880e0125   | available      |     8 | __DEFAULT__     |
| 39094eb4-bfc8-4c37-89c0-4c6326c2d7b4 | pvc-869037ab-b146-444c-8ff6-2b6e3d10b367   | available      |     8 | __DEFAULT__     |
| 417feeef-d79d-4a31-af13-f1bee971284b |                                            | available      |  4000 | __DEFAULT__     |
| 7aedaaa5-f547-401a-b4cb-1d2271eb1c7d |                                            | available      |  4000 | __DEFAULT__     |
| 46c6479d-a6c5-4efb-a7ab-fafa2cbb382e |                                            | available      |    40 | __DEFAULT__     |
| 57590c5a-e645-40a0-9237-3d69aca8c020 |                                            | available      |  4000 | __DEFAULT__     |
| f818b7e0-9d56-4df1-8416-6aa50cbeb606 |                                            | available      |   500 | __DEFAULT__     |
| 61192337-5f2f-400e-b04f-337ce502b326 |                                            | available      |   500 | __DEFAULT__     |
| 6ff869ee-8bd0-49fc-9a5e-fb0f6db9295e |                                            | available      |   500 | __DEFAULT__     |
| f465b10e-5d01-4b5d-a786-51cc217220ee |                                            | available      |   500 | __DEFAULT__     |
| 74e650ba-05ec-41fa-9a47-fe04ad1ab20b |                                            | available      |    40 | __DEFAULT__     |
| 9b81e22e-81d6-4e61-be54-fb990083c540 |                                            | available      |   250 | __DEFAULT__     |
| 095849e0-b0b6-4ba6-bb00-0f0030a91baf |                                            | available      |    40 | __DEFAULT__     |
| 083836a3-5437-4951-b066-66ff3fff9a68 |                                            | available      |   500 | __DEFAULT__     |
| 0a050de7-73b9-4239-ad5f-40c0cbb250f9 |                                            | available      |   500 | __DEFAULT__     |
| ea0b101d-86d3-4ab7-a01f-d9fa566e7174 |                                            | available      |   500 | __DEFAULT__     |
| b1a50f3e-6c7c-4fa8-a1ae-1a488fc4d3b3 |                                            | available      |   500 | __DEFAULT__     |
| a2727c56-fe70-4f02-8dcb-a23373bcbfe0 |                                            | available      |   500 | __DEFAULT__     |
| ef74477b-a345-4cc0-ae94-4b29b0375015 |                                            | available      |   160 | __DEFAULT__     |
| e50b1239-62da-4535-aa21-4e551c1eb2be |                                            | available      |   100 | __DEFAULT__     |
| 70d3019c-8085-4bcf-b120-b20652fcc835 | 2tb workspace                              | available      |  2000 | __DEFAULT__     |
| 94e97068-13e3-49a2-8bfe-4c4ba1d898fd |                                            | available      |    40 | __DEFAULT__     |
| f3417f0a-1e34-4de6-83e5-f4c86d526e48 |                                            | available      |    40 | __DEFAULT__     |
| 696d70cf-bf67-4a75-a3ff-75d1b445d1c9 |                                            | available      |    40 | __DEFAULT__     |
| 1c68a40b-beae-45cb-9e0b-519301fb34af |                                            | available      |   250 | __DEFAULT__     |
| 0ab6766c-6cde-4a03-ad46-03009066783e |                                            | available      |    40 | __DEFAULT__     |
| 0d9aaf09-12b2-4b3c-8e37-7bb389731bf0 |                                            | available      |    40 | __DEFAULT__     |
| 6efd9e53-4cdd-4b99-84b2-114bd7e276c3 |                                            | available      |   500 | __DEFAULT__     |
| 9ce7b127-acae-4e94-9b21-079633f008bd |                                            | available      |   500 | __DEFAULT__     |
| 3a6e0d75-62ce-4d8f-8235-850c5c0db61d |                                            | available      |  2000 | __DEFAULT__     |
| 403314ac-9ff0-4e66-85ef-9c225603744e |                                            | available      |   100 | __DEFAULT__     |
| 83764bd8-5840-4f0b-9895-8b5738fd02e0 |                                            | available      |   500 | __DEFAULT__     |
| 5fd36e09-b5b7-472a-aade-2861af3f1470 |                                            | available      |   500 | __DEFAULT__     |
| 87a5e7a8-9c05-4863-8326-963a0637a3ee |                                            | available      |    20 | __DEFAULT__     |
| ba54c714-f982-4ca8-a471-debdce00c85e |                                            | available      |    20 | __DEFAULT__     |
| ca02edcf-4490-428d-b338-90ca08aa8776 |                                            | available      |    20 | __DEFAULT__     |
| 8e50c207-ee1c-40a2-9396-20f982e6c6f9 |                                            | available      |    20 | __DEFAULT__     |
| 581bb8cf-c3c4-47b7-a278-831b2e25434d |                                            | available      |    40 | __DEFAULT__     |
| 8126e37e-4ff2-465a-aefd-0ab34b97380c |                                            | available      |   250 | __DEFAULT__     |
| ae27171c-f11e-44e7-8928-e72efe9b45c6 |                                            | available      |   100 | __DEFAULT__     |
| 2f174279-21ed-4f23-aa04-d967dea13898 |                                            | available      |   250 | __DEFAULT__     |
| d59e470e-b6b2-4101-a845-6ae361ffd2bb |                                            | available      |   100 | __DEFAULT__     |
| 2a7ffe62-54e7-45e4-a61e-b066d29acc50 |                                            | available      |   100 | __DEFAULT__     |
| 6ef47ed4-a802-4e65-bb07-8c36b4a1fe5e |                                            | available      |   100 | __DEFAULT__     |
| 41cd2ee2-e85d-4046-9582-2fb555ed16f4 |                                            | available      |   100 | __DEFAULT__     |
| f12c71ee-f344-4ce8-9acc-db8a0b2d7047 |                                            | available      |   100 | __DEFAULT__     |
| aa0bd72b-4061-422a-a6ff-3fefee7f9f0c |                                            | available      |   100 | __DEFAULT__     |
| 2adda04e-b32c-4c27-88f3-a632ef55ac7d |                                            | available      |   500 | __DEFAULT__     |
| b43d92b0-d35c-41a0-926a-c94abb8bee23 |                                            | available      |   500 | __DEFAULT__     |
| fda55e3d-d49e-487e-b081-932c358051e6 |                                            | available      |   500 | __DEFAULT__     |
| 92aca6e5-bf64-475a-a3c5-00cd941848b5 |                                            | available      |   500 | __DEFAULT__     |
| 1cbdf9bb-2f89-47a7-86d2-16d0739a8451 |                                            | available      |   100 | __DEFAULT__     |
| ee882f16-605d-433d-88e0-5900898ed68e | Chris-data-drive                           | available      |  2000 | __DEFAULT__     |
| 88f473eb-d2fa-4e37-82f2-54fd85c25d5e |                                            | available      |   100 | __DEFAULT__     |
| d4af50d5-2182-47b3-941c-997e74c9341c |                                            | available      |   100 | __DEFAULT__     |
| 0da62386-8eb4-4963-95ff-d9a2b561061a |                                            | available      |   100 | __DEFAULT__     |
| 2dfb47ee-307e-4f85-b593-160ffecfbfe4 |                                            | available      |    40 | __DEFAULT__     |
| c89b51d0-49dc-4923-aaec-5e5da27ed799 |                                            | available      |    40 | __DEFAULT__     |
| df4c895e-3459-46d8-9f32-77158e8a1d12 |                                            | available      |    40 | __DEFAULT__     |
| 2caf828b-2d45-4191-bffb-0b07c5200446 |                                            | available      |    40 | __DEFAULT__     |
| 1e121c44-0c85-41d0-9c3e-250a1e576802 |                                            | available      |   100 | __DEFAULT__     |
| 321e3628-170f-42c9-9f37-66413250c441 |                                            | available      |   100 | __DEFAULT__     |
| f035099d-a4c1-489d-9253-edebe3413fec |                                            | available      |   100 | __DEFAULT__     |
| a3a39c50-5fa8-4b50-8093-d166eda187b9 | 5tb02                                      | available      |  5000 | __DEFAULT__     |
| 13ef19ae-7050-4c2b-8701-b4e8fea5bf6c | 5tb                                        | available      |  5000 | __DEFAULT__     |
| 6d979565-8049-4180-8979-b293911cdf83 |                                            | available      |   100 | __DEFAULT__     |
| 626d4ef8-ee00-4199-89ca-69cc3a443cff |                                            | available      |   100 | __DEFAULT__     |
| dc48db05-56e2-448e-99cb-e07e65977b53 |                                            | available      |   100 | __DEFAULT__     |
| b6088fb5-2be7-41ee-b72c-0c8c9d9ab66b |                                            | available      |   100 | __DEFAULT__     |
| 5d9b7aa9-2c51-412c-94d0-15660bf1b0aa |                                            | available      |   100 | __DEFAULT__     |
| 3f79a1dd-6cf8-4dea-b98f-b7597d3a5a6d |                                            | available      |   100 | __DEFAULT__     |
| 10d11999-8baa-4d8e-be9d-a2576f4f1288 |                                            | available      |   100 | __DEFAULT__     |
| 06421bcb-4e38-472f-8d71-ac840e539016 |                                            | available      |   100 | __DEFAULT__     |
| a9cbb441-96e9-4d84-8b3d-d2907ec42833 |                                            | available      |   100 | __DEFAULT__     |
| c9bff6d5-5890-4bf8-82a6-c8171bf80d2a |                                            | available      |   100 | __DEFAULT__     |
| 1e560124-46c2-4cab-806f-64a5eacfaf08 |                                            | available      |   100 | __DEFAULT__     |
| 5cbae0c2-7fa2-4e56-bc26-633c66aa5d73 |                                            | available      |    40 | __DEFAULT__     |
| 881aebac-a73f-4b65-a719-147fd1472cb5 |                                            | available      |    40 | __DEFAULT__     |
| 7b8a8580-503b-4da8-8777-9654407c86a4 |                                            | available      |   100 | __DEFAULT__     |
| a33c5734-11e1-4fda-8e88-44e7f1eded93 |                                            | available      |   100 | __DEFAULT__     |
| 3776f713-e1bd-40bd-be2b-18bd945385e6 |                                            | available      |   100 | __DEFAULT__     |
| 129e0f0e-efbf-41c0-b0e2-3dec5aaeb945 |                                            | available      |   100 | __DEFAULT__     |
| e1a43d3b-cdaf-475a-909c-fc54123bd87f | blank                                      | available      |   250 | __DEFAULT__     |
| b9e1b8d6-f38e-4570-ac01-49a7309ea6ec |                                            | available      |    40 | __DEFAULT__     |
| 76860ded-c5ff-458c-b749-a07b3c8e03fd |                                            | available      |    40 | __DEFAULT__     |
| 5765a2d5-9b84-44c3-b590-3cf97a073318 |                                            | available      |    40 | __DEFAULT__     |
| 54a55171-5a45-47eb-9c93-6e2b1f8f1dcf |                                            | available      |    40 | __DEFAULT__     |
| 26a1c410-3996-427f-8b93-69b003784dcd |                                            | available      |    40 | __DEFAULT__     |
| 21f54cb3-33a0-4df8-9ba5-fca4ea15409d |                                            | available      |    40 | __DEFAULT__     |
| 125a7496-fdcd-4979-8dae-f342ee1f28a0 |                                            | available      |    40 | __DEFAULT__     |
| 00c3f24c-37b9-4923-a7e5-70ca06b3023e |                                            | available      |    40 | __DEFAULT__     |
| eda73b06-05eb-4b3c-9280-0851873f145c |                                            | available      |    40 | __DEFAULT__     |
| 9255c6e7-9f79-408e-b8be-61082cf88d1d | centos-volume                              | available      |    40 | __DEFAULT__     |
| fa3e9113-7fcf-4ba5-b44e-201d406e6d04 |                                            | available      |    40 | __DEFAULT__     |
+--------------------------------------+--------------------------------------------+----------------+-------+-----------------+
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume list --all --long -c ID -c Status -c Type -c Name -c Size | grep -v -i storpool > ceph_volumes.txt
(kolla-ansible-zed) define@kolla-deploy:~$ for i in $(cat volumes_to_delete.txt); do  if grep -q $i volumes_in_ceph.txt; then echo openstack volume delete $i; fi ;done
openstack volume delete 00c3f24c-37b9-4923-a7e5-70ca06b3023e
openstack volume delete 06421bcb-4e38-472f-8d71-ac840e539016
openstack volume delete 0677c8e8-f4e7-42f4-8a57-7439b70e5653
openstack volume delete 083836a3-5437-4951-b066-66ff3fff9a68
openstack volume delete 0929147c-7e6a-464d-aa43-54212b955a58
openstack volume delete 095849e0-b0b6-4ba6-bb00-0f0030a91baf
openstack volume delete 0a050de7-73b9-4239-ad5f-40c0cbb250f9
openstack volume delete 0ab6766c-6cde-4a03-ad46-03009066783e
openstack volume delete 0d9aaf09-12b2-4b3c-8e37-7bb389731bf0
openstack volume delete 0da62386-8eb4-4963-95ff-d9a2b561061a
openstack volume delete 0e993fe7-c7a5-467c-9d72-4532cc316741
openstack volume delete 101cf309-5eab-45e4-9128-3572e107d237
openstack volume delete 10d11999-8baa-4d8e-be9d-a2576f4f1288
openstack volume delete 125a7496-fdcd-4979-8dae-f342ee1f28a0
openstack volume delete 129e0f0e-efbf-41c0-b0e2-3dec5aaeb945
openstack volume delete 13ef19ae-7050-4c2b-8701-b4e8fea5bf6c
openstack volume delete 19186f06-f923-4416-89e3-f52f6c54570b
openstack volume delete 1c68a40b-beae-45cb-9e0b-519301fb34af
openstack volume delete 1cbdf9bb-2f89-47a7-86d2-16d0739a8451
openstack volume delete 1dcf464a-7efd-4ad4-a774-367188f55448
openstack volume delete 1e121c44-0c85-41d0-9c3e-250a1e576802
openstack volume delete 1e560124-46c2-4cab-806f-64a5eacfaf08
openstack volume delete 1fc5a6f4-88ee-43bc-afdc-acb762bbec9d
openstack volume delete 21622152-bc69-4d80-83e4-216d22bfb193
openstack volume delete 21f54cb3-33a0-4df8-9ba5-fca4ea15409d
openstack volume delete 26a1c410-3996-427f-8b93-69b003784dcd
openstack volume delete 27b56326-21fe-4514-9b7f-52e56e00002a
openstack volume delete 29c54d8a-5e6d-4523-98a7-5f30c793068f
openstack volume delete 29dacac9-93dc-408a-bf3f-327e8590acad
openstack volume delete 2a7ffe62-54e7-45e4-a61e-b066d29acc50
openstack volume delete 2adda04e-b32c-4c27-88f3-a632ef55ac7d
openstack volume delete 2caf828b-2d45-4191-bffb-0b07c5200446
openstack volume delete 2dfb47ee-307e-4f85-b593-160ffecfbfe4
openstack volume delete 2f174279-21ed-4f23-aa04-d967dea13898
openstack volume delete 321e3628-170f-42c9-9f37-66413250c441
openstack volume delete 3776f713-e1bd-40bd-be2b-18bd945385e6
openstack volume delete 39094eb4-bfc8-4c37-89c0-4c6326c2d7b4
openstack volume delete 3a6e0d75-62ce-4d8f-8235-850c5c0db61d
openstack volume delete 3a887d0d-4b7d-4ee7-8b83-dbc21d199822
openstack volume delete 3d53a2d8-2fc3-4475-b5aa-a8ae4ef8adb8
openstack volume delete 3f79a1dd-6cf8-4dea-b98f-b7597d3a5a6d
openstack volume delete 403314ac-9ff0-4e66-85ef-9c225603744e
openstack volume delete 4054433b-03cf-4cb1-8948-c6ffe6e2955c
openstack volume delete 417feeef-d79d-4a31-af13-f1bee971284b
openstack volume delete 41cd2ee2-e85d-4046-9582-2fb555ed16f4
openstack volume delete 467bb817-3c6c-4f44-b767-35109127cb7c
openstack volume delete 46c6479d-a6c5-4efb-a7ab-fafa2cbb382e
openstack volume delete 4f2d0656-d7ff-4e43-8b93-24e74ee04b72
openstack volume delete 4f80c812-cc7e-4b04-8148-ffd01ff69f14
openstack volume delete 4f9393d2-ee56-49b3-ac8c-3ba192ec8921
openstack volume delete 5408fbf0-46f6-41d9-937a-239be5055c4c
openstack volume delete 54a55171-5a45-47eb-9c93-6e2b1f8f1dcf
openstack volume delete 57590c5a-e645-40a0-9237-3d69aca8c020
openstack volume delete 5765a2d5-9b84-44c3-b590-3cf97a073318
openstack volume delete 581bb8cf-c3c4-47b7-a278-831b2e25434d
openstack volume delete 5cbae0c2-7fa2-4e56-bc26-633c66aa5d73
openstack volume delete 5d9b7aa9-2c51-412c-94d0-15660bf1b0aa
openstack volume delete 5fd36e09-b5b7-472a-aade-2861af3f1470
openstack volume delete 61192337-5f2f-400e-b04f-337ce502b326
openstack volume delete 62547f47-0a75-400b-848a-a13ba8523613
openstack volume delete 626d4ef8-ee00-4199-89ca-69cc3a443cff
openstack volume delete 639983da-294f-45ac-883f-b7ec3d0f4ff3
openstack volume delete 64187c05-3299-4a2e-ab1b-e527f483cb6c
openstack volume delete 68348ebf-fbb8-4d5a-8965-bc84f70a059a
openstack volume delete 683b6a60-45df-4366-a7b7-860b2338aa33
openstack volume delete 68fcd1c6-60d3-4e7d-a7e5-d13a8d7bd424
openstack volume delete 696d70cf-bf67-4a75-a3ff-75d1b445d1c9
openstack volume delete 6d979565-8049-4180-8979-b293911cdf83
openstack volume delete 6ef47ed4-a802-4e65-bb07-8c36b4a1fe5e
openstack volume delete 6efa66a3-89f9-4f44-9526-f6425c962a75
openstack volume delete 6efd9e53-4cdd-4b99-84b2-114bd7e276c3
openstack volume delete 6ff869ee-8bd0-49fc-9a5e-fb0f6db9295e
openstack volume delete 70d3019c-8085-4bcf-b120-b20652fcc835
openstack volume delete 723b7867-c8fc-48bd-a48b-e3956bddc173
openstack volume delete 74e650ba-05ec-41fa-9a47-fe04ad1ab20b
openstack volume delete 763547b6-db23-496f-b852-46f1126411d2
openstack volume delete 76860ded-c5ff-458c-b749-a07b3c8e03fd
openstack volume delete 77b5618b-91c3-428d-ba70-2e399c46878d
openstack volume delete 7894f6c4-c815-4132-896e-c4d5f41efdb9
openstack volume delete 79ccdd09-102d-42e6-bc6c-01b6a2d19892
openstack volume delete 7aedaaa5-f547-401a-b4cb-1d2271eb1c7d
openstack volume delete 7b8a8580-503b-4da8-8777-9654407c86a4
openstack volume delete 7cc5f04a-0ecf-4ddc-a2c4-5c0d3a0b5913
openstack volume delete 8126e37e-4ff2-465a-aefd-0ab34b97380c
openstack volume delete 83764bd8-5840-4f0b-9895-8b5738fd02e0
openstack volume delete 85ecb198-c2de-4ecf-90b6-7f658085d5f5
openstack volume delete 86056c2b-a904-4228-8dd1-c4cb01648f0f
openstack volume delete 87a5e7a8-9c05-4863-8326-963a0637a3ee
openstack volume delete 881aebac-a73f-4b65-a719-147fd1472cb5
openstack volume delete 8889e3b6-5d6c-4bf6-b17c-434ffb95154f
openstack volume delete 88f473eb-d2fa-4e37-82f2-54fd85c25d5e
openstack volume delete 8a8bf436-da79-49ce-979c-4f312e9015da
openstack volume delete 8e50c207-ee1c-40a2-9396-20f982e6c6f9
openstack volume delete 8eb7069c-5c07-43c6-a91f-f1d79a071a91
openstack volume delete 9255c6e7-9f79-408e-b8be-61082cf88d1d
openstack volume delete 92aca6e5-bf64-475a-a3c5-00cd941848b5
openstack volume delete 93f22739-5ac8-4a88-8c49-de4c2234e7e7
openstack volume delete 94e97068-13e3-49a2-8bfe-4c4ba1d898fd
openstack volume delete 998bcdd4-3588-4464-ac8b-88c5387402e0
openstack volume delete 9b1d7f51-7486-4b01-b269-2ed143fc7428
openstack volume delete 9b81e22e-81d6-4e61-be54-fb990083c540
openstack volume delete 9ce7b127-acae-4e94-9b21-079633f008bd
openstack volume delete 9dd70ba7-1387-4d9d-bde0-ae2c843e535f
openstack volume delete 9ead516b-aaf4-492e-9457-b4be23f10d7f
openstack volume delete 9fb3978c-050d-4bfc-b9ce-9cd213994d96
openstack volume delete a1468374-b1c3-4c26-a2da-c2cfeee1c27d
openstack volume delete a2727c56-fe70-4f02-8dcb-a23373bcbfe0
openstack volume delete a33c5734-11e1-4fda-8e88-44e7f1eded93
openstack volume delete a3a39c50-5fa8-4b50-8093-d166eda187b9
openstack volume delete a9cbb441-96e9-4d84-8b3d-d2907ec42833
openstack volume delete aa0bd72b-4061-422a-a6ff-3fefee7f9f0c
openstack volume delete ae27171c-f11e-44e7-8928-e72efe9b45c6
openstack volume delete b1a50f3e-6c7c-4fa8-a1ae-1a488fc4d3b3
openstack volume delete b43d92b0-d35c-41a0-926a-c94abb8bee23
openstack volume delete b6088fb5-2be7-41ee-b72c-0c8c9d9ab66b
openstack volume delete b96a70f0-7326-4a7d-9c36-928bf2abd1ce
openstack volume delete b9e1b8d6-f38e-4570-ac01-49a7309ea6ec
openstack volume delete ba54c714-f982-4ca8-a471-debdce00c85e
openstack volume delete bb55d0ba-7dec-458f-a7fc-b6d4b46a9936
openstack volume delete c4d0ce91-8f23-40cf-9772-cca3c77ae672
openstack volume delete c89b51d0-49dc-4923-aaec-5e5da27ed799
openstack volume delete c9bff6d5-5890-4bf8-82a6-c8171bf80d2a
openstack volume delete ca02edcf-4490-428d-b338-90ca08aa8776
openstack volume delete d2f33e0e-d7f5-4eae-8efb-27a71f335d12
openstack volume delete d4af50d5-2182-47b3-941c-997e74c9341c
openstack volume delete d4dda353-7cb0-4a60-9c76-edfa119d86bc
openstack volume delete d59e470e-b6b2-4101-a845-6ae361ffd2bb
openstack volume delete d698f7bf-fff5-450d-8ed6-2f62e4ca80cf
openstack volume delete dc275cb9-6709-4dc1-8e9d-5943a3b4964e
openstack volume delete dc48db05-56e2-448e-99cb-e07e65977b53
openstack volume delete df4c895e-3459-46d8-9f32-77158e8a1d12
openstack volume delete e1a43d3b-cdaf-475a-909c-fc54123bd87f
openstack volume delete e2c8bfb6-346b-485e-a621-15cb0b9f82c9
openstack volume delete e50b1239-62da-4535-aa21-4e551c1eb2be
openstack volume delete e5d32199-8115-43a8-a8df-fa2f98442f96
openstack volume delete e9638883-b712-4fcc-beec-b0fc7aa0ac75
openstack volume delete ea0b101d-86d3-4ab7-a01f-d9fa566e7174
openstack volume delete eda73b06-05eb-4b3c-9280-0851873f145c
openstack volume delete ee882f16-605d-433d-88e0-5900898ed68e
openstack volume delete ef74477b-a345-4cc0-ae94-4b29b0375015
openstack volume delete f035099d-a4c1-489d-9253-edebe3413fec
openstack volume delete f12c71ee-f344-4ce8-9acc-db8a0b2d7047
openstack volume delete f3417f0a-1e34-4de6-83e5-f4c86d526e48
openstack volume delete f465b10e-5d01-4b5d-a786-51cc217220ee
openstack volume delete f818b7e0-9d56-4df1-8416-6aa50cbeb606
openstack volume delete f8f47d4a-1baa-4a3e-a5e0-98d911f700b0
openstack volume delete fa3e9113-7fcf-4ba5-b44e-201d406e6d04
openstack volume delete fda55e3d-d49e-487e-b081-932c358051e6

(kolla-ansible-zed) define@kolla-deploy:~$ for i in $(cat volumes_to_delete.txt); do  if grep -q $i volumes_in_ceph.txt; then openstack volume delete $i; fi ;done
Failed to delete volume with name or ID '0ab6766c-6cde-4a03-ad46-03009066783e': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-4727a52a-83ef-4136-84b4-d5e390f14422)
1 of 1 volumes failed to delete.
Failed to delete volume with name or ID '101cf309-5eab-45e4-9128-3572e107d237': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-d701e69f-d4d5-47d9-a26c-92526586ed3a)
1 of 1 volumes failed to delete.
Failed to delete volume with name or ID '19186f06-f923-4416-89e3-f52f6c54570b': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-144a36ac-dbf0-4b29-a62e-2037d4453308)
1 of 1 volumes failed to delete.
Failed to delete volume with name or ID '1c68a40b-beae-45cb-9e0b-519301fb34af': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-07697f0a-dcf7-43cf-8a5b-2cbf41161acc)
1 of 1 volumes failed to delete.
Failed to delete volume with name or ID '1dcf464a-7efd-4ad4-a774-367188f55448': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-a34981f1-1f79-441f-a7f9-bd1fe985ccad)
1 of 1 volumes failed to delete.
Failed to delete volume with name or ID '29dacac9-93dc-408a-bf3f-327e8590acad': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-22de8f1d-ba09-4fba-9ce3-b81a027e3f06)
1 of 1 volumes failed to delete.
Failed to delete volume with name or ID '321e3628-170f-42c9-9f37-66413250c441': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-bbef874b-9752-472d-8530-3d9605f9c24b)
1 of 1 volumes failed to delete.
Failed to delete volume with name or ID '3776f713-e1bd-40bd-be2b-18bd945385e6': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-20e78bd2-f040-4017-b156-53cc2e515378)
1 of 1 volumes failed to delete.
Failed to delete volume with name or ID '4054433b-03cf-4cb1-8948-c6ffe6e2955c': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-de4f46ee-a021-4c0a-bce4-9f7575af4525)
1 of 1 volumes failed to delete.
Failed to delete volume with name or ID '417feeef-d79d-4a31-af13-f1bee971284b': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-c1b0a36e-6281-4f15-adfe-f049615dafdd)
1 of 1 volumes failed to delete.
Failed to delete volume with name or ID '5408fbf0-46f6-41d9-937a-239be5055c4c': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-1a4f961a-e1e7-4348-b5da-87a06c418fb3)
1 of 1 volumes failed to delete.
Failed to delete volume with name or ID '64187c05-3299-4a2e-ab1b-e527f483cb6c': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-60a9ef81-a7dc-4eb3-8379-315c6c2183dc)
1 of 1 volumes failed to delete.
Failed to delete volume with name or ID '6efd9e53-4cdd-4b99-84b2-114bd7e276c3': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-c674668e-b9e4-49ec-a970-7f8f91e393ec)
1 of 1 volumes failed to delete.
Failed to delete volume with name or ID '763547b6-db23-496f-b852-46f1126411d2': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-04b1274a-1105-4b7f-b057-9d921d2509a2)
1 of 1 volumes failed to delete.
Failed to delete volume with name or ID '7894f6c4-c815-4132-896e-c4d5f41efdb9': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-2ec2ff9f-49ca-4819-bd6a-ff0555b860fc)
1 of 1 volumes failed to delete.
Failed to delete volume with name or ID '79ccdd09-102d-42e6-bc6c-01b6a2d19892': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-550a660d-1b2e-44bf-9bd9-31938cc8679f)
1 of 1 volumes failed to delete.
Failed to delete volume with name or ID '7aedaaa5-f547-401a-b4cb-1d2271eb1c7d': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-076359be-32a0-4dc2-b014-a9654096664f)
1 of 1 volumes failed to delete.
Failed to delete volume with name or ID '85ecb198-c2de-4ecf-90b6-7f658085d5f5': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-c53d1d70-c7d6-4ef8-bf77-1a69e376eafa)
1 of 1 volumes failed to delete.
Failed to delete volume with name or ID '88f473eb-d2fa-4e37-82f2-54fd85c25d5e': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-9002f726-6ad1-43e9-8d86-7e1c1bc13ac8)
1 of 1 volumes failed to delete.
Failed to delete volume with name or ID '8eb7069c-5c07-43c6-a91f-f1d79a071a91': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-ed457830-1d54-44c4-8695-f5eb4e64db3b)
1 of 1 volumes failed to delete.
Failed to delete volume with name or ID '9255c6e7-9f79-408e-b8be-61082cf88d1d': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-c7dabe25-2ce7-43a0-a31c-31d219610f0e)
1 of 1 volumes failed to delete.
Failed to delete volume with name or ID '92aca6e5-bf64-475a-a3c5-00cd941848b5': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-b0810666-9328-4bb2-90a0-16bcbad93e55)
1 of 1 volumes failed to delete.
Failed to delete volume with name or ID '93f22739-5ac8-4a88-8c49-de4c2234e7e7': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-9a2800f5-adcc-4f55-83d0-8b9a5c030701)
1 of 1 volumes failed to delete.
Failed to delete volume with name or ID '998bcdd4-3588-4464-ac8b-88c5387402e0': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-f3fde1b3-4814-411e-abfb-b6cb69709d5c)
1 of 1 volumes failed to delete.
Failed to delete volume with name or ID '9fb3978c-050d-4bfc-b9ce-9cd213994d96': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-f0ed5a1a-6f66-41aa-8576-a8655308258a)
1 of 1 volumes failed to delete.
Failed to delete volume with name or ID 'b1a50f3e-6c7c-4fa8-a1ae-1a488fc4d3b3': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-4dd7bbea-0fed-44c7-b084-edee3c87cd57)
1 of 1 volumes failed to delete.
Failed to delete volume with name or ID 'b96a70f0-7326-4a7d-9c36-928bf2abd1ce': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-a8118825-ade3-4a87-8084-fd02709691bf)
1 of 1 volumes failed to delete.
Failed to delete volume with name or ID 'c4d0ce91-8f23-40cf-9772-cca3c77ae672': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-272d1c02-b4fb-4fb1-8f38-da10ea6d98bb)
1 of 1 volumes failed to delete.
Failed to delete volume with name or ID 'd4af50d5-2182-47b3-941c-997e74c9341c': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-f4600baa-9cd8-4e65-9b79-99097aac0a0a)
1 of 1 volumes failed to delete.
Failed to delete volume with name or ID 'dc275cb9-6709-4dc1-8e9d-5943a3b4964e': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-5b07202d-182c-47ea-8cc4-a483473c87ca)
1 of 1 volumes failed to delete.
Failed to delete volume with name or ID 'dc48db05-56e2-448e-99cb-e07e65977b53': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-0ddfe2a5-d3bf-4e10-aa99-a350eb244da2)
1 of 1 volumes failed to delete.
Failed to delete volume with name or ID 'e1a43d3b-cdaf-475a-909c-fc54123bd87f': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-1020606f-4e28-45a7-bb48-6475a14ef0e8)
1 of 1 volumes failed to delete.
Failed to delete volume with name or ID 'e50b1239-62da-4535-aa21-4e551c1eb2be': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-d82767b7-13f4-41b5-82af-42e427558b24)
1 of 1 volumes failed to delete.
(kolla-ansible-zed) define@kolla-deploy:~$ for i in $(cat volumes_to_delete.txt); do  if grep -q $i volumes_in_ceph.txt; then openstack volume delete $i; fi ;done


for i in $(rbd  list volumes); do echo the kids of $i  are:; rbd children volumes/$i; done





root@str-238:~# for i in $(rbd  list volumes); do echo the snaps of $i are:; rbd snap ls volumes/$i; done
the snaps of volume-0677c8e8-f4e7-42f4-8a57-7439b70e5653 are:
the snaps of volume-095849e0-b0b6-4ba6-bb00-0f0030a91baf are:
the snaps of volume-0ab6766c-6cde-4a03-ad46-03009066783e are:
SNAPID  NAME                                           SIZE    PROTECTED  TIMESTAMP
    19  snapshot-6a125870-d6ef-40af-a9c8-7c2c72ba0876  40 GiB  yes        Mon Jan 16 16:51:32 2023
the snaps of volume-0da62386-8eb4-4963-95ff-d9a2b561061a are:
the snaps of volume-101cf309-5eab-45e4-9128-3572e107d237 are:
SNAPID  NAME                                           SIZE     PROTECTED  TIMESTAMP
    30  snapshot-500e0ae0-b6ce-4aba-8434-2b73bb7251a6  5.9 TiB  yes        Fri Mar 10 19:40:59 2023
    31  snapshot-e68051c0-b3a0-4e65-8305-78f7aa3b2065  5.9 TiB  yes        Fri Mar 10 19:43:26 2023
    32  snapshot-e0364942-1bb3-493b-a84e-a736b29fb847  5.9 TiB  yes        Fri Mar 10 19:44:08 2023
the snaps of volume-125a7496-fdcd-4979-8dae-f342ee1f28a0 are:
the snaps of volume-16b5ad27-ecf5-46e7-81ef-b79af5618dd0.deleted are:
SNAPID  NAME                                                    SIZE     PROTECTED  TIMESTAMP
    65  volume-34daba88-c273-4b3b-8954-6ad6832bdd21.clone_snap  100 GiB  yes        Thu Apr 13 08:24:08 2023
the snaps of volume-19186f06-f923-4416-89e3-f52f6c54570b are:
SNAPID  NAME                                           SIZE     PROTECTED  TIMESTAMP
    41  snapshot-6d605146-428f-4feb-a0c8-b0a4a8733a2a  3.9 TiB  yes        Mon Mar 20 16:19:10 2023
the snaps of volume-1c68a40b-beae-45cb-9e0b-519301fb34af are:
SNAPID  NAME                                           SIZE     PROTECTED  TIMESTAMP
    28  snapshot-fd08c190-3022-4764-8722-8de79ec0969b  250 GiB  yes        Thu Mar  2 20:21:42 2023
    29  snapshot-3013ef2e-f749-4ff1-ae0c-ebf49a4deec3  250 GiB  yes        Fri Mar  3 14:58:53 2023
the snaps of volume-1cbdf9bb-2f89-47a7-86d2-16d0739a8451 are:
the snaps of volume-1dcf464a-7efd-4ad4-a774-367188f55448 are:
SNAPID  NAME                                           SIZE     PROTECTED  TIMESTAMP
    56  snapshot-7b0771a6-b06d-491e-86a0-940cd4383adb  100 GiB  yes        Thu Mar 30 13:23:18 2023
    59  snapshot-20864eb1-b532-422d-aa4b-bd40965842f4  100 GiB  yes        Tue Apr  4 16:28:00 2023
    78  snapshot-f19c6d99-4217-43cf-9a93-b1b9b60206be  100 GiB  yes        Thu Jun  8 15:31:44 2023
   105  snapshot-33bd9242-9441-471f-a4a6-8d46f2127ae4  100 GiB  yes        Tue Aug 29 16:58:13 2023
the snaps of volume-1fc5a6f4-88ee-43bc-afdc-acb762bbec9d are:
the snaps of volume-26a1c410-3996-427f-8b93-69b003784dcd are:
the snaps of volume-29dacac9-93dc-408a-bf3f-327e8590acad are:
SNAPID  NAME                                           SIZE     PROTECTED  TIMESTAMP
    94  snapshot-5d7cf008-d658-4d63-8f9a-39e754494038  500 GiB  yes        Fri Jul  7 14:51:22 2023
   114  snapshot-7c9b41ab-ee7b-4be6-832a-7a773ac6710a  500 GiB  yes        Tue Sep  5 12:25:16 2023
the snaps of volume-2a7ffe62-54e7-45e4-a61e-b066d29acc50 are:
the snaps of volume-2dfb47ee-307e-4f85-b593-160ffecfbfe4 are:
the snaps of volume-321e3628-170f-42c9-9f37-66413250c441 are:
SNAPID  NAME                                           SIZE     PROTECTED  TIMESTAMP
    13  snapshot-e52ae299-9eb6-4c72-85b8-691de1683f5c  100 GiB  yes        Thu Dec 15 15:25:31 2022
the snaps of volume-3776f713-e1bd-40bd-be2b-18bd945385e6 are:
SNAPID  NAME                                           SIZE     PROTECTED  TIMESTAMP
     5  snapshot-1396b706-51d0-40e2-b670-3b17b5c4dbe0  100 GiB  yes        Mon Oct 24 23:18:21 2022
     6  snapshot-ab5416a7-ea1d-43a5-b084-6f5a2ea09bf5  100 GiB  yes        Mon Oct 24 23:23:48 2022
     7  snapshot-435a8912-3cd4-4e93-b1b5-ef8056cf6bce  100 GiB  yes        Tue Oct 25 00:30:26 2022
     8  snapshot-f8cd87ae-afd4-4338-89d1-13a78eeab2b2  100 GiB  yes        Tue Oct 25 02:07:05 2022
the snaps of volume-39094eb4-bfc8-4c37-89c0-4c6326c2d7b4 are:
the snaps of volume-3a6e0d75-62ce-4d8f-8235-850c5c0db61d are:
the snaps of volume-3f79a1dd-6cf8-4dea-b98f-b7597d3a5a6d are:
the snaps of volume-4054433b-03cf-4cb1-8948-c6ffe6e2955c are:
the snaps of volume-417feeef-d79d-4a31-af13-f1bee971284b are:
SNAPID  NAME                                           SIZE     PROTECTED  TIMESTAMP
    25  snapshot-de4787b8-ce8f-48ce-b13c-2854f1bbab2e  3.9 TiB  yes        Mon Feb 27 16:33:30 2023
the snaps of volume-467bb817-3c6c-4f44-b767-35109127cb7c are:
the snaps of volume-4f80c812-cc7e-4b04-8148-ffd01ff69f14 are:
the snaps of volume-5408fbf0-46f6-41d9-937a-239be5055c4c are:
SNAPID  NAME                                           SIZE     PROTECTED  TIMESTAMP
    39  snapshot-e41485d1-c71d-45d3-9184-b40522e9e700  5.9 TiB  yes        Sat Mar 18 16:25:04 2023
the snaps of volume-56bb6be4-b0b5-44ca-8bad-9e4b55ac922e.deleted are:
SNAPID  NAME                                                    SIZE     PROTECTED  TIMESTAMP
    66  volume-3b794a23-c30f-44a8-a1c1-763ab4ee1635.clone_snap  250 GiB  yes        Thu Apr 13 21:56:54 2023
the snaps of volume-57590c5a-e645-40a0-9237-3d69aca8c020 are:
the snaps of volume-5cbae0c2-7fa2-4e56-bc26-633c66aa5d73 are:
the snaps of volume-61192337-5f2f-400e-b04f-337ce502b326 are:
the snaps of volume-639983da-294f-45ac-883f-b7ec3d0f4ff3 are:
the snaps of volume-64187c05-3299-4a2e-ab1b-e527f483cb6c are:
SNAPID  NAME                                           SIZE     PROTECTED  TIMESTAMP
   104  snapshot-b139b50d-4984-452b-8a04-6a3ae9ac13d2  250 GiB  yes        Fri Jul 28 13:21:57 2023
   113  snapshot-734d71df-fa23-4145-ad91-c40004d63797  250 GiB  yes        Tue Sep  5 12:22:33 2023
the snaps of volume-68fcd1c6-60d3-4e7d-a7e5-d13a8d7bd424 are:
the snaps of volume-6ef47ed4-a802-4e65-bb07-8c36b4a1fe5e are:
the snaps of volume-6efd9e53-4cdd-4b99-84b2-114bd7e276c3 are:
SNAPID  NAME                                           SIZE     PROTECTED  TIMESTAMP
    22  snapshot-19c4346b-9ae1-489f-b992-d0186bbd8154  500 GiB  yes        Tue Feb 14 17:53:37 2023
the snaps of volume-70d3019c-8085-4bcf-b120-b20652fcc835 are:
the snaps of volume-763547b6-db23-496f-b852-46f1126411d2 are:
SNAPID  NAME                                           SIZE     PROTECTED  TIMESTAMP
    38  snapshot-7eea23cd-6ae4-419b-82d8-132b32f6cad8  3.9 TiB  yes        Thu Mar 16 21:43:24 2023
the snaps of volume-76860ded-c5ff-458c-b749-a07b3c8e03fd are:
the snaps of volume-7894f6c4-c815-4132-896e-c4d5f41efdb9 are:
SNAPID  NAME                                           SIZE     PROTECTED  TIMESTAMP
    36  snapshot-001d43be-e82d-474b-bea4-1138ba325899  3.9 TiB  yes        Thu Mar 16 20:14:29 2023
the snaps of volume-79ccdd09-102d-42e6-bc6c-01b6a2d19892 are:
SNAPID  NAME                                           SIZE     PROTECTED  TIMESTAMP
    51  snapshot-e8c076ef-6b66-485b-9706-f83413929eda  500 GiB  yes        Thu Mar 23 19:29:20 2023
the snaps of volume-7aedaaa5-f547-401a-b4cb-1d2271eb1c7d are:
SNAPID  NAME                                           SIZE     PROTECTED  TIMESTAMP
    24  snapshot-509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824  3.9 TiB  yes        Tue Feb 14 19:35:36 2023
the snaps of volume-7cc5f04a-0ecf-4ddc-a2c4-5c0d3a0b5913 are:
the snaps of volume-85ecb198-c2de-4ecf-90b6-7f658085d5f5 are:
SNAPID  NAME                                           SIZE    PROTECTED  TIMESTAMP
   103  snapshot-98cd07e4-6b5f-4b5f-8608-15e759791dfb  50 GiB  yes        Thu Jul 13 02:30:44 2023
the snaps of volume-86056c2b-a904-4228-8dd1-c4cb01648f0f are:
the snaps of volume-8889e3b6-5d6c-4bf6-b17c-434ffb95154f are:
the snaps of volume-88f473eb-d2fa-4e37-82f2-54fd85c25d5e are:
SNAPID  NAME                                           SIZE     PROTECTED  TIMESTAMP
    15  snapshot-3b355a39-d82a-45e0-95fd-49d447ae5a34  100 GiB  yes        Tue Dec 27 03:24:20 2022
    17  snapshot-79c0a755-87e6-4b5f-bca1-da657e52331b  100 GiB  yes        Mon Jan  9 18:56:00 2023
the snaps of volume-8eb7069c-5c07-43c6-a91f-f1d79a071a91 are:
SNAPID  NAME                                           SIZE     PROTECTED  TIMESTAMP
    42  snapshot-14b5f169-1fe1-43f1-aacf-0c4589e8c469  200 GiB  yes        Mon Mar 20 20:18:37 2023
the snaps of volume-92aca6e5-bf64-475a-a3c5-00cd941848b5 are:
SNAPID  NAME                                           SIZE     PROTECTED  TIMESTAMP
    11  snapshot-96fdbeab-81a4-465d-87ae-0916c0950a88  500 GiB  yes        Tue Nov 29 02:12:25 2022
the snaps of volume-93f22739-5ac8-4a88-8c49-de4c2234e7e7 are:
SNAPID  NAME                                           SIZE     PROTECTED  TIMESTAMP
    80  snapshot-42d02435-04aa-42e5-9e92-a87161f50353  250 GiB  yes        Thu Jun  8 15:45:29 2023
the snaps of volume-94e97068-13e3-49a2-8bfe-4c4ba1d898fd are:
the snaps of volume-9ce7b127-acae-4e94-9b21-079633f008bd are:
the snaps of volume-9fb3978c-050d-4bfc-b9ce-9cd213994d96 are:
SNAPID  NAME                                           SIZE     PROTECTED  TIMESTAMP
    53  snapshot-69d9e3b8-eed3-47e8-854f-21147fbce2da  100 GiB  yes        Mon Mar 27 23:27:38 2023
the snaps of volume-a1468374-b1c3-4c26-a2da-c2cfeee1c27d are:
the snaps of volume-a3a39c50-5fa8-4b50-8093-d166eda187b9 are:
the snaps of volume-ae27171c-f11e-44e7-8928-e72efe9b45c6 are:
the snaps of volume-b1a50f3e-6c7c-4fa8-a1ae-1a488fc4d3b3 are:
SNAPID  NAME                                           SIZE     PROTECTED  TIMESTAMP
    37  snapshot-7bb8db32-e5f3-4ae5-92a0-9373cc543bff  500 GiB  yes        Thu Mar 16 21:43:24 2023
the snaps of volume-b96a70f0-7326-4a7d-9c36-928bf2abd1ce are:
SNAPID  NAME                                           SIZE     PROTECTED  TIMESTAMP
    44  snapshot-d44d6b1a-7a13-4cec-878e-ec9cef041fdb  3.9 TiB  yes        Tue Mar 21 17:42:47 2023
the snaps of volume-b9e1b8d6-f38e-4570-ac01-49a7309ea6ec are:
the snaps of volume-c4d0ce91-8f23-40cf-9772-cca3c77ae672 are:
SNAPID  NAME                                           SIZE     PROTECTED  TIMESTAMP
    83  snapshot-2df9ca3f-0340-4a4c-8995-9626627af7c5  5.9 TiB  yes        Mon Jun 12 20:12:29 2023
the snaps of volume-c89b51d0-49dc-4923-aaec-5e5da27ed799 are:
the snaps of volume-d2f33e0e-d7f5-4eae-8efb-27a71f335d12 are:
the snaps of volume-d4af50d5-2182-47b3-941c-997e74c9341c are:
SNAPID  NAME                                           SIZE     PROTECTED  TIMESTAMP
    10  snapshot-91d7fb79-f5ae-4c3f-bb1c-068c7c5a1adc  100 GiB  yes        Fri Nov 11 14:51:08 2022
the snaps of volume-d698f7bf-fff5-450d-8ed6-2f62e4ca80cf are:
the snaps of volume-dc275cb9-6709-4dc1-8e9d-5943a3b4964e are:
SNAPID  NAME                                           SIZE     PROTECTED  TIMESTAMP
    60  snapshot-85e73527-63b5-4f9c-a66f-70356e11a490  100 GiB  yes        Thu Apr 13 08:08:15 2023
the snaps of volume-dc48db05-56e2-448e-99cb-e07e65977b53 are:
SNAPID  NAME                                           SIZE     PROTECTED  TIMESTAMP
     9  snapshot-b9345b91-a5b5-4157-8838-4740b7e27788  100 GiB  yes        Tue Nov  8 21:07:28 2022
the snaps of volume-e1a43d3b-cdaf-475a-909c-fc54123bd87f are:
SNAPID  NAME                                           SIZE     PROTECTED  TIMESTAMP
    14  snapshot-0c867532-e1d9-429e-b402-b86db63f98c0  250 GiB  yes        Thu Dec 15 15:25:31 2022
the snaps of volume-e50b1239-62da-4535-aa21-4e551c1eb2be are:
SNAPID  NAME                                           SIZE     PROTECTED  TIMESTAMP
    33  snapshot-0e47cfa0-481b-4f8e-b405-8fb5f74b927b  100 GiB  yes        Wed Mar 15 14:43:45 2023
    43  snapshot-fc42e191-f086-487e-ba15-305830cac507  100 GiB  yes        Tue Mar 21 17:03:58 2023
    50  snapshot-7d89b23e-4620-4293-931a-1c47b11985bb  100 GiB  yes        Thu Mar 23 14:17:23 2023
the snaps of volume-e5d32199-8115-43a8-a8df-fa2f98442f96 are:
the snaps of volume-eda73b06-05eb-4b3c-9280-0851873f145c are:
the snaps of volume-ef74477b-a345-4cc0-ae94-4b29b0375015 are:
the snaps of volume-f035099d-a4c1-489d-9253-edebe3413fec are:
the snaps of volume-f465b10e-5d01-4b5d-a786-51cc217220ee are:
the snaps of volume-fa3e9113-7fcf-4ba5-b44e-201d406e6d04 are:


















##################################################### dev work complete 

checking volumes/volume-0677c8e8-f4e7-42f4-8a57-7439b70e5653 for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete 0677c8e8-f4e7-42f4-8a57-7439b70e5653
checking volumes/volume-095849e0-b0b6-4ba6-bb00-0f0030a91baf for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete 095849e0-b0b6-4ba6-bb00-0f0030a91baf
checking volumes/volume-0ab6766c-6cde-4a03-ad46-03009066783e for snaps

image volumes/volume-0ab6766c-6cde-4a03-ad46-03009066783e has a snapshot called snapshot-6a125870-d6ef-40af-a9c8-7c2c72ba0876 and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show 6a125870-d6ef-40af-a9c8-7c2c72ba0876
please check that this is a member of 0ab6766c-6cde-4a03-ad46-03009066783e before running any of the following

openstack volume snapshot delete 6a125870-d6ef-40af-a9c8-7c2c72ba0876

it should now be possible to delete the volume using the following commands

openstack volume delete 0ab6766c-6cde-4a03-ad46-03009066783e
checking volumes/volume-0da62386-8eb4-4963-95ff-d9a2b561061a for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete 0da62386-8eb4-4963-95ff-d9a2b561061a
checking volumes/volume-101cf309-5eab-45e4-9128-3572e107d237 for snaps

image volumes/volume-101cf309-5eab-45e4-9128-3572e107d237 has a snapshot called snapshot-500e0ae0-b6ce-4aba-8434-2b73bb7251a6 and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show 500e0ae0-b6ce-4aba-8434-2b73bb7251a6
please check that this is a member of 101cf309-5eab-45e4-9128-3572e107d237 before running any of the following

openstack volume snapshot delete 500e0ae0-b6ce-4aba-8434-2b73bb7251a6

image volumes/volume-101cf309-5eab-45e4-9128-3572e107d237 has a snapshot called snapshot-e68051c0-b3a0-4e65-8305-78f7aa3b2065 and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show e68051c0-b3a0-4e65-8305-78f7aa3b2065
please check that this is a member of 101cf309-5eab-45e4-9128-3572e107d237 before running any of the following

openstack volume snapshot delete e68051c0-b3a0-4e65-8305-78f7aa3b2065

image volumes/volume-101cf309-5eab-45e4-9128-3572e107d237 has a snapshot called snapshot-e0364942-1bb3-493b-a84e-a736b29fb847 and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show e0364942-1bb3-493b-a84e-a736b29fb847
please check that this is a member of 101cf309-5eab-45e4-9128-3572e107d237 before running any of the following

openstack volume snapshot delete e0364942-1bb3-493b-a84e-a736b29fb847

it should now be possible to delete the volume using the following commands

openstack volume delete 101cf309-5eab-45e4-9128-3572e107d237
checking volumes/volume-125a7496-fdcd-4979-8dae-f342ee1f28a0 for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete 125a7496-fdcd-4979-8dae-f342ee1f28a0
checking volumes/volume-16b5ad27-ecf5-46e7-81ef-b79af5618dd0.deleted for snaps

image volumes/volume-16b5ad27-ecf5-46e7-81ef-b79af5618dd0.deleted has a snapshot called volume-34daba88-c273-4b3b-8954-6ad6832bdd21.clone_snap and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show 34daba88-c273-4b3b-8954-6ad6832bdd21.clone_snap
please check that this is a member of 16b5ad27-ecf5-46e7-81ef-b79af5618dd0.deleted before running any of the following

openstack volume snapshot delete 34daba88-c273-4b3b-8954-6ad6832bdd21.clone_snap

it should now be possible to delete the volume using the following commands

openstack volume delete 16b5ad27-ecf5-46e7-81ef-b79af5618dd0.deleted
checking volumes/volume-19186f06-f923-4416-89e3-f52f6c54570b for snaps

image volumes/volume-19186f06-f923-4416-89e3-f52f6c54570b has a snapshot called snapshot-6d605146-428f-4feb-a0c8-b0a4a8733a2a and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show 6d605146-428f-4feb-a0c8-b0a4a8733a2a
please check that this is a member of 19186f06-f923-4416-89e3-f52f6c54570b before running any of the following

openstack volume snapshot delete 6d605146-428f-4feb-a0c8-b0a4a8733a2a

it should now be possible to delete the volume using the following commands

openstack volume delete 19186f06-f923-4416-89e3-f52f6c54570b
checking volumes/volume-1c68a40b-beae-45cb-9e0b-519301fb34af for snaps

image volumes/volume-1c68a40b-beae-45cb-9e0b-519301fb34af has a snapshot called snapshot-fd08c190-3022-4764-8722-8de79ec0969b and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show fd08c190-3022-4764-8722-8de79ec0969b
please check that this is a member of 1c68a40b-beae-45cb-9e0b-519301fb34af before running any of the following

openstack volume snapshot delete fd08c190-3022-4764-8722-8de79ec0969b

image volumes/volume-1c68a40b-beae-45cb-9e0b-519301fb34af has a snapshot called snapshot-3013ef2e-f749-4ff1-ae0c-ebf49a4deec3 and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show 3013ef2e-f749-4ff1-ae0c-ebf49a4deec3
please check that this is a member of 1c68a40b-beae-45cb-9e0b-519301fb34af before running any of the following

openstack volume snapshot delete 3013ef2e-f749-4ff1-ae0c-ebf49a4deec3

it should now be possible to delete the volume using the following commands

openstack volume delete 1c68a40b-beae-45cb-9e0b-519301fb34af
checking volumes/volume-1cbdf9bb-2f89-47a7-86d2-16d0739a8451 for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete 1cbdf9bb-2f89-47a7-86d2-16d0739a8451
checking volumes/volume-1dcf464a-7efd-4ad4-a774-367188f55448 for snaps

image volumes/volume-1dcf464a-7efd-4ad4-a774-367188f55448 has a snapshot called snapshot-7b0771a6-b06d-491e-86a0-940cd4383adb and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show 7b0771a6-b06d-491e-86a0-940cd4383adb
please check that this is a member of 1dcf464a-7efd-4ad4-a774-367188f55448 before running any of the following

openstack volume snapshot delete 7b0771a6-b06d-491e-86a0-940cd4383adb

image volumes/volume-1dcf464a-7efd-4ad4-a774-367188f55448 has a snapshot called snapshot-20864eb1-b532-422d-aa4b-bd40965842f4 and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show 20864eb1-b532-422d-aa4b-bd40965842f4
please check that this is a member of 1dcf464a-7efd-4ad4-a774-367188f55448 before running any of the following

openstack volume snapshot delete 20864eb1-b532-422d-aa4b-bd40965842f4

image volumes/volume-1dcf464a-7efd-4ad4-a774-367188f55448 has a snapshot called snapshot-f19c6d99-4217-43cf-9a93-b1b9b60206be and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show f19c6d99-4217-43cf-9a93-b1b9b60206be
please check that this is a member of 1dcf464a-7efd-4ad4-a774-367188f55448 before running any of the following

openstack volume snapshot delete f19c6d99-4217-43cf-9a93-b1b9b60206be

image volumes/volume-1dcf464a-7efd-4ad4-a774-367188f55448 has a snapshot called snapshot-33bd9242-9441-471f-a4a6-8d46f2127ae4 and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show 33bd9242-9441-471f-a4a6-8d46f2127ae4
please check that this is a member of 1dcf464a-7efd-4ad4-a774-367188f55448 before running any of the following

openstack volume snapshot delete 33bd9242-9441-471f-a4a6-8d46f2127ae4

it should now be possible to delete the volume using the following commands

openstack volume delete 1dcf464a-7efd-4ad4-a774-367188f55448
checking volumes/volume-1fc5a6f4-88ee-43bc-afdc-acb762bbec9d for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete 1fc5a6f4-88ee-43bc-afdc-acb762bbec9d
checking volumes/volume-26a1c410-3996-427f-8b93-69b003784dcd for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete 26a1c410-3996-427f-8b93-69b003784dcd
checking volumes/volume-29dacac9-93dc-408a-bf3f-327e8590acad for snaps

image volumes/volume-29dacac9-93dc-408a-bf3f-327e8590acad has a snapshot called snapshot-5d7cf008-d658-4d63-8f9a-39e754494038 and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show 5d7cf008-d658-4d63-8f9a-39e754494038
please check that this is a member of 29dacac9-93dc-408a-bf3f-327e8590acad before running any of the following

openstack volume snapshot delete 5d7cf008-d658-4d63-8f9a-39e754494038

image volumes/volume-29dacac9-93dc-408a-bf3f-327e8590acad has a snapshot called snapshot-7c9b41ab-ee7b-4be6-832a-7a773ac6710a and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show 7c9b41ab-ee7b-4be6-832a-7a773ac6710a
please check that this is a member of 29dacac9-93dc-408a-bf3f-327e8590acad before running any of the following

openstack volume snapshot delete 7c9b41ab-ee7b-4be6-832a-7a773ac6710a

it should now be possible to delete the volume using the following commands

openstack volume delete 29dacac9-93dc-408a-bf3f-327e8590acad
checking volumes/volume-2a7ffe62-54e7-45e4-a61e-b066d29acc50 for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete 2a7ffe62-54e7-45e4-a61e-b066d29acc50
checking volumes/volume-2dfb47ee-307e-4f85-b593-160ffecfbfe4 for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete 2dfb47ee-307e-4f85-b593-160ffecfbfe4
checking volumes/volume-321e3628-170f-42c9-9f37-66413250c441 for snaps

image volumes/volume-321e3628-170f-42c9-9f37-66413250c441 has a snapshot called snapshot-e52ae299-9eb6-4c72-85b8-691de1683f5c and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show e52ae299-9eb6-4c72-85b8-691de1683f5c
please check that this is a member of 321e3628-170f-42c9-9f37-66413250c441 before running any of the following

openstack volume snapshot delete e52ae299-9eb6-4c72-85b8-691de1683f5c

it should now be possible to delete the volume using the following commands

openstack volume delete 321e3628-170f-42c9-9f37-66413250c441
checking volumes/volume-3776f713-e1bd-40bd-be2b-18bd945385e6 for snaps

image volumes/volume-3776f713-e1bd-40bd-be2b-18bd945385e6 has a snapshot called snapshot-1396b706-51d0-40e2-b670-3b17b5c4dbe0 and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show 1396b706-51d0-40e2-b670-3b17b5c4dbe0
please check that this is a member of 3776f713-e1bd-40bd-be2b-18bd945385e6 before running any of the following

openstack volume snapshot delete 1396b706-51d0-40e2-b670-3b17b5c4dbe0

image volumes/volume-3776f713-e1bd-40bd-be2b-18bd945385e6 has a snapshot called snapshot-ab5416a7-ea1d-43a5-b084-6f5a2ea09bf5 and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show ab5416a7-ea1d-43a5-b084-6f5a2ea09bf5
please check that this is a member of 3776f713-e1bd-40bd-be2b-18bd945385e6 before running any of the following

openstack volume snapshot delete ab5416a7-ea1d-43a5-b084-6f5a2ea09bf5

image volumes/volume-3776f713-e1bd-40bd-be2b-18bd945385e6 has a snapshot called snapshot-435a8912-3cd4-4e93-b1b5-ef8056cf6bce and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show 435a8912-3cd4-4e93-b1b5-ef8056cf6bce
please check that this is a member of 3776f713-e1bd-40bd-be2b-18bd945385e6 before running any of the following

openstack volume snapshot delete 435a8912-3cd4-4e93-b1b5-ef8056cf6bce

image volumes/volume-3776f713-e1bd-40bd-be2b-18bd945385e6 has a snapshot called snapshot-f8cd87ae-afd4-4338-89d1-13a78eeab2b2 and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show f8cd87ae-afd4-4338-89d1-13a78eeab2b2
please check that this is a member of 3776f713-e1bd-40bd-be2b-18bd945385e6 before running any of the following

openstack volume snapshot delete f8cd87ae-afd4-4338-89d1-13a78eeab2b2

it should now be possible to delete the volume using the following commands

openstack volume delete 3776f713-e1bd-40bd-be2b-18bd945385e6
checking volumes/volume-39094eb4-bfc8-4c37-89c0-4c6326c2d7b4 for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete 39094eb4-bfc8-4c37-89c0-4c6326c2d7b4
checking volumes/volume-3a6e0d75-62ce-4d8f-8235-850c5c0db61d for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete 3a6e0d75-62ce-4d8f-8235-850c5c0db61d
checking volumes/volume-3f79a1dd-6cf8-4dea-b98f-b7597d3a5a6d for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete 3f79a1dd-6cf8-4dea-b98f-b7597d3a5a6d
checking volumes/volume-4054433b-03cf-4cb1-8948-c6ffe6e2955c for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete 4054433b-03cf-4cb1-8948-c6ffe6e2955c
checking volumes/volume-417feeef-d79d-4a31-af13-f1bee971284b for snaps

image volumes/volume-417feeef-d79d-4a31-af13-f1bee971284b has a snapshot called snapshot-de4787b8-ce8f-48ce-b13c-2854f1bbab2e and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show de4787b8-ce8f-48ce-b13c-2854f1bbab2e
please check that this is a member of 417feeef-d79d-4a31-af13-f1bee971284b before running any of the following

openstack volume snapshot delete de4787b8-ce8f-48ce-b13c-2854f1bbab2e

it should now be possible to delete the volume using the following commands

openstack volume delete 417feeef-d79d-4a31-af13-f1bee971284b
checking volumes/volume-467bb817-3c6c-4f44-b767-35109127cb7c for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete 467bb817-3c6c-4f44-b767-35109127cb7c
checking volumes/volume-4f80c812-cc7e-4b04-8148-ffd01ff69f14 for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete 4f80c812-cc7e-4b04-8148-ffd01ff69f14
checking volumes/volume-5408fbf0-46f6-41d9-937a-239be5055c4c for snaps

image volumes/volume-5408fbf0-46f6-41d9-937a-239be5055c4c has a snapshot called snapshot-e41485d1-c71d-45d3-9184-b40522e9e700 and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show e41485d1-c71d-45d3-9184-b40522e9e700
please check that this is a member of 5408fbf0-46f6-41d9-937a-239be5055c4c before running any of the following

openstack volume snapshot delete e41485d1-c71d-45d3-9184-b40522e9e700

it should now be possible to delete the volume using the following commands

openstack volume delete 5408fbf0-46f6-41d9-937a-239be5055c4c
checking volumes/volume-56bb6be4-b0b5-44ca-8bad-9e4b55ac922e.deleted for snaps

image volumes/volume-56bb6be4-b0b5-44ca-8bad-9e4b55ac922e.deleted has a snapshot called volume-3b794a23-c30f-44a8-a1c1-763ab4ee1635.clone_snap and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show 3b794a23-c30f-44a8-a1c1-763ab4ee1635.clone_snap
please check that this is a member of 56bb6be4-b0b5-44ca-8bad-9e4b55ac922e.deleted before running any of the following

openstack volume snapshot delete 3b794a23-c30f-44a8-a1c1-763ab4ee1635.clone_snap

it should now be possible to delete the volume using the following commands

openstack volume delete 56bb6be4-b0b5-44ca-8bad-9e4b55ac922e.deleted
checking volumes/volume-57590c5a-e645-40a0-9237-3d69aca8c020 for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete 57590c5a-e645-40a0-9237-3d69aca8c020
checking volumes/volume-5cbae0c2-7fa2-4e56-bc26-633c66aa5d73 for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete 5cbae0c2-7fa2-4e56-bc26-633c66aa5d73
checking volumes/volume-61192337-5f2f-400e-b04f-337ce502b326 for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete 61192337-5f2f-400e-b04f-337ce502b326
checking volumes/volume-639983da-294f-45ac-883f-b7ec3d0f4ff3 for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete 639983da-294f-45ac-883f-b7ec3d0f4ff3
checking volumes/volume-64187c05-3299-4a2e-ab1b-e527f483cb6c for snaps

image volumes/volume-64187c05-3299-4a2e-ab1b-e527f483cb6c has a snapshot called snapshot-b139b50d-4984-452b-8a04-6a3ae9ac13d2 and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show b139b50d-4984-452b-8a04-6a3ae9ac13d2
please check that this is a member of 64187c05-3299-4a2e-ab1b-e527f483cb6c before running any of the following

openstack volume snapshot delete b139b50d-4984-452b-8a04-6a3ae9ac13d2

image volumes/volume-64187c05-3299-4a2e-ab1b-e527f483cb6c has a snapshot called snapshot-734d71df-fa23-4145-ad91-c40004d63797 and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show 734d71df-fa23-4145-ad91-c40004d63797
please check that this is a member of 64187c05-3299-4a2e-ab1b-e527f483cb6c before running any of the following

openstack volume snapshot delete 734d71df-fa23-4145-ad91-c40004d63797

it should now be possible to delete the volume using the following commands

openstack volume delete 64187c05-3299-4a2e-ab1b-e527f483cb6c
checking volumes/volume-68fcd1c6-60d3-4e7d-a7e5-d13a8d7bd424 for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete 68fcd1c6-60d3-4e7d-a7e5-d13a8d7bd424
checking volumes/volume-6ef47ed4-a802-4e65-bb07-8c36b4a1fe5e for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete 6ef47ed4-a802-4e65-bb07-8c36b4a1fe5e
checking volumes/volume-6efd9e53-4cdd-4b99-84b2-114bd7e276c3 for snaps

image volumes/volume-6efd9e53-4cdd-4b99-84b2-114bd7e276c3 has a snapshot called snapshot-19c4346b-9ae1-489f-b992-d0186bbd8154 and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show 19c4346b-9ae1-489f-b992-d0186bbd8154
please check that this is a member of 6efd9e53-4cdd-4b99-84b2-114bd7e276c3 before running any of the following

openstack volume snapshot delete 19c4346b-9ae1-489f-b992-d0186bbd8154

it should now be possible to delete the volume using the following commands

openstack volume delete 6efd9e53-4cdd-4b99-84b2-114bd7e276c3
checking volumes/volume-70d3019c-8085-4bcf-b120-b20652fcc835 for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete 70d3019c-8085-4bcf-b120-b20652fcc835
checking volumes/volume-763547b6-db23-496f-b852-46f1126411d2 for snaps

image volumes/volume-763547b6-db23-496f-b852-46f1126411d2 has a snapshot called snapshot-7eea23cd-6ae4-419b-82d8-132b32f6cad8 and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show 7eea23cd-6ae4-419b-82d8-132b32f6cad8
please check that this is a member of 763547b6-db23-496f-b852-46f1126411d2 before running any of the following

openstack volume snapshot delete 7eea23cd-6ae4-419b-82d8-132b32f6cad8

it should now be possible to delete the volume using the following commands

openstack volume delete 763547b6-db23-496f-b852-46f1126411d2
checking volumes/volume-76860ded-c5ff-458c-b749-a07b3c8e03fd for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete 76860ded-c5ff-458c-b749-a07b3c8e03fd
checking volumes/volume-7894f6c4-c815-4132-896e-c4d5f41efdb9 for snaps

image volumes/volume-7894f6c4-c815-4132-896e-c4d5f41efdb9 has a snapshot called snapshot-001d43be-e82d-474b-bea4-1138ba325899 and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show 001d43be-e82d-474b-bea4-1138ba325899
please check that this is a member of 7894f6c4-c815-4132-896e-c4d5f41efdb9 before running any of the following

openstack volume snapshot delete 001d43be-e82d-474b-bea4-1138ba325899

it should now be possible to delete the volume using the following commands

openstack volume delete 7894f6c4-c815-4132-896e-c4d5f41efdb9
checking volumes/volume-79ccdd09-102d-42e6-bc6c-01b6a2d19892 for snaps

image volumes/volume-79ccdd09-102d-42e6-bc6c-01b6a2d19892 has a snapshot called snapshot-e8c076ef-6b66-485b-9706-f83413929eda and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show e8c076ef-6b66-485b-9706-f83413929eda
please check that this is a member of 79ccdd09-102d-42e6-bc6c-01b6a2d19892 before running any of the following

openstack volume snapshot delete e8c076ef-6b66-485b-9706-f83413929eda

it should now be possible to delete the volume using the following commands

openstack volume delete 79ccdd09-102d-42e6-bc6c-01b6a2d19892
checking volumes/volume-7aedaaa5-f547-401a-b4cb-1d2271eb1c7d for snaps

image volumes/volume-7aedaaa5-f547-401a-b4cb-1d2271eb1c7d has a snapshot called snapshot-509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824 and is protections state is true
checking volumes/volume-7aedaaa5-f547-401a-b4cb-1d2271eb1c7d@snapshot-509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824 for children

unprotecting volumes/volume-7aedaaa5-f547-401a-b4cb-1d2271eb1c7d@snapshot-509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824

rbd snap unprotect volumes/volume-7aedaaa5-f547-401a-b4cb-1d2271eb1c7d@snapshot-509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824
now possible to delete snapshot with the following commands:

openstack volume snapshot show 509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824
please check that this is a member of 7aedaaa5-f547-401a-b4cb-1d2271eb1c7d before running any of the following

openstack volume snapshot delete 509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824

it should now be possible to delete the volume using the following commands

openstack volume delete 7aedaaa5-f547-401a-b4cb-1d2271eb1c7d
checking volumes/volume-7cc5f04a-0ecf-4ddc-a2c4-5c0d3a0b5913 for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete 7cc5f04a-0ecf-4ddc-a2c4-5c0d3a0b5913
checking volumes/volume-85ecb198-c2de-4ecf-90b6-7f658085d5f5 for snaps

image volumes/volume-85ecb198-c2de-4ecf-90b6-7f658085d5f5 has a snapshot called snapshot-98cd07e4-6b5f-4b5f-8608-15e759791dfb and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show 98cd07e4-6b5f-4b5f-8608-15e759791dfb
please check that this is a member of 85ecb198-c2de-4ecf-90b6-7f658085d5f5 before running any of the following

openstack volume snapshot delete 98cd07e4-6b5f-4b5f-8608-15e759791dfb

it should now be possible to delete the volume using the following commands

openstack volume delete 85ecb198-c2de-4ecf-90b6-7f658085d5f5
checking volumes/volume-86056c2b-a904-4228-8dd1-c4cb01648f0f for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete 86056c2b-a904-4228-8dd1-c4cb01648f0f
checking volumes/volume-8889e3b6-5d6c-4bf6-b17c-434ffb95154f for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete 8889e3b6-5d6c-4bf6-b17c-434ffb95154f
checking volumes/volume-88f473eb-d2fa-4e37-82f2-54fd85c25d5e for snaps

image volumes/volume-88f473eb-d2fa-4e37-82f2-54fd85c25d5e has a snapshot called snapshot-3b355a39-d82a-45e0-95fd-49d447ae5a34 and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show 3b355a39-d82a-45e0-95fd-49d447ae5a34
please check that this is a member of 88f473eb-d2fa-4e37-82f2-54fd85c25d5e before running any of the following

openstack volume snapshot delete 3b355a39-d82a-45e0-95fd-49d447ae5a34

image volumes/volume-88f473eb-d2fa-4e37-82f2-54fd85c25d5e has a snapshot called snapshot-79c0a755-87e6-4b5f-bca1-da657e52331b and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show 79c0a755-87e6-4b5f-bca1-da657e52331b
please check that this is a member of 88f473eb-d2fa-4e37-82f2-54fd85c25d5e before running any of the following

openstack volume snapshot delete 79c0a755-87e6-4b5f-bca1-da657e52331b

it should now be possible to delete the volume using the following commands

openstack volume delete 88f473eb-d2fa-4e37-82f2-54fd85c25d5e
checking volumes/volume-8eb7069c-5c07-43c6-a91f-f1d79a071a91 for snaps

image volumes/volume-8eb7069c-5c07-43c6-a91f-f1d79a071a91 has a snapshot called snapshot-14b5f169-1fe1-43f1-aacf-0c4589e8c469 and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show 14b5f169-1fe1-43f1-aacf-0c4589e8c469
please check that this is a member of 8eb7069c-5c07-43c6-a91f-f1d79a071a91 before running any of the following

openstack volume snapshot delete 14b5f169-1fe1-43f1-aacf-0c4589e8c469

it should now be possible to delete the volume using the following commands

openstack volume delete 8eb7069c-5c07-43c6-a91f-f1d79a071a91
checking volumes/volume-92aca6e5-bf64-475a-a3c5-00cd941848b5 for snaps

image volumes/volume-92aca6e5-bf64-475a-a3c5-00cd941848b5 has a snapshot called snapshot-96fdbeab-81a4-465d-87ae-0916c0950a88 and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show 96fdbeab-81a4-465d-87ae-0916c0950a88
please check that this is a member of 92aca6e5-bf64-475a-a3c5-00cd941848b5 before running any of the following

openstack volume snapshot delete 96fdbeab-81a4-465d-87ae-0916c0950a88

it should now be possible to delete the volume using the following commands

openstack volume delete 92aca6e5-bf64-475a-a3c5-00cd941848b5
checking volumes/volume-93f22739-5ac8-4a88-8c49-de4c2234e7e7 for snaps

image volumes/volume-93f22739-5ac8-4a88-8c49-de4c2234e7e7 has a snapshot called snapshot-42d02435-04aa-42e5-9e92-a87161f50353 and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show 42d02435-04aa-42e5-9e92-a87161f50353
please check that this is a member of 93f22739-5ac8-4a88-8c49-de4c2234e7e7 before running any of the following

openstack volume snapshot delete 42d02435-04aa-42e5-9e92-a87161f50353

it should now be possible to delete the volume using the following commands

openstack volume delete 93f22739-5ac8-4a88-8c49-de4c2234e7e7
checking volumes/volume-94e97068-13e3-49a2-8bfe-4c4ba1d898fd for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete 94e97068-13e3-49a2-8bfe-4c4ba1d898fd
checking volumes/volume-9ce7b127-acae-4e94-9b21-079633f008bd for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete 9ce7b127-acae-4e94-9b21-079633f008bd
checking volumes/volume-9fb3978c-050d-4bfc-b9ce-9cd213994d96 for snaps

image volumes/volume-9fb3978c-050d-4bfc-b9ce-9cd213994d96 has a snapshot called snapshot-69d9e3b8-eed3-47e8-854f-21147fbce2da and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show 69d9e3b8-eed3-47e8-854f-21147fbce2da
please check that this is a member of 9fb3978c-050d-4bfc-b9ce-9cd213994d96 before running any of the following

openstack volume snapshot delete 69d9e3b8-eed3-47e8-854f-21147fbce2da

it should now be possible to delete the volume using the following commands

openstack volume delete 9fb3978c-050d-4bfc-b9ce-9cd213994d96
checking volumes/volume-a1468374-b1c3-4c26-a2da-c2cfeee1c27d for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete a1468374-b1c3-4c26-a2da-c2cfeee1c27d
checking volumes/volume-a3a39c50-5fa8-4b50-8093-d166eda187b9 for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete a3a39c50-5fa8-4b50-8093-d166eda187b9
checking volumes/volume-ae27171c-f11e-44e7-8928-e72efe9b45c6 for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete ae27171c-f11e-44e7-8928-e72efe9b45c6
checking volumes/volume-b1a50f3e-6c7c-4fa8-a1ae-1a488fc4d3b3 for snaps

image volumes/volume-b1a50f3e-6c7c-4fa8-a1ae-1a488fc4d3b3 has a snapshot called snapshot-7bb8db32-e5f3-4ae5-92a0-9373cc543bff and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show 7bb8db32-e5f3-4ae5-92a0-9373cc543bff
please check that this is a member of b1a50f3e-6c7c-4fa8-a1ae-1a488fc4d3b3 before running any of the following

openstack volume snapshot delete 7bb8db32-e5f3-4ae5-92a0-9373cc543bff

it should now be possible to delete the volume using the following commands

openstack volume delete b1a50f3e-6c7c-4fa8-a1ae-1a488fc4d3b3
checking volumes/volume-b9e1b8d6-f38e-4570-ac01-49a7309ea6ec for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete b9e1b8d6-f38e-4570-ac01-49a7309ea6ec
checking volumes/volume-c4d0ce91-8f23-40cf-9772-cca3c77ae672 for snaps

image volumes/volume-c4d0ce91-8f23-40cf-9772-cca3c77ae672 has a snapshot called snapshot-2df9ca3f-0340-4a4c-8995-9626627af7c5 and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show 2df9ca3f-0340-4a4c-8995-9626627af7c5
please check that this is a member of c4d0ce91-8f23-40cf-9772-cca3c77ae672 before running any of the following

openstack volume snapshot delete 2df9ca3f-0340-4a4c-8995-9626627af7c5

it should now be possible to delete the volume using the following commands

openstack volume delete c4d0ce91-8f23-40cf-9772-cca3c77ae672
checking volumes/volume-c89b51d0-49dc-4923-aaec-5e5da27ed799 for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete c89b51d0-49dc-4923-aaec-5e5da27ed799
checking volumes/volume-d2f33e0e-d7f5-4eae-8efb-27a71f335d12 for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete d2f33e0e-d7f5-4eae-8efb-27a71f335d12
checking volumes/volume-d4af50d5-2182-47b3-941c-997e74c9341c for snaps

image volumes/volume-d4af50d5-2182-47b3-941c-997e74c9341c has a snapshot called snapshot-91d7fb79-f5ae-4c3f-bb1c-068c7c5a1adc and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show 91d7fb79-f5ae-4c3f-bb1c-068c7c5a1adc
please check that this is a member of d4af50d5-2182-47b3-941c-997e74c9341c before running any of the following

openstack volume snapshot delete 91d7fb79-f5ae-4c3f-bb1c-068c7c5a1adc

it should now be possible to delete the volume using the following commands

openstack volume delete d4af50d5-2182-47b3-941c-997e74c9341c
checking volumes/volume-d698f7bf-fff5-450d-8ed6-2f62e4ca80cf for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete d698f7bf-fff5-450d-8ed6-2f62e4ca80cf
checking volumes/volume-dc275cb9-6709-4dc1-8e9d-5943a3b4964e for snaps

image volumes/volume-dc275cb9-6709-4dc1-8e9d-5943a3b4964e has a snapshot called snapshot-85e73527-63b5-4f9c-a66f-70356e11a490 and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show 85e73527-63b5-4f9c-a66f-70356e11a490
please check that this is a member of dc275cb9-6709-4dc1-8e9d-5943a3b4964e before running any of the following

openstack volume snapshot delete 85e73527-63b5-4f9c-a66f-70356e11a490

it should now be possible to delete the volume using the following commands

openstack volume delete dc275cb9-6709-4dc1-8e9d-5943a3b4964e
checking volumes/volume-dc48db05-56e2-448e-99cb-e07e65977b53 for snaps

image volumes/volume-dc48db05-56e2-448e-99cb-e07e65977b53 has a snapshot called snapshot-b9345b91-a5b5-4157-8838-4740b7e27788 and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show b9345b91-a5b5-4157-8838-4740b7e27788
please check that this is a member of dc48db05-56e2-448e-99cb-e07e65977b53 before running any of the following

openstack volume snapshot delete b9345b91-a5b5-4157-8838-4740b7e27788

it should now be possible to delete the volume using the following commands

openstack volume delete dc48db05-56e2-448e-99cb-e07e65977b53
checking volumes/volume-e1a43d3b-cdaf-475a-909c-fc54123bd87f for snaps

image volumes/volume-e1a43d3b-cdaf-475a-909c-fc54123bd87f has a snapshot called snapshot-0c867532-e1d9-429e-b402-b86db63f98c0 and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show 0c867532-e1d9-429e-b402-b86db63f98c0
please check that this is a member of e1a43d3b-cdaf-475a-909c-fc54123bd87f before running any of the following

openstack volume snapshot delete 0c867532-e1d9-429e-b402-b86db63f98c0

it should now be possible to delete the volume using the following commands

openstack volume delete e1a43d3b-cdaf-475a-909c-fc54123bd87f
checking volumes/volume-e50b1239-62da-4535-aa21-4e551c1eb2be for snaps

image volumes/volume-e50b1239-62da-4535-aa21-4e551c1eb2be has a snapshot called snapshot-0e47cfa0-481b-4f8e-b405-8fb5f74b927b and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show 0e47cfa0-481b-4f8e-b405-8fb5f74b927b
please check that this is a member of e50b1239-62da-4535-aa21-4e551c1eb2be before running any of the following

openstack volume snapshot delete 0e47cfa0-481b-4f8e-b405-8fb5f74b927b

image volumes/volume-e50b1239-62da-4535-aa21-4e551c1eb2be has a snapshot called snapshot-fc42e191-f086-487e-ba15-305830cac507 and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show fc42e191-f086-487e-ba15-305830cac507
please check that this is a member of e50b1239-62da-4535-aa21-4e551c1eb2be before running any of the following

openstack volume snapshot delete fc42e191-f086-487e-ba15-305830cac507

image volumes/volume-e50b1239-62da-4535-aa21-4e551c1eb2be has a snapshot called snapshot-7d89b23e-4620-4293-931a-1c47b11985bb and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show 7d89b23e-4620-4293-931a-1c47b11985bb
please check that this is a member of e50b1239-62da-4535-aa21-4e551c1eb2be before running any of the following

openstack volume snapshot delete 7d89b23e-4620-4293-931a-1c47b11985bb

it should now be possible to delete the volume using the following commands

openstack volume delete e50b1239-62da-4535-aa21-4e551c1eb2be
checking volumes/volume-e5d32199-8115-43a8-a8df-fa2f98442f96 for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete e5d32199-8115-43a8-a8df-fa2f98442f96
checking volumes/volume-eda73b06-05eb-4b3c-9280-0851873f145c for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete eda73b06-05eb-4b3c-9280-0851873f145c
checking volumes/volume-ef74477b-a345-4cc0-ae94-4b29b0375015 for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete ef74477b-a345-4cc0-ae94-4b29b0375015
checking volumes/volume-f035099d-a4c1-489d-9253-edebe3413fec for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete f035099d-a4c1-489d-9253-edebe3413fec
checking volumes/volume-f465b10e-5d01-4b5d-a786-51cc217220ee for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete f465b10e-5d01-4b5d-a786-51cc217220ee
checking volumes/volume-fa3e9113-7fcf-4ba5-b44e-201d406e6d04 for snaps

it should now be possible to delete the volume using the following commands

openstack volume delete fa3e9113-7fcf-4ba5-b44e-201d406e6d04
root@str-238:~# 




####### 
some quick manual checks first, lets cheack  a complex one with a snapshot and make sure we can rempove the snap and then the vol

lts pick this one:
###############################################################################
checking volumes/volume-0ab6766c-6cde-4a03-ad46-03009066783e for snaps

image volumes/volume-0ab6766c-6cde-4a03-ad46-03009066783e has a snapshot called snapshot-6a125870-d6ef-40af-a9c8-7c2c72ba0876 and is protections state is false
now possible to delete snapshot with the following commands:

openstack volume snapshot show 6a125870-d6ef-40af-a9c8-7c2c72ba0876
please check that this is a member of 0ab6766c-6cde-4a03-ad46-03009066783e before running any of the following

openstack volume snapshot delete 6a125870-d6ef-40af-a9c8-7c2c72ba0876

it should now be possible to delete the volume using the following commands

openstack volume delete 0ab6766c-6cde-4a03-ad46-03009066783e
###############################################################################

(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume snapshot show 6a125870-d6ef-40af-a9c8-7c2c72ba0876
+--------------------------------------------+--------------------------------------+
| Field                                      | Value                                |
+--------------------------------------------+--------------------------------------+
| created_at                                 | 2023-01-16T16:51:31.000000           |
| description                                |                                      |
| id                                         | 6a125870-d6ef-40af-a9c8-7c2c72ba0876 |
| name                                       | snapshot for ubuntu-with-biggpu-opts |
| os-extended-snapshot-attributes:progress   | 100%                                 |
| os-extended-snapshot-attributes:project_id | 6c8f3c23ab0649b0a10330f19ae04ab1     |
| properties                                 |                                      |
| size                                       | 40                                   |
| status                                     | available                            |
| updated_at                                 | 2023-01-16T16:51:32.000000           |
| volume_id                                  | 0ab6766c-6cde-4a03-ad46-03009066783e |
+--------------------------------------------+--------------------------------------+

it matches the volume it says it should
lets check that vol is on ceph (it will be)
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show  0ab6766c-6cde-4a03-ad46-03009066783e | grep 'os-vol-host-attr.host '
| os-vol-host-attr:host          | rbd:volumes@ceph-replicated#ceph-replicated                                                                                                                                                                                                                                                                                                                                                                                                                    |
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume snapshot delete 6a125870-d6ef-40af-a9c8-7c2c72ba0876
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume delete 0ab6766c-6cde-4a03-ad46-03009066783e
(kolla-ansible-zed) define@kolla-deploy:~$ openstack  volume show 0ab6766c-6cde-4a03-ad46-03009066783e
No volume with a name or ID of '0ab6766c-6cde-4a03-ad46-03009066783e' exists.

ok doing the lot






##################
added last layer of volume protection check vol is on rbd host before delete now we let rip


root@str-238:~# ./unprotect_and_flatten_snapshots.py
openstack volume show 0677c8e8-f4e7-42f4-8a57-7439b70e5653 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 0677c8e8-f4e7-42f4-8a57-7439b70e5653
openstack volume show 095849e0-b0b6-4ba6-bb00-0f0030a91baf -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 095849e0-b0b6-4ba6-bb00-0f0030a91baf
openstack volume show 0da62386-8eb4-4963-95ff-d9a2b561061a -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 0da62386-8eb4-4963-95ff-d9a2b561061a
openstack volume show 101cf309-5eab-45e4-9128-3572e107d237 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 500e0ae0-b6ce-4aba-8434-2b73bb7251a6

openstack volume show 101cf309-5eab-45e4-9128-3572e107d237 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete e68051c0-b3a0-4e65-8305-78f7aa3b2065

openstack volume show 101cf309-5eab-45e4-9128-3572e107d237 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete e0364942-1bb3-493b-a84e-a736b29fb847

openstack volume show 101cf309-5eab-45e4-9128-3572e107d237 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 101cf309-5eab-45e4-9128-3572e107d237
openstack volume show 125a7496-fdcd-4979-8dae-f342ee1f28a0 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 125a7496-fdcd-4979-8dae-f342ee1f28a0
openstack volume show 16b5ad27-ecf5-46e7-81ef-b79af5618dd0.deleted -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 34daba88-c273-4b3b-8954-6ad6832bdd21.clone_snap

openstack volume show 16b5ad27-ecf5-46e7-81ef-b79af5618dd0.deleted -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 16b5ad27-ecf5-46e7-81ef-b79af5618dd0.deleted
openstack volume show 19186f06-f923-4416-89e3-f52f6c54570b -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 6d605146-428f-4feb-a0c8-b0a4a8733a2a

openstack volume show 19186f06-f923-4416-89e3-f52f6c54570b -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 19186f06-f923-4416-89e3-f52f6c54570b
openstack volume show 1c68a40b-beae-45cb-9e0b-519301fb34af -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete fd08c190-3022-4764-8722-8de79ec0969b

openstack volume show 1c68a40b-beae-45cb-9e0b-519301fb34af -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 3013ef2e-f749-4ff1-ae0c-ebf49a4deec3

openstack volume show 1c68a40b-beae-45cb-9e0b-519301fb34af -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 1c68a40b-beae-45cb-9e0b-519301fb34af
openstack volume show 1cbdf9bb-2f89-47a7-86d2-16d0739a8451 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 1cbdf9bb-2f89-47a7-86d2-16d0739a8451
openstack volume show 1dcf464a-7efd-4ad4-a774-367188f55448 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 7b0771a6-b06d-491e-86a0-940cd4383adb

openstack volume show 1dcf464a-7efd-4ad4-a774-367188f55448 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 20864eb1-b532-422d-aa4b-bd40965842f4

openstack volume show 1dcf464a-7efd-4ad4-a774-367188f55448 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete f19c6d99-4217-43cf-9a93-b1b9b60206be

openstack volume show 1dcf464a-7efd-4ad4-a774-367188f55448 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 33bd9242-9441-471f-a4a6-8d46f2127ae4

openstack volume show 1dcf464a-7efd-4ad4-a774-367188f55448 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 1dcf464a-7efd-4ad4-a774-367188f55448
openstack volume show 1fc5a6f4-88ee-43bc-afdc-acb762bbec9d -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 1fc5a6f4-88ee-43bc-afdc-acb762bbec9d
openstack volume show 26a1c410-3996-427f-8b93-69b003784dcd -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 26a1c410-3996-427f-8b93-69b003784dcd
openstack volume show 29dacac9-93dc-408a-bf3f-327e8590acad -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 5d7cf008-d658-4d63-8f9a-39e754494038

openstack volume show 29dacac9-93dc-408a-bf3f-327e8590acad -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 7c9b41ab-ee7b-4be6-832a-7a773ac6710a

openstack volume show 29dacac9-93dc-408a-bf3f-327e8590acad -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 29dacac9-93dc-408a-bf3f-327e8590acad
openstack volume show 2a7ffe62-54e7-45e4-a61e-b066d29acc50 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 2a7ffe62-54e7-45e4-a61e-b066d29acc50
openstack volume show 2dfb47ee-307e-4f85-b593-160ffecfbfe4 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 2dfb47ee-307e-4f85-b593-160ffecfbfe4
openstack volume show 321e3628-170f-42c9-9f37-66413250c441 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete e52ae299-9eb6-4c72-85b8-691de1683f5c

openstack volume show 321e3628-170f-42c9-9f37-66413250c441 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 321e3628-170f-42c9-9f37-66413250c441
openstack volume show 3776f713-e1bd-40bd-be2b-18bd945385e6 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 1396b706-51d0-40e2-b670-3b17b5c4dbe0

openstack volume show 3776f713-e1bd-40bd-be2b-18bd945385e6 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete ab5416a7-ea1d-43a5-b084-6f5a2ea09bf5

openstack volume show 3776f713-e1bd-40bd-be2b-18bd945385e6 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 435a8912-3cd4-4e93-b1b5-ef8056cf6bce

openstack volume show 3776f713-e1bd-40bd-be2b-18bd945385e6 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete f8cd87ae-afd4-4338-89d1-13a78eeab2b2

openstack volume show 3776f713-e1bd-40bd-be2b-18bd945385e6 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 3776f713-e1bd-40bd-be2b-18bd945385e6
openstack volume show 39094eb4-bfc8-4c37-89c0-4c6326c2d7b4 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 39094eb4-bfc8-4c37-89c0-4c6326c2d7b4
openstack volume show 3a6e0d75-62ce-4d8f-8235-850c5c0db61d -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 3a6e0d75-62ce-4d8f-8235-850c5c0db61d
openstack volume show 3f79a1dd-6cf8-4dea-b98f-b7597d3a5a6d -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 3f79a1dd-6cf8-4dea-b98f-b7597d3a5a6d
openstack volume show 4054433b-03cf-4cb1-8948-c6ffe6e2955c -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 4054433b-03cf-4cb1-8948-c6ffe6e2955c
openstack volume show 417feeef-d79d-4a31-af13-f1bee971284b -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete de4787b8-ce8f-48ce-b13c-2854f1bbab2e

openstack volume show 417feeef-d79d-4a31-af13-f1bee971284b -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 417feeef-d79d-4a31-af13-f1bee971284b
openstack volume show 467bb817-3c6c-4f44-b767-35109127cb7c -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 467bb817-3c6c-4f44-b767-35109127cb7c
openstack volume show 4f80c812-cc7e-4b04-8148-ffd01ff69f14 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 4f80c812-cc7e-4b04-8148-ffd01ff69f14
openstack volume show 5408fbf0-46f6-41d9-937a-239be5055c4c -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete e41485d1-c71d-45d3-9184-b40522e9e700

openstack volume show 5408fbf0-46f6-41d9-937a-239be5055c4c -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 5408fbf0-46f6-41d9-937a-239be5055c4c
openstack volume show 56bb6be4-b0b5-44ca-8bad-9e4b55ac922e.deleted -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 3b794a23-c30f-44a8-a1c1-763ab4ee1635.clone_snap

openstack volume show 56bb6be4-b0b5-44ca-8bad-9e4b55ac922e.deleted -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 56bb6be4-b0b5-44ca-8bad-9e4b55ac922e.deleted
openstack volume show 57590c5a-e645-40a0-9237-3d69aca8c020 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 57590c5a-e645-40a0-9237-3d69aca8c020
openstack volume show 5cbae0c2-7fa2-4e56-bc26-633c66aa5d73 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 5cbae0c2-7fa2-4e56-bc26-633c66aa5d73
openstack volume show 61192337-5f2f-400e-b04f-337ce502b326 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 61192337-5f2f-400e-b04f-337ce502b326
openstack volume show 639983da-294f-45ac-883f-b7ec3d0f4ff3 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 639983da-294f-45ac-883f-b7ec3d0f4ff3
openstack volume show 64187c05-3299-4a2e-ab1b-e527f483cb6c -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete b139b50d-4984-452b-8a04-6a3ae9ac13d2

openstack volume show 64187c05-3299-4a2e-ab1b-e527f483cb6c -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 734d71df-fa23-4145-ad91-c40004d63797

openstack volume show 64187c05-3299-4a2e-ab1b-e527f483cb6c -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 64187c05-3299-4a2e-ab1b-e527f483cb6c
openstack volume show 68fcd1c6-60d3-4e7d-a7e5-d13a8d7bd424 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 68fcd1c6-60d3-4e7d-a7e5-d13a8d7bd424
openstack volume show 6ef47ed4-a802-4e65-bb07-8c36b4a1fe5e -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 6ef47ed4-a802-4e65-bb07-8c36b4a1fe5e
openstack volume show 6efd9e53-4cdd-4b99-84b2-114bd7e276c3 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 19c4346b-9ae1-489f-b992-d0186bbd8154

openstack volume show 6efd9e53-4cdd-4b99-84b2-114bd7e276c3 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 6efd9e53-4cdd-4b99-84b2-114bd7e276c3
openstack volume show 70d3019c-8085-4bcf-b120-b20652fcc835 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 70d3019c-8085-4bcf-b120-b20652fcc835
openstack volume show 763547b6-db23-496f-b852-46f1126411d2 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 7eea23cd-6ae4-419b-82d8-132b32f6cad8

openstack volume show 763547b6-db23-496f-b852-46f1126411d2 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 763547b6-db23-496f-b852-46f1126411d2
openstack volume show 76860ded-c5ff-458c-b749-a07b3c8e03fd -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 76860ded-c5ff-458c-b749-a07b3c8e03fd
openstack volume show 7894f6c4-c815-4132-896e-c4d5f41efdb9 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 001d43be-e82d-474b-bea4-1138ba325899

openstack volume show 7894f6c4-c815-4132-896e-c4d5f41efdb9 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 7894f6c4-c815-4132-896e-c4d5f41efdb9
openstack volume show 79ccdd09-102d-42e6-bc6c-01b6a2d19892 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete e8c076ef-6b66-485b-9706-f83413929eda

openstack volume show 79ccdd09-102d-42e6-bc6c-01b6a2d19892 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 79ccdd09-102d-42e6-bc6c-01b6a2d19892
rbd snap unprotect volumes/volume-7aedaaa5-f547-401a-b4cb-1d2271eb1c7d@snapshot-509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824
openstack volume show 7aedaaa5-f547-401a-b4cb-1d2271eb1c7d -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824

openstack volume show 7aedaaa5-f547-401a-b4cb-1d2271eb1c7d -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 7aedaaa5-f547-401a-b4cb-1d2271eb1c7d
openstack volume show 7cc5f04a-0ecf-4ddc-a2c4-5c0d3a0b5913 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 7cc5f04a-0ecf-4ddc-a2c4-5c0d3a0b5913
openstack volume show 85ecb198-c2de-4ecf-90b6-7f658085d5f5 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 98cd07e4-6b5f-4b5f-8608-15e759791dfb

openstack volume show 85ecb198-c2de-4ecf-90b6-7f658085d5f5 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 85ecb198-c2de-4ecf-90b6-7f658085d5f5
openstack volume show 86056c2b-a904-4228-8dd1-c4cb01648f0f -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 86056c2b-a904-4228-8dd1-c4cb01648f0f
openstack volume show 8889e3b6-5d6c-4bf6-b17c-434ffb95154f -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 8889e3b6-5d6c-4bf6-b17c-434ffb95154f
openstack volume show 88f473eb-d2fa-4e37-82f2-54fd85c25d5e -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 3b355a39-d82a-45e0-95fd-49d447ae5a34

openstack volume show 88f473eb-d2fa-4e37-82f2-54fd85c25d5e -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 79c0a755-87e6-4b5f-bca1-da657e52331b

openstack volume show 88f473eb-d2fa-4e37-82f2-54fd85c25d5e -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 88f473eb-d2fa-4e37-82f2-54fd85c25d5e
openstack volume show 8eb7069c-5c07-43c6-a91f-f1d79a071a91 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 14b5f169-1fe1-43f1-aacf-0c4589e8c469

openstack volume show 8eb7069c-5c07-43c6-a91f-f1d79a071a91 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 8eb7069c-5c07-43c6-a91f-f1d79a071a91
openstack volume show 92aca6e5-bf64-475a-a3c5-00cd941848b5 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 96fdbeab-81a4-465d-87ae-0916c0950a88

openstack volume show 92aca6e5-bf64-475a-a3c5-00cd941848b5 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 92aca6e5-bf64-475a-a3c5-00cd941848b5
openstack volume show 93f22739-5ac8-4a88-8c49-de4c2234e7e7 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 42d02435-04aa-42e5-9e92-a87161f50353

openstack volume show 93f22739-5ac8-4a88-8c49-de4c2234e7e7 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 93f22739-5ac8-4a88-8c49-de4c2234e7e7
openstack volume show 94e97068-13e3-49a2-8bfe-4c4ba1d898fd -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 94e97068-13e3-49a2-8bfe-4c4ba1d898fd
openstack volume show 9ce7b127-acae-4e94-9b21-079633f008bd -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 9ce7b127-acae-4e94-9b21-079633f008bd
openstack volume show 9fb3978c-050d-4bfc-b9ce-9cd213994d96 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 69d9e3b8-eed3-47e8-854f-21147fbce2da

openstack volume show 9fb3978c-050d-4bfc-b9ce-9cd213994d96 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 9fb3978c-050d-4bfc-b9ce-9cd213994d96
openstack volume show a1468374-b1c3-4c26-a2da-c2cfeee1c27d -c os-vol-host-attr:host | grep -e rbd && openstack volume delete a1468374-b1c3-4c26-a2da-c2cfeee1c27d
openstack volume show a3a39c50-5fa8-4b50-8093-d166eda187b9 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete a3a39c50-5fa8-4b50-8093-d166eda187b9
openstack volume show ae27171c-f11e-44e7-8928-e72efe9b45c6 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete ae27171c-f11e-44e7-8928-e72efe9b45c6
openstack volume show b1a50f3e-6c7c-4fa8-a1ae-1a488fc4d3b3 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 7bb8db32-e5f3-4ae5-92a0-9373cc543bff

openstack volume show b1a50f3e-6c7c-4fa8-a1ae-1a488fc4d3b3 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete b1a50f3e-6c7c-4fa8-a1ae-1a488fc4d3b3
openstack volume show b9e1b8d6-f38e-4570-ac01-49a7309ea6ec -c os-vol-host-attr:host | grep -e rbd && openstack volume delete b9e1b8d6-f38e-4570-ac01-49a7309ea6ec
openstack volume show c4d0ce91-8f23-40cf-9772-cca3c77ae672 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 2df9ca3f-0340-4a4c-8995-9626627af7c5

openstack volume show c4d0ce91-8f23-40cf-9772-cca3c77ae672 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete c4d0ce91-8f23-40cf-9772-cca3c77ae672
openstack volume show c89b51d0-49dc-4923-aaec-5e5da27ed799 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete c89b51d0-49dc-4923-aaec-5e5da27ed799
openstack volume show d2f33e0e-d7f5-4eae-8efb-27a71f335d12 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete d2f33e0e-d7f5-4eae-8efb-27a71f335d12
openstack volume show d4af50d5-2182-47b3-941c-997e74c9341c -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 91d7fb79-f5ae-4c3f-bb1c-068c7c5a1adc

openstack volume show d4af50d5-2182-47b3-941c-997e74c9341c -c os-vol-host-attr:host | grep -e rbd && openstack volume delete d4af50d5-2182-47b3-941c-997e74c9341c
openstack volume show d698f7bf-fff5-450d-8ed6-2f62e4ca80cf -c os-vol-host-attr:host | grep -e rbd && openstack volume delete d698f7bf-fff5-450d-8ed6-2f62e4ca80cf
openstack volume show dc275cb9-6709-4dc1-8e9d-5943a3b4964e -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 85e73527-63b5-4f9c-a66f-70356e11a490

openstack volume show dc275cb9-6709-4dc1-8e9d-5943a3b4964e -c os-vol-host-attr:host | grep -e rbd && openstack volume delete dc275cb9-6709-4dc1-8e9d-5943a3b4964e
openstack volume show dc48db05-56e2-448e-99cb-e07e65977b53 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete b9345b91-a5b5-4157-8838-4740b7e27788

openstack volume show dc48db05-56e2-448e-99cb-e07e65977b53 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete dc48db05-56e2-448e-99cb-e07e65977b53
openstack volume show e1a43d3b-cdaf-475a-909c-fc54123bd87f -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 0c867532-e1d9-429e-b402-b86db63f98c0

openstack volume show e1a43d3b-cdaf-475a-909c-fc54123bd87f -c os-vol-host-attr:host | grep -e rbd && openstack volume delete e1a43d3b-cdaf-475a-909c-fc54123bd87f
openstack volume show e50b1239-62da-4535-aa21-4e551c1eb2be -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 0e47cfa0-481b-4f8e-b405-8fb5f74b927b

openstack volume show e50b1239-62da-4535-aa21-4e551c1eb2be -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete fc42e191-f086-487e-ba15-305830cac507

openstack volume show e50b1239-62da-4535-aa21-4e551c1eb2be -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 7d89b23e-4620-4293-931a-1c47b11985bb

openstack volume show e50b1239-62da-4535-aa21-4e551c1eb2be -c os-vol-host-attr:host | grep -e rbd && openstack volume delete e50b1239-62da-4535-aa21-4e551c1eb2be
openstack volume show e5d32199-8115-43a8-a8df-fa2f98442f96 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete e5d32199-8115-43a8-a8df-fa2f98442f96
openstack volume show eda73b06-05eb-4b3c-9280-0851873f145c -c os-vol-host-attr:host | grep -e rbd && openstack volume delete eda73b06-05eb-4b3c-9280-0851873f145c
openstack volume show ef74477b-a345-4cc0-ae94-4b29b0375015 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete ef74477b-a345-4cc0-ae94-4b29b0375015
openstack volume show f035099d-a4c1-489d-9253-edebe3413fec -c os-vol-host-attr:host | grep -e rbd && openstack volume delete f035099d-a4c1-489d-9253-edebe3413fec
openstack volume show f465b10e-5d01-4b5d-a786-51cc217220ee -c os-vol-host-attr:host | grep -e rbd && openstack volume delete f465b10e-5d01-4b5d-a786-51cc217220ee
openstack volume show fa3e9113-7fcf-4ba5-b44e-201d406e6d04 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete fa3e9113-7fcf-4ba5-b44e-201d406e6d04




DO IT

(kolla-ansible-zed) define@kolla-deploy:~$
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show 7aedaaa5-f547-401a-b4cb-1d2271eb1c7d -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 7aedaaa5-f547-401a-b4cb-1d2271eb1c7d
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
Failed to delete volume with name or ID '7aedaaa5-f547-401a-b4cb-1d2271eb1c7d': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-9fd22892-043f-48d1-a5ec-9530f0e83ed0)
1 of 1 volumes failed to delete.
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show 7cc5f04a-0ecf-4ddc-a2c4-5c0d3a0b5913 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 7cc5f04a-0ecf-4ddc-a2c4-5c0d3a0b5913
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show 85ecb198-c2de-4ecf-90b6-7f658085d5f5 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 98cd07e4-6b5f-4b5f-8608-15e759791dfb
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show 85ecb198-c2de-4ecf-90b6-7f658085d5f5 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 85ecb198-c2de-4ecf-90b6-7f658085d5f5
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show 86056c2b-a904-4228-8dd1-c4cb01648f0f -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 86056c2b-a904-4228-8dd1-c4cb01648f0f
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show 8889e3b6-5d6c-4bf6-b17c-434ffb95154f -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 8889e3b6-5d6c-4bf6-b17c-434ffb95154f
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show 88f473eb-d2fa-4e37-82f2-54fd85c25d5e -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 3b355a39-d82a-45e0-95fd-49d447ae5a34
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show 88f473eb-d2fa-4e37-82f2-54fd85c25d5e -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 79c0a755-87e6-4b5f-bca1-da657e52331b
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show 88f473eb-d2fa-4e37-82f2-54fd85c25d5e -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 88f473eb-d2fa-4e37-82f2-54fd85c25d5e
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show 8eb7069c-5c07-43c6-a91f-f1d79a071a91 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 14b5f169-1fe1-43f1-aacf-0c4589e8c469
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show 8eb7069c-5c07-43c6-a91f-f1d79a071a91 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 8eb7069c-5c07-43c6-a91f-f1d79a071a91
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show 92aca6e5-bf64-475a-a3c5-00cd941848b5 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 96fdbeab-81a4-465d-87ae-0916c0950a88
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show 92aca6e5-bf64-475a-a3c5-00cd941848b5 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 92aca6e5-bf64-475a-a3c5-00cd941848b5
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
Failed to delete volume with name or ID '92aca6e5-bf64-475a-a3c5-00cd941848b5': Invalid volume: Volume status must be available or error or error_restoring or error_extending or error_managing and must not be migrating, attached, belong to a group, have snapshots, awaiting a transfer, or be disassociated from snapshots after volume transfer. (HTTP 400) (Request-ID: req-76ab0c88-7a9b-440e-bd27-9a1386766ad4)
1 of 1 volumes failed to delete.
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show 93f22739-5ac8-4a88-8c49-de4c2234e7e7 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 42d02435-04aa-42e5-9e92-a87161f50353
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show 93f22739-5ac8-4a88-8c49-de4c2234e7e7 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 93f22739-5ac8-4a88-8c49-de4c2234e7e7
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show 94e97068-13e3-49a2-8bfe-4c4ba1d898fd -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 94e97068-13e3-49a2-8bfe-4c4ba1d898fd
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show 9ce7b127-acae-4e94-9b21-079633f008bd -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 9ce7b127-acae-4e94-9b21-079633f008bd
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show 9fb3978c-050d-4bfc-b9ce-9cd213994d96 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 69d9e3b8-eed3-47e8-854f-21147fbce2da
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$
e(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show 9fb3978c-050d-4bfc-b9ce-9cd213994d96 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 9fb3978c-050d-4bfc-b9ce-9cd213994d96
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show a1468374-b1c3-4c26-a2da-c2cfeee1c27d -c os-vol-host-attr:host | grep -e rbd && openstack volume delete a1468374-b1c3-4c26-a2da-c2cfeee1c27d
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show a3a39c50-5fa8-4b50-8093-d166eda187b9 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete a3a39c50-5fa8-4b50-8093-d166eda187b9
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show ae27171c-f11e-44e7-8928-e72efe9b45c6 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete ae27171c-f11e-44e7-8928-e72efe9b45c6
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show b1a50f3e-6c7c-4fa8-a1ae-1a488fc4d3b3 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 7bb8db32-e5f3-4ae5-92a0-9373cc543bff
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show b1a50f3e-6c7c-4fa8-a1ae-1a488fc4d3b3 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete b1a50f3e-6c7c-4fa8-a1ae-1a488fc4d3b3
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show b9e1b8d6-f38e-4570-ac01-49a7309ea6ec -c os-vol-host-attr:host | grep -e rbd && openstack volume delete b9e1b8d6-f38e-4570-ac01-49a7309ea6ec
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show c4d0ce91-8f23-40cf-9772-cca3c77ae672 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 2df9ca3f-0340-4a4c-8995-9626627af7c5
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show c4d0ce91-8f23-40cf-9772-cca3c77ae672 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete c4d0ce91-8f23-40cf-9772-cca3c77ae672
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show c89b51d0-49dc-4923-aaec-5e5da27ed799 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete c89b51d0-49dc-4923-aaec-5e5da27ed799
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show d2f33e0e-d7f5-4eae-8efb-27a71f335d12 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete d2f33e0e-d7f5-4eae-8efb-27a71f335d12
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show d4af50d5-2182-47b3-941c-997e74c9341c -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 91d7fb79-f5ae-4c3f-bb1c-068c7c5a1adc
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show d4af50d5-2182-47b3-941c-997e74c9341c -c os-vol-host-attr:host | grep -e rbd && openstack volume delete d4af50d5-2182-47b3-941c-997e74c9341c
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show d698f7bf-fff5-450d-8ed6-2f62e4ca80cf -c os-vol-host-attr:host | grep -e rbd && openstack volume delete d698f7bf-fff5-450d-8ed6-2f62e4ca80cf
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show dc275cb9-6709-4dc1-8e9d-5943a3b4964e -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 85e73527-63b5-4f9c-a66f-70356e11a490
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show dc275cb9-6709-4dc1-8e9d-5943a3b4964e -c os-vol-host-attr:host | grep -e rbd && openstack volume delete dc275cb9-6709-4dc1-8e9d-5943a3b4964e
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show dc48db05-56e2-448e-99cb-e07e65977b53 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete b9345b91-a5b5-4157-8838-4740b7e27788
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show dc48db05-56e2-448e-99cb-e07e65977b53 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete dc48db05-56e2-448e-99cb-e07e65977b53
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show e1a43d3b-cdaf-475a-909c-fc54123bd87f -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 0c867532-e1d9-429e-b402-b86db63f98c0
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show e1a43d3b-cdaf-475a-909c-fc54123bd87f -c os-vol-host-attr:host | grep -e rbd && openstack volume delete e1a43d3b-cdaf-475a-909c-fc54123bd87f
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show e50b1239-62da-4535-aa21-4e551c1eb2be -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 0e47cfa0-481b-4f8e-b405-8fb5f74b927b
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show e50b1239-62da-4535-aa21-4e551c1eb2be -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete fc42e191-f086-487e-ba15-305830cac507
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show e50b1239-62da-4535-aa21-4e551c1eb2be -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 7d89b23e-4620-4293-931a-1c47b11985bb
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show e50b1239-62da-4535-aa21-4e551c1eb2be -c os-vol-host-attr:host | grep -e rbd && openstack volume delete e50b1239-62da-4535-aa21-4e551c1eb2be
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show e5d32199-8115-43a8-a8df-fa2f98442f96 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete e5d32199-8115-43a8-a8df-fa2f98442f96
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show eda73b06-05eb-4b3c-9280-0851873f145c -c os-vol-host-attr:host | grep -e rbd && openstack volume delete eda73b06-05eb-4b3c-9280-0851873f145c
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show ef74477b-a345-4cc0-ae94-4b29b0375015 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete ef74477b-a345-4cc0-ae94-4b29b0375015
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show f035099d-a4c1-489d-9253-edebe3413fec -c os-vol-host-attr:host | grep -e rbd && openstack volume delete f035099d-a4c1-489d-9253-edebe3413fec
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show f465b10e-5d01-4b5d-a786-51cc217220ee -c os-vol-host-attr:host | grep -e rbd && openstack volume delete f465b10e-5d01-4b5d-a786-51cc217220ee
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$ openstack volume show fa3e9113-7fcf-4ba5-b44e-201d406e6d04 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete fa3e9113-7fcf-4ba5-b44e-201d406e6d04
| os-vol-host-attr:host | rbd:volumes@ceph-replicated#ceph-replicated |
(kolla-ansible-zed) define@kolla-deploy:~$


some manual cleanup required to remove last 7 or so volumes with snapshots dependents in trash and then fix the volume stuck in migrating state


  309  vi unprotect_and_flatten_snapshots.py
  310  ./unprotect_and_flatten_snapshots.py
  311  ceph -s
  312  rbd ls volumes
  313  ceph -s
  314  rbd ls volumes
  315  rbd delete volums/volume-16b5ad27-ecf5-46e7-81ef-b79af5618dd0.deleted
  316  rbd mv volums/volume-16b5ad27-ecf5-46e7-81ef-b79af5618dd0.deleted
  317  rbd rm volumes/volume-16b5ad27-ecf5-46e7-81ef-b79af5618dd0.deleted
  318  rbd snap ls volumes/volume-16b5ad27-ecf5-46e7-81ef-b79af5618dd0.deleted
  319  rbd snap purge volumes/volume-16b5ad27-ecf5-46e7-81ef-b79af5618dd0.deleted
  320  rbd rm volumes/volume-16b5ad27-ecf5-46e7-81ef-b79af5618dd0.deleted
  321  rbd ls volumes
  322  rbd rm volume-56bb6be4-b0b5-44ca-8bad-9e4b55ac922e.deleted
  323  rbd rm volumes/volume-56bb6be4-b0b5-44ca-8bad-9e4b55ac922e.deleted
  324  rbd snap ls volumes/volume-56bb6be4-b0b5-44ca-8bad-9e4b55ac922e.deleted
  325  rbd snap purge volumes/volume-56bb6be4-b0b5-44ca-8bad-9e4b55ac922e.deleted
  326  rbd rm volumes/volume-56bb6be4-b0b5-44ca-8bad-9e4b55ac922e.deleted
  327  rbd ls volumes
  328  for i in $(rbd ls volumes); rbd snap ls volumes/$i;done
  329  for i in $(rbd ls volumes); do rbd snap ls volumes/$i;done
  330  for i in $(rbd ls volumes); do echo $i; rbd snap ls volumes/$i;done
  331  rbd children volume-7aedaaa5-f547-401a-b4cb-1d2271eb1c7d
  332  rbd children volumes/volume-7aedaaa5-f547-401a-b4cb-1d2271eb1c7d
  333  rbd children volumes/snapshot-96fdbeab-81a4-465d-87ae-0916c0950a88
  334  rbd children volumes/volume-92aca6e5-bf64-475a-a3c5-00cd941848b5
  335  rbd flatten volumes/volume-7aedaaa5-f547-401a-b4cb-1d2271eb1c7d@snapshot-509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824
  336  ceph -s
  337  rbd ls volumes
  338  ceph -s
  339  rbd ls volumes
  340  for i in $(rbd ls volumes); do echo $i; rbd lock list volumes/$i ;done
  341  rbd lock rm volumes/volume-39094eb4-bfc8-4c37-89c0-4c6326c2d7b4 "auto 139690858516416" client.6055435
  342  rbd lock rm volumes/volume-4f80c812-cc7e-4b04-8148-ffd01ff69f14 "auto 139692603504608" client.6055423
  343  rbd lock rm volumes/volume-92aca6e5-bf64-475a-a3c5-00cd941848b5 "auto 139648110593536" client.2073145
  344  rbd lock rm volumes/volume-ef74477b-a345-4cc0-ae94-4b29b0375015 "auto 140712859602272" client.5502657
  345  rbd lock rm volumes/volume-fa3e9113-7fcf-4ba5-b44e-201d406e6d04 "auto 140214039568736" client.143671
  346  for i in $(rbd ls volumes); do echo $i; rbd lock list volumes/$i ;done
  347  for i in $(rbd ls volumes); do echo $i; rbd snap ls volumes/$i ;done
  348  rbd ls volumes
  349  for i in $(rbd ls volumes); do echo $i; rbd snap ls volumes/$i ;done
  350  rbd snap unprotect volumes/volume-7aedaaa5-f547-401a-b4cb-1d2271eb1c7d@snapshot-509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824
  351  rbd snap shildren volumes/volume-7aedaaa5-f547-401a-b4cb-1d2271eb1c7d@snapshot-509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824
  352  rbd snap children volumes/volume-7aedaaa5-f547-401a-b4cb-1d2271eb1c7d@snapshot-509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824
  353  rbd children volumes/volume-7aedaaa5-f547-401a-b4cb-1d2271eb1c7d@snapshot-509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824
  354  rbd snap unprotect volumes/volume-7aedaaa5-f547-401a-b4cb-1d2271eb1c7d@snapshot-509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824
  355  for i in $(rbd ls volumes); do echo $i; rbd snap ls volumes/$i ;done
  356  rbd ls volumes
  357  for i in $(rbd ls volumes); do echo $i; rbd snap ls volumes/$i ;done
  358  rbd ls volumes
  359  for i in $(rbd ls volumes); do echo $i; rbd snap ls volumes/$i ;done
  360  rbd ls volumes
  361  for i in $(rbd ls volumes); do echo $i; rbd snap ls volumes/$i ;done
  362  rbd ls images
  363  for i in $(rbd ls volumes); do echo $i; rbd snap ls volumes/$i ;done
  364  rbd snap unprotect volumes/volume-7aedaaa5-f547-401a-b4cb-1d2271eb1c7d@snapshot-509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824
  365  rbd ls volumes
  366  rbd snap unprotect volumes/volume-7aedaaa5-f547-401a-b4cb-1d2271eb1c7d@snapshot-509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824
  367  rbd ceph -s
  368  ceph -s
  369  rbd ls volumes
  370  rbd ls images
  371  rbd ls volumes
  372  rbd ls images
  373  rbd snap unprotect volumes/volume-7aedaaa5-f547-401a-b4cb-1d2271eb1c7d@snapshot-509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824
  374  for i in $(rbd ls images); do echo $i; rbd snap ls images/$i ;done
  375  rbd snap unprotect images/812c6915-0748-4aab-8321-91b42e18118d@cc75032c-1c99-4186-b20d-3a39ac80bc90
  376  rbd snap ls 812c6915-0748-4aab-8321-91b42e18118d
  377  rbd snap ls images/812c6915-0748-4aab-8321-91b42e18118d
  378  rbd snap unprotect images/812c6915-0748-4aab-8321-91b42e18118d@snap
  379  rbd snap unprotect images@cc75032c-1c99-4186-b20d-3a39ac80bc90@snap
  380  rbd snap unprotect images/cc75032c-1c99-4186-b20d-3a39ac80bc90@snap
  381  ceph -s
  382  for i in $(rbd ls images); do echo $i; rbd snap ls images/$i ;done
  383  rbd ls images
  384  rbd ls volumes
  385  ceph -s
  386  rbd ls images
  387  rbd ls volumes
  388  for i in $(rbd ls volumes); do echo $i; rbd snap ls volumes/$i ;done
  389  rbd snap unprotect volumes/volume-7aedaaa5-f547-401a-b4cb-1d2271eb1c7d@snapshot-509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824
  390  rbd children volumes/volume-7aedaaa5-f547-401a-b4cb-1d2271eb1c7d@snapshot-509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824
  391  rbd info volumes/volume-7aedaaa5-f547-401a-b4cb-1d2271eb1c7d
  392  rbd snap ls volumes/volume-7aedaaa5-f547-401a-b4cb-1d2271eb1c7d
  393  rbd children volumes/volume-7aedaaa5-f547-401a-b4cb-1d2271eb1c7d -a
  394  rbd restore volumes/volume-7e5a90b5-bc54-4ec9-8573-0d73f0932771
  395  rbd children volumes/volume-7aedaaa5-f547-401a-b4cb-1d2271eb1c7d -a
  396  rbd  trash restore 75b3f22361394e
  397  rbd  -p volumes trash restore 75b3f22361394e
  398  rbd ls volumes
  399  rbd snap ls volumes/volume-7aedaaa5-f547-401a-b4cb-1d2271eb1c7d
  400  rbd children volumes/volume-7aedaaa5-f547-401a-b4cb-1d2271eb1c7d -a
  401  rbd children volumes/volume-7aedaaa5-f547-401a-b4cb-1d2271eb1c7d
  402  rbd rm volumes/volume-7e5a90b5-bc54-4ec9-8573-0d73f0932771
  403  rbd children volumes/volume-7aedaaa5-f547-401a-b4cb-1d2271eb1c7d
  404  rbd snap unprotect volumes/volume-7aedaaa5-f547-401a-b4cb-1d2271eb1c7d@snapshot-509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824
  405  for i in $(rbd ls iamges); do echo $i; rbd snap ls images/$i ;done
  406  for i in $(rbd ls images); do echo $i; rbd snap ls images/$i ;done
  407  rbd snap unprotect images/812c6915-0748-4aab-8321-91b42e18118d@snap
  408  rbd children images/812c6915-0748-4aab-8321-91b42e18118d -a
  409  rbd children images/812c6915-0748-4aab-8321-91b42e18118d
  410  rbd trash restore -p volumes c74489995faceb
  411  rbd children images/812c6915-0748-4aab-8321-91b42e18118d
  412  rbd rm volumes/volume-ae34dc48-f081-4e76-bd13-63d26db34519
  413  rbd snap unprotect images/812c6915-0748-4aab-8321-91b42e18118d@snap
  414  for i in $(rbd ls images); do echo $i;
  415  rbd info 812c6915-0748-4aab-8321-91b42e18118d
  416  rbd info images/812c6915-0748-4aab-8321-91b42e18118d
  417  rbd snap rm images/812c6915-0748-4aab-8321-91b42e18118d@snap
  418  rbd info images/812c6915-0748-4aab-8321-91b42e18118d
  419  rbd ls images
  420  rbd ls volumes
  421  rbd snap ls volumes/volume-4054433b-03cf-4cb1-8948-c6ffe6e2955c -
  422  rbd ls volumes
  423  rdb rm volume-4054433b-03cf-4cb1-8948-c6ffe6e2955c
  424  rbd rm volume-4054433b-03cf-4cb1-8948-c6ffe6e2955c
  425  rdb rm volumes/volume-4054433b-03cf-4cb1-8948-c6ffe6e2955c
  426  rbd rm volumes/volume-4054433b-03cf-4cb1-8948-c6ffe6e2955c
  427  rbd children volumes/volume-4054433b-03cf-4cb1-8948-c6ffe6e2955c
  428  rbd ls volumes
  429  rbd ls images
  430  ceph -s
  431  ceph df
  432  ceph -s
  433  history > backup


hist log from kolla node


 1643  source ~/kolla/venv/kolla-ansible-zed/bin/activate; source ~/kolla/config/admin-openrc.sh
 1644  for i in $(openstack image list --all --format value -c ID);do openstack image show $i --format value -c properties | grep -q "'rbd'" && openstack image show $i -c id -c name; done
 1645  touch images-to-delete.txt
 1646  for i in $(openstack image list --all --format value -c ID);do openstack image show $i --format value -c properties | grep -q "'rbd'" && openstack image show $i -c id --format value >>images-to-delete.txt; done
 1647  cat images-to-delete.txt
 1648  cat images-to-delete.txt  | wc -l
 1649  for i in images-to-delete.txt; do echo openstack image delete $i; done
 1650  for i in $(cat images-to-delete.txt); do echo openstack image delete $i; done
 1651  for i in images-to-delete.txt; do echo openstack image delete $i; done
 1652  for i in $(cat images-to-delete.txt); do openstack image delete $i; done
 1653  rm images-to-delete.txt; touch images-to-delete.txt
 1654  for i in $(openstack image list --all --format value -c ID);do openstack image show $i --format value -c properties | grep -q "'rbd'" && openstack image show $i -c id --format value >>images-to-delete.txt; done
 1655  cat images-to-delete.txt
 1656  for i in images-to-delete.txt; do echo openstack image delete $i; done
 1657  for i in $(cat images-to-delete.txt); do echo openstack image delete $i; done
 1658  openstack image delete 812c6915-0748-4aab-8321-91b42e18118d
 1659  openstack volume list
 1660  openstack volume list --status availiable
 1661  openstack volume list --help
 1662  openstack volume list --status Availiable
 1663  openstack volume list --status Avail
 1664  cat images-to-delete.txt
 1665  openstack volume list --help
 1666  openstack volume list --all --long
 1667  openstack volume list --all --long -c ID -c Status -c Type
 1668  openstack volume list --all --long -c ID -c Status -c Type -c Name -c Size
 1669  openstack volume list --all --long -c ID -c Status -c Type -c Name -c Size | grep -v -i storpool
 1670  openstack volume list --all --long -c ID -c Status -c Type -c Name -c Size | grep -v -i storpool > ceph_volumes.txt
 1671  cat ceph_volumes.txt  | awk -F'| ' '/availiable/ {print $1}'
 1672  cat ceph_volumes.txt  | awk -F'| ' '/available/ {print $1}'
 1673  cat ceph_volumes.txt
 1674  cat ceph_volumes.txt  | awk -F'| ' '/available/ {print $1}' > volumes_to delete.txt
 1675  cat ceph_volumes.txt  | awk -F'| ' '/available/ {print $1}' > volumes_to_delete.txt
 1676  cat ceph_volumes.txt  | awk -F'| ' '/available/ {print $1}' | sort > volumes_to_delete.txt
 1677  cat volumes_to_delete.txt
 1678  cat ceph_volumes.txt  | awk -F'| ' '/available/ {print $2}' | sort > volumes_to_delete.txt
 1679  cat volumes_to_delete.txt
 1680  vi volumes_in_ceph.txt
 1681  for i in $(cat volumes_to_delete.txt); if grep -q $i volumes_in_ceph.txt; then echo openstack volume delete $i; fi
 1682  for i in $(cat volumes_to_delete.txt); do  if grep -q $i volumes_in_ceph.txt; then openstack volume snapshot list $i; fi ;done
 1683  for i in $(cat volumes_to_delete.txt); do  if grep -q $i volumes_in_ceph.txt; then openstack volume delete $i; fi ;done
 1684  openstack volume list --all --long -c ID -c Status -c Type -c Name -c Size | grep -v -i storpool > ceph_volumes.txt
 1685  less ceph_volumes.txt
 1686  openstack volume list --all --long -c ID -c Status -c host
 1687  openstack volume list --all --long
 1688  openstack volume snapshot
 1689  source ~/kolla/venv/kolla-ansible-zed/bin/activate; source ~/kolla/config/admin-openrc.sh
 1690  wc -l volumes_in_ceph.txt
 1691  openstack volume snapshot list
 1692  openstack volume snapshot list --all
 1693  openstack volume snapshot list --all --long
 1694  openstack volume snapshot show ebf98ec7-2edb-474a-9a1a-b2de35a14aa9
 1695  openstack volume show 9255c6e7-9f79-408e-b8be-61082cf88d1d
 1696  openstack volume delet
 1697  openstack volume snapshot show ebf98ec7-2edb-474a-9a1a-b2de35a14aa9
 1698  openstack volume snapshot delete ebf98ec7-2edb-474a-9a1a-b2de35a14aa9
 1699  openstack volume snapshot show ebf98ec7-2edb-474a-9a1a-b2de35a14aa9
 1700  openstack volume snapshot list --all --long
 1701  openstack volume snapshot
 1702  openstack volume snapshot set --
 1703  openstack volume snapshot delete ebf98ec7-2edb-474a-9a1a-b2de35a14aa9
 1704  openstack volume snapshot list --all
 1705  openstack volume show
 1706  openstack volume show 9255c6e7-9f79-408e-b8be-61082cf88d1d
 1707  openstack volume delete 9255c6e7-9f79-408e-b8be-61082cf88d1d
 1708  openstack volume show 9255c6e7-9f79-408e-b8be-61082cf88d1d
 1709  openstack volume snapshot list --all | grep 44f3ee34-0784-484d-b2b4-fa833821c351
 1710  openstack volume snapshot delete44f3ee34-0784-484d-b2b4-fa833821c351
 1711  openstack volume snapshot delete 44f3ee34-0784-484d-b2b4-fa833821c351
 1712  openstack volume snapshot show 44f3ee34-0784-484d-b2b4-fa833821c351
 1713  openstack volume snapshot set status availiable
 1714  openstack volume snapshot set state availiable
 1715  openstack volume snapshot set state Available
 1716  openstack volume snapshot set state available
 1717  openstack volume snapshot show 96fdbeab-81a4-465d-87ae-0916c0950a88
 1718  openstack volume snapshot show 44f3ee34-0784-484d-b2b4-fa833821c351
 1719  openstack image list --all
 1720  openstack volume snapshot show 44f3ee34-0784-484d-b2b4-fa833821c351
 1721  openstack volume snapshot delete 44f3ee34-0784-484d-b2b4-fa833821c351
 1722  openstack volume snapshot show 44f3ee34-0784-484d-b2b4-fa833821c351
 1723  openstack volume list | grep
 1724  openstack volume list | grep 998bcdd4-3588-4464-ac8b-88c5387402e0
 1725  openstack volume delete 998bcdd4-3588-4464-ac8b-88c5387402e0
 1726  ssh ubuntu@str-238
 1727  ssh ubuntu@str-238
 1728  source ~/kolla/venv/kolla-ansible-zed/bin/activate; source ~/kolla/config/admin-openrc.sh
 1729  openstack volume snappshot show d44d6b1a-7a13-4cec-878e-ec9cef041fdb
 1730  openstack volume snapshot show d44d6b1a-7a13-4cec-878e-ec9cef041fdb
 1731  openstack volume snapshot delete d44d6b1a-7a13-4cec-878e-ec9cef041fdb
 1732  openstack volume snapshot show d44d6b1a-7a13-4cec-878e-ec9cef041fdb
 1733  openstack volume show
 1734  openstack volume show  b96a70f0-7326-4a7d-9c36-928bf2abd1ce
 1735  openstack volume snapshot show d44d6b1a-7a13-4cec-878e-ec9cef041fdb
 1736  openstack volume snapshot delete d44d6b1a-7a13-4cec-878e-ec9cef041fdb
 1737  openstack volume snapshot show d44d6b1a-7a13-4cec-878e-ec9cef041fdb
 1738  openstack volume show  b96a70f0-7326-4a7d-9c36-928bf2abd1ce
 1739  openstack volume delete  b96a70f0-7326-4a7d-9c36-928bf2abd1ce
 1740  ssh c001
 1741  ssh ubuntu@c001
 1742  source ~/kolla/venv/kolla-ansible-zed/bin/activate; source ~/kolla/config/admin-openrc.sh
 1743  openstack volume list --all --long -c ID -c Status -c Type -c Name -c Size | grep -v -i storpool
 1744  openstack volume show fa3e9113-7fcf-4ba5-b44e-201d406e6d04
 1745  openstack volume set
 1746  openstack volume set --state available fa3e9113-7fcf-4ba5-b44e-201d406e6d04
 1747  openstack volume show fa3e9113-7fcf-4ba5-b44e-201d406e6d04
 1748  openstack volume list --all --long -c ID -c Status -c Type -c Name -c Size | grep -v -i storpool | awk -F'| ' '/error_deleting/ {print $2}'
 1749  openstack volume list --all --long -c ID -c Status -c Type -c Name -c Size | grep -v -i storpool | awk -F'| ' '/error_deleting/ {print $2 $4}'
 1750  openstack volume list --all --long -c ID -c Status -c Type -c Name -c Size | grep -v -i storpool | awk -F'| ' '/error_deleting/ {print $2 $5}'
 1751  openstack volume list --all --long -c ID -c Status -c Type -c Name -c Size | grep -v -i storpool | awk -F'| ' '/error_deleting/ {print $2 $6}'
 1752  openstack volume list --all --long -c ID -c Status -c Type -c Name -c Size | grep -v -i storpool | awk -F'|' '/error_deleting/ {print $2 $6}'
 1753  openstack volume list --all --long -c ID -c Status -c Type -c Name -c Size | grep -v -i storpool | awk -F'|' '/error_deleting/ {print $2 $5}'
 1754  openstack volume list --all --long -c ID -c Status -c Type -c Name -c Size | grep -v -i storpool | awk -F'|' '/error_deleting/ {print $2 $4}'
 1755  openstack volume list --all --long -c ID -c Status -c Type -c Name -c Size | grep -v -i storpool | awk -F'|' '/error_deleting/ {print}'
 1756  openstack volume list --all --long -c ID -c Status -c Type -c Name -c Size | grep -v -i storpool | awk -F'|' '/error_deleting/ {print $2}'
 1757  openstack volume list --all --long -c ID -c Status -c Type -c Name -c Size | grep -v -i storpool | awk -F'|' '/error_deleting/ {print $2}' >ceph_volumes_to reset
 1758  openstack volume list --all --long -c ID -c Status -c Type -c Name -c Size | grep -v -i storpool | awk -F'|' '/error_deleting/ {print $2}' >ceph_volumes_to_reset
 1759  for i in (cat ceph_volumes_to_reset); do echo openstack volume set --state available $i; done
 1760  for i in $(cat ceph_volumes_to_reset); do echo openstack volume set --state available $i; done
 1761  for i in $(cat ceph_volumes_to_reset); do echo openstack volume set --state available $i; openstack volume set --state available $i; done
 1762  openstack volume snapshot show 6a125870-d6ef-40af-a9c8-7c2c72ba0876
 1763  openstack volume show  0ab6766c-6cde-4a03-ad46-03009066783e
 1764  openstack volume snapshot delete 6a125870-d6ef-40af-a9c8-7c2c72ba0876
 1765  openstack volume delete 0ab6766c-6cde-4a03-ad46-03009066783e
 1766  openstack  volume show 0ab6766c-6cde-4a03-ad46-03009066783e
 1767  openstack volume show e50b1239-62da-4535-aa21-4e551c1eb2be -cos-vol-host-attr:host
 1768  openstack volume show e50b1239-62da-4535-aa21-4e551c1eb2be -c os-vol-host-attr:host
 1769  openstack volume show e50b1239-62da-4535-aa21-4e551c1eb2be -c os-vol-host-attr:host | grep -q -e rbd && echo openstack volume delete e50b1239-62da-4535-aa21-4e551c1eb2be
 1770  openstack volume show e50b1239-62da-4535-aa21-4e551c1eb2be -c os-vol-host-attr:host | grep -q -e rbq && echo openstack volume delete e50b1239-62da-4535-aa21-4e551c1eb2be
 1771  openstack volume show e50b1239-62da-4535-aa21-4e551c1eb2be -c os-vol-host-attr:host -c ID
 1772  openstack volume show e50b1239-62da-4535-aa21-4e551c1eb2be -c os-vol-host-attr:host -c id
 1773  openstack volume show e50b1239-62da-4535-aa21-4e551c1eb2be -c os-vol-host-attr:host
 1774  openstack volume show 88f473eb-d2fa-4e37-82f2-54fd85c25d5e -c os-vol-host-attr:host | grep -e rbd && echo openstack volume snapshot delete 79c0a755-87e6-4b5f-bca1-da657e52331b
 1775  openstack volume show 88f473eb-d2fa-4e37-82f2-54fd85c25d5e -c os-vol-host-attr:host | grep -e rbd && echo openstack volume delete 88f473eb-d2fa-4e37-82f2-54fd85c25d5e
 1776  openstack volume show 0677c8e8-f4e7-42f4-8a57-7439b70e5653 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 0677c8e8-f4e7-42f4-8a57-7439b70e5653
 1777  openstack volume show 095849e0-b0b6-4ba6-bb00-0f0030a91baf -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 095849e0-b0b6-4ba6-bb00-0f0030a91baf
 1778  openstack volume show 0da62386-8eb4-4963-95ff-d9a2b561061a -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 0da62386-8eb4-4963-95ff-d9a2b561061a
 1779  openstack volume show 101cf309-5eab-45e4-9128-3572e107d237 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 500e0ae0-b6ce-4aba-8434-2b73bb7251a6
 1780  openstack volume show 101cf309-5eab-45e4-9128-3572e107d237 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete e68051c0-b3a0-4e65-8305-78f7aa3b2065
 1781  openstack volume show 101cf309-5eab-45e4-9128-3572e107d237 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete e0364942-1bb3-493b-a84e-a736b29fb847
 1782  openstack volume show 101cf309-5eab-45e4-9128-3572e107d237 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 101cf309-5eab-45e4-9128-3572e107d237
 1783  openstack volume show 125a7496-fdcd-4979-8dae-f342ee1f28a0 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 125a7496-fdcd-4979-8dae-f342ee1f28a0
 1784  openstack volume show 16b5ad27-ecf5-46e7-81ef-b79af5618dd0.deleted -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 34daba88-c273-4b3b-8954-6ad6832bdd21.clone_snap
 1785  openstack volume show 16b5ad27-ecf5-46e7-81ef-b79af5618dd0.deleted -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 16b5ad27-ecf5-46e7-81ef-b79af5618dd0.deleted
 1786  openstack volume show 19186f06-f923-4416-89e3-f52f6c54570b -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 6d605146-428f-4feb-a0c8-b0a4a8733a2a
 1787  openstack volume show 19186f06-f923-4416-89e3-f52f6c54570b -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 19186f06-f923-4416-89e3-f52f6c54570b
 1788  openstack volume show 1c68a40b-beae-45cb-9e0b-519301fb34af -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete fd08c190-3022-4764-8722-8de79ec0969b
 1789  openstack volume show 1c68a40b-beae-45cb-9e0b-519301fb34af -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 3013ef2e-f749-4ff1-ae0c-ebf49a4deec3
 1790  openstack volume show 1c68a40b-beae-45cb-9e0b-519301fb34af -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 1c68a40b-beae-45cb-9e0b-519301fb34af
 1791  openstack volume show 1cbdf9bb-2f89-47a7-86d2-16d0739a8451 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 1cbdf9bb-2f89-47a7-86d2-16d0739a8451
 1792  openstack volume show 1dcf464a-7efd-4ad4-a774-367188f55448 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 7b0771a6-b06d-491e-86a0-940cd4383adb
 1793  openstack volume show 1dcf464a-7efd-4ad4-a774-367188f55448 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 20864eb1-b532-422d-aa4b-bd40965842f4
 1794  openstack volume show 1dcf464a-7efd-4ad4-a774-367188f55448 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete f19c6d99-4217-43cf-9a93-b1b9b60206be
 1795  openstack volume show 1dcf464a-7efd-4ad4-a774-367188f55448 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 33bd9242-9441-471f-a4a6-8d46f2127ae4
 1796  openstack volume show 1dcf464a-7efd-4ad4-a774-367188f55448 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 1dcf464a-7efd-4ad4-a774-367188f55448
 1797  openstack volume show 1fc5a6f4-88ee-43bc-afdc-acb762bbec9d -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 1fc5a6f4-88ee-43bc-afdc-acb762bbec9d
 1798  openstack volume show 26a1c410-3996-427f-8b93-69b003784dcd -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 26a1c410-3996-427f-8b93-69b003784dcd
 1799  openstack volume show 29dacac9-93dc-408a-bf3f-327e8590acad -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 5d7cf008-d658-4d63-8f9a-39e754494038
 1800  openstack volume show 29dacac9-93dc-408a-bf3f-327e8590acad -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 7c9b41ab-ee7b-4be6-832a-7a773ac6710a
 1801  openstack volume show 29dacac9-93dc-408a-bf3f-327e8590acad -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 29dacac9-93dc-408a-bf3f-327e8590acad
 1802  openstack volume show 2a7ffe62-54e7-45e4-a61e-b066d29acc50 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 2a7ffe62-54e7-45e4-a61e-b066d29acc50
 1803  openstack volume show 2dfb47ee-307e-4f85-b593-160ffecfbfe4 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 2dfb47ee-307e-4f85-b593-160ffecfbfe4
 1804  openstack volume show 321e3628-170f-42c9-9f37-66413250c441 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete e52ae299-9eb6-4c72-85b8-691de1683f5c
 1805  openstack volume show 321e3628-170f-42c9-9f37-66413250c441 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 321e3628-170f-42c9-9f37-66413250c441
 1806  openstack volume show 3776f713-e1bd-40bd-be2b-18bd945385e6 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 1396b706-51d0-40e2-b670-3b17b5c4dbe0
 1807  openstack volume show 3776f713-e1bd-40bd-be2b-18bd945385e6 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete ab5416a7-ea1d-43a5-b084-6f5a2ea09bf5
 1808  openstack volume show 3776f713-e1bd-40bd-be2b-18bd945385e6 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 435a8912-3cd4-4e93-b1b5-ef8056cf6bce
 1809  openstack volume show 3776f713-e1bd-40bd-be2b-18bd945385e6 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete f8cd87ae-afd4-4338-89d1-13a78eeab2b2
 1810  openstack volume show 3776f713-e1bd-40bd-be2b-18bd945385e6 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 3776f713-e1bd-40bd-be2b-18bd945385e6
 1811  openstack volume show 39094eb4-bfc8-4c37-89c0-4c6326c2d7b4 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 39094eb4-bfc8-4c37-89c0-4c6326c2d7b4
 1812  openstack volume show 3a6e0d75-62ce-4d8f-8235-850c5c0db61d -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 3a6e0d75-62ce-4d8f-8235-850c5c0db61d
 1813  openstack volume show 3f79a1dd-6cf8-4dea-b98f-b7597d3a5a6d -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 3f79a1dd-6cf8-4dea-b98f-b7597d3a5a6d
 1814  openstack volume show 4054433b-03cf-4cb1-8948-c6ffe6e2955c -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 4054433b-03cf-4cb1-8948-c6ffe6e2955c
 1815  openstack volume show 417feeef-d79d-4a31-af13-f1bee971284b -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete de4787b8-ce8f-48ce-b13c-2854f1bbab2e
 1816  openstack volume show 417feeef-d79d-4a31-af13-f1bee971284b -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 417feeef-d79d-4a31-af13-f1bee971284b
 1817  openstack volume show 467bb817-3c6c-4f44-b767-35109127cb7c -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 467bb817-3c6c-4f44-b767-35109127cb7c
 1818  openstack volume show 4f80c812-cc7e-4b04-8148-ffd01ff69f14 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 4f80c812-cc7e-4b04-8148-ffd01ff69f14
 1819  openstack volume show 5408fbf0-46f6-41d9-937a-239be5055c4c -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete e41485d1-c71d-45d3-9184-b40522e9e700
 1820  openstack volume show 5408fbf0-46f6-41d9-937a-239be5055c4c -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 5408fbf0-46f6-41d9-937a-239be5055c4c
 1821  openstack volume show 56bb6be4-b0b5-44ca-8bad-9e4b55ac922e.deleted -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 3b794a23-c30f-44a8-a1c1-763ab4ee1635.clone_snap
 1822  openstack volume show 56bb6be4-b0b5-44ca-8bad-9e4b55ac922e.deleted -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 56bb6be4-b0b5-44ca-8bad-9e4b55ac922e.deleted
 1823  openstack volume show 57590c5a-e645-40a0-9237-3d69aca8c020 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 57590c5a-e645-40a0-9237-3d69aca8c020
 1824  openstack volume show 5cbae0c2-7fa2-4e56-bc26-633c66aa5d73 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 5cbae0c2-7fa2-4e56-bc26-633c66aa5d73
 1825  openstack volume show 61192337-5f2f-400e-b04f-337ce502b326 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 61192337-5f2f-400e-b04f-337ce502b326
 1826  openstack volume show 639983da-294f-45ac-883f-b7ec3d0f4ff3 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 639983da-294f-45ac-883f-b7ec3d0f4ff3
 1827  openstack volume show 64187c05-3299-4a2e-ab1b-e527f483cb6c -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete b139b50d-4984-452b-8a04-6a3ae9ac13d2
 1828  openstack volume show 64187c05-3299-4a2e-ab1b-e527f483cb6c -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 734d71df-fa23-4145-ad91-c40004d63797
 1829  openstack volume show 64187c05-3299-4a2e-ab1b-e527f483cb6c -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 64187c05-3299-4a2e-ab1b-e527f483cb6c
 1830  openstack volume show 68fcd1c6-60d3-4e7d-a7e5-d13a8d7bd424 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 68fcd1c6-60d3-4e7d-a7e5-d13a8d7bd424
 1831  openstack volume show 6ef47ed4-a802-4e65-bb07-8c36b4a1fe5e -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 6ef47ed4-a802-4e65-bb07-8c36b4a1fe5e
 1832  openstack volume show 6efd9e53-4cdd-4b99-84b2-114bd7e276c3 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 19c4346b-9ae1-489f-b992-d0186bbd8154
 1833  openstack volume show 6efd9e53-4cdd-4b99-84b2-114bd7e276c3 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 6efd9e53-4cdd-4b99-84b2-114bd7e276c3
 1834  openstack volume show 70d3019c-8085-4bcf-b120-b20652fcc835 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 70d3019c-8085-4bcf-b120-b20652fcc835
 1835  openstack volume show 763547b6-db23-496f-b852-46f1126411d2 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 7eea23cd-6ae4-419b-82d8-132b32f6cad8
 1836  openstack volume show 763547b6-db23-496f-b852-46f1126411d2 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 763547b6-db23-496f-b852-46f1126411d2
 1837  openstack volume show 76860ded-c5ff-458c-b749-a07b3c8e03fd -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 76860ded-c5ff-458c-b749-a07b3c8e03fd
 1838  openstack volume show 7894f6c4-c815-4132-896e-c4d5f41efdb9 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 001d43be-e82d-474b-bea4-1138ba325899
 1839  openstack volume show 7894f6c4-c815-4132-896e-c4d5f41efdb9 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 7894f6c4-c815-4132-896e-c4d5f41efdb9
 1840  openstack volume show 79ccdd09-102d-42e6-bc6c-01b6a2d19892 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete e8c076ef-6b66-485b-9706-f83413929eda
 1841  openstack volume show 79ccdd09-102d-42e6-bc6c-01b6a2d19892 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 79ccdd09-102d-42e6-bc6c-01b6a2d19892
 1842  rbd snap unprotect volumes/volume-7aedaaa5-f547-401a-b4cb-1d2271eb1c7d@snapshot-509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824
 1843  openstack volume show 7aedaaa5-f547-401a-b4cb-1d2271eb1c7d -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824
 1844  openstack volume show 7aedaaa5-f547-401a-b4cb-1d2271eb1c7d -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 7aedaaa5-f547-401a-b4cb-1d2271eb1c7d
 1845  openstack volume show 7cc5f04a-0ecf-4ddc-a2c4-5c0d3a0b5913 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 7cc5f04a-0ecf-4ddc-a2c4-5c0d3a0b5913
 1846  openstack volume show 85ecb198-c2de-4ecf-90b6-7f658085d5f5 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 98cd07e4-6b5f-4b5f-8608-15e759791dfb
 1847  openstack volume show 85ecb198-c2de-4ecf-90b6-7f658085d5f5 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 85ecb198-c2de-4ecf-90b6-7f658085d5f5
 1848  openstack volume show 86056c2b-a904-4228-8dd1-c4cb01648f0f -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 86056c2b-a904-4228-8dd1-c4cb01648f0f
 1849  openstack volume show 8889e3b6-5d6c-4bf6-b17c-434ffb95154f -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 8889e3b6-5d6c-4bf6-b17c-434ffb95154f
 1850  openstack volume show 88f473eb-d2fa-4e37-82f2-54fd85c25d5e -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 3b355a39-d82a-45e0-95fd-49d447ae5a34
 1851  openstack volume show 88f473eb-d2fa-4e37-82f2-54fd85c25d5e -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 79c0a755-87e6-4b5f-bca1-da657e52331b
 1852  openstack volume show 88f473eb-d2fa-4e37-82f2-54fd85c25d5e -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 88f473eb-d2fa-4e37-82f2-54fd85c25d5e
 1853  openstack volume show 8eb7069c-5c07-43c6-a91f-f1d79a071a91 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 14b5f169-1fe1-43f1-aacf-0c4589e8c469
 1854  openstack volume show 8eb7069c-5c07-43c6-a91f-f1d79a071a91 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 8eb7069c-5c07-43c6-a91f-f1d79a071a91
 1855  openstack volume show 92aca6e5-bf64-475a-a3c5-00cd941848b5 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 96fdbeab-81a4-465d-87ae-0916c0950a88
 1856  openstack volume show 92aca6e5-bf64-475a-a3c5-00cd941848b5 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 92aca6e5-bf64-475a-a3c5-00cd941848b5
 1857  openstack volume show 93f22739-5ac8-4a88-8c49-de4c2234e7e7 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 42d02435-04aa-42e5-9e92-a87161f50353
 1858  openstack volume show 93f22739-5ac8-4a88-8c49-de4c2234e7e7 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 93f22739-5ac8-4a88-8c49-de4c2234e7e7
 1859  openstack volume show 94e97068-13e3-49a2-8bfe-4c4ba1d898fd -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 94e97068-13e3-49a2-8bfe-4c4ba1d898fd
 1860  openstack volume show 9ce7b127-acae-4e94-9b21-079633f008bd -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 9ce7b127-acae-4e94-9b21-079633f008bd
 1861  openstack volume show 9fb3978c-050d-4bfc-b9ce-9cd213994d96 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 69d9e3b8-eed3-47e8-854f-21147fbce2da
 1862  openstack volume show 9fb3978c-050d-4bfc-b9ce-9cd213994d96 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete 9fb3978c-050d-4bfc-b9ce-9cd213994d96
 1863  openstack volume show a1468374-b1c3-4c26-a2da-c2cfeee1c27d -c os-vol-host-attr:host | grep -e rbd && openstack volume delete a1468374-b1c3-4c26-a2da-c2cfeee1c27d
 1864  openstack volume show a3a39c50-5fa8-4b50-8093-d166eda187b9 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete a3a39c50-5fa8-4b50-8093-d166eda187b9
 1865  openstack volume show ae27171c-f11e-44e7-8928-e72efe9b45c6 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete ae27171c-f11e-44e7-8928-e72efe9b45c6
 1866  openstack volume show b1a50f3e-6c7c-4fa8-a1ae-1a488fc4d3b3 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 7bb8db32-e5f3-4ae5-92a0-9373cc543bff
 1867  openstack volume show b1a50f3e-6c7c-4fa8-a1ae-1a488fc4d3b3 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete b1a50f3e-6c7c-4fa8-a1ae-1a488fc4d3b3
 1868  openstack volume show b9e1b8d6-f38e-4570-ac01-49a7309ea6ec -c os-vol-host-attr:host | grep -e rbd && openstack volume delete b9e1b8d6-f38e-4570-ac01-49a7309ea6ec
 1869  openstack volume show c4d0ce91-8f23-40cf-9772-cca3c77ae672 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 2df9ca3f-0340-4a4c-8995-9626627af7c5
 1870  openstack volume show c4d0ce91-8f23-40cf-9772-cca3c77ae672 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete c4d0ce91-8f23-40cf-9772-cca3c77ae672
 1871  openstack volume show c89b51d0-49dc-4923-aaec-5e5da27ed799 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete c89b51d0-49dc-4923-aaec-5e5da27ed799
 1872  openstack volume show d2f33e0e-d7f5-4eae-8efb-27a71f335d12 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete d2f33e0e-d7f5-4eae-8efb-27a71f335d12
 1873  openstack volume show d4af50d5-2182-47b3-941c-997e74c9341c -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 91d7fb79-f5ae-4c3f-bb1c-068c7c5a1adc
 1874  openstack volume show d4af50d5-2182-47b3-941c-997e74c9341c -c os-vol-host-attr:host | grep -e rbd && openstack volume delete d4af50d5-2182-47b3-941c-997e74c9341c
 1875  openstack volume show d698f7bf-fff5-450d-8ed6-2f62e4ca80cf -c os-vol-host-attr:host | grep -e rbd && openstack volume delete d698f7bf-fff5-450d-8ed6-2f62e4ca80cf
 1876  openstack volume show dc275cb9-6709-4dc1-8e9d-5943a3b4964e -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 85e73527-63b5-4f9c-a66f-70356e11a490
 1877  openstack volume show dc275cb9-6709-4dc1-8e9d-5943a3b4964e -c os-vol-host-attr:host | grep -e rbd && openstack volume delete dc275cb9-6709-4dc1-8e9d-5943a3b4964e
 1878  openstack volume show dc48db05-56e2-448e-99cb-e07e65977b53 -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete b9345b91-a5b5-4157-8838-4740b7e27788
 1879  openstack volume show dc48db05-56e2-448e-99cb-e07e65977b53 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete dc48db05-56e2-448e-99cb-e07e65977b53
 1880  openstack volume show e1a43d3b-cdaf-475a-909c-fc54123bd87f -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 0c867532-e1d9-429e-b402-b86db63f98c0
 1881  openstack volume show e1a43d3b-cdaf-475a-909c-fc54123bd87f -c os-vol-host-attr:host | grep -e rbd && openstack volume delete e1a43d3b-cdaf-475a-909c-fc54123bd87f
 1882  openstack volume show e50b1239-62da-4535-aa21-4e551c1eb2be -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 0e47cfa0-481b-4f8e-b405-8fb5f74b927b
 1883  openstack volume show e50b1239-62da-4535-aa21-4e551c1eb2be -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete fc42e191-f086-487e-ba15-305830cac507
 1884  openstack volume show e50b1239-62da-4535-aa21-4e551c1eb2be -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete 7d89b23e-4620-4293-931a-1c47b11985bb
 1885  openstack volume show e50b1239-62da-4535-aa21-4e551c1eb2be -c os-vol-host-attr:host | grep -e rbd && openstack volume delete e50b1239-62da-4535-aa21-4e551c1eb2be
 1886  openstack volume show e5d32199-8115-43a8-a8df-fa2f98442f96 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete e5d32199-8115-43a8-a8df-fa2f98442f96
 1887  openstack volume show eda73b06-05eb-4b3c-9280-0851873f145c -c os-vol-host-attr:host | grep -e rbd && openstack volume delete eda73b06-05eb-4b3c-9280-0851873f145c
 1888  openstack volume show ef74477b-a345-4cc0-ae94-4b29b0375015 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete ef74477b-a345-4cc0-ae94-4b29b0375015
 1889  openstack volume show f035099d-a4c1-489d-9253-edebe3413fec -c os-vol-host-attr:host | grep -e rbd && openstack volume delete f035099d-a4c1-489d-9253-edebe3413fec
 1890  openstack volume show f465b10e-5d01-4b5d-a786-51cc217220ee -c os-vol-host-attr:host | grep -e rbd && openstack volume delete f465b10e-5d01-4b5d-a786-51cc217220ee
 1891  openstack volume show fa3e9113-7fcf-4ba5-b44e-201d406e6d04 -c os-vol-host-attr:host | grep -e rbd && openstack volume delete fa3e9113-7fcf-4ba5-b44e-201d406e6d04
 1892  mkdr cleanup
 1893  mkdir cleanup
 1894  cd cleanup/
 1895  vi lastvols
 1896  for i in $(cat lastvols); do openstack volume show $i; done
 1897  for i in $(cat lastvols); do openstack volume show $i -c id -c  os-vol-host-attr:host -c status -c type; done
 1898  vi lastvols
 1899  for i in $(cat lastvols); do openstack volume show $i -c id -c  os-vol-host-attr:host -c status -c type; done
 1900  openstack volume show 39094eb4-bfc8-4c37-89c0-4c6326c2d7b4
 1901  openstack volume delete 39094eb4-bfc8-4c37-89c0-4c6326c2d7b4
 1902  for i in $(cat lastvols); do openstack volume delete $i; done
 1903  wc -l lastvols
 1904  vi lastvols
 1905  for i in $(cat lastvols); do openstack volume show $i -c id -c  os-vol-host-attr:host -c status -c type; done
 1906  for i in $(cat lastvols); do openstack volume delete $i; done
 1907  for i in $(cat lastvols); do openstack volume show $i -c id -c  os-vol-host-attr:host -c status -c type; done
 1908  openstack volume snapshot show 96fdbeab-81a4-465d-87ae-0916c0950a88
 1909  openstack volume snapshot set 96fdbeab-81a4-465d-87ae-0916c0950a88 --state available
 1910  openstack volume snapshot delete 96fdbeab-81a4-465d-87ae-0916c0950a88
 1911  openstack volume delete 92aca6e5-bf64-475a-a3c5-00cd941848b5
 1912  openstack volume show 4054433b-03cf-4cb1-8948-c6ffe6e2955c
 1913  openstack volume set 4054433b-03cf-4cb1-8948-c6ffe6e2955c --state error
 1914  openstack volume delete 4054433b-03cf-4cb1-8948-c6ffe6e2955c
 1915  openstack volume set 4054433b-03cf-4cb1-8948-c6ffe6e2955c --state availiable
 1916  openstack volume set 4054433b-03cf-4cb1-8948-c6ffe6e2955c --state available
 1917  openstack volume snapshot show 509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824
 1918  openstack volume snapshot rm  509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824
 1919  openstack volume snapshot delete  509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824
 1920  openstack volume snapshot show 509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824
 1921  openstack volume snapshot set 509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824 status=error
 1922  openstack volume snapshot set 509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824 --state error
 1923  openstack volume snapshot delete  509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824
 1924  openstack volume show 7aedaaa5-f547-401a-b4cb-1d2271eb1c7d
 1925  openstack volume delete 7aedaaa5-f547-401a-b4cb-1d2271eb1c7d
 1926  openstack volume set 7aedaaa5-f547-401a-b4cb-1d2271eb1c7d --state error
 1927  openstack volume delete 7aedaaa5-f547-401a-b4cb-1d2271eb1c7d
 1928  openstack volume snapshot list --all --long
 1929  openstack volume show 4054433b-03cf-4cb1-8948-c6ffe6e2955c
 1930  openstack image show 1ad8e143-725e-4e05-a9e2-80d8a7944613
 1931  openstack image delete 1ad8e143-725e-4e05-a9e2-80d8a7944613
 1932  openstack image show 1c89c13f-7a31-46b4-b11e-631451536343
 1933  openstack image delete 1c89c13f-7a31-46b4-b11e-631451536343
 1934  openstack image show 812c6915-0748-4aab-8321-91b42e18118d
 1935  openstack image delete 812c6915-0748-4aab-8321-91b42e18118d
 1936  openstack image show c265d6c6-e7f0-4f3e-9b60-2cc888edae62
 1937  openstack image delete c265d6c6-e7f0-4f3e-9b60-2cc888edae62
 1938  openstack image show cc75032c-1c99-4186-b20d-3a39ac80bc90
 1939  openstack image delete cc75032c-1c99-4186-b20d-3a39ac80bc90
 1940  openstack volume snapshot show 509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824
 1941  openstack volume snapshot rm 509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824
 1942  openstack volume snapshot delete 509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824
 1943  openstack volume snapshot show 509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824
 1944  openstack volume list --long --all
 1945  openstack volume list --long --all | grep -v storpool
 1946  openstack volume show 34daba88-c273-4b3b-8954-6ad6832bdd21
 1947  openstack volume deltee 34daba88-c273-4b3b-8954-6ad6832bdd21
 1948  openstack volume delete 34daba88-c273-4b3b-8954-6ad6832bdd21
 1949  openstack volume list --long --all | grep -v storpool
 1950  openstack volume show 3b794a23-c30f-44a8-a1c1-763ab4ee1635
 1951  openstack volume deltete 3b794a23-c30f-44a8-a1c1-763ab4ee1635
 1952  openstack volume show 3b794a23-c30f-44a8-a1c1-763ab4ee1635
 1953  openstack volume delete 3b794a23-c30f-44a8-a1c1-763ab4ee1635
 1954  openstack volume list --long --all | grep -v storpool
 1955  openstack image delete 812c6915-0748-4aab-8321-91b42e18118d
 1956  openstack image delete cc75032c-1c99-4186-b20d-3a39ac80bc90
 1957  openstack image show cc75032c-1c99-4186-b20d-3a39ac80bc90
 1958  openstack volume list --long --all | grep -v storpool
 1959  openstack volume set 4054433b-03cf-4cb1-8948-c6ffe6e2955c --state error
 1960  openstack volume show 4054433b-03cf-4cb1-8948-c6ffe6e2955c
 1961  openstack volume list --long --all | grep -v storpool
 1962  openstack image set
 1963  openstack volume list --long --all | grep -v storpool
 1964  openstack volume snapshot show 509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824
 1965  openstack volume snapshot delete 509cbdc3-4eeb-4acf-ae0e-0cf64aa9f824
 1966  openstack volume delete 7aedaaa5-f547-401a-b4cb-1d2271eb1c7d
 1967  openstack image delete cc75032c-1c99-4186-b20d-3a39ac80bc90
 1968  openstack image delete 812c6915-0748-4aab-8321-91b42e18118d
 1969  openstack volume list --long --all | grep -v storpool
 1970  openstack image delete cc75032c-1c99-4186-b20d-3a39ac80bc90
 1971  openstack image
 1972  openstack image set
 1973  openstack image set --unprotected cc75032c-1c99-4186-b20d-3a39ac80bc90
 1974  openstack image delete cc75032c-1c99-4186-b20d-3a39ac80bc90
 1975  openstack volume show volumes/volume-4054433b-03cf-4cb1-8948-c6ffe6e2955c
 1976  openstack volume show 4054433b-03cf-4cb1-8948-c6ffe6e2955c
 1977  openstack volume delete 4054433b-03cf-4cb1-8948-c6ffe6e2955c
 1978  cinder reset-state available 4054433b-03cf-4cb1-8948-c6ffe6e2955c
 1979  cinder reset-state
 1980  cinder reset-state --reset-migration-status
 1981  cinder reset-state --reset-migration-status 4054433b-03cf-4cb1-8948-c6ffe6e2955c
 1982  openstack volume delete 4054433b-03cf-4cb1-8948-c6ffe6e2955c
 1983  openstack volume list --long --all | grep -v storpool
 1984  openstack volume set --state error 4054433b-03cf-4cb1-8948-c6ffe6e2955c
 1985  openstack volume delete 4054433b-03cf-4cb1-8948-c6ffe6e2955c
 1986  openstack volume list --long --all | grep -v storpool
 1987  openstack image delete 812c6915-0748-4aab-8321-91b42e18118d
 1988  ssh ubuntu@str-238
 1989  ssh ubuntu@c003
 1990  ssh ubuntu@c002
 1991  ssh ubuntu@str-238
 1992  history > hist.bak; cat hist.bak




and last but not least the script to unprotect_and_flatten_snapshots
cat unprotect_and_flatten_snapshots.py
#!/usr/bin/env python3
import subprocess
import json

dev=False
verbose=0
mypool='volumes'
dry_run=True

def get_rbd_image_list(pool):
  if not dev:
    result=subprocess.run(['rbd', 'list', '--format', 'json', pool], stdout=subprocess.PIPE)
  else:
    result=subprocess.run(['cat', pool + '.json' ], stdout=subprocess.PIPE)
  if result.returncode==0:
    return json.loads(result.stdout.decode('utf-8'))
  else:
    raise SystemExit('Unable to get list of pool %s' % pool)

def get_snapshots(pool,image_name):
  if verbose > 0:
    print('checking %s/%s for snaps\n' % (mypool,volume))
  if not dev:
    result=subprocess.run(['rbd', 'snap', 'ls', '--format', 'json', pool+'/'+volume ], stdout=subprocess.PIPE)
  else:
    result=subprocess.run(['cat', volume + '.json' ], stdout=subprocess.PIPE)
  if result.returncode==0:
    return json.loads(result.stdout.decode('utf-8'))
  else:
    raise SystemExit('Unable to get list of snapshots for image %s in pool %s' % (image_name, pool))

def get_snapshot_children(pool,image_name,snap_name):
  if verbose > 0:
    print('checking %s/%s@%s for children\n' % (pool,image_name,snap_name))
  if not dev:
    result=subprocess.run(['rbd', 'children', '--format', 'json', pool+'/'+volume+'@'+snap_name ], stdout=subprocess.PIPE)
  else:
    result=subprocess.run(['cat', image_name +'_'+snap_name +'_kids.json' ], stdout=subprocess.PIPE)
  if result.returncode==0:
    return json.loads(result.stdout.decode('utf-8'))
  else:
    raise SystemExit('Unable to get list of children for image snapshot %s@%s in pool %s' % (image_name, snap_name, pool))
#def get_image_children(pool,image_name):
#

def flatten_image(full_image_path):
  if verbose > 0:
    print('flattening image %s' % (full_image_path))
  if dry_run or dev:
    print(' '.join(['rbd', 'flatten', full_image_path ]))
  else:
    raise SystemExit('UNTESTED CODE PATH QUITTING NOW')
    result=subprocess.run(['rbd', 'flatten', full_image_path ], stdout=subprocess.PIPE)
    if result.returncode==0:
      return json.loads(result.stdout.decode('utf-8'))
    else:
      raise SystemExit('Unable to flatten image %s'%full_image_path)
  return True

def unprotect_snapshot(full_image_path,snap_name):
  if verbose > 0:
    print('unprotecting %s@%s \n' % (full_image_path,snap_name))
  if '/' not in full_image_path:
    raise SystemExit('the image path %s has no / quitting' % full_image_path)
  if dry_run or dev:
    print(' '.join(['rbd', 'snap', 'unprotect', full_image_path+'@'+snap_name ]))
    return True
  else:
    result=subprocess.run(['rbd', 'snap', 'unprotect', full_image_path+'@'+snap_name ], stdout=subprocess.PIPE)
    if result.returncode==0:
      return True
    else:
      raise SystemExit('Unable to unprotect snapshot %s@%s \n' % (full_image_path,snap_name))

for volume in get_rbd_image_list(mypool):
  os_vol_uuid='-'.join((volume.split('-')[1:]))

  snaps=get_snapshots(mypool,volume)
  for snapshot in snaps:
    os_snap_uuid='-'.join((snapshot['name'].split('-')[1:]))
    if verbose > 0:
      print('image %s has a snapshot called %s and is protections state is %s'%(mypool+'/'+volume, snapshot['name'], snapshot['protected'] ))
    if snapshot['protected']=='true':
      snapshot_child_list=get_snapshot_children(mypool,volume,snapshot['name'])
      for child_image in snapshot_child_list:
        flatten_image(child_image)
      # if we get this far all child images are flattened and we can unprotect the image snapshot
      unprotect_snapshot('%s/%s' % (mypool,volume), snapshot['name'])
    if verbose > 0:
      print('now possible to delete snapshot with the following commands:\n')
      print('openstack volume snapshot show %s' % os_snap_uuid)
    if verbose > 0:
      print('please check that this is a member of %s before running any of the following\n' % os_vol_uuid)

    print('openstack volume show %s -c os-vol-host-attr:host | grep -e rbd && openstack volume snapshot delete %s\n' % (os_vol_uuid,os_snap_uuid))
  if verbose > 0:
    print('it should now be possible to delete the volume using the following commands\n')
    print('openstack volume show %s' % (os_vol_uuid))
  print('openstack volume show %s -c os-vol-host-attr:host | grep -e rbd && openstack volume delete %s' % (os_vol_uuid,os_vol_uuid))




#################################### log complete