CEPH: Ceph how to uninstall a Ceph cluster

So... ya f*cked it all up and want to start again - luckily, there's a command for that

# This was run on the ceph-deploy node, in the ~/ceph directory where the keys and ceph.conf were stored. 
cd ceph 
cp ceph.conf ceph.conf.bck
ceph-deploy purge ceph-osd1 ceph-osd2 ceph-osd3
ceph-deploy purgedata ceph-osd1 ceph-osd2 ceph-osd3 # this failed as device/resource busy. guess a reboot will sort but cant tests that atm. to be updated. 
ceph-deploy forgetkeys

And now go back and run ceph-deploy new...

One additional note:

• On disks that had already been installed with Ceph run:

#  on all nodes previously installed / disks
parted /dev/sda
mklabel gpt
quit
partprobe
# not sure if this is needed but previous commands would exist with 
#  /usr/sbin/sgdisk --verbose --largest-new=1 --change-name=1:ceph data --partition-guid=1:f4784a2e-9d59-47fd-949e-1ef6d434f108 --typecode=1:89c57f98-2fe5-4dc0-89c1-f3ad0ceff2be -- /dev/sdc

Some more notes on debugging permissions errors:

• Restarting the mon service regenerated the keyrings keys (/etc/ceph/*key /var/lib/ceph/booststrap-osd/ceph.keyring etc)

[root@node01 ~]# ceph auth list 
installed auth entries:

client.admin
        key: AQARRTtXdaUIAxAAYXwhQ4H2jvdLGKa+/UIFEQ==
        caps: [mds] allow
        caps: [mon] allow *
        caps: [osd] allow *
client.bootstrap-osd
        key: AQDtgDtXIQVaBhAAq/x86vJeK6DwscpzgDlrlA==
        caps: [mon] allow profile bootstrap-osd
[root@node01 ~]# cat /etc/ceph/ceph.client.admin.keyring 
[client.admin]
        key = AQARRTtXdaUIAxAAYXwhQ4H2jvdLGKa+/UIFEQ==
[root@node01 ~]# cat /var/lib/ceph/bootstrap-osd/ceph.keyring 
[client.bootstrap-osd]
        key = AQDtgDtXIQVaBhAAq/x86vJeK6DwscpzgDlrlA==
# i was getting permission errors as a result of the bootstrap keys being from old ceph cluster. remove them, restart mon, check the key files and should work ok 


# so lets check this with some ceph parameters (--name / --keyring) 

# default with be --name client.admin and --keyring /etc/ceph/ceph.client.admin.keyring
root@node01 ~]# ceph health 
HEALTH_ERR 64 pgs stuck inactive; 64 pgs stuck unclean; no osds

# keyring will default to /etc/ and client.bootstrap-osd is not defined in there: FAIL
[root@node01 ~]# ceph --name client.bootstrap-osd health                                                                                                                                     
2016-05-17 22:13:52.428328 7fba26bed700 -1 monclient(hunting): ERROR: missing keyring, cannot use cephx for authentication
2016-05-17 22:13:52.428332 7fba26bed700  0 librados: client.bootstrap-osd initialization error (2) No such file or directory
Error connecting to cluster: ObjectNotFound

# change the name, all OK
[root@node01 ~]# ceph --name client.admin health                                                                                                                                             
HEALTH_ERR 64 pgs stuck inactive; 64 pgs stuck unclean; no osds

# back to name=bootstrap but setting the right keyring: all OK 
[root@node01 ~]# ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring health                                                                                  
HEALTH_ERR 64 pgs stuck inactive; 64 pgs stuck unclean; no osds

# and finally - equivalent to no args as all default 
[root@node01 ~]# ceph --name client.admin --keyring /etc/ceph/ceph.client.admin.keyring health 
HEALTH_ERR 64 pgs stuck inactive; 64 pgs stuck unclean; no osds