CEPH: Ceph how to uninstall a Ceph cluster

So... ya f*cked it all up and want to start again - luckily, there's a command for that

# This was run on the ceph-deploy node, in the ~/ceph directory where the keys and ceph.conf were stored. 
cd ceph 
cp ceph.conf ceph.conf.bck
ceph-deploy purge ceph-osd1 ceph-osd2 ceph-osd3
ceph-deploy purgedata ceph-osd1 ceph-osd2 ceph-osd3 # this failed as device/resource busy. guess a reboot will sort but cant tests that atm. to be updated. 
ceph-deploy forgetkeys

And now go back and run ceph-deploy new...

One additional note:

• On disks that had already been installed with Ceph run:

#  on all nodes previously installed / disks
parted /dev/sda
mklabel gpt
quit
partprobe
# not sure if this is needed but previous commands would exist with 
#  /usr/sbin/sgdisk --verbose --largest-new=1 --change-name=1:ceph data --partition-guid=1:f4784a2e-9d59-47fd-949e-1ef6d434f108 --typecode=1:89c57f98-2fe5-4dc0-89c1-f3ad0ceff2be -- /dev/sdc

Some more notes on debugging permissions errors:

• Restarting the mon service regenerated the keyrings keys (/etc/ceph/*key /var/lib/ceph/booststrap-osd/ceph.keyring etc)

[root@node01 ~]# ceph auth list 
installed auth entries:

client.admin
        key: AQARRTtXdaUIAxAAYXwhQ4H2jvdLGKa+/UIFEQ==
        caps: [mds] allow
        caps: [mon] allow *
        caps: [osd] allow *
client.bootstrap-osd
        key: AQDtgDtXIQVaBhAAq/x86vJeK6DwscpzgDlrlA==
        caps: [mon] allow profile bootstrap-osd
[root@node01 ~]# cat /etc/ceph/ceph.client.admin.keyring 
[client.admin]
        key = AQARRTtXdaUIAxAAYXwhQ4H2jvdLGKa+/UIFEQ==
[root@node01 ~]# cat /var/lib/ceph/bootstrap-osd/ceph.keyring 
[client.bootstrap-osd]
        key = AQDtgDtXIQVaBhAAq/x86vJeK6DwscpzgDlrlA==
# i was getting permission errors as a result of the bootstrap keys being from old ceph cluster. remove them, restart mon, check the key files and should work ok 


# so lets check this with some ceph parameters (--name / --keyring) 

# default with be --name client.admin and --keyring /etc/ceph/ceph.client.admin.keyring
root@node01 ~]# ceph health 
HEALTH_ERR 64 pgs stuck inactive; 64 pgs stuck unclean; no osds

# keyring will default to /etc/ and client.bootstrap-osd is not defined in there: FAIL
[root@node01 ~]# ceph --name client.bootstrap-osd health                                                                                                                                     
2016-05-17 22:13:52.428328 7fba26bed700 -1 monclient(hunting): ERROR: missing keyring, cannot use cephx for authentication
2016-05-17 22:13:52.428332 7fba26bed700  0 librados: client.bootstrap-osd initialization error (2) No such file or directory
Error connecting to cluster: ObjectNotFound

# change the name, all OK
[root@node01 ~]# ceph --name client.admin health                                                                                                                                             
HEALTH_ERR 64 pgs stuck inactive; 64 pgs stuck unclean; no osds

# back to name=bootstrap but setting the right keyring: all OK 
[root@node01 ~]# ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring health                                                                                  
HEALTH_ERR 64 pgs stuck inactive; 64 pgs stuck unclean; no osds

# and finally - equivalent to no args as all default 
[root@node01 ~]# ceph --name client.admin --keyring /etc/ceph/ceph.client.admin.keyring health 
HEALTH_ERR 64 pgs stuck inactive; 64 pgs stuck unclean; no osds

Is the ceph mon daemon running and listening ok?

[root@head ~]# ceph --admin-daemon /var/run/ceph/ceph-mon.head.asok mon_status
{
    "name": "head",
    "rank": 0,
    "state": "leader",
    "election_epoch": 1,
    "quorum": [
        0
    ],
    "outside_quorum": [],
    "extra_probe_peers": [],
    "sync_provider": [],
    "monmap": {
        "epoch": 1,
        "fsid": "dabd24c7-fb4c-4d84-9ae9-ad8e39f5c856",
        "modified": "0.000000",
        "created": "0.000000",
        "mons": [
            {
                "rank": 0,
                "name": "head",
                "addr": "10.100.100.1:6789\/0"
            }
        ]
    }
}