User Tools

Site Tools


This is an old revision of the document!

Решение Ceph



nodeN# apt install ceph


Начальная конфигурация

nodeN# cat /etc/ceph/ceph.conf
	fsid = a7f64266-0894-4f1e-a635-d0aeaca0e993
	mon initial members = node3,node4,node5
	mon host =,,
	public network =
	auth cluster required = none
	auth service required = none
	auth client required = none
ssh-copy-id node4
ssh-copy-id node5
scp /etc/ceph/ceph.conf node4:/etc/ceph/
scp /etc/ceph/ceph.conf node5:/etc/ceph/
monmaptool --create --add node3 --fsid a7f64266-0894-4f1e-a635-d0aeaca0e993 /tmp/monmap
monmaptool --add node4 --fsid a7f64266-0894-4f1e-a635-d0aeaca0e993 /tmp/monmap
monmaptool --add node5 --fsid a7f64266-0894-4f1e-a635-d0aeaca0e993 /tmp/monmap
monmaptool --print /tmp/monmap
scp /tmp/monmap node4:/tmp/
scp /tmp/monmap node5:/tmp/
bash -c '
sudo -u ceph mkdir /var/lib/ceph/mon/ceph-node3
ssh node4 sudo -u ceph mkdir /var/lib/ceph/mon/ceph-node4
ssh node5 sudo -u ceph mkdir /var/lib/ceph/mon/ceph-node5
bash -c '
sudo -u ceph ceph-mon --mkfs -i node3 --monmap /tmp/monmap
ssh node4 sudo -u ceph ceph-mon --mkfs -i node4 --monmap /tmp/monmap
ssh node5 sudo -u ceph ceph-mon --mkfs -i node5 --monmap /tmp/monmap
ls -l /var/lib/ceph/mon/ceph-node3
bash -c '
systemctl start ceph-mon@node3
systemctl enable ceph-mon@node3
ssh node4 systemctl start ceph-mon@node4
ssh node4 systemctl enable ceph-mon@node4
ssh node5 systemctl start ceph-mon@node5
ssh node5 systemctl enable ceph-mon@node5
# ceph -s

# ps ax | grep ceph

Поиск проблемных узлов

# ceph health detail
MON_DOWN 1/3 mons down, quorum node4,node5
    mon.node3 (rank 0) addr is down (out of quorum)


bash -c '
sudo -u ceph mkdir /var/lib/ceph/mgr/ceph-node3
systemctl start ceph-mgr@node3
systemctl enable ceph-mgr@node3

ssh node4 sudo -u ceph mkdir /var/lib/ceph/mgr/ceph-node4
ssh node4 systemctl start ceph-mgr@node4
ssh node4 systemctl enable ceph-mgr@node4

ssh node5 sudo -u ceph mkdir /var/lib/ceph/mgr/ceph-node5
ssh node5 systemctl start ceph-mgr@node5
ssh node5 systemctl enable ceph-mgr@node5
# ceph -s

# ps ax | grep ceph


Начальная конфигурация

bash -c '
ceph-volume lvm create --data /dev/sdb
ssh node4 ceph-volume lvm create --data /dev/sdb
ssh node5 ceph-volume lvm create --data /dev/sdb
# vgs

# ps ax | grep ceph

# ceph -s

# ceph osd stat

# ceph osd tree

# ceph osd df

Добавление узла OSD


ssh-copy-id node6

scp /etc/ceph/ceph.conf node6:/etc/ceph/

ssh node6 ceph-volume lvm create --data /dev/sdb

watch ceph -s

Удаление узла OSD

ceph osd safe-to-destroy osd.0
OSD(s) 0 are safe to destroy without reducing data durability.

ceph osd destroy 0 --yes-i-really-mean-it
ceph osd purge 0 --yes-i-really-mean-it

Управление приоритетом использования узла OSD

ceph osd crush reweight osd.2 0
ceph osd crush reweight osd.2 0.00389

node3.corp13.un:~# ceph osd df
 0   hdd       0  0.83000 4.00GiB 1.01GiB 2.99GiB 25.30 0.37   0
 1   hdd 0.00389  0.83000 4.00GiB 2.78GiB 1.21GiB 69.66 1.02  74
 2   hdd 0.00389  0.83000 4.00GiB 3.04GiB  980MiB 76.04 1.11  81
 3   hdd 0.00389  1.00000 4.00GiB 3.43GiB  575MiB 85.96 1.26  99
 4   hdd 0.00389  1.00000 4.00GiB 3.38GiB  631MiB 84.58 1.24 102

ceph osd reweight osd.2 1


Расчет параметров и создание POOL

Total PGs = (Total_number_of_OSD * 100) / max_replication_count

5*100/3 = 166,6...

ceph osd pool create test-pool1 180

ceph -s
    pgs:     180 active+clean

ceph pg dump

ceph osd lspools

Изменение параметров POOL

ceph osd pool set test-pool1 pg_num 250
ceph osd pool set test-pool1 pgp_num 250

ceph osd pool get test-pool1 size
ceph osd pool get test-pool1 min_size

ceph osd pool set test-pool1 size 2
ceph osd pool set test-pool1 size 3


ceph df

rbd pool init test-pool1
rbd create -p test-pool1 rbd1 --size 1024

rbd list test-pool1
rbd info test-pool1/rbd1
rbd resize --size 3G test-pool1/rbd1
rbd resize --size 3584M test-pool1/rbd1

nodeN# systemctl restart tgt


nodeN# apt install tgt-rbd

nodeN# cat /etc/tgt/conf.d/ceph.conf
<target virtual-ceph-test-pool1-rbd1:iscsi>
    driver iscsi
    bs-type rbd
    backing-store test-pool1/rbd1
    initiator-address ALL
scp /etc/tgt/conf.d/ceph.conf node4:/etc/tgt/conf.d/
scp /etc/tgt/conf.d/ceph.conf node5:/etc/tgt/conf.d/
bash -c '
systemctl restart tgt
ssh node4 systemctl restart tgt
ssh node5 systemctl restart tgt
systemctl status tgt


# ceph osd find 0

ceph health detail


Case #1. Gracefully taking out OSDs from the Ceph cluster
ceph tell 'osd.*' injectargs --osd-max-backfills=1
решение_ceph.1622278866.txt.gz · Last modified: 2021/05/29 12:01 by val