CEPH

此材料正在开发中...

开发中
（最后编辑者：WhyNotHugo，于 2025 年 2 月 6 日。）

Ceph 是一个软件定义的存储平台。有多种方法和工具可以部署 Ceph，但据我所知，没有一种可以在 Alpine 上工作。自 Alpine 3.10 以来，Ceph 软件包已在社区仓库中提供。通常，会提供当前最新版本和以前的版本。

本指南介绍了如何通过 apk 手动部署 Ceph monitors、managers 和 OSD。

安装 MONITOR

引导第一个 MONITOR

在此示例中，我们使用 3 台主机作为 monitor 和 manager 角色。

MON_HOST1="mon01"
MON_HOST1_IP="%IP_ADDRESS%"
MON_HOST2="mon02"
MON_HOST2_IP="%IP_ADDRESS%"
MON_HOST3="mon03"
MON_HOST3_IP="%IP_ADDRESS%"
FSID=$(cat /proc/sys/kernel/random/uuid) #store this FSID
echo $FSID
CLUSTER_NAME="ceph" # default value if unspecified
HOSTNAME=$(cat /etc/hostname)
PUBLIC_NETWORK="%NETWORK_CIDR%"
MYNET="%NETWORK_CIDR%"
VERSION=17

#apk add ceph$VERSION-mon ceph$VERSION-mon-daemon ceph$VERSION-mon-tools ceph$VERSION-openrc sudo

cat << EOF > /etc/ceph/$CLUSTER_NAME.conf 
[global]
# Cluster unique identifier
fsid = $FSID
mon_initial_members = $MON_HOST1
mon_host = $MON_HOST1_IP, $MON_HOST2_IP, $MON_HOST3_IP
mon_allow_pool_delete = true
ms_bind_ipv4 = false # change as needed
ms_bind_ipv6 = true # change as needed
public_network = $PUBLIC_NETWORK
# Enable authentication
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
# https://docs.ceph.com/en/latest/rados/configuration/pool-pg-config-ref/#pool-pg-and-crush-config-reference
osd_pool_default_size = 3 # Write an object three times
osd_pool_default_min_size = 2 # Accept an I/O operation to a degraded PG that has two copies of an object
osd_pool_default_pg_num = 128 # total number of OSDs * 100 / osd_pool_default_size. Use nearest power of two.
osd_crush_chooseleaf_type = 1
rgw_data = /var/lib/ceph/radosgw/\$cluster-\$id # literal variables

[mon]
mon_data = /var/lib/ceph/mon/\$cluster-$FSID # "cluster" is a literal variable
EOF

ceph-authtool --create-keyring /tmp/$CLUSTER_NAME.mon.keyring --gen-key -n mon. --cap mon 'allow *'
ceph-authtool --create-keyring /etc/ceph/$CLUSTER_NAME.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'
ceph-authtool --create-keyring /var/lib/ceph/bootstrap-osd/$CLUSTER_NAME.keyring --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' --cap mgr 'allow r'

ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/$CLUSTER_NAME.client.admin.keyring
ceph-authtool /tmp/ceph.mon.keyring --import-keyring /var/lib/ceph/bootstrap-osd/$CLUSTER_NAME.keyring

chown ceph:ceph /tmp/$CLUSTER_NAME.mon.keyring

monmaptool --create --add $MON_HOST1 $MON_HOST1_IP --add $MON_HOST2 $MON_HOST2_IP --add $MON_HOST3 $MON_HOST3_IP --fsid $FSID /tmp/monmap

install -d -o ceph /var/lib/ceph/mon/$CLUSTER_NAME-$FSID

sudo -u ceph ceph-mon --cluster $CLUSTER_NAME --mkfs -i $HOSTNAME --inject-monmap /tmp/monmap --keyring /tmp/$CLUSTER_NAME.mon.keyring

创建 AWall 策略

cat << EOF > /etc/awall/optional/ceph-mon.json
{
  "description": "Ceph cluster monitor component",

  "service": {
    "ceph-mon": { "proto": "tcp", "port": [ 3300, 6789 ] }
  },

  "filter": [
    {
      "src": "\$MYNET",
      "out": "_fw",
      "service": "ceph-mon",
      "action": "accept"
    },
    {
      "in": "_fw",
      "dest": "\$MYNET",
      "service": "ceph-mon",
      "action": "accept"
    }
  ]
}
EOF

cat << EOF > /etc/awall/optional/ceph-client-osd.json
{
  "description": "Ceph cluster OSD client",
  
  "service": {
    "ceph-osd": { "proto": "tcp", "port": "6800-7300" }
  },

  "filter": [
    {
      "in": "_fw",
      "dest": "\$MYNET",
      "service": "ceph-osd",
      "action": "accept"
    }
  ]
}
EOF

awall enable ceph-mon
awall enable ceph-client-osd
awall activate -f
ln -s ceph /etc/init.d/ceph-mon.$HOSTNAME
rc-update add ceph-mon.$HOSTNAME
openrc

检查第一个节点是否已正确引导，运行

ceph -s

添加其他 MONITOR 节点

将 /etc/ceph/ceph.conf、/tmp/ceph.mon.keyring、/etc/ceph/ceph.client.admin.keyring、/var/lib/ceph/bootstrap-osd/ceph.keyring 从现有 monitor 复制到新的 monitor 节点。

install -d --owner ceph --group ceph /etc/ceph /var/lib/ceph/bootstrap-osd
install -d -o ceph /var/lib/ceph/mon/$CLUSTER_NAME-$FSID
chown ceph /tmp/$CLUSTER_NAME.mon.keyring

monmaptool --create --add $MON_HOST1 $MON_HOST1_IP --add $MON_HOST2 $MON_HOST2_IP --add $MON_HOST3 $MON_HOST3_IP --fsid $FSID /tmp/monmap

sudo -u ceph ceph-mon --cluster $CLUSTER_NAME --mkfs -i $HOSTNAME --inject-monmap /tmp/monmap --keyring /tmp/$CLUSTER_NAME.mon.keyring

ln -s ceph /etc/init.d/ceph-mon.$HOSTNAME
rc-update add ceph-mon.$HOSTNAME
openrc

如果 ceph -s 返回 mon is allowing insecure global_id reclaim，请使用以下命令修复：

ceph config set mon auth_allow_insecure_global_id_reclaim false

安装 MANAGER

install -d --owner ceph --group ceph /var/lib/ceph/mgr/$CLUSTER_NAME-$HOSTNAME
# note this keyring is unique per manager, does not need to be copied across the cluster!
ceph auth get-or-create mgr.$HOSTNAME mon 'allow profile mgr' osd 'allow *' mds 'allow *' > /var/lib/ceph/mgr/$CLUSTER_NAME-$HOSTNAME/keyring

cat << EOF > /etc/awall/optional/ceph-mgr.json
{
  "description": "Ceph cluster Manager component",
  
  "service": {
    "ceph-mgr": { "proto": "tcp", "port": "6800-7300" }
  },

  "filter": [
    {
      "src": "\$MYNET",
      "out": "_fw",
      "service": "ceph-mgr",
      "action": "accept"
    }
  ]
}
EOF

awall enable ceph-mgr
awall activate -f

apk add ceph$VERSION-mgr ceph$VERSION-mgr-dashboard
ln -s ceph /etc/init.d/ceph-mgr.$HOSTNAME
rc-update add ceph-mgr.$HOSTNAME
openrc

预期问题

模块 'restful' 依赖项失败：PyO3 模块在每个解释器进程中只能初始化一次

https://github.com/bazaah/aur-ceph/issues/20

https://lists.ceph.io/hyperkitty/list/ceph-users@ceph.io/thread/FB7XE6WYDK3EBJYPABSPX5B2LEILWWJA/#FB7XE6WYDK3EBJYPABSPX5B2LEILWWJA

解决方法：ceph mgr module disable restful

安装 OSD

setup-devd udev #alpine eudev drop-in replacement for udev
apk add ceph$VERSION-osd ceph$VERSION-osd-daemon ceph$VERSION-osd-tools eudev ceph$VERSION-openrc
mkdir -p /var/lib/ceph/bootstrap-osd /etc/ceph

将 /var/lib/ceph/bootstrap-osd/$CLUSTER_NAME.keyring 从 monitor 节点复制到 /var/lib/ceph/bootstrap-osd/$CLUSTER_NAME.keyring。

将 /etc/ceph/ceph.conf 从 monitor 节点复制到 /etc/ceph/ceph.conf。

cat << EOF > /etc/awall/optional/ceph-client-mon.json
{
  "description": "Ceph cluster monitor client",
  
  "service": {
    "ceph-mon": { "proto": "tcp", "port": [ 3300, 6789 ] }
  },

  "filter": [
    {
      "in": "_fw",
      "dest": "\$MYNET",
      "service": "ceph-mon",
      "action": "accept"
    }
  ]
}
EOF

cat << EOF > /etc/awall/optional/ceph-osd.json
{
  "description": "Ceph cluster OSD component",
  
  "service": {
    "ceph-osd": { "proto": "tcp", "port": "6800-7300" }
  },

  "filter": [
    {
      "src": "\$MYNET",
      "out": "_fw",
      "service": "ceph-osd",
      "action": "accept"
    },
    {
      "in": "_fw",
      "dest": "\$MYNET",
      "service": "ceph-osd",
      "action": "accept"
    }
  ]
}
EOF

awall enable ceph-client-mon
awall enable ceph-osd
awall activate -f

ln -s /var/lib/ceph/bootstrap-osd/ceph.keyring /etc/ceph/ceph.client.bootstrap-osd.keyring #it seems it wants the keyring in this location
ceph-volume lvm create --data /dev/sdX --no-systemd

参见