k8s && ceph 环境信息

ceph 部署以及配置

搭建ceph集群

# 添加host，每台都要添加
192.168.9.177    swarm01
192.168.9.178    swarm02
192.168.9.181    swarm03

# 创建目录
mkdir ceph-deploy && cd ceph-deploy
# 初始化集群
ceph-deploy --overwrite-conf new swarm01 swarm02 swarm03
echo 'public_network = 192.168.9.0/24' >> ceph.conf
# 安装ceph包
ceph-deploy --overwrite-conf install swarm01 swarm02 swarm03
# 初始化monitor节点
ceph-deploy --overwrite-conf mon create-initial
# 创建admin节点
ceph-deploy --overwrite-conf admin swarm01 swarm02 swarm03
# 创建mds（不创造这个就不能用mount挂载）
ceph-deploy --overwrite-conf mds create swarm01 swarm02 swarm03
# 创建管理节点
ceph-deploy mgr create swarm01
# 创建存储
ceph-deploy --overwrite-conf osd create --data /dev/sdb swarm01
ceph-deploy --overwrite-conf osd create --data /dev/sdb swarm02
ceph-deploy --overwrite-conf osd create --data /dev/sdb swarm03
# 查看集群状态是否正常
ceph -s

ceph 集群配置

# 使用cephfs必须创建元数据
ceph osd pool create cephfs_data 128 
ceph osd pool create cephfs_metadata 32
    * 若少于5个OSD， 设置pg_num为128。
    * 5~10个OSD，设置pg_num为512。
    * 10~50个OSD，设置pg_num为4096。
    * 超过50个OSD，可以参考pgcalc计算。
# 创建文件系统
ceph fs new cephfs cephfs_metadata cephfs_data

ceph 创建MySQL存储目录

# 创建y-mysql用户，并且对y-mysql目录赋权
ceph auth get-or-create client.y-mysql mon 'allow r' mds 'allow r, allow rw path=/y-mysql' osd 'allow rw pool=cephfs_data'
# 获取y-mysql用户令牌并base64一下，后面k8s要用到
ceph auth get-key client.y-mysql | base64
    -> QVFBWWZveGROM1ZiSXhBQVptcHNkbXRqQXRjenBnamNIZXV3Unc9PQ== # 执行结果

k8s

创建namespace

kubectl create namespace y

创建ceph令牌

vim ceph-y-mysql-secret.yml

apiVersion: v1
kind: Secret
metadata:
  name: ceph-y-mysql-secret
  namespace: "y"
data:
  key: QVFBWWZveGROM1ZiSXhBQVptcHNkbXRqQXRjenBnamNIZXV3Unc9PQ== # 上面的ceph令牌

创建pv

vim ceph-y-mysql-pv.yml

apiVersion: v1
kind: PersistentVolume
metadata:
  name: y-mysql-pv
  namespace: "y"
  labels:
    app: y-mysql
spec:
  capacity:
    storage: 2Gi
  accessModes:
    - ReadWriteMany
  cephfs:
    monitors:
      - 192.168.9.177:6789
      - 192.168.9.178:6789
      - 192.168.9.181:6789
    user: y-mysql
    secretRef:
      name: ceph-y-mysql-secret
    readOnly: false
  persistentVolumeReclaimPolicy: Recycle

创建pvc

vim ceph-y-mysql-pvc.yml

kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: y-mysql-pvc
  namespace: "y"
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 2Gi

创建deployment

vim y-mysql-deployment.yml

apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: mysql
    provider: "y"
    version: "5.7"
    group: "top.yuser"
    env: "cdp"
  name: y-mysql
  namespace: "y"
spec:
  replicas: 1
  selector:
    matchLabels:
      mysql-app: y-mysql-devops
      provider: "y"
      group: "top.yuser"
  template:
    metadata:
      labels:
        mysql-app: y-mysql-devops
        provider: "y"
        group: top.yuser
    spec:
      containers:
      - env:
        - name: MYSQL_ROOT_PASSWORD
          value: "XXXX"  # mysql root用户密码
        - name: MYSQL_USER
          value: "XXXX"  # mysql 用户名
        - name: MYSQL_PASSWORD
          value: "XXXX"  # mysql 密码
        image: "mysql:5.7"
        imagePullPolicy: IfNotPresent
        name: y-mysql
        ports:
        - containerPort: 3306
          protocol: TCP
          name: tcp
        volumeMounts:
        - name: mysqldata
          mountPath: /var/lib/mysql
          readOnly: false
          subPath: y-mysql/   # ceph目录做了权限，这么写为把ceph的/y-mysql挂载到pod的/var/lib/mysql
      volumes:
      - name: mysqldata
        persistentVolumeClaim:
          claimName: y-mysql-pvc

创建sevice

vim y-mysql-svc.yml

apiVersion: v1
kind: Service
metadata:
  name: y-mysql-service
  namespace: "y"
spec:
  ports:
  - port: 3306
    protocol: TCP
    targetPort: 3306
    nodePort: 30006
  type: NodePort  # 这里使用nodeport模式，可直接连接mysql
  selector:
    mysql-app: y-mysql-devops

apply以上几个yml文件

kubectl apply -f ceph-y-mysql-secret.yml
kubectl apply -f ceph-y-mysql-pv.yml
kubectl apply -f ceph-y-mysql-pvc.yml

kubectl apply -f y-mysql-deployment.yml
kubectl apply -f y-mysql-svc.yml

测试访问

用navicat 连接192.168.9.142的 30006端口，即可连接到MySQL