image.png

CI/CD生产实战项目

先把k8s的二进制命令行工具kubectl容器化备用

FROM alpine:3.13

MAINTAINER test

ENV TZ "Asia/Shanghai"

RUN sed -ri 's+dl-cdn.alpinelinux.org+mirrors.aliyun.com+g' /etc/apk/repositories \
 && apk add --no-cache curl tzdata ca-certificates \
 && cp -f /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
 && apk upgrade \
 && rm -rf /var/cache/apk/*

COPY kubectl /usr/local/bin/
RUN chmod +x /usr/local/bin/kubectl

ENTRYPOINT ["kubectl"]
CMD ["help"]

python的flask模块

准备好flask相关的代码文件上传到gitlab代码仓库

app.py

from flask import Flask
app = Flask(__name__)

@app.route('/')
def hello_world():
    return 'Hello,年.月.日.时'

@app.route('/gg/<username>')
def hello(username):
    return 'welcome' + ': ' + username + '!'

Dockerfile

FROM python:3.5-slim-stretch
MAINTAINER test

WORKDIR /kae/app

COPY requirements.txt .

RUN  sed -i 's/deb.debian.org/ftp.cn.debian.org/g' /etc/apt/sources.list \
  && sed -i 's/security.debian.org/ftp.cn.debian.org/g' /etc/apt/sources.list \
  && apt-get update -y \
  && apt-get install -y wget gcc libsm6 libxext6 libglib2.0-0 libxrender1 make \
  && apt-get clean && apt-get autoremove -y && rm -rf /var/lib/apt/lists/*
RUN pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple -r requirements.txt \
    && rm requirements.txt

COPY . .

EXPOSE 5000
HEALTHCHECK CMD curl --fail http://localhost:5000 || exit 1

ENTRYPOINT ["gunicorn", "app:app", "-c", "gunicorn_config.py"]

gunicorn_config.py

bind = '0.0.0.0:5000'
graceful_timeout = 3600
timeout = 1200
max_requests = 1200
workers = 1
worker_class = 'gevent'

requirements.txt

flask
gevent
gunicorn

在代码仓库变量配置里面配置如下变量值

Type        Key                        Value                    State        Masked
Variable   DOCKER_USER                 admin                   下面都关闭   下面都关闭
Variable   DOCKER_PASS                 test666
Variable   REGISTRY_URL                harbor.test.com
Variable   REGISTRY_NS                 product
File       KUBE_CONFIG_TEST            k8s相关config配置文件内容

gitlab页面设置变量值

b9ba5fd0e4bb1c022cbdf567d40830e.png

7408d555e9921cc28f4002328021d17.png

a5ad57ee5a1cd5e5392d903a823163e.png

a5ad57ee5a1cd5e5392d903a823163e.png

准备项目自动化配置文件.gitlab-ci.yml

stages:
  - build
  - deploy
  - rollback

# tag name need: 年.月.日.时
variables:
  namecb: "flask-test"
  svcport: "5000"
  replicanum: "2"
  ingress: "flask-test.test.com"
  certname: "mytls"
  CanarylIngressNum: "20"

.deploy_k8s: &deploy_k8s |
  if [ $CANARY_CB -eq 1 ];then cp -arf .project-name-canary.yaml ${namecb}-${CI_COMMIT_TAG}.yaml; sed -ri "s+CanarylIngressNum+${CanarylIngressNum}+g" ${namecb}-${CI_COMMIT_TAG}.yaml; sed -ri "s+NomalIngressNum+$(expr 100 - ${CanarylIngressNum})+g" ${namecb}-${CI_COMMIT_TAG}.yaml ;else cp -arf .project-name.yaml ${namecb}-${CI_COMMIT_TAG}.yaml;fi
  sed -ri "s+projectnamecb.test.com+${ingress}+g" ${namecb}-${CI_COMMIT_TAG}.yaml
  sed -ri "s+projectnamecb+${namecb}+g" ${namecb}-${CI_COMMIT_TAG}.yaml
  sed -ri "s+5000+${svcport}+g" ${namecb}-${CI_COMMIT_TAG}.yaml
  sed -ri "s+replicanum+${replicanum}+g" ${namecb}-${CI_COMMIT_TAG}.yaml
  sed -ri "s+mytls+${certname}+g" ${namecb}-${CI_COMMIT_TAG}.yaml
  sed -ri "s+mytagcb+${CI_COMMIT_TAG}+g" ${namecb}-${CI_COMMIT_TAG}.yaml
  sed -ri "s+harbor.test.com/library+${IMG_URL}+g" ${namecb}-${CI_COMMIT_TAG}.yaml
  cat ${namecb}-${CI_COMMIT_TAG}.yaml
  [ -d ~/.kube ] || mkdir ~/.kube
  echo "$KUBE_CONFIG" > ~/.kube/config
  if [ $NORMAL_CB -eq 1 ];then if kubectl get deployments.|grep -w ${namecb}-canary &>/dev/null;then kubectl delete deployments.,svc ${namecb}-canary ;fi;fi
  kubectl apply -f ${namecb}-${CI_COMMIT_TAG}.yaml --record
  echo
  echo
  echo "============================================================="
  echo "                    Rollback Indx List"
  echo "============================================================="
  kubectl rollout history deployment ${namecb}|tail -5|awk -F"[ =]+" '{print $1"\t"$5}'|sed '$d'|sed '$d'|sort -r|awk '{print $NF}'|awk '$0=""NR".   "$0'

.rollback_k8s: &rollback_k8s |
  [ -d ~/.kube ] || mkdir ~/.kube
  echo "$KUBE_CONFIG" > ~/.kube/config
  last_version_command=$( kubectl rollout history deployment ${namecb}|tail -5|awk -F"[ =]+" '{print $1"\t"$5}'|sed '$d'|sed '$d'|tail -${ROLL_NUM}|head -1 )
  last_version_num=$( echo ${last_version_command}|awk '{print $1}' )
  last_version_name=$( echo ${last_version_command}|awk '{print $2}' )
  kubectl rollout undo deployment ${namecb} --to-revision=$last_version_num
  echo $last_version_num
  echo $last_version_name
  kubectl rollout history deployment ${namecb}


build:
  stage: build
  retry: 2
  variables:
    # use dind.yaml to depoy dind'service on k8s
    DOCKER_HOST: tcp://10.68.86.33:2375/
    DOCKER_DRIVER: overlay2
    DOCKER_TLS_CERTDIR: ""
  ##services:
    ##- docker:dind
  before_script:
    - docker login ${REGISTRY_URL} -u "$DOCKER_USER" -p "$DOCKER_PASS"
  script:
    - docker pull ${REGISTRY_URL}/${REGISTRY_NS}/${namecb}:latest || true
    - docker build --network host --cache-from ${REGISTRY_URL}/${REGISTRY_NS}/${namecb}:latest --tag ${REGISTRY_URL}/${REGISTRY_NS}/${namecb}:$CI_COMMIT_TAG --tag ${REGISTRY_URL}/${REGISTRY_NS}/${namecb}:latest .
    - docker push ${REGISTRY_URL}/${REGISTRY_NS}/${namecb}:$CI_COMMIT_TAG
    - docker push ${REGISTRY_URL}/${REGISTRY_NS}/${namecb}:latest
  after_script:
    - docker logout ${REGISTRY_URL}
  tags:
    - "docker"
  only:
    - tags





#--------------------------K8S DEPLOY--------------------------------------------------

deploy:
  stage: deploy
  image: kubectl:v1.19.9
  variables:
    KUBE_CONFIG: "$KUBE_CONFIG_TEST"
    IMG_URL: "${REGISTRY_URL}/${REGISTRY_NS}"
    NORMAL_CB: 1
  script:
    - *deploy_k8s
  when: manual
  only:
    - tags

# canary start
canary-deploy:
  stage: deploy
  image: kubectl:v1.19.9
  variables:
    KUBE_CONFIG: "$KUBE_CONFIG_TEST"
    IMG_URL: "${REGISTRY_URL}/${REGISTRY_NS}"
    CANARY_CB: 1
  script:
    - *deploy_k8s
  when: manual
  only:
    - tags
# canary end

rollback-1:
  stage: rollback
  image: kubectl:v1.19.9
  variables:
    KUBE_CONFIG: "$KUBE_CONFIG_TEST"
    ROLL_NUM: 1
  script:
    - *rollback_k8s
  when: manual
  only:
    - tags


rollback-2:
  stage: rollback
  image: kubectl:v1.19.9
  variables:
    KUBE_CONFIG: "$KUBE_CONFIG_TEST"
    ROLL_NUM: 2
  script:
    - *rollback_k8s
  when: manual
  only:
    - tags


rollback-3:
  stage: rollback
  image: kubectl:v1.19.9
  variables:
    KUBE_CONFIG: "$KUBE_CONFIG_TEST"
    ROLL_NUM: 3
  script:
    - *rollback_k8s
  when: manual
  only:
    - tags

准备k8s的deployment模板文件 .project-name.yaml

这里要注意提前在K8S把harbor拉取的凭证secret给创建好，命令如下：

kubectl -n test create secret docker-registry test-secret --docker-server=harbor.test.com --docker-username=admin --docker-password=test666 --docker-email=admin@test.com

---
# SVC
kind: Service
apiVersion: v1
metadata:
  labels:
    kae: "true"
    kae-app-name: projectnamecb
    kae-type: app
  name: projectnamecb
spec:
  selector:
    kae: "true"
    kae-app-name: projectnamecb
    kae-type: app
  ports:
    - name: http-port
      port: 80
      protocol: TCP
      targetPort: 5000
#      nodePort: 12345
#  type: NodePort

---
# Ingress
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  labels:
    kae: "true"
    kae-app-name: projectnamecb
    kae-type: app
  name: projectnamecb
spec:
  tls:
  - hosts:
    - projectnamecb.test.com
    secretName: mytls
  rules:
  - host: projectnamecb.test.com
    http:
      paths:
      - path: /
        backend:
          serviceName: projectnamecb
          servicePort: 80

---
# Deployment
apiVersion: apps/v1
kind: Deployment
metadata:
  name: projectnamecb
  labels:
    kae: "true"
    kae-app-name: projectnamecb
    kae-type: app
spec:
  replicas: replicanum
  selector:
    matchLabels:
      kae-app-name: projectnamecb
  template:
    metadata:
      labels:
        kae: "true"
        kae-app-name: projectnamecb
        kae-type: app
    spec:
      containers:
      - name: projectnamecb
        image: projectnamecb:mytagcb
        env:
          - name: TZ
            value: Asia/Shanghai
        ports:
        - containerPort: 5000
        readinessProbe:
          httpGet:
            scheme: HTTP
            path: /
            port: 5000
          initialDelaySeconds: 10
          periodSeconds: 5
          timeoutSeconds: 3
          successThreshold: 1
          failureThreshold: 3
        livenessProbe:
          httpGet:
            scheme: HTTP
            path: /
            port: 5000
          initialDelaySeconds: 10
          periodSeconds: 5
          timeoutSeconds: 3
          successThreshold: 1
          failureThreshold: 3
        resources:
          requests:
            cpu: 0.3
            memory: 0.5Gi
          limits:
            cpu: 0.3
            memory: 0.5Gi
      imagePullSecrets:
      - name: test-secret

准备好K8S上金丝雀部署的模板文件 .project-name-canary.yaml

--
# SVC
kind: Service
apiVersion: v1
metadata:
  labels:
    kae: "true"
    kae-app-name: projectnamecb-canary
    kae-type: app
  name: projectnamecb-canary
spec:
  selector:
    kae: "true"
    kae-app-name: projectnamecb-canary
    kae-type: app
  ports:
    - name: http-port
      port: 80
      protocol: TCP
      targetPort: 5000
#      nodePort: 12345
#  type: NodePort

---
# Ingress
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  labels:
    kae: "true"
    kae-app-name: projectnamecb-canary
    kae-type: app
  name: projectnamecb
  annotations:
    nginx.ingress.kubernetes.io/service-weight: |
        projectnamecb: NomalIngressNum, projectnamecb-canary: CanarylIngressNum
spec:
  tls:
  - hosts:
    - projectnamecb.test.com
    secretName: mytls
  rules:
  - host: projectnamecb.test.com
    http:
      paths:
      - path: /
        backend:
          serviceName: projectnamecb
          servicePort: 80
      - path: /
        backend:
          serviceName: projectnamecb-canary
          servicePort: 80

---
# Deployment
apiVersion: apps/v1
kind: Deployment
metadata:
  name: projectnamecb-canary
  labels:
    kae: "true"
    kae-app-name: projectnamecb-canary
    kae-type: app
spec:
  replicas: replicanum
  selector:
    matchLabels:
      kae-app-name: projectnamecb-canary
  template:
    metadata:
      labels:
        kae: "true"
        kae-app-name: projectnamecb-canary
        kae-type: app
    spec:
      containers:
      - name: projectnamecb-canary
        image: projectnamecb:mytagcb
        env:
          - name: TZ
            value: Asia/Shanghai
        ports:
        - containerPort: 5000
        readinessProbe:
          httpGet:
            scheme: HTTP
            path: /
            port: 5000
          initialDelaySeconds: 10
          periodSeconds: 5
          timeoutSeconds: 3
          successThreshold: 1
          failureThreshold: 3
        livenessProbe:
          httpGet:
            scheme: HTTP
            path: /
            port: 5000
          initialDelaySeconds: 10
          periodSeconds: 5
          timeoutSeconds: 3
          successThreshold: 1
          failureThreshold: 3
        resources:
          requests:
            cpu: 0.3
            memory: 0.5Gi
          limits:
            cpu: 0.3
            memory: 0.5Gi
      imagePullSecrets:
      - name: test-secret

最后，在修改完代码，提交tag版本号后，即会触发CI/CD自动化流程