我的第一个开源项目-jenkins集成k8s项目

一、Jenkins 实现 K8S 持续集成项目架构图

1.gitlab项目部署

1.1 gitlab的Service资源清单

## 创建项目专用命名空间
[root@k8s-master01 ~]# kubectl create namespace devops
namespace/devops created

## gitlab是有状态服务，需要使用StatefulSet部署`在这里插入代码片`
## StatefulSet需要依赖svc无头服务

[root@k8s-master01 ~]# mkdir /devops
[root@k8s-master01 ~]# cd /devops/
[root@k8s-master01 devops]# vim 01-gitlab-svc.yaml    ## 创建无头svc
apiVersion: v1
kind: Service
metadata:
  name: svc-gitlab
  namespace: devops
spec:
  clusterIP: None
  selector:
    app: gitlab
  ports:
    - name: http
      port: 80
      targetPort: 80
    - name: https
      port: 443
      targetPort: 443

1.2 编写gitlab的StatefulSet资源清单

[root@k8s-master01 devops]# vim 02-gitlab-sts.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: gitlab
  namespace: devops
spec:
  serviceName: "svc-gitlab"
  selector:
    matchLabels:
      app: gitlab
  template:
    metadata:
      labels:
        app: gitlab
    spec:
      containers:
        - name: gitlab-ce
          image: 10.0.0.200/devops/gitlab-ce:14.6.0-ce.0
          imagePullPolicy: IfNotPresent
          env:
            - name: GITLAB_ROOT_PASSWORD
              value: "admin123"
            - name: GITLAB_OMNIBUS_CONFIG
              value: |
                external_url "http://www.gitlab.com"
                gitlab_rails['time_zone'] = 'Asia/Shanghai'
                node_exporter['enable'] = false
                redis_exporter['enable'] = false
                postgres_exporter['enable'] = false
                gitlab_exporter['enable'] = false
                grafana['enable'] = false
                grafana['reporting_enabled'] = false
                prometheus['enable'] = false
                prometheus['monitor_kubernetes'] = false
          ports:
            - name: http
              containerPort: 80
            - name: https
              containerPort: 443
          volumeMounts:
            - name: data
              mountPath: /etc/gitlab
              subPath: config
            - name: data
              mountPath: /var/opt/gitlab
              subPath: data
            - name: data
              mountPath: /var/log/gitlab
              subPath: logs
      volumes:
        - name: data
          persistentVolumeClaim:
            claimName: pvc-gitlab

1.3 创建gitlab的sc

[root@k8s-master01 devops]# cat /manifests/csi/sc-devops.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: sc-devops
provisioner: rbd.csi.ceph.com
parameters:
  clusterID: 571a3bbf-1ac1-4859-949c-2b9fc5655d3
  pool: devops
  imageFeatures: layering
  csi.storage.k8s.io/provisioner-secret-name: csi-rbd-secret
  csi.storage.k8s.io/provisioner-secret-namespace: default
  csi.storage.k8s.io/controller-expand-secret-name: csi-rbd-secret
  csi.storage.k8s.io/controller-expand-secret-namespace: default
  csi.storage.k8s.io/node-stage-secret-name: csi-rbd-secret
  csi.storage.k8s.io/node-stage-secret-namespace: default
reclaimPolicy: Delete
allowVolumeExpansion: true
mountOptions:
  - discard

1.4 创建gitlab的pvc

[root@k8s-master01 devops]# vim 03-gitlab-pvc.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: pvc-gitlab
  namespace: devops
spec:
  accessModes:
    - ReadWriteOnce
  volumeMode: Filesystem
  resources:
    requests:
      storage: 30Gi
  storageClassName: sc-devops

1.5 创建gitlab的ingress

[root@k8s-master01 devops]# vim 04-gitlab-ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-gitlab
  namespace: devops
spec:
  ingressClassName: "nginx"
  rules:
    - host: www.gitlab.com
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: svc-gitlab
                port:
                  name: http

## windows hosts 域名解析
10.103.236.201   www.gitlab.com

2.部署sonarqube

2.1 部署依赖的数据库 pgsql

[root@k8s-master01 devops]# mkdir sonarqube
[root@k8s-master01 devops]# cd sonarqube/

## pgsql的svc
[root@k8s-master01 sonarqube]# vim 01-pgsql-svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: svc-pgsql
  namespace: devops
spec:
  clusterIP: None
  selector:
    app: pgsql
  ports:
    - port: 5432
      targetPort: 5432

2.2 创建pgsql的sts

[root@k8s-master01 sonarqube]# vim 02-pgsql-sts.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: psetgresql
  namespace: devops
spec:
  serviceName: svc-pgsql
  replicas: 1
  selector:
    matchLabels:
      app: pgsql
  template:
    metadata:
      labels:
        app: pgsql
    spec:
      containers:
        - name: postgres
          image: 10.0.0.200/devops/postgres:13.8
          ports:
            - containerPort: 5432
          env:
            - name: POSTGRES_DB
              value: sonardb
            - name: POSTGRES_USER
              value: sonar
            - name: POSTGRES_PASSWORD
              value: "123456"
          volumeMounts:
            - name: db
              mountPath: /var/lib/postgresql/data
              subPath: data
            - name: db
              mountPath: /var/run/secrets/kubernetes.io/serviceaccount
              subPath: serviceaccount
      volumes:
        - name: db
          persistentVolumeClaim:
            claimName: pvc-pgsql

2.3 创建pgsql的sc

[root@k8s-master01 devops]# cat /manifests/csi/sc-pgsql.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: sc-pgsql
provisioner: rbd.csi.ceph.com
parameters:
  clusterID: 571a3bbf-1ac1-4859-949c-2b9fc5655d3
  pool: pgsql
  imageFeatures: layering
  csi.storage.k8s.io/provisioner-secret-name: csi-rbd-secret
  csi.storage.k8s.io/provisioner-secret-namespace: default
  csi.storage.k8s.io/controller-expand-secret-name: csi-rbd-secret
  csi.storage.k8s.io/controller-expand-secret-namespace: default
  csi.storage.k8s.io/node-stage-secret-name: csi-rbd-secret
  csi.storage.k8s.io/node-stage-secret-namespace: default
reclaimPolicy: Delete
allowVolumeExpansion: true
mountOptions:
  - discard

2.4 创建pgsql的pvc

[root@k8s-master01 devops]# vim 03-pgsql-pvc.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: pvc-pgsql
  namespace: devops
spec:
  accessModes:
    - ReadWriteOnce
  volumeMode: Filesystem
  resources:
    requests:
      storage: 30Gi
  storageClassName: sc-pgsql


## ceph 创建 pool
[root@node-1 ~]# ceph osd pool create pgsql 16 16
pool 'pgsql' created

2.5 创建sc

[root@k8s-master01 sonarqube]# cat sc-pgsql.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: sc-pgsql
provisioner: rbd.csi.ceph.com
parameters:
  clusterID: 571a3bbf-1ac1-4859-949c-2b9fc5655d3
  pool: pgsql
  imageFeatures: layering
  csi.storage.k8s.io/provisioner-secret-name: csi-rbd-secret
  csi.storage.k8s.io/provisioner-secret-namespace: default
  csi.storage.k8s.io/controller-expand-secret-name: csi-rbd-secret
  csi.storage.k8s.io/controller-expand-secret-namespace: default
  csi.storage.k8s.io/node-stage-secret-name: csi-rbd-secret
  csi.storage.k8s.io/node-stage-secret-namespace: default
reclaimPolicy: Delete
allowVolumeExpansion: true
mountOptions:
  - discard


## 进入pgsql验证
[root@k8s-master01 sonarqube]# kubectl exec -it -n devops postgresql-0 -- bash
root@postgresql-0:/# psql -Usonar -d sonardb
sonardb=# \l
                                  List of databases
   Name    | Owner | Encoding | Collate     | Ctype       | Access privileges
-----------+-------+----------+-------------+-------------+-------------------
 postgres  | sonar | UTF8     | en_US.utf8  | en_US.utf8  |
 sonardb   | sonar | UTF8     | en_US.utf8  | en_US.utf8  | =c/sonar         +
           |       |          |             |             | sonar=CTc/sonar
 template0 | sonar | UTF8     | en_US.utf8  | en_US.utf8  | =c/sonar         +
           |       |          |             |             | sonar=CTc/sonar
 template1 | sonar | UTF8     | en_US.utf8  | en_US.utf8  | =c/sonar         +
           |       |          |             |             | sonar=CTc/sonar

2.6 创建sonarqube-svc

[root@k8s-master01 sonarqube]# vim 04-sonarqube-svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: svc-sonarqube
  namespace: devops
spec:
  clusterIP: None
  selector:
    app: sonarqube
  ports:
    - name: web
      port: 9000
      targetPort: 9000

2.7 部署sonarqube

[root@k8s-master01 sonarqube]# vim 05-sonarqube-sts.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: sonarqube
  namespace: devops
spec:
  serviceName: svc-sonarqube
  selector:
    matchLabels:
      app: sonarqube
  template:
    metadata:
      labels:
        app: sonarqube
    spec:
      initContainers:
        - name: set-kernel
          image: busybox
          command: ["sh", "-c", "sysctl -w vm.max_map_count=524288 ; sysctl -w fs.file-max=131072 ; ulimit -n 131072 ; ulimit -u 8192"]
          securityContext:
            privileged: true
      containers:
        - name: sonarqube
          image: 10.0.0.200/devops/sonarqube:9.7-community
          ports:
            - name: web
              containerPort: 9000
          env:
            - name: JAVA_OPTS
              value: -Duser.timezone=Asia/Shanghai
            - name: SONARQUBE_JDBC_USERNAME
              value: sonar  ## 连接pgsql用户名
            - name: SONARQUBE_JDBC_PASSWORD
              value: "123456"  ## 连接pgsql密码
            - name: SONARQUBE_JDBC_URL
              value: jdbc:postgresql://svc-pgsql:5432/sonardb
          resources:
            limits:
              cpu: 1500m
              memory: 2048Mi
          volumeMounts:
            - name: data
              mountPath: /opt/sonarqube/data
              subPath: data
            - name: data
              mountPath: /opt/sonarqube/logs
              subPath: logs
            - name: data
              mountPath: /opt/sonarqube/extensions
              subPath: extensions
            - name: data
              mountPath: /var/run/secrets/kubernetes.io/serviceaccount
              subPath: serviceaccount
        volumeClaimTemplates:
          - metadata:
              name: data
              namespace: devops
            spec:
              accessModes:
                - ReadWriteOnce
              storageClassName: "sc-devops"
              resources:
                requests:
                  storage: 50Gi

2.8 创建sonarqube的ingress规则

[root@k8s-master01 sonarqube]# vim 06-sonarqube-ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-sonarqube
  namespace: devops
spec:
  ingressClassName: "nginx"
  rules:
    - host: www.sonarqube.com
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: svc-sonarqube
                port:
                  number: 9000

##web浏览器访问www.sonarqube.com(记得解析)
账号 admin 密码 admin 登陆后密码改为 123456

3.jenkins部署

3.1 (Jenkins不需要数据库，数据存在本地，StatefulSet部署)

Jenkins 需要创建 Slave Pod 来执行流水线构建，这就需要与 apiserver 交互，所以就需要 RBAC 权限

[root@k8s-master01 devops]# pwd
/devops
[root@k8s-master01 devops]# mkdir jenkins
[root@k8s-master01 devops]# cd jenkins/
[root@k8s-master01 jenkins]# vim 01-jenkins-rbac.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: jenkins
  namespace: devops
    
---

kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: jenkins
rules:
- apiGroups: ["extensions", "apps"]
  resources: ["deployments", "ingresses"]
  verbs: ["create", "delete", "get", "list", "watch", "patch", "update"]
- apiGroups: [""]
  resources: ["services"]
  verbs: ["create", "delete", "get", "list", "watch", "patch", "update"]
- apiGroups: [""]
  resources: ["pods"]
  verbs: ["create", "delete", "get", "list", "patch", "update", "watch"]
- apiGroups: [""]
  resources: ["pods/exec"]
  verbs: ["create", "delete", "get", "list", "patch", "update", "watch"]
- apiGroups: [""]
  resources: ["pods/log", "events"]
  verbs: ["get", "list", "watch"]
- apiGroups: [""]
  resources: ["secrets"]
  verbs: ["get"]

---

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: jenkins
  namespace: devops
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: jenkins
subjects:
- kind: ServiceAccount
  name: jenkins
  namespace: devops

[root@k8s-master01 jenkins]# kubectl create -f 01-jenkins-rbac.yaml

3.2 创建 Jenkins master 的 svc

[root@k8s-master01 jenkins]# vim 02-jenkins-svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: svc-jenkins
  namespace: devops
spec:
  clusterIP: None
  selector:
    app: jenkins
  ports:
  - name: http
    port: 8080
    targetPort: 8080
  - name: agent
    port: 50000
    targetPort: 50000

3.3 Jenkins 创建 statefulset

[root@k8s-master01 jenkins]# vim 03-jenkins-sts.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: jenkins
  namespace: devops
spec:
  serviceName: svc-jenkins
  selector:
    matchLabels:
      app: jenkins
  template:
    metadata:
      labels:
        app: jenkins
    spec:
      serviceAccount: jenkins
      containers:
      - name: jenkins
        image: 10.0.0.200/devops/jenkins:2.346.3-2-lts
        imagePullPolicy: IfNotPresent
        securityContext:
          privileged: true
          runAsUser: 0
        env:
        - name: JAVA_OPTS
          value: -Duser.timezone=Asia/Shanghai
        ports:
        - name: http
          containerPort: 8080
        - name: agent
          containerPort: 50000
        resources:
          limits:
            cpu: 1500m
            memory: 2048Mi
        readinessProbe:
          httpGet:
            path: /login
            port: 8080
          initialDelaySeconds: 60
          timeoutSeconds: 5
          failureThreshold: 12
        volumeMounts:
        - name: data
          mountPath: /var/jenkins_home
          subPath: jenkins_home
  volumeClaimTemplates:
  - metadata:
      name: data
    spec:
      accessModes: ["ReadWriteOnce"]
      storageClassName: "sc-devops"
      resources:
        requests:
          storage: 20Gi

3.4 创建 Jenkins 的 Ingress

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-jenkins
  namespace: devops
spec:
  ingressClassName: "nginx"
  rules:
  - host: www.jenkins.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: svc-jenkins
            port:
              name: http


## windows hosts 解析这个域名
## 查看jenkins密码

安装 Jenkins 插件：（这些又有的可能安装失败，其他安装完成后重启 jenkins 再装一遍，然后记得再已安装中降级一下，不然可能会有兼容性问题）

中文插件：Localization: Chinese (Simplified)  
Git 插件：git, gitlab  
Sonar 插件：SonarQube Scanner（这是一个）  
Pipeline 插件：pipeline、Stage View、Blue Ocean  
Kubernetes 插件：Kubernetes

4.dockerfile 镜像制作

4.1 Jenkins 镜像模板

## Jenkins
FROM 10.0.0.200/devops/jenkins:2.346.3-2-lts

USER root

# 设置时区
RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \
    echo "Asia/Shanghai" > /etc/timezone

# 安装必要工具
RUN apt-get update && \
    apt-get install -y fontconfig git curl unzip nfs-common && \
    rm -rf /var/lib/apt/lists/*

# 配置 Jenkins 插件（可选）
COPY plugins.txt /usr/share/jenkins/ref/plugins.txt
RUN jenkins-plugin-cli --plugin-file /usr/share/jenkins/ref/plugins.txt

# 挂载目录
VOLUME ["/var/jenkins_home"]

EXPOSE 8080 50000

ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/jenkins.sh"]

4.2 Maven 镜像模板

## Maven
FROM 10.0.0.200/pipeline/maven:3.8.6-openjdk-8

ADD ./settings_docker.xml /usr/share/maven/conf/settings.xml

RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \
    apt-get update && \
    apt-get install -y nfs-utils && \
    rm -rf /var/lib/apt/lists/*

# 构建命令
# docker build -t 10.0.0.200/pipeline/maven:3.8.6-openjdk-8 .

4.3 SonarQube Scanner 镜像模板

## SonarQube Scanner
FROM 10.0.0.200/devops/sonar-scanner:4.8

# 设置时区
RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \
    apt-get update && \
    apt-get install -y nfs-common && \
    rm -rf /var/lib/apt/lists/*

# 配置 SonarQube 扫描器参数
COPY sonar-scanner.properties /opt/sonar-scanner/conf/sonar-scanner.properties

ENTRYPOINT ["/opt/sonar-scanner/bin/sonar-scanner"]

4.4 Kubernetes Agent 镜像模板

## Kubernetes Jenkins Agent
FROM 10.0.0.200/devops/jenkins-agent:latest

USER root

RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \
    apt-get update && \
    apt-get install -y nfs-common git curl && \
    rm -rf /var/lib/apt/lists/*

ENTRYPOINT ["/usr/local/bin/jenkins-agent"]

4.5 GitLab 镜像模板

## GitLab
FROM 10.0.0.200/devops/gitlab-ce:15.3.2-ce.0

# 设置时区
RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \
    echo "Asia/Shanghai" > /etc/timezone

# 安装必要工具
RUN apt-get update && \
    apt-get install -y curl vim nfs-common && \
    rm -rf /var/lib/apt/lists/*

# 配置 GitLab（可选）
# COPY gitlab.rb /etc/gitlab/gitlab.rb
# RUN gitlab-ctl reconfigure

EXPOSE 80 443 22

VOLUME ["/etc/gitlab", "/var/log/gitlab", "/var/opt/gitlab"]

4.6 PostgreSQL 镜像模板

## PostgreSQL
FROM 10.0.0.200/devops/postgres:13

# 设置时区
RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \
    echo "Asia/Shanghai" > /etc/timezone

# 安装必要工具
RUN apt-get update && \
    apt-get install -y nfs-common && \
    rm -rf /var/lib/apt/lists/*

# 复制初始化 SQL（可选）
# COPY init.sql /docker-entrypoint-initdb.d/

ENV POSTGRES_USER=gitlab
ENV POSTGRES_PASSWORD=gitlab123
ENV POSTGRES_DB=gitlabhq_production

EXPOSE 5432

VOLUME ["/var/lib/postgresql/data"]

5. jenkins的pipeline流水线

pipeline {
    agent any

    environment {
        NAME = "全局变量"
        VERSION = "1.0.0"
        REGISTRY = "10.0.0.200"  // 镜像仓库地址
        IMAGE_NAME = "pipeline-app" // 项目镜像名称
        KUBE_CONFIG = credentials('kubeconfig-cred') // Jenkins 中配置的 kubeconfig 凭证
    }

    options {
        buildDiscarder(logRotator(daysToKeepStr: '5', numToKeepStr: '10'))
        disableConcurrentBuilds()
        skipDefaultCheckout(true)
        timestamps()
    }

    parameters {
        string(name: 'Version', defaultValue: '1.1.1', description: '版本号', trim: true)
        choice(name: 'EnvType', choices: ['dev', 'prod'], description: '选择部署环境')
    }

    stages {
        stage('Select Environment') {
            steps {
                script {
                    def userInput = input(
                        message: '选择部署的环境',
                        ok: '提交',
                        parameters: [
                            choice(name: 'EnvType', choices: ['dev', 'prod'], description: '部署环境')
                        ]
                    )
                    env.EnvType = userInput
                }
            }
        }

        stage('Checkout Code') {
            steps {
                checkout scm
            }
        }

        stage('Build Docker Image') {
            steps {
                script {
                    sh """
                    docker build -t ${REGISTRY}/${IMAGE_NAME}:${params.Version} .
                    """
                }
            }
        }

        stage('Push Docker Image') {
            steps {
                script {
                    sh """
                    docker login ${REGISTRY} -u admin -p admin123
                    docker push ${REGISTRY}/${IMAGE_NAME}:${params.Version}
                    """
                }
            }
        }

        stage('Deploy to Kubernetes') {
            steps {
                script {
                    writeFile file: 'k8s-deploy.yaml', text: """
apiVersion: apps/v1
kind: Deployment
metadata:
  name: ${IMAGE_NAME}
  namespace: ${params.EnvType}
spec:
  replicas: 1
  selector:
    matchLabels:
      app: ${IMAGE_NAME}
  template:
    metadata:
      labels:
        app: ${IMAGE_NAME}
    spec:
      containers:
      - name: ${IMAGE_NAME}
        image: ${REGISTRY}/${IMAGE_NAME}:${params.Version}
        ports:
        - containerPort: 8080
---
apiVersion: v1
kind: Service
metadata:
  name: ${IMAGE_NAME}
  namespace: ${params.EnvType}
spec:
  selector:
    app: ${IMAGE_NAME}
  ports:
    - port: 80
      targetPort: 8080
"""
                    sh """
                    kubectl --kubeconfig=${KUBE_CONFIG} apply -f k8s-deploy.yaml
                    """
                }
            }
        }
    }

    post {
        success {
            echo "部署成功: ${IMAGE_NAME}:${params.Version} -> 环境: ${params.EnvType}"
        }
        failure {
            echo "部署失败，请检查构建日志。"
        }
    }
}