我的第一个开源项目-jenkins集成k8s项目

一、Jenkins 实现 K8S 持续集成项目架构图

1.gitlab项目部署

1.1 gitlab的Service资源清单

## 创建项目专用命名空间
[root@k8s-master01 ~]# kubectl create namespace devops
namespace/devops created## gitlab是有状态服务，需要使用StatefulSet部署`在这里插入代码片`
## StatefulSet需要依赖svc无头服务[root@k8s-master01 ~]# mkdir /devops
[root@k8s-master01 ~]# cd /devops/
[root@k8s-master01 devops]# vim 01-gitlab-svc.yaml    ## 创建无头svc
apiVersion: v1
kind: Service
metadata:name: svc-gitlabnamespace: devops
spec:clusterIP: Noneselector:app: gitlabports:- name: httpport: 80targetPort: 80- name: httpsport: 443targetPort: 443

1.2 编写gitlab的StatefulSet资源清单

[root@k8s-master01 devops]# vim 02-gitlab-sts.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:name: gitlabnamespace: devops
spec:serviceName: "svc-gitlab"selector:matchLabels:app: gitlabtemplate:metadata:labels:app: gitlabspec:containers:- name: gitlab-ceimage: 10.0.0.200/devops/gitlab-ce:14.6.0-ce.0imagePullPolicy: IfNotPresentenv:- name: GITLAB_ROOT_PASSWORDvalue: "admin123"- name: GITLAB_OMNIBUS_CONFIGvalue: |external_url "http://www.gitlab.com"gitlab_rails['time_zone'] = 'Asia/Shanghai'node_exporter['enable'] = falseredis_exporter['enable'] = falsepostgres_exporter['enable'] = falsegitlab_exporter['enable'] = falsegrafana['enable'] = falsegrafana['reporting_enabled'] = falseprometheus['enable'] = falseprometheus['monitor_kubernetes'] = falseports:- name: httpcontainerPort: 80- name: httpscontainerPort: 443volumeMounts:- name: datamountPath: /etc/gitlabsubPath: config- name: datamountPath: /var/opt/gitlabsubPath: data- name: datamountPath: /var/log/gitlabsubPath: logsvolumes:- name: datapersistentVolumeClaim:claimName: pvc-gitlab

1.3 创建gitlab的sc

[root@k8s-master01 devops]# cat /manifests/csi/sc-devops.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:name: sc-devops
provisioner: rbd.csi.ceph.com
parameters:clusterID: 571a3bbf-1ac1-4859-949c-2b9fc5655d3pool: devopsimageFeatures: layeringcsi.storage.k8s.io/provisioner-secret-name: csi-rbd-secretcsi.storage.k8s.io/provisioner-secret-namespace: defaultcsi.storage.k8s.io/controller-expand-secret-name: csi-rbd-secretcsi.storage.k8s.io/controller-expand-secret-namespace: defaultcsi.storage.k8s.io/node-stage-secret-name: csi-rbd-secretcsi.storage.k8s.io/node-stage-secret-namespace: default
reclaimPolicy: Delete
allowVolumeExpansion: true
mountOptions:- discard

1.4 创建gitlab的pvc

[root@k8s-master01 devops]# vim 03-gitlab-pvc.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:name: pvc-gitlabnamespace: devops
spec:accessModes:- ReadWriteOncevolumeMode: Filesystemresources:requests:storage: 30GistorageClassName: sc-devops

1.5 创建gitlab的ingress

[root@k8s-master01 devops]# vim 04-gitlab-ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:name: ingress-gitlabnamespace: devops
spec:ingressClassName: "nginx"rules:- host: www.gitlab.comhttp:paths:- path: /pathType: Prefixbackend:service:name: svc-gitlabport:name: http## windows hosts 域名解析
10.103.236.201   www.gitlab.com

2.部署sonarqube

2.1 部署依赖的数据库 pgsql

[root@k8s-master01 devops]# mkdir sonarqube
[root@k8s-master01 devops]# cd sonarqube/## pgsql的svc
[root@k8s-master01 sonarqube]# vim 01-pgsql-svc.yaml
apiVersion: v1
kind: Service
metadata:name: svc-pgsqlnamespace: devops
spec:clusterIP: Noneselector:app: pgsqlports:- port: 5432targetPort: 5432

2.2 创建pgsql的sts

[root@k8s-master01 sonarqube]# vim 02-pgsql-sts.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:name: psetgresqlnamespace: devops
spec:serviceName: svc-pgsqlreplicas: 1selector:matchLabels:app: pgsqltemplate:metadata:labels:app: pgsqlspec:containers:- name: postgresimage: 10.0.0.200/devops/postgres:13.8ports:- containerPort: 5432env:- name: POSTGRES_DBvalue: sonardb- name: POSTGRES_USERvalue: sonar- name: POSTGRES_PASSWORDvalue: "123456"volumeMounts:- name: dbmountPath: /var/lib/postgresql/datasubPath: data- name: dbmountPath: /var/run/secrets/kubernetes.io/serviceaccountsubPath: serviceaccountvolumes:- name: dbpersistentVolumeClaim:claimName: pvc-pgsql

2.3 创建pgsql的sc

[root@k8s-master01 devops]# cat /manifests/csi/sc-pgsql.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:name: sc-pgsql
provisioner: rbd.csi.ceph.com
parameters:clusterID: 571a3bbf-1ac1-4859-949c-2b9fc5655d3pool: pgsqlimageFeatures: layeringcsi.storage.k8s.io/provisioner-secret-name: csi-rbd-secretcsi.storage.k8s.io/provisioner-secret-namespace: defaultcsi.storage.k8s.io/controller-expand-secret-name: csi-rbd-secretcsi.storage.k8s.io/controller-expand-secret-namespace: defaultcsi.storage.k8s.io/node-stage-secret-name: csi-rbd-secretcsi.storage.k8s.io/node-stage-secret-namespace: default
reclaimPolicy: Delete
allowVolumeExpansion: true
mountOptions:- discard

2.4 创建pgsql的pvc

[root@k8s-master01 devops]# vim 03-pgsql-pvc.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:name: pvc-pgsqlnamespace: devops
spec:accessModes:- ReadWriteOncevolumeMode: Filesystemresources:requests:storage: 30GistorageClassName: sc-pgsql## ceph 创建 pool
[root@node-1 ~]# ceph osd pool create pgsql 16 16
pool 'pgsql' created

2.5 创建sc

[root@k8s-master01 sonarqube]# cat sc-pgsql.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:name: sc-pgsql
provisioner: rbd.csi.ceph.com
parameters:clusterID: 571a3bbf-1ac1-4859-949c-2b9fc5655d3pool: pgsqlimageFeatures: layeringcsi.storage.k8s.io/provisioner-secret-name: csi-rbd-secretcsi.storage.k8s.io/provisioner-secret-namespace: defaultcsi.storage.k8s.io/controller-expand-secret-name: csi-rbd-secretcsi.storage.k8s.io/controller-expand-secret-namespace: defaultcsi.storage.k8s.io/node-stage-secret-name: csi-rbd-secretcsi.storage.k8s.io/node-stage-secret-namespace: default
reclaimPolicy: Delete
allowVolumeExpansion: true
mountOptions:- discard## 进入pgsql验证
[root@k8s-master01 sonarqube]# kubectl exec -it -n devops postgresql-0 -- bash
root@postgresql-0:/# psql -Usonar -d sonardb
sonardb=# \lList of databasesName    | Owner | Encoding | Collate     | Ctype       | Access privileges
-----------+-------+----------+-------------+-------------+-------------------postgres  | sonar | UTF8     | en_US.utf8  | en_US.utf8  |sonardb   | sonar | UTF8     | en_US.utf8  | en_US.utf8  | =c/sonar         +|       |          |             |             | sonar=CTc/sonartemplate0 | sonar | UTF8     | en_US.utf8  | en_US.utf8  | =c/sonar         +|       |          |             |             | sonar=CTc/sonartemplate1 | sonar | UTF8     | en_US.utf8  | en_US.utf8  | =c/sonar         +|       |          |             |             | sonar=CTc/sonar

2.6 创建sonarqube-svc

[root@k8s-master01 sonarqube]# vim 04-sonarqube-svc.yaml
apiVersion: v1
kind: Service
metadata:name: svc-sonarqubenamespace: devops
spec:clusterIP: Noneselector:app: sonarqubeports:- name: webport: 9000targetPort: 9000

2.7 部署sonarqube

[root@k8s-master01 sonarqube]# vim 05-sonarqube-sts.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:name: sonarqubenamespace: devops
spec:serviceName: svc-sonarqubeselector:matchLabels:app: sonarqubetemplate:metadata:labels:app: sonarqubespec:initContainers:- name: set-kernelimage: busyboxcommand: ["sh", "-c", "sysctl -w vm.max_map_count=524288 ; sysctl -w fs.file-max=131072 ; ulimit -n 131072 ; ulimit -u 8192"]securityContext:privileged: truecontainers:- name: sonarqubeimage: 10.0.0.200/devops/sonarqube:9.7-communityports:- name: webcontainerPort: 9000env:- name: JAVA_OPTSvalue: -Duser.timezone=Asia/Shanghai- name: SONARQUBE_JDBC_USERNAMEvalue: sonar  ## 连接pgsql用户名- name: SONARQUBE_JDBC_PASSWORDvalue: "123456"  ## 连接pgsql密码- name: SONARQUBE_JDBC_URLvalue: jdbc:postgresql://svc-pgsql:5432/sonardbresources:limits:cpu: 1500mmemory: 2048MivolumeMounts:- name: datamountPath: /opt/sonarqube/datasubPath: data- name: datamountPath: /opt/sonarqube/logssubPath: logs- name: datamountPath: /opt/sonarqube/extensionssubPath: extensions- name: datamountPath: /var/run/secrets/kubernetes.io/serviceaccountsubPath: serviceaccountvolumeClaimTemplates:- metadata:name: datanamespace: devopsspec:accessModes:- ReadWriteOncestorageClassName: "sc-devops"resources:requests:storage: 50Gi

2.8 创建sonarqube的ingress规则

[root@k8s-master01 sonarqube]# vim 06-sonarqube-ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:name: ingress-sonarqubenamespace: devops
spec:ingressClassName: "nginx"rules:- host: www.sonarqube.comhttp:paths:- path: /pathType: Prefixbackend:service:name: svc-sonarqubeport:number: 9000##web浏览器访问www.sonarqube.com(记得解析)
账号 admin 密码 admin 登陆后密码改为 123456

3.jenkins部署

3.1 (Jenkins不需要数据库，数据存在本地，StatefulSet部署)

Jenkins 需要创建 Slave Pod 来执行流水线构建，这就需要与 apiserver 交互，所以就需要 RBAC 权限[root@k8s-master01 devops]# pwd
/devops
[root@k8s-master01 devops]# mkdir jenkins
[root@k8s-master01 devops]# cd jenkins/
[root@k8s-master01 jenkins]# vim 01-jenkins-rbac.yamlapiVersion: v1
kind: ServiceAccount
metadata:name: jenkinsnamespace: devops---kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:name: jenkins
rules:
- apiGroups: ["extensions", "apps"]resources: ["deployments", "ingresses"]verbs: ["create", "delete", "get", "list", "watch", "patch", "update"]
- apiGroups: [""]resources: ["services"]verbs: ["create", "delete", "get", "list", "watch", "patch", "update"]
- apiGroups: [""]resources: ["pods"]verbs: ["create", "delete", "get", "list", "patch", "update", "watch"]
- apiGroups: [""]resources: ["pods/exec"]verbs: ["create", "delete", "get", "list", "patch", "update", "watch"]
- apiGroups: [""]resources: ["pods/log", "events"]verbs: ["get", "list", "watch"]
- apiGroups: [""]resources: ["secrets"]verbs: ["get"]---apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:name: jenkinsnamespace: devops
roleRef:apiGroup: rbac.authorization.k8s.iokind: ClusterRolename: jenkins
subjects:
- kind: ServiceAccountname: jenkinsnamespace: devops[root@k8s-master01 jenkins]# kubectl create -f 01-jenkins-rbac.yaml

3.2 创建 Jenkins master 的 svc

[root@k8s-master01 jenkins]# vim 02-jenkins-svc.yaml
apiVersion: v1
kind: Service
metadata:name: svc-jenkinsnamespace: devops
spec:clusterIP: Noneselector:app: jenkinsports:- name: httpport: 8080targetPort: 8080- name: agentport: 50000targetPort: 50000

3.3 Jenkins 创建 statefulset

[root@k8s-master01 jenkins]# vim 03-jenkins-sts.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:name: jenkinsnamespace: devops
spec:serviceName: svc-jenkinsselector:matchLabels:app: jenkinstemplate:metadata:labels:app: jenkinsspec:serviceAccount: jenkinscontainers:- name: jenkinsimage: 10.0.0.200/devops/jenkins:2.346.3-2-ltsimagePullPolicy: IfNotPresentsecurityContext:privileged: truerunAsUser: 0env:- name: JAVA_OPTSvalue: -Duser.timezone=Asia/Shanghaiports:- name: httpcontainerPort: 8080- name: agentcontainerPort: 50000resources:limits:cpu: 1500mmemory: 2048MireadinessProbe:httpGet:path: /loginport: 8080initialDelaySeconds: 60timeoutSeconds: 5failureThreshold: 12volumeMounts:- name: datamountPath: /var/jenkins_homesubPath: jenkins_homevolumeClaimTemplates:- metadata:name: dataspec:accessModes: ["ReadWriteOnce"]storageClassName: "sc-devops"resources:requests:storage: 20Gi

3.4 创建 Jenkins 的 Ingress

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:name: ingress-jenkinsnamespace: devops
spec:ingressClassName: "nginx"rules:- host: www.jenkins.comhttp:paths:- path: /pathType: Prefixbackend:service:name: svc-jenkinsport:name: http## windows hosts 解析这个域名
## 查看jenkins密码安装 Jenkins 插件：（这些又有的可能安装失败，其他安装完成后重启 jenkins 再装一遍，然后记得再已安装中降级一下，不然可能会有兼容性问题）中文插件：Localization: Chinese (Simplified)  
Git 插件：git, gitlab  
Sonar 插件：SonarQube Scanner（这是一个）  
Pipeline 插件：pipeline、Stage View、Blue Ocean  
Kubernetes 插件：Kubernetes

4.dockerfile 镜像制作

4.1 Jenkins 镜像模板

## Jenkins
FROM 10.0.0.200/devops/jenkins:2.346.3-2-ltsUSER root# 设置时区
RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \echo "Asia/Shanghai" > /etc/timezone# 安装必要工具
RUN apt-get update && \apt-get install -y fontconfig git curl unzip nfs-common && \rm -rf /var/lib/apt/lists/*# 配置 Jenkins 插件（可选）
COPY plugins.txt /usr/share/jenkins/ref/plugins.txt
RUN jenkins-plugin-cli --plugin-file /usr/share/jenkins/ref/plugins.txt# 挂载目录
VOLUME ["/var/jenkins_home"]EXPOSE 8080 50000ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/jenkins.sh"]

4.2 Maven 镜像模板

## Maven
FROM 10.0.0.200/pipeline/maven:3.8.6-openjdk-8ADD ./settings_docker.xml /usr/share/maven/conf/settings.xmlRUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \apt-get update && \apt-get install -y nfs-utils && \rm -rf /var/lib/apt/lists/*# 构建命令
# docker build -t 10.0.0.200/pipeline/maven:3.8.6-openjdk-8 .

4.3 SonarQube Scanner 镜像模板

## SonarQube Scanner
FROM 10.0.0.200/devops/sonar-scanner:4.8# 设置时区
RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \apt-get update && \apt-get install -y nfs-common && \rm -rf /var/lib/apt/lists/*# 配置 SonarQube 扫描器参数
COPY sonar-scanner.properties /opt/sonar-scanner/conf/sonar-scanner.propertiesENTRYPOINT ["/opt/sonar-scanner/bin/sonar-scanner"]

4.4 Kubernetes Agent 镜像模板

## Kubernetes Jenkins Agent
FROM 10.0.0.200/devops/jenkins-agent:latestUSER rootRUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \apt-get update && \apt-get install -y nfs-common git curl && \rm -rf /var/lib/apt/lists/*ENTRYPOINT ["/usr/local/bin/jenkins-agent"]

4.5 GitLab 镜像模板

## GitLab
FROM 10.0.0.200/devops/gitlab-ce:15.3.2-ce.0# 设置时区
RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \echo "Asia/Shanghai" > /etc/timezone# 安装必要工具
RUN apt-get update && \apt-get install -y curl vim nfs-common && \rm -rf /var/lib/apt/lists/*# 配置 GitLab（可选）
# COPY gitlab.rb /etc/gitlab/gitlab.rb
# RUN gitlab-ctl reconfigureEXPOSE 80 443 22VOLUME ["/etc/gitlab", "/var/log/gitlab", "/var/opt/gitlab"]

4.6 PostgreSQL 镜像模板

## PostgreSQL
FROM 10.0.0.200/devops/postgres:13# 设置时区
RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \echo "Asia/Shanghai" > /etc/timezone# 安装必要工具
RUN apt-get update && \apt-get install -y nfs-common && \rm -rf /var/lib/apt/lists/*# 复制初始化 SQL（可选）
# COPY init.sql /docker-entrypoint-initdb.d/ENV POSTGRES_USER=gitlab
ENV POSTGRES_PASSWORD=gitlab123
ENV POSTGRES_DB=gitlabhq_productionEXPOSE 5432VOLUME ["/var/lib/postgresql/data"]

5. jenkins的pipeline流水线

pipeline {agent anyenvironment {NAME = "全局变量"VERSION = "1.0.0"REGISTRY = "10.0.0.200"  // 镜像仓库地址IMAGE_NAME = "pipeline-app" // 项目镜像名称KUBE_CONFIG = credentials('kubeconfig-cred') // Jenkins 中配置的 kubeconfig 凭证}options {buildDiscarder(logRotator(daysToKeepStr: '5', numToKeepStr: '10'))disableConcurrentBuilds()skipDefaultCheckout(true)timestamps()}parameters {string(name: 'Version', defaultValue: '1.1.1', description: '版本号', trim: true)choice(name: 'EnvType', choices: ['dev', 'prod'], description: '选择部署环境')}stages {stage('Select Environment') {steps {script {def userInput = input(message: '选择部署的环境',ok: '提交',parameters: [choice(name: 'EnvType', choices: ['dev', 'prod'], description: '部署环境')])env.EnvType = userInput}}}stage('Checkout Code') {steps {checkout scm}}stage('Build Docker Image') {steps {script {sh """docker build -t ${REGISTRY}/${IMAGE_NAME}:${params.Version} ."""}}}stage('Push Docker Image') {steps {script {sh """docker login ${REGISTRY} -u admin -p admin123docker push ${REGISTRY}/${IMAGE_NAME}:${params.Version}"""}}}stage('Deploy to Kubernetes') {steps {script {writeFile file: 'k8s-deploy.yaml', text: """
apiVersion: apps/v1
kind: Deployment
metadata:name: ${IMAGE_NAME}namespace: ${params.EnvType}
spec:replicas: 1selector:matchLabels:app: ${IMAGE_NAME}template:metadata:labels:app: ${IMAGE_NAME}spec:containers:- name: ${IMAGE_NAME}image: ${REGISTRY}/${IMAGE_NAME}:${params.Version}ports:- containerPort: 8080
---
apiVersion: v1
kind: Service
metadata:name: ${IMAGE_NAME}namespace: ${params.EnvType}
spec:selector:app: ${IMAGE_NAME}ports:- port: 80targetPort: 8080
"""sh """kubectl --kubeconfig=${KUBE_CONFIG} apply -f k8s-deploy.yaml"""}}}}post {success {echo "部署成功: ${IMAGE_NAME}:${params.Version} -> 环境: ${params.EnvType}"}failure {echo "部署失败，请检查构建日志。"}}
}