企業級 kubernetes SpringCloud 持續整合，從0到1
2020-10-01 21:00:02
1 環境概述
192.168.38.50 master
192.168.38.51 node1
192.168.38.52 node2
192.168.38.100 harbor/nfs
192.168.38.45 gitlab/sonarqube
2 Kubernetes安裝Jenkins
2.1 PV/PVC
#建立名稱空間
kubectl create namespace jenkins-k8s
(1) 建立PV
vim jenkins-pv.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: jenkins-pv
spec:
capacity:
storage: 10Gi
accessModes:
- ReadWriteMany
nfs:
path: /usr/local/kubernetes/volumes/jenkins
server: 192.168.38.100
#應用
kubectl apply -f jenkins-pv.yaml
(2) 建立PVC
vim jenkins-pvc.yaml
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: jenkins-pvc
namespace: jenkins-k8s
spec:
resources:
requests:
storage: 10Gi
accessModes:
- ReadWriteMany
#應用
kubectl apply -f jenkins-pvc.yaml
2.2 ServiceAccount
#建立一個sa賬號
kubectl create sa jenkins-k8s-sa -n jenkins-k8s
#授權，kubectl create clusterrolebinding 名稱、名稱空間、繫結clusterrole=cluster-admin
kubectl create clusterrolebinding jenkins-k8s-sa-cluster -n jenkins-k8s --clusterrole=cluster-admin --serviceaccount=jenkins-k8s:jenkins-k8s-sa
2.3 整合Jenkins
2.3.1 下載映象
#node1、node2
docker pull jenkins/jenkins:latest
2.3.2 jenkins-deployment.yaml
vim jenkins-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: jenkins
namespace: jenkins-k8s
spec:
replicas: 1
selector:
matchLabels:
app: jenkins
template:
metadata:
labels:
app: jenkins
spec:
serviceAccountName: jenkins-k8s-sa
containers:
- name: jenkins
image: jenkins/jenkins:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 8080
name: web
protocol: TCP
- containerPort: 50000
name: agent
protocol: TCP
resources:
limits:
cpu: 1000m
memory: 1Gi
requests:
cpu: 500m
memory: 500Mi
livenessProbe:
httpGet:
port: 8080
path: /login
initialDelaySeconds: 60
timeoutSeconds: 5
failureThreshold: 12
readinessProbe:
httpGet:
port: 8080
path: /login
initialDelaySeconds: 60
volumeMounts:
- mountPath: /var/jenkins_home
name: jenkins-volume
subPath: jenkins-home
volumes:
- name: jenkins-volume
persistentVolumeClaim:
claimName: jenkins-pvc
#應用
kubectl apply -f jenkins-deployment.yaml
#檢視
kubectl get pod -n jenkins-k8s
#驗證
kubectl get pod -o wide -n jenkins-k8s
curl 10.244.2.23:8080
2.3.3 jenkins-svc.yaml
#建立
vim jenkins-svc.yaml
apiVersion: v1
kind: Service
metadata:
name: jenkins-service
namespace: jenkins-k8s
labels:
app: jenkins
spec:
selector:
app: jenkins
type: ClusterIP
ports:
- port: 8080
name: web
targetPort: web
- name: agent
port: 50000
targetPort: agent
#應用
kubectl apply -f jenkins-svc.yaml
#檢視
kubectl get svc -n jenkins-k8s
2.3.4 jenkins-ingress.yaml
vim jenkins-ingress.yaml
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
name: jenkins
namespace: jenkins-k8s
spec:
rules:
- host: rosh.jenkins.com
http:
paths:
- backend:
serviceName: jenkins-service
servicePort: 8080
#應用
kubectl apply -f jenkins-ingress.yaml
#檢視
kubectl get ingress -n jenkins-k8s
2.4 Jenkins 初始化
(1) 檢視初始化密碼
#檢視掛載紀錄檔,記錄初始化密碼，填入初始化頁面
84f98993d6594792b7e420c74934eb89
（2）初始化外掛
（3）初始化使用者
2.5 中文外掛
2.6 kubernetes外掛
(1) 安裝外掛
(2) 設定Kubernetes
2.7 Jenkins從節點設定
2.7.1 製作映象
(1) setting.xml
<?xml version="1.0" encoding="UTF-8"?>
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
<localRepository>/usr/local/apache-maven/repo</localRepository>
<pluginGroups></pluginGroups>
<proxies></proxies>
<servers></servers>
<profiles></profiles>
<mirrors>
<mirror>
<id>central</id>
<mirrorOf>central</mirrorOf>
<name>aliyun maven</name>
<url>https://maven.aliyun.com/repository/public</url>
</mirror>
</mirrors>
</settings>
(2) Dockerfile
FROM cnych/jenkins:jnlp6
MAINTAINER rosh
# 切換到 root 賬戶進行操作
USER root
# 安裝 maven
COPY apache-maven-3.6.2-bin.tar.gz .
RUN tar -zxf apache-maven-3.6.2-bin.tar.gz && \
mv apache-maven-3.6.2 /usr/local && \
rm -f apache-maven-3.6.2-bin.tar.gz && \
ln -s /usr/local/apache-maven-3.6.2/bin/mvn /usr/bin/mvn && \
ln -s /usr/local/apache-maven-3.6.2 /usr/local/apache-maven && \
mkdir -p /usr/local/apache-maven/repo
COPY settings.xml /usr/local/apache-maven/conf/settings.xml
(3) 製作映象
docker build -t jenkins-slave-maven .
(4) 上傳至harbor
docker tag jenkins-slave-maven:latest 192.168.38.100/library/jenkins-slave-maven:latest
docker push 192.168.38.100/library/jenkins-slave-maven:latest
(5) 檢視
2.7.2 設定模板
master的/root/.kube目錄拷貝到node1和node2節點：
scp -r /root/.kube node1:
scp -r /root/.kube node2:
2.7.3 測試
(1) 建立流水線
(2) 測試指令碼
//參照jenkins-slave的pod模組來構建Jenkins-Slave的pod
node("jenkins-slave"){
stage('k8s-測試'){
sh "kubectl get node"
}
stage('maven-測試'){
sh "mvn -v"
}
stage('docker 測試'){
sh "docker version"
}
}
(3) 結果
3 整合Gitlab
3.1 憑證
jenkins新增存取憑證
3.2 建立pipeline工程
3.3 整合Hook
3.4 Gitlab
3.5 Jenkinsfile
(1) 流水線設定
(2) 生成指令碼
(3) Jenkinsfile
//參照jenkins-slave的pod模組來構建Jenkins-Slave的pod
node("jenkins-slave"){
stage('拉取程式碼'){
git branch: 'develop', credentialsId: 'ad4d8da3-c362-4b92-b8da-e218f372d193', url: 'http://192.168.38.45/root/springboot-cloud-k8s.git'
echo "拉取程式碼成功"
}
}
(4) 推播程式碼到develop分支
(5) 檢視jenkins 紀錄檔
3.6 Generic Webhook Trigger
檢視Gitlab推播請求，防止頻繁構建，現在限制develop分支推播程式碼時，觸發自動構建，別的分支不觸發構建。
3.6.1 修改Pipeline設定
3.6.2 修改Gitlab勾點
3.6.3 測試
master分支merge develop分支程式碼，推播master分支，沒有觸發jenkins自動化構建，檢視gitlab響應資訊
4 製作映象、上傳Harbor
4.1 Dockerfile
#######################閘道器#######################
FROM java:8
RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
RUN echo "Asia/Shanghai" > /etc/timezone
RUN bash -c 'touch /app.jar'
ARG JAR_FILE
COPY ${JAR_FILE} app.jar
EXPOSE 9000
ENTRYPOINT ["java","-jar","/app.jar"]
#######################student#######################
FROM java:8
RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
RUN echo "Asia/Shanghai" > /etc/timezone
RUN bash -c 'touch /app.jar'
ARG JAR_FILE
COPY ${JAR_FILE} app.jar
EXPOSE 8001
ENTRYPOINT ["java","-jar","/app.jar"]
#######################teacher#######################
FROM java:8
RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
RUN echo "Asia/Shanghai" > /etc/timezone
RUN bash -c 'touch /app.jar'
ARG JAR_FILE
COPY ${JAR_FILE} app.jar
EXPOSE 8002
ENTRYPOINT ["java","-jar","/app.jar"]
4.2 Maven外掛
<!-- docker的maven外掛-->
<plugin>
<groupId>com.spotify</groupId>
<artifactId>dockerfile-maven-plugin</artifactId>
<version>1.3.6</version>
<configuration>
<repository>${project.artifactId}</repository>
<buildArgs>
<JAR_FILE>target/${project.build.finalName}.jar</JAR_FILE>
</buildArgs>
</configuration>
</plugin>
4.3 整合Harbor
(1) 憑證
(2) 生成指令碼
4.4 nfs建立maven目錄
#nfs伺服器
cd /usr/local/kubernetes/volumes
mkdir maven
chmod 777 ./maven/
chown nfsnobody ./maven/
vim /etc/exports
/usr/local/kubernetes/volumes/maven/ *(rw,sync,insecure,no_subtree_check,no_root_squash)
#重新啟動
systemctl restart rpcbind && systemctl restart nfs
#其它節點測試
showmount -e 192.168.38.100
4.5 Jenkinsfile
//harbor地址
def harborUrl = "192.168.38.100"
//harbor倉庫名稱
def harborRepository = "springcloud-k8s"
//構建版本tag
def tag = "latest"
//harbor 憑證
def harborAuth = "3db8138a-792d-4d9b-8b4e-c7c62d9f6cf4"
//微服務列表
def microServerList = ['gateway-server','student-server','teacher-server']
//參照jenkins-slave的pod模組來構建Jenkins-Slave的pod
node("jenkins-slave"){
stage('拉取程式碼'){
git branch: 'develop', credentialsId: 'ad4d8da3-c362-4b92-b8da-e218f372d193', url: 'http://192.168.38.45/root/springboot-cloud-k8s.git'
echo "拉取程式碼成功"
}
stage('製作映象,上傳harbor'){
for(int i=0;i<microServerList.size();i++){
echo "製作映象: ${microServerList[i]}"
//服務名稱
def serverName = "${microServerList[i]}"
//定義映象名稱
def imageName="${serverName}:${tag}"
//打包，構建映象
sh "mvn -f ${serverName} clean package dockerfile:build"
// 打tag
sh "docker tag ${imageName} ${harborUrl}/${harborRepository}/${imageName}"
// 上傳映象
withCredentials([usernamePassword(credentialsId: '3db8138a-792d-4d9b-8b4e-c7c62d9f6cf4', passwordVariable: 'password', usernameVariable: 'username')]) {
// some block
//登入
sh "docker login -u ${username} -p ${password} ${harborUrl}"
//上傳
sh "docker push ${harborUrl}/${harborRepository}/${imageName}"
}
//刪除
sh "docker rmi -f ${imageName}"
sh "docker rmi -f ${harborUrl}/${harborRepository}/${imageName}"
}
}
}
4.6 推播dev分支
4.7 jenkins檢視結果
5 SonarQube
5.1 Sonarqube token
#記錄token
c06945196968cf0a3e5e326e1e4b2646fc97b135
mvn sonar:sonar \
-Dsonar.host.url=http://192.168.38.45:9000/sonar \
-Dsonar.login=c06945196968cf0a3e5e326e1e4b2646fc97b135
5.2 Jenkins整合Sonarqube
(1) 安裝外掛
(2) 建立憑證
(3) 系統設定
(4) 全域性設定
5.3 sonar-project.properties
#################################閘道器##########################################
#SonarQube 範例名稱，唯一
sonar.projectKey=gateway-server
#專案名稱
sonar.projectName=gateway-server
#版本號
sonar.projectVersion=1.0
#指定掃描目錄
sonar.sources=.
sonar.exclusions=**/test/**
#jdk版本
sonar.java.source=1.8
sonar.java.target=1.8
#掃描編寫類的專案
sonar.java.binaries=target/classes
#編碼格式
sonar.sourceEncoding=UTF-8
#################################學生##########################################
#SonarQube 範例名稱，唯一
sonar.projectKey=student-server
#專案名稱
sonar.projectName=student-server
#版本號
sonar.projectVersion=1.0
#指定掃描目錄
sonar.sources=.
sonar.exclusions=**/test/**
#jdk版本
sonar.java.source=1.8
sonar.java.target=1.8
#掃描編寫類的專案
sonar.java.binaries=target/classes
#編碼格式
sonar.sourceEncoding=UTF-8
#################################老師##########################################
#SonarQube 範例名稱，唯一
sonar.projectKey=teacher-server
#專案名稱
sonar.projectName=teacher-server
#版本號
sonar.projectVersion=1.0
#指定掃描目錄
sonar.sources=.
sonar.exclusions=**/test/**
#jdk版本
sonar.java.source=1.8
sonar.java.target=1.8
#掃描編寫類的專案
sonar.java.binaries=target/classes
#編碼格式
sonar.sourceEncoding=UTF-8
5.4 Jenkinsfile
//harbor地址
def harborUrl = "192.168.38.100"
//harbor倉庫名稱
def harborRepository = "springcloud-k8s"
//構建版本tag
def tag = "latest"
//harbor 憑證
def harborAuth = "3db8138a-792d-4d9b-8b4e-c7c62d9f6cf4"
//微服務列表
def microServerList = ['gateway-server','student-server','teacher-server']
//參照jenkins-slave的pod模組來構建Jenkins-Slave的pod
node("jenkins-slave"){
stage('拉取程式碼'){
git branch: 'develop', credentialsId: 'ad4d8da3-c362-4b92-b8da-e218f372d193', url: 'http://192.168.38.45/root/springboot-cloud-k8s.git'
echo "拉取程式碼成功"
}
stage('製作映象,上傳harbor'){
for(int i=0;i<microServerList.size();i++){
echo "製作映象: ${microServerList[i]}"
//服務名稱
def serverName = "${microServerList[i]}"
//定義映象名稱
def imageName="${serverName}:${tag}"
//打包，構建映象
sh "mvn -f ${serverName} clean package dockerfile:build"
// 打tag
sh "docker tag ${imageName} ${harborUrl}/${harborRepository}/${imageName}"
// 上傳映象
withCredentials([usernamePassword(credentialsId: '3db8138a-792d-4d9b-8b4e-c7c62d9f6cf4', passwordVariable: 'password', usernameVariable: 'username')]) {
// some block
//登入
sh "docker login -u ${username} -p ${password} ${harborUrl}"
//上傳
sh "docker push ${harborUrl}/${harborRepository}/${imageName}"
}
//刪除
sh "docker rmi -f ${imageName}"
sh "docker rmi -f ${harborUrl}/${harborRepository}/${imageName}"
}
}
stage('程式碼審查'){
//定義當前Jenkins的SonarQubeScanner工具
def scannerHome = tool 'sonarqube-scanner'
for(int i=0;i<microServerList.size();i++){
//參照當前JenkinsSonarQube環境
withSonarQubeEnv('sonarqube') {
sh """cd ${microServerList[i]}
${scannerHome}/bin/sonar-scanner"""
}
}
}
}
5.5 推程式碼
5.6 檢視結果
6 部署
6.1建立紀錄檔PV/PVC
(1) gateway
vim gateway-pv-pvc.yaml
kubectl apply -f gateway-pv-pvc.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: gateway-logs-pv
spec:
capacity:
storage: 2Gi
accessModes:
- ReadWriteMany
nfs:
path: "/usr/local/kubernetes/volumes/logs/gateway"
server: 192.168.38.100
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: gateway-logs-pvc
namespace: jenkins-k8s
spec:
resources:
requests:
storage: 2Gi
accessModes:
- ReadWriteMany
(2) teacher
vim teacher-pv-pvc.yaml
kubectl apply -f teacher-pv-pvc.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: teacher-logs-pv
spec:
capacity:
storage: 2Gi
accessModes:
- ReadWriteMany
nfs:
path: "/usr/local/kubernetes/volumes/logs/teacher"
server: 192.168.38.100
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: teacher-logs-pvc
namespace: jenkins-k8s
spec:
resources:
requests:
storage: 2Gi
accessModes:
- ReadWriteMany
(3) student
vim student-pv-pvc.yaml
kubectl apply -f student-pv-pvc.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: student-logs-pv
spec:
capacity:
storage: 2Gi
accessModes:
- ReadWriteMany
nfs:
path: "/usr/local/kubernetes/volumes/logs/student"
server: 192.168.38.100
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: student-logs-pvc
namespace: jenkins-k8s
spec:
resources:
requests:
storage: 2Gi
accessModes:
- ReadWriteMany
6.2 建立許可權
kubectl create secret docker-registry registry-secret --namespace=jenkins-k8s \
--docker-server=192.168.38.100 --docker-username=admin \
--docker-password=Harbor12345
kubectl get secret
6.3 Deployment
(1) gateway
##############Deployment######################
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: gateway-server
name: gateway-server
namespace: jenkins-k8s
spec:
replicas: 2
selector:
matchLabels:
app: gateway-server
template:
metadata:
labels:
app: gateway-server
spec:
imagePullSecrets:
- name: registry-secret
containers:
- name: gateway-server
image: 192.168.38.100/springcloud-k8s/gateway-server:latest
imagePullPolicy: Always
volumeMounts:
- mountPath: /gateway-server/logs
name: gateway-server
volumes:
- name: gateway-server
persistentVolumeClaim:
claimName: gateway-logs-pvc
---
##############Service####################
apiVersion: v1
kind: Service
metadata:
labels:
app: gateway-server
name: gateway-server
spec:
ports:
- port: 9000
protocol: TCP
targetPort: 9000
selector:
app: gateway-server
type: ClusterIP
---
##############Ingress####################
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
name: gateway-server
spec:
rules:
- host: rosh.gateway-server.com
http:
paths:
- backend:
serviceName: gateway-server
servicePort: 9000
(2) student
##############Deployment####################
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: student-server
name: student-server
namespace: jenkins-k8s
spec:
replicas: 2
selector:
matchLabels:
app: student-server
template:
metadata:
labels:
app: student-server
spec:
imagePullSecrets:
- name: registry-secret
containers:
- name: student-server
image: 192.168.38.100/springcloud-k8s/student-server:latest
imagePullPolicy: Always
volumeMounts:
- mountPath: /student-server/logs
name: student-server
volumes:
- name: student-server
persistentVolumeClaim:
claimName: student-logs-pvc
---
##############Service####################
apiVersion: v1
kind: Service
metadata:
labels:
app: student-server
name: student-server
spec:
ports:
- port: 8001
protocol: TCP
targetPort: 8001
selector:
app: student-server
type: ClusterIP
---
##############Ingress####################
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
name: student-server
spec:
rules:
- host: rosh.student-server.com
http:
paths:
- backend:
serviceName: student-server
servicePort: 8001
(3) teacher
##############Deployment####################
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: teacher-server
name: teacher-server
namespace: jenkins-k8s
spec:
replicas: 2
selector:
matchLabels:
app: teacher-server
template:
metadata:
labels:
app: teacher-server
spec:
imagePullSecrets:
- name: registry-secret
containers:
- name: teacher-server
image: 192.168.38.100/springcloud-k8s/teacher-server:latest
imagePullPolicy: Always
volumeMounts:
- mountPath: /teacher-server/logs
name: teacher-server
volumes:
- name: teacher-server
persistentVolumeClaim:
claimName: teacher-logs-pvc
---
##############Service####################
apiVersion: v1
kind: Service
metadata:
labels:
app: teacher-server
name: teacher-server
spec:
ports:
- port: 8002
protocol: TCP
targetPort: 8002
selector:
app: teacher-server
type: ClusterIP
---
##############Ingress####################
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
name: teacher-server
spec:
rules:
- host: rosh.teacher-server.com
http:
paths:
- backend:
serviceName: teacher-server
servicePort: 8002
6.4 Jenkinsfile
//harbor地址
def harborUrl = "192.168.38.100"
//harbor倉庫名稱
def harborRepository = "springcloud-k8s"
//構建版本tag
def tag = "latest"
//harbor 憑證
def harborAuth = "3db8138a-792d-4d9b-8b4e-c7c62d9f6cf4"
//微服務列表
def microServerList = ['gateway-server','student-server','teacher-server']
//參照jenkins-slave的pod模組來構建Jenkins-Slave的pod
node("jenkins-slave"){
stage('拉取程式碼'){
git branch: 'develop', credentialsId: 'ad4d8da3-c362-4b92-b8da-e218f372d193', url: 'http://192.168.38.45/root/springboot-cloud-k8s.git'
echo "拉取程式碼成功"
}
stage('製作映象,上傳harbor'){
for(int i=0;i<microServerList.size();i++){
echo "製作映象: ${microServerList[i]}"
//服務名稱
def serverName = "${microServerList[i]}"
//定義映象名稱
def imageName="${serverName}:${tag}"
//打包，構建映象
sh "mvn -f ${serverName} clean package dockerfile:build"
// 打tag
sh "docker tag ${imageName} ${harborUrl}/${harborRepository}/${imageName}"
// 上傳映象
withCredentials([usernamePassword(credentialsId: '3db8138a-792d-4d9b-8b4e-c7c62d9f6cf4', passwordVariable: 'password', usernameVariable: 'username')]) {
// some block
//登入
sh "docker login -u ${username} -p ${password} ${harborUrl}"
//上傳
sh "docker push ${harborUrl}/${harborRepository}/${imageName}"
}
}
}
stage('程式碼審查'){
//定義當前Jenkins的SonarQubeScanner工具
def scannerHome = tool 'sonarqube-scanner'
for(int i=0;i<microServerList.size();i++){
//參照當前JenkinsSonarQube環境
withSonarQubeEnv('sonarqube') {
sh """cd ${microServerList[i]}
${scannerHome}/bin/sonar-scanner"""
}
}
}
stage('k8s 部署'){
for(int i=0;i<microServerList.size();i++){
sh "kubectl apply -f ${microServerList[i]}/Deployment.yaml"
}
}
stage('刪除本地映象 '){
for(int i=0;i<microServerList.size();i++){
//服務名稱
def serverName = "${microServerList[i]}"
//定義映象名稱
def imageName="${serverName}:${tag}"
//刪除
sh "docker rmi -f ${imageName}"
sh "docker rmi -f ${harborUrl}/${harborRepository}/${imageName}"
}
}
}
6.5 結果
