首先要確定部署的版本
查詢Kubernetes對Docker支援的情況
kubernetes/dependencies.yaml at master · kubernetes/kubernetes (github.com)
查詢Kubernetes Dashboard對Kubernetes支援的情況
Releases · kubernetes/dashboard (github.com)
名稱 | 版本 |
---|---|
kubernetes | 1.23 |
Docker | 20.10.22 |
Kubernetes Dashboard | 2.5.1 |
部署的步驟為
節點hostname | 作用 | IP |
---|---|---|
kubemaster | master | 192.168.1.4 |
kubeworker1 | work1 | 192.168.1.5 |
kubeworker2 | work2 | 192.168.1.6 |
如表格所示,將192.168.1.4伺服器的hostname
設定為kubemaster
,將192.168.1.5伺服器的hostname
設定為kubeworker1
,將192.168.1.6伺服器的hostname
設定為kubeworker2
。並將每個伺服器的網路卡設定為靜態IP,不使用DHCP
## 更改節點hostname
[root@localhost ~]# hostnamectl set-hostname kubemaster --static
## 獲取節點網路卡名
[root@localhost ~]# ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
link/ether fa:16:3e:0b:68:40 brd ff:ff:ff:ff:ff:ff
inet 192.168.1.4/24 brd 192.168.1.255 scope global noprefixroute dynamic eth0
valid_lft 77613sec preferred_lft 77613sec
inet6 fe80::f816:3eff:fe0b:6840/64 scope link
valid_lft forever preferred_lft forever
此時需要設定
eth0
網路卡,命令格式為vi /etc/sysconfig/network-scripts/ifcfg-[網路卡名稱]
## 設定eth0網路卡
[root@localhost ~]# vi /etc/sysconfig/network-scripts/ifcfg-eth0
修改以下內容
BOOTPROTO="static" # dhcp改為static
ONBOOT="yes" # 開機啟用本設定
IPADDR=192.168.1.4 # 靜態IP
GATEWAY=192.168.1.1 # 預設閘道器
NETMASK=255.255.255.0 # 子網掩碼
DNS1=114.114.114.114 # DNS 設定
DNS2=8.8.8.8 # DNS 設定【必須設定,否則SDK映象下載很慢】
隨後重啟伺服器並編輯hosts檔案
## 重啟伺服器
[root@localhost ~] reboot
## 檢視hostname是否生效
[root@kubemaster ~]# hostname
kubemaster
## 編輯/etc/hosts檔案,設定對映關係
[root@kubemaster ~]# vi /etc/hosts
新增hosts檔案的規則
192.168.1.4 kubemaster
192.168.1.5 kubeworker1
192.168.1.6 kubeworker2
# 更改節點hostname
[root@localhost ~]# hostnamectl set-hostname kubeworker1 --static
# 獲取節點網路卡名
[root@localhost ~]# ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
link/ether fa:16:3e:0b:68:40 brd ff:ff:ff:ff:ff:ff
inet 192.168.1.5/24 brd 192.168.1.255 scope global noprefixroute dynamic eth0
valid_lft 77613sec preferred_lft 77613sec
inet6 fe80::f816:3eff:fe0b:6840/64 scope link
valid_lft forever preferred_lft forever
此時需要設定eth0
網路卡,命令格式為vi /etc/sysconfig/network-scripts/ifcfg-[網路卡名稱]
# 設定eth0網路卡
[root@localhost ~]# vi /etc/sysconfig/network-scripts/ifcfg-eth0
修改以下內容
BOOTPROTO="static" #dhcp改為static
ONBOOT="yes" #開機啟用本設定
IPADDR=192.168.1.5 #靜態IP
GATEWAY=192.168.1.1 #預設閘道器
NETMASK=255.255.255.0 #子網掩碼
DNS1=114.114.114.114 #DNS 設定
DNS2=8.8.8.8 #DNS 設定【必須設定,否則SDK映象下載很慢】
隨後重啟伺服器並編輯hosts檔案
## 重啟伺服器
[root@localhost ~] reboot
## 檢視hostname是否生效
[root@kubeworker1 ~]# hostname
kubeworker1
## 編輯/etc/hosts檔案,設定對映關係
[root@kubeworker1 ~]# vi /etc/hosts
新增hosts檔案的規則
192.168.1.4 kubemaster
192.168.1.5 kubeworker1
192.168.1.6 kubeworker2
# 更改節點hostname
[root@localhost ~]# hostnamectl set-hostname kubeworker2 --static
# 獲取節點網路卡名
[root@localhost ~]# ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
link/ether fa:16:3e:0b:68:40 brd ff:ff:ff:ff:ff:ff
inet 192.168.1.6/24 brd 192.168.1.255 scope global noprefixroute dynamic eth0
valid_lft 77613sec preferred_lft 77613sec
inet6 fe80::f816:3eff:fe0b:6840/64 scope link
valid_lft forever preferred_lft forever
此時需要設定eth0
網路卡,命令格式為vi /etc/sysconfig/network-scripts/ifcfg-[網路卡名稱]
# 設定eth0網路卡
[root@localhost ~]# vi /etc/sysconfig/network-scripts/ifcfg-eth0
修改以下內容
BOOTPROTO="static" #dhcp改為static
ONBOOT="yes" #開機啟用本設定
IPADDR=192.168.1.6 #靜態IP
GATEWAY=192.168.1.1 #預設閘道器
NETMASK=255.255.255.0 #子網掩碼
DNS1=114.114.114.114 #DNS 設定
DNS2=8.8.8.8 #DNS 設定【必須設定,否則SDK映象下載很慢】
隨後重啟伺服器並編輯hosts檔案
## 重啟伺服器
[root@localhost ~] reboot
## 檢視hostname是否生效
[root@kubeworker2 ~]# hostname
kubeworker2
## 編輯/etc/hosts檔案,設定對映關係
[root@kubeworker2 ~]# vi /etc/hosts
新增hosts檔案的規則
192.168.1.4 kubemaster
192.168.1.5 kubeworker1
192.168.1.6 kubeworker2
注意:
此項需要每一臺機器都安裝
yum install -y conntrack ntpdate ntp ipvsadm ipset jq iptables curl sysstatlibseccomp wget vim net-tools git iproute lrzsz bash-completion tree bridge-utils unzip bind-utils gcc
用普通的noteport不行,必須用ingress
注意:
生產環境建議放行埠
systemctl stop firewalld && systemctl disable firewalld
iptables設定
注意:
iptables -F
命令為清空iptables規則,生產環境下會清空已有規則,需謹慎執行
安裝、啟動iptables,設定開機自啟,清空iptables規則,儲存當前規則到預設規則
yum -y install iptables-services && systemctl start iptables && systemctl enable iptables && iptables -F && service iptables save
注意:
關閉Selinux是為了放行指令碼(安裝的時候需要執行指令碼)
# 關閉swap分割區【虛擬記憶體】並且永久關閉虛擬記憶體
[root@kubemaster ~]# swapoff -a && sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
# 關閉selinux
[root@kubemaster ~]# swapoff -a && sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
[root@kubemaster ~]# setenforce 0 && sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config
setenforce: SELinux is disabled
注意:
此項需要每一臺機器都設定
K8s必須禁用ipv6
net.ipv6.conf.all.disable_ipv6=1
cat > kubernetes.conf <<EOF
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
net.ipv4.tcp_tw_recycle=0
vm.swappiness=0
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963
fs.nr_open=52706963
net.ipv6.conf.all.disable_ipv6=1
net.netfilter.nf_conntrack_max=2310720
EOF
#將優化核心檔案拷貝到/etc/sysctl.d/資料夾下,這樣優化檔案開機的時候能夠被呼叫
cp kubernetes.conf /etc/sysctl.d/kubernetes.conf
#自動載入br_netfilter模組
modprobe br_netfilter
#自動載入ip_conntrack模組
modprobe ip_conntrack
#手動重新整理,讓優化檔案立即生效
sysctl -p /etc/sysctl.d/kubernetes.conf
#設定系統時區為中國/上海
timedatectl set-timezone "Asia/Shanghai"
#將當前的UTC 時間寫入硬體時鐘
timedatectl set-local-rtc 0
#重啟依賴於系統時間的服務
systemctl restart rsyslog
systemctl restart crond
systemctl stop postfix && systemctl disable postfix
[root@kubemaster ~]# mkdir /var/log/journal
[root@kubemaster ~]# mkdir /etc/systemd/journald.conf.d
cat > /etc/systemd/journald.conf.d/99-prophet.conf <<EOF
[Journal]
Storage=persistent
Compress=yes
SyncIntervalSec=5m
RateLimitInterval=30s
RateLimitBurst=1000
SystemMaxUse=10G
SystemMaxFileSize=200M
MaxRetentionSec=2week
ForwardToSyslog=no
EOF
systemctl restart systemd-journald
echo "* soft nofile 65536" >> /etc/security/limits.conf
echo "* hard nofile 65536" >> /etc/security/limits.conf
注意:
kube-proxy 的ingress部署,需要開啟 ipvs
cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF
#使用lsmod命令檢視這些檔案是否被引導
chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack_ipv4
ip_vs_sh 12688 0
ip_vs_wrr 12697 0
ip_vs_rr 12600 0
ip_vs 145458 6 ip_vs_rr,ip_vs_sh,ip_vs_wrr
nf_conntrack_ipv4 15053 0
nf_defrag_ipv4 12729 1 nf_conntrack_ipv4
nf_conntrack 139264 2 ip_vs,nf_conntrack_ipv4
libcrc32c 12644 2 ip_vs,nf_conntrack
注意:
此項需要每一臺機器都安裝
#安裝依賴
yum update
yum install -y yum-utils device-mapper-persistent-data lvm2
#設定倉庫
yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
#安裝docker ce
yum install docker-ce-20.10.22
#建立/etc/docker目錄
mkdir /etc/docker
#更新daemon.json檔案
cat > /etc/docker/daemon.json <<EOF
{
"registry-mirrors": [
"https://ebkn7ykm.mirror.aliyuncs.com",
"https://docker.mirrors.ustc.edu.cn",
"http://f1361db2.m.daocloud.io",
"https://registry.docker-cn.com"
],
"exec-opts": ["native.cgroupdriver=systemd"],
"log-driver": "json-file",
"log-opts": {
"max-size": "100m"
},
"storage-driver": "overlay2"
}
EOF
#注意:一定注意編碼問題,出現錯誤---檢視命令:journalctl -amu docker 即可發現錯誤
#建立,儲存docker組態檔
# mkdir -p /etc/systemd/system/docker.service.d
[root@kubemaster ~]# systemctl daemon-reload && systemctl restart docker && systemctl enable docker
Created symlink from /etc/systemd/system/multi-user.target.wants/docker.service to /usr/lib/systemd/system/docker.service.
注意:
安裝後需使用docker info
檢視是否有網路警告,會影響後續k8s部署
注意:
此項需要每一臺機器都安裝
國內映象設定(國內建議設定)
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
官網映象設定
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
EOF
[root@kubemaster ~]# yum install -y kubelet-1.23.15 kubeadm-1.23.15 kubectl-1.23.15 --disableexcludes=kubernetes
[root@kubemaster ~]# systemctl enable kubelet && systemctl start kubelet
Created symlink from /etc/systemd/system/multi-user.target.wants/kubelet.service to /usr/lib/systemd/system/kubelet.service.
注意:
此項需要安裝在Master節點
# 初始化組態檔
kubeadm config print init-defaults > kubeadm-init.yaml
# 修改組態檔
vi kubeadm-init.yaml
# 檢視kubeadm版本
kubeadm version
需要修改的項
advertiseAddress: 1.2.3.4
修改為本地使用的IP地址,範例上使用的是192.168.1.4
,就修改為advertiseAddress: 192.168.1.4
kubernetesVersion: 1.23.0
修改為當前使用的版本,範例上使用的是1.23.15
,就修改為kubernetesVersion: 1.23.15
imageRepository: k8s.gcr.io
修改為imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers
networking
下新增podSubnet: 10.244.0.0/16
修改完畢後檔案如下
apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: abcdef.0123456789abcdef
ttl: 24h0m0s
usages:
- signing
- authentication
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 192.168.1.4 # 本機IP
bindPort: 6443
nodeRegistration:
criSocket: /var/run/dockershim.sock
name: k8s-master
taints:
- effect: NoSchedule
key: node-role.kubernetes.io/master
---
apiServer:
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns:
type: CoreDNS
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers #映象倉庫
kind: ClusterConfiguration
kubernetesVersion: v1.20.15
networking:
dnsDomain: cluster.local
serviceSubnet: 10.96.0.0/12
podSubnet: 10.244.0.0/16 # 新增Pod子網路
scheduler: {}
[root@kubemaster ~]# kubeadm config images pull --config kubeadm-init.yaml
[root@kubemaster ~]# kubeadm init --config kubeadm-init.yaml
初始化後,會出現以下命令,後面追加Node的時候需要用
kubeadm join 192.168.1.4:6443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:602efef33cee46c1aa6a95ddd0972606e826ef122f810930e835b4f536cddc14
當前Master節點的STATUS是NotReady,是因為沒有設定網路
## 設定kubectl執行命令環境
mkdir -p $HOME/.kube
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
chown $(id -u):$(id -g) $HOME/.kube/config
## 執行kubectl命令檢視機器節點
[root@kubemaster ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
node NotReady control-plane,master 14m v1.23.15
wget https://docs.projectcalico.org/manifests/calico.yaml
這裡需要指定網路卡(新增IP_AUTODETECTION_METHOD
)
## 編輯calico.yaml
vi calico.yaml
下面的範例擷取了部分組態檔,eth.*
的意思就是以eth
為開頭的網路卡,根據伺服器的不同,字首也會不同
# Cluster type to identify the deployment type
- name: CLUSTER_TYPE
value: "k8s,bgp"
# IP automatic detection
- name: IP_AUTODETECTION_METHOD
value: "interface=eth.*"
# Auto-detect the BGP IP address.
- name: IP
value: "autodetect"
# Enable IPIP
- name: CALICO_IPV4POOL_IPIP
value: "Always"
kubectl apply -f calico.yaml
此時檢視node資訊, Master的狀態已經是Ready
了.
[root@kubemaster ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
node Ready control-plane,master 14m v1.23.15
注意:
此項需要執行在Node節點
在其他Node執行以下命令即可
kubeadm join 192.168.1.4:6443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:602efef33cee46c1aa6a95ddd0972606e826ef122f810930e835b4f536cddc14
[root@kubemaster ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
kubemaster Ready control-plane,master 14m v1.23.15
kubeworker1 Ready <none> 5m37s v1.23.15
kubeworker2 Ready <none> 5m28s v1.23.15
[root@kubemaster ~]# kubectl get pod -n kube-system -o wide
## 如果看到下面的pod狀態都是Running狀態,說明K8S叢集環境就構建完成
wget https://raw.githubusercontent.com/kubernetes/dashboard/v2.5.1/aio/deploy/recommended.yaml
在recommended.yaml檔案中尋找kubernetes-dashboard
,新增存取方式為NodePort,埠為30443,範例為組態檔需要修改的部分,需要新增type: NodePort
和nodePort: 30443
kind: Service
apiVersion: v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kubernetes-dashboard
spec:
type: NodePort
ports:
- port: 443
targetPort: 8443
nodePort: 30443
selector:
k8s-app: kubernetes-dashboard
kubectl apply -f recommended.yaml
檢視dashboard是否進行了設定,443:30443/TCP
即證明已設定完成
[root@kubemaster ~]# kubectl get svc -A | grep kubernetes-dashboard
kubernetes-dashboard dashboard-metrics-scraper ClusterIP 10.110.95.223 <none> 8000/TCP 107m
kubernetes-dashboard kubernetes-dashboard NodePort 10.111.35.64 <none> 443:30443/TCP 107m
cat > dashboard-admin.yaml << EOF
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: admin
annotations:
rbac.authorization.kubernetes.io/autoupdate: "true"
roleRef:
kind: ClusterRole
name: cluster-admin
apiGroup: rbac.authorization.k8s.io
subjects:
- kind: ServiceAccount
name: admin
namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: admin
namespace: kube-system
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
EOF
[root@kubemaster ~]# kubectl apply -f dashboard-admin.yaml
clusterrolebinding.rbac.authorization.k8s.io/admin created
serviceaccount/admin created
[root@kubemaster ~]# kubectl -n kube-system get secret|grep admin-token
admin-token-w5gl9 kubernetes.io/service-account-token 3 2m20s
[root@kubemaster ~]# kubectl -n kube-system describe secret admin-token-w5gl9
Name: admin-token-w5gl9
Namespace: kube-system
Labels: <none>
Annotations: kubernetes.io/service-account.name: admin
kubernetes.io/service-account.uid: 958ae7a6-66b0-4685-b1d5-cf4be9523940
Type: kubernetes.io/service-account-token
Data
====
ca.crt: 1099 bytes
namespace: 11 bytes
token: eyJhbGciOiJSUzI1NiIsImtpZCI6InpQUjkxMXJYR1RaUEZMU1AtZV9rU3VLVEs3djVGNFdpWGZQMmtZTlRaQkEifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJhZG1pbi10b2tlbi13NWdsOSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50Lm5hbWUiOiJhZG1pbiIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6Ijk1OGFlN2E2LTY2YjAtNDY4NS1iMWQ1LWNmNGJlOTUyMzk0MCIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDprdWJlLXN5c3RlbTphZG1pbiJ9.cfELmWrVeLY4fJsR9b72_Uyy4HJ1sl9IIRCzje17l-ZOcyJq6TUKhIbfGt52YOa7b2ZNF-yjln-kcUKP5hlMEafPRyEy4UzFvOT3e9PW6PolTqB23NUPpcyu_sUflxVzOEZMXngqvvyxqgxk6fmoLOTRhLAnfhyI_cHidn4Pffen3uBMB1pAPXfNp9exDxMjHLhrJDsc9RGOe7gJqVTuvAOe2fV5A4Fd_pxiZmwKrZr4S4EpCHtBYWCz_xil5eclSzjBCvu_ZR9YSGRAsNt0OocEi4QnqPSIxYsm4KzVyDp9AWao9vGpDwmJ5RmFLm6E-0JQJc5hMSUwSbFkte8jHg
在瀏覽器輸入https://[yourIP]:30443
,填入IP地址並存取,會出現下圖,在下圖token處填入剛才獲取的token即可進入Dashboard