aliyun3-k8s-1.26安装步骤

准备6台主机，3master 3worker,主要安装步骤，1、系统时钟，关ufw;2、安装containerd;3、下载k8s images 4、增加k8s master init;5、增加k8s work节点

1、安装系统初始化
systemctl disable nfs-mountd.service rpcbind.service rpcbind.socket rpcbind.target systemd-resolved.service
systemctl stop nfs-mountd.service rpcbind.service rpcbind.socket rpcbind.target systemd-resolved.service

yum update
yum -y install perl gcc gcc-c++ cpp bzip2 openssl-devel bind-utils wget net-tools nmon iftop readline readline-devel
1)、swapoff -a

2)、setenforce 0 && sed -i ‘s/^SELINUX=.*/SELINUX=disabled/’ /etc/selinux/config

3)、chrony sources -v
4)、timedatectl set-timezone Asia/Shanghai
ls -l /etc/localtime

2、时间同步,修改hosts
#!/bin/sh

systemctl stop firewalld && systemctl disable firewalld
sed -i ‘s/SELINUX=enforcing/SELINUX=disabled/g’ /etc/sysconfig/selinux
setenforce 0

yum -y install chrony

sed -i ‘s/2.centos.pool.ntp.org/ntp3.aliyun.com/g’ /etc/chrony.conf
sed -i ‘s/#local/local/g’ /etc/chrony.conf
sed -i “/#allow/a allow all” /etc/chrony.conf

systemctl enable chronyd.service
systemctl restart chronyd.service
chronyc sources -v;date

3、关闭防火墙/setenforce 0
4、sysctl.d/k8s.conf和/etc/modules-load.d/ipvs.conf
#!/bin/sh

cat << EOF > /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1

vm.swappiness=0
vm.overcommit_memory=1
vm.panic_on_oom=0

fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963
fs.nr_open=52706963
fs.may_detach_mounts = 1
net.netfilter.nf_conntrack_max=2310720

net.core.somaxconn = 16384
net.ipv4.ip_forward=1
net.ipv4.tcp_tw_recycle=0
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_timestamps = 0
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_keepalive_intvl =15
net.ipv4.tcp_max_tw_buckets = 36000
net.ipv4.tcp_max_orphans = 327680
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.ip_conntrack_max = 65536
net.ipv4.tcp_max_syn_backlog = 16384

net.ipv6.conf.all.disable_ipv6=1
net.ipv6.conf.default.disable_ipv6 = 0
net.ipv6.conf.lo.disable_ipv6 = 0
net.ipv6.conf.all.forwarding = 1
EOF

sysctl -p /etc/sysctl.d/k8s.conf

yum -y install ipvsadm ipset sysstat conntrack libseccomp

sysctl –system

modprobe br_netfilter
cat > /etc/sysconfig/modules/ipvs.modules < /etc/modules-load.d/ipvs.conf ip_vs ip_vs_rr ip_vs_wrr ip_vs_sh nf_conntrack ip_tables ip_set xt_set ipt_set ipt_rpfilter ipt_REJECT ipip EOF systemctl restart systemd-modules-load.service lsmod | grep -e ip_vs -e nf_conntrack cat < /etc/yum.repos.d/kubernetes.repo [kubernetes] name=Kubernetes baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64 #baseurl=https://mirrors.tuna.tsinghua.edu.cn/kubernetes/yum/repos/kubernetes-el7-x86_64/ enabled=1 gpgcheck=0 repo_gpgcheck=0 gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg EOF ver=1.26.3 yum -y install kubeadm-${ver} kubectl-${ver} kubelet-${ver} cat << EOF > ~/restart_kubelet.sh
systemctl enable –now kubelet
EOF

echo ‘source <(kubectl completion bash)' >> ~/.bash_profile
5、每台都需要安装containerd非常重要
crictl
#!/bin/sh
cat << EOF | sudo tee /etc/modules-load.d/containerd.conf overlay br_netfilter EOF systemctl restart systemd-modules-load.service sudo modprobe overlay sudo modprobe br_netfilter wget https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo -O /etc/yum.repos.d/docker-ce.repo yum -y install containerd.io.x86_64 # 修改containerd配置，将sanbox_image镜像源设置为阿里云google_containers镜像源 mkdir /etc/containerd/ # 产生containerd默认配置文件 containerd config default > /etc/containerd/config.toml
#sed -i "s#k8s.gcr.io/pause#registry.aliyuncs.com/google_containers/pause#g" /etc/containerd/config.toml
#sed -i "s#https://registry-1.docker.io#https://0k0953tv.mirror.aliyuncs.com#g" /etc/containerd/config.toml

sed -i "s#registry.k8s.io/pause:3.6#registry.aliyuncs.com/google_containers/pause:3.9#g" /etc/containerd/config.toml

# 配置containerd cgroup驱动程序systemd
sed -i 's#SystemdCgroup = false#SystemdCgroup = true#g' /etc/containerd/config.toml

#vi /etc/containerd/config.toml
# 在此行下面加两行
#[plugins."io.containerd.grpc.v1.cri".registry.mirrors]
# [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
# endpoint = ["https://registry.aliyuncs.com"]

# 重启containerd
systemctl daemon-reload

# 服务启动
systemctl enable --now containerd
systemctl restart containerd
systemctl status containerd --no-pager -l

# 利用containerd的客户端工具ctr验证修改是否成功
#containerd --version
#ctr version

#设置crictl
cat << EOF >> /etc/crictl.yaml
runtime-endpoint: unix:///var/run/containerd/containerd.sock
image-endpoint: unix:///var/run/containerd/containerd.sock
timeout: 10
debug: false
EOF
6、每台都需要安装kubelet/kubeadm/kubectl，最新1.26.3,并systemctl enable –now kubelet
yum install kubelet kubeadm kubectl
7、docker pull images

#!/bin/sh
ver=1.26.4
src=registry.aliyuncs.com/google_containers
dst=k8s.gcr.io

kubeadm config images list --kubernetes-version=${ver} \
--image-repository ${src}

#kubeadm config images pull --kubernetes-version=${ver} \
# --image-repository ${src}

#ctr task ls
#crictl images list
#ctr image ls docker images
#ctr image pull pause docker pull pause pull 应该pause镜像
#ctr image push pause-test docker push pause-test 改名
#ctr image import pause.tar docker load 镜像 导入本地镜像
#ctr run -d pause-test pause docker run -d --name=pause pause-test 运行容器
#ctr image tag pause pause-test docker tag pause pause-test tag应该pause镜像 tag改标签

8、kubeadm init
#!/bin/sh
systemctl stop kubelet.service
kubeadm reset

#--cri-socket unix:///var/run/cri-dockerd.sock \
ver=v1.26.3

IP=$(hostname -I|awk '{print $1}')
kubeadm init \
--apiserver-advertise-address=10.10.8.1 \
--image-repository registry.aliyuncs.com/google_containers \
--kubernetes-version=${ver} \
--pod-network-cidr=10.244.0.0/16 \
--ignore-preflight-errors=Swap \
--cri-socket unix:///var/run/containerd/containerd.sock \
--v=5 |tee init.out

9、##master join
##复制/etc/pki/ca.crt
cd /etc/kubernetes
zip -r pki.zip admin.conf pki/ca* pki/front-proxy-c* pki/sa* pki/etcd/ca.*
复制到其他master节点初始化

kubeadm join 10.10.8.1:6443 --token 4qg53r.qrj0kl2e2jrvpcvj \
--discovery-token-ca-cert-hash sha256:1b29cb6a22455874ebb59600a44bdc2ef5b1a9b478d6f1446041fddbcf245714 \
--control-plane \
--cri-socket unix:///var/run/containerd/containerd.sock \
--v=5 |tee init.out

unable to add a new control plane instance to a cluster that doesn’t have a stable controlPlaneEndpoint address

##添加controlPlaneEndpoint这个参数

# kubectl edit cm kubeadm-config -n kube-system
kubernetesVersion: v1.26.3
controlPlaneEndpoint: 10.10.8.11:6443

10、##worker join
kubeadm join 10.10.8.1:6443 --token il3ox8.7yml00p14zpzhv4m \
--discovery-token-ca-cert-hash sha256:3843f3c22af462d577e6ad4d8cb87810fd90452934991972df81e33bbf43a94e \
--cri-socket unix:///var/run/containerd/containerd.sock \
--v=5 |tee init.out

11、##kubeconfig
#!/bin/sh

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

sed -i '/KUBECONFIG/d' ~/.bash_profile
echo "export KUBECONFIG=/etc/kubernetes/admin.conf" >> ~/.bash_profile

12、kalico或者flannel网络
wget -c https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
kubectl apply -f kube-flannel.yml

13、安装kuboard
docker run -d \
--restart=unless-stopped \
--name=kuboard-docker \
-p 88:80/tcp \
-p 10081:10081/tcp \
-e KUBOARD_ENDPOINT="http://10.10.10.118:88" \
-e KUBOARD_AGENT_SERVER_TCP_PORT="10081" \
-v /jesong/kuboard-data:/data \
swr.cn-east-2.myhuaweicloud.com/kuboard/kuboard:v3.5.2.3

14、获取token
kubeadm token create --ttl=0

get join token
kubeadm token create --print-join-command
#输出范例：
#jp6v6a.drmy4d9ri7cdlsa5

#如果你没有 --discovery-token-ca-cert-hash，你可以在control-plane节点执行如下命令获取：

#openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | \
# openssl dgst -sha256 -hex | sed 's/^.* //'
#输出范例：

#5af85d3f157c8d997bbde3adbe6cb6c975d699d10faf2ffdb2321cb9ee8fc84f

#kubeadm alpha certs renew all
kubeadm alpha certs check-expiration

kubeadm token list
###########################find pki expire
cd /etc/kubernetes/pki
for crt in $(find /etc/kubernetes/pki/ -name "*.crt"); do openssl x509 -in $crt -noout -dates; done

###update pki
kubeadm alpha certs check-expiration
kubeadm alpha certs renew all
mkdir -p /etc/kubernetes/bak
mv /etc/kubernetes/*.conf /etc/kubernetes/bak
kubeadm init phase kubeconfig all
systemctl restart kubelet
/bin/cp /etc/kubernetes/admin.conf ~/.kube/config

systemctl status kubelet
15、报错后重新搞
使用journalctl -xeu kubelet发现因为无法拉取k8s.gcr.io/pause:3.6导致pod创建失败
通过查文档，发现containerd默认配置中用到了该镜像，通过覆盖默认生成的文件，并重启containerd解决。

systemctl restart containerd

清理集群，重新初始化

kubeadm reset –cert-dir /etc/kubernetes/pki

16、后期join worker节点token如果没有就是过期了 要重新生成
kubeadm token list
#token如果没有就是过期了 要重新生成
vgih4q.i9fdwn0mjwa67r7n

kubeadm token create

openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed ‘s/^.* //’

932e86d9ca1a7444f9de70272a614271c93deacbd053c404743f637f8e669514

kubeadm join 127.0.0.1:8443 –token vgih4q.i9fdwn0mjwa67r7n \
–discovery-token-ca-cert-hash sha256:932e86d9ca1a7444f9de70272a614271c93deacbd053c404743f637f8e669514 \
–v=5

kubeadm config images list

17、污点处理
1）、让 master节点参与POD负载的命令为
kubectl taint nodes –all node-role.kubernetes.io/master-

2）、让 master节点恢复不参与POD负载的命令为
kubectl taint nodes node-role.kubernetes.io/master=:NoSchedule

参考文档：
https://kubernetes.io/docs/home/
https://kubernetes.io/zh-cn/docs/home/

相关日志