cat << EOF >> /etc/hosts
192.168.30.128 k8s-master01.xiaohui.cn k8s-master01
192.168.30.129 k8s-master02.xiaohui.cn k8s-master02
192.168.30.130 k8s-master03.xiaohui.cn k8s-master03
192.168.30.131 k8s-worker01.xiaohui.cn k8s-worker01
192.168.30.132 k8s-worker02.xiaohui.cn k8s-worker02
192.168.30.200 k8s.xiaohui.cn k8s
EOF

wget http://www.haproxy.org/download/2.6/src/haproxy-2.6.5.tar.gz
wget https://www.lua.org/ftp/lua-5.4.4.tar.gz
tar xf lua-5.4.4.tar.gz
tar xf haproxy-2.6.5.tar.gz

make -j $(nproc) TARGET=linux-glibc USE_OPENSSL=1 USE_LUA=1 USE_PCRE=1 USE_SYSTEMD=1
Makefile:661: *** unable to automatically detect the Lua library name, you can enforce its name with LUA_LIB_NAME=<name> (where <name> can be lua5.4, lua54, lua, ...).  Stop
include/haproxy/regex-t.h:31:18: fatal error: pcre.h: No such file or directory
include/haproxy/openssl-compat.h:5:24: fatal error: openssl/bn.h: No such file or directory
src/haproxy.c:80:31: fatal error: systemd/sd-daemon.h: No such file or directory

yum install gcc pcre-devel openssl-devel systemd-devel -y

cd lua-5.4.4
make
make install
cd ../haproxy-2.6.5/

make -j $(nproc) TARGET=linux-glibc USE_OPENSSL=1 USE_LUA=1 USE_PCRE=1 USE_SYSTEMD=1
make install
mkdir /etc/haproxy /var/lib/haproxy

cat << EOF > /etc/haproxy/haproxy.cfg
global
    log /dev/log local0
    log /dev/log local1 notice
    daemon
defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option http-server-close
    option forwardfor       except 127.0.0.0/8
    option                  redispatch
    retries                 1
    timeout http-request    10s
    timeout queue           20s
    timeout connect         5s
    timeout client          20s
    timeout server          20s
    timeout http-keep-alive 10s
    timeout check           10s
frontend apiserver
    bind *:64433 #非并置的情况下，这里写6443和k8s更贴近
    mode tcp
    option tcplog
    default_backend apiserver
backend apiserver
    option httpchk GET /healthz
    http-check expect status 200
    mode tcp
    option ssl-hello-chk
    balance     roundrobin
        server k8s-master01 k8s-master01.xiaohui.cn:6443 check
        server k8s-master02 k8s-master02.xiaohui.cn:6443 check
        server k8s-master03 k8s-master03.xiaohui.cn:6443 check
EOF

cat << EOF > /etc/systemd/system/haproxy.service
[Unit]
Description=HAProxy Load Balancer
After=syslog.target network.target
[Service]
ExecStartPre=/usr/local/sbin/haproxy -f /etc/haproxy/haproxy.cfg -c -q
ExecStart=/usr/local/sbin/haproxy -Ws -f /etc/haproxy/haproxy.cfg -p /var/lib/haproxy/haproxy.pid
ExecReload=/bin/kill -USR2 $MAINPID
[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable haproxy --now

wget https://www.keepalived.org/software/keepalived-2.2.7.tar.gz
tar xf keepalived-2.2.7.tar.gz
cd keepalived-2.2.7

yum install libnl libnl-devel libnl3 libnl3-devel libmnl-devel libnftnl-devel libnftnl psmisc -y

./configure
make
make install
mkdir /etc/keepalived

cat << EOF > /etc/keepalived/check_apiserver.sh
#!/bin/sh
systemctl is-active haproxy
if [ $? -eq 0 ];then
  exit 0
else
  systemctl poweroff
fi
EOF

cat << EOF > /etc/keepalived/keepalived.conf
global_defs {
    router_id lixiaohui
}
vrrp_script check_apiserver {
  script "/etc/keepalived/check_apiserver.sh"
  interval 3
  weight -2
  fall 10
  rise 2
}

vrrp_instance haproxy {
    state MASTER     # 其他机器要写成BACKUP
    interface ens33
    virtual_router_id 20
    priority 100
    authentication {
        auth_type PASS
        auth_pass lixiaohui
    }
    unicast_src_ip 192.168.30.128 #本机地址
    unicast_peer {
        192.168.30.129 #对端地址
        192.168.30.130 #对端地址
    }
    virtual_ipaddress {
        192.168.30.200
    }
    track_script {
        check_apiserver
    }
}

EOF

cat << EOF > /etc/systemd/system/keepalived.service
[Unit]
Description=LVS and VRRP High Availability Monitor
After=network-online.target syslog.target haproxy.service
Wants=network-online.target
Documentation=man:keepalived(8)
Documentation=man:keepalived.conf(5)
Documentation=man:genhash(1)
Documentation=https://keepalived.org

[Service]
Type=notify
NotifyAccess=all
PIDFile=/run/keepalived.pid
KillMode=process
EnvironmentFile=-/usr/local/etc/sysconfig/keepalived
ExecStart=/usr/local/sbin/keepalived --dont-fork $KEEPALIVED_OPTIONS
ExecReload=/bin/kill -HUP $MAINPID

[Install]
WantedBy=multi-user.target
EOF

chmod +x /etc/keepalived/check_apiserver.sh
systemctl daemon-reload
systemctl enable keepalived --now

firewall-cmd --add-port=6443/tcp --add-port=2379-2380/tcp --add-port=10250/tcp --add-port=10259/tcp --add-port=10257/tcp --add-port=64433/tcp --permanent
firewall-cmd --reload

firewall-cmd --add-port=30000-32767/tcp --add-port=10250/tcp --permanent
firewall-cmd --reload

sed -ri 's/.*swap.*/#&/' /etc/fstab
swapoff -a

wget https://github.com/containerd/nerdctl/releases/download/v0.22.2/nerdctl-full-0.22.2-linux-amd64.tar.gz
tar Cxzvvf /usr/local nerdctl-full-0.22.2-linux-amd64.tar.gz

mkdir /etc/containerd
containerd config default > /etc/containerd/config.toml
vim /etc/containerd/config.toml
...
# 如果是在国内部署，需要修改sandbox_image参数为阿里云，如果是海外的机器，就不需要修改sandbox_image
sandbox_image = "registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.6"
...
SystemdCgroup = true
...
# 添加加速器
[plugins."io.containerd.grpc.v1.cri".registry.mirrors]
  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
    endpoint = ["https://registry.cn-hangzhou.aliyuncs.com"]

systemctl enable --now containerd

cat << EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=kubernetes
baseurl=https://mirrors.tuna.tsinghua.edu.cn/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=0
EOF
setenforce 0
yum install -y kubelet kubeadm kubectl
systemctl enable kubelet && systemctl start kubelet
sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config

cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
br_netfilter
EOF
modprobe br_netfilter
cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
EOF

sudo sysctl --system

kubectl completion bash > /etc/bash_completion.d/kubectl
kubeadm completion bash > /etc/bash_completion.d/kubeadm
source /etc/bash_completion.d/kubectl
source /etc/bash_completion.d/kubeadm

crictl config runtime-endpoint unix:///run/containerd/containerd.sock
crictl images

kubeadm init \
--apiserver-advertise-address=192.168.30.128 \
--apiserver-bind-port=6443 \
--control-plane-endpoint=192.168.30.200:64433 \
--image-repository=registry.cn-hangzhou.aliyuncs.com/google_containers \
--kubernetes-version=v1.25.0 \
--service-cidr=10.96.0.0/12 \
--service-dns-domain=lixiaohui.cn \
--upload-certs

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

kubectl create -f https://docs.projectcalico.org/manifests/calico.yaml
kubectl get nodes
NAME   STATUS   ROLES           AGE     VERSION
k8s    Ready    control-plane   3m20s   v1.25.0

ssh root@k8s-master02 mkdir /etc/kubernetes/pki/etcd -p
scp -rp /etc/kubernetes/pki/ca.* k8s-master02:/etc/kubernetes/pki
scp -rp /etc/kubernetes/pki/sa.* k8s-master02:/etc/kubernetes/pki
scp -rp /etc/kubernetes/pki/front-proxy-ca.* k8s-master02:/etc/kubernetes/pki
scp -rp /etc/kubernetes/pki/etcd/ca* k8s-master02:/etc/kubernetes/pki/etcd
scp -rp /etc/kubernetes/pki/etcd/healthcheck-client.* k8s-master02:/etc/kubernetes/pki/etcd

kubeadm join 192.168.30.200:64433 --token v22v7i.901fgfr55b5mbfd3 --control-plane --discovery-token-ca-cert-hash sha256:94f73076dc062e0eba6ac502a2c54c4500a7b3a3a9062d544fe1afaf881a73a3

kubeadm join 192.168.30.200:64433 --token 389wbm.41yho95l251by9np --discovery-token-ca-cert-hash sha256:94f73076dc062e0eba6ac502a2c54c4500a7b3a3a9062d544fe1afaf881a73a3

kubectl label nodes k8s-worker01.xiaohui.cn node-role.kubernetes.io/Worker=
kubectl label nodes k8s-worker02.xiaohui.cn node-role.kubernetes.io/Worker=
kubectl get nodes
NAME                      STATUS   ROLES           AGE     VERSION
k8s-master01.xiaohui.cn   Ready    control-plane   37m     v1.25.0
k8s-master02.xiaohui.cn   Ready    control-plane   10m     v1.25.0
k8s-master03.xiaohui.cn   Ready    control-plane   6m4s    v1.25.0
k8s-worker01.xiaohui.cn   Ready    Worker          3m47s   v1.25.0
k8s-worker02.xiaohui.cn   Ready    Worker          3m56s   v1.25.0