# 确认唯一UUID
cat /sys/class/dmi/id/product_uuid

# 初始化变量定义
api=k8s-api.alfie.com
m1=k8s-master01.alfie.com
w1=k8s-node01.alfie.com
w2=k8s-node02.alfie.com
w3=k8s-node03.alfie.com

m1_ip=192.168.100.110
w1_ip=192.168.100.121
w2_ip=192.168.100.122
w3_ip=192.168.100.123

# hostname 修改
hostnamectl set-hostname ${hn}

# hosts记录更新
cat > /etc/hosts <<EOF
${m1_ip} ${api}
${m1_ip} ${m1}
${w1_ip} ${w1}
${w2_ip} ${w2}
${w3_ip} ${w3}
EOF

# 按主机修改ip
ip=${xxx}
hn=${yyy}

# Ubuntu系列修改IP
sed -i.bak "6c\        - ${ip}/24" \
/etc/netplan/00-installer-config.yaml
netplan apply

bash -c "cat < EOF > /etc/apt/sources.list && apt update
deb http://mirrors.aliyun.com/ubuntu/ jammy main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ jammy main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ jammy-security main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ jammy-security main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ jammy-updates main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ jammy-updates main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ jammy-proposed main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ jammy-proposed main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ jammy-backports main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ jammy-backports main restricted universe multiverse
EOF"

# 修改时区为中国时间
timedatectl set-timezone Asia/Shanghai

# 关闭此前的默认时间同步工具
systemctl unmask systemd-timesyncd.service
systemctl disable --now systemd-timesyncd.service

apt -y install chrony
mv /etc/chrony/chrony.conf /etc/chrony/chrony.conf.bak

cat > /etc/chrony/chrony.conf << EOF
server ntp1.aliyun.com iburst minpoll 4 maxpoll 10
server ntp2.aliyun.com iburst minpoll 4 maxpoll 10
server ntp3.aliyun.com iburst minpoll 4 maxpoll 10
server ntp4.aliyun.com iburst minpoll 4 maxpoll 10
server ntp5.aliyun.com iburst minpoll 4 maxpoll 10
server ntp6.aliyun.com iburst minpoll 4 maxpoll 10
server ntp7.aliyun.com iburst minpoll 4 maxpoll 10
driftfile /var/lib/chrony/drift
makestep 10 3
rtcsync
allow 0.0.0.0/0
local stratum 10
#keyfile /etc/chrony.keys
logdir /var/log/chrony
stratumweight 0.05
noclientlog
logchange 0.5
EOF

# 开启当前时间同步chrony
systemctl daemon-reload
systemctl enable --now chrony.service

sed -ri.bak 's/.*swap.*/#&/' /etc/fstab
swappoff -a
#sed -n '/swap/ s/\(.*\)/# \1/gp' /etc/fstab
systemctl disable --now swap.target 

# 检查是否关闭
swapon --show

ufw disable
ufw status

cat > /etc/sysctl.d/k8s.conf <<EOF
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
fs.file-max=52706963
fs.nr_open=52706963
net.netfilter.nf_conntrack_max=2310720
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_keepalive_intvl =15
net.ipv4.tcp_max_tw_buckets = 36000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_max_orphans = 327680
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.ip_conntrack_max = 65536
net.ipv4.tcp_max_syn_backlog = 16384
net.core.somaxconn = 16384
EOF

# 重新加载配置
sysctl --system

# 文件资源限制
sudo sh -c 'cat >> /etc/security/limits.d/k8s.conf <<EOF
* soft nofile 65535
* hard nofile 131070
EOF'

ulimit -Sn 
ulimit -Hn

apt -y install tree nmap sysstat \
lrzsz dos2unix telnet vim lsof net-tools \
rsync wget jq psmisc ipvsadm \
lvm2 git curl tar bash-completion expect \
apt-transport-https ca-certificates curl ipset iputils-ping

# 安装docker
#0. 卸载此前的docker
sudo apt purge docker-ce \
docker-ce-cli  \
docker-buildx-plugin \
docker-ce-rootless-extras \
docker-compose-plugin \
containerd.io \
docker-scan-plugin && sudo rm -rf /var/lib/docker && sudo apt autoremove

#1. 安装必备工具
sudo apt-get -y install apt-transport-https ca-certificates curl software-properties-common

#2. 安装GPG证书
curl -fsSL https://mirrors.aliyun.com/docker-ce/linux/ubuntu/gpg | sudo apt-key add -

#3. 写入仓库
sudo add-apt-repository "deb [arch=amd64] https://mirrors.aliyun.com/docker-ce/linux/ubuntu $(lsb_release -cs) stable"

#4. 查询版本
apt-cache madison docker-ce
>  docker-ce | 5:25.0.3-1~ubuntu.22.04~jammy | https://mirrors.aliyun.com/docker-ce/linux/ubuntu jammy/stable amd64 Packages

#5. 安装指定docker-ce (指定版本不能只指定docker-ce，否则其他都会使用最新版本造成不是适配)
v='5:25.0.3-1~ubuntu.22.04~jammy'
sudo apt-get install docker-ce=$v \
docker-ce-cli=$v \
docker-ce-rootless-extras=$v 

#6. 配置镜像加速
cat > /etc/docker/daemon.json <<EOF
{
  "registry-mirrors": [
    "https://registry.cn-hangzhou.aliyuncs.com"
  ],
  "exec-opts": [
    "native.cgroupdriver=systemd"
  ],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "200m"
  },
  "storage-driver": "overlay2"
}
EOF

#7. 配置代理
vim /lib/systemd/system/docker.service
[Service]
Environment="HTTP_PROXY=http://$PROXY_SERVER_IP:$PROXY_PORT"
Environment="HTTPS_PROXY=http://$PROXY_SERVER_IP:$PROXY_PORT"
Environment="NO_PROXY=127.0.0.0/8,192.168.100.0/24,10.244.0.0/16,192.168.0.0/16,10.96.0.0/12,alfie.com,cluster.local"

#8. 配置加载
systemctl daemon-reload && systemctl start docker

#9. 查看版本
docker version

# 下载安装包
curl -LO https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.10/cri-dockerd_0.3.10.3-0.ubuntu-jammy_amd64.deb

# 安装
dpkg -i ./cri-dockerd_0.3.10.3-0.ubuntu-jammy_amd64.deb
#apt install ./cri-dockerd_0.3.10.3-0.ubuntu-jammy_amd64.deb

# 查看其状态
systemctl status cri-docker.service

vim /usr/lib/systemd/system/cri-docker.service

ExecStart=/usr/bin/cri-dockerd --container-runtime-endpoint fd:// --network-plugin=cni --cni-bin-dir=/opt/cni/bin --cni-cache-dir=/var/lib/cni/cache --cni-conf-dir=/etc/cni/net.d --pod-infra-container-image=registry.k8s.io/pause:3.9

    --network-plugin：指定网络插件规范的类型，这里要使用CNI；
    --cni-bin-dir：指定CNI插件二进制程序文件的搜索目录； 
    --cni-cache-dir：CNI插件使用的缓存目录； 
    --cni-conf-dir：CNI插件加载配置文件的目录； 
    --pod-infra-container-image：Pod中的puase容器要使用的Image，默认为registry.k8s.io上的pause仓库中的镜像；不能直接获取到该Image时，需要明确指定为从指定的位置加载，例如“registry.aliyuncs.com/google_containers/pause:3.9”。

# 重启服务
systemctl daemon-reload && systemctl restart cri-docker.service

# 导入containerd密钥
curl -fsSL http://mirrors.aliyun.com/docker-ce/linux/ubuntu/gpg | apt-key add -

# 添加库
add-apt-repository "deb [arch=amd64] http://mirrors.aliyun.com/docker-ce/linux/ubuntu $(lsb_release -cs) stable"

# 更新库
apt update

# 安装
apt-get  install  containerd.io

# 配置
## 生成默认配置
mkdir /etc/containerd
containerd config default > /etc/containerd/config.toml

## 定制所需配置
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
  SystemdCgroup = true

[plugins."io.containerd.grpc.v1.cri"]
sandbox_image = "registry.aliyuncs.com/google_containers/pause:3.9"

[plugins."io.containerd.grpc.v1.cri".registry]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors]
  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
    endpoint = ["https://docker.mirrors.ustc.edu.cn", "https://registry.docker-cn.com"]

[plugins."io.containerd.grpc.v1.cri".registry.mirrors."registry.k8s.io"]
endpoint = ["https://registry.aliyuncs.com/google_containers"]

[plugins."io.containerd.grpc.v1.cri".registry]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors]
  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."registry.magedu.com"]
    endpoint = ["https://registry.magedu.com"]

systemctl daemon-reload  && systemctl restart containerd

vim /etc/crictl.yam
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout: 10
debug: true

# 导入gpg证书
curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.29/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg

# 写入源文件
echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.29/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list

# 查看当前支持版本
apt-cache madison kublet 
>   kubelet | 1.29.2-1.1 | https://pkgs.k8s.io/core:/stable:/v1.29/deb  Packages

# 安装 kublet, kubeadm, kubectl
v=1.29.2-1.1
apt-get update
apt-get install -y kubelet=$v kubeadm=$v kubectl=$v
apt-mark hold kubelet kubeadm kubectl

# 只能获得最新大版本下的镜像
kubeadm config images list

# 指定所需的版本镜像
kubeadm config images list --kubernetes-version v1.29.2
> registry.k8s.io/kube-apiserver:v1.29.2
> registry.k8s.io/kube-controller-manager:v1.29.2
> registry.k8s.io/kube-scheduler:v1.29.2
> registry.k8s.io/kube-proxy:v1.29.2
> registry.k8s.io/coredns/coredns:v1.11.1
> registry.k8s.io/pause:3.9
> registry.k8s.io/etcd:3.5.10-0

# 二选一
  # 查看国内源list
    kubeadm config images list --image-repository=registry.aliyuncs.com/google_containers --kubernetes-version v1.29.2
    ## 通过国内源下载
    kubeadm config images pull --image-repository=registry.aliyuncs.com/google_containers --kubernetes-version v1.29.2
    ## 修改镜像tag
    docker tag registry.aliyuncs.com/google_containers/kube-apiserver:v1.29.2  registry.k8s.io/kube-apiserver:v1.29.2
    ...

    ## 通过国外源下载
    ## 使用cri-dockerd 需要添加 --cri-socket
    kubeadm config images pull --kubernetes-version v1.29.2 --cri-socket=unix:///var/run/cri-dockerd.sock
    
    ## 使用contained则使用如下命令
    kubeadm config images pull --kubernetes-version v1.29.2

kubeadm init \
      --control-plane-endpoint="k8s-master01.alfie.com" \
      --kubernetes-version=v1.29.2 \
      --pod-network-cidr=10.244.0.0/16 \
      --service-cidr=10.96.0.0/12 \
      --token-ttl=0 \
      --upload-certs \
      --cri-socket=unix:///var/run/cri-dockerd.sock

kubeadm config print init-defaults

apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
  - groups:
      - system:bootstrappers:kubeadm:default-node-token
    token: alfie.comc4mu9kzd5q7ur
    ttl: 24h0m0s
    usages:
      - signing
      - authentication
kind: InitConfiguration
localAPIEndpoint:
  # 这里的地址即为初始化的控制平面第一个节点的IP地址；
  advertiseAddress: 192.168.100.110
  bindPort: 6443
nodeRegistration:
  # 注意，使用docker-ce和cri-dockerd时，要启用如下配置的cri socket文件的路径；
  criSocket: unix:///run/cri-dockerd.sock
  imagePullPolicy: IfNotPresent
  # 第一个控制平面节点的主机名称；
  name: k8s-master01.magedu.com
  taints:
    - effect: NoSchedule
      key: node-role.kubernetes.io/master
    - effect: NoSchedule
      key: node-role.kubernetes.io/control-plane
---
apiServer:
  timeoutForControlPlane: 4m0s
  # 将下面配置中的certSANS列表中的值，修改为客户端接入API Server时可能会使用的各类目标地址；
  certSANs:
    - kubeapi.alfie.com
    - 192.168.100.110
apiVersion: kubeadm.k8s.io/v1beta3
# 控制平面的接入端点，我们这里选择适配到kubeapi.alfie.com这一域名上；
controlPlaneEndpoint: "kubeapi.alfie.com:6443"
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns: {}
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: v1.29.2
networking:
  # 集群要使用的域名，默认为cluster.local
  dnsDomain: cluster.local
  # service网络的地址
  serviceSubnet: 10.96.0.0/12
  # pod网络的地址，flannel网络插件默认使用10.244.0.0/16
  podSubnet: 10.244.0.0/16
scheduler: {}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
# 用于配置kube-proxy上为Service指定的代理模式，默认为iptables；
mode: "ipvs"

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

# 部署flannel
kubectl apply -f https://github.com/flannel-io/flannel/releases/latest/download/kube-flannel.yml

# 验证cri容器安装
kubectl get pods -n kube-flannel

# 使用`docker+cri-dockered`作为运行时，需要使用 `--cri-socket`指定运行时
  kubeadm join k8s-master01.alfie.com:6443 --token n2fewy.h1fwv3svdrf9l6w8 \
    --discovery-token-ca-cert-hash sha256:d0b3fb23632bb02fddad5789e1cb09c34161a152421bf23e3263bce54800392f \
    --control-plane --certificate-key 927190558e3213e44841da9aa5de38391882db7a47bc83244df49a0f6f904d0b
    --cri-socket=unix:///var/run/cri-dockerd.sock

# 使用`docker+cri-dockered`作为运行时，需要使用 `--cri-socket`指定运行时
kubeadm join k8s-master01.alfie.com:6443 --token n2fewy.h1fwv3svdrf9l6w8 \
    --discovery-token-ca-cert-hash sha256:d0b3fb23632bb02fddad5789e1cb09c34161a152421bf23e3263bce54800392f \
    --cri-socket=unix:///var/run/cri-dockerd.sock

# 查看集群节点
kubectl get nodes

1. 配置configmap
kubectl edit configmap kube-proxy -n kube-system

2. 修改mode
    mode: "ipvs"
3. 删除所有kube-proxy pods
    kubectl get po -n kube-system
    kubectl delete po -n kube-system <pod-name>
4. 验证kube-proxy pod 是否采用了 ipvs
    kubectl logs [kube-proxy pod] -n kube-system| grep "Using ipvs Proxier"