努力挣扎的生活
yangfk
2022-12-16

kubeadm安装k8s(版本1.26.4)

//

系统：20.04.5 LTS (Focal Fossa)

# 基础环境初始优化

  • 1，配置时间同步（生产环境必须），添加hosts解析(必须)

  • 2，关闭swap,ufw防火墙

#关闭swap
sed -ri 's$(^/swapfile)(.*)$#\1\2$g' /etc/fstab
swapoff /swapfile

#关闭防火墙
systemctl stop ufw
systemctl disable ufw
1
2
3
4
5
6
7
  • 3，配置源
sed -i "s@http://.*archive.ubuntu.com@https://mirrors.tuna.tsinghua.edu.cn@g" /etc/apt/sources.list
sed -i "s@http://.*security.ubuntu.com@https://mirrors.tuna.tsinghua.edu.cn@g" /etc/apt/sources.list

apt update
apt upgrade #更新系统
apt install bash-completion telnet tree lrzsz net-tools vim iftop unzip wget openssh-server curl  ethtool #安装常用工具包
1
2
3
4
5
6

  • 4，安装kubeadm,kubectl

  • 指定版本1.26.4-00

#!/bin/bash
apt update && apt install apt-transport-https
curl -fsSL https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | apt-key add -
add-apt-repository "deb [arch=amd64] https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main"
apt-get update
apt-cache madison kubelet kubectl kubeadm |grep '1.26.4-00' 
apt install -y kubelet=1.26.4-00 kubectl=1.26.4-00 kubeadm=1.26.4-00
1
2
3
4
5
6
7
8
  • 默认安装最新（二选一）
#导入 gpg key
sudo mkdir -p /usr/share/keyrings
sudo curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg
#sudo curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://download.yfklife.cn/blog/cloud/k8s/ubuntu-k8s-apt-key.gpg

#新加源文件
echo 'deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://mirrors.tuna.tsinghua.edu.cn/kubernetes/apt kubernetes-xenial main
' >>/etc/apt/sources.list.d/kubernetes.list

#更新，安装
apt-get update
apt-get install  kubelet kubeadm kubectl
1
2
3
4
5
6
7
8
9
10
11
12
13
  • 修改网卡为eth0,不是必须
#修改grub引导

cp /etc/default/grub /etc/default/grub_bak
sed -i 's#GRUB_CMDLINE_LINUX=.*#GRUB_CMDLINE_LINUX="net.ifnames=0 biosdevname=0"#g' /etc/default/grub
grub-mkconfig -o /boot/grub/grub.cfg


#修改网卡设备名配置
root@master01:~# cat /etc/netplan/01-network-manager-all.yaml
# Let NetworkManager manage all devices on this system
#network:
#  version: 2
#  renderer: NetworkManager

network:
  version: 2
  renderer: networkd
  ethernets:
    eth0:
      dhcp4: no
      dhcp6: no
      addresses: [192.168.108.100/24]
      gateway4: 192.168.108.2
      nameservers:
        addresses: [114.114.114.114]

#重启服务器
reboot
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

# 转发IPv4并让iptables看到桥接流量（所有节点）

通过运行 lsmod | grep br_netfilter 来验证 br_netfilter 模块是否已加载。

若要显式加载此模块，请运行 sudo modprobe br_netfilter。

为了让 Linux 节点的 iptables 能够正确查看桥接流量，请确认 sysctl 配置中的 net.bridge.bridge-nf-call-iptables 设置为 1。例如：

cat >>/etc/modules-load.d/k8s.conf<<EOF
overlay
br_netfilter
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
EOF

for i in $(cat /etc/modules-load.d/k8s.conf);do
sudo modprobe $i
done


# https://github.com/kubernetes/kubernetes/blob/master/pkg/proxy/ipvs/README.md
lsmod | grep -e ip_vs -e nf_conntrack_ipv4

# 设置所需的 sysctl 参数，参数在重新启动后保持不变
cat >>/etc/sysctl.d/k8s.conf<<EOF
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
vm.swappiness = 0
fs.inotify.max_user_instances=81920
EOF

# 应用 sysctl 参数而不重新启动
sudo sysctl --system
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

# 容器运行时（所有节点）

  • 安装docker最新版本

清华大学开源站 (opens new window)

export DOWNLOAD_URL="https://mirrors.tuna.tsinghua.edu.cn/docker-ce"
curl -fsSL https://download.yfklife.cn/blog/cloud/k8s/get.docker.com | sh
#curl -fsSL https://get.docker.com/ | sh


#sudo apt-get install apt-transport-https ca-certificates curl gnupg2 software-properties-common
#curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
#备用：curl -fsSL https://download.yfklife.cn/blog/cloud/docker/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
#echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://mirrors.tuna.tsinghua.edu.cn/docker-ce/linux/ubuntu  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
#
#sudo apt-get update
#sudo apt-get install docker-ce containerd.io
1
2
3
4
5
6
7
8
9
10
11
12

结合 runc 使用 systemd cgroup 驱动，在 vi /etc/containerd/config.toml 中设置：

disabled_plugins = ["cri"] 从1.24开始，使用containerd 启动，使用cri 插件,默认是关闭，需要注释

  • 启用cri插件
cp /etc/containerd/config.toml{,.bak}

sed  -i 's#disabled_plugins#\#disabled_plugins#g' /etc/containerd/config.toml

cat >>/etc/containerd/config.toml<<EOF
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
    SystemdCgroup = true
EOF
sudo systemctl restart containerd
1
2
3
4
5
6
7
8
9
  • 参考配置 config.toml

ctr版本

[root@k8s-node144 pki]# ctr version
Client:
  Version:  1.6.25
  Revision: d8f198a4ed8892c764191ef7b3b06d8a2eeb5c7f
  Go version: go1.20.10

Server:
  Version:  1.6.25
  Revision: d8f198a4ed8892c764191ef7b3b06d8a2eeb5c7f
  UUID: 5f10a1c6-f227-43e4-b431-36eabeacf822
[root@k8s-node144 pki]#
[root@k8s-node144 pki]# crictl version
Version:  0.1.0
RuntimeName:  containerd
RuntimeVersion:  1.6.25
RuntimeApiVersion:  v1
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16

如果服务器手动要拉取自签证书，拷贝pem 到 /etc/ssl/certs/ 下信任，或者配置参数 --plain-http

ctr -n=k8s.io image pull --plain-http dev.idocker.io/yfk-test:v1

生成默认的配置文件: containerd config default > /etc/containerd/config.toml-default

#disabled_plugins = ["cri"]

root = "/data/containerd" #工作目录
state = "/run/containerd"
#subreaper = true
#oom_score = 0

[grpc]
  address = "/run/containerd/containerd.sock"
#  uid = 0
#  gid = 0

#[debug]
#  address = "/run/containerd/debug.sock"
#  uid = 0
#  gid = 0
#  level = "info"

# dev.idocker.io 是自签https证书，
[plugins."io.containerd.grpc.v1.cri".registry]
  [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
    [plugins."io.containerd.grpc.v1.cri".registry.mirrors."dev.idocker.io"]
      endpoint = ["https://dev.idocker.io"]
    [plugins."io.containerd.grpc.v1.cri".registry.mirrors."idocker.io"]
      endpoint = ["https://idocker.io"]
  [plugins."io.containerd.grpc.v1.cri".registry.configs]
    [plugins."io.containerd.grpc.v1.cri".registry.configs."dev.idocker.io".tls]
      insecure_skip_verify = true
    [plugins."io.containerd.grpc.v1.cri".registry.configs."idocker.io".tls]
      insecure_skip_verify = true

[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
    SystemdCgroup = true

[plugins."io.containerd.grpc.v1.cri"]
    insecure_skip_verify = true
    device_ownership_from_security_context = false
    disable_apparmor = false
    disable_cgroup = false
    disable_hugetlb_controller = true
    disable_proc_mount = false
    disable_tcp_service = true
    enable_selinux = false
    enable_tls_streaming = false
    enable_unprivileged_icmp = false
    enable_unprivileged_ports = false
    ignore_image_defined_volumes = false
    max_concurrent_downloads = 3
    max_container_log_line_size = 2000000
    netns_mounts_under_state_dir = false
    restrict_oom_score_adj = false
    sandbox_image = "registry.aliyuncs.com/google_containers/pause:3.6"
    selinux_category_range = 1024
    stats_collect_period = 10
    stream_idle_timeout = "4h0m0s"
    stream_server_address = "127.0.0.1"
    stream_server_port = "0"
    systemd_cgroup = false
    tolerate_missing_hugetlb_controller = true
    unset_seccomp_profile = ""
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60

# systemd cgroup 驱动（所有节点）

root@master01://opt# grep  cgroupDriver /var/lib/kubelet/config.yaml
cgroupDriver: systemd

kubeadm_1.jpg

# kubeadm安装k8s

# 提前拉取镜像

  • 拉取镜像

kubeadm config images pull --kubernetes-version=1.26.4 --image-repository registry.aliyuncs.com/google_containers

在国内网络，新版本很多人卡在这里，先别着急初始化，先了解新版本的不同

由于Docker Engine 没有实现（CRI）接口，所以在1.24版本开始，不在使用dockershim，但是从 docker build 生成的镜像将适用于所有 CRI 实现， 现有的所有镜像仍将完全相同。

也就说通常在v.1.23版本之前，我们离线安装k8s 都会提前把docker 镜像下载下来，但在1.24版本开始，docker命令不再适用，将使用 ctr 和 crtictl 这两个命令工具

Kubernetes在v1.24版移除了Dockershim常见问题官方文档 (opens new window)

# Container命令ctr crictl的image 区别

  • 版本：ctr containerd.io 1.4.3

containerd 相比于docker , 多了namespace概念, 每个image和container 都会在各自的namespace下可见, 目前k8s会使用k8s.io 作为命名空间

ctr和crict输出的镜像列表不一致 (opens new window)

# kubeadm init 失败（版本v1.26.4）

就算是指定了infra镜像，也始终去拉取 “registry.k8s.io/pause:3.6” 镜像，导致初始化不成功：kubeadm init --kubernetes-version=1.26.4 --image-repository registry.aliyuncs.com/google_containers

  • journalctl -xeu kubelet 报错日志

Failed to create sandbox for pod" err="rpc error: code = Unknown desc = failed to get sandbox image "registry.k8s.io/pause:3.6": failed to pull image "registry.k8s.io/pause:3.6" CreatePodSandbox for pod failed" err="rpc error: code = Unknown desc = failed to get sandbox image "registry.k8s.io/pause:3.6": failed to pull image "registry.k8s.io/pause:3.6"

# ctr 拉取镜像，重做tag

按照以往的方式，把镜像拉取到本地，但不是使用docker image了，而是使用 ctr

crictl images #查看镜像

#拉取镜像
ctr -n=k8s.io image pull registry.aliyuncs.com/google_containers/pause:3.6

#重命名tag
ctr  -n=k8s.io  images tag registry.aliyuncs.com/google_containers/pause:3.6 registry.k8s.io/pause:3.6

crictl images
1
2
3
4
5
6
7
8
9

crictl image 查看镜像，会出现一个 WARN，一个 ERRO ,可以通过下面两个配置去除

crictl config runtime-endpoint unix:///run/containerd/containerd.sock
crictl config image-endpoint unix:///run/containerd/containerd.sock
1
2

kubeadm_1.jpg

# kubeadm初始化

# 使用config初始化方式

  • 常用配置参数

kubeadm config print init-defaults > kubeadm.yaml

  • 详细配置参数

kubeadm config print init-defaults --kubeconfig ClusterConfiguration --component-configs KubeProxyConfiguration --component-configs KubeletConfiguration > kubeadm.yaml

ClusterConfiguration (opens new window)

ipvs

Github ipvs配置 (opens new window)

kubeadm_

kubelet 过期时间查看：openssl x509 -in ./ca.crt -noout -text |grep Not，1.26 版本证书过期时间为10年

#kubelet证书轮换
#https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/kubeadm-certs/

apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
serverTLSBootstrap: true
1
2
3
4
5
6
7
8
9

# init 参数方式

kubeadm init (opens new window)

 指定版本 --kubernetes-version=1.26.4 
 API Server将要广播的监听地址  --apiserver-advertise-address=10.0.0.215 
 API Server绑定的端口   --apiserver-bind-port 6443
 指定镜像仓库 --image-repository 10.0.0.200:80/google_containers 
 指定svc虚拟IP网段 --service-cidr=10.96.0.0/12 
 指定pod 网段 --pod-network-cidr=10.244.0.0/16 
 忽略未关闭swap --ignore-preflight-errors=Swap
 --v=5
1
2
3
4
5
6
7
8

建议指定pod网段,不然得修改网络POD addons配置：kubeadm init --kubernetes-version=1.26.4 --image-repository registry.aliyuncs.com/google_containers --pod-network-cidr=10.244.0.0/16 --service-cidr=10.96.0.0/12 --v=5

kubeadm_1.jpg

kubeadm_1.jpg


  mkdir -p $HOME/.kube
  test -f $HOME/.kube/config || sudo \cp  /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config
  
  export KUBECONFIG=/etc/kubernetes/admin.conf
  echo "KUBECONFIG=/etc/kubernetes/admin.conf" >> ~/.bashrc

#配置pod网络
You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/
1
2
3
4
5
6
7
8
9
10
11
12
13

# addons插件"Error registering network: failed to acquire lease"

在没有安装网络插件的时候coredns pod一直处于 ContainerCreating 状态

如果init 没有指定“--pod-network-cidr” calico会报错，Failed to create pod sandbox: rpc error: code = Unknown desc = failed to setup network for sandbox

curl https://raw.githubusercontent.com/projectcalico/calico/v3.24.5/manifests/canal.yaml -O
kubectl apply -f canal.yaml
1
2

如果已经创建了集群，查看pod子网：kubectl -n kube-system get cm kubeadm-config -o yaml

kubeadm_1.jpg

# kubeadm join过期

  • 方式一
kubeadm token create --print-join-command

#如果是添加控制节点
echo "$(kubeadm token create --print-join-command) --control-plane --certificate-key $(kubeadm init phase upload-certs --upload-certs |sed -n '$p')"
1
2
3
4
5
  • 方式二
kubeadm token create #默认token过期时间24H

kubeadm_join_token=$(kubeadm token list |grep system:bootstrappers:kubeadm:default-node-token |awk 'NR==1{print $1}')
kubeadm_join_cert_hash=$(openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //')

#把IP换成你的IP
echo "kubeadm join 192.168.108.100:6443 --token $kubeadm_join_token --discovery-token-ca-cert-hash sha256:$kubeadm_join_cert_hash"
1
2
3
4
5
6
7
8

kubeadm_1.jpg

//
如果此文章对您有帮助，点击 -->> 请博主喝咖啡 (opens new window)
上次更新: 2024/08/01, 11:00:01
k8s网络策略
kubeadm安装k8s-自签证书

← kubeadm安装k8s-自签证书→

最近更新
01
openStack配置GPU直通(zed)
 10-21
02
nginx禁用真实IP
 10-15
03
docker运行一个bind9和常用的网络命令
 09-27
更多文章>
Theme by Vdoing | Copyright © 2019-2024 yangfk | 湘ICP备2021014415号-1
×
//