Shia
Shia
HOME
BLOG
k8s部署
11月 21 2023

准备工作

[kiosk@k8s-master | k8s-worker1 | k8s-worker2]$

  1. 设置当前用户sudo免密 [选做]

    不想每次都输入密码 - 加速

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    # 当前用户的密码
    OS_PASS=ubuntu

    # 缓存 sudo 密码 ubuntu
    echo $OS_PASS | sudo -v -S 

    # 永久生效
    sudo tee /etc/sudoers.d/$USER >/dev/null <<EOF
    $USER ALL=(ALL) NOPASSWD: ALL
    EOF

    cat /etc/sudoers.d/$USER

  2. 使用国内镜像仓库 [选做]

    软件安装 - 加速

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    if ! curl --connect-timeout 2 google.com &>/dev/null; then
        # C. 国内
        MIRROR_URL=http://mirror.nju.edu.cn/ubuntu
        CODE_NAME=$(lsb_release -cs)
        COMPONENT="main restricted universe multiverse"

        # 生成软件仓库源
        sudo tee /etc/apt/sources.list >/dev/null <<EOF
    deb $MIRROR_URL $CODE_NAME $COMPONENT
    deb $MIRROR_URL $CODE_NAME-updates $COMPONENT
    deb $MIRROR_URL $CODE_NAME-backports $COMPONENT
    deb $MIRROR_URL $CODE_NAME-security $COMPONENT
    EOF
    fi

    cat /etc/apt/sources.list

  3. 安装相关软件 <必做>

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    # Which services should be restarted?
    NFILE=/etc/needrestart/needrestart.conf

    sudo sed -i \
        -e '/nrconf{restart}/{s+i+a+;s+#++}'  \
        $NFILE

    grep nrconf{restart} $NFILE

    # 手动更新
    sudo apt -y update

    ## 安装
    # -	远程, ssh 免交互, 编辑文件, storageClass
    # -	Tab 自动补全, nc, ping
    sudo apt -y install \
        openssh-server sshpass vim nfs-common \
        bash-completion netcat-openbsd iputils-ping

  4. 设置静态 IP <必做>

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    # 获取 IP
    NICP=$(ip a | awk '/inet / {print $2}' | grep -v ^127)

    if [ "$(echo $NICP | wc -w)" != "1" ]; then
        select IP1 in $NICP; do
            break
        done
    else
        IP1=$NICP
    fi

    echo -e " addresses: \e[1;34m${IP1}\e[0;0m"

    # 获取网卡名
    NICN=$(ip a | awk '/^2:/ {print $2}' | sed 's/://')

    echo -e " ethernets: \e[1;34m${NICN}\e[0;0m"

    # 获取网关
    NICG=$(ip route | awk '/^default/ {print $3}')

    echo -e " routes: \e[1;34m${NICG}\e[0;0m"

    # 获取 DNS
    unset DNS; unset DN
    for i in 114.114.114.114 8.8.4.4 8.8.8.8; do
        if nc -w 2 -zn $i 53 &>/dev/null; then
            DNS1=$i
            DNS="$DNS, $DNS1"
        fi
    done

    echo -e " nameservers: \e[1;34m${DNS#, }\e[0;0m"

    # 更新网卡配置文件
    NYML=/etc/netplan/00-installer-config.yaml

    sudo tee $NYML >/dev/null <<EOF
    network:
      ethernets:
        $NICN:
          dhcp4: false
          addresses: [$IP1]
          routes: 
          - to: default
            via: $NICG
          nameservers:
            addresses: [${DNS#, }]
      version: 2
    EOF

    cat $NYML

    # dns 解析文件链接
    sudo ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf

    # active
    sudo netplan apply

  5. 编辑 hosts <必做>

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    # 显示 ip 地址和主机名，方便复制
    echo $(hostname -I) $(hostname)
    sudo tee -a /etc/hosts >/dev/null <<EOF
    # K8s-cluster
    192.168.147.128 k8s-master
    192.168.147.129 k8s-worker1
    192.168.147.130 k8s-worker2
    EOF

    cat /etc/hosts

  6. 设置 root 密码 [选做]

    1
    2
    3
    4
    5
    6
    7
    8
    9
    ROOT_PASS=ubuntu

    (echo $ROOT_PASS; echo $ROOT_PASS) \
        | sudo passwd root

    echo PermitRootLogin yes \
        | sudo tee -a /etc/ssh/sshd_config

    sudo systemctl restart sshd

  7. ssh 免密 [选做]

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    # 生成 keypair
    ssh-keygen -f ~/.ssh/id_rsa -N ''

    # 拷贝公钥
    ROOT_PASS=ubuntu

    NODES=$(egrep -v '^[a-f]|^:|^#|^$|^127' /etc/hosts | awk '{print $2}')

    echo -e "NODES: \e[1;34m$(echo $NODES)\e[0;0m"

    for i in $NODES; do
        for j in kiosk root; do
            sshpass -p$ROOT_PASS \
                ssh-copy-id -o StrictHostKeyChecking=no \
                    $j@$i
        done
    done

  8. 禁用 swap <必做>

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    # 交换文件
    SWAPF=$(awk '/swap/ {print $1}' /etc/fstab)

    # 立即禁用
    sudo swapoff $SWAPF

    # 永久禁用
    sudo sed -i '/swap/d' /etc/fstab

    # 删除交换文件
    sudo rm $SWAPF

    # 确认
    free -h

  9. 扩容 <必做>

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    df -h / | egrep [0-9]+G

    # 逻辑卷名
    LVN=$(sudo lvdisplay | awk '/Path/ {print $3}')

    echo -e " LV Name: \e[1;34m${LVN}\e[0;0m"

    # 扩容
    sudo lvextend -r -l 100%PVS $LVN \
      > /dev/null

    df -h / | egrep [0-9]+G

  10. 模块支持 <必做>

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    ## 安装
    sudo apt -y install bridge-utils

    # 永久生效
    MFILE=/etc/modules-load.d/br.conf

    sudo tee $MFILE >/dev/null <<EOF
    br_netfilter
    EOF

    cat $MFILE

    # 立即生效
    sudo modprobe br_netfilter

    lsmod | grep br

    ## 内核支持
    sudo tee /etc/sysctl.d/k8s.conf >/dev/null <<EOF
    net.ipv4.ip_forward=1
    vm.swappiness=0
    vm.overcommit_memory=1
    vm.panic_on_oom=0
    EOF

    # 立即生效
    sudo sysctl -p /etc/sysctl.d/k8s.conf

    sudo sysctl -a | grep 'ip_forward '

  11. 安装运行时 <必做> 

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64
    65
    66
    67
    68
    69
    70
    71
    72
    73
    74
    75
    76
    77
    78
    79
    80
    81
    82
    83
    84
    85
    86
    87
    88
    ## 创建镜像仓库文件
    AFILE=/etc/apt/sources.list.d/docker.list

    if ! curl --connect-timeout 2 google.com &>/dev/null; then
        # C. 国内
        AURL=https://mirror.nju.edu.cn/docker-ce
    else
        # A. 国外
        AURL=https://download.docker.com
    fi

    sudo tee $AFILE >/dev/null <<EOF
    deb $AURL/linux/ubuntu $(lsb_release -cs) stable
    EOF

    cat $AFILE

    # 导入公钥
    curl -fsSL https://download.docker.com/linux/ubuntu/gpg \
        | sudo apt-key add -

    # W: Key is stored in legacy trusted.gpg keyring
    sudo cp /etc/apt/trusted.gpg /etc/apt/trusted.gpg.d

    # 更新索引
    sudo apt-get update

    # 安装 containerd
    sudo apt-get install containerd.io

    # 生成默认配置文件
    containerd config default \
        | sudo tee /etc/containerd/config.toml >/dev/null

    ## 修改配置文件
    #	"alpine"
    #	376ms  https://docker.nju.edu.cn
    #	623ms  http://hub-mirror.c.163.com 
    #	10.97s https://docker.mirrors.ustc.edu.cn
    sudo sed -i \
        -e '/SystemdCgroup/s+false+true+' \
        /etc/containerd/config.toml

    if ! curl --connect-timeout 2 google.com &>/dev/null; then
        # C. 国内
        REISTRY_OLD=registry.k8s.io
        REGISTRY_NEW=registry.aliyuncs.com/google_containers
        M1='[plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]'
        M2='endpoint = ["https://docker.nju.edu.cn"]'
      
        sudo sed -i \
            -e "/sandbox_image/s+$REISTRY_OLD+$REGISTRY_NEW+" \
            -e "/registry.mirrors/a\        $M1" \
            -e "/registry.mirrors/a\          $M2" \
            /etc/containerd/config.toml 
    fi

    # 服务重启
    sudo systemctl restart containerd
    ## 下载 crictl 压缩包
    if ! curl --connect-timeout 2 google.com &>/dev/null; then
        # C. 国内
        CURL=http://k8s.ruitong.cn:8080/K8s
    else
        # A. 国外
        CURL=https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.28.0
    fi

    TFILE=crictl-v1.28.0-linux-amd64.tar.gz

    curl -# -o $TFILE $CURL/$TFILE

    # 解压 crictl
    tar -xf $TFILE

    # 安装 crictl 命令
    sudo cp crictl /usr/bin/

    # 创建 crictl 配置文件
    sudo tee /etc/crictl.yaml <<EOF >/dev/null
    runtime-endpoint: unix:///run/containerd/containerd.sock
    image-endpoint: unix:///run/containerd/containerd.sock
    timeout: 10
    debug: false
    pull-image-on-create: true
    EOF

    sudo usermod -aG root $USER

安装 K8s

[kiosk@k8s-master | k8s-worker1 | k8s-worker2]$

  1. 安装 kubeadm、kubelet 和 kubectl
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    # 更新 apt 包索引并安装使用 Kubernetes apt 仓库所需要的包
    sudo apt -y install apt-transport-https ca-certificates curl

    ## 添加 Kubernetes apt 仓库
    sudo mkdir /etc/apt/keyrings &>/dev/null

    KFILE=/etc/apt/keyrings/kubernetes-archive-keyring.gpg

    if ! curl --connect-timeout 2 google.com &>/dev/null; then
        # C. 国内
        KURL=http://k8s.ruitong.cn:8080/K8s
        AURL=https://mirror.nju.edu.cn/kubernetes/apt
    else
        # A. 国外
        KURL=https://packages.cloud.google.com
        AURL=https://apt.kubernetes.io/
    fi

    sudo curl -fsSLo $KFILE $KURL/apt/doc/apt-key.gpg

    sudo tee /etc/apt/sources.list.d/kubernetes.list <<EOF >/dev/null
    deb [signed-by=$KFILE] $AURL kubernetes-xenial main
    EOF

    sudo apt -y update
  • CKA，CKS v1.27.1
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    # 官方考试版本-CKA
    CKx_URL=https://training.linuxfoundation.cn/certificates/1

    :<<EOF
    # 官方考试版本-CKS
    CKx_URL=https://training.linuxfoundation.cn/certificates/16
    EOF

    KV=$(curl -s $CKx_URL | grep -Eo 软件版本.*v[0-9].[0-9]+ | awk '{print $NF}')

    echo -e " The exam is based on Kubernetes: \e[1;34m${KV#v}\e[0;0m"

    # 列出所有小版本
    sudo apt-cache madison kubelet | grep ${KV#v}

    # 安装 kubelet、kubeadm 和 kubectl 考试版本
    sudo apt -y install \
        kubelet=${KV#v}.1-00 \
        kubeadm=${KV#v}.1-00 \
        kubectl=${KV#v}.1-00
  • CKAD v1.26.1
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    # 官方考试版本 - CKAD
    CKx_URL=https://training.linuxfoundation.cn/certificates/4

    KV=$(curl -s $CKx_URL | grep -Eo 软件版本.*v[0-9].[0-9]+ | awk '{print $NF}')

    echo -e " The exam is based on Kubernetes: \e[1;34m${KV#v}\e[0;0m"

    # 列出所有小版本
    sudo apt-cache madison kubelet | grep ${KV#v}

    # 安装 kubelet、kubeadm 和 kubectl 考试版本
    sudo apt -y install \
        kubelet=${KV#v}.1-00 \
        kubeadm=${KV#v}.1-00 \
        kubectl=${KV#v}.1-00
    # 锁定版本
    sudo apt-mark hold kubelet kubeadm kubectl

[kiosk@k8s-master]$
13. 初始化

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
# 生成初始文件
sudo kubeadm config print init-defaults \
    > kubeadm-config.yaml

# IP 地址
NICP=$(ip a | awk '/inet / {print $2}' | grep -v ^127 | sed 's+/.*++')

if [ "$(echo $NICP | wc -w)" != "1" ]; then
    select IP1 in $NICP; do
        break
    done
else
    IP1=$NICP
fi

echo -e " advertiseAddress: \e[1;34m${IP1}\e[0;0m"
# kubeadm-config.yaml
sudo sed -i \
    -e "/advertiseAddress/s+:.*+: $IP1+" \
    -e "/name/s+:.*+: $(hostname -s)+" \
    -e "/clusterName/s+:.*+: ck8s+" \
    kubeadm-config.yaml

# 手动拉取镜像 [必选]
# kubeadm init 时，会自动拉取
if ! curl --connect-timeout 2 google.com &>/dev/null; then
    # C. 国内
    RURL=registry.aliyuncs.com/google_containers
    
    sudo kubeadm config images pull \
        --image-repository $RURL \
        --kubernetes-version ${KV#v}.1
    
    sudo sed -i \
        -e "/imageRepository/s+:.*+: $RURL+" \
        kubeadm-config.yaml
fi

# 验证+k8s
sudo crictl images
# 验证-k8s
sudo ctr -n k8s.io image ls
# 使用初始文件，初始化集群
sudo kubeadm init \
    --config kubeadm-config.yaml
  1. 配置文件 - Client

    参考 init 的输出提示

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    # 创建目录
    mkdir -p ~/.kube

    # user 复制配置文件
    sudo \cp /etc/kubernetes/admin.conf \
        ~/.kube/config

    sudo chown $(id -u):$(id -g) ~/.kube/config

    ll ~/.kube/config | grep $USER

    # root 变量
    sudo tee -a ~root/.bashrc >/dev/null <<EOF
    export KUBECONFIG=/etc/kubernetes/admin.conf
    EOF

    sudo grep KUBECONFIG ~root/.bashrc
  2. 创建网络

参考 init 的输出提示
https://www.tigera.io/project-calico/ Home -> Calico Open Source Install -> CalicoKubernetesSelf-managed on-premises Install Calico -> Manifest -> Install Calico with Kubernetes API datastore, 50 nodes or less

1
curl https://raw.githubusercontent.com/projectcalico/calico/v3.26.1/manifests/calico.yaml -O
1
2
3
4
5
6
7
8
9
10
11
CFILE=calico/v3.26.1/manifests/calico.yaml

if ! curl --connect-timeout 2 google.com &>/dev/null; then
    # C. 国内
    CURL=http://k8s.ruitong.cn:8080/K8s
else
    # A. 国外
    CURL=https://raw.githubusercontent.com/projectcalico
fi

kubectl apply -f $CURL/$CFILE
  1. 命令补全 - Client [建议]
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    # 查看帮助
    $ kubectl completion --help
    ## 立即生效
    source <(kubectl completion bash)
        
    ## 永久生效
    mkdir ~/.kube 2>/dev/null

    kubectl completion bash \
        > ~/.kube/completion.bash.inc

    tee -a ~/.bashrc <<EOF >/dev/null
    # Kubectl shell completdion
    source ~/.kube/completion.bash.inc
    EOF

    source ~/.bashrc

    # kubectl <Tab><Tab>
  2. 命令别名 - Client[建议]

    参考网址 https://kubernetes.io/zh-cn/docs/reference/kubectl/cheatsheet/

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    # 永久生效
    tee -a ~/.bashrc <<EOF >/dev/null
    alias k='kubectl'
    complete -F __start_kubectl k
    EOF

    # 立即生效
    source ~/.bashrc

    # k <Tab><Tab>
    [kiosk@k8s-worker1 | k8s-worker2]$
  3. 加入集群
    • 参考 init 的输出提示
    • 找不到提示，参照附录 A1
    1
    2
    3
    sudo \
        kubeadm join 192.168.147.128:6443 --token abcdef.0123456789abcdef \
    	--discovery-token-ca-cert-hash sha256:49ff7a97c017153baee67c8f44fa84155b8cf06ca8cee067f766ec252cb8d1ac

确认环境正常

[kiosk@k8s-master]

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
$ kubectl get nodes
NAME          STATUS   ROLES           AGE     VERSION
k8s-master    Ready    control-plane   3m42s   v1.27.1
k8s-worker1   Ready    <none>          60s     v1.27.1
k8s-worker2   Ready    <none>          54s     v1.27.1

$ kubectl get pod -A
NAMESPACE     NAME                                       READY   STATUS    RESTARTS   AGE
kube-system   calico-kube-controllers-786b679988-jqmtn   1/1     Running   0          2m58s
kube-system   calico-node-4sbwf                          1/1     Running   0          71s
kube-system   calico-node-cfq42                          1/1     Running   0          65s
kube-system   calico-node-xgmqv                          1/1     Running   0          2m58s
kube-system   coredns-7bdc4cb885-grj7m                   1/1     Running   0          3m46s
kube-system   coredns-7bdc4cb885-hjdkp                   1/1     Running   0          3m46s
kube-system   etcd-k8s-master                            1/1     Running   0          3m50s
kube-system   kube-apiserver-k8s-master                  1/1     Running   0          3m50s
kube-system   kube-controller-manager-k8s-master         1/1     Running   0          3m50s
kube-system   kube-proxy-2cgvn                           1/1     Running   0          3m46s
kube-system   kube-proxy-2zh5k                           1/1     Running   0          65s
kube-system   kube-proxy-nd5z8                           1/1     Running   0          71s
kube-system   kube-scheduler-k8s-master                  1/1     Running   0          3m50s

$ kubectl get componentstatuses
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME                 STATUS    MESSAGE                         ERROR
etcd-0               Healthy   {"health":"true","reason":""}
controller-manager   Healthy   ok
scheduler            Healthy   ok

A. 附录

A1. 新节点加入 K8s 集群

  • 查看 kubeadm init 命令的输出
  • 使用 kubeadm token create 命令重新创建
    [kiosk@k8s-master]$
1
2
kubeadm token create --print-join-command

A2. kubeadm reset

还原通过==kubeadm init==或==kubeadm join==对此主机所做的更改

[kiosk@k8s-master]$

1
2
echo y | sudo kubeadm reset

A3. 国内需要加速

ID 国内 国外 说明
1 http://mirror.nju.edu.cn/ubuntu 默认 系统软件仓库
2 https://mirror.nju.edu.cn/docker-ce/linux/ubuntu https://download.docker.com/linux/ubuntu 运行时仓库
3 registry.aliyuncs.com/google_containers registry.k8s.io google容器镜像仓库
4 https://docker.nju.edu.cn/ 默认 docker容器镜像仓库
5 http://k8s.ruitong.cn:8080/K8s https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.24.2 crictl 命令
6 https://mirror.nju.edu.cn/kubernetes/apt/ https://apt.kubernetes.io/ kubeadm, kubelet 软件仓库
7 http://k8s.ruitong.cn:8080/K8s/calico/v3.25.0/manifests https://raw.githubusercontent.com/projectcalico/calico/v3.25.0/manifests calico.yaml 文件

A4 . Err1

The connection to the server localhost:8080 was refused - did you specify the right host or port?

[kiosk@k8s-worker2]$

1
2
3
4
5
6
mkdir ~/.kube

scp root@k8s-master:/etc/kubernetes/admin.conf \
    ~/.kube/config

kubectl get node

A5. RHEL布署

  • 关闭 firewalld
  • 关闭 SELinux
1
2
3
4
5
6
7
8
# firewalld
systemctl disable --now firewalld

# SELinux
sed -i '/^SELINUX=/s/=.*/=disabled/' \
    /etc/selinux/config

reboot
  1. 准备工作
  2. 安装 K8s
  3. 确认环境正常
  4. A. 附录

Power by Shia