小趙同學
0%

自建k8s的ingress组件pod更新后自动同步4层Nginx

发表于 更新于

自建k8s的ingress组件pod更新后4层解析的更新问题

背景

自建k8s的ingress组件使用k8s社区的ingress-nginx组件，使用nodeport的形式部署service，使用nginx 4层代理到ingress组件。

方案

shell脚本

  1. 安装kubectl
  2. 新增同步脚本
  3. 配置定时任务
  4. 观察效果
安装kubectl
配置yum源
1
2
3
4
5
6
7
8
9
10
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
       http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
安装
1
yum install kubectl -y
配置kubeconfig
1
2
mkdir -p ~/.kube
# 将kubeconfig 存入~/.kube/config
同步脚本

/data/scripts/sync_k8s_ingress_node_upstream.sh

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
#!/bin/bash
# 动态生成nginx配置文件
# 后续需调整为etcd confd nginx-template生成
# 赵宏业
# 20230222

btime=$(date "+%Y%m%d_%H%M%S")
log_file="/data/logs/sync_k8s_ingress_node_upstream.log"

make_config() {
    http_port=$(kubectl -n ingress-nginx describe svc ingress-nginx-controller | grep NodePort | grep http | grep -v https | awk '{print $3}' | awk -F '/' '{print $1}')
    https_port=$(kubectl -n ingress-nginx describe svc ingress-nginx-controller | grep NodePort | grep https | awk '{print $3}' | awk -F '/' '{print $1}')

    \rm -f /tmp/k8s-ingress.conf

    echo "upstream k8s_ingress_80 {" >>/tmp/k8s-ingress.conf
    node_list=$(kubectl -n ingress-nginx get po -o wide | grep ingress-nginx-controller | awk '{print $7}')
    for node in ${node_list}; do
        node_ip=$(kubectl describe no ${node} | grep InternalIP | awk -F ':' '{print $2}')
        echo "    server ${node_ip}:${http_port};" >>/tmp/k8s-ingress.conf
    done
    echo "}" >>/tmp/k8s-ingress.conf

    echo "upstream k8s_ingress_443 {" >>/tmp/k8s-ingress.conf
    node_list=$(kubectl -n ingress-nginx get po -o wide | grep ingress-nginx-controller | awk '{print $7}')
    for node in ${node_list}; do
        node_ip=$(kubectl describe no ${node} | grep InternalIP | awk -F ':' '{print $2}')
        echo "    server ${node_ip}:${https_port};" >>/tmp/k8s-ingress.conf
    done
    echo "}" >>/tmp/k8s-ingress.conf

    echo "

server {
    listen 80;
    proxy_pass k8s_ingress_80;
}

server {
    listen 443;
    proxy_pass k8s_ingress_443;
}
    " >>/tmp/k8s-ingress.conf
    echo "检测时间: ${btime}" >>${log_file}
    echo "-- file begin --" >>${log_file}
    cat /tmp/k8s-ingress.conf >>${log_file}
    echo "-- file end --" >>${log_file}
}

update_config() {
    config_name="/data/servers/nginx/conf/stream/k8s-ingress.conf"
    if [ -f ${config_name} ]; then
        diff /tmp/k8s-ingress.conf ${config_name}
        if [ $? == 0 ]; then
            echo "文件内容一致, 忽略" >>${log_file}
        else
            echo "替换文件" >>${log_file}
            mkdir -p /data/backup/nginx/
            mv ${config_name} /data/backup/nginx/k8s-ingress.conf_bak_${btime}
            nginx_reload
        fi
    else
        nginx_reload
    fi
}

nginx_reload() {
    cp /tmp/k8s-ingress.conf ${config_name}
    /usr/local/sbin/nginx -t

    if [ $? == 0 ]; then
        /usr/local/sbin/nginx -s reload
        echo "reload ok" >>${log_file}
    else
        echo "文件异常, 还原配置" >>${log_file}
        echo "异常回滚" >>${log_file}
        \rm -f ${config_name}
        cp /data/backup/nginx/k8s-ingress.conf_bak_${btime} ${config_name}
        /usr/local/sbin/nginx -t
    fi
}

echo "---- begin scripts ----" >>${log_file}
make_config
update_config
echo "---- end scripts ----" >>${log_file}

配置脚本权限
1
chmod +x sync_k8s_ingress_node_upstream.sh
新增定时任务
1
*/1 * * * * /data/scripts/sync_k8s_ingress_node_upstream.sh
观察日志
1
tail -f /data/logs/sync_k8s_ingress_node_upstream.log

etcd+confd

待研究