LB节点安装
部署API高可用集群 nginx及keepalived组件
安装nginx、keepalived
yum -y install gcc gcc-c++ make automake autoconf libtool pcre pcre-devel zlib zlib-devel openssl openssl-devel
yum -y install keepalived
useradd nginx -s /sbin/nologin
tar xvf nginx-1.14.0.tar.gz
cd nginx-1.14.0
./configure --prefix=/usr/local/nginx \
--user=nginx --group=nginx \
--with-http_ssl_module \
--with-http_realip_module \
--with-http_addition_module \
--with-http_sub_module \
--with-http_gunzip_module \
--with-http_gzip_static_module \
--with-http_stub_status_module \
--with-stream --with-stream_ssl_module
make
make install
配置nginx
##生成nginx主配置文件
cat > /data/k8s/script/config/nginx.conf << EOF
user nginx;
worker_processes auto;
pid logs/nginx.pid;
events {
worker_connections 10240;
}
stream {
upstream apiservers_https {
hash $remote_addr consistent;
server 192.168.255.190:6443;
server 192.168.255.191:6443;
server 192.168.255.192:6443;
}
upstream apiservers_http {
hash $remote_addr consistent;
server 192.168.255.190:8080;
server 192.168.255.191:8080;
server 192.168.255.192:8080;
}
server {
listen 6443;
proxy_pass apiservers_https;
}
server {
listen 8080;
proxy_pass apiservers_http;
}
}
EOF
配置keepalived
## 创建keepalived.conf配置文件
cat > /data/k8s/script/config/keepalived.conf << EOF
! Configuration File for keepalived
global_defs {
router_id LVS_DEVEL #不同keepalived要不同
}
# 设置检查nginx存活脚本
vrrp_script chk_nginx {
script "/data/k8s/script/nginx_check.sh"
interval 2
weight -20
}
vrrp_instance VI_1 {
state MASTER #master节点，备节点为BACKUP
interface ens33 #网卡接口名称
virtual_router_id 51 #设置VRID，相同的VRID为一个组，他将决定多播的MAC地址
mcast_src_ip 192.168.255.199 #发送多播数据包时的源IP地址，相当于heartbeat的心跳端口
#nopreempt #是否抢占资源，只需在备节点配置
priority 100 #设置本节点的优先级，优先级高的为master
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
track_script {
chk_nginx
}
#设置VIP
virtual_ipaddress {
192.168.255.197
}
}
EOF
创建check nginx脚本
cat > /data/k8s/script/nginx_check.sh << EOF
#!/bin/bash
source /data/k8s/script/config/env
cd $workdir
A=`ps -C nginx --no-header |wc -l`
if [ $A -eq 0 ];then
#/usr/local/nginx/sbin/nginx
sh nginx_ctl start
sleep 2
if [ `ps -C nginx --no-header |wc -l` -eq 0 ];then
killall keepalived
fi
fi
EOF
创建 nginx启动脚本
cat > /data/k8s/script/nginx_ctl << EOF
#!/bin/bash
source /data/k8s/script/config/env
nginxdir="/usr/local/nginx"
name="nginx"
pidfile="$nginxdir/logs/$name.pid"
conf_file="$workdir/config/nginx.conf"
test -d $workdir/log/$name || mkdir -p $workdir/log/$name
display_help(){
echo "Usage: `basename $0` (start|stop)"
exit 0
}
if [ $# -ne 1 ];then
display_help
fi
source $workdir/pid_utils/pid_util.sh
case $1 in
start)
exec $nginxdir/sbin/$name -c $conf_file \
1>>$workdir/log/$name/$name.log 2>&1 &
for try in $(seq 0 9);do
sleep $try
echo "wait $name pid (try: $try)"
pid=$(lsof -t $nginxdir/sbin/$name)
if [ -n "$pid" ]; then
echo "$name pid: $pid is running..."
break;
fi
done
;;
stop)
kill_and_wait $pidfile
;;
*)
display_help
;;
esac
EOF
创建keepalived启动脚本
cat > /data/k8s/script/keepalived_ctl << EOF
#!/bin/bash
source /data/k8s/script/config/env
name="keepalived"
#PIDFile=/var/run/keepalived.pid
pidfile="$workdir/run/$name.pid"
test -d $workdir/run || mkdir -p $workdir/run
test -d $workdir/log/$name || mkdir -p $workdir/log/$name
display_help(){
echo "Usage: `basename $0` (start|stop)"
exit 0
}
if [ $# -ne 1 ];then
display_help
fi
KEEPALIVED_OPTIONS="-l --use-file=$workdir/config/keepalived.conf --pid=$pidfile"
source $workdir/pid_utils/pid_util.sh
case $1 in
start)
exec /usr/sbin/keepalived $KEEPALIVED_OPTIONS \
1>>$workdir/log/$name/$name.log 2>&1 &
for try in $(seq 0 9);do
sleep $try
echo "wait $name pid (try: $try)"
pid=$(lsof -t /usr/sbin/keepalived|head -1)
if [ -n "$pid" ]; then
#echo "$pid" > $pidfile
run_pid=`cat $pidfile`
if [ "$pid" == "$run_pid" ];then
echo "$name pid: $pid is running..."
fi
break;
fi
done
;;
stop)
kill_and_wait $pidfile
;;
*)
display_help
;;
esac
EOF
启动keepalived及nginx
chmod +x /data/k8s/script/keepalived_ctl
chmod +x /data/k8s/script/nginx*
sh keepalived_ctl start
在LB 备节点上面操作参考上面步骤，只是要注意keepalived.conf配置文件里的部份内容
注意： 启动 keepalived后会自动拉起nginx服务
下载及部署kube-controller-manager,kube-scheduler组件
复制执行文件到 /usr/loca/bin目录
cp -v kube-controller-manager /usr/local/bin/
cp -v kube-scheduler /usr/local/bin/
chmod +x /usr/local/bin/kube-*
创建kube-controller-manager_ctl启动脚本
cat > /data/k8s/script/kube-controller-manager_ctl << EOF
#!/bin/bash
source /data/k8s/script/config/env
source $workdir/config/kube-config
name="kube-controller-manager"
pidfile="$workdir/run/$name.pid"
test -d $workdir/run || mkdir -p $workdir/run
test -d $workdir/log/$name || mkdir -p $workdir/log/$name
display_help(){
echo "Usage: `basename $0` (start|stop)"
exit 0
}
if [ $# -ne 1 ];then
display_help
fi
KUBE_CONTROLLER_MANAGER_ARGS="--address=127.0.0.1 \
--service-cluster-ip-range=10.254.0.0/16 \
--cluster-name=kubernetes \
--cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem \
--cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \
--service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem \
--root-ca-file=/etc/kubernetes/ssl/ca.pem \
--leader-elect=true"
source $workdir/pid_utils/pid_util.sh
case $1 in
start)
exec $bin_dir/$name \
$KUBE_LOGTOSTDERR \
$KUBE_LOG_LEVEL \
$KUBE_MASTER \
$KUBE_CONTROLLER_MANAGER_ARGS \
1>>$workdir/log/$name/$name.log 2>&1 &
for try in $(seq 0 9);do
sleep $try
echo "wait $name pid (try: $try)"
pid=$(lsof -t $bin_dir/$name)
if [ -n "$pid" ]; then
echo "$pid" > $pidfile
echo "$name pid: $pid is running..."
break;
fi
done
;;
stop)
kill_and_wait $pidfile
;;
*)
display_help
;;
esac
EOF
创建kube-scheduler_ctl启动脚本
cat > /data/k8s/script/kube-scheduler_ctl << EOF
#!/bin/bash
name="kube-scheduler"
source /data/k8s/script/config/env
source $workdir/config/kube-config
pidfile="$workdir/run/$name.pid"
test -d $workdir/run || mkdir -p $workdir/run
test -d $workdir/log/$name || mkdir -p $workdir/log/$name
display_help(){
echo "Usage: `basename $0` (start|stop)"
exit 0
}
if [ $# -ne 1 ];then
display_help
fi
KUBE_SCHEDULER_ARGS="--leader-elect=true --address=127.0.0.1"
source $workdir/pid_utils/pid_util.sh
case $1 in
start)
exec $bin_dir/$name \
$KUBE_LOGTOSTDERR \
$KUBE_LOG_LEVEL \
$KUBE_MASTER \
$KUBE_SCHEDULER_ARGS \
1>>$workdir/log/$name/$name.log 2>&1 &
for try in $(seq 0 9);do
sleep $try
echo "wait $name pid (try: $try)"
pid=$(lsof -t $bin_dir/$name)
if [ -n "$pid" ]; then
echo "$pid" > $pidfile
echo "$name pid: $pid is running..."
break;
fi
done
;;
stop)
kill_and_wait $pidfile
;;
*)
display_help
;;
esac
EOF
启动kube-scheduler及kube-controller-manager组件
chmod +x /data/k8s/script/kube-*
sh kube-controller-manager_ctl start
sh kube-scheduler_ctl start
验证安装
# kubectl get componentstatuses
NAME STATUS MESSAGE ERROR
scheduler Unhealthy Get http://127.0.0.1:10251/healthz: dial tcp 127.0.0.1:10251: getsockopt: connection refused
controller-manager Unhealthy Get http://127.0.0.1:10252/healthz: dial tcp 127.0.0.1:10252: getsockopt: connection refused
etcd-1 Healthy {"health": "true"}
etcd-0 Healthy {"health": "true"}
etcd-2 Healthy {"health": "true"}