lvs+keepalive配置Jenkins2高可用

yum -y install ipvsadm

systemctl start ipvsadm

/bin/bash: /etc/sysconfig/ipvsadm: No such file or directory

ipvsadm --save > /etc/sysconfig/ipvsadm

~]#systemctl status ipvsadm
● ipvsadm.service - Initialise the Linux Virtual Server
    Loaded: loaded (/usr/lib/systemd/system/ipvsadm.service; disabled; vendor preset: disabled)
    Active: active (exited) since 日 2020-02-09 12:39:02 CST; 4min 49s ago
    Process: 46374 ExecStart=/bin/bash -c exec /sbin/ipvsadm-restore < /etc/sysconfig/ipvsadm (code=exited, status=0/SUCCESS)
    Main PID: 46374 (code=exited, status=0/SUCCESS)
# 如上说明服务正常启动

#安装依赖应用（没有使用源码包方式安装，因为配置超麻烦）
[root@localhost ~]# yum install -y curl gcc openssl-devel libnl3-devel net-snmp-devel
[root@localhost ~]# yum install -y curl gcc openssl-devel libnl3-devel net-snmp-devel
[root@localhost ~]# yum install -y keepalived

#yum安装的 keepalived配置文件在 /etc/keepalived 目录中

[root@localhost ~]# systemctl start keepalived   //启动keepalived
[root@localhost ~]# systemctl restart keepalived  //重新启动keepalived
[root@localhost ~]# systemctl status keepalived   //查看keepalived状态

#如果主备服务器间网络不通，则需要关闭防火墙
systemctl stop firewalld
systemctl disable firewalld
setenforce 0

#启动keepalived以后，查看是否出现虚拟ip，主备上都要执行查看，正常应该只有主出现虚拟ip，备不出现，如果主备都出现了，说明脑裂了。
1、主备之间是否通信正常，若不正常是否有防火墙阻挡
2、主备对应的keepalived.conf配置是否有误，virtual_router_id配置不一致
3、硬件故障：心跳线坏了、网卡坏了、IP配置冲突等

! Configuration File for keepalived

global_defs {
    #全局routerid，可以设置为ip
    router_id 10.155.10.171
    vrrp_skip_check_adv_addr
    vrrp_garp_interval 0
    vrrp_gna_interval 0
    #如果需要调用脚本，需要配置执行脚本的用户
    script_user root
    #配置允许执行脚本
    enable_script_security
}

#虚拟实例
vrrp_instance JENKINS2_TRUNK {
    #主机设置为master
    state MASTER
    #选取ifconfig -a 后除了docker和lo外的第一个物理网卡配置
    interface em1
    # vrrp_instance的id，主备必须配置相同
    virtual_router_id 188
    priority 100
    advert_int 1
    authentication {
        # PASS 为密码类型
        auth_type PASS
        # 密码 主备keepalive配置要一致
        auth_pass StupidMan
    }
    virtual_ipaddress {
        # 虚拟ip地址
        10.155.10.167
    }
}
#虚拟服务， ip设置与  vrrp_instance的 virtual_ipaddress 一致，端口为应用服务的端口，如trunk jenkins服务端口8093
virtual_server 10.155.10.167 8093 {
    delay_loop 6
    lb_algo rr 
    lb_kind NAT
    persistence_timeout 50
    protocol TCP

    #真实服务器，有几台机器就配置几个，主备都要在这里包括，ip为真实服务的物理ip和应用的端口，本trunk配置的端口为8093
    real_server 10.155.10.171 8093 {
        weight 3
        #服务停掉时调用的脚本
        notify_down "/tol/data/jenkins-scripts/jenkins2-trunk/script_notify.sh down"
        #服务启动时调用的脚本
        notify_up "/tol/data/jenkins-scripts/jenkins2-trunk/script_notify.sh up"
        #对配置的此真实器的对应服务进行健康检查
        HTTP_GET {
            url {
                path /login
                status_code 200
            }
            connect_timeout 3
            //
            nb_get_retry 3
            delay_before_retry 3
        }
    }
    real_server 10.155.10.174 8093 {
        weight 3
        HTTP_GET {
            url {
                path /login
                status_code 200
            }
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }

}

    ! Configuration File for keepalived
    
    global_defs {
       router_id 10.155.10.172
       vrrp_skip_check_adv_addr
       vrrp_garp_interval 0
       vrrp_gna_interval 0
       script_user root
       enable_script_security
    }
    
    vrrp_instance JENKINS2_NEIBU {
        state MASTER
        interface em1
        virtual_router_id 185
        priority 100
        advert_int 1
        authentication {
            auth_type PASS
            auth_pass StupidMan
        }
        virtual_ipaddress {
            10.155.10.185
        }
    }
    
    virtual_server 10.155.10.185 8091 {
        delay_loop 6
        lb_algo rr 
        lb_kind NAT
        persistence_timeout 50
        protocol TCP
    
        #sorry_server 192.168.200.200 1358
    
        real_server 10.155.10.172 8091 {
    	weight 3
    	notify_down "/tol/data/jenkins-scripts/jenkins2-neibu/script_notify.sh down"
    	notify_up "/tol/data/jenkins-scripts/jenkins2-neibu/script_notify.sh up"
            HTTP_GET {
                url { 
                  path /login
                  status_code 200
                }
                connect_timeout 3
                nb_get_retry 3
                delay_before_retry 3
            }
        }
    
        real_server 10.155.10.174 8091 {
            weight 1
            HTTP_GET {
                url {
                  path /login
                  status_code 200
                }
                connect_timeout 3
                nb_get_retry 3
                delay_before_retry 3
            }
        } 
    }

    ! Configuration File for keepalived
    
    global_defs {
       router_id 10.155.10.173
       vrrp_skip_check_adv_addr
       vrrp_garp_interval 0
       vrrp_gna_interval 0
       script_user root
       enable_script_security
    }
    
    vrrp_instance JENKINS2_RELEASE {
        state MASTER
        interface em1
        virtual_router_id 186
        priority 100
        advert_int 1
        authentication {
            auth_type PASS
            auth_pass StupidMan
        }
        virtual_ipaddress {
            10.155.10.186
        }
    }
    
    virtual_server 10.155.10.186 8092 {
        delay_loop 6
        lb_algo rr 
        lb_kind NAT
        persistence_timeout 50
        protocol TCP
    
        real_server 10.155.10.173 8092 {
    	weight 3
    	notify_down "/tol/data/jenkins-scripts/jenkins2-release/script_notify.sh down"
    	notify_up "/tol/data/jenkins-scripts/jenkins2-release/script_notify.sh up"
            HTTP_GET {
                url { 
                  path /login
                  status_code 200
                }
                connect_timeout 3
                nb_get_retry 3
                delay_before_retry 3
            }
        }
    
        real_server 10.155.10.174 8092 {
            weight 1
            HTTP_GET {
                url {
                  path /login
                  status_code 200
                }
                connect_timeout 3
                nb_get_retry 3
                delay_before_retry 3
            }
        } 
    }

    ! Configuration File for keepalived
    
    global_defs {
       router_id 10.155.10.174
       vrrp_skip_check_adv_addr
       vrrp_garp_interval 0
       vrrp_gna_interval 0
       script_user root
       enable_script_security
    }
    
    vrrp_instance JENKINS2_TRUNK {
        #此处不同，默认给 BACKUP,主机down机或恢复时，会根据VIP争夺自动变为MASTER或BACKUP
        state BACKUP
        interface em1
        #此处的id必须与主机的virtual_router_id 一致，否则可能会导致虚拟ip不能访问的问题
        virtual_router_id 167
        #权重要配置的比主机小，主机配置的是100，这里配置了50
        priority 50
        advert_int 1
        #此处必须与主机一致，否则会出连通的问题
        authentication {
            auth_type PASS
            auth_pass StupidMan
        }
        virtual_ipaddress {
            10.155.10.167
        }
        #当该虚拟实例状态变为MASTER时要调用的脚本
        notify_master "/tol/app/jenkins2-trunk/script_notify.sh master"
        #当该虚拟实例状态变为BACKUP时要调用的脚本
        notify_backup "/tol/app/jenkins2-trunk/script_notify.sh backup"
    }
    
    virtual_server 10.155.10.167 8093 {
        delay_loop 6
        lb_algo rr 
        lb_kind NAT
        persistence_timeout 50
        protocol TCP
    
        real_server 10.155.10.171 8093 {
    	    weight 3
            HTTP_GET {
                url { 
                  path /login
                  status_code 200
                }
                connect_timeout 3
                nb_get_retry 3
                delay_before_retry 3
            }
        }
    
        real_server 10.155.10.174 8093 {
            weight 3
            HTTP_GET {
                url {
                  path /login
                  status_code 200
                }
                connect_timeout 3
                nb_get_retry 3
                delay_before_retry 3
            }
        }
    
    }
    
    vrrp_instance JENKINS2_NEIBU {
        state BACKUP
        interface em1
        virtual_router_id 185
        priority 50
        advert_int 1
        authentication {
            auth_type PASS
            auth_pass StupidMan
        }
        virtual_ipaddress {
            10.155.10.185
        }
        notify_master "/tol/app/jenkins2-neibu/script_notify.sh master"
        notify_backup "/tol/app/jenkins2-neibu/script_notify.sh backup"
    }
    
    virtual_server 10.155.10.185 8091 {
        delay_loop 6
        lb_algo rr 
        lb_kind NAT
        persistence_timeout 50
        protocol TCP
        
        #sorry_server 192.168.200.200 1358
        
        real_server 10.155.10.172 8091 {
            weight 3
            HTTP_GET { 
                url {  
                  path /login
                  status_code 200
                } 
                connect_timeout 3
                nb_get_retry 3
                delay_before_retry 3
            }   
        }   
        
        real_server 10.155.10.174 8091 {
            weight 1
            HTTP_GET {
                url {
                  path /login
                  status_code 200
                } 
                connect_timeout 3
                nb_get_retry 3
                delay_before_retry 3
            }   
        }   
        
    }
    vrrp_instance JENKINS2_RELEASE {
        state BACKUP
        interface em1
        virtual_router_id 186
        priority 50
        advert_int 1
        authentication {
            auth_type PASS
            auth_pass StupidMan
        }
        virtual_ipaddress {
            10.155.10.186
        }
        notify_master "/tol/app/jenkins2-release/script_notify.sh master"
        notify_backup "/tol/app/jenkins2-release/script_notify.sh backup"
    }
    
    virtual_server 10.155.10.186 8092 {
        delay_loop 6
        lb_algo rr 
        lb_kind NAT
        persistence_timeout 50
        protocol TCP
        
        real_server 10.155.10.173 8092 {
            weight 3
            HTTP_GET { 
                url {  
                  path /login
                  status_code 200
                } 
                connect_timeout 3
                nb_get_retry 3
                delay_before_retry 3
            }   
        }   
        
        real_server 10.155.10.174 8092 {
            weight 1
            HTTP_GET {
                url {
                  path /login
                  status_code 200
                } 
                connect_timeout 3
                nb_get_retry 3
                delay_before_retry 3
            }   
        }   
    }

[root@cygs-09-10.171-jiaogou.cn ~]# systemctl status keepalived
● keepalived.service - LVS and VRRP High Availability Monitor
    Loaded: loaded (/usr/lib/systemd/system/keepalived.service; enabled; vendor preset: disabled)
    Active: active (running) since 四 2020-02-20 01:19:45 CST; 17h ago
    Process: 47502 ExecStart=/usr/sbin/keepalived $KEEPALIVED_OPTIONS (code=exited, status=0/SUCCESS)
    Main PID: 47503 (keepalived)
    Tasks: 3
    Memory: 2.1M
    CGroup: /system.slice/keepalived.service
            ├─47503 /usr/sbin/keepalived -D
            ├─47504 /usr/sbin/keepalived -D
            └─47505 /usr/sbin/keepalived -D

#!/bin/bash
case $1 in
down)
echo `date +"%Y-%m-%d %H:%M:%S  jenkins service down"` >> /tol/app/jenkins-trunk/logs/jenkins-notify.log
echo `date +"%Y-%m-%d %H:%M:%S  stop keepalived"` >> /tol/app/jenkins-trunk/logs/jenkins-notify.log
#停掉keepalived服务，如果应用服务down掉但不停掉keepalived服务的话，VIP不会转移到备用服务器的服务器上
systemctl stop keepalived
#如果不加这句 在VIP切换以后，虚拟IP可能会无法访问
ipvsadm -C

exit 1
;;
up)
echo `date +"%Y-%m-%d %H:%M:%S this notify_script is called [up]"` >> /tol/app/jenkins-trunk/logs/jenkins-notify.log
echo `date +"%Y-%m-%d %H:%M:%S run /tol/app/jenkins-trunk/bin/shutdown.sh and sleep 10s"` >> /tol/app/jenkins-trunk/logs/jenkins-notify.log
#关闭jenkins服务
sh /tol/app/jenkins-trunk/bin/shutdown.sh
sleep 10s
echo `date +"%Y-%m-%d %H:%M:%S run /tol/app/jenkins-trunk/bin/startup.sh and sleep 45s"` >> /tol/app/jenkins-trunk/logs/jenkins-notify.log
#启动jenkins服务
sh /tol/app/jenkins-trunk/bin/startup.sh
sleep 45s
echo `date +"%Y-%m-%d %H:%M:%S start keepalived"` >> /tol/app/jenkins-trunk/logs/jenkins-notify.log
#启动keepalived，启动以后会自动进行应用状态检查，如果应用仍没有正常启动，会继续走notify_down逻辑
systemctl start keepalived
#清空虚拟服务ipvs规则，有时候虚拟ip访问不了，执行这个命令就可以
ipvsadm -C
exit 1
;;
esac

#!/bin/bash
#
case $1 in
backup)
echo `date +"%Y-%m-%d %H:%M:%S i became backup"` >> /tol/app/jenkins2-trunk/jenkins-notify.log
exit 1
;;
master)
echo `date +"%Y-%m-%d %H:%M:%S i became master"` >> /tol/app/jenkins2-trunk/jenkins-notify.log
ipvsadm -C
ssh 10.155.10.171 "sh /tol/data/jenkins-scripts/jenkins2-trunk/script_notify.sh up"
echo `date +"%Y-%m-%d %H:%M:%S called remote script"` >> /tol/app/jenkins2-trunk/jenkins-notify.log
exit 1
;;
down)
echo `date +"%Y-%m-%d %H:%M:%S  jenkins service down"` >> /tol/app/jenkins2-trunk/jenkins-notify.log
echo `date +"%Y-%m-%d %H:%M:%S  stop keepalived"` >> /tol/app/jenkins2-trunk/jenkins-notify.log
systemctl stop keepalived
ipvsadm -C

exit 1
;;
up)
echo `date +"%Y-%m-%d %H:%M:%S this script is called"` >> /tol/app/jenkins2-trunk/jenkins-notify.log
exit 1
;;
esac

#!/bin/bash
#清空name.txt，没有就新创建
> name.txt
#将jobs目录中的modules builds目录生成到name.txt中做为rsync的exclude用
for line in `ls /tol/htdocs/jenkins-trunk/.jenkins/jobs/`
do
    echo "jobs/"$line"/modules/" >> name.txt
    echo "jobs/"$line"/builds/" >> name.txt
done
#将jenkins工作空间的文件（除了modules和builds）都同步到备用服务器对应服务的工作空间中
rsync -avz --exclude-from=/tol/data/jenkins-scripts/jenkins2-trunk/name.txt  --exclude=workspace/ /tol/htdocs/jenkins-trunk/.jenkins/ 10.155.10.174:/tol/data/jenkins-trunk
#调用备用服务器脚本重启备用服务器的jenkins服务
ssh 10.155.10.174 "sh /tol/app/jenkins2-trunk/restart.sh"

#!/bin/bash
process_id=$(ps -ef|grep 8093|grep -v grep|awk '{print $2}')
if [[ ! -z "$process_id" ]]
then
    echo $process_id not empty
    echo "killed process $process_id"
    kill -9 $process_id
    echo "start jenkins"
    source /etc/profile
    JENKINS_HOME=/tol/data/jenkins-trunk
    export JENKINS_HOME
    echo $JAVA_HOME >> /tol/app/jenkins2-trunk/jenkins-notify.log
    JAVA_CMD="nohup $JAVA_HOME/bin/java -jar /tol/app/jenkins2-trunk/jenkins-2.204.2.war --httpPort=8093 > /tol/app/jenkins2-trunk/jenkins.log 2>&1 &"  
    eval $JAVA_CMD
    echo "jenkins started pid "$(ps -ef|grep 8093|grep -v grep|awk '{print $2}')
else
    echo empty
    source /etc/profile
    JENKINS_HOME=/tol/data/jenkins-trunk
    export JENKINS_HOME
    echo $JAVA_HOME >> /tol/app/jenkins2-trunk/jenkins-notify.log
    JAVA_CMD="nohup $JAVA_HOME/bin/java -jar /tol/app/jenkins2-trunk/jenkins-2.204.2.war --httpPort=8093 > /tol/app/jenkins2-trunk/jenkins.log 2>&1 &"
    eval $JAVA_CMD
    echo "jenkins started pid "$(ps -ef|grep 8093|grep -v grep|awk '{print $2}')
fi

    0 * * * * sh /tol/data/jenkins-scripts/jenkins2-trunk/rsyncto174.sh &>/dev/null

    script_user root   #指定脚本执行为root
    enable_script_security #允许执行脚本

    # 这个命令master机上不用执行 备机上不执行也没事，有keepalived了
    ipvsadm -A -t 10.155.10.188:8080 -s wrr
    #-A添加虚拟服务器 -t tcp -s指定负载调度算法
    ipvsadm -a -t 10.155.10.188:8080 -r 10.155.10.171:8080 -m
    ipvsadm -a -t 10.155.10.188:8080 -r 10.155.10.172:8080 -m
    #执行以上命令出现 no address or device的问题的时候，先使用 ipvsadm -ln 看是否已存在对应的设置 
    #删除配置
    [root@lb01 ~]# ipvsadm -D -t 10.188.188.188:80
    [root@lb01 ~]# ipvsadm -d -t 10.188.188.188:80 -r 10.155.10.171:8080
    [root@lb01 ~]# ipvsadm -d -t 10.188.188.188:80 -r 10.155.10.172:8081

! Configuration File for keepalived     #全局定义
    
global_defs {
    notification_email {  
        #指定keepalived在发生事件时(比如切换)发送通知邮件的邮箱
        ops@wangshibo.cn   #设置报警邮件地址，可以设置多个，每行一个。 需开启本机的sendmail服务
        tech@wangshibo.cn
    }
        
    notification_email_from ops@wangshibo.cn   #keepalived在发生诸如切换操作时需要发送email通知地址
    smtp_server 127.0.0.1      #指定发送email的smtp服务器
    smtp_connect_timeout 30    #设置连接smtp server的超时时间
    router_id master-node      #运行keepalived的机器的一个标识，通常可设为hostname。故障发生时，发邮件时显示在邮件主题中的信息。
}
    
vrrp_script chk_http_port {      #检测nginx服务是否在运行。有很多方式，比如进程，用脚本检测等等
    script "/opt/chk_nginx.sh"   #这里通过脚本监测
    interval 2                   #脚本执行间隔，每2s检测一次
    weight -5                    #脚本结果导致的优先级变更，检测失败（脚本返回非0）则优先级 -5
    fall 2                    #检测连续2次失败才算确定是真失败。会用weight减少优先级（1-255之间）
    rise 1                    #检测1次成功就算成功。但不修改优先级
}
    
vrrp_instance VI_1 {    #keepalived在同一virtual_router_id中priority（0-255）最大的会成为master，也就是接管VIP，当priority最大的主机发生故障后次priority将会接管
    state MASTER        #指定keepalived的角色，MASTER表示此主机是主服务器，BACKUP表示此主机是备用服务器。注意这里的state指定instance(Initial)的初始状态，就是说在配置好后，这台服务器的初始状态就是这里指定的，但这里指定的不算，还是得要通过竞选通过优先级来确定。如果这里设置为MASTER，但如若他的优先级不及另外一台，那么这台在发送通告时，会发送自己的优先级，另外一台发现优先级不如自己的高，那么他会就回抢占为MASTER
    interface em1          #指定HA监测网络的接口。实例绑定的网卡，因为在配置虚拟IP的时候必须是在已有的网卡上添加的
    mcast_src_ip 103.110.98.14  # 发送多播数据包时的源IP地址，这里注意了，这里实际上就是在哪个地址上发送VRRP通告，这个非常重要，一定要选择稳定的网卡端口来发送，这里相当于heartbeat的心跳端口，如果没有设置那么就用默认的绑定的网卡的IP，也就是interface指定的IP地址
    virtual_router_id 51         #虚拟路由标识，这个标识是一个数字，同一个vrrp实例使用唯一的标识。即同一vrrp_instance下，MASTER和BACKUP必须是一致的
    priority 101                 #定义优先级，数字越大，优先级越高，在同一个vrrp_instance下，MASTER的优先级必须大于BACKUP的优先级
    advert_int 1                 #设定MASTER与BACKUP负载均衡器之间同步检查的时间间隔，单位是秒
    authentication {             #设置验证类型和密码。主从必须一样
        auth_type PASS           #设置vrrp验证类型，主要有PASS和AH两种
        auth_pass 1111           #设置vrrp验证密码，在同一个vrrp_instance下，MASTER与BACKUP必须使用相同的密码才能正常通信
    }
    virtual_ipaddress {          #VRRP HA 虚拟地址 如果有多个VIP，继续换行填写
        103.110.98.20
    }
    
    track_script {                      #执行监控的服务。注意这个设置不能紧挨着写在vrrp_script配置块的后面（实验中碰过的坑），否则nginx监控失效！！
        chk_http_port                    #引用VRRP脚本，即在 vrrp_script 部分指定的名字。定期运行它们来改变优先级，并最终引发主备切换。
    }
    notify_master “xxxxx start”     # 当成为master时调用脚本
    notify_backup “xxxxx stop”      # 当成为backup时调用脚本
}