script | weskiller

这是我最近维护的线上数据库的备份脚本，断断续续code了不少。
就目前来看效果很不错，当然主要目的是抛(rang)砖(wo)引(zhuang)玉(bi)。

backup.sh
点击下载
backup.sh 会调用对应的配置文件mysql-cluster-list，public.conf，backuptime文件，调度多个备份脚本执行，并统计结果

#!/bin/bash
#Author		weskiller	2016-04-08	
#Email:weskiller@vip.qq.com

#定义环境变量
export PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin
export LANG="en_US.UTF-8"

#定义路径
basedir=/root/scripts/backup
log=$basedir/backup.log
date=`date +%F`
starttime_conf=$basedir/backuptime
mysql_cluster_conf=$basedir/mysql-cluster-list

#superadmin邮件地址,当脚本文件缺失的时候,会发送邮件
Semail='weskiller@vip.qq.com'

#下标变量
declare -i sub=0

#设置循环检查的间隔时间
check_sleep=60

#设置备份超时时间(每天下午6点,用户高峰前期)
deadtime=`date --date "18:00" +%s`

#获取mysql-cluster的列表
mysql_cluster_list=(192.168.1.1)

#或者通过远程的web页面获取列表
#cluster_curl=http://192.168.1.1/mysql_cluster.txt
#for list in `curl "$cluster_curl"`;do
#	mysql_cluster_list[$sub]=$list;
#done

#标记日志备份开始
echo "" >> $log
echo "$date backup.sh start" >> $log

#加载公有配置
if [ -f "$basedir/config/public.conf" ];then
	. $basedir/config/public.conf
else
	/bin/mail -r "backup.sh" -s "backup.sh scripts is interrupt" "$Semail" <<< "`date +%F\ %T` - backup scripts is not performing! the public.conf is miss."
	echo "`date +%T` - backup scripts is not performing! the public.conf is miss." >> $log
	exit 1
fi

#检查开始备份时间配置文件
if [ ! -f "$starttime_conf" ];then
	sendemail "backup.sh scripts is interrupt" "`date +%F\ %T` - backup scripts is not performing! can't find configuration with ${starttime_conf}."
	echo "`date +%T` - backup scripts is not performing! can't find configuration with ${starttime_conf}." >> $log
	exit 1
fi

#加载配置文件mysql_cluster_conf,如果存在的话
if [ -f "$mysql_cluster_conf" ];then
	mysql_cluster_list=(`cat "$mysql_cluster_conf"`)
fi

#检测后端mysql_cluster
declare -i change=0
for i in `seq 0 $((${#mysql_cluster_list[@]}-1))`;do
	nc -z ${mysql_cluster_list[$i]} 3306 >/dev/null 2>&1
	if [ $? -eq 0 ];then
		continue
	else
		echo "detected ${mysql_cluster_list[$i]} is down. will be remove from mysql cluster." >> $log
		unset mysql_cluster_list[$i]
		change=1
	fi 
done

#重新定义有效的mysql_cluster_list
if [ $change -eq 1 ];then
	mysql_cluster_list=(${mysql_cluster_list[@]})
fi

#极端情况下后端集群全部宕机
if [ ${#mysql_cluster_list[@]} -eq 0 ];then
	sendemail "backup.sh scripts is interrupt" "`date +%F\ %T` - backup scripts is not performing! without living mysql cluster."
	echo "`date +%T` - backup scripts is not performing! without living mysql cluster." >> $log
	exit 1
fi

#手动备份
if [ "x$1" == "x" ];then
	manual=0
elif [ "$1" == "manual" ];then
	manual=1
	echo "backup.sh is start with manual." >> $log
fi

#重定向所有标准错误信息
#exec 2>$basedir/backup.error

#备份循环
sub=0
for project in $basedir/config/*-*;do
	script=`basename $project`
	project_name=${script%-*}
#默认模式下,根据设定的时间备份
	if [ "$manual" -eq 0 ];then
#获取配置时间
		project_start_time="`awk 'BEGIN{FS="="}/'$script'/{print $2}' $starttime_conf`" 
		if [ $? -eq 0 ];then
			project_timestamp=`date --date "$project_start_time" +%s`
			if [ "$project_timestamp" -le "`date +%s`" ];then
				echo "`date +%F\ %T` - $script backuptime is error" >> $log
				continue
			fi
		else
			echo "`date +%F\ %T` - $script backuptime is not find " >> $log
			continue
		fi
		waittime="$((project_timestamp-`date +%s`))"
#手动备份,根据设定的时间延迟
	elif [ "$manual" -eq 1 ];then
		first=`awk 'BEGIN{FS="="}{print $2}' backuptime|grep -P "^[^ ]" |sort -n|head -1`
		lag=$((`date --date '1 minute' +%s`-`date --date "$first" +%s`))
		project_start_time="`awk 'BEGIN{FS="="}/'$script'/{print $2}' $starttime_conf`" 
                if [ $? -eq 0 ];then
                        project_timestamp=$((`date --date "$project_start_time" +%s`+$lag))
                else
                        echo "`date +%F\ %T` - $script backuptime is not find " >> $log
                        continue
                fi
                waittime="$((project_timestamp-`date +%s`))"
	fi
	host=${mysql_cluster_list[$((sub%${#mysql_cluster_list[@]}))]}
#使用screen,定时放入后台
	screen -S $project_name -d -m /bin/bash "$project" "$host" "$waittime"
#放入项目数组
	project_list[$sub]="$script"
	((sub++))
done

#监控今天的备份日志,当所有备份完成后发送邮件通知
line=`grep -Pn "$date backup.sh start" $log|sed -n '$p'|awk 'BEGIN{FS=":"}{print $1}'`
while sleep $check_sleep && [ ${#project_list[@]} -gt 0 ];do
	sub=0
	for i in ${project_list[@]};do
		if sed -n "$line,\$p" $log |grep -Pq "$i backup (success|failed)" ;then
			 unset project_list[$sub]
		fi  
		((sub++))
	done
	project_list=(${project_list[@]})
	if [ "`date +%s`" -ge "$deadtime" ];then
		sendemail "backup is overtime" "`date +%F\ %T` -- project:${project_list[*]} already overtime"
	fi
done

#判断是否有失败的项目
if sed -n "$line,\$p" $log |grep -Pq "backup failed";then
	backup_failed=1
else 
	backup_failed=0
fi

#标记日志备份结束
echo "$date backup.sh finish" >> $log

#统计失败项目邮件
if [ $backup_failed -eq 1 ];then
	sendemail "some project backup failed" "`date +%F\ %T` -- backup failed project:`sed -n $line,'$p' $log|grep -Po "\S*(?= backup failed)"|tr '\n' ' '`"
fi

#生成WEB网页
sed -n -e "$line,\$p" -e '1a<pre>' -e '$a</pre>' $log > /backup/backup.html

#日志整理
if [ `date --date '1 day' +%d` -eq 1 ];then
        .  $basedir/logrotate.sh
fi

#优雅的退出
exit 0

scripts
点击下载
scripts 是真正的备份执行脚本，加载public.conf获取公共配置信息，针对不同的线上业务和数据库可以自定义操作，或者根据需求修改，这里只是提供模板。

#!/bin/bash
#author weskiller 2016-04-08
#description:project backup scripts unit

#2015-05-16:新增隔天自动备份功能

#定义环境变量
export PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin
export LANG="en_US.UTF-8"
#获取脚本包含的信息
dir=/root/scripts/backup
scripts=`basename $0`
backup_host=192.168.1.${scripts#*-}
game_label=${scripts%-*}

#定义各种需要的时间戳
month=`date +%m`
last_month=`date --date "1 months ago" +%m`
day=`date +%d`
yesterday=`date --date "1 day ago" +%d`

#定义数据库信息
game_database="p${game_label}_logs1"
game_backup_database="p${game_label}_logs_backup1"
today="$(date --date @$(($(date --date "`date +%F`" +%s)-3600*8)) +%F\ %T)"
delete_gamelog_time="$(date --date @$((`date --date "$today" +%s`+7*3600)) +%F\ %T)"
three_months="`date --date '4 month ago' +%m`"

#加载公共配置
if [ -f "$dir/config/public.conf" ];then
	. $dir/config/public.conf
else
	echo "`date +%F\ %T`:not find public.conf" > $dir/config/$scripts.error 
	exit 1; 
fi

#获取backup.sh传递的参数
if [ "x$1" == "x" ];then
	echo "`date +%F\ %T`:not received parameter:mysql backup host " >> $dir/log/$scripts.log
	exit 1
else
	db_host="$1"
fi
if [ "x$2" == "x$2" ];then
	sleep $2
	if [ $? -ne 0 ];then
	echo "`date +%F\ %T`:received sleep time parameter,but sleep error " >> $dir/log/$scripts.log
	exit 1
	fi
else
	echo "`date +%F\ %T`:not received parameter:sleep time(s) " >> $dir/log/$scripts.log
	exit 1
fi

#开始时间戳
start_time="`date +%s`"

#创建文件夹，如果不存在的话
/usr/bin/ssh root@$backup_host "mkdir -p $backupdir/$game_label/months/$month"

#创建xtrabakup命令
if [ "$day" -eq 1 ];then
	innobackup_command="/usr/bin/innobackupex --no-timestamp $backupdir/$game_label/months/$month/$day"
	/usr/bin/ssh root@$backup_host "rm -f $backupdir/$game_label/days"
	/usr/bin/ssh root@$backup_host "ln -sf $backupdir/$game_label/months/$month $backupdir/$game_label/days"
#删除三个月之前的备份
	delete_three_backup="rm -rf $backupdir/$game_label/months/$three_months"
else
#获取days下的目录,确定日期
	uday=$(ssh root@$backup_host "basename \`ls -d $backupdir/$game_label/days/*|grep -Pv "tar.bz2$"\` 2>/dev/null")
	if [ ${uday:-0} -eq $yesterday ];then
	innobackup_command="/usr/bin/innobackupex --incremental --incremental-basedir $backupdir/$game_label/days/$yesterday --no-timestamp  $backupdir/$game_label/days/$day ; result=\$? ; if [ \$result -ne 0 -a ! -e $backupdir/$game_label/days/$day/xtrabackup_info ];then rm -rf $backupdir/$game_label/days/$day;fi ;exit \$result"
	else
	innobackup_command="/usr/bin/innobackupex --incremental --incremental-basedir $backupdir/$game_label/days/$uday --no-timestamp  $backupdir/$game_label/days/$day ; result=\$? ; if [ \$result -ne 0 -a ! -e $backupdir/$game_label/days/$day/xtrabackup_info ];then rm -rf $backupdir/$game_label/days/$day;fi ;exit \$result"
	jump=1	
	fi
fi

#创建mysqldump命令
mysqldump_command="mysqldump --single-transaction --skip-add-locks --skip-lock-tables  --no-create-info --replace --where \"create_time < '$today'\" $game_database logs  |mysql -u$db_user -h$db_host -p$db_pass $game_backup_database"

#创建清理日志命令
mysql_command="mysql -e \"delete from $game_database.logs where create_time < '$delete_gamelog_time';\""

#压缩前一天的备份文件
if  [ "$day" -eq 1 ];then
	tar_command="tar -jcf $backupdir/$game_label/months/$last_month/$yesterday.tar.bz2 -C $backupdir/$game_label/months/$last_month $yesterday --remove-files"
else
	if [ ${uday:-0} -eq $yesterday ];then
		tar_command="tar -jcf $backupdir/$game_label/days/$yesterday.tar.bz2 -C $backupdir/$game_label/days $yesterday --remove-files"
	else
		tar_command="tar -jcf $backupdir/$game_label/days/${uday}.tar.bz2 -C $backupdir/$game_label/days $uday --remove-files"
	fi
fi
#日志标记开始
echo "" >> $dir/log/$scripts.log
echo "`date +%F\ %T` $scripts start" >> $dir/log/$scripts.log
#在远程备份客服端上执行命令
for command in "$innobackup_command" "$mysqldump_command" "$mysql_command" "$tar_command" "$delete_three_backup" ;do
#先判断变量内容,确保ssh命令正常执行
if [ -z "$command " ] ||  echo "$command"|grep -Pq "^\s*$" ;then
	continue;
fi
ssh root@$backup_host "$command" 2>$dir/log/$scripts.error 1>/dev/null
if [ $? -eq 0 ];then
	echo "`date +%F\ %T` -- \"$command\" performing success." >> $dir/log/$scripts.log
else
	echo "`date +%F\ %T` -- \"$command\" performing failed." >> $dir/log/$scripts.log
#	sendemail "$scripts" "`cat $dir/log/$scripts.error`"
	log "failed" "$scripts"
	exit 1
fi
done
#删除错误的输出
rm -rf $dir/log/$scripts.error
#标记日志结束
echo "`date +%F` $scripts finish" >> $dir/log/$scripts.log
#输出backup.log日志
if  [ ${jump:-0} -eq 1 ];then
	log "success" "$scripts" "$((`date +%s`-start_time))" "$uday $day"
else
	log "success" "$scripts" "$((`date +%s`-start_time))"
fi
#优雅的退出
exit 0

public.conf
点击下载
public.conf 公共配置文件，包含了日志输出和维护邮件组，mysql验证

#this is public config that all scripts need to be loaded
#定位绝对路径
backupdir=/backup
#备份验证
db_user=
db_pass=

#负责人E-Mail地址
adminmail=(weskiller@vip.qq.com admin@xx.com)
sendemail () {
	for mail in ${adminmail[@]};do
	/bin/mail -r "backup.sh" -s "$1" "$mail" <<< "$2"
	done
}

#输出日志函数
log () {
case $1 in
	success)
	if [ $3 -lt 60 ];then
		consume="${3}s"
	elif [ $3 -lt 3600 ];then
		consume="$(($3/60))m$(($3%60))s"
	else
		consume="$(($3/(60*60)))h$((($3%(60*60))/60))m$(($3%60))s"
	fi
	if [ "x$4" == "x" ];then
		echo "`date +%F\ %T` - $2 backup $1. used ${consume}." >> /root/scripts/backup/backup.log
	else
		echo "`date +%F\ %T` - $2 backup $1. used ${consume}. ${4% *} => ${4#* } " >> /root/scripts/backup/backup.log
	fi
	;;
	failed)
	echo "`date +%F\ %T` - $2 backup $1" >> /root/scripts/backup/backup.log
	;;
esac
}

backuptime
点击下载
backuptime 主要用于设定备份数据库的时间

#测试数据库
test-1=01:00
test-2=00:00

#线上数据库
produce-101=03:00
produce-102=03:30

以前买过许多vps,国内，国外的都有。无论idc供应商防火墙做的多好，策略有多复杂，总是会有很多连续的IP段不停的扫描你的主机服务端口，特别是Linux系统的默认远程管理端口22
最开始博主也是放置不管，并没有起多大疑心
但是麻烦就来了。不到一周的时间 ssh系统日志达到了10G，当时我一看磁盘空间就懵了，刚买的主机，什么服务都没放，怎么磁盘就增加了這么多
细细检查下才发现是/var/log/secure 这个记录ssh登陆情况的日志占用了空间
再检查内容，发现全是记录的ssh登陆失败的日志，恐怖的是某几个IP的验证密码的次数达到了十几万次
吸取教训，删除了secure日志。但是治标不治本，决定对ssh安全花些心思

No1 修改ssh端口号

更换ssh是一个很实用的方法，直接略去了百分之80以上的恶意扫描。操作也很简单，缺点是剩下百分之20还是会侦探到远程端口，没有根治

#vi /etc/ssh/sshd_config
Port $port

No2 针对固定IP开发端口

这种方法是很多企业使用的方式，效果好，安全，能根治恶意扫描，操作也相对简单
在防火墙中添加规则

iptables -t filter -I INPUT -s $IP -p tcp -m tcp --dport 22 -m state --state NEW,ESTABLISHED -j ACCEPT

问题是如果没有固定的IP需要搭建VPN跳板

No3 编写脚本自动拒绝恶意攻击

博主最开始就是从这角度出发的，最开始在网上找了许多shell脚本，但是都不符合博主本意。偶然看到前辈一篇博客，备受启发，于是觉得编写一个适合自己的

#!/bin/bash
#function	    DROP all failed IP if  more than $Number(default is 99)
#author         weskiller 	2014-10-31
#drop this script on /root/ext_ssh_deny/ext_ssh_deny.sh
#and add crond like this
#crontab -e
#*/30 * * * * /root/ext_ssh_deny/ext_ssh_deny.sh
#service crond start

#Check whether the iptables installed
[ -e /usr/sbin/iptables ] || { echo "iptables-services not installed" && exit 1 ;}

#set limit number
Number=

#set  IP of regular expressions
Regular_Expression_Ip='(\<(22[0-3]|2[01][0-9]|1[3-9][0-9]|12[0-689]|1[0-1][0-9]|[1-9]?[0-9]|[1-9]))(\.(25[0-5]|2[0-4][0-9]|[0-1]?[0-9]?[0-9])){2}(\.(25[0-4]|2[0-4][0-9]|1[0-9]{2}|[1-9][0-9]|[1-9])\>)'

#set Centos ssh log file
File=/root/ext_ssh_deny/secure

#Extract failed information
FIND() {
	grep -Eo "((failure|Failed)(.*)$Regular_Expression_Ip|$Regular_Expression_Ip(.*)failed)" /var/log/secure > $File
}

#Find the attack IP
ATTACK_LIST() {
	grep -Eo "$Regular_Expression_Ip" $File |sort|uniq -c|awk '($1>'${Number:=99}') {printf $2"\n"}'|sort > /root/ext_ssh_deny/Attack_Ip
	
}

#Add iptables rule 
DENY_ATTACK_IP() {
	for IP in $1
	do
	Re_Ip=`echo $IP |sed  's|\.|\\\.|g'`
	TIMES=`grep "$Re_Ip" /root/ext_ssh_deny/secure |wc -l`
	/sbin/iptables -vnL|grep "$Re_Ip" >/dev/null 2>&1 || /sbin/iptables -I INPUT -s $IP -m state --state NEW,RELATED,ESTABLISHED -p tcp --dport 22 -j DROP && echo "`date +%Y%m%d-%H:%M:%S`    IP:$IP  TIMES=$TIMES" >> /root/ext_ssh_deny/deny.log
	done
}

#Extract iptables deny
IPTABLES_DENY_IP() {
 	/sbin/iptables -vnL|grep -Eo "DROP.*$Regular_Expression_Ip" |awk '{print $NF}'|sort > /root/ext_ssh_deny/Iptables_Deny_Ip
}

#the IP from the script log
LOG_IP() {
	grep -Eo "$Regular_Expression_Ip" /root/ext_ssh_deny/deny.log |sort > /root/ext_ssh_deny/Log_Ip 
}

#clear iptables rules and script log
CLEAR() {
	cat /dev/null > /root/ext_ssh_deny/deny.log
	sed -i '/\/32/'d /etc/sysconfig/iptables
	/sbin/service iptables restart >/dev/null 2>&1
}

#initialization
RESET(){
	CLEAR
	FIND
	ATTACK_LIST
	DENY_ATTACK_IP "`cat /root/ext_ssh_deny/Attack_Ip`"
	/sbin/service iptables save >/dev/null 2>&1
	exit 0
}

#order to refresh data
PREPARE() {
	IPTABLES_DENY_IP
	LOG_IP
}

#just add new Attack_Ip in iptables
UPDATE() {
	FIND
	ATTACK_LIST
	comm -13 /root/ext_ssh_deny/Log_Ip /root/ext_ssh_deny/Attack_Ip > /root/ext_ssh_deny/Update_Ip 
	[ -s /root/ext_ssh_deny/Update_Ip ] || exit 0
	DENY_ATTACK_IP "`cat /root/ext_ssh_deny/Update_Ip`"
	/sbin/service iptables save >/dev/null 2>&1
	exit 0
}

#check script health in order to Released  ip if a long time ago  
CHECK_HEALTH() {
	[ -f /var/log/secure ] || return 1
	[ -s /var/log/secure ] || return 1
	[ -f /root/ext_ssh_deny/deny.log ] || return 2
	[ -s /root/ext_ssh_deny/deny.log ] || return 2
	/usr/sbin/iptables -vnL|grep -Eo "DROP.*$Regular_Expression_Ip"  >/dev/null 2>&1 || return 3
	PREPARE
	[ -z "`comm -13 /root/ext_ssh_deny/Iptables_Deny_Ip /root/ext_ssh_deny/Log_Ip`" ] && return 0 || return 3
}

#start
MAIN() {
CHECK_HEALTH
case $? in
	1)
	CLEAR && return 1
	;;
	2|3)
	RESET 
	;;
	0)
	UPDATE
	;;
esac
}

MAIN

如上，脚本的功能是自动找出ssh登陆次数大于限定次数的ip,并加入防火墙中拒绝登陆，而且做到自动更新的功能
缺点是适应性不强，根据个人定制，无法满足所有人

No4 取消密码登陆，使用密钥验证

优点是操作简单，实用性很好。缺点是密钥文件需要本地存储，无法使用交互式命令如scp
生成密码文件

#ssh-keygen -t rsa -b 2048
Generating public/private rsa key pair.
Enter file in which to save the key (/root/.ssh/id_rsa):  
Created directory '/root/.ssh'.
Enter passphrase (empty for no passphrase): 
Enter same passphrase again: 
Your identification has been saved in /root/.ssh/id_rsa.
Your public key has been saved in /root/.ssh/id_rsa.pub.
The key fingerprint is:
40:86:b1:06:62:0b:ea:76:ce:2f:28:76:2c:55:23:b7 root@localhost.localdomain
The key's randomart image is:
+--[ RSA 2048]----+
|o.. .oo          |
|+....+           |
|..  o .          |
|.  o + .         |
| o .+ o S        |
|. +. E           |
|  +o             |
|.+ +.            |
|o o ..           |
+-----------------+
#cp ~/.ssh/id_rsa.pub ~/.ssh/authorized_keys
#chmod 400 ~/.ssh/authorized_keys
#vi /etc/ssh/sshd_config
PasswordAuthentication no
#service sshd restart

No5 使用vnc远程管理

这种方式直接弃用了ssh,十分彻底的屏蔽了恶意主机的扫描和攻击，随意设定的端口也为攻击带来了难度，而且使用的真实的用户命令接口。但是问题也很明显。依旧使用口令，远程管理，流量开销大，对系统资源占用也比ssh要多，导致经常性的网络中断。建议管理内网的服务器使用

weskiller

have a nice day ,

分类目录归档：script

shell-scripts(backup)

ssh安全措施