日志切割方法及脚本，日志切割方法脚本
日志切割方法及脚本，日志切割方法脚本
ansible-log脚本：
##日志分割集中管理
-
hosts: webs
become: yes
vars:
app:
tomcat_log_split
tasks:
-
name: "{{
app }}"
script: /opt/tags/ansible/playbooks/log_scripts/{{
app }}.sh {{ ENV }} {{ TIME }}
-
hosts: services
become: yes
vars:
app:
service_log_split
tasks:
-
name: "{{
app }}"
script: /opt/tags/ansible/playbooks/log_scripts/{{
app }}.sh {{ ENV }} {{ TIME }}
-
hosts: mongodb
become: yes
vars:
app:
mongo_log_split
tasks:
-
name: "{{
app }}"
script: /opt/tags/ansible/playbooks/log_scripts/{{
app }}.sh {{ ENV }} {{ TIME }}
-
hosts: openresty
become: yes
vars:
app:
openresty_log_split
tasks:
-
name: "{{
app }}"
script: /opt/tags/ansible/playbooks/log_scripts/{{
app }}.sh {{ ENV }} {{ TIME }}
-
hosts: nginx
become: yes
vars:
app:
nginx_log_split
tasks:
-
name: "{{
app }}"
script: /opt/tags/ansible/playbooks/log_scripts/{{
app }}.sh {{ ENV }} {{ TIME }}
Tomcat日志：
#!/bin/bash
set -x
#tomcat日志分割脚本
#判断参数个数
if [
$#
!= 2 ] ; then
echo "USAGE:
$0 ENV TIME"
exit 1;
fi
ENV=$1
TIME=$2
IP=`/sbin/ifconfig eth0
|grep "inet
addr" |awk {'print
$2'}
|awk -F ':' {'print
$2'}`
ACCESS_LOG_TIME=`date +%Y-%m-%d`
S3_DIR="s3://***-logs/${ENV}/web"
#日志文件存放目录
LOCAL_DIR="/opt/tomcat_websvr_*/logs"
#item目录
ITEM_DIR="/opt/tomcat_websvr_*"
#压缩后缀名
BAK_SUFFIX=".gz"
#判断日志备份目录是否存在
for logs_bak_dir in `ls ${LOCAL_DIR}
-d`;
do
if [
-d ${logs_bak_dir}/logs_bak ];then
echo "dir
exist!"
else
mkdir -p
${logs_bak_dir}/logs_bak
fi
done
#只处理以log、out、txt结尾的日志文件,排除try.log等3个日志
for logfile in `find ${LOCAL_DIR}
| grep -v "try.log$\|reg.log$\|login.log$" | egrep "*[a-z].log$|
*[a-z].out$| *${ACCESS_LOG_TIME}.txt$"`;
do
#强制cp并保留属性
/bin/cp -af
${logfile} ${logfile}.${TIME}
echo -n "" >
${logfile}
#压缩日志文件并保存到指定目录
gzip -c
${logfile}.${TIME} > ${logfile}+${TIME}+${BAK_SUFFIX}
mv ${logfile}+${TIME}+${BAK_SUFFIX}
${logfile%/*}/logs_bak/
done
#上传文件到s3,删除对应的文件
for project in `ls ${ITEM_DIR}
-d |awk -F '/' '{print
$NF}'`;
do
aws
s3 sync /opt/${project}/logs/logs_bak/ ${S3_DIR}/${project}/${IP}
--exclude "*" --include "*.log+${TIME}+${BAK_SUFFIX}"
aws
s3 rm ${S3_DIR}/${project}/${IP}
--recursive --exclude "*" --include "*`date
-d "-90
days"
+%Y%m%d`*${BAK_SUFFIX}"
aws
s3 rm ${S3_DIR}/${project}/${IP}
--recursive --exclude "*" --include "*`date
-d "-90
days"
+%Y-%m-%d`*${BAK_SUFFIX}"
done
#删除30天前的日志
find ${LOCAL_DIR}/
-name "*`date
-d "-7
days"
+%Y%m%d`*" |xargs rm -rf
Nginx日志：
#!/bin/bash
set -x
#判断参数个数
if [
$#
!= 2 ] ; then
echo "USAGE:
$0 ENV"
exit 1;
fi
#日志文件存放目录
ENV=$1
TIME=$2
IP=`/sbin/ifconfig eth0
|grep "inet
addr" |awk {'print
$2'}
|awk -F ':' {'print
$2'}`
S3_DIR="s3://***-logs/${ENV}/nginx"
LOG_FILE="/etc/nginx/logs"
#压缩后缀名
BAK_SUFFIX=".gz"
#判断日志备份目录是否存在
for logs_bak_dir in `ls ${LOG_FILE}
-d`;
do
if [
-d ${logs_bak_dir}/logs_bak ];then
echo "dir
exist!"
else
mkdir -p
${logs_bak_dir}/logs_bak
fi
done
#备份日志主要逻辑
for logfile in `find ${LOG_FILE}
|grep "log$"`;
do
#强制cp并保留属性
/bin/cp -af
${logfile} ${logfile}.${TIME}
echo -n "" >
${logfile}
#压缩日志文件并保存到指定目录
gzip -c
${logfile}.${TIME} > ${logfile}+${TIME}+${BAK_SUFFIX}
mv ${logfile}+${TIME}+${BAK_SUFFIX}
${logfile%/*}/logs_bak/
done
#重启服务
/etc/init.d/nginx reload
#s3上传并删除指定日期文件
aws
s3 sync ${LOG_FILE}/logs_bak/ ${S3_DIR}/${IP}
--exclude "*" --include "*${TIME}+${BAK_SUFFIX}"
aws
s3 rm ${S3_DIR}/${IP}
--recursive --exclude "*" --include "*`date
-d "-90
days"
+%Y%m%d`*${BAK_SUFFIX}"
#删除30天前的日志
find ${LOG_FILE}/
-name "*${BAK_SUFFIX}" -mtime
+30 -delete
find ${LOG_FILE}/
-name "*`date
-d "-3
days"
+%Y%m%d`*" |xargs rm -rf
Service日志：
#!/bin/bash
set -x
#判断参数个数
if [
$#
!= 2 ] ; then
echo "USAGE:
$0 ENV TIME"
exit 1;
fi
ENV=$1
TIME=$2
IP=`/sbin/ifconfig eth0
|grep "inet
addr" |awk {'print
$2'}
|awk -F ':' {'print
$2'}`
TYPE="service"
S3_DIR="s3://***-logs/${ENV}/service"
#日志文件存放目录
LOCAL_DIR="/opt/platform/services/*/*/logs"
#日志文件过滤
LOG_FILE="/opt/platform/services/*/*/logs/*.log"
#不同类型日志目录
TYPE_DIR="/opt/platform/services/*/*"
#压缩后缀名
BAK_SUFFIX=".gz"
#判断日志备份目录是否存在
for logs_bak_dir in `ls ${LOCAL_DIR}
-d`;
do
if [
-d ${logs_bak_dir}/logs_bak ];then
echo "dir
exist!"
else
mkdir -p
${logs_bak_dir}/logs_bak
fi
done
#备份日志主要逻辑
for logfile in `find ${LOG_FILE}
|grep "log$"`;
do
#强制cp并保留属性
/bin/cp -af
${logfile} ${logfile}.${TIME}
echo -n "" >
${logfile}
#压缩日志文件并保存到指定目录
gzip -c
${logfile}.${TIME} > ${logfile}+${TIME}+${BAK_SUFFIX}
mv ${logfile}+${TIME}+${BAK_SUFFIX}
${logfile%/*}/logs_bak/
done
#同步到s3并删除s3指定日期日志
for project in `ls ${TYPE_DIR}
-d |awk -F '/' '{print
$NF}'`;
do
aws
s3 sync /opt/platform/services/*/${project}/logs/logs_bak/ ${S3_DIR}/${project}/${IP}
--exclude "*" --include "*${TIME}+${BAK_SUFFIX}"
aws
s3 rm ${S3_DIR}/${project}/${IP}
--recursive --exclude "*" --include "*`date
-d "-90
days"
+%Y%m%d`*${BAK_SUFFIX}"
done
#删除7天前日志
find ${LOCAL_DIR}/
-name "*`date
-d "-3
days"
+%Y%m%d`*" |xargs rm -rf
Mongo日志：
#!/bin/bash
set -x
#判断参数个数
if [
$#
!= 2 ] ; then
echo "USAGE:
$0 ENV TIME"
exit 1;
fi
#日志文件存放目录
ENV=$1
TIME=$2
IP=`/sbin/ifconfig eth0
|grep "inet
addr" |awk {'print
$2'}
|awk -F ':' {'print
$2'}`
NAME="mongo"
S3_DIR="s3://***-logs/${ENV}/other"
LOG_FILE="/var/log/*/mongod*.log"
LOCAL_DIR="/var/log/mongo"
BAK_SUFFIX=".gz"
#判断日志备份目录是否存在
for logs_bak_dir in `ls ${LOCAL_DIR}
-d`;
do
if [
-d ${logs_bak_dir}/logs_bak ];then
echo "dir
exist!"
else
mkdir -p
${logs_bak_dir}/logs_bak
fi
done
#日志文件过滤
for LOG in `find ${LOG_FILE}
|grep "log$"`;do
#强制cp并保留属性
/bin/cp -af
${LOG} ${LOG}.${TIME}
echo -n "" >
${LOG}
#压缩日志文件
gzip -c
${LOG}.${TIME} > ${LOG}+${TIME}+${BAK_SUFFIX}
mv ${LOG}+${TIME}+${BAK_SUFFIX}
${LOG%/*}/logs_bak/
done;
#上传文件到s3
aws
s3 sync ${LOCAL_DIR}/logs_bak/ ${S3_DIR}/${NAME}/${IP}
--exclude "*" --include "*${TIME}+${BAK_SUFFIX}"
aws
s3 rm ${S3_DIR}/${NAME}/${IP}
--recursive --exclude "*" --include "*`date
-d "-90
days"
+%Y%m%d`*${BAK_SUFFIX}"
#删除7天前日志
find ${LOCAL_DIR}
-name "mongo*${BAK_SUFFIX}" -mtime
+7 |xargs rm -rf
Openresty日志：
#!/bin/bash
set -x
#判断参数个数
if [
$#
!= 2 ] ; then
echo "USAGE:
$0 ENV"
exit 1;
fi
#日志文件存放目录
ENV=$1
TIME=$2
IP=`/sbin/ifconfig eth0
|grep "inet
addr" |awk {'print
$2'}
|awk -F ':' {'print
$2'}`
S3_DIR="s3://***-logs/${ENV}/openresty"
LOG_FILE="/opt/openresty/log/nginx"
#压缩后缀名
BAK_SUFFIX=".gz"
#判断日志备份目录是否存在
for logs_bak_dir in `ls ${LOG_FILE}
-d`;
do
if [
-d ${logs_bak_dir}/logs_bak ];then
echo "dir
exist!"
else
mkdir -p
${logs_bak_dir}/logs_bak
fi
done
#备份日志主要逻辑
for logfile in `find ${LOG_FILE}
|grep "log$"`;
do
#强制cp并保留属性
/bin/cp -af
${logfile} ${logfile}.${TIME}
echo -n "" >
${logfile}
#压缩日志文件并保存到指定目录
gzip -c
${logfile}.${TIME} > ${logfile}+${TIME}+${BAK_SUFFIX}
mv ${logfile}+${TIME}+${BAK_SUFFIX}
${logfile%/*}/logs_bak/
done
#重启服务
docker exec openresty /usr/local/openresty/nginx/sbin/nginx -c /opt/openresty/nginx/nginx.conf
-s reopen
#s3上传并删除指定日期文件
aws
s3 sync ${LOG_FILE}/logs_bak/ ${S3_DIR}/${IP}
--exclude "*" --include "*${TIME}+${BAK_SUFFIX}"
aws
s3 rm ${S3_DIR}/${IP}
--recursive --exclude "*" --include "*`date
-d "-90
days"
+%Y%m%d`*${BAK_SUFFIX}"
#删除30天前的日志
find ${LOG_FILE}/
-name "*${BAK_SUFFIX}" -mtime
+30 -delete
find ${LOG_FILE}/
-name "*`date
-d "-3
days"
+%Y%m%d`*" |xargs rm -rf
Crontab
#59
23 * * * /bin/sh /opt/backup/nginx_log_split.sh >/dev/null 2>&1
#59
23 * * * /bin/sh /opt/backup/tomcat_log_split.sh >/dev/null 2>&1
#59
23 * * * /bin/sh /opt/backup/service_log_split.sh >/dev/null 2>&1
#59
23 * * * /bin/sh /opt/backup/mongo_log_split.sh >/tmp/mongo.log 2>&1
评论暂时关闭