一个支持正则表达式的日志关键字统计的shell工具
一个支持正则表达式的日志关键字统计的shell工具
一个支持正则表达式的日志关键字统计的shell工具
Usage: pafa_mon/statistics_pafa.sh -i|--instance <instance name> [-fn|--filename <file name>] [-ip|--instancepath] <instance path> [-kw|--keywords] <key words> [-bt|--begintime] <begin time> [-et|--endtime] <end time> [-t|--type] <progress type>
instance name: 实例英文名称
instance path: 实例所在路径，默认为/wls/applogs/rtlog/
file name: 文件名称，默认为pafa.log
key words: 匹配关键字,如有多个关键字请用冒号隔开，传参不支持带空格的关键字
从文件读取支持带空格关键字，多个关键字关键字用换行符分隔，支持正则表达式
默认从文件"/instance path/instance name/pafa.key"读取，如：/wls/applogs/rtlog/TOA-CORESF6802/pafa.key
begin time: 开始时间，默认从文件开头匹配
end time: 结束时间，默认匹配到文件末尾
progress type: 操作类型: 默认为1.
1，指定文件范围内扫描关键字，打印内容；（最后100w行）
2，指定文件范围内扫描关键字，统计次数；（最后100w行）
3，指定时间范围内扫描关键字，打印内容；
4，指定时间范围内扫描关键字，统计次数；
for example :
./statistics_pafa.sh -i TOA-CORESF6801
./statistics_pafa.sh -i TOA-CORESF6801 -t 2
./statistics_pafa.sh -i TOA-CORESF6801 -t 2 -kw ERROR
./statistics_pafa.sh -i TOA-CORESF6801 -kw ERROR
./statistics_pafa.sh -i TOA-CORESF6801 -fn TOA-CORESF6801.acc -t 2 -kw ERROR
./statistics_pafa.sh -i TOA-CORESF6801 -t 2 -kw ERROR -ip /tmp/ -bt 08:00:00
./statistics_pafa.sh -i TOA-CORESF6801 -t 2 -kw ERROR -bt 08:00:00 -et 18:00:00
./statistics_pafa.sh -i TOA-CORESF6801 -ip /wls/applogs/rtlog/ -fn pafa.log -kw ERROR:WARN -bt 08:00:00 -et 18:00:00 --type 2
#!/bin/bash
#
# File Name: statistics.sh
#
# Function: statistics the keywords from the specified log-file.
#
# Written by: stone
#
# Written on: 2013-01-16
#
#######################################################################
## default values for the variables below ##
HOST_NAME=$(hostname)
HOST_IP=$(hostname -i)
INSTANCE_NAME=''
INSTANCE_PATH="/wls/applogs/rtlog/"
FILE_NAME='pafa.log'
KEY_WORDS=''
MONTH=$(date +%m)
DAY=$(date +%e)
BEGIN_TIME=''
END_TIME=''
PROGRESS_TYPE=1
########################################################################
#### function for usage of the shell script ############
function Usage(){
echo "Usage: $0 -i|--instance <instance name> [-fn|--filename <file name>] [-ip|--instancepath] <instance path> [-kw|--keywords] <key words> [-bt|--begintime] <begin time> [-et|--endtime] <end time> [-t|--type] <progress type> "
echo "instance name: 实例英文名称"
echo "instance path: 实例所在路径，默认为/wls/applogs/rtlog/"
echo "file name: 文件名称，默认为pafa.log "
echo 'key words: 匹配关键字,如有多个关键字请用冒号隔开，传参不支持带空格的关键字'
echo ' 从文件读取支持带空格关键字，多个关键字关键字用换行符分隔'
echo ' 默认从文件"/instance path/instance name/pafa.key"读取，如：/wls/applogs/rtlog/TOA-CORESF6802/pafa.key'
echo "begin time: 开始时间，默认从文件开头匹配 "
echo "end time: 结束时间，默认匹配到文件末尾 "
echo 'progress type: 操作类型: 默认为1. '
echo ' 1，指定文件范围内扫描关键字，打印内容；（最后100w行）'
echo ' 2，指定文件范围内扫描关键字，统计次数；（最后100w行）'
echo ' 3，指定时间范围内扫描关键字，打印内容；'
echo ' 4，指定时间范围内扫描关键字，统计次数；'
echo 'for example :'
echo ' ./statistics_pafa.sh -i TOA-CORESF6801 '
echo ' ./statistics_pafa.sh -i TOA-CORESF6801 -t 2 '
echo ' ./statistics_pafa.sh -i TOA-CORESF6801 -t 2 -kw ERROR'
echo ' ./statistics_pafa.sh -i TOA-CORESF6801 -kw ERROR'
echo ' ./statistics_pafa.sh -i TOA-CORESF6801 -fn TOA-CORESF6801.acc -t 2 -kw ERROR'
echo ' ./statistics_pafa.sh -i TOA-CORESF6801 -t 2 -kw ERROR -ip /tmp/ -bt 08:00:00'
echo ' ./statistics_pafa.sh -i TOA-CORESF6801 -t 2 -kw ERROR -bt 08:00:00 -et 18:00:00'
echo ' ./statistics_pafa.sh -i TOA-CORESF6801 -ip /wls/applogs/rtlog/ -fn pafa.log -kw ERROR:WARN -bt 08:00:00 -et 18:00:00 --type 2'
exit 1
}
#### function for setting variables for the shell script #############
function Set_variables(){
if [[ $# == 0 ]]; then
if [[ -n $INSTANCE_NAME ]];then
FILE_PATH="${INSTANCE_PATH}/${INSTANCE_NAME}/"
Choose
exit 0
else
Usage
fi
fi
case "$1" in
"-i") shift
INSTANCE_NAME=$1
shift
Set_variables $@
;;
"--instance") shift
INSTANCE_NAME=$1
shift
Set_variables $@
;;
"-fn") shift
FILE_NAME=$1
shift
Set_variables $@
;;
"--filename") shift
FILE_NAME=$1
shift
Set_variables $@
;;
"-ip") shift
INSTANCE_PATH=$1
shift
Set_variables $@
;;
"--instancepath") shift
INSTANCE_PATH=$1
shift
Set_variables $@
;;
"-kw") shift
KEY_WORDS=$1
shift
Set_variables $@
;;
"--keywords") shift
KEY_WORDS=$1
shift
Set_variables $@
;;
"-bt") shift
BEGIN_TIME=$1
shift
Set_variables $@
;;
"--begintime") shift
BEGIN_TIME=$1
shift
Set_variables $@
;;
"-et") shift
END_TIME=$1
shift
Set_variables $@
;;
"--endtime") shift
END_TIME=$1
shift
Set_variables $@
;;
"-t") shift
PROGRESS_TYPE=$1
shift
Set_variables $@
;;
"--type") shift
PROGRESS_TYPE=$1
shift
Set_variables $@
;;
*) Usage
esac
}
#### function for show the value of the shell script's variables ############
function Show_variables(){
echo HOST_NAME=$HOST_NAME
echo HOST_IP=$HOST_IP
echo INSTANCE_NAME=$INSTANCE_NAME
echo FILE_PATH=$FILE_PATH
echo INSTANCE_PATH=$INSTANCE_PATH
echo FILE_NAME=$FILE_NAME
echo KEY_WORDS=$KEY_WORDS
echo MONTH=$MONTH
echo DAY=$DAY
echo BEGIN_TIME=$BEGIN_TIME
echo END_TIME=$END_TIME
}
#### function for statistic the key-words in the specified file from the last 1 million lines ############
function Statistic_contents(){
tail -1000000 ${FILE_PATH}${FILE_NAME} |awk '/^'${MONTH}'\/'${DAY}'/ && /'"${KEY}"'/ {OFS=""; print "KeyWord=","'"${KEY}:"'",$0} ' | tail -n 10
}
#### function for statistic the key-words arise times in the specified file from the last 1 million lines ############
function Statistic_times(){
tail -1000000 ${FILE_PATH}${FILE_NAME} |awk '/^'${MONTH}'\/'${DAY}'/ && /'"${KEY}"'/ {a++} END{OFS=""; print "KeyWord=","'"${KEY}:"'",a} '
}
#### function for statistic the key-words in the specified file by time ############
function Statistic_contents_bytime(){
if [[ -z $BEGIN_TIME ]];then
if [[ -z $END_TIME ]];then
awk '/^'${MONTH}'\/'${DAY}'/ && /'"${KEY}"'/ {OFS=""; print "KeyWord=","'"${KEY}:"'",$0} ' ${FILE_PATH}${FILE_NAME} | tail -n 10
else
awk '/^'${MONTH}'\/'${DAY}'/ && $2<"'$END_TIME'" && /'"${KEY}"'/ {OFS=""; print "KeyWord=","'"${KEY}:"'",$0}' ${FILE_PATH}${FILE_NAME} | tail -n 10
fi
else
if [[ -z $END_TIME ]];then
awk '/^'${MONTH}'\/'${DAY}'/ && $2>"'$BEGIN_TIME'" && /'"${KEY}"'/ {OFS=""; print "KeyWord=","'"${KEY}:"'",$0}' ${FILE_PATH}${FILE_NAME} | tail -n 10
else
awk '/^'${MONTH}'\/'${DAY}'/ && $2>"'$BEGIN_TIME'" && $2<"'$END_TIME'" && /'"${KEY}"'/ {OFS=""; print "KeyWord=","'"${KEY}:"'",$0}' ${FILE_PATH}${FILE_NAME} | tail -n 10
fi
fi
}
#### function for statistic the key-words arise times in the specified file by time ############
function Statistic_times_bytime(){
if [[ -z $BEGIN_TIME ]];then
if [[ -z $END_TIME ]];then
awk '/^'${MONTH}'\/'${DAY}'/ && /'"${KEY}"'/ {a++} END{OFS=""; print "KeyWord=","'"${KEY}:"'",a}' ${FILE_PATH}${FILE_NAME}
else
awk '/^'${MONTH}'\/'${DAY}'/ && $2<"'$END_TIME'" && /'"${KEY}"'/ {a++} END{OFS=""; print "KeyWord=","'"${KEY}:"'",a}' ${FILE_PATH}${FILE_NAME}
fi
else
if [[ -z $END_TIME ]];then
awk '/^'${MONTH}'\/'${DAY}'/ && $2>"'$BEGIN_TIME'" && /'"${KEY}"'/ {a++} END{OFS=""; print "KeyWord=","'"${KEY}:"'",a}' ${FILE_PATH}${FILE_NAME}
else
awk '/^'${MONTH}'\/'${DAY}'/ && $2>"'$BEGIN_TIME'" && $2<"'$END_TIME'" && /'"${KEY}"'/ {a++} END{OFS=""; print "KeyWord=","'"${KEY}:"'",a}' ${FILE_PATH}${FILE_NAME}
fi
fi
}
#### function for progressing for statistic ##########################
function Progress(){
if [[ -z $KEY_WORDS ]];then
if [[ -f ${FILE_PATH}/pafa.key ]];then
grep -v "^#" ${FILE_PATH}/pafa.key | while read KEY
do
$1
done
else
echo "Attention please !"
echo "You should create the key-word file named pafa.key in the directory ${FILE_PATH} at first !"
exit 1
fi
else
Key_Words="$(echo ${KEY_WORDS} |sed s/':'/' '/g )"
for KEY in $Key_Words
do
$1
done
fi
}
#### function for choose progress type for statistic ###############
function Choose(){
case "$PROGRESS_TYPE" in
"1") Progress "Statistic_contents"
;;
"2") Progress "Statistic_times"
;;
"3") Progress "Statistic_contents_bytime"
;;
"4") Progress "Statistic_times_bytime"
;;
*) Usage
;;
esac
}
#### function for the start of the shell script #################
function Main(){
Set_variables $@
}
Main $@
评论暂时关闭