Linux入门教程:OpenResty(Nginx Lua)统计网站访问信息,网页访问统计但之前的统计方法没有
背景
之前的一篇文章openresty(nginx lua)统计域名状态码、平均响应时间和流量实现了对域名状态码，平均响应时间和流量的统计。但之前的统计方法没有实现当某一域名404或500等状态码超过一定数量后发送具体的url来快速定位位置。这个功能我们其实是通过统计网站日志来实现了。为了摆脱对网站日志的依赖以及提高统计性能，我们尝试把此功能也用nginx lua来实现。具体的使用方法与之前的文章一样，这里只是更新了两个lua脚本。
使用方法
1、获取域名www.centos.bz 404状态码数量
curl -s "localhost/domain_status?count=status&host=www.centos.bz&status=404"
输出:
10 688
第一列为状态码数量，第二列为域名请求总数
2、获取当域名www.centos.bz 404状态码超过50个时,输出前10个url
curl -s "localhost/domain_status?count=statusUrl&host=www.centos.bz&status=404&exceed=50&output=10"
输出:
/hello-world 90
/centos 10
第一列为url，第二列为url请求次数。
3、获取域名www.centos.bz upstream一分钟内平均耗时
curl -s "localhost/domain_status?count=upT&host=www.centos.bz"
输出:
0.02 452
第一列为upstream平均耗时，第二列为域名总请求次数。
4、获取当域名www.centos.bz upstream平均耗时超过0.5秒时,输出其url
curl -s "localhost/domain_status?count=upTUrl&host=www.centos.bz&exceed=0.5"
输出:
/hello.php 0.82 52
第一列为url，第二列为此url平均耗时,第三列为此url请求次数。监控此接口数据可以快速定位出具体哪些url慢了。
5、获取域名www.centos.bz request time平均耗时
curl -s "localhost/domain_status?count=reqT&host=www.centos.bz"
输出:
1.82 52
第一列为平均耗时，第二列为域名请求数。request time是指完成整个请求所需要的时间（包括把数据传输到用户浏览器的时间）。对于php请求，upstream time指的是nginx把php请求传给fastcgi到完成数据接收所需时间。所以request time永远大于upstream time。
6、获取域名www.centos.bz占用的带宽(单位:字节/秒)
curl -s "localhost/domain_status?count=flow&host=www.centos.bz"
输出:
1024 52
第一列为此域名一分钟内平均传输速率，单位为字节/秒，第二列为域名请求总数。
相关脚本
log_acesss.lua
local access = ngx.shared.access
local host = ngx.var.host or "unknow"
local status = ngx.var.status
local body_bytes_sent = ngx.var.body_bytes_sent
local request_time = ngx.var.request_time
local upstream_response_time = ngx.var.upstream_response_time or 0
local request_uri = ngx.var.request_uri or "/unknow"
local timestamp = ngx.time()
local expire_time = 70
local status_key = table.concat({host,"-",status,"-",timestamp})
local flow_key = table.concat({host,"-flow-",timestamp})
local req_time_key = table.concat({host,"-reqt-",timestamp})
local up_time_key = table.concat({host,"-upt-",timestamp})
local total_key = table.concat({host,"-total-",timestamp})
-- 域名总请求数
local n,e = access:incr(total_key,1)
if not n then
access:set(total_key, 1, expire_time)
end
-- 域名状态码请求数
local n,e = access:incr(status_key,1)
if not n then
access:set(status_key, 1, expire_time)
end
-- 域名流量
local n,e = access:incr(flow_key,body_bytes_sent)
if not n then
access:set(flow_key, body_bytes_sent, expire_time)
end
-- 域名请求耗时
local n,e = access:incr(req_time_key,request_time)
if not n then
access:set(req_time_key, request_time, expire_time)
end
-- 域名upstream耗时
local n,e = access:incr(up_time_key,upstream_response_time)
if not n then
access:set(up_time_key, upstream_response_time, expire_time)
end
-- 获取不带参数的uri
local m, err = ngx.re.match(request_uri, "(.*?)\\?")
local request_without_args = m and m[1] or request_uri
-- 存储状态码大于400的url
if tonumber(status) >= 400 then
-- 拼接url,状态码,字节数等字段
local request_log_t = {}
table.insert(request_log_t,host)
table.insert(request_log_t,request_without_args)
table.insert(request_log_t,status)
local request_log = table.concat(request_log_t," ")
-- 把拼接的字段储存在字典中
local log_key = table.concat({"status-",timestamp})
local request_log_dict = access:get(log_key) or ""
if request_log_dict == "" then
request_log_dict = request_log
else
request_log_dict = table.concat({request_log_dict,"\n",request_log})
end
access:set(log_key, request_log_dict, expire_time)
end
-- 存储upstream time大于0.5的url
if tonumber(upstream_response_time) > 0.5 then
-- 拼接url,状态码,字节数等字段
local request_log_t = {}
table.insert(request_log_t,host)
table.insert(request_log_t,request_without_args)
table.insert(request_log_t,upstream_response_time)
local request_log = table.concat(request_log_t," ")
-- 把拼接的字段储存在字典中
local log_key = table.concat({"upt-",timestamp})
local request_log_dict = access:get(log_key) or ""
if request_log_dict == "" then
request_log_dict = request_log
else
request_log_dict = table.concat({request_log_dict,"\n",request_log})
end
access:set(log_key, request_log_dict, expire_time)
end
domain_status.lua
-- 各参数用法:
-- count=status,host=xxx.com,status=404 统计xxx.com域名一分钟内404状态码个数.
-- count=statusUrl,host=xxx.com,status=404,exceed=50,output=30 当xxx.com域名404状态码一分钟内超过50个时,输出前30个url，否则返回空.
-- count=upT,host=xxx.com 统计xxx.com域名一分钟内平均upsteam耗时
-- count=upTUrl,host=xxx.com,exceed=0.5 输出upstreamTime超过0.5秒的url,没有就返回空
-- count=reqT,host=xxx.com 统计xxx.com域名一分钟内平均请求耗时
-- count=flow,host=xxx.com 统计xxx.com域名一分钟内流量(单位字节/秒)
-- 函数: 获取迭代器值
local get_field = function(iterator)
local m,err = iterator
if err then
ngx.log(ngx.ERR, "get_field iterator error: ", err)
ngx.exit(ngx.HTTP_OK)
end
return m[0]
end
-- 函数: 按值排序table
local getKeysSortedByValue = function (tbl, sortFunction)
local keys = {}
for key in pairs(tbl) do
table.insert(keys, key)
end
table.sort(keys, function(a, b)
return sortFunction(tbl[a], tbl[b])
end)
return keys
end
-- 函数: 判断table是否存在某元素
local tbl_contain = function(table,element)
for k in pairs(table) do
if k == element then
return true
end
end
return false
end
local access = ngx.shared.access
local now = ngx.time()
local one_minute_ago = now - 60
-- 获取参数
local args = ngx.req.get_uri_args()
local count_arg = args["count"]
local host_arg = args["host"]
local status_arg = args["status"]
local exceed_arg = args["exceed"]
local output_arg = args["output"]
local count_t = {["status"]=0,["statusUrl"]=0,["upT"]=0,["upTUrl"]=0,["reqT"]=0,["flow"]=0}
-- 检查参数是否满足
if not tbl_contain(count_t,count_arg) then
ngx.print("count arg invalid.")
ngx.exit(ngx.HTTP_OK)
end
if not host_arg then
ngx.print("host arg not found.")
ngx.exit(ngx.HTTP_OK)
end
if count_arg == "status" and not status_arg then
ngx.print("status arg not found.")
ngx.exit(ngx.HTTP_OK)
end
if count_arg == "statusUrl" and not (status_arg and exceed_arg and output_arg) then
ngx.print("status or exceed or output arg not found.")
ngx.exit(ngx.HTTP_OK)
end
if count_arg == "upTUrl" and not exceed_arg then
ngx.print("exceed arg not found.")
ngx.exit(ngx.HTTP_OK)
end
-- 检查参数是否合法
if status_arg and ngx.re.find(status_arg, "^[0-9]{3}$") == nil then
ngx.print("status arg must be a valid httpd code.")
ngx.exit(ngx.HTTP_OK)
end
if exceed_arg and ngx.re.find(exceed_arg, "^[0-9.]+$") == nil then
ngx.print("exceed arg must be a number.")
ngx.exit(ngx.HTTP_OK)
end
if output_arg and ngx.re.find(output_arg, "^[0-9]+$") == nil then
ngx.print("output arg must be a number.")
ngx.exit(ngx.HTTP_OK)
end
-- 开始统计
local url
local status_code
local upstream_time
local status_total = 0
local host
local req_total = 0
local flow_total = 0
local reqtime_total = 0
local upstream_total = 0
local status_url_t = {}
local upstream_url_t = {}
local upstream_url_count_t = {}
local status_log
local upt_log
for second_num=one_minute_ago,now do
local flow_key = table.concat({host_arg,"-flow-",second_num})
local req_time_key = table.concat({host_arg,"-reqt-",second_num})
local up_time_key = table.concat({host_arg,"-upt-",second_num})
local total_req_key = table.concat({host_arg,"-total-",second_num})
local log_key
local log_line
-- 合并状态码大于等于400的请求日志到变量status_log
log_key = table.concat({"status-",second_num})
log_line = access:get(log_key) or ""
if not (log_line == "") then
status_log = table.concat({log_line,"\n",status_log})
end
-- 合并upstream time大于0.5秒的请求日志到变量upt_log
log_key = table.concat({"upt-",second_num})
log_line = access:get(log_key) or ""
if not (log_line == "") then
upt_log = table.concat({log_line,"\n",upt_log})
end
-- 域名总请求数
local req_sum = access:get(total_req_key) or 0
req_total = req_total + req_sum
if count_arg == "status" or count_arg == "statusUrl" then
local status_key = table.concat({host_arg,"-",status_arg,"-",second_num})
local status_sum = access:get(status_key) or 0
status_total = status_total + status_sum
end
if count_arg == "flow" then
local flow_sum = access:get(flow_key) or 0
flow_total = flow_total + flow_sum
end
if count_arg == "reqT" then
local req_time_sum = access:get(req_time_key) or 0
reqtime_total = reqtime_total + req_time_sum
end
if count_arg == "upT" then
local up_time_sum = access:get(up_time_key) or 0
upstream_total = upstream_total + up_time_sum
end
end
-- 统计状态码url
if count_arg == "statusUrl" and status_log and not (status_log == "") then
local iterator, err = ngx.re.gmatch(status_log,".+\n")
if not iterator then
ngx.log(ngx.ERR, "status_log iterator error: ", err)
return
end
for line in iterator do
if not line[0] then
ngx.log(ngx.ERR, "line[0] is nil")
return
end
local iterator, err = ngx.re.gmatch(line[0],"[^ \n]+")
if not iterator then
ngx.log(ngx.ERR, "line[0] iterator error: ", err)
return
end
host = get_field(iterator())
url = get_field(iterator())
status_code = get_field(iterator())
if status_code == status_arg then
if status_url_t[url] then
status_url_t[url] = status_url_t[url] + 1
else
status_url_t[url] = 1
end
end
end
end
-- 统计upstream time大于0.5秒url
if count_arg == "upTUrl" and upt_log and not (upt_log == "") then
local iterator, err = ngx.re.gmatch(upt_log,".+\n")
if not iterator then
ngx.log(ngx.ERR, "upt_log iterator error: ", err)
return
end
for line in iterator do
if not line[0] then
ngx.log(ngx.ERR, "line[0] is nil")
return
end
local iterator, err = ngx.re.gmatch(line[0],"[^ \n]+")
if not iterator then
ngx.log(ngx.ERR, "line[0] iterator error: ", err)
return
end
host = get_field(iterator())
url = get_field(iterator())
upstream_time = get_field(iterator())
upstream_time = tonumber(upstream_time) or 0
-- 统计各url upstream平均耗时
if host == host_arg then
if upstream_url_t[url] then
upstream_url_t[url] = upstream_url_t[url] + upstream_time
else
upstream_url_t[url] = upstream_time
end
if upstream_url_count_t[url] then
upstream_url_count_t[url] = upstream_url_count_t[url] + 1
else
upstream_url_count_t[url] = 1
end
end
end
end
-- 输出结果
if count_arg == "status" then
ngx.print(status_total," ",req_total)
elseif count_arg == "flow" then
ngx.print(flow_total," ",req_total)
elseif count_arg == "reqT" then
local reqt_avg = 0
if req_total == 0 then
reqt_avg = 0
else
reqt_avg = reqtime_total/req_total
end
ngx.print(reqt_avg," ",req_total)
elseif count_arg == "upT" then
local upt_avg = 0
if req_total == 0 then
upt_avg = 0
else
upt_avg = upstream_total/req_total
end
ngx.print(upt_avg," ",req_total)
elseif count_arg == "statusUrl" then
if status_total > tonumber(exceed_arg) then
-- 排序table
status_url_t_key = getKeysSortedByValue(status_url_t, function(a, b) return a > b end)
local output_body = ""
for i, uri in ipairs(status_url_t_key) do
if output_body == "" then
output_body = table.concat({uri," ",status_url_t[uri]})
else
output_body = table.concat({output_body,"\n",uri," ",status_url_t[uri]})
end
if i >= tonumber(output_arg) then
ngx.print(output_body)
ngx.exit(ngx.HTTP_OK)
end
end
ngx.print(output_body)
ngx.exit(ngx.HTTP_OK)
end
elseif count_arg == "upTUrl" then
local max_output = 30
local total_time = 0
local total_count = 0
local output_body = ""
local i = 0
for url in pairs(upstream_url_t) do
i = i + 1
total_time = upstream_url_t[url]
total_count = upstream_url_count_t[url]
avg_time = upstream_url_t[url] / upstream_url_count_t[url]
if avg_time > tonumber(exceed_arg) then
output_body = table.concat({url," ",avg_time," ",total_count,"\n",output_body})
end
if i >= max_output then
ngx.print(output_body)
ngx.exit(ngx.HTTP_OK)
end
end
ngx.print(output_body)
ngx.exit(ngx.HTTP_OK)
end
评论暂时关闭