添加文件黑名单和记录蜘蛛

Author: wenjun1055

config.lua

@@ -66,7 +66,7 @@ local Config = {
 	whiteTime = 600,
 
 	-- 用于生成token密码的key过期时间
-	keyExpire = 600,
+	keyExpire = 0,
 
 	-- 匹配url模式，可选值requestUri,uri
 	-- 值requestUri时,url-protect目录下的正则匹配的是浏览器最初请求的地址且没有被decode,带参数的链接
@@ -80,7 +80,10 @@ local Config = {
 	reCaptchaPage = baseDir.."html/reCatchaPage.html",
 
 	-- 白名单ip文件,文件内容为正则表达式。
-	whiteIpModules = { state = "Off", ipList = baseDir.."url-protect/white_ip_list.txt" },
+	whiteIpModules = { state = "On", ipList = baseDir.."url-protect/white_ip_list.txt" },
+
+	-- 黑名单ip文件,文件内容为正则表达式。
+	blackIpModules = { state = "On", ipList = baseDir.."url-protect/black_ip_list.txt" },
 
 	-- 如果需要从请求头获取真实ip,此值就需要设置,如x-forwarded-for
 	-- 当state为on时,此设置才有效

guard.lua

@@ -49,6 +49,32 @@ function Guard:ipInWhiteList(ip)
 	end
 end
 
+function Guard:ipInFileBlackList(ip)
+	if _Conf.fileBlackIpModulesIsOn then
+		self:debug("[IpInFileBlackList] fileBlackIpModules is on.",ip,"")
+
+		if ngx.re.match(ip, _Conf.fileBlackIpList) then --匹配黑名单列表
+			self:debug("[ipInFileBlackList] ip "..ip.. " match black list ".._Conf.fileBlackIpList,ip,"")
+			return true
+		else
+			return false
+		end	
+	end
+end
+
+
+--收集不在白名单中的蜘蛛ip
+function Guard:collectSpiderIp(ip, headers)
+	spiderPattern = "baiduspider|360spider|sogou web spider|sogou inst spider|mediapartners|adsbot-google|googlebot"
+	userAgent = string.lower(headers["user-agent"])
+	if ngx.re.match(userAgent, spiderPattern) then 
+		local filename = _Conf.logPath.."/spider_ip.log"
+		local file = io.open(filename, "a+")
+		file:write(os.date('%Y-%m-%d %H:%M:%S').." IP "..ip.." UA "..userAgent.."\n")
+		file:close()
+	end
+end
+
 --黑名单模块
 function Guard:blackListModules(ip,reqUri)
 	local blackKey = ip.."black"

init.lua

@@ -139,6 +139,7 @@ _Conf = {
 	--解析开关设置
 	limitReqModulesIsOn = optionIsOn(Config.limitReqModules.state),
 	whiteIpModulesIsOn = optionIsOn(Config.whiteIpModules.state),
+	fileBlackIpModulesIsOn = optionIsOn(Config.blackIpModules.state)
 	realIpFromHeaderIsOn = optionIsOn(Config.realIpFromHeader.state),
 	autoEnableIsOn = optionIsOn(Config.autoEnable.state),
 	redirectModulesIsOn = optionIsOn(Config.redirectModules.state),
@@ -151,6 +152,7 @@ _Conf = {
 	limitUrlProtect = parseRuleFile(Config.limitReqModules.urlProtect),
 	cookieUrlProtect = parseRuleFile(Config.cookieModules.urlProtect),
 	whiteIpList = parseRuleFile(Config.whiteIpModules.ipList),
+	fileBlackIpList = parseRuleFile(Config.blackIpModules.ipList),
 
 	--读取文件到内存
 	captchaPage = readFile2Mem(Config.captchaPage),

runtime.lua

@@ -28,9 +28,18 @@ else
 	if _Conf.autoEnableIsOn then
 		ngx.timer.at(0,Guard.autoSwitch)
 	end
+
+	--永久黑名单
+	if Guard:ipInFileBlackList(ip) then
+		ngx.exit(404)
+	end
+
 
 	--白名单模块
 	if not Guard:ipInWhiteList(ip) then
+		--收集不在白名单库里面的蜘蛛
+		Guard:collectSpiderIp(ip, headers)
+
 		--黑名单模块
 		Guard:blackListModules(ip,reqUri)
 

url-protect/black_ip_list.txt

@@ -0,0 +1,15 @@
+61.135.186.*
+61.155.149.*
+61.182.137.*
+117.27.149.*
+117.34.28.*
+119.188.132.*
+119.188.14.*
+119.63.193.*
+123.125.71.*
+180.76.5.*
+180.76.6.*
+183.60.235.*
+185.10.104.*
+220.181.108.*
+222.216.190.*