Merge fb986a9779 into 7b7fcbe078
This commit is contained in:
Михаил Че (Mikhail Che)
GitHub
commit
22becb895c
|
|
@ -0,0 +1,73 @@
|
|||
Ip_blacklist - counter and locker for IPs by limiting requests per time range.
|
||||
Locked IPs will get saved and may be fake content from Redis :) (or 403)
|
||||
Used the Redis storage only.
|
||||
Monitoring blacklist on redis-cli: lrange ip_blacklist 0 -1
|
||||
|
||||
Requires
|
||||
========
|
||||
|
||||
Requires openresty bundle http://openresty.org/ (tested on openresty/1.5.8.1)
|
||||
or ngx-lua module http://wiki.nginx.org/HttpLuaModule#Installation
|
||||
and lua-resty-redis from: https://github.com/agentzh/lua-resty-redis
|
||||
|
||||
|
||||
Configure
|
||||
=========
|
||||
|
||||
Change settings on ip_blacklist_conf.lua (see options desciption in file):
|
||||
|
||||
Redis: socket (or host, port), timeout, pool keepalive
|
||||
Mode: <whatever for full mode>|count|forbidden|nocapture
|
||||
Time range: seconds of timeline division
|
||||
Limit of requests on time range
|
||||
Time lock: seconds for lock when IPs exceeded limit (prolong also if continue)
|
||||
Debug: <bool> for log INFO messages into nginx.log
|
||||
Log lock: <bool> persistent log list when locking IPs
|
||||
|
||||
Replace the system distribution nginx with local installation of openresty
|
||||
==========================================================================
|
||||
|
||||
Install latest openresty on default location usr/local/openresty.
|
||||
|
||||
# service nginx stop
|
||||
# or
|
||||
# /etc/init.d/nginx stop
|
||||
|
||||
mv /usr/local/openresty/nginx/conf /usr/local/openresty/nginx/conf_install
|
||||
ln -s /etc/nginx /usr/local/openresty/nginx/conf
|
||||
mv /usr/sbin/nginx /usr/sbin/nginx-back
|
||||
ln -s /usr/local/openresty/nginx/sbin/nginx /usr/sbin/nginx
|
||||
|
||||
mkdir /etc/nginx/lua
|
||||
cp ip_blacklist_conf.lua /etc/nginx/lua/
|
||||
cp ip_blacklist.lua /etc/nginx/lua/
|
||||
|
||||
Configure NGINX
|
||||
===============
|
||||
|
||||
Add one line to http section:
|
||||
lua_package_path "/etc/nginx/lua/?.lua;;";
|
||||
|
||||
Add one line per need location section:
|
||||
access_by_lua_file '/etc/nginx/lua/ip_blacklist.lua';
|
||||
|
||||
...
|
||||
http {
|
||||
...
|
||||
# you do not need the following line if you are using the ngx_openresty bundle:
|
||||
#lua_package_path "/path/to/lua-resty-redis/lib/?.lua;;";
|
||||
lua_package_path "/etc/nginx/lua/?.lua;;";
|
||||
...
|
||||
server {
|
||||
...
|
||||
location <...> {
|
||||
access_by_lua_file '/etc/nginx/lua/ip_blacklist.lua';
|
||||
proxy_pass | uwsgi_pass | ...
|
||||
...
|
||||
}
|
||||
...
|
||||
}
|
||||
}
|
||||
|
||||
# service nginx start
|
||||
# /etc/init.d/nginx start
|
||||
|
|
@ -0,0 +1,147 @@
|
|||
--[[
|
||||
|
||||
¡ ¡ ¡ ALL GLORY TO GLORIA ! ! !
|
||||
===============================
|
||||
Settings on ip_tables_conf.lua
|
||||
|
||||
--]]
|
||||
|
||||
local _M = require "ip_blacklist_conf"
|
||||
local log_info = _M.log_info
|
||||
local log_err = _M.log_err
|
||||
local redis = _M.redis_connect()
|
||||
local redis_exists = _M.redis_exists
|
||||
local redis_expire = _M.redis_expire
|
||||
local redis_incr = _M.redis_incr
|
||||
local redis_end = _M.redis_end
|
||||
local key_count, key_lock, key_doc, time_range = _M.redis_keys()
|
||||
|
||||
|
||||
if redis_exists(redis, key_lock) then -- is IP locked?
|
||||
redis_expire(redis, key_lock, _M.time_lock) -- prolong the lock!
|
||||
|
||||
if _M.mode == 'forbidden' then
|
||||
redis_end(redis)
|
||||
ngx.exit(ngx.HTTP_FORBIDDEN)
|
||||
end
|
||||
|
||||
if redis_exists(redis, key_doc) then --~ found content ?
|
||||
local hash, err = redis:hmget(key_doc, "status", "content_type", "content_length", "body")
|
||||
if err then
|
||||
log_err(string.format("redis:hmget failed for key=[%s]: %s", key_doc, err))
|
||||
redis_end(redis)
|
||||
ngx.exit(ngx.OK)
|
||||
elseif not hash[1] then -- not status
|
||||
redis:del(key_doc)
|
||||
redis_end(redis)
|
||||
ngx.exit(ngx.OK)
|
||||
end
|
||||
|
||||
log_info(string.format("locked request content from redis:[hmget '%s']", key_doc))
|
||||
--~ for key,value in pairs(hash) do print(key,value) end
|
||||
ngx.status = hash[1] --"status"
|
||||
ngx.header.content_type = hash[2] -- "content_type"
|
||||
ngx.header.content_length = hash[3] or string.len(hash[4]) --"content_length"
|
||||
ngx.say(hash[4]) -- "body"
|
||||
redis_end(redis)
|
||||
ngx.exit(hash[1]) --"status"
|
||||
else
|
||||
log_info(string.format("locked request content not found on redis:[hmget '%s']", key_doc))
|
||||
redis_end(redis)
|
||||
ngx.exit(ngx.HTTP_FORBIDDEN)
|
||||
end
|
||||
end
|
||||
|
||||
--~ else -- counter block
|
||||
local count = redis_incr(redis, key_count)
|
||||
|
||||
if count == 1 then
|
||||
redis_expire(redis, key_count, _M.time_range - time_range + 1)
|
||||
end
|
||||
|
||||
|
||||
if count < _M.req_limit then
|
||||
log_info(string.format("allow request for key_count=[%s] count=[%s] ", key_count, count))
|
||||
redis_end(redis)
|
||||
ngx.exit(ngx.OK)
|
||||
end
|
||||
|
||||
log_info(string.format("locked request for key_count=[%s] count=[%s] ", key_count, count))
|
||||
-- redis-cli lrange <_M.prefix> 0 -1
|
||||
if _M.log_lock then
|
||||
local _, err = redis:rpush(_M.prefix, key_count)
|
||||
if err then log_err(string.format("[redis:rpush] failed for key=[%s] and value[%s]: %s", _M.prefix, key_count, err)) end
|
||||
log_info(string.format("logging the lock for key_count=[%s] ", key_count))
|
||||
end
|
||||
|
||||
if _M.mode == 'count' then
|
||||
log_info(string.format("mode count exit for key_count=[%s] count=[%s] ", key_count, count))
|
||||
redis_end(redis)
|
||||
ngx.exit(ngx.OK)
|
||||
end
|
||||
|
||||
redis_incr(redis, key_lock) -- add IP to lock
|
||||
redis_expire(redis, key_lock, _M.time_lock) -- for locking time
|
||||
--~ end -- counter block
|
||||
|
||||
|
||||
if _M.mode == 'nocapture' then
|
||||
log_info(string.format("mode nocapture exit for key_lock=[%s] ", key_lock))
|
||||
redis_end(redis)
|
||||
ngx.exit(ngx.OK)
|
||||
end
|
||||
|
||||
|
||||
|
||||
if redis_exists(redis, key_doc) then -- doc in Redis already
|
||||
log_info(string.format("content already exists in [redis:hgetall '%s']", key_doc))
|
||||
redis_end(redis)
|
||||
ngx.exit(ngx.OK)
|
||||
end
|
||||
|
||||
--[[
|
||||
Prepare store content for next locked requests
|
||||
The problem:
|
||||
location.capture and chunked content from Apache
|
||||
--]]
|
||||
|
||||
ngx.req.read_body() --You should always read the request body (by either calling ngx.req.read_body or configuring lua_need_request_body on) before initiating a subrequest.
|
||||
|
||||
log_info("location.capture=" .. ngx.var.uri)
|
||||
local cap = ngx.location.capture(ngx.var.uri)
|
||||
if cap.status >= 500 then
|
||||
log_info("bad location.capture 50X " .. ngx.var.uri)
|
||||
redis_end(redis)
|
||||
ngx.exit(cap.status)
|
||||
end
|
||||
|
||||
--~ for k, v in pairs(cap.header) do print(k,'=', v) end
|
||||
if not (cap.status == ngx.HTTP_OK and cap.header['Content-Type'] and (string.match(cap.header['Content-Type'], 'text') or string.match(cap.header['Content-Type'], 'json'))) then
|
||||
log_info(string.format("skip cache non HTTP_OK [%s]; non text content type [%s]", cap.status, cap.header['Content-Type']))
|
||||
ngx.status = cap.status
|
||||
for k, v in pairs(cap.header) do ngx.header[k] = v end
|
||||
ngx.say(cap.body)
|
||||
redis_end(redis)
|
||||
ngx.exit(cap.status)
|
||||
end
|
||||
|
||||
--~ local body = string.gsub(cap.body, "<body>", "<body><h1 style='color:red;'>Внимание! Вы находитесь в зоне выполнения регламентного обновления сайта. Пожалуйста, обновите страницу позже. Спасибо.</h1>")
|
||||
local body = string.gsub(cap.body, "<body>", "<body><h1 style='color:red;'>WARN! Your request is locked!</h1>")
|
||||
local _, err = redis:hmset(
|
||||
key_doc,
|
||||
"status", cap.status,
|
||||
"content_type", cap.header['Content-Type'],
|
||||
"content_length", cap.header['Content-Length'] or string.len(body),
|
||||
"body", body
|
||||
) -- "content_length", cap.header['Content-Length']
|
||||
if err then
|
||||
log_err(string.format("[redis:hmset] failed for key=[%s]: %s", key_doc, err))
|
||||
else
|
||||
log_info(string.format("content [%s%s] stored in redis:[hmset '%s'] for next locked requests", ngx.var.host, ngx.var.uri, key_doc))
|
||||
end
|
||||
redis_end(redis)
|
||||
ngx.exit(ngx.OK)
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,151 @@
|
|||
--[[
|
||||
|
||||
¡ ¡ ¡ ALL GLORY TO GLORIA ! ! !
|
||||
===============================
|
||||
|
||||
|
||||
--]]
|
||||
|
||||
|
||||
|
||||
local _M = {
|
||||
author='Mikhail Che <mche@cpan.org>',
|
||||
version='0.14',
|
||||
redis_sock = 'unix:/tmp/redis.sock', -- redis.conf: unixsocketperm 777!!!
|
||||
--~ or
|
||||
--~ redis_host = "127.0.0.1",
|
||||
--~ redis_port = 6379,
|
||||
redis_timeout = 200,-- connection timeout for redis in ms. don't set this too high!
|
||||
redis_poolsize = 15, -- maximal size of the pool. Puts the current Redis connection immediately into the ngx_lua cosocket connection pool.
|
||||
redis_keepalive = 0, -- ms. (0 - persistent) You can specify the max idle timeout (in ms) when the connection is in the pool and the maximal size of the pool every nginx worker process.
|
||||
prefix = "ip_blacklist", -- for Redis keys and for log strings
|
||||
key_delim = ':', -- for join Redis keys
|
||||
--[[
|
||||
There are four types of Redis keys:
|
||||
1. Counter: <prefix>:count:<time range>:<ip>:<uri>
|
||||
2. Locker: <prefix>:lock:<ip>:<uri>
|
||||
3. Saved content: <prefix>:content:<host>:<uri>
|
||||
4. Log of locking: <prefix>
|
||||
--]]
|
||||
time_range = 3, -- seconds of timeline sewgments разбивает временную линию на отрезки секунд и использует для ключа
|
||||
req_limit = 20, -- max count of requests on time_range столько раз может обратиться IP в текущем временном отрезке
|
||||
time_lock = 60, -- seconds for locking IP. If IP still make request then prolong lock time (expire Redis key) секунд держать в ловушке, если продолжает дергаться - пролонгировать expire этого ключа
|
||||
mode='count',
|
||||
--[[
|
||||
nil or <whatever>- full mode: monitoring, locking and getout content from the Redis or capture content once and self save to Redis for next one locked IP requests
|
||||
'count' - without lock IPs on blacklist, i.e. monitoring only. To see log of bad IPs on redis-cli: lrange <prefix> 0 -1
|
||||
'forbidden' - monitoring + 403 page for locked IP
|
||||
'nocapture' - disable self save content, i.e. monitoring + content from the Redis only. If content not found in Redis then 403
|
||||
Note: chunked content when ngx.location.capture() bad?
|
||||
--]]
|
||||
log_lock = true, -- (false not) push locked IPs in persistent list <prefix> when count of requests exceeded limit (see by redis-cli: lrange <prefix> 0 -1)
|
||||
["debug"]=true,-- false disable ngx.INFO log to nginx.log (errors always enabled as ngx.ERR level)
|
||||
redis = require "resty.redis"
|
||||
}
|
||||
--------------------------------- END CONFIG -----------------------------------------------------------------
|
||||
|
||||
--=================== REDIS FUNCS =============================================
|
||||
_M.redis_connect = function ()
|
||||
local redis = _M.redis:new()
|
||||
if not redis then
|
||||
_M.log_err("Failed new redis")
|
||||
ngx.exit(ngx.OK)
|
||||
end
|
||||
redis:set_timeout(_M.redis_timeout)
|
||||
local connected, err
|
||||
if _M.redis_sock then connected, err = redis:connect(_M.redis_sock)
|
||||
elseif _M.redis_host and _M.redis_port then connected, err = redis:connect(_M.redis_host, _M.redis_port)
|
||||
end
|
||||
if not connected then
|
||||
_M.log_err(string.format("could not connect to redis host=[%s:%s] sock=[%s] : %s", _M.redis_host, _M.redis_port, _M.redis_sock, err))
|
||||
ngx.exit(ngx.OK)
|
||||
end
|
||||
--~ _M.log_info("success connect to redis")
|
||||
return redis
|
||||
end
|
||||
----------------------------------------------------------------------------------------------------------------------------
|
||||
_M.redis_expire = function (redis, key, time)
|
||||
local result, err = redis:expire(key, time)
|
||||
if not result then
|
||||
_M.log_err(string.format("[redis:expire] failed for key=[%s]: %s", key, err))
|
||||
_M.redis_end(redis)
|
||||
ngx.exit(ngx.OK)
|
||||
end
|
||||
end
|
||||
|
||||
-------------------------------------------------------------------------------------------------------------------------------
|
||||
_M.redis_exists = function (redis, key)
|
||||
local key_ex, err = redis:exists(key)
|
||||
if err then
|
||||
_M.log_err(string.format("[redis:exists] failed for key=[%s]: %s", key, err))
|
||||
_M.redis_end(redis)
|
||||
ngx.exit(ngx.OK)
|
||||
end
|
||||
--~ _M.log_info(string.format("key=[%s] exists", key))
|
||||
return key_ex == 1
|
||||
end
|
||||
|
||||
------------------------------------------------------------------------------------------------------------------------------
|
||||
_M.redis_incr = function (redis, key)
|
||||
local count, err = redis:incr(key)
|
||||
if err then
|
||||
_M.log_err(string.format("[redis:incr] failed for key=[%s]: %s", key, err))
|
||||
_M.redis_end(redis)
|
||||
ngx.exit(ngx.OK)
|
||||
end
|
||||
return count
|
||||
end
|
||||
|
||||
-----------------------------------------------------------------------------------------------------------------------------
|
||||
_M.redis_end = function (redis)
|
||||
--[[
|
||||
syntax: ok, err = red:set_keepalive(max_idle_timeout, pool_size)
|
||||
Puts the current Redis connection immediately into the ngx_lua cosocket connection pool.
|
||||
You can specify the max idle timeout (in ms) when the connection is in the pool and the maximal size of the pool every nginx worker process.
|
||||
In case of success, returns 1. In case of errors, returns nil with a string describing the error.
|
||||
Only call this method in the place you would have called the close method instead. Calling this method will immediately turn the current redis object into the closed state. Any subsequent operations other than connect() on the current objet will return the closed error.
|
||||
]]
|
||||
local ok, err = redis:set_keepalive(_M.redis_keepalive, _M.redis_poolsize)
|
||||
if not ok then _M.log_err(string.format("failed to set set_keepalive: %s", err)) end
|
||||
end
|
||||
|
||||
--======================= OTHER FUNCS =====================================
|
||||
_M.redis_keys = function ()
|
||||
-- counter <prefix>:count:<time range>:<ip>:<host>:<uri>
|
||||
local now = os.time()
|
||||
local time_range = now % _M.time_range
|
||||
local key_count = _M.join(_M.prefix, 'count', now - time_range, ngx.var.remote_addr, ngx.var.host, ngx.var.uri)
|
||||
-- content <prefix>:content:<host>:<uri>
|
||||
local key_content = _M.join(_M.prefix, 'content', ngx.var.host, ngx.var.uri)
|
||||
--locker <prefix>:lock:<ip>:<host>:<uri>
|
||||
local key_lock = _M.join(_M.prefix, 'lock', ngx.var.remote_addr, ngx.var.host, ngx.var.uri)
|
||||
return key_count, key_lock, key_content, time_range
|
||||
end
|
||||
|
||||
--------------------------------------------------------------------------------------------------------------------
|
||||
_M.log_info = function (msg)
|
||||
--~ if _M.debug then ngx.log(ngx.INFO, (string.format("%s: %s", _M.prefix, msg))) end
|
||||
if _M.debug then ngx.log(ngx.INFO, msg) end
|
||||
end
|
||||
_M.log = _M.log_info
|
||||
--------------------------------------------------------------------------------------------------------------------
|
||||
_M.log_err = function (msg)
|
||||
--~ ngx.log(ngx.ERR, (string.format("%s: %s", _M.prefix, msg)))
|
||||
ngx.log(ngx.ERR, msg)
|
||||
end
|
||||
--------------------------------------------------------------------------------------------------------------------
|
||||
_M.join = function (...) -- a,b,c
|
||||
local t = {}
|
||||
--~ for _,v in pairs(list) do
|
||||
--~ for i = 1, arg.n do -- bad
|
||||
--~ t[#t+1] = tostring(arg[i]) -- v
|
||||
for i=1, select("#",...) do
|
||||
t[#t+1] = (select(i,...))
|
||||
end
|
||||
return table.concat(t,_M.key_delim)
|
||||
end
|
||||
---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
return _M
|
||||
|
||||
|
||||
Loading…
Reference in New Issue