nginx屏蔽网站攻击采集垃圾蜘蛛的方法总结

nginx配置文件应对网站攻击采集垃圾蜘蛛的方法总结

#禁垃圾蜘蛛

if ($http_user_agent ~* "CheckMarkNetwork|Synapse|Bingbot|Googlebot|Nimbostratus-Bot|Dark|scraper|LMAO|Hakai|Gemini|Wappalyzer|masscan|crawler4j|Mappy|Center|eright|aiohttp|MauiBot|Crawler|researchscan|Dispatch|AlphaBot|Census|ips-agent|NetcraftSurveyAgent|ToutiaoSpider|EasyHttp|Iframely|sysscan|fasthttp|muhstik|DeuSu|mstshash|HTTP_Request|ExtLinksBot|package|SafeDNSBot|CPython|SiteExplorer|SSH|MegaIndex|BUbiNG|CCBot|NetTrack|Digincore|aiHitBot|SurdotlyBot|null|SemrushBot|Test|Copied|ltx71|Nmap|DotBot|AdsBot|InetURL|Pcore-HTTP|PocketParser|Wotbox|newspaper|DnyzBot|redback|PiplBot|SMTBot|WinHTTP|Auto Spider 1.0|GrabNet|TurnitinBot|Go-Ahead-Got-It|Download Demon|Go!Zilla|GetWeb!|GetRight|libwww-perl|Cliqzbot|MailChimp|SMTBot|Dataprovider|XoviBot|linkdexbot|SeznamBot|Qwantify|spbot|evc-batch|zgrab|Go-http-client|FeedDemon|JikeSpider|Indy Library|Alexa Toolbar|AskTbFXTV|AhrefsBot|CrawlDaddy|CoolpadWebkit|Java|UniversalFeedParser|ApacheBench|Microsoft URL Control|Swiftbot|ZmEu|jaunty|Python-urllib|lightDeckReports Bot|YYSpider|DigExt|HttpClient|MJ12bot|EasouSpider|LinkpadBot|Ezooms|YoudaoBot|YandexBot|Rogerbot|exabot|ia_archiver|Teoma|gigabot|DOCOMO Sprider|AhrefsBot|SemrushBot|Sosospider|Yahoo! Slurp China|Yahoo! Slurp|MSNBot|MSNot-media|FlightDeckReports Bot|Bytespider|Mail.RU_Bot") {

return 403;

break;

}

#禁攻击采集

if ($http_user_agent ~* "FeedDemon|BOT/0.1 (BOT for JCE)|CrawlDaddy|Java|Jullo|Feedly|UniversalFeedParser|ApacheBench|

复制代码

#过滤url参数

set $URL $request_uri;

if ($URL ~* "member|plus|base|data|dede|public|plug|Vote|tool|feed|components|skin|tinyMCE|version|sysimage|wp-content|wp-admin|static|common|face|shell|swfupload|utility|convert|sitemap|siteserver|BackupDB|file|user|system|upimg|install|wap|multiupload|ewebeditor|office|wallet|backup|bitcoin|maccms|vendor|apply|bjebhgm|photo|module|external|Analytics|tools|subdomains|notes|md5|ckeditor|bbs|ajax|zhuitanyun|logbaak|help|weki|dxyylc|Somnus|manage|J4H7eFjWoBa3bO6U|SiteFiles|dowds|source|ucenter|phpcms|language|TeatchClass|taglib|sql|allowurl|shitan|root|wp-login|houtai|admin001|htadmin|clock2|webadmin"){

return 403;

break;

}

复制代码

#禁特殊后缀

location ~* \.(asp|xml|jsp|aspx|dev|aspx|ewebeditor|sql|xsl|asmx|htaccess|ini|env|git|project|cgi|md5|ajax.js|swf|tpl.php)$ {

return 403;

break;

}

复制代码

#禁止非GET方式的抓取

if ($request_method !~ ^(GET)$) {

    return 403;

}

#禁止特殊请求方式

if ($request_method ~* "HEAD|DELETE|OPTIONS|POST" ) {

    return 403;

    break;

}

复制代码

#禁特殊请求工具

if ($http_user_agent ~* "Wget|Curl" ) {

return 403;

break;

}

#禁部分爬取工具

if ($http_user_agent ~* "crawl|curb|git|Wtrace|Scrapy|python|http://www.snsbianpofanghu.com/" ) {

return 403;

break;

}

#禁压缩包

location ~* \.(tgz|bak|zip|rar|tar|gz|bz2|xz|tar.gz)$ {

return 403;

break;

}

复制代码

#UA 不全，十有八九不是正常访问，禁

if ($http_user_agent = "Mozilla") {

    return 403;

    break;

}

#UA 不全，十有八九不是正常访问，禁

if ($http_user_agent = "Mozilla/5\.0") {

    return 403;

    break;

}

#UA 不全，十有八九不是正常访问，禁

if ($http_user_agent = "Mozilla/4\.0") {

    return 403;

    break;

}

  #禁空 UA

  if (http_user_agent ~* ^

) {

  return 403;

  break;

  }

复制代码

#屏蔽ip

deny 113.92.157.0/24;

deny 223.199.0.0/16;

deny 192.74.225.105;

复制代码

限制指定蜘蛛访问频次

limit_req_zone  $anti_spider  zone=one:100m rate=30r/m;

limit_req zone=one burst=5 nodelay;

server

{

if ($http_user_agent ~* "Sogou web spider|YisouSpider") {

    set $anti_spider $http_user_agent;

}

复制代码

限制所有ip访问频次

limit_req_zone $binary_remote_addr zone=allips:100m rate=30r/m;

limit_req zone=allips burst=5 nodelay;