Improved crawler detection: add Twitterbot, Facebook, curl, Bing, Baidu.

2014-03-15 20:01:46 +05:30
parent 5e93d60e9b
commit e3702ecb30
2 changed files with 5 additions and 1 deletions
@@ -1,5 +1,5 @@
 module CrawlerDetection
  def self.crawler?(user_agent)
-    !/Googlebot|Mediapartners|AdsBot/.match(user_agent).nil?
+    !/Googlebot|Mediapartners|AdsBot|curl|Twitterbot|facebookexternalhit|bingbot|Baiduspider/.match(user_agent).nil?
  end
 end