From d28de23ac82ecb322f086bd05a6a8ceaf787da31 Mon Sep 17 00:00:00 2001
From: Sam Saffron <sam.saffron@gmail.com>
Date: Mon, 18 Nov 2019 16:47:47 +1100
Subject: [PATCH] FEATURE: block screaming frog SEO spider by default

This crawler is not respectful and often can make 1000s of reqs a minute.

Block by default, site operators can amend this default if they want to.
---
 config/site_settings.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/site_settings.yml b/config/site_settings.yml
index cb7890b8fa..ce91ea6237 100644
--- a/config/site_settings.yml
+++ b/config/site_settings.yml
@@ -1371,7 +1371,7 @@ security:
     list_type: compact
   blacklisted_crawler_user_agents:
     type: list
-    default: "mauibot|semrushbot|ahrefsbot|blexbot"
+    default: "mauibot|semrushbot|ahrefsbot|blexbot|seo spider"
     list_type: compact
   slow_down_crawler_user_agents:
     type: list