3a7b696703
Also moved to default crawl delay bing so no more than a req every 5 seconds is allowed New site settings: "slow_down_crawler_user_agents" - list of crawlers that will be slowed down "slow_down_crawler_rate" - how many seconds to wait between requests Not enforced server side yet |
||
|---|---|---|
| .. | ||
| admin | ||
| application_controller_spec.rb | ||
| categories_controller_spec.rb | ||
| composer_controller_spec.rb | ||
| directory_items_controller_spec.rb | ||
| email_controller_spec.rb | ||
| embed_controller_spec.rb | ||
| groups_controller_spec.rb | ||
| list_controller_spec.rb | ||
| omniauth_callbacks_controller_spec.rb | ||
| post_actions_controller_spec.rb | ||
| posts_controller_spec.rb | ||
| robots_txt_controller_spec.rb | ||
| safe_mode_controller_spec.rb | ||
| session_controller_spec.rb | ||
| static_controller_spec.rb | ||
| tags_controller_spec.rb | ||
| topics_controller_spec.rb | ||
| users_controller_spec.rb | ||
| users_email_controller_spec.rb | ||