From cda370db9fe97e1cd97a34a594274dbb124d08fc Mon Sep 17 00:00:00 2001
From: David Taylor <david@taylorhq.com>
Date: Fri, 12 Aug 2022 16:34:04 +0100
Subject: [PATCH] PERF: Add exponential backoff for DistributedMutex

Polling every 0.001s can cause extreme load on the redis instance, especially in scenarios where multiple app instances are waiting on the same lock. This commit introduces an exponential backoff starting from 0.001s and reaching a maximum interval of 1s.

Previously `CHECK_READONLY_ATTEMPTS` was 10, and resulted in a block for 0.01s. Under the new logic, 10 attempts take more than 1s. Therefore CHECK_READONLY_ATTEMPTS is reduced to 5, bringing its blocking time to around 0.031s
---
 lib/distributed_mutex.rb | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/lib/distributed_mutex.rb b/lib/distributed_mutex.rb
index b724948d23..bc29d14951 100644
--- a/lib/distributed_mutex.rb
+++ b/lib/distributed_mutex.rb
@@ -4,7 +4,7 @@
 # Expiration happens when the current time is greater than the expire time
 class DistributedMutex
   DEFAULT_VALIDITY = 60
-  CHECK_READONLY_ATTEMPTS = 10
+  CHECK_READONLY_ATTEMPTS = 5
 
   LOCK_SCRIPT = DiscourseRedis::EvalHelper.new <<~LUA
     local now = redis.call("time")[1]
@@ -85,7 +85,9 @@ class DistributedMutex
 
       return expire_time if expire_time
 
-      sleep 0.001
+      # Exponential backoff, max duration 1s
+      interval = attempts < 10 ? (0.001 * 2**attempts) : 1
+      sleep interval
 
       # in readonly we will never be able to get a lock
       if @using_global_redis && Discourse.recently_readonly?