diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py
index 9ab119f29e05962d15e3d9f290e0a19ab63b48fc..888b7ef2e91a8a3b250b3ba498e7d593c5176fb6 100644
--- a/synapse/util/retryutils.py
+++ b/synapse/util/retryutils.py
@@ -89,6 +89,9 @@ class RetryDestinationLimiter(object):
 
         if exc_type is None and exc_val is None and exc_tb is None:
             # We connected successfully.
+            if not self.retry_interval:
+                return
+
             retry_last_ts = 0
             self.retry_interval = 0
         else: