diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index 56115a87d7c9e774bdbb5c43473f45dd0c37043f..9ba31517130000406c53e650a7423d1c2d50f3ad 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -300,23 +300,13 @@ class FederationClient(FederationBase):
 
                         break
 
-            except SynapseError as e:
-                logger.info(
-                    "Failed to get PDU %s from %s because %s",
-                    event_id, destination, e,
-                )
-                continue
-            except CodeMessageException as e:
-                if 400 <= e.code < 500 and e.code != 404:
-                    raise
-
                 pdu_attempts[destination] = now
 
+            except SynapseError as e:
                 logger.info(
                     "Failed to get PDU %s from %s because %s",
                     event_id, destination, e,
                 )
-                continue
             except NotRetryingDestination as e:
                 logger.info(e.message)
                 continue