diff --git a/MANIFEST.in b/MANIFEST.in
index ad1523e387069a291a00b50f7ffe695837f3d852..2c59c7bdc29aa3773c79f74c2f2937ffb83313b7 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -18,8 +18,10 @@ recursive-include docs *
 recursive-include scripts *
 recursive-include scripts-dev *
 recursive-include synapse *.pyi
-recursive-include tests *.pem
 recursive-include tests *.py
+include tests/http/ca.crt
+include tests/http/ca.key
+include tests/http/server.key
 
 recursive-include synapse/res *
 recursive-include synapse/static *.css
diff --git a/changelog.d/5417.bugfix b/changelog.d/5417.bugfix
new file mode 100644
index 0000000000000000000000000000000000000000..54be963a4e8e904c7f50dd2e4239044587a4e2cd
--- /dev/null
+++ b/changelog.d/5417.bugfix
@@ -0,0 +1 @@
+Fix excessive memory using with default `federation_verify_certificates: true` configuration.
\ No newline at end of file