diff --git a/.travis.yml b/.travis.yml
index b34b17af758f028d146267379466f37400c677f4..318701c9f89fad70c69dfcd4dbcf913eb2673136 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,6 +8,9 @@ before_script:
- git remote set-branches --add origin develop
- git fetch origin develop
+services:
+ - postgresql
+
matrix:
fast_finish: true
include:
@@ -20,6 +23,9 @@ matrix:
- python: 2.7
env: TOX_ENV=py27
+ - python: 2.7
+ env: TOX_ENV=py27-postgres TRIAL_FLAGS="-j 4"
+
- python: 3.6
env: TOX_ENV=py36
@@ -29,6 +35,10 @@ matrix:
- python: 3.6
env: TOX_ENV=check-newsfragment
+ allow_failures:
+ - python: 2.7
+ env: TOX_ENV=py27-postgres TRIAL_FLAGS="-j 4"
+
install:
- pip install tox
diff --git a/changelog.d/3423.misc b/changelog.d/3423.misc
new file mode 100644
index 0000000000000000000000000000000000000000..51768c6d14efde17e8bac265b96d5eb539ab6a26
--- /dev/null
+++ b/changelog.d/3423.misc
@@ -0,0 +1 @@
+The test suite now can run under PostgreSQL.
diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index 20fc3b03230dab2acfbf240dfb73d1e28d66b3a4..3671d24f60d08743556986edf1da9efdcfbf7ccc 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -95,6 +95,7 @@ class PresenceHandler(object):
Args:
hs (synapse.server.HomeServer):
"""
+ self.hs = hs
self.is_mine = hs.is_mine
self.is_mine_id = hs.is_mine_id
self.clock = hs.get_clock()
@@ -230,6 +231,10 @@ class PresenceHandler(object):
earlier than they should when synapse is restarted. This affect of this
is some spurious presence changes that will self-correct.
"""
+ # If the DB pool has already terminated, don't try updating
+ if not self.hs.get_db_pool().running:
+ return
+
logger.info(
"Performing _on_shutdown. Persisting %d unpersisted changes",
len(self.user_to_current_state)
diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py
index 2489527f2cc8537f69ec4644e0232f9c68d4359f..8fc678fa67be97cff6203d5fe02534cc7248611c 100644
--- a/synapse/storage/client_ips.py
+++ b/synapse/storage/client_ips.py
@@ -96,6 +96,11 @@ class ClientIpStore(background_updates.BackgroundUpdateStore):
self._batch_row_update[key] = (user_agent, device_id, now)
def _update_client_ips_batch(self):
+
+ # If the DB pool has already terminated, don't try updating
+ if not self.hs.get_db_pool().running:
+ return
+
def update():
to_update = self._batch_row_update
self._batch_row_update = {}
diff --git a/tests/__init__.py b/tests/__init__.py
index 24006c949e48da22aa34716ab33e1e87907c7cdc..9d9ca2282928ba44251555a4ab27f9b308bf3303 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -15,4 +15,7 @@
from twisted.trial import util
+from tests import utils
+
util.DEFAULT_TIMEOUT_DURATION = 10
+utils.setupdb()
diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py
index f8e28876bb52a4afda0cdf936451d54efc520efe..a65689ba896e5d8037404903eb95c6e71bbcf074 100644
--- a/tests/api/test_auth.py
+++ b/tests/api/test_auth.py
@@ -39,7 +39,7 @@ class AuthTestCase(unittest.TestCase):
self.state_handler = Mock()
self.store = Mock()
- self.hs = yield setup_test_homeserver(handlers=None)
+ self.hs = yield setup_test_homeserver(self.addCleanup, handlers=None)
self.hs.get_datastore = Mock(return_value=self.store)
self.hs.handlers = TestHandlers(self.hs)
self.auth = Auth(self.hs)
diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py
index 1c2d71052cd01bfd0d910405282609d208104e74..48b2d3d663bbda881a4129cd37d1294d35409454 100644
--- a/tests/api/test_filtering.py
+++ b/tests/api/test_filtering.py
@@ -46,7 +46,10 @@ class FilteringTestCase(unittest.TestCase):
self.mock_http_client.put_json = DeferredMockCallable()
hs = yield setup_test_homeserver(
- handlers=None, http_client=self.mock_http_client, keyring=Mock()
+ self.addCleanup,
+ handlers=None,
+ http_client=self.mock_http_client,
+ keyring=Mock(),
)
self.filtering = hs.get_filtering()
diff --git a/tests/crypto/test_keyring.py b/tests/crypto/test_keyring.py
index 0c6f510d113ecc791bed92dda339aeb4098edc6b..8299dc72c87a9f1d271a01d10b4a892c4dc9bc8f 100644
--- a/tests/crypto/test_keyring.py
+++ b/tests/crypto/test_keyring.py
@@ -58,12 +58,10 @@ class KeyringTestCase(unittest.TestCase):
self.mock_perspective_server = MockPerspectiveServer()
self.http_client = Mock()
self.hs = yield utils.setup_test_homeserver(
- handlers=None, http_client=self.http_client
+ self.addCleanup, handlers=None, http_client=self.http_client
)
keys = self.mock_perspective_server.get_verify_keys()
- self.hs.config.perspectives = {
- self.mock_perspective_server.server_name: keys
- }
+ self.hs.config.perspectives = {self.mock_perspective_server.server_name: keys}
def check_context(self, _, expected):
self.assertEquals(
diff --git a/tests/handlers/test_auth.py b/tests/handlers/test_auth.py
index ede01f8099a63cf37f6ef9171b8535a82fa90401..56c0f87fb7723ba657cca4ca60f2117157013626 100644
--- a/tests/handlers/test_auth.py
+++ b/tests/handlers/test_auth.py
@@ -35,7 +35,7 @@ class AuthHandlers(object):
class AuthTestCase(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
- self.hs = yield setup_test_homeserver(handlers=None)
+ self.hs = yield setup_test_homeserver(self.addCleanup, handlers=None)
self.hs.handlers = AuthHandlers(self.hs)
self.auth_handler = self.hs.handlers.auth_handler
self.macaroon_generator = self.hs.get_macaroon_generator()
diff --git a/tests/handlers/test_device.py b/tests/handlers/test_device.py
index d70d64550403b656a38cb7de040d0be108108be4..56e7acd37cfba31f008e770a0dc1dacf974ae5c2 100644
--- a/tests/handlers/test_device.py
+++ b/tests/handlers/test_device.py
@@ -34,7 +34,7 @@ class DeviceTestCase(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
- hs = yield utils.setup_test_homeserver()
+ hs = yield utils.setup_test_homeserver(self.addCleanup)
self.handler = hs.get_device_handler()
self.store = hs.get_datastore()
self.clock = hs.get_clock()
diff --git a/tests/handlers/test_directory.py b/tests/handlers/test_directory.py
index 06de9f5eca2564fa27c8ddd0c170f2ed95576186..ec7355688b25435500526d96e33debee8723c8aa 100644
--- a/tests/handlers/test_directory.py
+++ b/tests/handlers/test_directory.py
@@ -46,6 +46,7 @@ class DirectoryTestCase(unittest.TestCase):
self.mock_registry.register_query_handler = register_query_handler
hs = yield setup_test_homeserver(
+ self.addCleanup,
http_client=None,
resource_for_federation=Mock(),
federation_client=self.mock_federation,
diff --git a/tests/handlers/test_e2e_keys.py b/tests/handlers/test_e2e_keys.py
index 57ab228455e453ac013d7880ca994d1634fe56c9..8dccc6826eb2bf4e327d24ce7da7fd9160c6aa1d 100644
--- a/tests/handlers/test_e2e_keys.py
+++ b/tests/handlers/test_e2e_keys.py
@@ -34,7 +34,7 @@ class E2eKeysHandlerTestCase(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
self.hs = yield utils.setup_test_homeserver(
- handlers=None, federation_client=mock.Mock()
+ self.addCleanup, handlers=None, federation_client=mock.Mock()
)
self.handler = synapse.handlers.e2e_keys.E2eKeysHandler(self.hs)
diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py
index 9268a6fe2b895bb150da4ddd038687627e824eae..62dc69003c85f326028bb23cca2987dc23fa5b2c 100644
--- a/tests/handlers/test_profile.py
+++ b/tests/handlers/test_profile.py
@@ -48,6 +48,7 @@ class ProfileTestCase(unittest.TestCase):
self.mock_registry.register_query_handler = register_query_handler
hs = yield setup_test_homeserver(
+ self.addCleanup,
http_client=None,
handlers=None,
resource_for_federation=Mock(),
diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py
index dbec81076f135a48b35ba6bf5ab9d04edfa30600..d48d40c8dd97d1ab6fd3b6504e949fb91f73e66b 100644
--- a/tests/handlers/test_register.py
+++ b/tests/handlers/test_register.py
@@ -40,6 +40,7 @@ class RegistrationTestCase(unittest.TestCase):
self.mock_distributor.declare("registered_user")
self.mock_captcha_client = Mock()
self.hs = yield setup_test_homeserver(
+ self.addCleanup,
handlers=None,
http_client=None,
expire_access_token=True,
diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py
index becfa77bfa9f99d910c4c574a0fb510e1adbe6bc..ad58073a140cb04fabff9d6b3dbcfe62ad95868c 100644
--- a/tests/handlers/test_typing.py
+++ b/tests/handlers/test_typing.py
@@ -67,6 +67,7 @@ class TypingNotificationsTestCase(unittest.TestCase):
self.state_handler = Mock()
hs = yield setup_test_homeserver(
+ self.addCleanup,
"test",
auth=self.auth,
clock=self.clock,
diff --git a/tests/replication/slave/storage/_base.py b/tests/replication/slave/storage/_base.py
index c23b6e2cfd7a191dfde8829da97264a0d8641bb1..65df116efc75c1233ff35de2bcc256a7367bc5cd 100644
--- a/tests/replication/slave/storage/_base.py
+++ b/tests/replication/slave/storage/_base.py
@@ -54,6 +54,7 @@ class BaseSlavedStoreTestCase(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
self.hs = yield setup_test_homeserver(
+ self.addCleanup,
"blue",
http_client=None,
federation_client=Mock(),
diff --git a/tests/rest/client/v1/test_admin.py b/tests/rest/client/v1/test_admin.py
index 67d9ab94e2c848fe75abbb0a31ad59285f9f8683..1a553fa3f92869f45fd947444019f2ab48a4081f 100644
--- a/tests/rest/client/v1/test_admin.py
+++ b/tests/rest/client/v1/test_admin.py
@@ -51,7 +51,7 @@ class UserRegisterTestCase(unittest.TestCase):
self.secrets = Mock()
self.hs = setup_test_homeserver(
- http_client=None, clock=self.hs_clock, reactor=self.clock
+ self.addCleanup, http_client=None, clock=self.hs_clock, reactor=self.clock
)
self.hs.config.registration_shared_secret = u"shared"
diff --git a/tests/rest/client/v1/test_events.py b/tests/rest/client/v1/test_events.py
index 0316b74fa1112e43a2136e53c7f27345db1f5968..956f7fc4c4d39363423cafe754087bd1379dcd3e 100644
--- a/tests/rest/client/v1/test_events.py
+++ b/tests/rest/client/v1/test_events.py
@@ -41,6 +41,7 @@ class EventStreamPermissionsTestCase(RestTestCase):
self.mock_resource = MockHttpResource(prefix=PATH_PREFIX)
hs = yield setup_test_homeserver(
+ self.addCleanup,
http_client=None,
federation_client=Mock(),
ratelimiter=NonCallableMock(spec_set=["send_message"]),
diff --git a/tests/rest/client/v1/test_profile.py b/tests/rest/client/v1/test_profile.py
index 9ba0ffc19f8a9f17ac26da16d9fbb0f7de81a99d..1eab9c3bdba23e3769e52d6ba9a6681f6b183cd3 100644
--- a/tests/rest/client/v1/test_profile.py
+++ b/tests/rest/client/v1/test_profile.py
@@ -46,6 +46,7 @@ class ProfileTestCase(unittest.TestCase):
)
hs = yield setup_test_homeserver(
+ self.addCleanup,
"test",
http_client=None,
resource_for_client=self.mock_resource,
diff --git a/tests/rest/client/v1/test_register.py b/tests/rest/client/v1/test_register.py
index 6f15d69ecd545e528ecab4468dffa8429faa171f..4be88b8a39fedbf3c9f873e9bfa02befd654ac98 100644
--- a/tests/rest/client/v1/test_register.py
+++ b/tests/rest/client/v1/test_register.py
@@ -49,7 +49,7 @@ class CreateUserServletTestCase(unittest.TestCase):
self.hs_clock = Clock(self.clock)
self.hs = self.hs = setup_test_homeserver(
- http_client=None, clock=self.hs_clock, reactor=self.clock
+ self.addCleanup, http_client=None, clock=self.hs_clock, reactor=self.clock
)
self.hs.get_datastore = Mock(return_value=self.datastore)
self.hs.get_handlers = Mock(return_value=handlers)
diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py
index 00fc796787b0b52cebadb83178116c673ff10361..9fe0760496de06fdd91cb4e5a172bd5a8ebe1b14 100644
--- a/tests/rest/client/v1/test_rooms.py
+++ b/tests/rest/client/v1/test_rooms.py
@@ -50,6 +50,7 @@ class RoomBase(unittest.TestCase):
self.hs_clock = Clock(self.clock)
self.hs = setup_test_homeserver(
+ self.addCleanup,
"red",
http_client=None,
clock=self.hs_clock,
diff --git a/tests/rest/client/v1/test_typing.py b/tests/rest/client/v1/test_typing.py
index 7f1a435e7ba3d6f58c1173d29ae012119187de32..677265edf602357c5459dc79b5825c547c5b42bb 100644
--- a/tests/rest/client/v1/test_typing.py
+++ b/tests/rest/client/v1/test_typing.py
@@ -44,6 +44,7 @@ class RoomTypingTestCase(RestTestCase):
self.auth_user_id = self.user_id
hs = yield setup_test_homeserver(
+ self.addCleanup,
"red",
clock=self.clock,
http_client=None,
diff --git a/tests/rest/client/v2_alpha/test_filter.py b/tests/rest/client/v2_alpha/test_filter.py
index de33b10a5f5e796264061f6361db5f146eeab6ad..8260c130f8e504849517a9af13208bd30a100167 100644
--- a/tests/rest/client/v2_alpha/test_filter.py
+++ b/tests/rest/client/v2_alpha/test_filter.py
@@ -43,7 +43,7 @@ class FilterTestCase(unittest.TestCase):
self.hs_clock = Clock(self.clock)
self.hs = setup_test_homeserver(
- http_client=None, clock=self.hs_clock, reactor=self.clock
+ self.addCleanup, http_client=None, clock=self.hs_clock, reactor=self.clock
)
self.auth = self.hs.get_auth()
diff --git a/tests/rest/client/v2_alpha/test_register.py b/tests/rest/client/v2_alpha/test_register.py
index 9487babac34325a26c70a83c270ccc6bb82d901f..b72bd0fb7fab351c104432c734d3ac7f9251d2d8 100644
--- a/tests/rest/client/v2_alpha/test_register.py
+++ b/tests/rest/client/v2_alpha/test_register.py
@@ -47,7 +47,7 @@ class RegisterRestServletTestCase(unittest.TestCase):
login_handler=self.login_handler,
)
self.hs = setup_test_homeserver(
- http_client=None, clock=self.hs_clock, reactor=self.clock
+ self.addCleanup, http_client=None, clock=self.hs_clock, reactor=self.clock
)
self.hs.get_auth = Mock(return_value=self.auth)
self.hs.get_handlers = Mock(return_value=self.handlers)
diff --git a/tests/rest/client/v2_alpha/test_sync.py b/tests/rest/client/v2_alpha/test_sync.py
index bafc0d1df07d434153a4a03d445576cfb1d978bd..2e1d06c50982594143491474f8963c2ac3e2a497 100644
--- a/tests/rest/client/v2_alpha/test_sync.py
+++ b/tests/rest/client/v2_alpha/test_sync.py
@@ -40,7 +40,7 @@ class FilterTestCase(unittest.TestCase):
self.hs_clock = Clock(self.clock)
self.hs = setup_test_homeserver(
- http_client=None, clock=self.hs_clock, reactor=self.clock
+ self.addCleanup, http_client=None, clock=self.hs_clock, reactor=self.clock
)
self.auth = self.hs.get_auth()
diff --git a/tests/server.py b/tests/server.py
index 05708be8b91706aeb6e3b6dce89d6ca351aec125..beb24cf0320a4d77959fdf6fb55066fc22430950 100644
--- a/tests/server.py
+++ b/tests/server.py
@@ -147,12 +147,15 @@ class ThreadedMemoryReactorClock(MemoryReactorClock):
return d
-def setup_test_homeserver(*args, **kwargs):
+def setup_test_homeserver(cleanup_func, *args, **kwargs):
"""
Set up a synchronous test server, driven by the reactor used by
the homeserver.
"""
- d = _sth(*args, **kwargs).result
+ d = _sth(cleanup_func, *args, **kwargs).result
+
+ if isinstance(d, Failure):
+ d.raiseException()
# Make the thread pool synchronous.
clock = d.get_clock()
@@ -189,6 +192,9 @@ def setup_test_homeserver(*args, **kwargs):
def start(self):
pass
+ def stop(self):
+ pass
+
def callInThreadWithCallback(self, onResult, function, *args, **kwargs):
def _(res):
if isinstance(res, Failure):
diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py
index fbb25a88441a5f56c066fc122a55b1b0b5454f9c..c893990454bbe0b78ca8bfba4a0d54eaf55a9845 100644
--- a/tests/storage/test_appservice.py
+++ b/tests/storage/test_appservice.py
@@ -43,7 +43,10 @@ class ApplicationServiceStoreTestCase(unittest.TestCase):
password_providers=[],
)
hs = yield setup_test_homeserver(
- config=config, federation_sender=Mock(), federation_client=Mock()
+ self.addCleanup,
+ config=config,
+ federation_sender=Mock(),
+ federation_client=Mock(),
)
self.as_token = "token1"
@@ -108,7 +111,10 @@ class ApplicationServiceTransactionStoreTestCase(unittest.TestCase):
password_providers=[],
)
hs = yield setup_test_homeserver(
- config=config, federation_sender=Mock(), federation_client=Mock()
+ self.addCleanup,
+ config=config,
+ federation_sender=Mock(),
+ federation_client=Mock(),
)
self.db_pool = hs.get_db_pool()
@@ -392,6 +398,7 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase):
app_service_config_files=[f1, f2], event_cache_size=1, password_providers=[]
)
hs = yield setup_test_homeserver(
+ self.addCleanup,
config=config,
datastore=Mock(),
federation_sender=Mock(),
@@ -409,6 +416,7 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase):
app_service_config_files=[f1, f2], event_cache_size=1, password_providers=[]
)
hs = yield setup_test_homeserver(
+ self.addCleanup,
config=config,
datastore=Mock(),
federation_sender=Mock(),
@@ -432,6 +440,7 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase):
app_service_config_files=[f1, f2], event_cache_size=1, password_providers=[]
)
hs = yield setup_test_homeserver(
+ self.addCleanup,
config=config,
datastore=Mock(),
federation_sender=Mock(),
diff --git a/tests/storage/test_background_update.py b/tests/storage/test_background_update.py
index b4f6baf4418a985b3b267e36dff5d90c8c645448..81403727c52b4397005d9814f3fd3c20866ddeaf 100644
--- a/tests/storage/test_background_update.py
+++ b/tests/storage/test_background_update.py
@@ -9,7 +9,9 @@ from tests.utils import setup_test_homeserver
class BackgroundUpdateTestCase(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
- hs = yield setup_test_homeserver() # type: synapse.server.HomeServer
+ hs = yield setup_test_homeserver(
+ self.addCleanup
+ ) # type: synapse.server.HomeServer
self.store = hs.get_datastore()
self.clock = hs.get_clock()
diff --git a/tests/storage/test_client_ips.py b/tests/storage/test_client_ips.py
index ea00bbe84cd4789130603d184b29f317ebdffa92..fa60d949baf8532a1df3e7744640fa55afefb3d5 100644
--- a/tests/storage/test_client_ips.py
+++ b/tests/storage/test_client_ips.py
@@ -28,7 +28,7 @@ class ClientIpStoreTestCase(tests.unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
- self.hs = yield tests.utils.setup_test_homeserver()
+ self.hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
self.store = self.hs.get_datastore()
self.clock = self.hs.get_clock()
diff --git a/tests/storage/test_devices.py b/tests/storage/test_devices.py
index 63bc42d9e0965734cd4f3dac25b5a3ea16b460ad..aef4dfaf57a0a0895f3b16aa8a39deed5b39ffb4 100644
--- a/tests/storage/test_devices.py
+++ b/tests/storage/test_devices.py
@@ -28,7 +28,7 @@ class DeviceStoreTestCase(tests.unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
- hs = yield tests.utils.setup_test_homeserver()
+ hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
self.store = hs.get_datastore()
diff --git a/tests/storage/test_directory.py b/tests/storage/test_directory.py
index 9a8ba2fcfe295fae680861cf6f2a16b744311ba7..b4510c1c8d2b81737b0b58bad6634b9a20ab9e87 100644
--- a/tests/storage/test_directory.py
+++ b/tests/storage/test_directory.py
@@ -26,7 +26,7 @@ from tests.utils import setup_test_homeserver
class DirectoryStoreTestCase(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
- hs = yield setup_test_homeserver()
+ hs = yield setup_test_homeserver(self.addCleanup)
self.store = DirectoryStore(None, hs)
diff --git a/tests/storage/test_end_to_end_keys.py b/tests/storage/test_end_to_end_keys.py
index d45c775c2d66b26870ab8878622a892fdbb1298d..8f0aaece4061f75abc670ceb0c470763cc24f3ce 100644
--- a/tests/storage/test_end_to_end_keys.py
+++ b/tests/storage/test_end_to_end_keys.py
@@ -26,8 +26,7 @@ class EndToEndKeyStoreTestCase(tests.unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
- hs = yield tests.utils.setup_test_homeserver()
-
+ hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
self.store = hs.get_datastore()
@defer.inlineCallbacks
diff --git a/tests/storage/test_event_federation.py b/tests/storage/test_event_federation.py
index 66eb11958107d961b1cef2584465715e23d961ca..2fdf34fdf63a7b0fc9203e39fbd66fe8fffa7e43 100644
--- a/tests/storage/test_event_federation.py
+++ b/tests/storage/test_event_federation.py
@@ -22,7 +22,7 @@ import tests.utils
class EventFederationWorkerStoreTestCase(tests.unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
- hs = yield tests.utils.setup_test_homeserver()
+ hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
self.store = hs.get_datastore()
@defer.inlineCallbacks
diff --git a/tests/storage/test_event_push_actions.py b/tests/storage/test_event_push_actions.py
index 5e87b4530df8e30f2081752d167db03badd9d66a..b114c6fb1d4ed8697a9cc0bbed27a5b41a0ef62c 100644
--- a/tests/storage/test_event_push_actions.py
+++ b/tests/storage/test_event_push_actions.py
@@ -33,7 +33,7 @@ HIGHLIGHT = [
class EventPushActionsStoreTestCase(tests.unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
- hs = yield tests.utils.setup_test_homeserver()
+ hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
self.store = hs.get_datastore()
@defer.inlineCallbacks
diff --git a/tests/storage/test_keys.py b/tests/storage/test_keys.py
index ad0a55b324b0801b5899bb341f81c248f8f26511..47f4a8ceac99e8196063f3dad7173df65f1c1edc 100644
--- a/tests/storage/test_keys.py
+++ b/tests/storage/test_keys.py
@@ -28,7 +28,7 @@ class KeyStoreTestCase(tests.unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
- hs = yield tests.utils.setup_test_homeserver()
+ hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
self.store = hs.get_datastore()
@defer.inlineCallbacks
diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py
index 22b1072d9f57d79d8092a569a3f2e050c514d457..0a2c859f26cad52ad72cb874d4acf3c06194b6aa 100644
--- a/tests/storage/test_monthly_active_users.py
+++ b/tests/storage/test_monthly_active_users.py
@@ -28,7 +28,7 @@ class MonthlyActiveUsersTestCase(tests.unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
- self.hs = yield setup_test_homeserver()
+ self.hs = yield setup_test_homeserver(self.addCleanup)
self.store = self.hs.get_datastore()
@defer.inlineCallbacks
diff --git a/tests/storage/test_presence.py b/tests/storage/test_presence.py
index 12c540dfab404689c4dabf2e76a733e371ac376e..b5b58ff660a33710088275f0c616df25c7c9bd0a 100644
--- a/tests/storage/test_presence.py
+++ b/tests/storage/test_presence.py
@@ -26,7 +26,7 @@ from tests.utils import MockClock, setup_test_homeserver
class PresenceStoreTestCase(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
- hs = yield setup_test_homeserver(clock=MockClock())
+ hs = yield setup_test_homeserver(self.addCleanup, clock=MockClock())
self.store = PresenceStore(None, hs)
diff --git a/tests/storage/test_profile.py b/tests/storage/test_profile.py
index 5acbc8be0c57f449e545a28ff295d565ff4d2a41..a1f6618bf94bcdc57a4b4b58e2b95deb09c6de24 100644
--- a/tests/storage/test_profile.py
+++ b/tests/storage/test_profile.py
@@ -26,7 +26,7 @@ from tests.utils import setup_test_homeserver
class ProfileStoreTestCase(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
- hs = yield setup_test_homeserver()
+ hs = yield setup_test_homeserver(self.addCleanup)
self.store = ProfileStore(None, hs)
diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py
index 85ce61e841cf244ae61162252bf56f2adc39d7bb..c4e9fb72bffff4f8f5a4c77cfe0f5981b4976bf3 100644
--- a/tests/storage/test_redaction.py
+++ b/tests/storage/test_redaction.py
@@ -29,7 +29,7 @@ class RedactionTestCase(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
hs = yield setup_test_homeserver(
- resource_for_federation=Mock(), http_client=None
+ self.addCleanup, resource_for_federation=Mock(), http_client=None
)
self.store = hs.get_datastore()
diff --git a/tests/storage/test_registration.py b/tests/storage/test_registration.py
index bd96896bb35e4d66d624131d397db42903d74108..4eda122edc39c2c368a4ec9982a2928bb7e08c55 100644
--- a/tests/storage/test_registration.py
+++ b/tests/storage/test_registration.py
@@ -23,7 +23,7 @@ from tests.utils import setup_test_homeserver
class RegistrationStoreTestCase(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
- hs = yield setup_test_homeserver()
+ hs = yield setup_test_homeserver(self.addCleanup)
self.db_pool = hs.get_db_pool()
self.store = hs.get_datastore()
diff --git a/tests/storage/test_room.py b/tests/storage/test_room.py
index 84d49b55c1cb78357f9ebcaf7fba916967a35b65..a1ea23b06896e4a39c11047984d015ff3b0be663 100644
--- a/tests/storage/test_room.py
+++ b/tests/storage/test_room.py
@@ -26,7 +26,7 @@ from tests.utils import setup_test_homeserver
class RoomStoreTestCase(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
- hs = yield setup_test_homeserver()
+ hs = yield setup_test_homeserver(self.addCleanup)
# We can't test RoomStore on its own without the DirectoryStore, for
# management of the 'room_aliases' table
@@ -57,7 +57,7 @@ class RoomStoreTestCase(unittest.TestCase):
class RoomEventsStoreTestCase(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
- hs = setup_test_homeserver()
+ hs = setup_test_homeserver(self.addCleanup)
# Room events need the full datastore, for persist_event() and
# get_room_state()
diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py
index 0d9908926a94f6547bbd92889bd985e66583d2c7..c83ef600625f413d16a804acbd97a26a954e4c4a 100644
--- a/tests/storage/test_roommember.py
+++ b/tests/storage/test_roommember.py
@@ -29,7 +29,7 @@ class RoomMemberStoreTestCase(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
hs = yield setup_test_homeserver(
- resource_for_federation=Mock(), http_client=None
+ self.addCleanup, resource_for_federation=Mock(), http_client=None
)
# We can't test the RoomMemberStore on its own without the other event
# storage logic
diff --git a/tests/storage/test_state.py b/tests/storage/test_state.py
index ed5b41644a5c89f774042b4ab69fa22424c8d6ae..6168c46248d0794e2239fafdb8b5f1bd54db6482 100644
--- a/tests/storage/test_state.py
+++ b/tests/storage/test_state.py
@@ -33,7 +33,7 @@ class StateStoreTestCase(tests.unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
- hs = yield tests.utils.setup_test_homeserver()
+ hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
self.store = hs.get_datastore()
self.event_builder_factory = hs.get_event_builder_factory()
diff --git a/tests/storage/test_user_directory.py b/tests/storage/test_user_directory.py
index 7a273eab48805d86cc21295e61ee7b321c6cd1e7..b46e0ea7e279db2e36ebba3f7d1141436f0bf14b 100644
--- a/tests/storage/test_user_directory.py
+++ b/tests/storage/test_user_directory.py
@@ -29,7 +29,7 @@ BOBBY = "@bobby:a"
class UserDirectoryStoreTestCase(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
- self.hs = yield setup_test_homeserver()
+ self.hs = yield setup_test_homeserver(self.addCleanup)
self.store = UserDirectoryStore(None, self.hs)
# alice and bob are both in !room_id. bobby is not but shares
diff --git a/tests/test_federation.py b/tests/test_federation.py
index f40ff29b52af6c574d1c693264e31b6e8d0a4cc9..2540604fcc91febd8430d68532b7f9f26bbe421f 100644
--- a/tests/test_federation.py
+++ b/tests/test_federation.py
@@ -18,7 +18,10 @@ class MessageAcceptTests(unittest.TestCase):
self.reactor = ThreadedMemoryReactorClock()
self.hs_clock = Clock(self.reactor)
self.homeserver = setup_test_homeserver(
- http_client=self.http_client, clock=self.hs_clock, reactor=self.reactor
+ self.addCleanup,
+ http_client=self.http_client,
+ clock=self.hs_clock,
+ reactor=self.reactor,
)
user_id = UserID("us", "test")
diff --git a/tests/test_server.py b/tests/test_server.py
index fc396226ea27f280a50946abd93699d77857850f..895e49040688f1134d55bf84cd69ef4a4e2765ea 100644
--- a/tests/test_server.py
+++ b/tests/test_server.py
@@ -16,7 +16,7 @@ class JsonResourceTests(unittest.TestCase):
self.reactor = MemoryReactorClock()
self.hs_clock = Clock(self.reactor)
self.homeserver = setup_test_homeserver(
- http_client=None, clock=self.hs_clock, reactor=self.reactor
+ self.addCleanup, http_client=None, clock=self.hs_clock, reactor=self.reactor
)
def test_handler_for_request(self):
diff --git a/tests/test_visibility.py b/tests/test_visibility.py
index 8643d631257072a6ca15ad622bb2053c7b455326..45a78338d6aac66f774200a19ade03ca14662cc6 100644
--- a/tests/test_visibility.py
+++ b/tests/test_visibility.py
@@ -31,7 +31,7 @@ TEST_ROOM_ID = "!TEST:ROOM"
class FilterEventsForServerTestCase(tests.unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
- self.hs = yield setup_test_homeserver()
+ self.hs = yield setup_test_homeserver(self.addCleanup)
self.event_creation_handler = self.hs.get_event_creation_handler()
self.event_builder_factory = self.hs.get_event_builder_factory()
self.store = self.hs.get_datastore()
diff --git a/tests/utils.py b/tests/utils.py
index 8668b5478fd8c7b03f13f107870676378ae70f2b..90378326f89844001f3ed818e4e370b867ed5452 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -13,7 +13,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import atexit
import hashlib
+import os
+import uuid
from inspect import getcallargs
from mock import Mock, patch
@@ -27,23 +30,80 @@ from synapse.http.server import HttpServer
from synapse.server import HomeServer
from synapse.storage import PostgresEngine
from synapse.storage.engines import create_engine
-from synapse.storage.prepare_database import prepare_database
+from synapse.storage.prepare_database import (
+ _get_or_create_schema_state,
+ _setup_new_database,
+ prepare_database,
+)
from synapse.util.logcontext import LoggingContext
from synapse.util.ratelimitutils import FederationRateLimiter
# set this to True to run the tests against postgres instead of sqlite.
-# It requires you to have a local postgres database called synapse_test, within
-# which ALL TABLES WILL BE DROPPED
-USE_POSTGRES_FOR_TESTS = False
+USE_POSTGRES_FOR_TESTS = os.environ.get("SYNAPSE_POSTGRES", False)
+POSTGRES_USER = os.environ.get("SYNAPSE_POSTGRES_USER", "postgres")
+POSTGRES_BASE_DB = "_synapse_unit_tests_base_%s" % (os.getpid(),)
+
+
+def setupdb():
+
+ # If we're using PostgreSQL, set up the db once
+ if USE_POSTGRES_FOR_TESTS:
+ pgconfig = {
+ "name": "psycopg2",
+ "args": {
+ "database": POSTGRES_BASE_DB,
+ "user": POSTGRES_USER,
+ "cp_min": 1,
+ "cp_max": 5,
+ },
+ }
+ config = Mock()
+ config.password_providers = []
+ config.database_config = pgconfig
+ db_engine = create_engine(pgconfig)
+ db_conn = db_engine.module.connect(user=POSTGRES_USER)
+ db_conn.autocommit = True
+ cur = db_conn.cursor()
+ cur.execute("DROP DATABASE IF EXISTS %s;" % (POSTGRES_BASE_DB,))
+ cur.execute("CREATE DATABASE %s;" % (POSTGRES_BASE_DB,))
+ cur.close()
+ db_conn.close()
+
+ # Set up in the db
+ db_conn = db_engine.module.connect(
+ database=POSTGRES_BASE_DB, user=POSTGRES_USER
+ )
+ cur = db_conn.cursor()
+ _get_or_create_schema_state(cur, db_engine)
+ _setup_new_database(cur, db_engine)
+ db_conn.commit()
+ cur.close()
+ db_conn.close()
+
+ def _cleanup():
+ db_conn = db_engine.module.connect(user=POSTGRES_USER)
+ db_conn.autocommit = True
+ cur = db_conn.cursor()
+ cur.execute("DROP DATABASE IF EXISTS %s;" % (POSTGRES_BASE_DB,))
+ cur.close()
+ db_conn.close()
+
+ atexit.register(_cleanup)
@defer.inlineCallbacks
def setup_test_homeserver(
- name="test", datastore=None, config=None, reactor=None, **kargs
+ cleanup_func, name="test", datastore=None, config=None, reactor=None, **kargs
):
- """Setup a homeserver suitable for running tests against. Keyword arguments
- are passed to the Homeserver constructor. If no datastore is supplied a
- datastore backed by an in-memory sqlite db will be given to the HS.
+ """
+ Setup a homeserver suitable for running tests against. Keyword arguments
+ are passed to the Homeserver constructor.
+
+ If no datastore is supplied, one is created and given to the homeserver.
+
+ Args:
+ cleanup_func : The function used to register a cleanup routine for
+ after the test.
"""
if reactor is None:
from twisted.internet import reactor
@@ -95,9 +155,11 @@ def setup_test_homeserver(
kargs["clock"] = MockClock()
if USE_POSTGRES_FOR_TESTS:
+ test_db = "synapse_test_%s" % uuid.uuid4().hex
+
config.database_config = {
"name": "psycopg2",
- "args": {"database": "synapse_test", "cp_min": 1, "cp_max": 5},
+ "args": {"database": test_db, "cp_min": 1, "cp_max": 5},
}
else:
config.database_config = {
@@ -107,6 +169,21 @@ def setup_test_homeserver(
db_engine = create_engine(config.database_config)
+ # Create the database before we actually try and connect to it, based off
+ # the template database we generate in setupdb()
+ if datastore is None and isinstance(db_engine, PostgresEngine):
+ db_conn = db_engine.module.connect(
+ database=POSTGRES_BASE_DB, user=POSTGRES_USER
+ )
+ db_conn.autocommit = True
+ cur = db_conn.cursor()
+ cur.execute("DROP DATABASE IF EXISTS %s;" % (test_db,))
+ cur.execute(
+ "CREATE DATABASE %s WITH TEMPLATE %s;" % (test_db, POSTGRES_BASE_DB)
+ )
+ cur.close()
+ db_conn.close()
+
# we need to configure the connection pool to run the on_new_connection
# function, so that we can test code that uses custom sqlite functions
# (like rank).
@@ -125,15 +202,35 @@ def setup_test_homeserver(
reactor=reactor,
**kargs
)
- db_conn = hs.get_db_conn()
- # make sure that the database is empty
- if isinstance(db_engine, PostgresEngine):
- cur = db_conn.cursor()
- cur.execute("SELECT tablename FROM pg_tables where schemaname='public'")
- rows = cur.fetchall()
- for r in rows:
- cur.execute("DROP TABLE %s CASCADE" % r[0])
- yield prepare_database(db_conn, db_engine, config)
+
+ # Prepare the DB on SQLite -- PostgreSQL is a copy of an already up to
+ # date db
+ if not isinstance(db_engine, PostgresEngine):
+ db_conn = hs.get_db_conn()
+ yield prepare_database(db_conn, db_engine, config)
+ db_conn.commit()
+ db_conn.close()
+
+ else:
+ # We need to do cleanup on PostgreSQL
+ def cleanup():
+ # Close all the db pools
+ hs.get_db_pool().close()
+
+ # Drop the test database
+ db_conn = db_engine.module.connect(
+ database=POSTGRES_BASE_DB, user=POSTGRES_USER
+ )
+ db_conn.autocommit = True
+ cur = db_conn.cursor()
+ cur.execute("DROP DATABASE IF EXISTS %s;" % (test_db,))
+ db_conn.commit()
+ cur.close()
+ db_conn.close()
+
+ # Register the cleanup hook
+ cleanup_func(cleanup)
+
hs.setup()
else:
hs = HomeServer(
diff --git a/tox.ini b/tox.ini
index ed26644bd9227cdac0258ecc6a08cd403793b3eb..085f438989089835e51bc32f3b564d74629ab87a 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,7 +1,7 @@
[tox]
envlist = packaging, py27, py36, pep8, check_isort
-[testenv]
+[base]
deps =
coverage
Twisted>=15.1
@@ -15,6 +15,15 @@ deps =
setenv =
PYTHONDONTWRITEBYTECODE = no_byte_code
+[testenv]
+deps =
+ {[base]deps}
+
+setenv =
+ {[base]setenv}
+
+passenv = *
+
commands =
/usr/bin/find "{toxinidir}" -name '*.pyc' -delete
coverage run {env:COVERAGE_OPTS:} --source="{toxinidir}/synapse" \
@@ -46,6 +55,15 @@ commands =
# )
usedevelop=true
+[testenv:py27-postgres]
+usedevelop=true
+deps =
+ {[base]deps}
+ psycopg2
+setenv =
+ {[base]setenv}
+ SYNAPSE_POSTGRES = 1
+
[testenv:py36]
usedevelop=true
commands =