diff --git a/MANIFEST.in b/MANIFEST.in
index 5668665db75f4e139e0fa59578c794cde9e3a263..211bde2fc496da5ab9cd5453b09ad5addafab9bb 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -21,5 +21,6 @@ recursive-include synapse/static *.html
recursive-include synapse/static *.js
exclude jenkins.sh
+exclude jenkins*.sh
prune demo/etc
diff --git a/README.rst b/README.rst
index 39a338c790b68910d86116cbef9b2739d83b1c46..8a745259bf404adfccc47e3726428b014691becb 100644
--- a/README.rst
+++ b/README.rst
@@ -565,4 +565,21 @@ sphinxcontrib-napoleon::
Building internal API documentation::
python setup.py build_sphinx
-
\ No newline at end of file
+
+
+
+Halp!! Synapse eats all my RAM!
+===============================
+
+Synapse's architecture is quite RAM hungry currently - we deliberately
+cache a lot of recent room data and metadata in RAM in order to speed up
+common requests. We'll improve this in future, but for now the easiest
+way to either reduce the RAM usage (at the risk of slowing things down)
+is to set the almost-undocumented ``SYNAPSE_CACHE_FACTOR`` environment
+variable. Roughly speaking, a SYNAPSE_CACHE_FACTOR of 1.0 will max out
+at around 3-4GB of resident memory - this is what we currently run the
+matrix.org on. The default setting is currently 0.1, which is probably
+around a ~700MB footprint. You can dial it down further to 0.02 if
+desired, which targets roughly ~512MB. Conversely you can dial it up if
+you need performance for lots of users and have a box with a lot of RAM.
+
diff --git a/jenkins-flake8.sh b/jenkins-flake8.sh
new file mode 100755
index 0000000000000000000000000000000000000000..11f1cab6c86314e517f4931c327d0a5de1072388
--- /dev/null
+++ b/jenkins-flake8.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -eux
+
+: ${WORKSPACE:="$(pwd)"}
+
+export PYTHONDONTWRITEBYTECODE=yep
+export SYNAPSE_CACHE_FACTOR=1
+
+# Output test results as junit xml
+export TRIAL_FLAGS="--reporter=subunit"
+export TOXSUFFIX="| subunit-1to2 | subunit2junitxml --no-passthrough --output-to=results.xml"
+# Write coverage reports to a separate file for each process
+export COVERAGE_OPTS="-p"
+export DUMP_COVERAGE_COMMAND="coverage help"
+
+# Output flake8 violations to violations.flake8.log
+export PEP8SUFFIX="--output-file=violations.flake8.log"
+
+rm .coverage* || echo "No coverage files to remove"
+
+tox -e packaging -e pep8
diff --git a/jenkins-postgres.sh b/jenkins-postgres.sh
new file mode 100755
index 0000000000000000000000000000000000000000..d1fed590a211bbb187f91c61740494ca306c9e68
--- /dev/null
+++ b/jenkins-postgres.sh
@@ -0,0 +1,92 @@
+#!/bin/bash
+
+set -eux
+
+: ${WORKSPACE:="$(pwd)"}
+
+export PYTHONDONTWRITEBYTECODE=yep
+export SYNAPSE_CACHE_FACTOR=1
+
+# Output test results as junit xml
+export TRIAL_FLAGS="--reporter=subunit"
+export TOXSUFFIX="| subunit-1to2 | subunit2junitxml --no-passthrough --output-to=results.xml"
+# Write coverage reports to a separate file for each process
+export COVERAGE_OPTS="-p"
+export DUMP_COVERAGE_COMMAND="coverage help"
+
+# Output flake8 violations to violations.flake8.log
+# Don't exit with non-0 status code on Jenkins,
+# so that the build steps continue and a later step can decided whether to
+# UNSTABLE or FAILURE this build.
+export PEP8SUFFIX="--output-file=violations.flake8.log || echo flake8 finished with status code \$?"
+
+rm .coverage* || echo "No coverage files to remove"
+
+tox --notest
+
+: ${GIT_BRANCH:="origin/$(git rev-parse --abbrev-ref HEAD)"}
+
+TOX_BIN=$WORKSPACE/.tox/py27/bin
+
+if [[ ! -e .sytest-base ]]; then
+ git clone https://github.com/matrix-org/sytest.git .sytest-base --mirror
+else
+ (cd .sytest-base; git fetch -p)
+fi
+
+rm -rf sytest
+git clone .sytest-base sytest --shared
+cd sytest
+
+git checkout "${GIT_BRANCH}" || (echo >&2 "No ref ${GIT_BRANCH} found, falling back to develop" ; git checkout develop)
+
+: ${PERL5LIB:=$WORKSPACE/perl5/lib/perl5}
+: ${PERL_MB_OPT:=--install_base=$WORKSPACE/perl5}
+: ${PERL_MM_OPT:=INSTALL_BASE=$WORKSPACE/perl5}
+export PERL5LIB PERL_MB_OPT PERL_MM_OPT
+
+./install-deps.pl
+
+: ${PORT_BASE:=8000}
+
+
+if [[ -z "$POSTGRES_DB_1" ]]; then
+ echo >&2 "Variable POSTGRES_DB_1 not set"
+ exit 1
+fi
+
+if [[ -z "$POSTGRES_DB_2" ]]; then
+ echo >&2 "Variable POSTGRES_DB_2 not set"
+ exit 1
+fi
+
+mkdir -p "localhost-$(($PORT_BASE + 1))"
+mkdir -p "localhost-$(($PORT_BASE + 2))"
+
+cat > localhost-$(($PORT_BASE + 1))/database.yaml << EOF
+name: psycopg2
+args:
+ database: $POSTGRES_DB_1
+EOF
+
+cat > localhost-$(($PORT_BASE + 2))/database.yaml << EOF
+name: psycopg2
+args:
+ database: $POSTGRES_DB_2
+EOF
+
+
+# Run if both postgresql databases exist
+echo >&2 "Running sytest with PostgreSQL";
+$TOX_BIN/pip install psycopg2
+./run-tests.pl --coverage -O tap --synapse-directory $WORKSPACE \
+ --python $TOX_BIN/python --all --port-base $PORT_BASE > results-postgresql.tap
+
+cd ..
+cp sytest/.coverage.* .
+
+# Combine the coverage reports
+echo "Combining:" .coverage.*
+$TOX_BIN/python -m coverage combine
+# Output coverage to coverage.xml
+$TOX_BIN/coverage xml -o coverage.xml
diff --git a/jenkins-sqlite.sh b/jenkins-sqlite.sh
new file mode 100755
index 0000000000000000000000000000000000000000..57fd8de54d4753dbfb52e7e32300331423d332ee
--- /dev/null
+++ b/jenkins-sqlite.sh
@@ -0,0 +1,63 @@
+#!/bin/bash
+
+set -eux
+
+: ${WORKSPACE:="$(pwd)"}
+
+export PYTHONDONTWRITEBYTECODE=yep
+export SYNAPSE_CACHE_FACTOR=1
+
+# Output test results as junit xml
+export TRIAL_FLAGS="--reporter=subunit"
+export TOXSUFFIX="| subunit-1to2 | subunit2junitxml --no-passthrough --output-to=results.xml"
+# Write coverage reports to a separate file for each process
+export COVERAGE_OPTS="-p"
+export DUMP_COVERAGE_COMMAND="coverage help"
+
+# Output flake8 violations to violations.flake8.log
+# Don't exit with non-0 status code on Jenkins,
+# so that the build steps continue and a later step can decided whether to
+# UNSTABLE or FAILURE this build.
+export PEP8SUFFIX="--output-file=violations.flake8.log || echo flake8 finished with status code \$?"
+
+rm .coverage* || echo "No coverage files to remove"
+
+tox --notest
+
+: ${GIT_BRANCH:="origin/$(git rev-parse --abbrev-ref HEAD)"}
+
+TOX_BIN=$WORKSPACE/.tox/py27/bin
+
+if [[ ! -e .sytest-base ]]; then
+ git clone https://github.com/matrix-org/sytest.git .sytest-base --mirror
+else
+ (cd .sytest-base; git fetch -p)
+fi
+
+rm -rf sytest
+git clone .sytest-base sytest --shared
+cd sytest
+
+git checkout "${GIT_BRANCH}" || (echo >&2 "No ref ${GIT_BRANCH} found, falling back to develop" ; git checkout develop)
+
+: ${PERL5LIB:=$WORKSPACE/perl5/lib/perl5}
+: ${PERL_MB_OPT:=--install_base=$WORKSPACE/perl5}
+: ${PERL_MM_OPT:=INSTALL_BASE=$WORKSPACE/perl5}
+export PERL5LIB PERL_MB_OPT PERL_MM_OPT
+
+./install-deps.pl
+
+: ${PORT_BASE:=8500}
+
+echo >&2 "Running sytest with SQLite3";
+./run-tests.pl --coverage -O tap --synapse-directory $WORKSPACE \
+ --python $TOX_BIN/python --all --port-base $PORT_BASE > results-sqlite3.tap
+
+cd ..
+cp sytest/.coverage.* .
+
+# Combine the coverage reports
+echo "Combining:" .coverage.*
+$TOX_BIN/python -m coverage combine
+# Output coverage to coverage.xml
+$TOX_BIN/coverage xml -o coverage.xml
diff --git a/jenkins-unittests.sh b/jenkins-unittests.sh
new file mode 100755
index 0000000000000000000000000000000000000000..104d511994740d6d495582bbf6b7f8085422552a
--- /dev/null
+++ b/jenkins-unittests.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -eux
+
+: ${WORKSPACE:="$(pwd)"}
+
+export PYTHONDONTWRITEBYTECODE=yep
+export SYNAPSE_CACHE_FACTOR=1
+
+# Output test results as junit xml
+export TRIAL_FLAGS="--reporter=subunit"
+export TOXSUFFIX="| subunit-1to2 | subunit2junitxml --no-passthrough --output-to=results.xml"
+# Write coverage reports to a separate file for each process
+export COVERAGE_OPTS="-p"
+export DUMP_COVERAGE_COMMAND="coverage help"
+
+# Output flake8 violations to violations.flake8.log
+# Don't exit with non-0 status code on Jenkins,
+# so that the build steps continue and a later step can decided whether to
+# UNSTABLE or FAILURE this build.
+export PEP8SUFFIX="--output-file=violations.flake8.log || echo flake8 finished with status code \$?"
+
+rm .coverage* || echo "No coverage files to remove"
+
+tox -e py27
diff --git a/jenkins.sh b/jenkins.sh
index 359fd22771123c15f1e24848fbb01a690570b88c..b826d510c9fe7ff34e01971c4d1561a26da12107 100755
--- a/jenkins.sh
+++ b/jenkins.sh
@@ -1,8 +1,11 @@
-#!/bin/bash -eu
+#!/bin/bash
+
+set -eux
: ${WORKSPACE:="$(pwd)"}
export PYTHONDONTWRITEBYTECODE=yep
+export SYNAPSE_CACHE_FACTOR=1
# Output test results as junit xml
export TRIAL_FLAGS="--reporter=subunit"
diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db
index fc92bbf2d87d6b6e7bbe5dbb0ded47065740d547..a2a0f364cfdbb285527abf6b5ee0628ba8063524 100755
--- a/scripts/synapse_port_db
+++ b/scripts/synapse_port_db
@@ -309,8 +309,8 @@ class Porter(object):
**self.postgres_config["args"]
)
- sqlite_engine = create_engine("sqlite3")
- postgres_engine = create_engine("psycopg2")
+ sqlite_engine = create_engine(FakeConfig(sqlite_config))
+ postgres_engine = create_engine(FakeConfig(postgres_config))
self.sqlite_store = Store(sqlite_db_pool, sqlite_engine)
self.postgres_store = Store(postgres_db_pool, postgres_engine)
@@ -792,3 +792,8 @@ if __name__ == "__main__":
if end_error_exec_info:
exc_type, exc_value, exc_traceback = end_error_exec_info
traceback.print_exception(exc_type, exc_value, exc_traceback)
+
+
+class FakeConfig:
+ def __init__(self, database_config):
+ self.database_config = database_config
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index de5ee988f1ab2c60770d8a9298487edc3ec114d4..021dc1d610f0ba2ae111bb7faef61eb17ec34aa0 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -386,7 +386,7 @@ def setup(config_options):
tls_server_context_factory = context_factory.ServerContextFactory(config)
- database_engine = create_engine(config.database_config["name"])
+ database_engine = create_engine(config)
config.database_config["args"]["cp_openfun"] = database_engine.on_new_connection
hs = SynapseHomeServer(
diff --git a/synapse/config/api.py b/synapse/config/api.py
new file mode 100644
index 0000000000000000000000000000000000000000..20ba33226a6b772cb81b4bf8b789b3ce4647cbb0
--- /dev/null
+++ b/synapse/config/api.py
@@ -0,0 +1,40 @@
+# Copyright 2015, 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ._base import Config
+
+from synapse.api.constants import EventTypes
+
+
+class ApiConfig(Config):
+
+ def read_config(self, config):
+ self.room_invite_state_types = config.get("room_invite_state_types", [
+ EventTypes.JoinRules,
+ EventTypes.CanonicalAlias,
+ EventTypes.RoomAvatar,
+ EventTypes.Name,
+ ])
+
+ def default_config(cls, **kwargs):
+ return """\
+ ## API Configuration ##
+
+ # A list of event types that will be included in the room_invite_state
+ room_invite_state_types:
+ - "{JoinRules}"
+ - "{CanonicalAlias}"
+ - "{RoomAvatar}"
+ - "{Name}"
+ """.format(**vars(EventTypes))
diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py
index 3c333b417298b3b15913073cbcaed3bf0c9a330e..a08c170f1ddc68290748b81b1b4800b9b32a6bed 100644
--- a/synapse/config/homeserver.py
+++ b/synapse/config/homeserver.py
@@ -23,6 +23,7 @@ from .captcha import CaptchaConfig
from .voip import VoipConfig
from .registration import RegistrationConfig
from .metrics import MetricsConfig
+from .api import ApiConfig
from .appservice import AppServiceConfig
from .key import KeyConfig
from .saml2 import SAML2Config
@@ -32,7 +33,7 @@ from .password import PasswordConfig
class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig,
RatelimitConfig, ContentRepositoryConfig, CaptchaConfig,
- VoipConfig, RegistrationConfig, MetricsConfig,
+ VoipConfig, RegistrationConfig, MetricsConfig, ApiConfig,
AppServiceConfig, KeyConfig, SAML2Config, CasConfig,
PasswordConfig,):
pass
diff --git a/synapse/config/repository.py b/synapse/config/repository.py
index 2fcf872449c1aa013e201be5ff2f61f4ebd19a9f..2e96c090130693704aac29d3cf4d1c646e3b5bcb 100644
--- a/synapse/config/repository.py
+++ b/synapse/config/repository.py
@@ -97,4 +97,7 @@ class ContentRepositoryConfig(Config):
- width: 640
height: 480
method: scale
+ - width: 800
+ height: 600
+ method: scale
""" % locals()
diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py
index bdade98bf7b2613e15bd2a8d2a170e2a43ac14d9..2d56af50277437e6cfd81704b505dfa3dceef54e 100644
--- a/synapse/handlers/_base.py
+++ b/synapse/handlers/_base.py
@@ -160,10 +160,10 @@ class BaseHandler(object):
)
defer.returnValue(res.get(user_id, []))
- def ratelimit(self, user_id):
+ def ratelimit(self, requester):
time_now = self.clock.time()
allowed, time_allowed = self.ratelimiter.send_message(
- user_id, time_now,
+ requester.user.to_string(), time_now,
msg_rate_hz=self.hs.config.rc_messages_per_second,
burst_count=self.hs.config.rc_message_burst_count,
)
@@ -199,8 +199,7 @@ class BaseHandler(object):
# events in the room, because we don't know enough about the graph
# fragment we received to treat it like a graph, so the above returned
# no relevant events. It may have returned some events (if we have
- # joined and left the room), but not useful ones, like the invite. So we
- # forcibly set our context to the invite we received over federation.
+ # joined and left the room), but not useful ones, like the invite.
if (
not self.is_host_in_room(context.current_state) and
builder.type == EventTypes.Member
@@ -208,7 +207,27 @@ class BaseHandler(object):
prev_member_event = yield self.store.get_room_member(
builder.sender, builder.room_id
)
- if prev_member_event:
+
+ # The prev_member_event may already be in context.current_state,
+ # despite us not being present in the room; in particular, if
+ # inviting user, and all other local users, have already left.
+ #
+ # In that case, we have all the information we need, and we don't
+ # want to drop "context" - not least because we may need to handle
+ # the invite locally, which will require us to have the whole
+ # context (not just prev_member_event) to auth it.
+ #
+ context_event_ids = (
+ e.event_id for e in context.current_state.values()
+ )
+
+ if (
+ prev_member_event and
+ prev_member_event.event_id not in context_event_ids
+ ):
+ # The prev_member_event is missing from context, so it must
+ # have arrived over federation and is an outlier. We forcibly
+ # set our context to the invite we received over federation
builder.prev_events = (
prev_member_event.event_id,
prev_member_event.prev_events
@@ -263,11 +282,18 @@ class BaseHandler(object):
return False
@defer.inlineCallbacks
- def handle_new_client_event(self, event, context, ratelimit=True, extra_users=[]):
+ def handle_new_client_event(
+ self,
+ requester,
+ event,
+ context,
+ ratelimit=True,
+ extra_users=[]
+ ):
# We now need to go and hit out to wherever we need to hit out to.
if ratelimit:
- self.ratelimit(event.sender)
+ self.ratelimit(requester)
self.auth.check(event, auth_events=context.current_state)
@@ -307,12 +333,8 @@ class BaseHandler(object):
"sender": e.sender,
}
for k, e in context.current_state.items()
- if e.type in (
- EventTypes.JoinRules,
- EventTypes.CanonicalAlias,
- EventTypes.RoomAvatar,
- EventTypes.Name,
- ) or is_inviter_member_event(e)
+ if e.type in self.hs.config.room_invite_state_types
+ or is_inviter_member_event(e)
]
invitee = UserID.from_string(event.state_key)
@@ -348,6 +370,12 @@ class BaseHandler(object):
"You don't have permission to redact events"
)
+ if event.type == EventTypes.Create and context.current_state:
+ raise AuthError(
+ 403,
+ "Changing the room create event is forbidden",
+ )
+
action_generator = ActionGenerator(self.hs)
yield action_generator.handle_push_actions_for_event(
event, context, self
diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
index 62e82a25702e6ab8b2005cbbbbc1e00702094a3e..7a4afe446da37a44b030111801a21abe01ae6f4b 100644
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -477,4 +477,4 @@ class AuthHandler(BaseHandler):
Returns:
Whether self.hash(password) == stored_hash (bool).
"""
- return bcrypt.checkpw(password, stored_hash)
+ return bcrypt.hashpw(password, stored_hash) == stored_hash
diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py
index e0a778e7ffcf6b89b748f6aea30ac08b84591a7b..c4aaa119188ca4f60d01c1d2e5d7996b219c1e47 100644
--- a/synapse/handlers/directory.py
+++ b/synapse/handlers/directory.py
@@ -17,9 +17,9 @@
from twisted.internet import defer
from ._base import BaseHandler
-from synapse.api.errors import SynapseError, Codes, CodeMessageException
+from synapse.api.errors import SynapseError, Codes, CodeMessageException, AuthError
from synapse.api.constants import EventTypes
-from synapse.types import RoomAlias
+from synapse.types import RoomAlias, UserID
import logging
import string
@@ -38,7 +38,7 @@ class DirectoryHandler(BaseHandler):
)
@defer.inlineCallbacks
- def _create_association(self, room_alias, room_id, servers=None):
+ def _create_association(self, room_alias, room_id, servers=None, creator=None):
# general association creation for both human users and app services
for wchar in string.whitespace:
@@ -60,7 +60,8 @@ class DirectoryHandler(BaseHandler):
yield self.store.create_room_alias_association(
room_alias,
room_id,
- servers
+ servers,
+ creator=creator,
)
@defer.inlineCallbacks
@@ -77,7 +78,7 @@ class DirectoryHandler(BaseHandler):
400, "This alias is reserved by an application service.",
errcode=Codes.EXCLUSIVE
)
- yield self._create_association(room_alias, room_id, servers)
+ yield self._create_association(room_alias, room_id, servers, creator=user_id)
@defer.inlineCallbacks
def create_appservice_association(self, service, room_alias, room_id,
@@ -95,7 +96,11 @@ class DirectoryHandler(BaseHandler):
def delete_association(self, user_id, room_alias):
# association deletion for human users
- # TODO Check if server admin
+ can_delete = yield self._user_can_delete_alias(room_alias, user_id)
+ if not can_delete:
+ raise AuthError(
+ 403, "You don't have permission to delete the alias.",
+ )
can_delete = yield self.can_modify_alias(
room_alias,
@@ -212,17 +217,21 @@ class DirectoryHandler(BaseHandler):
)
@defer.inlineCallbacks
- def send_room_alias_update_event(self, user_id, room_id):
+ def send_room_alias_update_event(self, requester, user_id, room_id):
aliases = yield self.store.get_aliases_for_room(room_id)
msg_handler = self.hs.get_handlers().message_handler
- yield msg_handler.create_and_send_nonmember_event({
- "type": EventTypes.Aliases,
- "state_key": self.hs.hostname,
- "room_id": room_id,
- "sender": user_id,
- "content": {"aliases": aliases},
- }, ratelimit=False)
+ yield msg_handler.create_and_send_nonmember_event(
+ requester,
+ {
+ "type": EventTypes.Aliases,
+ "state_key": self.hs.hostname,
+ "room_id": room_id,
+ "sender": user_id,
+ "content": {"aliases": aliases},
+ },
+ ratelimit=False
+ )
@defer.inlineCallbacks
def get_association_from_room_alias(self, room_alias):
@@ -257,3 +266,13 @@ class DirectoryHandler(BaseHandler):
return
# either no interested services, or no service with an exclusive lock
defer.returnValue(True)
+
+ @defer.inlineCallbacks
+ def _user_can_delete_alias(self, alias, user_id):
+ creator = yield self.store.get_room_alias_creator(alias.to_string())
+
+ if creator and creator == user_id:
+ defer.returnValue(True)
+
+ is_admin = yield self.auth.is_server_admin(UserID.from_string(user_id))
+ defer.returnValue(is_admin)
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 3655b9e5e2eb077bbc056c0761b852a35c614633..6e50b0963e7c9ef10e7afce291f24b4b6bdb59ef 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1657,7 +1657,7 @@ class FederationHandler(BaseHandler):
self.auth.check(event, context.current_state)
yield self._check_signature(event, auth_events=context.current_state)
member_handler = self.hs.get_handlers().room_member_handler
- yield member_handler.send_membership_event(event, context, from_client=False)
+ yield member_handler.send_membership_event(None, event, context)
else:
destinations = set(x.split(":", 1)[-1] for x in (sender_user_id, room_id))
yield self.replication_layer.forward_third_party_invite(
@@ -1686,7 +1686,7 @@ class FederationHandler(BaseHandler):
# TODO: Make sure the signatures actually are correct.
event.signatures.update(returned_invite.signatures)
member_handler = self.hs.get_handlers().room_member_handler
- yield member_handler.send_membership_event(event, context, from_client=False)
+ yield member_handler.send_membership_event(None, event, context)
@defer.inlineCallbacks
def add_display_name_to_third_party_invite(self, event_dict, event, context):
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 2fa12c8f2b14383a471ec97b83721d00e12cc8aa..e238579164f22cb8fc1d616e5429ea92073eaaaa 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -215,7 +215,7 @@ class MessageHandler(BaseHandler):
defer.returnValue((event, context))
@defer.inlineCallbacks
- def send_nonmember_event(self, event, context, ratelimit=True):
+ def send_nonmember_event(self, requester, event, context, ratelimit=True):
"""
Persists and notifies local clients and federation of an event.
@@ -241,6 +241,7 @@ class MessageHandler(BaseHandler):
defer.returnValue(prev_state)
yield self.handle_new_client_event(
+ requester=requester,
event=event,
context=context,
ratelimit=ratelimit,
@@ -268,9 +269,9 @@ class MessageHandler(BaseHandler):
@defer.inlineCallbacks
def create_and_send_nonmember_event(
self,
+ requester,
event_dict,
ratelimit=True,
- token_id=None,
txn_id=None
):
"""
@@ -280,10 +281,11 @@ class MessageHandler(BaseHandler):
"""
event, context = yield self.create_event(
event_dict,
- token_id=token_id,
+ token_id=requester.access_token_id,
txn_id=txn_id
)
yield self.send_nonmember_event(
+ requester,
event,
context,
ratelimit=ratelimit,
diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index d98e80086e3c327a357cd346ebf8b22b2bf50858..f6cf343174772c826e861bf93e3bfbdd452a6f30 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -130,6 +130,10 @@ class PresenceHandler(BaseHandler):
for state in active_presence
}
+ metrics.register_callback(
+ "user_to_current_state_size", lambda: len(self.user_to_current_state)
+ )
+
now = self.clock.time_msec()
for state in active_presence:
self.wheel_timer.insert(
diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py
index c9ad5944e6d8fea1d4640e94958efa2d922c7040..b45eafbb49ea18a28de7a8b5f5d4d4cc771fd99b 100644
--- a/synapse/handlers/profile.py
+++ b/synapse/handlers/profile.py
@@ -89,13 +89,13 @@ class ProfileHandler(BaseHandler):
defer.returnValue(result["displayname"])
@defer.inlineCallbacks
- def set_displayname(self, target_user, auth_user, new_displayname):
+ def set_displayname(self, target_user, requester, new_displayname):
"""target_user is the user whose displayname is to be changed;
auth_user is the user attempting to make this change."""
if not self.hs.is_mine(target_user):
raise SynapseError(400, "User is not hosted on this Home Server")
- if target_user != auth_user:
+ if target_user != requester.user:
raise AuthError(400, "Cannot set another user's displayname")
if new_displayname == '':
@@ -109,7 +109,7 @@ class ProfileHandler(BaseHandler):
"displayname": new_displayname,
})
- yield self._update_join_states(target_user)
+ yield self._update_join_states(requester)
@defer.inlineCallbacks
def get_avatar_url(self, target_user):
@@ -139,13 +139,13 @@ class ProfileHandler(BaseHandler):
defer.returnValue(result["avatar_url"])
@defer.inlineCallbacks
- def set_avatar_url(self, target_user, auth_user, new_avatar_url):
+ def set_avatar_url(self, target_user, requester, new_avatar_url):
"""target_user is the user whose avatar_url is to be changed;
auth_user is the user attempting to make this change."""
if not self.hs.is_mine(target_user):
raise SynapseError(400, "User is not hosted on this Home Server")
- if target_user != auth_user:
+ if target_user != requester.user:
raise AuthError(400, "Cannot set another user's avatar_url")
yield self.store.set_profile_avatar_url(
@@ -156,7 +156,7 @@ class ProfileHandler(BaseHandler):
"avatar_url": new_avatar_url,
})
- yield self._update_join_states(target_user)
+ yield self._update_join_states(requester)
@defer.inlineCallbacks
def collect_presencelike_data(self, user, state):
@@ -199,11 +199,12 @@ class ProfileHandler(BaseHandler):
defer.returnValue(response)
@defer.inlineCallbacks
- def _update_join_states(self, user):
+ def _update_join_states(self, requester):
+ user = requester.user
if not self.hs.is_mine(user):
return
- self.ratelimit(user.to_string())
+ self.ratelimit(requester)
joins = yield self.store.get_rooms_for_user(
user.to_string(),
diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py
index de4c694714e9127a73122e79cc9b0b90b077ca8e..935c339707354dd6cfbb80a307bfe04c68b241ec 100644
--- a/synapse/handlers/receipts.py
+++ b/synapse/handlers/receipts.py
@@ -36,8 +36,6 @@ class ReceiptsHandler(BaseHandler):
)
self.clock = self.hs.get_clock()
- self._receipt_cache = None
-
@defer.inlineCallbacks
def received_client_receipt(self, room_id, receipt_type, user_id,
event_id):
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index 6d155d57e7940da3706fb7fbb5f04a2a50daf1b2..c5e5b288119458724d36ed5243d946487ca5feb5 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -157,6 +157,7 @@ class RegistrationHandler(BaseHandler):
)
except SynapseError:
# if user id is taken, just generate another
+ user = None
user_id = None
token = None
attempts += 1
diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index d2de23a6cc10eb8ffe5aec6ed34d8806c1fd91f4..0cb6c521c4f969fcb2ee7a78ca0f5d52b1a0867a 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -18,7 +18,7 @@ from twisted.internet import defer
from ._base import BaseHandler
-from synapse.types import UserID, RoomAlias, RoomID, RoomStreamToken
+from synapse.types import UserID, RoomAlias, RoomID, RoomStreamToken, Requester
from synapse.api.constants import (
EventTypes, Membership, JoinRules, RoomCreationPreset,
)
@@ -90,7 +90,7 @@ class RoomCreationHandler(BaseHandler):
"""
user_id = requester.user.to_string()
- self.ratelimit(user_id)
+ self.ratelimit(requester)
if "room_alias_name" in config:
for wchar in string.whitespace:
@@ -185,26 +185,32 @@ class RoomCreationHandler(BaseHandler):
if "name" in config:
name = config["name"]
- yield msg_handler.create_and_send_nonmember_event({
- "type": EventTypes.Name,
- "room_id": room_id,
- "sender": user_id,
- "state_key": "",
- "content": {"name": name},
- }, ratelimit=False)
+ yield msg_handler.create_and_send_nonmember_event(
+ requester,
+ {
+ "type": EventTypes.Name,
+ "room_id": room_id,
+ "sender": user_id,
+ "state_key": "",
+ "content": {"name": name},
+ },
+ ratelimit=False)
if "topic" in config:
topic = config["topic"]
- yield msg_handler.create_and_send_nonmember_event({
- "type": EventTypes.Topic,
- "room_id": room_id,
- "sender": user_id,
- "state_key": "",
- "content": {"topic": topic},
- }, ratelimit=False)
+ yield msg_handler.create_and_send_nonmember_event(
+ requester,
+ {
+ "type": EventTypes.Topic,
+ "room_id": room_id,
+ "sender": user_id,
+ "state_key": "",
+ "content": {"topic": topic},
+ },
+ ratelimit=False)
for invitee in invite_list:
- room_member_handler.update_membership(
+ yield room_member_handler.update_membership(
requester,
UserID.from_string(invitee),
room_id,
@@ -231,7 +237,7 @@ class RoomCreationHandler(BaseHandler):
if room_alias:
result["room_alias"] = room_alias.to_string()
yield directory_handler.send_room_alias_update_event(
- user_id, room_id
+ requester, user_id, room_id
)
defer.returnValue(result)
@@ -263,7 +269,11 @@ class RoomCreationHandler(BaseHandler):
@defer.inlineCallbacks
def send(etype, content, **kwargs):
event = create(etype, content, **kwargs)
- yield msg_handler.create_and_send_nonmember_event(event, ratelimit=False)
+ yield msg_handler.create_and_send_nonmember_event(
+ creator,
+ event,
+ ratelimit=False
+ )
config = RoomCreationHandler.PRESETS_DICT[preset_config]
@@ -454,12 +464,11 @@ class RoomMemberHandler(BaseHandler):
member_handler = self.hs.get_handlers().room_member_handler
yield member_handler.send_membership_event(
+ requester,
event,
context,
- is_guest=requester.is_guest,
ratelimit=ratelimit,
remote_room_hosts=remote_room_hosts,
- from_client=True,
)
if action == "forget":
@@ -468,17 +477,19 @@ class RoomMemberHandler(BaseHandler):
@defer.inlineCallbacks
def send_membership_event(
self,
+ requester,
event,
context,
- is_guest=False,
remote_room_hosts=None,
ratelimit=True,
- from_client=True,
):
"""
Change the membership status of a user in a room.
Args:
+ requester (Requester): The local user who requested the membership
+ event. If None, certain checks, like whether this homeserver can
+ act as the sender, will be skipped.
event (SynapseEvent): The membership event.
context: The context of the event.
is_guest (bool): Whether the sender is a guest.
@@ -486,19 +497,23 @@ class RoomMemberHandler(BaseHandler):
the room, and could be danced with in order to join this
homeserver for the first time.
ratelimit (bool): Whether to rate limit this request.
- from_client (bool): Whether this request is the result of a local
- client request (rather than over federation). If so, we will
- perform extra checks, like that this homeserver can act as this
- client.
Raises:
SynapseError if there was a problem changing the membership.
"""
+ remote_room_hosts = remote_room_hosts or []
+
target_user = UserID.from_string(event.state_key)
room_id = event.room_id
- if from_client:
+ if requester is not None:
sender = UserID.from_string(event.sender)
+ assert sender == requester.user, (
+ "Sender (%s) must be same as requester (%s)" %
+ (sender, requester.user)
+ )
assert self.hs.is_mine(sender), "Sender must be our own: %s" % (sender,)
+ else:
+ requester = Requester(target_user, None, False)
message_handler = self.hs.get_handlers().message_handler
prev_event = message_handler.deduplicate_state_event(event, context)
@@ -508,7 +523,7 @@ class RoomMemberHandler(BaseHandler):
action = "send"
if event.membership == Membership.JOIN:
- if is_guest and not self._can_guest_join(context.current_state):
+ if requester.is_guest and not self._can_guest_join(context.current_state):
# This should be an auth check, but guests are a local concept,
# so don't really fit into the general auth process.
raise AuthError(403, "Guest access not allowed")
@@ -521,8 +536,24 @@ class RoomMemberHandler(BaseHandler):
action = "remote_join"
elif event.membership == Membership.LEAVE:
is_host_in_room = self.is_host_in_room(context.current_state)
+
if not is_host_in_room:
- action = "remote_reject"
+ # perhaps we've been invited
+ inviter = self.get_inviter(target_user.to_string(), context.current_state)
+ if not inviter:
+ raise SynapseError(404, "Not a known room")
+
+ if self.hs.is_mine(inviter):
+ # the inviter was on our server, but has now left. Carry on
+ # with the normal rejection codepath.
+ #
+ # This is a bit of a hack, because the room might still be
+ # active on other servers.
+ pass
+ else:
+ # send the rejection to the inviter's HS.
+ remote_room_hosts = remote_room_hosts + [inviter.domain]
+ action = "remote_reject"
federation_handler = self.hs.get_handlers().federation_handler
@@ -541,16 +572,14 @@ class RoomMemberHandler(BaseHandler):
event.content,
)
elif action == "remote_reject":
- inviter = self.get_inviter(target_user.to_string(), context.current_state)
- if not inviter:
- raise SynapseError(404, "No known servers")
yield federation_handler.do_remotely_reject_invite(
- [inviter.domain],
+ remote_room_hosts,
room_id,
event.user_id
)
else:
yield self.handle_new_client_event(
+ requester,
event,
context,
extra_users=[target_user],
@@ -669,12 +698,12 @@ class RoomMemberHandler(BaseHandler):
)
else:
yield self._make_and_store_3pid_invite(
+ requester,
id_server,
medium,
address,
room_id,
inviter,
- requester.access_token_id,
txn_id=txn_id
)
@@ -732,12 +761,12 @@ class RoomMemberHandler(BaseHandler):
@defer.inlineCallbacks
def _make_and_store_3pid_invite(
self,
+ requester,
id_server,
medium,
address,
room_id,
user,
- token_id,
txn_id
):
room_state = yield self.hs.get_state_handler().get_current_state(room_id)
@@ -787,6 +816,7 @@ class RoomMemberHandler(BaseHandler):
msg_handler = self.hs.get_handlers().message_handler
yield msg_handler.create_and_send_nonmember_event(
+ requester,
{
"type": EventTypes.ThirdPartyInvite,
"content": {
@@ -801,7 +831,6 @@ class RoomMemberHandler(BaseHandler):
"sender": user.to_string(),
"state_key": token,
},
- token_id=token_id,
txn_id=txn_id,
)
diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py
index 74ec1e50e08b191d61fbad3bee3af88d655a45af..8bfe9fdea850f3ef0c253c3326f867419837d720 100644
--- a/synapse/rest/client/v1/directory.py
+++ b/synapse/rest/client/v1/directory.py
@@ -75,7 +75,11 @@ class ClientDirectoryServer(ClientV1RestServlet):
yield dir_handler.create_association(
user_id, room_alias, room_id, servers
)
- yield dir_handler.send_room_alias_update_event(user_id, room_id)
+ yield dir_handler.send_room_alias_update_event(
+ requester,
+ user_id,
+ room_id
+ )
except SynapseError as e:
raise e
except:
@@ -118,9 +122,6 @@ class ClientDirectoryServer(ClientV1RestServlet):
requester = yield self.auth.get_user_by_req(request)
user = requester.user
- is_admin = yield self.auth.is_server_admin(user)
- if not is_admin:
- raise AuthError(403, "You need to be a server admin")
room_alias = RoomAlias.from_string(room_alias)
diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py
index 3c5a212920d4a07b867ef4a490a53ae73d814c72..953764bd8ef20de0047b103aef8950c5f77ffed5 100644
--- a/synapse/rest/client/v1/profile.py
+++ b/synapse/rest/client/v1/profile.py
@@ -51,7 +51,7 @@ class ProfileDisplaynameRestServlet(ClientV1RestServlet):
defer.returnValue((400, "Unable to parse name"))
yield self.handlers.profile_handler.set_displayname(
- user, requester.user, new_name)
+ user, requester, new_name)
defer.returnValue((200, {}))
@@ -88,7 +88,7 @@ class ProfileAvatarURLRestServlet(ClientV1RestServlet):
defer.returnValue((400, "Unable to parse name"))
yield self.handlers.profile_handler.set_avatar_url(
- user, requester.user, new_name)
+ user, requester, new_name)
defer.returnValue((200, {}))
diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py
index f5ed4f730296f7293761d7704d27bdcf58e42ac3..cbf3673eff5b9bcb533a4fbd3c1cee99f17afc47 100644
--- a/synapse/rest/client/v1/room.py
+++ b/synapse/rest/client/v1/room.py
@@ -158,12 +158,12 @@ class RoomStateEventRestServlet(ClientV1RestServlet):
if event_type == EventTypes.Member:
yield self.handlers.room_member_handler.send_membership_event(
+ requester,
event,
context,
- is_guest=requester.is_guest,
)
else:
- yield msg_handler.send_nonmember_event(event, context)
+ yield msg_handler.send_nonmember_event(requester, event, context)
defer.returnValue((200, {"event_id": event.event_id}))
@@ -183,13 +183,13 @@ class RoomSendEventRestServlet(ClientV1RestServlet):
msg_handler = self.handlers.message_handler
event = yield msg_handler.create_and_send_nonmember_event(
+ requester,
{
"type": event_type,
"content": content,
"room_id": room_id,
"sender": requester.user.to_string(),
},
- token_id=requester.access_token_id,
txn_id=txn_id,
)
@@ -504,6 +504,7 @@ class RoomRedactEventRestServlet(ClientV1RestServlet):
msg_handler = self.handlers.message_handler
event = yield msg_handler.create_and_send_nonmember_event(
+ requester,
{
"type": EventTypes.Redaction,
"content": content,
@@ -511,7 +512,6 @@ class RoomRedactEventRestServlet(ClientV1RestServlet):
"sender": requester.user.to_string(),
"redacts": event_id,
},
- token_id=requester.access_token_id,
txn_id=txn_id,
)
diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py
index 1100c67714974c48c2072a854cdac548e6d71177..371600eebb8bd4479183ad40b1a6a20252dc6303 100644
--- a/synapse/storage/appservice.py
+++ b/synapse/storage/appservice.py
@@ -34,8 +34,8 @@ class ApplicationServiceStore(SQLBaseStore):
def __init__(self, hs):
super(ApplicationServiceStore, self).__init__(hs)
self.hostname = hs.hostname
- self.services_cache = []
- self._populate_appservice_cache(
+ self.services_cache = ApplicationServiceStore.load_appservices(
+ hs.hostname,
hs.config.app_service_config_files
)
@@ -144,21 +144,23 @@ class ApplicationServiceStore(SQLBaseStore):
return rooms_for_user_matching_user_id
- def _load_appservice(self, as_info):
+ @classmethod
+ def _load_appservice(cls, hostname, as_info, config_filename):
required_string_fields = [
- # TODO: Add id here when it's stable to release
- "url", "as_token", "hs_token", "sender_localpart"
+ "id", "url", "as_token", "hs_token", "sender_localpart"
]
for field in required_string_fields:
if not isinstance(as_info.get(field), basestring):
- raise KeyError("Required string field: '%s'", field)
+ raise KeyError("Required string field: '%s' (%s)" % (
+ field, config_filename,
+ ))
localpart = as_info["sender_localpart"]
if urllib.quote(localpart) != localpart:
raise ValueError(
"sender_localpart needs characters which are not URL encoded."
)
- user = UserID(localpart, self.hostname)
+ user = UserID(localpart, hostname)
user_id = user.to_string()
# namespace checks
@@ -188,25 +190,30 @@ class ApplicationServiceStore(SQLBaseStore):
namespaces=as_info["namespaces"],
hs_token=as_info["hs_token"],
sender=user_id,
- id=as_info["id"] if "id" in as_info else as_info["as_token"],
+ id=as_info["id"],
)
- def _populate_appservice_cache(self, config_files):
- """Populates a cache of Application Services from the config files."""
+ @classmethod
+ def load_appservices(cls, hostname, config_files):
+ """Returns a list of Application Services from the config files."""
if not isinstance(config_files, list):
logger.warning(
"Expected %s to be a list of AS config files.", config_files
)
- return
+ return []
# Dicts of value -> filename
seen_as_tokens = {}
seen_ids = {}
+ appservices = []
+
for config_file in config_files:
try:
with open(config_file, 'r') as f:
- appservice = self._load_appservice(yaml.load(f))
+ appservice = ApplicationServiceStore._load_appservice(
+ hostname, yaml.load(f), config_file
+ )
if appservice.id in seen_ids:
raise ConfigError(
"Cannot reuse ID across application services: "
@@ -226,11 +233,12 @@ class ApplicationServiceStore(SQLBaseStore):
)
seen_as_tokens[appservice.token] = config_file
logger.info("Loaded application service: %s", appservice)
- self.services_cache.append(appservice)
+ appservices.append(appservice)
except Exception as e:
logger.error("Failed to load appservice from '%s'", config_file)
logger.exception(e)
raise
+ return appservices
class ApplicationServiceTransactionStore(SQLBaseStore):
diff --git a/synapse/storage/directory.py b/synapse/storage/directory.py
index 1556619d5e49b3860db3b80f937762b9c794e821..012a0b414aa7b201c575dec4d85ae8f49aaf4532 100644
--- a/synapse/storage/directory.py
+++ b/synapse/storage/directory.py
@@ -70,13 +70,14 @@ class DirectoryStore(SQLBaseStore):
)
@defer.inlineCallbacks
- def create_room_alias_association(self, room_alias, room_id, servers):
+ def create_room_alias_association(self, room_alias, room_id, servers, creator=None):
""" Creates an associatin between a room alias and room_id/servers
Args:
room_alias (RoomAlias)
room_id (str)
servers (list)
+ creator (str): Optional user_id of creator.
Returns:
Deferred
@@ -87,6 +88,7 @@ class DirectoryStore(SQLBaseStore):
{
"room_alias": room_alias.to_string(),
"room_id": room_id,
+ "creator": creator,
},
desc="create_room_alias_association",
)
@@ -107,6 +109,17 @@ class DirectoryStore(SQLBaseStore):
)
self.get_aliases_for_room.invalidate((room_id,))
+ def get_room_alias_creator(self, room_alias):
+ return self._simple_select_one_onecol(
+ table="room_aliases",
+ keyvalues={
+ "room_alias": room_alias,
+ },
+ retcol="creator",
+ desc="get_room_alias_creator",
+ allow_none=True
+ )
+
@defer.inlineCallbacks
def delete_room_alias(self, room_alias):
room_id = yield self.runInteraction(
diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py
index 4290aea83ae77b46137df957d846c6f932a04e15..a48230b93fee6123132042f564b9a7f1154b13e1 100644
--- a/synapse/storage/engines/__init__.py
+++ b/synapse/storage/engines/__init__.py
@@ -26,12 +26,13 @@ SUPPORTED_MODULE = {
}
-def create_engine(name):
+def create_engine(config):
+ name = config.database_config["name"]
engine_class = SUPPORTED_MODULE.get(name, None)
if engine_class:
module = importlib.import_module(name)
- return engine_class(module)
+ return engine_class(module, config=config)
raise RuntimeError(
"Unsupported database engine '%s'" % (name,)
diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py
index 17b7a9c0776546c54af51a39ac59caaceaee6eda..a09685b4df12dfd4d3a7a8f7a8a3f0f45ccb9520 100644
--- a/synapse/storage/engines/postgres.py
+++ b/synapse/storage/engines/postgres.py
@@ -21,9 +21,10 @@ from ._base import IncorrectDatabaseSetup
class PostgresEngine(object):
single_threaded = False
- def __init__(self, database_module):
+ def __init__(self, database_module, config):
self.module = database_module
self.module.extensions.register_type(self.module.extensions.UNICODE)
+ self.config = config
def check_database(self, txn):
txn.execute("SHOW SERVER_ENCODING")
@@ -44,7 +45,7 @@ class PostgresEngine(object):
)
def prepare_database(self, db_conn):
- prepare_database(db_conn, self)
+ prepare_database(db_conn, self, config=self.config)
def is_deadlock(self, error):
if isinstance(error, self.module.DatabaseError):
diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py
index 91fac33b8b97e18d685618069e0ec0cbf8c5bdc6..522b9059492e52bf438a22797abad88423d3c7e3 100644
--- a/synapse/storage/engines/sqlite3.py
+++ b/synapse/storage/engines/sqlite3.py
@@ -23,8 +23,9 @@ import struct
class Sqlite3Engine(object):
single_threaded = True
- def __init__(self, database_module):
+ def __init__(self, database_module, config):
self.module = database_module
+ self.config = config
def check_database(self, txn):
pass
@@ -38,7 +39,7 @@ class Sqlite3Engine(object):
def prepare_database(self, db_conn):
prepare_sqlite3_database(db_conn)
- prepare_database(db_conn, self)
+ prepare_database(db_conn, self, config=self.config)
def is_deadlock(self, error):
return False
diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py
index 0fd5d497ab0f77d42019d9200ae1268f64b644d9..3f29aad1e834ce6c99a6093c1ea9ed6aa1b3f768 100644
--- a/synapse/storage/prepare_database.py
+++ b/synapse/storage/prepare_database.py
@@ -50,7 +50,7 @@ class UpgradeDatabaseException(PrepareDatabaseException):
pass
-def prepare_database(db_conn, database_engine):
+def prepare_database(db_conn, database_engine, config):
"""Prepares a database for usage. Will either create all necessary tables
or upgrade from an older schema version.
"""
@@ -61,10 +61,10 @@ def prepare_database(db_conn, database_engine):
if version_info:
user_version, delta_files, upgraded = version_info
_upgrade_existing_database(
- cur, user_version, delta_files, upgraded, database_engine
+ cur, user_version, delta_files, upgraded, database_engine, config
)
else:
- _setup_new_database(cur, database_engine)
+ _setup_new_database(cur, database_engine, config)
# cur.execute("PRAGMA user_version = %d" % (SCHEMA_VERSION,))
@@ -75,7 +75,7 @@ def prepare_database(db_conn, database_engine):
raise
-def _setup_new_database(cur, database_engine):
+def _setup_new_database(cur, database_engine, config):
"""Sets up the database by finding a base set of "full schemas" and then
applying any necessary deltas.
@@ -148,11 +148,12 @@ def _setup_new_database(cur, database_engine):
applied_delta_files=[],
upgraded=False,
database_engine=database_engine,
+ config=config,
)
def _upgrade_existing_database(cur, current_version, applied_delta_files,
- upgraded, database_engine):
+ upgraded, database_engine, config):
"""Upgrades an existing database.
Delta files can either be SQL stored in *.sql files, or python modules
@@ -245,7 +246,7 @@ def _upgrade_existing_database(cur, current_version, applied_delta_files,
module_name, absolute_path, python_file
)
logger.debug("Running script %s", relative_path)
- module.run_upgrade(cur, database_engine)
+ module.run_upgrade(cur, database_engine, config=config)
elif ext == ".pyc":
# Sometimes .pyc files turn up anyway even though we've
# disabled their generation; e.g. from distribution package
diff --git a/synapse/storage/schema/delta/30/alias_creator.sql b/synapse/storage/schema/delta/30/alias_creator.sql
new file mode 100644
index 0000000000000000000000000000000000000000..c9d0dde638db809b0ab77d5b1f6301cb86abd605
--- /dev/null
+++ b/synapse/storage/schema/delta/30/alias_creator.sql
@@ -0,0 +1,16 @@
+/* Copyright 2016 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ALTER TABLE room_aliases ADD COLUMN creator TEXT;
diff --git a/synapse/storage/schema/delta/30/as_users.py b/synapse/storage/schema/delta/30/as_users.py
new file mode 100644
index 0000000000000000000000000000000000000000..4cf4dd09173014f16891a0f16fde8e7bb83481df
--- /dev/null
+++ b/synapse/storage/schema/delta/30/as_users.py
@@ -0,0 +1,59 @@
+# Copyright 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+from synapse.storage.appservice import ApplicationServiceStore
+
+
+logger = logging.getLogger(__name__)
+
+
+def run_upgrade(cur, database_engine, config, *args, **kwargs):
+ # NULL indicates user was not registered by an appservice.
+ cur.execute("ALTER TABLE users ADD COLUMN appservice_id TEXT")
+
+ cur.execute("SELECT name FROM users")
+ rows = cur.fetchall()
+
+ config_files = []
+ try:
+ config_files = config.app_service_config_files
+ except AttributeError:
+ logger.warning("Could not get app_service_config_files from config")
+ pass
+
+ appservices = ApplicationServiceStore.load_appservices(
+ config.server_name, config_files
+ )
+
+ owned = {}
+
+ for row in rows:
+ user_id = row[0]
+ for appservice in appservices:
+ if appservice.is_exclusive_user(user_id):
+ if user_id in owned.keys():
+ logger.error(
+ "user_id %s was owned by more than one application"
+ " service (IDs %s and %s); assigning arbitrarily to %s" %
+ (user_id, owned[user_id], appservice.id, owned[user_id])
+ )
+ owned[user_id] = appservice.id
+
+ for user_id, as_id in owned.items():
+ cur.execute(
+ database_engine.convert_param_style(
+ "UPDATE users SET appservice_id = ? WHERE name = ?"
+ ),
+ (as_id, user_id)
+ )
diff --git a/synapse/util/caches/descriptors.py b/synapse/util/caches/descriptors.py
index 277854ccbc999612f448a06b71436d8f6f737b18..35544b19fd121aa96265d768ebfc3ac0f40c3207 100644
--- a/synapse/util/caches/descriptors.py
+++ b/synapse/util/caches/descriptors.py
@@ -28,6 +28,7 @@ from twisted.internet import defer
from collections import OrderedDict
+import os
import functools
import inspect
import threading
@@ -38,6 +39,9 @@ logger = logging.getLogger(__name__)
_CacheSentinel = object()
+CACHE_SIZE_FACTOR = float(os.environ.get("SYNAPSE_CACHE_FACTOR", 0.1))
+
+
class Cache(object):
def __init__(self, name, max_entries=1000, keylen=1, lru=True, tree=False):
@@ -140,6 +144,8 @@ class CacheDescriptor(object):
"""
def __init__(self, orig, max_entries=1000, num_args=1, lru=True, tree=False,
inlineCallbacks=False):
+ max_entries = int(max_entries * CACHE_SIZE_FACTOR)
+
self.orig = orig
if inlineCallbacks:
diff --git a/synapse/util/caches/stream_change_cache.py b/synapse/util/caches/stream_change_cache.py
index 970488a19c8e707ae28b5a057e80202e57e0795f..a1aec7aa55f7085553abc9e162ef109abe42d7ef 100644
--- a/synapse/util/caches/stream_change_cache.py
+++ b/synapse/util/caches/stream_change_cache.py
@@ -18,11 +18,15 @@ from synapse.util.caches import cache_counter, caches_by_name
from blist import sorteddict
import logging
+import os
logger = logging.getLogger(__name__)
+CACHE_SIZE_FACTOR = float(os.environ.get("SYNAPSE_CACHE_FACTOR", 0.1))
+
+
class StreamChangeCache(object):
"""Keeps track of the stream positions of the latest change in a set of entities.
@@ -33,7 +37,7 @@ class StreamChangeCache(object):
old then the cache will simply return all given entities.
"""
def __init__(self, name, current_stream_pos, max_size=10000, prefilled_cache={}):
- self._max_size = max_size
+ self._max_size = int(max_size * CACHE_SIZE_FACTOR)
self._entity_to_key = {}
self._cache = sorteddict()
self._earliest_known_stream_pos = current_stream_pos
diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py
index a87703bbfd6260c0e5767258027190911240474d..4f2c14e4ff7c464df997c0bd053253abb43c2d91 100644
--- a/tests/handlers/test_profile.py
+++ b/tests/handlers/test_profile.py
@@ -23,7 +23,7 @@ from synapse.api.errors import AuthError
from synapse.handlers.profile import ProfileHandler
from synapse.types import UserID
-from tests.utils import setup_test_homeserver
+from tests.utils import setup_test_homeserver, requester_for_user
class ProfileHandlers(object):
@@ -84,7 +84,11 @@ class ProfileTestCase(unittest.TestCase):
@defer.inlineCallbacks
def test_set_my_name(self):
- yield self.handler.set_displayname(self.frank, self.frank, "Frank Jr.")
+ yield self.handler.set_displayname(
+ self.frank,
+ requester_for_user(self.frank),
+ "Frank Jr."
+ )
self.assertEquals(
(yield self.store.get_profile_displayname(self.frank.localpart)),
@@ -93,7 +97,11 @@ class ProfileTestCase(unittest.TestCase):
@defer.inlineCallbacks
def test_set_my_name_noauth(self):
- d = self.handler.set_displayname(self.frank, self.bob, "Frank Jr.")
+ d = self.handler.set_displayname(
+ self.frank,
+ requester_for_user(self.bob),
+ "Frank Jr."
+ )
yield self.assertFailure(d, AuthError)
@@ -136,7 +144,7 @@ class ProfileTestCase(unittest.TestCase):
@defer.inlineCallbacks
def test_set_my_avatar(self):
yield self.handler.set_avatar_url(
- self.frank, self.frank, "http://my.server/pic.gif"
+ self.frank, requester_for_user(self.frank), "http://my.server/pic.gif"
)
self.assertEquals(
diff --git a/tests/replication/test_resource.py b/tests/replication/test_resource.py
index a30d59a8655115aeb29d16d433351a8c01c443b1..4a42eb33652ccc340081b37af8ad8fa6558f71f0 100644
--- a/tests/replication/test_resource.py
+++ b/tests/replication/test_resource.py
@@ -18,7 +18,7 @@ from synapse.types import Requester, UserID
from twisted.internet import defer
from tests import unittest
-from tests.utils import setup_test_homeserver
+from tests.utils import setup_test_homeserver, requester_for_user
from mock import Mock, NonCallableMock
import json
import contextlib
@@ -135,12 +135,15 @@ class ReplicationResourceCase(unittest.TestCase):
@defer.inlineCallbacks
def send_text_message(self, room_id, message):
handler = self.hs.get_handlers().message_handler
- event = yield handler.create_and_send_nonmember_event({
- "type": "m.room.message",
- "content": {"body": "message", "msgtype": "m.text"},
- "room_id": room_id,
- "sender": self.user.to_string(),
- })
+ event = yield handler.create_and_send_nonmember_event(
+ requester_for_user(self.user),
+ {
+ "type": "m.room.message",
+ "content": {"body": "message", "msgtype": "m.text"},
+ "room_id": room_id,
+ "sender": self.user.to_string(),
+ }
+ )
defer.returnValue(event.event_id)
@defer.inlineCallbacks
diff --git a/tests/rest/client/v1/test_profile.py b/tests/rest/client/v1/test_profile.py
index 0785965de25181d103e6a0f77b5d5222ad603bdd..1d210f9bf8ca4b1555dfe6abdb6a1555056d6e64 100644
--- a/tests/rest/client/v1/test_profile.py
+++ b/tests/rest/client/v1/test_profile.py
@@ -86,7 +86,7 @@ class ProfileTestCase(unittest.TestCase):
self.assertEquals(200, code)
self.assertEquals(mocked_set.call_args[0][0].localpart, "1234ABCD")
- self.assertEquals(mocked_set.call_args[0][1].localpart, "1234ABCD")
+ self.assertEquals(mocked_set.call_args[0][1].user.localpart, "1234ABCD")
self.assertEquals(mocked_set.call_args[0][2], "Frank Jr.")
@defer.inlineCallbacks
@@ -155,5 +155,5 @@ class ProfileTestCase(unittest.TestCase):
self.assertEquals(200, code)
self.assertEquals(mocked_set.call_args[0][0].localpart, "1234ABCD")
- self.assertEquals(mocked_set.call_args[0][1].localpart, "1234ABCD")
+ self.assertEquals(mocked_set.call_args[0][1].user.localpart, "1234ABCD")
self.assertEquals(mocked_set.call_args[0][2], "http://my.server/pic.gif")
diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py
index ed8af10d87f9df66c5656ee312801aead1a7a1e0..5734198121c3fb054f574f12a453e904552ea8f3 100644
--- a/tests/storage/test_appservice.py
+++ b/tests/storage/test_appservice.py
@@ -35,7 +35,8 @@ class ApplicationServiceStoreTestCase(unittest.TestCase):
def setUp(self):
self.as_yaml_files = []
config = Mock(
- app_service_config_files=self.as_yaml_files
+ app_service_config_files=self.as_yaml_files,
+ event_cache_size=1,
)
hs = yield setup_test_homeserver(config=config)
@@ -109,7 +110,8 @@ class ApplicationServiceTransactionStoreTestCase(unittest.TestCase):
self.as_yaml_files = []
config = Mock(
- app_service_config_files=self.as_yaml_files
+ app_service_config_files=self.as_yaml_files,
+ event_cache_size=1,
)
hs = yield setup_test_homeserver(config=config)
self.db_pool = hs.get_db_pool()
@@ -438,7 +440,7 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase):
f1 = self._write_config(suffix="1")
f2 = self._write_config(suffix="2")
- config = Mock(app_service_config_files=[f1, f2])
+ config = Mock(app_service_config_files=[f1, f2], event_cache_size=1)
hs = yield setup_test_homeserver(config=config, datastore=Mock())
ApplicationServiceStore(hs)
@@ -448,7 +450,7 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase):
f1 = self._write_config(id="id", suffix="1")
f2 = self._write_config(id="id", suffix="2")
- config = Mock(app_service_config_files=[f1, f2])
+ config = Mock(app_service_config_files=[f1, f2], event_cache_size=1)
hs = yield setup_test_homeserver(config=config, datastore=Mock())
with self.assertRaises(ConfigError) as cm:
@@ -464,7 +466,7 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase):
f1 = self._write_config(as_token="as_token", suffix="1")
f2 = self._write_config(as_token="as_token", suffix="2")
- config = Mock(app_service_config_files=[f1, f2])
+ config = Mock(app_service_config_files=[f1, f2], event_cache_size=1)
hs = yield setup_test_homeserver(config=config, datastore=Mock())
with self.assertRaises(ConfigError) as cm:
diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py
index c76545be65bc9e77dd8aea5e4b9df0a90f7bf39f..2e33beb07c952ae9aa18538c4573b6e182a96b88 100644
--- a/tests/storage/test_base.py
+++ b/tests/storage/test_base.py
@@ -48,11 +48,12 @@ class SQLBaseStoreTestCase(unittest.TestCase):
config = Mock()
config.event_cache_size = 1
+ config.database_config = {"name": "sqlite3"}
hs = HomeServer(
"test",
db_pool=self.db_pool,
config=config,
- database_engine=create_engine("sqlite3"),
+ database_engine=create_engine(config),
)
self.datastore = SQLBaseStore(hs)
diff --git a/tests/utils.py b/tests/utils.py
index dfbee5c23ae48d93d17154d77e962b1578095ed6..52405502e9fdeee868224eba414d2ac43f630dc5 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -20,6 +20,7 @@ from synapse.storage.prepare_database import prepare_database
from synapse.storage.engines import create_engine
from synapse.server import HomeServer
from synapse.federation.transport import server
+from synapse.types import Requester
from synapse.util.ratelimitutils import FederationRateLimiter
from synapse.util.logcontext import LoggingContext
@@ -50,6 +51,9 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs):
config.macaroon_secret_key = "not even a little secret"
config.server_name = "server.under.test"
config.trusted_third_party_id_servers = []
+ config.room_invite_state_types = []
+
+ config.database_config = {"name": "sqlite3"}
if "clock" not in kargs:
kargs["clock"] = MockClock()
@@ -60,7 +64,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs):
hs = HomeServer(
name, db_pool=db_pool, config=config,
version_string="Synapse/tests",
- database_engine=create_engine("sqlite3"),
+ database_engine=create_engine(config),
get_db_conn=db_pool.get_db_conn,
**kargs
)
@@ -69,7 +73,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs):
hs = HomeServer(
name, db_pool=None, datastore=datastore, config=config,
version_string="Synapse/tests",
- database_engine=create_engine("sqlite3"),
+ database_engine=create_engine(config),
**kargs
)
@@ -278,18 +282,24 @@ class SQLiteMemoryDbPool(ConnectionPool, object):
cp_max=1,
)
+ self.config = Mock()
+ self.config.database_config = {"name": "sqlite3"}
+
def prepare(self):
- engine = create_engine("sqlite3")
+ engine = self.create_engine()
return self.runWithConnection(
- lambda conn: prepare_database(conn, engine)
+ lambda conn: prepare_database(conn, engine, self.config)
)
def get_db_conn(self):
conn = self.connect()
- engine = create_engine("sqlite3")
- prepare_database(conn, engine)
+ engine = self.create_engine()
+ prepare_database(conn, engine, self.config)
return conn
+ def create_engine(self):
+ return create_engine(self.config)
+
class MemoryDataStore(object):
@@ -502,3 +512,7 @@ class DeferredMockCallable(object):
"call(%s)" % _format_call(c[0], c[1]) for c in calls
])
)
+
+
+def requester_for_user(user):
+ return Requester(user, None, False)