Commit c31e324e81acb92090b208cfb402253e29297e4f - matrix-synapse

+0

-48

~~.buildkite/format_tap.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2019 The Matrix.org Foundation C.I.C.
2		#
3		# Licensed under the Apache License, Version 2.0 (the "License");
4		# you may not use this file except in compliance with the License.
5		# You may obtain a copy of the License at
6		#
7		# http://www.apache.org/licenses/LICENSE-2.0
8		#
9		# Unless required by applicable law or agreed to in writing, software
10		# distributed under the License is distributed on an "AS IS" BASIS,
11		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12		# See the License for the specific language governing permissions and
13		# limitations under the License.
14
15		import sys
16		from tap.parser import Parser
17		from tap.line import Result, Unknown, Diagnostic
18
19		out = ["### TAP Output for " + sys.argv[2]]
20
21		p = Parser()
22
23		in_error = False
24
25		for line in p.parse_file(sys.argv[1]):
26		if isinstance(line, Result):
27		if in_error:
28		out.append("")
29		out.append("</pre></code></details>")
30		out.append("")
31		out.append("----")
32		out.append("")
33		in_error = False
34
35		if not line.ok and not line.todo:
36		in_error = True
37
38		out.append("FAILURE Test #%d: ``%s``" % (line.number, line.description))
39		out.append("")
40		out.append("<details><summary>Show log</summary><code><pre>")
41
42		elif isinstance(line, Diagnostic) and in_error:
43		out.append(line.text)
44
45		if out:
46		for line in out[:-3]:
47		print(line)

+13

-9

.github/ISSUE_TEMPLATE/BUG_REPORT.md less more

6	6	<!--
7	7
8	8	IF YOU HAVE SUPPORT QUESTIONS ABOUT RUNNING OR CONFIGURING YOUR OWN HOME SERVER:
9		You will likely get better support more quickly if you ask in #matrix:matrix.org ;)
	9	You will likely get better support more quickly if you ask in #synapse:matrix.org ;)
10	10
11	11
12	12	This is a bug report template. By following the instructions below and

43	43	<!-- IMPORTANT: please answer the following questions, to help us narrow down the problem -->
44	44
45	45	<!-- Was this issue identified on matrix.org or another homeserver? -->
46		- Homeserver:
	46	- Homeserver:
47	47
48	48	If not matrix.org:
49	49
50	50	<!--
51		What version of Synapse is running?
52		You can find the Synapse version by inspecting the server headers (replace matrix.org with
53		your own homeserver domain):
54		$ curl -v https://matrix.org/_matrix/client/versions 2>&1 \| grep "Server:"
	51	What version of Synapse is running?
	52
	53	You can find the Synapse version with this command:
	54
	55	$ curl http://localhost:8008/_synapse/admin/v1/server_version
	56
	57	(You may need to replace `localhost:8008` if Synapse is not configured to
	58	listen on that port.)
55	59	-->
56		- Version:
	60	- Version:
57	61
58		- Install method:
	62	- Install method:
59	63	<!-- examples: package manager/git clone/pip -->
60	64
61		- Platform:
	65	- Platform:
62	66	<!--
63	67	Tell us about the environment in which your homeserver is operating
64	68	distro, hardware, if it's running in a vm/container, etc.

+2

-0

.gitignore less more

6	6	*.egg-info
7	7	*.lock
8	8	*.pyc
	9	*.snap
9	10	*.tac
10	11	_trial_temp/
11	12	_trial_temp*/
	13	/out
12	14
13	15	# stuff that is likely to exist when you run a server locally
14	16	/*.db

+96

-0

CHANGES.md less more

	0	Synapse 1.5.0rc1 (2019-10-24)
	1	==========================
	2
	3	This release includes a database migration step which may take a long time to complete:
	4
	5	- Allow devices to be marked as hidden, for use by features such as cross-signing.
	6	This adds a new field with a default value to the devices field in the database,
	7	and so the database upgrade may take a long time depending on how many devices
	8	are in the database. ([\#5759](https://github.com/matrix-org/synapse/issues/5759))
	9
	10	Features
	11	--------
	12
	13	- Improve quality of thumbnails for 1-bit/8-bit color palette images. ([\#2142](https://github.com/matrix-org/synapse/issues/2142))
	14	- Add ability to upload cross-signing signatures. ([\#5726](https://github.com/matrix-org/synapse/issues/5726))
	15	- Allow uploading of cross-signing keys. ([\#5769](https://github.com/matrix-org/synapse/issues/5769))
	16	- CAS login now provides a default display name for users if a `displayname_attribute` is set in the configuration file. ([\#6114](https://github.com/matrix-org/synapse/issues/6114))
	17	- Reject all pending invites for a user during deactivation. ([\#6125](https://github.com/matrix-org/synapse/issues/6125))
	18	- Add config option to suppress client side resource limit alerting. ([\#6173](https://github.com/matrix-org/synapse/issues/6173))
	19
	20
	21	Bugfixes
	22	--------
	23
	24	- Return an HTTP 404 instead of 400 when requesting a filter by ID that is unknown to the server. Thanks to @krombel for contributing this! ([\#2380](https://github.com/matrix-org/synapse/issues/2380))
	25	- Fix a bug where users could be invited twice to the same group. ([\#3436](https://github.com/matrix-org/synapse/issues/3436))
	26	- Fix `/createRoom` failing with badly-formatted MXIDs in the invitee list. Thanks to @wener291! ([\#4088](https://github.com/matrix-org/synapse/issues/4088))
	27	- Make the `synapse_port_db` script create the right indexes on a new PostgreSQL database. ([\#6102](https://github.com/matrix-org/synapse/issues/6102), [\#6178](https://github.com/matrix-org/synapse/issues/6178), [\#6243](https://github.com/matrix-org/synapse/issues/6243))
	28	- Fix bug when uploading a large file: Synapse responds with `M_UNKNOWN` while it should be `M_TOO_LARGE` according to spec. Contributed by Anshul Angaria. ([\#6109](https://github.com/matrix-org/synapse/issues/6109))
	29	- Fix user push rules being deleted from a room when it is upgraded. ([\#6144](https://github.com/matrix-org/synapse/issues/6144))
	30	- Don't 500 when trying to exchange a revoked 3PID invite. ([\#6147](https://github.com/matrix-org/synapse/issues/6147))
	31	- Fix transferring notifications and tags when joining an upgraded room that is new to your server. ([\#6155](https://github.com/matrix-org/synapse/issues/6155))
	32	- Fix bug where guest account registration can wedge after restart. ([\#6161](https://github.com/matrix-org/synapse/issues/6161))
	33	- Fix monthly active user reaping when reserved users are specified. ([\#6168](https://github.com/matrix-org/synapse/issues/6168))
	34	- Fix `/federation/v1/state` endpoint not supporting newer room versions. ([\#6170](https://github.com/matrix-org/synapse/issues/6170))
	35	- Fix bug where we were updating censored events as bytes rather than text, occaisonally causing invalid JSON being inserted breaking APIs that attempted to fetch such events. ([\#6186](https://github.com/matrix-org/synapse/issues/6186))
	36	- Fix occasional missed updates in the room and user directories. ([\#6187](https://github.com/matrix-org/synapse/issues/6187))
	37	- Fix tracing of non-JSON APIs, `/media`, `/key` etc. ([\#6195](https://github.com/matrix-org/synapse/issues/6195))
	38	- Fix bug where presence would not get timed out correctly if a synchrotron worker is used and restarted. ([\#6212](https://github.com/matrix-org/synapse/issues/6212))
	39	- synapse_port_db: Add 2 additional BOOLEAN_COLUMNS to be able to convert from database schema v56. ([\#6216](https://github.com/matrix-org/synapse/issues/6216))
	40	- Fix a bug where the Synapse demo script blacklisted `::1` (ipv6 localhost) from receiving federation traffic. ([\#6229](https://github.com/matrix-org/synapse/issues/6229))
	41
	42
	43	Updates to the Docker image
	44	---------------------------
	45
	46	- Fix logging getting lost for the docker image. ([\#6197](https://github.com/matrix-org/synapse/issues/6197))
	47
	48
	49	Internal Changes
	50	----------------
	51
	52	- Update `user_filters` table to have a unique index, and non-null columns. Thanks to @pik for contributing this. ([\#1172](https://github.com/matrix-org/synapse/issues/1172), [\#6175](https://github.com/matrix-org/synapse/issues/6175), [\#6184](https://github.com/matrix-org/synapse/issues/6184))
	53	- Move lookup-related functions from RoomMemberHandler to IdentityHandler. ([\#5978](https://github.com/matrix-org/synapse/issues/5978))
	54	- Improve performance of the public room list directory. ([\#6019](https://github.com/matrix-org/synapse/issues/6019), [\#6152](https://github.com/matrix-org/synapse/issues/6152), [\#6153](https://github.com/matrix-org/synapse/issues/6153), [\#6154](https://github.com/matrix-org/synapse/issues/6154))
	55	- Edit header dicts docstrings in `SimpleHttpClient` to note that `str` or `bytes` can be passed as header keys. ([\#6077](https://github.com/matrix-org/synapse/issues/6077))
	56	- Add snapcraft packaging information. Contributed by @devec0. ([\#6084](https://github.com/matrix-org/synapse/issues/6084), [\#6191](https://github.com/matrix-org/synapse/issues/6191))
	57	- Kill off half-implemented password-reset via sms. ([\#6101](https://github.com/matrix-org/synapse/issues/6101))
	58	- Remove `get_user_by_req` opentracing span and add some tags. ([\#6108](https://github.com/matrix-org/synapse/issues/6108))
	59	- Drop some unused database tables. ([\#6115](https://github.com/matrix-org/synapse/issues/6115))
	60	- Add env var to turn on tracking of log context changes. ([\#6127](https://github.com/matrix-org/synapse/issues/6127))
	61	- Refactor configuration loading to allow better typechecking. ([\#6137](https://github.com/matrix-org/synapse/issues/6137))
	62	- Log responder when responding to media request. ([\#6139](https://github.com/matrix-org/synapse/issues/6139))
	63	- Improve performance of `find_next_generated_user_id` DB query. ([\#6148](https://github.com/matrix-org/synapse/issues/6148))
	64	- Expand type-checking on modules imported by `synapse.config`. ([\#6150](https://github.com/matrix-org/synapse/issues/6150))
	65	- Use Postgres ANY for selecting many values. ([\#6156](https://github.com/matrix-org/synapse/issues/6156))
	66	- Add more caching to `_get_joined_users_from_context` DB query. ([\#6159](https://github.com/matrix-org/synapse/issues/6159))
	67	- Add some metrics on the federation sender. ([\#6160](https://github.com/matrix-org/synapse/issues/6160))
	68	- Add some logging to the rooms stats updates, to try to track down a flaky test. ([\#6167](https://github.com/matrix-org/synapse/issues/6167))
	69	- Remove unused `timeout` parameter from `_get_public_room_list`. ([\#6179](https://github.com/matrix-org/synapse/issues/6179))
	70	- Reject (accidental) attempts to insert bytes into postgres tables. ([\#6186](https://github.com/matrix-org/synapse/issues/6186))
	71	- Make `version` optional in body of `PUT /room_keys/version/{version}`, since it's redundant. ([\#6189](https://github.com/matrix-org/synapse/issues/6189))
	72	- Make storage layer responsible for adding device names to key, rather than the handler. ([\#6193](https://github.com/matrix-org/synapse/issues/6193))
	73	- Port `synapse.rest.admin` module to use async/await. ([\#6196](https://github.com/matrix-org/synapse/issues/6196))
	74	- Enforce that all boolean configuration values are lowercase in CI. ([\#6203](https://github.com/matrix-org/synapse/issues/6203))
	75	- Remove some unused event-auth code. ([\#6214](https://github.com/matrix-org/synapse/issues/6214))
	76	- Remove `Auth.check` method. ([\#6217](https://github.com/matrix-org/synapse/issues/6217))
	77	- Remove `format_tap.py` script in favour of a perl reimplementation in Sytest's repo. ([\#6219](https://github.com/matrix-org/synapse/issues/6219))
	78	- Refactor storage layer in preparation to support having multiple databases. ([\#6231](https://github.com/matrix-org/synapse/issues/6231))
	79	- Remove some extra quotation marks across the codebase. ([\#6236](https://github.com/matrix-org/synapse/issues/6236))
	80
	81
	82	Synapse 1.4.1 (2019-10-18)
	83	==========================
	84
	85	No changes since 1.4.1rc1.
	86
	87
	88	Synapse 1.4.1rc1 (2019-10-17)
	89	=============================
	90
	91	Bugfixes
	92	--------
	93
	94	- Fix bug where redacted events were sometimes incorrectly censored in the database, breaking APIs that attempted to fetch such events. ([\#6185](https://github.com/matrix-org/synapse/issues/6185), [5b0e9948](https://github.com/matrix-org/synapse/commit/5b0e9948eaae801643e594b5abc8ee4b10bd194e))
	95
0	96	Synapse 1.4.0 (2019-10-03)
1	97	==========================
2	98

+7

-0

INSTALL.md less more

347	347	sudo pip uninstall py-bcrypt
348	348	sudo pip install py-bcrypt
349	349	```
	350
	351	### Void Linux
	352
	353	Synapse can be found in the void repositories as 'synapse':
	354
	355	xbps-install -Su
	356	xbps-install -S synapse
350	357
351	358	### FreeBSD
352	359

+7

-8

MANIFEST.in less more

7	7	include demo/*.py
8	8	include demo/*.sh
9	9
10		recursive-include synapse/storage/schema *.sql
11		recursive-include synapse/storage/schema *.sql.postgres
12		recursive-include synapse/storage/schema *.sql.sqlite
13		recursive-include synapse/storage/schema *.py
14		recursive-include synapse/storage/schema *.txt
	10	recursive-include synapse/storage *.sql
	11	recursive-include synapse/storage *.sql.postgres
	12	recursive-include synapse/storage *.sql.sqlite
	13	recursive-include synapse/storage *.py
	14	recursive-include synapse/storage *.txt
	15	recursive-include synapse/storage *.md
15	16
16	17	recursive-include docs *
17	18	recursive-include scripts *

46	47	prune demo/etc
47	48	prune docker
48	49	prune mypy.ini
	50	prune snap
49	51	prune stubs
50
51		exclude jenkins*
52		recursive-exclude jenkins *.sh

+13

-0

README.rst less more

380	380	requests than can be accounted for by your users' activity, this is a
381	381	likely cause. The misbehavior can be worked around by setting
382	382	``use_presence: false`` in the Synapse config file.
	383
	384	People can't accept room invitations from me
	385	--------------------------------------------
	386
	387	The typical failure mode here is that you send an invitation to someone
	388	to join a room or direct chat, but when they go to accept it, they get an
	389	error (typically along the lines of "Invalid signature"). They might see
	390	something like the following in their logs::
	391
	392	2019-09-11 19:32:04,271 - synapse.federation.transport.server - 288 - WARNING - GET-11752 - authenticate_request failed: 401: Invalid signature for server <server> with key ed25519:a_EqML: Unable to verify signature for <server>
	393
	394	This is normally caused by a misconfiguration in your reverse-proxy. See
	395	`<docs/reverse_proxy.rst>`_ and double-check that your settings are correct.

+11

-24

contrib/docker/README.md less more

	0
0	1	# Synapse Docker
1	2
2		FIXME: this is out-of-date as of
3		https://github.com/matrix-org/synapse/issues/5518. Contributions to bring it up
4		to date would be welcome.
5
6		### Automated configuration
7
8		It is recommended that you use Docker Compose to run your containers, including
9		this image and a Postgres server. A sample ``docker-compose.yml`` is provided,
10		including example labels for reverse proxying and other artifacts.
11
12		Read the section about environment variables and set at least mandatory variables,
13		then run the server:
14
15		```
16		docker-compose up -d
17		```
18
19		If secrets are not specified in the environment variables, they will be generated
20		as part of the startup. Please ensure these secrets are kept between launches of the
21		Docker container, as their loss may require users to log in again.
22
23		### Manual configuration
	3	### Configuration
24	4
25	5	A sample ``docker-compose.yml`` is provided, including example labels for
26	6	reverse proxying and other artifacts. The docker-compose file is an example,
27	7	please comment/uncomment sections that are not suitable for your usecase.
28	8
29	9	Specify a ``SYNAPSE_CONFIG_PATH``, preferably to a persistent path,
30		to use manual configuration. To generate a fresh ``homeserver.yaml``, simply run:
	10	to use manual configuration.
	11
	12	To generate a fresh `homeserver.yaml`, you can use the `generate` command.
	13	(See the [documentation](../../docker/README.md#generating-a-configuration-file)
	14	for more information.) You will need to specify appropriate values for at least the
	15	`SYNAPSE_SERVER_NAME` and `SYNAPSE_REPORT_STATS` environment variables. For example:
31	16
32	17	```
33		docker-compose run --rm -e SYNAPSE_SERVER_NAME=my.matrix.host synapse generate
	18	docker-compose run --rm -e SYNAPSE_SERVER_NAME=my.matrix.host -e SYNAPSE_REPORT_STATS=yes synapse generate
34	19	```
	20
	21	(This will also generate necessary signing keys.)
35	22
36	23	Then, customize your configuration and run the server:
37	24

+16

-5

contrib/docker/docker-compose.yml less more

14	14	restart: unless-stopped
15	15	# See the readme for a full documentation of the environment settings
16	16	environment:
17		- SYNAPSE_SERVER_NAME=my.matrix.host
18		- SYNAPSE_REPORT_STATS=no
19		- SYNAPSE_ENABLE_REGISTRATION=yes
20		- SYNAPSE_LOG_LEVEL=INFO
21		- POSTGRES_PASSWORD=changeme
	17	- SYNAPSE_CONFIG_PATH=/etc/homeserver.yaml
22	18	volumes:
23	19	# You may either store all the files in a local folder
	20	- ./matrix-config:/etc
24	21	- ./files:/data
25	22	# .. or you may split this between different storage points
26	23	# - ./files:/data

34	31	- 8448:8448/tcp
35	32	# ... or use a reverse proxy, here is an example for traefik:
36	33	labels:
	34	# The following lines are valid for Traefik version 1.x:
37	35	- traefik.enable=true
38	36	- traefik.frontend.rule=Host:my.matrix.Host
39	37	- traefik.port=8008
	38	# Alternatively, for Traefik version 2.0:
	39	- traefik.enable=true
	40	- traefik.http.routers.http-synapse.entryPoints=http
	41	- traefik.http.routers.http-synapse.rule=Host(`my.matrix.host`)
	42	- traefik.http.middlewares.https_redirect.redirectscheme.scheme=https
	43	- traefik.http.middlewares.https_redirect.redirectscheme.permanent=true
	44	- traefik.http.routers.http-synapse.middlewares=https_redirect
	45	- traefik.http.routers.https-synapse.entryPoints=https
	46	- traefik.http.routers.https-synapse.rule=Host(`my.matrix.host`)
	47	- traefik.http.routers.https-synapse.service=synapse
	48	- traefik.http.routers.https-synapse.tls=true
	49	- traefik.http.services.synapse.loadbalancer.server.port=8008
	50	- traefik.http.routers.https-synapse.tls.certResolver=le-ssl
40	51
41	52	db:
42	53	image: docker.io/postgres:10-alpine

+1

-1

contrib/experiments/test_messaging.py less more

338	338	root_logger = logging.getLogger()
339	339
340	340	formatter = logging.Formatter(
341		"%(asctime)s - %(name)s - %(lineno)d - " "%(levelname)s - %(message)s"
	341	"%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s"
342	342	)
343	343	if not os.path.exists("logs"):
344	344	os.makedirs("logs")

+2

-2

contrib/graph/graph2.py less more

35	35	args = [room_id]
36	36
37	37	if limit:
38		sql += " ORDER BY topological_ordering DESC, stream_ordering DESC " "LIMIT ?"
	38	sql += " ORDER BY topological_ordering DESC, stream_ordering DESC LIMIT ?"
39	39
40	40	args.append(limit)
41	41

52	52
53	53	for event in events:
54	54	c = conn.execute(
55		"SELECT state_group FROM event_to_state_groups " "WHERE event_id = ?",
	55	"SELECT state_group FROM event_to_state_groups WHERE event_id = ?",
56	56	(event.event_id,),
57	57	)
58	58

+6

-0

debian/changelog less more

	0	matrix-synapse-py3 (1.4.1) stable; urgency=medium
	1
	2	* New synapse release 1.4.1.
	3
	4	-- Synapse Packaging team <packages@matrix.org> Fri, 18 Oct 2019 10:13:27 +0100
	5
0	6	matrix-synapse-py3 (1.4.0) stable; urgency=medium
1	7
2	8	* New synapse release 1.4.0.

+1

-2

demo/start.sh less more

76	76
77	77	# Reduce the blacklist
78	78	blacklist=$(cat <<-BLACK
79		# Set the blacklist so that it doesn't include 127.0.0.1
	79	# Set the blacklist so that it doesn't include 127.0.0.1, ::1
80	80	federation_ip_range_blacklist:
81	81	- '10.0.0.0/8'
82	82	- '172.16.0.0/12'
83	83	- '192.168.0.0/16'
84	84	- '100.64.0.0/10'
85	85	- '169.254.0.0/16'
86		- '::1/128'
87	86	- 'fe80::/64'
88	87	- 'fc00::/7'
89	88	BLACK

+2

-0

docker/conf/log.config less more

23	23	root:
24	24	level: {{ SYNAPSE_LOG_LEVEL or "INFO" }}
25	25	handlers: [console]
	26
	27	disable_existing_loggers: false

+12

-0

docs/admin_api/README.rst less more

9	9	``UPDATE users SET admin = 1 WHERE name = '@foo:bar.com'``
10	10
11	11	Restarting may be required for the changes to register.
	12
	13	Using an admin access_token
	14	###########################
	15
	16	Many of the API calls listed in the documentation here will require to include an admin `access_token`.
	17	Finding your user's `access_token` is client-dependent, but will usually be shown in the client's settings.
	18
	19	Once you have your `access_token`, to include it in a request, the best option is to add the token to a request header:
	20
	21	``curl --header "Authorization: Bearer <access_token>" <the_rest_of_your_API_request>``
	22
	23	Fore more details, please refer to the complete `matrix spec documentation <https://matrix.org/docs/spec/client_server/r0.5.0#using-access-tokens>`_.

+7

-3

docs/postgres.md less more

26	26
27	27	## Set up database
28	28
29		Assuming your PostgreSQL database user is called `postgres`, create a
30		user `synapse_user` with:
	29	Assuming your PostgreSQL database user is called `postgres`, first authenticate as the database user with:
31	30
32	31	su - postgres
	32	# Or, if your system uses sudo to get administrative rights
	33	sudo -u postgres bash
	34
	35	Then, create a user ``synapse_user`` with:
	36
33	37	createuser --pwprompt synapse_user
34	38
35	39	Before you can authenticate with the `synapse_user`, you must create a
36	40	database that it can access. To create a database, first connect to the
37	41	database with your database user:
38	42
39		su - postgres
	43	su - postgres # Or: sudo -u postgres bash
40	44	psql
41	45
42	46	and then run:

+23

-16

docs/sample_config.yaml less more

85	85	# Whether room invites to users on this server should be blocked
86	86	# (except those sent by local server admins). The default is False.
87	87	#
88		#block_non_admin_invites: True
	88	#block_non_admin_invites: true
89	89
90	90	# Room searching
91	91	#

238	238
239	239	# Global blocking
240	240	#
241		#hs_disabled: False
	241	#hs_disabled: false
242	242	#hs_disabled_message: 'Human readable reason for why the HS is blocked'
243		#hs_disabled_limit_type: 'error code(str), to help clients decode reason'
244	243
245	244	# Monthly Active User Blocking
246	245	#

260	259	# sign up in a short space of time never to return after their initial
261	260	# session.
262	261	#
263		#limit_usage_by_mau: False
	262	# 'mau_limit_alerting' is a means of limiting client side alerting
	263	# should the mau limit be reached. This is useful for small instances
	264	# where the admin has 5 mau seats (say) for 5 specific people and no
	265	# interest increasing the mau limit further. Defaults to True, which
	266	# means that alerting is enabled
	267	#
	268	#limit_usage_by_mau: false
264	269	#max_mau_value: 50
265	270	#mau_trial_days: 2
	271	#mau_limit_alerting: false
266	272
267	273	# If enabled, the metrics for the number of monthly active users will
268	274	# be populated, however no one will be limited. If limit_usage_by_mau
269	275	# is true, this is implied to be true.
270	276	#
271		#mau_stats_only: False
	277	#mau_stats_only: false
272	278
273	279	# Sometimes the server admin will want to ensure certain accounts are
274	280	# never blocked by mau checking. These accounts are specified here.

293	299	#
294	300	# Uncomment the below lines to enable:
295	301	#limit_remote_rooms:
296		# enabled: True
	302	# enabled: true
297	303	# complexity: 1.0
298	304	# complexity_error: "This room is too complex."
299	305

410	416	# ACME support is disabled by default. Set this to `true` and uncomment
411	417	# tls_certificate_path and tls_private_key_path above to enable it.
412	418	#
413		enabled: False
	419	enabled: false
414	420
415	421	# Endpoint to use to request certificates. If you only want to test,
416	422	# use Let's Encrypt's staging url:

785	791	# connect to arbitrary endpoints without having first signed up for a
786	792	# valid account (e.g. by passing a CAPTCHA).
787	793	#
788		#turn_allow_guests: True
	794	#turn_allow_guests: true
789	795
790	796
791	797	## Registration ##

828	834	# where d is equal to 10% of the validity period.
829	835	#
830	836	#account_validity:
831		# enabled: True
	837	# enabled: true
832	838	# period: 6w
833	839	# renew_at: 1w
834	840	# renew_email_subject: "Renew your %(app)s account"

970	976
971	977	# Enable collection and rendering of performance metrics
972	978	#
973		#enable_metrics: False
	979	#enable_metrics: false
974	980
975	981	# Enable sentry integration
976	982	# NOTE: While attempts are made to ensure that the logs don't contain

1022	1028	# Uncomment to enable tracking of application service IP addresses. Implicitly
1023	1029	# enables MAU tracking for application service users.
1024	1030	#
1025		#track_appservice_user_ips: True
	1031	#track_appservice_user_ips: true
1026	1032
1027	1033
1028	1034	# a secret which is used to sign access tokens. If none is specified,

1148	1154	# - url: https://our_idp/metadata.xml
1149	1155	#
1150	1156	# # By default, the user has to go to our login page first. If you'd like
1151		# # to allow IdP-initiated login, set 'allow_unsolicited: True' in a
	1157	# # to allow IdP-initiated login, set 'allow_unsolicited: true' in a
1152	1158	# # 'service.sp' section:
1153	1159	# #
1154	1160	# #service:

1219	1225	# enabled: true
1220	1226	# server_url: "https://cas-server.com"
1221	1227	# service_url: "https://homeserver.domain.com:8448"
	1228	# #displayname_attribute: name
1222	1229	# #required_attributes:
1223	1230	# # name: value
1224	1231

1261	1268	# smtp_port: 25 # SSL: 465, STARTTLS: 587
1262	1269	# smtp_user: "exampleusername"
1263	1270	# smtp_pass: "examplepassword"
1264		# require_transport_security: False
	1271	# require_transport_security: false
1265	1272	# notif_from: "Your Friendly %(app)s Home Server <noreply@example.com>"
1266	1273	# app_name: Matrix
1267	1274	#
1268	1275	# # Enable email notifications by default
1269	1276	# #
1270		# notif_for_new_users: True
	1277	# notif_for_new_users: true
1271	1278	#
1272	1279	# # Defining a custom URL for Riot is only needed if email notifications
1273	1280	# # should contain links to a self-hosted installation of Riot; when set

1445	1452	# body: >-
1446	1453	# To continue using this homeserver you must review and agree to the
1447	1454	# terms and conditions at %(consent_uri)s
1448		# send_server_notice_to_guests: True
	1455	# send_server_notice_to_guests: true
1449	1456	# block_events_error: >-
1450	1457	# To continue using this homeserver you must review and agree to the
1451	1458	# terms and conditions at %(consent_uri)s
1452		# require_at_registration: False
	1459	# require_at_registration: false
1453	1460	# policy_name: Privacy Policy
1454	1461	#
1455	1462

+12

-4

mypy.ini less more

2	2	plugins=mypy_zope:plugin
3	3	follow_imports=skip
4	4	mypy_path=stubs
5
6		[mypy-synapse.config.homeserver]
7		# this is a mess because of the metaclass shenanigans
8		ignore_errors = True
9	5
10	6	[mypy-zope]
11	7	ignore_missing_imports = True

51	47
52	48	[mypy-signedjson.*]
53	49	ignore_missing_imports = True
	50
	51	[mypy-prometheus_client.*]
	52	ignore_missing_imports = True
	53
	54	[mypy-service_identity.*]
	55	ignore_missing_imports = True
	56
	57	[mypy-daemonize]
	58	ignore_missing_imports = True
	59
	60	[mypy-sentry_sdk]
	61	ignore_missing_imports = True

+148

-58

scripts/synapse_port_db less more

1	1	# -- coding: utf-8 --
2	2	# Copyright 2015, 2016 OpenMarket Ltd
3	3	# Copyright 2018 New Vector Ltd
	4	# Copyright 2019 The Matrix.org Foundation C.I.C.
4	5	#
5	6	# Licensed under the Apache License, Version 2.0 (the "License");
6	7	# you may not use this file except in compliance with the License.

28	29	from twisted.enterprise import adbapi
29	30	from twisted.internet import defer, reactor
30	31
31		from synapse.storage._base import LoggingTransaction, SQLBaseStore
	32	from synapse.config.homeserver import HomeServerConfig
	33	from synapse.logging.context import PreserveLoggingContext
	34	from synapse.storage._base import LoggingTransaction
	35	from synapse.storage.data_stores.main.client_ips import ClientIpBackgroundUpdateStore
	36	from synapse.storage.data_stores.main.deviceinbox import (
	37	DeviceInboxBackgroundUpdateStore,
	38	)
	39	from synapse.storage.data_stores.main.devices import DeviceBackgroundUpdateStore
	40	from synapse.storage.data_stores.main.events_bg_updates import (
	41	EventsBackgroundUpdatesStore,
	42	)
	43	from synapse.storage.data_stores.main.media_repository import (
	44	MediaRepositoryBackgroundUpdateStore,
	45	)
	46	from synapse.storage.data_stores.main.registration import (
	47	RegistrationBackgroundUpdateStore,
	48	)
	49	from synapse.storage.data_stores.main.roommember import RoomMemberBackgroundUpdateStore
	50	from synapse.storage.data_stores.main.search import SearchBackgroundUpdateStore
	51	from synapse.storage.data_stores.main.state import StateBackgroundUpdateStore
	52	from synapse.storage.data_stores.main.stats import StatsStore
	53	from synapse.storage.data_stores.main.user_directory import (
	54	UserDirectoryBackgroundUpdateStore,
	55	)
32	56	from synapse.storage.engines import create_engine
33	57	from synapse.storage.prepare_database import prepare_database
	58	from synapse.util import Clock
34	59
35	60	logger = logging.getLogger("synapse_port_db")
36	61

54	79	"local_group_membership": ["is_publicised", "is_admin"],
55	80	"e2e_room_keys": ["is_verified"],
56	81	"account_validity": ["email_sent"],
	82	"redactions": ["have_censored"],
	83	"room_stats_state": ["is_federatable"],
57	84	}
58	85
59	86

95	122	end_error_exec_info = None
96	123
97	124
98		class Store(object):
99		"""This object is used to pull out some of the convenience API from the
100		Storage layer.
101
102		All database interactions should go through this object.
103		"""
104
105		def __init__(self, db_pool, engine):
106		self.db_pool = db_pool
107		self.database_engine = engine
108
109		_simple_insert_txn = SQLBaseStore.__dict__["_simple_insert_txn"]
110		_simple_insert = SQLBaseStore.__dict__["_simple_insert"]
111
112		_simple_select_onecol_txn = SQLBaseStore.__dict__["_simple_select_onecol_txn"]
113		_simple_select_onecol = SQLBaseStore.__dict__["_simple_select_onecol"]
114		_simple_select_one = SQLBaseStore.__dict__["_simple_select_one"]
115		_simple_select_one_txn = SQLBaseStore.__dict__["_simple_select_one_txn"]
116		_simple_select_one_onecol = SQLBaseStore.__dict__["_simple_select_one_onecol"]
117		_simple_select_one_onecol_txn = SQLBaseStore.__dict__[
118		"_simple_select_one_onecol_txn"
119		]
120
121		_simple_update_one = SQLBaseStore.__dict__["_simple_update_one"]
122		_simple_update_one_txn = SQLBaseStore.__dict__["_simple_update_one_txn"]
123		_simple_update_txn = SQLBaseStore.__dict__["_simple_update_txn"]
124
	125	class Store(
	126	ClientIpBackgroundUpdateStore,
	127	DeviceInboxBackgroundUpdateStore,
	128	DeviceBackgroundUpdateStore,
	129	EventsBackgroundUpdatesStore,
	130	MediaRepositoryBackgroundUpdateStore,
	131	RegistrationBackgroundUpdateStore,
	132	RoomMemberBackgroundUpdateStore,
	133	SearchBackgroundUpdateStore,
	134	StateBackgroundUpdateStore,
	135	UserDirectoryBackgroundUpdateStore,
	136	StatsStore,
	137	):
	138	def __init__(self, db_conn, hs):
	139	super().__init__(db_conn, hs)
	140	self.db_pool = hs.get_db_pool()
	141
	142	@defer.inlineCallbacks
125	143	def runInteraction(self, desc, func, args, *kwargs):
126	144	def r(conn):
127	145	try:

147	165	logger.debug("[TXN FAIL] {%s} %s", desc, e)
148	166	raise
149	167
150		return self.db_pool.runWithConnection(r)
	168	with PreserveLoggingContext():
	169	return (yield self.db_pool.runWithConnection(r))
151	170
152	171	def execute(self, f, args, *kwargs):
153	172	return self.runInteraction(f.__name__, f, args, *kwargs)

171	190	except Exception:
172	191	logger.exception("Failed to insert: %s", table)
173	192	raise
	193
	194
	195	class MockHomeserver:
	196	def __init__(self, config, database_engine, db_conn, db_pool):
	197	self.database_engine = database_engine
	198	self.db_conn = db_conn
	199	self.db_pool = db_pool
	200	self.clock = Clock(reactor)
	201	self.config = config
	202	self.hostname = config.server_name
	203
	204	def get_db_conn(self):
	205	return self.db_conn
	206
	207	def get_db_pool(self):
	208	return self.db_pool
	209
	210	def get_clock(self):
	211	return self.clock
174	212
175	213
176	214	class Porter(object):

444	482
445	483	db_conn.commit()
446	484
	485	return db_conn
	486
	487	@defer.inlineCallbacks
	488	def build_db_store(self, config):
	489	"""Builds and returns a database store using the provided configuration.
	490
	491	Args:
	492	config: The database configuration, i.e. a dict following the structure of
	493	the "database" section of Synapse's configuration file.
	494
	495	Returns:
	496	The built Store object.
	497	"""
	498	engine = create_engine(config)
	499
	500	self.progress.set_state("Preparing %s" % config["name"])
	501	conn = self.setup_db(config, engine)
	502
	503	db_pool = adbapi.ConnectionPool(
	504	config["name"], **config["args"]
	505	)
	506
	507	hs = MockHomeserver(self.hs_config, engine, conn, db_pool)
	508
	509	store = Store(conn, hs)
	510
	511	yield store.runInteraction(
	512	"%s_engine.check_database" % config["name"],
	513	engine.check_database,
	514	)
	515
	516	return store
	517
	518	@defer.inlineCallbacks
	519	def run_background_updates_on_postgres(self):
	520	# Manually apply all background updates on the PostgreSQL database.
	521	postgres_ready = yield self.postgres_store.has_completed_background_updates()
	522
	523	if not postgres_ready:
	524	# Only say that we're running background updates when there are background
	525	# updates to run.
	526	self.progress.set_state("Running background updates on PostgreSQL")
	527
	528	while not postgres_ready:
	529	yield self.postgres_store.do_next_background_update(100)
	530	postgres_ready = yield (
	531	self.postgres_store.has_completed_background_updates()
	532	)
	533
447	534	@defer.inlineCallbacks
448	535	def run(self):
449	536	try:
450		sqlite_db_pool = adbapi.ConnectionPool(
451		self.sqlite_config["name"], **self.sqlite_config["args"]
452		)
453
454		postgres_db_pool = adbapi.ConnectionPool(
455		self.postgres_config["name"], **self.postgres_config["args"]
456		)
457
458		sqlite_engine = create_engine(sqlite_config)
459		postgres_engine = create_engine(postgres_config)
460
461		self.sqlite_store = Store(sqlite_db_pool, sqlite_engine)
462		self.postgres_store = Store(postgres_db_pool, postgres_engine)
463
464		yield self.postgres_store.execute(postgres_engine.check_database)
465
466		# Step 1. Set up databases.
467		self.progress.set_state("Preparing SQLite3")
468		self.setup_db(sqlite_config, sqlite_engine)
469
470		self.progress.set_state("Preparing PostgreSQL")
471		self.setup_db(postgres_config, postgres_engine)
	537	self.sqlite_store = yield self.build_db_store(self.sqlite_config)
	538
	539	# Check if all background updates are done, abort if not.
	540	updates_complete = yield self.sqlite_store.has_completed_background_updates()
	541	if not updates_complete:
	542	sys.stderr.write(
	543	"Pending background updates exist in the SQLite3 database."
	544	" Please start Synapse again and wait until every update has finished"
	545	" before running this script.\n"
	546	)
	547	defer.returnValue(None)
	548
	549	self.postgres_store = yield self.build_db_store(
	550	self.hs_config.database_config
	551	)
	552
	553	yield self.run_background_updates_on_postgres()
472	554
473	555	self.progress.set_state("Creating port tables")
474	556

560	642	def conv(j, col):
561	643	if j in bool_cols:
562	644	return bool(col)
	645	if isinstance(col, bytes):
	646	return bytearray(col)
563	647	elif isinstance(col, string_types) and "\0" in col:
564	648	logger.warn(
565	649	"DROPPING ROW: NUL value in table %s col %s: %r",

923	1007	},
924	1008	}
925	1009
926		postgres_config = yaml.safe_load(args.postgres_config)
927
928		if "database" in postgres_config:
929		postgres_config = postgres_config["database"]
	1010	hs_config = yaml.safe_load(args.postgres_config)
	1011
	1012	if "database" not in hs_config:
	1013	sys.stderr.write("The configuration file must have a 'database' section.\n")
	1014	sys.exit(4)
	1015
	1016	postgres_config = hs_config["database"]
930	1017
931	1018	if "name" not in postgres_config:
932		sys.stderr.write("Malformed database config: no 'name'")
	1019	sys.stderr.write("Malformed database config: no 'name'\n")
933	1020	sys.exit(2)
934	1021	if postgres_config["name"] != "psycopg2":
935		sys.stderr.write("Database must use 'psycopg2' connector.")
	1022	sys.stderr.write("Database must use the 'psycopg2' connector.\n")
936	1023	sys.exit(3)
	1024
	1025	config = HomeServerConfig()
	1026	config.parse_config_dict(hs_config, "", "")
937	1027
938	1028	def start(stdscr=None):
939	1029	if stdscr:

943	1033
944	1034	porter = Porter(
945	1035	sqlite_config=sqlite_config,
946		postgres_config=postgres_config,
947	1036	progress=progress,
948	1037	batch_size=args.batch_size,
	1038	hs_config=config,
949	1039	)
950	1040
951	1041	reactor.callWhenRunning(porter.run)

+0

-58

~~scripts-dev/check_auth.py~~ less more

0		from __future__ import print_function
1
2		import argparse
3		import itertools
4		import json
5		import sys
6
7		from mock import Mock
8
9		from synapse.api.auth import Auth
10		from synapse.events import FrozenEvent
11
12
13		def check_auth(auth, auth_chain, events):
14		auth_chain.sort(key=lambda e: e.depth)
15
16		auth_map = {e.event_id: e for e in auth_chain}
17
18		create_events = {}
19		for e in auth_chain:
20		if e.type == "m.room.create":
21		create_events[e.room_id] = e
22
23		for e in itertools.chain(auth_chain, events):
24		auth_events_list = [auth_map[i] for i, _ in e.auth_events]
25
26		auth_events = {(e.type, e.state_key): e for e in auth_events_list}
27
28		auth_events[("m.room.create", "")] = create_events[e.room_id]
29
30		try:
31		auth.check(e, auth_events=auth_events)
32		except Exception as ex:
33		print("Failed:", e.event_id, e.type, e.state_key)
34		print("Auth_events:", auth_events)
35		print(ex)
36		print(json.dumps(e.get_dict(), sort_keys=True, indent=4))
37		# raise
38		print("Success:", e.event_id, e.type, e.state_key)
39
40
41		if __name__ == "__main__":
42		parser = argparse.ArgumentParser()
43
44		parser.add_argument(
45		"json", nargs="?", type=argparse.FileType("r"), default=sys.stdin
46		)
47
48		args = parser.parse_args()
49
50		js = json.load(args.json)
51
52		auth = Auth(Mock())
53		check_auth(
54		auth,
55		[FrozenEvent(d) for d in js["auth_chain"]],
56		[FrozenEvent(d) for d in js.get("pdus", [])],
57		)

+9

-0

scripts-dev/config-lint.sh less more

	0	#!/bin/bash
	1	# Find linting errors in Synapse's default config file.
	2	# Exits with 0 if there are no problems, or another code otherwise.
	3
	4	# Fix non-lowercase true/false values
	5	sed -i -E "s/: +True/: true/g; s/: +False/: false/g;" docs/sample_config.yaml
	6
	7	# Check if anything changed
	8	git diff --exit-code docs/sample_config.yaml

+1

-0

scripts-dev/lint.sh less more

9	9	isort -y -rc synapse tests scripts-dev scripts
10	10	flake8 synapse tests
11	11	python3 -m black synapse tests scripts-dev scripts
	12	./scripts-dev/config-lint.sh

+22

-0

snap/snapcraft.yaml less more

	0	name: matrix-synapse
	1	base: core18
	2	version: git
	3	summary: Reference Matrix homeserver
	4	description: \|
	5	Synapse is the reference Matrix homeserver.
	6	Matrix is a federated and decentralised instant messaging and VoIP system.
	7
	8	grade: stable
	9	confinement: strict
	10
	11	apps:
	12	matrix-synapse:
	13	command: synctl --no-daemonize start $SNAP_COMMON/homeserver.yaml
	14	stop-command: synctl -c $SNAP_COMMON stop
	15	plugs: [network-bind, network]
	16	daemon: simple
	17	parts:
	18	matrix-synapse:
	19	source: .
	20	plugin: python
	21	python-version: python3

+9

-1

synapse/__init__.py less more

16	16	""" This is a reference implementation of a Matrix home server.
17	17	"""
18	18
	19	import os
19	20	import sys
20	21
21	22	# Check that we're not running on an unsupported Python version.

34	35	except ImportError:
35	36	pass
36	37
37		__version__ = "1.4.0"
	38	__version__ = "1.5.0rc1"
	39
	40	if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)):
	41	# We import here so that we don't have to install a bunch of deps when
	42	# running the packaging tox test.
	43	from synapse.util.patch_inline_callbacks import do_patch
	44
	45	do_patch()

+13

-22

synapse/api/auth.py less more

24	24	import synapse.logging.opentracing as opentracing
25	25	import synapse.types
26	26	from synapse import event_auth
27		from synapse.api.constants import EventTypes, JoinRules, Membership, UserTypes
	27	from synapse.api.constants import (
	28	EventTypes,
	29	JoinRules,
	30	LimitBlockingTypes,
	31	Membership,
	32	UserTypes,
	33	)
28	34	from synapse.api.errors import (
29	35	AuthError,
30	36	Codes,

83	89	)
84	90	auth_events = yield self.store.get_events(auth_events_ids)
85	91	auth_events = {(e.type, e.state_key): e for e in itervalues(auth_events)}
86		self.check(
	92	event_auth.check(
87	93	room_version, event, auth_events=auth_events, do_sig_check=do_sig_check
88	94	)
89
90		def check(self, room_version, event, auth_events, do_sig_check=True):
91		""" Checks if this event is correctly authed.
92
93		Args:
94		room_version (str): version of the room
95		event: the event being checked.
96		auth_events (dict: event-key -> event): the existing room state.
97
98
99		Returns:
100		True if the auth checks pass.
101		"""
102		with Measure(self.clock, "auth.check"):
103		event_auth.check(
104		room_version, event, auth_events, do_sig_check=do_sig_check
105		)
106	95
107	96	@defer.inlineCallbacks
108	97	def check_joined_room(self, room_id, user_id, current_state=None):

178	167	def get_public_keys(self, invite_event):
179	168	return event_auth.get_public_keys(invite_event)
180	169
181		@opentracing.trace
182	170	@defer.inlineCallbacks
183	171	def get_user_by_req(
184	172	self, request, allow_guest=False, rights="access", allow_expired=False

211	199	if user_id:
212	200	request.authenticated_entity = user_id
213	201	opentracing.set_tag("authenticated_entity", user_id)
	202	opentracing.set_tag("appservice_id", app_service.id)
214	203
215	204	if ip_addr and self.hs.config.track_appservice_user_ips:
216	205	yield self.store.insert_client_ip(

262	251
263	252	request.authenticated_entity = user.to_string()
264	253	opentracing.set_tag("authenticated_entity", user.to_string())
	254	if device_id:
	255	opentracing.set_tag("device_id", device_id)
265	256
266	257	return synapse.types.create_requester(
267	258	user, token_id, is_guest, device_id, app_service=app_service

740	731	self.hs.config.hs_disabled_message,
741	732	errcode=Codes.RESOURCE_LIMIT_EXCEEDED,
742	733	admin_contact=self.hs.config.admin_contact,
743		limit_type=self.hs.config.hs_disabled_limit_type,
	734	limit_type=LimitBlockingTypes.HS_DISABLED,
744	735	)
745	736	if self.hs.config.limit_usage_by_mau is True:
746	737	assert not (user_id and threepid)

773	764	"Monthly Active User Limit Exceeded",
774	765	admin_contact=self.hs.config.admin_contact,
775	766	errcode=Codes.RESOURCE_LIMIT_EXCEEDED,
776		limit_type="monthly_active_user",
	767	limit_type=LimitBlockingTypes.MONTHLY_ACTIVE_USER,
777	768	)

+7

-2

synapse/api/constants.py less more

96	96
97	97	class RejectedReason(object):
98	98	AUTH_ERROR = "auth_error"
99		REPLACED = "replaced"
100		NOT_ANCESTOR = "not_ancestor"
101	99
102	100
103	101	class RoomCreationPreset(object):

132	130	ANNOTATION = "m.annotation"
133	131	REPLACE = "m.replace"
134	132	REFERENCE = "m.reference"
	133
	134
	135	class LimitBlockingTypes(object):
	136	"""Reasons that a server may be blocked"""
	137
	138	MONTHLY_ACTIVE_USER = "monthly_active_user"
	139	HS_DISABLED = "hs_disabled"

+3

-1

synapse/api/errors.py less more

16	16	"""Contains exceptions and error codes."""
17	17
18	18	import logging
	19	from typing import Dict
19	20
20	21	from six import iteritems
21	22	from six.moves import http_client

60	61	INCOMPATIBLE_ROOM_VERSION = "M_INCOMPATIBLE_ROOM_VERSION"
61	62	WRONG_ROOM_KEYS_VERSION = "M_WRONG_ROOM_KEYS_VERSION"
62	63	EXPIRED_ACCOUNT = "ORG_MATRIX_EXPIRED_ACCOUNT"
	64	INVALID_SIGNATURE = "M_INVALID_SIGNATURE"
63	65	USER_DEACTIVATED = "M_USER_DEACTIVATED"
64	66
65	67

110	112	def __init__(self, code, msg, errcode=Codes.UNKNOWN, additional_fields=None):
111	113	super(ProxiedRequestError, self).__init__(code, msg, errcode)
112	114	if additional_fields is None:
113		self._additional_fields = {}
	115	self._additional_fields = {} # type: Dict
114	116	else:
115	117	self._additional_fields = dict(additional_fields)
116	118

+4

-1

synapse/api/room_versions.py less more

11	11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	12	# See the License for the specific language governing permissions and
13	13	# limitations under the License.
	14
	15	from typing import Dict
	16
14	17	import attr
15	18
16	19

101	104	RoomVersions.V4,
102	105	RoomVersions.V5,
103	106	)
104		} # type: dict[str, RoomVersion]
	107	} # type: Dict[str, RoomVersion]

+3

-1

synapse/app/_base.py less more

262	262	refresh_certificate(hs)
263	263
264	264	# Start the tracer
265		synapse.logging.opentracing.init_tracer(hs.config)
	265	synapse.logging.opentracing.init_tracer( # type: ignore[attr-defined] # noqa
	266	hs.config
	267	)
266	268
267	269	# It is now safe to start your Synapse.
268	270	hs.start_listening(listeners)

+1

-1

synapse/app/event_creator.py less more

55	55	RoomStateEventRestServlet,
56	56	)
57	57	from synapse.server import HomeServer
	58	from synapse.storage.data_stores.main.user_directory import UserDirectoryStore
58	59	from synapse.storage.engines import create_engine
59		from synapse.storage.user_directory import UserDirectoryStore
60	60	from synapse.util.httpresourcetree import create_resource_tree
61	61	from synapse.util.manhole import manhole
62	62	from synapse.util.versionstring import get_version_string

+3

-3

synapse/app/homeserver.py less more

604	604	@defer.inlineCallbacks
605	605	def generate_monthly_active_users():
606	606	current_mau_count = 0
607		reserved_count = 0
	607	reserved_users = ()
608	608	store = hs.get_datastore()
609	609	if hs.config.limit_usage_by_mau or hs.config.mau_stats_only:
610	610	current_mau_count = yield store.get_monthly_active_count()
611		reserved_count = yield store.get_registered_reserved_users_count()
	611	reserved_users = yield store.get_registered_reserved_users()
612	612	current_mau_gauge.set(float(current_mau_count))
613		registered_reserved_users_mau_gauge.set(float(reserved_count))
	613	registered_reserved_users_mau_gauge.set(float(len(reserved_users)))
614	614	max_mau_gauge.set(float(hs.config.max_mau_value))
615	615
616	616	def start_generate_monthly_active_users():

+1

-1

synapse/app/media_repository.py less more

38	38	from synapse.rest.admin import register_servlets_for_media_repo
39	39	from synapse.rest.media.v0.content_repository import ContentRepoResource
40	40	from synapse.server import HomeServer
	41	from synapse.storage.data_stores.main.media_repository import MediaRepositoryStore
41	42	from synapse.storage.engines import create_engine
42		from synapse.storage.media_repository import MediaRepositoryStore
43	43	from synapse.util.httpresourcetree import create_resource_tree
44	44	from synapse.util.manhole import manhole
45	45	from synapse.util.versionstring import get_version_string

+1

-1

synapse/app/synchrotron.py less more

53	53	from synapse.rest.client.v1.room import RoomInitialSyncRestServlet
54	54	from synapse.rest.client.v2_alpha import sync
55	55	from synapse.server import HomeServer
	56	from synapse.storage.data_stores.main.presence import UserPresenceState
56	57	from synapse.storage.engines import create_engine
57		from synapse.storage.presence import UserPresenceState
58	58	from synapse.util.httpresourcetree import create_resource_tree
59	59	from synapse.util.manhole import manhole
60	60	from synapse.util.stringutils import random_string

+1

-1

synapse/app/user_dir.py less more

41	41	)
42	42	from synapse.rest.client.v2_alpha import user_directory
43	43	from synapse.server import HomeServer
	44	from synapse.storage.data_stores.main.user_directory import UserDirectoryStore
44	45	from synapse.storage.engines import create_engine
45		from synapse.storage.user_directory import UserDirectoryStore
46	46	from synapse.util.caches.stream_change_cache import StreamChangeCache
47	47	from synapse.util.httpresourcetree import create_resource_tree
48	48	from synapse.util.manhole import manhole

+152

-47

synapse/config/_base.py less more

17	17	import argparse
18	18	import errno
19	19	import os
	20	from collections import OrderedDict
20	21	from textwrap import dedent
	22	from typing import Any, MutableMapping, Optional
21	23
22	24	from six import integer_types
23	25

50	52	"""
51	53
52	54
	55	def path_exists(file_path):
	56	"""Check if a file exists
	57
	58	Unlike os.path.exists, this throws an exception if there is an error
	59	checking if the file exists (for example, if there is a perms error on
	60	the parent dir).
	61
	62	Returns:
	63	bool: True if the file exists; False if not.
	64	"""
	65	try:
	66	os.stat(file_path)
	67	return True
	68	except OSError as e:
	69	if e.errno != errno.ENOENT:
	70	raise e
	71	return False
	72
	73
53	74	class Config(object):
	75	"""
	76	A configuration section, containing configuration keys and values.
	77
	78	Attributes:
	79	section (str): The section title of this config object, such as
	80	"tls" or "logger". This is used to refer to it on the root
	81	logger (for example, `config.tls.some_option`). Must be
	82	defined in subclasses.
	83	"""
	84
	85	section = None
	86
	87	def __init__(self, root_config=None):
	88	self.root = root_config
	89
	90	def __getattr__(self, item: str) -> Any:
	91	"""
	92	Try and fetch a configuration option that does not exist on this class.
	93
	94	This is so that existing configs that rely on `self.value`, where value
	95	is actually from a different config section, continue to work.
	96	"""
	97	if item in ["generate_config_section", "read_config"]:
	98	raise AttributeError(item)
	99
	100	if self.root is None:
	101	raise AttributeError(item)
	102	else:
	103	return self.root._get_unclassed_config(self.section, item)
	104
54	105	@staticmethod
55	106	def parse_size(value):
56	107	if isinstance(value, integer_types):

87	138
88	139	@classmethod
89	140	def path_exists(cls, file_path):
90		"""Check if a file exists
91
92		Unlike os.path.exists, this throws an exception if there is an error
93		checking if the file exists (for example, if there is a perms error on
94		the parent dir).
95
96		Returns:
97		bool: True if the file exists; False if not.
98		"""
99		try:
100		os.stat(file_path)
101		return True
102		except OSError as e:
103		if e.errno != errno.ENOENT:
104		raise e
105		return False
	141	return path_exists(file_path)
106	142
107	143	@classmethod
108	144	def check_file(cls, file_path, config_name):

135	171	with open(file_path) as file_stream:
136	172	return file_stream.read()
137	173
138		def invoke_all(self, name, args, *kargs):
139		"""Invoke all instance methods with the given name and arguments in the
140		class's MRO.
	174
	175	class RootConfig(object):
	176	"""
	177	Holder of an application's configuration.
	178
	179	What configuration this object holds is defined by `config_classes`, a list
	180	of Config classes that will be instantiated and given the contents of a
	181	configuration file to read. They can then be accessed on this class by their
	182	section name, defined in the Config or dynamically set to be the name of the
	183	class, lower-cased and with "Config" removed.
	184	"""
	185
	186	config_classes = []
	187
	188	def __init__(self):
	189	self._configs = OrderedDict()
	190
	191	for config_class in self.config_classes:
	192	if config_class.section is None:
	193	raise ValueError("%r requires a section name" % (config_class,))
	194
	195	try:
	196	conf = config_class(self)
	197	except Exception as e:
	198	raise Exception("Failed making %s: %r" % (config_class.section, e))
	199	self._configs[config_class.section] = conf
	200
	201	def __getattr__(self, item: str) -> Any:
	202	"""
	203	Redirect lookups on this object either to config objects, or values on
	204	config objects, so that `config.tls.blah` works, as well as legacy uses
	205	of things like `config.server_name`. It will first look up the config
	206	section name, and then values on those config classes.
	207	"""
	208	if item in self._configs.keys():
	209	return self._configs[item]
	210
	211	return self._get_unclassed_config(None, item)
	212
	213	def _get_unclassed_config(self, asking_section: Optional[str], item: str):
	214	"""
	215	Fetch a config value from one of the instantiated config classes that
	216	has not been fetched directly.
141	217
142	218	Args:
143		name (str): Name of function to invoke
	219	asking_section: If this check is coming from a Config child, which
	220	one? This section will not be asked if it has the value.
	221	item: The configuration value key.
	222
	223	Raises:
	224	AttributeError if no config classes have the config key. The body
	225	will contain what sections were checked.
	226	"""
	227	for key, val in self._configs.items():
	228	if key == asking_section:
	229	continue
	230
	231	if item in dir(val):
	232	return getattr(val, item)
	233
	234	raise AttributeError(item, "not found in %s" % (list(self._configs.keys()),))
	235
	236	def invoke_all(self, func_name: str, args, *kwargs) -> MutableMapping[str, Any]:
	237	"""
	238	Invoke a function on all instantiated config objects this RootConfig is
	239	configured to use.
	240
	241	Args:
	242	func_name: Name of function to invoke
144	243	*args
145	244	**kwargs
146
147	245	Returns:
148		list: The list of the return values from each method called
149		"""
150		results = []
151		for cls in type(self).mro():
152		if name in cls.__dict__:
153		results.append(getattr(cls, name)(self, args, *kargs))
154		return results
155
156		@classmethod
157		def invoke_all_static(cls, name, args, *kargs):
158		"""Invoke all static methods with the given name and arguments in the
159		class's MRO.
	246	ordered dictionary of config section name and the result of the
	247	function from it.
	248	"""
	249	res = OrderedDict()
	250
	251	for name, config in self._configs.items():
	252	if hasattr(config, func_name):
	253	res[name] = getattr(config, func_name)(args, *kwargs)
	254
	255	return res
	256
	257	@classmethod
	258	def invoke_all_static(cls, func_name: str, args, *kwargs):
	259	"""
	260	Invoke a static function on config objects this RootConfig is
	261	configured to use.
160	262
161	263	Args:
162		name (str): Name of function to invoke
	264	func_name: Name of function to invoke
163	265	*args
164	266	**kwargs
165
166	267	Returns:
167		list: The list of the return values from each method called
168		"""
169		results = []
170		for c in cls.mro():
171		if name in c.__dict__:
172		results.append(getattr(c, name)(args, *kargs))
173		return results
	268	ordered dictionary of config section name and the result of the
	269	function from it.
	270	"""
	271	for config in cls.config_classes:
	272	if hasattr(config, func_name):
	273	getattr(config, func_name)(args, *kwargs)
174	274
175	275	def generate_config(
176	276	self,

186	286	tls_private_key_path=None,
187	287	acme_domain=None,
188	288	):
189		"""Build a default configuration file
	289	"""
	290	Build a default configuration file
190	291
191	292	This is used when the user explicitly asks us to generate a config file
192	293	(eg with --generate_config).

241	342	Returns:
242	343	str: the yaml config file
243	344	"""
	345
244	346	return "\n\n".join(
245	347	dedent(conf)
246	348	for conf in self.invoke_all(

256	358	tls_certificate_path=tls_certificate_path,
257	359	tls_private_key_path=tls_private_key_path,
258	360	acme_domain=acme_domain,
259		)
	361	).values()
260	362	)
261	363
262	364	@classmethod

443	545	)
444	546
445	547	(config_path,) = config_files
446		if not cls.path_exists(config_path):
	548	if not path_exists(config_path):
447	549	print("Generating config file %s" % (config_path,))
448	550
449	551	if config_args.data_directory:

468	570	open_private_ports=config_args.open_private_ports,
469	571	)
470	572
471		if not cls.path_exists(config_dir_path):
	573	if not path_exists(config_dir_path):
472	574	os.makedirs(config_dir_path)
473	575	with open(config_path, "w") as config_file:
474	576	config_file.write("# vim:ft=yaml\n\n")

517	619
518	620	return obj
519	621
520		def parse_config_dict(self, config_dict, config_dir_path, data_dir_path):
	622	def parse_config_dict(self, config_dict, config_dir_path=None, data_dir_path=None):
521	623	"""Read the information from the config dict into this Config object.
522	624
523	625	Args:

606	708	else:
607	709	config_files.append(config_path)
608	710	return config_files
	711
	712
	713	__all__ = ["Config", "RootConfig"]

+135

-0

synapse/config/_base.pyi less more

	0	from typing import Any, List, Optional
	1
	2	from synapse.config import (
	3	api,
	4	appservice,
	5	captcha,
	6	cas,
	7	consent_config,
	8	database,
	9	emailconfig,
	10	groups,
	11	jwt_config,
	12	key,
	13	logger,
	14	metrics,
	15	password,
	16	password_auth_providers,
	17	push,
	18	ratelimiting,
	19	registration,
	20	repository,
	21	room_directory,
	22	saml2_config,
	23	server,
	24	server_notices_config,
	25	spam_checker,
	26	stats,
	27	third_party_event_rules,
	28	tls,
	29	tracer,
	30	user_directory,
	31	voip,
	32	workers,
	33	)
	34
	35	class ConfigError(Exception): ...
	36
	37	MISSING_REPORT_STATS_CONFIG_INSTRUCTIONS: str
	38	MISSING_REPORT_STATS_SPIEL: str
	39	MISSING_SERVER_NAME: str
	40
	41	def path_exists(file_path: str): ...
	42
	43	class RootConfig:
	44	server: server.ServerConfig
	45	tls: tls.TlsConfig
	46	database: database.DatabaseConfig
	47	logging: logger.LoggingConfig
	48	ratelimit: ratelimiting.RatelimitConfig
	49	media: repository.ContentRepositoryConfig
	50	captcha: captcha.CaptchaConfig
	51	voip: voip.VoipConfig
	52	registration: registration.RegistrationConfig
	53	metrics: metrics.MetricsConfig
	54	api: api.ApiConfig
	55	appservice: appservice.AppServiceConfig
	56	key: key.KeyConfig
	57	saml2: saml2_config.SAML2Config
	58	cas: cas.CasConfig
	59	jwt: jwt_config.JWTConfig
	60	password: password.PasswordConfig
	61	email: emailconfig.EmailConfig
	62	worker: workers.WorkerConfig
	63	authproviders: password_auth_providers.PasswordAuthProviderConfig
	64	push: push.PushConfig
	65	spamchecker: spam_checker.SpamCheckerConfig
	66	groups: groups.GroupsConfig
	67	userdirectory: user_directory.UserDirectoryConfig
	68	consent: consent_config.ConsentConfig
	69	stats: stats.StatsConfig
	70	servernotices: server_notices_config.ServerNoticesConfig
	71	roomdirectory: room_directory.RoomDirectoryConfig
	72	thirdpartyrules: third_party_event_rules.ThirdPartyRulesConfig
	73	tracer: tracer.TracerConfig
	74
	75	config_classes: List = ...
	76	def __init__(self) -> None: ...
	77	def invoke_all(self, func_name: str, args: Any, *kwargs: Any): ...
	78	@classmethod
	79	def invoke_all_static(cls, func_name: str, args: Any, *kwargs: Any) -> None: ...
	80	def __getattr__(self, item: str): ...
	81	def parse_config_dict(
	82	self,
	83	config_dict: Any,
	84	config_dir_path: Optional[Any] = ...,
	85	data_dir_path: Optional[Any] = ...,
	86	) -> None: ...
	87	read_config: Any = ...
	88	def generate_config(
	89	self,
	90	config_dir_path: str,
	91	data_dir_path: str,
	92	server_name: str,
	93	generate_secrets: bool = ...,
	94	report_stats: Optional[str] = ...,
	95	open_private_ports: bool = ...,
	96	listeners: Optional[Any] = ...,
	97	database_conf: Optional[Any] = ...,
	98	tls_certificate_path: Optional[str] = ...,
	99	tls_private_key_path: Optional[str] = ...,
	100	acme_domain: Optional[str] = ...,
	101	): ...
	102	@classmethod
	103	def load_or_generate_config(cls, description: Any, argv: Any): ...
	104	@classmethod
	105	def load_config(cls, description: Any, argv: Any): ...
	106	@classmethod
	107	def add_arguments_to_parser(cls, config_parser: Any) -> None: ...
	108	@classmethod
	109	def load_config_with_parser(cls, parser: Any, argv: Any): ...
	110	def generate_missing_files(
	111	self, config_dict: dict, config_dir_path: str
	112	) -> None: ...
	113
	114	class Config:
	115	root: RootConfig
	116	def __init__(self, root_config: Optional[RootConfig] = ...) -> None: ...
	117	def __getattr__(self, item: str, from_root: bool = ...): ...
	118	@staticmethod
	119	def parse_size(value: Any): ...
	120	@staticmethod
	121	def parse_duration(value: Any): ...
	122	@staticmethod
	123	def abspath(file_path: Optional[str]): ...
	124	@classmethod
	125	def path_exists(cls, file_path: str): ...
	126	@classmethod
	127	def check_file(cls, file_path: str, config_name: str): ...
	128	@classmethod
	129	def ensure_directory(cls, dir_path: str): ...
	130	@classmethod
	131	def read_file(cls, file_path: str, config_name: str): ...
	132
	133	def read_config_files(config_files: List[str]): ...
	134	def find_config_files(search_paths: List[str]): ...

+2

-0

synapse/config/api.py less more

17	17
18	18
19	19	class ApiConfig(Config):
	20	section = "api"
	21
20	22	def read_config(self, config, **kwargs):
21	23	self.room_invite_state_types = config.get(
22	24	"room_invite_state_types",

+6

-3

synapse/config/appservice.py less more

12	12	# limitations under the License.
13	13
14	14	import logging
	15	from typing import Dict
15	16
16	17	from six import string_types
17	18	from six.moves.urllib import parse as urlparse

28	29
29	30
30	31	class AppServiceConfig(Config):
	32	section = "appservice"
	33
31	34	def read_config(self, config, **kwargs):
32	35	self.app_service_config_files = config.get("app_service_config_files", [])
33	36	self.notify_appservices = config.get("notify_appservices", True)

44	47	# Uncomment to enable tracking of application service IP addresses. Implicitly
45	48	# enables MAU tracking for application service users.
46	49	#
47		#track_appservice_user_ips: True
	50	#track_appservice_user_ips: true
48	51	"""
49	52
50	53

55	58	return []
56	59
57	60	# Dicts of value -> filename
58		seen_as_tokens = {}
59		seen_ids = {}
	61	seen_as_tokens = {} # type: Dict[str, str]
	62	seen_ids = {} # type: Dict[str, str]
60	63
61	64	appservices = []
62	65

+2

-0

synapse/config/captcha.py less more

15	15
16	16
17	17	class CaptchaConfig(Config):
	18	section = "captcha"
	19
18	20	def read_config(self, config, **kwargs):
19	21	self.recaptcha_private_key = config.get("recaptcha_private_key")
20	22	self.recaptcha_public_key = config.get("recaptcha_public_key")

+5

-0

synapse/config/cas.py less more

21	21	cas_server_url: URL of CAS server
22	22	"""
23	23
	24	section = "cas"
	25
24	26	def read_config(self, config, **kwargs):
25	27	cas_config = config.get("cas_config", None)
26	28	if cas_config:
27	29	self.cas_enabled = cas_config.get("enabled", True)
28	30	self.cas_server_url = cas_config["server_url"]
29	31	self.cas_service_url = cas_config["service_url"]
	32	self.cas_displayname_attribute = cas_config.get("displayname_attribute")
30	33	self.cas_required_attributes = cas_config.get("required_attributes", {})
31	34	else:
32	35	self.cas_enabled = False
33	36	self.cas_server_url = None
34	37	self.cas_service_url = None
	38	self.cas_displayname_attribute = None
35	39	self.cas_required_attributes = {}
36	40
37	41	def generate_config_section(self, config_dir_path, server_name, **kwargs):

42	46	# enabled: true
43	47	# server_url: "https://cas-server.com"
44	48	# service_url: "https://homeserver.domain.com:8448"
	49	# #displayname_attribute: name
45	50	# #required_attributes:
46	51	# # name: value
47	52	"""

+7

-4

synapse/config/consent_config.py less more

61	61	# body: >-
62	62	# To continue using this homeserver you must review and agree to the
63	63	# terms and conditions at %(consent_uri)s
64		# send_server_notice_to_guests: True
	64	# send_server_notice_to_guests: true
65	65	# block_events_error: >-
66	66	# To continue using this homeserver you must review and agree to the
67	67	# terms and conditions at %(consent_uri)s
68		# require_at_registration: False
	68	# require_at_registration: false
69	69	# policy_name: Privacy Policy
70	70	#
71	71	"""
72	72
73	73
74	74	class ConsentConfig(Config):
75		def __init__(self):
76		super(ConsentConfig, self).__init__()
	75
	76	section = "consent"
	77
	78	def __init__(self, *args):
	79	super(ConsentConfig, self).__init__(*args)
77	80
78	81	self.user_consent_version = None
79	82	self.user_consent_template_dir = None

+2

-0

synapse/config/database.py less more

20	20
21	21
22	22	class DatabaseConfig(Config):
	23	section = "database"
	24
23	25	def read_config(self, config, **kwargs):
24	26	self.event_cache_size = self.parse_size(config.get("event_cache_size", "10K"))
25	27

+4

-2

synapse/config/emailconfig.py less more

27	27
28	28
29	29	class EmailConfig(Config):
	30	section = "email"
	31
30	32	def read_config(self, config, **kwargs):
31	33	# TODO: We should separate better the email configuration from the notification
32	34	# and account validity config.

301	303	# smtp_port: 25 # SSL: 465, STARTTLS: 587
302	304	# smtp_user: "exampleusername"
303	305	# smtp_pass: "examplepassword"
304		# require_transport_security: False
	306	# require_transport_security: false
305	307	# notif_from: "Your Friendly %(app)s Home Server <noreply@example.com>"
306	308	# app_name: Matrix
307	309	#
308	310	# # Enable email notifications by default
309	311	# #
310		# notif_for_new_users: True
	312	# notif_for_new_users: true
311	313	#
312	314	# # Defining a custom URL for Riot is only needed if email notifications
313	315	# # should contain links to a self-hosted installation of Riot; when set

+2

-0

synapse/config/groups.py less more

16	16
17	17
18	18	class GroupsConfig(Config):
	19	section = "groups"
	20
19	21	def read_config(self, config, **kwargs):
20	22	self.enable_group_creation = config.get("enable_group_creation", False)
21	23	self.group_creation_prefix = config.get("group_creation_prefix", "")

+35

-33

synapse/config/homeserver.py less more

13	13	# See the License for the specific language governing permissions and
14	14	# limitations under the License.
15	15
	16	from ._base import RootConfig
16	17	from .api import ApiConfig
17	18	from .appservice import AppServiceConfig
18	19	from .captcha import CaptchaConfig

45	46	from .workers import WorkerConfig
46	47
47	48
48		class HomeServerConfig(
49		ServerConfig,
50		TlsConfig,
51		DatabaseConfig,
52		LoggingConfig,
53		RatelimitConfig,
54		ContentRepositoryConfig,
55		CaptchaConfig,
56		VoipConfig,
57		RegistrationConfig,
58		MetricsConfig,
59		ApiConfig,
60		AppServiceConfig,
61		KeyConfig,
62		SAML2Config,
63		CasConfig,
64		JWTConfig,
65		PasswordConfig,
66		EmailConfig,
67		WorkerConfig,
68		PasswordAuthProviderConfig,
69		PushConfig,
70		SpamCheckerConfig,
71		GroupsConfig,
72		UserDirectoryConfig,
73		ConsentConfig,
74		StatsConfig,
75		ServerNoticesConfig,
76		RoomDirectoryConfig,
77		ThirdPartyRulesConfig,
78		TracerConfig,
79		):
80		pass
	49	class HomeServerConfig(RootConfig):
	50
	51	config_classes = [
	52	ServerConfig,
	53	TlsConfig,
	54	DatabaseConfig,
	55	LoggingConfig,
	56	RatelimitConfig,
	57	ContentRepositoryConfig,
	58	CaptchaConfig,
	59	VoipConfig,
	60	RegistrationConfig,
	61	MetricsConfig,
	62	ApiConfig,
	63	AppServiceConfig,
	64	KeyConfig,
	65	SAML2Config,
	66	CasConfig,
	67	JWTConfig,
	68	PasswordConfig,
	69	EmailConfig,
	70	WorkerConfig,
	71	PasswordAuthProviderConfig,
	72	PushConfig,
	73	SpamCheckerConfig,
	74	GroupsConfig,
	75	UserDirectoryConfig,
	76	ConsentConfig,
	77	StatsConfig,
	78	ServerNoticesConfig,
	79	RoomDirectoryConfig,
	80	ThirdPartyRulesConfig,
	81	TracerConfig,
	82	]

+2

-0

synapse/config/jwt_config.py less more

22	22
23	23
24	24	class JWTConfig(Config):
	25	section = "jwt"
	26
25	27	def read_config(self, config, **kwargs):
26	28	jwt_config = config.get("jwt_config", None)
27	29	if jwt_config:

+2

-0

synapse/config/key.py less more

91	91
92	92
93	93	class KeyConfig(Config):
	94	section = "key"
	95
94	96	def read_config(self, config, config_dir_path, **kwargs):
95	97	# the signing key can be specified inline or in a separate file
96	98	if "signing_key" in config:

+4

-3

synapse/config/logger.py less more

67	67	filters: [context]
68	68
69	69	loggers:
70		synapse:
71		level: INFO
72
73	70	synapse.storage.SQL:
74	71	# beware: increasing this to DEBUG will make synapse log sensitive
75	72	# information such as access tokens.

78	75	root:
79	76	level: INFO
80	77	handlers: [file, console]
	78
	79	disable_existing_loggers: false
81	80	"""
82	81	)
83	82
84	83
85	84	class LoggingConfig(Config):
	85	section = "logging"
	86
86	87	def read_config(self, config, **kwargs):
87	88	self.log_config = self.abspath(config.get("log_config"))
88	89	self.no_redirect_stdio = config.get("no_redirect_stdio", False)

+3

-1

synapse/config/metrics.py less more

33	33
34	34
35	35	class MetricsConfig(Config):
	36	section = "metrics"
	37
36	38	def read_config(self, config, **kwargs):
37	39	self.enable_metrics = config.get("enable_metrics", False)
38	40	self.report_stats = config.get("report_stats", None)

67	69
68	70	# Enable collection and rendering of performance metrics
69	71	#
70		#enable_metrics: False
	72	#enable_metrics: false
71	73
72	74	# Enable sentry integration
73	75	# NOTE: While attempts are made to ensure that the logs don't contain

+2

-0

synapse/config/password.py less more

18	18	class PasswordConfig(Config):
19	19	"""Password login configuration
20	20	"""
	21
	22	section = "password"
21	23
22	24	def read_config(self, config, **kwargs):
23	25	password_config = config.get("password_config", {})

+5

-1

synapse/config/password_auth_providers.py less more

12	12	# See the License for the specific language governing permissions and
13	13	# limitations under the License.
14	14
	15	from typing import Any, List
	16
15	17	from synapse.util.module_loader import load_module
16	18
17	19	from ._base import Config

20	22
21	23
22	24	class PasswordAuthProviderConfig(Config):
	25	section = "authproviders"
	26
23	27	def read_config(self, config, **kwargs):
24		self.password_providers = []
	28	self.password_providers = [] # type: List[Any]
25	29	providers = []
26	30
27	31	# We want to be backwards compatible with the old `ldap_config`

+2

-0

synapse/config/push.py less more

17	17
18	18
19	19	class PushConfig(Config):
	20	section = "push"
	21
20	22	def read_config(self, config, **kwargs):
21	23	push_config = config.get("push", {})
22	24	self.push_include_content = push_config.get("include_content", True)

+2

-0

synapse/config/ratelimiting.py less more

35	35
36	36
37	37	class RatelimitConfig(Config):
	38	section = "ratelimiting"
	39
38	40	def read_config(self, config, **kwargs):
39	41
40	42	# Load the new-style messages config if it exists. Otherwise fall back

+5

-1

synapse/config/registration.py less more

23	23
24	24
25	25	class AccountValidityConfig(Config):
	26	section = "accountvalidity"
	27
26	28	def __init__(self, config, synapse_config):
27	29	self.enabled = config.get("enabled", False)
28	30	self.renew_by_email_enabled = "renew_at" in config

76	78
77	79
78	80	class RegistrationConfig(Config):
	81	section = "registration"
	82
79	83	def read_config(self, config, **kwargs):
80	84	self.enable_registration = bool(
81	85	strtobool(str(config.get("enable_registration", False)))

175	179	# where d is equal to 10%% of the validity period.
176	180	#
177	181	#account_validity:
178		# enabled: True
	182	# enabled: true
179	183	# period: 6w
180	184	# renew_at: 1w
181	185	# renew_email_subject: "Renew your %%(app)s account"

+5

-2

synapse/config/repository.py less more

14	14
15	15	import os
16	16	from collections import namedtuple
	17	from typing import Dict, List
17	18
18	19	from synapse.python_dependencies import DependencyException, check_requirements
19	20	from synapse.util.module_loader import load_module

60	61	Dictionary mapping from media type string to list of
61	62	ThumbnailRequirement tuples.
62	63	"""
63		requirements = {}
	64	requirements = {} # type: Dict[str, List]
64	65	for size in thumbnail_sizes:
65	66	width = size["width"]
66	67	height = size["height"]

76	77
77	78
78	79	class ContentRepositoryConfig(Config):
	80	section = "media"
	81
79	82	def read_config(self, config, **kwargs):
80	83
81	84	# Only enable the media repo if either the media repo is enabled or the

129	132	#
130	133	# We don't create the storage providers here as not all workers need
131	134	# them to be started.
132		self.media_storage_providers = []
	135	self.media_storage_providers = [] # type: List[tuple]
133	136
134	137	for provider_config in storage_providers:
135	138	# We special case the module "file_system" so as not to need to

+2

-0

synapse/config/room_directory.py less more

18	18
19	19
20	20	class RoomDirectoryConfig(Config):
	21	section = "roomdirectory"
	22
21	23	def read_config(self, config, **kwargs):
22	24	self.enable_room_list_search = config.get("enable_room_list_search", True)
23	25

+3

-1

synapse/config/saml2_config.py less more

54	54
55	55
56	56	class SAML2Config(Config):
	57	section = "saml2"
	58
57	59	def read_config(self, config, **kwargs):
58	60	self.saml2_enabled = False
59	61

173	175	# - url: https://our_idp/metadata.xml
174	176	#
175	177	# # By default, the user has to go to our login page first. If you'd like
176		# # to allow IdP-initiated login, set 'allow_unsolicited: True' in a
	178	# # to allow IdP-initiated login, set 'allow_unsolicited: true' in a
177	179	# # 'service.sp' section:
178	180	# #
179	181	# #service:

+22

-10

synapse/config/server.py less more

18	18	import os.path
19	19	import re
20	20	from textwrap import indent
	21	from typing import List
21	22
22	23	import attr
23	24	import yaml

56	57
57	58
58	59	class ServerConfig(Config):
	60	section = "server"
	61
59	62	def read_config(self, config, **kwargs):
60	63	self.server_name = config["server_name"]
61	64	self.server_context = config.get("server_context", None)

167	170	)
168	171
169	172	self.mau_trial_days = config.get("mau_trial_days", 0)
	173	self.mau_limit_alerting = config.get("mau_limit_alerting", True)
170	174
171	175	# How long to keep redacted events in the database in unredacted form
172	176	# before redacting them.

188	192	# Options to disable HS
189	193	self.hs_disabled = config.get("hs_disabled", False)
190	194	self.hs_disabled_message = config.get("hs_disabled_message", "")
191		self.hs_disabled_limit_type = config.get("hs_disabled_limit_type", "")
192	195
193	196	# Admin uri to direct users at should their instance become blocked
194	197	# due to resource constraints

242	245	# events with profile information that differ from the target's global profile.
243	246	self.allow_per_room_profiles = config.get("allow_per_room_profiles", True)
244	247
245		self.listeners = []
	248	self.listeners = [] # type: List[dict]
246	249	for listener in config.get("listeners", []):
247	250	if not isinstance(listener.get("port", None), int):
248	251	raise ConfigError(

286	289	validator=attr.validators.instance_of(bool), default=False
287	290	)
288	291	complexity = attr.ib(
289		validator=attr.validators.instance_of((int, float)), default=1.0
	292	validator=attr.validators.instance_of(
	293	(float, int) # type: ignore[arg-type] # noqa
	294	),
	295	default=1.0,
290	296	)
291	297	complexity_error = attr.ib(
292	298	validator=attr.validators.instance_of(str),

365	371	"cleanup_extremities_with_dummy_events", True
366	372	)
367	373
368		def has_tls_listener(self):
	374	def has_tls_listener(self) -> bool:
369	375	return any(l["tls"] for l in self.listeners)
370	376
371	377	def generate_config_section(

525	531	# Whether room invites to users on this server should be blocked
526	532	# (except those sent by local server admins). The default is False.
527	533	#
528		#block_non_admin_invites: True
	534	#block_non_admin_invites: true
529	535
530	536	# Room searching
531	537	#

666	672
667	673	# Global blocking
668	674	#
669		#hs_disabled: False
	675	#hs_disabled: false
670	676	#hs_disabled_message: 'Human readable reason for why the HS is blocked'
671		#hs_disabled_limit_type: 'error code(str), to help clients decode reason'
672	677
673	678	# Monthly Active User Blocking
674	679	#

688	693	# sign up in a short space of time never to return after their initial
689	694	# session.
690	695	#
691		#limit_usage_by_mau: False
	696	# 'mau_limit_alerting' is a means of limiting client side alerting
	697	# should the mau limit be reached. This is useful for small instances
	698	# where the admin has 5 mau seats (say) for 5 specific people and no
	699	# interest increasing the mau limit further. Defaults to True, which
	700	# means that alerting is enabled
	701	#
	702	#limit_usage_by_mau: false
692	703	#max_mau_value: 50
693	704	#mau_trial_days: 2
	705	#mau_limit_alerting: false
694	706
695	707	# If enabled, the metrics for the number of monthly active users will
696	708	# be populated, however no one will be limited. If limit_usage_by_mau
697	709	# is true, this is implied to be true.
698	710	#
699		#mau_stats_only: False
	711	#mau_stats_only: false
700	712
701	713	# Sometimes the server admin will want to ensure certain accounts are
702	714	# never blocked by mau checking. These accounts are specified here.

721	733	#
722	734	# Uncomment the below lines to enable:
723	735	#limit_remote_rooms:
724		# enabled: True
	736	# enabled: true
725	737	# complexity: 1.0
726	738	# complexity_error: "This room is too complex."
727	739

+4

-2

synapse/config/server_notices_config.py less more

58	58	None if server notices are not enabled.
59	59	"""
60	60
61		def __init__(self):
62		super(ServerNoticesConfig, self).__init__()
	61	section = "servernotices"
	62
	63	def __init__(self, *args):
	64	super(ServerNoticesConfig, self).__init__(*args)
63	65	self.server_notices_mxid = None
64	66	self.server_notices_mxid_display_name = None
65	67	self.server_notices_mxid_avatar_url = None

+2

-0

synapse/config/spam_checker.py less more

18	18
19	19
20	20	class SpamCheckerConfig(Config):
	21	section = "spamchecker"
	22
21	23	def read_config(self, config, **kwargs):
22	24	self.spam_checker = None
23	25

+2

-0

synapse/config/stats.py less more

23	23	"""Stats Configuration
24	24	Configuration for the behaviour of synapse's stats engine
25	25	"""
	26
	27	section = "stats"
26	28
27	29	def read_config(self, config, **kwargs):
28	30	self.stats_enabled = True

+2

-0

synapse/config/third_party_event_rules.py less more

18	18
19	19
20	20	class ThirdPartyRulesConfig(Config):
	21	section = "thirdpartyrules"
	22
21	23	def read_config(self, config, **kwargs):
22	24	self.third_party_event_rules = None
23	25

+14

-4

synapse/config/tls.py less more

17	17	import warnings
18	18	from datetime import datetime
19	19	from hashlib import sha256
	20	from typing import List
20	21
21	22	import six
22	23

32	33
33	34
34	35	class TlsConfig(Config):
35		def read_config(self, config, config_dir_path, **kwargs):
	36	section = "tls"
	37
	38	def read_config(self, config: dict, config_dir_path: str, **kwargs):
36	39
37	40	acme_config = config.get("acme", None)
38	41	if acme_config is None:

56	59	self.tls_certificate_file = self.abspath(config.get("tls_certificate_path"))
57	60	self.tls_private_key_file = self.abspath(config.get("tls_private_key_path"))
58	61
59		if self.has_tls_listener():
	62	if self.root.server.has_tls_listener():
60	63	if not self.tls_certificate_file:
61	64	raise ConfigError(
62	65	"tls_certificate_path must be specified if TLS-enabled listeners are "

107	110	)
108	111
109	112	# Support globs (*) in whitelist values
110		self.federation_certificate_verification_whitelist = []
	113	self.federation_certificate_verification_whitelist = [] # type: List[str]
111	114	for entry in fed_whitelist_entries:
112	115	try:
113	116	entry_regex = glob_to_regex(entry.encode("ascii").decode("ascii"))

285	288	"http://localhost:8009/.well-known/acme-challenge"
286	289	)
287	290
	291	# flake8 doesn't recognise that variables are used in the below string
	292	_ = tls_enabled, proxypassline, acme_enabled, default_acme_account_file
	293
288	294	return (
289	295	"""\
290	296	## TLS ##

447	453	#tls_fingerprints: [{"sha256": "<base64_encoded_sha256_fingerprint>"}]
448	454
449	455	"""
450		% locals()
	456	# Lowercase the string representation of boolean values
	457	% {
	458	x[0]: str(x[1]).lower() if isinstance(x[1], bool) else x[1]
	459	for x in locals().items()
	460	}
451	461	)
452	462
453	463	def read_tls_certificate(self):

+2

-0

synapse/config/tracer.py less more

18	18
19	19
20	20	class TracerConfig(Config):
	21	section = "tracing"
	22
21	23	def read_config(self, config, **kwargs):
22	24	opentracing_config = config.get("opentracing")
23	25	if opentracing_config is None:

+2

-0

synapse/config/user_directory.py less more

19	19	"""User Directory Configuration
20	20	Configuration for the behaviour of the /user_directory API
21	21	"""
	22
	23	section = "userdirectory"
22	24
23	25	def read_config(self, config, **kwargs):
24	26	self.user_directory_search_enabled = True

+3

-1

synapse/config/voip.py less more

15	15
16	16
17	17	class VoipConfig(Config):
	18	section = "voip"
	19
18	20	def read_config(self, config, **kwargs):
19	21	self.turn_uris = config.get("turn_uris", [])
20	22	self.turn_shared_secret = config.get("turn_shared_secret")

53	55	# connect to arbitrary endpoints without having first signed up for a
54	56	# valid account (e.g. by passing a CAPTCHA).
55	57	#
56		#turn_allow_guests: True
	58	#turn_allow_guests: true
57	59	"""

+2

-0

synapse/config/workers.py less more

19	19	"""The workers are processes run separately to the main synapse process.
20	20	They have their own pid_file and listener configuration. They use the
21	21	replication_url to talk to the main synapse process."""
	22
	23	section = "worker"
22	24
23	25	def read_config(self, config, **kwargs):
24	26	self.worker_app = config.get("worker_app")

+1

-2

synapse/event_auth.py less more

492	492	new_level_too_big = new_level is not None and new_level > user_level
493	493	if old_level_too_big or new_level_too_big:
494	494	raise AuthError(
495		403,
496		"You don't have permission to add ops level greater " "than your own",
	495	403, "You don't have permission to add ops level greater than your own"
497	496	)
498	497
499	498

+0

-38

synapse/federation/federation_client.py less more

878	878	)
879	879
880	880	@defer.inlineCallbacks
881		def query_auth(self, destination, room_id, event_id, local_auth):
882		"""
883		Params:
884		destination (str)
885		event_it (str)
886		local_auth (list)
887		"""
888		time_now = self._clock.time_msec()
889
890		send_content = {"auth_chain": [e.get_pdu_json(time_now) for e in local_auth]}
891
892		code, content = yield self.transport_layer.send_query_auth(
893		destination=destination,
894		room_id=room_id,
895		event_id=event_id,
896		content=send_content,
897		)
898
899		room_version = yield self.store.get_room_version(room_id)
900		format_ver = room_version_to_event_format(room_version)
901
902		auth_chain = [event_from_pdu_json(e, format_ver) for e in content["auth_chain"]]
903
904		signed_auth = yield self._check_sigs_and_hash_and_fetch(
905		destination, auth_chain, outlier=True, room_version=room_version
906		)
907
908		signed_auth.sort(key=lambda e: e.depth)
909
910		ret = {
911		"auth_chain": signed_auth,
912		"rejects": content.get("rejects", []),
913		"missing": content.get("missing", []),
914		}
915
916		return ret
917
918		@defer.inlineCallbacks
919	881	def get_missing_events(
920	882	self,
921	883	destination,

+0

-13

synapse/federation/federation_server.py less more

35	35	UnsupportedRoomVersionError,
36	36	)
37	37	from synapse.api.room_versions import KNOWN_ROOM_VERSIONS
38		from synapse.crypto.event_signing import compute_event_signature
39	38	from synapse.events import room_version_to_event_format
40	39	from synapse.federation.federation_base import FederationBase, event_from_pdu_json
41	40	from synapse.federation.persistence import TransactionActions

320	319	def _on_context_state_request_compute(self, room_id, event_id):
321	320	pdus = yield self.handler.get_state_for_pdu(room_id, event_id)
322	321	auth_chain = yield self.store.get_auth_chain([pdu.event_id for pdu in pdus])
323
324		for event in auth_chain:
325		# We sign these again because there was a bug where we
326		# incorrectly signed things the first time round
327		if self.hs.is_mine_id(event.event_id):
328		event.signatures.update(
329		compute_event_signature(
330		event.get_pdu_json(),
331		self.hs.hostname,
332		self.hs.config.signing_key[0],
333		)
334		)
335	322
336	323	return {
337	324	"pdus": [pdu.get_pdu_json() for pdu in pdus],

+6

-5

synapse/federation/sender/__init__.py less more

37	37	events_processed_counter,
38	38	)
39	39	from synapse.metrics.background_process_metrics import run_as_background_process
40		from synapse.util.metrics import measure_func
	40	from synapse.util.metrics import Measure, measure_func
41	41
42	42	logger = logging.getLogger(__name__)
43	43

182	182	# Otherwise if the last member on a server in a room is
183	183	# banned then it won't receive the event because it won't
184	184	# be in the room after the ban.
185		destinations = yield self.state.get_current_hosts_in_room(
186		event.room_id, latest_event_ids=event.prev_event_ids()
	185	destinations = yield self.state.get_hosts_in_room_at_events(
	186	event.room_id, event_ids=event.prev_event_ids()
187	187	)
188	188	except Exception:
189	189	logger.exception(

206	206
207	207	@defer.inlineCallbacks
208	208	def handle_room_events(events):
209		for event in events:
210		yield handle_event(event)
	209	with Measure(self.clock, "handle_room_events"):
	210	for event in events:
	211	yield handle_event(event)
211	212
212	213	events_by_room = {}
213	214	for event in events:

+1

-1

synapse/federation/sender/per_destination_queue.py less more

29	29	from synapse.handlers.presence import format_user_presence_state
30	30	from synapse.metrics import sent_transactions_counter
31	31	from synapse.metrics.background_process_metrics import run_as_background_process
32		from synapse.storage import UserPresenceState
	32	from synapse.storage.presence import UserPresenceState
33	33	from synapse.util.retryutils import NotRetryingDestination, get_retry_limiter
34	34
35	35	# This is defined in the Matrix spec and enforced by the receiver.

+0

-11

synapse/federation/transport/client.py less more

382	382
383	383	@defer.inlineCallbacks
384	384	@log_function
385		def send_query_auth(self, destination, room_id, event_id, content):
386		path = _create_v1_path("/query_auth/%s/%s", room_id, event_id)
387
388		content = yield self.client.post_json(
389		destination=destination, path=path, data=content
390		)
391
392		return content
393
394		@defer.inlineCallbacks
395		@log_function
396	385	def query_client_keys(self, destination, query_content, timeout):
397	386	"""Query the device keys for a list of user ids hosted on a remote
398	387	server.

+8

-0

synapse/federation/transport/server.py less more

764	764	else:
765	765	network_tuple = ThirdPartyInstanceID(None, None)
766	766
	767	if limit == 0:
	768	# zero is a special value which corresponds to no limit.
	769	limit = None
	770
767	771	data = await maybeDeferred(
768	772	self.handler.get_local_public_room_list,
769	773	limit,

798	802
799	803	if search_filter is None:
800	804	logger.warning("Nonefilter")
	805
	806	if limit == 0:
	807	# zero is a special value which corresponds to no limit.
	808	limit = None
801	809
802	810	data = await self.handler.get_local_public_room_list(
803	811	limit=limit,

+16

-4

synapse/groups/groups_server.py less more

0	0	# -- coding: utf-8 --
1	1	# Copyright 2017 Vector Creations Ltd
2	2	# Copyright 2018 New Vector Ltd
	3	# Copyright 2019 Michael Telatynski <7t3chguy@gmail.com>
3	4	#
4	5	# Licensed under the Apache License, Version 2.0 (the "License");
5	6	# you may not use this file except in compliance with the License.

19	20
20	21	from twisted.internet import defer
21	22
22		from synapse.api.errors import SynapseError
	23	from synapse.api.errors import Codes, SynapseError
23	24	from synapse.types import GroupID, RoomID, UserID, get_domain_from_id
24	25	from synapse.util.async_helpers import concurrently_execute
25	26
26	27	logger = logging.getLogger(__name__)
27	28
28	29
29		# TODO: Allow users to "knock" or simpkly join depending on rules
	30	# TODO: Allow users to "knock" or simply join depending on rules
30	31	# TODO: Federation admin APIs
31		# TODO: is_priveged flag to users and is_public to users and rooms
	32	# TODO: is_privileged flag to users and is_public to users and rooms
32	33	# TODO: Audit log for admins (profile updates, membership changes, users who tried
33	34	# to join but were rejected, etc)
34	35	# TODO: Flairs

589	590	)
590	591
591	592	# TODO: Check if user knocked
592		# TODO: Check if user is already invited
	593
	594	invited_users = yield self.store.get_invited_users_in_group(group_id)
	595	if user_id in invited_users:
	596	raise SynapseError(
	597	400, "User already invited to group", errcode=Codes.BAD_STATE
	598	)
	599
	600	user_results = yield self.store.get_users_in_group(
	601	group_id, include_private=True
	602	)
	603	if user_id in [user_result["user_id"] for user_result in user_results]:
	604	raise SynapseError(400, "User already in group")
593	605
594	606	content = {
595	607	"profile": {"name": group["name"], "avatar_url": group["avatar_url"]},

+37

-0

synapse/handlers/deactivate_account.py less more

119	119	# parts users from rooms (if it isn't already running)
120	120	self._start_user_parting()
121	121
	122	# Reject all pending invites for the user, so that the user doesn't show up in the
	123	# "invited" section of rooms' members list.
	124	yield self._reject_pending_invites_for_user(user_id)
	125
122	126	# Remove all information on the user from the account_validity table.
123	127	if self._account_validity_enabled:
124	128	yield self.store.delete_account_validity_for_user(user_id)

127	131	yield self.store.set_user_deactivated_status(user_id, True)
128	132
129	133	return identity_server_supports_unbinding
	134
	135	@defer.inlineCallbacks
	136	def _reject_pending_invites_for_user(self, user_id):
	137	"""Reject pending invites addressed to a given user ID.
	138
	139	Args:
	140	user_id (str): The user ID to reject pending invites for.
	141	"""
	142	user = UserID.from_string(user_id)
	143	pending_invites = yield self.store.get_invited_rooms_for_user(user_id)
	144
	145	for room in pending_invites:
	146	try:
	147	yield self._room_member_handler.update_membership(
	148	create_requester(user),
	149	user,
	150	room.room_id,
	151	"leave",
	152	ratelimit=False,
	153	require_consent=False,
	154	)
	155	logger.info(
	156	"Rejected invite for deactivated user %r in room %r",
	157	user_id,
	158	room.room_id,
	159	)
	160	except Exception:
	161	logger.exception(
	162	"Failed to reject invite for user %r in room %r:"
	163	" ignoring and continuing",
	164	user_id,
	165	room.room_id,
	166	)
130	167
131	168	def _start_user_parting(self):
132	169	"""

+17

-0

synapse/handlers/device.py less more

0	0	# -- coding: utf-8 --
1	1	# Copyright 2016 OpenMarket Ltd
	2	# Copyright 2019 New Vector Ltd
	3	# Copyright 2019 The Matrix.org Foundation C.I.C.
2	4	#
3	5	# Licensed under the Apache License, Version 2.0 (the "License");
4	6	# you may not use this file except in compliance with the License.

436	438	for host in hosts:
437	439	self.federation_sender.send_device_messages(host)
438	440	log_kv({"message": "sent device update to host", "host": host})
	441
	442	@defer.inlineCallbacks
	443	def notify_user_signature_update(self, from_user_id, user_ids):
	444	"""Notify a user that they have made new signatures of other users.
	445
	446	Args:
	447	from_user_id (str): the user who made the signature
	448	user_ids (list[str]): the users IDs that have new signatures
	449	"""
	450
	451	position = yield self.store.add_user_signature_change_to_streams(
	452	from_user_id, user_ids
	453	)
	454
	455	self.notifier.on_new_event("device_list_key", position, users=[from_user_id])
439	456
440	457	@defer.inlineCallbacks
441	458	def on_federation_query_user_devices(self, user_id):

+584

-13

synapse/handlers/e2e_keys.py less more

0	0	# -- coding: utf-8 --
1	1	# Copyright 2016 OpenMarket Ltd
2		# Copyright 2018 New Vector Ltd
	2	# Copyright 2018-2019 New Vector Ltd
	3	# Copyright 2019 The Matrix.org Foundation C.I.C.
3	4	#
4	5	# Licensed under the Apache License, Version 2.0 (the "License");
5	6	# you may not use this file except in compliance with the License.

17	18
18	19	from six import iteritems
19	20
	21	import attr
20	22	from canonicaljson import encode_canonical_json, json
	23	from signedjson.key import decode_verify_key_bytes
	24	from signedjson.sign import SignatureVerifyException, verify_signed_json
	25	from unpaddedbase64 import decode_base64
21	26
22	27	from twisted.internet import defer
23	28
24		from synapse.api.errors import CodeMessageException, SynapseError
	29	from synapse.api.errors import CodeMessageException, Codes, NotFoundError, SynapseError
25	30	from synapse.logging.context import make_deferred_yieldable, run_in_background
26	31	from synapse.logging.opentracing import log_kv, set_tag, tag_args, trace
27		from synapse.types import UserID, get_domain_from_id
	32	from synapse.types import (
	33	UserID,
	34	get_domain_from_id,
	35	get_verify_key_from_cross_signing_key,
	36	)
28	37	from synapse.util import unwrapFirstError
29	38	from synapse.util.retryutils import NotRetryingDestination
30	39

48	57
49	58	@trace
50	59	@defer.inlineCallbacks
51		def query_devices(self, query_body, timeout):
	60	def query_devices(self, query_body, timeout, from_user_id):
52	61	""" Handle a device key query from a client
53	62
54	63	{

66	75	}
67	76	}
68	77	}
	78
	79	Args:
	80	from_user_id (str): the user making the query. This is used when
	81	adding cross-signing signatures to limit what signatures users
	82	can see.
69	83	"""
70	84
71	85	device_keys_query = query_body.get("device_keys", {})

123	137	domain = get_domain_from_id(user_id)
124	138	r = remote_queries_not_in_cache.setdefault(domain, {})
125	139	r[user_id] = remote_queries[user_id]
	140
	141	# Get cached cross-signing keys
	142	cross_signing_keys = yield self.get_cross_signing_keys_from_cache(
	143	device_keys_query, from_user_id
	144	)
126	145
127	146	# Now fetch any devices that we don't have in our cache
128	147	@trace

187	206	if user_id in destination_query:
188	207	results[user_id] = keys
189	208
	209	for user_id, key in remote_result["master_keys"].items():
	210	if user_id in destination_query:
	211	cross_signing_keys["master_keys"][user_id] = key
	212
	213	for user_id, key in remote_result["self_signing_keys"].items():
	214	if user_id in destination_query:
	215	cross_signing_keys["self_signing_keys"][user_id] = key
	216
190	217	except Exception as e:
191	218	failure = _exception_to_failure(e)
192	219	failures[destination] = failure

203	230	).addErrback(unwrapFirstError)
204	231	)
205	232
206		return {"device_keys": results, "failures": failures}
	233	ret = {"device_keys": results, "failures": failures}
	234
	235	ret.update(cross_signing_keys)
	236
	237	return ret
	238
	239	@defer.inlineCallbacks
	240	def get_cross_signing_keys_from_cache(self, query, from_user_id):
	241	"""Get cross-signing keys for users from the database
	242
	243	Args:
	244	query (Iterable[string]) an iterable of user IDs. A dict whose keys
	245	are user IDs satisfies this, so the query format used for
	246	query_devices can be used here.
	247	from_user_id (str): the user making the query. This is used when
	248	adding cross-signing signatures to limit what signatures users
	249	can see.
	250
	251	Returns:
	252	defer.Deferred[dict[str, dict[str, dict]]]: map from
	253	(master\|self_signing\|user_signing) -> user_id -> key
	254	"""
	255	master_keys = {}
	256	self_signing_keys = {}
	257	user_signing_keys = {}
	258
	259	for user_id in query:
	260	# XXX: consider changing the store functions to allow querying
	261	# multiple users simultaneously.
	262	key = yield self.store.get_e2e_cross_signing_key(
	263	user_id, "master", from_user_id
	264	)
	265	if key:
	266	master_keys[user_id] = key
	267
	268	key = yield self.store.get_e2e_cross_signing_key(
	269	user_id, "self_signing", from_user_id
	270	)
	271	if key:
	272	self_signing_keys[user_id] = key
	273
	274	# users can see other users' master and self-signing keys, but can
	275	# only see their own user-signing keys
	276	if from_user_id == user_id:
	277	key = yield self.store.get_e2e_cross_signing_key(
	278	user_id, "user_signing", from_user_id
	279	)
	280	if key:
	281	user_signing_keys[user_id] = key
	282
	283	return {
	284	"master_keys": master_keys,
	285	"self_signing_keys": self_signing_keys,
	286	"user_signing_keys": user_signing_keys,
	287	}
207	288
208	289	@trace
209	290	@defer.inlineCallbacks

247	328
248	329	results = yield self.store.get_e2e_device_keys(local_query)
249	330
250		# Build the result structure, un-jsonify the results, and add the
251		# "unsigned" section
	331	# Build the result structure
252	332	for user_id, device_keys in results.items():
253	333	for device_id, device_info in device_keys.items():
254		r = dict(device_info["keys"])
255		r["unsigned"] = {}
256		display_name = device_info["device_display_name"]
257		if display_name is not None:
258		r["unsigned"]["device_display_name"] = display_name
259		result_dict[user_id][device_id] = r
	334	result_dict[user_id][device_id] = device_info
260	335
261	336	log_kv(results)
262	337	return result_dict

446	521	log_kv({"message": "Inserting new one_time_keys.", "keys": new_keys})
447	522	yield self.store.add_e2e_one_time_keys(user_id, device_id, time_now, new_keys)
448	523
	524	@defer.inlineCallbacks
	525	def upload_signing_keys_for_user(self, user_id, keys):
	526	"""Upload signing keys for cross-signing
	527
	528	Args:
	529	user_id (string): the user uploading the keys
	530	keys (dict[string, dict]): the signing keys
	531	"""
	532
	533	# if a master key is uploaded, then check it. Otherwise, load the
	534	# stored master key, to check signatures on other keys
	535	if "master_key" in keys:
	536	master_key = keys["master_key"]
	537
	538	_check_cross_signing_key(master_key, user_id, "master")
	539	else:
	540	master_key = yield self.store.get_e2e_cross_signing_key(user_id, "master")
	541
	542	# if there is no master key, then we can't do anything, because all the
	543	# other cross-signing keys need to be signed by the master key
	544	if not master_key:
	545	raise SynapseError(400, "No master key available", Codes.MISSING_PARAM)
	546
	547	try:
	548	master_key_id, master_verify_key = get_verify_key_from_cross_signing_key(
	549	master_key
	550	)
	551	except ValueError:
	552	if "master_key" in keys:
	553	# the invalid key came from the request
	554	raise SynapseError(400, "Invalid master key", Codes.INVALID_PARAM)
	555	else:
	556	# the invalid key came from the database
	557	logger.error("Invalid master key found for user %s", user_id)
	558	raise SynapseError(500, "Invalid master key")
	559
	560	# for the other cross-signing keys, make sure that they have valid
	561	# signatures from the master key
	562	if "self_signing_key" in keys:
	563	self_signing_key = keys["self_signing_key"]
	564
	565	_check_cross_signing_key(
	566	self_signing_key, user_id, "self_signing", master_verify_key
	567	)
	568
	569	if "user_signing_key" in keys:
	570	user_signing_key = keys["user_signing_key"]
	571
	572	_check_cross_signing_key(
	573	user_signing_key, user_id, "user_signing", master_verify_key
	574	)
	575
	576	# if everything checks out, then store the keys and send notifications
	577	deviceids = []
	578	if "master_key" in keys:
	579	yield self.store.set_e2e_cross_signing_key(user_id, "master", master_key)
	580	deviceids.append(master_verify_key.version)
	581	if "self_signing_key" in keys:
	582	yield self.store.set_e2e_cross_signing_key(
	583	user_id, "self_signing", self_signing_key
	584	)
	585	try:
	586	deviceids.append(
	587	get_verify_key_from_cross_signing_key(self_signing_key)[1].version
	588	)
	589	except ValueError:
	590	raise SynapseError(400, "Invalid self-signing key", Codes.INVALID_PARAM)
	591	if "user_signing_key" in keys:
	592	yield self.store.set_e2e_cross_signing_key(
	593	user_id, "user_signing", user_signing_key
	594	)
	595	# the signature stream matches the semantics that we want for
	596	# user-signing key updates: only the user themselves is notified of
	597	# their own user-signing key updates
	598	yield self.device_handler.notify_user_signature_update(user_id, [user_id])
	599
	600	# master key and self-signing key updates match the semantics of device
	601	# list updates: all users who share an encrypted room are notified
	602	if len(deviceids):
	603	yield self.device_handler.notify_device_update(user_id, deviceids)
	604
	605	return {}
	606
	607	@defer.inlineCallbacks
	608	def upload_signatures_for_device_keys(self, user_id, signatures):
	609	"""Upload device signatures for cross-signing
	610
	611	Args:
	612	user_id (string): the user uploading the signatures
	613	signatures (dict[string, dict[string, dict]]): map of users to
	614	devices to signed keys. This is the submission from the user; an
	615	exception will be raised if it is malformed.
	616	Returns:
	617	dict: response to be sent back to the client. The response will have
	618	a "failures" key, which will be a dict mapping users to devices
	619	to errors for the signatures that failed.
	620	Raises:
	621	SynapseError: if the signatures dict is not valid.
	622	"""
	623	failures = {}
	624
	625	# signatures to be stored. Each item will be a SignatureListItem
	626	signature_list = []
	627
	628	# split between checking signatures for own user and signatures for
	629	# other users, since we verify them with different keys
	630	self_signatures = signatures.get(user_id, {})
	631	other_signatures = {k: v for k, v in signatures.items() if k != user_id}
	632
	633	self_signature_list, self_failures = yield self._process_self_signatures(
	634	user_id, self_signatures
	635	)
	636	signature_list.extend(self_signature_list)
	637	failures.update(self_failures)
	638
	639	other_signature_list, other_failures = yield self._process_other_signatures(
	640	user_id, other_signatures
	641	)
	642	signature_list.extend(other_signature_list)
	643	failures.update(other_failures)
	644
	645	# store the signature, and send the appropriate notifications for sync
	646	logger.debug("upload signature failures: %r", failures)
	647	yield self.store.store_e2e_cross_signing_signatures(user_id, signature_list)
	648
	649	self_device_ids = [item.target_device_id for item in self_signature_list]
	650	if self_device_ids:
	651	yield self.device_handler.notify_device_update(user_id, self_device_ids)
	652	signed_users = [item.target_user_id for item in other_signature_list]
	653	if signed_users:
	654	yield self.device_handler.notify_user_signature_update(
	655	user_id, signed_users
	656	)
	657
	658	return {"failures": failures}
	659
	660	@defer.inlineCallbacks
	661	def _process_self_signatures(self, user_id, signatures):
	662	"""Process uploaded signatures of the user's own keys.
	663
	664	Signatures of the user's own keys from this API come in two forms:
	665	- signatures of the user's devices by the user's self-signing key,
	666	- signatures of the user's master key by the user's devices.
	667
	668	Args:
	669	user_id (string): the user uploading the keys
	670	signatures (dict[string, dict]): map of devices to signed keys
	671
	672	Returns:
	673	(list[SignatureListItem], dict[string, dict[string, dict]]):
	674	a list of signatures to store, and a map of users to devices to failure
	675	reasons
	676
	677	Raises:
	678	SynapseError: if the input is malformed
	679	"""
	680	signature_list = []
	681	failures = {}
	682	if not signatures:
	683	return signature_list, failures
	684
	685	if not isinstance(signatures, dict):
	686	raise SynapseError(400, "Invalid parameter", Codes.INVALID_PARAM)
	687
	688	try:
	689	# get our self-signing key to verify the signatures
	690	_, self_signing_key_id, self_signing_verify_key = yield self._get_e2e_cross_signing_verify_key(
	691	user_id, "self_signing"
	692	)
	693
	694	# get our master key, since we may have received a signature of it.
	695	# We need to fetch it here so that we know what its key ID is, so
	696	# that we can check if a signature that was sent is a signature of
	697	# the master key or of a device
	698	master_key, _, master_verify_key = yield self._get_e2e_cross_signing_verify_key(
	699	user_id, "master"
	700	)
	701
	702	# fetch our stored devices. This is used to 1. verify
	703	# signatures on the master key, and 2. to compare with what
	704	# was sent if the device was signed
	705	devices = yield self.store.get_e2e_device_keys([(user_id, None)])
	706
	707	if user_id not in devices:
	708	raise NotFoundError("No device keys found")
	709
	710	devices = devices[user_id]
	711	except SynapseError as e:
	712	failure = _exception_to_failure(e)
	713	failures[user_id] = {device: failure for device in signatures.keys()}
	714	return signature_list, failures
	715
	716	for device_id, device in signatures.items():
	717	# make sure submitted data is in the right form
	718	if not isinstance(device, dict):
	719	raise SynapseError(400, "Invalid parameter", Codes.INVALID_PARAM)
	720
	721	try:
	722	if "signatures" not in device or user_id not in device["signatures"]:
	723	# no signature was sent
	724	raise SynapseError(
	725	400, "Invalid signature", Codes.INVALID_SIGNATURE
	726	)
	727
	728	if device_id == master_verify_key.version:
	729	# The signature is of the master key. This needs to be
	730	# handled differently from signatures of normal devices.
	731	master_key_signature_list = self._check_master_key_signature(
	732	user_id, device_id, device, master_key, devices
	733	)
	734	signature_list.extend(master_key_signature_list)
	735	continue
	736
	737	# at this point, we have a device that should be signed
	738	# by the self-signing key
	739	if self_signing_key_id not in device["signatures"][user_id]:
	740	# no signature was sent
	741	raise SynapseError(
	742	400, "Invalid signature", Codes.INVALID_SIGNATURE
	743	)
	744
	745	try:
	746	stored_device = devices[device_id]
	747	except KeyError:
	748	raise NotFoundError("Unknown device")
	749	if self_signing_key_id in stored_device.get("signatures", {}).get(
	750	user_id, {}
	751	):
	752	# we already have a signature on this device, so we
	753	# can skip it, since it should be exactly the same
	754	continue
	755
	756	_check_device_signature(
	757	user_id, self_signing_verify_key, device, stored_device
	758	)
	759
	760	signature = device["signatures"][user_id][self_signing_key_id]
	761	signature_list.append(
	762	SignatureListItem(
	763	self_signing_key_id, user_id, device_id, signature
	764	)
	765	)
	766	except SynapseError as e:
	767	failures.setdefault(user_id, {})[device_id] = _exception_to_failure(e)
	768
	769	return signature_list, failures
	770
	771	def _check_master_key_signature(
	772	self, user_id, master_key_id, signed_master_key, stored_master_key, devices
	773	):
	774	"""Check signatures of a user's master key made by their devices.
	775
	776	Args:
	777	user_id (string): the user whose master key is being checked
	778	master_key_id (string): the ID of the user's master key
	779	signed_master_key (dict): the user's signed master key that was uploaded
	780	stored_master_key (dict): our previously-stored copy of the user's master key
	781	devices (iterable(dict)): the user's devices
	782
	783	Returns:
	784	list[SignatureListItem]: a list of signatures to store
	785
	786	Raises:
	787	SynapseError: if a signature is invalid
	788	"""
	789	# for each device that signed the master key, check the signature.
	790	master_key_signature_list = []
	791	sigs = signed_master_key["signatures"]
	792	for signing_key_id, signature in sigs[user_id].items():
	793	_, signing_device_id = signing_key_id.split(":", 1)
	794	if (
	795	signing_device_id not in devices
	796	or signing_key_id not in devices[signing_device_id]["keys"]
	797	):
	798	# signed by an unknown device, or the
	799	# device does not have the key
	800	raise SynapseError(400, "Invalid signature", Codes.INVALID_SIGNATURE)
	801
	802	# get the key and check the signature
	803	pubkey = devices[signing_device_id]["keys"][signing_key_id]
	804	verify_key = decode_verify_key_bytes(signing_key_id, decode_base64(pubkey))
	805	_check_device_signature(
	806	user_id, verify_key, signed_master_key, stored_master_key
	807	)
	808
	809	master_key_signature_list.append(
	810	SignatureListItem(signing_key_id, user_id, master_key_id, signature)
	811	)
	812
	813	return master_key_signature_list
	814
	815	@defer.inlineCallbacks
	816	def _process_other_signatures(self, user_id, signatures):
	817	"""Process uploaded signatures of other users' keys. These will be the
	818	target user's master keys, signed by the uploading user's user-signing
	819	key.
	820
	821	Args:
	822	user_id (string): the user uploading the keys
	823	signatures (dict[string, dict]): map of users to devices to signed keys
	824
	825	Returns:
	826	(list[SignatureListItem], dict[string, dict[string, dict]]):
	827	a list of signatures to store, and a map of users to devices to failure
	828	reasons
	829
	830	Raises:
	831	SynapseError: if the input is malformed
	832	"""
	833	signature_list = []
	834	failures = {}
	835	if not signatures:
	836	return signature_list, failures
	837
	838	try:
	839	# get our user-signing key to verify the signatures
	840	user_signing_key, user_signing_key_id, user_signing_verify_key = yield self._get_e2e_cross_signing_verify_key(
	841	user_id, "user_signing"
	842	)
	843	except SynapseError as e:
	844	failure = _exception_to_failure(e)
	845	for user, devicemap in signatures.items():
	846	failures[user] = {device_id: failure for device_id in devicemap.keys()}
	847	return signature_list, failures
	848
	849	for target_user, devicemap in signatures.items():
	850	# make sure submitted data is in the right form
	851	if not isinstance(devicemap, dict):
	852	raise SynapseError(400, "Invalid parameter", Codes.INVALID_PARAM)
	853	for device in devicemap.values():
	854	if not isinstance(device, dict):
	855	raise SynapseError(400, "Invalid parameter", Codes.INVALID_PARAM)
	856
	857	device_id = None
	858	try:
	859	# get the target user's master key, to make sure it matches
	860	# what was sent
	861	master_key, master_key_id, _ = yield self._get_e2e_cross_signing_verify_key(
	862	target_user, "master", user_id
	863	)
	864
	865	# make sure that the target user's master key is the one that
	866	# was signed (and no others)
	867	device_id = master_key_id.split(":", 1)[1]
	868	if device_id not in devicemap:
	869	logger.debug(
	870	"upload signature: could not find signature for device %s",
	871	device_id,
	872	)
	873	# set device to None so that the failure gets
	874	# marked on all the signatures
	875	device_id = None
	876	raise NotFoundError("Unknown device")
	877	key = devicemap[device_id]
	878	other_devices = [k for k in devicemap.keys() if k != device_id]
	879	if other_devices:
	880	# other devices were signed -- mark those as failures
	881	logger.debug("upload signature: too many devices specified")
	882	failure = _exception_to_failure(NotFoundError("Unknown device"))
	883	failures[target_user] = {
	884	device: failure for device in other_devices
	885	}
	886
	887	if user_signing_key_id in master_key.get("signatures", {}).get(
	888	user_id, {}
	889	):
	890	# we already have the signature, so we can skip it
	891	continue
	892
	893	_check_device_signature(
	894	user_id, user_signing_verify_key, key, master_key
	895	)
	896
	897	signature = key["signatures"][user_id][user_signing_key_id]
	898	signature_list.append(
	899	SignatureListItem(
	900	user_signing_key_id, target_user, device_id, signature
	901	)
	902	)
	903	except SynapseError as e:
	904	failure = _exception_to_failure(e)
	905	if device_id is None:
	906	failures[target_user] = {
	907	device_id: failure for device_id in devicemap.keys()
	908	}
	909	else:
	910	failures.setdefault(target_user, {})[device_id] = failure
	911
	912	return signature_list, failures
	913
	914	@defer.inlineCallbacks
	915	def _get_e2e_cross_signing_verify_key(self, user_id, key_type, from_user_id=None):
	916	"""Fetch the cross-signing public key from storage and interpret it.
	917
	918	Args:
	919	user_id (str): the user whose key should be fetched
	920	key_type (str): the type of key to fetch
	921	from_user_id (str): the user that we are fetching the keys for.
	922	This affects what signatures are fetched.
	923
	924	Returns:
	925	dict, str, VerifyKey: the raw key data, the key ID, and the
	926	signedjson verify key
	927
	928	Raises:
	929	NotFoundError: if the key is not found
	930	"""
	931	key = yield self.store.get_e2e_cross_signing_key(
	932	user_id, key_type, from_user_id
	933	)
	934	if key is None:
	935	logger.debug("no %s key found for %s", key_type, user_id)
	936	raise NotFoundError("No %s key found for %s" % (key_type, user_id))
	937	key_id, verify_key = get_verify_key_from_cross_signing_key(key)
	938	return key, key_id, verify_key
	939
	940
	941	def _check_cross_signing_key(key, user_id, key_type, signing_key=None):
	942	"""Check a cross-signing key uploaded by a user. Performs some basic sanity
	943	checking, and ensures that it is signed, if a signature is required.
	944
	945	Args:
	946	key (dict): the key data to verify
	947	user_id (str): the user whose key is being checked
	948	key_type (str): the type of key that the key should be
	949	signing_key (VerifyKey): (optional) the signing key that the key should
	950	be signed with. If omitted, signatures will not be checked.
	951	"""
	952	if (
	953	key.get("user_id") != user_id
	954	or key_type not in key.get("usage", [])
	955	or len(key.get("keys", {})) != 1
	956	):
	957	raise SynapseError(400, ("Invalid %s key" % (key_type,)), Codes.INVALID_PARAM)
	958
	959	if signing_key:
	960	try:
	961	verify_signed_json(key, user_id, signing_key)
	962	except SignatureVerifyException:
	963	raise SynapseError(
	964	400, ("Invalid signature on %s key" % key_type), Codes.INVALID_SIGNATURE
	965	)
	966
	967
	968	def _check_device_signature(user_id, verify_key, signed_device, stored_device):
	969	"""Check that a signature on a device or cross-signing key is correct and
	970	matches the copy of the device/key that we have stored. Throws an
	971	exception if an error is detected.
	972
	973	Args:
	974	user_id (str): the user ID whose signature is being checked
	975	verify_key (VerifyKey): the key to verify the device with
	976	signed_device (dict): the uploaded signed device data
	977	stored_device (dict): our previously stored copy of the device
	978
	979	Raises:
	980	SynapseError: if the signature was invalid or the sent device is not the
	981	same as the stored device
	982
	983	"""
	984
	985	# make sure that the device submitted matches what we have stored
	986	stripped_signed_device = {
	987	k: v for k, v in signed_device.items() if k not in ["signatures", "unsigned"]
	988	}
	989	stripped_stored_device = {
	990	k: v for k, v in stored_device.items() if k not in ["signatures", "unsigned"]
	991	}
	992	if stripped_signed_device != stripped_stored_device:
	993	logger.debug(
	994	"upload signatures: key does not match %s vs %s",
	995	signed_device,
	996	stored_device,
	997	)
	998	raise SynapseError(400, "Key does not match")
	999
	1000	try:
	1001	verify_signed_json(signed_device, user_id, verify_key)
	1002	except SignatureVerifyException:
	1003	logger.debug("invalid signature on key")
	1004	raise SynapseError(400, "Invalid signature", Codes.INVALID_SIGNATURE)
	1005
449	1006
450	1007	def _exception_to_failure(e):
	1008	if isinstance(e, SynapseError):
	1009	return {"status": e.code, "errcode": e.errcode, "message": str(e)}
	1010
451	1011	if isinstance(e, CodeMessageException):
452	1012	return {"status": e.code, "message": str(e)}
453	1013

475	1035	new_key_copy.pop("signatures", None)
476	1036
477	1037	return old_key == new_key_copy
	1038
	1039
	1040	@attr.s
	1041	class SignatureListItem:
	1042	"""An item in the signature list as used by upload_signatures_for_device_keys.
	1043	"""
	1044
	1045	signing_key_id = attr.ib()
	1046	target_user_id = attr.ib()
	1047	target_device_id = attr.ib()
	1048	signature = attr.ib()

+2

-2

synapse/handlers/e2e_room_keys.py less more

351	351	A deferred of an empty dict.
352	352	"""
353	353	if "version" not in version_info:
354		raise SynapseError(400, "Missing version in body", Codes.MISSING_PARAM)
355		if version_info["version"] != version:
	354	version_info["version"] = version
	355	elif version_info["version"] != version:
356	356	raise SynapseError(
357	357	400, "Version in body does not match", Codes.INVALID_PARAM
358	358	)

+11

-107

synapse/handlers/federation.py less more

29	29
30	30	from twisted.internet import defer
31	31
	32	from synapse import event_auth
32	33	from synapse.api.constants import EventTypes, Membership, RejectedReason
33	34	from synapse.api.errors import (
34	35	AuthError,

1762	1763	auth_for_e[(EventTypes.Create, "")] = create_event
1763	1764
1764	1765	try:
1765		self.auth.check(room_version, e, auth_events=auth_for_e)
	1766	event_auth.check(room_version, e, auth_events=auth_for_e)
1766	1767	except SynapseError as err:
1767	1768	# we may get SynapseErrors here as well as AuthErrors. For
1768	1769	# instance, there are a couple of (ancient) events in some

1918	1919	}
1919	1920
1920	1921	try:
1921		self.auth.check(room_version, event, auth_events=current_auth_events)
	1922	event_auth.check(room_version, event, auth_events=current_auth_events)
1922	1923	except AuthError as e:
1923	1924	logger.warn("Soft-failing %r because %s", event, e)
1924	1925	event.internal_metadata.soft_failed = True

2017	2018	)
2018	2019
2019	2020	try:
2020		self.auth.check(room_version, event, auth_events=auth_events)
	2021	event_auth.check(room_version, event, auth_events=auth_events)
2021	2022	except AuthError as e:
2022	2023	logger.warn("Failed auth resolution for %r because %s", event, e)
2023	2024	raise e

2180	2181
2181	2182	auth_events.update(new_state)
2182	2183
2183		different_auth = event_auth_events.difference(
2184		e.event_id for e in auth_events.values()
2185		)
2186
2187	2184	yield self._update_context_for_auth_events(
2188	2185	event, context, auth_events, event_key
2189	2186	)
2190
2191		if not different_auth:
2192		# we're done
2193		return
2194
2195		logger.info(
2196		"auth_events still refers to events which are not in the calculated auth "
2197		"chain after state resolution: %s",
2198		different_auth,
2199		)
2200
2201		# Only do auth resolution if we have something new to say.
2202		# We can't prove an auth failure.
2203		do_resolution = False
2204
2205		for e_id in different_auth:
2206		if e_id in have_events:
2207		if have_events[e_id] == RejectedReason.NOT_ANCESTOR:
2208		do_resolution = True
2209		break
2210
2211		if not do_resolution:
2212		logger.info(
2213		"Skipping auth resolution due to lack of provable rejection reasons"
2214		)
2215		return
2216
2217		logger.info("Doing auth resolution")
2218
2219		prev_state_ids = yield context.get_prev_state_ids(self.store)
2220
2221		# 1. Get what we think is the auth chain.
2222		auth_ids = yield self.auth.compute_auth_events(event, prev_state_ids)
2223		local_auth_chain = yield self.store.get_auth_chain(auth_ids, include_given=True)
2224
2225		try:
2226		# 2. Get remote difference.
2227		try:
2228		result = yield self.federation_client.query_auth(
2229		origin, event.room_id, event.event_id, local_auth_chain
2230		)
2231		except RequestSendFailed as e:
2232		# The other side isn't around or doesn't implement the
2233		# endpoint, so lets just bail out.
2234		logger.info("Failed to query auth from remote: %s", e)
2235		return
2236
2237		seen_remotes = yield self.store.have_seen_events(
2238		[e.event_id for e in result["auth_chain"]]
2239		)
2240
2241		# 3. Process any remote auth chain events we haven't seen.
2242		for ev in result["auth_chain"]:
2243		if ev.event_id in seen_remotes:
2244		continue
2245
2246		if ev.event_id == event.event_id:
2247		continue
2248
2249		try:
2250		auth_ids = ev.auth_event_ids()
2251		auth = {
2252		(e.type, e.state_key): e
2253		for e in result["auth_chain"]
2254		if e.event_id in auth_ids or event.type == EventTypes.Create
2255		}
2256		ev.internal_metadata.outlier = True
2257
2258		logger.debug(
2259		"do_auth %s different_auth: %s", event.event_id, e.event_id
2260		)
2261
2262		yield self._handle_new_event(origin, ev, auth_events=auth)
2263
2264		if ev.event_id in event_auth_events:
2265		auth_events[(ev.type, ev.state_key)] = ev
2266		except AuthError:
2267		pass
2268
2269		except Exception:
2270		# FIXME:
2271		logger.exception("Failed to query auth chain")
2272
2273		# 4. Look at rejects and their proofs.
2274		# TODO.
2275
2276		yield self._update_context_for_auth_events(
2277		event, context, auth_events, event_key
2278		)
2279	2187
2280	2188	@defer.inlineCallbacks
2281	2189	def _update_context_for_auth_events(self, event, context, auth_events, event_key):

2443	2351
2444	2352	reason_map[e.event_id] = reason
2445	2353
2446		if reason == RejectedReason.AUTH_ERROR:
2447		pass
2448		elif reason == RejectedReason.REPLACED:
2449		# TODO: Get proof
2450		pass
2451		elif reason == RejectedReason.NOT_ANCESTOR:
2452		# TODO: Get proof.
2453		pass
2454
2455	2354	logger.debug("construct_auth_difference returning")
2456	2355
2457	2356	return {

2569	2468	)
2570	2469
2571	2470	try:
2572		self.auth.check_from_context(room_version, event, context)
	2471	yield self.auth.check_from_context(room_version, event, context)
2573	2472	except AuthError as e:
2574	2473	logger.warn("Denying third party invite %r because %s", event, e)
2575	2474	raise e

2598	2497	original_invite_id, allow_none=True
2599	2498	)
2600	2499	if original_invite:
2601		display_name = original_invite.content["display_name"]
	2500	# If the m.room.third_party_invite event's content is empty, it means the
	2501	# invite has been revoked. In this case, we don't have to raise an error here
	2502	# because the auth check will fail on the invite (because it's not able to
	2503	# fetch public keys from the m.room.third_party_invite event's content, which
	2504	# is empty).
	2505	display_name = original_invite.content.get("display_name")
2602	2506	event_dict["content"]["third_party_invite"]["display_name"] = display_name
2603	2507	else:
2604	2508	logger.info(

+353

-0

synapse/handlers/identity.py less more

20	20	import urllib
21	21
22	22	from canonicaljson import json
	23	from signedjson.key import decode_verify_key_bytes
	24	from signedjson.sign import verify_signed_json
	25	from unpaddedbase64 import decode_base64
23	26
24	27	from twisted.internet import defer
25	28	from twisted.internet.error import TimeoutError
26	29
27	30	from synapse.api.errors import (
	31	AuthError,
28	32	CodeMessageException,
29	33	Codes,
30	34	HttpResponseException,

32	36	)
33	37	from synapse.config.emailconfig import ThreepidBehaviour
34	38	from synapse.http.client import SimpleHttpClient
	39	from synapse.util.hash import sha256_and_url_safe_base64
35	40	from synapse.util.stringutils import random_string
36	41
37	42	from ._base import BaseHandler
38	43
39	44	logger = logging.getLogger(__name__)
	45
	46	id_server_scheme = "https://"
40	47
41	48
42	49	class IdentityHandler(BaseHandler):

556	563	logger.warning("Error contacting msisdn account_threepid_delegate: %s", e)
557	564	raise SynapseError(400, "Error contacting the identity server")
558	565
	566	@defer.inlineCallbacks
	567	def lookup_3pid(self, id_server, medium, address, id_access_token=None):
	568	"""Looks up a 3pid in the passed identity server.
	569
	570	Args:
	571	id_server (str): The server name (including port, if required)
	572	of the identity server to use.
	573	medium (str): The type of the third party identifier (e.g. "email").
	574	address (str): The third party identifier (e.g. "foo@example.com").
	575	id_access_token (str\|None): The access token to authenticate to the identity
	576	server with
	577
	578	Returns:
	579	str\|None: the matrix ID of the 3pid, or None if it is not recognized.
	580	"""
	581	if id_access_token is not None:
	582	try:
	583	results = yield self._lookup_3pid_v2(
	584	id_server, id_access_token, medium, address
	585	)
	586	return results
	587
	588	except Exception as e:
	589	# Catch HttpResponseExcept for a non-200 response code
	590	# Check if this identity server does not know about v2 lookups
	591	if isinstance(e, HttpResponseException) and e.code == 404:
	592	# This is an old identity server that does not yet support v2 lookups
	593	logger.warning(
	594	"Attempted v2 lookup on v1 identity server %s. Falling "
	595	"back to v1",
	596	id_server,
	597	)
	598	else:
	599	logger.warning("Error when looking up hashing details: %s", e)
	600	return None
	601
	602	return (yield self._lookup_3pid_v1(id_server, medium, address))
	603
	604	@defer.inlineCallbacks
	605	def _lookup_3pid_v1(self, id_server, medium, address):
	606	"""Looks up a 3pid in the passed identity server using v1 lookup.
	607
	608	Args:
	609	id_server (str): The server name (including port, if required)
	610	of the identity server to use.
	611	medium (str): The type of the third party identifier (e.g. "email").
	612	address (str): The third party identifier (e.g. "foo@example.com").
	613
	614	Returns:
	615	str: the matrix ID of the 3pid, or None if it is not recognized.
	616	"""
	617	try:
	618	data = yield self.blacklisting_http_client.get_json(
	619	"%s%s/_matrix/identity/api/v1/lookup" % (id_server_scheme, id_server),
	620	{"medium": medium, "address": address},
	621	)
	622
	623	if "mxid" in data:
	624	if "signatures" not in data:
	625	raise AuthError(401, "No signatures on 3pid binding")
	626	yield self._verify_any_signature(data, id_server)
	627	return data["mxid"]
	628	except TimeoutError:
	629	raise SynapseError(500, "Timed out contacting identity server")
	630	except IOError as e:
	631	logger.warning("Error from v1 identity server lookup: %s" % (e,))
	632
	633	return None
	634
	635	@defer.inlineCallbacks
	636	def _lookup_3pid_v2(self, id_server, id_access_token, medium, address):
	637	"""Looks up a 3pid in the passed identity server using v2 lookup.
	638
	639	Args:
	640	id_server (str): The server name (including port, if required)
	641	of the identity server to use.
	642	id_access_token (str): The access token to authenticate to the identity server with
	643	medium (str): The type of the third party identifier (e.g. "email").
	644	address (str): The third party identifier (e.g. "foo@example.com").
	645
	646	Returns:
	647	Deferred[str\|None]: the matrix ID of the 3pid, or None if it is not recognised.
	648	"""
	649	# Check what hashing details are supported by this identity server
	650	try:
	651	hash_details = yield self.blacklisting_http_client.get_json(
	652	"%s%s/_matrix/identity/v2/hash_details" % (id_server_scheme, id_server),
	653	{"access_token": id_access_token},
	654	)
	655	except TimeoutError:
	656	raise SynapseError(500, "Timed out contacting identity server")
	657
	658	if not isinstance(hash_details, dict):
	659	logger.warning(
	660	"Got non-dict object when checking hash details of %s%s: %s",
	661	id_server_scheme,
	662	id_server,
	663	hash_details,
	664	)
	665	raise SynapseError(
	666	400,
	667	"Non-dict object from %s%s during v2 hash_details request: %s"
	668	% (id_server_scheme, id_server, hash_details),
	669	)
	670
	671	# Extract information from hash_details
	672	supported_lookup_algorithms = hash_details.get("algorithms")
	673	lookup_pepper = hash_details.get("lookup_pepper")
	674	if (
	675	not supported_lookup_algorithms
	676	or not isinstance(supported_lookup_algorithms, list)
	677	or not lookup_pepper
	678	or not isinstance(lookup_pepper, str)
	679	):
	680	raise SynapseError(
	681	400,
	682	"Invalid hash details received from identity server %s%s: %s"
	683	% (id_server_scheme, id_server, hash_details),
	684	)
	685
	686	# Check if any of the supported lookup algorithms are present
	687	if LookupAlgorithm.SHA256 in supported_lookup_algorithms:
	688	# Perform a hashed lookup
	689	lookup_algorithm = LookupAlgorithm.SHA256
	690
	691	# Hash address, medium and the pepper with sha256
	692	to_hash = "%s %s %s" % (address, medium, lookup_pepper)
	693	lookup_value = sha256_and_url_safe_base64(to_hash)
	694
	695	elif LookupAlgorithm.NONE in supported_lookup_algorithms:
	696	# Perform a non-hashed lookup
	697	lookup_algorithm = LookupAlgorithm.NONE
	698
	699	# Combine together plaintext address and medium
	700	lookup_value = "%s %s" % (address, medium)
	701
	702	else:
	703	logger.warning(
	704	"None of the provided lookup algorithms of %s are supported: %s",
	705	id_server,
	706	supported_lookup_algorithms,
	707	)
	708	raise SynapseError(
	709	400,
	710	"Provided identity server does not support any v2 lookup "
	711	"algorithms that this homeserver supports.",
	712	)
	713
	714	# Authenticate with identity server given the access token from the client
	715	headers = {"Authorization": create_id_access_token_header(id_access_token)}
	716
	717	try:
	718	lookup_results = yield self.blacklisting_http_client.post_json_get_json(
	719	"%s%s/_matrix/identity/v2/lookup" % (id_server_scheme, id_server),
	720	{
	721	"addresses": [lookup_value],
	722	"algorithm": lookup_algorithm,
	723	"pepper": lookup_pepper,
	724	},
	725	headers=headers,
	726	)
	727	except TimeoutError:
	728	raise SynapseError(500, "Timed out contacting identity server")
	729	except Exception as e:
	730	logger.warning("Error when performing a v2 3pid lookup: %s", e)
	731	raise SynapseError(
	732	500, "Unknown error occurred during identity server lookup"
	733	)
	734
	735	# Check for a mapping from what we looked up to an MXID
	736	if "mappings" not in lookup_results or not isinstance(
	737	lookup_results["mappings"], dict
	738	):
	739	logger.warning("No results from 3pid lookup")
	740	return None
	741
	742	# Return the MXID if it's available, or None otherwise
	743	mxid = lookup_results["mappings"].get(lookup_value)
	744	return mxid
	745
	746	@defer.inlineCallbacks
	747	def _verify_any_signature(self, data, server_hostname):
	748	if server_hostname not in data["signatures"]:
	749	raise AuthError(401, "No signature from server %s" % (server_hostname,))
	750	for key_name, signature in data["signatures"][server_hostname].items():
	751	try:
	752	key_data = yield self.blacklisting_http_client.get_json(
	753	"%s%s/_matrix/identity/api/v1/pubkey/%s"
	754	% (id_server_scheme, server_hostname, key_name)
	755	)
	756	except TimeoutError:
	757	raise SynapseError(500, "Timed out contacting identity server")
	758	if "public_key" not in key_data:
	759	raise AuthError(
	760	401, "No public key named %s from %s" % (key_name, server_hostname)
	761	)
	762	verify_signed_json(
	763	data,
	764	server_hostname,
	765	decode_verify_key_bytes(
	766	key_name, decode_base64(key_data["public_key"])
	767	),
	768	)
	769	return
	770
	771	@defer.inlineCallbacks
	772	def ask_id_server_for_third_party_invite(
	773	self,
	774	requester,
	775	id_server,
	776	medium,
	777	address,
	778	room_id,
	779	inviter_user_id,
	780	room_alias,
	781	room_avatar_url,
	782	room_join_rules,
	783	room_name,
	784	inviter_display_name,
	785	inviter_avatar_url,
	786	id_access_token=None,
	787	):
	788	"""
	789	Asks an identity server for a third party invite.
	790
	791	Args:
	792	requester (Requester)
	793	id_server (str): hostname + optional port for the identity server.
	794	medium (str): The literal string "email".
	795	address (str): The third party address being invited.
	796	room_id (str): The ID of the room to which the user is invited.
	797	inviter_user_id (str): The user ID of the inviter.
	798	room_alias (str): An alias for the room, for cosmetic notifications.
	799	room_avatar_url (str): The URL of the room's avatar, for cosmetic
	800	notifications.
	801	room_join_rules (str): The join rules of the email (e.g. "public").
	802	room_name (str): The m.room.name of the room.
	803	inviter_display_name (str): The current display name of the
	804	inviter.
	805	inviter_avatar_url (str): The URL of the inviter's avatar.
	806	id_access_token (str\|None): The access token to authenticate to the identity
	807	server with
	808
	809	Returns:
	810	A deferred tuple containing:
	811	token (str): The token which must be signed to prove authenticity.
	812	public_keys ([{"public_key": str, "key_validity_url": str}]):
	813	public_key is a base64-encoded ed25519 public key.
	814	fallback_public_key: One element from public_keys.
	815	display_name (str): A user-friendly name to represent the invited
	816	user.
	817	"""
	818	invite_config = {
	819	"medium": medium,
	820	"address": address,
	821	"room_id": room_id,
	822	"room_alias": room_alias,
	823	"room_avatar_url": room_avatar_url,
	824	"room_join_rules": room_join_rules,
	825	"room_name": room_name,
	826	"sender": inviter_user_id,
	827	"sender_display_name": inviter_display_name,
	828	"sender_avatar_url": inviter_avatar_url,
	829	}
	830
	831	# Add the identity service access token to the JSON body and use the v2
	832	# Identity Service endpoints if id_access_token is present
	833	data = None
	834	base_url = "%s%s/_matrix/identity" % (id_server_scheme, id_server)
	835
	836	if id_access_token:
	837	key_validity_url = "%s%s/_matrix/identity/v2/pubkey/isvalid" % (
	838	id_server_scheme,
	839	id_server,
	840	)
	841
	842	# Attempt a v2 lookup
	843	url = base_url + "/v2/store-invite"
	844	try:
	845	data = yield self.blacklisting_http_client.post_json_get_json(
	846	url,
	847	invite_config,
	848	{"Authorization": create_id_access_token_header(id_access_token)},
	849	)
	850	except TimeoutError:
	851	raise SynapseError(500, "Timed out contacting identity server")
	852	except HttpResponseException as e:
	853	if e.code != 404:
	854	logger.info("Failed to POST %s with JSON: %s", url, e)
	855	raise e
	856
	857	if data is None:
	858	key_validity_url = "%s%s/_matrix/identity/api/v1/pubkey/isvalid" % (
	859	id_server_scheme,
	860	id_server,
	861	)
	862	url = base_url + "/api/v1/store-invite"
	863
	864	try:
	865	data = yield self.blacklisting_http_client.post_json_get_json(
	866	url, invite_config
	867	)
	868	except TimeoutError:
	869	raise SynapseError(500, "Timed out contacting identity server")
	870	except HttpResponseException as e:
	871	logger.warning(
	872	"Error trying to call /store-invite on %s%s: %s",
	873	id_server_scheme,
	874	id_server,
	875	e,
	876	)
	877
	878	if data is None:
	879	# Some identity servers may only support application/x-www-form-urlencoded
	880	# types. This is especially true with old instances of Sydent, see
	881	# https://github.com/matrix-org/sydent/pull/170
	882	try:
	883	data = yield self.blacklisting_http_client.post_urlencoded_get_json(
	884	url, invite_config
	885	)
	886	except HttpResponseException as e:
	887	logger.warning(
	888	"Error calling /store-invite on %s%s with fallback "
	889	"encoding: %s",
	890	id_server_scheme,
	891	id_server,
	892	e,
	893	)
	894	raise e
	895
	896	# TODO: Check for success
	897	token = data["token"]
	898	public_keys = data.get("public_keys", [])
	899	if "public_key" in data:
	900	fallback_public_key = {
	901	"public_key": data["public_key"],
	902	"key_validity_url": key_validity_url,
	903	}
	904	else:
	905	fallback_public_key = public_keys[0]
	906
	907	if not public_keys:
	908	public_keys.append(fallback_public_key)
	909	display_name = data["display_name"]
	910	return token, public_keys, fallback_public_key, display_name
	911
559	912
560	913	def create_id_access_token_header(id_access_token):
561	914	"""Create an Authorization header for passing to SimpleHttpClient as the header value

+22

-9

synapse/handlers/presence.py less more

23	23
24	24	import logging
25	25	from contextlib import contextmanager
	26	from typing import Dict, Set
26	27
27	28	from six import iteritems, itervalues
28	29

178	179	# we assume that all the sync requests on that process have stopped.
179	180	# Stored as a dict from process_id to set of user_id, and a dict of
180	181	# process_id to millisecond timestamp last updated.
181		self.external_process_to_current_syncs = {}
182		self.external_process_last_updated_ms = {}
	182	self.external_process_to_current_syncs = {} # type: Dict[int, Set[str]]
	183	self.external_process_last_updated_ms = {} # type: Dict[int, int]
	184
183	185	self.external_sync_linearizer = Linearizer(name="external_sync_linearizer")
184	186
185	187	# Start a LoopingCall in 30s that fires every 5s.

348	350	if now - last_update > EXTERNAL_PROCESS_EXPIRY
349	351	]
350	352	for process_id in expired_process_ids:
	353	# For each expired process drop tracking info and check the users
	354	# that were syncing on that process to see if they need to be timed
	355	# out.
351	356	users_to_check.update(
352		self.external_process_last_updated_ms.pop(process_id, ())
353		)
354		self.external_process_last_update.pop(process_id)
	357	self.external_process_to_current_syncs.pop(process_id, ())
	358	)
	359	self.external_process_last_updated_ms.pop(process_id)
355	360
356	361	states = [
357	362	self.user_to_current_state.get(user_id, UserPresenceState.default(user_id))

802	807	# Loop round handling deltas until we're up to date
803	808	while True:
804	809	with Measure(self.clock, "presence_delta"):
805		deltas = yield self.store.get_current_state_deltas(self._event_pos)
806		if not deltas:
	810	room_max_stream_ordering = self.store.get_room_max_stream_ordering()
	811	if self._event_pos == room_max_stream_ordering:
807	812	return
808	813
	814	logger.debug(
	815	"Processing presence stats %s->%s",
	816	self._event_pos,
	817	room_max_stream_ordering,
	818	)
	819	max_pos, deltas = yield self.store.get_current_state_deltas(
	820	self._event_pos, room_max_stream_ordering
	821	)
809	822	yield self._handle_state_delta(deltas)
810	823
811		self._event_pos = deltas[-1]["stream_id"]
	824	self._event_pos = max_pos
812	825
813	826	# Expose current event processing position to prometheus
814	827	synapse.metrics.event_processing_positions.labels("presence").set(
815		self._event_pos
	828	max_pos
816	829	)
817	830
818	831	@defer.inlineCallbacks

+4

-6

synapse/handlers/register.py less more

216	216
217	217	else:
218	218	# autogen a sequential user ID
219		attempts = 0
220	219	user = None
221	220	while not user:
222		localpart = yield self._generate_user_id(attempts > 0)
	221	localpart = yield self._generate_user_id()
223	222	user = UserID(localpart, self.hs.hostname)
224	223	user_id = user.to_string()
225	224	yield self.check_user_id_not_appservice_exclusive(user_id)

237	236	# if user id is taken, just generate another
238	237	user = None
239	238	user_id = None
240		attempts += 1
241	239
242	240	if not self.hs.config.user_consent_at_registration:
243	241	yield self._auto_join_rooms(user_id)

378	376	)
379	377
380	378	@defer.inlineCallbacks
381		def _generate_user_id(self, reseed=False):
382		if reseed or self._next_generated_user_id is None:
	379	def _generate_user_id(self):
	380	if self._next_generated_user_id is None:
383	381	with (yield self._generate_user_id_linearizer.queue(())):
384		if reseed or self._next_generated_user_id is None:
	382	if self._next_generated_user_id is None:
385	383	self._next_generated_user_id = (
386	384	yield self.store.find_next_generated_user_id_localpart()
387	385	)

+3

-1

synapse/handlers/room.py less more

27	27	from synapse.api.constants import EventTypes, JoinRules, RoomCreationPreset
28	28	from synapse.api.errors import AuthError, Codes, NotFoundError, StoreError, SynapseError
29	29	from synapse.api.room_versions import KNOWN_ROOM_VERSIONS
	30	from synapse.http.endpoint import parse_and_validate_server_name
30	31	from synapse.storage.state import StateFilter
31	32	from synapse.types import RoomAlias, RoomID, RoomStreamToken, StreamToken, UserID
32	33	from synapse.util import stringutils

553	554	invite_list = config.get("invite", [])
554	555	for i in invite_list:
555	556	try:
556		UserID.from_string(i)
	557	uid = UserID.from_string(i)
	558	parse_and_validate_server_name(uid.domain)
557	559	except Exception:
558	560	raise SynapseError(400, "Invalid user_id: %s" % (i,))
559	561

+115

-258

synapse/handlers/room_list.py less more

15	15	import logging
16	16	from collections import namedtuple
17	17
18		from six import PY3, iteritems
19		from six.moves import range
	18	from six import iteritems
20	19
21	20	import msgpack
22	21	from unpaddedbase64 import decode_base64, encode_base64

26	25	from synapse.api.constants import EventTypes, JoinRules
27	26	from synapse.api.errors import Codes, HttpResponseException
28	27	from synapse.types import ThirdPartyInstanceID
29		from synapse.util.async_helpers import concurrently_execute
30	28	from synapse.util.caches.descriptors import cachedInlineCallbacks
31	29	from synapse.util.caches.response_cache import ResponseCache
32	30

35	33	logger = logging.getLogger(__name__)
36	34
37	35	REMOTE_ROOM_LIST_POLL_INTERVAL = 60 * 1000
38
39	36
40	37	# This is used to indicate we should only return rooms published to the main list.
41	38	EMPTY_THIRD_PARTY_ID = ThirdPartyInstanceID(None, None)

71	68	This can be (None, None) to indicate the main list, or a particular
72	69	appservice and network id to use an appservice specific one.
73	70	Setting to None returns all public rooms across all lists.
	71	from_federation (bool): true iff the request comes from the federation
	72	API
74	73	"""
75	74	if not self.enable_room_list_search:
76	75	return defer.succeed({"chunk": [], "total_room_count_estimate": 0})

88	87	# appservice specific lists.
89	88	logger.info("Bypassing cache as search request.")
90	89
91		# XXX: Quick hack to stop room directory queries taking too long.
92		# Timeout request after 60s. Probably want a more fundamental
93		# solution at some point
94		timeout = self.clock.time() + 60
95	90	return self._get_public_room_list(
96		limit,
97		since_token,
98		search_filter,
99		network_tuple=network_tuple,
100		timeout=timeout,
	91	limit, since_token, search_filter, network_tuple=network_tuple
101	92	)
102	93
103	94	key = (limit, since_token, network_tuple)

118	109	search_filter=None,
119	110	network_tuple=EMPTY_THIRD_PARTY_ID,
120	111	from_federation=False,
121		timeout=None,
122	112	):
123	113	"""Generate a public room list.
124	114	Args:

131	121	Setting to None returns all public rooms across all lists.
132	122	from_federation (bool): Whether this request originated from a
133	123	federating server or a client. Used for room filtering.
134		timeout (int\|None): Amount of seconds to wait for a response before
135		timing out.
136	124	"""
137		if since_token and since_token != "END":
138		since_token = RoomListNextBatch.from_token(since_token)
	125
	126	# Pagination tokens work by storing the room ID sent in the last batch,
	127	# plus the direction (forwards or backwards). Next batch tokens always
	128	# go forwards, prev batch tokens always go backwards.
	129
	130	if since_token:
	131	batch_token = RoomListNextBatch.from_token(since_token)
	132
	133	bounds = (batch_token.last_joined_members, batch_token.last_room_id)
	134	forwards = batch_token.direction_is_forward
139	135	else:
140		since_token = None
141
142		rooms_to_order_value = {}
143		rooms_to_num_joined = {}
144
145		newly_visible = []
146		newly_unpublished = []
147		if since_token:
148		stream_token = since_token.stream_ordering
149		current_public_id = yield self.store.get_current_public_room_stream_id()
150		public_room_stream_id = since_token.public_room_stream_id
151		newly_visible, newly_unpublished = yield self.store.get_public_room_changes(
152		public_room_stream_id, current_public_id, network_tuple=network_tuple
153		)
	136	batch_token = None
	137	bounds = None
	138
	139	forwards = True
	140
	141	# we request one more than wanted to see if there are more pages to come
	142	probing_limit = limit + 1 if limit is not None else None
	143
	144	results = yield self.store.get_largest_public_rooms(
	145	network_tuple,
	146	search_filter,
	147	probing_limit,
	148	bounds=bounds,
	149	forwards=forwards,
	150	ignore_non_federatable=from_federation,
	151	)
	152
	153	def build_room_entry(room):
	154	entry = {
	155	"room_id": room["room_id"],
	156	"name": room["name"],
	157	"topic": room["topic"],
	158	"canonical_alias": room["canonical_alias"],
	159	"num_joined_members": room["joined_members"],
	160	"avatar_url": room["avatar"],
	161	"world_readable": room["history_visibility"] == "world_readable",
	162	"guest_can_join": room["guest_access"] == "can_join",
	163	}
	164
	165	# Filter out Nones – rather omit the field altogether
	166	return {k: v for k, v in entry.items() if v is not None}
	167
	168	results = [build_room_entry(r) for r in results]
	169
	170	response = {}
	171	num_results = len(results)
	172	if limit is not None:
	173	more_to_come = num_results == probing_limit
	174
	175	# Depending on direction we trim either the front or back.
	176	if forwards:
	177	results = results[:limit]
	178	else:
	179	results = results[-limit:]
154	180	else:
155		stream_token = yield self.store.get_room_max_stream_ordering()
156		public_room_stream_id = yield self.store.get_current_public_room_stream_id()
157
158		room_ids = yield self.store.get_public_room_ids_at_stream_id(
159		public_room_stream_id, network_tuple=network_tuple
160		)
161
162		# We want to return rooms in a particular order: the number of joined
163		# users. We then arbitrarily use the room_id as a tie breaker.
164
165		@defer.inlineCallbacks
166		def get_order_for_room(room_id):
167		# Most of the rooms won't have changed between the since token and
168		# now (especially if the since token is "now"). So, we can ask what
169		# the current users are in a room (that will hit a cache) and then
170		# check if the room has changed since the since token. (We have to
171		# do it in that order to avoid races).
172		# If things have changed then fall back to getting the current state
173		# at the since token.
174		joined_users = yield self.store.get_users_in_room(room_id)
175		if self.store.has_room_changed_since(room_id, stream_token):
176		latest_event_ids = yield self.store.get_forward_extremeties_for_room(
177		room_id, stream_token
178		)
179
180		if not latest_event_ids:
181		return
182
183		joined_users = yield self.state_handler.get_current_users_in_room(
184		room_id, latest_event_ids
185		)
186
187		num_joined_users = len(joined_users)
188		rooms_to_num_joined[room_id] = num_joined_users
189
190		if num_joined_users == 0:
191		return
192
193		# We want larger rooms to be first, hence negating num_joined_users
194		rooms_to_order_value[room_id] = (-num_joined_users, room_id)
195
196		logger.info(
197		"Getting ordering for %i rooms since %s", len(room_ids), stream_token
198		)
199		yield concurrently_execute(get_order_for_room, room_ids, 10)
200
201		sorted_entries = sorted(rooms_to_order_value.items(), key=lambda e: e[1])
202		sorted_rooms = [room_id for room_id, _ in sorted_entries]
203
204		# `sorted_rooms` should now be a list of all public room ids that is
205		# stable across pagination. Therefore, we can use indices into this
206		# list as our pagination tokens.
207
208		# Filter out rooms that we don't want to return
209		rooms_to_scan = [
210		r
211		for r in sorted_rooms
212		if r not in newly_unpublished and rooms_to_num_joined[r] > 0
213		]
214
215		total_room_count = len(rooms_to_scan)
216
217		if since_token:
218		# Filter out rooms we've already returned previously
219		# `since_token.current_limit` is the index of the last room we
220		# sent down, so we exclude it and everything before/after it.
221		if since_token.direction_is_forward:
222		rooms_to_scan = rooms_to_scan[since_token.current_limit + 1 :]
	181	more_to_come = False
	182
	183	if num_results > 0:
	184	final_entry = results[-1]
	185	initial_entry = results[0]
	186
	187	if forwards:
	188	if batch_token:
	189	# If there was a token given then we assume that there
	190	# must be previous results.
	191	response["prev_batch"] = RoomListNextBatch(
	192	last_joined_members=initial_entry["num_joined_members"],
	193	last_room_id=initial_entry["room_id"],
	194	direction_is_forward=False,
	195	).to_token()
	196
	197	if more_to_come:
	198	response["next_batch"] = RoomListNextBatch(
	199	last_joined_members=final_entry["num_joined_members"],
	200	last_room_id=final_entry["room_id"],
	201	direction_is_forward=True,
	202	).to_token()
223	203	else:
224		rooms_to_scan = rooms_to_scan[: since_token.current_limit]
225		rooms_to_scan.reverse()
226
227		logger.info("After sorting and filtering, %i rooms remain", len(rooms_to_scan))
228
229		# _append_room_entry_to_chunk will append to chunk but will stop if
230		# len(chunk) > limit
231		#
232		# Normally we will generate enough results on the first iteration here,
233		# but if there is a search filter, _append_room_entry_to_chunk may
234		# filter some results out, in which case we loop again.
235		#
236		# We don't want to scan over the entire range either as that
237		# would potentially waste a lot of work.
238		#
239		# XXX if there is no limit, we may end up DoSing the server with
240		# calls to get_current_state_ids for every single room on the
241		# server. Surely we should cap this somehow?
242		#
243		if limit:
244		step = limit + 1
245		else:
246		# step cannot be zero
247		step = len(rooms_to_scan) if len(rooms_to_scan) != 0 else 1
248
249		chunk = []
250		for i in range(0, len(rooms_to_scan), step):
251		if timeout and self.clock.time() > timeout:
252		raise Exception("Timed out searching room directory")
253
254		batch = rooms_to_scan[i : i + step]
255		logger.info("Processing %i rooms for result", len(batch))
256		yield concurrently_execute(
257		lambda r: self._append_room_entry_to_chunk(
258		r,
259		rooms_to_num_joined[r],
260		chunk,
261		limit,
262		search_filter,
263		from_federation=from_federation,
264		),
265		batch,
266		5,
267		)
268		logger.info("Now %i rooms in result", len(chunk))
269		if len(chunk) >= limit + 1:
270		break
271
272		chunk.sort(key=lambda e: (-e["num_joined_members"], e["room_id"]))
273
274		# Work out the new limit of the batch for pagination, or None if we
275		# know there are no more results that would be returned.
276		# i.e., [since_token.current_limit..new_limit] is the batch of rooms
277		# we've returned (or the reverse if we paginated backwards)
278		# We tried to pull out limit + 1 rooms above, so if we have <= limit
279		# then we know there are no more results to return
280		new_limit = None
281		if chunk and (not limit or len(chunk) > limit):
282
283		if not since_token or since_token.direction_is_forward:
284		if limit:
285		chunk = chunk[:limit]
286		last_room_id = chunk[-1]["room_id"]
287		else:
288		if limit:
289		chunk = chunk[-limit:]
290		last_room_id = chunk[0]["room_id"]
291
292		new_limit = sorted_rooms.index(last_room_id)
293
294		results = {"chunk": chunk, "total_room_count_estimate": total_room_count}
295
296		if since_token:
297		results["new_rooms"] = bool(newly_visible)
298
299		if not since_token or since_token.direction_is_forward:
300		if new_limit is not None:
301		results["next_batch"] = RoomListNextBatch(
302		stream_ordering=stream_token,
303		public_room_stream_id=public_room_stream_id,
304		current_limit=new_limit,
305		direction_is_forward=True,
306		).to_token()
307
308		if since_token:
309		results["prev_batch"] = since_token.copy_and_replace(
310		direction_is_forward=False,
311		current_limit=since_token.current_limit + 1,
312		).to_token()
313		else:
314		if new_limit is not None:
315		results["prev_batch"] = RoomListNextBatch(
316		stream_ordering=stream_token,
317		public_room_stream_id=public_room_stream_id,
318		current_limit=new_limit,
319		direction_is_forward=False,
320		).to_token()
321
322		if since_token:
323		results["next_batch"] = since_token.copy_and_replace(
324		direction_is_forward=True,
325		current_limit=since_token.current_limit - 1,
326		).to_token()
327
328		return results
329
330		@defer.inlineCallbacks
331		def _append_room_entry_to_chunk(
332		self,
333		room_id,
334		num_joined_users,
335		chunk,
336		limit,
337		search_filter,
338		from_federation=False,
339		):
340		"""Generate the entry for a room in the public room list and append it
341		to the `chunk` if it matches the search filter
342
343		Args:
344		room_id (str): The ID of the room.
345		num_joined_users (int): The number of joined users in the room.
346		chunk (list)
347		limit (int\|None): Maximum amount of rooms to display. Function will
348		return if length of chunk is greater than limit + 1.
349		search_filter (dict\|None)
350		from_federation (bool): Whether this request originated from a
351		federating server or a client. Used for room filtering.
352		"""
353		if limit and len(chunk) > limit + 1:
354		# We've already got enough, so lets just drop it.
355		return
356
357		result = yield self.generate_room_entry(room_id, num_joined_users)
358		if not result:
359		return
360
361		if from_federation and not result.get("m.federate", True):
362		# This is a room that other servers cannot join. Do not show them
363		# this room.
364		return
365
366		if _matches_room_entry(result, search_filter):
367		chunk.append(result)
	204	if batch_token:
	205	response["next_batch"] = RoomListNextBatch(
	206	last_joined_members=final_entry["num_joined_members"],
	207	last_room_id=final_entry["room_id"],
	208	direction_is_forward=True,
	209	).to_token()
	210
	211	if more_to_come:
	212	response["prev_batch"] = RoomListNextBatch(
	213	last_joined_members=initial_entry["num_joined_members"],
	214	last_room_id=initial_entry["room_id"],
	215	direction_is_forward=False,
	216	).to_token()
	217
	218	for room in results:
	219	# populate search result entries with additional fields, namely
	220	# 'aliases'
	221	room_id = room["room_id"]
	222
	223	aliases = yield self.store.get_aliases_for_room(room_id)
	224	if aliases:
	225	room["aliases"] = aliases
	226
	227	response["chunk"] = results
	228
	229	response["total_room_count_estimate"] = yield self.store.count_public_rooms(
	230	network_tuple, ignore_non_federatable=from_federation
	231	)
	232
	233	return response
368	234
369	235	@cachedInlineCallbacks(num_args=1, cache_context=True)
370	236	def generate_room_entry(

579	445	namedtuple(
580	446	"RoomListNextBatch",
581	447	(
582		"stream_ordering", # stream_ordering of the first public room list
583		"public_room_stream_id", # public room stream id for first public room list
584		"current_limit", # The number of previous rooms returned
	448	"last_joined_members", # The count to get rooms after/before
	449	"last_room_id", # The room_id to get rooms after/before
585	450	"direction_is_forward", # Bool if this is a next_batch, false if prev_batch
586	451	),
587	452	)
588	453	):
589
590	454	KEY_DICT = {
591		"stream_ordering": "s",
592		"public_room_stream_id": "p",
593		"current_limit": "n",
	455	"last_joined_members": "m",
	456	"last_room_id": "r",
594	457	"direction_is_forward": "d",
595	458	}
596	459

598	461
599	462	@classmethod
600	463	def from_token(cls, token):
601		if PY3:
602		# The argument raw=False is only available on new versions of
603		# msgpack, and only really needed on Python 3. Gate it behind
604		# a PY3 check to avoid causing issues on Debian-packaged versions.
605		decoded = msgpack.loads(decode_base64(token), raw=False)
606		else:
607		decoded = msgpack.loads(decode_base64(token))
	464	decoded = msgpack.loads(decode_base64(token), raw=False)
608	465	return RoomListNextBatch(
609	466	**{cls.REVERSE_KEY_DICT[key]: val for key, val in decoded.items()}
610	467	)

+50

-382

synapse/handlers/room_member.py less more

19	19
20	20	from six.moves import http_client
21	21
22		from signedjson.key import decode_verify_key_bytes
23		from signedjson.sign import verify_signed_json
24		from unpaddedbase64 import decode_base64
25
26	22	from twisted.internet import defer
27		from twisted.internet.error import TimeoutError
28	23
29	24	from synapse import types
30	25	from synapse.api.constants import EventTypes, Membership
31		from synapse.api.errors import AuthError, Codes, HttpResponseException, SynapseError
32		from synapse.handlers.identity import LookupAlgorithm, create_id_access_token_header
33		from synapse.http.client import SimpleHttpClient
	26	from synapse.api.errors import AuthError, Codes, SynapseError
34	27	from synapse.types import RoomID, UserID
35	28	from synapse.util.async_helpers import Linearizer
36	29	from synapse.util.distributor import user_joined_room, user_left_room
37		from synapse.util.hash import sha256_and_url_safe_base64
38	30
39	31	from ._base import BaseHandler
40	32
41	33	logger = logging.getLogger(__name__)
42
43		id_server_scheme = "https://"
44	34
45	35
46	36	class RoomMemberHandler(object):

62	52	self.auth = hs.get_auth()
63	53	self.state_handler = hs.get_state_handler()
64	54	self.config = hs.config
65		# We create a blacklisting instance of SimpleHttpClient for contacting identity
66		# servers specified by clients
67		self.simple_http_client = SimpleHttpClient(
68		hs, ip_blacklist=hs.config.federation_ip_range_blacklist
69		)
70	55
71	56	self.federation_handler = hs.get_handlers().federation_handler
72	57	self.directory_handler = hs.get_handlers().directory_handler
	58	self.identity_handler = hs.get_handlers().identity_handler
73	59	self.registration_handler = hs.get_registration_handler()
74	60	self.profile_handler = hs.get_profile_handler()
75	61	self.event_creation_handler = hs.get_event_creation_handler()

216	202	prev_member_event = yield self.store.get_event(prev_member_event_id)
217	203	newly_joined = prev_member_event.membership != Membership.JOIN
218	204	if newly_joined:
	205	# Copy over user state if we're joining an upgraded room
	206	yield self.copy_user_state_if_room_upgrade(
	207	room_id, requester.user.to_string()
	208	)
219	209	yield self._user_joined_room(target, room_id)
220
221		# Copy over direct message status and room tags if this is a join
222		# on an upgraded room
223
224		# Check if this is an upgraded room
225		predecessor = yield self.store.get_room_predecessor(room_id)
226
227		if predecessor:
228		# It is an upgraded room. Copy over old tags
229		self.copy_room_tags_and_direct_to_room(
230		predecessor["room_id"], room_id, user_id
231		)
232		# Move over old push rules
233		self.store.move_push_rules_from_room_to_room_for_user(
234		predecessor["room_id"], room_id, user_id
235		)
236	210	elif event.membership == Membership.LEAVE:
237	211	if prev_member_event_id:
238	212	prev_member_event = yield self.store.get_event(prev_member_event_id)

476	450	if requester.is_guest:
477	451	content["kind"] = "guest"
478	452
479		ret = yield self._remote_join(
	453	remote_join_response = yield self._remote_join(
480	454	requester, remote_room_hosts, room_id, target, content
481	455	)
482		return ret
	456
	457	# Copy over user state if this is a join on an remote upgraded room
	458	yield self.copy_user_state_if_room_upgrade(
	459	room_id, requester.user.to_string()
	460	)
	461
	462	return remote_join_response
483	463
484	464	elif effective_membership_state == Membership.LEAVE:
485	465	if not is_host_in_room:

515	495	require_consent=require_consent,
516	496	)
517	497	return res
	498
	499	@defer.inlineCallbacks
	500	def copy_user_state_if_room_upgrade(self, new_room_id, user_id):
	501	"""Copy user-specific information when they join a new room if that new room is the
	502	result of a room upgrade
	503
	504	Args:
	505	new_room_id (str): The ID of the room the user is joining
	506	user_id (str): The ID of the user
	507
	508	Returns:
	509	Deferred
	510	"""
	511	# Check if the new room is an upgraded room
	512	predecessor = yield self.store.get_room_predecessor(new_room_id)
	513	if not predecessor:
	514	return
	515
	516	logger.debug(
	517	"Found predecessor for %s: %s. Copying over room tags and push " "rules",
	518	new_room_id,
	519	predecessor,
	520	)
	521
	522	# It is an upgraded room. Copy over old tags
	523	yield self.copy_room_tags_and_direct_to_room(
	524	predecessor["room_id"], new_room_id, user_id
	525	)
	526	# Copy over push rules
	527	yield self.store.copy_push_rules_from_room_to_room_for_user(
	528	predecessor["room_id"], new_room_id, user_id
	529	)
518	530
519	531	@defer.inlineCallbacks
520	532	def send_membership_event(self, requester, event, context, ratelimit=True):

681	693	403, "Looking up third-party identifiers is denied from this server"
682	694	)
683	695
684		invitee = yield self._lookup_3pid(id_server, medium, address, id_access_token)
	696	invitee = yield self.identity_handler.lookup_3pid(
	697	id_server, medium, address, id_access_token
	698	)
685	699
686	700	if invitee:
687	701	yield self.update_membership(

698	712	txn_id=txn_id,
699	713	id_access_token=id_access_token,
700	714	)
701
702		@defer.inlineCallbacks
703		def _lookup_3pid(self, id_server, medium, address, id_access_token=None):
704		"""Looks up a 3pid in the passed identity server.
705
706		Args:
707		id_server (str): The server name (including port, if required)
708		of the identity server to use.
709		medium (str): The type of the third party identifier (e.g. "email").
710		address (str): The third party identifier (e.g. "foo@example.com").
711		id_access_token (str\|None): The access token to authenticate to the identity
712		server with
713
714		Returns:
715		str\|None: the matrix ID of the 3pid, or None if it is not recognized.
716		"""
717		if id_access_token is not None:
718		try:
719		results = yield self._lookup_3pid_v2(
720		id_server, id_access_token, medium, address
721		)
722		return results
723
724		except Exception as e:
725		# Catch HttpResponseExcept for a non-200 response code
726		# Check if this identity server does not know about v2 lookups
727		if isinstance(e, HttpResponseException) and e.code == 404:
728		# This is an old identity server that does not yet support v2 lookups
729		logger.warning(
730		"Attempted v2 lookup on v1 identity server %s. Falling "
731		"back to v1",
732		id_server,
733		)
734		else:
735		logger.warning("Error when looking up hashing details: %s", e)
736		return None
737
738		return (yield self._lookup_3pid_v1(id_server, medium, address))
739
740		@defer.inlineCallbacks
741		def _lookup_3pid_v1(self, id_server, medium, address):
742		"""Looks up a 3pid in the passed identity server using v1 lookup.
743
744		Args:
745		id_server (str): The server name (including port, if required)
746		of the identity server to use.
747		medium (str): The type of the third party identifier (e.g. "email").
748		address (str): The third party identifier (e.g. "foo@example.com").
749
750		Returns:
751		str: the matrix ID of the 3pid, or None if it is not recognized.
752		"""
753		try:
754		data = yield self.simple_http_client.get_json(
755		"%s%s/_matrix/identity/api/v1/lookup" % (id_server_scheme, id_server),
756		{"medium": medium, "address": address},
757		)
758
759		if "mxid" in data:
760		if "signatures" not in data:
761		raise AuthError(401, "No signatures on 3pid binding")
762		yield self._verify_any_signature(data, id_server)
763		return data["mxid"]
764		except TimeoutError:
765		raise SynapseError(500, "Timed out contacting identity server")
766		except IOError as e:
767		logger.warning("Error from v1 identity server lookup: %s" % (e,))
768
769		return None
770
771		@defer.inlineCallbacks
772		def _lookup_3pid_v2(self, id_server, id_access_token, medium, address):
773		"""Looks up a 3pid in the passed identity server using v2 lookup.
774
775		Args:
776		id_server (str): The server name (including port, if required)
777		of the identity server to use.
778		id_access_token (str): The access token to authenticate to the identity server with
779		medium (str): The type of the third party identifier (e.g. "email").
780		address (str): The third party identifier (e.g. "foo@example.com").
781
782		Returns:
783		Deferred[str\|None]: the matrix ID of the 3pid, or None if it is not recognised.
784		"""
785		# Check what hashing details are supported by this identity server
786		try:
787		hash_details = yield self.simple_http_client.get_json(
788		"%s%s/_matrix/identity/v2/hash_details" % (id_server_scheme, id_server),
789		{"access_token": id_access_token},
790		)
791		except TimeoutError:
792		raise SynapseError(500, "Timed out contacting identity server")
793
794		if not isinstance(hash_details, dict):
795		logger.warning(
796		"Got non-dict object when checking hash details of %s%s: %s",
797		id_server_scheme,
798		id_server,
799		hash_details,
800		)
801		raise SynapseError(
802		400,
803		"Non-dict object from %s%s during v2 hash_details request: %s"
804		% (id_server_scheme, id_server, hash_details),
805		)
806
807		# Extract information from hash_details
808		supported_lookup_algorithms = hash_details.get("algorithms")
809		lookup_pepper = hash_details.get("lookup_pepper")
810		if (
811		not supported_lookup_algorithms
812		or not isinstance(supported_lookup_algorithms, list)
813		or not lookup_pepper
814		or not isinstance(lookup_pepper, str)
815		):
816		raise SynapseError(
817		400,
818		"Invalid hash details received from identity server %s%s: %s"
819		% (id_server_scheme, id_server, hash_details),
820		)
821
822		# Check if any of the supported lookup algorithms are present
823		if LookupAlgorithm.SHA256 in supported_lookup_algorithms:
824		# Perform a hashed lookup
825		lookup_algorithm = LookupAlgorithm.SHA256
826
827		# Hash address, medium and the pepper with sha256
828		to_hash = "%s %s %s" % (address, medium, lookup_pepper)
829		lookup_value = sha256_and_url_safe_base64(to_hash)
830
831		elif LookupAlgorithm.NONE in supported_lookup_algorithms:
832		# Perform a non-hashed lookup
833		lookup_algorithm = LookupAlgorithm.NONE
834
835		# Combine together plaintext address and medium
836		lookup_value = "%s %s" % (address, medium)
837
838		else:
839		logger.warning(
840		"None of the provided lookup algorithms of %s are supported: %s",
841		id_server,
842		supported_lookup_algorithms,
843		)
844		raise SynapseError(
845		400,
846		"Provided identity server does not support any v2 lookup "
847		"algorithms that this homeserver supports.",
848		)
849
850		# Authenticate with identity server given the access token from the client
851		headers = {"Authorization": create_id_access_token_header(id_access_token)}
852
853		try:
854		lookup_results = yield self.simple_http_client.post_json_get_json(
855		"%s%s/_matrix/identity/v2/lookup" % (id_server_scheme, id_server),
856		{
857		"addresses": [lookup_value],
858		"algorithm": lookup_algorithm,
859		"pepper": lookup_pepper,
860		},
861		headers=headers,
862		)
863		except TimeoutError:
864		raise SynapseError(500, "Timed out contacting identity server")
865		except Exception as e:
866		logger.warning("Error when performing a v2 3pid lookup: %s", e)
867		raise SynapseError(
868		500, "Unknown error occurred during identity server lookup"
869		)
870
871		# Check for a mapping from what we looked up to an MXID
872		if "mappings" not in lookup_results or not isinstance(
873		lookup_results["mappings"], dict
874		):
875		logger.warning("No results from 3pid lookup")
876		return None
877
878		# Return the MXID if it's available, or None otherwise
879		mxid = lookup_results["mappings"].get(lookup_value)
880		return mxid
881
882		@defer.inlineCallbacks
883		def _verify_any_signature(self, data, server_hostname):
884		if server_hostname not in data["signatures"]:
885		raise AuthError(401, "No signature from server %s" % (server_hostname,))
886		for key_name, signature in data["signatures"][server_hostname].items():
887		try:
888		key_data = yield self.simple_http_client.get_json(
889		"%s%s/_matrix/identity/api/v1/pubkey/%s"
890		% (id_server_scheme, server_hostname, key_name)
891		)
892		except TimeoutError:
893		raise SynapseError(500, "Timed out contacting identity server")
894		if "public_key" not in key_data:
895		raise AuthError(
896		401, "No public key named %s from %s" % (key_name, server_hostname)
897		)
898		verify_signed_json(
899		data,
900		server_hostname,
901		decode_verify_key_bytes(
902		key_name, decode_base64(key_data["public_key"])
903		),
904		)
905		return
906	715
907	716	@defer.inlineCallbacks
908	717	def _make_and_store_3pid_invite(

950	759	room_avatar_url = room_avatar_event.content.get("url", "")
951	760
952	761	token, public_keys, fallback_public_key, display_name = (
953		yield self._ask_id_server_for_third_party_invite(
	762	yield self.identity_handler.ask_id_server_for_third_party_invite(
954	763	requester=requester,
955	764	id_server=id_server,
956	765	medium=medium,

987	796	)
988	797
989	798	@defer.inlineCallbacks
990		def _ask_id_server_for_third_party_invite(
991		self,
992		requester,
993		id_server,
994		medium,
995		address,
996		room_id,
997		inviter_user_id,
998		room_alias,
999		room_avatar_url,
1000		room_join_rules,
1001		room_name,
1002		inviter_display_name,
1003		inviter_avatar_url,
1004		id_access_token=None,
1005		):
1006		"""
1007		Asks an identity server for a third party invite.
1008
1009		Args:
1010		requester (Requester)
1011		id_server (str): hostname + optional port for the identity server.
1012		medium (str): The literal string "email".
1013		address (str): The third party address being invited.
1014		room_id (str): The ID of the room to which the user is invited.
1015		inviter_user_id (str): The user ID of the inviter.
1016		room_alias (str): An alias for the room, for cosmetic notifications.
1017		room_avatar_url (str): The URL of the room's avatar, for cosmetic
1018		notifications.
1019		room_join_rules (str): The join rules of the email (e.g. "public").
1020		room_name (str): The m.room.name of the room.
1021		inviter_display_name (str): The current display name of the
1022		inviter.
1023		inviter_avatar_url (str): The URL of the inviter's avatar.
1024		id_access_token (str\|None): The access token to authenticate to the identity
1025		server with
1026
1027		Returns:
1028		A deferred tuple containing:
1029		token (str): The token which must be signed to prove authenticity.
1030		public_keys ([{"public_key": str, "key_validity_url": str}]):
1031		public_key is a base64-encoded ed25519 public key.
1032		fallback_public_key: One element from public_keys.
1033		display_name (str): A user-friendly name to represent the invited
1034		user.
1035		"""
1036		invite_config = {
1037		"medium": medium,
1038		"address": address,
1039		"room_id": room_id,
1040		"room_alias": room_alias,
1041		"room_avatar_url": room_avatar_url,
1042		"room_join_rules": room_join_rules,
1043		"room_name": room_name,
1044		"sender": inviter_user_id,
1045		"sender_display_name": inviter_display_name,
1046		"sender_avatar_url": inviter_avatar_url,
1047		}
1048
1049		# Add the identity service access token to the JSON body and use the v2
1050		# Identity Service endpoints if id_access_token is present
1051		data = None
1052		base_url = "%s%s/_matrix/identity" % (id_server_scheme, id_server)
1053
1054		if id_access_token:
1055		key_validity_url = "%s%s/_matrix/identity/v2/pubkey/isvalid" % (
1056		id_server_scheme,
1057		id_server,
1058		)
1059
1060		# Attempt a v2 lookup
1061		url = base_url + "/v2/store-invite"
1062		try:
1063		data = yield self.simple_http_client.post_json_get_json(
1064		url,
1065		invite_config,
1066		{"Authorization": create_id_access_token_header(id_access_token)},
1067		)
1068		except TimeoutError:
1069		raise SynapseError(500, "Timed out contacting identity server")
1070		except HttpResponseException as e:
1071		if e.code != 404:
1072		logger.info("Failed to POST %s with JSON: %s", url, e)
1073		raise e
1074
1075		if data is None:
1076		key_validity_url = "%s%s/_matrix/identity/api/v1/pubkey/isvalid" % (
1077		id_server_scheme,
1078		id_server,
1079		)
1080		url = base_url + "/api/v1/store-invite"
1081
1082		try:
1083		data = yield self.simple_http_client.post_json_get_json(
1084		url, invite_config
1085		)
1086		except TimeoutError:
1087		raise SynapseError(500, "Timed out contacting identity server")
1088		except HttpResponseException as e:
1089		logger.warning(
1090		"Error trying to call /store-invite on %s%s: %s",
1091		id_server_scheme,
1092		id_server,
1093		e,
1094		)
1095
1096		if data is None:
1097		# Some identity servers may only support application/x-www-form-urlencoded
1098		# types. This is especially true with old instances of Sydent, see
1099		# https://github.com/matrix-org/sydent/pull/170
1100		try:
1101		data = yield self.simple_http_client.post_urlencoded_get_json(
1102		url, invite_config
1103		)
1104		except HttpResponseException as e:
1105		logger.warning(
1106		"Error calling /store-invite on %s%s with fallback "
1107		"encoding: %s",
1108		id_server_scheme,
1109		id_server,
1110		e,
1111		)
1112		raise e
1113
1114		# TODO: Check for success
1115		token = data["token"]
1116		public_keys = data.get("public_keys", [])
1117		if "public_key" in data:
1118		fallback_public_key = {
1119		"public_key": data["public_key"],
1120		"key_validity_url": key_validity_url,
1121		}
1122		else:
1123		fallback_public_key = public_keys[0]
1124
1125		if not public_keys:
1126		public_keys.append(fallback_public_key)
1127		display_name = data["display_name"]
1128		return token, public_keys, fallback_public_key, display_name
1129
1130		@defer.inlineCallbacks
1131	799	def _is_host_in_room(self, current_state_ids):
1132	800	# Have we just created the room, and is this about to be the very
1133	801	# first member event?

+8

-5

synapse/handlers/stats.py less more

86	86	# Be sure to read the max stream_ordering before checking if there are any outstanding
87	87	# deltas, since there is otherwise a chance that we could miss updates which arrive
88	88	# after we check the deltas.
89		room_max_stream_ordering = yield self.store.get_room_max_stream_ordering()
	89	room_max_stream_ordering = self.store.get_room_max_stream_ordering()
90	90	if self.pos == room_max_stream_ordering:
91	91	break
92	92
93		deltas = yield self.store.get_current_state_deltas(self.pos)
	93	logger.debug(
	94	"Processing room stats %s->%s", self.pos, room_max_stream_ordering
	95	)
	96	max_pos, deltas = yield self.store.get_current_state_deltas(
	97	self.pos, room_max_stream_ordering
	98	)
94	99
95	100	if deltas:
96	101	logger.debug("Handling %d state deltas", len(deltas))
97	102	room_deltas, user_deltas = yield self._handle_deltas(deltas)
98
99		max_pos = deltas[-1]["stream_id"]
100	103	else:
101	104	room_deltas = {}
102	105	user_deltas = {}
103		max_pos = room_max_stream_ordering
104	106
105	107	# Then count deltas for total_events and total_event_bytes.
106	108	room_count, user_count = yield self.store.get_changes_room_total_events_and_bytes(

292	294	room_state["guest_access"] = event_content.get("guest_access")
293	295
294	296	for room_id, state in room_to_state_updates.items():
	297	logger.info("Updating room_stats_state for %s: %s", room_id, state)
295	298	yield self.store.update_room_state(room_id, state)
296	299
297	300	return room_to_stats_deltas, user_to_stats_deltas

+6

-1

synapse/handlers/sync.py less more

0	0	# -- coding: utf-8 --
1	1	# Copyright 2015, 2016 OpenMarket Ltd
2		# Copyright 2018 New Vector Ltd
	2	# Copyright 2018, 2019 New Vector Ltd
3	3	#
4	4	# Licensed under the Apache License, Version 2.0 (the "License");
5	5	# you may not use this file except in compliance with the License.

1123	1123	# weren't in the previous sync or they left and rejoined.
1124	1124	users_that_have_changed.update(newly_joined_or_invited_users)
1125	1125
	1126	user_signatures_changed = yield self.store.get_users_whose_signatures_changed(
	1127	user_id, since_token.device_list_key
	1128	)
	1129	users_that_have_changed.update(user_signatures_changed)
	1130
1126	1131	# Now find users that we no longer track
1127	1132	for room_id in newly_left_rooms:
1128	1133	left_users = yield self.state.get_current_users_in_room(room_id)

+14

-7

synapse/handlers/user_directory.py less more

137	137	# Loop round handling deltas until we're up to date
138	138	while True:
139	139	with Measure(self.clock, "user_dir_delta"):
140		deltas = yield self.store.get_current_state_deltas(self.pos)
141		if not deltas:
	140	room_max_stream_ordering = self.store.get_room_max_stream_ordering()
	141	if self.pos == room_max_stream_ordering:
142	142	return
	143
	144	logger.debug(
	145	"Processing user stats %s->%s", self.pos, room_max_stream_ordering
	146	)
	147	max_pos, deltas = yield self.store.get_current_state_deltas(
	148	self.pos, room_max_stream_ordering
	149	)
143	150
144	151	logger.info("Handling %d state deltas", len(deltas))
145	152	yield self._handle_deltas(deltas)
146	153
147		self.pos = deltas[-1]["stream_id"]
	154	self.pos = max_pos
148	155
149	156	# Expose current event processing position to prometheus
150	157	synapse.metrics.event_processing_positions.labels("user_dir").set(
151		self.pos
152		)
153
154		yield self.store.update_user_directory_stream_pos(self.pos)
	158	max_pos
	159	)
	160
	161	yield self.store.update_user_directory_stream_pos(max_pos)
155	162
156	163	@defer.inlineCallbacks
157	164	def _handle_deltas(self, deltas):

+6

-6

synapse/http/client.py less more

326	326	Args:
327	327	uri (str):
328	328	args (dict[str, str\|List[str]]): query params
329		headers (dict[str, List[str]]\|None): If not None, a map from
	329	headers (dict[str\|bytes, List[str\|bytes]]\|None): If not None, a map from
330	330	header name to a list of values for that header
331	331
332	332	Returns:

370	370	Args:
371	371	uri (str):
372	372	post_json (object):
373		headers (dict[str, List[str]]\|None): If not None, a map from
	373	headers (dict[str\|bytes, List[str\|bytes]]\|None): If not None, a map from
374	374	header name to a list of values for that header
375	375
376	376	Returns:

413	413	None.
414	414	Note: The value of each key is assumed to be an iterable
415	415	and not a string.
416		headers (dict[str, List[str]]\|None): If not None, a map from
	416	headers (dict[str\|bytes, List[str\|bytes]]\|None): If not None, a map from
417	417	header name to a list of values for that header
418	418	Returns:
419	419	Deferred: Succeeds when we get any 2xx HTTP response, with the

437	437	None.
438	438	Note: The value of each key is assumed to be an iterable
439	439	and not a string.
440		headers (dict[str, List[str]]\|None): If not None, a map from
	440	headers (dict[str\|bytes, List[str\|bytes]]\|None): If not None, a map from
441	441	header name to a list of values for that header
442	442	Returns:
443	443	Deferred: Succeeds when we get any 2xx HTTP response, with the

481	481	None.
482	482	Note: The value of each key is assumed to be an iterable
483	483	and not a string.
484		headers (dict[str, List[str]]\|None): If not None, a map from
	484	headers (dict[str\|bytes, List[str\|bytes]]\|None): If not None, a map from
485	485	header name to a list of values for that header
486	486	Returns:
487	487	Deferred: Succeeds when we get any 2xx HTTP response, with the

515	515	Args:
516	516	url (str): The URL to GET
517	517	output_stream (file): File to write the response body to.
518		headers (dict[str, List[str]]\|None): If not None, a map from
	518	headers (dict[str\|bytes, List[str\|bytes]]\|None): If not None, a map from
519	519	header name to a list of values for that header
520	520	Returns:
521	521	A (int,dict,string,int) tuple of the file length, dict of the response

+1

-1

synapse/http/server.py less more

387	387	if not callback:
388	388	return super().render(request)
389	389
390		resp = callback(request)
	390	resp = trace_servlet(self.__class__.__name__)(callback)(request)
391	391
392	392	# If it's a coroutine, turn it into a Deferred
393	393	if isinstance(resp, types.CoroutineType):

+15

-8

synapse/logging/opentracing.py less more

168	168	import inspect
169	169	import logging
170	170	import re
	171	import types
171	172	from functools import wraps
	173	from typing import Dict
172	174
173	175	from canonicaljson import json
174	176

546	548	return
547	549
548	550	span = opentracing.tracer.active_span
549		carrier = {}
	551	carrier = {} # type: Dict[str, str]
550	552	opentracing.tracer.inject(span, opentracing.Format.HTTP_HEADERS, carrier)
551	553
552	554	for key, value in carrier.items():

583	585
584	586	span = opentracing.tracer.active_span
585	587
586		carrier = {}
	588	carrier = {} # type: Dict[str, str]
587	589	opentracing.tracer.inject(span, opentracing.Format.HTTP_HEADERS, carrier)
588	590
589	591	for key, value in carrier.items():

638	640	if destination and not whitelisted_homeserver(destination):
639	641	return {}
640	642
641		carrier = {}
	643	carrier = {} # type: Dict[str, str]
642	644	opentracing.tracer.inject(
643	645	opentracing.tracer.active_span, opentracing.Format.TEXT_MAP, carrier
644	646	)

652	654	Returns:
653	655	The active span context encoded as a string.
654	656	"""
655		carrier = {}
	657	carrier = {} # type: Dict[str, str]
656	658	if opentracing:
657	659	opentracing.tracer.inject(
658	660	opentracing.tracer.active_span, opentracing.Format.TEXT_MAP, carrier

776	778	return func
777	779
778	780	@wraps(func)
779		@defer.inlineCallbacks
780		def _trace_servlet_inner(request, args, *kwargs):
	781	async def _trace_servlet_inner(request, args, *kwargs):
781	782	request_tags = {
782	783	"request_id": request.get_request_id(),
783	784	tags.SPAN_KIND: tags.SPAN_KIND_RPC_SERVER,

794	795	scope = start_active_span(servlet_name, tags=request_tags)
795	796
796	797	with scope:
797		result = yield defer.maybeDeferred(func, request, args, *kwargs)
798		return result
	798	result = func(request, args, *kwargs)
	799
	800	if not isinstance(result, (types.CoroutineType, defer.Deferred)):
	801	# Some servlets aren't async and just return results
	802	# directly, so we handle that here.
	803	return result
	804
	805	return await result
799	806
800	807	return _trace_servlet_inner
801	808

+16

-4

synapse/logging/utils.py less more

118	118	logger = logging.getLogger(name)
119	119	level = logging.DEBUG
120	120
121		s = inspect.currentframe().f_back
	121	frame = inspect.currentframe()
	122	if frame is None:
	123	raise Exception("Can't get current frame!")
	124
	125	s = frame.f_back
122	126
123	127	to_print = [
124	128	"\t%s:%s %s. Args: args=%s, kwargs=%s"

143	147	pathname=pathname,
144	148	lineno=lineno,
145	149	msg=msg,
146		args=None,
	150	args=tuple(),
147	151	exc_info=None,
148	152	)
149	153

156	160
157	161
158	162	def get_previous_frames():
159		s = inspect.currentframe().f_back.f_back
	163
	164	frame = inspect.currentframe()
	165	if frame is None:
	166	raise Exception("Can't get current frame!")
	167
	168	s = frame.f_back.f_back
160	169	to_return = []
161	170	while s:
162	171	if s.f_globals["__name__"].startswith("synapse"):

173	182
174	183
175	184	def get_previous_frame(ignore=[]):
176		s = inspect.currentframe().f_back.f_back
	185	frame = inspect.currentframe()
	186	if frame is None:
	187	raise Exception("Can't get current frame!")
	188	s = frame.f_back.f_back
177	189
178	190	while s:
179	191	if s.f_globals["__name__"].startswith("synapse"):

+2

-2

synapse/metrics/__init__.py less more

124	124	)
125	125
126	126	# Counts number of in flight blocks for a given set of label values
127		self._registrations = {}
	127	self._registrations = {} # type: Dict
128	128
129	129	# Protects access to _registrations
130	130	self._lock = threading.Lock()

225	225	# Fetch the data -- this must be synchronous!
226	226	data = self.data_collector()
227	227
228		buckets = {}
	228	buckets = {} # type: Dict[float, int]
229	229
230	230	res = []
231	231	for x in data.keys():

+2

-2

synapse/metrics/_exposition.py less more

35	35	try:
36	36	from prometheus_client.samples import Sample
37	37	except ImportError:
38		Sample = namedtuple(
	38	Sample = namedtuple( # type: ignore[no-redef] # noqa
39	39	"Sample", ["name", "labels", "value", "timestamp", "exemplar"]
40		) # type: ignore
	40	)
41	41
42	42
43	43	CONTENT_TYPE_LATEST = str("text/plain; version=0.0.4; charset=utf-8")

+13

-4

synapse/python_dependencies.py less more

14	14	# limitations under the License.
15	15
16	16	import logging
17		from typing import Set
	17	from typing import List, Set
18	18
19	19	from pkg_resources import (
20	20	DistributionNotFound,

72	72	"netaddr>=0.7.18",
73	73	"Jinja2>=2.9",
74	74	"bleach>=1.4.3",
	75	"typing-extensions>=3.7.4",
75	76	]
76	77
77	78	CONDITIONAL_REQUIREMENTS = {

143	144	deps_needed.append(dependency)
144	145	errors.append(
145	146	"Needed %s, got %s==%s"
146		% (dependency, e.dist.project_name, e.dist.version)
	147	% (
	148	dependency,
	149	e.dist.project_name, # type: ignore[attr-defined] # noqa
	150	e.dist.version, # type: ignore[attr-defined] # noqa
	151	)
147	152	)
148	153	except DistributionNotFound:
149	154	deps_needed.append(dependency)

158	163	if not for_feature:
159	164	# Check the optional dependencies are up to date. We allow them to not be
160	165	# installed.
161		OPTS = sum(CONDITIONAL_REQUIREMENTS.values(), [])
	166	OPTS = sum(CONDITIONAL_REQUIREMENTS.values(), []) # type: List[str]
162	167
163	168	for dependency in OPTS:
164	169	try:

167	172	deps_needed.append(dependency)
168	173	errors.append(
169	174	"Needed optional %s, got %s==%s"
170		% (dependency, e.dist.project_name, e.dist.version)
	175	% (
	176	dependency,
	177	e.dist.project_name, # type: ignore[attr-defined] # noqa
	178	e.dist.version, # type: ignore[attr-defined] # noqa
	179	)
171	180	)
172	181	except DistributionNotFound:
173	182	# If it's not found, we don't care

+2

-2

synapse/replication/slave/storage/account_data.py less more

15	15
16	16	from synapse.replication.slave.storage._base import BaseSlavedStore
17	17	from synapse.replication.slave.storage._slaved_id_tracker import SlavedIdTracker
18		from synapse.storage.account_data import AccountDataWorkerStore
19		from synapse.storage.tags import TagsWorkerStore
	18	from synapse.storage.data_stores.main.account_data import AccountDataWorkerStore
	19	from synapse.storage.data_stores.main.tags import TagsWorkerStore
20	20
21	21
22	22	class SlavedAccountDataStore(TagsWorkerStore, AccountDataWorkerStore, BaseSlavedStore):

+1

-1

synapse/replication/slave/storage/appservice.py less more

13	13	# See the License for the specific language governing permissions and
14	14	# limitations under the License.
15	15
16		from synapse.storage.appservice import (
	16	from synapse.storage.data_stores.main.appservice import (
17	17	ApplicationServiceTransactionWorkerStore,
18	18	ApplicationServiceWorkerStore,
19	19	)

+1

-1

synapse/replication/slave/storage/client_ips.py less more

12	12	# See the License for the specific language governing permissions and
13	13	# limitations under the License.
14	14
15		from synapse.storage.client_ips import LAST_SEEN_GRANULARITY
	15	from synapse.storage.data_stores.main.client_ips import LAST_SEEN_GRANULARITY
16	16	from synapse.util.caches import CACHE_SIZE_FACTOR
17	17	from synapse.util.caches.descriptors import Cache
18	18

+1

-1

synapse/replication/slave/storage/deviceinbox.py less more

14	14
15	15	from synapse.replication.slave.storage._base import BaseSlavedStore
16	16	from synapse.replication.slave.storage._slaved_id_tracker import SlavedIdTracker
17		from synapse.storage.deviceinbox import DeviceInboxWorkerStore
	17	from synapse.storage.data_stores.main.deviceinbox import DeviceInboxWorkerStore
18	18	from synapse.util.caches.expiringcache import ExpiringCache
19	19	from synapse.util.caches.stream_change_cache import StreamChangeCache
20	20

+5

-2

synapse/replication/slave/storage/devices.py less more

14	14
15	15	from synapse.replication.slave.storage._base import BaseSlavedStore
16	16	from synapse.replication.slave.storage._slaved_id_tracker import SlavedIdTracker
17		from synapse.storage.devices import DeviceWorkerStore
18		from synapse.storage.end_to_end_keys import EndToEndKeyWorkerStore
	17	from synapse.storage.data_stores.main.devices import DeviceWorkerStore
	18	from synapse.storage.data_stores.main.end_to_end_keys import EndToEndKeyWorkerStore
19	19	from synapse.util.caches.stream_change_cache import StreamChangeCache
20	20
21	21

31	31	device_list_max = self._device_list_id_gen.get_current_token()
32	32	self._device_list_stream_cache = StreamChangeCache(
33	33	"DeviceListStreamChangeCache", device_list_max
	34	)
	35	self._user_signature_stream_cache = StreamChangeCache(
	36	"UserSignatureStreamChangeCache", device_list_max
34	37	)
35	38	self._device_list_federation_stream_cache = StreamChangeCache(
36	39	"DeviceListFederationStreamChangeCache", device_list_max

+1

-1

synapse/replication/slave/storage/directory.py less more

12	12	# See the License for the specific language governing permissions and
13	13	# limitations under the License.
14	14
15		from synapse.storage.directory import DirectoryWorkerStore
	15	from synapse.storage.data_stores.main.directory import DirectoryWorkerStore
16	16
17	17	from ._base import BaseSlavedStore
18	18

+11

-9

synapse/replication/slave/storage/events.py less more

19	19	EventsStreamCurrentStateRow,
20	20	EventsStreamEventRow,
21	21	)
22		from synapse.storage.event_federation import EventFederationWorkerStore
23		from synapse.storage.event_push_actions import EventPushActionsWorkerStore
24		from synapse.storage.events_worker import EventsWorkerStore
25		from synapse.storage.relations import RelationsWorkerStore
26		from synapse.storage.roommember import RoomMemberWorkerStore
27		from synapse.storage.signatures import SignatureWorkerStore
28		from synapse.storage.state import StateGroupWorkerStore
29		from synapse.storage.stream import StreamWorkerStore
30		from synapse.storage.user_erasure_store import UserErasureWorkerStore
	22	from synapse.storage.data_stores.main.event_federation import EventFederationWorkerStore
	23	from synapse.storage.data_stores.main.event_push_actions import (
	24	EventPushActionsWorkerStore,
	25	)
	26	from synapse.storage.data_stores.main.events_worker import EventsWorkerStore
	27	from synapse.storage.data_stores.main.relations import RelationsWorkerStore
	28	from synapse.storage.data_stores.main.roommember import RoomMemberWorkerStore
	29	from synapse.storage.data_stores.main.signatures import SignatureWorkerStore
	30	from synapse.storage.data_stores.main.state import StateGroupWorkerStore
	31	from synapse.storage.data_stores.main.stream import StreamWorkerStore
	32	from synapse.storage.data_stores.main.user_erasure_store import UserErasureWorkerStore
31	33
32	34	from ._base import BaseSlavedStore
33	35	from ._slaved_id_tracker import SlavedIdTracker

+1

-1

synapse/replication/slave/storage/filtering.py less more

12	12	# See the License for the specific language governing permissions and
13	13	# limitations under the License.
14	14
15		from synapse.storage.filtering import FilteringStore
	15	from synapse.storage.data_stores.main.filtering import FilteringStore
16	16
17	17	from ._base import BaseSlavedStore
18	18

+1

-1

synapse/replication/slave/storage/keys.py less more

12	12	# See the License for the specific language governing permissions and
13	13	# limitations under the License.
14	14
15		from synapse.storage import KeyStore
	15	from synapse.storage.data_stores.main.keys import KeyStore
16	16
17	17	# KeyStore isn't really safe to use from a worker, but for now we do so and hope that
18	18	# the races it creates aren't too bad.

+1

-1

synapse/replication/slave/storage/presence.py less more

13	13	# limitations under the License.
14	14
15	15	from synapse.storage import DataStore
16		from synapse.storage.presence import PresenceStore
	16	from synapse.storage.data_stores.main.presence import PresenceStore
17	17	from synapse.util.caches.stream_change_cache import StreamChangeCache
18	18
19	19	from ._base import BaseSlavedStore, __func__

+1

-1

synapse/replication/slave/storage/profile.py less more

13	13	# limitations under the License.
14	14
15	15	from synapse.replication.slave.storage._base import BaseSlavedStore
16		from synapse.storage.profile import ProfileWorkerStore
	16	from synapse.storage.data_stores.main.profile import ProfileWorkerStore
17	17
18	18
19	19	class SlavedProfileStore(ProfileWorkerStore, BaseSlavedStore):

+1

-1

synapse/replication/slave/storage/push_rule.py less more

13	13	# See the License for the specific language governing permissions and
14	14	# limitations under the License.
15	15
16		from synapse.storage.push_rule import PushRulesWorkerStore
	16	from synapse.storage.data_stores.main.push_rule import PushRulesWorkerStore
17	17
18	18	from ._slaved_id_tracker import SlavedIdTracker
19	19	from .events import SlavedEventStore

+1

-1

synapse/replication/slave/storage/pushers.py less more

13	13	# See the License for the specific language governing permissions and
14	14	# limitations under the License.
15	15
16		from synapse.storage.pusher import PusherWorkerStore
	16	from synapse.storage.data_stores.main.pusher import PusherWorkerStore
17	17
18	18	from ._base import BaseSlavedStore
19	19	from ._slaved_id_tracker import SlavedIdTracker

+1

-1

synapse/replication/slave/storage/receipts.py less more

13	13	# See the License for the specific language governing permissions and
14	14	# limitations under the License.
15	15
16		from synapse.storage.receipts import ReceiptsWorkerStore
	16	from synapse.storage.data_stores.main.receipts import ReceiptsWorkerStore
17	17
18	18	from ._base import BaseSlavedStore
19	19	from ._slaved_id_tracker import SlavedIdTracker

+1

-1

synapse/replication/slave/storage/registration.py less more

12	12	# See the License for the specific language governing permissions and
13	13	# limitations under the License.
14	14
15		from synapse.storage.registration import RegistrationWorkerStore
	15	from synapse.storage.data_stores.main.registration import RegistrationWorkerStore
16	16
17	17	from ._base import BaseSlavedStore
18	18

+1

-1

synapse/replication/slave/storage/room.py less more

12	12	# See the License for the specific language governing permissions and
13	13	# limitations under the License.
14	14
15		from synapse.storage.room import RoomWorkerStore
	15	from synapse.storage.data_stores.main.room import RoomWorkerStore
16	16
17	17	from ._base import BaseSlavedStore
18	18	from ._slaved_id_tracker import SlavedIdTracker

+1

-1

synapse/replication/slave/storage/transactions.py less more

12	12	# See the License for the specific language governing permissions and
13	13	# limitations under the License.
14	14
15		from synapse.storage.transactions import TransactionStore
	15	from synapse.storage.data_stores.main.transactions import TransactionStore
16	16
17	17	from ._base import BaseSlavedStore
18	18

+62

-76

synapse/rest/admin/__init__.py less more

21	21
22	22	from six import text_type
23	23	from six.moves import http_client
24
25		from twisted.internet import defer
26	24
27	25	import synapse
28	26	from synapse.api.constants import Membership, UserTypes

45	43	from synapse.rest.admin.server_notice_servlet import SendServerNoticeServlet
46	44	from synapse.rest.admin.users import UserAdminServlet
47	45	from synapse.types import UserID, create_requester
	46	from synapse.util.async_helpers import maybe_awaitable
48	47	from synapse.util.versionstring import get_version_string
49	48
50	49	logger = logging.getLogger(__name__)

58	57	self.auth = hs.get_auth()
59	58	self.handlers = hs.get_handlers()
60	59
61		@defer.inlineCallbacks
62		def on_GET(self, request, user_id):
	60	async def on_GET(self, request, user_id):
63	61	target_user = UserID.from_string(user_id)
64		yield assert_requester_is_admin(self.auth, request)
	62	await assert_requester_is_admin(self.auth, request)
65	63
66	64	if not self.hs.is_mine(target_user):
67	65	raise SynapseError(400, "Can only users a local user")
68	66
69		ret = yield self.handlers.admin_handler.get_users()
	67	ret = await self.handlers.admin_handler.get_users()
70	68
71	69	return 200, ret
72	70

121	119	self.nonces[nonce] = int(self.reactor.seconds())
122	120	return 200, {"nonce": nonce}
123	121
124		@defer.inlineCallbacks
125		def on_POST(self, request):
	122	async def on_POST(self, request):
126	123	self._clear_old_nonces()
127	124
128	125	if not self.hs.config.registration_shared_secret:

203	200
204	201	register = RegisterRestServlet(self.hs)
205	202
206		user_id = yield register.registration_handler.register_user(
	203	user_id = await register.registration_handler.register_user(
207	204	localpart=body["username"].lower(),
208	205	password=body["password"],
209	206	admin=bool(admin),
210	207	user_type=user_type,
211	208	)
212	209
213		result = yield register._create_registration_details(user_id, body)
	210	result = await register._create_registration_details(user_id, body)
214	211	return 200, result
215	212
216	213

222	219	self.auth = hs.get_auth()
223	220	self.handlers = hs.get_handlers()
224	221
225		@defer.inlineCallbacks
226		def on_GET(self, request, user_id):
	222	async def on_GET(self, request, user_id):
227	223	target_user = UserID.from_string(user_id)
228		requester = yield self.auth.get_user_by_req(request)
	224	requester = await self.auth.get_user_by_req(request)
229	225	auth_user = requester.user
230	226
231	227	if target_user != auth_user:
232		yield assert_user_is_admin(self.auth, auth_user)
	228	await assert_user_is_admin(self.auth, auth_user)
233	229
234	230	if not self.hs.is_mine(target_user):
235	231	raise SynapseError(400, "Can only whois a local user")
236	232
237		ret = yield self.handlers.admin_handler.get_whois(target_user)
	233	ret = await self.handlers.admin_handler.get_whois(target_user)
238	234
239	235	return 200, ret
240	236

254	250	self.store = hs.get_datastore()
255	251	self.auth = hs.get_auth()
256	252
257		@defer.inlineCallbacks
258		def on_POST(self, request, room_id, event_id):
259		yield assert_requester_is_admin(self.auth, request)
	253	async def on_POST(self, request, room_id, event_id):
	254	await assert_requester_is_admin(self.auth, request)
260	255
261	256	body = parse_json_object_from_request(request, allow_empty_body=True)
262	257

269	264	event_id = body.get("purge_up_to_event_id")
270	265
271	266	if event_id is not None:
272		event = yield self.store.get_event(event_id)
	267	event = await self.store.get_event(event_id)
273	268
274	269	if event.room_id != room_id:
275	270	raise SynapseError(400, "Event is for wrong room.")
276	271
277		token = yield self.store.get_topological_token_for_event(event_id)
	272	token = await self.store.get_topological_token_for_event(event_id)
278	273
279	274	logger.info("[purge] purging up to token %s (event_id %s)", token, event_id)
280	275	elif "purge_up_to_ts" in body:

284	279	400, "purge_up_to_ts must be an int", errcode=Codes.BAD_JSON
285	280	)
286	281
287		stream_ordering = (yield self.store.find_first_stream_ordering_after_ts(ts))
288
289		r = (
290		yield self.store.get_room_event_after_stream_ordering(
291		room_id, stream_ordering
292		)
	282	stream_ordering = await self.store.find_first_stream_ordering_after_ts(ts)
	283
	284	r = await self.store.get_room_event_after_stream_ordering(
	285	room_id, stream_ordering
293	286	)
294	287	if not r:
295	288	logger.warn(

317	310	errcode=Codes.BAD_JSON,
318	311	)
319	312
320		purge_id = yield self.pagination_handler.start_purge_history(
	313	purge_id = self.pagination_handler.start_purge_history(
321	314	room_id, token, delete_local_events=delete_local_events
322	315	)
323	316

338	331	self.pagination_handler = hs.get_pagination_handler()
339	332	self.auth = hs.get_auth()
340	333
341		@defer.inlineCallbacks
342		def on_GET(self, request, purge_id):
343		yield assert_requester_is_admin(self.auth, request)
	334	async def on_GET(self, request, purge_id):
	335	await assert_requester_is_admin(self.auth, request)
344	336
345	337	purge_status = self.pagination_handler.get_purge_status(purge_id)
346	338	if purge_status is None:

356	348	self._deactivate_account_handler = hs.get_deactivate_account_handler()
357	349	self.auth = hs.get_auth()
358	350
359		@defer.inlineCallbacks
360		def on_POST(self, request, target_user_id):
361		yield assert_requester_is_admin(self.auth, request)
	351	async def on_POST(self, request, target_user_id):
	352	await assert_requester_is_admin(self.auth, request)
362	353	body = parse_json_object_from_request(request, allow_empty_body=True)
363	354	erase = body.get("erase", False)
364	355	if not isinstance(erase, bool):

370	361
371	362	UserID.from_string(target_user_id)
372	363
373		result = yield self._deactivate_account_handler.deactivate_account(
	364	result = await self._deactivate_account_handler.deactivate_account(
374	365	target_user_id, erase
375	366	)
376	367	if result:

404	395	self.room_member_handler = hs.get_room_member_handler()
405	396	self.auth = hs.get_auth()
406	397
407		@defer.inlineCallbacks
408		def on_POST(self, request, room_id):
409		requester = yield self.auth.get_user_by_req(request)
410		yield assert_user_is_admin(self.auth, requester.user)
	398	async def on_POST(self, request, room_id):
	399	requester = await self.auth.get_user_by_req(request)
	400	await assert_user_is_admin(self.auth, requester.user)
411	401
412	402	content = parse_json_object_from_request(request)
413	403	assert_params_in_dict(content, ["new_room_user_id"])

418	408	message = content.get("message", self.DEFAULT_MESSAGE)
419	409	room_name = content.get("room_name", "Content Violation Notification")
420	410
421		info = yield self._room_creation_handler.create_room(
	411	info = await self._room_creation_handler.create_room(
422	412	room_creator_requester,
423	413	config={
424	414	"preset": "public_chat",

437	427
438	428	# This will work even if the room is already blocked, but that is
439	429	# desirable in case the first attempt at blocking the room failed below.
440		yield self.store.block_room(room_id, requester_user_id)
441
442		users = yield self.state.get_current_users_in_room(room_id)
	430	await self.store.block_room(room_id, requester_user_id)
	431
	432	users = await self.state.get_current_users_in_room(room_id)
443	433	kicked_users = []
444	434	failed_to_kick_users = []
445	435	for user_id in users:

450	440
451	441	try:
452	442	target_requester = create_requester(user_id)
453		yield self.room_member_handler.update_membership(
	443	await self.room_member_handler.update_membership(
454	444	requester=target_requester,
455	445	target=target_requester.user,
456	446	room_id=room_id,

460	450	require_consent=False,
461	451	)
462	452
463		yield self.room_member_handler.forget(target_requester.user, room_id)
464
465		yield self.room_member_handler.update_membership(
	453	await self.room_member_handler.forget(target_requester.user, room_id)
	454
	455	await self.room_member_handler.update_membership(
466	456	requester=target_requester,
467	457	target=target_requester.user,
468	458	room_id=new_room_id,

479	469	)
480	470	failed_to_kick_users.append(user_id)
481	471
482		yield self.event_creation_handler.create_and_send_nonmember_event(
	472	await self.event_creation_handler.create_and_send_nonmember_event(
483	473	room_creator_requester,
484	474	{
485	475	"type": "m.room.message",

490	480	ratelimit=False,
491	481	)
492	482
493		aliases_for_room = yield self.store.get_aliases_for_room(room_id)
494
495		yield self.store.update_aliases_for_room(
	483	aliases_for_room = await maybe_awaitable(
	484	self.store.get_aliases_for_room(room_id)
	485	)
	486
	487	await self.store.update_aliases_for_room(
496	488	room_id, new_room_id, requester_user_id
497	489	)
498	490

531	523	self.auth = hs.get_auth()
532	524	self._set_password_handler = hs.get_set_password_handler()
533	525
534		@defer.inlineCallbacks
535		def on_POST(self, request, target_user_id):
	526	async def on_POST(self, request, target_user_id):
536	527	"""Post request to allow an administrator reset password for a user.
537	528	This needs user to have administrator access in Synapse.
538	529	"""
539		requester = yield self.auth.get_user_by_req(request)
540		yield assert_user_is_admin(self.auth, requester.user)
	530	requester = await self.auth.get_user_by_req(request)
	531	await assert_user_is_admin(self.auth, requester.user)
541	532
542	533	UserID.from_string(target_user_id)
543	534

545	536	assert_params_in_dict(params, ["new_password"])
546	537	new_password = params["new_password"]
547	538
548		yield self._set_password_handler.set_password(
	539	await self._set_password_handler.set_password(
549	540	target_user_id, new_password, requester
550	541	)
551	542	return 200, {}

571	562	self.auth = hs.get_auth()
572	563	self.handlers = hs.get_handlers()
573	564
574		@defer.inlineCallbacks
575		def on_GET(self, request, target_user_id):
	565	async def on_GET(self, request, target_user_id):
576	566	"""Get request to get specific number of users from Synapse.
577	567	This needs user to have administrator access in Synapse.
578	568	"""
579		yield assert_requester_is_admin(self.auth, request)
	569	await assert_requester_is_admin(self.auth, request)
580	570
581	571	target_user = UserID.from_string(target_user_id)
582	572

589	579
590	580	logger.info("limit: %s, start: %s", limit, start)
591	581
592		ret = yield self.handlers.admin_handler.get_users_paginate(order, start, limit)
	582	ret = await self.handlers.admin_handler.get_users_paginate(order, start, limit)
593	583	return 200, ret
594	584
595		@defer.inlineCallbacks
596		def on_POST(self, request, target_user_id):
	585	async def on_POST(self, request, target_user_id):
597	586	"""Post request to get specific number of users from Synapse..
598	587	This needs user to have administrator access in Synapse.
599	588	Example:

607	596	Returns:
608	597	200 OK with json object {list[dict[str, Any]], count} or empty object.
609	598	"""
610		yield assert_requester_is_admin(self.auth, request)
	599	await assert_requester_is_admin(self.auth, request)
611	600	UserID.from_string(target_user_id)
612	601
613	602	order = "name" # order by name in user table

617	606	start = params["start"]
618	607	logger.info("limit: %s, start: %s", limit, start)
619	608
620		ret = yield self.handlers.admin_handler.get_users_paginate(order, start, limit)
	609	ret = await self.handlers.admin_handler.get_users_paginate(order, start, limit)
621	610	return 200, ret
622	611
623	612

640	629	self.auth = hs.get_auth()
641	630	self.handlers = hs.get_handlers()
642	631
643		@defer.inlineCallbacks
644		def on_GET(self, request, target_user_id):
	632	async def on_GET(self, request, target_user_id):
645	633	"""Get request to search user table for specific users according to
646	634	search term.
647	635	This needs user to have a administrator access in Synapse.
648	636	"""
649		yield assert_requester_is_admin(self.auth, request)
	637	await assert_requester_is_admin(self.auth, request)
650	638
651	639	target_user = UserID.from_string(target_user_id)
652	640

660	648	term = parse_string(request, "term", required=True)
661	649	logger.info("term: %s ", term)
662	650
663		ret = yield self.handlers.admin_handler.search_users(term)
	651	ret = await self.handlers.admin_handler.search_users(term)
664	652	return 200, ret
665	653
666	654

675	663	self.is_mine_id = hs.is_mine_id
676	664	self.auth = hs.get_auth()
677	665
678		@defer.inlineCallbacks
679		def on_POST(self, request, group_id):
680		requester = yield self.auth.get_user_by_req(request)
681		yield assert_user_is_admin(self.auth, requester.user)
	666	async def on_POST(self, request, group_id):
	667	requester = await self.auth.get_user_by_req(request)
	668	await assert_user_is_admin(self.auth, requester.user)
682	669
683	670	if not self.is_mine_id(group_id):
684	671	raise SynapseError(400, "Can only delete local groups")
685	672
686		yield self.group_server.delete_group(group_id, requester.user.to_string())
	673	await self.group_server.delete_group(group_id, requester.user.to_string())
687	674	return 200, {}
688	675
689	676

699	686	self.account_activity_handler = hs.get_account_validity_handler()
700	687	self.auth = hs.get_auth()
701	688
702		@defer.inlineCallbacks
703		def on_POST(self, request):
704		yield assert_requester_is_admin(self.auth, request)
	689	async def on_POST(self, request):
	690	await assert_requester_is_admin(self.auth, request)
705	691
706	692	body = parse_json_object_from_request(request)
707	693
708	694	if "user_id" not in body:
709	695	raise SynapseError(400, "Missing property 'user_id' in the request body")
710	696
711		expiration_ts = yield self.account_activity_handler.renew_account_for_user(
	697	expiration_ts = await self.account_activity_handler.renew_account_for_user(
712	698	body["user_id"],
713	699	body.get("expiration_ts"),
714	700	not body.get("enable_renewal_emails", True),

+5

-9

synapse/rest/admin/_base.py less more

13	13	# limitations under the License.
14	14
15	15	import re
16
17		from twisted.internet import defer
18	16
19	17	from synapse.api.errors import AuthError
20	18

41	39	)
42	40
43	41
44		@defer.inlineCallbacks
45		def assert_requester_is_admin(auth, request):
	42	async def assert_requester_is_admin(auth, request):
46	43	"""Verify that the requester is an admin user
47	44
48	45	WARNING: MAKE SURE YOU YIELD ON THE RESULT!

57	54	Raises:
58	55	AuthError if the requester is not an admin
59	56	"""
60		requester = yield auth.get_user_by_req(request)
61		yield assert_user_is_admin(auth, requester.user)
	57	requester = await auth.get_user_by_req(request)
	58	await assert_user_is_admin(auth, requester.user)
62	59
63	60
64		@defer.inlineCallbacks
65		def assert_user_is_admin(auth, user_id):
	61	async def assert_user_is_admin(auth, user_id):
66	62	"""Verify that the given user is an admin user
67	63
68	64	WARNING: MAKE SURE YOU YIELD ON THE RESULT!

78	74	AuthError if the user is not an admin
79	75	"""
80	76
81		is_admin = yield auth.is_server_admin(user_id)
	77	is_admin = await auth.is_server_admin(user_id)
82	78	if not is_admin:
83	79	raise AuthError(403, "You are not a server admin")

+11

-16

synapse/rest/admin/media.py less more

14	14	# limitations under the License.
15	15
16	16	import logging
17
18		from twisted.internet import defer
19	17
20	18	from synapse.api.errors import AuthError
21	19	from synapse.http.servlet import RestServlet, parse_integer

39	37	self.store = hs.get_datastore()
40	38	self.auth = hs.get_auth()
41	39
42		@defer.inlineCallbacks
43		def on_POST(self, request, room_id):
44		requester = yield self.auth.get_user_by_req(request)
45		yield assert_user_is_admin(self.auth, requester.user)
	40	async def on_POST(self, request, room_id):
	41	requester = await self.auth.get_user_by_req(request)
	42	await assert_user_is_admin(self.auth, requester.user)
46	43
47		num_quarantined = yield self.store.quarantine_media_ids_in_room(
	44	num_quarantined = await self.store.quarantine_media_ids_in_room(
48	45	room_id, requester.user.to_string()
49	46	)
50	47

61	58	self.store = hs.get_datastore()
62	59	self.auth = hs.get_auth()
63	60
64		@defer.inlineCallbacks
65		def on_GET(self, request, room_id):
66		requester = yield self.auth.get_user_by_req(request)
67		is_admin = yield self.auth.is_server_admin(requester.user)
	61	async def on_GET(self, request, room_id):
	62	requester = await self.auth.get_user_by_req(request)
	63	is_admin = await self.auth.is_server_admin(requester.user)
68	64	if not is_admin:
69	65	raise AuthError(403, "You are not a server admin")
70	66
71		local_mxcs, remote_mxcs = yield self.store.get_media_mxcs_in_room(room_id)
	67	local_mxcs, remote_mxcs = await self.store.get_media_mxcs_in_room(room_id)
72	68
73	69	return 200, {"local": local_mxcs, "remote": remote_mxcs}
74	70

80	76	self.media_repository = hs.get_media_repository()
81	77	self.auth = hs.get_auth()
82	78
83		@defer.inlineCallbacks
84		def on_POST(self, request):
85		yield assert_requester_is_admin(self.auth, request)
	79	async def on_POST(self, request):
	80	await assert_requester_is_admin(self.auth, request)
86	81
87	82	before_ts = parse_integer(request, "before_ts", required=True)
88	83	logger.info("before_ts: %r", before_ts)
89	84
90		ret = yield self.media_repository.delete_old_remote_media(before_ts)
	85	ret = await self.media_repository.delete_old_remote_media(before_ts)
91	86
92	87	return 200, ret
93	88

+3

-6

synapse/rest/admin/server_notice_servlet.py less more

12	12	# See the License for the specific language governing permissions and
13	13	# limitations under the License.
14	14	import re
15
16		from twisted.internet import defer
17	15
18	16	from synapse.api.constants import EventTypes
19	17	from synapse.api.errors import SynapseError

68	66	self.__class__.__name__,
69	67	)
70	68
71		@defer.inlineCallbacks
72		def on_POST(self, request, txn_id=None):
73		yield assert_requester_is_admin(self.auth, request)
	69	async def on_POST(self, request, txn_id=None):
	70	await assert_requester_is_admin(self.auth, request)
74	71	body = parse_json_object_from_request(request)
75	72	assert_params_in_dict(body, ("user_id", "content"))
76	73	event_type = body.get("type", EventTypes.Message)

84	81	if not self.hs.is_mine_id(user_id):
85	82	raise SynapseError(400, "Server notices can only be sent to local users")
86	83
87		event = yield self.snm.send_notice(
	84	event = await self.snm.send_notice(
88	85	user_id=body["user_id"],
89	86	type=event_type,
90	87	state_key=state_key,

+7

-11

synapse/rest/admin/users.py less more

12	12	# See the License for the specific language governing permissions and
13	13	# limitations under the License.
14	14	import re
15
16		from twisted.internet import defer
17	15
18	16	from synapse.api.errors import SynapseError
19	17	from synapse.http.servlet import (

58	56	self.auth = hs.get_auth()
59	57	self.handlers = hs.get_handlers()
60	58
61		@defer.inlineCallbacks
62		def on_GET(self, request, user_id):
63		yield assert_requester_is_admin(self.auth, request)
	59	async def on_GET(self, request, user_id):
	60	await assert_requester_is_admin(self.auth, request)
64	61
65	62	target_user = UserID.from_string(user_id)
66	63
67	64	if not self.hs.is_mine(target_user):
68	65	raise SynapseError(400, "Only local users can be admins of this homeserver")
69	66
70		is_admin = yield self.handlers.admin_handler.get_user_server_admin(target_user)
	67	is_admin = await self.handlers.admin_handler.get_user_server_admin(target_user)
71	68	is_admin = bool(is_admin)
72	69
73	70	return 200, {"admin": is_admin}
74	71
75		@defer.inlineCallbacks
76		def on_PUT(self, request, user_id):
77		requester = yield self.auth.get_user_by_req(request)
78		yield assert_user_is_admin(self.auth, requester.user)
	72	async def on_PUT(self, request, user_id):
	73	requester = await self.auth.get_user_by_req(request)
	74	await assert_user_is_admin(self.auth, requester.user)
79	75	auth_user = requester.user
80	76
81	77	target_user = UserID.from_string(user_id)

92	88	if target_user == auth_user and not set_admin_to:
93	89	raise SynapseError(400, "You may not demote yourself.")
94	90
95		yield self.handlers.admin_handler.set_user_server_admin(
	91	await self.handlers.admin_handler.set_user_server_admin(
96	92	target_user, set_admin_to
97	93	)
98	94

+3

-1

synapse/rest/client/v1/login.py less more

376	376	super(CasTicketServlet, self).__init__()
377	377	self.cas_server_url = hs.config.cas_server_url
378	378	self.cas_service_url = hs.config.cas_service_url
	379	self.cas_displayname_attribute = hs.config.cas_displayname_attribute
379	380	self.cas_required_attributes = hs.config.cas_required_attributes
380	381	self._sso_auth_handler = SSOAuthHandler(hs)
381	382	self._http_client = hs.get_simple_http_client()

399	400
400	401	def handle_cas_response(self, request, cas_response_body, client_redirect_url):
401	402	user, attributes = self.parse_cas_response(cas_response_body)
	403	displayname = attributes.pop(self.cas_displayname_attribute, None)
402	404
403	405	for required_attribute, required_value in self.cas_required_attributes.items():
404	406	# If required attribute was not in CAS Response - Forbidden

413	415	raise LoginError(401, "Unauthorized", errcode=Codes.UNAUTHORIZED)
414	416
415	417	return self._sso_auth_handler.on_successful_auth(
416		user, request, client_redirect_url
	418	user, request, client_redirect_url, displayname
417	419	)
418	420
419	421	def parse_cas_response(self, cas_response_body):

+26

-0

synapse/rest/client/v1/room.py less more

38	38	parse_json_object_from_request,
39	39	parse_string,
40	40	)
	41	from synapse.logging.opentracing import set_tag
41	42	from synapse.rest.client.transactions import HttpTransactionCache
42	43	from synapse.rest.client.v2_alpha._base import client_patterns
43	44	from synapse.storage.state import StateFilter

80	81	)
81	82
82	83	def on_PUT(self, request, txn_id):
	84	set_tag("txn_id", txn_id)
83	85	return self.txns.fetch_or_execute_request(request, self.on_POST, request)
84	86
85	87	@defer.inlineCallbacks

179	181	@defer.inlineCallbacks
180	182	def on_PUT(self, request, room_id, event_type, state_key, txn_id=None):
181	183	requester = yield self.auth.get_user_by_req(request)
	184
	185	if txn_id:
	186	set_tag("txn_id", txn_id)
182	187
183	188	content = parse_json_object_from_request(request)
184	189

208	213
209	214	ret = {}
210	215	if event:
	216	set_tag("event_id", event.event_id)
211	217	ret = {"event_id": event.event_id}
212	218	return 200, ret
213	219

243	249	requester, event_dict, txn_id=txn_id
244	250	)
245	251
	252	set_tag("event_id", event.event_id)
246	253	return 200, {"event_id": event.event_id}
247	254
248	255	def on_GET(self, request, room_id, event_type, txn_id):
249	256	return 200, "Not implemented"
250	257
251	258	def on_PUT(self, request, room_id, event_type, txn_id):
	259	set_tag("txn_id", txn_id)
	260
252	261	return self.txns.fetch_or_execute_request(
253	262	request, self.on_POST, request, room_id, event_type, txn_id
254	263	)

309	318	return 200, {"room_id": room_id}
310	319
311	320	def on_PUT(self, request, room_identifier, txn_id):
	321	set_tag("txn_id", txn_id)
	322
312	323	return self.txns.fetch_or_execute_request(
313	324	request, self.on_POST, request, room_identifier, txn_id
314	325	)

348	359
349	360	limit = parse_integer(request, "limit", 0)
350	361	since_token = parse_string(request, "since", None)
	362
	363	if limit == 0:
	364	# zero is a special value which corresponds to no limit.
	365	limit = None
351	366
352	367	handler = self.hs.get_room_list_handler()
353	368	if server:

385	400	network_tuple = ThirdPartyInstanceID(None, None)
386	401	else:
387	402	network_tuple = ThirdPartyInstanceID.from_string(third_party_instance_id)
	403
	404	if limit == 0:
	405	# zero is a special value which corresponds to no limit.
	406	limit = None
388	407
389	408	handler = self.hs.get_room_list_handler()
390	409	if server:

654	673	return 200, {}
655	674
656	675	def on_PUT(self, request, room_id, txn_id):
	676	set_tag("txn_id", txn_id)
	677
657	678	return self.txns.fetch_or_execute_request(
658	679	request, self.on_POST, request, room_id, txn_id
659	680	)

737	758	return True
738	759
739	760	def on_PUT(self, request, room_id, membership_action, txn_id):
	761	set_tag("txn_id", txn_id)
	762
740	763	return self.txns.fetch_or_execute_request(
741	764	request, self.on_POST, request, room_id, membership_action, txn_id
742	765	)

770	793	txn_id=txn_id,
771	794	)
772	795
	796	set_tag("event_id", event.event_id)
773	797	return 200, {"event_id": event.event_id}
774	798
775	799	def on_PUT(self, request, room_id, event_id, txn_id):
	800	set_tag("txn_id", txn_id)
	801
776	802	return self.txns.fetch_or_execute_request(
777	803	request, self.on_POST, request, room_id, event_id, txn_id
778	804	)

+1

-64

synapse/rest/client/v2_alpha/account.py less more

128	128	return 200, ret
129	129
130	130
131		class MsisdnPasswordRequestTokenRestServlet(RestServlet):
132		PATTERNS = client_patterns("/account/password/msisdn/requestToken$")
133
134		def __init__(self, hs):
135		super(MsisdnPasswordRequestTokenRestServlet, self).__init__()
136		self.hs = hs
137		self.datastore = self.hs.get_datastore()
138		self.identity_handler = hs.get_handlers().identity_handler
139
140		@defer.inlineCallbacks
141		def on_POST(self, request):
142		body = parse_json_object_from_request(request)
143
144		assert_params_in_dict(
145		body, ["client_secret", "country", "phone_number", "send_attempt"]
146		)
147		client_secret = body["client_secret"]
148		country = body["country"]
149		phone_number = body["phone_number"]
150		send_attempt = body["send_attempt"]
151		next_link = body.get("next_link") # Optional param
152
153		msisdn = phone_number_to_msisdn(country, phone_number)
154
155		if not check_3pid_allowed(self.hs, "msisdn", msisdn):
156		raise SynapseError(
157		403,
158		"Account phone numbers are not authorized on this server",
159		Codes.THREEPID_DENIED,
160		)
161
162		existing_user_id = yield self.datastore.get_user_id_by_threepid(
163		"msisdn", msisdn
164		)
165
166		if existing_user_id is None:
167		raise SynapseError(400, "MSISDN not found", Codes.THREEPID_NOT_FOUND)
168
169		if not self.hs.config.account_threepid_delegate_msisdn:
170		logger.warn(
171		"No upstream msisdn account_threepid_delegate configured on the server to "
172		"handle this request"
173		)
174		raise SynapseError(
175		400,
176		"Password reset by phone number is not supported on this homeserver",
177		)
178
179		ret = yield self.identity_handler.requestMsisdnToken(
180		self.hs.config.account_threepid_delegate_msisdn,
181		country,
182		phone_number,
183		client_secret,
184		send_attempt,
185		next_link,
186		)
187
188		return 200, ret
189
190
191	131	class PasswordResetSubmitTokenServlet(RestServlet):
192	132	"""Handles 3PID validation token submission"""
193	133

300	240	else:
301	241	requester = None
302	242	result, params, _ = yield self.auth_handler.check_auth(
303		[[LoginType.EMAIL_IDENTITY], [LoginType.MSISDN]],
304		body,
305		self.hs.get_ip_from_request(request),
	243	[[LoginType.EMAIL_IDENTITY]], body, self.hs.get_ip_from_request(request)
306	244	)
307	245
308	246	if LoginType.EMAIL_IDENTITY in result:

842	780
843	781	def register_servlets(hs, http_server):
844	782	EmailPasswordRequestTokenRestServlet(hs).register(http_server)
845		MsisdnPasswordRequestTokenRestServlet(hs).register(http_server)
846	783	PasswordResetSubmitTokenServlet(hs).register(http_server)
847	784	PasswordRestServlet(hs).register(http_server)
848	785	DeactivateAccountRestServlet(hs).register(http_server)

+7

-5

synapse/rest/client/v2_alpha/filter.py less more

16	16
17	17	from twisted.internet import defer
18	18
19		from synapse.api.errors import AuthError, Codes, StoreError, SynapseError
	19	from synapse.api.errors import AuthError, NotFoundError, StoreError, SynapseError
20	20	from synapse.http.servlet import RestServlet, parse_json_object_from_request
21	21	from synapse.types import UserID
22	22

51	51	raise SynapseError(400, "Invalid filter_id")
52	52
53	53	try:
54		filter = yield self.filtering.get_user_filter(
	54	filter_collection = yield self.filtering.get_user_filter(
55	55	user_localpart=target_user.localpart, filter_id=filter_id
56	56	)
	57	except StoreError as e:
	58	if e.code != 404:
	59	raise
	60	raise NotFoundError("No such filter")
57	61
58		return 200, filter.get_filter_json()
59		except (KeyError, StoreError):
60		raise SynapseError(400, "No such filter", errcode=Codes.NOT_FOUND)
	62	return 200, filter_collection.get_filter_json()
61	63
62	64
63	65	class CreateFilterRestServlet(RestServlet):

+94

-3

synapse/rest/client/v2_alpha/keys.py less more

0	0	# -- coding: utf-8 --
1	1	# Copyright 2015, 2016 OpenMarket Ltd
	2	# Copyright 2019 New Vector Ltd
2	3	#
3	4	# Licensed under the Apache License, Version 2.0 (the "License");
4	5	# you may not use this file except in compliance with the License.

26	27	from synapse.logging.opentracing import log_kv, set_tag, trace
27	28	from synapse.types import StreamToken
28	29
29		from ._base import client_patterns
	30	from ._base import client_patterns, interactive_auth_handler
30	31
31	32	logger = logging.getLogger(__name__)
32	33

154	155
155	156	@defer.inlineCallbacks
156	157	def on_POST(self, request):
157		yield self.auth.get_user_by_req(request, allow_guest=True)
	158	requester = yield self.auth.get_user_by_req(request, allow_guest=True)
	159	user_id = requester.user.to_string()
158	160	timeout = parse_integer(request, "timeout", 10 * 1000)
159	161	body = parse_json_object_from_request(request)
160		result = yield self.e2e_keys_handler.query_devices(body, timeout)
	162	result = yield self.e2e_keys_handler.query_devices(body, timeout, user_id)
161	163	return 200, result
162	164
163	165

237	239	return 200, result
238	240
239	241
	242	class SigningKeyUploadServlet(RestServlet):
	243	"""
	244	POST /keys/device_signing/upload HTTP/1.1
	245	Content-Type: application/json
	246
	247	{
	248	}
	249	"""
	250
	251	PATTERNS = client_patterns("/keys/device_signing/upload$", releases=())
	252
	253	def __init__(self, hs):
	254	"""
	255	Args:
	256	hs (synapse.server.HomeServer): server
	257	"""
	258	super(SigningKeyUploadServlet, self).__init__()
	259	self.hs = hs
	260	self.auth = hs.get_auth()
	261	self.e2e_keys_handler = hs.get_e2e_keys_handler()
	262	self.auth_handler = hs.get_auth_handler()
	263
	264	@interactive_auth_handler
	265	@defer.inlineCallbacks
	266	def on_POST(self, request):
	267	requester = yield self.auth.get_user_by_req(request)
	268	user_id = requester.user.to_string()
	269	body = parse_json_object_from_request(request)
	270
	271	yield self.auth_handler.validate_user_via_ui_auth(
	272	requester, body, self.hs.get_ip_from_request(request)
	273	)
	274
	275	result = yield self.e2e_keys_handler.upload_signing_keys_for_user(user_id, body)
	276	return 200, result
	277
	278
	279	class SignaturesUploadServlet(RestServlet):
	280	"""
	281	POST /keys/signatures/upload HTTP/1.1
	282	Content-Type: application/json
	283
	284	{
	285	"@alice:example.com": {
	286	"<device_id>": {
	287	"user_id": "<user_id>",
	288	"device_id": "<device_id>",
	289	"algorithms": [
	290	"m.olm.curve25519-aes-sha256",
	291	"m.megolm.v1.aes-sha"
	292	],
	293	"keys": {
	294	"<algorithm>:<device_id>": "<key_base64>",
	295	},
	296	"signatures": {
	297	"<signing_user_id>": {
	298	"<algorithm>:<signing_key_base64>": "<signature_base64>>"
	299	}
	300	}
	301	}
	302	}
	303	}
	304	"""
	305
	306	PATTERNS = client_patterns("/keys/signatures/upload$")
	307
	308	def __init__(self, hs):
	309	"""
	310	Args:
	311	hs (synapse.server.HomeServer): server
	312	"""
	313	super(SignaturesUploadServlet, self).__init__()
	314	self.auth = hs.get_auth()
	315	self.e2e_keys_handler = hs.get_e2e_keys_handler()
	316
	317	@defer.inlineCallbacks
	318	def on_POST(self, request):
	319	requester = yield self.auth.get_user_by_req(request, allow_guest=True)
	320	user_id = requester.user.to_string()
	321	body = parse_json_object_from_request(request)
	322
	323	result = yield self.e2e_keys_handler.upload_signatures_for_device_keys(
	324	user_id, body
	325	)
	326	return 200, result
	327
	328
240	329	def register_servlets(hs, http_server):
241	330	KeyUploadServlet(hs).register(http_server)
242	331	KeyQueryServlet(hs).register(http_server)
243	332	KeyChangesServlet(hs).register(http_server)
244	333	OneTimeKeyServlet(hs).register(http_server)
	334	SigningKeyUploadServlet(hs).register(http_server)
	335	SignaturesUploadServlet(hs).register(http_server)

+1

-1

synapse/rest/client/v2_alpha/room_keys.py less more

374	374	"ed25519:something": "hijklmnop"
375	375	}
376	376	},
377		"version": "42"
	377	"version": "12345"
378	378	}
379	379
380	380	HTTP/1.1 200 OK

+24

-17

synapse/rest/client/v2_alpha/sync.py less more

20	20	from twisted.internet import defer
21	21
22	22	from synapse.api.constants import PresenceState
23		from synapse.api.errors import SynapseError
	23	from synapse.api.errors import Codes, StoreError, SynapseError
24	24	from synapse.api.filtering import DEFAULT_FILTER_COLLECTION, FilterCollection
25	25	from synapse.events.utils import (
26	26	format_event_for_client_v2_without_room_id,

118	118
119	119	request_key = (user, timeout, since, filter_id, full_state, device_id)
120	120
121		if filter_id:
122		if filter_id.startswith("{"):
123		try:
124		filter_object = json.loads(filter_id)
125		set_timeline_upper_limit(
126		filter_object, self.hs.config.filter_timeline_limit
127		)
128		except Exception:
129		raise SynapseError(400, "Invalid filter JSON")
130		self.filtering.check_valid_filter(filter_object)
131		filter = FilterCollection(filter_object)
132		else:
133		filter = yield self.filtering.get_user_filter(user.localpart, filter_id)
	121	if filter_id is None:
	122	filter_collection = DEFAULT_FILTER_COLLECTION
	123	elif filter_id.startswith("{"):
	124	try:
	125	filter_object = json.loads(filter_id)
	126	set_timeline_upper_limit(
	127	filter_object, self.hs.config.filter_timeline_limit
	128	)
	129	except Exception:
	130	raise SynapseError(400, "Invalid filter JSON")
	131	self.filtering.check_valid_filter(filter_object)
	132	filter_collection = FilterCollection(filter_object)
134	133	else:
135		filter = DEFAULT_FILTER_COLLECTION
	134	try:
	135	filter_collection = yield self.filtering.get_user_filter(
	136	user.localpart, filter_id
	137	)
	138	except StoreError as err:
	139	if err.code != 404:
	140	raise
	141	# fix up the description and errcode to be more useful
	142	raise SynapseError(400, "No such filter", errcode=Codes.INVALID_PARAM)
136	143
137	144	sync_config = SyncConfig(
138	145	user=user,
139		filter_collection=filter,
	146	filter_collection=filter_collection,
140	147	is_guest=requester.is_guest,
141	148	request_key=request_key,
142	149	device_id=device_id,

170	177
171	178	time_now = self.clock.time_msec()
172	179	response_content = yield self.encode_response(
173		time_now, sync_result, requester.access_token_id, filter
	180	time_now, sync_result, requester.access_token_id, filter_collection
174	181	)
175	182
176	183	return 200, response_content

+1

-1

synapse/rest/media/v1/_base.py less more

194	194	respond_404(request)
195	195	return
196	196
197		logger.debug("Responding to media request with responder %s")
	197	logger.debug("Responding to media request with responder %s", responder)
198	198	add_file_headers(request, media_type, file_size, upload_name)
199	199	try:
200	200	with responder:

+2

-2

synapse/rest/media/v1/preview_url_resource.py less more

269	269
270	270	logger.debug("Calculated OG for %s as %s" % (url, og))
271	271
272		jsonog = json.dumps(og).encode("utf8")
	272	jsonog = json.dumps(og)
273	273
274	274	# store OG in history-aware DB cache
275	275	yield self.store.store_url_cache(

282	282	media_info["created_ts"],
283	283	)
284	284
285		return jsonog
	285	return jsonog.encode("utf8")
286	286
287	287	@defer.inlineCallbacks
288	288	def _download_url(self, url, user):

+11

-3

synapse/rest/media/v1/thumbnailer.py less more

81	81	else:
82	82	return (max_height * self.width) // self.height, max_height
83	83
	84	def _resize(self, width, height):
	85	# 1-bit or 8-bit color palette images need converting to RGB
	86	# otherwise they will be scaled using nearest neighbour which
	87	# looks awful
	88	if self.image.mode in ["1", "P"]:
	89	self.image = self.image.convert("RGB")
	90	return self.image.resize((width, height), Image.ANTIALIAS)
	91
84	92	def scale(self, width, height, output_type):
85	93	"""Rescales the image to the given dimensions.
86	94
87	95	Returns:
88	96	BytesIO: the bytes of the encoded image ready to be written to disk
89	97	"""
90		scaled = self.image.resize((width, height), Image.ANTIALIAS)
	98	scaled = self._resize(width, height)
91	99	return self._encode_image(scaled, output_type)
92	100
93	101	def crop(self, width, height, output_type):

106	114	"""
107	115	if width * self.height > height * self.width:
108	116	scaled_height = (width * self.height) // self.width
109		scaled_image = self.image.resize((width, scaled_height), Image.ANTIALIAS)
	117	scaled_image = self._resize(width, scaled_height)
110	118	crop_top = (scaled_height - height) // 2
111	119	crop_bottom = height + crop_top
112	120	cropped = scaled_image.crop((0, crop_top, width, crop_bottom))
113	121	else:
114	122	scaled_width = (height * self.width) // self.height
115		scaled_image = self.image.resize((scaled_width, height), Image.ANTIALIAS)
	123	scaled_image = self._resize(scaled_width, height)
116	124	crop_left = (scaled_width - width) // 2
117	125	crop_right = width + crop_left
118	126	cropped = scaled_image.crop((crop_left, 0, crop_right, height))

+6

-2

synapse/rest/media/v1/upload_resource.py less more

16	16
17	17	from twisted.web.server import NOT_DONE_YET
18	18
19		from synapse.api.errors import SynapseError
	19	from synapse.api.errors import Codes, SynapseError
20	20	from synapse.http.server import (
21	21	DirectServeResource,
22	22	respond_with_json,

55	55	if content_length is None:
56	56	raise SynapseError(msg="Request must specify a Content-Length", code=400)
57	57	if int(content_length) > self.max_upload_size:
58		raise SynapseError(msg="Upload request body is too large", code=413)
	58	raise SynapseError(
	59	msg="Upload request body is too large",
	60	code=413,
	61	errcode=Codes.TOO_LARGE,
	62	)
59	63
60	64	upload_name = parse_string(request, b"filename", encoding=None)
61	65	if upload_name:

+73

-39

synapse/server_notices/resource_limits_server_notices.py less more

19	19
20	20	from synapse.api.constants import (
21	21	EventTypes,
	22	LimitBlockingTypes,
22	23	ServerNoticeLimitReached,
23	24	ServerNoticeMsgType,
24	25	)

69	70	return
70	71
71	72	if not self._server_notices_manager.is_enabled():
72		# Don't try and send server notices unles they've been enabled
	73	# Don't try and send server notices unless they've been enabled
73	74	return
74	75
75	76	timestamp = yield self._store.user_last_seen_monthly_active(user_id)

78	79	# In practice, not sure we can ever get here
79	80	return
80	81
81		# Determine current state of room
82
83	82	room_id = yield self._server_notices_manager.get_notice_room_for_user(user_id)
84	83
85	84	if not room_id:

87	86	return
88	87
89	88	yield self._check_and_set_tags(user_id, room_id)
	89
	90	# Determine current state of room
90	91	currently_blocked, ref_events = yield self._is_room_currently_blocked(room_id)
91	92
	93	limit_msg = None
	94	limit_type = None
92	95	try:
93		# Normally should always pass in user_id if you have it, but in
94		# this case are checking what would happen to other users if they
95		# were to arrive.
96		try:
97		yield self._auth.check_auth_blocking()
98		is_auth_blocking = False
99		except ResourceLimitError as e:
100		is_auth_blocking = True
101		event_content = e.msg
102		event_limit_type = e.limit_type
103
104		if currently_blocked and not is_auth_blocking:
	96	# Normally should always pass in user_id to check_auth_blocking
	97	# if you have it, but in this case are checking what would happen
	98	# to other users if they were to arrive.
	99	yield self._auth.check_auth_blocking()
	100	except ResourceLimitError as e:
	101	limit_msg = e.msg
	102	limit_type = e.limit_type
	103
	104	try:
	105	if (
	106	limit_type == LimitBlockingTypes.MONTHLY_ACTIVE_USER
	107	and not self._config.mau_limit_alerting
	108	):
	109	# We have hit the MAU limit, but MAU alerting is disabled:
	110	# reset room if necessary and return
	111	if currently_blocked:
	112	self._remove_limit_block_notification(user_id, ref_events)
	113	return
	114
	115	if currently_blocked and not limit_msg:
105	116	# Room is notifying of a block, when it ought not to be.
106		# Remove block notification
107		content = {"pinned": ref_events}
108		yield self._server_notices_manager.send_notice(
109		user_id, content, EventTypes.Pinned, ""
	117	yield self._remove_limit_block_notification(user_id, ref_events)
	118	elif not currently_blocked and limit_msg:
	119	# Room is not notifying of a block, when it ought to be.
	120	yield self._apply_limit_block_notification(
	121	user_id, limit_msg, limit_type
110	122	)
111
112		elif not currently_blocked and is_auth_blocking:
113		# Room is not notifying of a block, when it ought to be.
114		# Add block notification
115		content = {
116		"body": event_content,
117		"msgtype": ServerNoticeMsgType,
118		"server_notice_type": ServerNoticeLimitReached,
119		"admin_contact": self._config.admin_contact,
120		"limit_type": event_limit_type,
121		}
122		event = yield self._server_notices_manager.send_notice(
123		user_id, content, EventTypes.Message
124		)
125
126		content = {"pinned": [event.event_id]}
127		yield self._server_notices_manager.send_notice(
128		user_id, content, EventTypes.Pinned, ""
129		)
130
131	123	except SynapseError as e:
132	124	logger.error("Error sending resource limits server notice: %s", e)
	125
	126	@defer.inlineCallbacks
	127	def _remove_limit_block_notification(self, user_id, ref_events):
	128	"""Utility method to remove limit block notifications from the server
	129	notices room.
	130
	131	Args:
	132	user_id (str): user to notify
	133	ref_events (list[str]): The event_ids of pinned events that are unrelated to
	134	limit blocking and need to be preserved.
	135	"""
	136	content = {"pinned": ref_events}
	137	yield self._server_notices_manager.send_notice(
	138	user_id, content, EventTypes.Pinned, ""
	139	)
	140
	141	@defer.inlineCallbacks
	142	def _apply_limit_block_notification(self, user_id, event_body, event_limit_type):
	143	"""Utility method to apply limit block notifications in the server
	144	notices room.
	145
	146	Args:
	147	user_id (str): user to notify
	148	event_body(str): The human readable text that describes the block.
	149	event_limit_type(str): Specifies the type of block e.g. monthly active user
	150	limit has been exceeded.
	151	"""
	152	content = {
	153	"body": event_body,
	154	"msgtype": ServerNoticeMsgType,
	155	"server_notice_type": ServerNoticeLimitReached,
	156	"admin_contact": self._config.admin_contact,
	157	"limit_type": event_limit_type,
	158	}
	159	event = yield self._server_notices_manager.send_notice(
	160	user_id, content, EventTypes.Message
	161	)
	162
	163	content = {"pinned": [event.event_id]}
	164	yield self._server_notices_manager.send_notice(
	165	user_id, content, EventTypes.Pinned, ""
	166	)
133	167
134	168	@defer.inlineCallbacks
135	169	def _check_and_set_tags(self, user_id, room_id):

+18

-6

synapse/state/__init__.py less more

32	32	from synapse.util.async_helpers import Linearizer
33	33	from synapse.util.caches import get_cache_factor_for
34	34	from synapse.util.caches.expiringcache import ExpiringCache
35		from synapse.util.metrics import Measure
	35	from synapse.util.metrics import Measure, measure_func
36	36
37	37	logger = logging.getLogger(__name__)
38	38

190	190	return joined_users
191	191
192	192	@defer.inlineCallbacks
193		def get_current_hosts_in_room(self, room_id, latest_event_ids=None):
194		if not latest_event_ids:
195		latest_event_ids = yield self.store.get_latest_event_ids_in_room(room_id)
196		logger.debug("calling resolve_state_groups from get_current_hosts_in_room")
197		entry = yield self.resolve_state_groups_for_events(room_id, latest_event_ids)
	193	def get_current_hosts_in_room(self, room_id):
	194	event_ids = yield self.store.get_latest_event_ids_in_room(room_id)
	195	return (yield self.get_hosts_in_room_at_events(room_id, event_ids))
	196
	197	@defer.inlineCallbacks
	198	def get_hosts_in_room_at_events(self, room_id, event_ids):
	199	"""Get the hosts that were in a room at the given event ids
	200
	201	Args:
	202	room_id (str):
	203	event_ids (list[str]):
	204
	205	Returns:
	206	Deferred[list[str]]: the hosts in the room at the given events
	207	"""
	208	entry = yield self.resolve_state_groups_for_events(room_id, event_ids)
198	209	joined_hosts = yield self.store.get_joined_hosts(room_id, entry)
199	210	return joined_hosts
200	211

343	354
344	355	return context
345	356
	357	@measure_func()
346	358	@defer.inlineCallbacks
347	359	def resolve_state_groups_for_events(self, room_id, event_ids):
348	360	""" Given a list of event_ids this method fetches the state at each

+13

-502

synapse/storage/__init__.py less more

0	0	# -- coding: utf-8 --
1	1	# Copyright 2014-2016 OpenMarket Ltd
2		# Copyright 2018 New Vector Ltd
	2	# Copyright 2018,2019 New Vector Ltd
3	3	#
4	4	# Licensed under the Apache License, Version 2.0 (the "License");
5	5	# you may not use this file except in compliance with the License.

13	13	# See the License for the specific language governing permissions and
14	14	# limitations under the License.
15	15
16		import calendar
17		import logging
18		import time
	16	"""
	17	The storage layer is split up into multiple parts to allow Synapse to run
	18	against different configurations of databases (e.g. single or multiple
	19	databases). The `data_stores` are classes that talk directly to a single
	20	database and have associated schemas, background updates, etc. On top of those
	21	there are (or will be) classes that provide high level interfaces that combine
	22	calls to multiple `data_stores`.
19	23
20		from twisted.internet import defer
	24	There are also schemas that get applied to every database, regardless of the
	25	data stores associated with them (e.g. the schema version tables), which are
	26	stored in `synapse.storage.schema`.
	27	"""
21	28
22		from synapse.api.constants import PresenceState
23		from synapse.storage.devices import DeviceStore
24		from synapse.storage.user_erasure_store import UserErasureStore
25		from synapse.util.caches.stream_change_cache import StreamChangeCache
26
27		from .account_data import AccountDataStore
28		from .appservice import ApplicationServiceStore, ApplicationServiceTransactionStore
29		from .client_ips import ClientIpStore
30		from .deviceinbox import DeviceInboxStore
31		from .directory import DirectoryStore
32		from .e2e_room_keys import EndToEndRoomKeyStore
33		from .end_to_end_keys import EndToEndKeyStore
34		from .engines import PostgresEngine
35		from .event_federation import EventFederationStore
36		from .event_push_actions import EventPushActionsStore
37		from .events import EventsStore
38		from .events_bg_updates import EventsBackgroundUpdatesStore
39		from .filtering import FilteringStore
40		from .group_server import GroupServerStore
41		from .keys import KeyStore
42		from .media_repository import MediaRepositoryStore
43		from .monthly_active_users import MonthlyActiveUsersStore
44		from .openid import OpenIdStore
45		from .presence import PresenceStore, UserPresenceState
46		from .profile import ProfileStore
47		from .push_rule import PushRuleStore
48		from .pusher import PusherStore
49		from .receipts import ReceiptsStore
50		from .registration import RegistrationStore
51		from .rejections import RejectionsStore
52		from .relations import RelationsStore
53		from .room import RoomStore
54		from .roommember import RoomMemberStore
55		from .search import SearchStore
56		from .signatures import SignatureStore
57		from .state import StateStore
58		from .stats import StatsStore
59		from .stream import StreamStore
60		from .tags import TagsStore
61		from .transactions import TransactionStore
62		from .user_directory import UserDirectoryStore
63		from .util.id_generators import ChainedIdGenerator, IdGenerator, StreamIdGenerator
64
65		logger = logging.getLogger(__name__)
66
67
68		class DataStore(
69		EventsBackgroundUpdatesStore,
70		RoomMemberStore,
71		RoomStore,
72		RegistrationStore,
73		StreamStore,
74		ProfileStore,
75		PresenceStore,
76		TransactionStore,
77		DirectoryStore,
78		KeyStore,
79		StateStore,
80		SignatureStore,
81		ApplicationServiceStore,
82		EventsStore,
83		EventFederationStore,
84		MediaRepositoryStore,
85		RejectionsStore,
86		FilteringStore,
87		PusherStore,
88		PushRuleStore,
89		ApplicationServiceTransactionStore,
90		ReceiptsStore,
91		EndToEndKeyStore,
92		EndToEndRoomKeyStore,
93		SearchStore,
94		TagsStore,
95		AccountDataStore,
96		EventPushActionsStore,
97		OpenIdStore,
98		ClientIpStore,
99		DeviceStore,
100		DeviceInboxStore,
101		UserDirectoryStore,
102		GroupServerStore,
103		UserErasureStore,
104		MonthlyActiveUsersStore,
105		StatsStore,
106		RelationsStore,
107		):
108		def __init__(self, db_conn, hs):
109		self.hs = hs
110		self._clock = hs.get_clock()
111		self.database_engine = hs.database_engine
112
113		self._stream_id_gen = StreamIdGenerator(
114		db_conn,
115		"events",
116		"stream_ordering",
117		extra_tables=[("local_invites", "stream_id")],
118		)
119		self._backfill_id_gen = StreamIdGenerator(
120		db_conn,
121		"events",
122		"stream_ordering",
123		step=-1,
124		extra_tables=[("ex_outlier_stream", "event_stream_ordering")],
125		)
126		self._presence_id_gen = StreamIdGenerator(
127		db_conn, "presence_stream", "stream_id"
128		)
129		self._device_inbox_id_gen = StreamIdGenerator(
130		db_conn, "device_max_stream_id", "stream_id"
131		)
132		self._public_room_id_gen = StreamIdGenerator(
133		db_conn, "public_room_list_stream", "stream_id"
134		)
135		self._device_list_id_gen = StreamIdGenerator(
136		db_conn, "device_lists_stream", "stream_id"
137		)
138
139		self._access_tokens_id_gen = IdGenerator(db_conn, "access_tokens", "id")
140		self._event_reports_id_gen = IdGenerator(db_conn, "event_reports", "id")
141		self._push_rule_id_gen = IdGenerator(db_conn, "push_rules", "id")
142		self._push_rules_enable_id_gen = IdGenerator(db_conn, "push_rules_enable", "id")
143		self._push_rules_stream_id_gen = ChainedIdGenerator(
144		self._stream_id_gen, db_conn, "push_rules_stream", "stream_id"
145		)
146		self._pushers_id_gen = StreamIdGenerator(
147		db_conn, "pushers", "id", extra_tables=[("deleted_pushers", "stream_id")]
148		)
149		self._group_updates_id_gen = StreamIdGenerator(
150		db_conn, "local_group_updates", "stream_id"
151		)
152
153		if isinstance(self.database_engine, PostgresEngine):
154		self._cache_id_gen = StreamIdGenerator(
155		db_conn, "cache_invalidation_stream", "stream_id"
156		)
157		else:
158		self._cache_id_gen = None
159
160		self._presence_on_startup = self._get_active_presence(db_conn)
161
162		presence_cache_prefill, min_presence_val = self._get_cache_dict(
163		db_conn,
164		"presence_stream",
165		entity_column="user_id",
166		stream_column="stream_id",
167		max_value=self._presence_id_gen.get_current_token(),
168		)
169		self.presence_stream_cache = StreamChangeCache(
170		"PresenceStreamChangeCache",
171		min_presence_val,
172		prefilled_cache=presence_cache_prefill,
173		)
174
175		max_device_inbox_id = self._device_inbox_id_gen.get_current_token()
176		device_inbox_prefill, min_device_inbox_id = self._get_cache_dict(
177		db_conn,
178		"device_inbox",
179		entity_column="user_id",
180		stream_column="stream_id",
181		max_value=max_device_inbox_id,
182		limit=1000,
183		)
184		self._device_inbox_stream_cache = StreamChangeCache(
185		"DeviceInboxStreamChangeCache",
186		min_device_inbox_id,
187		prefilled_cache=device_inbox_prefill,
188		)
189		# The federation outbox and the local device inbox uses the same
190		# stream_id generator.
191		device_outbox_prefill, min_device_outbox_id = self._get_cache_dict(
192		db_conn,
193		"device_federation_outbox",
194		entity_column="destination",
195		stream_column="stream_id",
196		max_value=max_device_inbox_id,
197		limit=1000,
198		)
199		self._device_federation_outbox_stream_cache = StreamChangeCache(
200		"DeviceFederationOutboxStreamChangeCache",
201		min_device_outbox_id,
202		prefilled_cache=device_outbox_prefill,
203		)
204
205		device_list_max = self._device_list_id_gen.get_current_token()
206		self._device_list_stream_cache = StreamChangeCache(
207		"DeviceListStreamChangeCache", device_list_max
208		)
209		self._device_list_federation_stream_cache = StreamChangeCache(
210		"DeviceListFederationStreamChangeCache", device_list_max
211		)
212
213		events_max = self._stream_id_gen.get_current_token()
214		curr_state_delta_prefill, min_curr_state_delta_id = self._get_cache_dict(
215		db_conn,
216		"current_state_delta_stream",
217		entity_column="room_id",
218		stream_column="stream_id",
219		max_value=events_max, # As we share the stream id with events token
220		limit=1000,
221		)
222		self._curr_state_delta_stream_cache = StreamChangeCache(
223		"_curr_state_delta_stream_cache",
224		min_curr_state_delta_id,
225		prefilled_cache=curr_state_delta_prefill,
226		)
227
228		_group_updates_prefill, min_group_updates_id = self._get_cache_dict(
229		db_conn,
230		"local_group_updates",
231		entity_column="user_id",
232		stream_column="stream_id",
233		max_value=self._group_updates_id_gen.get_current_token(),
234		limit=1000,
235		)
236		self._group_updates_stream_cache = StreamChangeCache(
237		"_group_updates_stream_cache",
238		min_group_updates_id,
239		prefilled_cache=_group_updates_prefill,
240		)
241
242		self._stream_order_on_start = self.get_room_max_stream_ordering()
243		self._min_stream_order_on_start = self.get_room_min_stream_ordering()
244
245		# Used in _generate_user_daily_visits to keep track of progress
246		self._last_user_visit_update = self._get_start_of_day()
247
248		super(DataStore, self).__init__(db_conn, hs)
249
250		def take_presence_startup_info(self):
251		active_on_startup = self._presence_on_startup
252		self._presence_on_startup = None
253		return active_on_startup
254
255		def _get_active_presence(self, db_conn):
256		"""Fetch non-offline presence from the database so that we can register
257		the appropriate time outs.
258		"""
259
260		sql = (
261		"SELECT user_id, state, last_active_ts, last_federation_update_ts,"
262		" last_user_sync_ts, status_msg, currently_active FROM presence_stream"
263		" WHERE state != ?"
264		)
265		sql = self.database_engine.convert_param_style(sql)
266
267		txn = db_conn.cursor()
268		txn.execute(sql, (PresenceState.OFFLINE,))
269		rows = self.cursor_to_dict(txn)
270		txn.close()
271
272		for row in rows:
273		row["currently_active"] = bool(row["currently_active"])
274
275		return [UserPresenceState(**row) for row in rows]
276
277		def count_daily_users(self):
278		"""
279		Counts the number of users who used this homeserver in the last 24 hours.
280		"""
281		yesterday = int(self._clock.time_msec()) - (1000 * 60 * 60 * 24)
282		return self.runInteraction("count_daily_users", self._count_users, yesterday)
283
284		def count_monthly_users(self):
285		"""
286		Counts the number of users who used this homeserver in the last 30 days.
287		Note this method is intended for phonehome metrics only and is different
288		from the mau figure in synapse.storage.monthly_active_users which,
289		amongst other things, includes a 3 day grace period before a user counts.
290		"""
291		thirty_days_ago = int(self._clock.time_msec()) - (1000 * 60 * 60 * 24 * 30)
292		return self.runInteraction(
293		"count_monthly_users", self._count_users, thirty_days_ago
294		)
295
296		def _count_users(self, txn, time_from):
297		"""
298		Returns number of users seen in the past time_from period
299		"""
300		sql = """
301		SELECT COALESCE(count(*), 0) FROM (
302		SELECT user_id FROM user_ips
303		WHERE last_seen > ?
304		GROUP BY user_id
305		) u
306		"""
307		txn.execute(sql, (time_from,))
308		count, = txn.fetchone()
309		return count
310
311		def count_r30_users(self):
312		"""
313		Counts the number of 30 day retained users, defined as:-
314		* Users who have created their accounts more than 30 days ago
315		* Where last seen at most 30 days ago
316		* Where account creation and last_seen are > 30 days apart
317
318		Returns counts globaly for a given user as well as breaking
319		by platform
320		"""
321
322		def _count_r30_users(txn):
323		thirty_days_in_secs = 86400 * 30
324		now = int(self._clock.time())
325		thirty_days_ago_in_secs = now - thirty_days_in_secs
326
327		sql = """
328		SELECT platform, COALESCE(count(*), 0) FROM (
329		SELECT
330		users.name, platform, users.creation_ts * 1000,
331		MAX(uip.last_seen)
332		FROM users
333		INNER JOIN (
334		SELECT
335		user_id,
336		last_seen,
337		CASE
338		WHEN user_agent LIKE '%%Android%%' THEN 'android'
339		WHEN user_agent LIKE '%%iOS%%' THEN 'ios'
340		WHEN user_agent LIKE '%%Electron%%' THEN 'electron'
341		WHEN user_agent LIKE '%%Mozilla%%' THEN 'web'
342		WHEN user_agent LIKE '%%Gecko%%' THEN 'web'
343		ELSE 'unknown'
344		END
345		AS platform
346		FROM user_ips
347		) uip
348		ON users.name = uip.user_id
349		AND users.appservice_id is NULL
350		AND users.creation_ts < ?
351		AND uip.last_seen/1000 > ?
352		AND (uip.last_seen/1000) - users.creation_ts > 86400 * 30
353		GROUP BY users.name, platform, users.creation_ts
354		) u GROUP BY platform
355		"""
356
357		results = {}
358		txn.execute(sql, (thirty_days_ago_in_secs, thirty_days_ago_in_secs))
359
360		for row in txn:
361		if row[0] == "unknown":
362		pass
363		results[row[0]] = row[1]
364
365		sql = """
366		SELECT COALESCE(count(*), 0) FROM (
367		SELECT users.name, users.creation_ts * 1000,
368		MAX(uip.last_seen)
369		FROM users
370		INNER JOIN (
371		SELECT
372		user_id,
373		last_seen
374		FROM user_ips
375		) uip
376		ON users.name = uip.user_id
377		AND appservice_id is NULL
378		AND users.creation_ts < ?
379		AND uip.last_seen/1000 > ?
380		AND (uip.last_seen/1000) - users.creation_ts > 86400 * 30
381		GROUP BY users.name, users.creation_ts
382		) u
383		"""
384
385		txn.execute(sql, (thirty_days_ago_in_secs, thirty_days_ago_in_secs))
386
387		count, = txn.fetchone()
388		results["all"] = count
389
390		return results
391
392		return self.runInteraction("count_r30_users", _count_r30_users)
393
394		def _get_start_of_day(self):
395		"""
396		Returns millisecond unixtime for start of UTC day.
397		"""
398		now = time.gmtime()
399		today_start = calendar.timegm((now.tm_year, now.tm_mon, now.tm_mday, 0, 0, 0))
400		return today_start * 1000
401
402		def generate_user_daily_visits(self):
403		"""
404		Generates daily visit data for use in cohort/ retention analysis
405		"""
406
407		def _generate_user_daily_visits(txn):
408		logger.info("Calling _generate_user_daily_visits")
409		today_start = self._get_start_of_day()
410		a_day_in_milliseconds = 24 * 60 * 60 * 1000
411		now = self.clock.time_msec()
412
413		sql = """
414		INSERT INTO user_daily_visits (user_id, device_id, timestamp)
415		SELECT u.user_id, u.device_id, ?
416		FROM user_ips AS u
417		LEFT JOIN (
418		SELECT user_id, device_id, timestamp FROM user_daily_visits
419		WHERE timestamp = ?
420		) udv
421		ON u.user_id = udv.user_id AND u.device_id=udv.device_id
422		INNER JOIN users ON users.name=u.user_id
423		WHERE last_seen > ? AND last_seen <= ?
424		AND udv.timestamp IS NULL AND users.is_guest=0
425		AND users.appservice_id IS NULL
426		GROUP BY u.user_id, u.device_id
427		"""
428
429		# This means that the day has rolled over but there could still
430		# be entries from the previous day. There is an edge case
431		# where if the user logs in at 23:59 and overwrites their
432		# last_seen at 00:01 then they will not be counted in the
433		# previous day's stats - it is important that the query is run
434		# often to minimise this case.
435		if today_start > self._last_user_visit_update:
436		yesterday_start = today_start - a_day_in_milliseconds
437		txn.execute(
438		sql,
439		(
440		yesterday_start,
441		yesterday_start,
442		self._last_user_visit_update,
443		today_start,
444		),
445		)
446		self._last_user_visit_update = today_start
447
448		txn.execute(
449		sql, (today_start, today_start, self._last_user_visit_update, now)
450		)
451		# Update _last_user_visit_update to now. The reason to do this
452		# rather just clamping to the beginning of the day is to limit
453		# the size of the join - meaning that the query can be run more
454		# frequently
455		self._last_user_visit_update = now
456
457		return self.runInteraction(
458		"generate_user_daily_visits", _generate_user_daily_visits
459		)
460
461		def get_users(self):
462		"""Function to reterive a list of users in users table.
463
464		Args:
465		Returns:
466		defer.Deferred: resolves to list[dict[str, Any]]
467		"""
468		return self._simple_select_list(
469		table="users",
470		keyvalues={},
471		retcols=["name", "password_hash", "is_guest", "admin", "user_type"],
472		desc="get_users",
473		)
474
475		@defer.inlineCallbacks
476		def get_users_paginate(self, order, start, limit):
477		"""Function to reterive a paginated list of users from
478		users list. This will return a json object, which contains
479		list of users and the total number of users in users table.
480
481		Args:
482		order (str): column name to order the select by this column
483		start (int): start number to begin the query from
484		limit (int): number of rows to reterive
485		Returns:
486		defer.Deferred: resolves to json object {list[dict[str, Any]], count}
487		"""
488		users = yield self.runInteraction(
489		"get_users_paginate",
490		self._simple_select_list_paginate_txn,
491		table="users",
492		keyvalues={"is_guest": False},
493		orderby=order,
494		start=start,
495		limit=limit,
496		retcols=["name", "password_hash", "is_guest", "admin", "user_type"],
497		)
498		count = yield self.runInteraction("get_users_paginate", self.get_user_count_txn)
499		retval = {"users": users, "total": count}
500		return retval
501
502		def search_users(self, term):
503		"""Function to search users list for one or more users with
504		the matched term.
505
506		Args:
507		term (str): search term
508		col (str): column to query term should be matched to
509		Returns:
510		defer.Deferred: resolves to list[dict[str, Any]]
511		"""
512		return self._simple_search_list(
513		table="users",
514		term=term,
515		col="name",
516		retcols=["name", "password_hash", "is_guest", "admin", "user_type"],
517		desc="search_users",
518		)
	29	from synapse.storage.data_stores.main import DataStore # noqa: F401
519	30
520	31
521	32	def are_all_users_on_domain(txn, database_engine, domain):

+41

-15

synapse/storage/_base.py less more

19	19	import sys
20	20	import threading
21	21	import time
	22	from typing import Iterable, Tuple
22	23
23	24	from six import PY2, iteritems, iterkeys, itervalues
24	25	from six.moves import builtins, intern, range

29	30	from twisted.internet import defer
30	31
31	32	from synapse.api.errors import StoreError
32		from synapse.logging.context import LoggingContext, PreserveLoggingContext
	33	from synapse.logging.context import LoggingContext, make_deferred_yieldable
33	34	from synapse.metrics.background_process_metrics import run_as_background_process
34	35	from synapse.storage.engines import PostgresEngine, Sqlite3Engine
35	36	from synapse.types import get_domain_from_id

549	550
550	551	return func(conn, args, *kwargs)
551	552
552		with PreserveLoggingContext():
553		result = yield self._db_pool.runWithConnection(inner_func, args, *kwargs)
	553	result = yield make_deferred_yieldable(
	554	self._db_pool.runWithConnection(inner_func, args, *kwargs)
	555	)
554	556
555	557	return result
556	558

1161	1163	if not iterable:
1162	1164	return []
1163	1165
1164		sql = "SELECT %s FROM %s" % (", ".join(retcols), table)
1165
1166		clauses = []
1167		values = []
1168		clauses.append("%s IN (%s)" % (column, ",".join("?" for _ in iterable)))
1169		values.extend(iterable)
	1166	clause, values = make_in_list_sql_clause(txn.database_engine, column, iterable)
	1167	clauses = [clause]
1170	1168
1171	1169	for key, value in iteritems(keyvalues):
1172	1170	clauses.append("%s = ?" % (key,))
1173	1171	values.append(value)
1174	1172
1175		if clauses:
1176		sql = "%s WHERE %s" % (sql, " AND ".join(clauses))
	1173	sql = "SELECT %s FROM %s WHERE %s" % (
	1174	", ".join(retcols),
	1175	table,
	1176	" AND ".join(clauses),
	1177	)
1177	1178
1178	1179	txn.execute(sql, values)
1179	1180	return cls.cursor_to_dict(txn)

1322	1323
1323	1324	sql = "DELETE FROM %s" % table
1324	1325
1325		clauses = []
1326		values = []
1327		clauses.append("%s IN (%s)" % (column, ",".join("?" for _ in iterable)))
1328		values.extend(iterable)
	1326	clause, values = make_in_list_sql_clause(txn.database_engine, column, iterable)
	1327	clauses = [clause]
1329	1328
1330	1329	for key, value in iteritems(keyvalues):
1331	1330	clauses.append("%s = ?" % (key,))

1692	1691	except Exception:
1693	1692	logging.warning("Tried to decode '%r' as JSON and failed", db_content)
1694	1693	raise
	1694
	1695
	1696	def make_in_list_sql_clause(
	1697	database_engine, column: str, iterable: Iterable
	1698	) -> Tuple[str, Iterable]:
	1699	"""Returns an SQL clause that checks the given column is in the iterable.
	1700
	1701	On SQLite this expands to `column IN (?, ?, ...)`, whereas on Postgres
	1702	it expands to `column = ANY(?)`. While both DBs support the `IN` form,
	1703	using the `ANY` form on postgres means that it views queries with
	1704	different length iterables as the same, helping the query stats.
	1705
	1706	Args:
	1707	database_engine
	1708	column: Name of the column
	1709	iterable: The values to check the column against.
	1710
	1711	Returns:
	1712	A tuple of SQL query and the args
	1713	"""
	1714
	1715	if database_engine.supports_using_any_list:
	1716	# This should hopefully be faster, but also makes postgres query
	1717	# stats easier to understand.
	1718	return "%s = ANY(?)" % (column,), [list(iterable)]
	1719	else:
	1720	return "%s IN (%s)" % (column, ",".join("?" for _ in iterable)), list(iterable)

+0

-391

~~synapse/storage/account_data.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2014-2016 OpenMarket Ltd
2		# Copyright 2018 New Vector Ltd
3		#
4		# Licensed under the Apache License, Version 2.0 (the "License");
5		# you may not use this file except in compliance with the License.
6		# You may obtain a copy of the License at
7		#
8		# http://www.apache.org/licenses/LICENSE-2.0
9		#
10		# Unless required by applicable law or agreed to in writing, software
11		# distributed under the License is distributed on an "AS IS" BASIS,
12		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13		# See the License for the specific language governing permissions and
14		# limitations under the License.
15
16		import abc
17		import logging
18
19		from canonicaljson import json
20
21		from twisted.internet import defer
22
23		from synapse.storage._base import SQLBaseStore
24		from synapse.storage.util.id_generators import StreamIdGenerator
25		from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
26		from synapse.util.caches.stream_change_cache import StreamChangeCache
27
28		logger = logging.getLogger(__name__)
29
30
31		class AccountDataWorkerStore(SQLBaseStore):
32		"""This is an abstract base class where subclasses must implement
33		`get_max_account_data_stream_id` which can be called in the initializer.
34		"""
35
36		# This ABCMeta metaclass ensures that we cannot be instantiated without
37		# the abstract methods being implemented.
38		__metaclass__ = abc.ABCMeta
39
40		def __init__(self, db_conn, hs):
41		account_max = self.get_max_account_data_stream_id()
42		self._account_data_stream_cache = StreamChangeCache(
43		"AccountDataAndTagsChangeCache", account_max
44		)
45
46		super(AccountDataWorkerStore, self).__init__(db_conn, hs)
47
48		@abc.abstractmethod
49		def get_max_account_data_stream_id(self):
50		"""Get the current max stream ID for account data stream
51
52		Returns:
53		int
54		"""
55		raise NotImplementedError()
56
57		@cached()
58		def get_account_data_for_user(self, user_id):
59		"""Get all the client account_data for a user.
60
61		Args:
62		user_id(str): The user to get the account_data for.
63		Returns:
64		A deferred pair of a dict of global account_data and a dict
65		mapping from room_id string to per room account_data dicts.
66		"""
67
68		def get_account_data_for_user_txn(txn):
69		rows = self._simple_select_list_txn(
70		txn,
71		"account_data",
72		{"user_id": user_id},
73		["account_data_type", "content"],
74		)
75
76		global_account_data = {
77		row["account_data_type"]: json.loads(row["content"]) for row in rows
78		}
79
80		rows = self._simple_select_list_txn(
81		txn,
82		"room_account_data",
83		{"user_id": user_id},
84		["room_id", "account_data_type", "content"],
85		)
86
87		by_room = {}
88		for row in rows:
89		room_data = by_room.setdefault(row["room_id"], {})
90		room_data[row["account_data_type"]] = json.loads(row["content"])
91
92		return global_account_data, by_room
93
94		return self.runInteraction(
95		"get_account_data_for_user", get_account_data_for_user_txn
96		)
97
98		@cachedInlineCallbacks(num_args=2, max_entries=5000)
99		def get_global_account_data_by_type_for_user(self, data_type, user_id):
100		"""
101		Returns:
102		Deferred: A dict
103		"""
104		result = yield self._simple_select_one_onecol(
105		table="account_data",
106		keyvalues={"user_id": user_id, "account_data_type": data_type},
107		retcol="content",
108		desc="get_global_account_data_by_type_for_user",
109		allow_none=True,
110		)
111
112		if result:
113		return json.loads(result)
114		else:
115		return None
116
117		@cached(num_args=2)
118		def get_account_data_for_room(self, user_id, room_id):
119		"""Get all the client account_data for a user for a room.
120
121		Args:
122		user_id(str): The user to get the account_data for.
123		room_id(str): The room to get the account_data for.
124		Returns:
125		A deferred dict of the room account_data
126		"""
127
128		def get_account_data_for_room_txn(txn):
129		rows = self._simple_select_list_txn(
130		txn,
131		"room_account_data",
132		{"user_id": user_id, "room_id": room_id},
133		["account_data_type", "content"],
134		)
135
136		return {
137		row["account_data_type"]: json.loads(row["content"]) for row in rows
138		}
139
140		return self.runInteraction(
141		"get_account_data_for_room", get_account_data_for_room_txn
142		)
143
144		@cached(num_args=3, max_entries=5000)
145		def get_account_data_for_room_and_type(self, user_id, room_id, account_data_type):
146		"""Get the client account_data of given type for a user for a room.
147
148		Args:
149		user_id(str): The user to get the account_data for.
150		room_id(str): The room to get the account_data for.
151		account_data_type (str): The account data type to get.
152		Returns:
153		A deferred of the room account_data for that type, or None if
154		there isn't any set.
155		"""
156
157		def get_account_data_for_room_and_type_txn(txn):
158		content_json = self._simple_select_one_onecol_txn(
159		txn,
160		table="room_account_data",
161		keyvalues={
162		"user_id": user_id,
163		"room_id": room_id,
164		"account_data_type": account_data_type,
165		},
166		retcol="content",
167		allow_none=True,
168		)
169
170		return json.loads(content_json) if content_json else None
171
172		return self.runInteraction(
173		"get_account_data_for_room_and_type", get_account_data_for_room_and_type_txn
174		)
175
176		def get_all_updated_account_data(
177		self, last_global_id, last_room_id, current_id, limit
178		):
179		"""Get all the client account_data that has changed on the server
180		Args:
181		last_global_id(int): The position to fetch from for top level data
182		last_room_id(int): The position to fetch from for per room data
183		current_id(int): The position to fetch up to.
184		Returns:
185		A deferred pair of lists of tuples of stream_id int, user_id string,
186		room_id string, type string, and content string.
187		"""
188		if last_room_id == current_id and last_global_id == current_id:
189		return defer.succeed(([], []))
190
191		def get_updated_account_data_txn(txn):
192		sql = (
193		"SELECT stream_id, user_id, account_data_type, content"
194		" FROM account_data WHERE ? < stream_id AND stream_id <= ?"
195		" ORDER BY stream_id ASC LIMIT ?"
196		)
197		txn.execute(sql, (last_global_id, current_id, limit))
198		global_results = txn.fetchall()
199
200		sql = (
201		"SELECT stream_id, user_id, room_id, account_data_type, content"
202		" FROM room_account_data WHERE ? < stream_id AND stream_id <= ?"
203		" ORDER BY stream_id ASC LIMIT ?"
204		)
205		txn.execute(sql, (last_room_id, current_id, limit))
206		room_results = txn.fetchall()
207		return global_results, room_results
208
209		return self.runInteraction(
210		"get_all_updated_account_data_txn", get_updated_account_data_txn
211		)
212
213		def get_updated_account_data_for_user(self, user_id, stream_id):
214		"""Get all the client account_data for a that's changed for a user
215
216		Args:
217		user_id(str): The user to get the account_data for.
218		stream_id(int): The point in the stream since which to get updates
219		Returns:
220		A deferred pair of a dict of global account_data and a dict
221		mapping from room_id string to per room account_data dicts.
222		"""
223
224		def get_updated_account_data_for_user_txn(txn):
225		sql = (
226		"SELECT account_data_type, content FROM account_data"
227		" WHERE user_id = ? AND stream_id > ?"
228		)
229
230		txn.execute(sql, (user_id, stream_id))
231
232		global_account_data = {row[0]: json.loads(row[1]) for row in txn}
233
234		sql = (
235		"SELECT room_id, account_data_type, content FROM room_account_data"
236		" WHERE user_id = ? AND stream_id > ?"
237		)
238
239		txn.execute(sql, (user_id, stream_id))
240
241		account_data_by_room = {}
242		for row in txn:
243		room_account_data = account_data_by_room.setdefault(row[0], {})
244		room_account_data[row[1]] = json.loads(row[2])
245
246		return global_account_data, account_data_by_room
247
248		changed = self._account_data_stream_cache.has_entity_changed(
249		user_id, int(stream_id)
250		)
251		if not changed:
252		return {}, {}
253
254		return self.runInteraction(
255		"get_updated_account_data_for_user", get_updated_account_data_for_user_txn
256		)
257
258		@cachedInlineCallbacks(num_args=2, cache_context=True, max_entries=5000)
259		def is_ignored_by(self, ignored_user_id, ignorer_user_id, cache_context):
260		ignored_account_data = yield self.get_global_account_data_by_type_for_user(
261		"m.ignored_user_list",
262		ignorer_user_id,
263		on_invalidate=cache_context.invalidate,
264		)
265		if not ignored_account_data:
266		return False
267
268		return ignored_user_id in ignored_account_data.get("ignored_users", {})
269
270
271		class AccountDataStore(AccountDataWorkerStore):
272		def __init__(self, db_conn, hs):
273		self._account_data_id_gen = StreamIdGenerator(
274		db_conn, "account_data_max_stream_id", "stream_id"
275		)
276
277		super(AccountDataStore, self).__init__(db_conn, hs)
278
279		def get_max_account_data_stream_id(self):
280		"""Get the current max stream id for the private user data stream
281
282		Returns:
283		A deferred int.
284		"""
285		return self._account_data_id_gen.get_current_token()
286
287		@defer.inlineCallbacks
288		def add_account_data_to_room(self, user_id, room_id, account_data_type, content):
289		"""Add some account_data to a room for a user.
290		Args:
291		user_id(str): The user to add a tag for.
292		room_id(str): The room to add a tag for.
293		account_data_type(str): The type of account_data to add.
294		content(dict): A json object to associate with the tag.
295		Returns:
296		A deferred that completes once the account_data has been added.
297		"""
298		content_json = json.dumps(content)
299
300		with self._account_data_id_gen.get_next() as next_id:
301		# no need to lock here as room_account_data has a unique constraint
302		# on (user_id, room_id, account_data_type) so _simple_upsert will
303		# retry if there is a conflict.
304		yield self._simple_upsert(
305		desc="add_room_account_data",
306		table="room_account_data",
307		keyvalues={
308		"user_id": user_id,
309		"room_id": room_id,
310		"account_data_type": account_data_type,
311		},
312		values={"stream_id": next_id, "content": content_json},
313		lock=False,
314		)
315
316		# it's theoretically possible for the above to succeed and the
317		# below to fail - in which case we might reuse a stream id on
318		# restart, and the above update might not get propagated. That
319		# doesn't sound any worse than the whole update getting lost,
320		# which is what would happen if we combined the two into one
321		# transaction.
322		yield self._update_max_stream_id(next_id)
323
324		self._account_data_stream_cache.entity_has_changed(user_id, next_id)
325		self.get_account_data_for_user.invalidate((user_id,))
326		self.get_account_data_for_room.invalidate((user_id, room_id))
327		self.get_account_data_for_room_and_type.prefill(
328		(user_id, room_id, account_data_type), content
329		)
330
331		result = self._account_data_id_gen.get_current_token()
332		return result
333
334		@defer.inlineCallbacks
335		def add_account_data_for_user(self, user_id, account_data_type, content):
336		"""Add some account_data to a room for a user.
337		Args:
338		user_id(str): The user to add a tag for.
339		account_data_type(str): The type of account_data to add.
340		content(dict): A json object to associate with the tag.
341		Returns:
342		A deferred that completes once the account_data has been added.
343		"""
344		content_json = json.dumps(content)
345
346		with self._account_data_id_gen.get_next() as next_id:
347		# no need to lock here as account_data has a unique constraint on
348		# (user_id, account_data_type) so _simple_upsert will retry if
349		# there is a conflict.
350		yield self._simple_upsert(
351		desc="add_user_account_data",
352		table="account_data",
353		keyvalues={"user_id": user_id, "account_data_type": account_data_type},
354		values={"stream_id": next_id, "content": content_json},
355		lock=False,
356		)
357
358		# it's theoretically possible for the above to succeed and the
359		# below to fail - in which case we might reuse a stream id on
360		# restart, and the above update might not get propagated. That
361		# doesn't sound any worse than the whole update getting lost,
362		# which is what would happen if we combined the two into one
363		# transaction.
364		yield self._update_max_stream_id(next_id)
365
366		self._account_data_stream_cache.entity_has_changed(user_id, next_id)
367		self.get_account_data_for_user.invalidate((user_id,))
368		self.get_global_account_data_by_type_for_user.invalidate(
369		(account_data_type, user_id)
370		)
371
372		result = self._account_data_id_gen.get_current_token()
373		return result
374
375		def _update_max_stream_id(self, next_id):
376		"""Update the max stream_id
377
378		Args:
379		next_id(int): The the revision to advance to.
380		"""
381
382		def _update(txn):
383		update_max_id_sql = (
384		"UPDATE account_data_max_stream_id"
385		" SET stream_id = ?"
386		" WHERE stream_id < ?"
387		)
388		txn.execute(update_max_id_sql, (next_id, next_id))
389
390		return self.runInteraction("update_account_data_max_stream_id", _update)

+0

-368

~~synapse/storage/appservice.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2015, 2016 OpenMarket Ltd
2		# Copyright 2018 New Vector Ltd
3		#
4		# Licensed under the Apache License, Version 2.0 (the "License");
5		# you may not use this file except in compliance with the License.
6		# You may obtain a copy of the License at
7		#
8		# http://www.apache.org/licenses/LICENSE-2.0
9		#
10		# Unless required by applicable law or agreed to in writing, software
11		# distributed under the License is distributed on an "AS IS" BASIS,
12		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13		# See the License for the specific language governing permissions and
14		# limitations under the License.
15		import logging
16		import re
17
18		from canonicaljson import json
19
20		from twisted.internet import defer
21
22		from synapse.appservice import AppServiceTransaction
23		from synapse.config.appservice import load_appservices
24		from synapse.storage.events_worker import EventsWorkerStore
25
26		from ._base import SQLBaseStore
27
28		logger = logging.getLogger(__name__)
29
30
31		def _make_exclusive_regex(services_cache):
32		# We precompie a regex constructed from all the regexes that the AS's
33		# have registered for exclusive users.
34		exclusive_user_regexes = [
35		regex.pattern
36		for service in services_cache
37		for regex in service.get_exlusive_user_regexes()
38		]
39		if exclusive_user_regexes:
40		exclusive_user_regex = "\|".join("(" + r + ")" for r in exclusive_user_regexes)
41		exclusive_user_regex = re.compile(exclusive_user_regex)
42		else:
43		# We handle this case specially otherwise the constructed regex
44		# will always match
45		exclusive_user_regex = None
46
47		return exclusive_user_regex
48
49
50		class ApplicationServiceWorkerStore(SQLBaseStore):
51		def __init__(self, db_conn, hs):
52		self.services_cache = load_appservices(
53		hs.hostname, hs.config.app_service_config_files
54		)
55		self.exclusive_user_regex = _make_exclusive_regex(self.services_cache)
56
57		super(ApplicationServiceWorkerStore, self).__init__(db_conn, hs)
58
59		def get_app_services(self):
60		return self.services_cache
61
62		def get_if_app_services_interested_in_user(self, user_id):
63		"""Check if the user is one associated with an app service (exclusively)
64		"""
65		if self.exclusive_user_regex:
66		return bool(self.exclusive_user_regex.match(user_id))
67		else:
68		return False
69
70		def get_app_service_by_user_id(self, user_id):
71		"""Retrieve an application service from their user ID.
72
73		All application services have associated with them a particular user ID.
74		There is no distinguishing feature on the user ID which indicates it
75		represents an application service. This function allows you to map from
76		a user ID to an application service.
77
78		Args:
79		user_id(str): The user ID to see if it is an application service.
80		Returns:
81		synapse.appservice.ApplicationService or None.
82		"""
83		for service in self.services_cache:
84		if service.sender == user_id:
85		return service
86		return None
87
88		def get_app_service_by_token(self, token):
89		"""Get the application service with the given appservice token.
90
91		Args:
92		token (str): The application service token.
93		Returns:
94		synapse.appservice.ApplicationService or None.
95		"""
96		for service in self.services_cache:
97		if service.token == token:
98		return service
99		return None
100
101		def get_app_service_by_id(self, as_id):
102		"""Get the application service with the given appservice ID.
103
104		Args:
105		as_id (str): The application service ID.
106		Returns:
107		synapse.appservice.ApplicationService or None.
108		"""
109		for service in self.services_cache:
110		if service.id == as_id:
111		return service
112		return None
113
114
115		class ApplicationServiceStore(ApplicationServiceWorkerStore):
116		# This is currently empty due to there not being any AS storage functions
117		# that can't be run on the workers. Since this may change in future, and
118		# to keep consistency with the other stores, we keep this empty class for
119		# now.
120		pass
121
122
123		class ApplicationServiceTransactionWorkerStore(
124		ApplicationServiceWorkerStore, EventsWorkerStore
125		):
126		@defer.inlineCallbacks
127		def get_appservices_by_state(self, state):
128		"""Get a list of application services based on their state.
129
130		Args:
131		state(ApplicationServiceState): The state to filter on.
132		Returns:
133		A Deferred which resolves to a list of ApplicationServices, which
134		may be empty.
135		"""
136		results = yield self._simple_select_list(
137		"application_services_state", dict(state=state), ["as_id"]
138		)
139		# NB: This assumes this class is linked with ApplicationServiceStore
140		as_list = self.get_app_services()
141		services = []
142
143		for res in results:
144		for service in as_list:
145		if service.id == res["as_id"]:
146		services.append(service)
147		return services
148
149		@defer.inlineCallbacks
150		def get_appservice_state(self, service):
151		"""Get the application service state.
152
153		Args:
154		service(ApplicationService): The service whose state to set.
155		Returns:
156		A Deferred which resolves to ApplicationServiceState.
157		"""
158		result = yield self._simple_select_one(
159		"application_services_state",
160		dict(as_id=service.id),
161		["state"],
162		allow_none=True,
163		desc="get_appservice_state",
164		)
165		if result:
166		return result.get("state")
167		return None
168
169		def set_appservice_state(self, service, state):
170		"""Set the application service state.
171
172		Args:
173		service(ApplicationService): The service whose state to set.
174		state(ApplicationServiceState): The connectivity state to apply.
175		Returns:
176		A Deferred which resolves when the state was set successfully.
177		"""
178		return self._simple_upsert(
179		"application_services_state", dict(as_id=service.id), dict(state=state)
180		)
181
182		def create_appservice_txn(self, service, events):
183		"""Atomically creates a new transaction for this application service
184		with the given list of events.
185
186		Args:
187		service(ApplicationService): The service who the transaction is for.
188		events(list<Event>): A list of events to put in the transaction.
189		Returns:
190		AppServiceTransaction: A new transaction.
191		"""
192
193		def _create_appservice_txn(txn):
194		# work out new txn id (highest txn id for this service += 1)
195		# The highest id may be the last one sent (in which case it is last_txn)
196		# or it may be the highest in the txns list (which are waiting to be/are
197		# being sent)
198		last_txn_id = self._get_last_txn(txn, service.id)
199
200		txn.execute(
201		"SELECT MAX(txn_id) FROM application_services_txns WHERE as_id=?",
202		(service.id,),
203		)
204		highest_txn_id = txn.fetchone()[0]
205		if highest_txn_id is None:
206		highest_txn_id = 0
207
208		new_txn_id = max(highest_txn_id, last_txn_id) + 1
209
210		# Insert new txn into txn table
211		event_ids = json.dumps([e.event_id for e in events])
212		txn.execute(
213		"INSERT INTO application_services_txns(as_id, txn_id, event_ids) "
214		"VALUES(?,?,?)",
215		(service.id, new_txn_id, event_ids),
216		)
217		return AppServiceTransaction(service=service, id=new_txn_id, events=events)
218
219		return self.runInteraction("create_appservice_txn", _create_appservice_txn)
220
221		def complete_appservice_txn(self, txn_id, service):
222		"""Completes an application service transaction.
223
224		Args:
225		txn_id(str): The transaction ID being completed.
226		service(ApplicationService): The application service which was sent
227		this transaction.
228		Returns:
229		A Deferred which resolves if this transaction was stored
230		successfully.
231		"""
232		txn_id = int(txn_id)
233
234		def _complete_appservice_txn(txn):
235		# Debugging query: Make sure the txn being completed is EXACTLY +1 from
236		# what was there before. If it isn't, we've got problems (e.g. the AS
237		# has probably missed some events), so whine loudly but still continue,
238		# since it shouldn't fail completion of the transaction.
239		last_txn_id = self._get_last_txn(txn, service.id)
240		if (last_txn_id + 1) != txn_id:
241		logger.error(
242		"appservice: Completing a transaction which has an ID > 1 from "
243		"the last ID sent to this AS. We've either dropped events or "
244		"sent it to the AS out of order. FIX ME. last_txn=%s "
245		"completing_txn=%s service_id=%s",
246		last_txn_id,
247		txn_id,
248		service.id,
249		)
250
251		# Set current txn_id for AS to 'txn_id'
252		self._simple_upsert_txn(
253		txn,
254		"application_services_state",
255		dict(as_id=service.id),
256		dict(last_txn=txn_id),
257		)
258
259		# Delete txn
260		self._simple_delete_txn(
261		txn, "application_services_txns", dict(txn_id=txn_id, as_id=service.id)
262		)
263
264		return self.runInteraction("complete_appservice_txn", _complete_appservice_txn)
265
266		@defer.inlineCallbacks
267		def get_oldest_unsent_txn(self, service):
268		"""Get the oldest transaction which has not been sent for this
269		service.
270
271		Args:
272		service(ApplicationService): The app service to get the oldest txn.
273		Returns:
274		A Deferred which resolves to an AppServiceTransaction or
275		None.
276		"""
277
278		def _get_oldest_unsent_txn(txn):
279		# Monotonically increasing txn ids, so just select the smallest
280		# one in the txns table (we delete them when they are sent)
281		txn.execute(
282		"SELECT * FROM application_services_txns WHERE as_id=?"
283		" ORDER BY txn_id ASC LIMIT 1",
284		(service.id,),
285		)
286		rows = self.cursor_to_dict(txn)
287		if not rows:
288		return None
289
290		entry = rows[0]
291
292		return entry
293
294		entry = yield self.runInteraction(
295		"get_oldest_unsent_appservice_txn", _get_oldest_unsent_txn
296		)
297
298		if not entry:
299		return None
300
301		event_ids = json.loads(entry["event_ids"])
302
303		events = yield self.get_events_as_list(event_ids)
304
305		return AppServiceTransaction(service=service, id=entry["txn_id"], events=events)
306
307		def _get_last_txn(self, txn, service_id):
308		txn.execute(
309		"SELECT last_txn FROM application_services_state WHERE as_id=?",
310		(service_id,),
311		)
312		last_txn_id = txn.fetchone()
313		if last_txn_id is None or last_txn_id[0] is None: # no row exists
314		return 0
315		else:
316		return int(last_txn_id[0]) # select 'last_txn' col
317
318		def set_appservice_last_pos(self, pos):
319		def set_appservice_last_pos_txn(txn):
320		txn.execute(
321		"UPDATE appservice_stream_position SET stream_ordering = ?", (pos,)
322		)
323
324		return self.runInteraction(
325		"set_appservice_last_pos", set_appservice_last_pos_txn
326		)
327
328		@defer.inlineCallbacks
329		def get_new_events_for_appservice(self, current_id, limit):
330		"""Get all new evnets"""
331
332		def get_new_events_for_appservice_txn(txn):
333		sql = (
334		"SELECT e.stream_ordering, e.event_id"
335		" FROM events AS e"
336		" WHERE"
337		" (SELECT stream_ordering FROM appservice_stream_position)"
338		" < e.stream_ordering"
339		" AND e.stream_ordering <= ?"
340		" ORDER BY e.stream_ordering ASC"
341		" LIMIT ?"
342		)
343
344		txn.execute(sql, (current_id, limit))
345		rows = txn.fetchall()
346
347		upper_bound = current_id
348		if len(rows) == limit:
349		upper_bound = rows[-1][0]
350
351		return upper_bound, [row[1] for row in rows]
352
353		upper_bound, event_ids = yield self.runInteraction(
354		"get_new_events_for_appservice", get_new_events_for_appservice_txn
355		)
356
357		events = yield self.get_events_as_list(event_ids)
358
359		return upper_bound, events
360
361
362		class ApplicationServiceTransactionStore(ApplicationServiceTransactionWorkerStore):
363		# This is currently empty due to there not being any AS storage functions
364		# that can't be run on the workers. Since this may change in future, and
365		# to keep consistency with the other stores, we keep this empty class for
366		# now.
367		pass

+0

-576

~~synapse/storage/client_ips.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2016 OpenMarket Ltd
2		#
3		# Licensed under the Apache License, Version 2.0 (the "License");
4		# you may not use this file except in compliance with the License.
5		# You may obtain a copy of the License at
6		#
7		# http://www.apache.org/licenses/LICENSE-2.0
8		#
9		# Unless required by applicable law or agreed to in writing, software
10		# distributed under the License is distributed on an "AS IS" BASIS,
11		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12		# See the License for the specific language governing permissions and
13		# limitations under the License.
14
15		import logging
16
17		from six import iteritems
18
19		from twisted.internet import defer
20
21		from synapse.metrics.background_process_metrics import wrap_as_background_process
22		from synapse.util.caches import CACHE_SIZE_FACTOR
23
24		from . import background_updates
25		from ._base import Cache
26
27		logger = logging.getLogger(__name__)
28
29		# Number of msec of granularity to store the user IP 'last seen' time. Smaller
30		# times give more inserts into the database even for readonly API hits
31		# 120 seconds == 2 minutes
32		LAST_SEEN_GRANULARITY = 120 * 1000
33
34
35		class ClientIpStore(background_updates.BackgroundUpdateStore):
36		def __init__(self, db_conn, hs):
37
38		self.client_ip_last_seen = Cache(
39		name="client_ip_last_seen", keylen=4, max_entries=50000 * CACHE_SIZE_FACTOR
40		)
41
42		super(ClientIpStore, self).__init__(db_conn, hs)
43
44		self.user_ips_max_age = hs.config.user_ips_max_age
45
46		self.register_background_index_update(
47		"user_ips_device_index",
48		index_name="user_ips_device_id",
49		table="user_ips",
50		columns=["user_id", "device_id", "last_seen"],
51		)
52
53		self.register_background_index_update(
54		"user_ips_last_seen_index",
55		index_name="user_ips_last_seen",
56		table="user_ips",
57		columns=["user_id", "last_seen"],
58		)
59
60		self.register_background_index_update(
61		"user_ips_last_seen_only_index",
62		index_name="user_ips_last_seen_only",
63		table="user_ips",
64		columns=["last_seen"],
65		)
66
67		self.register_background_update_handler(
68		"user_ips_analyze", self._analyze_user_ip
69		)
70
71		self.register_background_update_handler(
72		"user_ips_remove_dupes", self._remove_user_ip_dupes
73		)
74
75		# Register a unique index
76		self.register_background_index_update(
77		"user_ips_device_unique_index",
78		index_name="user_ips_user_token_ip_unique_index",
79		table="user_ips",
80		columns=["user_id", "access_token", "ip"],
81		unique=True,
82		)
83
84		# Drop the old non-unique index
85		self.register_background_update_handler(
86		"user_ips_drop_nonunique_index", self._remove_user_ip_nonunique
87		)
88
89		# Update the last seen info in devices.
90		self.register_background_update_handler(
91		"devices_last_seen", self._devices_last_seen_update
92		)
93
94		# (user_id, access_token, ip,) -> (user_agent, device_id, last_seen)
95		self._batch_row_update = {}
96
97		self._client_ip_looper = self._clock.looping_call(
98		self._update_client_ips_batch, 5 * 1000
99		)
100		self.hs.get_reactor().addSystemEventTrigger(
101		"before", "shutdown", self._update_client_ips_batch
102		)
103
104		if self.user_ips_max_age:
105		self._clock.looping_call(self._prune_old_user_ips, 5 * 1000)
106
107		@defer.inlineCallbacks
108		def _remove_user_ip_nonunique(self, progress, batch_size):
109		def f(conn):
110		txn = conn.cursor()
111		txn.execute("DROP INDEX IF EXISTS user_ips_user_ip")
112		txn.close()
113
114		yield self.runWithConnection(f)
115		yield self._end_background_update("user_ips_drop_nonunique_index")
116		return 1
117
118		@defer.inlineCallbacks
119		def _analyze_user_ip(self, progress, batch_size):
120		# Background update to analyze user_ips table before we run the
121		# deduplication background update. The table may not have been analyzed
122		# for ages due to the table locks.
123		#
124		# This will lock out the naive upserts to user_ips while it happens, but
125		# the analyze should be quick (28GB table takes ~10s)
126		def user_ips_analyze(txn):
127		txn.execute("ANALYZE user_ips")
128
129		yield self.runInteraction("user_ips_analyze", user_ips_analyze)
130
131		yield self._end_background_update("user_ips_analyze")
132
133		return 1
134
135		@defer.inlineCallbacks
136		def _remove_user_ip_dupes(self, progress, batch_size):
137		# This works function works by scanning the user_ips table in batches
138		# based on `last_seen`. For each row in a batch it searches the rest of
139		# the table to see if there are any duplicates, if there are then they
140		# are removed and replaced with a suitable row.
141
142		# Fetch the start of the batch
143		begin_last_seen = progress.get("last_seen", 0)
144
145		def get_last_seen(txn):
146		txn.execute(
147		"""
148		SELECT last_seen FROM user_ips
149		WHERE last_seen > ?
150		ORDER BY last_seen
151		LIMIT 1
152		OFFSET ?
153		""",
154		(begin_last_seen, batch_size),
155		)
156		row = txn.fetchone()
157		if row:
158		return row[0]
159		else:
160		return None
161
162		# Get a last seen that has roughly `batch_size` since `begin_last_seen`
163		end_last_seen = yield self.runInteraction(
164		"user_ips_dups_get_last_seen", get_last_seen
165		)
166
167		# If it returns None, then we're processing the last batch
168		last = end_last_seen is None
169
170		logger.info(
171		"Scanning for duplicate 'user_ips' rows in range: %s <= last_seen < %s",
172		begin_last_seen,
173		end_last_seen,
174		)
175
176		def remove(txn):
177		# This works by looking at all entries in the given time span, and
178		# then for each (user_id, access_token, ip) tuple in that range
179		# checking for any duplicates in the rest of the table (via a join).
180		# It then only returns entries which have duplicates, and the max
181		# last_seen across all duplicates, which can the be used to delete
182		# all other duplicates.
183		# It is efficient due to the existence of (user_id, access_token,
184		# ip) and (last_seen) indices.
185
186		# Define the search space, which requires handling the last batch in
187		# a different way
188		if last:
189		clause = "? <= last_seen"
190		args = (begin_last_seen,)
191		else:
192		clause = "? <= last_seen AND last_seen < ?"
193		args = (begin_last_seen, end_last_seen)
194
195		# (Note: The DISTINCT in the inner query is important to ensure that
196		# the COUNT(*) is accurate, otherwise double counting may happen due
197		# to the join effectively being a cross product)
198		txn.execute(
199		"""
200		SELECT user_id, access_token, ip,
201		MAX(device_id), MAX(user_agent), MAX(last_seen),
202		COUNT(*)
203		FROM (
204		SELECT DISTINCT user_id, access_token, ip
205		FROM user_ips
206		WHERE {}
207		) c
208		INNER JOIN user_ips USING (user_id, access_token, ip)
209		GROUP BY user_id, access_token, ip
210		HAVING count(*) > 1
211		""".format(
212		clause
213		),
214		args,
215		)
216		res = txn.fetchall()
217
218		# We've got some duplicates
219		for i in res:
220		user_id, access_token, ip, device_id, user_agent, last_seen, count = i
221
222		# We want to delete the duplicates so we end up with only a
223		# single row.
224		#
225		# The naive way of doing this would be just to delete all rows
226		# and reinsert a constructed row. However, if there are a lot of
227		# duplicate rows this can cause the table to grow a lot, which
228		# can be problematic in two ways:
229		# 1. If user_ips is already large then this can cause the
230		# table to rapidly grow, potentially filling the disk.
231		# 2. Reinserting a lot of rows can confuse the table
232		# statistics for postgres, causing it to not use the
233		# correct indices for the query above, resulting in a full
234		# table scan. This is incredibly slow for large tables and
235		# can kill database performance. (This seems to mainly
236		# happen for the last query where the clause is simply `? <
237		# last_seen`)
238		#
239		# So instead we want to delete all but one of the duplicate
240		# rows. That is hard to do reliably, so we cheat and do a two
241		# step process:
242		# 1. Delete all rows with a last_seen strictly less than the
243		# max last_seen. This hopefully results in deleting all but
244		# one row the majority of the time, but there may be
245		# duplicate last_seen
246		# 2. If multiple rows remain, we fall back to the naive method
247		# and simply delete all rows and reinsert.
248		#
249		# Note that this relies on no new duplicate rows being inserted,
250		# but if that is happening then this entire process is futile
251		# anyway.
252
253		# Do step 1:
254
255		txn.execute(
256		"""
257		DELETE FROM user_ips
258		WHERE user_id = ? AND access_token = ? AND ip = ? AND last_seen < ?
259		""",
260		(user_id, access_token, ip, last_seen),
261		)
262		if txn.rowcount == count - 1:
263		# We deleted all but one of the duplicate rows, i.e. there
264		# is exactly one remaining and so there is nothing left to
265		# do.
266		continue
267		elif txn.rowcount >= count:
268		raise Exception(
269		"We deleted more duplicate rows from 'user_ips' than expected"
270		)
271
272		# The previous step didn't delete enough rows, so we fallback to
273		# step 2:
274
275		# Drop all the duplicates
276		txn.execute(
277		"""
278		DELETE FROM user_ips
279		WHERE user_id = ? AND access_token = ? AND ip = ?
280		""",
281		(user_id, access_token, ip),
282		)
283
284		# Add in one to be the last_seen
285		txn.execute(
286		"""
287		INSERT INTO user_ips
288		(user_id, access_token, ip, device_id, user_agent, last_seen)
289		VALUES (?, ?, ?, ?, ?, ?)
290		""",
291		(user_id, access_token, ip, device_id, user_agent, last_seen),
292		)
293
294		self._background_update_progress_txn(
295		txn, "user_ips_remove_dupes", {"last_seen": end_last_seen}
296		)
297
298		yield self.runInteraction("user_ips_dups_remove", remove)
299
300		if last:
301		yield self._end_background_update("user_ips_remove_dupes")
302
303		return batch_size
304
305		@defer.inlineCallbacks
306		def insert_client_ip(
307		self, user_id, access_token, ip, user_agent, device_id, now=None
308		):
309		if not now:
310		now = int(self._clock.time_msec())
311		key = (user_id, access_token, ip)
312
313		try:
314		last_seen = self.client_ip_last_seen.get(key)
315		except KeyError:
316		last_seen = None
317		yield self.populate_monthly_active_users(user_id)
318		# Rate-limited inserts
319		if last_seen is not None and (now - last_seen) < LAST_SEEN_GRANULARITY:
320		return
321
322		self.client_ip_last_seen.prefill(key, now)
323
324		self._batch_row_update[key] = (user_agent, device_id, now)
325
326		@wrap_as_background_process("update_client_ips")
327		def _update_client_ips_batch(self):
328
329		# If the DB pool has already terminated, don't try updating
330		if not self.hs.get_db_pool().running:
331		return
332
333		to_update = self._batch_row_update
334		self._batch_row_update = {}
335
336		return self.runInteraction(
337		"_update_client_ips_batch", self._update_client_ips_batch_txn, to_update
338		)
339
340		def _update_client_ips_batch_txn(self, txn, to_update):
341		if "user_ips" in self._unsafe_to_upsert_tables or (
342		not self.database_engine.can_native_upsert
343		):
344		self.database_engine.lock_table(txn, "user_ips")
345
346		for entry in iteritems(to_update):
347		(user_id, access_token, ip), (user_agent, device_id, last_seen) = entry
348
349		try:
350		self._simple_upsert_txn(
351		txn,
352		table="user_ips",
353		keyvalues={
354		"user_id": user_id,
355		"access_token": access_token,
356		"ip": ip,
357		},
358		values={
359		"user_agent": user_agent,
360		"device_id": device_id,
361		"last_seen": last_seen,
362		},
363		lock=False,
364		)
365
366		# Technically an access token might not be associated with
367		# a device so we need to check.
368		if device_id:
369		self._simple_upsert_txn(
370		txn,
371		table="devices",
372		keyvalues={"user_id": user_id, "device_id": device_id},
373		values={
374		"user_agent": user_agent,
375		"last_seen": last_seen,
376		"ip": ip,
377		},
378		lock=False,
379		)
380		except Exception as e:
381		# Failed to upsert, log and continue
382		logger.error("Failed to insert client IP %r: %r", entry, e)
383
384		@defer.inlineCallbacks
385		def get_last_client_ip_by_device(self, user_id, device_id):
386		"""For each device_id listed, give the user_ip it was last seen on
387
388		Args:
389		user_id (str)
390		device_id (str): If None fetches all devices for the user
391
392		Returns:
393		defer.Deferred: resolves to a dict, where the keys
394		are (user_id, device_id) tuples. The values are also dicts, with
395		keys giving the column names
396		"""
397
398		keyvalues = {"user_id": user_id}
399		if device_id is not None:
400		keyvalues["device_id"] = device_id
401
402		res = yield self._simple_select_list(
403		table="devices",
404		keyvalues=keyvalues,
405		retcols=("user_id", "ip", "user_agent", "device_id", "last_seen"),
406		)
407
408		ret = {(d["user_id"], d["device_id"]): d for d in res}
409		for key in self._batch_row_update:
410		uid, access_token, ip = key
411		if uid == user_id:
412		user_agent, did, last_seen = self._batch_row_update[key]
413		if not device_id or did == device_id:
414		ret[(user_id, device_id)] = {
415		"user_id": user_id,
416		"access_token": access_token,
417		"ip": ip,
418		"user_agent": user_agent,
419		"device_id": did,
420		"last_seen": last_seen,
421		}
422		return ret
423
424		@defer.inlineCallbacks
425		def get_user_ip_and_agents(self, user):
426		user_id = user.to_string()
427		results = {}
428
429		for key in self._batch_row_update:
430		uid, access_token, ip, = key
431		if uid == user_id:
432		user_agent, _, last_seen = self._batch_row_update[key]
433		results[(access_token, ip)] = (user_agent, last_seen)
434
435		rows = yield self._simple_select_list(
436		table="user_ips",
437		keyvalues={"user_id": user_id},
438		retcols=["access_token", "ip", "user_agent", "last_seen"],
439		desc="get_user_ip_and_agents",
440		)
441
442		results.update(
443		((row["access_token"], row["ip"]), (row["user_agent"], row["last_seen"]))
444		for row in rows
445		)
446		return list(
447		{
448		"access_token": access_token,
449		"ip": ip,
450		"user_agent": user_agent,
451		"last_seen": last_seen,
452		}
453		for (access_token, ip), (user_agent, last_seen) in iteritems(results)
454		)
455
456		@defer.inlineCallbacks
457		def _devices_last_seen_update(self, progress, batch_size):
458		"""Background update to insert last seen info into devices table
459		"""
460
461		last_user_id = progress.get("last_user_id", "")
462		last_device_id = progress.get("last_device_id", "")
463
464		def _devices_last_seen_update_txn(txn):
465		# This consists of two queries:
466		#
467		# 1. The sub-query searches for the next N devices and joins
468		# against user_ips to find the max last_seen associated with
469		# that device.
470		# 2. The outer query then joins again against user_ips on
471		# user/device/last_seen. This should hopefully only
472		# return one row, but if it does return more than one then
473		# we'll just end up updating the same device row multiple
474		# times, which is fine.
475
476		if self.database_engine.supports_tuple_comparison:
477		where_clause = "(user_id, device_id) > (?, ?)"
478		where_args = [last_user_id, last_device_id]
479		else:
480		# We explicitly do a `user_id >= ? AND (...)` here to ensure
481		# that an index is used, as doing `user_id > ? OR (user_id = ? AND ...)`
482		# makes it hard for query optimiser to tell that it can use the
483		# index on user_id
484		where_clause = "user_id >= ? AND (user_id > ? OR device_id > ?)"
485		where_args = [last_user_id, last_user_id, last_device_id]
486
487		sql = """
488		SELECT
489		last_seen, ip, user_agent, user_id, device_id
490		FROM (
491		SELECT
492		user_id, device_id, MAX(u.last_seen) AS last_seen
493		FROM devices
494		INNER JOIN user_ips AS u USING (user_id, device_id)
495		WHERE %(where_clause)s
496		GROUP BY user_id, device_id
497		ORDER BY user_id ASC, device_id ASC
498		LIMIT ?
499		) c
500		INNER JOIN user_ips AS u USING (user_id, device_id, last_seen)
501		""" % {
502		"where_clause": where_clause
503		}
504		txn.execute(sql, where_args + [batch_size])
505
506		rows = txn.fetchall()
507		if not rows:
508		return 0
509
510		sql = """
511		UPDATE devices
512		SET last_seen = ?, ip = ?, user_agent = ?
513		WHERE user_id = ? AND device_id = ?
514		"""
515		txn.execute_batch(sql, rows)
516
517		_, _, _, user_id, device_id = rows[-1]
518		self._background_update_progress_txn(
519		txn,
520		"devices_last_seen",
521		{"last_user_id": user_id, "last_device_id": device_id},
522		)
523
524		return len(rows)
525
526		updated = yield self.runInteraction(
527		"_devices_last_seen_update", _devices_last_seen_update_txn
528		)
529
530		if not updated:
531		yield self._end_background_update("devices_last_seen")
532
533		return updated
534
535		@wrap_as_background_process("prune_old_user_ips")
536		async def _prune_old_user_ips(self):
537		"""Removes entries in user IPs older than the configured period.
538		"""
539
540		if self.user_ips_max_age is None:
541		# Nothing to do
542		return
543
544		if not await self.has_completed_background_update("devices_last_seen"):
545		# Only start pruning if we have finished populating the devices
546		# last seen info.
547		return
548
549		# We do a slightly funky SQL delete to ensure we don't try and delete
550		# too much at once (as the table may be very large from before we
551		# started pruning).
552		#
553		# This works by finding the max last_seen that is less than the given
554		# time, but has no more than N rows before it, deleting all rows with
555		# a lesser last_seen time. (We COALESCE so that the sub-SELECT always
556		# returns exactly one row).
557		sql = """
558		DELETE FROM user_ips
559		WHERE last_seen <= (
560		SELECT COALESCE(MAX(last_seen), -1)
561		FROM (
562		SELECT last_seen FROM user_ips
563		WHERE last_seen <= ?
564		ORDER BY last_seen ASC
565		LIMIT 5000
566		) AS u
567		)
568		"""
569
570		timestamp = self.clock.time_msec() - self.user_ips_max_age
571
572		def _prune_old_user_ips_txn(txn):
573		txn.execute(sql, (timestamp,))
574
575		await self.runInteraction("_prune_old_user_ips", _prune_old_user_ips_txn)

+14

-0

synapse/storage/data_stores/__init__.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2019 The Matrix.org Foundation C.I.C.
	2	#
	3	# Licensed under the Apache License, Version 2.0 (the "License");
	4	# you may not use this file except in compliance with the License.
	5	# You may obtain a copy of the License at
	6	#
	7	# http://www.apache.org/licenses/LICENSE-2.0
	8	#
	9	# Unless required by applicable law or agreed to in writing, software
	10	# distributed under the License is distributed on an "AS IS" BASIS,
	11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	# See the License for the specific language governing permissions and
	13	# limitations under the License.

+530

-0

synapse/storage/data_stores/main/__init__.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2014-2016 OpenMarket Ltd
	2	# Copyright 2018 New Vector Ltd
	3	# Copyright 2019 The Matrix.org Foundation C.I.C.
	4	#
	5	# Licensed under the Apache License, Version 2.0 (the "License");
	6	# you may not use this file except in compliance with the License.
	7	# You may obtain a copy of the License at
	8	#
	9	# http://www.apache.org/licenses/LICENSE-2.0
	10	#
	11	# Unless required by applicable law or agreed to in writing, software
	12	# distributed under the License is distributed on an "AS IS" BASIS,
	13	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	14	# See the License for the specific language governing permissions and
	15	# limitations under the License.
	16
	17	import calendar
	18	import logging
	19	import time
	20
	21	from twisted.internet import defer
	22
	23	from synapse.api.constants import PresenceState
	24	from synapse.storage.engines import PostgresEngine
	25	from synapse.storage.util.id_generators import (
	26	ChainedIdGenerator,
	27	IdGenerator,
	28	StreamIdGenerator,
	29	)
	30	from synapse.util.caches.stream_change_cache import StreamChangeCache
	31
	32	from .account_data import AccountDataStore
	33	from .appservice import ApplicationServiceStore, ApplicationServiceTransactionStore
	34	from .client_ips import ClientIpStore
	35	from .deviceinbox import DeviceInboxStore
	36	from .devices import DeviceStore
	37	from .directory import DirectoryStore
	38	from .e2e_room_keys import EndToEndRoomKeyStore
	39	from .end_to_end_keys import EndToEndKeyStore
	40	from .event_federation import EventFederationStore
	41	from .event_push_actions import EventPushActionsStore
	42	from .events import EventsStore
	43	from .events_bg_updates import EventsBackgroundUpdatesStore
	44	from .filtering import FilteringStore
	45	from .group_server import GroupServerStore
	46	from .keys import KeyStore
	47	from .media_repository import MediaRepositoryStore
	48	from .monthly_active_users import MonthlyActiveUsersStore
	49	from .openid import OpenIdStore
	50	from .presence import PresenceStore, UserPresenceState
	51	from .profile import ProfileStore
	52	from .push_rule import PushRuleStore
	53	from .pusher import PusherStore
	54	from .receipts import ReceiptsStore
	55	from .registration import RegistrationStore
	56	from .rejections import RejectionsStore
	57	from .relations import RelationsStore
	58	from .room import RoomStore
	59	from .roommember import RoomMemberStore
	60	from .search import SearchStore
	61	from .signatures import SignatureStore
	62	from .state import StateStore
	63	from .stats import StatsStore
	64	from .stream import StreamStore
	65	from .tags import TagsStore
	66	from .transactions import TransactionStore
	67	from .user_directory import UserDirectoryStore
	68	from .user_erasure_store import UserErasureStore
	69
	70	logger = logging.getLogger(__name__)
	71
	72
	73	class DataStore(
	74	EventsBackgroundUpdatesStore,
	75	RoomMemberStore,
	76	RoomStore,
	77	RegistrationStore,
	78	StreamStore,
	79	ProfileStore,
	80	PresenceStore,
	81	TransactionStore,
	82	DirectoryStore,
	83	KeyStore,
	84	StateStore,
	85	SignatureStore,
	86	ApplicationServiceStore,
	87	EventsStore,
	88	EventFederationStore,
	89	MediaRepositoryStore,
	90	RejectionsStore,
	91	FilteringStore,
	92	PusherStore,
	93	PushRuleStore,
	94	ApplicationServiceTransactionStore,
	95	ReceiptsStore,
	96	EndToEndKeyStore,
	97	EndToEndRoomKeyStore,
	98	SearchStore,
	99	TagsStore,
	100	AccountDataStore,
	101	EventPushActionsStore,
	102	OpenIdStore,
	103	ClientIpStore,
	104	DeviceStore,
	105	DeviceInboxStore,
	106	UserDirectoryStore,
	107	GroupServerStore,
	108	UserErasureStore,
	109	MonthlyActiveUsersStore,
	110	StatsStore,
	111	RelationsStore,
	112	):
	113	def __init__(self, db_conn, hs):
	114	self.hs = hs
	115	self._clock = hs.get_clock()
	116	self.database_engine = hs.database_engine
	117
	118	self._stream_id_gen = StreamIdGenerator(
	119	db_conn,
	120	"events",
	121	"stream_ordering",
	122	extra_tables=[("local_invites", "stream_id")],
	123	)
	124	self._backfill_id_gen = StreamIdGenerator(
	125	db_conn,
	126	"events",
	127	"stream_ordering",
	128	step=-1,
	129	extra_tables=[("ex_outlier_stream", "event_stream_ordering")],
	130	)
	131	self._presence_id_gen = StreamIdGenerator(
	132	db_conn, "presence_stream", "stream_id"
	133	)
	134	self._device_inbox_id_gen = StreamIdGenerator(
	135	db_conn, "device_max_stream_id", "stream_id"
	136	)
	137	self._public_room_id_gen = StreamIdGenerator(
	138	db_conn, "public_room_list_stream", "stream_id"
	139	)
	140	self._device_list_id_gen = StreamIdGenerator(
	141	db_conn, "device_lists_stream", "stream_id"
	142	)
	143	self._cross_signing_id_gen = StreamIdGenerator(
	144	db_conn, "e2e_cross_signing_keys", "stream_id"
	145	)
	146
	147	self._access_tokens_id_gen = IdGenerator(db_conn, "access_tokens", "id")
	148	self._event_reports_id_gen = IdGenerator(db_conn, "event_reports", "id")
	149	self._push_rule_id_gen = IdGenerator(db_conn, "push_rules", "id")
	150	self._push_rules_enable_id_gen = IdGenerator(db_conn, "push_rules_enable", "id")
	151	self._push_rules_stream_id_gen = ChainedIdGenerator(
	152	self._stream_id_gen, db_conn, "push_rules_stream", "stream_id"
	153	)
	154	self._pushers_id_gen = StreamIdGenerator(
	155	db_conn, "pushers", "id", extra_tables=[("deleted_pushers", "stream_id")]
	156	)
	157	self._group_updates_id_gen = StreamIdGenerator(
	158	db_conn, "local_group_updates", "stream_id"
	159	)
	160
	161	if isinstance(self.database_engine, PostgresEngine):
	162	self._cache_id_gen = StreamIdGenerator(
	163	db_conn, "cache_invalidation_stream", "stream_id"
	164	)
	165	else:
	166	self._cache_id_gen = None
	167
	168	self._presence_on_startup = self._get_active_presence(db_conn)
	169
	170	presence_cache_prefill, min_presence_val = self._get_cache_dict(
	171	db_conn,
	172	"presence_stream",
	173	entity_column="user_id",
	174	stream_column="stream_id",
	175	max_value=self._presence_id_gen.get_current_token(),
	176	)
	177	self.presence_stream_cache = StreamChangeCache(
	178	"PresenceStreamChangeCache",
	179	min_presence_val,
	180	prefilled_cache=presence_cache_prefill,
	181	)
	182
	183	max_device_inbox_id = self._device_inbox_id_gen.get_current_token()
	184	device_inbox_prefill, min_device_inbox_id = self._get_cache_dict(
	185	db_conn,
	186	"device_inbox",
	187	entity_column="user_id",
	188	stream_column="stream_id",
	189	max_value=max_device_inbox_id,
	190	limit=1000,
	191	)
	192	self._device_inbox_stream_cache = StreamChangeCache(
	193	"DeviceInboxStreamChangeCache",
	194	min_device_inbox_id,
	195	prefilled_cache=device_inbox_prefill,
	196	)
	197	# The federation outbox and the local device inbox uses the same
	198	# stream_id generator.
	199	device_outbox_prefill, min_device_outbox_id = self._get_cache_dict(
	200	db_conn,
	201	"device_federation_outbox",
	202	entity_column="destination",
	203	stream_column="stream_id",
	204	max_value=max_device_inbox_id,
	205	limit=1000,
	206	)
	207	self._device_federation_outbox_stream_cache = StreamChangeCache(
	208	"DeviceFederationOutboxStreamChangeCache",
	209	min_device_outbox_id,
	210	prefilled_cache=device_outbox_prefill,
	211	)
	212
	213	device_list_max = self._device_list_id_gen.get_current_token()
	214	self._device_list_stream_cache = StreamChangeCache(
	215	"DeviceListStreamChangeCache", device_list_max
	216	)
	217	self._user_signature_stream_cache = StreamChangeCache(
	218	"UserSignatureStreamChangeCache", device_list_max
	219	)
	220	self._device_list_federation_stream_cache = StreamChangeCache(
	221	"DeviceListFederationStreamChangeCache", device_list_max
	222	)
	223
	224	events_max = self._stream_id_gen.get_current_token()
	225	curr_state_delta_prefill, min_curr_state_delta_id = self._get_cache_dict(
	226	db_conn,
	227	"current_state_delta_stream",
	228	entity_column="room_id",
	229	stream_column="stream_id",
	230	max_value=events_max, # As we share the stream id with events token
	231	limit=1000,
	232	)
	233	self._curr_state_delta_stream_cache = StreamChangeCache(
	234	"_curr_state_delta_stream_cache",
	235	min_curr_state_delta_id,
	236	prefilled_cache=curr_state_delta_prefill,
	237	)
	238
	239	_group_updates_prefill, min_group_updates_id = self._get_cache_dict(
	240	db_conn,
	241	"local_group_updates",
	242	entity_column="user_id",
	243	stream_column="stream_id",
	244	max_value=self._group_updates_id_gen.get_current_token(),
	245	limit=1000,
	246	)
	247	self._group_updates_stream_cache = StreamChangeCache(
	248	"_group_updates_stream_cache",
	249	min_group_updates_id,
	250	prefilled_cache=_group_updates_prefill,
	251	)
	252
	253	self._stream_order_on_start = self.get_room_max_stream_ordering()
	254	self._min_stream_order_on_start = self.get_room_min_stream_ordering()
	255
	256	# Used in _generate_user_daily_visits to keep track of progress
	257	self._last_user_visit_update = self._get_start_of_day()
	258
	259	super(DataStore, self).__init__(db_conn, hs)
	260
	261	def take_presence_startup_info(self):
	262	active_on_startup = self._presence_on_startup
	263	self._presence_on_startup = None
	264	return active_on_startup
	265
	266	def _get_active_presence(self, db_conn):
	267	"""Fetch non-offline presence from the database so that we can register
	268	the appropriate time outs.
	269	"""
	270
	271	sql = (
	272	"SELECT user_id, state, last_active_ts, last_federation_update_ts,"
	273	" last_user_sync_ts, status_msg, currently_active FROM presence_stream"
	274	" WHERE state != ?"
	275	)
	276	sql = self.database_engine.convert_param_style(sql)
	277
	278	txn = db_conn.cursor()
	279	txn.execute(sql, (PresenceState.OFFLINE,))
	280	rows = self.cursor_to_dict(txn)
	281	txn.close()
	282
	283	for row in rows:
	284	row["currently_active"] = bool(row["currently_active"])
	285
	286	return [UserPresenceState(**row) for row in rows]
	287
	288	def count_daily_users(self):
	289	"""
	290	Counts the number of users who used this homeserver in the last 24 hours.
	291	"""
	292	yesterday = int(self._clock.time_msec()) - (1000 * 60 * 60 * 24)
	293	return self.runInteraction("count_daily_users", self._count_users, yesterday)
	294
	295	def count_monthly_users(self):
	296	"""
	297	Counts the number of users who used this homeserver in the last 30 days.
	298	Note this method is intended for phonehome metrics only and is different
	299	from the mau figure in synapse.storage.monthly_active_users which,
	300	amongst other things, includes a 3 day grace period before a user counts.
	301	"""
	302	thirty_days_ago = int(self._clock.time_msec()) - (1000 * 60 * 60 * 24 * 30)
	303	return self.runInteraction(
	304	"count_monthly_users", self._count_users, thirty_days_ago
	305	)
	306
	307	def _count_users(self, txn, time_from):
	308	"""
	309	Returns number of users seen in the past time_from period
	310	"""
	311	sql = """
	312	SELECT COALESCE(count(*), 0) FROM (
	313	SELECT user_id FROM user_ips
	314	WHERE last_seen > ?
	315	GROUP BY user_id
	316	) u
	317	"""
	318	txn.execute(sql, (time_from,))
	319	count, = txn.fetchone()
	320	return count
	321
	322	def count_r30_users(self):
	323	"""
	324	Counts the number of 30 day retained users, defined as:-
	325	* Users who have created their accounts more than 30 days ago
	326	* Where last seen at most 30 days ago
	327	* Where account creation and last_seen are > 30 days apart
	328
	329	Returns counts globaly for a given user as well as breaking
	330	by platform
	331	"""
	332
	333	def _count_r30_users(txn):
	334	thirty_days_in_secs = 86400 * 30
	335	now = int(self._clock.time())
	336	thirty_days_ago_in_secs = now - thirty_days_in_secs
	337
	338	sql = """
	339	SELECT platform, COALESCE(count(*), 0) FROM (
	340	SELECT
	341	users.name, platform, users.creation_ts * 1000,
	342	MAX(uip.last_seen)
	343	FROM users
	344	INNER JOIN (
	345	SELECT
	346	user_id,
	347	last_seen,
	348	CASE
	349	WHEN user_agent LIKE '%%Android%%' THEN 'android'
	350	WHEN user_agent LIKE '%%iOS%%' THEN 'ios'
	351	WHEN user_agent LIKE '%%Electron%%' THEN 'electron'
	352	WHEN user_agent LIKE '%%Mozilla%%' THEN 'web'
	353	WHEN user_agent LIKE '%%Gecko%%' THEN 'web'
	354	ELSE 'unknown'
	355	END
	356	AS platform
	357	FROM user_ips
	358	) uip
	359	ON users.name = uip.user_id
	360	AND users.appservice_id is NULL
	361	AND users.creation_ts < ?
	362	AND uip.last_seen/1000 > ?
	363	AND (uip.last_seen/1000) - users.creation_ts > 86400 * 30
	364	GROUP BY users.name, platform, users.creation_ts
	365	) u GROUP BY platform
	366	"""
	367
	368	results = {}
	369	txn.execute(sql, (thirty_days_ago_in_secs, thirty_days_ago_in_secs))
	370
	371	for row in txn:
	372	if row[0] == "unknown":
	373	pass
	374	results[row[0]] = row[1]
	375
	376	sql = """
	377	SELECT COALESCE(count(*), 0) FROM (
	378	SELECT users.name, users.creation_ts * 1000,
	379	MAX(uip.last_seen)
	380	FROM users
	381	INNER JOIN (
	382	SELECT
	383	user_id,
	384	last_seen
	385	FROM user_ips
	386	) uip
	387	ON users.name = uip.user_id
	388	AND appservice_id is NULL
	389	AND users.creation_ts < ?
	390	AND uip.last_seen/1000 > ?
	391	AND (uip.last_seen/1000) - users.creation_ts > 86400 * 30
	392	GROUP BY users.name, users.creation_ts
	393	) u
	394	"""
	395
	396	txn.execute(sql, (thirty_days_ago_in_secs, thirty_days_ago_in_secs))
	397
	398	count, = txn.fetchone()
	399	results["all"] = count
	400
	401	return results
	402
	403	return self.runInteraction("count_r30_users", _count_r30_users)
	404
	405	def _get_start_of_day(self):
	406	"""
	407	Returns millisecond unixtime for start of UTC day.
	408	"""
	409	now = time.gmtime()
	410	today_start = calendar.timegm((now.tm_year, now.tm_mon, now.tm_mday, 0, 0, 0))
	411	return today_start * 1000
	412
	413	def generate_user_daily_visits(self):
	414	"""
	415	Generates daily visit data for use in cohort/ retention analysis
	416	"""
	417
	418	def _generate_user_daily_visits(txn):
	419	logger.info("Calling _generate_user_daily_visits")
	420	today_start = self._get_start_of_day()
	421	a_day_in_milliseconds = 24 * 60 * 60 * 1000
	422	now = self.clock.time_msec()
	423
	424	sql = """
	425	INSERT INTO user_daily_visits (user_id, device_id, timestamp)
	426	SELECT u.user_id, u.device_id, ?
	427	FROM user_ips AS u
	428	LEFT JOIN (
	429	SELECT user_id, device_id, timestamp FROM user_daily_visits
	430	WHERE timestamp = ?
	431	) udv
	432	ON u.user_id = udv.user_id AND u.device_id=udv.device_id
	433	INNER JOIN users ON users.name=u.user_id
	434	WHERE last_seen > ? AND last_seen <= ?
	435	AND udv.timestamp IS NULL AND users.is_guest=0
	436	AND users.appservice_id IS NULL
	437	GROUP BY u.user_id, u.device_id
	438	"""
	439
	440	# This means that the day has rolled over but there could still
	441	# be entries from the previous day. There is an edge case
	442	# where if the user logs in at 23:59 and overwrites their
	443	# last_seen at 00:01 then they will not be counted in the
	444	# previous day's stats - it is important that the query is run
	445	# often to minimise this case.
	446	if today_start > self._last_user_visit_update:
	447	yesterday_start = today_start - a_day_in_milliseconds
	448	txn.execute(
	449	sql,
	450	(
	451	yesterday_start,
	452	yesterday_start,
	453	self._last_user_visit_update,
	454	today_start,
	455	),
	456	)
	457	self._last_user_visit_update = today_start
	458
	459	txn.execute(
	460	sql, (today_start, today_start, self._last_user_visit_update, now)
	461	)
	462	# Update _last_user_visit_update to now. The reason to do this
	463	# rather just clamping to the beginning of the day is to limit
	464	# the size of the join - meaning that the query can be run more
	465	# frequently
	466	self._last_user_visit_update = now
	467
	468	return self.runInteraction(
	469	"generate_user_daily_visits", _generate_user_daily_visits
	470	)
	471
	472	def get_users(self):
	473	"""Function to reterive a list of users in users table.
	474
	475	Args:
	476	Returns:
	477	defer.Deferred: resolves to list[dict[str, Any]]
	478	"""
	479	return self._simple_select_list(
	480	table="users",
	481	keyvalues={},
	482	retcols=["name", "password_hash", "is_guest", "admin", "user_type"],
	483	desc="get_users",
	484	)
	485
	486	@defer.inlineCallbacks
	487	def get_users_paginate(self, order, start, limit):
	488	"""Function to reterive a paginated list of users from
	489	users list. This will return a json object, which contains
	490	list of users and the total number of users in users table.
	491
	492	Args:
	493	order (str): column name to order the select by this column
	494	start (int): start number to begin the query from
	495	limit (int): number of rows to reterive
	496	Returns:
	497	defer.Deferred: resolves to json object {list[dict[str, Any]], count}
	498	"""
	499	users = yield self.runInteraction(
	500	"get_users_paginate",
	501	self._simple_select_list_paginate_txn,
	502	table="users",
	503	keyvalues={"is_guest": False},
	504	orderby=order,
	505	start=start,
	506	limit=limit,
	507	retcols=["name", "password_hash", "is_guest", "admin", "user_type"],
	508	)
	509	count = yield self.runInteraction("get_users_paginate", self.get_user_count_txn)
	510	retval = {"users": users, "total": count}
	511	return retval
	512
	513	def search_users(self, term):
	514	"""Function to search users list for one or more users with
	515	the matched term.
	516
	517	Args:
	518	term (str): search term
	519	col (str): column to query term should be matched to
	520	Returns:
	521	defer.Deferred: resolves to list[dict[str, Any]]
	522	"""
	523	return self._simple_search_list(
	524	table="users",
	525	term=term,
	526	col="name",
	527	retcols=["name", "password_hash", "is_guest", "admin", "user_type"],
	528	desc="search_users",
	529	)

+391

-0

synapse/storage/data_stores/main/account_data.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2014-2016 OpenMarket Ltd
	2	# Copyright 2018 New Vector Ltd
	3	#
	4	# Licensed under the Apache License, Version 2.0 (the "License");
	5	# you may not use this file except in compliance with the License.
	6	# You may obtain a copy of the License at
	7	#
	8	# http://www.apache.org/licenses/LICENSE-2.0
	9	#
	10	# Unless required by applicable law or agreed to in writing, software
	11	# distributed under the License is distributed on an "AS IS" BASIS,
	12	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	# See the License for the specific language governing permissions and
	14	# limitations under the License.
	15
	16	import abc
	17	import logging
	18
	19	from canonicaljson import json
	20
	21	from twisted.internet import defer
	22
	23	from synapse.storage._base import SQLBaseStore
	24	from synapse.storage.util.id_generators import StreamIdGenerator
	25	from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
	26	from synapse.util.caches.stream_change_cache import StreamChangeCache
	27
	28	logger = logging.getLogger(__name__)
	29
	30
	31	class AccountDataWorkerStore(SQLBaseStore):
	32	"""This is an abstract base class where subclasses must implement
	33	`get_max_account_data_stream_id` which can be called in the initializer.
	34	"""
	35
	36	# This ABCMeta metaclass ensures that we cannot be instantiated without
	37	# the abstract methods being implemented.
	38	__metaclass__ = abc.ABCMeta
	39
	40	def __init__(self, db_conn, hs):
	41	account_max = self.get_max_account_data_stream_id()
	42	self._account_data_stream_cache = StreamChangeCache(
	43	"AccountDataAndTagsChangeCache", account_max
	44	)
	45
	46	super(AccountDataWorkerStore, self).__init__(db_conn, hs)
	47
	48	@abc.abstractmethod
	49	def get_max_account_data_stream_id(self):
	50	"""Get the current max stream ID for account data stream
	51
	52	Returns:
	53	int
	54	"""
	55	raise NotImplementedError()
	56
	57	@cached()
	58	def get_account_data_for_user(self, user_id):
	59	"""Get all the client account_data for a user.
	60
	61	Args:
	62	user_id(str): The user to get the account_data for.
	63	Returns:
	64	A deferred pair of a dict of global account_data and a dict
	65	mapping from room_id string to per room account_data dicts.
	66	"""
	67
	68	def get_account_data_for_user_txn(txn):
	69	rows = self._simple_select_list_txn(
	70	txn,
	71	"account_data",
	72	{"user_id": user_id},
	73	["account_data_type", "content"],
	74	)
	75
	76	global_account_data = {
	77	row["account_data_type"]: json.loads(row["content"]) for row in rows
	78	}
	79
	80	rows = self._simple_select_list_txn(
	81	txn,
	82	"room_account_data",
	83	{"user_id": user_id},
	84	["room_id", "account_data_type", "content"],
	85	)
	86
	87	by_room = {}
	88	for row in rows:
	89	room_data = by_room.setdefault(row["room_id"], {})
	90	room_data[row["account_data_type"]] = json.loads(row["content"])
	91
	92	return global_account_data, by_room
	93
	94	return self.runInteraction(
	95	"get_account_data_for_user", get_account_data_for_user_txn
	96	)
	97
	98	@cachedInlineCallbacks(num_args=2, max_entries=5000)
	99	def get_global_account_data_by_type_for_user(self, data_type, user_id):
	100	"""
	101	Returns:
	102	Deferred: A dict
	103	"""
	104	result = yield self._simple_select_one_onecol(
	105	table="account_data",
	106	keyvalues={"user_id": user_id, "account_data_type": data_type},
	107	retcol="content",
	108	desc="get_global_account_data_by_type_for_user",
	109	allow_none=True,
	110	)
	111
	112	if result:
	113	return json.loads(result)
	114	else:
	115	return None
	116
	117	@cached(num_args=2)
	118	def get_account_data_for_room(self, user_id, room_id):
	119	"""Get all the client account_data for a user for a room.
	120
	121	Args:
	122	user_id(str): The user to get the account_data for.
	123	room_id(str): The room to get the account_data for.
	124	Returns:
	125	A deferred dict of the room account_data
	126	"""
	127
	128	def get_account_data_for_room_txn(txn):
	129	rows = self._simple_select_list_txn(
	130	txn,
	131	"room_account_data",
	132	{"user_id": user_id, "room_id": room_id},
	133	["account_data_type", "content"],
	134	)
	135
	136	return {
	137	row["account_data_type"]: json.loads(row["content"]) for row in rows
	138	}
	139
	140	return self.runInteraction(
	141	"get_account_data_for_room", get_account_data_for_room_txn
	142	)
	143
	144	@cached(num_args=3, max_entries=5000)
	145	def get_account_data_for_room_and_type(self, user_id, room_id, account_data_type):
	146	"""Get the client account_data of given type for a user for a room.
	147
	148	Args:
	149	user_id(str): The user to get the account_data for.
	150	room_id(str): The room to get the account_data for.
	151	account_data_type (str): The account data type to get.
	152	Returns:
	153	A deferred of the room account_data for that type, or None if
	154	there isn't any set.
	155	"""
	156
	157	def get_account_data_for_room_and_type_txn(txn):
	158	content_json = self._simple_select_one_onecol_txn(
	159	txn,
	160	table="room_account_data",
	161	keyvalues={
	162	"user_id": user_id,
	163	"room_id": room_id,
	164	"account_data_type": account_data_type,
	165	},
	166	retcol="content",
	167	allow_none=True,
	168	)
	169
	170	return json.loads(content_json) if content_json else None
	171
	172	return self.runInteraction(
	173	"get_account_data_for_room_and_type", get_account_data_for_room_and_type_txn
	174	)
	175
	176	def get_all_updated_account_data(
	177	self, last_global_id, last_room_id, current_id, limit
	178	):
	179	"""Get all the client account_data that has changed on the server
	180	Args:
	181	last_global_id(int): The position to fetch from for top level data
	182	last_room_id(int): The position to fetch from for per room data
	183	current_id(int): The position to fetch up to.
	184	Returns:
	185	A deferred pair of lists of tuples of stream_id int, user_id string,
	186	room_id string, type string, and content string.
	187	"""
	188	if last_room_id == current_id and last_global_id == current_id:
	189	return defer.succeed(([], []))
	190
	191	def get_updated_account_data_txn(txn):
	192	sql = (
	193	"SELECT stream_id, user_id, account_data_type, content"
	194	" FROM account_data WHERE ? < stream_id AND stream_id <= ?"
	195	" ORDER BY stream_id ASC LIMIT ?"
	196	)
	197	txn.execute(sql, (last_global_id, current_id, limit))
	198	global_results = txn.fetchall()
	199
	200	sql = (
	201	"SELECT stream_id, user_id, room_id, account_data_type, content"
	202	" FROM room_account_data WHERE ? < stream_id AND stream_id <= ?"
	203	" ORDER BY stream_id ASC LIMIT ?"
	204	)
	205	txn.execute(sql, (last_room_id, current_id, limit))
	206	room_results = txn.fetchall()
	207	return global_results, room_results
	208
	209	return self.runInteraction(
	210	"get_all_updated_account_data_txn", get_updated_account_data_txn
	211	)
	212
	213	def get_updated_account_data_for_user(self, user_id, stream_id):
	214	"""Get all the client account_data for a that's changed for a user
	215
	216	Args:
	217	user_id(str): The user to get the account_data for.
	218	stream_id(int): The point in the stream since which to get updates
	219	Returns:
	220	A deferred pair of a dict of global account_data and a dict
	221	mapping from room_id string to per room account_data dicts.
	222	"""
	223
	224	def get_updated_account_data_for_user_txn(txn):
	225	sql = (
	226	"SELECT account_data_type, content FROM account_data"
	227	" WHERE user_id = ? AND stream_id > ?"
	228	)
	229
	230	txn.execute(sql, (user_id, stream_id))
	231
	232	global_account_data = {row[0]: json.loads(row[1]) for row in txn}
	233
	234	sql = (
	235	"SELECT room_id, account_data_type, content FROM room_account_data"
	236	" WHERE user_id = ? AND stream_id > ?"
	237	)
	238
	239	txn.execute(sql, (user_id, stream_id))
	240
	241	account_data_by_room = {}
	242	for row in txn:
	243	room_account_data = account_data_by_room.setdefault(row[0], {})
	244	room_account_data[row[1]] = json.loads(row[2])
	245
	246	return global_account_data, account_data_by_room
	247
	248	changed = self._account_data_stream_cache.has_entity_changed(
	249	user_id, int(stream_id)
	250	)
	251	if not changed:
	252	return {}, {}
	253
	254	return self.runInteraction(
	255	"get_updated_account_data_for_user", get_updated_account_data_for_user_txn
	256	)
	257
	258	@cachedInlineCallbacks(num_args=2, cache_context=True, max_entries=5000)
	259	def is_ignored_by(self, ignored_user_id, ignorer_user_id, cache_context):
	260	ignored_account_data = yield self.get_global_account_data_by_type_for_user(
	261	"m.ignored_user_list",
	262	ignorer_user_id,
	263	on_invalidate=cache_context.invalidate,
	264	)
	265	if not ignored_account_data:
	266	return False
	267
	268	return ignored_user_id in ignored_account_data.get("ignored_users", {})
	269
	270
	271	class AccountDataStore(AccountDataWorkerStore):
	272	def __init__(self, db_conn, hs):
	273	self._account_data_id_gen = StreamIdGenerator(
	274	db_conn, "account_data_max_stream_id", "stream_id"
	275	)
	276
	277	super(AccountDataStore, self).__init__(db_conn, hs)
	278
	279	def get_max_account_data_stream_id(self):
	280	"""Get the current max stream id for the private user data stream
	281
	282	Returns:
	283	A deferred int.
	284	"""
	285	return self._account_data_id_gen.get_current_token()
	286
	287	@defer.inlineCallbacks
	288	def add_account_data_to_room(self, user_id, room_id, account_data_type, content):
	289	"""Add some account_data to a room for a user.
	290	Args:
	291	user_id(str): The user to add a tag for.
	292	room_id(str): The room to add a tag for.
	293	account_data_type(str): The type of account_data to add.
	294	content(dict): A json object to associate with the tag.
	295	Returns:
	296	A deferred that completes once the account_data has been added.
	297	"""
	298	content_json = json.dumps(content)
	299
	300	with self._account_data_id_gen.get_next() as next_id:
	301	# no need to lock here as room_account_data has a unique constraint
	302	# on (user_id, room_id, account_data_type) so _simple_upsert will
	303	# retry if there is a conflict.
	304	yield self._simple_upsert(
	305	desc="add_room_account_data",
	306	table="room_account_data",
	307	keyvalues={
	308	"user_id": user_id,
	309	"room_id": room_id,
	310	"account_data_type": account_data_type,
	311	},
	312	values={"stream_id": next_id, "content": content_json},
	313	lock=False,
	314	)
	315
	316	# it's theoretically possible for the above to succeed and the
	317	# below to fail - in which case we might reuse a stream id on
	318	# restart, and the above update might not get propagated. That
	319	# doesn't sound any worse than the whole update getting lost,
	320	# which is what would happen if we combined the two into one
	321	# transaction.
	322	yield self._update_max_stream_id(next_id)
	323
	324	self._account_data_stream_cache.entity_has_changed(user_id, next_id)
	325	self.get_account_data_for_user.invalidate((user_id,))
	326	self.get_account_data_for_room.invalidate((user_id, room_id))
	327	self.get_account_data_for_room_and_type.prefill(
	328	(user_id, room_id, account_data_type), content
	329	)
	330
	331	result = self._account_data_id_gen.get_current_token()
	332	return result
	333
	334	@defer.inlineCallbacks
	335	def add_account_data_for_user(self, user_id, account_data_type, content):
	336	"""Add some account_data to a room for a user.
	337	Args:
	338	user_id(str): The user to add a tag for.
	339	account_data_type(str): The type of account_data to add.
	340	content(dict): A json object to associate with the tag.
	341	Returns:
	342	A deferred that completes once the account_data has been added.
	343	"""
	344	content_json = json.dumps(content)
	345
	346	with self._account_data_id_gen.get_next() as next_id:
	347	# no need to lock here as account_data has a unique constraint on
	348	# (user_id, account_data_type) so _simple_upsert will retry if
	349	# there is a conflict.
	350	yield self._simple_upsert(
	351	desc="add_user_account_data",
	352	table="account_data",
	353	keyvalues={"user_id": user_id, "account_data_type": account_data_type},
	354	values={"stream_id": next_id, "content": content_json},
	355	lock=False,
	356	)
	357
	358	# it's theoretically possible for the above to succeed and the
	359	# below to fail - in which case we might reuse a stream id on
	360	# restart, and the above update might not get propagated. That
	361	# doesn't sound any worse than the whole update getting lost,
	362	# which is what would happen if we combined the two into one
	363	# transaction.
	364	yield self._update_max_stream_id(next_id)
	365
	366	self._account_data_stream_cache.entity_has_changed(user_id, next_id)
	367	self.get_account_data_for_user.invalidate((user_id,))
	368	self.get_global_account_data_by_type_for_user.invalidate(
	369	(account_data_type, user_id)
	370	)
	371
	372	result = self._account_data_id_gen.get_current_token()
	373	return result
	374
	375	def _update_max_stream_id(self, next_id):
	376	"""Update the max stream_id
	377
	378	Args:
	379	next_id(int): The the revision to advance to.
	380	"""
	381
	382	def _update(txn):
	383	update_max_id_sql = (
	384	"UPDATE account_data_max_stream_id"
	385	" SET stream_id = ?"
	386	" WHERE stream_id < ?"
	387	)
	388	txn.execute(update_max_id_sql, (next_id, next_id))
	389
	390	return self.runInteraction("update_account_data_max_stream_id", _update)

+367

-0

synapse/storage/data_stores/main/appservice.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2015, 2016 OpenMarket Ltd
	2	# Copyright 2018 New Vector Ltd
	3	#
	4	# Licensed under the Apache License, Version 2.0 (the "License");
	5	# you may not use this file except in compliance with the License.
	6	# You may obtain a copy of the License at
	7	#
	8	# http://www.apache.org/licenses/LICENSE-2.0
	9	#
	10	# Unless required by applicable law or agreed to in writing, software
	11	# distributed under the License is distributed on an "AS IS" BASIS,
	12	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	# See the License for the specific language governing permissions and
	14	# limitations under the License.
	15	import logging
	16	import re
	17
	18	from canonicaljson import json
	19
	20	from twisted.internet import defer
	21
	22	from synapse.appservice import AppServiceTransaction
	23	from synapse.config.appservice import load_appservices
	24	from synapse.storage._base import SQLBaseStore
	25	from synapse.storage.data_stores.main.events_worker import EventsWorkerStore
	26
	27	logger = logging.getLogger(__name__)
	28
	29
	30	def _make_exclusive_regex(services_cache):
	31	# We precompie a regex constructed from all the regexes that the AS's
	32	# have registered for exclusive users.
	33	exclusive_user_regexes = [
	34	regex.pattern
	35	for service in services_cache
	36	for regex in service.get_exlusive_user_regexes()
	37	]
	38	if exclusive_user_regexes:
	39	exclusive_user_regex = "\|".join("(" + r + ")" for r in exclusive_user_regexes)
	40	exclusive_user_regex = re.compile(exclusive_user_regex)
	41	else:
	42	# We handle this case specially otherwise the constructed regex
	43	# will always match
	44	exclusive_user_regex = None
	45
	46	return exclusive_user_regex
	47
	48
	49	class ApplicationServiceWorkerStore(SQLBaseStore):
	50	def __init__(self, db_conn, hs):
	51	self.services_cache = load_appservices(
	52	hs.hostname, hs.config.app_service_config_files
	53	)
	54	self.exclusive_user_regex = _make_exclusive_regex(self.services_cache)
	55
	56	super(ApplicationServiceWorkerStore, self).__init__(db_conn, hs)
	57
	58	def get_app_services(self):
	59	return self.services_cache
	60
	61	def get_if_app_services_interested_in_user(self, user_id):
	62	"""Check if the user is one associated with an app service (exclusively)
	63	"""
	64	if self.exclusive_user_regex:
	65	return bool(self.exclusive_user_regex.match(user_id))
	66	else:
	67	return False
	68
	69	def get_app_service_by_user_id(self, user_id):
	70	"""Retrieve an application service from their user ID.
	71
	72	All application services have associated with them a particular user ID.
	73	There is no distinguishing feature on the user ID which indicates it
	74	represents an application service. This function allows you to map from
	75	a user ID to an application service.
	76
	77	Args:
	78	user_id(str): The user ID to see if it is an application service.
	79	Returns:
	80	synapse.appservice.ApplicationService or None.
	81	"""
	82	for service in self.services_cache:
	83	if service.sender == user_id:
	84	return service
	85	return None
	86
	87	def get_app_service_by_token(self, token):
	88	"""Get the application service with the given appservice token.
	89
	90	Args:
	91	token (str): The application service token.
	92	Returns:
	93	synapse.appservice.ApplicationService or None.
	94	"""
	95	for service in self.services_cache:
	96	if service.token == token:
	97	return service
	98	return None
	99
	100	def get_app_service_by_id(self, as_id):
	101	"""Get the application service with the given appservice ID.
	102
	103	Args:
	104	as_id (str): The application service ID.
	105	Returns:
	106	synapse.appservice.ApplicationService or None.
	107	"""
	108	for service in self.services_cache:
	109	if service.id == as_id:
	110	return service
	111	return None
	112
	113
	114	class ApplicationServiceStore(ApplicationServiceWorkerStore):
	115	# This is currently empty due to there not being any AS storage functions
	116	# that can't be run on the workers. Since this may change in future, and
	117	# to keep consistency with the other stores, we keep this empty class for
	118	# now.
	119	pass
	120
	121
	122	class ApplicationServiceTransactionWorkerStore(
	123	ApplicationServiceWorkerStore, EventsWorkerStore
	124	):
	125	@defer.inlineCallbacks
	126	def get_appservices_by_state(self, state):
	127	"""Get a list of application services based on their state.
	128
	129	Args:
	130	state(ApplicationServiceState): The state to filter on.
	131	Returns:
	132	A Deferred which resolves to a list of ApplicationServices, which
	133	may be empty.
	134	"""
	135	results = yield self._simple_select_list(
	136	"application_services_state", dict(state=state), ["as_id"]
	137	)
	138	# NB: This assumes this class is linked with ApplicationServiceStore
	139	as_list = self.get_app_services()
	140	services = []
	141
	142	for res in results:
	143	for service in as_list:
	144	if service.id == res["as_id"]:
	145	services.append(service)
	146	return services
	147
	148	@defer.inlineCallbacks
	149	def get_appservice_state(self, service):
	150	"""Get the application service state.
	151
	152	Args:
	153	service(ApplicationService): The service whose state to set.
	154	Returns:
	155	A Deferred which resolves to ApplicationServiceState.
	156	"""
	157	result = yield self._simple_select_one(
	158	"application_services_state",
	159	dict(as_id=service.id),
	160	["state"],
	161	allow_none=True,
	162	desc="get_appservice_state",
	163	)
	164	if result:
	165	return result.get("state")
	166	return None
	167
	168	def set_appservice_state(self, service, state):
	169	"""Set the application service state.
	170
	171	Args:
	172	service(ApplicationService): The service whose state to set.
	173	state(ApplicationServiceState): The connectivity state to apply.
	174	Returns:
	175	A Deferred which resolves when the state was set successfully.
	176	"""
	177	return self._simple_upsert(
	178	"application_services_state", dict(as_id=service.id), dict(state=state)
	179	)
	180
	181	def create_appservice_txn(self, service, events):
	182	"""Atomically creates a new transaction for this application service
	183	with the given list of events.
	184
	185	Args:
	186	service(ApplicationService): The service who the transaction is for.
	187	events(list<Event>): A list of events to put in the transaction.
	188	Returns:
	189	AppServiceTransaction: A new transaction.
	190	"""
	191
	192	def _create_appservice_txn(txn):
	193	# work out new txn id (highest txn id for this service += 1)
	194	# The highest id may be the last one sent (in which case it is last_txn)
	195	# or it may be the highest in the txns list (which are waiting to be/are
	196	# being sent)
	197	last_txn_id = self._get_last_txn(txn, service.id)
	198
	199	txn.execute(
	200	"SELECT MAX(txn_id) FROM application_services_txns WHERE as_id=?",
	201	(service.id,),
	202	)
	203	highest_txn_id = txn.fetchone()[0]
	204	if highest_txn_id is None:
	205	highest_txn_id = 0
	206
	207	new_txn_id = max(highest_txn_id, last_txn_id) + 1
	208
	209	# Insert new txn into txn table
	210	event_ids = json.dumps([e.event_id for e in events])
	211	txn.execute(
	212	"INSERT INTO application_services_txns(as_id, txn_id, event_ids) "
	213	"VALUES(?,?,?)",
	214	(service.id, new_txn_id, event_ids),
	215	)
	216	return AppServiceTransaction(service=service, id=new_txn_id, events=events)
	217
	218	return self.runInteraction("create_appservice_txn", _create_appservice_txn)
	219
	220	def complete_appservice_txn(self, txn_id, service):
	221	"""Completes an application service transaction.
	222
	223	Args:
	224	txn_id(str): The transaction ID being completed.
	225	service(ApplicationService): The application service which was sent
	226	this transaction.
	227	Returns:
	228	A Deferred which resolves if this transaction was stored
	229	successfully.
	230	"""
	231	txn_id = int(txn_id)
	232
	233	def _complete_appservice_txn(txn):
	234	# Debugging query: Make sure the txn being completed is EXACTLY +1 from
	235	# what was there before. If it isn't, we've got problems (e.g. the AS
	236	# has probably missed some events), so whine loudly but still continue,
	237	# since it shouldn't fail completion of the transaction.
	238	last_txn_id = self._get_last_txn(txn, service.id)
	239	if (last_txn_id + 1) != txn_id:
	240	logger.error(
	241	"appservice: Completing a transaction which has an ID > 1 from "
	242	"the last ID sent to this AS. We've either dropped events or "
	243	"sent it to the AS out of order. FIX ME. last_txn=%s "
	244	"completing_txn=%s service_id=%s",
	245	last_txn_id,
	246	txn_id,
	247	service.id,
	248	)
	249
	250	# Set current txn_id for AS to 'txn_id'
	251	self._simple_upsert_txn(
	252	txn,
	253	"application_services_state",
	254	dict(as_id=service.id),
	255	dict(last_txn=txn_id),
	256	)
	257
	258	# Delete txn
	259	self._simple_delete_txn(
	260	txn, "application_services_txns", dict(txn_id=txn_id, as_id=service.id)
	261	)
	262
	263	return self.runInteraction("complete_appservice_txn", _complete_appservice_txn)
	264
	265	@defer.inlineCallbacks
	266	def get_oldest_unsent_txn(self, service):
	267	"""Get the oldest transaction which has not been sent for this
	268	service.
	269
	270	Args:
	271	service(ApplicationService): The app service to get the oldest txn.
	272	Returns:
	273	A Deferred which resolves to an AppServiceTransaction or
	274	None.
	275	"""
	276
	277	def _get_oldest_unsent_txn(txn):
	278	# Monotonically increasing txn ids, so just select the smallest
	279	# one in the txns table (we delete them when they are sent)
	280	txn.execute(
	281	"SELECT * FROM application_services_txns WHERE as_id=?"
	282	" ORDER BY txn_id ASC LIMIT 1",
	283	(service.id,),
	284	)
	285	rows = self.cursor_to_dict(txn)
	286	if not rows:
	287	return None
	288
	289	entry = rows[0]
	290
	291	return entry
	292
	293	entry = yield self.runInteraction(
	294	"get_oldest_unsent_appservice_txn", _get_oldest_unsent_txn
	295	)
	296
	297	if not entry:
	298	return None
	299
	300	event_ids = json.loads(entry["event_ids"])
	301
	302	events = yield self.get_events_as_list(event_ids)
	303
	304	return AppServiceTransaction(service=service, id=entry["txn_id"], events=events)
	305
	306	def _get_last_txn(self, txn, service_id):
	307	txn.execute(
	308	"SELECT last_txn FROM application_services_state WHERE as_id=?",
	309	(service_id,),
	310	)
	311	last_txn_id = txn.fetchone()
	312	if last_txn_id is None or last_txn_id[0] is None: # no row exists
	313	return 0
	314	else:
	315	return int(last_txn_id[0]) # select 'last_txn' col
	316
	317	def set_appservice_last_pos(self, pos):
	318	def set_appservice_last_pos_txn(txn):
	319	txn.execute(
	320	"UPDATE appservice_stream_position SET stream_ordering = ?", (pos,)
	321	)
	322
	323	return self.runInteraction(
	324	"set_appservice_last_pos", set_appservice_last_pos_txn
	325	)
	326
	327	@defer.inlineCallbacks
	328	def get_new_events_for_appservice(self, current_id, limit):
	329	"""Get all new evnets"""
	330
	331	def get_new_events_for_appservice_txn(txn):
	332	sql = (
	333	"SELECT e.stream_ordering, e.event_id"
	334	" FROM events AS e"
	335	" WHERE"
	336	" (SELECT stream_ordering FROM appservice_stream_position)"
	337	" < e.stream_ordering"
	338	" AND e.stream_ordering <= ?"
	339	" ORDER BY e.stream_ordering ASC"
	340	" LIMIT ?"
	341	)
	342
	343	txn.execute(sql, (current_id, limit))
	344	rows = txn.fetchall()
	345
	346	upper_bound = current_id
	347	if len(rows) == limit:
	348	upper_bound = rows[-1][0]
	349
	350	return upper_bound, [row[1] for row in rows]
	351
	352	upper_bound, event_ids = yield self.runInteraction(
	353	"get_new_events_for_appservice", get_new_events_for_appservice_txn
	354	)
	355
	356	events = yield self.get_events_as_list(event_ids)
	357
	358	return upper_bound, events
	359
	360
	361	class ApplicationServiceTransactionStore(ApplicationServiceTransactionWorkerStore):
	362	# This is currently empty due to there not being any AS storage functions
	363	# that can't be run on the workers. Since this may change in future, and
	364	# to keep consistency with the other stores, we keep this empty class for
	365	# now.
	366	pass

+580

-0

synapse/storage/data_stores/main/client_ips.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2016 OpenMarket Ltd
	2	#
	3	# Licensed under the Apache License, Version 2.0 (the "License");
	4	# you may not use this file except in compliance with the License.
	5	# You may obtain a copy of the License at
	6	#
	7	# http://www.apache.org/licenses/LICENSE-2.0
	8	#
	9	# Unless required by applicable law or agreed to in writing, software
	10	# distributed under the License is distributed on an "AS IS" BASIS,
	11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	# See the License for the specific language governing permissions and
	13	# limitations under the License.
	14
	15	import logging
	16
	17	from six import iteritems
	18
	19	from twisted.internet import defer
	20
	21	from synapse.metrics.background_process_metrics import wrap_as_background_process
	22	from synapse.storage import background_updates
	23	from synapse.storage._base import Cache
	24	from synapse.util.caches import CACHE_SIZE_FACTOR
	25
	26	logger = logging.getLogger(__name__)
	27
	28	# Number of msec of granularity to store the user IP 'last seen' time. Smaller
	29	# times give more inserts into the database even for readonly API hits
	30	# 120 seconds == 2 minutes
	31	LAST_SEEN_GRANULARITY = 120 * 1000
	32
	33
	34	class ClientIpBackgroundUpdateStore(background_updates.BackgroundUpdateStore):
	35	def __init__(self, db_conn, hs):
	36	super(ClientIpBackgroundUpdateStore, self).__init__(db_conn, hs)
	37
	38	self.register_background_index_update(
	39	"user_ips_device_index",
	40	index_name="user_ips_device_id",
	41	table="user_ips",
	42	columns=["user_id", "device_id", "last_seen"],
	43	)
	44
	45	self.register_background_index_update(
	46	"user_ips_last_seen_index",
	47	index_name="user_ips_last_seen",
	48	table="user_ips",
	49	columns=["user_id", "last_seen"],
	50	)
	51
	52	self.register_background_index_update(
	53	"user_ips_last_seen_only_index",
	54	index_name="user_ips_last_seen_only",
	55	table="user_ips",
	56	columns=["last_seen"],
	57	)
	58
	59	self.register_background_update_handler(
	60	"user_ips_analyze", self._analyze_user_ip
	61	)
	62
	63	self.register_background_update_handler(
	64	"user_ips_remove_dupes", self._remove_user_ip_dupes
	65	)
	66
	67	# Register a unique index
	68	self.register_background_index_update(
	69	"user_ips_device_unique_index",
	70	index_name="user_ips_user_token_ip_unique_index",
	71	table="user_ips",
	72	columns=["user_id", "access_token", "ip"],
	73	unique=True,
	74	)
	75
	76	# Drop the old non-unique index
	77	self.register_background_update_handler(
	78	"user_ips_drop_nonunique_index", self._remove_user_ip_nonunique
	79	)
	80
	81	# Update the last seen info in devices.
	82	self.register_background_update_handler(
	83	"devices_last_seen", self._devices_last_seen_update
	84	)
	85
	86	@defer.inlineCallbacks
	87	def _remove_user_ip_nonunique(self, progress, batch_size):
	88	def f(conn):
	89	txn = conn.cursor()
	90	txn.execute("DROP INDEX IF EXISTS user_ips_user_ip")
	91	txn.close()
	92
	93	yield self.runWithConnection(f)
	94	yield self._end_background_update("user_ips_drop_nonunique_index")
	95	return 1
	96
	97	@defer.inlineCallbacks
	98	def _analyze_user_ip(self, progress, batch_size):
	99	# Background update to analyze user_ips table before we run the
	100	# deduplication background update. The table may not have been analyzed
	101	# for ages due to the table locks.
	102	#
	103	# This will lock out the naive upserts to user_ips while it happens, but
	104	# the analyze should be quick (28GB table takes ~10s)
	105	def user_ips_analyze(txn):
	106	txn.execute("ANALYZE user_ips")
	107
	108	yield self.runInteraction("user_ips_analyze", user_ips_analyze)
	109
	110	yield self._end_background_update("user_ips_analyze")
	111
	112	return 1
	113
	114	@defer.inlineCallbacks
	115	def _remove_user_ip_dupes(self, progress, batch_size):
	116	# This works function works by scanning the user_ips table in batches
	117	# based on `last_seen`. For each row in a batch it searches the rest of
	118	# the table to see if there are any duplicates, if there are then they
	119	# are removed and replaced with a suitable row.
	120
	121	# Fetch the start of the batch
	122	begin_last_seen = progress.get("last_seen", 0)
	123
	124	def get_last_seen(txn):
	125	txn.execute(
	126	"""
	127	SELECT last_seen FROM user_ips
	128	WHERE last_seen > ?
	129	ORDER BY last_seen
	130	LIMIT 1
	131	OFFSET ?
	132	""",
	133	(begin_last_seen, batch_size),
	134	)
	135	row = txn.fetchone()
	136	if row:
	137	return row[0]
	138	else:
	139	return None
	140
	141	# Get a last seen that has roughly `batch_size` since `begin_last_seen`
	142	end_last_seen = yield self.runInteraction(
	143	"user_ips_dups_get_last_seen", get_last_seen
	144	)
	145
	146	# If it returns None, then we're processing the last batch
	147	last = end_last_seen is None
	148
	149	logger.info(
	150	"Scanning for duplicate 'user_ips' rows in range: %s <= last_seen < %s",
	151	begin_last_seen,
	152	end_last_seen,
	153	)
	154
	155	def remove(txn):
	156	# This works by looking at all entries in the given time span, and
	157	# then for each (user_id, access_token, ip) tuple in that range
	158	# checking for any duplicates in the rest of the table (via a join).
	159	# It then only returns entries which have duplicates, and the max
	160	# last_seen across all duplicates, which can the be used to delete
	161	# all other duplicates.
	162	# It is efficient due to the existence of (user_id, access_token,
	163	# ip) and (last_seen) indices.
	164
	165	# Define the search space, which requires handling the last batch in
	166	# a different way
	167	if last:
	168	clause = "? <= last_seen"
	169	args = (begin_last_seen,)
	170	else:
	171	clause = "? <= last_seen AND last_seen < ?"
	172	args = (begin_last_seen, end_last_seen)
	173
	174	# (Note: The DISTINCT in the inner query is important to ensure that
	175	# the COUNT(*) is accurate, otherwise double counting may happen due
	176	# to the join effectively being a cross product)
	177	txn.execute(
	178	"""
	179	SELECT user_id, access_token, ip,
	180	MAX(device_id), MAX(user_agent), MAX(last_seen),
	181	COUNT(*)
	182	FROM (
	183	SELECT DISTINCT user_id, access_token, ip
	184	FROM user_ips
	185	WHERE {}
	186	) c
	187	INNER JOIN user_ips USING (user_id, access_token, ip)
	188	GROUP BY user_id, access_token, ip
	189	HAVING count(*) > 1
	190	""".format(
	191	clause
	192	),
	193	args,
	194	)
	195	res = txn.fetchall()
	196
	197	# We've got some duplicates
	198	for i in res:
	199	user_id, access_token, ip, device_id, user_agent, last_seen, count = i
	200
	201	# We want to delete the duplicates so we end up with only a
	202	# single row.
	203	#
	204	# The naive way of doing this would be just to delete all rows
	205	# and reinsert a constructed row. However, if there are a lot of
	206	# duplicate rows this can cause the table to grow a lot, which
	207	# can be problematic in two ways:
	208	# 1. If user_ips is already large then this can cause the
	209	# table to rapidly grow, potentially filling the disk.
	210	# 2. Reinserting a lot of rows can confuse the table
	211	# statistics for postgres, causing it to not use the
	212	# correct indices for the query above, resulting in a full
	213	# table scan. This is incredibly slow for large tables and
	214	# can kill database performance. (This seems to mainly
	215	# happen for the last query where the clause is simply `? <
	216	# last_seen`)
	217	#
	218	# So instead we want to delete all but one of the duplicate
	219	# rows. That is hard to do reliably, so we cheat and do a two
	220	# step process:
	221	# 1. Delete all rows with a last_seen strictly less than the
	222	# max last_seen. This hopefully results in deleting all but
	223	# one row the majority of the time, but there may be
	224	# duplicate last_seen
	225	# 2. If multiple rows remain, we fall back to the naive method
	226	# and simply delete all rows and reinsert.
	227	#
	228	# Note that this relies on no new duplicate rows being inserted,
	229	# but if that is happening then this entire process is futile
	230	# anyway.
	231
	232	# Do step 1:
	233
	234	txn.execute(
	235	"""
	236	DELETE FROM user_ips
	237	WHERE user_id = ? AND access_token = ? AND ip = ? AND last_seen < ?
	238	""",
	239	(user_id, access_token, ip, last_seen),
	240	)
	241	if txn.rowcount == count - 1:
	242	# We deleted all but one of the duplicate rows, i.e. there
	243	# is exactly one remaining and so there is nothing left to
	244	# do.
	245	continue
	246	elif txn.rowcount >= count:
	247	raise Exception(
	248	"We deleted more duplicate rows from 'user_ips' than expected"
	249	)
	250
	251	# The previous step didn't delete enough rows, so we fallback to
	252	# step 2:
	253
	254	# Drop all the duplicates
	255	txn.execute(
	256	"""
	257	DELETE FROM user_ips
	258	WHERE user_id = ? AND access_token = ? AND ip = ?
	259	""",
	260	(user_id, access_token, ip),
	261	)
	262
	263	# Add in one to be the last_seen
	264	txn.execute(
	265	"""
	266	INSERT INTO user_ips
	267	(user_id, access_token, ip, device_id, user_agent, last_seen)
	268	VALUES (?, ?, ?, ?, ?, ?)
	269	""",
	270	(user_id, access_token, ip, device_id, user_agent, last_seen),
	271	)
	272
	273	self._background_update_progress_txn(
	274	txn, "user_ips_remove_dupes", {"last_seen": end_last_seen}
	275	)
	276
	277	yield self.runInteraction("user_ips_dups_remove", remove)
	278
	279	if last:
	280	yield self._end_background_update("user_ips_remove_dupes")
	281
	282	return batch_size
	283
	284	@defer.inlineCallbacks
	285	def _devices_last_seen_update(self, progress, batch_size):
	286	"""Background update to insert last seen info into devices table
	287	"""
	288
	289	last_user_id = progress.get("last_user_id", "")
	290	last_device_id = progress.get("last_device_id", "")
	291
	292	def _devices_last_seen_update_txn(txn):
	293	# This consists of two queries:
	294	#
	295	# 1. The sub-query searches for the next N devices and joins
	296	# against user_ips to find the max last_seen associated with
	297	# that device.
	298	# 2. The outer query then joins again against user_ips on
	299	# user/device/last_seen. This should hopefully only
	300	# return one row, but if it does return more than one then
	301	# we'll just end up updating the same device row multiple
	302	# times, which is fine.
	303
	304	if self.database_engine.supports_tuple_comparison:
	305	where_clause = "(user_id, device_id) > (?, ?)"
	306	where_args = [last_user_id, last_device_id]
	307	else:
	308	# We explicitly do a `user_id >= ? AND (...)` here to ensure
	309	# that an index is used, as doing `user_id > ? OR (user_id = ? AND ...)`
	310	# makes it hard for query optimiser to tell that it can use the
	311	# index on user_id
	312	where_clause = "user_id >= ? AND (user_id > ? OR device_id > ?)"
	313	where_args = [last_user_id, last_user_id, last_device_id]
	314
	315	sql = """
	316	SELECT
	317	last_seen, ip, user_agent, user_id, device_id
	318	FROM (
	319	SELECT
	320	user_id, device_id, MAX(u.last_seen) AS last_seen
	321	FROM devices
	322	INNER JOIN user_ips AS u USING (user_id, device_id)
	323	WHERE %(where_clause)s
	324	GROUP BY user_id, device_id
	325	ORDER BY user_id ASC, device_id ASC
	326	LIMIT ?
	327	) c
	328	INNER JOIN user_ips AS u USING (user_id, device_id, last_seen)
	329	""" % {
	330	"where_clause": where_clause
	331	}
	332	txn.execute(sql, where_args + [batch_size])
	333
	334	rows = txn.fetchall()
	335	if not rows:
	336	return 0
	337
	338	sql = """
	339	UPDATE devices
	340	SET last_seen = ?, ip = ?, user_agent = ?
	341	WHERE user_id = ? AND device_id = ?
	342	"""
	343	txn.execute_batch(sql, rows)
	344
	345	_, _, _, user_id, device_id = rows[-1]
	346	self._background_update_progress_txn(
	347	txn,
	348	"devices_last_seen",
	349	{"last_user_id": user_id, "last_device_id": device_id},
	350	)
	351
	352	return len(rows)
	353
	354	updated = yield self.runInteraction(
	355	"_devices_last_seen_update", _devices_last_seen_update_txn
	356	)
	357
	358	if not updated:
	359	yield self._end_background_update("devices_last_seen")
	360
	361	return updated
	362
	363
	364	class ClientIpStore(ClientIpBackgroundUpdateStore):
	365	def __init__(self, db_conn, hs):
	366
	367	self.client_ip_last_seen = Cache(
	368	name="client_ip_last_seen", keylen=4, max_entries=50000 * CACHE_SIZE_FACTOR
	369	)
	370
	371	super(ClientIpStore, self).__init__(db_conn, hs)
	372
	373	self.user_ips_max_age = hs.config.user_ips_max_age
	374
	375	# (user_id, access_token, ip,) -> (user_agent, device_id, last_seen)
	376	self._batch_row_update = {}
	377
	378	self._client_ip_looper = self._clock.looping_call(
	379	self._update_client_ips_batch, 5 * 1000
	380	)
	381	self.hs.get_reactor().addSystemEventTrigger(
	382	"before", "shutdown", self._update_client_ips_batch
	383	)
	384
	385	if self.user_ips_max_age:
	386	self._clock.looping_call(self._prune_old_user_ips, 5 * 1000)
	387
	388	@defer.inlineCallbacks
	389	def insert_client_ip(
	390	self, user_id, access_token, ip, user_agent, device_id, now=None
	391	):
	392	if not now:
	393	now = int(self._clock.time_msec())
	394	key = (user_id, access_token, ip)
	395
	396	try:
	397	last_seen = self.client_ip_last_seen.get(key)
	398	except KeyError:
	399	last_seen = None
	400	yield self.populate_monthly_active_users(user_id)
	401	# Rate-limited inserts
	402	if last_seen is not None and (now - last_seen) < LAST_SEEN_GRANULARITY:
	403	return
	404
	405	self.client_ip_last_seen.prefill(key, now)
	406
	407	self._batch_row_update[key] = (user_agent, device_id, now)
	408
	409	@wrap_as_background_process("update_client_ips")
	410	def _update_client_ips_batch(self):
	411
	412	# If the DB pool has already terminated, don't try updating
	413	if not self.hs.get_db_pool().running:
	414	return
	415
	416	to_update = self._batch_row_update
	417	self._batch_row_update = {}
	418
	419	return self.runInteraction(
	420	"_update_client_ips_batch", self._update_client_ips_batch_txn, to_update
	421	)
	422
	423	def _update_client_ips_batch_txn(self, txn, to_update):
	424	if "user_ips" in self._unsafe_to_upsert_tables or (
	425	not self.database_engine.can_native_upsert
	426	):
	427	self.database_engine.lock_table(txn, "user_ips")
	428
	429	for entry in iteritems(to_update):
	430	(user_id, access_token, ip), (user_agent, device_id, last_seen) = entry
	431
	432	try:
	433	self._simple_upsert_txn(
	434	txn,
	435	table="user_ips",
	436	keyvalues={
	437	"user_id": user_id,
	438	"access_token": access_token,
	439	"ip": ip,
	440	},
	441	values={
	442	"user_agent": user_agent,
	443	"device_id": device_id,
	444	"last_seen": last_seen,
	445	},
	446	lock=False,
	447	)
	448
	449	# Technically an access token might not be associated with
	450	# a device so we need to check.
	451	if device_id:
	452	self._simple_upsert_txn(
	453	txn,
	454	table="devices",
	455	keyvalues={"user_id": user_id, "device_id": device_id},
	456	values={
	457	"user_agent": user_agent,
	458	"last_seen": last_seen,
	459	"ip": ip,
	460	},
	461	lock=False,
	462	)
	463	except Exception as e:
	464	# Failed to upsert, log and continue
	465	logger.error("Failed to insert client IP %r: %r", entry, e)
	466
	467	@defer.inlineCallbacks
	468	def get_last_client_ip_by_device(self, user_id, device_id):
	469	"""For each device_id listed, give the user_ip it was last seen on
	470
	471	Args:
	472	user_id (str)
	473	device_id (str): If None fetches all devices for the user
	474
	475	Returns:
	476	defer.Deferred: resolves to a dict, where the keys
	477	are (user_id, device_id) tuples. The values are also dicts, with
	478	keys giving the column names
	479	"""
	480
	481	keyvalues = {"user_id": user_id}
	482	if device_id is not None:
	483	keyvalues["device_id"] = device_id
	484
	485	res = yield self._simple_select_list(
	486	table="devices",
	487	keyvalues=keyvalues,
	488	retcols=("user_id", "ip", "user_agent", "device_id", "last_seen"),
	489	)
	490
	491	ret = {(d["user_id"], d["device_id"]): d for d in res}
	492	for key in self._batch_row_update:
	493	uid, access_token, ip = key
	494	if uid == user_id:
	495	user_agent, did, last_seen = self._batch_row_update[key]
	496	if not device_id or did == device_id:
	497	ret[(user_id, device_id)] = {
	498	"user_id": user_id,
	499	"access_token": access_token,
	500	"ip": ip,
	501	"user_agent": user_agent,
	502	"device_id": did,
	503	"last_seen": last_seen,
	504	}
	505	return ret
	506
	507	@defer.inlineCallbacks
	508	def get_user_ip_and_agents(self, user):
	509	user_id = user.to_string()
	510	results = {}
	511
	512	for key in self._batch_row_update:
	513	uid, access_token, ip, = key
	514	if uid == user_id:
	515	user_agent, _, last_seen = self._batch_row_update[key]
	516	results[(access_token, ip)] = (user_agent, last_seen)
	517
	518	rows = yield self._simple_select_list(
	519	table="user_ips",
	520	keyvalues={"user_id": user_id},
	521	retcols=["access_token", "ip", "user_agent", "last_seen"],
	522	desc="get_user_ip_and_agents",
	523	)
	524
	525	results.update(
	526	((row["access_token"], row["ip"]), (row["user_agent"], row["last_seen"]))
	527	for row in rows
	528	)
	529	return list(
	530	{
	531	"access_token": access_token,
	532	"ip": ip,
	533	"user_agent": user_agent,
	534	"last_seen": last_seen,
	535	}
	536	for (access_token, ip), (user_agent, last_seen) in iteritems(results)
	537	)
	538
	539	@wrap_as_background_process("prune_old_user_ips")
	540	async def _prune_old_user_ips(self):
	541	"""Removes entries in user IPs older than the configured period.
	542	"""
	543
	544	if self.user_ips_max_age is None:
	545	# Nothing to do
	546	return
	547
	548	if not await self.has_completed_background_update("devices_last_seen"):
	549	# Only start pruning if we have finished populating the devices
	550	# last seen info.
	551	return
	552
	553	# We do a slightly funky SQL delete to ensure we don't try and delete
	554	# too much at once (as the table may be very large from before we
	555	# started pruning).
	556	#
	557	# This works by finding the max last_seen that is less than the given
	558	# time, but has no more than N rows before it, deleting all rows with
	559	# a lesser last_seen time. (We COALESCE so that the sub-SELECT always
	560	# returns exactly one row).
	561	sql = """
	562	DELETE FROM user_ips
	563	WHERE last_seen <= (
	564	SELECT COALESCE(MAX(last_seen), -1)
	565	FROM (
	566	SELECT last_seen FROM user_ips
	567	WHERE last_seen <= ?
	568	ORDER BY last_seen ASC
	569	LIMIT 5000
	570	) AS u
	571	)
	572	"""
	573
	574	timestamp = self.clock.time_msec() - self.user_ips_max_age
	575
	576	def _prune_old_user_ips_txn(txn):
	577	txn.execute(sql, (timestamp,))
	578
	579	await self.runInteraction("_prune_old_user_ips", _prune_old_user_ips_txn)

+457

-0

synapse/storage/data_stores/main/deviceinbox.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2016 OpenMarket Ltd
	2	#
	3	# Licensed under the Apache License, Version 2.0 (the "License");
	4	# you may not use this file except in compliance with the License.
	5	# You may obtain a copy of the License at
	6	#
	7	# http://www.apache.org/licenses/LICENSE-2.0
	8	#
	9	# Unless required by applicable law or agreed to in writing, software
	10	# distributed under the License is distributed on an "AS IS" BASIS,
	11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	# See the License for the specific language governing permissions and
	13	# limitations under the License.
	14
	15	import logging
	16
	17	from canonicaljson import json
	18
	19	from twisted.internet import defer
	20
	21	from synapse.logging.opentracing import log_kv, set_tag, trace
	22	from synapse.storage._base import SQLBaseStore, make_in_list_sql_clause
	23	from synapse.storage.background_updates import BackgroundUpdateStore
	24	from synapse.util.caches.expiringcache import ExpiringCache
	25
	26	logger = logging.getLogger(__name__)
	27
	28
	29	class DeviceInboxWorkerStore(SQLBaseStore):
	30	def get_to_device_stream_token(self):
	31	return self._device_inbox_id_gen.get_current_token()
	32
	33	def get_new_messages_for_device(
	34	self, user_id, device_id, last_stream_id, current_stream_id, limit=100
	35	):
	36	"""
	37	Args:
	38	user_id(str): The recipient user_id.
	39	device_id(str): The recipient device_id.
	40	current_stream_id(int): The current position of the to device
	41	message stream.
	42	Returns:
	43	Deferred ([dict], int): List of messages for the device and where
	44	in the stream the messages got to.
	45	"""
	46	has_changed = self._device_inbox_stream_cache.has_entity_changed(
	47	user_id, last_stream_id
	48	)
	49	if not has_changed:
	50	return defer.succeed(([], current_stream_id))
	51
	52	def get_new_messages_for_device_txn(txn):
	53	sql = (
	54	"SELECT stream_id, message_json FROM device_inbox"
	55	" WHERE user_id = ? AND device_id = ?"
	56	" AND ? < stream_id AND stream_id <= ?"
	57	" ORDER BY stream_id ASC"
	58	" LIMIT ?"
	59	)
	60	txn.execute(
	61	sql, (user_id, device_id, last_stream_id, current_stream_id, limit)
	62	)
	63	messages = []
	64	for row in txn:
	65	stream_pos = row[0]
	66	messages.append(json.loads(row[1]))
	67	if len(messages) < limit:
	68	stream_pos = current_stream_id
	69	return messages, stream_pos
	70
	71	return self.runInteraction(
	72	"get_new_messages_for_device", get_new_messages_for_device_txn
	73	)
	74
	75	@trace
	76	@defer.inlineCallbacks
	77	def delete_messages_for_device(self, user_id, device_id, up_to_stream_id):
	78	"""
	79	Args:
	80	user_id(str): The recipient user_id.
	81	device_id(str): The recipient device_id.
	82	up_to_stream_id(int): Where to delete messages up to.
	83	Returns:
	84	A deferred that resolves to the number of messages deleted.
	85	"""
	86	# If we have cached the last stream id we've deleted up to, we can
	87	# check if there is likely to be anything that needs deleting
	88	last_deleted_stream_id = self._last_device_delete_cache.get(
	89	(user_id, device_id), None
	90	)
	91
	92	set_tag("last_deleted_stream_id", last_deleted_stream_id)
	93
	94	if last_deleted_stream_id:
	95	has_changed = self._device_inbox_stream_cache.has_entity_changed(
	96	user_id, last_deleted_stream_id
	97	)
	98	if not has_changed:
	99	log_kv({"message": "No changes in cache since last check"})
	100	return 0
	101
	102	def delete_messages_for_device_txn(txn):
	103	sql = (
	104	"DELETE FROM device_inbox"
	105	" WHERE user_id = ? AND device_id = ?"
	106	" AND stream_id <= ?"
	107	)
	108	txn.execute(sql, (user_id, device_id, up_to_stream_id))
	109	return txn.rowcount
	110
	111	count = yield self.runInteraction(
	112	"delete_messages_for_device", delete_messages_for_device_txn
	113	)
	114
	115	log_kv(
	116	{"message": "deleted {} messages for device".format(count), "count": count}
	117	)
	118
	119	# Update the cache, ensuring that we only ever increase the value
	120	last_deleted_stream_id = self._last_device_delete_cache.get(
	121	(user_id, device_id), 0
	122	)
	123	self._last_device_delete_cache[(user_id, device_id)] = max(
	124	last_deleted_stream_id, up_to_stream_id
	125	)
	126
	127	return count
	128
	129	@trace
	130	def get_new_device_msgs_for_remote(
	131	self, destination, last_stream_id, current_stream_id, limit
	132	):
	133	"""
	134	Args:
	135	destination(str): The name of the remote server.
	136	last_stream_id(int\|long): The last position of the device message stream
	137	that the server sent up to.
	138	current_stream_id(int\|long): The current position of the device
	139	message stream.
	140	Returns:
	141	Deferred ([dict], int\|long): List of messages for the device and where
	142	in the stream the messages got to.
	143	"""
	144
	145	set_tag("destination", destination)
	146	set_tag("last_stream_id", last_stream_id)
	147	set_tag("current_stream_id", current_stream_id)
	148	set_tag("limit", limit)
	149
	150	has_changed = self._device_federation_outbox_stream_cache.has_entity_changed(
	151	destination, last_stream_id
	152	)
	153	if not has_changed or last_stream_id == current_stream_id:
	154	log_kv({"message": "No new messages in stream"})
	155	return defer.succeed(([], current_stream_id))
	156
	157	if limit <= 0:
	158	# This can happen if we run out of room for EDUs in the transaction.
	159	return defer.succeed(([], last_stream_id))
	160
	161	@trace
	162	def get_new_messages_for_remote_destination_txn(txn):
	163	sql = (
	164	"SELECT stream_id, messages_json FROM device_federation_outbox"
	165	" WHERE destination = ?"
	166	" AND ? < stream_id AND stream_id <= ?"
	167	" ORDER BY stream_id ASC"
	168	" LIMIT ?"
	169	)
	170	txn.execute(sql, (destination, last_stream_id, current_stream_id, limit))
	171	messages = []
	172	for row in txn:
	173	stream_pos = row[0]
	174	messages.append(json.loads(row[1]))
	175	if len(messages) < limit:
	176	log_kv({"message": "Set stream position to current position"})
	177	stream_pos = current_stream_id
	178	return messages, stream_pos
	179
	180	return self.runInteraction(
	181	"get_new_device_msgs_for_remote",
	182	get_new_messages_for_remote_destination_txn,
	183	)
	184
	185	@trace
	186	def delete_device_msgs_for_remote(self, destination, up_to_stream_id):
	187	"""Used to delete messages when the remote destination acknowledges
	188	their receipt.
	189
	190	Args:
	191	destination(str): The destination server_name
	192	up_to_stream_id(int): Where to delete messages up to.
	193	Returns:
	194	A deferred that resolves when the messages have been deleted.
	195	"""
	196
	197	def delete_messages_for_remote_destination_txn(txn):
	198	sql = (
	199	"DELETE FROM device_federation_outbox"
	200	" WHERE destination = ?"
	201	" AND stream_id <= ?"
	202	)
	203	txn.execute(sql, (destination, up_to_stream_id))
	204
	205	return self.runInteraction(
	206	"delete_device_msgs_for_remote", delete_messages_for_remote_destination_txn
	207	)
	208
	209
	210	class DeviceInboxBackgroundUpdateStore(BackgroundUpdateStore):
	211	DEVICE_INBOX_STREAM_ID = "device_inbox_stream_drop"
	212
	213	def __init__(self, db_conn, hs):
	214	super(DeviceInboxBackgroundUpdateStore, self).__init__(db_conn, hs)
	215
	216	self.register_background_index_update(
	217	"device_inbox_stream_index",
	218	index_name="device_inbox_stream_id_user_id",
	219	table="device_inbox",
	220	columns=["stream_id", "user_id"],
	221	)
	222
	223	self.register_background_update_handler(
	224	self.DEVICE_INBOX_STREAM_ID, self._background_drop_index_device_inbox
	225	)
	226
	227	@defer.inlineCallbacks
	228	def _background_drop_index_device_inbox(self, progress, batch_size):
	229	def reindex_txn(conn):
	230	txn = conn.cursor()
	231	txn.execute("DROP INDEX IF EXISTS device_inbox_stream_id")
	232	txn.close()
	233
	234	yield self.runWithConnection(reindex_txn)
	235
	236	yield self._end_background_update(self.DEVICE_INBOX_STREAM_ID)
	237
	238	return 1
	239
	240
	241	class DeviceInboxStore(DeviceInboxWorkerStore, DeviceInboxBackgroundUpdateStore):
	242	DEVICE_INBOX_STREAM_ID = "device_inbox_stream_drop"
	243
	244	def __init__(self, db_conn, hs):
	245	super(DeviceInboxStore, self).__init__(db_conn, hs)
	246
	247	# Map of (user_id, device_id) to the last stream_id that has been
	248	# deleted up to. This is so that we can no op deletions.
	249	self._last_device_delete_cache = ExpiringCache(
	250	cache_name="last_device_delete_cache",
	251	clock=self._clock,
	252	max_len=10000,
	253	expiry_ms=30 * 60 * 1000,
	254	)
	255
	256	@trace
	257	@defer.inlineCallbacks
	258	def add_messages_to_device_inbox(
	259	self, local_messages_by_user_then_device, remote_messages_by_destination
	260	):
	261	"""Used to send messages from this server.
	262
	263	Args:
	264	sender_user_id(str): The ID of the user sending these messages.
	265	local_messages_by_user_and_device(dict):
	266	Dictionary of user_id to device_id to message.
	267	remote_messages_by_destination(dict):
	268	Dictionary of destination server_name to the EDU JSON to send.
	269	Returns:
	270	A deferred stream_id that resolves when the messages have been
	271	inserted.
	272	"""
	273
	274	def add_messages_txn(txn, now_ms, stream_id):
	275	# Add the local messages directly to the local inbox.
	276	self._add_messages_to_local_device_inbox_txn(
	277	txn, stream_id, local_messages_by_user_then_device
	278	)
	279
	280	# Add the remote messages to the federation outbox.
	281	# We'll send them to a remote server when we next send a
	282	# federation transaction to that destination.
	283	sql = (
	284	"INSERT INTO device_federation_outbox"
	285	" (destination, stream_id, queued_ts, messages_json)"
	286	" VALUES (?,?,?,?)"
	287	)
	288	rows = []
	289	for destination, edu in remote_messages_by_destination.items():
	290	edu_json = json.dumps(edu)
	291	rows.append((destination, stream_id, now_ms, edu_json))
	292	txn.executemany(sql, rows)
	293
	294	with self._device_inbox_id_gen.get_next() as stream_id:
	295	now_ms = self.clock.time_msec()
	296	yield self.runInteraction(
	297	"add_messages_to_device_inbox", add_messages_txn, now_ms, stream_id
	298	)
	299	for user_id in local_messages_by_user_then_device.keys():
	300	self._device_inbox_stream_cache.entity_has_changed(user_id, stream_id)
	301	for destination in remote_messages_by_destination.keys():
	302	self._device_federation_outbox_stream_cache.entity_has_changed(
	303	destination, stream_id
	304	)
	305
	306	return self._device_inbox_id_gen.get_current_token()
	307
	308	@defer.inlineCallbacks
	309	def add_messages_from_remote_to_device_inbox(
	310	self, origin, message_id, local_messages_by_user_then_device
	311	):
	312	def add_messages_txn(txn, now_ms, stream_id):
	313	# Check if we've already inserted a matching message_id for that
	314	# origin. This can happen if the origin doesn't receive our
	315	# acknowledgement from the first time we received the message.
	316	already_inserted = self._simple_select_one_txn(
	317	txn,
	318	table="device_federation_inbox",
	319	keyvalues={"origin": origin, "message_id": message_id},
	320	retcols=("message_id",),
	321	allow_none=True,
	322	)
	323	if already_inserted is not None:
	324	return
	325
	326	# Add an entry for this message_id so that we know we've processed
	327	# it.
	328	self._simple_insert_txn(
	329	txn,
	330	table="device_federation_inbox",
	331	values={
	332	"origin": origin,
	333	"message_id": message_id,
	334	"received_ts": now_ms,
	335	},
	336	)
	337
	338	# Add the messages to the approriate local device inboxes so that
	339	# they'll be sent to the devices when they next sync.
	340	self._add_messages_to_local_device_inbox_txn(
	341	txn, stream_id, local_messages_by_user_then_device
	342	)
	343
	344	with self._device_inbox_id_gen.get_next() as stream_id:
	345	now_ms = self.clock.time_msec()
	346	yield self.runInteraction(
	347	"add_messages_from_remote_to_device_inbox",
	348	add_messages_txn,
	349	now_ms,
	350	stream_id,
	351	)
	352	for user_id in local_messages_by_user_then_device.keys():
	353	self._device_inbox_stream_cache.entity_has_changed(user_id, stream_id)
	354
	355	return stream_id
	356
	357	def _add_messages_to_local_device_inbox_txn(
	358	self, txn, stream_id, messages_by_user_then_device
	359	):
	360	sql = "UPDATE device_max_stream_id" " SET stream_id = ?" " WHERE stream_id < ?"
	361	txn.execute(sql, (stream_id, stream_id))
	362
	363	local_by_user_then_device = {}
	364	for user_id, messages_by_device in messages_by_user_then_device.items():
	365	messages_json_for_user = {}
	366	devices = list(messages_by_device.keys())
	367	if len(devices) == 1 and devices[0] == "*":
	368	# Handle wildcard device_ids.
	369	sql = "SELECT device_id FROM devices" " WHERE user_id = ?"
	370	txn.execute(sql, (user_id,))
	371	message_json = json.dumps(messages_by_device["*"])
	372	for row in txn:
	373	# Add the message for all devices for this user on this
	374	# server.
	375	device = row[0]
	376	messages_json_for_user[device] = message_json
	377	else:
	378	if not devices:
	379	continue
	380
	381	clause, args = make_in_list_sql_clause(
	382	txn.database_engine, "device_id", devices
	383	)
	384	sql = "SELECT device_id FROM devices WHERE user_id = ? AND " + clause
	385
	386	# TODO: Maybe this needs to be done in batches if there are
	387	# too many local devices for a given user.
	388	txn.execute(sql, [user_id] + list(args))
	389	for row in txn:
	390	# Only insert into the local inbox if the device exists on
	391	# this server
	392	device = row[0]
	393	message_json = json.dumps(messages_by_device[device])
	394	messages_json_for_user[device] = message_json
	395
	396	if messages_json_for_user:
	397	local_by_user_then_device[user_id] = messages_json_for_user
	398
	399	if not local_by_user_then_device:
	400	return
	401
	402	sql = (
	403	"INSERT INTO device_inbox"
	404	" (user_id, device_id, stream_id, message_json)"
	405	" VALUES (?,?,?,?)"
	406	)
	407	rows = []
	408	for user_id, messages_by_device in local_by_user_then_device.items():
	409	for device_id, message_json in messages_by_device.items():
	410	rows.append((user_id, device_id, stream_id, message_json))
	411
	412	txn.executemany(sql, rows)
	413
	414	def get_all_new_device_messages(self, last_pos, current_pos, limit):
	415	"""
	416	Args:
	417	last_pos(int):
	418	current_pos(int):
	419	limit(int):
	420	Returns:
	421	A deferred list of rows from the device inbox
	422	"""
	423	if last_pos == current_pos:
	424	return defer.succeed([])
	425
	426	def get_all_new_device_messages_txn(txn):
	427	# We limit like this as we might have multiple rows per stream_id, and
	428	# we want to make sure we always get all entries for any stream_id
	429	# we return.
	430	upper_pos = min(current_pos, last_pos + limit)
	431	sql = (
	432	"SELECT max(stream_id), user_id"
	433	" FROM device_inbox"
	434	" WHERE ? < stream_id AND stream_id <= ?"
	435	" GROUP BY user_id"
	436	)
	437	txn.execute(sql, (last_pos, upper_pos))
	438	rows = txn.fetchall()
	439
	440	sql = (
	441	"SELECT max(stream_id), destination"
	442	" FROM device_federation_outbox"
	443	" WHERE ? < stream_id AND stream_id <= ?"
	444	" GROUP BY destination"
	445	)
	446	txn.execute(sql, (last_pos, upper_pos))
	447	rows.extend(txn)
	448
	449	# Order by ascending stream ordering
	450	rows.sort()
	451
	452	return rows
	453
	454	return self.runInteraction(
	455	"get_all_new_device_messages", get_all_new_device_messages_txn
	456	)

+1014

-0

synapse/storage/data_stores/main/devices.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2016 OpenMarket Ltd
	2	# Copyright 2019 New Vector Ltd
	3	# Copyright 2019 The Matrix.org Foundation C.I.C.
	4	#
	5	# Licensed under the Apache License, Version 2.0 (the "License");
	6	# you may not use this file except in compliance with the License.
	7	# You may obtain a copy of the License at
	8	#
	9	# http://www.apache.org/licenses/LICENSE-2.0
	10	#
	11	# Unless required by applicable law or agreed to in writing, software
	12	# distributed under the License is distributed on an "AS IS" BASIS,
	13	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	14	# See the License for the specific language governing permissions and
	15	# limitations under the License.
	16	import logging
	17
	18	from six import iteritems
	19
	20	from canonicaljson import json
	21
	22	from twisted.internet import defer
	23
	24	from synapse.api.errors import Codes, StoreError
	25	from synapse.logging.opentracing import (
	26	get_active_span_text_map,
	27	set_tag,
	28	trace,
	29	whitelisted_homeserver,
	30	)
	31	from synapse.metrics.background_process_metrics import run_as_background_process
	32	from synapse.storage._base import (
	33	Cache,
	34	SQLBaseStore,
	35	db_to_json,
	36	make_in_list_sql_clause,
	37	)
	38	from synapse.storage.background_updates import BackgroundUpdateStore
	39	from synapse.util import batch_iter
	40	from synapse.util.caches.descriptors import cached, cachedInlineCallbacks, cachedList
	41
	42	logger = logging.getLogger(__name__)
	43
	44	DROP_DEVICE_LIST_STREAMS_NON_UNIQUE_INDEXES = (
	45	"drop_device_list_streams_non_unique_indexes"
	46	)
	47
	48
	49	class DeviceWorkerStore(SQLBaseStore):
	50	def get_device(self, user_id, device_id):
	51	"""Retrieve a device. Only returns devices that are not marked as
	52	hidden.
	53
	54	Args:
	55	user_id (str): The ID of the user which owns the device
	56	device_id (str): The ID of the device to retrieve
	57	Returns:
	58	defer.Deferred for a dict containing the device information
	59	Raises:
	60	StoreError: if the device is not found
	61	"""
	62	return self._simple_select_one(
	63	table="devices",
	64	keyvalues={"user_id": user_id, "device_id": device_id, "hidden": False},
	65	retcols=("user_id", "device_id", "display_name"),
	66	desc="get_device",
	67	)
	68
	69	@defer.inlineCallbacks
	70	def get_devices_by_user(self, user_id):
	71	"""Retrieve all of a user's registered devices. Only returns devices
	72	that are not marked as hidden.
	73
	74	Args:
	75	user_id (str):
	76	Returns:
	77	defer.Deferred: resolves to a dict from device_id to a dict
	78	containing "device_id", "user_id" and "display_name" for each
	79	device.
	80	"""
	81	devices = yield self._simple_select_list(
	82	table="devices",
	83	keyvalues={"user_id": user_id, "hidden": False},
	84	retcols=("user_id", "device_id", "display_name"),
	85	desc="get_devices_by_user",
	86	)
	87
	88	return {d["device_id"]: d for d in devices}
	89
	90	@trace
	91	@defer.inlineCallbacks
	92	def get_devices_by_remote(self, destination, from_stream_id, limit):
	93	"""Get stream of updates to send to remote servers
	94
	95	Returns:
	96	Deferred[tuple[int, list[dict]]]:
	97	current stream id (ie, the stream id of the last update included in the
	98	response), and the list of updates
	99	"""
	100	now_stream_id = self._device_list_id_gen.get_current_token()
	101
	102	has_changed = self._device_list_federation_stream_cache.has_entity_changed(
	103	destination, int(from_stream_id)
	104	)
	105	if not has_changed:
	106	return now_stream_id, []
	107
	108	# We retrieve n+1 devices from the list of outbound pokes where n is
	109	# our outbound device update limit. We then check if the very last
	110	# device has the same stream_id as the second-to-last device. If so,
	111	# then we ignore all devices with that stream_id and only send the
	112	# devices with a lower stream_id.
	113	#
	114	# If when culling the list we end up with no devices afterwards, we
	115	# consider the device update to be too large, and simply skip the
	116	# stream_id; the rationale being that such a large device list update
	117	# is likely an error.
	118	updates = yield self.runInteraction(
	119	"get_devices_by_remote",
	120	self._get_devices_by_remote_txn,
	121	destination,
	122	from_stream_id,
	123	now_stream_id,
	124	limit + 1,
	125	)
	126
	127	# Return an empty list if there are no updates
	128	if not updates:
	129	return now_stream_id, []
	130
	131	# if we have exceeded the limit, we need to exclude any results with the
	132	# same stream_id as the last row.
	133	if len(updates) > limit:
	134	stream_id_cutoff = updates[-1][2]
	135	now_stream_id = stream_id_cutoff - 1
	136	else:
	137	stream_id_cutoff = None
	138
	139	# Perform the equivalent of a GROUP BY
	140	#
	141	# Iterate through the updates list and copy non-duplicate
	142	# (user_id, device_id) entries into a map, with the value being
	143	# the max stream_id across each set of duplicate entries
	144	#
	145	# maps (user_id, device_id) -> (stream_id, opentracing_context)
	146	# as long as their stream_id does not match that of the last row
	147	#
	148	# opentracing_context contains the opentracing metadata for the request
	149	# that created the poke
	150	#
	151	# The most recent request's opentracing_context is used as the
	152	# context which created the Edu.
	153
	154	query_map = {}
	155	for update in updates:
	156	if stream_id_cutoff is not None and update[2] >= stream_id_cutoff:
	157	# Stop processing updates
	158	break
	159
	160	key = (update[0], update[1])
	161
	162	update_context = update[3]
	163	update_stream_id = update[2]
	164
	165	previous_update_stream_id, _ = query_map.get(key, (0, None))
	166
	167	if update_stream_id > previous_update_stream_id:
	168	query_map[key] = (update_stream_id, update_context)
	169
	170	# If we didn't find any updates with a stream_id lower than the cutoff, it
	171	# means that there are more than limit updates all of which have the same
	172	# steam_id.
	173
	174	# That should only happen if a client is spamming the server with new
	175	# devices, in which case E2E isn't going to work well anyway. We'll just
	176	# skip that stream_id and return an empty list, and continue with the next
	177	# stream_id next time.
	178	if not query_map:
	179	return stream_id_cutoff, []
	180
	181	results = yield self._get_device_update_edus_by_remote(
	182	destination, from_stream_id, query_map
	183	)
	184
	185	return now_stream_id, results
	186
	187	def _get_devices_by_remote_txn(
	188	self, txn, destination, from_stream_id, now_stream_id, limit
	189	):
	190	"""Return device update information for a given remote destination
	191
	192	Args:
	193	txn (LoggingTransaction): The transaction to execute
	194	destination (str): The host the device updates are intended for
	195	from_stream_id (int): The minimum stream_id to filter updates by, exclusive
	196	now_stream_id (int): The maximum stream_id to filter updates by, inclusive
	197	limit (int): Maximum number of device updates to return
	198
	199	Returns:
	200	List: List of device updates
	201	"""
	202	sql = """
	203	SELECT user_id, device_id, stream_id, opentracing_context FROM device_lists_outbound_pokes
	204	WHERE destination = ? AND ? < stream_id AND stream_id <= ? AND sent = ?
	205	ORDER BY stream_id
	206	LIMIT ?
	207	"""
	208	txn.execute(sql, (destination, from_stream_id, now_stream_id, False, limit))
	209
	210	return list(txn)
	211
	212	@defer.inlineCallbacks
	213	def _get_device_update_edus_by_remote(self, destination, from_stream_id, query_map):
	214	"""Returns a list of device update EDUs as well as E2EE keys
	215
	216	Args:
	217	destination (str): The host the device updates are intended for
	218	from_stream_id (int): The minimum stream_id to filter updates by, exclusive
	219	query_map (Dict[(str, str): (int, str\|None)]): Dictionary mapping
	220	user_id/device_id to update stream_id and the relevent json-encoded
	221	opentracing context
	222
	223	Returns:
	224	List[Dict]: List of objects representing an device update EDU
	225
	226	"""
	227	devices = yield self.runInteraction(
	228	"_get_e2e_device_keys_txn",
	229	self._get_e2e_device_keys_txn,
	230	query_map.keys(),
	231	include_all_devices=True,
	232	include_deleted_devices=True,
	233	)
	234
	235	results = []
	236	for user_id, user_devices in iteritems(devices):
	237	# The prev_id for the first row is always the last row before
	238	# `from_stream_id`
	239	prev_id = yield self._get_last_device_update_for_remote_user(
	240	destination, user_id, from_stream_id
	241	)
	242	for device_id, device in iteritems(user_devices):
	243	stream_id, opentracing_context = query_map[(user_id, device_id)]
	244	result = {
	245	"user_id": user_id,
	246	"device_id": device_id,
	247	"prev_id": [prev_id] if prev_id else [],
	248	"stream_id": stream_id,
	249	"org.matrix.opentracing_context": opentracing_context,
	250	}
	251
	252	prev_id = stream_id
	253
	254	if device is not None:
	255	key_json = device.get("key_json", None)
	256	if key_json:
	257	result["keys"] = db_to_json(key_json)
	258	device_display_name = device.get("device_display_name", None)
	259	if device_display_name:
	260	result["device_display_name"] = device_display_name
	261	else:
	262	result["deleted"] = True
	263
	264	results.append(result)
	265
	266	return results
	267
	268	def _get_last_device_update_for_remote_user(
	269	self, destination, user_id, from_stream_id
	270	):
	271	def f(txn):
	272	prev_sent_id_sql = """
	273	SELECT coalesce(max(stream_id), 0) as stream_id
	274	FROM device_lists_outbound_last_success
	275	WHERE destination = ? AND user_id = ? AND stream_id <= ?
	276	"""
	277	txn.execute(prev_sent_id_sql, (destination, user_id, from_stream_id))
	278	rows = txn.fetchall()
	279	return rows[0][0]
	280
	281	return self.runInteraction("get_last_device_update_for_remote_user", f)
	282
	283	def mark_as_sent_devices_by_remote(self, destination, stream_id):
	284	"""Mark that updates have successfully been sent to the destination.
	285	"""
	286	return self.runInteraction(
	287	"mark_as_sent_devices_by_remote",
	288	self._mark_as_sent_devices_by_remote_txn,
	289	destination,
	290	stream_id,
	291	)
	292
	293	def _mark_as_sent_devices_by_remote_txn(self, txn, destination, stream_id):
	294	# We update the device_lists_outbound_last_success with the successfully
	295	# poked users. We do the join to see which users need to be inserted and
	296	# which updated.
	297	sql = """
	298	SELECT user_id, coalesce(max(o.stream_id), 0), (max(s.stream_id) IS NOT NULL)
	299	FROM device_lists_outbound_pokes as o
	300	LEFT JOIN device_lists_outbound_last_success as s
	301	USING (destination, user_id)
	302	WHERE destination = ? AND o.stream_id <= ?
	303	GROUP BY user_id
	304	"""
	305	txn.execute(sql, (destination, stream_id))
	306	rows = txn.fetchall()
	307
	308	sql = """
	309	UPDATE device_lists_outbound_last_success
	310	SET stream_id = ?
	311	WHERE destination = ? AND user_id = ?
	312	"""
	313	txn.executemany(sql, ((row[1], destination, row[0]) for row in rows if row[2]))
	314
	315	sql = """
	316	INSERT INTO device_lists_outbound_last_success
	317	(destination, user_id, stream_id) VALUES (?, ?, ?)
	318	"""
	319	txn.executemany(
	320	sql, ((destination, row[0], row[1]) for row in rows if not row[2])
	321	)
	322
	323	# Delete all sent outbound pokes
	324	sql = """
	325	DELETE FROM device_lists_outbound_pokes
	326	WHERE destination = ? AND stream_id <= ?
	327	"""
	328	txn.execute(sql, (destination, stream_id))
	329
	330	@defer.inlineCallbacks
	331	def add_user_signature_change_to_streams(self, from_user_id, user_ids):
	332	"""Persist that a user has made new signatures
	333
	334	Args:
	335	from_user_id (str): the user who made the signatures
	336	user_ids (list[str]): the users who were signed
	337	"""
	338
	339	with self._device_list_id_gen.get_next() as stream_id:
	340	yield self.runInteraction(
	341	"add_user_sig_change_to_streams",
	342	self._add_user_signature_change_txn,
	343	from_user_id,
	344	user_ids,
	345	stream_id,
	346	)
	347	return stream_id
	348
	349	def _add_user_signature_change_txn(self, txn, from_user_id, user_ids, stream_id):
	350	txn.call_after(
	351	self._user_signature_stream_cache.entity_has_changed,
	352	from_user_id,
	353	stream_id,
	354	)
	355	self._simple_insert_txn(
	356	txn,
	357	"user_signature_stream",
	358	values={
	359	"stream_id": stream_id,
	360	"from_user_id": from_user_id,
	361	"user_ids": json.dumps(user_ids),
	362	},
	363	)
	364
	365	def get_device_stream_token(self):
	366	return self._device_list_id_gen.get_current_token()
	367
	368	@trace
	369	@defer.inlineCallbacks
	370	def get_user_devices_from_cache(self, query_list):
	371	"""Get the devices (and keys if any) for remote users from the cache.
	372
	373	Args:
	374	query_list(list): List of (user_id, device_ids), if device_ids is
	375	falsey then return all device ids for that user.
	376
	377	Returns:
	378	(user_ids_not_in_cache, results_map), where user_ids_not_in_cache is
	379	a set of user_ids and results_map is a mapping of
	380	user_id -> device_id -> device_info
	381	"""
	382	user_ids = set(user_id for user_id, _ in query_list)
	383	user_map = yield self.get_device_list_last_stream_id_for_remotes(list(user_ids))
	384	user_ids_in_cache = set(
	385	user_id for user_id, stream_id in user_map.items() if stream_id
	386	)
	387	user_ids_not_in_cache = user_ids - user_ids_in_cache
	388
	389	results = {}
	390	for user_id, device_id in query_list:
	391	if user_id not in user_ids_in_cache:
	392	continue
	393
	394	if device_id:
	395	device = yield self._get_cached_user_device(user_id, device_id)
	396	results.setdefault(user_id, {})[device_id] = device
	397	else:
	398	results[user_id] = yield self._get_cached_devices_for_user(user_id)
	399
	400	set_tag("in_cache", results)
	401	set_tag("not_in_cache", user_ids_not_in_cache)
	402
	403	return user_ids_not_in_cache, results
	404
	405	@cachedInlineCallbacks(num_args=2, tree=True)
	406	def _get_cached_user_device(self, user_id, device_id):
	407	content = yield self._simple_select_one_onecol(
	408	table="device_lists_remote_cache",
	409	keyvalues={"user_id": user_id, "device_id": device_id},
	410	retcol="content",
	411	desc="_get_cached_user_device",
	412	)
	413	return db_to_json(content)
	414
	415	@cachedInlineCallbacks()
	416	def _get_cached_devices_for_user(self, user_id):
	417	devices = yield self._simple_select_list(
	418	table="device_lists_remote_cache",
	419	keyvalues={"user_id": user_id},
	420	retcols=("device_id", "content"),
	421	desc="_get_cached_devices_for_user",
	422	)
	423	return {
	424	device["device_id"]: db_to_json(device["content"]) for device in devices
	425	}
	426
	427	def get_devices_with_keys_by_user(self, user_id):
	428	"""Get all devices (with any device keys) for a user
	429
	430	Returns:
	431	(stream_id, devices)
	432	"""
	433	return self.runInteraction(
	434	"get_devices_with_keys_by_user",
	435	self._get_devices_with_keys_by_user_txn,
	436	user_id,
	437	)
	438
	439	def _get_devices_with_keys_by_user_txn(self, txn, user_id):
	440	now_stream_id = self._device_list_id_gen.get_current_token()
	441
	442	devices = self._get_e2e_device_keys_txn(
	443	txn, [(user_id, None)], include_all_devices=True
	444	)
	445
	446	if devices:
	447	user_devices = devices[user_id]
	448	results = []
	449	for device_id, device in iteritems(user_devices):
	450	result = {"device_id": device_id}
	451
	452	key_json = device.get("key_json", None)
	453	if key_json:
	454	result["keys"] = db_to_json(key_json)
	455	device_display_name = device.get("device_display_name", None)
	456	if device_display_name:
	457	result["device_display_name"] = device_display_name
	458
	459	results.append(result)
	460
	461	return now_stream_id, results
	462
	463	return now_stream_id, []
	464
	465	def get_users_whose_devices_changed(self, from_key, user_ids):
	466	"""Get set of users whose devices have changed since `from_key` that
	467	are in the given list of user_ids.
	468
	469	Args:
	470	from_key (str): The device lists stream token
	471	user_ids (Iterable[str])
	472
	473	Returns:
	474	Deferred[set[str]]: The set of user_ids whose devices have changed
	475	since `from_key`
	476	"""
	477	from_key = int(from_key)
	478
	479	# Get set of users who may have changed. Users not in the returned
	480	# list have definitely not changed.
	481	to_check = list(
	482	self._device_list_stream_cache.get_entities_changed(user_ids, from_key)
	483	)
	484
	485	if not to_check:
	486	return defer.succeed(set())
	487
	488	def _get_users_whose_devices_changed_txn(txn):
	489	changes = set()
	490
	491	sql = """
	492	SELECT DISTINCT user_id FROM device_lists_stream
	493	WHERE stream_id > ?
	494	AND
	495	"""
	496
	497	for chunk in batch_iter(to_check, 100):
	498	clause, args = make_in_list_sql_clause(
	499	txn.database_engine, "user_id", chunk
	500	)
	501	txn.execute(sql + clause, (from_key,) + tuple(args))
	502	changes.update(user_id for user_id, in txn)
	503
	504	return changes
	505
	506	return self.runInteraction(
	507	"get_users_whose_devices_changed", _get_users_whose_devices_changed_txn
	508	)
	509
	510	@defer.inlineCallbacks
	511	def get_users_whose_signatures_changed(self, user_id, from_key):
	512	"""Get the users who have new cross-signing signatures made by `user_id` since
	513	`from_key`.
	514
	515	Args:
	516	user_id (str): the user who made the signatures
	517	from_key (str): The device lists stream token
	518	"""
	519	from_key = int(from_key)
	520	if self._user_signature_stream_cache.has_entity_changed(user_id, from_key):
	521	sql = """
	522	SELECT DISTINCT user_ids FROM user_signature_stream
	523	WHERE from_user_id = ? AND stream_id > ?
	524	"""
	525	rows = yield self._execute(
	526	"get_users_whose_signatures_changed", None, sql, user_id, from_key
	527	)
	528	return set(user for row in rows for user in json.loads(row[0]))
	529	else:
	530	return set()
	531
	532	def get_all_device_list_changes_for_remotes(self, from_key, to_key):
	533	"""Return a list of `(stream_id, user_id, destination)` which is the
	534	combined list of changes to devices, and which destinations need to be
	535	poked. `destination` may be None if no destinations need to be poked.
	536	"""
	537	# We do a group by here as there can be a large number of duplicate
	538	# entries, since we throw away device IDs.
	539	sql = """
	540	SELECT MAX(stream_id) AS stream_id, user_id, destination
	541	FROM device_lists_stream
	542	LEFT JOIN device_lists_outbound_pokes USING (stream_id, user_id, device_id)
	543	WHERE ? < stream_id AND stream_id <= ?
	544	GROUP BY user_id, destination
	545	"""
	546	return self._execute(
	547	"get_all_device_list_changes_for_remotes", None, sql, from_key, to_key
	548	)
	549
	550	@cached(max_entries=10000)
	551	def get_device_list_last_stream_id_for_remote(self, user_id):
	552	"""Get the last stream_id we got for a user. May be None if we haven't
	553	got any information for them.
	554	"""
	555	return self._simple_select_one_onecol(
	556	table="device_lists_remote_extremeties",
	557	keyvalues={"user_id": user_id},
	558	retcol="stream_id",
	559	desc="get_device_list_last_stream_id_for_remote",
	560	allow_none=True,
	561	)
	562
	563	@cachedList(
	564	cached_method_name="get_device_list_last_stream_id_for_remote",
	565	list_name="user_ids",
	566	inlineCallbacks=True,
	567	)
	568	def get_device_list_last_stream_id_for_remotes(self, user_ids):
	569	rows = yield self._simple_select_many_batch(
	570	table="device_lists_remote_extremeties",
	571	column="user_id",
	572	iterable=user_ids,
	573	retcols=("user_id", "stream_id"),
	574	desc="get_device_list_last_stream_id_for_remotes",
	575	)
	576
	577	results = {user_id: None for user_id in user_ids}
	578	results.update({row["user_id"]: row["stream_id"] for row in rows})
	579
	580	return results
	581
	582
	583	class DeviceBackgroundUpdateStore(BackgroundUpdateStore):
	584	def __init__(self, db_conn, hs):
	585	super(DeviceBackgroundUpdateStore, self).__init__(db_conn, hs)
	586
	587	self.register_background_index_update(
	588	"device_lists_stream_idx",
	589	index_name="device_lists_stream_user_id",
	590	table="device_lists_stream",
	591	columns=["user_id", "device_id"],
	592	)
	593
	594	# create a unique index on device_lists_remote_cache
	595	self.register_background_index_update(
	596	"device_lists_remote_cache_unique_idx",
	597	index_name="device_lists_remote_cache_unique_id",
	598	table="device_lists_remote_cache",
	599	columns=["user_id", "device_id"],
	600	unique=True,
	601	)
	602
	603	# And one on device_lists_remote_extremeties
	604	self.register_background_index_update(
	605	"device_lists_remote_extremeties_unique_idx",
	606	index_name="device_lists_remote_extremeties_unique_idx",
	607	table="device_lists_remote_extremeties",
	608	columns=["user_id"],
	609	unique=True,
	610	)
	611
	612	# once they complete, we can remove the old non-unique indexes.
	613	self.register_background_update_handler(
	614	DROP_DEVICE_LIST_STREAMS_NON_UNIQUE_INDEXES,
	615	self._drop_device_list_streams_non_unique_indexes,
	616	)
	617
	618	@defer.inlineCallbacks
	619	def _drop_device_list_streams_non_unique_indexes(self, progress, batch_size):
	620	def f(conn):
	621	txn = conn.cursor()
	622	txn.execute("DROP INDEX IF EXISTS device_lists_remote_cache_id")
	623	txn.execute("DROP INDEX IF EXISTS device_lists_remote_extremeties_id")
	624	txn.close()
	625
	626	yield self.runWithConnection(f)
	627	yield self._end_background_update(DROP_DEVICE_LIST_STREAMS_NON_UNIQUE_INDEXES)
	628	return 1
	629
	630
	631	class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore):
	632	def __init__(self, db_conn, hs):
	633	super(DeviceStore, self).__init__(db_conn, hs)
	634
	635	# Map of (user_id, device_id) -> bool. If there is an entry that implies
	636	# the device exists.
	637	self.device_id_exists_cache = Cache(
	638	name="device_id_exists", keylen=2, max_entries=10000
	639	)
	640
	641	self._clock.looping_call(self._prune_old_outbound_device_pokes, 60 * 60 * 1000)
	642
	643	@defer.inlineCallbacks
	644	def store_device(self, user_id, device_id, initial_device_display_name):
	645	"""Ensure the given device is known; add it to the store if not
	646
	647	Args:
	648	user_id (str): id of user associated with the device
	649	device_id (str): id of device
	650	initial_device_display_name (str): initial displayname of the
	651	device. Ignored if device exists.
	652	Returns:
	653	defer.Deferred: boolean whether the device was inserted or an
	654	existing device existed with that ID.
	655	Raises:
	656	StoreError: if the device is already in use
	657	"""
	658	key = (user_id, device_id)
	659	if self.device_id_exists_cache.get(key, None):
	660	return False
	661
	662	try:
	663	inserted = yield self._simple_insert(
	664	"devices",
	665	values={
	666	"user_id": user_id,
	667	"device_id": device_id,
	668	"display_name": initial_device_display_name,
	669	"hidden": False,
	670	},
	671	desc="store_device",
	672	or_ignore=True,
	673	)
	674	if not inserted:
	675	# if the device already exists, check if it's a real device, or
	676	# if the device ID is reserved by something else
	677	hidden = yield self._simple_select_one_onecol(
	678	"devices",
	679	keyvalues={"user_id": user_id, "device_id": device_id},
	680	retcol="hidden",
	681	)
	682	if hidden:
	683	raise StoreError(400, "The device ID is in use", Codes.FORBIDDEN)
	684	self.device_id_exists_cache.prefill(key, True)
	685	return inserted
	686	except StoreError:
	687	raise
	688	except Exception as e:
	689	logger.error(
	690	"store_device with device_id=%s(%r) user_id=%s(%r)"
	691	" display_name=%s(%r) failed: %s",
	692	type(device_id).__name__,
	693	device_id,
	694	type(user_id).__name__,
	695	user_id,
	696	type(initial_device_display_name).__name__,
	697	initial_device_display_name,
	698	e,
	699	)
	700	raise StoreError(500, "Problem storing device.")
	701
	702	@defer.inlineCallbacks
	703	def delete_device(self, user_id, device_id):
	704	"""Delete a device.
	705
	706	Args:
	707	user_id (str): The ID of the user which owns the device
	708	device_id (str): The ID of the device to delete
	709	Returns:
	710	defer.Deferred
	711	"""
	712	yield self._simple_delete_one(
	713	table="devices",
	714	keyvalues={"user_id": user_id, "device_id": device_id, "hidden": False},
	715	desc="delete_device",
	716	)
	717
	718	self.device_id_exists_cache.invalidate((user_id, device_id))
	719
	720	@defer.inlineCallbacks
	721	def delete_devices(self, user_id, device_ids):
	722	"""Deletes several devices.
	723
	724	Args:
	725	user_id (str): The ID of the user which owns the devices
	726	device_ids (list): The IDs of the devices to delete
	727	Returns:
	728	defer.Deferred
	729	"""
	730	yield self._simple_delete_many(
	731	table="devices",
	732	column="device_id",
	733	iterable=device_ids,
	734	keyvalues={"user_id": user_id, "hidden": False},
	735	desc="delete_devices",
	736	)
	737	for device_id in device_ids:
	738	self.device_id_exists_cache.invalidate((user_id, device_id))
	739
	740	def update_device(self, user_id, device_id, new_display_name=None):
	741	"""Update a device. Only updates the device if it is not marked as
	742	hidden.
	743
	744	Args:
	745	user_id (str): The ID of the user which owns the device
	746	device_id (str): The ID of the device to update
	747	new_display_name (str\|None): new displayname for device; None
	748	to leave unchanged
	749	Raises:
	750	StoreError: if the device is not found
	751	Returns:
	752	defer.Deferred
	753	"""
	754	updates = {}
	755	if new_display_name is not None:
	756	updates["display_name"] = new_display_name
	757	if not updates:
	758	return defer.succeed(None)
	759	return self._simple_update_one(
	760	table="devices",
	761	keyvalues={"user_id": user_id, "device_id": device_id, "hidden": False},
	762	updatevalues=updates,
	763	desc="update_device",
	764	)
	765
	766	@defer.inlineCallbacks
	767	def mark_remote_user_device_list_as_unsubscribed(self, user_id):
	768	"""Mark that we no longer track device lists for remote user.
	769	"""
	770	yield self._simple_delete(
	771	table="device_lists_remote_extremeties",
	772	keyvalues={"user_id": user_id},
	773	desc="mark_remote_user_device_list_as_unsubscribed",
	774	)
	775	self.get_device_list_last_stream_id_for_remote.invalidate((user_id,))
	776
	777	def update_remote_device_list_cache_entry(
	778	self, user_id, device_id, content, stream_id
	779	):
	780	"""Updates a single device in the cache of a remote user's devicelist.
	781
	782	Note: assumes that we are the only thread that can be updating this user's
	783	device list.
	784
	785	Args:
	786	user_id (str): User to update device list for
	787	device_id (str): ID of decivice being updated
	788	content (dict): new data on this device
	789	stream_id (int): the version of the device list
	790
	791	Returns:
	792	Deferred[None]
	793	"""
	794	return self.runInteraction(
	795	"update_remote_device_list_cache_entry",
	796	self._update_remote_device_list_cache_entry_txn,
	797	user_id,
	798	device_id,
	799	content,
	800	stream_id,
	801	)
	802
	803	def _update_remote_device_list_cache_entry_txn(
	804	self, txn, user_id, device_id, content, stream_id
	805	):
	806	if content.get("deleted"):
	807	self._simple_delete_txn(
	808	txn,
	809	table="device_lists_remote_cache",
	810	keyvalues={"user_id": user_id, "device_id": device_id},
	811	)
	812
	813	txn.call_after(self.device_id_exists_cache.invalidate, (user_id, device_id))
	814	else:
	815	self._simple_upsert_txn(
	816	txn,
	817	table="device_lists_remote_cache",
	818	keyvalues={"user_id": user_id, "device_id": device_id},
	819	values={"content": json.dumps(content)},
	820	# we don't need to lock, because we assume we are the only thread
	821	# updating this user's devices.
	822	lock=False,
	823	)
	824
	825	txn.call_after(self._get_cached_user_device.invalidate, (user_id, device_id))
	826	txn.call_after(self._get_cached_devices_for_user.invalidate, (user_id,))
	827	txn.call_after(
	828	self.get_device_list_last_stream_id_for_remote.invalidate, (user_id,)
	829	)
	830
	831	self._simple_upsert_txn(
	832	txn,
	833	table="device_lists_remote_extremeties",
	834	keyvalues={"user_id": user_id},
	835	values={"stream_id": stream_id},
	836	# again, we can assume we are the only thread updating this user's
	837	# extremity.
	838	lock=False,
	839	)
	840
	841	def update_remote_device_list_cache(self, user_id, devices, stream_id):
	842	"""Replace the entire cache of the remote user's devices.
	843
	844	Note: assumes that we are the only thread that can be updating this user's
	845	device list.
	846
	847	Args:
	848	user_id (str): User to update device list for
	849	devices (list[dict]): list of device objects supplied over federation
	850	stream_id (int): the version of the device list
	851
	852	Returns:
	853	Deferred[None]
	854	"""
	855	return self.runInteraction(
	856	"update_remote_device_list_cache",
	857	self._update_remote_device_list_cache_txn,
	858	user_id,
	859	devices,
	860	stream_id,
	861	)
	862
	863	def _update_remote_device_list_cache_txn(self, txn, user_id, devices, stream_id):
	864	self._simple_delete_txn(
	865	txn, table="device_lists_remote_cache", keyvalues={"user_id": user_id}
	866	)
	867
	868	self._simple_insert_many_txn(
	869	txn,
	870	table="device_lists_remote_cache",
	871	values=[
	872	{
	873	"user_id": user_id,
	874	"device_id": content["device_id"],
	875	"content": json.dumps(content),
	876	}
	877	for content in devices
	878	],
	879	)
	880
	881	txn.call_after(self._get_cached_devices_for_user.invalidate, (user_id,))
	882	txn.call_after(self._get_cached_user_device.invalidate_many, (user_id,))
	883	txn.call_after(
	884	self.get_device_list_last_stream_id_for_remote.invalidate, (user_id,)
	885	)
	886
	887	self._simple_upsert_txn(
	888	txn,
	889	table="device_lists_remote_extremeties",
	890	keyvalues={"user_id": user_id},
	891	values={"stream_id": stream_id},
	892	# we don't need to lock, because we can assume we are the only thread
	893	# updating this user's extremity.
	894	lock=False,
	895	)
	896
	897	@defer.inlineCallbacks
	898	def add_device_change_to_streams(self, user_id, device_ids, hosts):
	899	"""Persist that a user's devices have been updated, and which hosts
	900	(if any) should be poked.
	901	"""
	902	with self._device_list_id_gen.get_next() as stream_id:
	903	yield self.runInteraction(
	904	"add_device_change_to_streams",
	905	self._add_device_change_txn,
	906	user_id,
	907	device_ids,
	908	hosts,
	909	stream_id,
	910	)
	911	return stream_id
	912
	913	def _add_device_change_txn(self, txn, user_id, device_ids, hosts, stream_id):
	914	now = self._clock.time_msec()
	915
	916	txn.call_after(
	917	self._device_list_stream_cache.entity_has_changed, user_id, stream_id
	918	)
	919	for host in hosts:
	920	txn.call_after(
	921	self._device_list_federation_stream_cache.entity_has_changed,
	922	host,
	923	stream_id,
	924	)
	925
	926	# Delete older entries in the table, as we really only care about
	927	# when the latest change happened.
	928	txn.executemany(
	929	"""
	930	DELETE FROM device_lists_stream
	931	WHERE user_id = ? AND device_id = ? AND stream_id < ?
	932	""",
	933	[(user_id, device_id, stream_id) for device_id in device_ids],
	934	)
	935
	936	self._simple_insert_many_txn(
	937	txn,
	938	table="device_lists_stream",
	939	values=[
	940	{"stream_id": stream_id, "user_id": user_id, "device_id": device_id}
	941	for device_id in device_ids
	942	],
	943	)
	944
	945	context = get_active_span_text_map()
	946
	947	self._simple_insert_many_txn(
	948	txn,
	949	table="device_lists_outbound_pokes",
	950	values=[
	951	{
	952	"destination": destination,
	953	"stream_id": stream_id,
	954	"user_id": user_id,
	955	"device_id": device_id,
	956	"sent": False,
	957	"ts": now,
	958	"opentracing_context": json.dumps(context)
	959	if whitelisted_homeserver(destination)
	960	else "{}",
	961	}
	962	for destination in hosts
	963	for device_id in device_ids
	964	],
	965	)
	966
	967	def _prune_old_outbound_device_pokes(self):
	968	"""Delete old entries out of the device_lists_outbound_pokes to ensure
	969	that we don't fill up due to dead servers. We keep one entry per
	970	(destination, user_id) tuple to ensure that the prev_ids remain correct
	971	if the server does come back.
	972	"""
	973	yesterday = self._clock.time_msec() - 24 * 60 * 60 * 1000
	974
	975	def _prune_txn(txn):
	976	select_sql = """
	977	SELECT destination, user_id, max(stream_id) as stream_id
	978	FROM device_lists_outbound_pokes
	979	GROUP BY destination, user_id
	980	HAVING min(ts) < ? AND count(*) > 1
	981	"""
	982
	983	txn.execute(select_sql, (yesterday,))
	984	rows = txn.fetchall()
	985
	986	if not rows:
	987	return
	988
	989	delete_sql = """
	990	DELETE FROM device_lists_outbound_pokes
	991	WHERE ts < ? AND destination = ? AND user_id = ? AND stream_id < ?
	992	"""
	993
	994	txn.executemany(
	995	delete_sql, ((yesterday, row[0], row[1], row[2]) for row in rows)
	996	)
	997
	998	# Since we've deleted unsent deltas, we need to remove the entry
	999	# of last successful sent so that the prev_ids are correctly set.
	1000	sql = """
	1001	DELETE FROM device_lists_outbound_last_success
	1002	WHERE destination = ? AND user_id = ?
	1003	"""
	1004	txn.executemany(sql, ((row[0], row[1]) for row in rows))
	1005
	1006	logger.info("Pruned %d device list outbound pokes", txn.rowcount)
	1007
	1008	return run_as_background_process(
	1009	"prune_old_outbound_device_pokes",
	1010	self.runInteraction,
	1011	"_prune_old_outbound_device_pokes",
	1012	_prune_txn,
	1013	)

+173

-0

synapse/storage/data_stores/main/directory.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2014-2016 OpenMarket Ltd
	2	#
	3	# Licensed under the Apache License, Version 2.0 (the "License");
	4	# you may not use this file except in compliance with the License.
	5	# You may obtain a copy of the License at
	6	#
	7	# http://www.apache.org/licenses/LICENSE-2.0
	8	#
	9	# Unless required by applicable law or agreed to in writing, software
	10	# distributed under the License is distributed on an "AS IS" BASIS,
	11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	# See the License for the specific language governing permissions and
	13	# limitations under the License.
	14
	15	from collections import namedtuple
	16
	17	from twisted.internet import defer
	18
	19	from synapse.api.errors import SynapseError
	20	from synapse.storage._base import SQLBaseStore
	21	from synapse.util.caches.descriptors import cached
	22
	23	RoomAliasMapping = namedtuple("RoomAliasMapping", ("room_id", "room_alias", "servers"))
	24
	25
	26	class DirectoryWorkerStore(SQLBaseStore):
	27	@defer.inlineCallbacks
	28	def get_association_from_room_alias(self, room_alias):
	29	""" Get's the room_id and server list for a given room_alias
	30
	31	Args:
	32	room_alias (RoomAlias)
	33
	34	Returns:
	35	Deferred: results in namedtuple with keys "room_id" and
	36	"servers" or None if no association can be found
	37	"""
	38	room_id = yield self._simple_select_one_onecol(
	39	"room_aliases",
	40	{"room_alias": room_alias.to_string()},
	41	"room_id",
	42	allow_none=True,
	43	desc="get_association_from_room_alias",
	44	)
	45
	46	if not room_id:
	47	return None
	48
	49	servers = yield self._simple_select_onecol(
	50	"room_alias_servers",
	51	{"room_alias": room_alias.to_string()},
	52	"server",
	53	desc="get_association_from_room_alias",
	54	)
	55
	56	if not servers:
	57	return None
	58
	59	return RoomAliasMapping(room_id, room_alias.to_string(), servers)
	60
	61	def get_room_alias_creator(self, room_alias):
	62	return self._simple_select_one_onecol(
	63	table="room_aliases",
	64	keyvalues={"room_alias": room_alias},
	65	retcol="creator",
	66	desc="get_room_alias_creator",
	67	)
	68
	69	@cached(max_entries=5000)
	70	def get_aliases_for_room(self, room_id):
	71	return self._simple_select_onecol(
	72	"room_aliases",
	73	{"room_id": room_id},
	74	"room_alias",
	75	desc="get_aliases_for_room",
	76	)
	77
	78
	79	class DirectoryStore(DirectoryWorkerStore):
	80	@defer.inlineCallbacks
	81	def create_room_alias_association(self, room_alias, room_id, servers, creator=None):
	82	""" Creates an association between a room alias and room_id/servers
	83
	84	Args:
	85	room_alias (RoomAlias)
	86	room_id (str)
	87	servers (list)
	88	creator (str): Optional user_id of creator.
	89
	90	Returns:
	91	Deferred
	92	"""
	93
	94	def alias_txn(txn):
	95	self._simple_insert_txn(
	96	txn,
	97	"room_aliases",
	98	{
	99	"room_alias": room_alias.to_string(),
	100	"room_id": room_id,
	101	"creator": creator,
	102	},
	103	)
	104
	105	self._simple_insert_many_txn(
	106	txn,
	107	table="room_alias_servers",
	108	values=[
	109	{"room_alias": room_alias.to_string(), "server": server}
	110	for server in servers
	111	],
	112	)
	113
	114	self._invalidate_cache_and_stream(
	115	txn, self.get_aliases_for_room, (room_id,)
	116	)
	117
	118	try:
	119	ret = yield self.runInteraction("create_room_alias_association", alias_txn)
	120	except self.database_engine.module.IntegrityError:
	121	raise SynapseError(
	122	409, "Room alias %s already exists" % room_alias.to_string()
	123	)
	124	return ret
	125
	126	@defer.inlineCallbacks
	127	def delete_room_alias(self, room_alias):
	128	room_id = yield self.runInteraction(
	129	"delete_room_alias", self._delete_room_alias_txn, room_alias
	130	)
	131
	132	return room_id
	133
	134	def _delete_room_alias_txn(self, txn, room_alias):
	135	txn.execute(
	136	"SELECT room_id FROM room_aliases WHERE room_alias = ?",
	137	(room_alias.to_string(),),
	138	)
	139
	140	res = txn.fetchone()
	141	if res:
	142	room_id = res[0]
	143	else:
	144	return None
	145
	146	txn.execute(
	147	"DELETE FROM room_aliases WHERE room_alias = ?", (room_alias.to_string(),)
	148	)
	149
	150	txn.execute(
	151	"DELETE FROM room_alias_servers WHERE room_alias = ?",
	152	(room_alias.to_string(),),
	153	)
	154
	155	self._invalidate_cache_and_stream(txn, self.get_aliases_for_room, (room_id,))
	156
	157	return room_id
	158
	159	def update_aliases_for_room(self, old_room_id, new_room_id, creator):
	160	def _update_aliases_for_room_txn(txn):
	161	sql = "UPDATE room_aliases SET room_id = ?, creator = ? WHERE room_id = ?"
	162	txn.execute(sql, (new_room_id, creator, old_room_id))
	163	self._invalidate_cache_and_stream(
	164	txn, self.get_aliases_for_room, (old_room_id,)
	165	)
	166	self._invalidate_cache_and_stream(
	167	txn, self.get_aliases_for_room, (new_room_id,)
	168	)
	169
	170	return self.runInteraction(
	171	"_update_aliases_for_room_txn", _update_aliases_for_room_txn
	172	)

+336

-0

synapse/storage/data_stores/main/e2e_room_keys.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2017 New Vector Ltd
	2	#
	3	# Licensed under the Apache License, Version 2.0 (the "License");
	4	# you may not use this file except in compliance with the License.
	5	# You may obtain a copy of the License at
	6	#
	7	# http://www.apache.org/licenses/LICENSE-2.0
	8	#
	9	# Unless required by applicable law or agreed to in writing, software
	10	# distributed under the License is distributed on an "AS IS" BASIS,
	11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	# See the License for the specific language governing permissions and
	13	# limitations under the License.
	14
	15	import json
	16
	17	from twisted.internet import defer
	18
	19	from synapse.api.errors import StoreError
	20	from synapse.logging.opentracing import log_kv, trace
	21	from synapse.storage._base import SQLBaseStore
	22
	23
	24	class EndToEndRoomKeyStore(SQLBaseStore):
	25	@defer.inlineCallbacks
	26	def get_e2e_room_key(self, user_id, version, room_id, session_id):
	27	"""Get the encrypted E2E room key for a given session from a given
	28	backup version of room_keys. We only store the 'best' room key for a given
	29	session at a given time, as determined by the handler.
	30
	31	Args:
	32	user_id(str): the user whose backup we're querying
	33	version(str): the version ID of the backup for the set of keys we're querying
	34	room_id(str): the ID of the room whose keys we're querying.
	35	This is a bit redundant as it's implied by the session_id, but
	36	we include for consistency with the rest of the API.
	37	session_id(str): the session whose room_key we're querying.
	38
	39	Returns:
	40	A deferred dict giving the session_data and message metadata for
	41	this room key.
	42	"""
	43
	44	row = yield self._simple_select_one(
	45	table="e2e_room_keys",
	46	keyvalues={
	47	"user_id": user_id,
	48	"version": version,
	49	"room_id": room_id,
	50	"session_id": session_id,
	51	},
	52	retcols=(
	53	"first_message_index",
	54	"forwarded_count",
	55	"is_verified",
	56	"session_data",
	57	),
	58	desc="get_e2e_room_key",
	59	)
	60
	61	row["session_data"] = json.loads(row["session_data"])
	62
	63	return row
	64
	65	@defer.inlineCallbacks
	66	def set_e2e_room_key(self, user_id, version, room_id, session_id, room_key):
	67	"""Replaces or inserts the encrypted E2E room key for a given session in
	68	a given backup
	69
	70	Args:
	71	user_id(str): the user whose backup we're setting
	72	version(str): the version ID of the backup we're updating
	73	room_id(str): the ID of the room whose keys we're setting
	74	session_id(str): the session whose room_key we're setting
	75	room_key(dict): the room_key being set
	76	Raises:
	77	StoreError
	78	"""
	79
	80	yield self._simple_upsert(
	81	table="e2e_room_keys",
	82	keyvalues={
	83	"user_id": user_id,
	84	"version": version,
	85	"room_id": room_id,
	86	"session_id": session_id,
	87	},
	88	values={
	89	"first_message_index": room_key["first_message_index"],
	90	"forwarded_count": room_key["forwarded_count"],
	91	"is_verified": room_key["is_verified"],
	92	"session_data": json.dumps(room_key["session_data"]),
	93	},
	94	lock=False,
	95	)
	96	log_kv(
	97	{
	98	"message": "Set room key",
	99	"room_id": room_id,
	100	"session_id": session_id,
	101	"room_key": room_key,
	102	}
	103	)
	104
	105	@trace
	106	@defer.inlineCallbacks
	107	def get_e2e_room_keys(self, user_id, version, room_id=None, session_id=None):
	108	"""Bulk get the E2E room keys for a given backup, optionally filtered to a given
	109	room, or a given session.
	110
	111	Args:
	112	user_id(str): the user whose backup we're querying
	113	version(str): the version ID of the backup for the set of keys we're querying
	114	room_id(str): Optional. the ID of the room whose keys we're querying, if any.
	115	If not specified, we return the keys for all the rooms in the backup.
	116	session_id(str): Optional. the session whose room_key we're querying, if any.
	117	If specified, we also require the room_id to be specified.
	118	If not specified, we return all the keys in this version of
	119	the backup (or for the specified room)
	120
	121	Returns:
	122	A deferred list of dicts giving the session_data and message metadata for
	123	these room keys.
	124	"""
	125
	126	try:
	127	version = int(version)
	128	except ValueError:
	129	return {"rooms": {}}
	130
	131	keyvalues = {"user_id": user_id, "version": version}
	132	if room_id:
	133	keyvalues["room_id"] = room_id
	134	if session_id:
	135	keyvalues["session_id"] = session_id
	136
	137	rows = yield self._simple_select_list(
	138	table="e2e_room_keys",
	139	keyvalues=keyvalues,
	140	retcols=(
	141	"user_id",
	142	"room_id",
	143	"session_id",
	144	"first_message_index",
	145	"forwarded_count",
	146	"is_verified",
	147	"session_data",
	148	),
	149	desc="get_e2e_room_keys",
	150	)
	151
	152	sessions = {"rooms": {}}
	153	for row in rows:
	154	room_entry = sessions["rooms"].setdefault(row["room_id"], {"sessions": {}})
	155	room_entry["sessions"][row["session_id"]] = {
	156	"first_message_index": row["first_message_index"],
	157	"forwarded_count": row["forwarded_count"],
	158	"is_verified": row["is_verified"],
	159	"session_data": json.loads(row["session_data"]),
	160	}
	161
	162	return sessions
	163
	164	@trace
	165	@defer.inlineCallbacks
	166	def delete_e2e_room_keys(self, user_id, version, room_id=None, session_id=None):
	167	"""Bulk delete the E2E room keys for a given backup, optionally filtered to a given
	168	room or a given session.
	169
	170	Args:
	171	user_id(str): the user whose backup we're deleting from
	172	version(str): the version ID of the backup for the set of keys we're deleting
	173	room_id(str): Optional. the ID of the room whose keys we're deleting, if any.
	174	If not specified, we delete the keys for all the rooms in the backup.
	175	session_id(str): Optional. the session whose room_key we're querying, if any.
	176	If specified, we also require the room_id to be specified.
	177	If not specified, we delete all the keys in this version of
	178	the backup (or for the specified room)
	179
	180	Returns:
	181	A deferred of the deletion transaction
	182	"""
	183
	184	keyvalues = {"user_id": user_id, "version": int(version)}
	185	if room_id:
	186	keyvalues["room_id"] = room_id
	187	if session_id:
	188	keyvalues["session_id"] = session_id
	189
	190	yield self._simple_delete(
	191	table="e2e_room_keys", keyvalues=keyvalues, desc="delete_e2e_room_keys"
	192	)
	193
	194	@staticmethod
	195	def _get_current_version(txn, user_id):
	196	txn.execute(
	197	"SELECT MAX(version) FROM e2e_room_keys_versions "
	198	"WHERE user_id=? AND deleted=0",
	199	(user_id,),
	200	)
	201	row = txn.fetchone()
	202	if not row:
	203	raise StoreError(404, "No current backup version")
	204	return row[0]
	205
	206	def get_e2e_room_keys_version_info(self, user_id, version=None):
	207	"""Get info metadata about a version of our room_keys backup.
	208
	209	Args:
	210	user_id(str): the user whose backup we're querying
	211	version(str): Optional. the version ID of the backup we're querying about
	212	If missing, we return the information about the current version.
	213	Raises:
	214	StoreError: with code 404 if there are no e2e_room_keys_versions present
	215	Returns:
	216	A deferred dict giving the info metadata for this backup version, with
	217	fields including:
	218	version(str)
	219	algorithm(str)
	220	auth_data(object): opaque dict supplied by the client
	221	"""
	222
	223	def _get_e2e_room_keys_version_info_txn(txn):
	224	if version is None:
	225	this_version = self._get_current_version(txn, user_id)
	226	else:
	227	try:
	228	this_version = int(version)
	229	except ValueError:
	230	# Our versions are all ints so if we can't convert it to an integer,
	231	# it isn't there.
	232	raise StoreError(404, "No row found")
	233
	234	result = self._simple_select_one_txn(
	235	txn,
	236	table="e2e_room_keys_versions",
	237	keyvalues={"user_id": user_id, "version": this_version, "deleted": 0},
	238	retcols=("version", "algorithm", "auth_data"),
	239	)
	240	result["auth_data"] = json.loads(result["auth_data"])
	241	result["version"] = str(result["version"])
	242	return result
	243
	244	return self.runInteraction(
	245	"get_e2e_room_keys_version_info", _get_e2e_room_keys_version_info_txn
	246	)
	247
	248	@trace
	249	def create_e2e_room_keys_version(self, user_id, info):
	250	"""Atomically creates a new version of this user's e2e_room_keys store
	251	with the given version info.
	252
	253	Args:
	254	user_id(str): the user whose backup we're creating a version
	255	info(dict): the info about the backup version to be created
	256
	257	Returns:
	258	A deferred string for the newly created version ID
	259	"""
	260
	261	def _create_e2e_room_keys_version_txn(txn):
	262	txn.execute(
	263	"SELECT MAX(version) FROM e2e_room_keys_versions WHERE user_id=?",
	264	(user_id,),
	265	)
	266	current_version = txn.fetchone()[0]
	267	if current_version is None:
	268	current_version = "0"
	269
	270	new_version = str(int(current_version) + 1)
	271
	272	self._simple_insert_txn(
	273	txn,
	274	table="e2e_room_keys_versions",
	275	values={
	276	"user_id": user_id,
	277	"version": new_version,
	278	"algorithm": info["algorithm"],
	279	"auth_data": json.dumps(info["auth_data"]),
	280	},
	281	)
	282
	283	return new_version
	284
	285	return self.runInteraction(
	286	"create_e2e_room_keys_version_txn", _create_e2e_room_keys_version_txn
	287	)
	288
	289	@trace
	290	def update_e2e_room_keys_version(self, user_id, version, info):
	291	"""Update a given backup version
	292
	293	Args:
	294	user_id(str): the user whose backup version we're updating
	295	version(str): the version ID of the backup version we're updating
	296	info(dict): the new backup version info to store
	297	"""
	298
	299	return self._simple_update(
	300	table="e2e_room_keys_versions",
	301	keyvalues={"user_id": user_id, "version": version},
	302	updatevalues={"auth_data": json.dumps(info["auth_data"])},
	303	desc="update_e2e_room_keys_version",
	304	)
	305
	306	@trace
	307	def delete_e2e_room_keys_version(self, user_id, version=None):
	308	"""Delete a given backup version of the user's room keys.
	309	Doesn't delete their actual key data.
	310
	311	Args:
	312	user_id(str): the user whose backup version we're deleting
	313	version(str): Optional. the version ID of the backup version we're deleting
	314	If missing, we delete the current backup version info.
	315	Raises:
	316	StoreError: with code 404 if there are no e2e_room_keys_versions present,
	317	or if the version requested doesn't exist.
	318	"""
	319
	320	def _delete_e2e_room_keys_version_txn(txn):
	321	if version is None:
	322	this_version = self._get_current_version(txn, user_id)
	323	else:
	324	this_version = version
	325
	326	return self._simple_update_one_txn(
	327	txn,
	328	table="e2e_room_keys_versions",
	329	keyvalues={"user_id": user_id, "version": this_version},
	330	updatevalues={"deleted": 1},
	331	)
	332
	333	return self.runInteraction(
	334	"delete_e2e_room_keys_version", _delete_e2e_room_keys_version_txn
	335	)

+516

-0

synapse/storage/data_stores/main/end_to_end_keys.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2015, 2016 OpenMarket Ltd
	2	# Copyright 2019 New Vector Ltd
	3	# Copyright 2019 The Matrix.org Foundation C.I.C.
	4	#
	5	# Licensed under the Apache License, Version 2.0 (the "License");
	6	# you may not use this file except in compliance with the License.
	7	# You may obtain a copy of the License at
	8	#
	9	# http://www.apache.org/licenses/LICENSE-2.0
	10	#
	11	# Unless required by applicable law or agreed to in writing, software
	12	# distributed under the License is distributed on an "AS IS" BASIS,
	13	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	14	# See the License for the specific language governing permissions and
	15	# limitations under the License.
	16	from six import iteritems
	17
	18	from canonicaljson import encode_canonical_json, json
	19
	20	from twisted.internet import defer
	21
	22	from synapse.logging.opentracing import log_kv, set_tag, trace
	23	from synapse.storage._base import SQLBaseStore, db_to_json
	24	from synapse.util.caches.descriptors import cached
	25
	26
	27	class EndToEndKeyWorkerStore(SQLBaseStore):
	28	@trace
	29	@defer.inlineCallbacks
	30	def get_e2e_device_keys(
	31	self, query_list, include_all_devices=False, include_deleted_devices=False
	32	):
	33	"""Fetch a list of device keys.
	34	Args:
	35	query_list(list): List of pairs of user_ids and device_ids.
	36	include_all_devices (bool): whether to include entries for devices
	37	that don't have device keys
	38	include_deleted_devices (bool): whether to include null entries for
	39	devices which no longer exist (but were in the query_list).
	40	This option only takes effect if include_all_devices is true.
	41	Returns:
	42	Dict mapping from user-id to dict mapping from device_id to
	43	key data. The key data will be a dict in the same format as the
	44	DeviceKeys type returned by POST /_matrix/client/r0/keys/query.
	45	"""
	46	set_tag("query_list", query_list)
	47	if not query_list:
	48	return {}
	49
	50	results = yield self.runInteraction(
	51	"get_e2e_device_keys",
	52	self._get_e2e_device_keys_txn,
	53	query_list,
	54	include_all_devices,
	55	include_deleted_devices,
	56	)
	57
	58	# Build the result structure, un-jsonify the results, and add the
	59	# "unsigned" section
	60	rv = {}
	61	for user_id, device_keys in iteritems(results):
	62	rv[user_id] = {}
	63	for device_id, device_info in iteritems(device_keys):
	64	r = db_to_json(device_info.pop("key_json"))
	65	r["unsigned"] = {}
	66	display_name = device_info["device_display_name"]
	67	if display_name is not None:
	68	r["unsigned"]["device_display_name"] = display_name
	69	if "signatures" in device_info:
	70	for sig_user_id, sigs in device_info["signatures"].items():
	71	r.setdefault("signatures", {}).setdefault(
	72	sig_user_id, {}
	73	).update(sigs)
	74	rv[user_id][device_id] = r
	75
	76	return rv
	77
	78	@trace
	79	def _get_e2e_device_keys_txn(
	80	self, txn, query_list, include_all_devices=False, include_deleted_devices=False
	81	):
	82	set_tag("include_all_devices", include_all_devices)
	83	set_tag("include_deleted_devices", include_deleted_devices)
	84
	85	query_clauses = []
	86	query_params = []
	87	signature_query_clauses = []
	88	signature_query_params = []
	89
	90	if include_all_devices is False:
	91	include_deleted_devices = False
	92
	93	if include_deleted_devices:
	94	deleted_devices = set(query_list)
	95
	96	for (user_id, device_id) in query_list:
	97	query_clause = "user_id = ?"
	98	query_params.append(user_id)
	99	signature_query_clause = "target_user_id = ?"
	100	signature_query_params.append(user_id)
	101
	102	if device_id is not None:
	103	query_clause += " AND device_id = ?"
	104	query_params.append(device_id)
	105	signature_query_clause += " AND target_device_id = ?"
	106	signature_query_params.append(device_id)
	107
	108	signature_query_clause += " AND user_id = ?"
	109	signature_query_params.append(user_id)
	110
	111	query_clauses.append(query_clause)
	112	signature_query_clauses.append(signature_query_clause)
	113
	114	sql = (
	115	"SELECT user_id, device_id, "
	116	" d.display_name AS device_display_name, "
	117	" k.key_json"
	118	" FROM devices d"
	119	" %s JOIN e2e_device_keys_json k USING (user_id, device_id)"
	120	" WHERE %s AND NOT d.hidden"
	121	) % (
	122	"LEFT" if include_all_devices else "INNER",
	123	" OR ".join("(" + q + ")" for q in query_clauses),
	124	)
	125
	126	txn.execute(sql, query_params)
	127	rows = self.cursor_to_dict(txn)
	128
	129	result = {}
	130	for row in rows:
	131	if include_deleted_devices:
	132	deleted_devices.remove((row["user_id"], row["device_id"]))
	133	result.setdefault(row["user_id"], {})[row["device_id"]] = row
	134
	135	if include_deleted_devices:
	136	for user_id, device_id in deleted_devices:
	137	result.setdefault(user_id, {})[device_id] = None
	138
	139	# get signatures on the device
	140	signature_sql = (
	141	"SELECT * " " FROM e2e_cross_signing_signatures " " WHERE %s"
	142	) % (" OR ".join("(" + q + ")" for q in signature_query_clauses))
	143
	144	txn.execute(signature_sql, signature_query_params)
	145	rows = self.cursor_to_dict(txn)
	146
	147	for row in rows:
	148	target_user_id = row["target_user_id"]
	149	target_device_id = row["target_device_id"]
	150	if target_user_id in result and target_device_id in result[target_user_id]:
	151	result[target_user_id][target_device_id].setdefault(
	152	"signatures", {}
	153	).setdefault(row["user_id"], {})[row["key_id"]] = row["signature"]
	154
	155	log_kv(result)
	156	return result
	157
	158	@defer.inlineCallbacks
	159	def get_e2e_one_time_keys(self, user_id, device_id, key_ids):
	160	"""Retrieve a number of one-time keys for a user
	161
	162	Args:
	163	user_id(str): id of user to get keys for
	164	device_id(str): id of device to get keys for
	165	key_ids(list[str]): list of key ids (excluding algorithm) to
	166	retrieve
	167
	168	Returns:
	169	deferred resolving to Dict[(str, str), str]: map from (algorithm,
	170	key_id) to json string for key
	171	"""
	172
	173	rows = yield self._simple_select_many_batch(
	174	table="e2e_one_time_keys_json",
	175	column="key_id",
	176	iterable=key_ids,
	177	retcols=("algorithm", "key_id", "key_json"),
	178	keyvalues={"user_id": user_id, "device_id": device_id},
	179	desc="add_e2e_one_time_keys_check",
	180	)
	181	result = {(row["algorithm"], row["key_id"]): row["key_json"] for row in rows}
	182	log_kv({"message": "Fetched one time keys for user", "one_time_keys": result})
	183	return result
	184
	185	@defer.inlineCallbacks
	186	def add_e2e_one_time_keys(self, user_id, device_id, time_now, new_keys):
	187	"""Insert some new one time keys for a device. Errors if any of the
	188	keys already exist.
	189
	190	Args:
	191	user_id(str): id of user to get keys for
	192	device_id(str): id of device to get keys for
	193	time_now(long): insertion time to record (ms since epoch)
	194	new_keys(iterable[(str, str, str)]: keys to add - each a tuple of
	195	(algorithm, key_id, key json)
	196	"""
	197
	198	def _add_e2e_one_time_keys(txn):
	199	set_tag("user_id", user_id)
	200	set_tag("device_id", device_id)
	201	set_tag("new_keys", new_keys)
	202	# We are protected from race between lookup and insertion due to
	203	# a unique constraint. If there is a race of two calls to
	204	# `add_e2e_one_time_keys` then they'll conflict and we will only
	205	# insert one set.
	206	self._simple_insert_many_txn(
	207	txn,
	208	table="e2e_one_time_keys_json",
	209	values=[
	210	{
	211	"user_id": user_id,
	212	"device_id": device_id,
	213	"algorithm": algorithm,
	214	"key_id": key_id,
	215	"ts_added_ms": time_now,
	216	"key_json": json_bytes,
	217	}
	218	for algorithm, key_id, json_bytes in new_keys
	219	],
	220	)
	221	self._invalidate_cache_and_stream(
	222	txn, self.count_e2e_one_time_keys, (user_id, device_id)
	223	)
	224
	225	yield self.runInteraction(
	226	"add_e2e_one_time_keys_insert", _add_e2e_one_time_keys
	227	)
	228
	229	@cached(max_entries=10000)
	230	def count_e2e_one_time_keys(self, user_id, device_id):
	231	""" Count the number of one time keys the server has for a device
	232	Returns:
	233	Dict mapping from algorithm to number of keys for that algorithm.
	234	"""
	235
	236	def _count_e2e_one_time_keys(txn):
	237	sql = (
	238	"SELECT algorithm, COUNT(key_id) FROM e2e_one_time_keys_json"
	239	" WHERE user_id = ? AND device_id = ?"
	240	" GROUP BY algorithm"
	241	)
	242	txn.execute(sql, (user_id, device_id))
	243	result = {}
	244	for algorithm, key_count in txn:
	245	result[algorithm] = key_count
	246	return result
	247
	248	return self.runInteraction("count_e2e_one_time_keys", _count_e2e_one_time_keys)
	249
	250
	251	class EndToEndKeyStore(EndToEndKeyWorkerStore, SQLBaseStore):
	252	def set_e2e_device_keys(self, user_id, device_id, time_now, device_keys):
	253	"""Stores device keys for a device. Returns whether there was a change
	254	or the keys were already in the database.
	255	"""
	256
	257	def _set_e2e_device_keys_txn(txn):
	258	set_tag("user_id", user_id)
	259	set_tag("device_id", device_id)
	260	set_tag("time_now", time_now)
	261	set_tag("device_keys", device_keys)
	262
	263	old_key_json = self._simple_select_one_onecol_txn(
	264	txn,
	265	table="e2e_device_keys_json",
	266	keyvalues={"user_id": user_id, "device_id": device_id},
	267	retcol="key_json",
	268	allow_none=True,
	269	)
	270
	271	# In py3 we need old_key_json to match new_key_json type. The DB
	272	# returns unicode while encode_canonical_json returns bytes.
	273	new_key_json = encode_canonical_json(device_keys).decode("utf-8")
	274
	275	if old_key_json == new_key_json:
	276	log_kv({"Message": "Device key already stored."})
	277	return False
	278
	279	self._simple_upsert_txn(
	280	txn,
	281	table="e2e_device_keys_json",
	282	keyvalues={"user_id": user_id, "device_id": device_id},
	283	values={"ts_added_ms": time_now, "key_json": new_key_json},
	284	)
	285	log_kv({"message": "Device keys stored."})
	286	return True
	287
	288	return self.runInteraction("set_e2e_device_keys", _set_e2e_device_keys_txn)
	289
	290	def claim_e2e_one_time_keys(self, query_list):
	291	"""Take a list of one time keys out of the database"""
	292
	293	@trace
	294	def _claim_e2e_one_time_keys(txn):
	295	sql = (
	296	"SELECT key_id, key_json FROM e2e_one_time_keys_json"
	297	" WHERE user_id = ? AND device_id = ? AND algorithm = ?"
	298	" LIMIT 1"
	299	)
	300	result = {}
	301	delete = []
	302	for user_id, device_id, algorithm in query_list:
	303	user_result = result.setdefault(user_id, {})
	304	device_result = user_result.setdefault(device_id, {})
	305	txn.execute(sql, (user_id, device_id, algorithm))
	306	for key_id, key_json in txn:
	307	device_result[algorithm + ":" + key_id] = key_json
	308	delete.append((user_id, device_id, algorithm, key_id))
	309	sql = (
	310	"DELETE FROM e2e_one_time_keys_json"
	311	" WHERE user_id = ? AND device_id = ? AND algorithm = ?"
	312	" AND key_id = ?"
	313	)
	314	for user_id, device_id, algorithm, key_id in delete:
	315	log_kv(
	316	{
	317	"message": "Executing claim e2e_one_time_keys transaction on database."
	318	}
	319	)
	320	txn.execute(sql, (user_id, device_id, algorithm, key_id))
	321	log_kv({"message": "finished executing and invalidating cache"})
	322	self._invalidate_cache_and_stream(
	323	txn, self.count_e2e_one_time_keys, (user_id, device_id)
	324	)
	325	return result
	326
	327	return self.runInteraction("claim_e2e_one_time_keys", _claim_e2e_one_time_keys)
	328
	329	def delete_e2e_keys_by_device(self, user_id, device_id):
	330	def delete_e2e_keys_by_device_txn(txn):
	331	log_kv(
	332	{
	333	"message": "Deleting keys for device",
	334	"device_id": device_id,
	335	"user_id": user_id,
	336	}
	337	)
	338	self._simple_delete_txn(
	339	txn,
	340	table="e2e_device_keys_json",
	341	keyvalues={"user_id": user_id, "device_id": device_id},
	342	)
	343	self._simple_delete_txn(
	344	txn,
	345	table="e2e_one_time_keys_json",
	346	keyvalues={"user_id": user_id, "device_id": device_id},
	347	)
	348	self._invalidate_cache_and_stream(
	349	txn, self.count_e2e_one_time_keys, (user_id, device_id)
	350	)
	351
	352	return self.runInteraction(
	353	"delete_e2e_keys_by_device", delete_e2e_keys_by_device_txn
	354	)
	355
	356	def _set_e2e_cross_signing_key_txn(self, txn, user_id, key_type, key):
	357	"""Set a user's cross-signing key.
	358
	359	Args:
	360	txn (twisted.enterprise.adbapi.Connection): db connection
	361	user_id (str): the user to set the signing key for
	362	key_type (str): the type of key that is being set: either 'master'
	363	for a master key, 'self_signing' for a self-signing key, or
	364	'user_signing' for a user-signing key
	365	key (dict): the key data
	366	"""
	367	# the cross-signing keys need to occupy the same namespace as devices,
	368	# since signatures are identified by device ID. So add an entry to the
	369	# device table to make sure that we don't have a collision with device
	370	# IDs
	371
	372	# the 'key' dict will look something like:
	373	# {
	374	# "user_id": "@alice:example.com",
	375	# "usage": ["self_signing"],
	376	# "keys": {
	377	# "ed25519:base64+self+signing+public+key": "base64+self+signing+public+key",
	378	# },
	379	# "signatures": {
	380	# "@alice:example.com": {
	381	# "ed25519:base64+master+public+key": "base64+signature"
	382	# }
	383	# }
	384	# }
	385	# The "keys" property must only have one entry, which will be the public
	386	# key, so we just grab the first value in there
	387	pubkey = next(iter(key["keys"].values()))
	388	self._simple_insert_txn(
	389	txn,
	390	"devices",
	391	values={
	392	"user_id": user_id,
	393	"device_id": pubkey,
	394	"display_name": key_type + " signing key",
	395	"hidden": True,
	396	},
	397	)
	398
	399	# and finally, store the key itself
	400	with self._cross_signing_id_gen.get_next() as stream_id:
	401	self._simple_insert_txn(
	402	txn,
	403	"e2e_cross_signing_keys",
	404	values={
	405	"user_id": user_id,
	406	"keytype": key_type,
	407	"keydata": json.dumps(key),
	408	"stream_id": stream_id,
	409	},
	410	)
	411
	412	def set_e2e_cross_signing_key(self, user_id, key_type, key):
	413	"""Set a user's cross-signing key.
	414
	415	Args:
	416	user_id (str): the user to set the user-signing key for
	417	key_type (str): the type of cross-signing key to set
	418	key (dict): the key data
	419	"""
	420	return self.runInteraction(
	421	"add_e2e_cross_signing_key",
	422	self._set_e2e_cross_signing_key_txn,
	423	user_id,
	424	key_type,
	425	key,
	426	)
	427
	428	def _get_e2e_cross_signing_key_txn(self, txn, user_id, key_type, from_user_id=None):
	429	"""Returns a user's cross-signing key.
	430
	431	Args:
	432	txn (twisted.enterprise.adbapi.Connection): db connection
	433	user_id (str): the user whose key is being requested
	434	key_type (str): the type of key that is being set: either 'master'
	435	for a master key, 'self_signing' for a self-signing key, or
	436	'user_signing' for a user-signing key
	437	from_user_id (str): if specified, signatures made by this user on
	438	the key will be included in the result
	439
	440	Returns:
	441	dict of the key data or None if not found
	442	"""
	443	sql = (
	444	"SELECT keydata "
	445	" FROM e2e_cross_signing_keys "
	446	" WHERE user_id = ? AND keytype = ? ORDER BY stream_id DESC LIMIT 1"
	447	)
	448	txn.execute(sql, (user_id, key_type))
	449	row = txn.fetchone()
	450	if not row:
	451	return None
	452	key = json.loads(row[0])
	453
	454	device_id = None
	455	for k in key["keys"].values():
	456	device_id = k
	457
	458	if from_user_id is not None:
	459	sql = (
	460	"SELECT key_id, signature "
	461	" FROM e2e_cross_signing_signatures "
	462	" WHERE user_id = ? "
	463	" AND target_user_id = ? "
	464	" AND target_device_id = ? "
	465	)
	466	txn.execute(sql, (from_user_id, user_id, device_id))
	467	row = txn.fetchone()
	468	if row:
	469	key.setdefault("signatures", {}).setdefault(from_user_id, {})[
	470	row[0]
	471	] = row[1]
	472
	473	return key
	474
	475	def get_e2e_cross_signing_key(self, user_id, key_type, from_user_id=None):
	476	"""Returns a user's cross-signing key.
	477
	478	Args:
	479	user_id (str): the user whose self-signing key is being requested
	480	key_type (str): the type of cross-signing key to get
	481	from_user_id (str): if specified, signatures made by this user on
	482	the self-signing key will be included in the result
	483
	484	Returns:
	485	dict of the key data or None if not found
	486	"""
	487	return self.runInteraction(
	488	"get_e2e_cross_signing_key",
	489	self._get_e2e_cross_signing_key_txn,
	490	user_id,
	491	key_type,
	492	from_user_id,
	493	)
	494
	495	def store_e2e_cross_signing_signatures(self, user_id, signatures):
	496	"""Stores cross-signing signatures.
	497
	498	Args:
	499	user_id (str): the user who made the signatures
	500	signatures (iterable[SignatureListItem]): signatures to add
	501	"""
	502	return self._simple_insert_many(
	503	"e2e_cross_signing_signatures",
	504	[
	505	{
	506	"user_id": user_id,
	507	"key_id": item.signing_key_id,
	508	"target_user_id": item.target_user_id,
	509	"target_device_id": item.target_device_id,
	510	"signature": item.signature,
	511	}
	512	for item in signatures
	513	],
	514	"add_e2e_signing_key",
	515	)

+672

-0

synapse/storage/data_stores/main/event_federation.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2014-2016 OpenMarket Ltd
	2	#
	3	# Licensed under the Apache License, Version 2.0 (the "License");
	4	# you may not use this file except in compliance with the License.
	5	# You may obtain a copy of the License at
	6	#
	7	# http://www.apache.org/licenses/LICENSE-2.0
	8	#
	9	# Unless required by applicable law or agreed to in writing, software
	10	# distributed under the License is distributed on an "AS IS" BASIS,
	11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	# See the License for the specific language governing permissions and
	13	# limitations under the License.
	14	import itertools
	15	import logging
	16	import random
	17
	18	from six.moves import range
	19	from six.moves.queue import Empty, PriorityQueue
	20
	21	from unpaddedbase64 import encode_base64
	22
	23	from twisted.internet import defer
	24
	25	from synapse.api.errors import StoreError
	26	from synapse.metrics.background_process_metrics import run_as_background_process
	27	from synapse.storage._base import SQLBaseStore, make_in_list_sql_clause
	28	from synapse.storage.data_stores.main.events_worker import EventsWorkerStore
	29	from synapse.storage.data_stores.main.signatures import SignatureWorkerStore
	30	from synapse.util.caches.descriptors import cached
	31
	32	logger = logging.getLogger(__name__)
	33
	34
	35	class EventFederationWorkerStore(EventsWorkerStore, SignatureWorkerStore, SQLBaseStore):
	36	def get_auth_chain(self, event_ids, include_given=False):
	37	"""Get auth events for given event_ids. The events must be state events.
	38
	39	Args:
	40	event_ids (list): state events
	41	include_given (bool): include the given events in result
	42
	43	Returns:
	44	list of events
	45	"""
	46	return self.get_auth_chain_ids(
	47	event_ids, include_given=include_given
	48	).addCallback(self.get_events_as_list)
	49
	50	def get_auth_chain_ids(self, event_ids, include_given=False):
	51	"""Get auth events for given event_ids. The events must be state events.
	52
	53	Args:
	54	event_ids (list): state events
	55	include_given (bool): include the given events in result
	56
	57	Returns:
	58	list of event_ids
	59	"""
	60	return self.runInteraction(
	61	"get_auth_chain_ids", self._get_auth_chain_ids_txn, event_ids, include_given
	62	)
	63
	64	def _get_auth_chain_ids_txn(self, txn, event_ids, include_given):
	65	if include_given:
	66	results = set(event_ids)
	67	else:
	68	results = set()
	69
	70	base_sql = "SELECT auth_id FROM event_auth WHERE "
	71
	72	front = set(event_ids)
	73	while front:
	74	new_front = set()
	75	front_list = list(front)
	76	chunks = [front_list[x : x + 100] for x in range(0, len(front), 100)]
	77	for chunk in chunks:
	78	clause, args = make_in_list_sql_clause(
	79	txn.database_engine, "event_id", chunk
	80	)
	81	txn.execute(base_sql + clause, list(args))
	82	new_front.update([r[0] for r in txn])
	83
	84	new_front -= results
	85
	86	front = new_front
	87	results.update(front)
	88
	89	return list(results)
	90
	91	def get_oldest_events_in_room(self, room_id):
	92	return self.runInteraction(
	93	"get_oldest_events_in_room", self._get_oldest_events_in_room_txn, room_id
	94	)
	95
	96	def get_oldest_events_with_depth_in_room(self, room_id):
	97	return self.runInteraction(
	98	"get_oldest_events_with_depth_in_room",
	99	self.get_oldest_events_with_depth_in_room_txn,
	100	room_id,
	101	)
	102
	103	def get_oldest_events_with_depth_in_room_txn(self, txn, room_id):
	104	sql = (
	105	"SELECT b.event_id, MAX(e.depth) FROM events as e"
	106	" INNER JOIN event_edges as g"
	107	" ON g.event_id = e.event_id"
	108	" INNER JOIN event_backward_extremities as b"
	109	" ON g.prev_event_id = b.event_id"
	110	" WHERE b.room_id = ? AND g.is_state is ?"
	111	" GROUP BY b.event_id"
	112	)
	113
	114	txn.execute(sql, (room_id, False))
	115
	116	return dict(txn)
	117
	118	@defer.inlineCallbacks
	119	def get_max_depth_of(self, event_ids):
	120	"""Returns the max depth of a set of event IDs
	121
	122	Args:
	123	event_ids (list[str])
	124
	125	Returns
	126	Deferred[int]
	127	"""
	128	rows = yield self._simple_select_many_batch(
	129	table="events",
	130	column="event_id",
	131	iterable=event_ids,
	132	retcols=("depth",),
	133	desc="get_max_depth_of",
	134	)
	135
	136	if not rows:
	137	return 0
	138	else:
	139	return max(row["depth"] for row in rows)
	140
	141	def _get_oldest_events_in_room_txn(self, txn, room_id):
	142	return self._simple_select_onecol_txn(
	143	txn,
	144	table="event_backward_extremities",
	145	keyvalues={"room_id": room_id},
	146	retcol="event_id",
	147	)
	148
	149	@defer.inlineCallbacks
	150	def get_prev_events_for_room(self, room_id):
	151	"""
	152	Gets a subset of the current forward extremities in the given room.
	153
	154	Limits the result to 10 extremities, so that we can avoid creating
	155	events which refer to hundreds of prev_events.
	156
	157	Args:
	158	room_id (str): room_id
	159
	160	Returns:
	161	Deferred[list[(str, dict[str, str], int)]]
	162	for each event, a tuple of (event_id, hashes, depth)
	163	where hashes is a map from algorithm to hash.
	164	"""
	165	res = yield self.get_latest_event_ids_and_hashes_in_room(room_id)
	166	if len(res) > 10:
	167	# Sort by reverse depth, so we point to the most recent.
	168	res.sort(key=lambda a: -a[2])
	169
	170	# we use half of the limit for the actual most recent events, and
	171	# the other half to randomly point to some of the older events, to
	172	# make sure that we don't completely ignore the older events.
	173	res = res[0:5] + random.sample(res[5:], 5)
	174
	175	return res
	176
	177	def get_latest_event_ids_and_hashes_in_room(self, room_id):
	178	"""
	179	Gets the current forward extremities in the given room
	180
	181	Args:
	182	room_id (str): room_id
	183
	184	Returns:
	185	Deferred[list[(str, dict[str, str], int)]]
	186	for each event, a tuple of (event_id, hashes, depth)
	187	where hashes is a map from algorithm to hash.
	188	"""
	189
	190	return self.runInteraction(
	191	"get_latest_event_ids_and_hashes_in_room",
	192	self._get_latest_event_ids_and_hashes_in_room,
	193	room_id,
	194	)
	195
	196	def get_rooms_with_many_extremities(self, min_count, limit, room_id_filter):
	197	"""Get the top rooms with at least N extremities.
	198
	199	Args:
	200	min_count (int): The minimum number of extremities
	201	limit (int): The maximum number of rooms to return.
	202	room_id_filter (iterable[str]): room_ids to exclude from the results
	203
	204	Returns:
	205	Deferred[list]: At most `limit` room IDs that have at least
	206	`min_count` extremities, sorted by extremity count.
	207	"""
	208
	209	def _get_rooms_with_many_extremities_txn(txn):
	210	where_clause = "1=1"
	211	if room_id_filter:
	212	where_clause = "room_id NOT IN (%s)" % (
	213	",".join("?" for _ in room_id_filter),
	214	)
	215
	216	sql = """
	217	SELECT room_id FROM event_forward_extremities
	218	WHERE %s
	219	GROUP BY room_id
	220	HAVING count(*) > ?
	221	ORDER BY count(*) DESC
	222	LIMIT ?
	223	""" % (
	224	where_clause,
	225	)
	226
	227	query_args = list(itertools.chain(room_id_filter, [min_count, limit]))
	228	txn.execute(sql, query_args)
	229	return [room_id for room_id, in txn]
	230
	231	return self.runInteraction(
	232	"get_rooms_with_many_extremities", _get_rooms_with_many_extremities_txn
	233	)
	234
	235	@cached(max_entries=5000, iterable=True)
	236	def get_latest_event_ids_in_room(self, room_id):
	237	return self._simple_select_onecol(
	238	table="event_forward_extremities",
	239	keyvalues={"room_id": room_id},
	240	retcol="event_id",
	241	desc="get_latest_event_ids_in_room",
	242	)
	243
	244	def _get_latest_event_ids_and_hashes_in_room(self, txn, room_id):
	245	sql = (
	246	"SELECT e.event_id, e.depth FROM events as e "
	247	"INNER JOIN event_forward_extremities as f "
	248	"ON e.event_id = f.event_id "
	249	"AND e.room_id = f.room_id "
	250	"WHERE f.room_id = ?"
	251	)
	252
	253	txn.execute(sql, (room_id,))
	254
	255	results = []
	256	for event_id, depth in txn.fetchall():
	257	hashes = self._get_event_reference_hashes_txn(txn, event_id)
	258	prev_hashes = {
	259	k: encode_base64(v) for k, v in hashes.items() if k == "sha256"
	260	}
	261	results.append((event_id, prev_hashes, depth))
	262
	263	return results
	264
	265	def get_min_depth(self, room_id):
	266	""" For hte given room, get the minimum depth we have seen for it.
	267	"""
	268	return self.runInteraction(
	269	"get_min_depth", self._get_min_depth_interaction, room_id
	270	)
	271
	272	def _get_min_depth_interaction(self, txn, room_id):
	273	min_depth = self._simple_select_one_onecol_txn(
	274	txn,
	275	table="room_depth",
	276	keyvalues={"room_id": room_id},
	277	retcol="min_depth",
	278	allow_none=True,
	279	)
	280
	281	return int(min_depth) if min_depth is not None else None
	282
	283	def get_forward_extremeties_for_room(self, room_id, stream_ordering):
	284	"""For a given room_id and stream_ordering, return the forward
	285	extremeties of the room at that point in "time".
	286
	287	Throws a StoreError if we have since purged the index for
	288	stream_orderings from that point.
	289
	290	Args:
	291	room_id (str):
	292	stream_ordering (int):
	293
	294	Returns:
	295	deferred, which resolves to a list of event_ids
	296	"""
	297	# We want to make the cache more effective, so we clamp to the last
	298	# change before the given ordering.
	299	last_change = self._events_stream_cache.get_max_pos_of_last_change(room_id)
	300
	301	# We don't always have a full stream_to_exterm_id table, e.g. after
	302	# the upgrade that introduced it, so we make sure we never ask for a
	303	# stream_ordering from before a restart
	304	last_change = max(self._stream_order_on_start, last_change)
	305
	306	# provided the last_change is recent enough, we now clamp the requested
	307	# stream_ordering to it.
	308	if last_change > self.stream_ordering_month_ago:
	309	stream_ordering = min(last_change, stream_ordering)
	310
	311	return self._get_forward_extremeties_for_room(room_id, stream_ordering)
	312
	313	@cached(max_entries=5000, num_args=2)
	314	def _get_forward_extremeties_for_room(self, room_id, stream_ordering):
	315	"""For a given room_id and stream_ordering, return the forward
	316	extremeties of the room at that point in "time".
	317
	318	Throws a StoreError if we have since purged the index for
	319	stream_orderings from that point.
	320	"""
	321
	322	if stream_ordering <= self.stream_ordering_month_ago:
	323	raise StoreError(400, "stream_ordering too old")
	324
	325	sql = """
	326	SELECT event_id FROM stream_ordering_to_exterm
	327	INNER JOIN (
	328	SELECT room_id, MAX(stream_ordering) AS stream_ordering
	329	FROM stream_ordering_to_exterm
	330	WHERE stream_ordering <= ? GROUP BY room_id
	331	) AS rms USING (room_id, stream_ordering)
	332	WHERE room_id = ?
	333	"""
	334
	335	def get_forward_extremeties_for_room_txn(txn):
	336	txn.execute(sql, (stream_ordering, room_id))
	337	return [event_id for event_id, in txn]
	338
	339	return self.runInteraction(
	340	"get_forward_extremeties_for_room", get_forward_extremeties_for_room_txn
	341	)
	342
	343	def get_backfill_events(self, room_id, event_list, limit):
	344	"""Get a list of Events for a given topic that occurred before (and
	345	including) the events in event_list. Return a list of max size `limit`
	346
	347	Args:
	348	txn
	349	room_id (str)
	350	event_list (list)
	351	limit (int)
	352	"""
	353	return (
	354	self.runInteraction(
	355	"get_backfill_events",
	356	self._get_backfill_events,
	357	room_id,
	358	event_list,
	359	limit,
	360	)
	361	.addCallback(self.get_events_as_list)
	362	.addCallback(lambda l: sorted(l, key=lambda e: -e.depth))
	363	)
	364
	365	def _get_backfill_events(self, txn, room_id, event_list, limit):
	366	logger.debug(
	367	"_get_backfill_events: %s, %s, %s", room_id, repr(event_list), limit
	368	)
	369
	370	event_results = set()
	371
	372	# We want to make sure that we do a breadth-first, "depth" ordered
	373	# search.
	374
	375	query = (
	376	"SELECT depth, prev_event_id FROM event_edges"
	377	" INNER JOIN events"
	378	" ON prev_event_id = events.event_id"
	379	" WHERE event_edges.event_id = ?"
	380	" AND event_edges.is_state = ?"
	381	" LIMIT ?"
	382	)
	383
	384	queue = PriorityQueue()
	385
	386	for event_id in event_list:
	387	depth = self._simple_select_one_onecol_txn(
	388	txn,
	389	table="events",
	390	keyvalues={"event_id": event_id, "room_id": room_id},
	391	retcol="depth",
	392	allow_none=True,
	393	)
	394
	395	if depth:
	396	queue.put((-depth, event_id))
	397
	398	while not queue.empty() and len(event_results) < limit:
	399	try:
	400	_, event_id = queue.get_nowait()
	401	except Empty:
	402	break
	403
	404	if event_id in event_results:
	405	continue
	406
	407	event_results.add(event_id)
	408
	409	txn.execute(query, (event_id, False, limit - len(event_results)))
	410
	411	for row in txn:
	412	if row[1] not in event_results:
	413	queue.put((-row[0], row[1]))
	414
	415	return event_results
	416
	417	@defer.inlineCallbacks
	418	def get_missing_events(self, room_id, earliest_events, latest_events, limit):
	419	ids = yield self.runInteraction(
	420	"get_missing_events",
	421	self._get_missing_events,
	422	room_id,
	423	earliest_events,
	424	latest_events,
	425	limit,
	426	)
	427	events = yield self.get_events_as_list(ids)
	428	return events
	429
	430	def _get_missing_events(self, txn, room_id, earliest_events, latest_events, limit):
	431
	432	seen_events = set(earliest_events)
	433	front = set(latest_events) - seen_events
	434	event_results = []
	435
	436	query = (
	437	"SELECT prev_event_id FROM event_edges "
	438	"WHERE room_id = ? AND event_id = ? AND is_state = ? "
	439	"LIMIT ?"
	440	)
	441
	442	while front and len(event_results) < limit:
	443	new_front = set()
	444	for event_id in front:
	445	txn.execute(
	446	query, (room_id, event_id, False, limit - len(event_results))
	447	)
	448
	449	new_results = set(t[0] for t in txn) - seen_events
	450
	451	new_front \|= new_results
	452	seen_events \|= new_results
	453	event_results.extend(new_results)
	454
	455	front = new_front
	456
	457	# we built the list working backwards from latest_events; we now need to
	458	# reverse it so that the events are approximately chronological.
	459	event_results.reverse()
	460	return event_results
	461
	462	@defer.inlineCallbacks
	463	def get_successor_events(self, event_ids):
	464	"""Fetch all events that have the given events as a prev event
	465
	466	Args:
	467	event_ids (iterable[str])
	468
	469	Returns:
	470	Deferred[list[str]]
	471	"""
	472	rows = yield self._simple_select_many_batch(
	473	table="event_edges",
	474	column="prev_event_id",
	475	iterable=event_ids,
	476	retcols=("event_id",),
	477	desc="get_successor_events",
	478	)
	479
	480	return [row["event_id"] for row in rows]
	481
	482
	483	class EventFederationStore(EventFederationWorkerStore):
	484	""" Responsible for storing and serving up the various graphs associated
	485	with an event. Including the main event graph and the auth chains for an
	486	event.
	487
	488	Also has methods for getting the front (latest) and back (oldest) edges
	489	of the event graphs. These are used to generate the parents for new events
	490	and backfilling from another server respectively.
	491	"""
	492
	493	EVENT_AUTH_STATE_ONLY = "event_auth_state_only"
	494
	495	def __init__(self, db_conn, hs):
	496	super(EventFederationStore, self).__init__(db_conn, hs)
	497
	498	self.register_background_update_handler(
	499	self.EVENT_AUTH_STATE_ONLY, self._background_delete_non_state_event_auth
	500	)
	501
	502	hs.get_clock().looping_call(
	503	self._delete_old_forward_extrem_cache, 60 * 60 * 1000
	504	)
	505
	506	def _update_min_depth_for_room_txn(self, txn, room_id, depth):
	507	min_depth = self._get_min_depth_interaction(txn, room_id)
	508
	509	if min_depth and depth >= min_depth:
	510	return
	511
	512	self._simple_upsert_txn(
	513	txn,
	514	table="room_depth",
	515	keyvalues={"room_id": room_id},
	516	values={"min_depth": depth},
	517	)
	518
	519	def _handle_mult_prev_events(self, txn, events):
	520	"""
	521	For the given event, update the event edges table and forward and
	522	backward extremities tables.
	523	"""
	524	self._simple_insert_many_txn(
	525	txn,
	526	table="event_edges",
	527	values=[
	528	{
	529	"event_id": ev.event_id,
	530	"prev_event_id": e_id,
	531	"room_id": ev.room_id,
	532	"is_state": False,
	533	}
	534	for ev in events
	535	for e_id in ev.prev_event_ids()
	536	],
	537	)
	538
	539	self._update_backward_extremeties(txn, events)
	540
	541	def _update_backward_extremeties(self, txn, events):
	542	"""Updates the event_backward_extremities tables based on the new/updated
	543	events being persisted.
	544
	545	This is called for new events and for events that were outliers, but
	546	are now being persisted as non-outliers.
	547
	548	Forward extremities are handled when we first start persisting the events.
	549	"""
	550	events_by_room = {}
	551	for ev in events:
	552	events_by_room.setdefault(ev.room_id, []).append(ev)
	553
	554	query = (
	555	"INSERT INTO event_backward_extremities (event_id, room_id)"
	556	" SELECT ?, ? WHERE NOT EXISTS ("
	557	" SELECT 1 FROM event_backward_extremities"
	558	" WHERE event_id = ? AND room_id = ?"
	559	" )"
	560	" AND NOT EXISTS ("
	561	" SELECT 1 FROM events WHERE event_id = ? AND room_id = ? "
	562	" AND outlier = ?"
	563	" )"
	564	)
	565
	566	txn.executemany(
	567	query,
	568	[
	569	(e_id, ev.room_id, e_id, ev.room_id, e_id, ev.room_id, False)
	570	for ev in events
	571	for e_id in ev.prev_event_ids()
	572	if not ev.internal_metadata.is_outlier()
	573	],
	574	)
	575
	576	query = (
	577	"DELETE FROM event_backward_extremities"
	578	" WHERE event_id = ? AND room_id = ?"
	579	)
	580	txn.executemany(
	581	query,
	582	[
	583	(ev.event_id, ev.room_id)
	584	for ev in events
	585	if not ev.internal_metadata.is_outlier()
	586	],
	587	)
	588
	589	def _delete_old_forward_extrem_cache(self):
	590	def _delete_old_forward_extrem_cache_txn(txn):
	591	# Delete entries older than a month, while making sure we don't delete
	592	# the only entries for a room.
	593	sql = """
	594	DELETE FROM stream_ordering_to_exterm
	595	WHERE
	596	room_id IN (
	597	SELECT room_id
	598	FROM stream_ordering_to_exterm
	599	WHERE stream_ordering > ?
	600	) AND stream_ordering < ?
	601	"""
	602	txn.execute(
	603	sql, (self.stream_ordering_month_ago, self.stream_ordering_month_ago)
	604	)
	605
	606	return run_as_background_process(
	607	"delete_old_forward_extrem_cache",
	608	self.runInteraction,
	609	"_delete_old_forward_extrem_cache",
	610	_delete_old_forward_extrem_cache_txn,
	611	)
	612
	613	def clean_room_for_join(self, room_id):
	614	return self.runInteraction(
	615	"clean_room_for_join", self._clean_room_for_join_txn, room_id
	616	)
	617
	618	def _clean_room_for_join_txn(self, txn, room_id):
	619	query = "DELETE FROM event_forward_extremities WHERE room_id = ?"
	620
	621	txn.execute(query, (room_id,))
	622	txn.call_after(self.get_latest_event_ids_in_room.invalidate, (room_id,))
	623
	624	@defer.inlineCallbacks
	625	def _background_delete_non_state_event_auth(self, progress, batch_size):
	626	def delete_event_auth(txn):
	627	target_min_stream_id = progress.get("target_min_stream_id_inclusive")
	628	max_stream_id = progress.get("max_stream_id_exclusive")
	629
	630	if not target_min_stream_id or not max_stream_id:
	631	txn.execute("SELECT COALESCE(MIN(stream_ordering), 0) FROM events")
	632	rows = txn.fetchall()
	633	target_min_stream_id = rows[0][0]
	634
	635	txn.execute("SELECT COALESCE(MAX(stream_ordering), 0) FROM events")
	636	rows = txn.fetchall()
	637	max_stream_id = rows[0][0]
	638
	639	min_stream_id = max_stream_id - batch_size
	640
	641	sql = """
	642	DELETE FROM event_auth
	643	WHERE event_id IN (
	644	SELECT event_id FROM events
	645	LEFT JOIN state_events USING (room_id, event_id)
	646	WHERE ? <= stream_ordering AND stream_ordering < ?
	647	AND state_key IS null
	648	)
	649	"""
	650
	651	txn.execute(sql, (min_stream_id, max_stream_id))
	652
	653	new_progress = {
	654	"target_min_stream_id_inclusive": target_min_stream_id,
	655	"max_stream_id_exclusive": min_stream_id,
	656	}
	657
	658	self._background_update_progress_txn(
	659	txn, self.EVENT_AUTH_STATE_ONLY, new_progress
	660	)
	661
	662	return min_stream_id >= target_min_stream_id
	663
	664	result = yield self.runInteraction(
	665	self.EVENT_AUTH_STATE_ONLY, delete_event_auth
	666	)
	667
	668	if not result:
	669	yield self._end_background_update(self.EVENT_AUTH_STATE_ONLY)
	670
	671	return batch_size

+960

-0

synapse/storage/data_stores/main/event_push_actions.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2015 OpenMarket Ltd
	2	# Copyright 2018 New Vector Ltd
	3	#
	4	# Licensed under the Apache License, Version 2.0 (the "License");
	5	# you may not use this file except in compliance with the License.
	6	# You may obtain a copy of the License at
	7	#
	8	# http://www.apache.org/licenses/LICENSE-2.0
	9	#
	10	# Unless required by applicable law or agreed to in writing, software
	11	# distributed under the License is distributed on an "AS IS" BASIS,
	12	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	# See the License for the specific language governing permissions and
	14	# limitations under the License.
	15
	16	import logging
	17
	18	from six import iteritems
	19
	20	from canonicaljson import json
	21
	22	from twisted.internet import defer
	23
	24	from synapse.metrics.background_process_metrics import run_as_background_process
	25	from synapse.storage._base import LoggingTransaction, SQLBaseStore
	26	from synapse.util.caches.descriptors import cachedInlineCallbacks
	27
	28	logger = logging.getLogger(__name__)
	29
	30
	31	DEFAULT_NOTIF_ACTION = ["notify", {"set_tweak": "highlight", "value": False}]
	32	DEFAULT_HIGHLIGHT_ACTION = [
	33	"notify",
	34	{"set_tweak": "sound", "value": "default"},
	35	{"set_tweak": "highlight"},
	36	]
	37
	38
	39	def _serialize_action(actions, is_highlight):
	40	"""Custom serializer for actions. This allows us to "compress" common actions.
	41
	42	We use the fact that most users have the same actions for notifs (and for
	43	highlights).
	44	We store these default actions as the empty string rather than the full JSON.
	45	Since the empty string isn't valid JSON there is no risk of this clashing with
	46	any real JSON actions
	47	"""
	48	if is_highlight:
	49	if actions == DEFAULT_HIGHLIGHT_ACTION:
	50	return "" # We use empty string as the column is non-NULL
	51	else:
	52	if actions == DEFAULT_NOTIF_ACTION:
	53	return ""
	54	return json.dumps(actions)
	55
	56
	57	def _deserialize_action(actions, is_highlight):
	58	"""Custom deserializer for actions. This allows us to "compress" common actions
	59	"""
	60	if actions:
	61	return json.loads(actions)
	62
	63	if is_highlight:
	64	return DEFAULT_HIGHLIGHT_ACTION
	65	else:
	66	return DEFAULT_NOTIF_ACTION
	67
	68
	69	class EventPushActionsWorkerStore(SQLBaseStore):
	70	def __init__(self, db_conn, hs):
	71	super(EventPushActionsWorkerStore, self).__init__(db_conn, hs)
	72
	73	# These get correctly set by _find_stream_orderings_for_times_txn
	74	self.stream_ordering_month_ago = None
	75	self.stream_ordering_day_ago = None
	76
	77	cur = LoggingTransaction(
	78	db_conn.cursor(),
	79	name="_find_stream_orderings_for_times_txn",
	80	database_engine=self.database_engine,
	81	)
	82	self._find_stream_orderings_for_times_txn(cur)
	83	cur.close()
	84
	85	self.find_stream_orderings_looping_call = self._clock.looping_call(
	86	self._find_stream_orderings_for_times, 10 * 60 * 1000
	87	)
	88	self._rotate_delay = 3
	89	self._rotate_count = 10000
	90
	91	@cachedInlineCallbacks(num_args=3, tree=True, max_entries=5000)
	92	def get_unread_event_push_actions_by_room_for_user(
	93	self, room_id, user_id, last_read_event_id
	94	):
	95	ret = yield self.runInteraction(
	96	"get_unread_event_push_actions_by_room",
	97	self._get_unread_counts_by_receipt_txn,
	98	room_id,
	99	user_id,
	100	last_read_event_id,
	101	)
	102	return ret
	103
	104	def _get_unread_counts_by_receipt_txn(
	105	self, txn, room_id, user_id, last_read_event_id
	106	):
	107	sql = (
	108	"SELECT stream_ordering"
	109	" FROM events"
	110	" WHERE room_id = ? AND event_id = ?"
	111	)
	112	txn.execute(sql, (room_id, last_read_event_id))
	113	results = txn.fetchall()
	114	if len(results) == 0:
	115	return {"notify_count": 0, "highlight_count": 0}
	116
	117	stream_ordering = results[0][0]
	118
	119	return self._get_unread_counts_by_pos_txn(
	120	txn, room_id, user_id, stream_ordering
	121	)
	122
	123	def _get_unread_counts_by_pos_txn(self, txn, room_id, user_id, stream_ordering):
	124
	125	# First get number of notifications.
	126	# We don't need to put a notif=1 clause as all rows always have
	127	# notif=1
	128	sql = (
	129	"SELECT count(*)"
	130	" FROM event_push_actions ea"
	131	" WHERE"
	132	" user_id = ?"
	133	" AND room_id = ?"
	134	" AND stream_ordering > ?"
	135	)
	136
	137	txn.execute(sql, (user_id, room_id, stream_ordering))
	138	row = txn.fetchone()
	139	notify_count = row[0] if row else 0
	140
	141	txn.execute(
	142	"""
	143	SELECT notif_count FROM event_push_summary
	144	WHERE room_id = ? AND user_id = ? AND stream_ordering > ?
	145	""",
	146	(room_id, user_id, stream_ordering),
	147	)
	148	rows = txn.fetchall()
	149	if rows:
	150	notify_count += rows[0][0]
	151
	152	# Now get the number of highlights
	153	sql = (
	154	"SELECT count(*)"
	155	" FROM event_push_actions ea"
	156	" WHERE"
	157	" highlight = 1"
	158	" AND user_id = ?"
	159	" AND room_id = ?"
	160	" AND stream_ordering > ?"
	161	)
	162
	163	txn.execute(sql, (user_id, room_id, stream_ordering))
	164	row = txn.fetchone()
	165	highlight_count = row[0] if row else 0
	166
	167	return {"notify_count": notify_count, "highlight_count": highlight_count}
	168
	169	@defer.inlineCallbacks
	170	def get_push_action_users_in_range(self, min_stream_ordering, max_stream_ordering):
	171	def f(txn):
	172	sql = (
	173	"SELECT DISTINCT(user_id) FROM event_push_actions WHERE"
	174	" stream_ordering >= ? AND stream_ordering <= ?"
	175	)
	176	txn.execute(sql, (min_stream_ordering, max_stream_ordering))
	177	return [r[0] for r in txn]
	178
	179	ret = yield self.runInteraction("get_push_action_users_in_range", f)
	180	return ret
	181
	182	@defer.inlineCallbacks
	183	def get_unread_push_actions_for_user_in_range_for_http(
	184	self, user_id, min_stream_ordering, max_stream_ordering, limit=20
	185	):
	186	"""Get a list of the most recent unread push actions for a given user,
	187	within the given stream ordering range. Called by the httppusher.
	188
	189	Args:
	190	user_id (str): The user to fetch push actions for.
	191	min_stream_ordering(int): The exclusive lower bound on the
	192	stream ordering of event push actions to fetch.
	193	max_stream_ordering(int): The inclusive upper bound on the
	194	stream ordering of event push actions to fetch.
	195	limit (int): The maximum number of rows to return.
	196	Returns:
	197	A promise which resolves to a list of dicts with the keys "event_id",
	198	"room_id", "stream_ordering", "actions".
	199	The list will be ordered by ascending stream_ordering.
	200	The list will have between 0~limit entries.
	201	"""
	202	# find rooms that have a read receipt in them and return the next
	203	# push actions
	204	def get_after_receipt(txn):
	205	# find rooms that have a read receipt in them and return the next
	206	# push actions
	207	sql = (
	208	"SELECT ep.event_id, ep.room_id, ep.stream_ordering, ep.actions,"
	209	" ep.highlight "
	210	" FROM ("
	211	" SELECT room_id,"
	212	" MAX(stream_ordering) as stream_ordering"
	213	" FROM events"
	214	" INNER JOIN receipts_linearized USING (room_id, event_id)"
	215	" WHERE receipt_type = 'm.read' AND user_id = ?"
	216	" GROUP BY room_id"
	217	") AS rl,"
	218	" event_push_actions AS ep"
	219	" WHERE"
	220	" ep.room_id = rl.room_id"
	221	" AND ep.stream_ordering > rl.stream_ordering"
	222	" AND ep.user_id = ?"
	223	" AND ep.stream_ordering > ?"
	224	" AND ep.stream_ordering <= ?"
	225	" ORDER BY ep.stream_ordering ASC LIMIT ?"
	226	)
	227	args = [user_id, user_id, min_stream_ordering, max_stream_ordering, limit]
	228	txn.execute(sql, args)
	229	return txn.fetchall()
	230
	231	after_read_receipt = yield self.runInteraction(
	232	"get_unread_push_actions_for_user_in_range_http_arr", get_after_receipt
	233	)
	234
	235	# There are rooms with push actions in them but you don't have a read receipt in
	236	# them e.g. rooms you've been invited to, so get push actions for rooms which do
	237	# not have read receipts in them too.
	238	def get_no_receipt(txn):
	239	sql = (
	240	"SELECT ep.event_id, ep.room_id, ep.stream_ordering, ep.actions,"
	241	" ep.highlight "
	242	" FROM event_push_actions AS ep"
	243	" INNER JOIN events AS e USING (room_id, event_id)"
	244	" WHERE"
	245	" ep.room_id NOT IN ("
	246	" SELECT room_id FROM receipts_linearized"
	247	" WHERE receipt_type = 'm.read' AND user_id = ?"
	248	" GROUP BY room_id"
	249	" )"
	250	" AND ep.user_id = ?"
	251	" AND ep.stream_ordering > ?"
	252	" AND ep.stream_ordering <= ?"
	253	" ORDER BY ep.stream_ordering ASC LIMIT ?"
	254	)
	255	args = [user_id, user_id, min_stream_ordering, max_stream_ordering, limit]
	256	txn.execute(sql, args)
	257	return txn.fetchall()
	258
	259	no_read_receipt = yield self.runInteraction(
	260	"get_unread_push_actions_for_user_in_range_http_nrr", get_no_receipt
	261	)
	262
	263	notifs = [
	264	{
	265	"event_id": row[0],
	266	"room_id": row[1],
	267	"stream_ordering": row[2],
	268	"actions": _deserialize_action(row[3], row[4]),
	269	}
	270	for row in after_read_receipt + no_read_receipt
	271	]
	272
	273	# Now sort it so it's ordered correctly, since currently it will
	274	# contain results from the first query, correctly ordered, followed
	275	# by results from the second query, but we want them all ordered
	276	# by stream_ordering, oldest first.
	277	notifs.sort(key=lambda r: r["stream_ordering"])
	278
	279	# Take only up to the limit. We have to stop at the limit because
	280	# one of the subqueries may have hit the limit.
	281	return notifs[:limit]
	282
	283	@defer.inlineCallbacks
	284	def get_unread_push_actions_for_user_in_range_for_email(
	285	self, user_id, min_stream_ordering, max_stream_ordering, limit=20
	286	):
	287	"""Get a list of the most recent unread push actions for a given user,
	288	within the given stream ordering range. Called by the emailpusher
	289
	290	Args:
	291	user_id (str): The user to fetch push actions for.
	292	min_stream_ordering(int): The exclusive lower bound on the
	293	stream ordering of event push actions to fetch.
	294	max_stream_ordering(int): The inclusive upper bound on the
	295	stream ordering of event push actions to fetch.
	296	limit (int): The maximum number of rows to return.
	297	Returns:
	298	A promise which resolves to a list of dicts with the keys "event_id",
	299	"room_id", "stream_ordering", "actions", "received_ts".
	300	The list will be ordered by descending received_ts.
	301	The list will have between 0~limit entries.
	302	"""
	303	# find rooms that have a read receipt in them and return the most recent
	304	# push actions
	305	def get_after_receipt(txn):
	306	sql = (
	307	"SELECT ep.event_id, ep.room_id, ep.stream_ordering, ep.actions,"
	308	" ep.highlight, e.received_ts"
	309	" FROM ("
	310	" SELECT room_id,"
	311	" MAX(stream_ordering) as stream_ordering"
	312	" FROM events"
	313	" INNER JOIN receipts_linearized USING (room_id, event_id)"
	314	" WHERE receipt_type = 'm.read' AND user_id = ?"
	315	" GROUP BY room_id"
	316	") AS rl,"
	317	" event_push_actions AS ep"
	318	" INNER JOIN events AS e USING (room_id, event_id)"
	319	" WHERE"
	320	" ep.room_id = rl.room_id"
	321	" AND ep.stream_ordering > rl.stream_ordering"
	322	" AND ep.user_id = ?"
	323	" AND ep.stream_ordering > ?"
	324	" AND ep.stream_ordering <= ?"
	325	" ORDER BY ep.stream_ordering DESC LIMIT ?"
	326	)
	327	args = [user_id, user_id, min_stream_ordering, max_stream_ordering, limit]
	328	txn.execute(sql, args)
	329	return txn.fetchall()
	330
	331	after_read_receipt = yield self.runInteraction(
	332	"get_unread_push_actions_for_user_in_range_email_arr", get_after_receipt
	333	)
	334
	335	# There are rooms with push actions in them but you don't have a read receipt in
	336	# them e.g. rooms you've been invited to, so get push actions for rooms which do
	337	# not have read receipts in them too.
	338	def get_no_receipt(txn):
	339	sql = (
	340	"SELECT ep.event_id, ep.room_id, ep.stream_ordering, ep.actions,"
	341	" ep.highlight, e.received_ts"
	342	" FROM event_push_actions AS ep"
	343	" INNER JOIN events AS e USING (room_id, event_id)"
	344	" WHERE"
	345	" ep.room_id NOT IN ("
	346	" SELECT room_id FROM receipts_linearized"
	347	" WHERE receipt_type = 'm.read' AND user_id = ?"
	348	" GROUP BY room_id"
	349	" )"
	350	" AND ep.user_id = ?"
	351	" AND ep.stream_ordering > ?"
	352	" AND ep.stream_ordering <= ?"
	353	" ORDER BY ep.stream_ordering DESC LIMIT ?"
	354	)
	355	args = [user_id, user_id, min_stream_ordering, max_stream_ordering, limit]
	356	txn.execute(sql, args)
	357	return txn.fetchall()
	358
	359	no_read_receipt = yield self.runInteraction(
	360	"get_unread_push_actions_for_user_in_range_email_nrr", get_no_receipt
	361	)
	362
	363	# Make a list of dicts from the two sets of results.
	364	notifs = [
	365	{
	366	"event_id": row[0],
	367	"room_id": row[1],
	368	"stream_ordering": row[2],
	369	"actions": _deserialize_action(row[3], row[4]),
	370	"received_ts": row[5],
	371	}
	372	for row in after_read_receipt + no_read_receipt
	373	]
	374
	375	# Now sort it so it's ordered correctly, since currently it will
	376	# contain results from the first query, correctly ordered, followed
	377	# by results from the second query, but we want them all ordered
	378	# by received_ts (most recent first)
	379	notifs.sort(key=lambda r: -(r["received_ts"] or 0))
	380
	381	# Now return the first `limit`
	382	return notifs[:limit]
	383
	384	def get_if_maybe_push_in_range_for_user(self, user_id, min_stream_ordering):
	385	"""A fast check to see if there might be something to push for the
	386	user since the given stream ordering. May return false positives.
	387
	388	Useful to know whether to bother starting a pusher on start up or not.
	389
	390	Args:
	391	user_id (str)
	392	min_stream_ordering (int)
	393
	394	Returns:
	395	Deferred[bool]: True if there may be push to process, False if
	396	there definitely isn't.
	397	"""
	398
	399	def _get_if_maybe_push_in_range_for_user_txn(txn):
	400	sql = """
	401	SELECT 1 FROM event_push_actions
	402	WHERE user_id = ? AND stream_ordering > ?
	403	LIMIT 1
	404	"""
	405
	406	txn.execute(sql, (user_id, min_stream_ordering))
	407	return bool(txn.fetchone())
	408
	409	return self.runInteraction(
	410	"get_if_maybe_push_in_range_for_user",
	411	_get_if_maybe_push_in_range_for_user_txn,
	412	)
	413
	414	def add_push_actions_to_staging(self, event_id, user_id_actions):
	415	"""Add the push actions for the event to the push action staging area.
	416
	417	Args:
	418	event_id (str)
	419	user_id_actions (dict[str, list[dict\|str])]): A dictionary mapping
	420	user_id to list of push actions, where an action can either be
	421	a string or dict.
	422
	423	Returns:
	424	Deferred
	425	"""
	426
	427	if not user_id_actions:
	428	return
	429
	430	# This is a helper function for generating the necessary tuple that
	431	# can be used to inert into the `event_push_actions_staging` table.
	432	def _gen_entry(user_id, actions):
	433	is_highlight = 1 if _action_has_highlight(actions) else 0
	434	return (
	435	event_id, # event_id column
	436	user_id, # user_id column
	437	_serialize_action(actions, is_highlight), # actions column
	438	1, # notif column
	439	is_highlight, # highlight column
	440	)
	441
	442	def _add_push_actions_to_staging_txn(txn):
	443	# We don't use _simple_insert_many here to avoid the overhead
	444	# of generating lists of dicts.
	445
	446	sql = """
	447	INSERT INTO event_push_actions_staging
	448	(event_id, user_id, actions, notif, highlight)
	449	VALUES (?, ?, ?, ?, ?)
	450	"""
	451
	452	txn.executemany(
	453	sql,
	454	(
	455	_gen_entry(user_id, actions)
	456	for user_id, actions in iteritems(user_id_actions)
	457	),
	458	)
	459
	460	return self.runInteraction(
	461	"add_push_actions_to_staging", _add_push_actions_to_staging_txn
	462	)
	463
	464	@defer.inlineCallbacks
	465	def remove_push_actions_from_staging(self, event_id):
	466	"""Called if we failed to persist the event to ensure that stale push
	467	actions don't build up in the DB
	468
	469	Args:
	470	event_id (str)
	471	"""
	472
	473	try:
	474	res = yield self._simple_delete(
	475	table="event_push_actions_staging",
	476	keyvalues={"event_id": event_id},
	477	desc="remove_push_actions_from_staging",
	478	)
	479	return res
	480	except Exception:
	481	# this method is called from an exception handler, so propagating
	482	# another exception here really isn't helpful - there's nothing
	483	# the caller can do about it. Just log the exception and move on.
	484	logger.exception(
	485	"Error removing push actions after event persistence failure"
	486	)
	487
	488	def _find_stream_orderings_for_times(self):
	489	return run_as_background_process(
	490	"event_push_action_stream_orderings",
	491	self.runInteraction,
	492	"_find_stream_orderings_for_times",
	493	self._find_stream_orderings_for_times_txn,
	494	)
	495
	496	def _find_stream_orderings_for_times_txn(self, txn):
	497	logger.info("Searching for stream ordering 1 month ago")
	498	self.stream_ordering_month_ago = self._find_first_stream_ordering_after_ts_txn(
	499	txn, self._clock.time_msec() - 30 * 24 * 60 * 60 * 1000
	500	)
	501	logger.info(
	502	"Found stream ordering 1 month ago: it's %d", self.stream_ordering_month_ago
	503	)
	504	logger.info("Searching for stream ordering 1 day ago")
	505	self.stream_ordering_day_ago = self._find_first_stream_ordering_after_ts_txn(
	506	txn, self._clock.time_msec() - 24 * 60 * 60 * 1000
	507	)
	508	logger.info(
	509	"Found stream ordering 1 day ago: it's %d", self.stream_ordering_day_ago
	510	)
	511
	512	def find_first_stream_ordering_after_ts(self, ts):
	513	"""Gets the stream ordering corresponding to a given timestamp.
	514
	515	Specifically, finds the stream_ordering of the first event that was
	516	received on or after the timestamp. This is done by a binary search on
	517	the events table, since there is no index on received_ts, so is
	518	relatively slow.
	519
	520	Args:
	521	ts (int): timestamp in millis
	522
	523	Returns:
	524	Deferred[int]: stream ordering of the first event received on/after
	525	the timestamp
	526	"""
	527	return self.runInteraction(
	528	"_find_first_stream_ordering_after_ts_txn",
	529	self._find_first_stream_ordering_after_ts_txn,
	530	ts,
	531	)
	532
	533	@staticmethod
	534	def _find_first_stream_ordering_after_ts_txn(txn, ts):
	535	"""
	536	Find the stream_ordering of the first event that was received on or
	537	after a given timestamp. This is relatively slow as there is no index
	538	on received_ts but we can then use this to delete push actions before
	539	this.
	540
	541	received_ts must necessarily be in the same order as stream_ordering
	542	and stream_ordering is indexed, so we manually binary search using
	543	stream_ordering
	544
	545	Args:
	546	txn (twisted.enterprise.adbapi.Transaction):
	547	ts (int): timestamp to search for
	548
	549	Returns:
	550	int: stream ordering
	551	"""
	552	txn.execute("SELECT MAX(stream_ordering) FROM events")
	553	max_stream_ordering = txn.fetchone()[0]
	554
	555	if max_stream_ordering is None:
	556	return 0
	557
	558	# We want the first stream_ordering in which received_ts is greater
	559	# than or equal to ts. Call this point X.
	560	#
	561	# We maintain the invariants:
	562	#
	563	# range_start <= X <= range_end
	564	#
	565	range_start = 0
	566	range_end = max_stream_ordering + 1
	567
	568	# Given a stream_ordering, look up the timestamp at that
	569	# stream_ordering.
	570	#
	571	# The array may be sparse (we may be missing some stream_orderings).
	572	# We treat the gaps as the same as having the same value as the
	573	# preceding entry, because we will pick the lowest stream_ordering
	574	# which satisfies our requirement of received_ts >= ts.
	575	#
	576	# For example, if our array of events indexed by stream_ordering is
	577	# [10, <none>, 20], we should treat this as being equivalent to
	578	# [10, 10, 20].
	579	#
	580	sql = (
	581	"SELECT received_ts FROM events"
	582	" WHERE stream_ordering <= ?"
	583	" ORDER BY stream_ordering DESC"
	584	" LIMIT 1"
	585	)
	586
	587	while range_end - range_start > 0:
	588	middle = (range_end + range_start) // 2
	589	txn.execute(sql, (middle,))
	590	row = txn.fetchone()
	591	if row is None:
	592	# no rows with stream_ordering<=middle
	593	range_start = middle + 1
	594	continue
	595
	596	middle_ts = row[0]
	597	if ts > middle_ts:
	598	# we got a timestamp lower than the one we were looking for.
	599	# definitely need to look higher: X > middle.
	600	range_start = middle + 1
	601	else:
	602	# we got a timestamp higher than (or the same as) the one we
	603	# were looking for. We aren't yet sure about the point we
	604	# looked up, but we can be sure that X <= middle.
	605	range_end = middle
	606
	607	return range_end
	608
	609
	610	class EventPushActionsStore(EventPushActionsWorkerStore):
	611	EPA_HIGHLIGHT_INDEX = "epa_highlight_index"
	612
	613	def __init__(self, db_conn, hs):
	614	super(EventPushActionsStore, self).__init__(db_conn, hs)
	615
	616	self.register_background_index_update(
	617	self.EPA_HIGHLIGHT_INDEX,
	618	index_name="event_push_actions_u_highlight",
	619	table="event_push_actions",
	620	columns=["user_id", "stream_ordering"],
	621	)
	622
	623	self.register_background_index_update(
	624	"event_push_actions_highlights_index",
	625	index_name="event_push_actions_highlights_index",
	626	table="event_push_actions",
	627	columns=["user_id", "room_id", "topological_ordering", "stream_ordering"],
	628	where_clause="highlight=1",
	629	)
	630
	631	self._doing_notif_rotation = False
	632	self._rotate_notif_loop = self._clock.looping_call(
	633	self._start_rotate_notifs, 30 * 60 * 1000
	634	)
	635
	636	def _set_push_actions_for_event_and_users_txn(
	637	self, txn, events_and_contexts, all_events_and_contexts
	638	):
	639	"""Handles moving push actions from staging table to main
	640	event_push_actions table for all events in `events_and_contexts`.
	641
	642	Also ensures that all events in `all_events_and_contexts` are removed
	643	from the push action staging area.
	644
	645	Args:
	646	events_and_contexts (list[(EventBase, EventContext)]): events
	647	we are persisting
	648	all_events_and_contexts (list[(EventBase, EventContext)]): all
	649	events that we were going to persist. This includes events
	650	we've already persisted, etc, that wouldn't appear in
	651	events_and_context.
	652	"""
	653
	654	sql = """
	655	INSERT INTO event_push_actions (
	656	room_id, event_id, user_id, actions, stream_ordering,
	657	topological_ordering, notif, highlight
	658	)
	659	SELECT ?, event_id, user_id, actions, ?, ?, notif, highlight
	660	FROM event_push_actions_staging
	661	WHERE event_id = ?
	662	"""
	663
	664	if events_and_contexts:
	665	txn.executemany(
	666	sql,
	667	(
	668	(
	669	event.room_id,
	670	event.internal_metadata.stream_ordering,
	671	event.depth,
	672	event.event_id,
	673	)
	674	for event, _ in events_and_contexts
	675	),
	676	)
	677
	678	for event, _ in events_and_contexts:
	679	user_ids = self._simple_select_onecol_txn(
	680	txn,
	681	table="event_push_actions_staging",
	682	keyvalues={"event_id": event.event_id},
	683	retcol="user_id",
	684	)
	685
	686	for uid in user_ids:
	687	txn.call_after(
	688	self.get_unread_event_push_actions_by_room_for_user.invalidate_many,
	689	(event.room_id, uid),
	690	)
	691
	692	# Now we delete the staging area for all events that were being
	693	# persisted.
	694	txn.executemany(
	695	"DELETE FROM event_push_actions_staging WHERE event_id = ?",
	696	((event.event_id,) for event, _ in all_events_and_contexts),
	697	)
	698
	699	@defer.inlineCallbacks
	700	def get_push_actions_for_user(
	701	self, user_id, before=None, limit=50, only_highlight=False
	702	):
	703	def f(txn):
	704	before_clause = ""
	705	if before:
	706	before_clause = "AND epa.stream_ordering < ?"
	707	args = [user_id, before, limit]
	708	else:
	709	args = [user_id, limit]
	710
	711	if only_highlight:
	712	if len(before_clause) > 0:
	713	before_clause += " "
	714	before_clause += "AND epa.highlight = 1"
	715
	716	# NB. This assumes event_ids are globally unique since
	717	# it makes the query easier to index
	718	sql = (
	719	"SELECT epa.event_id, epa.room_id,"
	720	" epa.stream_ordering, epa.topological_ordering,"
	721	" epa.actions, epa.highlight, epa.profile_tag, e.received_ts"
	722	" FROM event_push_actions epa, events e"
	723	" WHERE epa.event_id = e.event_id"
	724	" AND epa.user_id = ? %s"
	725	" ORDER BY epa.stream_ordering DESC"
	726	" LIMIT ?" % (before_clause,)
	727	)
	728	txn.execute(sql, args)
	729	return self.cursor_to_dict(txn)
	730
	731	push_actions = yield self.runInteraction("get_push_actions_for_user", f)
	732	for pa in push_actions:
	733	pa["actions"] = _deserialize_action(pa["actions"], pa["highlight"])
	734	return push_actions
	735
	736	@defer.inlineCallbacks
	737	def get_time_of_last_push_action_before(self, stream_ordering):
	738	def f(txn):
	739	sql = (
	740	"SELECT e.received_ts"
	741	" FROM event_push_actions AS ep"
	742	" JOIN events e ON ep.room_id = e.room_id AND ep.event_id = e.event_id"
	743	" WHERE ep.stream_ordering > ?"
	744	" ORDER BY ep.stream_ordering ASC"
	745	" LIMIT 1"
	746	)
	747	txn.execute(sql, (stream_ordering,))
	748	return txn.fetchone()
	749
	750	result = yield self.runInteraction("get_time_of_last_push_action_before", f)
	751	return result[0] if result else None
	752
	753	@defer.inlineCallbacks
	754	def get_latest_push_action_stream_ordering(self):
	755	def f(txn):
	756	txn.execute("SELECT MAX(stream_ordering) FROM event_push_actions")
	757	return txn.fetchone()
	758
	759	result = yield self.runInteraction("get_latest_push_action_stream_ordering", f)
	760	return result[0] or 0
	761
	762	def _remove_push_actions_for_event_id_txn(self, txn, room_id, event_id):
	763	# Sad that we have to blow away the cache for the whole room here
	764	txn.call_after(
	765	self.get_unread_event_push_actions_by_room_for_user.invalidate_many,
	766	(room_id,),
	767	)
	768	txn.execute(
	769	"DELETE FROM event_push_actions WHERE room_id = ? AND event_id = ?",
	770	(room_id, event_id),
	771	)
	772
	773	def _remove_old_push_actions_before_txn(
	774	self, txn, room_id, user_id, stream_ordering
	775	):
	776	"""
	777	Purges old push actions for a user and room before a given
	778	stream_ordering.
	779
	780	We however keep a months worth of highlighted notifications, so that
	781	users can still get a list of recent highlights.
	782
	783	Args:
	784	txn: The transcation
	785	room_id: Room ID to delete from
	786	user_id: user ID to delete for
	787	stream_ordering: The lowest stream ordering which will
	788	not be deleted.
	789	"""
	790	txn.call_after(
	791	self.get_unread_event_push_actions_by_room_for_user.invalidate_many,
	792	(room_id, user_id),
	793	)
	794
	795	# We need to join on the events table to get the received_ts for
	796	# event_push_actions and sqlite won't let us use a join in a delete so
	797	# we can't just delete where received_ts < x. Furthermore we can
	798	# only identify event_push_actions by a tuple of room_id, event_id
	799	# we we can't use a subquery.
	800	# Instead, we look up the stream ordering for the last event in that
	801	# room received before the threshold time and delete event_push_actions
	802	# in the room with a stream_odering before that.
	803	txn.execute(
	804	"DELETE FROM event_push_actions "
	805	" WHERE user_id = ? AND room_id = ? AND "
	806	" stream_ordering <= ?"
	807	" AND ((stream_ordering < ? AND highlight = 1) or highlight = 0)",
	808	(user_id, room_id, stream_ordering, self.stream_ordering_month_ago),
	809	)
	810
	811	txn.execute(
	812	"""
	813	DELETE FROM event_push_summary
	814	WHERE room_id = ? AND user_id = ? AND stream_ordering <= ?
	815	""",
	816	(room_id, user_id, stream_ordering),
	817	)
	818
	819	def _start_rotate_notifs(self):
	820	return run_as_background_process("rotate_notifs", self._rotate_notifs)
	821
	822	@defer.inlineCallbacks
	823	def _rotate_notifs(self):
	824	if self._doing_notif_rotation or self.stream_ordering_day_ago is None:
	825	return
	826	self._doing_notif_rotation = True
	827
	828	try:
	829	while True:
	830	logger.info("Rotating notifications")
	831
	832	caught_up = yield self.runInteraction(
	833	"_rotate_notifs", self._rotate_notifs_txn
	834	)
	835	if caught_up:
	836	break
	837	yield self.hs.get_clock().sleep(self._rotate_delay)
	838	finally:
	839	self._doing_notif_rotation = False
	840
	841	def _rotate_notifs_txn(self, txn):
	842	"""Archives older notifications into event_push_summary. Returns whether
	843	the archiving process has caught up or not.
	844	"""
	845
	846	old_rotate_stream_ordering = self._simple_select_one_onecol_txn(
	847	txn,
	848	table="event_push_summary_stream_ordering",
	849	keyvalues={},
	850	retcol="stream_ordering",
	851	)
	852
	853	# We don't to try and rotate millions of rows at once, so we cap the
	854	# maximum stream ordering we'll rotate before.
	855	txn.execute(
	856	"""
	857	SELECT stream_ordering FROM event_push_actions
	858	WHERE stream_ordering > ?
	859	ORDER BY stream_ordering ASC LIMIT 1 OFFSET ?
	860	""",
	861	(old_rotate_stream_ordering, self._rotate_count),
	862	)
	863	stream_row = txn.fetchone()
	864	if stream_row:
	865	offset_stream_ordering, = stream_row
	866	rotate_to_stream_ordering = min(
	867	self.stream_ordering_day_ago, offset_stream_ordering
	868	)
	869	caught_up = offset_stream_ordering >= self.stream_ordering_day_ago
	870	else:
	871	rotate_to_stream_ordering = self.stream_ordering_day_ago
	872	caught_up = True
	873
	874	logger.info("Rotating notifications up to: %s", rotate_to_stream_ordering)
	875
	876	self._rotate_notifs_before_txn(txn, rotate_to_stream_ordering)
	877
	878	# We have caught up iff we were limited by `stream_ordering_day_ago`
	879	return caught_up
	880
	881	def _rotate_notifs_before_txn(self, txn, rotate_to_stream_ordering):
	882	old_rotate_stream_ordering = self._simple_select_one_onecol_txn(
	883	txn,
	884	table="event_push_summary_stream_ordering",
	885	keyvalues={},
	886	retcol="stream_ordering",
	887	)
	888
	889	# Calculate the new counts that should be upserted into event_push_summary
	890	sql = """
	891	SELECT user_id, room_id,
	892	coalesce(old.notif_count, 0) + upd.notif_count,
	893	upd.stream_ordering,
	894	old.user_id
	895	FROM (
	896	SELECT user_id, room_id, count(*) as notif_count,
	897	max(stream_ordering) as stream_ordering
	898	FROM event_push_actions
	899	WHERE ? <= stream_ordering AND stream_ordering < ?
	900	AND highlight = 0
	901	GROUP BY user_id, room_id
	902	) AS upd
	903	LEFT JOIN event_push_summary AS old USING (user_id, room_id)
	904	"""
	905
	906	txn.execute(sql, (old_rotate_stream_ordering, rotate_to_stream_ordering))
	907	rows = txn.fetchall()
	908
	909	logger.info("Rotating notifications, handling %d rows", len(rows))
	910
	911	# If the `old.user_id` above is NULL then we know there isn't already an
	912	# entry in the table, so we simply insert it. Otherwise we update the
	913	# existing table.
	914	self._simple_insert_many_txn(
	915	txn,
	916	table="event_push_summary",
	917	values=[
	918	{
	919	"user_id": row[0],
	920	"room_id": row[1],
	921	"notif_count": row[2],
	922	"stream_ordering": row[3],
	923	}
	924	for row in rows
	925	if row[4] is None
	926	],
	927	)
	928
	929	txn.executemany(
	930	"""
	931	UPDATE event_push_summary SET notif_count = ?, stream_ordering = ?
	932	WHERE user_id = ? AND room_id = ?
	933	""",
	934	((row[2], row[3], row[0], row[1]) for row in rows if row[4] is not None),
	935	)
	936
	937	txn.execute(
	938	"DELETE FROM event_push_actions"
	939	" WHERE ? <= stream_ordering AND stream_ordering < ? AND highlight = 0",
	940	(old_rotate_stream_ordering, rotate_to_stream_ordering),
	941	)
	942
	943	logger.info("Rotating notifications, deleted %s push actions", txn.rowcount)
	944
	945	txn.execute(
	946	"UPDATE event_push_summary_stream_ordering SET stream_ordering = ?",
	947	(rotate_to_stream_ordering,),
	948	)
	949
	950
	951	def _action_has_highlight(actions):
	952	for action in actions:
	953	try:
	954	if action.get("set_tweak", None) == "highlight":
	955	return action.get("value", True)
	956	except AttributeError:
	957	pass
	958
	959	return False

+2489

-0

synapse/storage/data_stores/main/events.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2014-2016 OpenMarket Ltd
	2	# Copyright 2018-2019 New Vector Ltd
	3	# Copyright 2019 The Matrix.org Foundation C.I.C.
	4	#
	5	# Licensed under the Apache License, Version 2.0 (the "License");
	6	# you may not use this file except in compliance with the License.
	7	# You may obtain a copy of the License at
	8	#
	9	# http://www.apache.org/licenses/LICENSE-2.0
	10	#
	11	# Unless required by applicable law or agreed to in writing, software
	12	# distributed under the License is distributed on an "AS IS" BASIS,
	13	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	14	# See the License for the specific language governing permissions and
	15	# limitations under the License.
	16
	17	import itertools
	18	import logging
	19	from collections import Counter as c_counter, OrderedDict, deque, namedtuple
	20	from functools import wraps
	21
	22	from six import iteritems, text_type
	23	from six.moves import range
	24
	25	from canonicaljson import json
	26	from prometheus_client import Counter, Histogram
	27
	28	from twisted.internet import defer
	29
	30	import synapse.metrics
	31	from synapse.api.constants import EventTypes
	32	from synapse.api.errors import SynapseError
	33	from synapse.events import EventBase # noqa: F401
	34	from synapse.events.snapshot import EventContext # noqa: F401
	35	from synapse.events.utils import prune_event_dict
	36	from synapse.logging.context import PreserveLoggingContext, make_deferred_yieldable
	37	from synapse.logging.utils import log_function
	38	from synapse.metrics import BucketCollector
	39	from synapse.metrics.background_process_metrics import run_as_background_process
	40	from synapse.state import StateResolutionStore
	41	from synapse.storage._base import make_in_list_sql_clause
	42	from synapse.storage.background_updates import BackgroundUpdateStore
	43	from synapse.storage.data_stores.main.event_federation import EventFederationStore
	44	from synapse.storage.data_stores.main.events_worker import EventsWorkerStore
	45	from synapse.storage.data_stores.main.state import StateGroupWorkerStore
	46	from synapse.types import RoomStreamToken, get_domain_from_id
	47	from synapse.util import batch_iter
	48	from synapse.util.async_helpers import ObservableDeferred
	49	from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
	50	from synapse.util.frozenutils import frozendict_json_encoder
	51	from synapse.util.metrics import Measure
	52
	53	logger = logging.getLogger(__name__)
	54
	55	persist_event_counter = Counter("synapse_storage_events_persisted_events", "")
	56	event_counter = Counter(
	57	"synapse_storage_events_persisted_events_sep",
	58	"",
	59	["type", "origin_type", "origin_entity"],
	60	)
	61
	62	# The number of times we are recalculating the current state
	63	state_delta_counter = Counter("synapse_storage_events_state_delta", "")
	64
	65	# The number of times we are recalculating state when there is only a
	66	# single forward extremity
	67	state_delta_single_event_counter = Counter(
	68	"synapse_storage_events_state_delta_single_event", ""
	69	)
	70
	71	# The number of times we are reculating state when we could have resonably
	72	# calculated the delta when we calculated the state for an event we were
	73	# persisting.
	74	state_delta_reuse_delta_counter = Counter(
	75	"synapse_storage_events_state_delta_reuse_delta", ""
	76	)
	77
	78	# The number of forward extremities for each new event.
	79	forward_extremities_counter = Histogram(
	80	"synapse_storage_events_forward_extremities_persisted",
	81	"Number of forward extremities for each new event",
	82	buckets=(1, 2, 3, 5, 7, 10, 15, 20, 50, 100, 200, 500, "+Inf"),
	83	)
	84
	85	# The number of stale forward extremities for each new event. Stale extremities
	86	# are those that were in the previous set of extremities as well as the new.
	87	stale_forward_extremities_counter = Histogram(
	88	"synapse_storage_events_stale_forward_extremities_persisted",
	89	"Number of unchanged forward extremities for each new event",
	90	buckets=(0, 1, 2, 3, 5, 7, 10, 15, 20, 50, 100, 200, 500, "+Inf"),
	91	)
	92
	93
	94	def encode_json(json_object):
	95	"""
	96	Encode a Python object as JSON and return it in a Unicode string.
	97	"""
	98	out = frozendict_json_encoder.encode(json_object)
	99	if isinstance(out, bytes):
	100	out = out.decode("utf8")
	101	return out
	102
	103
	104	class _EventPeristenceQueue(object):
	105	"""Queues up events so that they can be persisted in bulk with only one
	106	concurrent transaction per room.
	107	"""
	108
	109	_EventPersistQueueItem = namedtuple(
	110	"_EventPersistQueueItem", ("events_and_contexts", "backfilled", "deferred")
	111	)
	112
	113	def __init__(self):
	114	self._event_persist_queues = {}
	115	self._currently_persisting_rooms = set()
	116
	117	def add_to_queue(self, room_id, events_and_contexts, backfilled):
	118	"""Add events to the queue, with the given persist_event options.
	119
	120	NB: due to the normal usage pattern of this method, it does not
	121	follow the synapse logcontext rules, and leaves the logcontext in
	122	place whether or not the returned deferred is ready.
	123
	124	Args:
	125	room_id (str):
	126	events_and_contexts (list[(EventBase, EventContext)]):
	127	backfilled (bool):
	128
	129	Returns:
	130	defer.Deferred: a deferred which will resolve once the events are
	131	persisted. Runs its callbacks without a logcontext.
	132	"""
	133	queue = self._event_persist_queues.setdefault(room_id, deque())
	134	if queue:
	135	# if the last item in the queue has the same `backfilled` setting,
	136	# we can just add these new events to that item.
	137	end_item = queue[-1]
	138	if end_item.backfilled == backfilled:
	139	end_item.events_and_contexts.extend(events_and_contexts)
	140	return end_item.deferred.observe()
	141
	142	deferred = ObservableDeferred(defer.Deferred(), consumeErrors=True)
	143
	144	queue.append(
	145	self._EventPersistQueueItem(
	146	events_and_contexts=events_and_contexts,
	147	backfilled=backfilled,
	148	deferred=deferred,
	149	)
	150	)
	151
	152	return deferred.observe()
	153
	154	def handle_queue(self, room_id, per_item_callback):
	155	"""Attempts to handle the queue for a room if not already being handled.
	156
	157	The given callback will be invoked with for each item in the queue,
	158	of type _EventPersistQueueItem. The per_item_callback will continuously
	159	be called with new items, unless the queue becomnes empty. The return
	160	value of the function will be given to the deferreds waiting on the item,
	161	exceptions will be passed to the deferreds as well.
	162
	163	This function should therefore be called whenever anything is added
	164	to the queue.
	165
	166	If another callback is currently handling the queue then it will not be
	167	invoked.
	168	"""
	169
	170	if room_id in self._currently_persisting_rooms:
	171	return
	172
	173	self._currently_persisting_rooms.add(room_id)
	174
	175	@defer.inlineCallbacks
	176	def handle_queue_loop():
	177	try:
	178	queue = self._get_drainining_queue(room_id)
	179	for item in queue:
	180	try:
	181	ret = yield per_item_callback(item)
	182	except Exception:
	183	with PreserveLoggingContext():
	184	item.deferred.errback()
	185	else:
	186	with PreserveLoggingContext():
	187	item.deferred.callback(ret)
	188	finally:
	189	queue = self._event_persist_queues.pop(room_id, None)
	190	if queue:
	191	self._event_persist_queues[room_id] = queue
	192	self._currently_persisting_rooms.discard(room_id)
	193
	194	# set handle_queue_loop off in the background
	195	run_as_background_process("persist_events", handle_queue_loop)
	196
	197	def _get_drainining_queue(self, room_id):
	198	queue = self._event_persist_queues.setdefault(room_id, deque())
	199
	200	try:
	201	while True:
	202	yield queue.popleft()
	203	except IndexError:
	204	# Queue has been drained.
	205	pass
	206
	207
	208	_EventCacheEntry = namedtuple("_EventCacheEntry", ("event", "redacted_event"))
	209
	210
	211	def _retry_on_integrity_error(func):
	212	"""Wraps a database function so that it gets retried on IntegrityError,
	213	with `delete_existing=True` passed in.
	214
	215	Args:
	216	func: function that returns a Deferred and accepts a `delete_existing` arg
	217	"""
	218
	219	@wraps(func)
	220	@defer.inlineCallbacks
	221	def f(self, args, *kwargs):
	222	try:
	223	res = yield func(self, args, *kwargs)
	224	except self.database_engine.module.IntegrityError:
	225	logger.exception("IntegrityError, retrying.")
	226	res = yield func(self, args, delete_existing=True, *kwargs)
	227	return res
	228
	229	return f
	230
	231
	232	# inherits from EventFederationStore so that we can call _update_backward_extremities
	233	# and _handle_mult_prev_events (though arguably those could both be moved in here)
	234	class EventsStore(
	235	StateGroupWorkerStore,
	236	EventFederationStore,
	237	EventsWorkerStore,
	238	BackgroundUpdateStore,
	239	):
	240	def __init__(self, db_conn, hs):
	241	super(EventsStore, self).__init__(db_conn, hs)
	242
	243	self._event_persist_queue = _EventPeristenceQueue()
	244	self._state_resolution_handler = hs.get_state_resolution_handler()
	245
	246	# Collect metrics on the number of forward extremities that exist.
	247	# Counter of number of extremities to count
	248	self._current_forward_extremities_amount = c_counter()
	249
	250	BucketCollector(
	251	"synapse_forward_extremities",
	252	lambda: self._current_forward_extremities_amount,
	253	buckets=[1, 2, 3, 5, 7, 10, 15, 20, 50, 100, 200, 500, "+Inf"],
	254	)
	255
	256	# Read the extrems every 60 minutes
	257	def read_forward_extremities():
	258	# run as a background process to make sure that the database transactions
	259	# have a logcontext to report to
	260	return run_as_background_process(
	261	"read_forward_extremities", self._read_forward_extremities
	262	)
	263
	264	hs.get_clock().looping_call(read_forward_extremities, 60 * 60 * 1000)
	265
	266	def _censor_redactions():
	267	return run_as_background_process(
	268	"_censor_redactions", self._censor_redactions
	269	)
	270
	271	if self.hs.config.redaction_retention_period is not None:
	272	hs.get_clock().looping_call(_censor_redactions, 5 * 60 * 1000)
	273
	274	@defer.inlineCallbacks
	275	def _read_forward_extremities(self):
	276	def fetch(txn):
	277	txn.execute(
	278	"""
	279	select count(*) c from event_forward_extremities
	280	group by room_id
	281	"""
	282	)
	283	return txn.fetchall()
	284
	285	res = yield self.runInteraction("read_forward_extremities", fetch)
	286	self._current_forward_extremities_amount = c_counter(list(x[0] for x in res))
	287
	288	@defer.inlineCallbacks
	289	def persist_events(self, events_and_contexts, backfilled=False):
	290	"""
	291	Write events to the database
	292	Args:
	293	events_and_contexts: list of tuples of (event, context)
	294	backfilled (bool): Whether the results are retrieved from federation
	295	via backfill or not. Used to determine if they're "new" events
	296	which might update the current state etc.
	297
	298	Returns:
	299	Deferred[int]: the stream ordering of the latest persisted event
	300	"""
	301	partitioned = {}
	302	for event, ctx in events_and_contexts:
	303	partitioned.setdefault(event.room_id, []).append((event, ctx))
	304
	305	deferreds = []
	306	for room_id, evs_ctxs in iteritems(partitioned):
	307	d = self._event_persist_queue.add_to_queue(
	308	room_id, evs_ctxs, backfilled=backfilled
	309	)
	310	deferreds.append(d)
	311
	312	for room_id in partitioned:
	313	self._maybe_start_persisting(room_id)
	314
	315	yield make_deferred_yieldable(
	316	defer.gatherResults(deferreds, consumeErrors=True)
	317	)
	318
	319	max_persisted_id = yield self._stream_id_gen.get_current_token()
	320
	321	return max_persisted_id
	322
	323	@defer.inlineCallbacks
	324	@log_function
	325	def persist_event(self, event, context, backfilled=False):
	326	"""
	327
	328	Args:
	329	event (EventBase):
	330	context (EventContext):
	331	backfilled (bool):
	332
	333	Returns:
	334	Deferred: resolves to (int, int): the stream ordering of ``event``,
	335	and the stream ordering of the latest persisted event
	336	"""
	337	deferred = self._event_persist_queue.add_to_queue(
	338	event.room_id, [(event, context)], backfilled=backfilled
	339	)
	340
	341	self._maybe_start_persisting(event.room_id)
	342
	343	yield make_deferred_yieldable(deferred)
	344
	345	max_persisted_id = yield self._stream_id_gen.get_current_token()
	346	return (event.internal_metadata.stream_ordering, max_persisted_id)
	347
	348	def _maybe_start_persisting(self, room_id):
	349	@defer.inlineCallbacks
	350	def persisting_queue(item):
	351	with Measure(self._clock, "persist_events"):
	352	yield self._persist_events(
	353	item.events_and_contexts, backfilled=item.backfilled
	354	)
	355
	356	self._event_persist_queue.handle_queue(room_id, persisting_queue)
	357
	358	@_retry_on_integrity_error
	359	@defer.inlineCallbacks
	360	def _persist_events(
	361	self, events_and_contexts, backfilled=False, delete_existing=False
	362	):
	363	"""Persist events to db
	364
	365	Args:
	366	events_and_contexts (list[(EventBase, EventContext)]):
	367	backfilled (bool):
	368	delete_existing (bool):
	369
	370	Returns:
	371	Deferred: resolves when the events have been persisted
	372	"""
	373	if not events_and_contexts:
	374	return
	375
	376	chunks = [
	377	events_and_contexts[x : x + 100]
	378	for x in range(0, len(events_and_contexts), 100)
	379	]
	380
	381	for chunk in chunks:
	382	# We can't easily parallelize these since different chunks
	383	# might contain the same event. :(
	384
	385	# NB: Assumes that we are only persisting events for one room
	386	# at a time.
	387
	388	# map room_id->list[event_ids] giving the new forward
	389	# extremities in each room
	390	new_forward_extremeties = {}
	391
	392	# map room_id->(type,state_key)->event_id tracking the full
	393	# state in each room after adding these events.
	394	# This is simply used to prefill the get_current_state_ids
	395	# cache
	396	current_state_for_room = {}
	397
	398	# map room_id->(to_delete, to_insert) where to_delete is a list
	399	# of type/state keys to remove from current state, and to_insert
	400	# is a map (type,key)->event_id giving the state delta in each
	401	# room
	402	state_delta_for_room = {}
	403
	404	if not backfilled:
	405	with Measure(self._clock, "_calculate_state_and_extrem"):
	406	# Work out the new "current state" for each room.
	407	# We do this by working out what the new extremities are and then
	408	# calculating the state from that.
	409	events_by_room = {}
	410	for event, context in chunk:
	411	events_by_room.setdefault(event.room_id, []).append(
	412	(event, context)
	413	)
	414
	415	for room_id, ev_ctx_rm in iteritems(events_by_room):
	416	latest_event_ids = yield self.get_latest_event_ids_in_room(
	417	room_id
	418	)
	419	new_latest_event_ids = yield self._calculate_new_extremities(
	420	room_id, ev_ctx_rm, latest_event_ids
	421	)
	422
	423	latest_event_ids = set(latest_event_ids)
	424	if new_latest_event_ids == latest_event_ids:
	425	# No change in extremities, so no change in state
	426	continue
	427
	428	# there should always be at least one forward extremity.
	429	# (except during the initial persistence of the send_join
	430	# results, in which case there will be no existing
	431	# extremities, so we'll `continue` above and skip this bit.)
	432	assert new_latest_event_ids, "No forward extremities left!"
	433
	434	new_forward_extremeties[room_id] = new_latest_event_ids
	435
	436	len_1 = (
	437	len(latest_event_ids) == 1
	438	and len(new_latest_event_ids) == 1
	439	)
	440	if len_1:
	441	all_single_prev_not_state = all(
	442	len(event.prev_event_ids()) == 1
	443	and not event.is_state()
	444	for event, ctx in ev_ctx_rm
	445	)
	446	# Don't bother calculating state if they're just
	447	# a long chain of single ancestor non-state events.
	448	if all_single_prev_not_state:
	449	continue
	450
	451	state_delta_counter.inc()
	452	if len(new_latest_event_ids) == 1:
	453	state_delta_single_event_counter.inc()
	454
	455	# This is a fairly handwavey check to see if we could
	456	# have guessed what the delta would have been when
	457	# processing one of these events.
	458	# What we're interested in is if the latest extremities
	459	# were the same when we created the event as they are
	460	# now. When this server creates a new event (as opposed
	461	# to receiving it over federation) it will use the
	462	# forward extremities as the prev_events, so we can
	463	# guess this by looking at the prev_events and checking
	464	# if they match the current forward extremities.
	465	for ev, _ in ev_ctx_rm:
	466	prev_event_ids = set(ev.prev_event_ids())
	467	if latest_event_ids == prev_event_ids:
	468	state_delta_reuse_delta_counter.inc()
	469	break
	470
	471	logger.info("Calculating state delta for room %s", room_id)
	472	with Measure(
	473	self._clock, "persist_events.get_new_state_after_events"
	474	):
	475	res = yield self._get_new_state_after_events(
	476	room_id,
	477	ev_ctx_rm,
	478	latest_event_ids,
	479	new_latest_event_ids,
	480	)
	481	current_state, delta_ids = res
	482
	483	# If either are not None then there has been a change,
	484	# and we need to work out the delta (or use that
	485	# given)
	486	if delta_ids is not None:
	487	# If there is a delta we know that we've
	488	# only added or replaced state, never
	489	# removed keys entirely.
	490	state_delta_for_room[room_id] = ([], delta_ids)
	491	elif current_state is not None:
	492	with Measure(
	493	self._clock, "persist_events.calculate_state_delta"
	494	):
	495	delta = yield self._calculate_state_delta(
	496	room_id, current_state
	497	)
	498	state_delta_for_room[room_id] = delta
	499
	500	# If we have the current_state then lets prefill
	501	# the cache with it.
	502	if current_state is not None:
	503	current_state_for_room[room_id] = current_state
	504
	505	# We want to calculate the stream orderings as late as possible, as
	506	# we only notify after all events with a lesser stream ordering have
	507	# been persisted. I.e. if we spend 10s inside the with block then
	508	# that will delay all subsequent events from being notified about.
	509	# Hence why we do it down here rather than wrapping the entire
	510	# function.
	511	#
	512	# Its safe to do this after calculating the state deltas etc as we
	513	# only need to protect the persistence of the events. This is to
	514	# ensure that queries of the form "fetch events since X" don't
	515	# return events and stream positions after events that are still in
	516	# flight, as otherwise subsequent requests "fetch event since Y"
	517	# will not return those events.
	518	#
	519	# Note: Multiple instances of this function cannot be in flight at
	520	# the same time for the same room.
	521	if backfilled:
	522	stream_ordering_manager = self._backfill_id_gen.get_next_mult(
	523	len(chunk)
	524	)
	525	else:
	526	stream_ordering_manager = self._stream_id_gen.get_next_mult(len(chunk))
	527
	528	with stream_ordering_manager as stream_orderings:
	529	for (event, context), stream in zip(chunk, stream_orderings):
	530	event.internal_metadata.stream_ordering = stream
	531
	532	yield self.runInteraction(
	533	"persist_events",
	534	self._persist_events_txn,
	535	events_and_contexts=chunk,
	536	backfilled=backfilled,
	537	delete_existing=delete_existing,
	538	state_delta_for_room=state_delta_for_room,
	539	new_forward_extremeties=new_forward_extremeties,
	540	)
	541	persist_event_counter.inc(len(chunk))
	542
	543	if not backfilled:
	544	# backfilled events have negative stream orderings, so we don't
	545	# want to set the event_persisted_position to that.
	546	synapse.metrics.event_persisted_position.set(
	547	chunk[-1][0].internal_metadata.stream_ordering
	548	)
	549
	550	for event, context in chunk:
	551	if context.app_service:
	552	origin_type = "local"
	553	origin_entity = context.app_service.id
	554	elif self.hs.is_mine_id(event.sender):
	555	origin_type = "local"
	556	origin_entity = "client"
	557	else:
	558	origin_type = "remote"
	559	origin_entity = get_domain_from_id(event.sender)
	560
	561	event_counter.labels(event.type, origin_type, origin_entity).inc()
	562
	563	for room_id, new_state in iteritems(current_state_for_room):
	564	self.get_current_state_ids.prefill((room_id,), new_state)
	565
	566	for room_id, latest_event_ids in iteritems(new_forward_extremeties):
	567	self.get_latest_event_ids_in_room.prefill(
	568	(room_id,), list(latest_event_ids)
	569	)
	570
	571	@defer.inlineCallbacks
	572	def _calculate_new_extremities(self, room_id, event_contexts, latest_event_ids):
	573	"""Calculates the new forward extremities for a room given events to
	574	persist.
	575
	576	Assumes that we are only persisting events for one room at a time.
	577	"""
	578
	579	# we're only interested in new events which aren't outliers and which aren't
	580	# being rejected.
	581	new_events = [
	582	event
	583	for event, ctx in event_contexts
	584	if not event.internal_metadata.is_outlier()
	585	and not ctx.rejected
	586	and not event.internal_metadata.is_soft_failed()
	587	]
	588
	589	latest_event_ids = set(latest_event_ids)
	590
	591	# start with the existing forward extremities
	592	result = set(latest_event_ids)
	593
	594	# add all the new events to the list
	595	result.update(event.event_id for event in new_events)
	596
	597	# Now remove all events which are prev_events of any of the new events
	598	result.difference_update(
	599	e_id for event in new_events for e_id in event.prev_event_ids()
	600	)
	601
	602	# Remove any events which are prev_events of any existing events.
	603	existing_prevs = yield self._get_events_which_are_prevs(result)
	604	result.difference_update(existing_prevs)
	605
	606	# Finally handle the case where the new events have soft-failed prev
	607	# events. If they do we need to remove them and their prev events,
	608	# otherwise we end up with dangling extremities.
	609	existing_prevs = yield self._get_prevs_before_rejected(
	610	e_id for event in new_events for e_id in event.prev_event_ids()
	611	)
	612	result.difference_update(existing_prevs)
	613
	614	# We only update metrics for events that change forward extremities
	615	# (e.g. we ignore backfill/outliers/etc)
	616	if result != latest_event_ids:
	617	forward_extremities_counter.observe(len(result))
	618	stale = latest_event_ids & result
	619	stale_forward_extremities_counter.observe(len(stale))
	620
	621	return result
	622
	623	@defer.inlineCallbacks
	624	def _get_events_which_are_prevs(self, event_ids):
	625	"""Filter the supplied list of event_ids to get those which are prev_events of
	626	existing (non-outlier/rejected) events.
	627
	628	Args:
	629	event_ids (Iterable[str]): event ids to filter
	630
	631	Returns:
	632	Deferred[List[str]]: filtered event ids
	633	"""
	634	results = []
	635
	636	def _get_events_which_are_prevs_txn(txn, batch):
	637	sql = """
	638	SELECT prev_event_id, internal_metadata
	639	FROM event_edges
	640	INNER JOIN events USING (event_id)
	641	LEFT JOIN rejections USING (event_id)
	642	LEFT JOIN event_json USING (event_id)
	643	WHERE
	644	NOT events.outlier
	645	AND rejections.event_id IS NULL
	646	AND
	647	"""
	648
	649	clause, args = make_in_list_sql_clause(
	650	self.database_engine, "prev_event_id", batch
	651	)
	652
	653	txn.execute(sql + clause, args)
	654	results.extend(r[0] for r in txn if not json.loads(r[1]).get("soft_failed"))
	655
	656	for chunk in batch_iter(event_ids, 100):
	657	yield self.runInteraction(
	658	"_get_events_which_are_prevs", _get_events_which_are_prevs_txn, chunk
	659	)
	660
	661	return results
	662
	663	@defer.inlineCallbacks
	664	def _get_prevs_before_rejected(self, event_ids):
	665	"""Get soft-failed ancestors to remove from the extremities.
	666
	667	Given a set of events, find all those that have been soft-failed or
	668	rejected. Returns those soft failed/rejected events and their prev
	669	events (whether soft-failed/rejected or not), and recurses up the
	670	prev-event graph until it finds no more soft-failed/rejected events.
	671
	672	This is used to find extremities that are ancestors of new events, but
	673	are separated by soft failed events.
	674
	675	Args:
	676	event_ids (Iterable[str]): Events to find prev events for. Note
	677	that these must have already been persisted.
	678
	679	Returns:
	680	Deferred[set[str]]
	681	"""
	682
	683	# The set of event_ids to return. This includes all soft-failed events
	684	# and their prev events.
	685	existing_prevs = set()
	686
	687	def _get_prevs_before_rejected_txn(txn, batch):
	688	to_recursively_check = batch
	689
	690	while to_recursively_check:
	691	sql = """
	692	SELECT
	693	event_id, prev_event_id, internal_metadata,
	694	rejections.event_id IS NOT NULL
	695	FROM event_edges
	696	INNER JOIN events USING (event_id)
	697	LEFT JOIN rejections USING (event_id)
	698	LEFT JOIN event_json USING (event_id)
	699	WHERE
	700	NOT events.outlier
	701	AND
	702	"""
	703
	704	clause, args = make_in_list_sql_clause(
	705	self.database_engine, "event_id", to_recursively_check
	706	)
	707
	708	txn.execute(sql + clause, args)
	709	to_recursively_check = []
	710
	711	for event_id, prev_event_id, metadata, rejected in txn:
	712	if prev_event_id in existing_prevs:
	713	continue
	714
	715	soft_failed = json.loads(metadata).get("soft_failed")
	716	if soft_failed or rejected:
	717	to_recursively_check.append(prev_event_id)
	718	existing_prevs.add(prev_event_id)
	719
	720	for chunk in batch_iter(event_ids, 100):
	721	yield self.runInteraction(
	722	"_get_prevs_before_rejected", _get_prevs_before_rejected_txn, chunk
	723	)
	724
	725	return existing_prevs
	726
	727	@defer.inlineCallbacks
	728	def _get_new_state_after_events(
	729	self, room_id, events_context, old_latest_event_ids, new_latest_event_ids
	730	):
	731	"""Calculate the current state dict after adding some new events to
	732	a room
	733
	734	Args:
	735	room_id (str):
	736	room to which the events are being added. Used for logging etc
	737
	738	events_context (list[(EventBase, EventContext)]):
	739	events and contexts which are being added to the room
	740
	741	old_latest_event_ids (iterable[str]):
	742	the old forward extremities for the room.
	743
	744	new_latest_event_ids (iterable[str]):
	745	the new forward extremities for the room.
	746
	747	Returns:
	748	Deferred[tuple[dict[(str,str), str]\|None, dict[(str,str), str]\|None]]:
	749	Returns a tuple of two state maps, the first being the full new current
	750	state and the second being the delta to the existing current state.
	751	If both are None then there has been no change.
	752
	753	If there has been a change then we only return the delta if its
	754	already been calculated. Conversely if we do know the delta then
	755	the new current state is only returned if we've already calculated
	756	it.
	757	"""
	758	# map from state_group to ((type, key) -> event_id) state map
	759	state_groups_map = {}
	760
	761	# Map from (prev state group, new state group) -> delta state dict
	762	state_group_deltas = {}
	763
	764	for ev, ctx in events_context:
	765	if ctx.state_group is None:
	766	# This should only happen for outlier events.
	767	if not ev.internal_metadata.is_outlier():
	768	raise Exception(
	769	"Context for new event %s has no state "
	770	"group" % (ev.event_id,)
	771	)
	772	continue
	773
	774	if ctx.state_group in state_groups_map:
	775	continue
	776
	777	# We're only interested in pulling out state that has already
	778	# been cached in the context. We'll pull stuff out of the DB later
	779	# if necessary.
	780	current_state_ids = ctx.get_cached_current_state_ids()
	781	if current_state_ids is not None:
	782	state_groups_map[ctx.state_group] = current_state_ids
	783
	784	if ctx.prev_group:
	785	state_group_deltas[(ctx.prev_group, ctx.state_group)] = ctx.delta_ids
	786
	787	# We need to map the event_ids to their state groups. First, let's
	788	# check if the event is one we're persisting, in which case we can
	789	# pull the state group from its context.
	790	# Otherwise we need to pull the state group from the database.
	791
	792	# Set of events we need to fetch groups for. (We know none of the old
	793	# extremities are going to be in events_context).
	794	missing_event_ids = set(old_latest_event_ids)
	795
	796	event_id_to_state_group = {}
	797	for event_id in new_latest_event_ids:
	798	# First search in the list of new events we're adding.
	799	for ev, ctx in events_context:
	800	if event_id == ev.event_id and ctx.state_group is not None:
	801	event_id_to_state_group[event_id] = ctx.state_group
	802	break
	803	else:
	804	# If we couldn't find it, then we'll need to pull
	805	# the state from the database
	806	missing_event_ids.add(event_id)
	807
	808	if missing_event_ids:
	809	# Now pull out the state groups for any missing events from DB
	810	event_to_groups = yield self._get_state_group_for_events(missing_event_ids)
	811	event_id_to_state_group.update(event_to_groups)
	812
	813	# State groups of old_latest_event_ids
	814	old_state_groups = set(
	815	event_id_to_state_group[evid] for evid in old_latest_event_ids
	816	)
	817
	818	# State groups of new_latest_event_ids
	819	new_state_groups = set(
	820	event_id_to_state_group[evid] for evid in new_latest_event_ids
	821	)
	822
	823	# If they old and new groups are the same then we don't need to do
	824	# anything.
	825	if old_state_groups == new_state_groups:
	826	return None, None
	827
	828	if len(new_state_groups) == 1 and len(old_state_groups) == 1:
	829	# If we're going from one state group to another, lets check if
	830	# we have a delta for that transition. If we do then we can just
	831	# return that.
	832
	833	new_state_group = next(iter(new_state_groups))
	834	old_state_group = next(iter(old_state_groups))
	835
	836	delta_ids = state_group_deltas.get((old_state_group, new_state_group), None)
	837	if delta_ids is not None:
	838	# We have a delta from the existing to new current state,
	839	# so lets just return that. If we happen to already have
	840	# the current state in memory then lets also return that,
	841	# but it doesn't matter if we don't.
	842	new_state = state_groups_map.get(new_state_group)
	843	return new_state, delta_ids
	844
	845	# Now that we have calculated new_state_groups we need to get
	846	# their state IDs so we can resolve to a single state set.
	847	missing_state = new_state_groups - set(state_groups_map)
	848	if missing_state:
	849	group_to_state = yield self._get_state_for_groups(missing_state)
	850	state_groups_map.update(group_to_state)
	851
	852	if len(new_state_groups) == 1:
	853	# If there is only one state group, then we know what the current
	854	# state is.
	855	return state_groups_map[new_state_groups.pop()], None
	856
	857	# Ok, we need to defer to the state handler to resolve our state sets.
	858
	859	state_groups = {sg: state_groups_map[sg] for sg in new_state_groups}
	860
	861	events_map = {ev.event_id: ev for ev, _ in events_context}
	862
	863	# We need to get the room version, which is in the create event.
	864	# Normally that'd be in the database, but its also possible that we're
	865	# currently trying to persist it.
	866	room_version = None
	867	for ev, _ in events_context:
	868	if ev.type == EventTypes.Create and ev.state_key == "":
	869	room_version = ev.content.get("room_version", "1")
	870	break
	871
	872	if not room_version:
	873	room_version = yield self.get_room_version(room_id)
	874
	875	logger.debug("calling resolve_state_groups from preserve_events")
	876	res = yield self._state_resolution_handler.resolve_state_groups(
	877	room_id,
	878	room_version,
	879	state_groups,
	880	events_map,
	881	state_res_store=StateResolutionStore(self),
	882	)
	883
	884	return res.state, None
	885
	886	@defer.inlineCallbacks
	887	def _calculate_state_delta(self, room_id, current_state):
	888	"""Calculate the new state deltas for a room.
	889
	890	Assumes that we are only persisting events for one room at a time.
	891
	892	Returns:
	893	tuple[list, dict] (to_delete, to_insert): where to_delete are the
	894	type/state_keys to remove from current_state_events and `to_insert`
	895	are the updates to current_state_events.
	896	"""
	897	existing_state = yield self.get_current_state_ids(room_id)
	898
	899	to_delete = [key for key in existing_state if key not in current_state]
	900
	901	to_insert = {
	902	key: ev_id
	903	for key, ev_id in iteritems(current_state)
	904	if ev_id != existing_state.get(key)
	905	}
	906
	907	return to_delete, to_insert
	908
	909	@log_function
	910	def _persist_events_txn(
	911	self,
	912	txn,
	913	events_and_contexts,
	914	backfilled,
	915	delete_existing=False,
	916	state_delta_for_room={},
	917	new_forward_extremeties={},
	918	):
	919	"""Insert some number of room events into the necessary database tables.
	920
	921	Rejected events are only inserted into the events table, the events_json table,
	922	and the rejections table. Things reading from those table will need to check
	923	whether the event was rejected.
	924
	925	Args:
	926	txn (twisted.enterprise.adbapi.Connection): db connection
	927	events_and_contexts (list[(EventBase, EventContext)]):
	928	events to persist
	929	backfilled (bool): True if the events were backfilled
	930	delete_existing (bool): True to purge existing table rows for the
	931	events from the database. This is useful when retrying due to
	932	IntegrityError.
	933	state_delta_for_room (dict[str, (list, dict)]):
	934	The current-state delta for each room. For each room, a tuple
	935	(to_delete, to_insert), being a list of type/state keys to be
	936	removed from the current state, and a state set to be added to
	937	the current state.
	938	new_forward_extremeties (dict[str, list[str]]):
	939	The new forward extremities for each room. For each room, a
	940	list of the event ids which are the forward extremities.
	941
	942	"""
	943	all_events_and_contexts = events_and_contexts
	944
	945	min_stream_order = events_and_contexts[0][0].internal_metadata.stream_ordering
	946	max_stream_order = events_and_contexts[-1][0].internal_metadata.stream_ordering
	947
	948	self._update_forward_extremities_txn(
	949	txn,
	950	new_forward_extremities=new_forward_extremeties,
	951	max_stream_order=max_stream_order,
	952	)
	953
	954	# Ensure that we don't have the same event twice.
	955	events_and_contexts = self._filter_events_and_contexts_for_duplicates(
	956	events_and_contexts
	957	)
	958
	959	self._update_room_depths_txn(
	960	txn, events_and_contexts=events_and_contexts, backfilled=backfilled
	961	)
	962
	963	# _update_outliers_txn filters out any events which have already been
	964	# persisted, and returns the filtered list.
	965	events_and_contexts = self._update_outliers_txn(
	966	txn, events_and_contexts=events_and_contexts
	967	)
	968
	969	# From this point onwards the events are only events that we haven't
	970	# seen before.
	971
	972	if delete_existing:
	973	# For paranoia reasons, we go and delete all the existing entries
	974	# for these events so we can reinsert them.
	975	# This gets around any problems with some tables already having
	976	# entries.
	977	self._delete_existing_rows_txn(txn, events_and_contexts=events_and_contexts)
	978
	979	self._store_event_txn(txn, events_and_contexts=events_and_contexts)
	980
	981	# Insert into event_to_state_groups.
	982	self._store_event_state_mappings_txn(txn, events_and_contexts)
	983
	984	# We want to store event_auth mappings for rejected events, as they're
	985	# used in state res v2.
	986	# This is only necessary if the rejected event appears in an accepted
	987	# event's auth chain, but its easier for now just to store them (and
	988	# it doesn't take much storage compared to storing the entire event
	989	# anyway).
	990	self._simple_insert_many_txn(
	991	txn,
	992	table="event_auth",
	993	values=[
	994	{
	995	"event_id": event.event_id,
	996	"room_id": event.room_id,
	997	"auth_id": auth_id,
	998	}
	999	for event, _ in events_and_contexts
	1000	for auth_id in event.auth_event_ids()
	1001	if event.is_state()
	1002	],
	1003	)
	1004
	1005	# _store_rejected_events_txn filters out any events which were
	1006	# rejected, and returns the filtered list.
	1007	events_and_contexts = self._store_rejected_events_txn(
	1008	txn, events_and_contexts=events_and_contexts
	1009	)
	1010
	1011	# From this point onwards the events are only ones that weren't
	1012	# rejected.
	1013
	1014	self._update_metadata_tables_txn(
	1015	txn,
	1016	events_and_contexts=events_and_contexts,
	1017	all_events_and_contexts=all_events_and_contexts,
	1018	backfilled=backfilled,
	1019	)
	1020
	1021	# We call this last as it assumes we've inserted the events into
	1022	# room_memberships, where applicable.
	1023	self._update_current_state_txn(txn, state_delta_for_room, min_stream_order)
	1024
	1025	def _update_current_state_txn(self, txn, state_delta_by_room, stream_id):
	1026	for room_id, current_state_tuple in iteritems(state_delta_by_room):
	1027	to_delete, to_insert = current_state_tuple
	1028
	1029	# First we add entries to the current_state_delta_stream. We
	1030	# do this before updating the current_state_events table so
	1031	# that we can use it to calculate the `prev_event_id`. (This
	1032	# allows us to not have to pull out the existing state
	1033	# unnecessarily).
	1034	#
	1035	# The stream_id for the update is chosen to be the minimum of the stream_ids
	1036	# for the batch of the events that we are persisting; that means we do not
	1037	# end up in a situation where workers see events before the
	1038	# current_state_delta updates.
	1039	#
	1040	sql = """
	1041	INSERT INTO current_state_delta_stream
	1042	(stream_id, room_id, type, state_key, event_id, prev_event_id)
	1043	SELECT ?, ?, ?, ?, ?, (
	1044	SELECT event_id FROM current_state_events
	1045	WHERE room_id = ? AND type = ? AND state_key = ?
	1046	)
	1047	"""
	1048	txn.executemany(
	1049	sql,
	1050	(
	1051	(
	1052	stream_id,
	1053	room_id,
	1054	etype,
	1055	state_key,
	1056	None,
	1057	room_id,
	1058	etype,
	1059	state_key,
	1060	)
	1061	for etype, state_key in to_delete
	1062	# We sanity check that we're deleting rather than updating
	1063	if (etype, state_key) not in to_insert
	1064	),
	1065	)
	1066	txn.executemany(
	1067	sql,
	1068	(
	1069	(
	1070	stream_id,
	1071	room_id,
	1072	etype,
	1073	state_key,
	1074	ev_id,
	1075	room_id,
	1076	etype,
	1077	state_key,
	1078	)
	1079	for (etype, state_key), ev_id in iteritems(to_insert)
	1080	),
	1081	)
	1082
	1083	# Now we actually update the current_state_events table
	1084
	1085	txn.executemany(
	1086	"DELETE FROM current_state_events"
	1087	" WHERE room_id = ? AND type = ? AND state_key = ?",
	1088	(
	1089	(room_id, etype, state_key)
	1090	for etype, state_key in itertools.chain(to_delete, to_insert)
	1091	),
	1092	)
	1093
	1094	# We include the membership in the current state table, hence we do
	1095	# a lookup when we insert. This assumes that all events have already
	1096	# been inserted into room_memberships.
	1097	txn.executemany(
	1098	"""INSERT INTO current_state_events
	1099	(room_id, type, state_key, event_id, membership)
	1100	VALUES (?, ?, ?, ?, (SELECT membership FROM room_memberships WHERE event_id = ?))
	1101	""",
	1102	[
	1103	(room_id, key[0], key[1], ev_id, ev_id)
	1104	for key, ev_id in iteritems(to_insert)
	1105	],
	1106	)
	1107
	1108	txn.call_after(
	1109	self._curr_state_delta_stream_cache.entity_has_changed,
	1110	room_id,
	1111	stream_id,
	1112	)
	1113
	1114	# Invalidate the various caches
	1115
	1116	# Figure out the changes of membership to invalidate the
	1117	# `get_rooms_for_user` cache.
	1118	# We find out which membership events we may have deleted
	1119	# and which we have added, then we invlidate the caches for all
	1120	# those users.
	1121	members_changed = set(
	1122	state_key
	1123	for ev_type, state_key in itertools.chain(to_delete, to_insert)
	1124	if ev_type == EventTypes.Member
	1125	)
	1126
	1127	for member in members_changed:
	1128	txn.call_after(
	1129	self.get_rooms_for_user_with_stream_ordering.invalidate, (member,)
	1130	)
	1131
	1132	self._invalidate_state_caches_and_stream(txn, room_id, members_changed)
	1133
	1134	def _update_forward_extremities_txn(
	1135	self, txn, new_forward_extremities, max_stream_order
	1136	):
	1137	for room_id, new_extrem in iteritems(new_forward_extremities):
	1138	self._simple_delete_txn(
	1139	txn, table="event_forward_extremities", keyvalues={"room_id": room_id}
	1140	)
	1141	txn.call_after(self.get_latest_event_ids_in_room.invalidate, (room_id,))
	1142
	1143	self._simple_insert_many_txn(
	1144	txn,
	1145	table="event_forward_extremities",
	1146	values=[
	1147	{"event_id": ev_id, "room_id": room_id}
	1148	for room_id, new_extrem in iteritems(new_forward_extremities)
	1149	for ev_id in new_extrem
	1150	],
	1151	)
	1152	# We now insert into stream_ordering_to_exterm a mapping from room_id,
	1153	# new stream_ordering to new forward extremeties in the room.
	1154	# This allows us to later efficiently look up the forward extremeties
	1155	# for a room before a given stream_ordering
	1156	self._simple_insert_many_txn(
	1157	txn,
	1158	table="stream_ordering_to_exterm",
	1159	values=[
	1160	{
	1161	"room_id": room_id,
	1162	"event_id": event_id,
	1163	"stream_ordering": max_stream_order,
	1164	}
	1165	for room_id, new_extrem in iteritems(new_forward_extremities)
	1166	for event_id in new_extrem
	1167	],
	1168	)
	1169
	1170	@classmethod
	1171	def _filter_events_and_contexts_for_duplicates(cls, events_and_contexts):
	1172	"""Ensure that we don't have the same event twice.
	1173
	1174	Pick the earliest non-outlier if there is one, else the earliest one.
	1175
	1176	Args:
	1177	events_and_contexts (list[(EventBase, EventContext)]):
	1178	Returns:
	1179	list[(EventBase, EventContext)]: filtered list
	1180	"""
	1181	new_events_and_contexts = OrderedDict()
	1182	for event, context in events_and_contexts:
	1183	prev_event_context = new_events_and_contexts.get(event.event_id)
	1184	if prev_event_context:
	1185	if not event.internal_metadata.is_outlier():
	1186	if prev_event_context[0].internal_metadata.is_outlier():
	1187	# To ensure correct ordering we pop, as OrderedDict is
	1188	# ordered by first insertion.
	1189	new_events_and_contexts.pop(event.event_id, None)
	1190	new_events_and_contexts[event.event_id] = (event, context)
	1191	else:
	1192	new_events_and_contexts[event.event_id] = (event, context)
	1193	return list(new_events_and_contexts.values())
	1194
	1195	def _update_room_depths_txn(self, txn, events_and_contexts, backfilled):
	1196	"""Update min_depth for each room
	1197
	1198	Args:
	1199	txn (twisted.enterprise.adbapi.Connection): db connection
	1200	events_and_contexts (list[(EventBase, EventContext)]): events
	1201	we are persisting
	1202	backfilled (bool): True if the events were backfilled
	1203	"""
	1204	depth_updates = {}
	1205	for event, context in events_and_contexts:
	1206	# Remove the any existing cache entries for the event_ids
	1207	txn.call_after(self._invalidate_get_event_cache, event.event_id)
	1208	if not backfilled:
	1209	txn.call_after(
	1210	self._events_stream_cache.entity_has_changed,
	1211	event.room_id,
	1212	event.internal_metadata.stream_ordering,
	1213	)
	1214
	1215	if not event.internal_metadata.is_outlier() and not context.rejected:
	1216	depth_updates[event.room_id] = max(
	1217	event.depth, depth_updates.get(event.room_id, event.depth)
	1218	)
	1219
	1220	for room_id, depth in iteritems(depth_updates):
	1221	self._update_min_depth_for_room_txn(txn, room_id, depth)
	1222
	1223	def _update_outliers_txn(self, txn, events_and_contexts):
	1224	"""Update any outliers with new event info.
	1225
	1226	This turns outliers into ex-outliers (unless the new event was
	1227	rejected).
	1228
	1229	Args:
	1230	txn (twisted.enterprise.adbapi.Connection): db connection
	1231	events_and_contexts (list[(EventBase, EventContext)]): events
	1232	we are persisting
	1233
	1234	Returns:
	1235	list[(EventBase, EventContext)] new list, without events which
	1236	are already in the events table.
	1237	"""
	1238	txn.execute(
	1239	"SELECT event_id, outlier FROM events WHERE event_id in (%s)"
	1240	% (",".join(["?"] * len(events_and_contexts)),),
	1241	[event.event_id for event, _ in events_and_contexts],
	1242	)
	1243
	1244	have_persisted = {event_id: outlier for event_id, outlier in txn}
	1245
	1246	to_remove = set()
	1247	for event, context in events_and_contexts:
	1248	if event.event_id not in have_persisted:
	1249	continue
	1250
	1251	to_remove.add(event)
	1252
	1253	if context.rejected:
	1254	# If the event is rejected then we don't care if the event
	1255	# was an outlier or not.
	1256	continue
	1257
	1258	outlier_persisted = have_persisted[event.event_id]
	1259	if not event.internal_metadata.is_outlier() and outlier_persisted:
	1260	# We received a copy of an event that we had already stored as
	1261	# an outlier in the database. We now have some state at that
	1262	# so we need to update the state_groups table with that state.
	1263
	1264	# insert into event_to_state_groups.
	1265	try:
	1266	self._store_event_state_mappings_txn(txn, ((event, context),))
	1267	except Exception:
	1268	logger.exception("")
	1269	raise
	1270
	1271	metadata_json = encode_json(event.internal_metadata.get_dict())
	1272
	1273	sql = (
	1274	"UPDATE event_json SET internal_metadata = ?" " WHERE event_id = ?"
	1275	)
	1276	txn.execute(sql, (metadata_json, event.event_id))
	1277
	1278	# Add an entry to the ex_outlier_stream table to replicate the
	1279	# change in outlier status to our workers.
	1280	stream_order = event.internal_metadata.stream_ordering
	1281	state_group_id = context.state_group
	1282	self._simple_insert_txn(
	1283	txn,
	1284	table="ex_outlier_stream",
	1285	values={
	1286	"event_stream_ordering": stream_order,
	1287	"event_id": event.event_id,
	1288	"state_group": state_group_id,
	1289	},
	1290	)
	1291
	1292	sql = "UPDATE events SET outlier = ?" " WHERE event_id = ?"
	1293	txn.execute(sql, (False, event.event_id))
	1294
	1295	# Update the event_backward_extremities table now that this
	1296	# event isn't an outlier any more.
	1297	self._update_backward_extremeties(txn, [event])
	1298
	1299	return [ec for ec in events_and_contexts if ec[0] not in to_remove]
	1300
	1301	@classmethod
	1302	def _delete_existing_rows_txn(cls, txn, events_and_contexts):
	1303	if not events_and_contexts:
	1304	# nothing to do here
	1305	return
	1306
	1307	logger.info("Deleting existing")
	1308
	1309	for table in (
	1310	"events",
	1311	"event_auth",
	1312	"event_json",
	1313	"event_edges",
	1314	"event_forward_extremities",
	1315	"event_reference_hashes",
	1316	"event_search",
	1317	"event_to_state_groups",
	1318	"local_invites",
	1319	"state_events",
	1320	"rejections",
	1321	"redactions",
	1322	"room_memberships",
	1323	):
	1324	txn.executemany(
	1325	"DELETE FROM %s WHERE event_id = ?" % (table,),
	1326	[(ev.event_id,) for ev, _ in events_and_contexts],
	1327	)
	1328
	1329	for table in ("event_push_actions",):
	1330	txn.executemany(
	1331	"DELETE FROM %s WHERE room_id = ? AND event_id = ?" % (table,),
	1332	[(ev.room_id, ev.event_id) for ev, _ in events_and_contexts],
	1333	)
	1334
	1335	def _store_event_txn(self, txn, events_and_contexts):
	1336	"""Insert new events into the event and event_json tables
	1337
	1338	Args:
	1339	txn (twisted.enterprise.adbapi.Connection): db connection
	1340	events_and_contexts (list[(EventBase, EventContext)]): events
	1341	we are persisting
	1342	"""
	1343
	1344	if not events_and_contexts:
	1345	# nothing to do here
	1346	return
	1347
	1348	def event_dict(event):
	1349	d = event.get_dict()
	1350	d.pop("redacted", None)
	1351	d.pop("redacted_because", None)
	1352	return d
	1353
	1354	self._simple_insert_many_txn(
	1355	txn,
	1356	table="event_json",
	1357	values=[
	1358	{
	1359	"event_id": event.event_id,
	1360	"room_id": event.room_id,
	1361	"internal_metadata": encode_json(
	1362	event.internal_metadata.get_dict()
	1363	),
	1364	"json": encode_json(event_dict(event)),
	1365	"format_version": event.format_version,
	1366	}
	1367	for event, _ in events_and_contexts
	1368	],
	1369	)
	1370
	1371	self._simple_insert_many_txn(
	1372	txn,
	1373	table="events",
	1374	values=[
	1375	{
	1376	"stream_ordering": event.internal_metadata.stream_ordering,
	1377	"topological_ordering": event.depth,
	1378	"depth": event.depth,
	1379	"event_id": event.event_id,
	1380	"room_id": event.room_id,
	1381	"type": event.type,
	1382	"processed": True,
	1383	"outlier": event.internal_metadata.is_outlier(),
	1384	"origin_server_ts": int(event.origin_server_ts),
	1385	"received_ts": self._clock.time_msec(),
	1386	"sender": event.sender,
	1387	"contains_url": (
	1388	"url" in event.content
	1389	and isinstance(event.content["url"], text_type)
	1390	),
	1391	}
	1392	for event, _ in events_and_contexts
	1393	],
	1394	)
	1395
	1396	for event, _ in events_and_contexts:
	1397	if not event.internal_metadata.is_redacted():
	1398	# If we're persisting an unredacted event we go and ensure
	1399	# that we mark any redactions that reference this event as
	1400	# requiring censoring.
	1401	self._simple_update_txn(
	1402	txn,
	1403	table="redactions",
	1404	keyvalues={"redacts": event.event_id},
	1405	updatevalues={"have_censored": False},
	1406	)
	1407
	1408	def _store_rejected_events_txn(self, txn, events_and_contexts):
	1409	"""Add rows to the 'rejections' table for received events which were
	1410	rejected
	1411
	1412	Args:
	1413	txn (twisted.enterprise.adbapi.Connection): db connection
	1414	events_and_contexts (list[(EventBase, EventContext)]): events
	1415	we are persisting
	1416
	1417	Returns:
	1418	list[(EventBase, EventContext)] new list, without the rejected
	1419	events.
	1420	"""
	1421	# Remove the rejected events from the list now that we've added them
	1422	# to the events table and the events_json table.
	1423	to_remove = set()
	1424	for event, context in events_and_contexts:
	1425	if context.rejected:
	1426	# Insert the event_id into the rejections table
	1427	self._store_rejections_txn(txn, event.event_id, context.rejected)
	1428	to_remove.add(event)
	1429
	1430	return [ec for ec in events_and_contexts if ec[0] not in to_remove]
	1431
	1432	def _update_metadata_tables_txn(
	1433	self, txn, events_and_contexts, all_events_and_contexts, backfilled
	1434	):
	1435	"""Update all the miscellaneous tables for new events
	1436
	1437	Args:
	1438	txn (twisted.enterprise.adbapi.Connection): db connection
	1439	events_and_contexts (list[(EventBase, EventContext)]): events
	1440	we are persisting
	1441	all_events_and_contexts (list[(EventBase, EventContext)]): all
	1442	events that we were going to persist. This includes events
	1443	we've already persisted, etc, that wouldn't appear in
	1444	events_and_context.
	1445	backfilled (bool): True if the events were backfilled
	1446	"""
	1447
	1448	# Insert all the push actions into the event_push_actions table.
	1449	self._set_push_actions_for_event_and_users_txn(
	1450	txn,
	1451	events_and_contexts=events_and_contexts,
	1452	all_events_and_contexts=all_events_and_contexts,
	1453	)
	1454
	1455	if not events_and_contexts:
	1456	# nothing to do here
	1457	return
	1458
	1459	for event, context in events_and_contexts:
	1460	if event.type == EventTypes.Redaction and event.redacts is not None:
	1461	# Remove the entries in the event_push_actions table for the
	1462	# redacted event.
	1463	self._remove_push_actions_for_event_id_txn(
	1464	txn, event.room_id, event.redacts
	1465	)
	1466
	1467	# Remove from relations table.
	1468	self._handle_redaction(txn, event.redacts)
	1469
	1470	# Update the event_forward_extremities, event_backward_extremities and
	1471	# event_edges tables.
	1472	self._handle_mult_prev_events(
	1473	txn, events=[event for event, _ in events_and_contexts]
	1474	)
	1475
	1476	for event, _ in events_and_contexts:
	1477	if event.type == EventTypes.Name:
	1478	# Insert into the event_search table.
	1479	self._store_room_name_txn(txn, event)
	1480	elif event.type == EventTypes.Topic:
	1481	# Insert into the event_search table.
	1482	self._store_room_topic_txn(txn, event)
	1483	elif event.type == EventTypes.Message:
	1484	# Insert into the event_search table.
	1485	self._store_room_message_txn(txn, event)
	1486	elif event.type == EventTypes.Redaction:
	1487	# Insert into the redactions table.
	1488	self._store_redaction(txn, event)
	1489
	1490	self._handle_event_relations(txn, event)
	1491
	1492	# Insert into the room_memberships table.
	1493	self._store_room_members_txn(
	1494	txn,
	1495	[
	1496	event
	1497	for event, _ in events_and_contexts
	1498	if event.type == EventTypes.Member
	1499	],
	1500	backfilled=backfilled,
	1501	)
	1502
	1503	# Insert event_reference_hashes table.
	1504	self._store_event_reference_hashes_txn(
	1505	txn, [event for event, _ in events_and_contexts]
	1506	)
	1507
	1508	state_events_and_contexts = [
	1509	ec for ec in events_and_contexts if ec[0].is_state()
	1510	]
	1511
	1512	state_values = []
	1513	for event, context in state_events_and_contexts:
	1514	vals = {
	1515	"event_id": event.event_id,
	1516	"room_id": event.room_id,
	1517	"type": event.type,
	1518	"state_key": event.state_key,
	1519	}
	1520
	1521	# TODO: How does this work with backfilling?
	1522	if hasattr(event, "replaces_state"):
	1523	vals["prev_state"] = event.replaces_state
	1524
	1525	state_values.append(vals)
	1526
	1527	self._simple_insert_many_txn(txn, table="state_events", values=state_values)
	1528
	1529	# Prefill the event cache
	1530	self._add_to_cache(txn, events_and_contexts)
	1531
	1532	def _add_to_cache(self, txn, events_and_contexts):
	1533	to_prefill = []
	1534
	1535	rows = []
	1536	N = 200
	1537	for i in range(0, len(events_and_contexts), N):
	1538	ev_map = {e[0].event_id: e[0] for e in events_and_contexts[i : i + N]}
	1539	if not ev_map:
	1540	break
	1541
	1542	sql = (
	1543	"SELECT "
	1544	" e.event_id as event_id, "
	1545	" r.redacts as redacts,"
	1546	" rej.event_id as rejects "
	1547	" FROM events as e"
	1548	" LEFT JOIN rejections as rej USING (event_id)"
	1549	" LEFT JOIN redactions as r ON e.event_id = r.redacts"
	1550	" WHERE "
	1551	)
	1552
	1553	clause, args = make_in_list_sql_clause(
	1554	self.database_engine, "e.event_id", list(ev_map)
	1555	)
	1556
	1557	txn.execute(sql + clause, args)
	1558	rows = self.cursor_to_dict(txn)
	1559	for row in rows:
	1560	event = ev_map[row["event_id"]]
	1561	if not row["rejects"] and not row["redacts"]:
	1562	to_prefill.append(
	1563	_EventCacheEntry(event=event, redacted_event=None)
	1564	)
	1565
	1566	def prefill():
	1567	for cache_entry in to_prefill:
	1568	self._get_event_cache.prefill((cache_entry[0].event_id,), cache_entry)
	1569
	1570	txn.call_after(prefill)
	1571
	1572	def _store_redaction(self, txn, event):
	1573	# invalidate the cache for the redacted event
	1574	txn.call_after(self._invalidate_get_event_cache, event.redacts)
	1575
	1576	self._simple_insert_txn(
	1577	txn,
	1578	table="redactions",
	1579	values={
	1580	"event_id": event.event_id,
	1581	"redacts": event.redacts,
	1582	"received_ts": self._clock.time_msec(),
	1583	},
	1584	)
	1585
	1586	@defer.inlineCallbacks
	1587	def _censor_redactions(self):
	1588	"""Censors all redactions older than the configured period that haven't
	1589	been censored yet.
	1590
	1591	By censor we mean update the event_json table with the redacted event.
	1592
	1593	Returns:
	1594	Deferred
	1595	"""
	1596
	1597	if self.hs.config.redaction_retention_period is None:
	1598	return
	1599
	1600	before_ts = self._clock.time_msec() - self.hs.config.redaction_retention_period
	1601
	1602	# We fetch all redactions that:
	1603	# 1. point to an event we have,
	1604	# 2. has a received_ts from before the cut off, and
	1605	# 3. we haven't yet censored.
	1606	#
	1607	# This is limited to 100 events to ensure that we don't try and do too
	1608	# much at once. We'll get called again so this should eventually catch
	1609	# up.
	1610	sql = """
	1611	SELECT redactions.event_id, redacts FROM redactions
	1612	LEFT JOIN events AS original_event ON (
	1613	redacts = original_event.event_id
	1614	)
	1615	WHERE NOT have_censored
	1616	AND redactions.received_ts <= ?
	1617	ORDER BY redactions.received_ts ASC
	1618	LIMIT ?
	1619	"""
	1620
	1621	rows = yield self._execute(
	1622	"_censor_redactions_fetch", None, sql, before_ts, 100
	1623	)
	1624
	1625	updates = []
	1626
	1627	for redaction_id, event_id in rows:
	1628	redaction_event = yield self.get_event(redaction_id, allow_none=True)
	1629	original_event = yield self.get_event(
	1630	event_id, allow_rejected=True, allow_none=True
	1631	)
	1632
	1633	# The SQL above ensures that we have both the redaction and
	1634	# original event, so if the `get_event` calls return None it
	1635	# means that the redaction wasn't allowed. Either way we know that
	1636	# the result won't change so we mark the fact that we've checked.
	1637	if (
	1638	redaction_event
	1639	and original_event
	1640	and original_event.internal_metadata.is_redacted()
	1641	):
	1642	# Redaction was allowed
	1643	pruned_json = encode_json(prune_event_dict(original_event.get_dict()))
	1644	else:
	1645	# Redaction wasn't allowed
	1646	pruned_json = None
	1647
	1648	updates.append((redaction_id, event_id, pruned_json))
	1649
	1650	def _update_censor_txn(txn):
	1651	for redaction_id, event_id, pruned_json in updates:
	1652	if pruned_json:
	1653	self._simple_update_one_txn(
	1654	txn,
	1655	table="event_json",
	1656	keyvalues={"event_id": event_id},
	1657	updatevalues={"json": pruned_json},
	1658	)
	1659
	1660	self._simple_update_one_txn(
	1661	txn,
	1662	table="redactions",
	1663	keyvalues={"event_id": redaction_id},
	1664	updatevalues={"have_censored": True},
	1665	)
	1666
	1667	yield self.runInteraction("_update_censor_txn", _update_censor_txn)
	1668
	1669	@defer.inlineCallbacks
	1670	def count_daily_messages(self):
	1671	"""
	1672	Returns an estimate of the number of messages sent in the last day.
	1673
	1674	If it has been significantly less or more than one day since the last
	1675	call to this function, it will return None.
	1676	"""
	1677
	1678	def _count_messages(txn):
	1679	sql = """
	1680	SELECT COALESCE(COUNT(*), 0) FROM events
	1681	WHERE type = 'm.room.message'
	1682	AND stream_ordering > ?
	1683	"""
	1684	txn.execute(sql, (self.stream_ordering_day_ago,))
	1685	count, = txn.fetchone()
	1686	return count
	1687
	1688	ret = yield self.runInteraction("count_messages", _count_messages)
	1689	return ret
	1690
	1691	@defer.inlineCallbacks
	1692	def count_daily_sent_messages(self):
	1693	def _count_messages(txn):
	1694	# This is good enough as if you have silly characters in your own
	1695	# hostname then thats your own fault.
	1696	like_clause = "%:" + self.hs.hostname
	1697
	1698	sql = """
	1699	SELECT COALESCE(COUNT(*), 0) FROM events
	1700	WHERE type = 'm.room.message'
	1701	AND sender LIKE ?
	1702	AND stream_ordering > ?
	1703	"""
	1704
	1705	txn.execute(sql, (like_clause, self.stream_ordering_day_ago))
	1706	count, = txn.fetchone()
	1707	return count
	1708
	1709	ret = yield self.runInteraction("count_daily_sent_messages", _count_messages)
	1710	return ret
	1711
	1712	@defer.inlineCallbacks
	1713	def count_daily_active_rooms(self):
	1714	def _count(txn):
	1715	sql = """
	1716	SELECT COALESCE(COUNT(DISTINCT room_id), 0) FROM events
	1717	WHERE type = 'm.room.message'
	1718	AND stream_ordering > ?
	1719	"""
	1720	txn.execute(sql, (self.stream_ordering_day_ago,))
	1721	count, = txn.fetchone()
	1722	return count
	1723
	1724	ret = yield self.runInteraction("count_daily_active_rooms", _count)
	1725	return ret
	1726
	1727	def get_current_backfill_token(self):
	1728	"""The current minimum token that backfilled events have reached"""
	1729	return -self._backfill_id_gen.get_current_token()
	1730
	1731	def get_current_events_token(self):
	1732	"""The current maximum token that events have reached"""
	1733	return self._stream_id_gen.get_current_token()
	1734
	1735	def get_all_new_forward_event_rows(self, last_id, current_id, limit):
	1736	if last_id == current_id:
	1737	return defer.succeed([])
	1738
	1739	def get_all_new_forward_event_rows(txn):
	1740	sql = (
	1741	"SELECT e.stream_ordering, e.event_id, e.room_id, e.type,"
	1742	" state_key, redacts, relates_to_id"
	1743	" FROM events AS e"
	1744	" LEFT JOIN redactions USING (event_id)"
	1745	" LEFT JOIN state_events USING (event_id)"
	1746	" LEFT JOIN event_relations USING (event_id)"
	1747	" WHERE ? < stream_ordering AND stream_ordering <= ?"
	1748	" ORDER BY stream_ordering ASC"
	1749	" LIMIT ?"
	1750	)
	1751	txn.execute(sql, (last_id, current_id, limit))
	1752	new_event_updates = txn.fetchall()
	1753
	1754	if len(new_event_updates) == limit:
	1755	upper_bound = new_event_updates[-1][0]
	1756	else:
	1757	upper_bound = current_id
	1758
	1759	sql = (
	1760	"SELECT event_stream_ordering, e.event_id, e.room_id, e.type,"
	1761	" state_key, redacts, relates_to_id"
	1762	" FROM events AS e"
	1763	" INNER JOIN ex_outlier_stream USING (event_id)"
	1764	" LEFT JOIN redactions USING (event_id)"
	1765	" LEFT JOIN state_events USING (event_id)"
	1766	" LEFT JOIN event_relations USING (event_id)"
	1767	" WHERE ? < event_stream_ordering"
	1768	" AND event_stream_ordering <= ?"
	1769	" ORDER BY event_stream_ordering DESC"
	1770	)
	1771	txn.execute(sql, (last_id, upper_bound))
	1772	new_event_updates.extend(txn)
	1773
	1774	return new_event_updates
	1775
	1776	return self.runInteraction(
	1777	"get_all_new_forward_event_rows", get_all_new_forward_event_rows
	1778	)
	1779
	1780	def get_all_new_backfill_event_rows(self, last_id, current_id, limit):
	1781	if last_id == current_id:
	1782	return defer.succeed([])
	1783
	1784	def get_all_new_backfill_event_rows(txn):
	1785	sql = (
	1786	"SELECT -e.stream_ordering, e.event_id, e.room_id, e.type,"
	1787	" state_key, redacts, relates_to_id"
	1788	" FROM events AS e"
	1789	" LEFT JOIN redactions USING (event_id)"
	1790	" LEFT JOIN state_events USING (event_id)"
	1791	" LEFT JOIN event_relations USING (event_id)"
	1792	" WHERE ? > stream_ordering AND stream_ordering >= ?"
	1793	" ORDER BY stream_ordering ASC"
	1794	" LIMIT ?"
	1795	)
	1796	txn.execute(sql, (-last_id, -current_id, limit))
	1797	new_event_updates = txn.fetchall()
	1798
	1799	if len(new_event_updates) == limit:
	1800	upper_bound = new_event_updates[-1][0]
	1801	else:
	1802	upper_bound = current_id
	1803
	1804	sql = (
	1805	"SELECT -event_stream_ordering, e.event_id, e.room_id, e.type,"
	1806	" state_key, redacts, relates_to_id"
	1807	" FROM events AS e"
	1808	" INNER JOIN ex_outlier_stream USING (event_id)"
	1809	" LEFT JOIN redactions USING (event_id)"
	1810	" LEFT JOIN state_events USING (event_id)"
	1811	" LEFT JOIN event_relations USING (event_id)"
	1812	" WHERE ? > event_stream_ordering"
	1813	" AND event_stream_ordering >= ?"
	1814	" ORDER BY event_stream_ordering DESC"
	1815	)
	1816	txn.execute(sql, (-last_id, -upper_bound))
	1817	new_event_updates.extend(txn.fetchall())
	1818
	1819	return new_event_updates
	1820
	1821	return self.runInteraction(
	1822	"get_all_new_backfill_event_rows", get_all_new_backfill_event_rows
	1823	)
	1824
	1825	@cached(num_args=5, max_entries=10)
	1826	def get_all_new_events(
	1827	self,
	1828	last_backfill_id,
	1829	last_forward_id,
	1830	current_backfill_id,
	1831	current_forward_id,
	1832	limit,
	1833	):
	1834	"""Get all the new events that have arrived at the server either as
	1835	new events or as backfilled events"""
	1836	have_backfill_events = last_backfill_id != current_backfill_id
	1837	have_forward_events = last_forward_id != current_forward_id
	1838
	1839	if not have_backfill_events and not have_forward_events:
	1840	return defer.succeed(AllNewEventsResult([], [], [], [], []))
	1841
	1842	def get_all_new_events_txn(txn):
	1843	sql = (
	1844	"SELECT e.stream_ordering, e.event_id, e.room_id, e.type,"
	1845	" state_key, redacts"
	1846	" FROM events AS e"
	1847	" LEFT JOIN redactions USING (event_id)"
	1848	" LEFT JOIN state_events USING (event_id)"
	1849	" WHERE ? < stream_ordering AND stream_ordering <= ?"
	1850	" ORDER BY stream_ordering ASC"
	1851	" LIMIT ?"
	1852	)
	1853	if have_forward_events:
	1854	txn.execute(sql, (last_forward_id, current_forward_id, limit))
	1855	new_forward_events = txn.fetchall()
	1856
	1857	if len(new_forward_events) == limit:
	1858	upper_bound = new_forward_events[-1][0]
	1859	else:
	1860	upper_bound = current_forward_id
	1861
	1862	sql = (
	1863	"SELECT event_stream_ordering, event_id, state_group"
	1864	" FROM ex_outlier_stream"
	1865	" WHERE ? > event_stream_ordering"
	1866	" AND event_stream_ordering >= ?"
	1867	" ORDER BY event_stream_ordering DESC"
	1868	)
	1869	txn.execute(sql, (last_forward_id, upper_bound))
	1870	forward_ex_outliers = txn.fetchall()
	1871	else:
	1872	new_forward_events = []
	1873	forward_ex_outliers = []
	1874
	1875	sql = (
	1876	"SELECT -e.stream_ordering, e.event_id, e.room_id, e.type,"
	1877	" state_key, redacts"
	1878	" FROM events AS e"
	1879	" LEFT JOIN redactions USING (event_id)"
	1880	" LEFT JOIN state_events USING (event_id)"
	1881	" WHERE ? > stream_ordering AND stream_ordering >= ?"
	1882	" ORDER BY stream_ordering DESC"
	1883	" LIMIT ?"
	1884	)
	1885	if have_backfill_events:
	1886	txn.execute(sql, (-last_backfill_id, -current_backfill_id, limit))
	1887	new_backfill_events = txn.fetchall()
	1888
	1889	if len(new_backfill_events) == limit:
	1890	upper_bound = new_backfill_events[-1][0]
	1891	else:
	1892	upper_bound = current_backfill_id
	1893
	1894	sql = (
	1895	"SELECT -event_stream_ordering, event_id, state_group"
	1896	" FROM ex_outlier_stream"
	1897	" WHERE ? > event_stream_ordering"
	1898	" AND event_stream_ordering >= ?"
	1899	" ORDER BY event_stream_ordering DESC"
	1900	)
	1901	txn.execute(sql, (-last_backfill_id, -upper_bound))
	1902	backward_ex_outliers = txn.fetchall()
	1903	else:
	1904	new_backfill_events = []
	1905	backward_ex_outliers = []
	1906
	1907	return AllNewEventsResult(
	1908	new_forward_events,
	1909	new_backfill_events,
	1910	forward_ex_outliers,
	1911	backward_ex_outliers,
	1912	)
	1913
	1914	return self.runInteraction("get_all_new_events", get_all_new_events_txn)
	1915
	1916	def purge_history(self, room_id, token, delete_local_events):
	1917	"""Deletes room history before a certain point
	1918
	1919	Args:
	1920	room_id (str):
	1921
	1922	token (str): A topological token to delete events before
	1923
	1924	delete_local_events (bool):
	1925	if True, we will delete local events as well as remote ones
	1926	(instead of just marking them as outliers and deleting their
	1927	state groups).
	1928	"""
	1929
	1930	return self.runInteraction(
	1931	"purge_history",
	1932	self._purge_history_txn,
	1933	room_id,
	1934	token,
	1935	delete_local_events,
	1936	)
	1937
	1938	def _purge_history_txn(self, txn, room_id, token_str, delete_local_events):
	1939	token = RoomStreamToken.parse(token_str)
	1940
	1941	# Tables that should be pruned:
	1942	# event_auth
	1943	# event_backward_extremities
	1944	# event_edges
	1945	# event_forward_extremities
	1946	# event_json
	1947	# event_push_actions
	1948	# event_reference_hashes
	1949	# event_search
	1950	# event_to_state_groups
	1951	# events
	1952	# rejections
	1953	# room_depth
	1954	# state_groups
	1955	# state_groups_state
	1956
	1957	# we will build a temporary table listing the events so that we don't
	1958	# have to keep shovelling the list back and forth across the
	1959	# connection. Annoyingly the python sqlite driver commits the
	1960	# transaction on CREATE, so let's do this first.
	1961	#
	1962	# furthermore, we might already have the table from a previous (failed)
	1963	# purge attempt, so let's drop the table first.
	1964
	1965	txn.execute("DROP TABLE IF EXISTS events_to_purge")
	1966
	1967	txn.execute(
	1968	"CREATE TEMPORARY TABLE events_to_purge ("
	1969	" event_id TEXT NOT NULL,"
	1970	" should_delete BOOLEAN NOT NULL"
	1971	")"
	1972	)
	1973
	1974	# First ensure that we're not about to delete all the forward extremeties
	1975	txn.execute(
	1976	"SELECT e.event_id, e.depth FROM events as e "
	1977	"INNER JOIN event_forward_extremities as f "
	1978	"ON e.event_id = f.event_id "
	1979	"AND e.room_id = f.room_id "
	1980	"WHERE f.room_id = ?",
	1981	(room_id,),
	1982	)
	1983	rows = txn.fetchall()
	1984	max_depth = max(row[1] for row in rows)
	1985
	1986	if max_depth < token.topological:
	1987	# We need to ensure we don't delete all the events from the database
	1988	# otherwise we wouldn't be able to send any events (due to not
	1989	# having any backwards extremeties)
	1990	raise SynapseError(
	1991	400, "topological_ordering is greater than forward extremeties"
	1992	)
	1993
	1994	logger.info("[purge] looking for events to delete")
	1995
	1996	should_delete_expr = "state_key IS NULL"
	1997	should_delete_params = ()
	1998	if not delete_local_events:
	1999	should_delete_expr += " AND event_id NOT LIKE ?"
	2000
	2001	# We include the parameter twice since we use the expression twice
	2002	should_delete_params += ("%:" + self.hs.hostname, "%:" + self.hs.hostname)
	2003
	2004	should_delete_params += (room_id, token.topological)
	2005
	2006	# Note that we insert events that are outliers and aren't going to be
	2007	# deleted, as nothing will happen to them.
	2008	txn.execute(
	2009	"INSERT INTO events_to_purge"
	2010	" SELECT event_id, %s"
	2011	" FROM events AS e LEFT JOIN state_events USING (event_id)"
	2012	" WHERE (NOT outlier OR (%s)) AND e.room_id = ? AND topological_ordering < ?"
	2013	% (should_delete_expr, should_delete_expr),
	2014	should_delete_params,
	2015	)
	2016
	2017	# We create the indices after insertion as that's a lot faster.
	2018
	2019	# create an index on should_delete because later we'll be looking for
	2020	# the should_delete / shouldn't_delete subsets
	2021	txn.execute(
	2022	"CREATE INDEX events_to_purge_should_delete"
	2023	" ON events_to_purge(should_delete)"
	2024	)
	2025
	2026	# We do joins against events_to_purge for e.g. calculating state
	2027	# groups to purge, etc., so lets make an index.
	2028	txn.execute("CREATE INDEX events_to_purge_id" " ON events_to_purge(event_id)")
	2029
	2030	txn.execute("SELECT event_id, should_delete FROM events_to_purge")
	2031	event_rows = txn.fetchall()
	2032	logger.info(
	2033	"[purge] found %i events before cutoff, of which %i can be deleted",
	2034	len(event_rows),
	2035	sum(1 for e in event_rows if e[1]),
	2036	)
	2037
	2038	logger.info("[purge] Finding new backward extremities")
	2039
	2040	# We calculate the new entries for the backward extremeties by finding
	2041	# events to be purged that are pointed to by events we're not going to
	2042	# purge.
	2043	txn.execute(
	2044	"SELECT DISTINCT e.event_id FROM events_to_purge AS e"
	2045	" INNER JOIN event_edges AS ed ON e.event_id = ed.prev_event_id"
	2046	" LEFT JOIN events_to_purge AS ep2 ON ed.event_id = ep2.event_id"
	2047	" WHERE ep2.event_id IS NULL"
	2048	)
	2049	new_backwards_extrems = txn.fetchall()
	2050
	2051	logger.info("[purge] replacing backward extremities: %r", new_backwards_extrems)
	2052
	2053	txn.execute(
	2054	"DELETE FROM event_backward_extremities WHERE room_id = ?", (room_id,)
	2055	)
	2056
	2057	# Update backward extremeties
	2058	txn.executemany(
	2059	"INSERT INTO event_backward_extremities (room_id, event_id)"
	2060	" VALUES (?, ?)",
	2061	[(room_id, event_id) for event_id, in new_backwards_extrems],
	2062	)
	2063
	2064	logger.info("[purge] finding redundant state groups")
	2065
	2066	# Get all state groups that are referenced by events that are to be
	2067	# deleted. We then go and check if they are referenced by other events
	2068	# or state groups, and if not we delete them.
	2069	txn.execute(
	2070	"""
	2071	SELECT DISTINCT state_group FROM events_to_purge
	2072	INNER JOIN event_to_state_groups USING (event_id)
	2073	"""
	2074	)
	2075
	2076	referenced_state_groups = set(sg for sg, in txn)
	2077	logger.info(
	2078	"[purge] found %i referenced state groups", len(referenced_state_groups)
	2079	)
	2080
	2081	logger.info("[purge] finding state groups that can be deleted")
	2082
	2083	_ = self._find_unreferenced_groups_during_purge(txn, referenced_state_groups)
	2084	state_groups_to_delete, remaining_state_groups = _
	2085
	2086	logger.info(
	2087	"[purge] found %i state groups to delete", len(state_groups_to_delete)
	2088	)
	2089
	2090	logger.info(
	2091	"[purge] de-delta-ing %i remaining state groups",
	2092	len(remaining_state_groups),
	2093	)
	2094
	2095	# Now we turn the state groups that reference to-be-deleted state
	2096	# groups to non delta versions.
	2097	for sg in remaining_state_groups:
	2098	logger.info("[purge] de-delta-ing remaining state group %s", sg)
	2099	curr_state = self._get_state_groups_from_groups_txn(txn, [sg])
	2100	curr_state = curr_state[sg]
	2101
	2102	self._simple_delete_txn(
	2103	txn, table="state_groups_state", keyvalues={"state_group": sg}
	2104	)
	2105
	2106	self._simple_delete_txn(
	2107	txn, table="state_group_edges", keyvalues={"state_group": sg}
	2108	)
	2109
	2110	self._simple_insert_many_txn(
	2111	txn,
	2112	table="state_groups_state",
	2113	values=[
	2114	{
	2115	"state_group": sg,
	2116	"room_id": room_id,
	2117	"type": key[0],
	2118	"state_key": key[1],
	2119	"event_id": state_id,
	2120	}
	2121	for key, state_id in iteritems(curr_state)
	2122	],
	2123	)
	2124
	2125	logger.info("[purge] removing redundant state groups")
	2126	txn.executemany(
	2127	"DELETE FROM state_groups_state WHERE state_group = ?",
	2128	((sg,) for sg in state_groups_to_delete),
	2129	)
	2130	txn.executemany(
	2131	"DELETE FROM state_groups WHERE id = ?",
	2132	((sg,) for sg in state_groups_to_delete),
	2133	)
	2134
	2135	logger.info("[purge] removing events from event_to_state_groups")
	2136	txn.execute(
	2137	"DELETE FROM event_to_state_groups "
	2138	"WHERE event_id IN (SELECT event_id from events_to_purge)"
	2139	)
	2140	for event_id, _ in event_rows:
	2141	txn.call_after(self._get_state_group_for_event.invalidate, (event_id,))
	2142
	2143	# Delete all remote non-state events
	2144	for table in (
	2145	"events",
	2146	"event_json",
	2147	"event_auth",
	2148	"event_edges",
	2149	"event_forward_extremities",
	2150	"event_reference_hashes",
	2151	"event_search",
	2152	"rejections",
	2153	):
	2154	logger.info("[purge] removing events from %s", table)
	2155
	2156	txn.execute(
	2157	"DELETE FROM %s WHERE event_id IN ("
	2158	" SELECT event_id FROM events_to_purge WHERE should_delete"
	2159	")" % (table,)
	2160	)
	2161
	2162	# event_push_actions lacks an index on event_id, and has one on
	2163	# (room_id, event_id) instead.
	2164	for table in ("event_push_actions",):
	2165	logger.info("[purge] removing events from %s", table)
	2166
	2167	txn.execute(
	2168	"DELETE FROM %s WHERE room_id = ? AND event_id IN ("
	2169	" SELECT event_id FROM events_to_purge WHERE should_delete"
	2170	")" % (table,),
	2171	(room_id,),
	2172	)
	2173
	2174	# Mark all state and own events as outliers
	2175	logger.info("[purge] marking remaining events as outliers")
	2176	txn.execute(
	2177	"UPDATE events SET outlier = ?"
	2178	" WHERE event_id IN ("
	2179	" SELECT event_id FROM events_to_purge "
	2180	" WHERE NOT should_delete"
	2181	")",
	2182	(True,),
	2183	)
	2184
	2185	# synapse tries to take out an exclusive lock on room_depth whenever it
	2186	# persists events (because upsert), and once we run this update, we
	2187	# will block that for the rest of our transaction.
	2188	#
	2189	# So, let's stick it at the end so that we don't block event
	2190	# persistence.
	2191	#
	2192	# We do this by calculating the minimum depth of the backwards
	2193	# extremities. However, the events in event_backward_extremities
	2194	# are ones we don't have yet so we need to look at the events that
	2195	# point to it via event_edges table.
	2196	txn.execute(
	2197	"""
	2198	SELECT COALESCE(MIN(depth), 0)
	2199	FROM event_backward_extremities AS eb
	2200	INNER JOIN event_edges AS eg ON eg.prev_event_id = eb.event_id
	2201	INNER JOIN events AS e ON e.event_id = eg.event_id
	2202	WHERE eb.room_id = ?
	2203	""",
	2204	(room_id,),
	2205	)
	2206	min_depth, = txn.fetchone()
	2207
	2208	logger.info("[purge] updating room_depth to %d", min_depth)
	2209
	2210	txn.execute(
	2211	"UPDATE room_depth SET min_depth = ? WHERE room_id = ?",
	2212	(min_depth, room_id),
	2213	)
	2214
	2215	# finally, drop the temp table. this will commit the txn in sqlite,
	2216	# so make sure to keep this actually last.
	2217	txn.execute("DROP TABLE events_to_purge")
	2218
	2219	logger.info("[purge] done")
	2220
	2221	def _find_unreferenced_groups_during_purge(self, txn, state_groups):
	2222	"""Used when purging history to figure out which state groups can be
	2223	deleted and which need to be de-delta'ed (due to one of its prev groups
	2224	being scheduled for deletion).
	2225
	2226	Args:
	2227	txn
	2228	state_groups (set[int]): Set of state groups referenced by events
	2229	that are going to be deleted.
	2230
	2231	Returns:
	2232	tuple[set[int], set[int]]: The set of state groups that can be
	2233	deleted and the set of state groups that need to be de-delta'ed
	2234	"""
	2235	# Graph of state group -> previous group
	2236	graph = {}
	2237
	2238	# Set of events that we have found to be referenced by events
	2239	referenced_groups = set()
	2240
	2241	# Set of state groups we've already seen
	2242	state_groups_seen = set(state_groups)
	2243
	2244	# Set of state groups to handle next.
	2245	next_to_search = set(state_groups)
	2246	while next_to_search:
	2247	# We bound size of groups we're looking up at once, to stop the
	2248	# SQL query getting too big
	2249	if len(next_to_search) < 100:
	2250	current_search = next_to_search
	2251	next_to_search = set()
	2252	else:
	2253	current_search = set(itertools.islice(next_to_search, 100))
	2254	next_to_search -= current_search
	2255
	2256	# Check if state groups are referenced
	2257	sql = """
	2258	SELECT DISTINCT state_group FROM event_to_state_groups
	2259	LEFT JOIN events_to_purge AS ep USING (event_id)
	2260	WHERE ep.event_id IS NULL AND
	2261	"""
	2262	clause, args = make_in_list_sql_clause(
	2263	txn.database_engine, "state_group", current_search
	2264	)
	2265	txn.execute(sql + clause, list(args))
	2266
	2267	referenced = set(sg for sg, in txn)
	2268	referenced_groups \|= referenced
	2269
	2270	# We don't continue iterating up the state group graphs for state
	2271	# groups that are referenced.
	2272	current_search -= referenced
	2273
	2274	rows = self._simple_select_many_txn(
	2275	txn,
	2276	table="state_group_edges",
	2277	column="prev_state_group",
	2278	iterable=current_search,
	2279	keyvalues={},
	2280	retcols=("prev_state_group", "state_group"),
	2281	)
	2282
	2283	prevs = set(row["state_group"] for row in rows)
	2284	# We don't bother re-handling groups we've already seen
	2285	prevs -= state_groups_seen
	2286	next_to_search \|= prevs
	2287	state_groups_seen \|= prevs
	2288
	2289	for row in rows:
	2290	# Note: Each state group can have at most one prev group
	2291	graph[row["state_group"]] = row["prev_state_group"]
	2292
	2293	to_delete = state_groups_seen - referenced_groups
	2294
	2295	to_dedelta = set()
	2296	for sg in referenced_groups:
	2297	prev_sg = graph.get(sg)
	2298	if prev_sg and prev_sg in to_delete:
	2299	to_dedelta.add(sg)
	2300
	2301	return to_delete, to_dedelta
	2302
	2303	def purge_room(self, room_id):
	2304	"""Deletes all record of a room
	2305
	2306	Args:
	2307	room_id (str):
	2308	"""
	2309
	2310	return self.runInteraction("purge_room", self._purge_room_txn, room_id)
	2311
	2312	def _purge_room_txn(self, txn, room_id):
	2313	# first we have to delete the state groups states
	2314	logger.info("[purge] removing %s from state_groups_state", room_id)
	2315
	2316	txn.execute(
	2317	"""
	2318	DELETE FROM state_groups_state WHERE state_group IN (
	2319	SELECT state_group FROM events JOIN event_to_state_groups USING(event_id)
	2320	WHERE events.room_id=?
	2321	)
	2322	""",
	2323	(room_id,),
	2324	)
	2325
	2326	# ... and the state group edges
	2327	logger.info("[purge] removing %s from state_group_edges", room_id)
	2328
	2329	txn.execute(
	2330	"""
	2331	DELETE FROM state_group_edges WHERE state_group IN (
	2332	SELECT state_group FROM events JOIN event_to_state_groups USING(event_id)
	2333	WHERE events.room_id=?
	2334	)
	2335	""",
	2336	(room_id,),
	2337	)
	2338
	2339	# ... and the state groups
	2340	logger.info("[purge] removing %s from state_groups", room_id)
	2341
	2342	txn.execute(
	2343	"""
	2344	DELETE FROM state_groups WHERE id IN (
	2345	SELECT state_group FROM events JOIN event_to_state_groups USING(event_id)
	2346	WHERE events.room_id=?
	2347	)
	2348	""",
	2349	(room_id,),
	2350	)
	2351
	2352	# and then tables which lack an index on room_id but have one on event_id
	2353	for table in (
	2354	"event_auth",
	2355	"event_edges",
	2356	"event_push_actions_staging",
	2357	"event_reference_hashes",
	2358	"event_relations",
	2359	"event_to_state_groups",
	2360	"redactions",
	2361	"rejections",
	2362	"state_events",
	2363	):
	2364	logger.info("[purge] removing %s from %s", room_id, table)
	2365
	2366	txn.execute(
	2367	"""
	2368	DELETE FROM %s WHERE event_id IN (
	2369	SELECT event_id FROM events WHERE room_id=?
	2370	)
	2371	"""
	2372	% (table,),
	2373	(room_id,),
	2374	)
	2375
	2376	# and finally, the tables with an index on room_id (or no useful index)
	2377	for table in (
	2378	"current_state_events",
	2379	"event_backward_extremities",
	2380	"event_forward_extremities",
	2381	"event_json",
	2382	"event_push_actions",
	2383	"event_search",
	2384	"events",
	2385	"group_rooms",
	2386	"public_room_list_stream",
	2387	"receipts_graph",
	2388	"receipts_linearized",
	2389	"room_aliases",
	2390	"room_depth",
	2391	"room_memberships",
	2392	"room_stats_state",
	2393	"room_stats_current",
	2394	"room_stats_historical",
	2395	"room_stats_earliest_token",
	2396	"rooms",
	2397	"stream_ordering_to_exterm",
	2398	"topics",
	2399	"users_in_public_rooms",
	2400	"users_who_share_private_rooms",
	2401	# no useful index, but let's clear them anyway
	2402	"appservice_room_list",
	2403	"e2e_room_keys",
	2404	"event_push_summary",
	2405	"pusher_throttle",
	2406	"group_summary_rooms",
	2407	"local_invites",
	2408	"room_account_data",
	2409	"room_tags",
	2410	):
	2411	logger.info("[purge] removing %s from %s", room_id, table)
	2412	txn.execute("DELETE FROM %s WHERE room_id=?" % (table,), (room_id,))
	2413
	2414	# Other tables we do NOT need to clear out:
	2415	#
	2416	# - blocked_rooms
	2417	# This is important, to make sure that we don't accidentally rejoin a blocked
	2418	# room after it was purged
	2419	#
	2420	# - user_directory
	2421	# This has a room_id column, but it is unused
	2422	#
	2423
	2424	# Other tables that we might want to consider clearing out include:
	2425	#
	2426	# - event_reports
	2427	# Given that these are intended for abuse management my initial
	2428	# inclination is to leave them in place.
	2429	#
	2430	# - current_state_delta_stream
	2431	# - ex_outlier_stream
	2432	# - room_tags_revisions
	2433	# The problem with these is that they are largeish and there is no room_id
	2434	# index on them. In any case we should be clearing out 'stream' tables
	2435	# periodically anyway (#5888)
	2436
	2437	# TODO: we could probably usefully do a bunch of cache invalidation here
	2438
	2439	logger.info("[purge] done")
	2440
	2441	@defer.inlineCallbacks
	2442	def is_event_after(self, event_id1, event_id2):
	2443	"""Returns True if event_id1 is after event_id2 in the stream
	2444	"""
	2445	to_1, so_1 = yield self._get_event_ordering(event_id1)
	2446	to_2, so_2 = yield self._get_event_ordering(event_id2)
	2447	return (to_1, so_1) > (to_2, so_2)
	2448
	2449	@cachedInlineCallbacks(max_entries=5000)
	2450	def _get_event_ordering(self, event_id):
	2451	res = yield self._simple_select_one(
	2452	table="events",
	2453	retcols=["topological_ordering", "stream_ordering"],
	2454	keyvalues={"event_id": event_id},
	2455	allow_none=True,
	2456	)
	2457
	2458	if not res:
	2459	raise SynapseError(404, "Could not find event %s" % (event_id,))
	2460
	2461	return (int(res["topological_ordering"]), int(res["stream_ordering"]))
	2462
	2463	def get_all_updated_current_state_deltas(self, from_token, to_token, limit):
	2464	def get_all_updated_current_state_deltas_txn(txn):
	2465	sql = """
	2466	SELECT stream_id, room_id, type, state_key, event_id
	2467	FROM current_state_delta_stream
	2468	WHERE ? < stream_id AND stream_id <= ?
	2469	ORDER BY stream_id ASC LIMIT ?
	2470	"""
	2471	txn.execute(sql, (from_token, to_token, limit))
	2472	return txn.fetchall()
	2473
	2474	return self.runInteraction(
	2475	"get_all_updated_current_state_deltas",
	2476	get_all_updated_current_state_deltas_txn,
	2477	)
	2478
	2479
	2480	AllNewEventsResult = namedtuple(
	2481	"AllNewEventsResult",
	2482	[
	2483	"new_forward_events",
	2484	"new_backfill_events",
	2485	"forward_ex_outliers",
	2486	"backward_ex_outliers",
	2487	],
	2488	)

+505

-0

synapse/storage/data_stores/main/events_bg_updates.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2019 The Matrix.org Foundation C.I.C.
	2	#
	3	# Licensed under the Apache License, Version 2.0 (the "License");
	4	# you may not use this file except in compliance with the License.
	5	# You may obtain a copy of the License at
	6	#
	7	# http://www.apache.org/licenses/LICENSE-2.0
	8	#
	9	# Unless required by applicable law or agreed to in writing, software
	10	# distributed under the License is distributed on an "AS IS" BASIS,
	11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	# See the License for the specific language governing permissions and
	13	# limitations under the License.
	14
	15	import logging
	16
	17	from six import text_type
	18
	19	from canonicaljson import json
	20
	21	from twisted.internet import defer
	22
	23	from synapse.storage._base import make_in_list_sql_clause
	24	from synapse.storage.background_updates import BackgroundUpdateStore
	25
	26	logger = logging.getLogger(__name__)
	27
	28
	29	class EventsBackgroundUpdatesStore(BackgroundUpdateStore):
	30
	31	EVENT_ORIGIN_SERVER_TS_NAME = "event_origin_server_ts"
	32	EVENT_FIELDS_SENDER_URL_UPDATE_NAME = "event_fields_sender_url"
	33	DELETE_SOFT_FAILED_EXTREMITIES = "delete_soft_failed_extremities"
	34
	35	def __init__(self, db_conn, hs):
	36	super(EventsBackgroundUpdatesStore, self).__init__(db_conn, hs)
	37
	38	self.register_background_update_handler(
	39	self.EVENT_ORIGIN_SERVER_TS_NAME, self._background_reindex_origin_server_ts
	40	)
	41	self.register_background_update_handler(
	42	self.EVENT_FIELDS_SENDER_URL_UPDATE_NAME,
	43	self._background_reindex_fields_sender,
	44	)
	45
	46	self.register_background_index_update(
	47	"event_contains_url_index",
	48	index_name="event_contains_url_index",
	49	table="events",
	50	columns=["room_id", "topological_ordering", "stream_ordering"],
	51	where_clause="contains_url = true AND outlier = false",
	52	)
	53
	54	# an event_id index on event_search is useful for the purge_history
	55	# api. Plus it means we get to enforce some integrity with a UNIQUE
	56	# clause
	57	self.register_background_index_update(
	58	"event_search_event_id_idx",
	59	index_name="event_search_event_id_idx",
	60	table="event_search",
	61	columns=["event_id"],
	62	unique=True,
	63	psql_only=True,
	64	)
	65
	66	self.register_background_update_handler(
	67	self.DELETE_SOFT_FAILED_EXTREMITIES, self._cleanup_extremities_bg_update
	68	)
	69
	70	self.register_background_update_handler(
	71	"redactions_received_ts", self._redactions_received_ts
	72	)
	73
	74	# This index gets deleted in `event_fix_redactions_bytes` update
	75	self.register_background_index_update(
	76	"event_fix_redactions_bytes_create_index",
	77	index_name="redactions_censored_redacts",
	78	table="redactions",
	79	columns=["redacts"],
	80	where_clause="have_censored",
	81	)
	82
	83	self.register_background_update_handler(
	84	"event_fix_redactions_bytes", self._event_fix_redactions_bytes
	85	)
	86
	87	@defer.inlineCallbacks
	88	def _background_reindex_fields_sender(self, progress, batch_size):
	89	target_min_stream_id = progress["target_min_stream_id_inclusive"]
	90	max_stream_id = progress["max_stream_id_exclusive"]
	91	rows_inserted = progress.get("rows_inserted", 0)
	92
	93	INSERT_CLUMP_SIZE = 1000
	94
	95	def reindex_txn(txn):
	96	sql = (
	97	"SELECT stream_ordering, event_id, json FROM events"
	98	" INNER JOIN event_json USING (event_id)"
	99	" WHERE ? <= stream_ordering AND stream_ordering < ?"
	100	" ORDER BY stream_ordering DESC"
	101	" LIMIT ?"
	102	)
	103
	104	txn.execute(sql, (target_min_stream_id, max_stream_id, batch_size))
	105
	106	rows = txn.fetchall()
	107	if not rows:
	108	return 0
	109
	110	min_stream_id = rows[-1][0]
	111
	112	update_rows = []
	113	for row in rows:
	114	try:
	115	event_id = row[1]
	116	event_json = json.loads(row[2])
	117	sender = event_json["sender"]
	118	content = event_json["content"]
	119
	120	contains_url = "url" in content
	121	if contains_url:
	122	contains_url &= isinstance(content["url"], text_type)
	123	except (KeyError, AttributeError):
	124	# If the event is missing a necessary field then
	125	# skip over it.
	126	continue
	127
	128	update_rows.append((sender, contains_url, event_id))
	129
	130	sql = "UPDATE events SET sender = ?, contains_url = ? WHERE event_id = ?"
	131
	132	for index in range(0, len(update_rows), INSERT_CLUMP_SIZE):
	133	clump = update_rows[index : index + INSERT_CLUMP_SIZE]
	134	txn.executemany(sql, clump)
	135
	136	progress = {
	137	"target_min_stream_id_inclusive": target_min_stream_id,
	138	"max_stream_id_exclusive": min_stream_id,
	139	"rows_inserted": rows_inserted + len(rows),
	140	}
	141
	142	self._background_update_progress_txn(
	143	txn, self.EVENT_FIELDS_SENDER_URL_UPDATE_NAME, progress
	144	)
	145
	146	return len(rows)
	147
	148	result = yield self.runInteraction(
	149	self.EVENT_FIELDS_SENDER_URL_UPDATE_NAME, reindex_txn
	150	)
	151
	152	if not result:
	153	yield self._end_background_update(self.EVENT_FIELDS_SENDER_URL_UPDATE_NAME)
	154
	155	return result
	156
	157	@defer.inlineCallbacks
	158	def _background_reindex_origin_server_ts(self, progress, batch_size):
	159	target_min_stream_id = progress["target_min_stream_id_inclusive"]
	160	max_stream_id = progress["max_stream_id_exclusive"]
	161	rows_inserted = progress.get("rows_inserted", 0)
	162
	163	INSERT_CLUMP_SIZE = 1000
	164
	165	def reindex_search_txn(txn):
	166	sql = (
	167	"SELECT stream_ordering, event_id FROM events"
	168	" WHERE ? <= stream_ordering AND stream_ordering < ?"
	169	" ORDER BY stream_ordering DESC"
	170	" LIMIT ?"
	171	)
	172
	173	txn.execute(sql, (target_min_stream_id, max_stream_id, batch_size))
	174
	175	rows = txn.fetchall()
	176	if not rows:
	177	return 0
	178
	179	min_stream_id = rows[-1][0]
	180	event_ids = [row[1] for row in rows]
	181
	182	rows_to_update = []
	183
	184	chunks = [event_ids[i : i + 100] for i in range(0, len(event_ids), 100)]
	185	for chunk in chunks:
	186	ev_rows = self._simple_select_many_txn(
	187	txn,
	188	table="event_json",
	189	column="event_id",
	190	iterable=chunk,
	191	retcols=["event_id", "json"],
	192	keyvalues={},
	193	)
	194
	195	for row in ev_rows:
	196	event_id = row["event_id"]
	197	event_json = json.loads(row["json"])
	198	try:
	199	origin_server_ts = event_json["origin_server_ts"]
	200	except (KeyError, AttributeError):
	201	# If the event is missing a necessary field then
	202	# skip over it.
	203	continue
	204
	205	rows_to_update.append((origin_server_ts, event_id))
	206
	207	sql = "UPDATE events SET origin_server_ts = ? WHERE event_id = ?"
	208
	209	for index in range(0, len(rows_to_update), INSERT_CLUMP_SIZE):
	210	clump = rows_to_update[index : index + INSERT_CLUMP_SIZE]
	211	txn.executemany(sql, clump)
	212
	213	progress = {
	214	"target_min_stream_id_inclusive": target_min_stream_id,
	215	"max_stream_id_exclusive": min_stream_id,
	216	"rows_inserted": rows_inserted + len(rows_to_update),
	217	}
	218
	219	self._background_update_progress_txn(
	220	txn, self.EVENT_ORIGIN_SERVER_TS_NAME, progress
	221	)
	222
	223	return len(rows_to_update)
	224
	225	result = yield self.runInteraction(
	226	self.EVENT_ORIGIN_SERVER_TS_NAME, reindex_search_txn
	227	)
	228
	229	if not result:
	230	yield self._end_background_update(self.EVENT_ORIGIN_SERVER_TS_NAME)
	231
	232	return result
	233
	234	@defer.inlineCallbacks
	235	def _cleanup_extremities_bg_update(self, progress, batch_size):
	236	"""Background update to clean out extremities that should have been
	237	deleted previously.
	238
	239	Mainly used to deal with the aftermath of #5269.
	240	"""
	241
	242	# This works by first copying all existing forward extremities into the
	243	# `_extremities_to_check` table at start up, and then checking each
	244	# event in that table whether we have any descendants that are not
	245	# soft-failed/rejected. If that is the case then we delete that event
	246	# from the forward extremities table.
	247	#
	248	# For efficiency, we do this in batches by recursively pulling out all
	249	# descendants of a batch until we find the non soft-failed/rejected
	250	# events, i.e. the set of descendants whose chain of prev events back
	251	# to the batch of extremities are all soft-failed or rejected.
	252	# Typically, we won't find any such events as extremities will rarely
	253	# have any descendants, but if they do then we should delete those
	254	# extremities.
	255
	256	def _cleanup_extremities_bg_update_txn(txn):
	257	# The set of extremity event IDs that we're checking this round
	258	original_set = set()
	259
	260	# A dict[str, set[str]] of event ID to their prev events.
	261	graph = {}
	262
	263	# The set of descendants of the original set that are not rejected
	264	# nor soft-failed. Ancestors of these events should be removed
	265	# from the forward extremities table.
	266	non_rejected_leaves = set()
	267
	268	# Set of event IDs that have been soft failed, and for which we
	269	# should check if they have descendants which haven't been soft
	270	# failed.
	271	soft_failed_events_to_lookup = set()
	272
	273	# First, we get `batch_size` events from the table, pulling out
	274	# their successor events, if any, and the successor events'
	275	# rejection status.
	276	txn.execute(
	277	"""SELECT prev_event_id, event_id, internal_metadata,
	278	rejections.event_id IS NOT NULL, events.outlier
	279	FROM (
	280	SELECT event_id AS prev_event_id
	281	FROM _extremities_to_check
	282	LIMIT ?
	283	) AS f
	284	LEFT JOIN event_edges USING (prev_event_id)
	285	LEFT JOIN events USING (event_id)
	286	LEFT JOIN event_json USING (event_id)
	287	LEFT JOIN rejections USING (event_id)
	288	""",
	289	(batch_size,),
	290	)
	291
	292	for prev_event_id, event_id, metadata, rejected, outlier in txn:
	293	original_set.add(prev_event_id)
	294
	295	if not event_id or outlier:
	296	# Common case where the forward extremity doesn't have any
	297	# descendants.
	298	continue
	299
	300	graph.setdefault(event_id, set()).add(prev_event_id)
	301
	302	soft_failed = False
	303	if metadata:
	304	soft_failed = json.loads(metadata).get("soft_failed")
	305
	306	if soft_failed or rejected:
	307	soft_failed_events_to_lookup.add(event_id)
	308	else:
	309	non_rejected_leaves.add(event_id)
	310
	311	# Now we recursively check all the soft-failed descendants we
	312	# found above in the same way, until we have nothing left to
	313	# check.
	314	while soft_failed_events_to_lookup:
	315	# We only want to do 100 at a time, so we split given list
	316	# into two.
	317	batch = list(soft_failed_events_to_lookup)
	318	to_check, to_defer = batch[:100], batch[100:]
	319	soft_failed_events_to_lookup = set(to_defer)
	320
	321	sql = """SELECT prev_event_id, event_id, internal_metadata,
	322	rejections.event_id IS NOT NULL
	323	FROM event_edges
	324	INNER JOIN events USING (event_id)
	325	INNER JOIN event_json USING (event_id)
	326	LEFT JOIN rejections USING (event_id)
	327	WHERE
	328	NOT events.outlier
	329	AND
	330	"""
	331	clause, args = make_in_list_sql_clause(
	332	self.database_engine, "prev_event_id", to_check
	333	)
	334	txn.execute(sql + clause, list(args))
	335
	336	for prev_event_id, event_id, metadata, rejected in txn:
	337	if event_id in graph:
	338	# Already handled this event previously, but we still
	339	# want to record the edge.
	340	graph[event_id].add(prev_event_id)
	341	continue
	342
	343	graph[event_id] = {prev_event_id}
	344
	345	soft_failed = json.loads(metadata).get("soft_failed")
	346	if soft_failed or rejected:
	347	soft_failed_events_to_lookup.add(event_id)
	348	else:
	349	non_rejected_leaves.add(event_id)
	350
	351	# We have a set of non-soft-failed descendants, so we recurse up
	352	# the graph to find all ancestors and add them to the set of event
	353	# IDs that we can delete from forward extremities table.
	354	to_delete = set()
	355	while non_rejected_leaves:
	356	event_id = non_rejected_leaves.pop()
	357	prev_event_ids = graph.get(event_id, set())
	358	non_rejected_leaves.update(prev_event_ids)
	359	to_delete.update(prev_event_ids)
	360
	361	to_delete.intersection_update(original_set)
	362
	363	deleted = self._simple_delete_many_txn(
	364	txn=txn,
	365	table="event_forward_extremities",
	366	column="event_id",
	367	iterable=to_delete,
	368	keyvalues={},
	369	)
	370
	371	logger.info(
	372	"Deleted %d forward extremities of %d checked, to clean up #5269",
	373	deleted,
	374	len(original_set),
	375	)
	376
	377	if deleted:
	378	# We now need to invalidate the caches of these rooms
	379	rows = self._simple_select_many_txn(
	380	txn,
	381	table="events",
	382	column="event_id",
	383	iterable=to_delete,
	384	keyvalues={},
	385	retcols=("room_id",),
	386	)
	387	room_ids = set(row["room_id"] for row in rows)
	388	for room_id in room_ids:
	389	txn.call_after(
	390	self.get_latest_event_ids_in_room.invalidate, (room_id,)
	391	)
	392
	393	self._simple_delete_many_txn(
	394	txn=txn,
	395	table="_extremities_to_check",
	396	column="event_id",
	397	iterable=original_set,
	398	keyvalues={},
	399	)
	400
	401	return len(original_set)
	402
	403	num_handled = yield self.runInteraction(
	404	"_cleanup_extremities_bg_update", _cleanup_extremities_bg_update_txn
	405	)
	406
	407	if not num_handled:
	408	yield self._end_background_update(self.DELETE_SOFT_FAILED_EXTREMITIES)
	409
	410	def _drop_table_txn(txn):
	411	txn.execute("DROP TABLE _extremities_to_check")
	412
	413	yield self.runInteraction(
	414	"_cleanup_extremities_bg_update_drop_table", _drop_table_txn
	415	)
	416
	417	return num_handled
	418
	419	@defer.inlineCallbacks
	420	def _redactions_received_ts(self, progress, batch_size):
	421	"""Handles filling out the `received_ts` column in redactions.
	422	"""
	423	last_event_id = progress.get("last_event_id", "")
	424
	425	def _redactions_received_ts_txn(txn):
	426	# Fetch the set of event IDs that we want to update
	427	sql = """
	428	SELECT event_id FROM redactions
	429	WHERE event_id > ?
	430	ORDER BY event_id ASC
	431	LIMIT ?
	432	"""
	433
	434	txn.execute(sql, (last_event_id, batch_size))
	435
	436	rows = txn.fetchall()
	437	if not rows:
	438	return 0
	439
	440	upper_event_id, = rows[-1]
	441
	442	# Update the redactions with the received_ts.
	443	#
	444	# Note: Not all events have an associated received_ts, so we
	445	# fallback to using origin_server_ts. If we for some reason don't
	446	# have an origin_server_ts, lets just use the current timestamp.
	447	#
	448	# We don't want to leave it null, as then we'll never try and
	449	# censor those redactions.
	450	sql = """
	451	UPDATE redactions
	452	SET received_ts = (
	453	SELECT COALESCE(received_ts, origin_server_ts, ?) FROM events
	454	WHERE events.event_id = redactions.event_id
	455	)
	456	WHERE ? <= event_id AND event_id <= ?
	457	"""
	458
	459	txn.execute(sql, (self._clock.time_msec(), last_event_id, upper_event_id))
	460
	461	self._background_update_progress_txn(
	462	txn, "redactions_received_ts", {"last_event_id": upper_event_id}
	463	)
	464
	465	return len(rows)
	466
	467	count = yield self.runInteraction(
	468	"_redactions_received_ts", _redactions_received_ts_txn
	469	)
	470
	471	if not count:
	472	yield self._end_background_update("redactions_received_ts")
	473
	474	return count
	475
	476	@defer.inlineCallbacks
	477	def _event_fix_redactions_bytes(self, progress, batch_size):
	478	"""Undoes hex encoded censored redacted event JSON.
	479	"""
	480
	481	def _event_fix_redactions_bytes_txn(txn):
	482	# This update is quite fast due to new index.
	483	txn.execute(
	484	"""
	485	UPDATE event_json
	486	SET
	487	json = convert_from(json::bytea, 'utf8')
	488	FROM redactions
	489	WHERE
	490	redactions.have_censored
	491	AND event_json.event_id = redactions.redacts
	492	AND json NOT LIKE '{%';
	493	"""
	494	)
	495
	496	txn.execute("DROP INDEX redactions_censored_redacts")
	497
	498	yield self.runInteraction(
	499	"_event_fix_redactions_bytes", _event_fix_redactions_bytes_txn
	500	)
	501
	502	yield self._end_background_update("event_fix_redactions_bytes")
	503
	504	return 1

+882

-0

synapse/storage/data_stores/main/events_worker.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2018 New Vector Ltd
	2	#
	3	# Licensed under the Apache License, Version 2.0 (the "License");
	4	# you may not use this file except in compliance with the License.
	5	# You may obtain a copy of the License at
	6	#
	7	# http://www.apache.org/licenses/LICENSE-2.0
	8	#
	9	# Unless required by applicable law or agreed to in writing, software
	10	# distributed under the License is distributed on an "AS IS" BASIS,
	11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	# See the License for the specific language governing permissions and
	13	# limitations under the License.
	14
	15	from __future__ import division
	16
	17	import itertools
	18	import logging
	19	from collections import namedtuple
	20
	21	from canonicaljson import json
	22
	23	from twisted.internet import defer
	24
	25	from synapse.api.constants import EventTypes
	26	from synapse.api.errors import NotFoundError
	27	from synapse.api.room_versions import EventFormatVersions
	28	from synapse.events import FrozenEvent, event_type_from_format_version # noqa: F401
	29	from synapse.events.snapshot import EventContext # noqa: F401
	30	from synapse.events.utils import prune_event
	31	from synapse.logging.context import LoggingContext, PreserveLoggingContext
	32	from synapse.metrics.background_process_metrics import run_as_background_process
	33	from synapse.storage._base import SQLBaseStore, make_in_list_sql_clause
	34	from synapse.types import get_domain_from_id
	35	from synapse.util import batch_iter
	36	from synapse.util.metrics import Measure
	37
	38	logger = logging.getLogger(__name__)
	39
	40
	41	# These values are used in the `enqueus_event` and `_do_fetch` methods to
	42	# control how we batch/bulk fetch events from the database.
	43	# The values are plucked out of thing air to make initial sync run faster
	44	# on jki.re
	45	# TODO: Make these configurable.
	46	EVENT_QUEUE_THREADS = 3 # Max number of threads that will fetch events
	47	EVENT_QUEUE_ITERATIONS = 3 # No. times we block waiting for requests for events
	48	EVENT_QUEUE_TIMEOUT_S = 0.1 # Timeout when waiting for requests for events
	49
	50
	51	_EventCacheEntry = namedtuple("_EventCacheEntry", ("event", "redacted_event"))
	52
	53
	54	class EventsWorkerStore(SQLBaseStore):
	55	def get_received_ts(self, event_id):
	56	"""Get received_ts (when it was persisted) for the event.
	57
	58	Raises an exception for unknown events.
	59
	60	Args:
	61	event_id (str)
	62
	63	Returns:
	64	Deferred[int\|None]: Timestamp in milliseconds, or None for events
	65	that were persisted before received_ts was implemented.
	66	"""
	67	return self._simple_select_one_onecol(
	68	table="events",
	69	keyvalues={"event_id": event_id},
	70	retcol="received_ts",
	71	desc="get_received_ts",
	72	)
	73
	74	def get_received_ts_by_stream_pos(self, stream_ordering):
	75	"""Given a stream ordering get an approximate timestamp of when it
	76	happened.
	77
	78	This is done by simply taking the received ts of the first event that
	79	has a stream ordering greater than or equal to the given stream pos.
	80	If none exists returns the current time, on the assumption that it must
	81	have happened recently.
	82
	83	Args:
	84	stream_ordering (int)
	85
	86	Returns:
	87	Deferred[int]
	88	"""
	89
	90	def _get_approximate_received_ts_txn(txn):
	91	sql = """
	92	SELECT received_ts FROM events
	93	WHERE stream_ordering >= ?
	94	LIMIT 1
	95	"""
	96
	97	txn.execute(sql, (stream_ordering,))
	98	row = txn.fetchone()
	99	if row and row[0]:
	100	ts = row[0]
	101	else:
	102	ts = self.clock.time_msec()
	103
	104	return ts
	105
	106	return self.runInteraction(
	107	"get_approximate_received_ts", _get_approximate_received_ts_txn
	108	)
	109
	110	@defer.inlineCallbacks
	111	def get_event(
	112	self,
	113	event_id,
	114	check_redacted=True,
	115	get_prev_content=False,
	116	allow_rejected=False,
	117	allow_none=False,
	118	check_room_id=None,
	119	):
	120	"""Get an event from the database by event_id.
	121
	122	Args:
	123	event_id (str): The event_id of the event to fetch
	124	check_redacted (bool): If True, check if event has been redacted
	125	and redact it.
	126	get_prev_content (bool): If True and event is a state event,
	127	include the previous states content in the unsigned field.
	128	allow_rejected (bool): If True return rejected events.
	129	allow_none (bool): If True, return None if no event found, if
	130	False throw a NotFoundError
	131	check_room_id (str\|None): if not None, check the room of the found event.
	132	If there is a mismatch, behave as per allow_none.
	133
	134	Returns:
	135	Deferred[EventBase\|None]
	136	"""
	137	if not isinstance(event_id, str):
	138	raise TypeError("Invalid event event_id %r" % (event_id,))
	139
	140	events = yield self.get_events_as_list(
	141	[event_id],
	142	check_redacted=check_redacted,
	143	get_prev_content=get_prev_content,
	144	allow_rejected=allow_rejected,
	145	)
	146
	147	event = events[0] if events else None
	148
	149	if event is not None and check_room_id is not None:
	150	if event.room_id != check_room_id:
	151	event = None
	152
	153	if event is None and not allow_none:
	154	raise NotFoundError("Could not find event %s" % (event_id,))
	155
	156	return event
	157
	158	@defer.inlineCallbacks
	159	def get_events(
	160	self,
	161	event_ids,
	162	check_redacted=True,
	163	get_prev_content=False,
	164	allow_rejected=False,
	165	):
	166	"""Get events from the database
	167
	168	Args:
	169	event_ids (list): The event_ids of the events to fetch
	170	check_redacted (bool): If True, check if event has been redacted
	171	and redact it.
	172	get_prev_content (bool): If True and event is a state event,
	173	include the previous states content in the unsigned field.
	174	allow_rejected (bool): If True return rejected events.
	175
	176	Returns:
	177	Deferred : Dict from event_id to event.
	178	"""
	179	events = yield self.get_events_as_list(
	180	event_ids,
	181	check_redacted=check_redacted,
	182	get_prev_content=get_prev_content,
	183	allow_rejected=allow_rejected,
	184	)
	185
	186	return {e.event_id: e for e in events}
	187
	188	@defer.inlineCallbacks
	189	def get_events_as_list(
	190	self,
	191	event_ids,
	192	check_redacted=True,
	193	get_prev_content=False,
	194	allow_rejected=False,
	195	):
	196	"""Get events from the database and return in a list in the same order
	197	as given by `event_ids` arg.
	198
	199	Args:
	200	event_ids (list): The event_ids of the events to fetch
	201	check_redacted (bool): If True, check if event has been redacted
	202	and redact it.
	203	get_prev_content (bool): If True and event is a state event,
	204	include the previous states content in the unsigned field.
	205	allow_rejected (bool): If True return rejected events.
	206
	207	Returns:
	208	Deferred[list[EventBase]]: List of events fetched from the database. The
	209	events are in the same order as `event_ids` arg.
	210
	211	Note that the returned list may be smaller than the list of event
	212	IDs if not all events could be fetched.
	213	"""
	214
	215	if not event_ids:
	216	return []
	217
	218	# there may be duplicates so we cast the list to a set
	219	event_entry_map = yield self._get_events_from_cache_or_db(
	220	set(event_ids), allow_rejected=allow_rejected
	221	)
	222
	223	events = []
	224	for event_id in event_ids:
	225	entry = event_entry_map.get(event_id, None)
	226	if not entry:
	227	continue
	228
	229	if not allow_rejected:
	230	assert not entry.event.rejected_reason, (
	231	"rejected event returned from _get_events_from_cache_or_db despite "
	232	"allow_rejected=False"
	233	)
	234
	235	# We may not have had the original event when we received a redaction, so
	236	# we have to recheck auth now.
	237
	238	if not allow_rejected and entry.event.type == EventTypes.Redaction:
	239	if not hasattr(entry.event, "redacts"):
	240	# A redacted redaction doesn't have a `redacts` key, in
	241	# which case lets just withhold the event.
	242	#
	243	# Note: Most of the time if the redactions has been
	244	# redacted we still have the un-redacted event in the DB
	245	# and so we'll still see the `redacts` key. However, this
	246	# isn't always true e.g. if we have censored the event.
	247	logger.debug(
	248	"Withholding redaction event %s as we don't have redacts key",
	249	event_id,
	250	)
	251	continue
	252
	253	redacted_event_id = entry.event.redacts
	254	event_map = yield self._get_events_from_cache_or_db([redacted_event_id])
	255	original_event_entry = event_map.get(redacted_event_id)
	256	if not original_event_entry:
	257	# we don't have the redacted event (or it was rejected).
	258	#
	259	# We assume that the redaction isn't authorized for now; if the
	260	# redacted event later turns up, the redaction will be re-checked,
	261	# and if it is found valid, the original will get redacted before it
	262	# is served to the client.
	263	logger.debug(
	264	"Withholding redaction event %s since we don't (yet) have the "
	265	"original %s",
	266	event_id,
	267	redacted_event_id,
	268	)
	269	continue
	270
	271	original_event = original_event_entry.event
	272	if original_event.type == EventTypes.Create:
	273	# we never serve redactions of Creates to clients.
	274	logger.info(
	275	"Withholding redaction %s of create event %s",
	276	event_id,
	277	redacted_event_id,
	278	)
	279	continue
	280
	281	if original_event.room_id != entry.event.room_id:
	282	logger.info(
	283	"Withholding redaction %s of event %s from a different room",
	284	event_id,
	285	redacted_event_id,
	286	)
	287	continue
	288
	289	if entry.event.internal_metadata.need_to_check_redaction():
	290	original_domain = get_domain_from_id(original_event.sender)
	291	redaction_domain = get_domain_from_id(entry.event.sender)
	292	if original_domain != redaction_domain:
	293	# the senders don't match, so this is forbidden
	294	logger.info(
	295	"Withholding redaction %s whose sender domain %s doesn't "
	296	"match that of redacted event %s %s",
	297	event_id,
	298	redaction_domain,
	299	redacted_event_id,
	300	original_domain,
	301	)
	302	continue
	303
	304	# Update the cache to save doing the checks again.
	305	entry.event.internal_metadata.recheck_redaction = False
	306
	307	if check_redacted and entry.redacted_event:
	308	event = entry.redacted_event
	309	else:
	310	event = entry.event
	311
	312	events.append(event)
	313
	314	if get_prev_content:
	315	if "replaces_state" in event.unsigned:
	316	prev = yield self.get_event(
	317	event.unsigned["replaces_state"],
	318	get_prev_content=False,
	319	allow_none=True,
	320	)
	321	if prev:
	322	event.unsigned = dict(event.unsigned)
	323	event.unsigned["prev_content"] = prev.content
	324	event.unsigned["prev_sender"] = prev.sender
	325
	326	return events
	327
	328	@defer.inlineCallbacks
	329	def _get_events_from_cache_or_db(self, event_ids, allow_rejected=False):
	330	"""Fetch a bunch of events from the cache or the database.
	331
	332	If events are pulled from the database, they will be cached for future lookups.
	333
	334	Args:
	335	event_ids (Iterable[str]): The event_ids of the events to fetch
	336	allow_rejected (bool): Whether to include rejected events
	337
	338	Returns:
	339	Deferred[Dict[str, _EventCacheEntry]]:
	340	map from event id to result
	341	"""
	342	event_entry_map = self._get_events_from_cache(
	343	event_ids, allow_rejected=allow_rejected
	344	)
	345
	346	missing_events_ids = [e for e in event_ids if e not in event_entry_map]
	347
	348	if missing_events_ids:
	349	log_ctx = LoggingContext.current_context()
	350	log_ctx.record_event_fetch(len(missing_events_ids))
	351
	352	# Note that _get_events_from_db is also responsible for turning db rows
	353	# into FrozenEvents (via _get_event_from_row), which involves seeing if
	354	# the events have been redacted, and if so pulling the redaction event out
	355	# of the database to check it.
	356	#
	357	missing_events = yield self._get_events_from_db(
	358	missing_events_ids, allow_rejected=allow_rejected
	359	)
	360
	361	event_entry_map.update(missing_events)
	362
	363	return event_entry_map
	364
	365	def _invalidate_get_event_cache(self, event_id):
	366	self._get_event_cache.invalidate((event_id,))
	367
	368	def _get_events_from_cache(self, events, allow_rejected, update_metrics=True):
	369	"""Fetch events from the caches
	370
	371	Args:
	372	events (Iterable[str]): list of event_ids to fetch
	373	allow_rejected (bool): Whether to return events that were rejected
	374	update_metrics (bool): Whether to update the cache hit ratio metrics
	375
	376	Returns:
	377	dict of event_id -> _EventCacheEntry for each event_id in cache. If
	378	allow_rejected is `False` then there will still be an entry but it
	379	will be `None`
	380	"""
	381	event_map = {}
	382
	383	for event_id in events:
	384	ret = self._get_event_cache.get(
	385	(event_id,), None, update_metrics=update_metrics
	386	)
	387	if not ret:
	388	continue
	389
	390	if allow_rejected or not ret.event.rejected_reason:
	391	event_map[event_id] = ret
	392	else:
	393	event_map[event_id] = None
	394
	395	return event_map
	396
	397	def _do_fetch(self, conn):
	398	"""Takes a database connection and waits for requests for events from
	399	the _event_fetch_list queue.
	400	"""
	401	i = 0
	402	while True:
	403	with self._event_fetch_lock:
	404	event_list = self._event_fetch_list
	405	self._event_fetch_list = []
	406
	407	if not event_list:
	408	single_threaded = self.database_engine.single_threaded
	409	if single_threaded or i > EVENT_QUEUE_ITERATIONS:
	410	self._event_fetch_ongoing -= 1
	411	return
	412	else:
	413	self._event_fetch_lock.wait(EVENT_QUEUE_TIMEOUT_S)
	414	i += 1
	415	continue
	416	i = 0
	417
	418	self._fetch_event_list(conn, event_list)
	419
	420	def _fetch_event_list(self, conn, event_list):
	421	"""Handle a load of requests from the _event_fetch_list queue
	422
	423	Args:
	424	conn (twisted.enterprise.adbapi.Connection): database connection
	425
	426	event_list (list[Tuple[list[str], Deferred]]):
	427	The fetch requests. Each entry consists of a list of event
	428	ids to be fetched, and a deferred to be completed once the
	429	events have been fetched.
	430
	431	The deferreds are callbacked with a dictionary mapping from event id
	432	to event row. Note that it may well contain additional events that
	433	were not part of this request.
	434	"""
	435	with Measure(self._clock, "_fetch_event_list"):
	436	try:
	437	events_to_fetch = set(
	438	event_id for events, _ in event_list for event_id in events
	439	)
	440
	441	row_dict = self._new_transaction(
	442	conn, "do_fetch", [], [], self._fetch_event_rows, events_to_fetch
	443	)
	444
	445	# We only want to resolve deferreds from the main thread
	446	def fire():
	447	for _, d in event_list:
	448	d.callback(row_dict)
	449
	450	with PreserveLoggingContext():
	451	self.hs.get_reactor().callFromThread(fire)
	452	except Exception as e:
	453	logger.exception("do_fetch")
	454
	455	# We only want to resolve deferreds from the main thread
	456	def fire(evs, exc):
	457	for _, d in evs:
	458	if not d.called:
	459	with PreserveLoggingContext():
	460	d.errback(exc)
	461
	462	with PreserveLoggingContext():
	463	self.hs.get_reactor().callFromThread(fire, event_list, e)
	464
	465	@defer.inlineCallbacks
	466	def _get_events_from_db(self, event_ids, allow_rejected=False):
	467	"""Fetch a bunch of events from the database.
	468
	469	Returned events will be added to the cache for future lookups.
	470
	471	Args:
	472	event_ids (Iterable[str]): The event_ids of the events to fetch
	473	allow_rejected (bool): Whether to include rejected events
	474
	475	Returns:
	476	Deferred[Dict[str, _EventCacheEntry]]:
	477	map from event id to result. May return extra events which
	478	weren't asked for.
	479	"""
	480	fetched_events = {}
	481	events_to_fetch = event_ids
	482
	483	while events_to_fetch:
	484	row_map = yield self._enqueue_events(events_to_fetch)
	485
	486	# we need to recursively fetch any redactions of those events
	487	redaction_ids = set()
	488	for event_id in events_to_fetch:
	489	row = row_map.get(event_id)
	490	fetched_events[event_id] = row
	491	if row:
	492	redaction_ids.update(row["redactions"])
	493
	494	events_to_fetch = redaction_ids.difference(fetched_events.keys())
	495	if events_to_fetch:
	496	logger.debug("Also fetching redaction events %s", events_to_fetch)
	497
	498	# build a map from event_id to EventBase
	499	event_map = {}
	500	for event_id, row in fetched_events.items():
	501	if not row:
	502	continue
	503	assert row["event_id"] == event_id
	504
	505	rejected_reason = row["rejected_reason"]
	506
	507	if not allow_rejected and rejected_reason:
	508	continue
	509
	510	d = json.loads(row["json"])
	511	internal_metadata = json.loads(row["internal_metadata"])
	512
	513	format_version = row["format_version"]
	514	if format_version is None:
	515	# This means that we stored the event before we had the concept
	516	# of a event format version, so it must be a V1 event.
	517	format_version = EventFormatVersions.V1
	518
	519	original_ev = event_type_from_format_version(format_version)(
	520	event_dict=d,
	521	internal_metadata_dict=internal_metadata,
	522	rejected_reason=rejected_reason,
	523	)
	524
	525	event_map[event_id] = original_ev
	526
	527	# finally, we can decide whether each one nededs redacting, and build
	528	# the cache entries.
	529	result_map = {}
	530	for event_id, original_ev in event_map.items():
	531	redactions = fetched_events[event_id]["redactions"]
	532	redacted_event = self._maybe_redact_event_row(
	533	original_ev, redactions, event_map
	534	)
	535
	536	cache_entry = _EventCacheEntry(
	537	event=original_ev, redacted_event=redacted_event
	538	)
	539
	540	self._get_event_cache.prefill((event_id,), cache_entry)
	541	result_map[event_id] = cache_entry
	542
	543	return result_map
	544
	545	@defer.inlineCallbacks
	546	def _enqueue_events(self, events):
	547	"""Fetches events from the database using the _event_fetch_list. This
	548	allows batch and bulk fetching of events - it allows us to fetch events
	549	without having to create a new transaction for each request for events.
	550
	551	Args:
	552	events (Iterable[str]): events to be fetched.
	553
	554	Returns:
	555	Deferred[Dict[str, Dict]]: map from event id to row data from the database.
	556	May contain events that weren't requested.
	557	"""
	558
	559	events_d = defer.Deferred()
	560	with self._event_fetch_lock:
	561	self._event_fetch_list.append((events, events_d))
	562
	563	self._event_fetch_lock.notify()
	564
	565	if self._event_fetch_ongoing < EVENT_QUEUE_THREADS:
	566	self._event_fetch_ongoing += 1
	567	should_start = True
	568	else:
	569	should_start = False
	570
	571	if should_start:
	572	run_as_background_process(
	573	"fetch_events", self.runWithConnection, self._do_fetch
	574	)
	575
	576	logger.debug("Loading %d events: %s", len(events), events)
	577	with PreserveLoggingContext():
	578	row_map = yield events_d
	579	logger.debug("Loaded %d events (%d rows)", len(events), len(row_map))
	580
	581	return row_map
	582
	583	def _fetch_event_rows(self, txn, event_ids):
	584	"""Fetch event rows from the database
	585
	586	Events which are not found are omitted from the result.
	587
	588	The returned per-event dicts contain the following keys:
	589
	590	* event_id (str)
	591
	592	* json (str): json-encoded event structure
	593
	594	* internal_metadata (str): json-encoded internal metadata dict
	595
	596	* format_version (int\|None): The format of the event. Hopefully one
	597	of EventFormatVersions. 'None' means the event predates
	598	EventFormatVersions (so the event is format V1).
	599
	600	* rejected_reason (str\|None): if the event was rejected, the reason
	601	why.
	602
	603	* redactions (List[str]): a list of event-ids which (claim to) redact
	604	this event.
	605
	606	Args:
	607	txn (twisted.enterprise.adbapi.Connection):
	608	event_ids (Iterable[str]): event IDs to fetch
	609
	610	Returns:
	611	Dict[str, Dict]: a map from event id to event info.
	612	"""
	613	event_dict = {}
	614	for evs in batch_iter(event_ids, 200):
	615	sql = (
	616	"SELECT "
	617	" e.event_id, "
	618	" e.internal_metadata,"
	619	" e.json,"
	620	" e.format_version, "
	621	" rej.reason "
	622	" FROM event_json as e"
	623	" LEFT JOIN rejections as rej USING (event_id)"
	624	" WHERE "
	625	)
	626
	627	clause, args = make_in_list_sql_clause(
	628	txn.database_engine, "e.event_id", evs
	629	)
	630
	631	txn.execute(sql + clause, args)
	632
	633	for row in txn:
	634	event_id = row[0]
	635	event_dict[event_id] = {
	636	"event_id": event_id,
	637	"internal_metadata": row[1],
	638	"json": row[2],
	639	"format_version": row[3],
	640	"rejected_reason": row[4],
	641	"redactions": [],
	642	}
	643
	644	# check for redactions
	645	redactions_sql = "SELECT event_id, redacts FROM redactions WHERE "
	646
	647	clause, args = make_in_list_sql_clause(txn.database_engine, "redacts", evs)
	648
	649	txn.execute(redactions_sql + clause, args)
	650
	651	for (redacter, redacted) in txn:
	652	d = event_dict.get(redacted)
	653	if d:
	654	d["redactions"].append(redacter)
	655
	656	return event_dict
	657
	658	def _maybe_redact_event_row(self, original_ev, redactions, event_map):
	659	"""Given an event object and a list of possible redacting event ids,
	660	determine whether to honour any of those redactions and if so return a redacted
	661	event.
	662
	663	Args:
	664	original_ev (EventBase):
	665	redactions (iterable[str]): list of event ids of potential redaction events
	666	event_map (dict[str, EventBase]): other events which have been fetched, in
	667	which we can look up the redaaction events. Map from event id to event.
	668
	669	Returns:
	670	Deferred[EventBase\|None]: if the event should be redacted, a pruned
	671	event object. Otherwise, None.
	672	"""
	673	if original_ev.type == "m.room.create":
	674	# we choose to ignore redactions of m.room.create events.
	675	return None
	676
	677	for redaction_id in redactions:
	678	redaction_event = event_map.get(redaction_id)
	679	if not redaction_event or redaction_event.rejected_reason:
	680	# we don't have the redaction event, or the redaction event was not
	681	# authorized.
	682	logger.debug(
	683	"%s was redacted by %s but redaction not found/authed",
	684	original_ev.event_id,
	685	redaction_id,
	686	)
	687	continue
	688
	689	if redaction_event.room_id != original_ev.room_id:
	690	logger.debug(
	691	"%s was redacted by %s but redaction was in a different room!",
	692	original_ev.event_id,
	693	redaction_id,
	694	)
	695	continue
	696
	697	# Starting in room version v3, some redactions need to be
	698	# rechecked if we didn't have the redacted event at the
	699	# time, so we recheck on read instead.
	700	if redaction_event.internal_metadata.need_to_check_redaction():
	701	expected_domain = get_domain_from_id(original_ev.sender)
	702	if get_domain_from_id(redaction_event.sender) == expected_domain:
	703	# This redaction event is allowed. Mark as not needing a recheck.
	704	redaction_event.internal_metadata.recheck_redaction = False
	705	else:
	706	# Senders don't match, so the event isn't actually redacted
	707	logger.debug(
	708	"%s was redacted by %s but the senders don't match",
	709	original_ev.event_id,
	710	redaction_id,
	711	)
	712	continue
	713
	714	logger.debug("Redacting %s due to %s", original_ev.event_id, redaction_id)
	715
	716	# we found a good redaction event. Redact!
	717	redacted_event = prune_event(original_ev)
	718	redacted_event.unsigned["redacted_by"] = redaction_id
	719
	720	# It's fine to add the event directly, since get_pdu_json
	721	# will serialise this field correctly
	722	redacted_event.unsigned["redacted_because"] = redaction_event
	723
	724	return redacted_event
	725
	726	# no valid redaction found for this event
	727	return None
	728
	729	@defer.inlineCallbacks
	730	def have_events_in_timeline(self, event_ids):
	731	"""Given a list of event ids, check if we have already processed and
	732	stored them as non outliers.
	733	"""
	734	rows = yield self._simple_select_many_batch(
	735	table="events",
	736	retcols=("event_id",),
	737	column="event_id",
	738	iterable=list(event_ids),
	739	keyvalues={"outlier": False},
	740	desc="have_events_in_timeline",
	741	)
	742
	743	return set(r["event_id"] for r in rows)
	744
	745	@defer.inlineCallbacks
	746	def have_seen_events(self, event_ids):
	747	"""Given a list of event ids, check if we have already processed them.
	748
	749	Args:
	750	event_ids (iterable[str]):
	751
	752	Returns:
	753	Deferred[set[str]]: The events we have already seen.
	754	"""
	755	results = set()
	756
	757	def have_seen_events_txn(txn, chunk):
	758	sql = "SELECT event_id FROM events as e WHERE "
	759	clause, args = make_in_list_sql_clause(
	760	txn.database_engine, "e.event_id", chunk
	761	)
	762	txn.execute(sql + clause, args)
	763	for (event_id,) in txn:
	764	results.add(event_id)
	765
	766	# break the input up into chunks of 100
	767	input_iterator = iter(event_ids)
	768	for chunk in iter(lambda: list(itertools.islice(input_iterator, 100)), []):
	769	yield self.runInteraction("have_seen_events", have_seen_events_txn, chunk)
	770	return results
	771
	772	def get_seen_events_with_rejections(self, event_ids):
	773	"""Given a list of event ids, check if we rejected them.
	774
	775	Args:
	776	event_ids (list[str])
	777
	778	Returns:
	779	Deferred[dict[str, str\|None):
	780	Has an entry for each event id we already have seen. Maps to
	781	the rejected reason string if we rejected the event, else maps
	782	to None.
	783	"""
	784	if not event_ids:
	785	return defer.succeed({})
	786
	787	def f(txn):
	788	sql = (
	789	"SELECT e.event_id, reason FROM events as e "
	790	"LEFT JOIN rejections as r ON e.event_id = r.event_id "
	791	"WHERE e.event_id = ?"
	792	)
	793
	794	res = {}
	795	for event_id in event_ids:
	796	txn.execute(sql, (event_id,))
	797	row = txn.fetchone()
	798	if row:
	799	_, rejected = row
	800	res[event_id] = rejected
	801
	802	return res
	803
	804	return self.runInteraction("get_seen_events_with_rejections", f)
	805
	806	def _get_total_state_event_counts_txn(self, txn, room_id):
	807	"""
	808	See get_total_state_event_counts.
	809	"""
	810	# We join against the events table as that has an index on room_id
	811	sql = """
	812	SELECT COUNT(*) FROM state_events
	813	INNER JOIN events USING (room_id, event_id)
	814	WHERE room_id=?
	815	"""
	816	txn.execute(sql, (room_id,))
	817	row = txn.fetchone()
	818	return row[0] if row else 0
	819
	820	def get_total_state_event_counts(self, room_id):
	821	"""
	822	Gets the total number of state events in a room.
	823
	824	Args:
	825	room_id (str)
	826
	827	Returns:
	828	Deferred[int]
	829	"""
	830	return self.runInteraction(
	831	"get_total_state_event_counts",
	832	self._get_total_state_event_counts_txn,
	833	room_id,
	834	)
	835
	836	def _get_current_state_event_counts_txn(self, txn, room_id):
	837	"""
	838	See get_current_state_event_counts.
	839	"""
	840	sql = "SELECT COUNT(*) FROM current_state_events WHERE room_id=?"
	841	txn.execute(sql, (room_id,))
	842	row = txn.fetchone()
	843	return row[0] if row else 0
	844
	845	def get_current_state_event_counts(self, room_id):
	846	"""
	847	Gets the current number of state events in a room.
	848
	849	Args:
	850	room_id (str)
	851
	852	Returns:
	853	Deferred[int]
	854	"""
	855	return self.runInteraction(
	856	"get_current_state_event_counts",
	857	self._get_current_state_event_counts_txn,
	858	room_id,
	859	)
	860
	861	@defer.inlineCallbacks
	862	def get_room_complexity(self, room_id):
	863	"""
	864	Get a rough approximation of the complexity of the room. This is used by
	865	remote servers to decide whether they wish to join the room or not.
	866	Higher complexity value indicates that being in the room will consume
	867	more resources.
	868
	869	Args:
	870	room_id (str)
	871
	872	Returns:
	873	Deferred[dict[str:int]] of complexity version to complexity.
	874	"""
	875	state_events = yield self.get_current_state_event_counts(room_id)
	876
	877	# Call this one "v1", so we can introduce new ones as we want to develop
	878	# it.
	879	complexity_v1 = round(state_events / 500, 2)
	880
	881	return {"v1": complexity_v1}

+74

-0

synapse/storage/data_stores/main/filtering.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2015, 2016 OpenMarket Ltd
	2	#
	3	# Licensed under the Apache License, Version 2.0 (the "License");
	4	# you may not use this file except in compliance with the License.
	5	# You may obtain a copy of the License at
	6	#
	7	# http://www.apache.org/licenses/LICENSE-2.0
	8	#
	9	# Unless required by applicable law or agreed to in writing, software
	10	# distributed under the License is distributed on an "AS IS" BASIS,
	11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	# See the License for the specific language governing permissions and
	13	# limitations under the License.
	14
	15	from canonicaljson import encode_canonical_json
	16
	17	from synapse.api.errors import Codes, SynapseError
	18	from synapse.storage._base import SQLBaseStore, db_to_json
	19	from synapse.util.caches.descriptors import cachedInlineCallbacks
	20
	21
	22	class FilteringStore(SQLBaseStore):
	23	@cachedInlineCallbacks(num_args=2)
	24	def get_user_filter(self, user_localpart, filter_id):
	25	# filter_id is BIGINT UNSIGNED, so if it isn't a number, fail
	26	# with a coherent error message rather than 500 M_UNKNOWN.
	27	try:
	28	int(filter_id)
	29	except ValueError:
	30	raise SynapseError(400, "Invalid filter ID", Codes.INVALID_PARAM)
	31
	32	def_json = yield self._simple_select_one_onecol(
	33	table="user_filters",
	34	keyvalues={"user_id": user_localpart, "filter_id": filter_id},
	35	retcol="filter_json",
	36	allow_none=False,
	37	desc="get_user_filter",
	38	)
	39
	40	return db_to_json(def_json)
	41
	42	def add_user_filter(self, user_localpart, user_filter):
	43	def_json = encode_canonical_json(user_filter)
	44
	45	# Need an atomic transaction to SELECT the maximal ID so far then
	46	# INSERT a new one
	47	def _do_txn(txn):
	48	sql = (
	49	"SELECT filter_id FROM user_filters "
	50	"WHERE user_id = ? AND filter_json = ?"
	51	)
	52	txn.execute(sql, (user_localpart, bytearray(def_json)))
	53	filter_id_response = txn.fetchone()
	54	if filter_id_response is not None:
	55	return filter_id_response[0]
	56
	57	sql = "SELECT MAX(filter_id) FROM user_filters " "WHERE user_id = ?"
	58	txn.execute(sql, (user_localpart,))
	59	max_id = txn.fetchone()[0]
	60	if max_id is None:
	61	filter_id = 0
	62	else:
	63	filter_id = max_id + 1
	64
	65	sql = (
	66	"INSERT INTO user_filters (user_id, filter_id, filter_json)"
	67	"VALUES(?, ?, ?)"
	68	)
	69	txn.execute(sql, (user_localpart, filter_id, bytearray(def_json)))
	70
	71	return filter_id
	72
	73	return self.runInteraction("add_user_filter", _do_txn)

+1180

-0

synapse/storage/data_stores/main/group_server.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2017 Vector Creations Ltd
	2	# Copyright 2018 New Vector Ltd
	3	#
	4	# Licensed under the Apache License, Version 2.0 (the "License");
	5	# you may not use this file except in compliance with the License.
	6	# You may obtain a copy of the License at
	7	#
	8	# http://www.apache.org/licenses/LICENSE-2.0
	9	#
	10	# Unless required by applicable law or agreed to in writing, software
	11	# distributed under the License is distributed on an "AS IS" BASIS,
	12	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	# See the License for the specific language governing permissions and
	14	# limitations under the License.
	15
	16	from canonicaljson import json
	17
	18	from twisted.internet import defer
	19
	20	from synapse.api.errors import SynapseError
	21	from synapse.storage._base import SQLBaseStore
	22
	23	# The category ID for the "default" category. We don't store as null in the
	24	# database to avoid the fun of null != null
	25	_DEFAULT_CATEGORY_ID = ""
	26	_DEFAULT_ROLE_ID = ""
	27
	28
	29	class GroupServerStore(SQLBaseStore):
	30	def set_group_join_policy(self, group_id, join_policy):
	31	"""Set the join policy of a group.
	32
	33	join_policy can be one of:
	34	* "invite"
	35	* "open"
	36	"""
	37	return self._simple_update_one(
	38	table="groups",
	39	keyvalues={"group_id": group_id},
	40	updatevalues={"join_policy": join_policy},
	41	desc="set_group_join_policy",
	42	)
	43
	44	def get_group(self, group_id):
	45	return self._simple_select_one(
	46	table="groups",
	47	keyvalues={"group_id": group_id},
	48	retcols=(
	49	"name",
	50	"short_description",
	51	"long_description",
	52	"avatar_url",
	53	"is_public",
	54	"join_policy",
	55	),
	56	allow_none=True,
	57	desc="get_group",
	58	)
	59
	60	def get_users_in_group(self, group_id, include_private=False):
	61	# TODO: Pagination
	62
	63	keyvalues = {"group_id": group_id}
	64	if not include_private:
	65	keyvalues["is_public"] = True
	66
	67	return self._simple_select_list(
	68	table="group_users",
	69	keyvalues=keyvalues,
	70	retcols=("user_id", "is_public", "is_admin"),
	71	desc="get_users_in_group",
	72	)
	73
	74	def get_invited_users_in_group(self, group_id):
	75	# TODO: Pagination
	76
	77	return self._simple_select_onecol(
	78	table="group_invites",
	79	keyvalues={"group_id": group_id},
	80	retcol="user_id",
	81	desc="get_invited_users_in_group",
	82	)
	83
	84	def get_rooms_in_group(self, group_id, include_private=False):
	85	# TODO: Pagination
	86
	87	keyvalues = {"group_id": group_id}
	88	if not include_private:
	89	keyvalues["is_public"] = True
	90
	91	return self._simple_select_list(
	92	table="group_rooms",
	93	keyvalues=keyvalues,
	94	retcols=("room_id", "is_public"),
	95	desc="get_rooms_in_group",
	96	)
	97
	98	def get_rooms_for_summary_by_category(self, group_id, include_private=False):
	99	"""Get the rooms and categories that should be included in a summary request
	100
	101	Returns ([rooms], [categories])
	102	"""
	103
	104	def _get_rooms_for_summary_txn(txn):
	105	keyvalues = {"group_id": group_id}
	106	if not include_private:
	107	keyvalues["is_public"] = True
	108
	109	sql = """
	110	SELECT room_id, is_public, category_id, room_order
	111	FROM group_summary_rooms
	112	WHERE group_id = ?
	113	"""
	114
	115	if not include_private:
	116	sql += " AND is_public = ?"
	117	txn.execute(sql, (group_id, True))
	118	else:
	119	txn.execute(sql, (group_id,))
	120
	121	rooms = [
	122	{
	123	"room_id": row[0],
	124	"is_public": row[1],
	125	"category_id": row[2] if row[2] != _DEFAULT_CATEGORY_ID else None,
	126	"order": row[3],
	127	}
	128	for row in txn
	129	]
	130
	131	sql = """
	132	SELECT category_id, is_public, profile, cat_order
	133	FROM group_summary_room_categories
	134	INNER JOIN group_room_categories USING (group_id, category_id)
	135	WHERE group_id = ?
	136	"""
	137
	138	if not include_private:
	139	sql += " AND is_public = ?"
	140	txn.execute(sql, (group_id, True))
	141	else:
	142	txn.execute(sql, (group_id,))
	143
	144	categories = {
	145	row[0]: {
	146	"is_public": row[1],
	147	"profile": json.loads(row[2]),
	148	"order": row[3],
	149	}
	150	for row in txn
	151	}
	152
	153	return rooms, categories
	154
	155	return self.runInteraction("get_rooms_for_summary", _get_rooms_for_summary_txn)
	156
	157	def add_room_to_summary(self, group_id, room_id, category_id, order, is_public):
	158	return self.runInteraction(
	159	"add_room_to_summary",
	160	self._add_room_to_summary_txn,
	161	group_id,
	162	room_id,
	163	category_id,
	164	order,
	165	is_public,
	166	)
	167
	168	def _add_room_to_summary_txn(
	169	self, txn, group_id, room_id, category_id, order, is_public
	170	):
	171	"""Add (or update) room's entry in summary.
	172
	173	Args:
	174	group_id (str)
	175	room_id (str)
	176	category_id (str): If not None then adds the category to the end of
	177	the summary if its not already there. [Optional]
	178	order (int): If not None inserts the room at that position, e.g.
	179	an order of 1 will put the room first. Otherwise, the room gets
	180	added to the end.
	181	"""
	182	room_in_group = self._simple_select_one_onecol_txn(
	183	txn,
	184	table="group_rooms",
	185	keyvalues={"group_id": group_id, "room_id": room_id},
	186	retcol="room_id",
	187	allow_none=True,
	188	)
	189	if not room_in_group:
	190	raise SynapseError(400, "room not in group")
	191
	192	if category_id is None:
	193	category_id = _DEFAULT_CATEGORY_ID
	194	else:
	195	cat_exists = self._simple_select_one_onecol_txn(
	196	txn,
	197	table="group_room_categories",
	198	keyvalues={"group_id": group_id, "category_id": category_id},
	199	retcol="group_id",
	200	allow_none=True,
	201	)
	202	if not cat_exists:
	203	raise SynapseError(400, "Category doesn't exist")
	204
	205	# TODO: Check category is part of summary already
	206	cat_exists = self._simple_select_one_onecol_txn(
	207	txn,
	208	table="group_summary_room_categories",
	209	keyvalues={"group_id": group_id, "category_id": category_id},
	210	retcol="group_id",
	211	allow_none=True,
	212	)
	213	if not cat_exists:
	214	# If not, add it with an order larger than all others
	215	txn.execute(
	216	"""
	217	INSERT INTO group_summary_room_categories
	218	(group_id, category_id, cat_order)
	219	SELECT ?, ?, COALESCE(MAX(cat_order), 0) + 1
	220	FROM group_summary_room_categories
	221	WHERE group_id = ? AND category_id = ?
	222	""",
	223	(group_id, category_id, group_id, category_id),
	224	)
	225
	226	existing = self._simple_select_one_txn(
	227	txn,
	228	table="group_summary_rooms",
	229	keyvalues={
	230	"group_id": group_id,
	231	"room_id": room_id,
	232	"category_id": category_id,
	233	},
	234	retcols=("room_order", "is_public"),
	235	allow_none=True,
	236	)
	237
	238	if order is not None:
	239	# Shuffle other room orders that come after the given order
	240	sql = """
	241	UPDATE group_summary_rooms SET room_order = room_order + 1
	242	WHERE group_id = ? AND category_id = ? AND room_order >= ?
	243	"""
	244	txn.execute(sql, (group_id, category_id, order))
	245	elif not existing:
	246	sql = """
	247	SELECT COALESCE(MAX(room_order), 0) + 1 FROM group_summary_rooms
	248	WHERE group_id = ? AND category_id = ?
	249	"""
	250	txn.execute(sql, (group_id, category_id))
	251	order, = txn.fetchone()
	252
	253	if existing:
	254	to_update = {}
	255	if order is not None:
	256	to_update["room_order"] = order
	257	if is_public is not None:
	258	to_update["is_public"] = is_public
	259	self._simple_update_txn(
	260	txn,
	261	table="group_summary_rooms",
	262	keyvalues={
	263	"group_id": group_id,
	264	"category_id": category_id,
	265	"room_id": room_id,
	266	},
	267	values=to_update,
	268	)
	269	else:
	270	if is_public is None:
	271	is_public = True
	272
	273	self._simple_insert_txn(
	274	txn,
	275	table="group_summary_rooms",
	276	values={
	277	"group_id": group_id,
	278	"category_id": category_id,
	279	"room_id": room_id,
	280	"room_order": order,
	281	"is_public": is_public,
	282	},
	283	)
	284
	285	def remove_room_from_summary(self, group_id, room_id, category_id):
	286	if category_id is None:
	287	category_id = _DEFAULT_CATEGORY_ID
	288
	289	return self._simple_delete(
	290	table="group_summary_rooms",
	291	keyvalues={
	292	"group_id": group_id,
	293	"category_id": category_id,
	294	"room_id": room_id,
	295	},
	296	desc="remove_room_from_summary",
	297	)
	298
	299	@defer.inlineCallbacks
	300	def get_group_categories(self, group_id):
	301	rows = yield self._simple_select_list(
	302	table="group_room_categories",
	303	keyvalues={"group_id": group_id},
	304	retcols=("category_id", "is_public", "profile"),
	305	desc="get_group_categories",
	306	)
	307
	308	return {
	309	row["category_id"]: {
	310	"is_public": row["is_public"],
	311	"profile": json.loads(row["profile"]),
	312	}
	313	for row in rows
	314	}
	315
	316	@defer.inlineCallbacks
	317	def get_group_category(self, group_id, category_id):
	318	category = yield self._simple_select_one(
	319	table="group_room_categories",
	320	keyvalues={"group_id": group_id, "category_id": category_id},
	321	retcols=("is_public", "profile"),
	322	desc="get_group_category",
	323	)
	324
	325	category["profile"] = json.loads(category["profile"])
	326
	327	return category
	328
	329	def upsert_group_category(self, group_id, category_id, profile, is_public):
	330	"""Add/update room category for group
	331	"""
	332	insertion_values = {}
	333	update_values = {"category_id": category_id} # This cannot be empty
	334
	335	if profile is None:
	336	insertion_values["profile"] = "{}"
	337	else:
	338	update_values["profile"] = json.dumps(profile)
	339
	340	if is_public is None:
	341	insertion_values["is_public"] = True
	342	else:
	343	update_values["is_public"] = is_public
	344
	345	return self._simple_upsert(
	346	table="group_room_categories",
	347	keyvalues={"group_id": group_id, "category_id": category_id},
	348	values=update_values,
	349	insertion_values=insertion_values,
	350	desc="upsert_group_category",
	351	)
	352
	353	def remove_group_category(self, group_id, category_id):
	354	return self._simple_delete(
	355	table="group_room_categories",
	356	keyvalues={"group_id": group_id, "category_id": category_id},
	357	desc="remove_group_category",
	358	)
	359
	360	@defer.inlineCallbacks
	361	def get_group_roles(self, group_id):
	362	rows = yield self._simple_select_list(
	363	table="group_roles",
	364	keyvalues={"group_id": group_id},
	365	retcols=("role_id", "is_public", "profile"),
	366	desc="get_group_roles",
	367	)
	368
	369	return {
	370	row["role_id"]: {
	371	"is_public": row["is_public"],
	372	"profile": json.loads(row["profile"]),
	373	}
	374	for row in rows
	375	}
	376
	377	@defer.inlineCallbacks
	378	def get_group_role(self, group_id, role_id):
	379	role = yield self._simple_select_one(
	380	table="group_roles",
	381	keyvalues={"group_id": group_id, "role_id": role_id},
	382	retcols=("is_public", "profile"),
	383	desc="get_group_role",
	384	)
	385
	386	role["profile"] = json.loads(role["profile"])
	387
	388	return role
	389
	390	def upsert_group_role(self, group_id, role_id, profile, is_public):
	391	"""Add/remove user role
	392	"""
	393	insertion_values = {}
	394	update_values = {"role_id": role_id} # This cannot be empty
	395
	396	if profile is None:
	397	insertion_values["profile"] = "{}"
	398	else:
	399	update_values["profile"] = json.dumps(profile)
	400
	401	if is_public is None:
	402	insertion_values["is_public"] = True
	403	else:
	404	update_values["is_public"] = is_public
	405
	406	return self._simple_upsert(
	407	table="group_roles",
	408	keyvalues={"group_id": group_id, "role_id": role_id},
	409	values=update_values,
	410	insertion_values=insertion_values,
	411	desc="upsert_group_role",
	412	)
	413
	414	def remove_group_role(self, group_id, role_id):
	415	return self._simple_delete(
	416	table="group_roles",
	417	keyvalues={"group_id": group_id, "role_id": role_id},
	418	desc="remove_group_role",
	419	)
	420
	421	def add_user_to_summary(self, group_id, user_id, role_id, order, is_public):
	422	return self.runInteraction(
	423	"add_user_to_summary",
	424	self._add_user_to_summary_txn,
	425	group_id,
	426	user_id,
	427	role_id,
	428	order,
	429	is_public,
	430	)
	431
	432	def _add_user_to_summary_txn(
	433	self, txn, group_id, user_id, role_id, order, is_public
	434	):
	435	"""Add (or update) user's entry in summary.
	436
	437	Args:
	438	group_id (str)
	439	user_id (str)
	440	role_id (str): If not None then adds the role to the end of
	441	the summary if its not already there. [Optional]
	442	order (int): If not None inserts the user at that position, e.g.
	443	an order of 1 will put the user first. Otherwise, the user gets
	444	added to the end.
	445	"""
	446	user_in_group = self._simple_select_one_onecol_txn(
	447	txn,
	448	table="group_users",
	449	keyvalues={"group_id": group_id, "user_id": user_id},
	450	retcol="user_id",
	451	allow_none=True,
	452	)
	453	if not user_in_group:
	454	raise SynapseError(400, "user not in group")
	455
	456	if role_id is None:
	457	role_id = _DEFAULT_ROLE_ID
	458	else:
	459	role_exists = self._simple_select_one_onecol_txn(
	460	txn,
	461	table="group_roles",
	462	keyvalues={"group_id": group_id, "role_id": role_id},
	463	retcol="group_id",
	464	allow_none=True,
	465	)
	466	if not role_exists:
	467	raise SynapseError(400, "Role doesn't exist")
	468
	469	# TODO: Check role is part of the summary already
	470	role_exists = self._simple_select_one_onecol_txn(
	471	txn,
	472	table="group_summary_roles",
	473	keyvalues={"group_id": group_id, "role_id": role_id},
	474	retcol="group_id",
	475	allow_none=True,
	476	)
	477	if not role_exists:
	478	# If not, add it with an order larger than all others
	479	txn.execute(
	480	"""
	481	INSERT INTO group_summary_roles
	482	(group_id, role_id, role_order)
	483	SELECT ?, ?, COALESCE(MAX(role_order), 0) + 1
	484	FROM group_summary_roles
	485	WHERE group_id = ? AND role_id = ?
	486	""",
	487	(group_id, role_id, group_id, role_id),
	488	)
	489
	490	existing = self._simple_select_one_txn(
	491	txn,
	492	table="group_summary_users",
	493	keyvalues={"group_id": group_id, "user_id": user_id, "role_id": role_id},
	494	retcols=("user_order", "is_public"),
	495	allow_none=True,
	496	)
	497
	498	if order is not None:
	499	# Shuffle other users orders that come after the given order
	500	sql = """
	501	UPDATE group_summary_users SET user_order = user_order + 1
	502	WHERE group_id = ? AND role_id = ? AND user_order >= ?
	503	"""
	504	txn.execute(sql, (group_id, role_id, order))
	505	elif not existing:
	506	sql = """
	507	SELECT COALESCE(MAX(user_order), 0) + 1 FROM group_summary_users
	508	WHERE group_id = ? AND role_id = ?
	509	"""
	510	txn.execute(sql, (group_id, role_id))
	511	order, = txn.fetchone()
	512
	513	if existing:
	514	to_update = {}
	515	if order is not None:
	516	to_update["user_order"] = order
	517	if is_public is not None:
	518	to_update["is_public"] = is_public
	519	self._simple_update_txn(
	520	txn,
	521	table="group_summary_users",
	522	keyvalues={
	523	"group_id": group_id,
	524	"role_id": role_id,
	525	"user_id": user_id,
	526	},
	527	values=to_update,
	528	)
	529	else:
	530	if is_public is None:
	531	is_public = True
	532
	533	self._simple_insert_txn(
	534	txn,
	535	table="group_summary_users",
	536	values={
	537	"group_id": group_id,
	538	"role_id": role_id,
	539	"user_id": user_id,
	540	"user_order": order,
	541	"is_public": is_public,
	542	},
	543	)
	544
	545	def remove_user_from_summary(self, group_id, user_id, role_id):
	546	if role_id is None:
	547	role_id = _DEFAULT_ROLE_ID
	548
	549	return self._simple_delete(
	550	table="group_summary_users",
	551	keyvalues={"group_id": group_id, "role_id": role_id, "user_id": user_id},
	552	desc="remove_user_from_summary",
	553	)
	554
	555	def get_users_for_summary_by_role(self, group_id, include_private=False):
	556	"""Get the users and roles that should be included in a summary request
	557
	558	Returns ([users], [roles])
	559	"""
	560
	561	def _get_users_for_summary_txn(txn):
	562	keyvalues = {"group_id": group_id}
	563	if not include_private:
	564	keyvalues["is_public"] = True
	565
	566	sql = """
	567	SELECT user_id, is_public, role_id, user_order
	568	FROM group_summary_users
	569	WHERE group_id = ?
	570	"""
	571
	572	if not include_private:
	573	sql += " AND is_public = ?"
	574	txn.execute(sql, (group_id, True))
	575	else:
	576	txn.execute(sql, (group_id,))
	577
	578	users = [
	579	{
	580	"user_id": row[0],
	581	"is_public": row[1],
	582	"role_id": row[2] if row[2] != _DEFAULT_ROLE_ID else None,
	583	"order": row[3],
	584	}
	585	for row in txn
	586	]
	587
	588	sql = """
	589	SELECT role_id, is_public, profile, role_order
	590	FROM group_summary_roles
	591	INNER JOIN group_roles USING (group_id, role_id)
	592	WHERE group_id = ?
	593	"""
	594
	595	if not include_private:
	596	sql += " AND is_public = ?"
	597	txn.execute(sql, (group_id, True))
	598	else:
	599	txn.execute(sql, (group_id,))
	600
	601	roles = {
	602	row[0]: {
	603	"is_public": row[1],
	604	"profile": json.loads(row[2]),
	605	"order": row[3],
	606	}
	607	for row in txn
	608	}
	609
	610	return users, roles
	611
	612	return self.runInteraction(
	613	"get_users_for_summary_by_role", _get_users_for_summary_txn
	614	)
	615
	616	def is_user_in_group(self, user_id, group_id):
	617	return self._simple_select_one_onecol(
	618	table="group_users",
	619	keyvalues={"group_id": group_id, "user_id": user_id},
	620	retcol="user_id",
	621	allow_none=True,
	622	desc="is_user_in_group",
	623	).addCallback(lambda r: bool(r))
	624
	625	def is_user_admin_in_group(self, group_id, user_id):
	626	return self._simple_select_one_onecol(
	627	table="group_users",
	628	keyvalues={"group_id": group_id, "user_id": user_id},
	629	retcol="is_admin",
	630	allow_none=True,
	631	desc="is_user_admin_in_group",
	632	)
	633
	634	def add_group_invite(self, group_id, user_id):
	635	"""Record that the group server has invited a user
	636	"""
	637	return self._simple_insert(
	638	table="group_invites",
	639	values={"group_id": group_id, "user_id": user_id},
	640	desc="add_group_invite",
	641	)
	642
	643	def is_user_invited_to_local_group(self, group_id, user_id):
	644	"""Has the group server invited a user?
	645	"""
	646	return self._simple_select_one_onecol(
	647	table="group_invites",
	648	keyvalues={"group_id": group_id, "user_id": user_id},
	649	retcol="user_id",
	650	desc="is_user_invited_to_local_group",
	651	allow_none=True,
	652	)
	653
	654	def get_users_membership_info_in_group(self, group_id, user_id):
	655	"""Get a dict describing the membership of a user in a group.
	656
	657	Example if joined:
	658
	659	{
	660	"membership": "join",
	661	"is_public": True,
	662	"is_privileged": False,
	663	}
	664
	665	Returns an empty dict if the user is not join/invite/etc
	666	"""
	667
	668	def _get_users_membership_in_group_txn(txn):
	669	row = self._simple_select_one_txn(
	670	txn,
	671	table="group_users",
	672	keyvalues={"group_id": group_id, "user_id": user_id},
	673	retcols=("is_admin", "is_public"),
	674	allow_none=True,
	675	)
	676
	677	if row:
	678	return {
	679	"membership": "join",
	680	"is_public": row["is_public"],
	681	"is_privileged": row["is_admin"],
	682	}
	683
	684	row = self._simple_select_one_onecol_txn(
	685	txn,
	686	table="group_invites",
	687	keyvalues={"group_id": group_id, "user_id": user_id},
	688	retcol="user_id",
	689	allow_none=True,
	690	)
	691
	692	if row:
	693	return {"membership": "invite"}
	694
	695	return {}
	696
	697	return self.runInteraction(
	698	"get_users_membership_info_in_group", _get_users_membership_in_group_txn
	699	)
	700
	701	def add_user_to_group(
	702	self,
	703	group_id,
	704	user_id,
	705	is_admin=False,
	706	is_public=True,
	707	local_attestation=None,
	708	remote_attestation=None,
	709	):
	710	"""Add a user to the group server.
	711
	712	Args:
	713	group_id (str)
	714	user_id (str)
	715	is_admin (bool)
	716	is_public (bool)
	717	local_attestation (dict): The attestation the GS created to give
	718	to the remote server. Optional if the user and group are on the
	719	same server
	720	remote_attestation (dict): The attestation given to GS by remote
	721	server. Optional if the user and group are on the same server
	722	"""
	723
	724	def _add_user_to_group_txn(txn):
	725	self._simple_insert_txn(
	726	txn,
	727	table="group_users",
	728	values={
	729	"group_id": group_id,
	730	"user_id": user_id,
	731	"is_admin": is_admin,
	732	"is_public": is_public,
	733	},
	734	)
	735
	736	self._simple_delete_txn(
	737	txn,
	738	table="group_invites",
	739	keyvalues={"group_id": group_id, "user_id": user_id},
	740	)
	741
	742	if local_attestation:
	743	self._simple_insert_txn(
	744	txn,
	745	table="group_attestations_renewals",
	746	values={
	747	"group_id": group_id,
	748	"user_id": user_id,
	749	"valid_until_ms": local_attestation["valid_until_ms"],
	750	},
	751	)
	752	if remote_attestation:
	753	self._simple_insert_txn(
	754	txn,
	755	table="group_attestations_remote",
	756	values={
	757	"group_id": group_id,
	758	"user_id": user_id,
	759	"valid_until_ms": remote_attestation["valid_until_ms"],
	760	"attestation_json": json.dumps(remote_attestation),
	761	},
	762	)
	763
	764	return self.runInteraction("add_user_to_group", _add_user_to_group_txn)
	765
	766	def remove_user_from_group(self, group_id, user_id):
	767	def _remove_user_from_group_txn(txn):
	768	self._simple_delete_txn(
	769	txn,
	770	table="group_users",
	771	keyvalues={"group_id": group_id, "user_id": user_id},
	772	)
	773	self._simple_delete_txn(
	774	txn,
	775	table="group_invites",
	776	keyvalues={"group_id": group_id, "user_id": user_id},
	777	)
	778	self._simple_delete_txn(
	779	txn,
	780	table="group_attestations_renewals",
	781	keyvalues={"group_id": group_id, "user_id": user_id},
	782	)
	783	self._simple_delete_txn(
	784	txn,
	785	table="group_attestations_remote",
	786	keyvalues={"group_id": group_id, "user_id": user_id},
	787	)
	788	self._simple_delete_txn(
	789	txn,
	790	table="group_summary_users",
	791	keyvalues={"group_id": group_id, "user_id": user_id},
	792	)
	793
	794	return self.runInteraction(
	795	"remove_user_from_group", _remove_user_from_group_txn
	796	)
	797
	798	def add_room_to_group(self, group_id, room_id, is_public):
	799	return self._simple_insert(
	800	table="group_rooms",
	801	values={"group_id": group_id, "room_id": room_id, "is_public": is_public},
	802	desc="add_room_to_group",
	803	)
	804
	805	def update_room_in_group_visibility(self, group_id, room_id, is_public):
	806	return self._simple_update(
	807	table="group_rooms",
	808	keyvalues={"group_id": group_id, "room_id": room_id},
	809	updatevalues={"is_public": is_public},
	810	desc="update_room_in_group_visibility",
	811	)
	812
	813	def remove_room_from_group(self, group_id, room_id):
	814	def _remove_room_from_group_txn(txn):
	815	self._simple_delete_txn(
	816	txn,
	817	table="group_rooms",
	818	keyvalues={"group_id": group_id, "room_id": room_id},
	819	)
	820
	821	self._simple_delete_txn(
	822	txn,
	823	table="group_summary_rooms",
	824	keyvalues={"group_id": group_id, "room_id": room_id},
	825	)
	826
	827	return self.runInteraction(
	828	"remove_room_from_group", _remove_room_from_group_txn
	829	)
	830
	831	def get_publicised_groups_for_user(self, user_id):
	832	"""Get all groups a user is publicising
	833	"""
	834	return self._simple_select_onecol(
	835	table="local_group_membership",
	836	keyvalues={"user_id": user_id, "membership": "join", "is_publicised": True},
	837	retcol="group_id",
	838	desc="get_publicised_groups_for_user",
	839	)
	840
	841	def update_group_publicity(self, group_id, user_id, publicise):
	842	"""Update whether the user is publicising their membership of the group
	843	"""
	844	return self._simple_update_one(
	845	table="local_group_membership",
	846	keyvalues={"group_id": group_id, "user_id": user_id},
	847	updatevalues={"is_publicised": publicise},
	848	desc="update_group_publicity",
	849	)
	850
	851	@defer.inlineCallbacks
	852	def register_user_group_membership(
	853	self,
	854	group_id,
	855	user_id,
	856	membership,
	857	is_admin=False,
	858	content={},
	859	local_attestation=None,
	860	remote_attestation=None,
	861	is_publicised=False,
	862	):
	863	"""Registers that a local user is a member of a (local or remote) group.
	864
	865	Args:
	866	group_id (str)
	867	user_id (str)
	868	membership (str)
	869	is_admin (bool)
	870	content (dict): Content of the membership, e.g. includes the inviter
	871	if the user has been invited.
	872	local_attestation (dict): If remote group then store the fact that we
	873	have given out an attestation, else None.
	874	remote_attestation (dict): If remote group then store the remote
	875	attestation from the group, else None.
	876	"""
	877
	878	def _register_user_group_membership_txn(txn, next_id):
	879	# TODO: Upsert?
	880	self._simple_delete_txn(
	881	txn,
	882	table="local_group_membership",
	883	keyvalues={"group_id": group_id, "user_id": user_id},
	884	)
	885	self._simple_insert_txn(
	886	txn,
	887	table="local_group_membership",
	888	values={
	889	"group_id": group_id,
	890	"user_id": user_id,
	891	"is_admin": is_admin,
	892	"membership": membership,
	893	"is_publicised": is_publicised,
	894	"content": json.dumps(content),
	895	},
	896	)
	897
	898	self._simple_insert_txn(
	899	txn,
	900	table="local_group_updates",
	901	values={
	902	"stream_id": next_id,
	903	"group_id": group_id,
	904	"user_id": user_id,
	905	"type": "membership",
	906	"content": json.dumps(
	907	{"membership": membership, "content": content}
	908	),
	909	},
	910	)
	911	self._group_updates_stream_cache.entity_has_changed(user_id, next_id)
	912
	913	# TODO: Insert profile to ensure it comes down stream if its a join.
	914
	915	if membership == "join":
	916	if local_attestation:
	917	self._simple_insert_txn(
	918	txn,
	919	table="group_attestations_renewals",
	920	values={
	921	"group_id": group_id,
	922	"user_id": user_id,
	923	"valid_until_ms": local_attestation["valid_until_ms"],
	924	},
	925	)
	926	if remote_attestation:
	927	self._simple_insert_txn(
	928	txn,
	929	table="group_attestations_remote",
	930	values={
	931	"group_id": group_id,
	932	"user_id": user_id,
	933	"valid_until_ms": remote_attestation["valid_until_ms"],
	934	"attestation_json": json.dumps(remote_attestation),
	935	},
	936	)
	937	else:
	938	self._simple_delete_txn(
	939	txn,
	940	table="group_attestations_renewals",
	941	keyvalues={"group_id": group_id, "user_id": user_id},
	942	)
	943	self._simple_delete_txn(
	944	txn,
	945	table="group_attestations_remote",
	946	keyvalues={"group_id": group_id, "user_id": user_id},
	947	)
	948
	949	return next_id
	950
	951	with self._group_updates_id_gen.get_next() as next_id:
	952	res = yield self.runInteraction(
	953	"register_user_group_membership",
	954	_register_user_group_membership_txn,
	955	next_id,
	956	)
	957	return res
	958
	959	@defer.inlineCallbacks
	960	def create_group(
	961	self, group_id, user_id, name, avatar_url, short_description, long_description
	962	):
	963	yield self._simple_insert(
	964	table="groups",
	965	values={
	966	"group_id": group_id,
	967	"name": name,
	968	"avatar_url": avatar_url,
	969	"short_description": short_description,
	970	"long_description": long_description,
	971	"is_public": True,
	972	},
	973	desc="create_group",
	974	)
	975
	976	@defer.inlineCallbacks
	977	def update_group_profile(self, group_id, profile):
	978	yield self._simple_update_one(
	979	table="groups",
	980	keyvalues={"group_id": group_id},
	981	updatevalues=profile,
	982	desc="update_group_profile",
	983	)
	984
	985	def get_attestations_need_renewals(self, valid_until_ms):
	986	"""Get all attestations that need to be renewed until givent time
	987	"""
	988
	989	def _get_attestations_need_renewals_txn(txn):
	990	sql = """
	991	SELECT group_id, user_id FROM group_attestations_renewals
	992	WHERE valid_until_ms <= ?
	993	"""
	994	txn.execute(sql, (valid_until_ms,))
	995	return self.cursor_to_dict(txn)
	996
	997	return self.runInteraction(
	998	"get_attestations_need_renewals", _get_attestations_need_renewals_txn
	999	)
	1000
	1001	def update_attestation_renewal(self, group_id, user_id, attestation):
	1002	"""Update an attestation that we have renewed
	1003	"""
	1004	return self._simple_update_one(
	1005	table="group_attestations_renewals",
	1006	keyvalues={"group_id": group_id, "user_id": user_id},
	1007	updatevalues={"valid_until_ms": attestation["valid_until_ms"]},
	1008	desc="update_attestation_renewal",
	1009	)
	1010
	1011	def update_remote_attestion(self, group_id, user_id, attestation):
	1012	"""Update an attestation that a remote has renewed
	1013	"""
	1014	return self._simple_update_one(
	1015	table="group_attestations_remote",
	1016	keyvalues={"group_id": group_id, "user_id": user_id},
	1017	updatevalues={
	1018	"valid_until_ms": attestation["valid_until_ms"],
	1019	"attestation_json": json.dumps(attestation),
	1020	},
	1021	desc="update_remote_attestion",
	1022	)
	1023
	1024	def remove_attestation_renewal(self, group_id, user_id):
	1025	"""Remove an attestation that we thought we should renew, but actually
	1026	shouldn't. Ideally this would never get called as we would never
	1027	incorrectly try and do attestations for local users on local groups.
	1028
	1029	Args:
	1030	group_id (str)
	1031	user_id (str)
	1032	"""
	1033	return self._simple_delete(
	1034	table="group_attestations_renewals",
	1035	keyvalues={"group_id": group_id, "user_id": user_id},
	1036	desc="remove_attestation_renewal",
	1037	)
	1038
	1039	@defer.inlineCallbacks
	1040	def get_remote_attestation(self, group_id, user_id):
	1041	"""Get the attestation that proves the remote agrees that the user is
	1042	in the group.
	1043	"""
	1044	row = yield self._simple_select_one(
	1045	table="group_attestations_remote",
	1046	keyvalues={"group_id": group_id, "user_id": user_id},
	1047	retcols=("valid_until_ms", "attestation_json"),
	1048	desc="get_remote_attestation",
	1049	allow_none=True,
	1050	)
	1051
	1052	now = int(self._clock.time_msec())
	1053	if row and now < row["valid_until_ms"]:
	1054	return json.loads(row["attestation_json"])
	1055
	1056	return None
	1057
	1058	def get_joined_groups(self, user_id):
	1059	return self._simple_select_onecol(
	1060	table="local_group_membership",
	1061	keyvalues={"user_id": user_id, "membership": "join"},
	1062	retcol="group_id",
	1063	desc="get_joined_groups",
	1064	)
	1065
	1066	def get_all_groups_for_user(self, user_id, now_token):
	1067	def _get_all_groups_for_user_txn(txn):
	1068	sql = """
	1069	SELECT group_id, type, membership, u.content
	1070	FROM local_group_updates AS u
	1071	INNER JOIN local_group_membership USING (group_id, user_id)
	1072	WHERE user_id = ? AND membership != 'leave'
	1073	AND stream_id <= ?
	1074	"""
	1075	txn.execute(sql, (user_id, now_token))
	1076	return [
	1077	{
	1078	"group_id": row[0],
	1079	"type": row[1],
	1080	"membership": row[2],
	1081	"content": json.loads(row[3]),
	1082	}
	1083	for row in txn
	1084	]
	1085
	1086	return self.runInteraction(
	1087	"get_all_groups_for_user", _get_all_groups_for_user_txn
	1088	)
	1089
	1090	def get_groups_changes_for_user(self, user_id, from_token, to_token):
	1091	from_token = int(from_token)
	1092	has_changed = self._group_updates_stream_cache.has_entity_changed(
	1093	user_id, from_token
	1094	)
	1095	if not has_changed:
	1096	return []
	1097
	1098	def _get_groups_changes_for_user_txn(txn):
	1099	sql = """
	1100	SELECT group_id, membership, type, u.content
	1101	FROM local_group_updates AS u
	1102	INNER JOIN local_group_membership USING (group_id, user_id)
	1103	WHERE user_id = ? AND ? < stream_id AND stream_id <= ?
	1104	"""
	1105	txn.execute(sql, (user_id, from_token, to_token))
	1106	return [
	1107	{
	1108	"group_id": group_id,
	1109	"membership": membership,
	1110	"type": gtype,
	1111	"content": json.loads(content_json),
	1112	}
	1113	for group_id, membership, gtype, content_json in txn
	1114	]
	1115
	1116	return self.runInteraction(
	1117	"get_groups_changes_for_user", _get_groups_changes_for_user_txn
	1118	)
	1119
	1120	def get_all_groups_changes(self, from_token, to_token, limit):
	1121	from_token = int(from_token)
	1122	has_changed = self._group_updates_stream_cache.has_any_entity_changed(
	1123	from_token
	1124	)
	1125	if not has_changed:
	1126	return []
	1127
	1128	def _get_all_groups_changes_txn(txn):
	1129	sql = """
	1130	SELECT stream_id, group_id, user_id, type, content
	1131	FROM local_group_updates
	1132	WHERE ? < stream_id AND stream_id <= ?
	1133	LIMIT ?
	1134	"""
	1135	txn.execute(sql, (from_token, to_token, limit))
	1136	return [
	1137	(stream_id, group_id, user_id, gtype, json.loads(content_json))
	1138	for stream_id, group_id, user_id, gtype, content_json in txn
	1139	]
	1140
	1141	return self.runInteraction(
	1142	"get_all_groups_changes", _get_all_groups_changes_txn
	1143	)
	1144
	1145	def get_group_stream_token(self):
	1146	return self._group_updates_id_gen.get_current_token()
	1147
	1148	def delete_group(self, group_id):
	1149	"""Deletes a group fully from the database.
	1150
	1151	Args:
	1152	group_id (str)
	1153
	1154	Returns:
	1155	Deferred
	1156	"""
	1157
	1158	def _delete_group_txn(txn):
	1159	tables = [
	1160	"groups",
	1161	"group_users",
	1162	"group_invites",
	1163	"group_rooms",
	1164	"group_summary_rooms",
	1165	"group_summary_room_categories",
	1166	"group_room_categories",
	1167	"group_summary_users",
	1168	"group_summary_roles",
	1169	"group_roles",
	1170	"group_attestations_renewals",
	1171	"group_attestations_remote",
	1172	]
	1173
	1174	for table in tables:
	1175	self._simple_delete_txn(
	1176	txn, table=table, keyvalues={"group_id": group_id}
	1177	)
	1178
	1179	return self.runInteraction("delete_group", _delete_group_txn)

+214

-0

synapse/storage/data_stores/main/keys.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2014-2016 OpenMarket Ltd
	2	# Copyright 2019 New Vector Ltd.
	3	#
	4	# Licensed under the Apache License, Version 2.0 (the "License");
	5	# you may not use this file except in compliance with the License.
	6	# You may obtain a copy of the License at
	7	#
	8	# http://www.apache.org/licenses/LICENSE-2.0
	9	#
	10	# Unless required by applicable law or agreed to in writing, software
	11	# distributed under the License is distributed on an "AS IS" BASIS,
	12	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	# See the License for the specific language governing permissions and
	14	# limitations under the License.
	15
	16	import itertools
	17	import logging
	18
	19	import six
	20
	21	from signedjson.key import decode_verify_key_bytes
	22
	23	from synapse.storage._base import SQLBaseStore
	24	from synapse.storage.keys import FetchKeyResult
	25	from synapse.util import batch_iter
	26	from synapse.util.caches.descriptors import cached, cachedList
	27
	28	logger = logging.getLogger(__name__)
	29
	30	# py2 sqlite has buffer hardcoded as only binary type, so we must use it,
	31	# despite being deprecated and removed in favor of memoryview
	32	if six.PY2:
	33	db_binary_type = six.moves.builtins.buffer
	34	else:
	35	db_binary_type = memoryview
	36
	37
	38	class KeyStore(SQLBaseStore):
	39	"""Persistence for signature verification keys
	40	"""
	41
	42	@cached()
	43	def _get_server_verify_key(self, server_name_and_key_id):
	44	raise NotImplementedError()
	45
	46	@cachedList(
	47	cached_method_name="_get_server_verify_key", list_name="server_name_and_key_ids"
	48	)
	49	def get_server_verify_keys(self, server_name_and_key_ids):
	50	"""
	51	Args:
	52	server_name_and_key_ids (iterable[Tuple[str, str]]):
	53	iterable of (server_name, key-id) tuples to fetch keys for
	54
	55	Returns:
	56	Deferred: resolves to dict[Tuple[str, str], FetchKeyResult\|None]:
	57	map from (server_name, key_id) -> FetchKeyResult, or None if the key is
	58	unknown
	59	"""
	60	keys = {}
	61
	62	def _get_keys(txn, batch):
	63	"""Processes a batch of keys to fetch, and adds the result to `keys`."""
	64
	65	# batch_iter always returns tuples so it's safe to do len(batch)
	66	sql = (
	67	"SELECT server_name, key_id, verify_key, ts_valid_until_ms "
	68	"FROM server_signature_keys WHERE 1=0"
	69	) + " OR (server_name=? AND key_id=?)" * len(batch)
	70
	71	txn.execute(sql, tuple(itertools.chain.from_iterable(batch)))
	72
	73	for row in txn:
	74	server_name, key_id, key_bytes, ts_valid_until_ms = row
	75
	76	if ts_valid_until_ms is None:
	77	# Old keys may be stored with a ts_valid_until_ms of null,
	78	# in which case we treat this as if it was set to `0`, i.e.
	79	# it won't match key requests that define a minimum
	80	# `ts_valid_until_ms`.
	81	ts_valid_until_ms = 0
	82
	83	res = FetchKeyResult(
	84	verify_key=decode_verify_key_bytes(key_id, bytes(key_bytes)),
	85	valid_until_ts=ts_valid_until_ms,
	86	)
	87	keys[(server_name, key_id)] = res
	88
	89	def _txn(txn):
	90	for batch in batch_iter(server_name_and_key_ids, 50):
	91	_get_keys(txn, batch)
	92	return keys
	93
	94	return self.runInteraction("get_server_verify_keys", _txn)
	95
	96	def store_server_verify_keys(self, from_server, ts_added_ms, verify_keys):
	97	"""Stores NACL verification keys for remote servers.
	98	Args:
	99	from_server (str): Where the verification keys were looked up
	100	ts_added_ms (int): The time to record that the key was added
	101	verify_keys (iterable[tuple[str, str, FetchKeyResult]]):
	102	keys to be stored. Each entry is a triplet of
	103	(server_name, key_id, key).
	104	"""
	105	key_values = []
	106	value_values = []
	107	invalidations = []
	108	for server_name, key_id, fetch_result in verify_keys:
	109	key_values.append((server_name, key_id))
	110	value_values.append(
	111	(
	112	from_server,
	113	ts_added_ms,
	114	fetch_result.valid_until_ts,
	115	db_binary_type(fetch_result.verify_key.encode()),
	116	)
	117	)
	118	# invalidate takes a tuple corresponding to the params of
	119	# _get_server_verify_key. _get_server_verify_key only takes one
	120	# param, which is itself the 2-tuple (server_name, key_id).
	121	invalidations.append((server_name, key_id))
	122
	123	def _invalidate(res):
	124	f = self._get_server_verify_key.invalidate
	125	for i in invalidations:
	126	f((i,))
	127	return res
	128
	129	return self.runInteraction(
	130	"store_server_verify_keys",
	131	self._simple_upsert_many_txn,
	132	table="server_signature_keys",
	133	key_names=("server_name", "key_id"),
	134	key_values=key_values,
	135	value_names=(
	136	"from_server",
	137	"ts_added_ms",
	138	"ts_valid_until_ms",
	139	"verify_key",
	140	),
	141	value_values=value_values,
	142	).addCallback(_invalidate)
	143
	144	def store_server_keys_json(
	145	self, server_name, key_id, from_server, ts_now_ms, ts_expires_ms, key_json_bytes
	146	):
	147	"""Stores the JSON bytes for a set of keys from a server
	148	The JSON should be signed by the originating server, the intermediate
	149	server, and by this server. Updates the value for the
	150	(server_name, key_id, from_server) triplet if one already existed.
	151	Args:
	152	server_name (str): The name of the server.
	153	key_id (str): The identifer of the key this JSON is for.
	154	from_server (str): The server this JSON was fetched from.
	155	ts_now_ms (int): The time now in milliseconds.
	156	ts_valid_until_ms (int): The time when this json stops being valid.
	157	key_json (bytes): The encoded JSON.
	158	"""
	159	return self._simple_upsert(
	160	table="server_keys_json",
	161	keyvalues={
	162	"server_name": server_name,
	163	"key_id": key_id,
	164	"from_server": from_server,
	165	},
	166	values={
	167	"server_name": server_name,
	168	"key_id": key_id,
	169	"from_server": from_server,
	170	"ts_added_ms": ts_now_ms,
	171	"ts_valid_until_ms": ts_expires_ms,
	172	"key_json": db_binary_type(key_json_bytes),
	173	},
	174	desc="store_server_keys_json",
	175	)
	176
	177	def get_server_keys_json(self, server_keys):
	178	"""Retrive the key json for a list of server_keys and key ids.
	179	If no keys are found for a given server, key_id and source then
	180	that server, key_id, and source triplet entry will be an empty list.
	181	The JSON is returned as a byte array so that it can be efficiently
	182	used in an HTTP response.
	183	Args:
	184	server_keys (list): List of (server_name, key_id, source) triplets.
	185	Returns:
	186	Deferred[dict[Tuple[str, str, str\|None], list[dict]]]:
	187	Dict mapping (server_name, key_id, source) triplets to lists of dicts
	188	"""
	189
	190	def _get_server_keys_json_txn(txn):
	191	results = {}
	192	for server_name, key_id, from_server in server_keys:
	193	keyvalues = {"server_name": server_name}
	194	if key_id is not None:
	195	keyvalues["key_id"] = key_id
	196	if from_server is not None:
	197	keyvalues["from_server"] = from_server
	198	rows = self._simple_select_list_txn(
	199	txn,
	200	"server_keys_json",
	201	keyvalues=keyvalues,
	202	retcols=(
	203	"key_id",
	204	"from_server",
	205	"ts_added_ms",
	206	"ts_valid_until_ms",
	207	"key_json",
	208	),
	209	)
	210	results[(server_name, key_id, from_server)] = rows
	211	return results
	212
	213	return self.runInteraction("get_server_keys_json", _get_server_keys_json_txn)

+378

-0

synapse/storage/data_stores/main/media_repository.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2014-2016 OpenMarket Ltd
	2	#
	3	# Licensed under the Apache License, Version 2.0 (the "License");
	4	# you may not use this file except in compliance with the License.
	5	# You may obtain a copy of the License at
	6	#
	7	# http://www.apache.org/licenses/LICENSE-2.0
	8	#
	9	# Unless required by applicable law or agreed to in writing, software
	10	# distributed under the License is distributed on an "AS IS" BASIS,
	11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	# See the License for the specific language governing permissions and
	13	# limitations under the License.
	14	from synapse.storage.background_updates import BackgroundUpdateStore
	15
	16
	17	class MediaRepositoryBackgroundUpdateStore(BackgroundUpdateStore):
	18	def __init__(self, db_conn, hs):
	19	super(MediaRepositoryBackgroundUpdateStore, self).__init__(db_conn, hs)
	20
	21	self.register_background_index_update(
	22	update_name="local_media_repository_url_idx",
	23	index_name="local_media_repository_url_idx",
	24	table="local_media_repository",
	25	columns=["created_ts"],
	26	where_clause="url_cache IS NOT NULL",
	27	)
	28
	29
	30	class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore):
	31	"""Persistence for attachments and avatars"""
	32
	33	def __init__(self, db_conn, hs):
	34	super(MediaRepositoryStore, self).__init__(db_conn, hs)
	35
	36	def get_local_media(self, media_id):
	37	"""Get the metadata for a local piece of media
	38	Returns:
	39	None if the media_id doesn't exist.
	40	"""
	41	return self._simple_select_one(
	42	"local_media_repository",
	43	{"media_id": media_id},
	44	(
	45	"media_type",
	46	"media_length",
	47	"upload_name",
	48	"created_ts",
	49	"quarantined_by",
	50	"url_cache",
	51	),
	52	allow_none=True,
	53	desc="get_local_media",
	54	)
	55
	56	def store_local_media(
	57	self,
	58	media_id,
	59	media_type,
	60	time_now_ms,
	61	upload_name,
	62	media_length,
	63	user_id,
	64	url_cache=None,
	65	):
	66	return self._simple_insert(
	67	"local_media_repository",
	68	{
	69	"media_id": media_id,
	70	"media_type": media_type,
	71	"created_ts": time_now_ms,
	72	"upload_name": upload_name,
	73	"media_length": media_length,
	74	"user_id": user_id.to_string(),
	75	"url_cache": url_cache,
	76	},
	77	desc="store_local_media",
	78	)
	79
	80	def get_url_cache(self, url, ts):
	81	"""Get the media_id and ts for a cached URL as of the given timestamp
	82	Returns:
	83	None if the URL isn't cached.
	84	"""
	85
	86	def get_url_cache_txn(txn):
	87	# get the most recently cached result (relative to the given ts)
	88	sql = (
	89	"SELECT response_code, etag, expires_ts, og, media_id, download_ts"
	90	" FROM local_media_repository_url_cache"
	91	" WHERE url = ? AND download_ts <= ?"
	92	" ORDER BY download_ts DESC LIMIT 1"
	93	)
	94	txn.execute(sql, (url, ts))
	95	row = txn.fetchone()
	96
	97	if not row:
	98	# ...or if we've requested a timestamp older than the oldest
	99	# copy in the cache, return the oldest copy (if any)
	100	sql = (
	101	"SELECT response_code, etag, expires_ts, og, media_id, download_ts"
	102	" FROM local_media_repository_url_cache"
	103	" WHERE url = ? AND download_ts > ?"
	104	" ORDER BY download_ts ASC LIMIT 1"
	105	)
	106	txn.execute(sql, (url, ts))
	107	row = txn.fetchone()
	108
	109	if not row:
	110	return None
	111
	112	return dict(
	113	zip(
	114	(
	115	"response_code",
	116	"etag",
	117	"expires_ts",
	118	"og",
	119	"media_id",
	120	"download_ts",
	121	),
	122	row,
	123	)
	124	)
	125
	126	return self.runInteraction("get_url_cache", get_url_cache_txn)
	127
	128	def store_url_cache(
	129	self, url, response_code, etag, expires_ts, og, media_id, download_ts
	130	):
	131	return self._simple_insert(
	132	"local_media_repository_url_cache",
	133	{
	134	"url": url,
	135	"response_code": response_code,
	136	"etag": etag,
	137	"expires_ts": expires_ts,
	138	"og": og,
	139	"media_id": media_id,
	140	"download_ts": download_ts,
	141	},
	142	desc="store_url_cache",
	143	)
	144
	145	def get_local_media_thumbnails(self, media_id):
	146	return self._simple_select_list(
	147	"local_media_repository_thumbnails",
	148	{"media_id": media_id},
	149	(
	150	"thumbnail_width",
	151	"thumbnail_height",
	152	"thumbnail_method",
	153	"thumbnail_type",
	154	"thumbnail_length",
	155	),
	156	desc="get_local_media_thumbnails",
	157	)
	158
	159	def store_local_thumbnail(
	160	self,
	161	media_id,
	162	thumbnail_width,
	163	thumbnail_height,
	164	thumbnail_type,
	165	thumbnail_method,
	166	thumbnail_length,
	167	):
	168	return self._simple_insert(
	169	"local_media_repository_thumbnails",
	170	{
	171	"media_id": media_id,
	172	"thumbnail_width": thumbnail_width,
	173	"thumbnail_height": thumbnail_height,
	174	"thumbnail_method": thumbnail_method,
	175	"thumbnail_type": thumbnail_type,
	176	"thumbnail_length": thumbnail_length,
	177	},
	178	desc="store_local_thumbnail",
	179	)
	180
	181	def get_cached_remote_media(self, origin, media_id):
	182	return self._simple_select_one(
	183	"remote_media_cache",
	184	{"media_origin": origin, "media_id": media_id},
	185	(
	186	"media_type",
	187	"media_length",
	188	"upload_name",
	189	"created_ts",
	190	"filesystem_id",
	191	"quarantined_by",
	192	),
	193	allow_none=True,
	194	desc="get_cached_remote_media",
	195	)
	196
	197	def store_cached_remote_media(
	198	self,
	199	origin,
	200	media_id,
	201	media_type,
	202	media_length,
	203	time_now_ms,
	204	upload_name,
	205	filesystem_id,
	206	):
	207	return self._simple_insert(
	208	"remote_media_cache",
	209	{
	210	"media_origin": origin,
	211	"media_id": media_id,
	212	"media_type": media_type,
	213	"media_length": media_length,
	214	"created_ts": time_now_ms,
	215	"upload_name": upload_name,
	216	"filesystem_id": filesystem_id,
	217	"last_access_ts": time_now_ms,
	218	},
	219	desc="store_cached_remote_media",
	220	)
	221
	222	def update_cached_last_access_time(self, local_media, remote_media, time_ms):
	223	"""Updates the last access time of the given media
	224
	225	Args:
	226	local_media (iterable[str]): Set of media_ids
	227	remote_media (iterable[(str, str)]): Set of (server_name, media_id)
	228	time_ms: Current time in milliseconds
	229	"""
	230
	231	def update_cache_txn(txn):
	232	sql = (
	233	"UPDATE remote_media_cache SET last_access_ts = ?"
	234	" WHERE media_origin = ? AND media_id = ?"
	235	)
	236
	237	txn.executemany(
	238	sql,
	239	(
	240	(time_ms, media_origin, media_id)
	241	for media_origin, media_id in remote_media
	242	),
	243	)
	244
	245	sql = (
	246	"UPDATE local_media_repository SET last_access_ts = ?"
	247	" WHERE media_id = ?"
	248	)
	249
	250	txn.executemany(sql, ((time_ms, media_id) for media_id in local_media))
	251
	252	return self.runInteraction("update_cached_last_access_time", update_cache_txn)
	253
	254	def get_remote_media_thumbnails(self, origin, media_id):
	255	return self._simple_select_list(
	256	"remote_media_cache_thumbnails",
	257	{"media_origin": origin, "media_id": media_id},
	258	(
	259	"thumbnail_width",
	260	"thumbnail_height",
	261	"thumbnail_method",
	262	"thumbnail_type",
	263	"thumbnail_length",
	264	"filesystem_id",
	265	),
	266	desc="get_remote_media_thumbnails",
	267	)
	268
	269	def store_remote_media_thumbnail(
	270	self,
	271	origin,
	272	media_id,
	273	filesystem_id,
	274	thumbnail_width,
	275	thumbnail_height,
	276	thumbnail_type,
	277	thumbnail_method,
	278	thumbnail_length,
	279	):
	280	return self._simple_insert(
	281	"remote_media_cache_thumbnails",
	282	{
	283	"media_origin": origin,
	284	"media_id": media_id,
	285	"thumbnail_width": thumbnail_width,
	286	"thumbnail_height": thumbnail_height,
	287	"thumbnail_method": thumbnail_method,
	288	"thumbnail_type": thumbnail_type,
	289	"thumbnail_length": thumbnail_length,
	290	"filesystem_id": filesystem_id,
	291	},
	292	desc="store_remote_media_thumbnail",
	293	)
	294
	295	def get_remote_media_before(self, before_ts):
	296	sql = (
	297	"SELECT media_origin, media_id, filesystem_id"
	298	" FROM remote_media_cache"
	299	" WHERE last_access_ts < ?"
	300	)
	301
	302	return self._execute(
	303	"get_remote_media_before", self.cursor_to_dict, sql, before_ts
	304	)
	305
	306	def delete_remote_media(self, media_origin, media_id):
	307	def delete_remote_media_txn(txn):
	308	self._simple_delete_txn(
	309	txn,
	310	"remote_media_cache",
	311	keyvalues={"media_origin": media_origin, "media_id": media_id},
	312	)
	313	self._simple_delete_txn(
	314	txn,
	315	"remote_media_cache_thumbnails",
	316	keyvalues={"media_origin": media_origin, "media_id": media_id},
	317	)
	318
	319	return self.runInteraction("delete_remote_media", delete_remote_media_txn)
	320
	321	def get_expired_url_cache(self, now_ts):
	322	sql = (
	323	"SELECT media_id FROM local_media_repository_url_cache"
	324	" WHERE expires_ts < ?"
	325	" ORDER BY expires_ts ASC"
	326	" LIMIT 500"
	327	)
	328
	329	def _get_expired_url_cache_txn(txn):
	330	txn.execute(sql, (now_ts,))
	331	return [row[0] for row in txn]
	332
	333	return self.runInteraction("get_expired_url_cache", _get_expired_url_cache_txn)
	334
	335	def delete_url_cache(self, media_ids):
	336	if len(media_ids) == 0:
	337	return
	338
	339	sql = "DELETE FROM local_media_repository_url_cache" " WHERE media_id = ?"
	340
	341	def _delete_url_cache_txn(txn):
	342	txn.executemany(sql, [(media_id,) for media_id in media_ids])
	343
	344	return self.runInteraction("delete_url_cache", _delete_url_cache_txn)
	345
	346	def get_url_cache_media_before(self, before_ts):
	347	sql = (
	348	"SELECT media_id FROM local_media_repository"
	349	" WHERE created_ts < ? AND url_cache IS NOT NULL"
	350	" ORDER BY created_ts ASC"
	351	" LIMIT 500"
	352	)
	353
	354	def _get_url_cache_media_before_txn(txn):
	355	txn.execute(sql, (before_ts,))
	356	return [row[0] for row in txn]
	357
	358	return self.runInteraction(
	359	"get_url_cache_media_before", _get_url_cache_media_before_txn
	360	)
	361
	362	def delete_url_cache_media(self, media_ids):
	363	if len(media_ids) == 0:
	364	return
	365
	366	def _delete_url_cache_media_txn(txn):
	367	sql = "DELETE FROM local_media_repository" " WHERE media_id = ?"
	368
	369	txn.executemany(sql, [(media_id,) for media_id in media_ids])
	370
	371	sql = "DELETE FROM local_media_repository_thumbnails" " WHERE media_id = ?"
	372
	373	txn.executemany(sql, [(media_id,) for media_id in media_ids])
	374
	375	return self.runInteraction(
	376	"delete_url_cache_media", _delete_url_cache_media_txn
	377	)

+328

-0

synapse/storage/data_stores/main/monthly_active_users.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2018 New Vector
	2	#
	3	# Licensed under the Apache License, Version 2.0 (the "License");
	4	# you may not use this file except in compliance with the License.
	5	# You may obtain a copy of the License at
	6	#
	7	# http://www.apache.org/licenses/LICENSE-2.0
	8	#
	9	# Unless required by applicable law or agreed to in writing, software
	10	# distributed under the License is distributed on an "AS IS" BASIS,
	11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	# See the License for the specific language governing permissions and
	13	# limitations under the License.
	14	import logging
	15
	16	from twisted.internet import defer
	17
	18	from synapse.storage._base import SQLBaseStore
	19	from synapse.util.caches.descriptors import cached
	20
	21	logger = logging.getLogger(__name__)
	22
	23	# Number of msec of granularity to store the monthly_active_user timestamp
	24	# This means it is not necessary to update the table on every request
	25	LAST_SEEN_GRANULARITY = 60 * 60 * 1000
	26
	27
	28	class MonthlyActiveUsersStore(SQLBaseStore):
	29	def __init__(self, dbconn, hs):
	30	super(MonthlyActiveUsersStore, self).__init__(None, hs)
	31	self._clock = hs.get_clock()
	32	self.hs = hs
	33	# Do not add more reserved users than the total allowable number
	34	self._new_transaction(
	35	dbconn,
	36	"initialise_mau_threepids",
	37	[],
	38	[],
	39	self._initialise_reserved_users,
	40	hs.config.mau_limits_reserved_threepids[: self.hs.config.max_mau_value],
	41	)
	42
	43	def _initialise_reserved_users(self, txn, threepids):
	44	"""Ensures that reserved threepids are accounted for in the MAU table, should
	45	be called on start up.
	46
	47	Args:
	48	txn (cursor):
	49	threepids (list[dict]): List of threepid dicts to reserve
	50	"""
	51
	52	for tp in threepids:
	53	user_id = self.get_user_id_by_threepid_txn(txn, tp["medium"], tp["address"])
	54
	55	if user_id:
	56	is_support = self.is_support_user_txn(txn, user_id)
	57	if not is_support:
	58	self.upsert_monthly_active_user_txn(txn, user_id)
	59	else:
	60	logger.warning("mau limit reserved threepid %s not found in db" % tp)
	61
	62	@defer.inlineCallbacks
	63	def reap_monthly_active_users(self):
	64	"""Cleans out monthly active user table to ensure that no stale
	65	entries exist.
	66
	67	Returns:
	68	Deferred[]
	69	"""
	70
	71	def _reap_users(txn, reserved_users):
	72	"""
	73	Args:
	74	reserved_users (tuple): reserved users to preserve
	75	"""
	76
	77	thirty_days_ago = int(self._clock.time_msec()) - (1000 * 60 * 60 * 24 * 30)
	78	query_args = [thirty_days_ago]
	79	base_sql = "DELETE FROM monthly_active_users WHERE timestamp < ?"
	80
	81	# Need if/else since 'AND user_id NOT IN ({})' fails on Postgres
	82	# when len(reserved_users) == 0. Works fine on sqlite.
	83	if len(reserved_users) > 0:
	84	# questionmarks is a hack to overcome sqlite not supporting
	85	# tuples in 'WHERE IN %s'
	86	question_marks = ",".join("?" * len(reserved_users))
	87
	88	query_args.extend(reserved_users)
	89	sql = base_sql + " AND user_id NOT IN ({})".format(question_marks)
	90	else:
	91	sql = base_sql
	92
	93	txn.execute(sql, query_args)
	94
	95	max_mau_value = self.hs.config.max_mau_value
	96	if self.hs.config.limit_usage_by_mau:
	97	# If MAU user count still exceeds the MAU threshold, then delete on
	98	# a least recently active basis.
	99	# Note it is not possible to write this query using OFFSET due to
	100	# incompatibilities in how sqlite and postgres support the feature.
	101	# sqlite requires 'LIMIT -1 OFFSET ?', the LIMIT must be present
	102	# While Postgres does not require 'LIMIT', but also does not support
	103	# negative LIMIT values. So there is no way to write it that both can
	104	# support
	105	if len(reserved_users) == 0:
	106	sql = """
	107	DELETE FROM monthly_active_users
	108	WHERE user_id NOT IN (
	109	SELECT user_id FROM monthly_active_users
	110	ORDER BY timestamp DESC
	111	LIMIT ?
	112	)
	113	"""
	114	txn.execute(sql, (max_mau_value,))
	115	# Need if/else since 'AND user_id NOT IN ({})' fails on Postgres
	116	# when len(reserved_users) == 0. Works fine on sqlite.
	117	else:
	118	# Must be >= 0 for postgres
	119	num_of_non_reserved_users_to_remove = max(
	120	max_mau_value - len(reserved_users), 0
	121	)
	122
	123	# It is important to filter reserved users twice to guard
	124	# against the case where the reserved user is present in the
	125	# SELECT, meaning that a legitmate mau is deleted.
	126	sql = """
	127	DELETE FROM monthly_active_users
	128	WHERE user_id NOT IN (
	129	SELECT user_id FROM monthly_active_users
	130	WHERE user_id NOT IN ({})
	131	ORDER BY timestamp DESC
	132	LIMIT ?
	133	)
	134	AND user_id NOT IN ({})
	135	""".format(
	136	question_marks, question_marks
	137	)
	138
	139	query_args = [
	140	*reserved_users,
	141	num_of_non_reserved_users_to_remove,
	142	*reserved_users,
	143	]
	144
	145	txn.execute(sql, query_args)
	146
	147	reserved_users = yield self.get_registered_reserved_users()
	148	yield self.runInteraction(
	149	"reap_monthly_active_users", _reap_users, reserved_users
	150	)
	151	# It seems poor to invalidate the whole cache, Postgres supports
	152	# 'Returning' which would allow me to invalidate only the
	153	# specific users, but sqlite has no way to do this and instead
	154	# I would need to SELECT and the DELETE which without locking
	155	# is racy.
	156	# Have resolved to invalidate the whole cache for now and do
	157	# something about it if and when the perf becomes significant
	158	self.user_last_seen_monthly_active.invalidate_all()
	159	self.get_monthly_active_count.invalidate_all()
	160
	161	@cached(num_args=0)
	162	def get_monthly_active_count(self):
	163	"""Generates current count of monthly active users
	164
	165	Returns:
	166	Defered[int]: Number of current monthly active users
	167	"""
	168
	169	def _count_users(txn):
	170	sql = "SELECT COALESCE(count(*), 0) FROM monthly_active_users"
	171
	172	txn.execute(sql)
	173	count, = txn.fetchone()
	174	return count
	175
	176	return self.runInteraction("count_users", _count_users)
	177
	178	@defer.inlineCallbacks
	179	def get_registered_reserved_users(self):
	180	"""Of the reserved threepids defined in config, which are associated
	181	with registered users?
	182
	183	Returns:
	184	Defered[list]: Real reserved users
	185	"""
	186	users = []
	187
	188	for tp in self.hs.config.mau_limits_reserved_threepids[
	189	: self.hs.config.max_mau_value
	190	]:
	191	user_id = yield self.hs.get_datastore().get_user_id_by_threepid(
	192	tp["medium"], tp["address"]
	193	)
	194	if user_id:
	195	users.append(user_id)
	196
	197	return users
	198
	199	@defer.inlineCallbacks
	200	def upsert_monthly_active_user(self, user_id):
	201	"""Updates or inserts the user into the monthly active user table, which
	202	is used to track the current MAU usage of the server
	203
	204	Args:
	205	user_id (str): user to add/update
	206	"""
	207	# Support user never to be included in MAU stats. Note I can't easily call this
	208	# from upsert_monthly_active_user_txn because then I need a _txn form of
	209	# is_support_user which is complicated because I want to cache the result.
	210	# Therefore I call it here and ignore the case where
	211	# upsert_monthly_active_user_txn is called directly from
	212	# _initialise_reserved_users reasoning that it would be very strange to
	213	# include a support user in this context.
	214
	215	is_support = yield self.is_support_user(user_id)
	216	if is_support:
	217	return
	218
	219	yield self.runInteraction(
	220	"upsert_monthly_active_user", self.upsert_monthly_active_user_txn, user_id
	221	)
	222
	223	user_in_mau = self.user_last_seen_monthly_active.cache.get(
	224	(user_id,), None, update_metrics=False
	225	)
	226	if user_in_mau is None:
	227	self.get_monthly_active_count.invalidate(())
	228
	229	self.user_last_seen_monthly_active.invalidate((user_id,))
	230
	231	def upsert_monthly_active_user_txn(self, txn, user_id):
	232	"""Updates or inserts monthly active user member
	233
	234	Note that, after calling this method, it will generally be necessary
	235	to invalidate the caches on user_last_seen_monthly_active and
	236	get_monthly_active_count. We can't do that here, because we are running
	237	in a database thread rather than the main thread, and we can't call
	238	txn.call_after because txn may not be a LoggingTransaction.
	239
	240	We consciously do not call is_support_txn from this method because it
	241	is not possible to cache the response. is_support_txn will be false in
	242	almost all cases, so it seems reasonable to call it only for
	243	upsert_monthly_active_user and to call is_support_txn manually
	244	for cases where upsert_monthly_active_user_txn is called directly,
	245	like _initialise_reserved_users
	246
	247	In short, don't call this method with support users. (Support users
	248	should not appear in the MAU stats).
	249
	250	Args:
	251	txn (cursor):
	252	user_id (str): user to add/update
	253
	254	Returns:
	255	bool: True if a new entry was created, False if an
	256	existing one was updated.
	257	"""
	258
	259	# Am consciously deciding to lock the table on the basis that is ought
	260	# never be a big table and alternative approaches (batching multiple
	261	# upserts into a single txn) introduced a lot of extra complexity.
	262	# See https://github.com/matrix-org/synapse/issues/3854 for more
	263	is_insert = self._simple_upsert_txn(
	264	txn,
	265	table="monthly_active_users",
	266	keyvalues={"user_id": user_id},
	267	values={"timestamp": int(self._clock.time_msec())},
	268	)
	269
	270	return is_insert
	271
	272	@cached(num_args=1)
	273	def user_last_seen_monthly_active(self, user_id):
	274	"""
	275	Checks if a given user is part of the monthly active user group
	276	Arguments:
	277	user_id (str): user to add/update
	278	Return:
	279	Deferred[int] : timestamp since last seen, None if never seen
	280
	281	"""
	282
	283	return self._simple_select_one_onecol(
	284	table="monthly_active_users",
	285	keyvalues={"user_id": user_id},
	286	retcol="timestamp",
	287	allow_none=True,
	288	desc="user_last_seen_monthly_active",
	289	)
	290
	291	@defer.inlineCallbacks
	292	def populate_monthly_active_users(self, user_id):
	293	"""Checks on the state of monthly active user limits and optionally
	294	add the user to the monthly active tables
	295
	296	Args:
	297	user_id(str): the user_id to query
	298	"""
	299	if self.hs.config.limit_usage_by_mau or self.hs.config.mau_stats_only:
	300	# Trial users and guests should not be included as part of MAU group
	301	is_guest = yield self.is_guest(user_id)
	302	if is_guest:
	303	return
	304	is_trial = yield self.is_trial_user(user_id)
	305	if is_trial:
	306	return
	307
	308	last_seen_timestamp = yield self.user_last_seen_monthly_active(user_id)
	309	now = self.hs.get_clock().time_msec()
	310
	311	# We want to reduce to the total number of db writes, and are happy
	312	# to trade accuracy of timestamp in order to lighten load. This means
	313	# We always insert new users (where MAU threshold has not been reached),
	314	# but only update if we have not previously seen the user for
	315	# LAST_SEEN_GRANULARITY ms
	316	if last_seen_timestamp is None:
	317	# In the case where mau_stats_only is True and limit_usage_by_mau is
	318	# False, there is no point in checking get_monthly_active_count - it
	319	# adds no value and will break the logic if max_mau_value is exceeded.
	320	if not self.hs.config.limit_usage_by_mau:
	321	yield self.upsert_monthly_active_user(user_id)
	322	else:
	323	count = yield self.get_monthly_active_count()
	324	if count < self.hs.config.max_mau_value:
	325	yield self.upsert_monthly_active_user(user_id)
	326	elif now - last_seen_timestamp > LAST_SEEN_GRANULARITY:
	327	yield self.upsert_monthly_active_user(user_id)

+31

-0

synapse/storage/data_stores/main/openid.py less more

	0	from synapse.storage._base import SQLBaseStore
	1
	2
	3	class OpenIdStore(SQLBaseStore):
	4	def insert_open_id_token(self, token, ts_valid_until_ms, user_id):
	5	return self._simple_insert(
	6	table="open_id_tokens",
	7	values={
	8	"token": token,
	9	"ts_valid_until_ms": ts_valid_until_ms,
	10	"user_id": user_id,
	11	},
	12	desc="insert_open_id_token",
	13	)
	14
	15	def get_user_id_for_open_id_token(self, token, ts_now_ms):
	16	def get_user_id_for_token_txn(txn):
	17	sql = (
	18	"SELECT user_id FROM open_id_tokens"
	19	" WHERE token = ? AND ? <= ts_valid_until_ms"
	20	)
	21
	22	txn.execute(sql, (token, ts_now_ms))
	23
	24	rows = txn.fetchall()
	25	if not rows:
	26	return None
	27	else:
	28	return rows[0][0]
	29
	30	return self.runInteraction("get_user_id_for_token", get_user_id_for_token_txn)

+150

-0

synapse/storage/data_stores/main/presence.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2014-2016 OpenMarket Ltd
	2	#
	3	# Licensed under the Apache License, Version 2.0 (the "License");
	4	# you may not use this file except in compliance with the License.
	5	# You may obtain a copy of the License at
	6	#
	7	# http://www.apache.org/licenses/LICENSE-2.0
	8	#
	9	# Unless required by applicable law or agreed to in writing, software
	10	# distributed under the License is distributed on an "AS IS" BASIS,
	11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	# See the License for the specific language governing permissions and
	13	# limitations under the License.
	14
	15	from twisted.internet import defer
	16
	17	from synapse.storage._base import SQLBaseStore, make_in_list_sql_clause
	18	from synapse.storage.presence import UserPresenceState
	19	from synapse.util import batch_iter
	20	from synapse.util.caches.descriptors import cached, cachedList
	21
	22
	23	class PresenceStore(SQLBaseStore):
	24	@defer.inlineCallbacks
	25	def update_presence(self, presence_states):
	26	stream_ordering_manager = self._presence_id_gen.get_next_mult(
	27	len(presence_states)
	28	)
	29
	30	with stream_ordering_manager as stream_orderings:
	31	yield self.runInteraction(
	32	"update_presence",
	33	self._update_presence_txn,
	34	stream_orderings,
	35	presence_states,
	36	)
	37
	38	return stream_orderings[-1], self._presence_id_gen.get_current_token()
	39
	40	def _update_presence_txn(self, txn, stream_orderings, presence_states):
	41	for stream_id, state in zip(stream_orderings, presence_states):
	42	txn.call_after(
	43	self.presence_stream_cache.entity_has_changed, state.user_id, stream_id
	44	)
	45	txn.call_after(self._get_presence_for_user.invalidate, (state.user_id,))
	46
	47	# Actually insert new rows
	48	self._simple_insert_many_txn(
	49	txn,
	50	table="presence_stream",
	51	values=[
	52	{
	53	"stream_id": stream_id,
	54	"user_id": state.user_id,
	55	"state": state.state,
	56	"last_active_ts": state.last_active_ts,
	57	"last_federation_update_ts": state.last_federation_update_ts,
	58	"last_user_sync_ts": state.last_user_sync_ts,
	59	"status_msg": state.status_msg,
	60	"currently_active": state.currently_active,
	61	}
	62	for state in presence_states
	63	],
	64	)
	65
	66	# Delete old rows to stop database from getting really big
	67	sql = "DELETE FROM presence_stream WHERE stream_id < ? AND "
	68
	69	for states in batch_iter(presence_states, 50):
	70	clause, args = make_in_list_sql_clause(
	71	self.database_engine, "user_id", [s.user_id for s in states]
	72	)
	73	txn.execute(sql + clause, [stream_id] + list(args))
	74
	75	def get_all_presence_updates(self, last_id, current_id):
	76	if last_id == current_id:
	77	return defer.succeed([])
	78
	79	def get_all_presence_updates_txn(txn):
	80	sql = (
	81	"SELECT stream_id, user_id, state, last_active_ts,"
	82	" last_federation_update_ts, last_user_sync_ts, status_msg,"
	83	" currently_active"
	84	" FROM presence_stream"
	85	" WHERE ? < stream_id AND stream_id <= ?"
	86	)
	87	txn.execute(sql, (last_id, current_id))
	88	return txn.fetchall()
	89
	90	return self.runInteraction(
	91	"get_all_presence_updates", get_all_presence_updates_txn
	92	)
	93
	94	@cached()
	95	def _get_presence_for_user(self, user_id):
	96	raise NotImplementedError()
	97
	98	@cachedList(
	99	cached_method_name="_get_presence_for_user",
	100	list_name="user_ids",
	101	num_args=1,
	102	inlineCallbacks=True,
	103	)
	104	def get_presence_for_users(self, user_ids):
	105	rows = yield self._simple_select_many_batch(
	106	table="presence_stream",
	107	column="user_id",
	108	iterable=user_ids,
	109	keyvalues={},
	110	retcols=(
	111	"user_id",
	112	"state",
	113	"last_active_ts",
	114	"last_federation_update_ts",
	115	"last_user_sync_ts",
	116	"status_msg",
	117	"currently_active",
	118	),
	119	desc="get_presence_for_users",
	120	)
	121
	122	for row in rows:
	123	row["currently_active"] = bool(row["currently_active"])
	124
	125	return {row["user_id"]: UserPresenceState(**row) for row in rows}
	126
	127	def get_current_presence_token(self):
	128	return self._presence_id_gen.get_current_token()
	129
	130	def allow_presence_visible(self, observed_localpart, observer_userid):
	131	return self._simple_insert(
	132	table="presence_allow_inbound",
	133	values={
	134	"observed_user_id": observed_localpart,
	135	"observer_user_id": observer_userid,
	136	},
	137	desc="allow_presence_visible",
	138	or_ignore=True,
	139	)
	140
	141	def disallow_presence_visible(self, observed_localpart, observer_userid):
	142	return self._simple_delete_one(
	143	table="presence_allow_inbound",
	144	keyvalues={
	145	"observed_user_id": observed_localpart,
	146	"observer_user_id": observer_userid,
	147	},
	148	desc="disallow_presence_visible",
	149	)

+178

-0

synapse/storage/data_stores/main/profile.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2014-2016 OpenMarket Ltd
	2	#
	3	# Licensed under the Apache License, Version 2.0 (the "License");
	4	# you may not use this file except in compliance with the License.
	5	# You may obtain a copy of the License at
	6	#
	7	# http://www.apache.org/licenses/LICENSE-2.0
	8	#
	9	# Unless required by applicable law or agreed to in writing, software
	10	# distributed under the License is distributed on an "AS IS" BASIS,
	11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	# See the License for the specific language governing permissions and
	13	# limitations under the License.
	14
	15	from twisted.internet import defer
	16
	17	from synapse.api.errors import StoreError
	18	from synapse.storage._base import SQLBaseStore
	19	from synapse.storage.data_stores.main.roommember import ProfileInfo
	20
	21
	22	class ProfileWorkerStore(SQLBaseStore):
	23	@defer.inlineCallbacks
	24	def get_profileinfo(self, user_localpart):
	25	try:
	26	profile = yield self._simple_select_one(
	27	table="profiles",
	28	keyvalues={"user_id": user_localpart},
	29	retcols=("displayname", "avatar_url"),
	30	desc="get_profileinfo",
	31	)
	32	except StoreError as e:
	33	if e.code == 404:
	34	# no match
	35	return ProfileInfo(None, None)
	36	else:
	37	raise
	38
	39	return ProfileInfo(
	40	avatar_url=profile["avatar_url"], display_name=profile["displayname"]
	41	)
	42
	43	def get_profile_displayname(self, user_localpart):
	44	return self._simple_select_one_onecol(
	45	table="profiles",
	46	keyvalues={"user_id": user_localpart},
	47	retcol="displayname",
	48	desc="get_profile_displayname",
	49	)
	50
	51	def get_profile_avatar_url(self, user_localpart):
	52	return self._simple_select_one_onecol(
	53	table="profiles",
	54	keyvalues={"user_id": user_localpart},
	55	retcol="avatar_url",
	56	desc="get_profile_avatar_url",
	57	)
	58
	59	def get_from_remote_profile_cache(self, user_id):
	60	return self._simple_select_one(
	61	table="remote_profile_cache",
	62	keyvalues={"user_id": user_id},
	63	retcols=("displayname", "avatar_url"),
	64	allow_none=True,
	65	desc="get_from_remote_profile_cache",
	66	)
	67
	68	def create_profile(self, user_localpart):
	69	return self._simple_insert(
	70	table="profiles", values={"user_id": user_localpart}, desc="create_profile"
	71	)
	72
	73	def set_profile_displayname(self, user_localpart, new_displayname):
	74	return self._simple_update_one(
	75	table="profiles",
	76	keyvalues={"user_id": user_localpart},
	77	updatevalues={"displayname": new_displayname},
	78	desc="set_profile_displayname",
	79	)
	80
	81	def set_profile_avatar_url(self, user_localpart, new_avatar_url):
	82	return self._simple_update_one(
	83	table="profiles",
	84	keyvalues={"user_id": user_localpart},
	85	updatevalues={"avatar_url": new_avatar_url},
	86	desc="set_profile_avatar_url",
	87	)
	88
	89
	90	class ProfileStore(ProfileWorkerStore):
	91	def add_remote_profile_cache(self, user_id, displayname, avatar_url):
	92	"""Ensure we are caching the remote user's profiles.
	93
	94	This should only be called when `is_subscribed_remote_profile_for_user`
	95	would return true for the user.
	96	"""
	97	return self._simple_upsert(
	98	table="remote_profile_cache",
	99	keyvalues={"user_id": user_id},
	100	values={
	101	"displayname": displayname,
	102	"avatar_url": avatar_url,
	103	"last_check": self._clock.time_msec(),
	104	},
	105	desc="add_remote_profile_cache",
	106	)
	107
	108	def update_remote_profile_cache(self, user_id, displayname, avatar_url):
	109	return self._simple_update(
	110	table="remote_profile_cache",
	111	keyvalues={"user_id": user_id},
	112	values={
	113	"displayname": displayname,
	114	"avatar_url": avatar_url,
	115	"last_check": self._clock.time_msec(),
	116	},
	117	desc="update_remote_profile_cache",
	118	)
	119
	120	@defer.inlineCallbacks
	121	def maybe_delete_remote_profile_cache(self, user_id):
	122	"""Check if we still care about the remote user's profile, and if we
	123	don't then remove their profile from the cache
	124	"""
	125	subscribed = yield self.is_subscribed_remote_profile_for_user(user_id)
	126	if not subscribed:
	127	yield self._simple_delete(
	128	table="remote_profile_cache",
	129	keyvalues={"user_id": user_id},
	130	desc="delete_remote_profile_cache",
	131	)
	132
	133	def get_remote_profile_cache_entries_that_expire(self, last_checked):
	134	"""Get all users who haven't been checked since `last_checked`
	135	"""
	136
	137	def _get_remote_profile_cache_entries_that_expire_txn(txn):
	138	sql = """
	139	SELECT user_id, displayname, avatar_url
	140	FROM remote_profile_cache
	141	WHERE last_check < ?
	142	"""
	143
	144	txn.execute(sql, (last_checked,))
	145
	146	return self.cursor_to_dict(txn)
	147
	148	return self.runInteraction(
	149	"get_remote_profile_cache_entries_that_expire",
	150	_get_remote_profile_cache_entries_that_expire_txn,
	151	)
	152
	153	@defer.inlineCallbacks
	154	def is_subscribed_remote_profile_for_user(self, user_id):
	155	"""Check whether we are interested in a remote user's profile.
	156	"""
	157	res = yield self._simple_select_one_onecol(
	158	table="group_users",
	159	keyvalues={"user_id": user_id},
	160	retcol="user_id",
	161	allow_none=True,
	162	desc="should_update_remote_profile_cache_for_user",
	163	)
	164
	165	if res:
	166	return True
	167
	168	res = yield self._simple_select_one_onecol(
	169	table="group_invites",
	170	keyvalues={"user_id": user_id},
	171	retcol="user_id",
	172	allow_none=True,
	173	desc="should_update_remote_profile_cache_for_user",
	174	)
	175
	176	if res:
	177	return True

+713

-0

synapse/storage/data_stores/main/push_rule.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2014-2016 OpenMarket Ltd
	2	# Copyright 2018 New Vector Ltd
	3	#
	4	# Licensed under the Apache License, Version 2.0 (the "License");
	5	# you may not use this file except in compliance with the License.
	6	# You may obtain a copy of the License at
	7	#
	8	# http://www.apache.org/licenses/LICENSE-2.0
	9	#
	10	# Unless required by applicable law or agreed to in writing, software
	11	# distributed under the License is distributed on an "AS IS" BASIS,
	12	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	# See the License for the specific language governing permissions and
	14	# limitations under the License.
	15
	16	import abc
	17	import logging
	18
	19	from canonicaljson import json
	20
	21	from twisted.internet import defer
	22
	23	from synapse.push.baserules import list_with_base_rules
	24	from synapse.storage._base import SQLBaseStore
	25	from synapse.storage.data_stores.main.appservice import ApplicationServiceWorkerStore
	26	from synapse.storage.data_stores.main.pusher import PusherWorkerStore
	27	from synapse.storage.data_stores.main.receipts import ReceiptsWorkerStore
	28	from synapse.storage.data_stores.main.roommember import RoomMemberWorkerStore
	29	from synapse.storage.push_rule import InconsistentRuleException, RuleNotFoundException
	30	from synapse.util.caches.descriptors import cachedInlineCallbacks, cachedList
	31	from synapse.util.caches.stream_change_cache import StreamChangeCache
	32
	33	logger = logging.getLogger(__name__)
	34
	35
	36	def _load_rules(rawrules, enabled_map):
	37	ruleslist = []
	38	for rawrule in rawrules:
	39	rule = dict(rawrule)
	40	rule["conditions"] = json.loads(rawrule["conditions"])
	41	rule["actions"] = json.loads(rawrule["actions"])
	42	ruleslist.append(rule)
	43
	44	# We're going to be mutating this a lot, so do a deep copy
	45	rules = list(list_with_base_rules(ruleslist))
	46
	47	for i, rule in enumerate(rules):
	48	rule_id = rule["rule_id"]
	49	if rule_id in enabled_map:
	50	if rule.get("enabled", True) != bool(enabled_map[rule_id]):
	51	# Rules are cached across users.
	52	rule = dict(rule)
	53	rule["enabled"] = bool(enabled_map[rule_id])
	54	rules[i] = rule
	55
	56	return rules
	57
	58
	59	class PushRulesWorkerStore(
	60	ApplicationServiceWorkerStore,
	61	ReceiptsWorkerStore,
	62	PusherWorkerStore,
	63	RoomMemberWorkerStore,
	64	SQLBaseStore,
	65	):
	66	"""This is an abstract base class where subclasses must implement
	67	`get_max_push_rules_stream_id` which can be called in the initializer.
	68	"""
	69
	70	# This ABCMeta metaclass ensures that we cannot be instantiated without
	71	# the abstract methods being implemented.
	72	__metaclass__ = abc.ABCMeta
	73
	74	def __init__(self, db_conn, hs):
	75	super(PushRulesWorkerStore, self).__init__(db_conn, hs)
	76
	77	push_rules_prefill, push_rules_id = self._get_cache_dict(
	78	db_conn,
	79	"push_rules_stream",
	80	entity_column="user_id",
	81	stream_column="stream_id",
	82	max_value=self.get_max_push_rules_stream_id(),
	83	)
	84
	85	self.push_rules_stream_cache = StreamChangeCache(
	86	"PushRulesStreamChangeCache",
	87	push_rules_id,
	88	prefilled_cache=push_rules_prefill,
	89	)
	90
	91	@abc.abstractmethod
	92	def get_max_push_rules_stream_id(self):
	93	"""Get the position of the push rules stream.
	94
	95	Returns:
	96	int
	97	"""
	98	raise NotImplementedError()
	99
	100	@cachedInlineCallbacks(max_entries=5000)
	101	def get_push_rules_for_user(self, user_id):
	102	rows = yield self._simple_select_list(
	103	table="push_rules",
	104	keyvalues={"user_name": user_id},
	105	retcols=(
	106	"user_name",
	107	"rule_id",
	108	"priority_class",
	109	"priority",
	110	"conditions",
	111	"actions",
	112	),
	113	desc="get_push_rules_enabled_for_user",
	114	)
	115
	116	rows.sort(key=lambda row: (-int(row["priority_class"]), -int(row["priority"])))
	117
	118	enabled_map = yield self.get_push_rules_enabled_for_user(user_id)
	119
	120	rules = _load_rules(rows, enabled_map)
	121
	122	return rules
	123
	124	@cachedInlineCallbacks(max_entries=5000)
	125	def get_push_rules_enabled_for_user(self, user_id):
	126	results = yield self._simple_select_list(
	127	table="push_rules_enable",
	128	keyvalues={"user_name": user_id},
	129	retcols=("user_name", "rule_id", "enabled"),
	130	desc="get_push_rules_enabled_for_user",
	131	)
	132	return {r["rule_id"]: False if r["enabled"] == 0 else True for r in results}
	133
	134	def have_push_rules_changed_for_user(self, user_id, last_id):
	135	if not self.push_rules_stream_cache.has_entity_changed(user_id, last_id):
	136	return defer.succeed(False)
	137	else:
	138
	139	def have_push_rules_changed_txn(txn):
	140	sql = (
	141	"SELECT COUNT(stream_id) FROM push_rules_stream"
	142	" WHERE user_id = ? AND ? < stream_id"
	143	)
	144	txn.execute(sql, (user_id, last_id))
	145	count, = txn.fetchone()
	146	return bool(count)
	147
	148	return self.runInteraction(
	149	"have_push_rules_changed", have_push_rules_changed_txn
	150	)
	151
	152	@cachedList(
	153	cached_method_name="get_push_rules_for_user",
	154	list_name="user_ids",
	155	num_args=1,
	156	inlineCallbacks=True,
	157	)
	158	def bulk_get_push_rules(self, user_ids):
	159	if not user_ids:
	160	return {}
	161
	162	results = {user_id: [] for user_id in user_ids}
	163
	164	rows = yield self._simple_select_many_batch(
	165	table="push_rules",
	166	column="user_name",
	167	iterable=user_ids,
	168	retcols=("*",),
	169	desc="bulk_get_push_rules",
	170	)
	171
	172	rows.sort(key=lambda row: (-int(row["priority_class"]), -int(row["priority"])))
	173
	174	for row in rows:
	175	results.setdefault(row["user_name"], []).append(row)
	176
	177	enabled_map_by_user = yield self.bulk_get_push_rules_enabled(user_ids)
	178
	179	for user_id, rules in results.items():
	180	results[user_id] = _load_rules(rules, enabled_map_by_user.get(user_id, {}))
	181
	182	return results
	183
	184	@defer.inlineCallbacks
	185	def copy_push_rule_from_room_to_room(self, new_room_id, user_id, rule):
	186	"""Copy a single push rule from one room to another for a specific user.
	187
	188	Args:
	189	new_room_id (str): ID of the new room.
	190	user_id (str): ID of user the push rule belongs to.
	191	rule (Dict): A push rule.
	192	"""
	193	# Create new rule id
	194	rule_id_scope = "/".join(rule["rule_id"].split("/")[:-1])
	195	new_rule_id = rule_id_scope + "/" + new_room_id
	196
	197	# Change room id in each condition
	198	for condition in rule.get("conditions", []):
	199	if condition.get("key") == "room_id":
	200	condition["pattern"] = new_room_id
	201
	202	# Add the rule for the new room
	203	yield self.add_push_rule(
	204	user_id=user_id,
	205	rule_id=new_rule_id,
	206	priority_class=rule["priority_class"],
	207	conditions=rule["conditions"],
	208	actions=rule["actions"],
	209	)
	210
	211	@defer.inlineCallbacks
	212	def copy_push_rules_from_room_to_room_for_user(
	213	self, old_room_id, new_room_id, user_id
	214	):
	215	"""Copy all of the push rules from one room to another for a specific
	216	user.
	217
	218	Args:
	219	old_room_id (str): ID of the old room.
	220	new_room_id (str): ID of the new room.
	221	user_id (str): ID of user to copy push rules for.
	222	"""
	223	# Retrieve push rules for this user
	224	user_push_rules = yield self.get_push_rules_for_user(user_id)
	225
	226	# Get rules relating to the old room and copy them to the new room
	227	for rule in user_push_rules:
	228	conditions = rule.get("conditions", [])
	229	if any(
	230	(c.get("key") == "room_id" and c.get("pattern") == old_room_id)
	231	for c in conditions
	232	):
	233	yield self.copy_push_rule_from_room_to_room(new_room_id, user_id, rule)
	234
	235	@defer.inlineCallbacks
	236	def bulk_get_push_rules_for_room(self, event, context):
	237	state_group = context.state_group
	238	if not state_group:
	239	# If state_group is None it means it has yet to be assigned a
	240	# state group, i.e. we need to make sure that calls with a state_group
	241	# of None don't hit previous cached calls with a None state_group.
	242	# To do this we set the state_group to a new object as object() != object()
	243	state_group = object()
	244
	245	current_state_ids = yield context.get_current_state_ids(self)
	246	result = yield self._bulk_get_push_rules_for_room(
	247	event.room_id, state_group, current_state_ids, event=event
	248	)
	249	return result
	250
	251	@cachedInlineCallbacks(num_args=2, cache_context=True)
	252	def _bulk_get_push_rules_for_room(
	253	self, room_id, state_group, current_state_ids, cache_context, event=None
	254	):
	255	# We don't use `state_group`, its there so that we can cache based
	256	# on it. However, its important that its never None, since two current_state's
	257	# with a state_group of None are likely to be different.
	258	# See bulk_get_push_rules_for_room for how we work around this.
	259	assert state_group is not None
	260
	261	# We also will want to generate notifs for other people in the room so
	262	# their unread countss are correct in the event stream, but to avoid
	263	# generating them for bot / AS users etc, we only do so for people who've
	264	# sent a read receipt into the room.
	265
	266	users_in_room = yield self._get_joined_users_from_context(
	267	room_id,
	268	state_group,
	269	current_state_ids,
	270	on_invalidate=cache_context.invalidate,
	271	event=event,
	272	)
	273
	274	# We ignore app service users for now. This is so that we don't fill
	275	# up the `get_if_users_have_pushers` cache with AS entries that we
	276	# know don't have pushers, nor even read receipts.
	277	local_users_in_room = set(
	278	u
	279	for u in users_in_room
	280	if self.hs.is_mine_id(u)
	281	and not self.get_if_app_services_interested_in_user(u)
	282	)
	283
	284	# users in the room who have pushers need to get push rules run because
	285	# that's how their pushers work
	286	if_users_with_pushers = yield self.get_if_users_have_pushers(
	287	local_users_in_room, on_invalidate=cache_context.invalidate
	288	)
	289	user_ids = set(
	290	uid for uid, have_pusher in if_users_with_pushers.items() if have_pusher
	291	)
	292
	293	users_with_receipts = yield self.get_users_with_read_receipts_in_room(
	294	room_id, on_invalidate=cache_context.invalidate
	295	)
	296
	297	# any users with pushers must be ours: they have pushers
	298	for uid in users_with_receipts:
	299	if uid in local_users_in_room:
	300	user_ids.add(uid)
	301
	302	rules_by_user = yield self.bulk_get_push_rules(
	303	user_ids, on_invalidate=cache_context.invalidate
	304	)
	305
	306	rules_by_user = {k: v for k, v in rules_by_user.items() if v is not None}
	307
	308	return rules_by_user
	309
	310	@cachedList(
	311	cached_method_name="get_push_rules_enabled_for_user",
	312	list_name="user_ids",
	313	num_args=1,
	314	inlineCallbacks=True,
	315	)
	316	def bulk_get_push_rules_enabled(self, user_ids):
	317	if not user_ids:
	318	return {}
	319
	320	results = {user_id: {} for user_id in user_ids}
	321
	322	rows = yield self._simple_select_many_batch(
	323	table="push_rules_enable",
	324	column="user_name",
	325	iterable=user_ids,
	326	retcols=("user_name", "rule_id", "enabled"),
	327	desc="bulk_get_push_rules_enabled",
	328	)
	329	for row in rows:
	330	enabled = bool(row["enabled"])
	331	results.setdefault(row["user_name"], {})[row["rule_id"]] = enabled
	332	return results
	333
	334
	335	class PushRuleStore(PushRulesWorkerStore):
	336	@defer.inlineCallbacks
	337	def add_push_rule(
	338	self,
	339	user_id,
	340	rule_id,
	341	priority_class,
	342	conditions,
	343	actions,
	344	before=None,
	345	after=None,
	346	):
	347	conditions_json = json.dumps(conditions)
	348	actions_json = json.dumps(actions)
	349	with self._push_rules_stream_id_gen.get_next() as ids:
	350	stream_id, event_stream_ordering = ids
	351	if before or after:
	352	yield self.runInteraction(
	353	"_add_push_rule_relative_txn",
	354	self._add_push_rule_relative_txn,
	355	stream_id,
	356	event_stream_ordering,
	357	user_id,
	358	rule_id,
	359	priority_class,
	360	conditions_json,
	361	actions_json,
	362	before,
	363	after,
	364	)
	365	else:
	366	yield self.runInteraction(
	367	"_add_push_rule_highest_priority_txn",
	368	self._add_push_rule_highest_priority_txn,
	369	stream_id,
	370	event_stream_ordering,
	371	user_id,
	372	rule_id,
	373	priority_class,
	374	conditions_json,
	375	actions_json,
	376	)
	377
	378	def _add_push_rule_relative_txn(
	379	self,
	380	txn,
	381	stream_id,
	382	event_stream_ordering,
	383	user_id,
	384	rule_id,
	385	priority_class,
	386	conditions_json,
	387	actions_json,
	388	before,
	389	after,
	390	):
	391	# Lock the table since otherwise we'll have annoying races between the
	392	# SELECT here and the UPSERT below.
	393	self.database_engine.lock_table(txn, "push_rules")
	394
	395	relative_to_rule = before or after
	396
	397	res = self._simple_select_one_txn(
	398	txn,
	399	table="push_rules",
	400	keyvalues={"user_name": user_id, "rule_id": relative_to_rule},
	401	retcols=["priority_class", "priority"],
	402	allow_none=True,
	403	)
	404
	405	if not res:
	406	raise RuleNotFoundException(
	407	"before/after rule not found: %s" % (relative_to_rule,)
	408	)
	409
	410	base_priority_class = res["priority_class"]
	411	base_rule_priority = res["priority"]
	412
	413	if base_priority_class != priority_class:
	414	raise InconsistentRuleException(
	415	"Given priority class does not match class of relative rule"
	416	)
	417
	418	if before:
	419	# Higher priority rules are executed first, So adding a rule before
	420	# a rule means giving it a higher priority than that rule.
	421	new_rule_priority = base_rule_priority + 1
	422	else:
	423	# We increment the priority of the existing rules to make space for
	424	# the new rule. Therefore if we want this rule to appear after
	425	# an existing rule we give it the priority of the existing rule,
	426	# and then increment the priority of the existing rule.
	427	new_rule_priority = base_rule_priority
	428
	429	sql = (
	430	"UPDATE push_rules SET priority = priority + 1"
	431	" WHERE user_name = ? AND priority_class = ? AND priority >= ?"
	432	)
	433
	434	txn.execute(sql, (user_id, priority_class, new_rule_priority))
	435
	436	self._upsert_push_rule_txn(
	437	txn,
	438	stream_id,
	439	event_stream_ordering,
	440	user_id,
	441	rule_id,
	442	priority_class,
	443	new_rule_priority,
	444	conditions_json,
	445	actions_json,
	446	)
	447
	448	def _add_push_rule_highest_priority_txn(
	449	self,
	450	txn,
	451	stream_id,
	452	event_stream_ordering,
	453	user_id,
	454	rule_id,
	455	priority_class,
	456	conditions_json,
	457	actions_json,
	458	):
	459	# Lock the table since otherwise we'll have annoying races between the
	460	# SELECT here and the UPSERT below.
	461	self.database_engine.lock_table(txn, "push_rules")
	462
	463	# find the highest priority rule in that class
	464	sql = (
	465	"SELECT COUNT(*), MAX(priority) FROM push_rules"
	466	" WHERE user_name = ? and priority_class = ?"
	467	)
	468	txn.execute(sql, (user_id, priority_class))
	469	res = txn.fetchall()
	470	(how_many, highest_prio) = res[0]
	471
	472	new_prio = 0
	473	if how_many > 0:
	474	new_prio = highest_prio + 1
	475
	476	self._upsert_push_rule_txn(
	477	txn,
	478	stream_id,
	479	event_stream_ordering,
	480	user_id,
	481	rule_id,
	482	priority_class,
	483	new_prio,
	484	conditions_json,
	485	actions_json,
	486	)
	487
	488	def _upsert_push_rule_txn(
	489	self,
	490	txn,
	491	stream_id,
	492	event_stream_ordering,
	493	user_id,
	494	rule_id,
	495	priority_class,
	496	priority,
	497	conditions_json,
	498	actions_json,
	499	update_stream=True,
	500	):
	501	"""Specialised version of _simple_upsert_txn that picks a push_rule_id
	502	using the _push_rule_id_gen if it needs to insert the rule. It assumes
	503	that the "push_rules" table is locked"""
	504
	505	sql = (
	506	"UPDATE push_rules"
	507	" SET priority_class = ?, priority = ?, conditions = ?, actions = ?"
	508	" WHERE user_name = ? AND rule_id = ?"
	509	)
	510
	511	txn.execute(
	512	sql,
	513	(priority_class, priority, conditions_json, actions_json, user_id, rule_id),
	514	)
	515
	516	if txn.rowcount == 0:
	517	# We didn't update a row with the given rule_id so insert one
	518	push_rule_id = self._push_rule_id_gen.get_next()
	519
	520	self._simple_insert_txn(
	521	txn,
	522	table="push_rules",
	523	values={
	524	"id": push_rule_id,
	525	"user_name": user_id,
	526	"rule_id": rule_id,
	527	"priority_class": priority_class,
	528	"priority": priority,
	529	"conditions": conditions_json,
	530	"actions": actions_json,
	531	},
	532	)
	533
	534	if update_stream:
	535	self._insert_push_rules_update_txn(
	536	txn,
	537	stream_id,
	538	event_stream_ordering,
	539	user_id,
	540	rule_id,
	541	op="ADD",
	542	data={
	543	"priority_class": priority_class,
	544	"priority": priority,
	545	"conditions": conditions_json,
	546	"actions": actions_json,
	547	},
	548	)
	549
	550	@defer.inlineCallbacks
	551	def delete_push_rule(self, user_id, rule_id):
	552	"""
	553	Delete a push rule. Args specify the row to be deleted and can be
	554	any of the columns in the push_rule table, but below are the
	555	standard ones
	556
	557	Args:
	558	user_id (str): The matrix ID of the push rule owner
	559	rule_id (str): The rule_id of the rule to be deleted
	560	"""
	561
	562	def delete_push_rule_txn(txn, stream_id, event_stream_ordering):
	563	self._simple_delete_one_txn(
	564	txn, "push_rules", {"user_name": user_id, "rule_id": rule_id}
	565	)
	566
	567	self._insert_push_rules_update_txn(
	568	txn, stream_id, event_stream_ordering, user_id, rule_id, op="DELETE"
	569	)
	570
	571	with self._push_rules_stream_id_gen.get_next() as ids:
	572	stream_id, event_stream_ordering = ids
	573	yield self.runInteraction(
	574	"delete_push_rule",
	575	delete_push_rule_txn,
	576	stream_id,
	577	event_stream_ordering,
	578	)
	579
	580	@defer.inlineCallbacks
	581	def set_push_rule_enabled(self, user_id, rule_id, enabled):
	582	with self._push_rules_stream_id_gen.get_next() as ids:
	583	stream_id, event_stream_ordering = ids
	584	yield self.runInteraction(
	585	"_set_push_rule_enabled_txn",
	586	self._set_push_rule_enabled_txn,
	587	stream_id,
	588	event_stream_ordering,
	589	user_id,
	590	rule_id,
	591	enabled,
	592	)
	593
	594	def _set_push_rule_enabled_txn(
	595	self, txn, stream_id, event_stream_ordering, user_id, rule_id, enabled
	596	):
	597	new_id = self._push_rules_enable_id_gen.get_next()
	598	self._simple_upsert_txn(
	599	txn,
	600	"push_rules_enable",
	601	{"user_name": user_id, "rule_id": rule_id},
	602	{"enabled": 1 if enabled else 0},
	603	{"id": new_id},
	604	)
	605
	606	self._insert_push_rules_update_txn(
	607	txn,
	608	stream_id,
	609	event_stream_ordering,
	610	user_id,
	611	rule_id,
	612	op="ENABLE" if enabled else "DISABLE",
	613	)
	614
	615	@defer.inlineCallbacks
	616	def set_push_rule_actions(self, user_id, rule_id, actions, is_default_rule):
	617	actions_json = json.dumps(actions)
	618
	619	def set_push_rule_actions_txn(txn, stream_id, event_stream_ordering):
	620	if is_default_rule:
	621	# Add a dummy rule to the rules table with the user specified
	622	# actions.
	623	priority_class = -1
	624	priority = 1
	625	self._upsert_push_rule_txn(
	626	txn,
	627	stream_id,
	628	event_stream_ordering,
	629	user_id,
	630	rule_id,
	631	priority_class,
	632	priority,
	633	"[]",
	634	actions_json,
	635	update_stream=False,
	636	)
	637	else:
	638	self._simple_update_one_txn(
	639	txn,
	640	"push_rules",
	641	{"user_name": user_id, "rule_id": rule_id},
	642	{"actions": actions_json},
	643	)
	644
	645	self._insert_push_rules_update_txn(
	646	txn,
	647	stream_id,
	648	event_stream_ordering,
	649	user_id,
	650	rule_id,
	651	op="ACTIONS",
	652	data={"actions": actions_json},
	653	)
	654
	655	with self._push_rules_stream_id_gen.get_next() as ids:
	656	stream_id, event_stream_ordering = ids
	657	yield self.runInteraction(
	658	"set_push_rule_actions",
	659	set_push_rule_actions_txn,
	660	stream_id,
	661	event_stream_ordering,
	662	)
	663
	664	def _insert_push_rules_update_txn(
	665	self, txn, stream_id, event_stream_ordering, user_id, rule_id, op, data=None
	666	):
	667	values = {
	668	"stream_id": stream_id,
	669	"event_stream_ordering": event_stream_ordering,
	670	"user_id": user_id,
	671	"rule_id": rule_id,
	672	"op": op,
	673	}
	674	if data is not None:
	675	values.update(data)
	676
	677	self._simple_insert_txn(txn, "push_rules_stream", values=values)
	678
	679	txn.call_after(self.get_push_rules_for_user.invalidate, (user_id,))
	680	txn.call_after(self.get_push_rules_enabled_for_user.invalidate, (user_id,))
	681	txn.call_after(
	682	self.push_rules_stream_cache.entity_has_changed, user_id, stream_id
	683	)
	684
	685	def get_all_push_rule_updates(self, last_id, current_id, limit):
	686	"""Get all the push rules changes that have happend on the server"""
	687	if last_id == current_id:
	688	return defer.succeed([])
	689
	690	def get_all_push_rule_updates_txn(txn):
	691	sql = (
	692	"SELECT stream_id, event_stream_ordering, user_id, rule_id,"
	693	" op, priority_class, priority, conditions, actions"
	694	" FROM push_rules_stream"
	695	" WHERE ? < stream_id AND stream_id <= ?"
	696	" ORDER BY stream_id ASC LIMIT ?"
	697	)
	698	txn.execute(sql, (last_id, current_id, limit))
	699	return txn.fetchall()
	700
	701	return self.runInteraction(
	702	"get_all_push_rule_updates", get_all_push_rule_updates_txn
	703	)
	704
	705	def get_push_rules_stream_token(self):
	706	"""Get the position of the push rules stream.
	707	Returns a pair of a stream id for the push_rules stream and the
	708	room stream ordering it corresponds to."""
	709	return self._push_rules_stream_id_gen.get_current_token()
	710
	711	def get_max_push_rules_stream_id(self):
	712	return self.get_push_rules_stream_token()[0]

+371

-0

synapse/storage/data_stores/main/pusher.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2014-2016 OpenMarket Ltd
	2	# Copyright 2018 New Vector Ltd
	3	#
	4	# Licensed under the Apache License, Version 2.0 (the "License");
	5	# you may not use this file except in compliance with the License.
	6	# You may obtain a copy of the License at
	7	#
	8	# http://www.apache.org/licenses/LICENSE-2.0
	9	#
	10	# Unless required by applicable law or agreed to in writing, software
	11	# distributed under the License is distributed on an "AS IS" BASIS,
	12	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	# See the License for the specific language governing permissions and
	14	# limitations under the License.
	15
	16	import logging
	17
	18	import six
	19
	20	from canonicaljson import encode_canonical_json, json
	21
	22	from twisted.internet import defer
	23
	24	from synapse.storage._base import SQLBaseStore
	25	from synapse.util.caches.descriptors import cachedInlineCallbacks, cachedList
	26
	27	logger = logging.getLogger(__name__)
	28
	29	if six.PY2:
	30	db_binary_type = six.moves.builtins.buffer
	31	else:
	32	db_binary_type = memoryview
	33
	34
	35	class PusherWorkerStore(SQLBaseStore):
	36	def _decode_pushers_rows(self, rows):
	37	for r in rows:
	38	dataJson = r["data"]
	39	r["data"] = None
	40	try:
	41	if isinstance(dataJson, db_binary_type):
	42	dataJson = str(dataJson).decode("UTF8")
	43
	44	r["data"] = json.loads(dataJson)
	45	except Exception as e:
	46	logger.warn(
	47	"Invalid JSON in data for pusher %d: %s, %s",
	48	r["id"],
	49	dataJson,
	50	e.args[0],
	51	)
	52	pass
	53
	54	if isinstance(r["pushkey"], db_binary_type):
	55	r["pushkey"] = str(r["pushkey"]).decode("UTF8")
	56
	57	return rows
	58
	59	@defer.inlineCallbacks
	60	def user_has_pusher(self, user_id):
	61	ret = yield self._simple_select_one_onecol(
	62	"pushers", {"user_name": user_id}, "id", allow_none=True
	63	)
	64	return ret is not None
	65
	66	def get_pushers_by_app_id_and_pushkey(self, app_id, pushkey):
	67	return self.get_pushers_by({"app_id": app_id, "pushkey": pushkey})
	68
	69	def get_pushers_by_user_id(self, user_id):
	70	return self.get_pushers_by({"user_name": user_id})
	71
	72	@defer.inlineCallbacks
	73	def get_pushers_by(self, keyvalues):
	74	ret = yield self._simple_select_list(
	75	"pushers",
	76	keyvalues,
	77	[
	78	"id",
	79	"user_name",
	80	"access_token",
	81	"profile_tag",
	82	"kind",
	83	"app_id",
	84	"app_display_name",
	85	"device_display_name",
	86	"pushkey",
	87	"ts",
	88	"lang",
	89	"data",
	90	"last_stream_ordering",
	91	"last_success",
	92	"failing_since",
	93	],
	94	desc="get_pushers_by",
	95	)
	96	return self._decode_pushers_rows(ret)
	97
	98	@defer.inlineCallbacks
	99	def get_all_pushers(self):
	100	def get_pushers(txn):
	101	txn.execute("SELECT * FROM pushers")
	102	rows = self.cursor_to_dict(txn)
	103
	104	return self._decode_pushers_rows(rows)
	105
	106	rows = yield self.runInteraction("get_all_pushers", get_pushers)
	107	return rows
	108
	109	def get_all_updated_pushers(self, last_id, current_id, limit):
	110	if last_id == current_id:
	111	return defer.succeed(([], []))
	112
	113	def get_all_updated_pushers_txn(txn):
	114	sql = (
	115	"SELECT id, user_name, access_token, profile_tag, kind,"
	116	" app_id, app_display_name, device_display_name, pushkey, ts,"
	117	" lang, data"
	118	" FROM pushers"
	119	" WHERE ? < id AND id <= ?"
	120	" ORDER BY id ASC LIMIT ?"
	121	)
	122	txn.execute(sql, (last_id, current_id, limit))
	123	updated = txn.fetchall()
	124
	125	sql = (
	126	"SELECT stream_id, user_id, app_id, pushkey"
	127	" FROM deleted_pushers"
	128	" WHERE ? < stream_id AND stream_id <= ?"
	129	" ORDER BY stream_id ASC LIMIT ?"
	130	)
	131	txn.execute(sql, (last_id, current_id, limit))
	132	deleted = txn.fetchall()
	133
	134	return updated, deleted
	135
	136	return self.runInteraction(
	137	"get_all_updated_pushers", get_all_updated_pushers_txn
	138	)
	139
	140	def get_all_updated_pushers_rows(self, last_id, current_id, limit):
	141	"""Get all the pushers that have changed between the given tokens.
	142
	143	Returns:
	144	Deferred(list(tuple)): each tuple consists of:
	145	stream_id (str)
	146	user_id (str)
	147	app_id (str)
	148	pushkey (str)
	149	was_deleted (bool): whether the pusher was added/updated (False)
	150	or deleted (True)
	151	"""
	152
	153	if last_id == current_id:
	154	return defer.succeed([])
	155
	156	def get_all_updated_pushers_rows_txn(txn):
	157	sql = (
	158	"SELECT id, user_name, app_id, pushkey"
	159	" FROM pushers"
	160	" WHERE ? < id AND id <= ?"
	161	" ORDER BY id ASC LIMIT ?"
	162	)
	163	txn.execute(sql, (last_id, current_id, limit))
	164	results = [list(row) + [False] for row in txn]
	165
	166	sql = (
	167	"SELECT stream_id, user_id, app_id, pushkey"
	168	" FROM deleted_pushers"
	169	" WHERE ? < stream_id AND stream_id <= ?"
	170	" ORDER BY stream_id ASC LIMIT ?"
	171	)
	172	txn.execute(sql, (last_id, current_id, limit))
	173
	174	results.extend(list(row) + [True] for row in txn)
	175	results.sort() # Sort so that they're ordered by stream id
	176
	177	return results
	178
	179	return self.runInteraction(
	180	"get_all_updated_pushers_rows", get_all_updated_pushers_rows_txn
	181	)
	182
	183	@cachedInlineCallbacks(num_args=1, max_entries=15000)
	184	def get_if_user_has_pusher(self, user_id):
	185	# This only exists for the cachedList decorator
	186	raise NotImplementedError()
	187
	188	@cachedList(
	189	cached_method_name="get_if_user_has_pusher",
	190	list_name="user_ids",
	191	num_args=1,
	192	inlineCallbacks=True,
	193	)
	194	def get_if_users_have_pushers(self, user_ids):
	195	rows = yield self._simple_select_many_batch(
	196	table="pushers",
	197	column="user_name",
	198	iterable=user_ids,
	199	retcols=["user_name"],
	200	desc="get_if_users_have_pushers",
	201	)
	202
	203	result = {user_id: False for user_id in user_ids}
	204	result.update({r["user_name"]: True for r in rows})
	205
	206	return result
	207
	208
	209	class PusherStore(PusherWorkerStore):
	210	def get_pushers_stream_token(self):
	211	return self._pushers_id_gen.get_current_token()
	212
	213	@defer.inlineCallbacks
	214	def add_pusher(
	215	self,
	216	user_id,
	217	access_token,
	218	kind,
	219	app_id,
	220	app_display_name,
	221	device_display_name,
	222	pushkey,
	223	pushkey_ts,
	224	lang,
	225	data,
	226	last_stream_ordering,
	227	profile_tag="",
	228	):
	229	with self._pushers_id_gen.get_next() as stream_id:
	230	# no need to lock because `pushers` has a unique key on
	231	# (app_id, pushkey, user_name) so _simple_upsert will retry
	232	yield self._simple_upsert(
	233	table="pushers",
	234	keyvalues={"app_id": app_id, "pushkey": pushkey, "user_name": user_id},
	235	values={
	236	"access_token": access_token,
	237	"kind": kind,
	238	"app_display_name": app_display_name,
	239	"device_display_name": device_display_name,
	240	"ts": pushkey_ts,
	241	"lang": lang,
	242	"data": bytearray(encode_canonical_json(data)),
	243	"last_stream_ordering": last_stream_ordering,
	244	"profile_tag": profile_tag,
	245	"id": stream_id,
	246	},
	247	desc="add_pusher",
	248	lock=False,
	249	)
	250
	251	user_has_pusher = self.get_if_user_has_pusher.cache.get(
	252	(user_id,), None, update_metrics=False
	253	)
	254
	255	if user_has_pusher is not True:
	256	# invalidate, since we the user might not have had a pusher before
	257	yield self.runInteraction(
	258	"add_pusher",
	259	self._invalidate_cache_and_stream,
	260	self.get_if_user_has_pusher,
	261	(user_id,),
	262	)
	263
	264	@defer.inlineCallbacks
	265	def delete_pusher_by_app_id_pushkey_user_id(self, app_id, pushkey, user_id):
	266	def delete_pusher_txn(txn, stream_id):
	267	self._invalidate_cache_and_stream(
	268	txn, self.get_if_user_has_pusher, (user_id,)
	269	)
	270
	271	self._simple_delete_one_txn(
	272	txn,
	273	"pushers",
	274	{"app_id": app_id, "pushkey": pushkey, "user_name": user_id},
	275	)
	276
	277	# it's possible for us to end up with duplicate rows for
	278	# (app_id, pushkey, user_id) at different stream_ids, but that
	279	# doesn't really matter.
	280	self._simple_insert_txn(
	281	txn,
	282	table="deleted_pushers",
	283	values={
	284	"stream_id": stream_id,
	285	"app_id": app_id,
	286	"pushkey": pushkey,
	287	"user_id": user_id,
	288	},
	289	)
	290
	291	with self._pushers_id_gen.get_next() as stream_id:
	292	yield self.runInteraction("delete_pusher", delete_pusher_txn, stream_id)
	293
	294	@defer.inlineCallbacks
	295	def update_pusher_last_stream_ordering(
	296	self, app_id, pushkey, user_id, last_stream_ordering
	297	):
	298	yield self._simple_update_one(
	299	"pushers",
	300	{"app_id": app_id, "pushkey": pushkey, "user_name": user_id},
	301	{"last_stream_ordering": last_stream_ordering},
	302	desc="update_pusher_last_stream_ordering",
	303	)
	304
	305	@defer.inlineCallbacks
	306	def update_pusher_last_stream_ordering_and_success(
	307	self, app_id, pushkey, user_id, last_stream_ordering, last_success
	308	):
	309	"""Update the last stream ordering position we've processed up to for
	310	the given pusher.
	311
	312	Args:
	313	app_id (str)
	314	pushkey (str)
	315	last_stream_ordering (int)
	316	last_success (int)
	317
	318	Returns:
	319	Deferred[bool]: True if the pusher still exists; False if it has been deleted.
	320	"""
	321	updated = yield self._simple_update(
	322	table="pushers",
	323	keyvalues={"app_id": app_id, "pushkey": pushkey, "user_name": user_id},
	324	updatevalues={
	325	"last_stream_ordering": last_stream_ordering,
	326	"last_success": last_success,
	327	},
	328	desc="update_pusher_last_stream_ordering_and_success",
	329	)
	330
	331	return bool(updated)
	332
	333	@defer.inlineCallbacks
	334	def update_pusher_failing_since(self, app_id, pushkey, user_id, failing_since):
	335	yield self._simple_update(
	336	table="pushers",
	337	keyvalues={"app_id": app_id, "pushkey": pushkey, "user_name": user_id},
	338	updatevalues={"failing_since": failing_since},
	339	desc="update_pusher_failing_since",
	340	)
	341
	342	@defer.inlineCallbacks
	343	def get_throttle_params_by_room(self, pusher_id):
	344	res = yield self._simple_select_list(
	345	"pusher_throttle",
	346	{"pusher": pusher_id},
	347	["room_id", "last_sent_ts", "throttle_ms"],
	348	desc="get_throttle_params_by_room",
	349	)
	350
	351	params_by_room = {}
	352	for row in res:
	353	params_by_room[row["room_id"]] = {
	354	"last_sent_ts": row["last_sent_ts"],
	355	"throttle_ms": row["throttle_ms"],
	356	}
	357
	358	return params_by_room
	359
	360	@defer.inlineCallbacks
	361	def set_throttle_params(self, pusher_id, room_id, params):
	362	# no need to lock because `pusher_throttle` has a primary key on
	363	# (pusher, room_id) so _simple_upsert will retry
	364	yield self._simple_upsert(
	365	"pusher_throttle",
	366	{"pusher": pusher_id, "room_id": room_id},
	367	params,
	368	desc="set_throttle_params",
	369	lock=False,
	370	)

+536

-0

synapse/storage/data_stores/main/receipts.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2014-2016 OpenMarket Ltd
	2	# Copyright 2018 New Vector Ltd
	3	#
	4	# Licensed under the Apache License, Version 2.0 (the "License");
	5	# you may not use this file except in compliance with the License.
	6	# You may obtain a copy of the License at
	7	#
	8	# http://www.apache.org/licenses/LICENSE-2.0
	9	#
	10	# Unless required by applicable law or agreed to in writing, software
	11	# distributed under the License is distributed on an "AS IS" BASIS,
	12	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	# See the License for the specific language governing permissions and
	14	# limitations under the License.
	15
	16	import abc
	17	import logging
	18
	19	from canonicaljson import json
	20
	21	from twisted.internet import defer
	22
	23	from synapse.storage._base import SQLBaseStore, make_in_list_sql_clause
	24	from synapse.storage.util.id_generators import StreamIdGenerator
	25	from synapse.util.caches.descriptors import cached, cachedInlineCallbacks, cachedList
	26	from synapse.util.caches.stream_change_cache import StreamChangeCache
	27
	28	logger = logging.getLogger(__name__)
	29
	30
	31	class ReceiptsWorkerStore(SQLBaseStore):
	32	"""This is an abstract base class where subclasses must implement
	33	`get_max_receipt_stream_id` which can be called in the initializer.
	34	"""
	35
	36	# This ABCMeta metaclass ensures that we cannot be instantiated without
	37	# the abstract methods being implemented.
	38	__metaclass__ = abc.ABCMeta
	39
	40	def __init__(self, db_conn, hs):
	41	super(ReceiptsWorkerStore, self).__init__(db_conn, hs)
	42
	43	self._receipts_stream_cache = StreamChangeCache(
	44	"ReceiptsRoomChangeCache", self.get_max_receipt_stream_id()
	45	)
	46
	47	@abc.abstractmethod
	48	def get_max_receipt_stream_id(self):
	49	"""Get the current max stream ID for receipts stream
	50
	51	Returns:
	52	int
	53	"""
	54	raise NotImplementedError()
	55
	56	@cachedInlineCallbacks()
	57	def get_users_with_read_receipts_in_room(self, room_id):
	58	receipts = yield self.get_receipts_for_room(room_id, "m.read")
	59	return set(r["user_id"] for r in receipts)
	60
	61	@cached(num_args=2)
	62	def get_receipts_for_room(self, room_id, receipt_type):
	63	return self._simple_select_list(
	64	table="receipts_linearized",
	65	keyvalues={"room_id": room_id, "receipt_type": receipt_type},
	66	retcols=("user_id", "event_id"),
	67	desc="get_receipts_for_room",
	68	)
	69
	70	@cached(num_args=3)
	71	def get_last_receipt_event_id_for_user(self, user_id, room_id, receipt_type):
	72	return self._simple_select_one_onecol(
	73	table="receipts_linearized",
	74	keyvalues={
	75	"room_id": room_id,
	76	"receipt_type": receipt_type,
	77	"user_id": user_id,
	78	},
	79	retcol="event_id",
	80	desc="get_own_receipt_for_user",
	81	allow_none=True,
	82	)
	83
	84	@cachedInlineCallbacks(num_args=2)
	85	def get_receipts_for_user(self, user_id, receipt_type):
	86	rows = yield self._simple_select_list(
	87	table="receipts_linearized",
	88	keyvalues={"user_id": user_id, "receipt_type": receipt_type},
	89	retcols=("room_id", "event_id"),
	90	desc="get_receipts_for_user",
	91	)
	92
	93	return {row["room_id"]: row["event_id"] for row in rows}
	94
	95	@defer.inlineCallbacks
	96	def get_receipts_for_user_with_orderings(self, user_id, receipt_type):
	97	def f(txn):
	98	sql = (
	99	"SELECT rl.room_id, rl.event_id,"
	100	" e.topological_ordering, e.stream_ordering"
	101	" FROM receipts_linearized AS rl"
	102	" INNER JOIN events AS e USING (room_id, event_id)"
	103	" WHERE rl.room_id = e.room_id"
	104	" AND rl.event_id = e.event_id"
	105	" AND user_id = ?"
	106	)
	107	txn.execute(sql, (user_id,))
	108	return txn.fetchall()
	109
	110	rows = yield self.runInteraction("get_receipts_for_user_with_orderings", f)
	111	return {
	112	row[0]: {
	113	"event_id": row[1],
	114	"topological_ordering": row[2],
	115	"stream_ordering": row[3],
	116	}
	117	for row in rows
	118	}
	119
	120	@defer.inlineCallbacks
	121	def get_linearized_receipts_for_rooms(self, room_ids, to_key, from_key=None):
	122	"""Get receipts for multiple rooms for sending to clients.
	123
	124	Args:
	125	room_ids (list): List of room_ids.
	126	to_key (int): Max stream id to fetch receipts upto.
	127	from_key (int): Min stream id to fetch receipts from. None fetches
	128	from the start.
	129
	130	Returns:
	131	list: A list of receipts.
	132	"""
	133	room_ids = set(room_ids)
	134
	135	if from_key is not None:
	136	# Only ask the database about rooms where there have been new
	137	# receipts added since `from_key`
	138	room_ids = yield self._receipts_stream_cache.get_entities_changed(
	139	room_ids, from_key
	140	)
	141
	142	results = yield self._get_linearized_receipts_for_rooms(
	143	room_ids, to_key, from_key=from_key
	144	)
	145
	146	return [ev for res in results.values() for ev in res]
	147
	148	def get_linearized_receipts_for_room(self, room_id, to_key, from_key=None):
	149	"""Get receipts for a single room for sending to clients.
	150
	151	Args:
	152	room_ids (str): The room id.
	153	to_key (int): Max stream id to fetch receipts upto.
	154	from_key (int): Min stream id to fetch receipts from. None fetches
	155	from the start.
	156
	157	Returns:
	158	Deferred[list]: A list of receipts.
	159	"""
	160	if from_key is not None:
	161	# Check the cache first to see if any new receipts have been added
	162	# since`from_key`. If not we can no-op.
	163	if not self._receipts_stream_cache.has_entity_changed(room_id, from_key):
	164	defer.succeed([])
	165
	166	return self._get_linearized_receipts_for_room(room_id, to_key, from_key)
	167
	168	@cachedInlineCallbacks(num_args=3, tree=True)
	169	def _get_linearized_receipts_for_room(self, room_id, to_key, from_key=None):
	170	"""See get_linearized_receipts_for_room
	171	"""
	172
	173	def f(txn):
	174	if from_key:
	175	sql = (
	176	"SELECT * FROM receipts_linearized WHERE"
	177	" room_id = ? AND stream_id > ? AND stream_id <= ?"
	178	)
	179
	180	txn.execute(sql, (room_id, from_key, to_key))
	181	else:
	182	sql = (
	183	"SELECT * FROM receipts_linearized WHERE"
	184	" room_id = ? AND stream_id <= ?"
	185	)
	186
	187	txn.execute(sql, (room_id, to_key))
	188
	189	rows = self.cursor_to_dict(txn)
	190
	191	return rows
	192
	193	rows = yield self.runInteraction("get_linearized_receipts_for_room", f)
	194
	195	if not rows:
	196	return []
	197
	198	content = {}
	199	for row in rows:
	200	content.setdefault(row["event_id"], {}).setdefault(row["receipt_type"], {})[
	201	row["user_id"]
	202	] = json.loads(row["data"])
	203
	204	return [{"type": "m.receipt", "room_id": room_id, "content": content}]
	205
	206	@cachedList(
	207	cached_method_name="_get_linearized_receipts_for_room",
	208	list_name="room_ids",
	209	num_args=3,
	210	inlineCallbacks=True,
	211	)
	212	def _get_linearized_receipts_for_rooms(self, room_ids, to_key, from_key=None):
	213	if not room_ids:
	214	return {}
	215
	216	def f(txn):
	217	if from_key:
	218	sql = """
	219	SELECT * FROM receipts_linearized WHERE
	220	stream_id > ? AND stream_id <= ? AND
	221	"""
	222	clause, args = make_in_list_sql_clause(
	223	self.database_engine, "room_id", room_ids
	224	)
	225
	226	txn.execute(sql + clause, [from_key, to_key] + list(args))
	227	else:
	228	sql = """
	229	SELECT * FROM receipts_linearized WHERE
	230	stream_id <= ? AND
	231	"""
	232
	233	clause, args = make_in_list_sql_clause(
	234	self.database_engine, "room_id", room_ids
	235	)
	236
	237	txn.execute(sql + clause, [to_key] + list(args))
	238
	239	return self.cursor_to_dict(txn)
	240
	241	txn_results = yield self.runInteraction("_get_linearized_receipts_for_rooms", f)
	242
	243	results = {}
	244	for row in txn_results:
	245	# We want a single event per room, since we want to batch the
	246	# receipts by room, event and type.
	247	room_event = results.setdefault(
	248	row["room_id"],
	249	{"type": "m.receipt", "room_id": row["room_id"], "content": {}},
	250	)
	251
	252	# The content is of the form:
	253	# {"$foo:bar": { "read": { "@user:host": <receipt> }, .. }, .. }
	254	event_entry = room_event["content"].setdefault(row["event_id"], {})
	255	receipt_type = event_entry.setdefault(row["receipt_type"], {})
	256
	257	receipt_type[row["user_id"]] = json.loads(row["data"])
	258
	259	results = {
	260	room_id: [results[room_id]] if room_id in results else []
	261	for room_id in room_ids
	262	}
	263	return results
	264
	265	def get_all_updated_receipts(self, last_id, current_id, limit=None):
	266	if last_id == current_id:
	267	return defer.succeed([])
	268
	269	def get_all_updated_receipts_txn(txn):
	270	sql = (
	271	"SELECT stream_id, room_id, receipt_type, user_id, event_id, data"
	272	" FROM receipts_linearized"
	273	" WHERE ? < stream_id AND stream_id <= ?"
	274	" ORDER BY stream_id ASC"
	275	)
	276	args = [last_id, current_id]
	277	if limit is not None:
	278	sql += " LIMIT ?"
	279	args.append(limit)
	280	txn.execute(sql, args)
	281
	282	return (r[0:5] + (json.loads(r[5]),) for r in txn)
	283
	284	return self.runInteraction(
	285	"get_all_updated_receipts", get_all_updated_receipts_txn
	286	)
	287
	288	def _invalidate_get_users_with_receipts_in_room(
	289	self, room_id, receipt_type, user_id
	290	):
	291	if receipt_type != "m.read":
	292	return
	293
	294	# Returns either an ObservableDeferred or the raw result
	295	res = self.get_users_with_read_receipts_in_room.cache.get(
	296	room_id, None, update_metrics=False
	297	)
	298
	299	# first handle the Deferred case
	300	if isinstance(res, defer.Deferred):
	301	if res.called:
	302	res = res.result
	303	else:
	304	res = None
	305
	306	if res and user_id in res:
	307	# We'd only be adding to the set, so no point invalidating if the
	308	# user is already there
	309	return
	310
	311	self.get_users_with_read_receipts_in_room.invalidate((room_id,))
	312
	313
	314	class ReceiptsStore(ReceiptsWorkerStore):
	315	def __init__(self, db_conn, hs):
	316	# We instantiate this first as the ReceiptsWorkerStore constructor
	317	# needs to be able to call get_max_receipt_stream_id
	318	self._receipts_id_gen = StreamIdGenerator(
	319	db_conn, "receipts_linearized", "stream_id"
	320	)
	321
	322	super(ReceiptsStore, self).__init__(db_conn, hs)
	323
	324	def get_max_receipt_stream_id(self):
	325	return self._receipts_id_gen.get_current_token()
	326
	327	def insert_linearized_receipt_txn(
	328	self, txn, room_id, receipt_type, user_id, event_id, data, stream_id
	329	):
	330	"""Inserts a read-receipt into the database if it's newer than the current RR
	331
	332	Returns: int\|None
	333	None if the RR is older than the current RR
	334	otherwise, the rx timestamp of the event that the RR corresponds to
	335	(or 0 if the event is unknown)
	336	"""
	337	res = self._simple_select_one_txn(
	338	txn,
	339	table="events",
	340	retcols=["stream_ordering", "received_ts"],
	341	keyvalues={"event_id": event_id},
	342	allow_none=True,
	343	)
	344
	345	stream_ordering = int(res["stream_ordering"]) if res else None
	346	rx_ts = res["received_ts"] if res else 0
	347
	348	# We don't want to clobber receipts for more recent events, so we
	349	# have to compare orderings of existing receipts
	350	if stream_ordering is not None:
	351	sql = (
	352	"SELECT stream_ordering, event_id FROM events"
	353	" INNER JOIN receipts_linearized as r USING (event_id, room_id)"
	354	" WHERE r.room_id = ? AND r.receipt_type = ? AND r.user_id = ?"
	355	)
	356	txn.execute(sql, (room_id, receipt_type, user_id))
	357
	358	for so, eid in txn:
	359	if int(so) >= stream_ordering:
	360	logger.debug(
	361	"Ignoring new receipt for %s in favour of existing "
	362	"one for later event %s",
	363	event_id,
	364	eid,
	365	)
	366	return None
	367
	368	txn.call_after(self.get_receipts_for_room.invalidate, (room_id, receipt_type))
	369	txn.call_after(
	370	self._invalidate_get_users_with_receipts_in_room,
	371	room_id,
	372	receipt_type,
	373	user_id,
	374	)
	375	txn.call_after(self.get_receipts_for_user.invalidate, (user_id, receipt_type))
	376	# FIXME: This shouldn't invalidate the whole cache
	377	txn.call_after(
	378	self._get_linearized_receipts_for_room.invalidate_many, (room_id,)
	379	)
	380
	381	txn.call_after(
	382	self._receipts_stream_cache.entity_has_changed, room_id, stream_id
	383	)
	384
	385	txn.call_after(
	386	self.get_last_receipt_event_id_for_user.invalidate,
	387	(user_id, room_id, receipt_type),
	388	)
	389
	390	self._simple_delete_txn(
	391	txn,
	392	table="receipts_linearized",
	393	keyvalues={
	394	"room_id": room_id,
	395	"receipt_type": receipt_type,
	396	"user_id": user_id,
	397	},
	398	)
	399
	400	self._simple_insert_txn(
	401	txn,
	402	table="receipts_linearized",
	403	values={
	404	"stream_id": stream_id,
	405	"room_id": room_id,
	406	"receipt_type": receipt_type,
	407	"user_id": user_id,
	408	"event_id": event_id,
	409	"data": json.dumps(data),
	410	},
	411	)
	412
	413	if receipt_type == "m.read" and stream_ordering is not None:
	414	self._remove_old_push_actions_before_txn(
	415	txn, room_id=room_id, user_id=user_id, stream_ordering=stream_ordering
	416	)
	417
	418	return rx_ts
	419
	420	@defer.inlineCallbacks
	421	def insert_receipt(self, room_id, receipt_type, user_id, event_ids, data):
	422	"""Insert a receipt, either from local client or remote server.
	423
	424	Automatically does conversion between linearized and graph
	425	representations.
	426	"""
	427	if not event_ids:
	428	return
	429
	430	if len(event_ids) == 1:
	431	linearized_event_id = event_ids[0]
	432	else:
	433	# we need to points in graph -> linearized form.
	434	# TODO: Make this better.
	435	def graph_to_linear(txn):
	436	clause, args = make_in_list_sql_clause(
	437	self.database_engine, "event_id", event_ids
	438	)
	439
	440	sql = """
	441	SELECT event_id WHERE room_id = ? AND stream_ordering IN (
	442	SELECT max(stream_ordering) WHERE %s
	443	)
	444	""" % (
	445	clause,
	446	)
	447
	448	txn.execute(sql, [room_id] + list(args))
	449	rows = txn.fetchall()
	450	if rows:
	451	return rows[0][0]
	452	else:
	453	raise RuntimeError("Unrecognized event_ids: %r" % (event_ids,))
	454
	455	linearized_event_id = yield self.runInteraction(
	456	"insert_receipt_conv", graph_to_linear
	457	)
	458
	459	stream_id_manager = self._receipts_id_gen.get_next()
	460	with stream_id_manager as stream_id:
	461	event_ts = yield self.runInteraction(
	462	"insert_linearized_receipt",
	463	self.insert_linearized_receipt_txn,
	464	room_id,
	465	receipt_type,
	466	user_id,
	467	linearized_event_id,
	468	data,
	469	stream_id=stream_id,
	470	)
	471
	472	if event_ts is None:
	473	return None
	474
	475	now = self._clock.time_msec()
	476	logger.debug(
	477	"RR for event %s in %s (%i ms old)",
	478	linearized_event_id,
	479	room_id,
	480	now - event_ts,
	481	)
	482
	483	yield self.insert_graph_receipt(room_id, receipt_type, user_id, event_ids, data)
	484
	485	max_persisted_id = self._receipts_id_gen.get_current_token()
	486
	487	return stream_id, max_persisted_id
	488
	489	def insert_graph_receipt(self, room_id, receipt_type, user_id, event_ids, data):
	490	return self.runInteraction(
	491	"insert_graph_receipt",
	492	self.insert_graph_receipt_txn,
	493	room_id,
	494	receipt_type,
	495	user_id,
	496	event_ids,
	497	data,
	498	)
	499
	500	def insert_graph_receipt_txn(
	501	self, txn, room_id, receipt_type, user_id, event_ids, data
	502	):
	503	txn.call_after(self.get_receipts_for_room.invalidate, (room_id, receipt_type))
	504	txn.call_after(
	505	self._invalidate_get_users_with_receipts_in_room,
	506	room_id,
	507	receipt_type,
	508	user_id,
	509	)
	510	txn.call_after(self.get_receipts_for_user.invalidate, (user_id, receipt_type))
	511	# FIXME: This shouldn't invalidate the whole cache
	512	txn.call_after(
	513	self._get_linearized_receipts_for_room.invalidate_many, (room_id,)
	514	)
	515
	516	self._simple_delete_txn(
	517	txn,
	518	table="receipts_graph",
	519	keyvalues={
	520	"room_id": room_id,
	521	"receipt_type": receipt_type,
	522	"user_id": user_id,
	523	},
	524	)
	525	self._simple_insert_txn(
	526	txn,
	527	table="receipts_graph",
	528	values={
	529	"room_id": room_id,
	530	"receipt_type": receipt_type,
	531	"user_id": user_id,
	532	"event_ids": json.dumps(event_ids),
	533	"data": json.dumps(data),
	534	},
	535	)

+1499

-0

synapse/storage/data_stores/main/registration.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2014-2016 OpenMarket Ltd
	2	# Copyright 2017-2018 New Vector Ltd
	3	# Copyright 2019 The Matrix.org Foundation C.I.C.
	4	#
	5	# Licensed under the Apache License, Version 2.0 (the "License");
	6	# you may not use this file except in compliance with the License.
	7	# You may obtain a copy of the License at
	8	#
	9	# http://www.apache.org/licenses/LICENSE-2.0
	10	#
	11	# Unless required by applicable law or agreed to in writing, software
	12	# distributed under the License is distributed on an "AS IS" BASIS,
	13	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	14	# See the License for the specific language governing permissions and
	15	# limitations under the License.
	16
	17	import logging
	18	import re
	19
	20	from six import iterkeys
	21	from six.moves import range
	22
	23	from twisted.internet import defer
	24	from twisted.internet.defer import Deferred
	25
	26	from synapse.api.constants import UserTypes
	27	from synapse.api.errors import Codes, StoreError, SynapseError, ThreepidValidationError
	28	from synapse.metrics.background_process_metrics import run_as_background_process
	29	from synapse.storage import background_updates
	30	from synapse.storage._base import SQLBaseStore
	31	from synapse.types import UserID
	32	from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
	33
	34	THIRTY_MINUTES_IN_MS = 30 * 60 * 1000
	35
	36	logger = logging.getLogger(__name__)
	37
	38
	39	class RegistrationWorkerStore(SQLBaseStore):
	40	def __init__(self, db_conn, hs):
	41	super(RegistrationWorkerStore, self).__init__(db_conn, hs)
	42
	43	self.config = hs.config
	44	self.clock = hs.get_clock()
	45
	46	@cached()
	47	def get_user_by_id(self, user_id):
	48	return self._simple_select_one(
	49	table="users",
	50	keyvalues={"name": user_id},
	51	retcols=[
	52	"name",
	53	"password_hash",
	54	"is_guest",
	55	"consent_version",
	56	"consent_server_notice_sent",
	57	"appservice_id",
	58	"creation_ts",
	59	"user_type",
	60	],
	61	allow_none=True,
	62	desc="get_user_by_id",
	63	)
	64
	65	@defer.inlineCallbacks
	66	def is_trial_user(self, user_id):
	67	"""Checks if user is in the "trial" period, i.e. within the first
	68	N days of registration defined by `mau_trial_days` config
	69
	70	Args:
	71	user_id (str)
	72
	73	Returns:
	74	Deferred[bool]
	75	"""
	76
	77	info = yield self.get_user_by_id(user_id)
	78	if not info:
	79	return False
	80
	81	now = self.clock.time_msec()
	82	trial_duration_ms = self.config.mau_trial_days * 24 * 60 * 60 * 1000
	83	is_trial = (now - info["creation_ts"] * 1000) < trial_duration_ms
	84	return is_trial
	85
	86	@cached()
	87	def get_user_by_access_token(self, token):
	88	"""Get a user from the given access token.
	89
	90	Args:
	91	token (str): The access token of a user.
	92	Returns:
	93	defer.Deferred: None, if the token did not match, otherwise dict
	94	including the keys `name`, `is_guest`, `device_id`, `token_id`,
	95	`valid_until_ms`.
	96	"""
	97	return self.runInteraction(
	98	"get_user_by_access_token", self._query_for_auth, token
	99	)
	100
	101	@cachedInlineCallbacks()
	102	def get_expiration_ts_for_user(self, user_id):
	103	"""Get the expiration timestamp for the account bearing a given user ID.
	104
	105	Args:
	106	user_id (str): The ID of the user.
	107	Returns:
	108	defer.Deferred: None, if the account has no expiration timestamp,
	109	otherwise int representation of the timestamp (as a number of
	110	milliseconds since epoch).
	111	"""
	112	res = yield self._simple_select_one_onecol(
	113	table="account_validity",
	114	keyvalues={"user_id": user_id},
	115	retcol="expiration_ts_ms",
	116	allow_none=True,
	117	desc="get_expiration_ts_for_user",
	118	)
	119	return res
	120
	121	@defer.inlineCallbacks
	122	def set_account_validity_for_user(
	123	self, user_id, expiration_ts, email_sent, renewal_token=None
	124	):
	125	"""Updates the account validity properties of the given account, with the
	126	given values.
	127
	128	Args:
	129	user_id (str): ID of the account to update properties for.
	130	expiration_ts (int): New expiration date, as a timestamp in milliseconds
	131	since epoch.
	132	email_sent (bool): True means a renewal email has been sent for this
	133	account and there's no need to send another one for the current validity
	134	period.
	135	renewal_token (str): Renewal token the user can use to extend the validity
	136	of their account. Defaults to no token.
	137	"""
	138
	139	def set_account_validity_for_user_txn(txn):
	140	self._simple_update_txn(
	141	txn=txn,
	142	table="account_validity",
	143	keyvalues={"user_id": user_id},
	144	updatevalues={
	145	"expiration_ts_ms": expiration_ts,
	146	"email_sent": email_sent,
	147	"renewal_token": renewal_token,
	148	},
	149	)
	150	self._invalidate_cache_and_stream(
	151	txn, self.get_expiration_ts_for_user, (user_id,)
	152	)
	153
	154	yield self.runInteraction(
	155	"set_account_validity_for_user", set_account_validity_for_user_txn
	156	)
	157
	158	@defer.inlineCallbacks
	159	def set_renewal_token_for_user(self, user_id, renewal_token):
	160	"""Defines a renewal token for a given user.
	161
	162	Args:
	163	user_id (str): ID of the user to set the renewal token for.
	164	renewal_token (str): Random unique string that will be used to renew the
	165	user's account.
	166
	167	Raises:
	168	StoreError: The provided token is already set for another user.
	169	"""
	170	yield self._simple_update_one(
	171	table="account_validity",
	172	keyvalues={"user_id": user_id},
	173	updatevalues={"renewal_token": renewal_token},
	174	desc="set_renewal_token_for_user",
	175	)
	176
	177	@defer.inlineCallbacks
	178	def get_user_from_renewal_token(self, renewal_token):
	179	"""Get a user ID from a renewal token.
	180
	181	Args:
	182	renewal_token (str): The renewal token to perform the lookup with.
	183
	184	Returns:
	185	defer.Deferred[str]: The ID of the user to which the token belongs.
	186	"""
	187	res = yield self._simple_select_one_onecol(
	188	table="account_validity",
	189	keyvalues={"renewal_token": renewal_token},
	190	retcol="user_id",
	191	desc="get_user_from_renewal_token",
	192	)
	193
	194	return res
	195
	196	@defer.inlineCallbacks
	197	def get_renewal_token_for_user(self, user_id):
	198	"""Get the renewal token associated with a given user ID.
	199
	200	Args:
	201	user_id (str): The user ID to lookup a token for.
	202
	203	Returns:
	204	defer.Deferred[str]: The renewal token associated with this user ID.
	205	"""
	206	res = yield self._simple_select_one_onecol(
	207	table="account_validity",
	208	keyvalues={"user_id": user_id},
	209	retcol="renewal_token",
	210	desc="get_renewal_token_for_user",
	211	)
	212
	213	return res
	214
	215	@defer.inlineCallbacks
	216	def get_users_expiring_soon(self):
	217	"""Selects users whose account will expire in the [now, now + renew_at] time
	218	window (see configuration for account_validity for information on what renew_at
	219	refers to).
	220
	221	Returns:
	222	Deferred: Resolves to a list[dict[user_id (str), expiration_ts_ms (int)]]
	223	"""
	224
	225	def select_users_txn(txn, now_ms, renew_at):
	226	sql = (
	227	"SELECT user_id, expiration_ts_ms FROM account_validity"
	228	" WHERE email_sent = ? AND (expiration_ts_ms - ?) <= ?"
	229	)
	230	values = [False, now_ms, renew_at]
	231	txn.execute(sql, values)
	232	return self.cursor_to_dict(txn)
	233
	234	res = yield self.runInteraction(
	235	"get_users_expiring_soon",
	236	select_users_txn,
	237	self.clock.time_msec(),
	238	self.config.account_validity.renew_at,
	239	)
	240
	241	return res
	242
	243	@defer.inlineCallbacks
	244	def set_renewal_mail_status(self, user_id, email_sent):
	245	"""Sets or unsets the flag that indicates whether a renewal email has been sent
	246	to the user (and the user hasn't renewed their account yet).
	247
	248	Args:
	249	user_id (str): ID of the user to set/unset the flag for.
	250	email_sent (bool): Flag which indicates whether a renewal email has been sent
	251	to this user.
	252	"""
	253	yield self._simple_update_one(
	254	table="account_validity",
	255	keyvalues={"user_id": user_id},
	256	updatevalues={"email_sent": email_sent},
	257	desc="set_renewal_mail_status",
	258	)
	259
	260	@defer.inlineCallbacks
	261	def delete_account_validity_for_user(self, user_id):
	262	"""Deletes the entry for the given user in the account validity table, removing
	263	their expiration date and renewal token.
	264
	265	Args:
	266	user_id (str): ID of the user to remove from the account validity table.
	267	"""
	268	yield self._simple_delete_one(
	269	table="account_validity",
	270	keyvalues={"user_id": user_id},
	271	desc="delete_account_validity_for_user",
	272	)
	273
	274	@defer.inlineCallbacks
	275	def is_server_admin(self, user):
	276	"""Determines if a user is an admin of this homeserver.
	277
	278	Args:
	279	user (UserID): user ID of the user to test
	280
	281	Returns (bool):
	282	true iff the user is a server admin, false otherwise.
	283	"""
	284	res = yield self._simple_select_one_onecol(
	285	table="users",
	286	keyvalues={"name": user.to_string()},
	287	retcol="admin",
	288	allow_none=True,
	289	desc="is_server_admin",
	290	)
	291
	292	return res if res else False
	293
	294	def set_server_admin(self, user, admin):
	295	"""Sets whether a user is an admin of this homeserver.
	296
	297	Args:
	298	user (UserID): user ID of the user to test
	299	admin (bool): true iff the user is to be a server admin,
	300	false otherwise.
	301	"""
	302	return self._simple_update_one(
	303	table="users",
	304	keyvalues={"name": user.to_string()},
	305	updatevalues={"admin": 1 if admin else 0},
	306	desc="set_server_admin",
	307	)
	308
	309	def _query_for_auth(self, txn, token):
	310	sql = (
	311	"SELECT users.name, users.is_guest, access_tokens.id as token_id,"
	312	" access_tokens.device_id, access_tokens.valid_until_ms"
	313	" FROM users"
	314	" INNER JOIN access_tokens on users.name = access_tokens.user_id"
	315	" WHERE token = ?"
	316	)
	317
	318	txn.execute(sql, (token,))
	319	rows = self.cursor_to_dict(txn)
	320	if rows:
	321	return rows[0]
	322
	323	return None
	324
	325	@cachedInlineCallbacks()
	326	def is_real_user(self, user_id):
	327	"""Determines if the user is a real user, ie does not have a 'user_type'.
	328
	329	Args:
	330	user_id (str): user id to test
	331
	332	Returns:
	333	Deferred[bool]: True if user 'user_type' is null or empty string
	334	"""
	335	res = yield self.runInteraction("is_real_user", self.is_real_user_txn, user_id)
	336	return res
	337
	338	@cachedInlineCallbacks()
	339	def is_support_user(self, user_id):
	340	"""Determines if the user is of type UserTypes.SUPPORT
	341
	342	Args:
	343	user_id (str): user id to test
	344
	345	Returns:
	346	Deferred[bool]: True if user is of type UserTypes.SUPPORT
	347	"""
	348	res = yield self.runInteraction(
	349	"is_support_user", self.is_support_user_txn, user_id
	350	)
	351	return res
	352
	353	def is_real_user_txn(self, txn, user_id):
	354	res = self._simple_select_one_onecol_txn(
	355	txn=txn,
	356	table="users",
	357	keyvalues={"name": user_id},
	358	retcol="user_type",
	359	allow_none=True,
	360	)
	361	return res is None
	362
	363	def is_support_user_txn(self, txn, user_id):
	364	res = self._simple_select_one_onecol_txn(
	365	txn=txn,
	366	table="users",
	367	keyvalues={"name": user_id},
	368	retcol="user_type",
	369	allow_none=True,
	370	)
	371	return True if res == UserTypes.SUPPORT else False
	372
	373	def get_users_by_id_case_insensitive(self, user_id):
	374	"""Gets users that match user_id case insensitively.
	375	Returns a mapping of user_id -> password_hash.
	376	"""
	377
	378	def f(txn):
	379	sql = (
	380	"SELECT name, password_hash FROM users" " WHERE lower(name) = lower(?)"
	381	)
	382	txn.execute(sql, (user_id,))
	383	return dict(txn)
	384
	385	return self.runInteraction("get_users_by_id_case_insensitive", f)
	386
	387	async def get_user_by_external_id(
	388	self, auth_provider: str, external_id: str
	389	) -> str:
	390	"""Look up a user by their external auth id
	391
	392	Args:
	393	auth_provider: identifier for the remote auth provider
	394	external_id: id on that system
	395
	396	Returns:
	397	str\|None: the mxid of the user, or None if they are not known
	398	"""
	399	return await self._simple_select_one_onecol(
	400	table="user_external_ids",
	401	keyvalues={"auth_provider": auth_provider, "external_id": external_id},
	402	retcol="user_id",
	403	allow_none=True,
	404	desc="get_user_by_external_id",
	405	)
	406
	407	@defer.inlineCallbacks
	408	def count_all_users(self):
	409	"""Counts all users registered on the homeserver."""
	410
	411	def _count_users(txn):
	412	txn.execute("SELECT COUNT(*) AS users FROM users")
	413	rows = self.cursor_to_dict(txn)
	414	if rows:
	415	return rows[0]["users"]
	416	return 0
	417
	418	ret = yield self.runInteraction("count_users", _count_users)
	419	return ret
	420
	421	def count_daily_user_type(self):
	422	"""
	423	Counts 1) native non guest users
	424	2) native guests users
	425	3) bridged users
	426	who registered on the homeserver in the past 24 hours
	427	"""
	428
	429	def _count_daily_user_type(txn):
	430	yesterday = int(self._clock.time()) - (60 * 60 * 24)
	431
	432	sql = """
	433	SELECT user_type, COALESCE(count(*), 0) AS count FROM (
	434	SELECT
	435	CASE
	436	WHEN is_guest=0 AND appservice_id IS NULL THEN 'native'
	437	WHEN is_guest=1 AND appservice_id IS NULL THEN 'guest'
	438	WHEN is_guest=0 AND appservice_id IS NOT NULL THEN 'bridged'
	439	END AS user_type
	440	FROM users
	441	WHERE creation_ts > ?
	442	) AS t GROUP BY user_type
	443	"""
	444	results = {"native": 0, "guest": 0, "bridged": 0}
	445	txn.execute(sql, (yesterday,))
	446	for row in txn:
	447	results[row[0]] = row[1]
	448	return results
	449
	450	return self.runInteraction("count_daily_user_type", _count_daily_user_type)
	451
	452	@defer.inlineCallbacks
	453	def count_nonbridged_users(self):
	454	def _count_users(txn):
	455	txn.execute(
	456	"""
	457	SELECT COALESCE(COUNT(*), 0) FROM users
	458	WHERE appservice_id IS NULL
	459	"""
	460	)
	461	count, = txn.fetchone()
	462	return count
	463
	464	ret = yield self.runInteraction("count_users", _count_users)
	465	return ret
	466
	467	@defer.inlineCallbacks
	468	def count_real_users(self):
	469	"""Counts all users without a special user_type registered on the homeserver."""
	470
	471	def _count_users(txn):
	472	txn.execute("SELECT COUNT(*) AS users FROM users where user_type is null")
	473	rows = self.cursor_to_dict(txn)
	474	if rows:
	475	return rows[0]["users"]
	476	return 0
	477
	478	ret = yield self.runInteraction("count_real_users", _count_users)
	479	return ret
	480
	481	@defer.inlineCallbacks
	482	def find_next_generated_user_id_localpart(self):
	483	"""
	484	Gets the localpart of the next generated user ID.
	485
	486	Generated user IDs are integers, and we aim for them to be as small as
	487	we can. Unfortunately, it's possible some of them are already taken by
	488	existing users, and there may be gaps in the already taken range. This
	489	function returns the start of the first allocatable gap. This is to
	490	avoid the case of ID 10000000 being pre-allocated, so us wasting the
	491	first (and shortest) many generated user IDs.
	492	"""
	493
	494	def _find_next_generated_user_id(txn):
	495	# We bound between '@1' and '@a' to avoid pulling the entire table
	496	# out.
	497	txn.execute("SELECT name FROM users WHERE '@1' <= name AND name < '@a'")
	498
	499	regex = re.compile(r"^@(\d+):")
	500
	501	found = set()
	502
	503	for (user_id,) in txn:
	504	match = regex.search(user_id)
	505	if match:
	506	found.add(int(match.group(1)))
	507	for i in range(len(found) + 1):
	508	if i not in found:
	509	return i
	510
	511	return (
	512	(
	513	yield self.runInteraction(
	514	"find_next_generated_user_id", _find_next_generated_user_id
	515	)
	516	)
	517	)
	518
	519	@defer.inlineCallbacks
	520	def get_user_id_by_threepid(self, medium, address):
	521	"""Returns user id from threepid
	522
	523	Args:
	524	medium (str): threepid medium e.g. email
	525	address (str): threepid address e.g. me@example.com
	526
	527	Returns:
	528	Deferred[str\|None]: user id or None if no user id/threepid mapping exists
	529	"""
	530	user_id = yield self.runInteraction(
	531	"get_user_id_by_threepid", self.get_user_id_by_threepid_txn, medium, address
	532	)
	533	return user_id
	534
	535	def get_user_id_by_threepid_txn(self, txn, medium, address):
	536	"""Returns user id from threepid
	537
	538	Args:
	539	txn (cursor):
	540	medium (str): threepid medium e.g. email
	541	address (str): threepid address e.g. me@example.com
	542
	543	Returns:
	544	str\|None: user id or None if no user id/threepid mapping exists
	545	"""
	546	ret = self._simple_select_one_txn(
	547	txn,
	548	"user_threepids",
	549	{"medium": medium, "address": address},
	550	["user_id"],
	551	True,
	552	)
	553	if ret:
	554	return ret["user_id"]
	555	return None
	556
	557	@defer.inlineCallbacks
	558	def user_add_threepid(self, user_id, medium, address, validated_at, added_at):
	559	yield self._simple_upsert(
	560	"user_threepids",
	561	{"medium": medium, "address": address},
	562	{"user_id": user_id, "validated_at": validated_at, "added_at": added_at},
	563	)
	564
	565	@defer.inlineCallbacks
	566	def user_get_threepids(self, user_id):
	567	ret = yield self._simple_select_list(
	568	"user_threepids",
	569	{"user_id": user_id},
	570	["medium", "address", "validated_at", "added_at"],
	571	"user_get_threepids",
	572	)
	573	return ret
	574
	575	def user_delete_threepid(self, user_id, medium, address):
	576	return self._simple_delete(
	577	"user_threepids",
	578	keyvalues={"user_id": user_id, "medium": medium, "address": address},
	579	desc="user_delete_threepids",
	580	)
	581
	582	def add_user_bound_threepid(self, user_id, medium, address, id_server):
	583	"""The server proxied a bind request to the given identity server on
	584	behalf of the given user. We need to remember this in case the user
	585	asks us to unbind the threepid.
	586
	587	Args:
	588	user_id (str)
	589	medium (str)
	590	address (str)
	591	id_server (str)
	592
	593	Returns:
	594	Deferred
	595	"""
	596	# We need to use an upsert, in case they user had already bound the
	597	# threepid
	598	return self._simple_upsert(
	599	table="user_threepid_id_server",
	600	keyvalues={
	601	"user_id": user_id,
	602	"medium": medium,
	603	"address": address,
	604	"id_server": id_server,
	605	},
	606	values={},
	607	insertion_values={},
	608	desc="add_user_bound_threepid",
	609	)
	610
	611	def user_get_bound_threepids(self, user_id):
	612	"""Get the threepids that a user has bound to an identity server through the homeserver
	613	The homeserver remembers where binds to an identity server occurred. Using this
	614	method can retrieve those threepids.
	615
	616	Args:
	617	user_id (str): The ID of the user to retrieve threepids for
	618
	619	Returns:
	620	Deferred[list[dict]]: List of dictionaries containing the following:
	621	medium (str): The medium of the threepid (e.g "email")
	622	address (str): The address of the threepid (e.g "bob@example.com")
	623	"""
	624	return self._simple_select_list(
	625	table="user_threepid_id_server",
	626	keyvalues={"user_id": user_id},
	627	retcols=["medium", "address"],
	628	desc="user_get_bound_threepids",
	629	)
	630
	631	def remove_user_bound_threepid(self, user_id, medium, address, id_server):
	632	"""The server proxied an unbind request to the given identity server on
	633	behalf of the given user, so we remove the mapping of threepid to
	634	identity server.
	635
	636	Args:
	637	user_id (str)
	638	medium (str)
	639	address (str)
	640	id_server (str)
	641
	642	Returns:
	643	Deferred
	644	"""
	645	return self._simple_delete(
	646	table="user_threepid_id_server",
	647	keyvalues={
	648	"user_id": user_id,
	649	"medium": medium,
	650	"address": address,
	651	"id_server": id_server,
	652	},
	653	desc="remove_user_bound_threepid",
	654	)
	655
	656	def get_id_servers_user_bound(self, user_id, medium, address):
	657	"""Get the list of identity servers that the server proxied bind
	658	requests to for given user and threepid
	659
	660	Args:
	661	user_id (str)
	662	medium (str)
	663	address (str)
	664
	665	Returns:
	666	Deferred[list[str]]: Resolves to a list of identity servers
	667	"""
	668	return self._simple_select_onecol(
	669	table="user_threepid_id_server",
	670	keyvalues={"user_id": user_id, "medium": medium, "address": address},
	671	retcol="id_server",
	672	desc="get_id_servers_user_bound",
	673	)
	674
	675	@cachedInlineCallbacks()
	676	def get_user_deactivated_status(self, user_id):
	677	"""Retrieve the value for the `deactivated` property for the provided user.
	678
	679	Args:
	680	user_id (str): The ID of the user to retrieve the status for.
	681
	682	Returns:
	683	defer.Deferred(bool): The requested value.
	684	"""
	685
	686	res = yield self._simple_select_one_onecol(
	687	table="users",
	688	keyvalues={"name": user_id},
	689	retcol="deactivated",
	690	desc="get_user_deactivated_status",
	691	)
	692
	693	# Convert the integer into a boolean.
	694	return res == 1
	695
	696	def get_threepid_validation_session(
	697	self, medium, client_secret, address=None, sid=None, validated=True
	698	):
	699	"""Gets a session_id and last_send_attempt (if available) for a
	700	combination of validation metadata
	701
	702	Args:
	703	medium (str\|None): The medium of the 3PID
	704	address (str\|None): The address of the 3PID
	705	sid (str\|None): The ID of the validation session
	706	client_secret (str): A unique string provided by the client to help identify this
	707	validation attempt
	708	validated (bool\|None): Whether sessions should be filtered by
	709	whether they have been validated already or not. None to
	710	perform no filtering
	711
	712	Returns:
	713	Deferred[dict\|None]: A dict containing the following:
	714	* address - address of the 3pid
	715	* medium - medium of the 3pid
	716	* client_secret - a secret provided by the client for this validation session
	717	* session_id - ID of the validation session
	718	* send_attempt - a number serving to dedupe send attempts for this session
	719	* validated_at - timestamp of when this session was validated if so
	720
	721	Otherwise None if a validation session is not found
	722	"""
	723	if not client_secret:
	724	raise SynapseError(
	725	400, "Missing parameter: client_secret", errcode=Codes.MISSING_PARAM
	726	)
	727
	728	keyvalues = {"client_secret": client_secret}
	729	if medium:
	730	keyvalues["medium"] = medium
	731	if address:
	732	keyvalues["address"] = address
	733	if sid:
	734	keyvalues["session_id"] = sid
	735
	736	assert address or sid
	737
	738	def get_threepid_validation_session_txn(txn):
	739	sql = """
	740	SELECT address, session_id, medium, client_secret,
	741	last_send_attempt, validated_at
	742	FROM threepid_validation_session WHERE %s
	743	""" % (
	744	" AND ".join("%s = ?" % k for k in iterkeys(keyvalues)),
	745	)
	746
	747	if validated is not None:
	748	sql += " AND validated_at IS " + ("NOT NULL" if validated else "NULL")
	749
	750	sql += " LIMIT 1"
	751
	752	txn.execute(sql, list(keyvalues.values()))
	753	rows = self.cursor_to_dict(txn)
	754	if not rows:
	755	return None
	756
	757	return rows[0]
	758
	759	return self.runInteraction(
	760	"get_threepid_validation_session", get_threepid_validation_session_txn
	761	)
	762
	763	def delete_threepid_session(self, session_id):
	764	"""Removes a threepid validation session from the database. This can
	765	be done after validation has been performed and whatever action was
	766	waiting on it has been carried out
	767
	768	Args:
	769	session_id (str): The ID of the session to delete
	770	"""
	771
	772	def delete_threepid_session_txn(txn):
	773	self._simple_delete_txn(
	774	txn,
	775	table="threepid_validation_token",
	776	keyvalues={"session_id": session_id},
	777	)
	778	self._simple_delete_txn(
	779	txn,
	780	table="threepid_validation_session",
	781	keyvalues={"session_id": session_id},
	782	)
	783
	784	return self.runInteraction(
	785	"delete_threepid_session", delete_threepid_session_txn
	786	)
	787
	788
	789	class RegistrationBackgroundUpdateStore(
	790	RegistrationWorkerStore, background_updates.BackgroundUpdateStore
	791	):
	792	def __init__(self, db_conn, hs):
	793	super(RegistrationBackgroundUpdateStore, self).__init__(db_conn, hs)
	794
	795	self.clock = hs.get_clock()
	796	self.config = hs.config
	797
	798	self.register_background_index_update(
	799	"access_tokens_device_index",
	800	index_name="access_tokens_device_id",
	801	table="access_tokens",
	802	columns=["user_id", "device_id"],
	803	)
	804
	805	self.register_background_index_update(
	806	"users_creation_ts",
	807	index_name="users_creation_ts",
	808	table="users",
	809	columns=["creation_ts"],
	810	)
	811
	812	# we no longer use refresh tokens, but it's possible that some people
	813	# might have a background update queued to build this index. Just
	814	# clear the background update.
	815	self.register_noop_background_update("refresh_tokens_device_index")
	816
	817	self.register_background_update_handler(
	818	"user_threepids_grandfather", self._bg_user_threepids_grandfather
	819	)
	820
	821	self.register_background_update_handler(
	822	"users_set_deactivated_flag", self._background_update_set_deactivated_flag
	823	)
	824
	825	@defer.inlineCallbacks
	826	def _background_update_set_deactivated_flag(self, progress, batch_size):
	827	"""Retrieves a list of all deactivated users and sets the 'deactivated' flag to 1
	828	for each of them.
	829	"""
	830
	831	last_user = progress.get("user_id", "")
	832
	833	def _background_update_set_deactivated_flag_txn(txn):
	834	txn.execute(
	835	"""
	836	SELECT
	837	users.name,
	838	COUNT(access_tokens.token) AS count_tokens,
	839	COUNT(user_threepids.address) AS count_threepids
	840	FROM users
	841	LEFT JOIN access_tokens ON (access_tokens.user_id = users.name)
	842	LEFT JOIN user_threepids ON (user_threepids.user_id = users.name)
	843	WHERE (users.password_hash IS NULL OR users.password_hash = '')
	844	AND (users.appservice_id IS NULL OR users.appservice_id = '')
	845	AND users.is_guest = 0
	846	AND users.name > ?
	847	GROUP BY users.name
	848	ORDER BY users.name ASC
	849	LIMIT ?;
	850	""",
	851	(last_user, batch_size),
	852	)
	853
	854	rows = self.cursor_to_dict(txn)
	855
	856	if not rows:
	857	return True, 0
	858
	859	rows_processed_nb = 0
	860
	861	for user in rows:
	862	if not user["count_tokens"] and not user["count_threepids"]:
	863	self.set_user_deactivated_status_txn(txn, user["name"], True)
	864	rows_processed_nb += 1
	865
	866	logger.info("Marked %d rows as deactivated", rows_processed_nb)
	867
	868	self._background_update_progress_txn(
	869	txn, "users_set_deactivated_flag", {"user_id": rows[-1]["name"]}
	870	)
	871
	872	if batch_size > len(rows):
	873	return True, len(rows)
	874	else:
	875	return False, len(rows)
	876
	877	end, nb_processed = yield self.runInteraction(
	878	"users_set_deactivated_flag", _background_update_set_deactivated_flag_txn
	879	)
	880
	881	if end:
	882	yield self._end_background_update("users_set_deactivated_flag")
	883
	884	return nb_processed
	885
	886	@defer.inlineCallbacks
	887	def _bg_user_threepids_grandfather(self, progress, batch_size):
	888	"""We now track which identity servers a user binds their 3PID to, so
	889	we need to handle the case of existing bindings where we didn't track
	890	this.
	891
	892	We do this by grandfathering in existing user threepids assuming that
	893	they used one of the server configured trusted identity servers.
	894	"""
	895	id_servers = set(self.config.trusted_third_party_id_servers)
	896
	897	def _bg_user_threepids_grandfather_txn(txn):
	898	sql = """
	899	INSERT INTO user_threepid_id_server
	900	(user_id, medium, address, id_server)
	901	SELECT user_id, medium, address, ?
	902	FROM user_threepids
	903	"""
	904
	905	txn.executemany(sql, [(id_server,) for id_server in id_servers])
	906
	907	if id_servers:
	908	yield self.runInteraction(
	909	"_bg_user_threepids_grandfather", _bg_user_threepids_grandfather_txn
	910	)
	911
	912	yield self._end_background_update("user_threepids_grandfather")
	913
	914	return 1
	915
	916
	917	class RegistrationStore(RegistrationBackgroundUpdateStore):
	918	def __init__(self, db_conn, hs):
	919	super(RegistrationStore, self).__init__(db_conn, hs)
	920
	921	self._account_validity = hs.config.account_validity
	922
	923	# Create a background job for culling expired 3PID validity tokens
	924	def start_cull():
	925	# run as a background process to make sure that the database transactions
	926	# have a logcontext to report to
	927	return run_as_background_process(
	928	"cull_expired_threepid_validation_tokens",
	929	self.cull_expired_threepid_validation_tokens,
	930	)
	931
	932	hs.get_clock().looping_call(start_cull, THIRTY_MINUTES_IN_MS)
	933
	934	@defer.inlineCallbacks
	935	def add_access_token_to_user(self, user_id, token, device_id, valid_until_ms):
	936	"""Adds an access token for the given user.
	937
	938	Args:
	939	user_id (str): The user ID.
	940	token (str): The new access token to add.
	941	device_id (str): ID of the device to associate with the access
	942	token
	943	valid_until_ms (int\|None): when the token is valid until. None for
	944	no expiry.
	945	Raises:
	946	StoreError if there was a problem adding this.
	947	"""
	948	next_id = self._access_tokens_id_gen.get_next()
	949
	950	yield self._simple_insert(
	951	"access_tokens",
	952	{
	953	"id": next_id,
	954	"user_id": user_id,
	955	"token": token,
	956	"device_id": device_id,
	957	"valid_until_ms": valid_until_ms,
	958	},
	959	desc="add_access_token_to_user",
	960	)
	961
	962	def register_user(
	963	self,
	964	user_id,
	965	password_hash=None,
	966	was_guest=False,
	967	make_guest=False,
	968	appservice_id=None,
	969	create_profile_with_displayname=None,
	970	admin=False,
	971	user_type=None,
	972	):
	973	"""Attempts to register an account.
	974
	975	Args:
	976	user_id (str): The desired user ID to register.
	977	password_hash (str): Optional. The password hash for this user.
	978	was_guest (bool): Optional. Whether this is a guest account being
	979	upgraded to a non-guest account.
	980	make_guest (boolean): True if the the new user should be guest,
	981	false to add a regular user account.
	982	appservice_id (str): The ID of the appservice registering the user.
	983	create_profile_with_displayname (unicode): Optionally create a profile for
	984	the user, setting their displayname to the given value
	985	admin (boolean): is an admin user?
	986	user_type (str\|None): type of user. One of the values from
	987	api.constants.UserTypes, or None for a normal user.
	988
	989	Raises:
	990	StoreError if the user_id could not be registered.
	991	"""
	992	return self.runInteraction(
	993	"register_user",
	994	self._register_user,
	995	user_id,
	996	password_hash,
	997	was_guest,
	998	make_guest,
	999	appservice_id,
	1000	create_profile_with_displayname,
	1001	admin,
	1002	user_type,
	1003	)
	1004
	1005	def _register_user(
	1006	self,
	1007	txn,
	1008	user_id,
	1009	password_hash,
	1010	was_guest,
	1011	make_guest,
	1012	appservice_id,
	1013	create_profile_with_displayname,
	1014	admin,
	1015	user_type,
	1016	):
	1017	user_id_obj = UserID.from_string(user_id)
	1018
	1019	now = int(self.clock.time())
	1020
	1021	try:
	1022	if was_guest:
	1023	# Ensure that the guest user actually exists
	1024	# ``allow_none=False`` makes this raise an exception
	1025	# if the row isn't in the database.
	1026	self._simple_select_one_txn(
	1027	txn,
	1028	"users",
	1029	keyvalues={"name": user_id, "is_guest": 1},
	1030	retcols=("name",),
	1031	allow_none=False,
	1032	)
	1033
	1034	self._simple_update_one_txn(
	1035	txn,
	1036	"users",
	1037	keyvalues={"name": user_id, "is_guest": 1},
	1038	updatevalues={
	1039	"password_hash": password_hash,
	1040	"upgrade_ts": now,
	1041	"is_guest": 1 if make_guest else 0,
	1042	"appservice_id": appservice_id,
	1043	"admin": 1 if admin else 0,
	1044	"user_type": user_type,
	1045	},
	1046	)
	1047	else:
	1048	self._simple_insert_txn(
	1049	txn,
	1050	"users",
	1051	values={
	1052	"name": user_id,
	1053	"password_hash": password_hash,
	1054	"creation_ts": now,
	1055	"is_guest": 1 if make_guest else 0,
	1056	"appservice_id": appservice_id,
	1057	"admin": 1 if admin else 0,
	1058	"user_type": user_type,
	1059	},
	1060	)
	1061
	1062	except self.database_engine.module.IntegrityError:
	1063	raise StoreError(400, "User ID already taken.", errcode=Codes.USER_IN_USE)
	1064
	1065	if self._account_validity.enabled:
	1066	self.set_expiration_date_for_user_txn(txn, user_id)
	1067
	1068	if create_profile_with_displayname:
	1069	# set a default displayname serverside to avoid ugly race
	1070	# between auto-joins and clients trying to set displaynames
	1071	#
	1072	# obviously the 'profiles' table uses localpart for user_id
	1073	# while everything else uses the full mxid.
	1074	txn.execute(
	1075	"INSERT INTO profiles(user_id, displayname) VALUES (?,?)",
	1076	(user_id_obj.localpart, create_profile_with_displayname),
	1077	)
	1078
	1079	if self.hs.config.stats_enabled:
	1080	# we create a new completed user statistics row
	1081
	1082	# we don't strictly need current_token since this user really can't
	1083	# have any state deltas before now (as it is a new user), but still,
	1084	# we include it for completeness.
	1085	current_token = self._get_max_stream_id_in_current_state_deltas_txn(txn)
	1086	self._update_stats_delta_txn(
	1087	txn, now, "user", user_id, {}, complete_with_stream_id=current_token
	1088	)
	1089
	1090	self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,))
	1091	txn.call_after(self.is_guest.invalidate, (user_id,))
	1092
	1093	def record_user_external_id(
	1094	self, auth_provider: str, external_id: str, user_id: str
	1095	) -> Deferred:
	1096	"""Record a mapping from an external user id to a mxid
	1097
	1098	Args:
	1099	auth_provider: identifier for the remote auth provider
	1100	external_id: id on that system
	1101	user_id: complete mxid that it is mapped to
	1102	"""
	1103	return self._simple_insert(
	1104	table="user_external_ids",
	1105	values={
	1106	"auth_provider": auth_provider,
	1107	"external_id": external_id,
	1108	"user_id": user_id,
	1109	},
	1110	desc="record_user_external_id",
	1111	)
	1112
	1113	def user_set_password_hash(self, user_id, password_hash):
	1114	"""
	1115	NB. This does not evict any cache because the one use for this
	1116	removes most of the entries subsequently anyway so it would be
	1117	pointless. Use flush_user separately.
	1118	"""
	1119
	1120	def user_set_password_hash_txn(txn):
	1121	self._simple_update_one_txn(
	1122	txn, "users", {"name": user_id}, {"password_hash": password_hash}
	1123	)
	1124	self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,))
	1125
	1126	return self.runInteraction("user_set_password_hash", user_set_password_hash_txn)
	1127
	1128	def user_set_consent_version(self, user_id, consent_version):
	1129	"""Updates the user table to record privacy policy consent
	1130
	1131	Args:
	1132	user_id (str): full mxid of the user to update
	1133	consent_version (str): version of the policy the user has consented
	1134	to
	1135
	1136	Raises:
	1137	StoreError(404) if user not found
	1138	"""
	1139
	1140	def f(txn):
	1141	self._simple_update_one_txn(
	1142	txn,
	1143	table="users",
	1144	keyvalues={"name": user_id},
	1145	updatevalues={"consent_version": consent_version},
	1146	)
	1147	self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,))
	1148
	1149	return self.runInteraction("user_set_consent_version", f)
	1150
	1151	def user_set_consent_server_notice_sent(self, user_id, consent_version):
	1152	"""Updates the user table to record that we have sent the user a server
	1153	notice about privacy policy consent
	1154
	1155	Args:
	1156	user_id (str): full mxid of the user to update
	1157	consent_version (str): version of the policy we have notified the
	1158	user about
	1159
	1160	Raises:
	1161	StoreError(404) if user not found
	1162	"""
	1163
	1164	def f(txn):
	1165	self._simple_update_one_txn(
	1166	txn,
	1167	table="users",
	1168	keyvalues={"name": user_id},
	1169	updatevalues={"consent_server_notice_sent": consent_version},
	1170	)
	1171	self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,))
	1172
	1173	return self.runInteraction("user_set_consent_server_notice_sent", f)
	1174
	1175	def user_delete_access_tokens(self, user_id, except_token_id=None, device_id=None):
	1176	"""
	1177	Invalidate access tokens belonging to a user
	1178
	1179	Args:
	1180	user_id (str): ID of user the tokens belong to
	1181	except_token_id (str): list of access_tokens IDs which should
	1182	not be deleted
	1183	device_id (str\|None): ID of device the tokens are associated with.
	1184	If None, tokens associated with any device (or no device) will
	1185	be deleted
	1186	Returns:
	1187	defer.Deferred[list[str, int, str\|None, int]]: a list of
	1188	(token, token id, device id) for each of the deleted tokens
	1189	"""
	1190
	1191	def f(txn):
	1192	keyvalues = {"user_id": user_id}
	1193	if device_id is not None:
	1194	keyvalues["device_id"] = device_id
	1195
	1196	items = keyvalues.items()
	1197	where_clause = " AND ".join(k + " = ?" for k, _ in items)
	1198	values = [v for _, v in items]
	1199	if except_token_id:
	1200	where_clause += " AND id != ?"
	1201	values.append(except_token_id)
	1202
	1203	txn.execute(
	1204	"SELECT token, id, device_id FROM access_tokens WHERE %s"
	1205	% where_clause,
	1206	values,
	1207	)
	1208	tokens_and_devices = [(r[0], r[1], r[2]) for r in txn]
	1209
	1210	for token, _, _ in tokens_and_devices:
	1211	self._invalidate_cache_and_stream(
	1212	txn, self.get_user_by_access_token, (token,)
	1213	)
	1214
	1215	txn.execute("DELETE FROM access_tokens WHERE %s" % where_clause, values)
	1216
	1217	return tokens_and_devices
	1218
	1219	return self.runInteraction("user_delete_access_tokens", f)
	1220
	1221	def delete_access_token(self, access_token):
	1222	def f(txn):
	1223	self._simple_delete_one_txn(
	1224	txn, table="access_tokens", keyvalues={"token": access_token}
	1225	)
	1226
	1227	self._invalidate_cache_and_stream(
	1228	txn, self.get_user_by_access_token, (access_token,)
	1229	)
	1230
	1231	return self.runInteraction("delete_access_token", f)
	1232
	1233	@cachedInlineCallbacks()
	1234	def is_guest(self, user_id):
	1235	res = yield self._simple_select_one_onecol(
	1236	table="users",
	1237	keyvalues={"name": user_id},
	1238	retcol="is_guest",
	1239	allow_none=True,
	1240	desc="is_guest",
	1241	)
	1242
	1243	return res if res else False
	1244
	1245	def add_user_pending_deactivation(self, user_id):
	1246	"""
	1247	Adds a user to the table of users who need to be parted from all the rooms they're
	1248	in
	1249	"""
	1250	return self._simple_insert(
	1251	"users_pending_deactivation",
	1252	values={"user_id": user_id},
	1253	desc="add_user_pending_deactivation",
	1254	)
	1255
	1256	def del_user_pending_deactivation(self, user_id):
	1257	"""
	1258	Removes the given user to the table of users who need to be parted from all the
	1259	rooms they're in, effectively marking that user as fully deactivated.
	1260	"""
	1261	# XXX: This should be simple_delete_one but we failed to put a unique index on
	1262	# the table, so somehow duplicate entries have ended up in it.
	1263	return self._simple_delete(
	1264	"users_pending_deactivation",
	1265	keyvalues={"user_id": user_id},
	1266	desc="del_user_pending_deactivation",
	1267	)
	1268
	1269	def get_user_pending_deactivation(self):
	1270	"""
	1271	Gets one user from the table of users waiting to be parted from all the rooms
	1272	they're in.
	1273	"""
	1274	return self._simple_select_one_onecol(
	1275	"users_pending_deactivation",
	1276	keyvalues={},
	1277	retcol="user_id",
	1278	allow_none=True,
	1279	desc="get_users_pending_deactivation",
	1280	)
	1281
	1282	def validate_threepid_session(self, session_id, client_secret, token, current_ts):
	1283	"""Attempt to validate a threepid session using a token
	1284
	1285	Args:
	1286	session_id (str): The id of a validation session
	1287	client_secret (str): A unique string provided by the client to
	1288	help identify this validation attempt
	1289	token (str): A validation token
	1290	current_ts (int): The current unix time in milliseconds. Used for
	1291	checking token expiry status
	1292
	1293	Raises:
	1294	ThreepidValidationError: if a matching validation token was not found or has
	1295	expired
	1296
	1297	Returns:
	1298	deferred str\|None: A str representing a link to redirect the user
	1299	to if there is one.
	1300	"""
	1301
	1302	# Insert everything into a transaction in order to run atomically
	1303	def validate_threepid_session_txn(txn):
	1304	row = self._simple_select_one_txn(
	1305	txn,
	1306	table="threepid_validation_session",
	1307	keyvalues={"session_id": session_id},
	1308	retcols=["client_secret", "validated_at"],
	1309	allow_none=True,
	1310	)
	1311
	1312	if not row:
	1313	raise ThreepidValidationError(400, "Unknown session_id")
	1314	retrieved_client_secret = row["client_secret"]
	1315	validated_at = row["validated_at"]
	1316
	1317	if retrieved_client_secret != client_secret:
	1318	raise ThreepidValidationError(
	1319	400, "This client_secret does not match the provided session_id"
	1320	)
	1321
	1322	row = self._simple_select_one_txn(
	1323	txn,
	1324	table="threepid_validation_token",
	1325	keyvalues={"session_id": session_id, "token": token},
	1326	retcols=["expires", "next_link"],
	1327	allow_none=True,
	1328	)
	1329
	1330	if not row:
	1331	raise ThreepidValidationError(
	1332	400, "Validation token not found or has expired"
	1333	)
	1334	expires = row["expires"]
	1335	next_link = row["next_link"]
	1336
	1337	# If the session is already validated, no need to revalidate
	1338	if validated_at:
	1339	return next_link
	1340
	1341	if expires <= current_ts:
	1342	raise ThreepidValidationError(
	1343	400, "This token has expired. Please request a new one"
	1344	)
	1345
	1346	# Looks good. Validate the session
	1347	self._simple_update_txn(
	1348	txn,
	1349	table="threepid_validation_session",
	1350	keyvalues={"session_id": session_id},
	1351	updatevalues={"validated_at": self.clock.time_msec()},
	1352	)
	1353
	1354	return next_link
	1355
	1356	# Return next_link if it exists
	1357	return self.runInteraction(
	1358	"validate_threepid_session_txn", validate_threepid_session_txn
	1359	)
	1360
	1361	def upsert_threepid_validation_session(
	1362	self,
	1363	medium,
	1364	address,
	1365	client_secret,
	1366	send_attempt,
	1367	session_id,
	1368	validated_at=None,
	1369	):
	1370	"""Upsert a threepid validation session
	1371	Args:
	1372	medium (str): The medium of the 3PID
	1373	address (str): The address of the 3PID
	1374	client_secret (str): A unique string provided by the client to
	1375	help identify this validation attempt
	1376	send_attempt (int): The latest send_attempt on this session
	1377	session_id (str): The id of this validation session
	1378	validated_at (int\|None): The unix timestamp in milliseconds of
	1379	when the session was marked as valid
	1380	"""
	1381	insertion_values = {
	1382	"medium": medium,
	1383	"address": address,
	1384	"client_secret": client_secret,
	1385	}
	1386
	1387	if validated_at:
	1388	insertion_values["validated_at"] = validated_at
	1389
	1390	return self._simple_upsert(
	1391	table="threepid_validation_session",
	1392	keyvalues={"session_id": session_id},
	1393	values={"last_send_attempt": send_attempt},
	1394	insertion_values=insertion_values,
	1395	desc="upsert_threepid_validation_session",
	1396	)
	1397
	1398	def start_or_continue_validation_session(
	1399	self,
	1400	medium,
	1401	address,
	1402	session_id,
	1403	client_secret,
	1404	send_attempt,
	1405	next_link,
	1406	token,
	1407	token_expires,
	1408	):
	1409	"""Creates a new threepid validation session if it does not already
	1410	exist and associates a new validation token with it
	1411
	1412	Args:
	1413	medium (str): The medium of the 3PID
	1414	address (str): The address of the 3PID
	1415	session_id (str): The id of this validation session
	1416	client_secret (str): A unique string provided by the client to
	1417	help identify this validation attempt
	1418	send_attempt (int): The latest send_attempt on this session
	1419	next_link (str\|None): The link to redirect the user to upon
	1420	successful validation
	1421	token (str): The validation token
	1422	token_expires (int): The timestamp for which after the token
	1423	will no longer be valid
	1424	"""
	1425
	1426	def start_or_continue_validation_session_txn(txn):
	1427	# Create or update a validation session
	1428	self._simple_upsert_txn(
	1429	txn,
	1430	table="threepid_validation_session",
	1431	keyvalues={"session_id": session_id},
	1432	values={"last_send_attempt": send_attempt},
	1433	insertion_values={
	1434	"medium": medium,
	1435	"address": address,
	1436	"client_secret": client_secret,
	1437	},
	1438	)
	1439
	1440	# Create a new validation token with this session ID
	1441	self._simple_insert_txn(
	1442	txn,
	1443	table="threepid_validation_token",
	1444	values={
	1445	"session_id": session_id,
	1446	"token": token,
	1447	"next_link": next_link,
	1448	"expires": token_expires,
	1449	},
	1450	)
	1451
	1452	return self.runInteraction(
	1453	"start_or_continue_validation_session",
	1454	start_or_continue_validation_session_txn,
	1455	)
	1456
	1457	def cull_expired_threepid_validation_tokens(self):
	1458	"""Remove threepid validation tokens with expiry dates that have passed"""
	1459
	1460	def cull_expired_threepid_validation_tokens_txn(txn, ts):
	1461	sql = """
	1462	DELETE FROM threepid_validation_token WHERE
	1463	expires < ?
	1464	"""
	1465	return txn.execute(sql, (ts,))
	1466
	1467	return self.runInteraction(
	1468	"cull_expired_threepid_validation_tokens",
	1469	cull_expired_threepid_validation_tokens_txn,
	1470	self.clock.time_msec(),
	1471	)
	1472
	1473	@defer.inlineCallbacks
	1474	def set_user_deactivated_status(self, user_id, deactivated):
	1475	"""Set the `deactivated` property for the provided user to the provided value.
	1476
	1477	Args:
	1478	user_id (str): The ID of the user to set the status for.
	1479	deactivated (bool): The value to set for `deactivated`.
	1480	"""
	1481
	1482	yield self.runInteraction(
	1483	"set_user_deactivated_status",
	1484	self.set_user_deactivated_status_txn,
	1485	user_id,
	1486	deactivated,
	1487	)
	1488
	1489	def set_user_deactivated_status_txn(self, txn, user_id, deactivated):
	1490	self._simple_update_one_txn(
	1491	txn=txn,
	1492	table="users",
	1493	keyvalues={"name": user_id},
	1494	updatevalues={"deactivated": 1 if deactivated else 0},
	1495	)
	1496	self._invalidate_cache_and_stream(
	1497	txn, self.get_user_deactivated_status, (user_id,)
	1498	)

+42

-0

synapse/storage/data_stores/main/rejections.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2014-2016 OpenMarket Ltd
	2	#
	3	# Licensed under the Apache License, Version 2.0 (the "License");
	4	# you may not use this file except in compliance with the License.
	5	# You may obtain a copy of the License at
	6	#
	7	# http://www.apache.org/licenses/LICENSE-2.0
	8	#
	9	# Unless required by applicable law or agreed to in writing, software
	10	# distributed under the License is distributed on an "AS IS" BASIS,
	11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	# See the License for the specific language governing permissions and
	13	# limitations under the License.
	14
	15	import logging
	16
	17	from synapse.storage._base import SQLBaseStore
	18
	19	logger = logging.getLogger(__name__)
	20
	21
	22	class RejectionsStore(SQLBaseStore):
	23	def _store_rejections_txn(self, txn, event_id, reason):
	24	self._simple_insert_txn(
	25	txn,
	26	table="rejections",
	27	values={
	28	"event_id": event_id,
	29	"reason": reason,
	30	"last_check": self._clock.time_msec(),
	31	},
	32	)
	33
	34	def get_rejection_reason(self, event_id):
	35	return self._simple_select_one_onecol(
	36	table="rejections",
	37	retcol="reason",
	38	keyvalues={"event_id": event_id},
	39	allow_none=True,
	40	desc="get_rejection_reason",
	41	)

+385

-0

synapse/storage/data_stores/main/relations.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2019 New Vector Ltd
	2	#
	3	# Licensed under the Apache License, Version 2.0 (the "License");
	4	# you may not use this file except in compliance with the License.
	5	# You may obtain a copy of the License at
	6	#
	7	# http://www.apache.org/licenses/LICENSE-2.0
	8	#
	9	# Unless required by applicable law or agreed to in writing, software
	10	# distributed under the License is distributed on an "AS IS" BASIS,
	11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	# See the License for the specific language governing permissions and
	13	# limitations under the License.
	14
	15	import logging
	16
	17	import attr
	18
	19	from synapse.api.constants import RelationTypes
	20	from synapse.storage._base import SQLBaseStore
	21	from synapse.storage.data_stores.main.stream import generate_pagination_where_clause
	22	from synapse.storage.relations import (
	23	AggregationPaginationToken,
	24	PaginationChunk,
	25	RelationPaginationToken,
	26	)
	27	from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
	28
	29	logger = logging.getLogger(__name__)
	30
	31
	32	class RelationsWorkerStore(SQLBaseStore):
	33	@cached(tree=True)
	34	def get_relations_for_event(
	35	self,
	36	event_id,
	37	relation_type=None,
	38	event_type=None,
	39	aggregation_key=None,
	40	limit=5,
	41	direction="b",
	42	from_token=None,
	43	to_token=None,
	44	):
	45	"""Get a list of relations for an event, ordered by topological ordering.
	46
	47	Args:
	48	event_id (str): Fetch events that relate to this event ID.
	49	relation_type (str\|None): Only fetch events with this relation
	50	type, if given.
	51	event_type (str\|None): Only fetch events with this event type, if
	52	given.
	53	aggregation_key (str\|None): Only fetch events with this aggregation
	54	key, if given.
	55	limit (int): Only fetch the most recent `limit` events.
	56	direction (str): Whether to fetch the most recent first (`"b"`) or
	57	the oldest first (`"f"`).
	58	from_token (RelationPaginationToken\|None): Fetch rows from the given
	59	token, or from the start if None.
	60	to_token (RelationPaginationToken\|None): Fetch rows up to the given
	61	token, or up to the end if None.
	62
	63	Returns:
	64	Deferred[PaginationChunk]: List of event IDs that match relations
	65	requested. The rows are of the form `{"event_id": "..."}`.
	66	"""
	67
	68	where_clause = ["relates_to_id = ?"]
	69	where_args = [event_id]
	70
	71	if relation_type is not None:
	72	where_clause.append("relation_type = ?")
	73	where_args.append(relation_type)
	74
	75	if event_type is not None:
	76	where_clause.append("type = ?")
	77	where_args.append(event_type)
	78
	79	if aggregation_key:
	80	where_clause.append("aggregation_key = ?")
	81	where_args.append(aggregation_key)
	82
	83	pagination_clause = generate_pagination_where_clause(
	84	direction=direction,
	85	column_names=("topological_ordering", "stream_ordering"),
	86	from_token=attr.astuple(from_token) if from_token else None,
	87	to_token=attr.astuple(to_token) if to_token else None,
	88	engine=self.database_engine,
	89	)
	90
	91	if pagination_clause:
	92	where_clause.append(pagination_clause)
	93
	94	if direction == "b":
	95	order = "DESC"
	96	else:
	97	order = "ASC"
	98
	99	sql = """
	100	SELECT event_id, topological_ordering, stream_ordering
	101	FROM event_relations
	102	INNER JOIN events USING (event_id)
	103	WHERE %s
	104	ORDER BY topological_ordering %s, stream_ordering %s
	105	LIMIT ?
	106	""" % (
	107	" AND ".join(where_clause),
	108	order,
	109	order,
	110	)
	111
	112	def _get_recent_references_for_event_txn(txn):
	113	txn.execute(sql, where_args + [limit + 1])
	114
	115	last_topo_id = None
	116	last_stream_id = None
	117	events = []
	118	for row in txn:
	119	events.append({"event_id": row[0]})
	120	last_topo_id = row[1]
	121	last_stream_id = row[2]
	122
	123	next_batch = None
	124	if len(events) > limit and last_topo_id and last_stream_id:
	125	next_batch = RelationPaginationToken(last_topo_id, last_stream_id)
	126
	127	return PaginationChunk(
	128	chunk=list(events[:limit]), next_batch=next_batch, prev_batch=from_token
	129	)
	130
	131	return self.runInteraction(
	132	"get_recent_references_for_event", _get_recent_references_for_event_txn
	133	)
	134
	135	@cached(tree=True)
	136	def get_aggregation_groups_for_event(
	137	self,
	138	event_id,
	139	event_type=None,
	140	limit=5,
	141	direction="b",
	142	from_token=None,
	143	to_token=None,
	144	):
	145	"""Get a list of annotations on the event, grouped by event type and
	146	aggregation key, sorted by count.
	147
	148	This is used e.g. to get the what and how many reactions have happend
	149	on an event.
	150
	151	Args:
	152	event_id (str): Fetch events that relate to this event ID.
	153	event_type (str\|None): Only fetch events with this event type, if
	154	given.
	155	limit (int): Only fetch the `limit` groups.
	156	direction (str): Whether to fetch the highest count first (`"b"`) or
	157	the lowest count first (`"f"`).
	158	from_token (AggregationPaginationToken\|None): Fetch rows from the
	159	given token, or from the start if None.
	160	to_token (AggregationPaginationToken\|None): Fetch rows up to the
	161	given token, or up to the end if None.
	162
	163
	164	Returns:
	165	Deferred[PaginationChunk]: List of groups of annotations that
	166	match. Each row is a dict with `type`, `key` and `count` fields.
	167	"""
	168
	169	where_clause = ["relates_to_id = ?", "relation_type = ?"]
	170	where_args = [event_id, RelationTypes.ANNOTATION]
	171
	172	if event_type:
	173	where_clause.append("type = ?")
	174	where_args.append(event_type)
	175
	176	having_clause = generate_pagination_where_clause(
	177	direction=direction,
	178	column_names=("COUNT(*)", "MAX(stream_ordering)"),
	179	from_token=attr.astuple(from_token) if from_token else None,
	180	to_token=attr.astuple(to_token) if to_token else None,
	181	engine=self.database_engine,
	182	)
	183
	184	if direction == "b":
	185	order = "DESC"
	186	else:
	187	order = "ASC"
	188
	189	if having_clause:
	190	having_clause = "HAVING " + having_clause
	191	else:
	192	having_clause = ""
	193
	194	sql = """
	195	SELECT type, aggregation_key, COUNT(DISTINCT sender), MAX(stream_ordering)
	196	FROM event_relations
	197	INNER JOIN events USING (event_id)
	198	WHERE {where_clause}
	199	GROUP BY relation_type, type, aggregation_key
	200	{having_clause}
	201	ORDER BY COUNT(*) {order}, MAX(stream_ordering) {order}
	202	LIMIT ?
	203	""".format(
	204	where_clause=" AND ".join(where_clause),
	205	order=order,
	206	having_clause=having_clause,
	207	)
	208
	209	def _get_aggregation_groups_for_event_txn(txn):
	210	txn.execute(sql, where_args + [limit + 1])
	211
	212	next_batch = None
	213	events = []
	214	for row in txn:
	215	events.append({"type": row[0], "key": row[1], "count": row[2]})
	216	next_batch = AggregationPaginationToken(row[2], row[3])
	217
	218	if len(events) <= limit:
	219	next_batch = None
	220
	221	return PaginationChunk(
	222	chunk=list(events[:limit]), next_batch=next_batch, prev_batch=from_token
	223	)
	224
	225	return self.runInteraction(
	226	"get_aggregation_groups_for_event", _get_aggregation_groups_for_event_txn
	227	)
	228
	229	@cachedInlineCallbacks()
	230	def get_applicable_edit(self, event_id):
	231	"""Get the most recent edit (if any) that has happened for the given
	232	event.
	233
	234	Correctly handles checking whether edits were allowed to happen.
	235
	236	Args:
	237	event_id (str): The original event ID
	238
	239	Returns:
	240	Deferred[EventBase\|None]: Returns the most recent edit, if any.
	241	"""
	242
	243	# We only allow edits for `m.room.message` events that have the same sender
	244	# and event type. We can't assert these things during regular event auth so
	245	# we have to do the checks post hoc.
	246
	247	# Fetches latest edit that has the same type and sender as the
	248	# original, and is an `m.room.message`.
	249	sql = """
	250	SELECT edit.event_id FROM events AS edit
	251	INNER JOIN event_relations USING (event_id)
	252	INNER JOIN events AS original ON
	253	original.event_id = relates_to_id
	254	AND edit.type = original.type
	255	AND edit.sender = original.sender
	256	WHERE
	257	relates_to_id = ?
	258	AND relation_type = ?
	259	AND edit.type = 'm.room.message'
	260	ORDER by edit.origin_server_ts DESC, edit.event_id DESC
	261	LIMIT 1
	262	"""
	263
	264	def _get_applicable_edit_txn(txn):
	265	txn.execute(sql, (event_id, RelationTypes.REPLACE))
	266	row = txn.fetchone()
	267	if row:
	268	return row[0]
	269
	270	edit_id = yield self.runInteraction(
	271	"get_applicable_edit", _get_applicable_edit_txn
	272	)
	273
	274	if not edit_id:
	275	return
	276
	277	edit_event = yield self.get_event(edit_id, allow_none=True)
	278	return edit_event
	279
	280	def has_user_annotated_event(self, parent_id, event_type, aggregation_key, sender):
	281	"""Check if a user has already annotated an event with the same key
	282	(e.g. already liked an event).
	283
	284	Args:
	285	parent_id (str): The event being annotated
	286	event_type (str): The event type of the annotation
	287	aggregation_key (str): The aggregation key of the annotation
	288	sender (str): The sender of the annotation
	289
	290	Returns:
	291	Deferred[bool]
	292	"""
	293
	294	sql = """
	295	SELECT 1 FROM event_relations
	296	INNER JOIN events USING (event_id)
	297	WHERE
	298	relates_to_id = ?
	299	AND relation_type = ?
	300	AND type = ?
	301	AND sender = ?
	302	AND aggregation_key = ?
	303	LIMIT 1;
	304	"""
	305
	306	def _get_if_user_has_annotated_event(txn):
	307	txn.execute(
	308	sql,
	309	(
	310	parent_id,
	311	RelationTypes.ANNOTATION,
	312	event_type,
	313	sender,
	314	aggregation_key,
	315	),
	316	)
	317
	318	return bool(txn.fetchone())
	319
	320	return self.runInteraction(
	321	"get_if_user_has_annotated_event", _get_if_user_has_annotated_event
	322	)
	323
	324
	325	class RelationsStore(RelationsWorkerStore):
	326	def _handle_event_relations(self, txn, event):
	327	"""Handles inserting relation data during peristence of events
	328
	329	Args:
	330	txn
	331	event (EventBase)
	332	"""
	333	relation = event.content.get("m.relates_to")
	334	if not relation:
	335	# No relations
	336	return
	337
	338	rel_type = relation.get("rel_type")
	339	if rel_type not in (
	340	RelationTypes.ANNOTATION,
	341	RelationTypes.REFERENCE,
	342	RelationTypes.REPLACE,
	343	):
	344	# Unknown relation type
	345	return
	346
	347	parent_id = relation.get("event_id")
	348	if not parent_id:
	349	# Invalid relation
	350	return
	351
	352	aggregation_key = relation.get("key")
	353
	354	self._simple_insert_txn(
	355	txn,
	356	table="event_relations",
	357	values={
	358	"event_id": event.event_id,
	359	"relates_to_id": parent_id,
	360	"relation_type": rel_type,
	361	"aggregation_key": aggregation_key,
	362	},
	363	)
	364
	365	txn.call_after(self.get_relations_for_event.invalidate_many, (parent_id,))
	366	txn.call_after(
	367	self.get_aggregation_groups_for_event.invalidate_many, (parent_id,)
	368	)
	369
	370	if rel_type == RelationTypes.REPLACE:
	371	txn.call_after(self.get_applicable_edit.invalidate, (parent_id,))
	372
	373	def _handle_redaction(self, txn, redacted_event_id):
	374	"""Handles receiving a redaction and checking whether we need to remove
	375	any redacted relations from the database.
	376
	377	Args:
	378	txn
	379	redacted_event_id (str): The event that was redacted.
	380	"""
	381
	382	self._simple_delete_txn(
	383	txn, table="event_relations", keyvalues={"event_id": redacted_event_id}
	384	)

+681

-0

synapse/storage/data_stores/main/room.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2014-2016 OpenMarket Ltd
	2	# Copyright 2019 The Matrix.org Foundation C.I.C.
	3	#
	4	# Licensed under the Apache License, Version 2.0 (the "License");
	5	# you may not use this file except in compliance with the License.
	6	# You may obtain a copy of the License at
	7	#
	8	# http://www.apache.org/licenses/LICENSE-2.0
	9	#
	10	# Unless required by applicable law or agreed to in writing, software
	11	# distributed under the License is distributed on an "AS IS" BASIS,
	12	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	# See the License for the specific language governing permissions and
	14	# limitations under the License.
	15
	16	import collections
	17	import logging
	18	import re
	19	from typing import Optional, Tuple
	20
	21	from canonicaljson import json
	22
	23	from twisted.internet import defer
	24
	25	from synapse.api.errors import StoreError
	26	from synapse.storage._base import SQLBaseStore
	27	from synapse.storage.data_stores.main.search import SearchStore
	28	from synapse.types import ThirdPartyInstanceID
	29	from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
	30
	31	logger = logging.getLogger(__name__)
	32
	33
	34	OpsLevel = collections.namedtuple(
	35	"OpsLevel", ("ban_level", "kick_level", "redact_level")
	36	)
	37
	38	RatelimitOverride = collections.namedtuple(
	39	"RatelimitOverride", ("messages_per_second", "burst_count")
	40	)
	41
	42
	43	class RoomWorkerStore(SQLBaseStore):
	44	def get_room(self, room_id):
	45	"""Retrieve a room.
	46
	47	Args:
	48	room_id (str): The ID of the room to retrieve.
	49	Returns:
	50	A dict containing the room information, or None if the room is unknown.
	51	"""
	52	return self._simple_select_one(
	53	table="rooms",
	54	keyvalues={"room_id": room_id},
	55	retcols=("room_id", "is_public", "creator"),
	56	desc="get_room",
	57	allow_none=True,
	58	)
	59
	60	def get_public_room_ids(self):
	61	return self._simple_select_onecol(
	62	table="rooms",
	63	keyvalues={"is_public": True},
	64	retcol="room_id",
	65	desc="get_public_room_ids",
	66	)
	67
	68	def count_public_rooms(self, network_tuple, ignore_non_federatable):
	69	"""Counts the number of public rooms as tracked in the room_stats_current
	70	and room_stats_state table.
	71
	72	Args:
	73	network_tuple (ThirdPartyInstanceID\|None)
	74	ignore_non_federatable (bool): If true filters out non-federatable rooms
	75	"""
	76
	77	def _count_public_rooms_txn(txn):
	78	query_args = []
	79
	80	if network_tuple:
	81	if network_tuple.appservice_id:
	82	published_sql = """
	83	SELECT room_id from appservice_room_list
	84	WHERE appservice_id = ? AND network_id = ?
	85	"""
	86	query_args.append(network_tuple.appservice_id)
	87	query_args.append(network_tuple.network_id)
	88	else:
	89	published_sql = """
	90	SELECT room_id FROM rooms WHERE is_public
	91	"""
	92	else:
	93	published_sql = """
	94	SELECT room_id FROM rooms WHERE is_public
	95	UNION SELECT room_id from appservice_room_list
	96	"""
	97
	98	sql = """
	99	SELECT
	100	COALESCE(COUNT(*), 0)
	101	FROM (
	102	%(published_sql)s
	103	) published
	104	INNER JOIN room_stats_state USING (room_id)
	105	INNER JOIN room_stats_current USING (room_id)
	106	WHERE
	107	(
	108	join_rules = 'public' OR history_visibility = 'world_readable'
	109	)
	110	AND joined_members > 0
	111	""" % {
	112	"published_sql": published_sql
	113	}
	114
	115	txn.execute(sql, query_args)
	116	return txn.fetchone()[0]
	117
	118	return self.runInteraction("count_public_rooms", _count_public_rooms_txn)
	119
	120	@defer.inlineCallbacks
	121	def get_largest_public_rooms(
	122	self,
	123	network_tuple: Optional[ThirdPartyInstanceID],
	124	search_filter: Optional[dict],
	125	limit: Optional[int],
	126	bounds: Optional[Tuple[int, str]],
	127	forwards: bool,
	128	ignore_non_federatable: bool = False,
	129	):
	130	"""Gets the largest public rooms (where largest is in terms of joined
	131	members, as tracked in the statistics table).
	132
	133	Args:
	134	network_tuple
	135	search_filter
	136	limit: Maxmimum number of rows to return, unlimited otherwise.
	137	bounds: An uppoer or lower bound to apply to result set if given,
	138	consists of a joined member count and room_id (these are
	139	excluded from result set).
	140	forwards: true iff going forwards, going backwards otherwise
	141	ignore_non_federatable: If true filters out non-federatable rooms.
	142
	143	Returns:
	144	Rooms in order: biggest number of joined users first.
	145	We then arbitrarily use the room_id as a tie breaker.
	146
	147	"""
	148
	149	where_clauses = []
	150	query_args = []
	151
	152	if network_tuple:
	153	if network_tuple.appservice_id:
	154	published_sql = """
	155	SELECT room_id from appservice_room_list
	156	WHERE appservice_id = ? AND network_id = ?
	157	"""
	158	query_args.append(network_tuple.appservice_id)
	159	query_args.append(network_tuple.network_id)
	160	else:
	161	published_sql = """
	162	SELECT room_id FROM rooms WHERE is_public
	163	"""
	164	else:
	165	published_sql = """
	166	SELECT room_id FROM rooms WHERE is_public
	167	UNION SELECT room_id from appservice_room_list
	168	"""
	169
	170	# Work out the bounds if we're given them, these bounds look slightly
	171	# odd, but are designed to help query planner use indices by pulling
	172	# out a common bound.
	173	if bounds:
	174	last_joined_members, last_room_id = bounds
	175	if forwards:
	176	where_clauses.append(
	177	"""
	178	joined_members <= ? AND (
	179	joined_members < ? OR room_id < ?
	180	)
	181	"""
	182	)
	183	else:
	184	where_clauses.append(
	185	"""
	186	joined_members >= ? AND (
	187	joined_members > ? OR room_id > ?
	188	)
	189	"""
	190	)
	191
	192	query_args += [last_joined_members, last_joined_members, last_room_id]
	193
	194	if ignore_non_federatable:
	195	where_clauses.append("is_federatable")
	196
	197	if search_filter and search_filter.get("generic_search_term", None):
	198	search_term = "%" + search_filter["generic_search_term"] + "%"
	199
	200	where_clauses.append(
	201	"""
	202	(
	203	name LIKE ?
	204	OR topic LIKE ?
	205	OR canonical_alias LIKE ?
	206	)
	207	"""
	208	)
	209	query_args += [search_term, search_term, search_term]
	210
	211	where_clause = ""
	212	if where_clauses:
	213	where_clause = " AND " + " AND ".join(where_clauses)
	214
	215	sql = """
	216	SELECT
	217	room_id, name, topic, canonical_alias, joined_members,
	218	avatar, history_visibility, joined_members, guest_access
	219	FROM (
	220	%(published_sql)s
	221	) published
	222	INNER JOIN room_stats_state USING (room_id)
	223	INNER JOIN room_stats_current USING (room_id)
	224	WHERE
	225	(
	226	join_rules = 'public' OR history_visibility = 'world_readable'
	227	)
	228	AND joined_members > 0
	229	%(where_clause)s
	230	ORDER BY joined_members %(dir)s, room_id %(dir)s
	231	""" % {
	232	"published_sql": published_sql,
	233	"where_clause": where_clause,
	234	"dir": "DESC" if forwards else "ASC",
	235	}
	236
	237	if limit is not None:
	238	query_args.append(limit)
	239
	240	sql += """
	241	LIMIT ?
	242	"""
	243
	244	def _get_largest_public_rooms_txn(txn):
	245	txn.execute(sql, query_args)
	246
	247	results = self.cursor_to_dict(txn)
	248
	249	if not forwards:
	250	results.reverse()
	251
	252	return results
	253
	254	ret_val = yield self.runInteraction(
	255	"get_largest_public_rooms", _get_largest_public_rooms_txn
	256	)
	257	defer.returnValue(ret_val)
	258
	259	@cached(max_entries=10000)
	260	def is_room_blocked(self, room_id):
	261	return self._simple_select_one_onecol(
	262	table="blocked_rooms",
	263	keyvalues={"room_id": room_id},
	264	retcol="1",
	265	allow_none=True,
	266	desc="is_room_blocked",
	267	)
	268
	269	@cachedInlineCallbacks(max_entries=10000)
	270	def get_ratelimit_for_user(self, user_id):
	271	"""Check if there are any overrides for ratelimiting for the given
	272	user
	273
	274	Args:
	275	user_id (str)
	276
	277	Returns:
	278	RatelimitOverride if there is an override, else None. If the contents
	279	of RatelimitOverride are None or 0 then ratelimitng has been
	280	disabled for that user entirely.
	281	"""
	282	row = yield self._simple_select_one(
	283	table="ratelimit_override",
	284	keyvalues={"user_id": user_id},
	285	retcols=("messages_per_second", "burst_count"),
	286	allow_none=True,
	287	desc="get_ratelimit_for_user",
	288	)
	289
	290	if row:
	291	return RatelimitOverride(
	292	messages_per_second=row["messages_per_second"],
	293	burst_count=row["burst_count"],
	294	)
	295	else:
	296	return None
	297
	298
	299	class RoomStore(RoomWorkerStore, SearchStore):
	300	@defer.inlineCallbacks
	301	def store_room(self, room_id, room_creator_user_id, is_public):
	302	"""Stores a room.
	303
	304	Args:
	305	room_id (str): The desired room ID, can be None.
	306	room_creator_user_id (str): The user ID of the room creator.
	307	is_public (bool): True to indicate that this room should appear in
	308	public room lists.
	309	Raises:
	310	StoreError if the room could not be stored.
	311	"""
	312	try:
	313
	314	def store_room_txn(txn, next_id):
	315	self._simple_insert_txn(
	316	txn,
	317	"rooms",
	318	{
	319	"room_id": room_id,
	320	"creator": room_creator_user_id,
	321	"is_public": is_public,
	322	},
	323	)
	324	if is_public:
	325	self._simple_insert_txn(
	326	txn,
	327	table="public_room_list_stream",
	328	values={
	329	"stream_id": next_id,
	330	"room_id": room_id,
	331	"visibility": is_public,
	332	},
	333	)
	334
	335	with self._public_room_id_gen.get_next() as next_id:
	336	yield self.runInteraction("store_room_txn", store_room_txn, next_id)
	337	except Exception as e:
	338	logger.error("store_room with room_id=%s failed: %s", room_id, e)
	339	raise StoreError(500, "Problem creating room.")
	340
	341	@defer.inlineCallbacks
	342	def set_room_is_public(self, room_id, is_public):
	343	def set_room_is_public_txn(txn, next_id):
	344	self._simple_update_one_txn(
	345	txn,
	346	table="rooms",
	347	keyvalues={"room_id": room_id},
	348	updatevalues={"is_public": is_public},
	349	)
	350
	351	entries = self._simple_select_list_txn(
	352	txn,
	353	table="public_room_list_stream",
	354	keyvalues={
	355	"room_id": room_id,
	356	"appservice_id": None,
	357	"network_id": None,
	358	},
	359	retcols=("stream_id", "visibility"),
	360	)
	361
	362	entries.sort(key=lambda r: r["stream_id"])
	363
	364	add_to_stream = True
	365	if entries:
	366	add_to_stream = bool(entries[-1]["visibility"]) != is_public
	367
	368	if add_to_stream:
	369	self._simple_insert_txn(
	370	txn,
	371	table="public_room_list_stream",
	372	values={
	373	"stream_id": next_id,
	374	"room_id": room_id,
	375	"visibility": is_public,
	376	"appservice_id": None,
	377	"network_id": None,
	378	},
	379	)
	380
	381	with self._public_room_id_gen.get_next() as next_id:
	382	yield self.runInteraction(
	383	"set_room_is_public", set_room_is_public_txn, next_id
	384	)
	385	self.hs.get_notifier().on_new_replication_data()
	386
	387	@defer.inlineCallbacks
	388	def set_room_is_public_appservice(
	389	self, room_id, appservice_id, network_id, is_public
	390	):
	391	"""Edit the appservice/network specific public room list.
	392
	393	Each appservice can have a number of published room lists associated
	394	with them, keyed off of an appservice defined `network_id`, which
	395	basically represents a single instance of a bridge to a third party
	396	network.
	397
	398	Args:
	399	room_id (str)
	400	appservice_id (str)
	401	network_id (str)
	402	is_public (bool): Whether to publish or unpublish the room from the
	403	list.
	404	"""
	405
	406	def set_room_is_public_appservice_txn(txn, next_id):
	407	if is_public:
	408	try:
	409	self._simple_insert_txn(
	410	txn,
	411	table="appservice_room_list",
	412	values={
	413	"appservice_id": appservice_id,
	414	"network_id": network_id,
	415	"room_id": room_id,
	416	},
	417	)
	418	except self.database_engine.module.IntegrityError:
	419	# We've already inserted, nothing to do.
	420	return
	421	else:
	422	self._simple_delete_txn(
	423	txn,
	424	table="appservice_room_list",
	425	keyvalues={
	426	"appservice_id": appservice_id,
	427	"network_id": network_id,
	428	"room_id": room_id,
	429	},
	430	)
	431
	432	entries = self._simple_select_list_txn(
	433	txn,
	434	table="public_room_list_stream",
	435	keyvalues={
	436	"room_id": room_id,
	437	"appservice_id": appservice_id,
	438	"network_id": network_id,
	439	},
	440	retcols=("stream_id", "visibility"),
	441	)
	442
	443	entries.sort(key=lambda r: r["stream_id"])
	444
	445	add_to_stream = True
	446	if entries:
	447	add_to_stream = bool(entries[-1]["visibility"]) != is_public
	448
	449	if add_to_stream:
	450	self._simple_insert_txn(
	451	txn,
	452	table="public_room_list_stream",
	453	values={
	454	"stream_id": next_id,
	455	"room_id": room_id,
	456	"visibility": is_public,
	457	"appservice_id": appservice_id,
	458	"network_id": network_id,
	459	},
	460	)
	461
	462	with self._public_room_id_gen.get_next() as next_id:
	463	yield self.runInteraction(
	464	"set_room_is_public_appservice",
	465	set_room_is_public_appservice_txn,
	466	next_id,
	467	)
	468	self.hs.get_notifier().on_new_replication_data()
	469
	470	def get_room_count(self):
	471	"""Retrieve a list of all rooms
	472	"""
	473
	474	def f(txn):
	475	sql = "SELECT count(*) FROM rooms"
	476	txn.execute(sql)
	477	row = txn.fetchone()
	478	return row[0] or 0
	479
	480	return self.runInteraction("get_rooms", f)
	481
	482	def _store_room_topic_txn(self, txn, event):
	483	if hasattr(event, "content") and "topic" in event.content:
	484	self.store_event_search_txn(
	485	txn, event, "content.topic", event.content["topic"]
	486	)
	487
	488	def _store_room_name_txn(self, txn, event):
	489	if hasattr(event, "content") and "name" in event.content:
	490	self.store_event_search_txn(
	491	txn, event, "content.name", event.content["name"]
	492	)
	493
	494	def _store_room_message_txn(self, txn, event):
	495	if hasattr(event, "content") and "body" in event.content:
	496	self.store_event_search_txn(
	497	txn, event, "content.body", event.content["body"]
	498	)
	499
	500	def add_event_report(
	501	self, room_id, event_id, user_id, reason, content, received_ts
	502	):
	503	next_id = self._event_reports_id_gen.get_next()
	504	return self._simple_insert(
	505	table="event_reports",
	506	values={
	507	"id": next_id,
	508	"received_ts": received_ts,
	509	"room_id": room_id,
	510	"event_id": event_id,
	511	"user_id": user_id,
	512	"reason": reason,
	513	"content": json.dumps(content),
	514	},
	515	desc="add_event_report",
	516	)
	517
	518	def get_current_public_room_stream_id(self):
	519	return self._public_room_id_gen.get_current_token()
	520
	521	def get_all_new_public_rooms(self, prev_id, current_id, limit):
	522	def get_all_new_public_rooms(txn):
	523	sql = """
	524	SELECT stream_id, room_id, visibility, appservice_id, network_id
	525	FROM public_room_list_stream
	526	WHERE stream_id > ? AND stream_id <= ?
	527	ORDER BY stream_id ASC
	528	LIMIT ?
	529	"""
	530
	531	txn.execute(sql, (prev_id, current_id, limit))
	532	return txn.fetchall()
	533
	534	if prev_id == current_id:
	535	return defer.succeed([])
	536
	537	return self.runInteraction("get_all_new_public_rooms", get_all_new_public_rooms)
	538
	539	@defer.inlineCallbacks
	540	def block_room(self, room_id, user_id):
	541	"""Marks the room as blocked. Can be called multiple times.
	542
	543	Args:
	544	room_id (str): Room to block
	545	user_id (str): Who blocked it
	546
	547	Returns:
	548	Deferred
	549	"""
	550	yield self._simple_upsert(
	551	table="blocked_rooms",
	552	keyvalues={"room_id": room_id},
	553	values={},
	554	insertion_values={"user_id": user_id},
	555	desc="block_room",
	556	)
	557	yield self.runInteraction(
	558	"block_room_invalidation",
	559	self._invalidate_cache_and_stream,
	560	self.is_room_blocked,
	561	(room_id,),
	562	)
	563
	564	def get_media_mxcs_in_room(self, room_id):
	565	"""Retrieves all the local and remote media MXC URIs in a given room
	566
	567	Args:
	568	room_id (str)
	569
	570	Returns:
	571	The local and remote media as a lists of tuples where the key is
	572	the hostname and the value is the media ID.
	573	"""
	574
	575	def _get_media_mxcs_in_room_txn(txn):
	576	local_mxcs, remote_mxcs = self._get_media_mxcs_in_room_txn(txn, room_id)
	577	local_media_mxcs = []
	578	remote_media_mxcs = []
	579
	580	# Convert the IDs to MXC URIs
	581	for media_id in local_mxcs:
	582	local_media_mxcs.append("mxc://%s/%s" % (self.hs.hostname, media_id))
	583	for hostname, media_id in remote_mxcs:
	584	remote_media_mxcs.append("mxc://%s/%s" % (hostname, media_id))
	585
	586	return local_media_mxcs, remote_media_mxcs
	587
	588	return self.runInteraction("get_media_ids_in_room", _get_media_mxcs_in_room_txn)
	589
	590	def quarantine_media_ids_in_room(self, room_id, quarantined_by):
	591	"""For a room loops through all events with media and quarantines
	592	the associated media
	593	"""
	594
	595	def _quarantine_media_in_room_txn(txn):
	596	local_mxcs, remote_mxcs = self._get_media_mxcs_in_room_txn(txn, room_id)
	597	total_media_quarantined = 0
	598
	599	# Now update all the tables to set the quarantined_by flag
	600
	601	txn.executemany(
	602	"""
	603	UPDATE local_media_repository
	604	SET quarantined_by = ?
	605	WHERE media_id = ?
	606	""",
	607	((quarantined_by, media_id) for media_id in local_mxcs),
	608	)
	609
	610	txn.executemany(
	611	"""
	612	UPDATE remote_media_cache
	613	SET quarantined_by = ?
	614	WHERE media_origin = ? AND media_id = ?
	615	""",
	616	(
	617	(quarantined_by, origin, media_id)
	618	for origin, media_id in remote_mxcs
	619	),
	620	)
	621
	622	total_media_quarantined += len(local_mxcs)
	623	total_media_quarantined += len(remote_mxcs)
	624
	625	return total_media_quarantined
	626
	627	return self.runInteraction(
	628	"quarantine_media_in_room", _quarantine_media_in_room_txn
	629	)
	630
	631	def _get_media_mxcs_in_room_txn(self, txn, room_id):
	632	"""Retrieves all the local and remote media MXC URIs in a given room
	633
	634	Args:
	635	txn (cursor)
	636	room_id (str)
	637
	638	Returns:
	639	The local and remote media as a lists of tuples where the key is
	640	the hostname and the value is the media ID.
	641	"""
	642	mxc_re = re.compile("^mxc://([^/]+)/([^/#?]+)")
	643
	644	next_token = self.get_current_events_token() + 1
	645	local_media_mxcs = []
	646	remote_media_mxcs = []
	647
	648	while next_token:
	649	sql = """
	650	SELECT stream_ordering, json FROM events
	651	JOIN event_json USING (room_id, event_id)
	652	WHERE room_id = ?
	653	AND stream_ordering < ?
	654	AND contains_url = ? AND outlier = ?
	655	ORDER BY stream_ordering DESC
	656	LIMIT ?
	657	"""
	658	txn.execute(sql, (room_id, next_token, True, False, 100))
	659
	660	next_token = None
	661	for stream_ordering, content_json in txn:
	662	next_token = stream_ordering
	663	event_json = json.loads(content_json)
	664	content = event_json["content"]
	665	content_url = content.get("url")
	666	thumbnail_url = content.get("info", {}).get("thumbnail_url")
	667
	668	for url in (content_url, thumbnail_url):
	669	if not url:
	670	continue
	671	matches = mxc_re.match(url)
	672	if matches:
	673	hostname = matches.group(1)
	674	media_id = matches.group(2)
	675	if hostname == self.hs.hostname:
	676	local_media_mxcs.append(media_id)
	677	else:
	678	remote_media_mxcs.append((hostname, media_id))
	679
	680	return local_media_mxcs, remote_media_mxcs

+1145

-0

synapse/storage/data_stores/main/roommember.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2014-2016 OpenMarket Ltd
	2	# Copyright 2018 New Vector Ltd
	3	#
	4	# Licensed under the Apache License, Version 2.0 (the "License");
	5	# you may not use this file except in compliance with the License.
	6	# You may obtain a copy of the License at
	7	#
	8	# http://www.apache.org/licenses/LICENSE-2.0
	9	#
	10	# Unless required by applicable law or agreed to in writing, software
	11	# distributed under the License is distributed on an "AS IS" BASIS,
	12	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	# See the License for the specific language governing permissions and
	14	# limitations under the License.
	15
	16	import logging
	17
	18	from six import iteritems, itervalues
	19
	20	from canonicaljson import json
	21
	22	from twisted.internet import defer
	23
	24	from synapse.api.constants import EventTypes, Membership
	25	from synapse.metrics import LaterGauge
	26	from synapse.metrics.background_process_metrics import run_as_background_process
	27	from synapse.storage._base import LoggingTransaction, make_in_list_sql_clause
	28	from synapse.storage.background_updates import BackgroundUpdateStore
	29	from synapse.storage.data_stores.main.events_worker import EventsWorkerStore
	30	from synapse.storage.engines import Sqlite3Engine
	31	from synapse.storage.roommember import (
	32	GetRoomsForUserWithStreamOrdering,
	33	MemberSummary,
	34	ProfileInfo,
	35	RoomsForUser,
	36	)
	37	from synapse.types import get_domain_from_id
	38	from synapse.util.async_helpers import Linearizer
	39	from synapse.util.caches import intern_string
	40	from synapse.util.caches.descriptors import cached, cachedInlineCallbacks, cachedList
	41	from synapse.util.metrics import Measure
	42	from synapse.util.stringutils import to_ascii
	43
	44	logger = logging.getLogger(__name__)
	45
	46
	47	_MEMBERSHIP_PROFILE_UPDATE_NAME = "room_membership_profile_update"
	48	_CURRENT_STATE_MEMBERSHIP_UPDATE_NAME = "current_state_events_membership"
	49
	50
	51	class RoomMemberWorkerStore(EventsWorkerStore):
	52	def __init__(self, db_conn, hs):
	53	super(RoomMemberWorkerStore, self).__init__(db_conn, hs)
	54
	55	# Is the current_state_events.membership up to date? Or is the
	56	# background update still running?
	57	self._current_state_events_membership_up_to_date = False
	58
	59	txn = LoggingTransaction(
	60	db_conn.cursor(),
	61	name="_check_safe_current_state_events_membership_updated",
	62	database_engine=self.database_engine,
	63	)
	64	self._check_safe_current_state_events_membership_updated_txn(txn)
	65	txn.close()
	66
	67	if self.hs.config.metrics_flags.known_servers:
	68	self._known_servers_count = 1
	69	self.hs.get_clock().looping_call(
	70	run_as_background_process,
	71	60 * 1000,
	72	"_count_known_servers",
	73	self._count_known_servers,
	74	)
	75	self.hs.get_clock().call_later(
	76	1000,
	77	run_as_background_process,
	78	"_count_known_servers",
	79	self._count_known_servers,
	80	)
	81	LaterGauge(
	82	"synapse_federation_known_servers",
	83	"",
	84	[],
	85	lambda: self._known_servers_count,
	86	)
	87
	88	@defer.inlineCallbacks
	89	def _count_known_servers(self):
	90	"""
	91	Count the servers that this server knows about.
	92
	93	The statistic is stored on the class for the
	94	`synapse_federation_known_servers` LaterGauge to collect.
	95	"""
	96
	97	def _transact(txn):
	98	if isinstance(self.database_engine, Sqlite3Engine):
	99	query = """
	100	SELECT COUNT(DISTINCT substr(out.user_id, pos+1))
	101	FROM (
	102	SELECT rm.user_id as user_id, instr(rm.user_id, ':')
	103	AS pos FROM room_memberships as rm
	104	INNER JOIN current_state_events as c ON rm.event_id = c.event_id
	105	WHERE c.type = 'm.room.member'
	106	) as out
	107	"""
	108	else:
	109	query = """
	110	SELECT COUNT(DISTINCT split_part(state_key, ':', 2))
	111	FROM current_state_events
	112	WHERE type = 'm.room.member' AND membership = 'join';
	113	"""
	114	txn.execute(query)
	115	return list(txn)[0][0]
	116
	117	count = yield self.runInteraction("get_known_servers", _transact)
	118
	119	# We always know about ourselves, even if we have nothing in
	120	# room_memberships (for example, the server is new).
	121	self._known_servers_count = max([count, 1])
	122	return self._known_servers_count
	123
	124	def _check_safe_current_state_events_membership_updated_txn(self, txn):
	125	"""Checks if it is safe to assume the new current_state_events
	126	membership column is up to date
	127	"""
	128
	129	pending_update = self._simple_select_one_txn(
	130	txn,
	131	table="background_updates",
	132	keyvalues={"update_name": _CURRENT_STATE_MEMBERSHIP_UPDATE_NAME},
	133	retcols=["update_name"],
	134	allow_none=True,
	135	)
	136
	137	self._current_state_events_membership_up_to_date = not pending_update
	138
	139	# If the update is still running, reschedule to run.
	140	if pending_update:
	141	self._clock.call_later(
	142	15.0,
	143	run_as_background_process,
	144	"_check_safe_current_state_events_membership_updated",
	145	self.runInteraction,
	146	"_check_safe_current_state_events_membership_updated",
	147	self._check_safe_current_state_events_membership_updated_txn,
	148	)
	149
	150	@cachedInlineCallbacks(max_entries=100000, iterable=True, cache_context=True)
	151	def get_hosts_in_room(self, room_id, cache_context):
	152	"""Returns the set of all hosts currently in the room
	153	"""
	154	user_ids = yield self.get_users_in_room(
	155	room_id, on_invalidate=cache_context.invalidate
	156	)
	157	hosts = frozenset(get_domain_from_id(user_id) for user_id in user_ids)
	158	return hosts
	159
	160	@cached(max_entries=100000, iterable=True)
	161	def get_users_in_room(self, room_id):
	162	return self.runInteraction(
	163	"get_users_in_room", self.get_users_in_room_txn, room_id
	164	)
	165
	166	def get_users_in_room_txn(self, txn, room_id):
	167	# If we can assume current_state_events.membership is up to date
	168	# then we can avoid a join, which is a Very Good Thing given how
	169	# frequently this function gets called.
	170	if self._current_state_events_membership_up_to_date:
	171	sql = """
	172	SELECT state_key FROM current_state_events
	173	WHERE type = 'm.room.member' AND room_id = ? AND membership = ?
	174	"""
	175	else:
	176	sql = """
	177	SELECT state_key FROM room_memberships as m
	178	INNER JOIN current_state_events as c
	179	ON m.event_id = c.event_id
	180	AND m.room_id = c.room_id
	181	AND m.user_id = c.state_key
	182	WHERE c.type = 'm.room.member' AND c.room_id = ? AND m.membership = ?
	183	"""
	184
	185	txn.execute(sql, (room_id, Membership.JOIN))
	186	return [to_ascii(r[0]) for r in txn]
	187
	188	@cached(max_entries=100000)
	189	def get_room_summary(self, room_id):
	190	""" Get the details of a room roughly suitable for use by the room
	191	summary extension to /sync. Useful when lazy loading room members.
	192	Args:
	193	room_id (str): The room ID to query
	194	Returns:
	195	Deferred[dict[str, MemberSummary]:
	196	dict of membership states, pointing to a MemberSummary named tuple.
	197	"""
	198
	199	def _get_room_summary_txn(txn):
	200	# first get counts.
	201	# We do this all in one transaction to keep the cache small.
	202	# FIXME: get rid of this when we have room_stats
	203
	204	# If we can assume current_state_events.membership is up to date
	205	# then we can avoid a join, which is a Very Good Thing given how
	206	# frequently this function gets called.
	207	if self._current_state_events_membership_up_to_date:
	208	# Note, rejected events will have a null membership field, so
	209	# we we manually filter them out.
	210	sql = """
	211	SELECT count(*), membership FROM current_state_events
	212	WHERE type = 'm.room.member' AND room_id = ?
	213	AND membership IS NOT NULL
	214	GROUP BY membership
	215	"""
	216	else:
	217	sql = """
	218	SELECT count(*), m.membership FROM room_memberships as m
	219	INNER JOIN current_state_events as c
	220	ON m.event_id = c.event_id
	221	AND m.room_id = c.room_id
	222	AND m.user_id = c.state_key
	223	WHERE c.type = 'm.room.member' AND c.room_id = ?
	224	GROUP BY m.membership
	225	"""
	226
	227	txn.execute(sql, (room_id,))
	228	res = {}
	229	for count, membership in txn:
	230	summary = res.setdefault(to_ascii(membership), MemberSummary([], count))
	231
	232	# we order by membership and then fairly arbitrarily by event_id so
	233	# heroes are consistent
	234	if self._current_state_events_membership_up_to_date:
	235	# Note, rejected events will have a null membership field, so
	236	# we we manually filter them out.
	237	sql = """
	238	SELECT state_key, membership, event_id
	239	FROM current_state_events
	240	WHERE type = 'm.room.member' AND room_id = ?
	241	AND membership IS NOT NULL
	242	ORDER BY
	243	CASE membership WHEN ? THEN 1 WHEN ? THEN 2 ELSE 3 END ASC,
	244	event_id ASC
	245	LIMIT ?
	246	"""
	247	else:
	248	sql = """
	249	SELECT c.state_key, m.membership, c.event_id
	250	FROM room_memberships as m
	251	INNER JOIN current_state_events as c USING (room_id, event_id)
	252	WHERE c.type = 'm.room.member' AND c.room_id = ?
	253	ORDER BY
	254	CASE m.membership WHEN ? THEN 1 WHEN ? THEN 2 ELSE 3 END ASC,
	255	c.event_id ASC
	256	LIMIT ?
	257	"""
	258
	259	# 6 is 5 (number of heroes) plus 1, in case one of them is the calling user.
	260	txn.execute(sql, (room_id, Membership.JOIN, Membership.INVITE, 6))
	261	for user_id, membership, event_id in txn:
	262	summary = res[to_ascii(membership)]
	263	# we will always have a summary for this membership type at this
	264	# point given the summary currently contains the counts.
	265	members = summary.members
	266	members.append((to_ascii(user_id), to_ascii(event_id)))
	267
	268	return res
	269
	270	return self.runInteraction("get_room_summary", _get_room_summary_txn)
	271
	272	def _get_user_counts_in_room_txn(self, txn, room_id):
	273	"""
	274	Get the user count in a room by membership.
	275
	276	Args:
	277	room_id (str)
	278	membership (Membership)
	279
	280	Returns:
	281	Deferred[int]
	282	"""
	283	sql = """
	284	SELECT m.membership, count(*) FROM room_memberships as m
	285	INNER JOIN current_state_events as c USING(event_id)
	286	WHERE c.type = 'm.room.member' AND c.room_id = ?
	287	GROUP BY m.membership
	288	"""
	289
	290	txn.execute(sql, (room_id,))
	291	return {row[0]: row[1] for row in txn}
	292
	293	@cached()
	294	def get_invited_rooms_for_user(self, user_id):
	295	""" Get all the rooms the user is invited to
	296	Args:
	297	user_id (str): The user ID.
	298	Returns:
	299	A deferred list of RoomsForUser.
	300	"""
	301
	302	return self.get_rooms_for_user_where_membership_is(user_id, [Membership.INVITE])
	303
	304	@defer.inlineCallbacks
	305	def get_invite_for_user_in_room(self, user_id, room_id):
	306	"""Gets the invite for the given user and room
	307
	308	Args:
	309	user_id (str)
	310	room_id (str)
	311
	312	Returns:
	313	Deferred: Resolves to either a RoomsForUser or None if no invite was
	314	found.
	315	"""
	316	invites = yield self.get_invited_rooms_for_user(user_id)
	317	for invite in invites:
	318	if invite.room_id == room_id:
	319	return invite
	320	return None
	321
	322	@defer.inlineCallbacks
	323	def get_rooms_for_user_where_membership_is(self, user_id, membership_list):
	324	""" Get all the rooms for this user where the membership for this user
	325	matches one in the membership list.
	326
	327	Filters out forgotten rooms.
	328
	329	Args:
	330	user_id (str): The user ID.
	331	membership_list (list): A list of synapse.api.constants.Membership
	332	values which the user must be in.
	333
	334	Returns:
	335	Deferred[list[RoomsForUser]]
	336	"""
	337	if not membership_list:
	338	return defer.succeed(None)
	339
	340	rooms = yield self.runInteraction(
	341	"get_rooms_for_user_where_membership_is",
	342	self._get_rooms_for_user_where_membership_is_txn,
	343	user_id,
	344	membership_list,
	345	)
	346
	347	# Now we filter out forgotten rooms
	348	forgotten_rooms = yield self.get_forgotten_rooms_for_user(user_id)
	349	return [room for room in rooms if room.room_id not in forgotten_rooms]
	350
	351	def _get_rooms_for_user_where_membership_is_txn(
	352	self, txn, user_id, membership_list
	353	):
	354
	355	do_invite = Membership.INVITE in membership_list
	356	membership_list = [m for m in membership_list if m != Membership.INVITE]
	357
	358	results = []
	359	if membership_list:
	360	if self._current_state_events_membership_up_to_date:
	361	clause, args = make_in_list_sql_clause(
	362	self.database_engine, "c.membership", membership_list
	363	)
	364	sql = """
	365	SELECT room_id, e.sender, c.membership, event_id, e.stream_ordering
	366	FROM current_state_events AS c
	367	INNER JOIN events AS e USING (room_id, event_id)
	368	WHERE
	369	c.type = 'm.room.member'
	370	AND state_key = ?
	371	AND %s
	372	""" % (
	373	clause,
	374	)
	375	else:
	376	clause, args = make_in_list_sql_clause(
	377	self.database_engine, "m.membership", membership_list
	378	)
	379	sql = """
	380	SELECT room_id, e.sender, m.membership, event_id, e.stream_ordering
	381	FROM current_state_events AS c
	382	INNER JOIN room_memberships AS m USING (room_id, event_id)
	383	INNER JOIN events AS e USING (room_id, event_id)
	384	WHERE
	385	c.type = 'm.room.member'
	386	AND state_key = ?
	387	AND %s
	388	""" % (
	389	clause,
	390	)
	391
	392	txn.execute(sql, (user_id, *args))
	393	results = [RoomsForUser(**r) for r in self.cursor_to_dict(txn)]
	394
	395	if do_invite:
	396	sql = (
	397	"SELECT i.room_id, inviter, i.event_id, e.stream_ordering"
	398	" FROM local_invites as i"
	399	" INNER JOIN events as e USING (event_id)"
	400	" WHERE invitee = ? AND locally_rejected is NULL"
	401	" AND replaced_by is NULL"
	402	)
	403
	404	txn.execute(sql, (user_id,))
	405	results.extend(
	406	RoomsForUser(
	407	room_id=r["room_id"],
	408	sender=r["inviter"],
	409	event_id=r["event_id"],
	410	stream_ordering=r["stream_ordering"],
	411	membership=Membership.INVITE,
	412	)
	413	for r in self.cursor_to_dict(txn)
	414	)
	415
	416	return results
	417
	418	@cachedInlineCallbacks(max_entries=500000, iterable=True)
	419	def get_rooms_for_user_with_stream_ordering(self, user_id):
	420	"""Returns a set of room_ids the user is currently joined to
	421
	422	Args:
	423	user_id (str)
	424
	425	Returns:
	426	Deferred[frozenset[GetRoomsForUserWithStreamOrdering]]: Returns
	427	the rooms the user is in currently, along with the stream ordering
	428	of the most recent join for that user and room.
	429	"""
	430	rooms = yield self.get_rooms_for_user_where_membership_is(
	431	user_id, membership_list=[Membership.JOIN]
	432	)
	433	return frozenset(
	434	GetRoomsForUserWithStreamOrdering(r.room_id, r.stream_ordering)
	435	for r in rooms
	436	)
	437
	438	@defer.inlineCallbacks
	439	def get_rooms_for_user(self, user_id, on_invalidate=None):
	440	"""Returns a set of room_ids the user is currently joined to
	441	"""
	442	rooms = yield self.get_rooms_for_user_with_stream_ordering(
	443	user_id, on_invalidate=on_invalidate
	444	)
	445	return frozenset(r.room_id for r in rooms)
	446
	447	@cachedInlineCallbacks(max_entries=500000, cache_context=True, iterable=True)
	448	def get_users_who_share_room_with_user(self, user_id, cache_context):
	449	"""Returns the set of users who share a room with `user_id`
	450	"""
	451	room_ids = yield self.get_rooms_for_user(
	452	user_id, on_invalidate=cache_context.invalidate
	453	)
	454
	455	user_who_share_room = set()
	456	for room_id in room_ids:
	457	user_ids = yield self.get_users_in_room(
	458	room_id, on_invalidate=cache_context.invalidate
	459	)
	460	user_who_share_room.update(user_ids)
	461
	462	return user_who_share_room
	463
	464	@defer.inlineCallbacks
	465	def get_joined_users_from_context(self, event, context):
	466	state_group = context.state_group
	467	if not state_group:
	468	# If state_group is None it means it has yet to be assigned a
	469	# state group, i.e. we need to make sure that calls with a state_group
	470	# of None don't hit previous cached calls with a None state_group.
	471	# To do this we set the state_group to a new object as object() != object()
	472	state_group = object()
	473
	474	current_state_ids = yield context.get_current_state_ids(self)
	475	result = yield self._get_joined_users_from_context(
	476	event.room_id, state_group, current_state_ids, event=event, context=context
	477	)
	478	return result
	479
	480	@defer.inlineCallbacks
	481	def get_joined_users_from_state(self, room_id, state_entry):
	482	state_group = state_entry.state_group
	483	if not state_group:
	484	# If state_group is None it means it has yet to be assigned a
	485	# state group, i.e. we need to make sure that calls with a state_group
	486	# of None don't hit previous cached calls with a None state_group.
	487	# To do this we set the state_group to a new object as object() != object()
	488	state_group = object()
	489
	490	with Measure(self._clock, "get_joined_users_from_state"):
	491	return (
	492	yield self._get_joined_users_from_context(
	493	room_id, state_group, state_entry.state, context=state_entry
	494	)
	495	)
	496
	497	@cachedInlineCallbacks(
	498	num_args=2, cache_context=True, iterable=True, max_entries=100000
	499	)
	500	def _get_joined_users_from_context(
	501	self,
	502	room_id,
	503	state_group,
	504	current_state_ids,
	505	cache_context,
	506	event=None,
	507	context=None,
	508	):
	509	# We don't use `state_group`, it's there so that we can cache based
	510	# on it. However, it's important that it's never None, since two current_states
	511	# with a state_group of None are likely to be different.
	512	# See bulk_get_push_rules_for_room for how we work around this.
	513	assert state_group is not None
	514
	515	users_in_room = {}
	516	member_event_ids = [
	517	e_id
	518	for key, e_id in iteritems(current_state_ids)
	519	if key[0] == EventTypes.Member
	520	]
	521
	522	if context is not None:
	523	# If we have a context with a delta from a previous state group,
	524	# check if we also have the result from the previous group in cache.
	525	# If we do then we can reuse that result and simply update it with
	526	# any membership changes in `delta_ids`
	527	if context.prev_group and context.delta_ids:
	528	prev_res = self._get_joined_users_from_context.cache.get(
	529	(room_id, context.prev_group), None
	530	)
	531	if prev_res and isinstance(prev_res, dict):
	532	users_in_room = dict(prev_res)
	533	member_event_ids = [
	534	e_id
	535	for key, e_id in iteritems(context.delta_ids)
	536	if key[0] == EventTypes.Member
	537	]
	538	for etype, state_key in context.delta_ids:
	539	users_in_room.pop(state_key, None)
	540
	541	# We check if we have any of the member event ids in the event cache
	542	# before we ask the DB
	543
	544	# We don't update the event cache hit ratio as it completely throws off
	545	# the hit ratio counts. After all, we don't populate the cache if we
	546	# miss it here
	547	event_map = self._get_events_from_cache(
	548	member_event_ids, allow_rejected=False, update_metrics=False
	549	)
	550
	551	missing_member_event_ids = []
	552	for event_id in member_event_ids:
	553	ev_entry = event_map.get(event_id)
	554	if ev_entry:
	555	if ev_entry.event.membership == Membership.JOIN:
	556	users_in_room[to_ascii(ev_entry.event.state_key)] = ProfileInfo(
	557	display_name=to_ascii(
	558	ev_entry.event.content.get("displayname", None)
	559	),
	560	avatar_url=to_ascii(
	561	ev_entry.event.content.get("avatar_url", None)
	562	),
	563	)
	564	else:
	565	missing_member_event_ids.append(event_id)
	566
	567	if missing_member_event_ids:
	568	event_to_memberships = yield self._get_joined_profiles_from_event_ids(
	569	missing_member_event_ids
	570	)
	571	users_in_room.update((row for row in event_to_memberships.values() if row))
	572
	573	if event is not None and event.type == EventTypes.Member:
	574	if event.membership == Membership.JOIN:
	575	if event.event_id in member_event_ids:
	576	users_in_room[to_ascii(event.state_key)] = ProfileInfo(
	577	display_name=to_ascii(event.content.get("displayname", None)),
	578	avatar_url=to_ascii(event.content.get("avatar_url", None)),
	579	)
	580
	581	return users_in_room
	582
	583	@cached(max_entries=10000)
	584	def _get_joined_profile_from_event_id(self, event_id):
	585	raise NotImplementedError()
	586
	587	@cachedList(
	588	cached_method_name="_get_joined_profile_from_event_id",
	589	list_name="event_ids",
	590	inlineCallbacks=True,
	591	)
	592	def _get_joined_profiles_from_event_ids(self, event_ids):
	593	"""For given set of member event_ids check if they point to a join
	594	event and if so return the associated user and profile info.
	595
	596	Args:
	597	event_ids (Iterable[str]): The member event IDs to lookup
	598
	599	Returns:
	600	Deferred[dict[str, Tuple[str, ProfileInfo]\|None]]: Map from event ID
	601	to `user_id` and ProfileInfo (or None if not join event).
	602	"""
	603
	604	rows = yield self._simple_select_many_batch(
	605	table="room_memberships",
	606	column="event_id",
	607	iterable=event_ids,
	608	retcols=("user_id", "display_name", "avatar_url", "event_id"),
	609	keyvalues={"membership": Membership.JOIN},
	610	batch_size=500,
	611	desc="_get_membership_from_event_ids",
	612	)
	613
	614	return {
	615	row["event_id"]: (
	616	row["user_id"],
	617	ProfileInfo(
	618	avatar_url=row["avatar_url"], display_name=row["display_name"]
	619	),
	620	)
	621	for row in rows
	622	}
	623
	624	@cachedInlineCallbacks(max_entries=10000)
	625	def is_host_joined(self, room_id, host):
	626	if "%" in host or "_" in host:
	627	raise Exception("Invalid host name")
	628
	629	sql = """
	630	SELECT state_key FROM current_state_events AS c
	631	INNER JOIN room_memberships AS m USING (event_id)
	632	WHERE m.membership = 'join'
	633	AND type = 'm.room.member'
	634	AND c.room_id = ?
	635	AND state_key LIKE ?
	636	LIMIT 1
	637	"""
	638
	639	# We do need to be careful to ensure that host doesn't have any wild cards
	640	# in it, but we checked above for known ones and we'll check below that
	641	# the returned user actually has the correct domain.
	642	like_clause = "%:" + host
	643
	644	rows = yield self._execute("is_host_joined", None, sql, room_id, like_clause)
	645
	646	if not rows:
	647	return False
	648
	649	user_id = rows[0][0]
	650	if get_domain_from_id(user_id) != host:
	651	# This can only happen if the host name has something funky in it
	652	raise Exception("Invalid host name")
	653
	654	return True
	655
	656	@cachedInlineCallbacks()
	657	def was_host_joined(self, room_id, host):
	658	"""Check whether the server is or ever was in the room.
	659
	660	Args:
	661	room_id (str)
	662	host (str)
	663
	664	Returns:
	665	Deferred: Resolves to True if the host is/was in the room, otherwise
	666	False.
	667	"""
	668	if "%" in host or "_" in host:
	669	raise Exception("Invalid host name")
	670
	671	sql = """
	672	SELECT user_id FROM room_memberships
	673	WHERE room_id = ?
	674	AND user_id LIKE ?
	675	AND membership = 'join'
	676	LIMIT 1
	677	"""
	678
	679	# We do need to be careful to ensure that host doesn't have any wild cards
	680	# in it, but we checked above for known ones and we'll check below that
	681	# the returned user actually has the correct domain.
	682	like_clause = "%:" + host
	683
	684	rows = yield self._execute("was_host_joined", None, sql, room_id, like_clause)
	685
	686	if not rows:
	687	return False
	688
	689	user_id = rows[0][0]
	690	if get_domain_from_id(user_id) != host:
	691	# This can only happen if the host name has something funky in it
	692	raise Exception("Invalid host name")
	693
	694	return True
	695
	696	@defer.inlineCallbacks
	697	def get_joined_hosts(self, room_id, state_entry):
	698	state_group = state_entry.state_group
	699	if not state_group:
	700	# If state_group is None it means it has yet to be assigned a
	701	# state group, i.e. we need to make sure that calls with a state_group
	702	# of None don't hit previous cached calls with a None state_group.
	703	# To do this we set the state_group to a new object as object() != object()
	704	state_group = object()
	705
	706	with Measure(self._clock, "get_joined_hosts"):
	707	return (
	708	yield self._get_joined_hosts(
	709	room_id, state_group, state_entry.state, state_entry=state_entry
	710	)
	711	)
	712
	713	@cachedInlineCallbacks(num_args=2, max_entries=10000, iterable=True)
	714	# @defer.inlineCallbacks
	715	def _get_joined_hosts(self, room_id, state_group, current_state_ids, state_entry):
	716	# We don't use `state_group`, its there so that we can cache based
	717	# on it. However, its important that its never None, since two current_state's
	718	# with a state_group of None are likely to be different.
	719	# See bulk_get_push_rules_for_room for how we work around this.
	720	assert state_group is not None
	721
	722	cache = self._get_joined_hosts_cache(room_id)
	723	joined_hosts = yield cache.get_destinations(state_entry)
	724
	725	return joined_hosts
	726
	727	@cached(max_entries=10000)
	728	def _get_joined_hosts_cache(self, room_id):
	729	return _JoinedHostsCache(self, room_id)
	730
	731	@cachedInlineCallbacks(num_args=2)
	732	def did_forget(self, user_id, room_id):
	733	"""Returns whether user_id has elected to discard history for room_id.
	734
	735	Returns False if they have since re-joined."""
	736
	737	def f(txn):
	738	sql = (
	739	"SELECT"
	740	" COUNT(*)"
	741	" FROM"
	742	" room_memberships"
	743	" WHERE"
	744	" user_id = ?"
	745	" AND"
	746	" room_id = ?"
	747	" AND"
	748	" forgotten = 0"
	749	)
	750	txn.execute(sql, (user_id, room_id))
	751	rows = txn.fetchall()
	752	return rows[0][0]
	753
	754	count = yield self.runInteraction("did_forget_membership", f)
	755	return count == 0
	756
	757	@cached()
	758	def get_forgotten_rooms_for_user(self, user_id):
	759	"""Gets all rooms the user has forgotten.
	760
	761	Args:
	762	user_id (str)
	763
	764	Returns:
	765	Deferred[set[str]]
	766	"""
	767
	768	def _get_forgotten_rooms_for_user_txn(txn):
	769	# This is a slightly convoluted query that first looks up all rooms
	770	# that the user has forgotten in the past, then rechecks that list
	771	# to see if any have subsequently been updated. This is done so that
	772	# we can use a partial index on `forgotten = 1` on the assumption
	773	# that few users will actually forget many rooms.
	774	#
	775	# Note that a room is considered "forgotten" if all membership
	776	# events for that user and room have the forgotten field set (as
	777	# when a user forgets a room we update all rows for that user and
	778	# room, not just the current one).
	779	sql = """
	780	SELECT room_id, (
	781	SELECT count(*) FROM room_memberships
	782	WHERE room_id = m.room_id AND user_id = m.user_id AND forgotten = 0
	783	) AS count
	784	FROM room_memberships AS m
	785	WHERE user_id = ? AND forgotten = 1
	786	GROUP BY room_id, user_id;
	787	"""
	788	txn.execute(sql, (user_id,))
	789	return set(row[0] for row in txn if row[1] == 0)
	790
	791	return self.runInteraction(
	792	"get_forgotten_rooms_for_user", _get_forgotten_rooms_for_user_txn
	793	)
	794
	795	@defer.inlineCallbacks
	796	def get_rooms_user_has_been_in(self, user_id):
	797	"""Get all rooms that the user has ever been in.
	798
	799	Args:
	800	user_id (str)
	801
	802	Returns:
	803	Deferred[set[str]]: Set of room IDs.
	804	"""
	805
	806	room_ids = yield self._simple_select_onecol(
	807	table="room_memberships",
	808	keyvalues={"membership": Membership.JOIN, "user_id": user_id},
	809	retcol="room_id",
	810	desc="get_rooms_user_has_been_in",
	811	)
	812
	813	return set(room_ids)
	814
	815
	816	class RoomMemberBackgroundUpdateStore(BackgroundUpdateStore):
	817	def __init__(self, db_conn, hs):
	818	super(RoomMemberBackgroundUpdateStore, self).__init__(db_conn, hs)
	819	self.register_background_update_handler(
	820	_MEMBERSHIP_PROFILE_UPDATE_NAME, self._background_add_membership_profile
	821	)
	822	self.register_background_update_handler(
	823	_CURRENT_STATE_MEMBERSHIP_UPDATE_NAME,
	824	self._background_current_state_membership,
	825	)
	826	self.register_background_index_update(
	827	"room_membership_forgotten_idx",
	828	index_name="room_memberships_user_room_forgotten",
	829	table="room_memberships",
	830	columns=["user_id", "room_id"],
	831	where_clause="forgotten = 1",
	832	)
	833
	834	@defer.inlineCallbacks
	835	def _background_add_membership_profile(self, progress, batch_size):
	836	target_min_stream_id = progress.get(
	837	"target_min_stream_id_inclusive", self._min_stream_order_on_start
	838	)
	839	max_stream_id = progress.get(
	840	"max_stream_id_exclusive", self._stream_order_on_start + 1
	841	)
	842
	843	INSERT_CLUMP_SIZE = 1000
	844
	845	def add_membership_profile_txn(txn):
	846	sql = """
	847	SELECT stream_ordering, event_id, events.room_id, event_json.json
	848	FROM events
	849	INNER JOIN event_json USING (event_id)
	850	INNER JOIN room_memberships USING (event_id)
	851	WHERE ? <= stream_ordering AND stream_ordering < ?
	852	AND type = 'm.room.member'
	853	ORDER BY stream_ordering DESC
	854	LIMIT ?
	855	"""
	856
	857	txn.execute(sql, (target_min_stream_id, max_stream_id, batch_size))
	858
	859	rows = self.cursor_to_dict(txn)
	860	if not rows:
	861	return 0
	862
	863	min_stream_id = rows[-1]["stream_ordering"]
	864
	865	to_update = []
	866	for row in rows:
	867	event_id = row["event_id"]
	868	room_id = row["room_id"]
	869	try:
	870	event_json = json.loads(row["json"])
	871	content = event_json["content"]
	872	except Exception:
	873	continue
	874
	875	display_name = content.get("displayname", None)
	876	avatar_url = content.get("avatar_url", None)
	877
	878	if display_name or avatar_url:
	879	to_update.append((display_name, avatar_url, event_id, room_id))
	880
	881	to_update_sql = """
	882	UPDATE room_memberships SET display_name = ?, avatar_url = ?
	883	WHERE event_id = ? AND room_id = ?
	884	"""
	885	for index in range(0, len(to_update), INSERT_CLUMP_SIZE):
	886	clump = to_update[index : index + INSERT_CLUMP_SIZE]
	887	txn.executemany(to_update_sql, clump)
	888
	889	progress = {
	890	"target_min_stream_id_inclusive": target_min_stream_id,
	891	"max_stream_id_exclusive": min_stream_id,
	892	}
	893
	894	self._background_update_progress_txn(
	895	txn, _MEMBERSHIP_PROFILE_UPDATE_NAME, progress
	896	)
	897
	898	return len(rows)
	899
	900	result = yield self.runInteraction(
	901	_MEMBERSHIP_PROFILE_UPDATE_NAME, add_membership_profile_txn
	902	)
	903
	904	if not result:
	905	yield self._end_background_update(_MEMBERSHIP_PROFILE_UPDATE_NAME)
	906
	907	return result
	908
	909	@defer.inlineCallbacks
	910	def _background_current_state_membership(self, progress, batch_size):
	911	"""Update the new membership column on current_state_events.
	912
	913	This works by iterating over all rooms in alphebetical order.
	914	"""
	915
	916	def _background_current_state_membership_txn(txn, last_processed_room):
	917	processed = 0
	918	while processed < batch_size:
	919	txn.execute(
	920	"""
	921	SELECT MIN(room_id) FROM current_state_events WHERE room_id > ?
	922	""",
	923	(last_processed_room,),
	924	)
	925	row = txn.fetchone()
	926	if not row or not row[0]:
	927	return processed, True
	928
	929	next_room, = row
	930
	931	sql = """
	932	UPDATE current_state_events
	933	SET membership = (
	934	SELECT membership FROM room_memberships
	935	WHERE event_id = current_state_events.event_id
	936	)
	937	WHERE room_id = ?
	938	"""
	939	txn.execute(sql, (next_room,))
	940	processed += txn.rowcount
	941
	942	last_processed_room = next_room
	943
	944	self._background_update_progress_txn(
	945	txn,
	946	_CURRENT_STATE_MEMBERSHIP_UPDATE_NAME,
	947	{"last_processed_room": last_processed_room},
	948	)
	949
	950	return processed, False
	951
	952	# If we haven't got a last processed room then just use the empty
	953	# string, which will compare before all room IDs correctly.
	954	last_processed_room = progress.get("last_processed_room", "")
	955
	956	row_count, finished = yield self.runInteraction(
	957	"_background_current_state_membership_update",
	958	_background_current_state_membership_txn,
	959	last_processed_room,
	960	)
	961
	962	if finished:
	963	yield self._end_background_update(_CURRENT_STATE_MEMBERSHIP_UPDATE_NAME)
	964
	965	return row_count
	966
	967
	968	class RoomMemberStore(RoomMemberWorkerStore, RoomMemberBackgroundUpdateStore):
	969	def __init__(self, db_conn, hs):
	970	super(RoomMemberStore, self).__init__(db_conn, hs)
	971
	972	def _store_room_members_txn(self, txn, events, backfilled):
	973	"""Store a room member in the database.
	974	"""
	975	self._simple_insert_many_txn(
	976	txn,
	977	table="room_memberships",
	978	values=[
	979	{
	980	"event_id": event.event_id,
	981	"user_id": event.state_key,
	982	"sender": event.user_id,
	983	"room_id": event.room_id,
	984	"membership": event.membership,
	985	"display_name": event.content.get("displayname", None),
	986	"avatar_url": event.content.get("avatar_url", None),
	987	}
	988	for event in events
	989	],
	990	)
	991
	992	for event in events:
	993	txn.call_after(
	994	self._membership_stream_cache.entity_has_changed,
	995	event.state_key,
	996	event.internal_metadata.stream_ordering,
	997	)
	998	txn.call_after(
	999	self.get_invited_rooms_for_user.invalidate, (event.state_key,)
	1000	)
	1001
	1002	# We update the local_invites table only if the event is "current",
	1003	# i.e., its something that has just happened. If the event is an
	1004	# outlier it is only current if its an "out of band membership",
	1005	# like a remote invite or a rejection of a remote invite.
	1006	is_new_state = not backfilled and (
	1007	not event.internal_metadata.is_outlier()
	1008	or event.internal_metadata.is_out_of_band_membership()
	1009	)
	1010	is_mine = self.hs.is_mine_id(event.state_key)
	1011	if is_new_state and is_mine:
	1012	if event.membership == Membership.INVITE:
	1013	self._simple_insert_txn(
	1014	txn,
	1015	table="local_invites",
	1016	values={
	1017	"event_id": event.event_id,
	1018	"invitee": event.state_key,
	1019	"inviter": event.sender,
	1020	"room_id": event.room_id,
	1021	"stream_id": event.internal_metadata.stream_ordering,
	1022	},
	1023	)
	1024	else:
	1025	sql = (
	1026	"UPDATE local_invites SET stream_id = ?, replaced_by = ? WHERE"
	1027	" room_id = ? AND invitee = ? AND locally_rejected is NULL"
	1028	" AND replaced_by is NULL"
	1029	)
	1030
	1031	txn.execute(
	1032	sql,
	1033	(
	1034	event.internal_metadata.stream_ordering,
	1035	event.event_id,
	1036	event.room_id,
	1037	event.state_key,
	1038	),
	1039	)
	1040
	1041	@defer.inlineCallbacks
	1042	def locally_reject_invite(self, user_id, room_id):
	1043	sql = (
	1044	"UPDATE local_invites SET stream_id = ?, locally_rejected = ? WHERE"
	1045	" room_id = ? AND invitee = ? AND locally_rejected is NULL"
	1046	" AND replaced_by is NULL"
	1047	)
	1048
	1049	def f(txn, stream_ordering):
	1050	txn.execute(sql, (stream_ordering, True, room_id, user_id))
	1051
	1052	with self._stream_id_gen.get_next() as stream_ordering:
	1053	yield self.runInteraction("locally_reject_invite", f, stream_ordering)
	1054
	1055	def forget(self, user_id, room_id):
	1056	"""Indicate that user_id wishes to discard history for room_id."""
	1057
	1058	def f(txn):
	1059	sql = (
	1060	"UPDATE"
	1061	" room_memberships"
	1062	" SET"
	1063	" forgotten = 1"
	1064	" WHERE"
	1065	" user_id = ?"
	1066	" AND"
	1067	" room_id = ?"
	1068	)
	1069	txn.execute(sql, (user_id, room_id))
	1070
	1071	self._invalidate_cache_and_stream(txn, self.did_forget, (user_id, room_id))
	1072	self._invalidate_cache_and_stream(
	1073	txn, self.get_forgotten_rooms_for_user, (user_id,)
	1074	)
	1075
	1076	return self.runInteraction("forget_membership", f)
	1077
	1078
	1079	class _JoinedHostsCache(object):
	1080	"""Cache for joined hosts in a room that is optimised to handle updates
	1081	via state deltas.
	1082	"""
	1083
	1084	def __init__(self, store, room_id):
	1085	self.store = store
	1086	self.room_id = room_id
	1087
	1088	self.hosts_to_joined_users = {}
	1089
	1090	self.state_group = object()
	1091
	1092	self.linearizer = Linearizer("_JoinedHostsCache")
	1093
	1094	self._len = 0
	1095
	1096	@defer.inlineCallbacks
	1097	def get_destinations(self, state_entry):
	1098	"""Get set of destinations for a state entry
	1099
	1100	Args:
	1101	state_entry(synapse.state._StateCacheEntry)
	1102	"""
	1103	if state_entry.state_group == self.state_group:
	1104	return frozenset(self.hosts_to_joined_users)
	1105
	1106	with (yield self.linearizer.queue(())):
	1107	if state_entry.state_group == self.state_group:
	1108	pass
	1109	elif state_entry.prev_group == self.state_group:
	1110	for (typ, state_key), event_id in iteritems(state_entry.delta_ids):
	1111	if typ != EventTypes.Member:
	1112	continue
	1113
	1114	host = intern_string(get_domain_from_id(state_key))
	1115	user_id = state_key
	1116	known_joins = self.hosts_to_joined_users.setdefault(host, set())
	1117
	1118	event = yield self.store.get_event(event_id)
	1119	if event.membership == Membership.JOIN:
	1120	known_joins.add(user_id)
	1121	else:
	1122	known_joins.discard(user_id)
	1123
	1124	if not known_joins:
	1125	self.hosts_to_joined_users.pop(host, None)
	1126	else:
	1127	joined_users = yield self.store.get_joined_users_from_state(
	1128	self.room_id, state_entry
	1129	)
	1130
	1131	self.hosts_to_joined_users = {}
	1132	for user_id in joined_users:
	1133	host = intern_string(get_domain_from_id(user_id))
	1134	self.hosts_to_joined_users.setdefault(host, set()).add(user_id)
	1135
	1136	if state_entry.state_group:
	1137	self.state_group = state_entry.state_group
	1138	else:
	1139	self.state_group = object()
	1140	self._len = sum(len(v) for v in itervalues(self.hosts_to_joined_users))
	1141	return frozenset(self.hosts_to_joined_users)
	1142
	1143	def __len__(self):
	1144	return self._len

+63

-0

synapse/storage/data_stores/main/schema/delta/12/v12.sql less more

	0	/* Copyright 2015, 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	CREATE TABLE IF NOT EXISTS rejections(
	16	event_id TEXT NOT NULL,
	17	reason TEXT NOT NULL,
	18	last_check TEXT NOT NULL,
	19	UNIQUE (event_id)
	20	);
	21
	22	-- Push notification endpoints that users have configured
	23	CREATE TABLE IF NOT EXISTS pushers (
	24	id INTEGER PRIMARY KEY AUTOINCREMENT,
	25	user_name TEXT NOT NULL,
	26	profile_tag VARCHAR(32) NOT NULL,
	27	kind VARCHAR(8) NOT NULL,
	28	app_id VARCHAR(64) NOT NULL,
	29	app_display_name VARCHAR(64) NOT NULL,
	30	device_display_name VARCHAR(128) NOT NULL,
	31	pushkey VARBINARY(512) NOT NULL,
	32	ts BIGINT UNSIGNED NOT NULL,
	33	lang VARCHAR(8),
	34	data LONGBLOB,
	35	last_token TEXT,
	36	last_success BIGINT UNSIGNED,
	37	failing_since BIGINT UNSIGNED,
	38	UNIQUE (app_id, pushkey)
	39	);
	40
	41	CREATE TABLE IF NOT EXISTS push_rules (
	42	id INTEGER PRIMARY KEY AUTOINCREMENT,
	43	user_name TEXT NOT NULL,
	44	rule_id TEXT NOT NULL,
	45	priority_class TINYINT NOT NULL,
	46	priority INTEGER NOT NULL DEFAULT 0,
	47	conditions TEXT NOT NULL,
	48	actions TEXT NOT NULL,
	49	UNIQUE(user_name, rule_id)
	50	);
	51
	52	CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name);
	53
	54	CREATE TABLE IF NOT EXISTS user_filters(
	55	user_id TEXT,
	56	filter_id BIGINT UNSIGNED,
	57	filter_json LONGBLOB
	58	);
	59
	60	CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters(
	61	user_id, filter_id
	62	);

+19

-0

synapse/storage/data_stores/main/schema/delta/13/v13.sql less more

	0	/* Copyright 2015, 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	/* We used to create a tables called application_services and
	16	* application_services_regex, but these are no longer used and are removed in
	17	* delta 54.
	18	*/

+23

-0

synapse/storage/data_stores/main/schema/delta/14/v14.sql less more

	0	/* Copyright 2015, 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14	CREATE TABLE IF NOT EXISTS push_rules_enable (
	15	id INTEGER PRIMARY KEY AUTOINCREMENT,
	16	user_name TEXT NOT NULL,
	17	rule_id TEXT NOT NULL,
	18	enabled TINYINT,
	19	UNIQUE(user_name, rule_id)
	20	);
	21
	22	CREATE INDEX IF NOT EXISTS push_rules_enable_user_name on push_rules_enable (user_name);

+31

-0

synapse/storage/data_stores/main/schema/delta/15/appservice_txns.sql less more

	0	/* Copyright 2015, 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	CREATE TABLE IF NOT EXISTS application_services_state(
	16	as_id TEXT PRIMARY KEY,
	17	state VARCHAR(5),
	18	last_txn INTEGER
	19	);
	20
	21	CREATE TABLE IF NOT EXISTS application_services_txns(
	22	as_id TEXT NOT NULL,
	23	txn_id INTEGER NOT NULL,
	24	event_ids TEXT NOT NULL,
	25	UNIQUE(as_id, txn_id)
	26	);
	27
	28	CREATE INDEX IF NOT EXISTS application_services_txns_id ON application_services_txns (
	29	as_id
	30	);

+2

-0

synapse/storage/data_stores/main/schema/delta/15/presence_indices.sql less more

	0
	1	CREATE INDEX IF NOT EXISTS presence_list_user_id ON presence_list (user_id);

+24

-0

synapse/storage/data_stores/main/schema/delta/15/v15.sql less more

	0	-- Drop, copy & recreate pushers table to change unique key
	1	-- Also add access_token column at the same time
	2	CREATE TABLE IF NOT EXISTS pushers2 (
	3	id BIGINT PRIMARY KEY,
	4	user_name TEXT NOT NULL,
	5	access_token BIGINT DEFAULT NULL,
	6	profile_tag VARCHAR(32) NOT NULL,
	7	kind VARCHAR(8) NOT NULL,
	8	app_id VARCHAR(64) NOT NULL,
	9	app_display_name VARCHAR(64) NOT NULL,
	10	device_display_name VARCHAR(128) NOT NULL,
	11	pushkey bytea NOT NULL,
	12	ts BIGINT NOT NULL,
	13	lang VARCHAR(8),
	14	data bytea,
	15	last_token TEXT,
	16	last_success BIGINT,
	17	failing_since BIGINT,
	18	UNIQUE (app_id, pushkey)
	19	);
	20	INSERT INTO pushers2 (id, user_name, profile_tag, kind, app_id, app_display_name, device_display_name, pushkey, ts, lang, data, last_token, last_success, failing_since)
	21	SELECT id, user_name, profile_tag, kind, app_id, app_display_name, device_display_name, pushkey, ts, lang, data, last_token, last_success, failing_since FROM pushers;
	22	DROP TABLE pushers;
	23	ALTER TABLE pushers2 RENAME TO pushers;

+4

-0

synapse/storage/data_stores/main/schema/delta/16/events_order_index.sql less more

	0	CREATE INDEX events_order ON events (topological_ordering, stream_ordering);
	1	CREATE INDEX events_order_room ON events (
	2	room_id, topological_ordering, stream_ordering
	3	);

+2

-0

synapse/storage/data_stores/main/schema/delta/16/remote_media_cache_index.sql less more

	0	CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id
	1	ON remote_media_cache_thumbnails (media_id);⏎

+9

-0

synapse/storage/data_stores/main/schema/delta/16/remove_duplicates.sql less more

	0
	1
	2	DELETE FROM event_to_state_groups WHERE state_group not in (
	3	SELECT MAX(state_group) FROM event_to_state_groups GROUP BY event_id
	4	);
	5
	6	DELETE FROM event_to_state_groups WHERE rowid not in (
	7	SELECT MIN(rowid) FROM event_to_state_groups GROUP BY event_id
	8	);

+3

-0

synapse/storage/data_stores/main/schema/delta/16/room_alias_index.sql less more

	0
	1	CREATE INDEX IF NOT EXISTS room_aliases_id ON room_aliases(room_id);
	2	CREATE INDEX IF NOT EXISTS room_alias_servers_alias ON room_alias_servers(room_alias);

+72

-0

synapse/storage/data_stores/main/schema/delta/16/unique_constraints.sql less more

	0
	1	-- We can use SQLite features here, since other db support was only added in v16
	2
	3	--
	4	DELETE FROM current_state_events WHERE rowid not in (
	5	SELECT MIN(rowid) FROM current_state_events GROUP BY event_id
	6	);
	7
	8	DROP INDEX IF EXISTS current_state_events_event_id;
	9	CREATE UNIQUE INDEX current_state_events_event_id ON current_state_events(event_id);
	10
	11	--
	12	DELETE FROM room_memberships WHERE rowid not in (
	13	SELECT MIN(rowid) FROM room_memberships GROUP BY event_id
	14	);
	15
	16	DROP INDEX IF EXISTS room_memberships_event_id;
	17	CREATE UNIQUE INDEX room_memberships_event_id ON room_memberships(event_id);
	18
	19	--
	20	DELETE FROM topics WHERE rowid not in (
	21	SELECT MIN(rowid) FROM topics GROUP BY event_id
	22	);
	23
	24	DROP INDEX IF EXISTS topics_event_id;
	25	CREATE UNIQUE INDEX topics_event_id ON topics(event_id);
	26
	27	--
	28	DELETE FROM room_names WHERE rowid not in (
	29	SELECT MIN(rowid) FROM room_names GROUP BY event_id
	30	);
	31
	32	DROP INDEX IF EXISTS room_names_id;
	33	CREATE UNIQUE INDEX room_names_id ON room_names(event_id);
	34
	35	--
	36	DELETE FROM presence WHERE rowid not in (
	37	SELECT MIN(rowid) FROM presence GROUP BY user_id
	38	);
	39
	40	DROP INDEX IF EXISTS presence_id;
	41	CREATE UNIQUE INDEX presence_id ON presence(user_id);
	42
	43	--
	44	DELETE FROM presence_allow_inbound WHERE rowid not in (
	45	SELECT MIN(rowid) FROM presence_allow_inbound
	46	GROUP BY observed_user_id, observer_user_id
	47	);
	48
	49	DROP INDEX IF EXISTS presence_allow_inbound_observers;
	50	CREATE UNIQUE INDEX presence_allow_inbound_observers ON presence_allow_inbound(
	51	observed_user_id, observer_user_id
	52	);
	53
	54	--
	55	DELETE FROM presence_list WHERE rowid not in (
	56	SELECT MIN(rowid) FROM presence_list
	57	GROUP BY user_id, observed_user_id
	58	);
	59
	60	DROP INDEX IF EXISTS presence_list_observers;
	61	CREATE UNIQUE INDEX presence_list_observers ON presence_list(
	62	user_id, observed_user_id
	63	);
	64
	65	--
	66	DELETE FROM room_aliases WHERE rowid not in (
	67	SELECT MIN(rowid) FROM room_aliases GROUP BY room_alias
	68	);
	69
	70	DROP INDEX IF EXISTS room_aliases_id;
	71	CREATE INDEX room_aliases_id ON room_aliases(room_id);

+56

-0

synapse/storage/data_stores/main/schema/delta/16/users.sql less more

	0	-- Convert `access_tokens`.user from rowids to user strings.
	1	-- MUST BE DONE BEFORE REMOVING ID COLUMN FROM USERS TABLE BELOW
	2	CREATE TABLE IF NOT EXISTS new_access_tokens(
	3	id BIGINT UNSIGNED PRIMARY KEY,
	4	user_id TEXT NOT NULL,
	5	device_id TEXT,
	6	token TEXT NOT NULL,
	7	last_used BIGINT UNSIGNED,
	8	UNIQUE(token)
	9	);
	10
	11	INSERT INTO new_access_tokens
	12	SELECT a.id, u.name, a.device_id, a.token, a.last_used
	13	FROM access_tokens as a
	14	INNER JOIN users as u ON u.id = a.user_id;
	15
	16	DROP TABLE access_tokens;
	17
	18	ALTER TABLE new_access_tokens RENAME TO access_tokens;
	19
	20	-- Remove ID column from `users` table
	21	CREATE TABLE IF NOT EXISTS new_users(
	22	name TEXT,
	23	password_hash TEXT,
	24	creation_ts BIGINT UNSIGNED,
	25	admin BOOL DEFAULT 0 NOT NULL,
	26	UNIQUE(name)
	27	);
	28
	29	INSERT INTO new_users SELECT name, password_hash, creation_ts, admin FROM users;
	30
	31	DROP TABLE users;
	32
	33	ALTER TABLE new_users RENAME TO users;
	34
	35
	36	-- Remove UNIQUE constraint from `user_ips` table
	37	CREATE TABLE IF NOT EXISTS new_user_ips (
	38	user_id TEXT NOT NULL,
	39	access_token TEXT NOT NULL,
	40	device_id TEXT,
	41	ip TEXT NOT NULL,
	42	user_agent TEXT NOT NULL,
	43	last_seen BIGINT UNSIGNED NOT NULL
	44	);
	45
	46	INSERT INTO new_user_ips
	47	SELECT user, access_token, device_id, ip, user_agent, last_seen FROM user_ips;
	48
	49	DROP TABLE user_ips;
	50
	51	ALTER TABLE new_user_ips RENAME TO user_ips;
	52
	53	CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user_id);
	54	CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user_id, access_token, ip);
	55

+18

-0

synapse/storage/data_stores/main/schema/delta/17/drop_indexes.sql less more

	0	/* Copyright 2015, 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	DROP INDEX IF EXISTS sent_transaction_dest;
	16	DROP INDEX IF EXISTS sent_transaction_sent;
	17	DROP INDEX IF EXISTS user_ips_user;

+24

-0

synapse/storage/data_stores/main/schema/delta/17/server_keys.sql less more

	0	/* Copyright 2015, 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	CREATE TABLE IF NOT EXISTS server_keys_json (
	16	server_name TEXT, -- Server name.
	17	key_id TEXT, -- Requested key id.
	18	from_server TEXT, -- Which server the keys were fetched from.
	19	ts_added_ms INTEGER, -- When the keys were fetched
	20	ts_valid_until_ms INTEGER, -- When this version of the keys exipires.
	21	key_json bytea, -- JSON certificate for the remote server.
	22	CONSTRAINT uniqueness UNIQUE (server_name, key_id, from_server)
	23	);

+9

-0

synapse/storage/data_stores/main/schema/delta/17/user_threepids.sql less more

	0	CREATE TABLE user_threepids (
	1	user_id TEXT NOT NULL,
	2	medium TEXT NOT NULL,
	3	address TEXT NOT NULL,
	4	validated_at BIGINT NOT NULL,
	5	added_at BIGINT NOT NULL,
	6	CONSTRAINT user_medium_address UNIQUE (user_id, medium, address)
	7	);
	8	CREATE INDEX user_threepids_user_id ON user_threepids(user_id);

+32

-0

synapse/storage/data_stores/main/schema/delta/18/server_keys_bigger_ints.sql less more

	0	/* Copyright 2015, 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15
	16	CREATE TABLE IF NOT EXISTS new_server_keys_json (
	17	server_name TEXT NOT NULL, -- Server name.
	18	key_id TEXT NOT NULL, -- Requested key id.
	19	from_server TEXT NOT NULL, -- Which server the keys were fetched from.
	20	ts_added_ms BIGINT NOT NULL, -- When the keys were fetched
	21	ts_valid_until_ms BIGINT NOT NULL, -- When this version of the keys exipires.
	22	key_json bytea NOT NULL, -- JSON certificate for the remote server.
	23	CONSTRAINT server_keys_json_uniqueness UNIQUE (server_name, key_id, from_server)
	24	);
	25
	26	INSERT INTO new_server_keys_json
	27	SELECT server_name, key_id, from_server,ts_added_ms, ts_valid_until_ms, key_json FROM server_keys_json ;
	28
	29	DROP TABLE server_keys_json;
	30
	31	ALTER TABLE new_server_keys_json RENAME TO server_keys_json;

+19

-0

synapse/storage/data_stores/main/schema/delta/19/event_index.sql less more

	0	/* Copyright 2015, 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15
	16	CREATE INDEX events_order_topo_stream_room ON events(
	17	topological_ordering, stream_ordering, room_id
	18	);

+1

-0

synapse/storage/data_stores/main/schema/delta/20/dummy.sql less more

0

SELECT 1;

+88

-0

synapse/storage/data_stores/main/schema/delta/20/pushers.py less more

	0	# Copyright 2015, 2016 OpenMarket Ltd
	1	#
	2	# Licensed under the Apache License, Version 2.0 (the "License");
	3	# you may not use this file except in compliance with the License.
	4	# You may obtain a copy of the License at
	5	#
	6	# http://www.apache.org/licenses/LICENSE-2.0
	7	#
	8	# Unless required by applicable law or agreed to in writing, software
	9	# distributed under the License is distributed on an "AS IS" BASIS,
	10	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	# See the License for the specific language governing permissions and
	12	# limitations under the License.
	13
	14
	15	"""
	16	Main purpose of this upgrade is to change the unique key on the
	17	pushers table again (it was missed when the v16 full schema was
	18	made) but this also changes the pushkey and data columns to text.
	19	When selecting a bytea column into a text column, postgres inserts
	20	the hex encoded data, and there's no portable way of getting the
	21	UTF-8 bytes, so we have to do it in Python.
	22	"""
	23
	24	import logging
	25
	26	logger = logging.getLogger(__name__)
	27
	28
	29	def run_create(cur, database_engine, args, *kwargs):
	30	logger.info("Porting pushers table...")
	31	cur.execute(
	32	"""
	33	CREATE TABLE IF NOT EXISTS pushers2 (
	34	id BIGINT PRIMARY KEY,
	35	user_name TEXT NOT NULL,
	36	access_token BIGINT DEFAULT NULL,
	37	profile_tag VARCHAR(32) NOT NULL,
	38	kind VARCHAR(8) NOT NULL,
	39	app_id VARCHAR(64) NOT NULL,
	40	app_display_name VARCHAR(64) NOT NULL,
	41	device_display_name VARCHAR(128) NOT NULL,
	42	pushkey TEXT NOT NULL,
	43	ts BIGINT NOT NULL,
	44	lang VARCHAR(8),
	45	data TEXT,
	46	last_token TEXT,
	47	last_success BIGINT,
	48	failing_since BIGINT,
	49	UNIQUE (app_id, pushkey, user_name)
	50	)
	51	"""
	52	)
	53	cur.execute(
	54	"""SELECT
	55	id, user_name, access_token, profile_tag, kind,
	56	app_id, app_display_name, device_display_name,
	57	pushkey, ts, lang, data, last_token, last_success,
	58	failing_since
	59	FROM pushers
	60	"""
	61	)
	62	count = 0
	63	for row in cur.fetchall():
	64	row = list(row)
	65	row[8] = bytes(row[8]).decode("utf-8")
	66	row[11] = bytes(row[11]).decode("utf-8")
	67	cur.execute(
	68	database_engine.convert_param_style(
	69	"""
	70	INSERT into pushers2 (
	71	id, user_name, access_token, profile_tag, kind,
	72	app_id, app_display_name, device_display_name,
	73	pushkey, ts, lang, data, last_token, last_success,
	74	failing_since
	75	) values (%s)"""
	76	% (",".join(["?" for _ in range(len(row))]))
	77	),
	78	row,
	79	)
	80	count += 1
	81	cur.execute("DROP TABLE pushers")
	82	cur.execute("ALTER TABLE pushers2 RENAME TO pushers")
	83	logger.info("Moved %d pushers to new table", count)
	84
	85
	86	def run_upgrade(args, *kwargs):
	87	pass

+34

-0

synapse/storage/data_stores/main/schema/delta/21/end_to_end_keys.sql less more

	0	/* Copyright 2015, 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15
	16	CREATE TABLE IF NOT EXISTS e2e_device_keys_json (
	17	user_id TEXT NOT NULL, -- The user these keys are for.
	18	device_id TEXT NOT NULL, -- Which of the user's devices these keys are for.
	19	ts_added_ms BIGINT NOT NULL, -- When the keys were uploaded.
	20	key_json TEXT NOT NULL, -- The keys for the device as a JSON blob.
	21	CONSTRAINT e2e_device_keys_json_uniqueness UNIQUE (user_id, device_id)
	22	);
	23
	24
	25	CREATE TABLE IF NOT EXISTS e2e_one_time_keys_json (
	26	user_id TEXT NOT NULL, -- The user this one-time key is for.
	27	device_id TEXT NOT NULL, -- The device this one-time key is for.
	28	algorithm TEXT NOT NULL, -- Which algorithm this one-time key is for.
	29	key_id TEXT NOT NULL, -- An id for suppressing duplicate uploads.
	30	ts_added_ms BIGINT NOT NULL, -- When this key was uploaded.
	31	key_json TEXT NOT NULL, -- The key as a JSON blob.
	32	CONSTRAINT e2e_one_time_keys_json_uniqueness UNIQUE (user_id, device_id, algorithm, key_id)
	33	);

+38

-0

synapse/storage/data_stores/main/schema/delta/21/receipts.sql less more

	0	/* Copyright 2015, 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15
	16	CREATE TABLE IF NOT EXISTS receipts_graph(
	17	room_id TEXT NOT NULL,
	18	receipt_type TEXT NOT NULL,
	19	user_id TEXT NOT NULL,
	20	event_ids TEXT NOT NULL,
	21	data TEXT NOT NULL,
	22	CONSTRAINT receipts_graph_uniqueness UNIQUE (room_id, receipt_type, user_id)
	23	);
	24
	25	CREATE TABLE IF NOT EXISTS receipts_linearized (
	26	stream_id BIGINT NOT NULL,
	27	room_id TEXT NOT NULL,
	28	receipt_type TEXT NOT NULL,
	29	user_id TEXT NOT NULL,
	30	event_id TEXT NOT NULL,
	31	data TEXT NOT NULL,
	32	CONSTRAINT receipts_linearized_uniqueness UNIQUE (room_id, receipt_type, user_id)
	33	);
	34
	35	CREATE INDEX receipts_linearized_id ON receipts_linearized(
	36	stream_id
	37	);

+22

-0

synapse/storage/data_stores/main/schema/delta/22/receipts_index.sql less more

	0	/* Copyright 2015, 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	/** Using CREATE INDEX directly is deprecated in favour of using background
	16	* update see synapse/storage/schema/delta/33/access_tokens_device_index.sql
	17	* and synapse/storage/registration.py for an example using
	18	* "access_tokens_device_index" **/
	19	CREATE INDEX receipts_linearized_room_stream ON receipts_linearized(
	20	room_id, stream_id
	21	);

+19

-0

synapse/storage/data_stores/main/schema/delta/22/user_threepids_unique.sql less more

	0	CREATE TABLE IF NOT EXISTS user_threepids2 (
	1	user_id TEXT NOT NULL,
	2	medium TEXT NOT NULL,
	3	address TEXT NOT NULL,
	4	validated_at BIGINT NOT NULL,
	5	added_at BIGINT NOT NULL,
	6	CONSTRAINT medium_address UNIQUE (medium, address)
	7	);
	8
	9	INSERT INTO user_threepids2
	10	SELECT * FROM user_threepids WHERE added_at IN (
	11	SELECT max(added_at) FROM user_threepids GROUP BY medium, address
	12	)
	13	;
	14
	15	DROP TABLE user_threepids;
	16	ALTER TABLE user_threepids2 RENAME TO user_threepids;
	17
	18	CREATE INDEX user_threepids_user_id ON user_threepids(user_id);

+16

-0

synapse/storage/data_stores/main/schema/delta/23/drop_state_index.sql less more

	0	/* Copyright 2015, 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	DROP INDEX IF EXISTS state_groups_state_tuple;

+18

-0

synapse/storage/data_stores/main/schema/delta/24/stats_reporting.sql less more

	0	/* Copyright 2019 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	/* We used to create a table called stats_reporting, but this is no longer
	16	* used and is removed in delta 54.
	17	*/⏎

+82

-0

synapse/storage/data_stores/main/schema/delta/25/fts.py less more

	0	# Copyright 2015, 2016 OpenMarket Ltd
	1	#
	2	# Licensed under the Apache License, Version 2.0 (the "License");
	3	# you may not use this file except in compliance with the License.
	4	# You may obtain a copy of the License at
	5	#
	6	# http://www.apache.org/licenses/LICENSE-2.0
	7	#
	8	# Unless required by applicable law or agreed to in writing, software
	9	# distributed under the License is distributed on an "AS IS" BASIS,
	10	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	# See the License for the specific language governing permissions and
	12	# limitations under the License.
	13
	14	import logging
	15
	16	import simplejson
	17
	18	from synapse.storage.engines import PostgresEngine, Sqlite3Engine
	19	from synapse.storage.prepare_database import get_statements
	20
	21	logger = logging.getLogger(__name__)
	22
	23
	24	POSTGRES_TABLE = """
	25	CREATE TABLE IF NOT EXISTS event_search (
	26	event_id TEXT,
	27	room_id TEXT,
	28	sender TEXT,
	29	key TEXT,
	30	vector tsvector
	31	);
	32
	33	CREATE INDEX event_search_fts_idx ON event_search USING gin(vector);
	34	CREATE INDEX event_search_ev_idx ON event_search(event_id);
	35	CREATE INDEX event_search_ev_ridx ON event_search(room_id);
	36	"""
	37
	38
	39	SQLITE_TABLE = (
	40	"CREATE VIRTUAL TABLE event_search"
	41	" USING fts4 ( event_id, room_id, sender, key, value )"
	42	)
	43
	44
	45	def run_create(cur, database_engine, args, *kwargs):
	46	if isinstance(database_engine, PostgresEngine):
	47	for statement in get_statements(POSTGRES_TABLE.splitlines()):
	48	cur.execute(statement)
	49	elif isinstance(database_engine, Sqlite3Engine):
	50	cur.execute(SQLITE_TABLE)
	51	else:
	52	raise Exception("Unrecognized database engine")
	53
	54	cur.execute("SELECT MIN(stream_ordering) FROM events")
	55	rows = cur.fetchall()
	56	min_stream_id = rows[0][0]
	57
	58	cur.execute("SELECT MAX(stream_ordering) FROM events")
	59	rows = cur.fetchall()
	60	max_stream_id = rows[0][0]
	61
	62	if min_stream_id is not None and max_stream_id is not None:
	63	progress = {
	64	"target_min_stream_id_inclusive": min_stream_id,
	65	"max_stream_id_exclusive": max_stream_id + 1,
	66	"rows_inserted": 0,
	67	}
	68	progress_json = simplejson.dumps(progress)
	69
	70	sql = (
	71	"INSERT into background_updates (update_name, progress_json)"
	72	" VALUES (?, ?)"
	73	)
	74
	75	sql = database_engine.convert_param_style(sql)
	76
	77	cur.execute(sql, ("event_search", progress_json))
	78
	79
	80	def run_upgrade(args, *kwargs):
	81	pass

+25

-0

synapse/storage/data_stores/main/schema/delta/25/guest_access.sql less more

	0	/* Copyright 2015, 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	/*
	16	* This is a manual index of guest_access content of state events,
	17	* so that we can join on them in SELECT statements.
	18	*/
	19	CREATE TABLE IF NOT EXISTS guest_access(
	20	event_id TEXT NOT NULL,
	21	room_id TEXT NOT NULL,
	22	guest_access TEXT NOT NULL,
	23	UNIQUE (event_id)
	24	);

+25

-0

synapse/storage/data_stores/main/schema/delta/25/history_visibility.sql less more

	0	/* Copyright 2015, 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	/*
	16	* This is a manual index of history_visibility content of state events,
	17	* so that we can join on them in SELECT statements.
	18	*/
	19	CREATE TABLE IF NOT EXISTS history_visibility(
	20	event_id TEXT NOT NULL,
	21	room_id TEXT NOT NULL,
	22	history_visibility TEXT NOT NULL,
	23	UNIQUE (event_id)
	24	);

+38

-0

synapse/storage/data_stores/main/schema/delta/25/tags.sql less more

	0	/* Copyright 2015, 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15
	16	CREATE TABLE IF NOT EXISTS room_tags(
	17	user_id TEXT NOT NULL,
	18	room_id TEXT NOT NULL,
	19	tag TEXT NOT NULL, -- The name of the tag.
	20	content TEXT NOT NULL, -- The JSON content of the tag.
	21	CONSTRAINT room_tag_uniqueness UNIQUE (user_id, room_id, tag)
	22	);
	23
	24	CREATE TABLE IF NOT EXISTS room_tags_revisions (
	25	user_id TEXT NOT NULL,
	26	room_id TEXT NOT NULL,
	27	stream_id BIGINT NOT NULL, -- The current version of the room tags.
	28	CONSTRAINT room_tag_revisions_uniqueness UNIQUE (user_id, room_id)
	29	);
	30
	31	CREATE TABLE IF NOT EXISTS private_user_data_max_stream_id(
	32	Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, -- Makes sure this table only has one row.
	33	stream_id BIGINT NOT NULL,
	34	CHECK (Lock='X')
	35	);
	36
	37	INSERT INTO private_user_data_max_stream_id (stream_id) VALUES (0);

+17

-0

synapse/storage/data_stores/main/schema/delta/26/account_data.sql less more

	0	/* Copyright 2015, 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15
	16	ALTER TABLE private_user_data_max_stream_id RENAME TO account_data_max_stream_id;

+36

-0

synapse/storage/data_stores/main/schema/delta/27/account_data.sql less more

	0	/* Copyright 2015, 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	CREATE TABLE IF NOT EXISTS account_data(
	16	user_id TEXT NOT NULL,
	17	account_data_type TEXT NOT NULL, -- The type of the account_data.
	18	stream_id BIGINT NOT NULL, -- The version of the account_data.
	19	content TEXT NOT NULL, -- The JSON content of the account_data
	20	CONSTRAINT account_data_uniqueness UNIQUE (user_id, account_data_type)
	21	);
	22
	23
	24	CREATE TABLE IF NOT EXISTS room_account_data(
	25	user_id TEXT NOT NULL,
	26	room_id TEXT NOT NULL,
	27	account_data_type TEXT NOT NULL, -- The type of the account_data.
	28	stream_id BIGINT NOT NULL, -- The version of the account_data.
	29	content TEXT NOT NULL, -- The JSON content of the account_data
	30	CONSTRAINT room_account_data_uniqueness UNIQUE (user_id, room_id, account_data_type)
	31	);
	32
	33
	34	CREATE INDEX account_data_stream_id on account_data(user_id, stream_id);
	35	CREATE INDEX room_account_data_stream_id on room_account_data(user_id, stream_id);

+26

-0

synapse/storage/data_stores/main/schema/delta/27/forgotten_memberships.sql less more

	0	/* Copyright 2015, 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	/*
	16	* Keeps track of what rooms users have left and don't want to be able to
	17	* access again.
	18	*
	19	* If all users on this server have left a room, we can delete the room
	20	* entirely.
	21	*
	22	* This column should always contain either 0 or 1.
	23	*/
	24
	25	ALTER TABLE room_memberships ADD COLUMN forgotten INTEGER DEFAULT 0;

+61

-0

synapse/storage/data_stores/main/schema/delta/27/ts.py less more

	0	# Copyright 2015, 2016 OpenMarket Ltd
	1	#
	2	# Licensed under the Apache License, Version 2.0 (the "License");
	3	# you may not use this file except in compliance with the License.
	4	# You may obtain a copy of the License at
	5	#
	6	# http://www.apache.org/licenses/LICENSE-2.0
	7	#
	8	# Unless required by applicable law or agreed to in writing, software
	9	# distributed under the License is distributed on an "AS IS" BASIS,
	10	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	# See the License for the specific language governing permissions and
	12	# limitations under the License.
	13
	14	import logging
	15
	16	import simplejson
	17
	18	from synapse.storage.prepare_database import get_statements
	19
	20	logger = logging.getLogger(__name__)
	21
	22
	23	ALTER_TABLE = (
	24	"ALTER TABLE events ADD COLUMN origin_server_ts BIGINT;"
	25	"CREATE INDEX events_ts ON events(origin_server_ts, stream_ordering);"
	26	)
	27
	28
	29	def run_create(cur, database_engine, args, *kwargs):
	30	for statement in get_statements(ALTER_TABLE.splitlines()):
	31	cur.execute(statement)
	32
	33	cur.execute("SELECT MIN(stream_ordering) FROM events")
	34	rows = cur.fetchall()
	35	min_stream_id = rows[0][0]
	36
	37	cur.execute("SELECT MAX(stream_ordering) FROM events")
	38	rows = cur.fetchall()
	39	max_stream_id = rows[0][0]
	40
	41	if min_stream_id is not None and max_stream_id is not None:
	42	progress = {
	43	"target_min_stream_id_inclusive": min_stream_id,
	44	"max_stream_id_exclusive": max_stream_id + 1,
	45	"rows_inserted": 0,
	46	}
	47	progress_json = simplejson.dumps(progress)
	48
	49	sql = (
	50	"INSERT into background_updates (update_name, progress_json)"
	51	" VALUES (?, ?)"
	52	)
	53
	54	sql = database_engine.convert_param_style(sql)
	55
	56	cur.execute(sql, ("event_origin_server_ts", progress_json))
	57
	58
	59	def run_upgrade(args, *kwargs):
	60	pass

+27

-0

synapse/storage/data_stores/main/schema/delta/28/event_push_actions.sql less more

	0	/* Copyright 2015 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	CREATE TABLE IF NOT EXISTS event_push_actions(
	16	room_id TEXT NOT NULL,
	17	event_id TEXT NOT NULL,
	18	user_id TEXT NOT NULL,
	19	profile_tag VARCHAR(32),
	20	actions TEXT NOT NULL,
	21	CONSTRAINT event_id_user_id_profile_tag_uniqueness UNIQUE (room_id, event_id, user_id, profile_tag)
	22	);
	23
	24
	25	CREATE INDEX event_push_actions_room_id_event_id_user_id_profile_tag on event_push_actions(room_id, event_id, user_id, profile_tag);
	26	CREATE INDEX event_push_actions_room_id_user_id on event_push_actions(room_id, user_id);

+20

-0

synapse/storage/data_stores/main/schema/delta/28/events_room_stream.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	/** Using CREATE INDEX directly is deprecated in favour of using background
	16	* update see synapse/storage/schema/delta/33/access_tokens_device_index.sql
	17	* and synapse/storage/registration.py for an example using
	18	* "access_tokens_device_index" **/
	19	CREATE INDEX events_room_stream on events(room_id, stream_ordering);

+20

-0

synapse/storage/data_stores/main/schema/delta/28/public_roms_index.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	/** Using CREATE INDEX directly is deprecated in favour of using background
	16	* update see synapse/storage/schema/delta/33/access_tokens_device_index.sql
	17	* and synapse/storage/registration.py for an example using
	18	* "access_tokens_device_index" **/
	19	CREATE INDEX public_room_index on rooms(is_public);

+22

-0

synapse/storage/data_stores/main/schema/delta/28/receipts_user_id_index.sql less more

	0	/* Copyright 2015, 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	/** Using CREATE INDEX directly is deprecated in favour of using background
	16	* update see synapse/storage/schema/delta/33/access_tokens_device_index.sql
	17	* and synapse/storage/registration.py for an example using
	18	* "access_tokens_device_index" **/
	19	CREATE INDEX receipts_linearized_user ON receipts_linearized(
	20	user_id
	21	);

+21

-0

synapse/storage/data_stores/main/schema/delta/28/upgrade_times.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	/*
	16	* Stores the timestamp when a user upgraded from a guest to a full user, if
	17	* that happened.
	18	*/
	19
	20	ALTER TABLE users ADD COLUMN upgrade_ts BIGINT;

+22

-0

synapse/storage/data_stores/main/schema/delta/28/users_is_guest.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	ALTER TABLE users ADD is_guest SMALLINT DEFAULT 0 NOT NULL;
	16	/*
	17	* NB: any guest users created between 27 and 28 will be incorrectly
	18	* marked as not guests: we don't bother to fill these in correctly
	19	* because guest access is not really complete in 27 anyway so it's
	20	* very unlikley there will be any guest users created.
	21	*/

+35

-0

synapse/storage/data_stores/main/schema/delta/29/push_actions.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	ALTER TABLE event_push_actions ADD COLUMN topological_ordering BIGINT;
	16	ALTER TABLE event_push_actions ADD COLUMN stream_ordering BIGINT;
	17	ALTER TABLE event_push_actions ADD COLUMN notif SMALLINT;
	18	ALTER TABLE event_push_actions ADD COLUMN highlight SMALLINT;
	19
	20	UPDATE event_push_actions SET stream_ordering = (
	21	SELECT stream_ordering FROM events WHERE event_id = event_push_actions.event_id
	22	), topological_ordering = (
	23	SELECT topological_ordering FROM events WHERE event_id = event_push_actions.event_id
	24	);
	25
	26	UPDATE event_push_actions SET notif = 1, highlight = 0;
	27
	28	/** Using CREATE INDEX directly is deprecated in favour of using background
	29	* update see synapse/storage/schema/delta/33/access_tokens_device_index.sql
	30	* and synapse/storage/registration.py for an example using
	31	* "access_tokens_device_index" **/
	32	CREATE INDEX event_push_actions_rm_tokens on event_push_actions(
	33	user_id, room_id, topological_ordering, stream_ordering
	34	);

+16

-0

synapse/storage/data_stores/main/schema/delta/30/alias_creator.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	ALTER TABLE room_aliases ADD COLUMN creator TEXT;

+69

-0

synapse/storage/data_stores/main/schema/delta/30/as_users.py less more

	0	# Copyright 2016 OpenMarket Ltd
	1	#
	2	# Licensed under the Apache License, Version 2.0 (the "License");
	3	# you may not use this file except in compliance with the License.
	4	# You may obtain a copy of the License at
	5	#
	6	# http://www.apache.org/licenses/LICENSE-2.0
	7	#
	8	# Unless required by applicable law or agreed to in writing, software
	9	# distributed under the License is distributed on an "AS IS" BASIS,
	10	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	# See the License for the specific language governing permissions and
	12	# limitations under the License.
	13	import logging
	14
	15	from six.moves import range
	16
	17	from synapse.config.appservice import load_appservices
	18
	19	logger = logging.getLogger(__name__)
	20
	21
	22	def run_create(cur, database_engine, args, *kwargs):
	23	# NULL indicates user was not registered by an appservice.
	24	try:
	25	cur.execute("ALTER TABLE users ADD COLUMN appservice_id TEXT")
	26	except Exception:
	27	# Maybe we already added the column? Hope so...
	28	pass
	29
	30
	31	def run_upgrade(cur, database_engine, config, args, *kwargs):
	32	cur.execute("SELECT name FROM users")
	33	rows = cur.fetchall()
	34
	35	config_files = []
	36	try:
	37	config_files = config.app_service_config_files
	38	except AttributeError:
	39	logger.warning("Could not get app_service_config_files from config")
	40	pass
	41
	42	appservices = load_appservices(config.server_name, config_files)
	43
	44	owned = {}
	45
	46	for row in rows:
	47	user_id = row[0]
	48	for appservice in appservices:
	49	if appservice.is_exclusive_user(user_id):
	50	if user_id in owned.keys():
	51	logger.error(
	52	"user_id %s was owned by more than one application"
	53	" service (IDs %s and %s); assigning arbitrarily to %s"
	54	% (user_id, owned[user_id], appservice.id, owned[user_id])
	55	)
	56	owned.setdefault(appservice.id, []).append(user_id)
	57
	58	for as_id, user_ids in owned.items():
	59	n = 100
	60	user_chunks = (user_ids[i : i + 100] for i in range(0, len(user_ids), n))
	61	for chunk in user_chunks:
	62	cur.execute(
	63	database_engine.convert_param_style(
	64	"UPDATE users SET appservice_id = ? WHERE name IN (%s)"
	65	% (",".join("?" for _ in chunk),)
	66	),
	67	[as_id] + chunk,
	68	)

+25

-0

synapse/storage/data_stores/main/schema/delta/30/deleted_pushers.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	CREATE TABLE IF NOT EXISTS deleted_pushers(
	16	stream_id BIGINT NOT NULL,
	17	app_id TEXT NOT NULL,
	18	pushkey TEXT NOT NULL,
	19	user_id TEXT NOT NULL,
	20	/* We only track the most recent delete for each app_id, pushkey and user_id. */
	21	UNIQUE (app_id, pushkey, user_id)
	22	);
	23
	24	CREATE INDEX deleted_pushers_stream_id ON deleted_pushers (stream_id);

+30

-0

synapse/storage/data_stores/main/schema/delta/30/presence_stream.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15
	16	CREATE TABLE presence_stream(
	17	stream_id BIGINT,
	18	user_id TEXT,
	19	state TEXT,
	20	last_active_ts BIGINT,
	21	last_federation_update_ts BIGINT,
	22	last_user_sync_ts BIGINT,
	23	status_msg TEXT,
	24	currently_active BOOLEAN
	25	);
	26
	27	CREATE INDEX presence_stream_id ON presence_stream(stream_id, user_id);
	28	CREATE INDEX presence_stream_user_id ON presence_stream(user_id);
	29	CREATE INDEX presence_stream_state ON presence_stream(state);

+23

-0

synapse/storage/data_stores/main/schema/delta/30/public_rooms.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15
	16	/* This release removes the restriction that published rooms must have an alias,
	17	* so we go back and ensure the only 'public' rooms are ones with an alias.
	18	* We use (1 = 0) and (1 = 1) so that it works in both postgres and sqlite
	19	*/
	20	UPDATE rooms SET is_public = (1 = 0) WHERE is_public = (1 = 1) AND room_id not in (
	21	SELECT room_id FROM room_aliases
	22	);

+38

-0

synapse/storage/data_stores/main/schema/delta/30/push_rule_stream.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15
	16
	17	CREATE TABLE push_rules_stream(
	18	stream_id BIGINT NOT NULL,
	19	event_stream_ordering BIGINT NOT NULL,
	20	user_id TEXT NOT NULL,
	21	rule_id TEXT NOT NULL,
	22	op TEXT NOT NULL, -- One of "ENABLE", "DISABLE", "ACTIONS", "ADD", "DELETE"
	23	priority_class SMALLINT,
	24	priority INTEGER,
	25	conditions TEXT,
	26	actions TEXT
	27	);
	28
	29	-- The extra data for each operation is:
	30	-- * ENABLE, DISABLE, DELETE: []
	31	-- * ACTIONS: ["actions"]
	32	-- * ADD: ["priority_class", "priority", "actions", "conditions"]
	33
	34	-- Index for replication queries.
	35	CREATE INDEX push_rules_stream_id ON push_rules_stream(stream_id);
	36	-- Index for /sync queries.
	37	CREATE INDEX push_rules_stream_user_stream_id on push_rules_stream(user_id, stream_id);

+33

-0

synapse/storage/data_stores/main/schema/delta/30/state_stream.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15
	16	/* We used to create a table called current_state_resets, but this is no
	17	* longer used and is removed in delta 54.
	18	*/
	19
	20	/* The outlier events that have aquired a state group typically through
	21	* backfill. This is tracked separately to the events table, as assigning a
	22	* state group change the position of the existing event in the stream
	23	* ordering.
	24	* However since a stream_ordering is assigned in persist_event for the
	25	* (event, state) pair, we can use that stream_ordering to identify when
	26	* the new state was assigned for the event.
	27	*/
	28	CREATE TABLE IF NOT EXISTS ex_outlier_stream(
	29	event_stream_ordering BIGINT PRIMARY KEY NOT NULL,
	30	event_id TEXT NOT NULL,
	31	state_group BIGINT NOT NULL
	32	);

+24

-0

synapse/storage/data_stores/main/schema/delta/30/threepid_guest_access_tokens.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- Stores guest account access tokens generated for unbound 3pids.
	16	CREATE TABLE threepid_guest_access_tokens(
	17	medium TEXT, -- The medium of the 3pid. Must be "email".
	18	address TEXT, -- The 3pid address.
	19	guest_access_token TEXT, -- The access token for a guest user for this 3pid.
	20	first_inviter TEXT -- User ID of the first user to invite this 3pid to a room.
	21	);
	22
	23	CREATE UNIQUE INDEX threepid_guest_access_tokens_index ON threepid_guest_access_tokens(medium, address);

+42

-0

synapse/storage/data_stores/main/schema/delta/31/invites.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15
	16	CREATE TABLE local_invites(
	17	stream_id BIGINT NOT NULL,
	18	inviter TEXT NOT NULL,
	19	invitee TEXT NOT NULL,
	20	event_id TEXT NOT NULL,
	21	room_id TEXT NOT NULL,
	22	locally_rejected TEXT,
	23	replaced_by TEXT
	24	);
	25
	26	-- Insert all invites for local users into new `invites` table
	27	INSERT INTO local_invites SELECT
	28	stream_ordering as stream_id,
	29	sender as inviter,
	30	state_key as invitee,
	31	event_id,
	32	room_id,
	33	NULL as locally_rejected,
	34	NULL as replaced_by
	35	FROM events
	36	NATURAL JOIN current_state_events
	37	NATURAL JOIN room_memberships
	38	WHERE membership = 'invite' AND state_key IN (SELECT name FROM users);
	39
	40	CREATE INDEX local_invites_id ON local_invites(stream_id);
	41	CREATE INDEX local_invites_for_user_idx ON local_invites(invitee, locally_rejected, replaced_by, room_id);

+27

-0

synapse/storage/data_stores/main/schema/delta/31/local_media_repository_url_cache.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	CREATE TABLE local_media_repository_url_cache(
	16	url TEXT, -- the URL being cached
	17	response_code INTEGER, -- the HTTP response code of this download attempt
	18	etag TEXT, -- the etag header of this response
	19	expires INTEGER, -- the number of ms this response was valid for
	20	og TEXT, -- cache of the OG metadata of this URL as JSON
	21	media_id TEXT, -- the media_id, if any, of the URL's content in the repo
	22	download_ts BIGINT -- the timestamp of this download attempt
	23	);
	24
	25	CREATE INDEX local_media_repository_url_cache_by_url_download_ts
	26	ON local_media_repository_url_cache(url, download_ts);

+87

-0

synapse/storage/data_stores/main/schema/delta/31/pushers.py less more

	0	# Copyright 2016 OpenMarket Ltd
	1	#
	2	# Licensed under the Apache License, Version 2.0 (the "License");
	3	# you may not use this file except in compliance with the License.
	4	# You may obtain a copy of the License at
	5	#
	6	# http://www.apache.org/licenses/LICENSE-2.0
	7	#
	8	# Unless required by applicable law or agreed to in writing, software
	9	# distributed under the License is distributed on an "AS IS" BASIS,
	10	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	# See the License for the specific language governing permissions and
	12	# limitations under the License.
	13
	14
	15	# Change the last_token to last_stream_ordering now that pushers no longer
	16	# listen on an event stream but instead select out of the event_push_actions
	17	# table.
	18
	19
	20	import logging
	21
	22	logger = logging.getLogger(__name__)
	23
	24
	25	def token_to_stream_ordering(token):
	26	return int(token[1:].split("_")[0])
	27
	28
	29	def run_create(cur, database_engine, args, *kwargs):
	30	logger.info("Porting pushers table, delta 31...")
	31	cur.execute(
	32	"""
	33	CREATE TABLE IF NOT EXISTS pushers2 (
	34	id BIGINT PRIMARY KEY,
	35	user_name TEXT NOT NULL,
	36	access_token BIGINT DEFAULT NULL,
	37	profile_tag VARCHAR(32) NOT NULL,
	38	kind VARCHAR(8) NOT NULL,
	39	app_id VARCHAR(64) NOT NULL,
	40	app_display_name VARCHAR(64) NOT NULL,
	41	device_display_name VARCHAR(128) NOT NULL,
	42	pushkey TEXT NOT NULL,
	43	ts BIGINT NOT NULL,
	44	lang VARCHAR(8),
	45	data TEXT,
	46	last_stream_ordering INTEGER,
	47	last_success BIGINT,
	48	failing_since BIGINT,
	49	UNIQUE (app_id, pushkey, user_name)
	50	)
	51	"""
	52	)
	53	cur.execute(
	54	"""SELECT
	55	id, user_name, access_token, profile_tag, kind,
	56	app_id, app_display_name, device_display_name,
	57	pushkey, ts, lang, data, last_token, last_success,
	58	failing_since
	59	FROM pushers
	60	"""
	61	)
	62	count = 0
	63	for row in cur.fetchall():
	64	row = list(row)
	65	row[12] = token_to_stream_ordering(row[12])
	66	cur.execute(
	67	database_engine.convert_param_style(
	68	"""
	69	INSERT into pushers2 (
	70	id, user_name, access_token, profile_tag, kind,
	71	app_id, app_display_name, device_display_name,
	72	pushkey, ts, lang, data, last_stream_ordering, last_success,
	73	failing_since
	74	) values (%s)"""
	75	% (",".join(["?" for _ in range(len(row))]))
	76	),
	77	row,
	78	)
	79	count += 1
	80	cur.execute("DROP TABLE pushers")
	81	cur.execute("ALTER TABLE pushers2 RENAME TO pushers")
	82	logger.info("Moved %d pushers to new table", count)
	83
	84
	85	def run_upgrade(cur, database_engine, args, *kwargs):
	86	pass

+22

-0

synapse/storage/data_stores/main/schema/delta/31/pushers_index.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	/** Using CREATE INDEX directly is deprecated in favour of using background
	16	* update see synapse/storage/schema/delta/33/access_tokens_device_index.sql
	17	* and synapse/storage/registration.py for an example using
	18	* "access_tokens_device_index" **/
	19	CREATE INDEX event_push_actions_stream_ordering on event_push_actions(
	20	stream_ordering, user_id
	21	);

+66

-0

synapse/storage/data_stores/main/schema/delta/31/search_update.py less more

	0	# Copyright 2016 OpenMarket Ltd
	1	#
	2	# Licensed under the Apache License, Version 2.0 (the "License");
	3	# you may not use this file except in compliance with the License.
	4	# You may obtain a copy of the License at
	5	#
	6	# http://www.apache.org/licenses/LICENSE-2.0
	7	#
	8	# Unless required by applicable law or agreed to in writing, software
	9	# distributed under the License is distributed on an "AS IS" BASIS,
	10	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	# See the License for the specific language governing permissions and
	12	# limitations under the License.
	13
	14	import logging
	15
	16	import simplejson
	17
	18	from synapse.storage.engines import PostgresEngine
	19	from synapse.storage.prepare_database import get_statements
	20
	21	logger = logging.getLogger(__name__)
	22
	23
	24	ALTER_TABLE = """
	25	ALTER TABLE event_search ADD COLUMN origin_server_ts BIGINT;
	26	ALTER TABLE event_search ADD COLUMN stream_ordering BIGINT;
	27	"""
	28
	29
	30	def run_create(cur, database_engine, args, *kwargs):
	31	if not isinstance(database_engine, PostgresEngine):
	32	return
	33
	34	for statement in get_statements(ALTER_TABLE.splitlines()):
	35	cur.execute(statement)
	36
	37	cur.execute("SELECT MIN(stream_ordering) FROM events")
	38	rows = cur.fetchall()
	39	min_stream_id = rows[0][0]
	40
	41	cur.execute("SELECT MAX(stream_ordering) FROM events")
	42	rows = cur.fetchall()
	43	max_stream_id = rows[0][0]
	44
	45	if min_stream_id is not None and max_stream_id is not None:
	46	progress = {
	47	"target_min_stream_id_inclusive": min_stream_id,
	48	"max_stream_id_exclusive": max_stream_id + 1,
	49	"rows_inserted": 0,
	50	"have_added_indexes": False,
	51	}
	52	progress_json = simplejson.dumps(progress)
	53
	54	sql = (
	55	"INSERT into background_updates (update_name, progress_json)"
	56	" VALUES (?, ?)"
	57	)
	58
	59	sql = database_engine.convert_param_style(sql)
	60
	61	cur.execute(sql, ("event_search_order", progress_json))
	62
	63
	64	def run_upgrade(cur, database_engine, args, *kwargs):
	65	pass

+16

-0

synapse/storage/data_stores/main/schema/delta/32/events.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	ALTER TABLE events ADD COLUMN received_ts BIGINT;

+9

-0

synapse/storage/data_stores/main/schema/delta/32/openid.sql less more

	0
	1	CREATE TABLE open_id_tokens (
	2	token TEXT NOT NULL PRIMARY KEY,
	3	ts_valid_until_ms bigint NOT NULL,
	4	user_id TEXT NOT NULL,
	5	UNIQUE (token)
	6	);
	7
	8	CREATE index open_id_tokens_ts_valid_until_ms ON open_id_tokens(ts_valid_until_ms);

+23

-0

synapse/storage/data_stores/main/schema/delta/32/pusher_throttle.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15
	16	CREATE TABLE pusher_throttle(
	17	pusher BIGINT NOT NULL,
	18	room_id TEXT NOT NULL,
	19	last_sent_ts BIGINT,
	20	throttle_ms BIGINT,
	21	PRIMARY KEY (pusher, room_id)
	22	);

+34

-0

synapse/storage/data_stores/main/schema/delta/32/remove_indices.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15
	16	-- The following indices are redundant, other indices are equivalent or
	17	-- supersets
	18	DROP INDEX IF EXISTS events_room_id; -- Prefix of events_room_stream
	19	DROP INDEX IF EXISTS events_order; -- Prefix of events_order_topo_stream_room
	20	DROP INDEX IF EXISTS events_topological_ordering; -- Prefix of events_order_topo_stream_room
	21	DROP INDEX IF EXISTS events_stream_ordering; -- Duplicate of PRIMARY KEY
	22	DROP INDEX IF EXISTS state_groups_id; -- Duplicate of PRIMARY KEY
	23	DROP INDEX IF EXISTS event_to_state_groups_id; -- Duplicate of PRIMARY KEY
	24	DROP INDEX IF EXISTS event_push_actions_room_id_event_id_user_id_profile_tag; -- Duplicate of UNIQUE CONSTRAINT
	25
	26	DROP INDEX IF EXISTS st_extrem_id; -- Prefix of UNIQUE CONSTRAINT
	27	DROP INDEX IF EXISTS event_signatures_id; -- Prefix of UNIQUE CONSTRAINT
	28	DROP INDEX IF EXISTS redactions_event_id; -- Duplicate of UNIQUE CONSTRAINT
	29
	30	-- The following indices were unused
	31	DROP INDEX IF EXISTS remote_media_cache_thumbnails_media_id;
	32	DROP INDEX IF EXISTS evauth_edges_auth_id;
	33	DROP INDEX IF EXISTS presence_stream_state;

+25

-0

synapse/storage/data_stores/main/schema/delta/32/reports.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15
	16	CREATE TABLE event_reports(
	17	id BIGINT NOT NULL PRIMARY KEY,
	18	received_ts BIGINT NOT NULL,
	19	room_id TEXT NOT NULL,
	20	event_id TEXT NOT NULL,
	21	user_id TEXT NOT NULL,
	22	reason TEXT,
	23	content TEXT
	24	);

+17

-0

synapse/storage/data_stores/main/schema/delta/33/access_tokens_device_index.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	INSERT INTO background_updates (update_name, progress_json) VALUES
	16	('access_tokens_device_index', '{}');

+21

-0

synapse/storage/data_stores/main/schema/delta/33/devices.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	CREATE TABLE devices (
	16	user_id TEXT NOT NULL,
	17	device_id TEXT NOT NULL,
	18	display_name TEXT,
	19	CONSTRAINT device_uniqueness UNIQUE (user_id, device_id)
	20	);

+19

-0

synapse/storage/data_stores/main/schema/delta/33/devices_for_e2e_keys.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- make sure that we have a device record for each set of E2E keys, so that the
	16	-- user can delete them if they like.
	17	INSERT INTO devices
	18	SELECT user_id, device_id, NULL FROM e2e_device_keys_json;

+20

-0

synapse/storage/data_stores/main/schema/delta/33/devices_for_e2e_keys_clear_unknown_device.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- a previous version of the "devices_for_e2e_keys" delta set all the device
	16	-- names to "unknown device". This wasn't terribly helpful
	17	UPDATE devices
	18	SET display_name = NULL
	19	WHERE display_name = 'unknown device';

+61

-0

synapse/storage/data_stores/main/schema/delta/33/event_fields.py less more

	0	# Copyright 2016 OpenMarket Ltd
	1	#
	2	# Licensed under the Apache License, Version 2.0 (the "License");
	3	# you may not use this file except in compliance with the License.
	4	# You may obtain a copy of the License at
	5	#
	6	# http://www.apache.org/licenses/LICENSE-2.0
	7	#
	8	# Unless required by applicable law or agreed to in writing, software
	9	# distributed under the License is distributed on an "AS IS" BASIS,
	10	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	# See the License for the specific language governing permissions and
	12	# limitations under the License.
	13
	14	import logging
	15
	16	import simplejson
	17
	18	from synapse.storage.prepare_database import get_statements
	19
	20	logger = logging.getLogger(__name__)
	21
	22
	23	ALTER_TABLE = """
	24	ALTER TABLE events ADD COLUMN sender TEXT;
	25	ALTER TABLE events ADD COLUMN contains_url BOOLEAN;
	26	"""
	27
	28
	29	def run_create(cur, database_engine, args, *kwargs):
	30	for statement in get_statements(ALTER_TABLE.splitlines()):
	31	cur.execute(statement)
	32
	33	cur.execute("SELECT MIN(stream_ordering) FROM events")
	34	rows = cur.fetchall()
	35	min_stream_id = rows[0][0]
	36
	37	cur.execute("SELECT MAX(stream_ordering) FROM events")
	38	rows = cur.fetchall()
	39	max_stream_id = rows[0][0]
	40
	41	if min_stream_id is not None and max_stream_id is not None:
	42	progress = {
	43	"target_min_stream_id_inclusive": min_stream_id,
	44	"max_stream_id_exclusive": max_stream_id + 1,
	45	"rows_inserted": 0,
	46	}
	47	progress_json = simplejson.dumps(progress)
	48
	49	sql = (
	50	"INSERT into background_updates (update_name, progress_json)"
	51	" VALUES (?, ?)"
	52	)
	53
	54	sql = database_engine.convert_param_style(sql)
	55
	56	cur.execute(sql, ("event_fields_sender_url", progress_json))
	57
	58
	59	def run_upgrade(cur, database_engine, args, *kwargs):
	60	pass

+30

-0

synapse/storage/data_stores/main/schema/delta/33/remote_media_ts.py less more

	0	# Copyright 2016 OpenMarket Ltd
	1	#
	2	# Licensed under the Apache License, Version 2.0 (the "License");
	3	# you may not use this file except in compliance with the License.
	4	# You may obtain a copy of the License at
	5	#
	6	# http://www.apache.org/licenses/LICENSE-2.0
	7	#
	8	# Unless required by applicable law or agreed to in writing, software
	9	# distributed under the License is distributed on an "AS IS" BASIS,
	10	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	# See the License for the specific language governing permissions and
	12	# limitations under the License.
	13
	14	import time
	15
	16	ALTER_TABLE = "ALTER TABLE remote_media_cache ADD COLUMN last_access_ts BIGINT"
	17
	18
	19	def run_create(cur, database_engine, args, *kwargs):
	20	cur.execute(ALTER_TABLE)
	21
	22
	23	def run_upgrade(cur, database_engine, args, *kwargs):
	24	cur.execute(
	25	database_engine.convert_param_style(
	26	"UPDATE remote_media_cache SET last_access_ts = ?"
	27	),
	28	(int(time.time() * 1000),),
	29	)

+17

-0

synapse/storage/data_stores/main/schema/delta/33/user_ips_index.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	INSERT INTO background_updates (update_name, progress_json) VALUES
	16	('user_ips_device_index', '{}');

+23

-0

synapse/storage/data_stores/main/schema/delta/34/appservice_stream.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	CREATE TABLE IF NOT EXISTS appservice_stream_position(
	16	Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, -- Makes sure this table only has one row.
	17	stream_ordering BIGINT,
	18	CHECK (Lock='X')
	19	);
	20
	21	INSERT INTO appservice_stream_position (stream_ordering)
	22	SELECT COALESCE(MAX(stream_ordering), 0) FROM events;

+46

-0

synapse/storage/data_stores/main/schema/delta/34/cache_stream.py less more

	0	# Copyright 2016 OpenMarket Ltd
	1	#
	2	# Licensed under the Apache License, Version 2.0 (the "License");
	3	# you may not use this file except in compliance with the License.
	4	# You may obtain a copy of the License at
	5	#
	6	# http://www.apache.org/licenses/LICENSE-2.0
	7	#
	8	# Unless required by applicable law or agreed to in writing, software
	9	# distributed under the License is distributed on an "AS IS" BASIS,
	10	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	# See the License for the specific language governing permissions and
	12	# limitations under the License.
	13
	14	import logging
	15
	16	from synapse.storage.engines import PostgresEngine
	17	from synapse.storage.prepare_database import get_statements
	18
	19	logger = logging.getLogger(__name__)
	20
	21
	22	# This stream is used to notify replication slaves that some caches have
	23	# been invalidated that they cannot infer from the other streams.
	24	CREATE_TABLE = """
	25	CREATE TABLE cache_invalidation_stream (
	26	stream_id BIGINT,
	27	cache_func TEXT,
	28	keys TEXT[],
	29	invalidation_ts BIGINT
	30	);
	31
	32	CREATE INDEX cache_invalidation_stream_id ON cache_invalidation_stream(stream_id);
	33	"""
	34
	35
	36	def run_create(cur, database_engine, args, *kwargs):
	37	if not isinstance(database_engine, PostgresEngine):
	38	return
	39
	40	for statement in get_statements(CREATE_TABLE.splitlines()):
	41	cur.execute(statement)
	42
	43
	44	def run_upgrade(cur, database_engine, args, *kwargs):
	45	pass

+24

-0

synapse/storage/data_stores/main/schema/delta/34/device_inbox.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	CREATE TABLE device_inbox (
	16	user_id TEXT NOT NULL,
	17	device_id TEXT NOT NULL,
	18	stream_id BIGINT NOT NULL,
	19	message_json TEXT NOT NULL -- {"type":, "sender":, "content",}
	20	);
	21
	22	CREATE INDEX device_inbox_user_stream_id ON device_inbox(user_id, device_id, stream_id);
	23	CREATE INDEX device_inbox_stream_id ON device_inbox(stream_id);

+20

-0

synapse/storage/data_stores/main/schema/delta/34/push_display_name_rename.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	DELETE FROM push_rules WHERE rule_id = 'global/override/.m.rule.contains_display_name';
	16	UPDATE push_rules SET rule_id = 'global/override/.m.rule.contains_display_name' WHERE rule_id = 'global/underride/.m.rule.contains_display_name';
	17
	18	DELETE FROM push_rules_enable WHERE rule_id = 'global/override/.m.rule.contains_display_name';
	19	UPDATE push_rules_enable SET rule_id = 'global/override/.m.rule.contains_display_name' WHERE rule_id = 'global/underride/.m.rule.contains_display_name';

+32

-0

synapse/storage/data_stores/main/schema/delta/34/received_txn_purge.py less more

	0	# Copyright 2016 OpenMarket Ltd
	1	#
	2	# Licensed under the Apache License, Version 2.0 (the "License");
	3	# you may not use this file except in compliance with the License.
	4	# You may obtain a copy of the License at
	5	#
	6	# http://www.apache.org/licenses/LICENSE-2.0
	7	#
	8	# Unless required by applicable law or agreed to in writing, software
	9	# distributed under the License is distributed on an "AS IS" BASIS,
	10	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	# See the License for the specific language governing permissions and
	12	# limitations under the License.
	13
	14	import logging
	15
	16	from synapse.storage.engines import PostgresEngine
	17
	18	logger = logging.getLogger(__name__)
	19
	20
	21	def run_create(cur, database_engine, args, *kwargs):
	22	if isinstance(database_engine, PostgresEngine):
	23	cur.execute("TRUNCATE received_transactions")
	24	else:
	25	cur.execute("DELETE FROM received_transactions")
	26
	27	cur.execute("CREATE INDEX received_transactions_ts ON received_transactions(ts)")
	28
	29
	30	def run_upgrade(cur, database_engine, args, *kwargs):
	31	pass

+17

-0

synapse/storage/data_stores/main/schema/delta/35/add_state_index.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	INSERT into background_updates (update_name, progress_json, depends_on)
	16	VALUES ('state_group_state_type_index', '{}', 'state_group_state_deduplication');

+17

-0

synapse/storage/data_stores/main/schema/delta/35/contains_url.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	INSERT into background_updates (update_name, progress_json)
	16	VALUES ('event_contains_url_index', '{}');

+39

-0

synapse/storage/data_stores/main/schema/delta/35/device_outbox.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	DROP TABLE IF EXISTS device_federation_outbox;
	16	CREATE TABLE device_federation_outbox (
	17	destination TEXT NOT NULL,
	18	stream_id BIGINT NOT NULL,
	19	queued_ts BIGINT NOT NULL,
	20	messages_json TEXT NOT NULL
	21	);
	22
	23
	24	DROP INDEX IF EXISTS device_federation_outbox_destination_id;
	25	CREATE INDEX device_federation_outbox_destination_id
	26	ON device_federation_outbox(destination, stream_id);
	27
	28
	29	DROP TABLE IF EXISTS device_federation_inbox;
	30	CREATE TABLE device_federation_inbox (
	31	origin TEXT NOT NULL,
	32	message_id TEXT NOT NULL,
	33	received_ts BIGINT NOT NULL
	34	);
	35
	36	DROP INDEX IF EXISTS device_federation_inbox_sender_id;
	37	CREATE INDEX device_federation_inbox_sender_id
	38	ON device_federation_inbox(origin, message_id);

+21

-0

synapse/storage/data_stores/main/schema/delta/35/device_stream_id.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	CREATE TABLE device_max_stream_id (
	16	stream_id BIGINT NOT NULL
	17	);
	18
	19	INSERT INTO device_max_stream_id (stream_id)
	20	SELECT COALESCE(MAX(stream_id), 0) FROM device_inbox;

+17

-0

synapse/storage/data_stores/main/schema/delta/35/event_push_actions_index.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	INSERT into background_updates (update_name, progress_json)
	16	VALUES ('epa_highlight_index', '{}');

+33

-0

synapse/storage/data_stores/main/schema/delta/35/public_room_list_change_stream.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15
	16	CREATE TABLE public_room_list_stream (
	17	stream_id BIGINT NOT NULL,
	18	room_id TEXT NOT NULL,
	19	visibility BOOLEAN NOT NULL
	20	);
	21
	22	INSERT INTO public_room_list_stream (stream_id, room_id, visibility)
	23	SELECT 1, room_id, is_public FROM rooms
	24	WHERE is_public = CAST(1 AS BOOLEAN);
	25
	26	CREATE INDEX public_room_list_stream_idx on public_room_list_stream(
	27	stream_id
	28	);
	29
	30	CREATE INDEX public_room_list_stream_rm_idx on public_room_list_stream(
	31	room_id, stream_id
	32	);

+22

-0

synapse/storage/data_stores/main/schema/delta/35/state.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	CREATE TABLE state_group_edges(
	16	state_group BIGINT NOT NULL,
	17	prev_state_group BIGINT NOT NULL
	18	);
	19
	20	CREATE INDEX state_group_edges_idx ON state_group_edges(state_group);
	21	CREATE INDEX state_group_edges_prev_idx ON state_group_edges(prev_state_group);

+17

-0

synapse/storage/data_stores/main/schema/delta/35/state_dedupe.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	INSERT into background_updates (update_name, progress_json)
	16	VALUES ('state_group_state_deduplication', '{}');

+37

-0

synapse/storage/data_stores/main/schema/delta/35/stream_order_to_extrem.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15
	16	CREATE TABLE stream_ordering_to_exterm (
	17	stream_ordering BIGINT NOT NULL,
	18	room_id TEXT NOT NULL,
	19	event_id TEXT NOT NULL
	20	);
	21
	22	INSERT INTO stream_ordering_to_exterm (stream_ordering, room_id, event_id)
	23	SELECT stream_ordering, room_id, event_id FROM event_forward_extremities
	24	INNER JOIN (
	25	SELECT room_id, max(stream_ordering) as stream_ordering FROM events
	26	INNER JOIN event_forward_extremities USING (room_id, event_id)
	27	GROUP BY room_id
	28	) AS rms USING (room_id);
	29
	30	CREATE INDEX stream_ordering_to_exterm_idx on stream_ordering_to_exterm(
	31	stream_ordering
	32	);
	33
	34	CREATE INDEX stream_ordering_to_exterm_rm_idx on stream_ordering_to_exterm(
	35	room_id, stream_ordering
	36	);

+26

-0

synapse/storage/data_stores/main/schema/delta/36/readd_public_rooms.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- Re-add some entries to stream_ordering_to_exterm that were incorrectly deleted
	16	INSERT INTO stream_ordering_to_exterm (stream_ordering, room_id, event_id)
	17	SELECT
	18	(SELECT stream_ordering FROM events where event_id = e.event_id) AS stream_ordering,
	19	room_id,
	20	event_id
	21	FROM event_forward_extremities AS e
	22	WHERE NOT EXISTS (
	23	SELECT room_id FROM stream_ordering_to_exterm AS s
	24	WHERE s.room_id = e.room_id
	25	);

+85

-0

synapse/storage/data_stores/main/schema/delta/37/remove_auth_idx.py less more

	0	# Copyright 2016 OpenMarket Ltd
	1	#
	2	# Licensed under the Apache License, Version 2.0 (the "License");
	3	# you may not use this file except in compliance with the License.
	4	# You may obtain a copy of the License at
	5	#
	6	# http://www.apache.org/licenses/LICENSE-2.0
	7	#
	8	# Unless required by applicable law or agreed to in writing, software
	9	# distributed under the License is distributed on an "AS IS" BASIS,
	10	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	# See the License for the specific language governing permissions and
	12	# limitations under the License.
	13
	14	import logging
	15
	16	from synapse.storage.engines import PostgresEngine
	17	from synapse.storage.prepare_database import get_statements
	18
	19	logger = logging.getLogger(__name__)
	20
	21	DROP_INDICES = """
	22	-- We only ever query based on event_id
	23	DROP INDEX IF EXISTS state_events_room_id;
	24	DROP INDEX IF EXISTS state_events_type;
	25	DROP INDEX IF EXISTS state_events_state_key;
	26
	27	-- room_id is indexed elsewhere
	28	DROP INDEX IF EXISTS current_state_events_room_id;
	29	DROP INDEX IF EXISTS current_state_events_state_key;
	30	DROP INDEX IF EXISTS current_state_events_type;
	31
	32	DROP INDEX IF EXISTS transactions_have_ref;
	33
	34	-- (topological_ordering, stream_ordering, room_id) seems like a strange index,
	35	-- and is used incredibly rarely.
	36	DROP INDEX IF EXISTS events_order_topo_stream_room;
	37
	38	-- an equivalent index to this actually gets re-created in delta 41, because it
	39	-- turned out that deleting it wasn't a great plan :/. In any case, let's
	40	-- delete it here, and delta 41 will create a new one with an added UNIQUE
	41	-- constraint
	42	DROP INDEX IF EXISTS event_search_ev_idx;
	43	"""
	44
	45	POSTGRES_DROP_CONSTRAINT = """
	46	ALTER TABLE event_auth DROP CONSTRAINT IF EXISTS event_auth_event_id_auth_id_room_id_key;
	47	"""
	48
	49	SQLITE_DROP_CONSTRAINT = """
	50	DROP INDEX IF EXISTS evauth_edges_id;
	51
	52	CREATE TABLE IF NOT EXISTS event_auth_new(
	53	event_id TEXT NOT NULL,
	54	auth_id TEXT NOT NULL,
	55	room_id TEXT NOT NULL
	56	);
	57
	58	INSERT INTO event_auth_new
	59	SELECT event_id, auth_id, room_id
	60	FROM event_auth;
	61
	62	DROP TABLE event_auth;
	63
	64	ALTER TABLE event_auth_new RENAME TO event_auth;
	65
	66	CREATE INDEX evauth_edges_id ON event_auth(event_id);
	67	"""
	68
	69
	70	def run_create(cur, database_engine, args, *kwargs):
	71	for statement in get_statements(DROP_INDICES.splitlines()):
	72	cur.execute(statement)
	73
	74	if isinstance(database_engine, PostgresEngine):
	75	drop_constraint = POSTGRES_DROP_CONSTRAINT
	76	else:
	77	drop_constraint = SQLITE_DROP_CONSTRAINT
	78
	79	for statement in get_statements(drop_constraint.splitlines()):
	80	cur.execute(statement)
	81
	82
	83	def run_upgrade(cur, database_engine, args, *kwargs):
	84	pass

+52

-0

synapse/storage/data_stores/main/schema/delta/37/user_threepids.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	/*
	16	* Update any email addresses that were stored with mixed case into all
	17	* lowercase
	18	*/
	19
	20	-- There may be "duplicate" emails (with different case) already in the table,
	21	-- so we find them and move all but the most recently used account.
	22	UPDATE user_threepids
	23	SET medium = 'email_old'
	24	WHERE medium = 'email'
	25	AND address IN (
	26	-- We select all the addresses that are linked to the user_id that is NOT
	27	-- the most recently created.
	28	SELECT u.address
	29	FROM
	30	user_threepids AS u,
	31	-- `duplicate_addresses` is a table of all the email addresses that
	32	-- appear multiple times and when the binding was created
	33	(
	34	SELECT lower(u1.address) AS address, max(u1.added_at) AS max_ts
	35	FROM user_threepids AS u1
	36	INNER JOIN user_threepids AS u2 ON u1.medium = u2.medium AND lower(u1.address) = lower(u2.address) AND u1.address != u2.address
	37	WHERE u1.medium = 'email' AND u2.medium = 'email'
	38	GROUP BY lower(u1.address)
	39	) AS duplicate_addresses
	40	WHERE
	41	lower(u.address) = duplicate_addresses.address
	42	AND u.added_at != max_ts -- NOT the most recently created
	43	);
	44
	45
	46	-- This update is now safe since we've removed the duplicate addresses.
	47	UPDATE user_threepids SET address = LOWER(address) WHERE medium = 'email';
	48
	49
	50	/* Add an index for the select we do on passwored reset */
	51	CREATE INDEX user_threepids_medium_address on user_threepids (medium, address);

+19

-0

synapse/storage/data_stores/main/schema/delta/38/postgres_fts_gist.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- We no longer do this given we back it out again in schema 47
	16
	17	-- INSERT into background_updates (update_name, progress_json)
	18	-- VALUES ('event_search_postgres_gist', '{}');

+29

-0

synapse/storage/data_stores/main/schema/delta/39/appservice_room_list.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	CREATE TABLE appservice_room_list(
	16	appservice_id TEXT NOT NULL,
	17	network_id TEXT NOT NULL,
	18	room_id TEXT NOT NULL
	19	);
	20
	21	-- Each appservice can have multiple published room lists associated with them,
	22	-- keyed of a particular network_id
	23	CREATE UNIQUE INDEX appservice_room_list_idx ON appservice_room_list(
	24	appservice_id, network_id, room_id
	25	);
	26
	27	ALTER TABLE public_room_list_stream ADD COLUMN appservice_id TEXT;
	28	ALTER TABLE public_room_list_stream ADD COLUMN network_id TEXT;

+16

-0

synapse/storage/data_stores/main/schema/delta/39/device_federation_stream_idx.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	CREATE INDEX device_federation_outbox_id ON device_federation_outbox(stream_id);

+17

-0

synapse/storage/data_stores/main/schema/delta/39/event_push_index.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	INSERT INTO background_updates (update_name, progress_json) VALUES
	16	('event_push_actions_highlights_index', '{}');

+22

-0

synapse/storage/data_stores/main/schema/delta/39/federation_out_position.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	CREATE TABLE federation_stream_position(
	16	type TEXT NOT NULL,
	17	stream_id INTEGER NOT NULL
	18	);
	19
	20	INSERT INTO federation_stream_position (type, stream_id) VALUES ('federation', -1);
	21	INSERT INTO federation_stream_position (type, stream_id) SELECT 'events', coalesce(max(stream_ordering), -1) FROM events;

+20

-0

synapse/storage/data_stores/main/schema/delta/39/membership_profile.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	ALTER TABLE room_memberships ADD COLUMN display_name TEXT;
	16	ALTER TABLE room_memberships ADD COLUMN avatar_url TEXT;
	17
	18	INSERT into background_updates (update_name, progress_json)
	19	VALUES ('room_membership_profile_update', '{}');

+17

-0

synapse/storage/data_stores/main/schema/delta/40/current_state_idx.sql less more

	0	/* Copyright 2017 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	INSERT INTO background_updates (update_name, progress_json) VALUES
	16	('current_state_members_idx', '{}');

+21

-0

synapse/storage/data_stores/main/schema/delta/40/device_inbox.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- turn the pre-fill startup query into a index-only scan on postgresql.
	16	INSERT into background_updates (update_name, progress_json)
	17	VALUES ('device_inbox_stream_index', '{}');
	18
	19	INSERT into background_updates (update_name, progress_json, depends_on)
	20	VALUES ('device_inbox_stream_drop', '{}', 'device_inbox_stream_index');

+60

-0

synapse/storage/data_stores/main/schema/delta/40/device_list_streams.sql less more

	0	/* Copyright 2017 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- Cache of remote devices.
	16	CREATE TABLE device_lists_remote_cache (
	17	user_id TEXT NOT NULL,
	18	device_id TEXT NOT NULL,
	19	content TEXT NOT NULL
	20	);
	21
	22	-- The last update we got for a user. Empty if we're not receiving updates for
	23	-- that user.
	24	CREATE TABLE device_lists_remote_extremeties (
	25	user_id TEXT NOT NULL,
	26	stream_id TEXT NOT NULL
	27	);
	28
	29	-- we used to create non-unique indexes on these tables, but as of update 52 we create
	30	-- unique indexes concurrently:
	31	--
	32	-- CREATE INDEX device_lists_remote_cache_id ON device_lists_remote_cache(user_id, device_id);
	33	-- CREATE INDEX device_lists_remote_extremeties_id ON device_lists_remote_extremeties(user_id, stream_id);
	34
	35
	36	-- Stream of device lists updates. Includes both local and remotes
	37	CREATE TABLE device_lists_stream (
	38	stream_id BIGINT NOT NULL,
	39	user_id TEXT NOT NULL,
	40	device_id TEXT NOT NULL
	41	);
	42
	43	CREATE INDEX device_lists_stream_id ON device_lists_stream(stream_id, user_id);
	44
	45
	46	-- The stream of updates to send to other servers. We keep at least one row
	47	-- per user that was sent so that the prev_id for any new updates can be
	48	-- calculated
	49	CREATE TABLE device_lists_outbound_pokes (
	50	destination TEXT NOT NULL,
	51	stream_id BIGINT NOT NULL,
	52	user_id TEXT NOT NULL,
	53	device_id TEXT NOT NULL,
	54	sent BOOLEAN NOT NULL,
	55	ts BIGINT NOT NULL -- So that in future we can clear out pokes to dead servers
	56	);
	57
	58	CREATE INDEX device_lists_outbound_pokes_id ON device_lists_outbound_pokes(destination, stream_id);
	59	CREATE INDEX device_lists_outbound_pokes_user ON device_lists_outbound_pokes(destination, user_id);

+37

-0

synapse/storage/data_stores/main/schema/delta/40/event_push_summary.sql less more

	0	/* Copyright 2017 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- Aggregate of old notification counts that have been deleted out of the
	16	-- main event_push_actions table. This count does not include those that were
	17	-- highlights, as they remain in the event_push_actions table.
	18	CREATE TABLE event_push_summary (
	19	user_id TEXT NOT NULL,
	20	room_id TEXT NOT NULL,
	21	notif_count BIGINT NOT NULL,
	22	stream_ordering BIGINT NOT NULL
	23	);
	24
	25	CREATE INDEX event_push_summary_user_rm ON event_push_summary(user_id, room_id);
	26
	27
	28	-- The stream ordering up to which we have aggregated the event_push_actions
	29	-- table into event_push_summary
	30	CREATE TABLE event_push_summary_stream_ordering (
	31	Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, -- Makes sure this table only has one row.
	32	stream_ordering BIGINT NOT NULL,
	33	CHECK (Lock='X')
	34	);
	35
	36	INSERT INTO event_push_summary_stream_ordering (stream_ordering) VALUES (0);

+39

-0

synapse/storage/data_stores/main/schema/delta/40/pushers.sql less more

	0	/* Copyright 2017 Vector Creations Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	CREATE TABLE IF NOT EXISTS pushers2 (
	16	id BIGINT PRIMARY KEY,
	17	user_name TEXT NOT NULL,
	18	access_token BIGINT DEFAULT NULL,
	19	profile_tag TEXT NOT NULL,
	20	kind TEXT NOT NULL,
	21	app_id TEXT NOT NULL,
	22	app_display_name TEXT NOT NULL,
	23	device_display_name TEXT NOT NULL,
	24	pushkey TEXT NOT NULL,
	25	ts BIGINT NOT NULL,
	26	lang TEXT,
	27	data TEXT,
	28	last_stream_ordering INTEGER,
	29	last_success BIGINT,
	30	failing_since BIGINT,
	31	UNIQUE (app_id, pushkey, user_name)
	32	);
	33
	34	INSERT INTO pushers2 SELECT * FROM PUSHERS;
	35
	36	DROP TABLE PUSHERS;
	37
	38	ALTER TABLE pushers2 RENAME TO pushers;

+17

-0

synapse/storage/data_stores/main/schema/delta/41/device_list_stream_idx.sql less more

	0	/* Copyright 2017 Vector Creations Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	INSERT into background_updates (update_name, progress_json)
	16	VALUES ('device_lists_stream_idx', '{}');

+16

-0

synapse/storage/data_stores/main/schema/delta/41/device_outbound_index.sql less more

	0	/* Copyright 2017 Vector Creations Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	CREATE INDEX device_lists_outbound_pokes_stream ON device_lists_outbound_pokes(stream_id);

+17

-0

synapse/storage/data_stores/main/schema/delta/41/event_search_event_id_idx.sql less more

	0	/* Copyright 2017 Vector Creations Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	INSERT into background_updates (update_name, progress_json)
	16	VALUES ('event_search_event_id_idx', '{}');

+22

-0

synapse/storage/data_stores/main/schema/delta/41/ratelimit.sql less more

	0	/* Copyright 2017 Vector Creations Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	CREATE TABLE ratelimit_override (
	16	user_id TEXT NOT NULL,
	17	messages_per_second BIGINT,
	18	burst_count BIGINT
	19	);
	20
	21	CREATE UNIQUE INDEX ratelimit_override_idx ON ratelimit_override(user_id);

+26

-0

synapse/storage/data_stores/main/schema/delta/42/current_state_delta.sql less more

	0	/* Copyright 2017 Vector Creations Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15
	16	CREATE TABLE current_state_delta_stream (
	17	stream_id BIGINT NOT NULL,
	18	room_id TEXT NOT NULL,
	19	type TEXT NOT NULL,
	20	state_key TEXT NOT NULL,
	21	event_id TEXT, -- Is null if the key was removed
	22	prev_event_id TEXT -- Is null if the key was added
	23	);
	24
	25	CREATE INDEX current_state_delta_stream_idx ON current_state_delta_stream(stream_id);

+33

-0

synapse/storage/data_stores/main/schema/delta/42/device_list_last_id.sql less more

	0	/* Copyright 2017 Vector Creations Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15
	16	-- Table of last stream_id that we sent to destination for user_id. This is
	17	-- used to fill out the `prev_id` fields of outbound device list updates.
	18	CREATE TABLE device_lists_outbound_last_success (
	19	destination TEXT NOT NULL,
	20	user_id TEXT NOT NULL,
	21	stream_id BIGINT NOT NULL
	22	);
	23
	24	INSERT INTO device_lists_outbound_last_success
	25	SELECT destination, user_id, coalesce(max(stream_id), 0) as stream_id
	26	FROM device_lists_outbound_pokes
	27	WHERE sent = (1 = 1) -- sqlite doesn't have inbuilt boolean values
	28	GROUP BY destination, user_id;
	29
	30	CREATE INDEX device_lists_outbound_last_success_idx ON device_lists_outbound_last_success(
	31	destination, user_id, stream_id
	32	);

+17

-0

synapse/storage/data_stores/main/schema/delta/42/event_auth_state_only.sql less more

	0	/* Copyright 2017 Vector Creations Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	INSERT INTO background_updates (update_name, progress_json) VALUES
	16	('event_auth_state_only', '{}');

+84

-0

synapse/storage/data_stores/main/schema/delta/42/user_dir.py less more

	0	# Copyright 2017 Vector Creations Ltd
	1	#
	2	# Licensed under the Apache License, Version 2.0 (the "License");
	3	# you may not use this file except in compliance with the License.
	4	# You may obtain a copy of the License at
	5	#
	6	# http://www.apache.org/licenses/LICENSE-2.0
	7	#
	8	# Unless required by applicable law or agreed to in writing, software
	9	# distributed under the License is distributed on an "AS IS" BASIS,
	10	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	# See the License for the specific language governing permissions and
	12	# limitations under the License.
	13
	14	import logging
	15
	16	from synapse.storage.engines import PostgresEngine, Sqlite3Engine
	17	from synapse.storage.prepare_database import get_statements
	18
	19	logger = logging.getLogger(__name__)
	20
	21
	22	BOTH_TABLES = """
	23	CREATE TABLE user_directory_stream_pos (
	24	Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, -- Makes sure this table only has one row.
	25	stream_id BIGINT,
	26	CHECK (Lock='X')
	27	);
	28
	29	INSERT INTO user_directory_stream_pos (stream_id) VALUES (null);
	30
	31	CREATE TABLE user_directory (
	32	user_id TEXT NOT NULL,
	33	room_id TEXT NOT NULL, -- A room_id that we know the user is joined to
	34	display_name TEXT,
	35	avatar_url TEXT
	36	);
	37
	38	CREATE INDEX user_directory_room_idx ON user_directory(room_id);
	39	CREATE UNIQUE INDEX user_directory_user_idx ON user_directory(user_id);
	40
	41	CREATE TABLE users_in_pubic_room (
	42	user_id TEXT NOT NULL,
	43	room_id TEXT NOT NULL -- A room_id that we know is public
	44	);
	45
	46	CREATE INDEX users_in_pubic_room_room_idx ON users_in_pubic_room(room_id);
	47	CREATE UNIQUE INDEX users_in_pubic_room_user_idx ON users_in_pubic_room(user_id);
	48	"""
	49
	50
	51	POSTGRES_TABLE = """
	52	CREATE TABLE user_directory_search (
	53	user_id TEXT NOT NULL,
	54	vector tsvector
	55	);
	56
	57	CREATE INDEX user_directory_search_fts_idx ON user_directory_search USING gin(vector);
	58	CREATE UNIQUE INDEX user_directory_search_user_idx ON user_directory_search(user_id);
	59	"""
	60
	61
	62	SQLITE_TABLE = """
	63	CREATE VIRTUAL TABLE user_directory_search
	64	USING fts4 ( user_id, value );
	65	"""
	66
	67
	68	def run_create(cur, database_engine, args, *kwargs):
	69	for statement in get_statements(BOTH_TABLES.splitlines()):
	70	cur.execute(statement)
	71
	72	if isinstance(database_engine, PostgresEngine):
	73	for statement in get_statements(POSTGRES_TABLE.splitlines()):
	74	cur.execute(statement)
	75	elif isinstance(database_engine, Sqlite3Engine):
	76	for statement in get_statements(SQLITE_TABLE.splitlines()):
	77	cur.execute(statement)
	78	else:
	79	raise Exception("Unrecognized database engine")
	80
	81
	82	def run_upgrade(args, *kwargs):
	83	pass

+21

-0

synapse/storage/data_stores/main/schema/delta/43/blocked_rooms.sql less more

	0	/* Copyright 2017 Vector Creations Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	CREATE TABLE blocked_rooms (
	16	room_id TEXT NOT NULL,
	17	user_id TEXT NOT NULL -- Admin who blocked the room
	18	);
	19
	20	CREATE UNIQUE INDEX blocked_rooms_idx ON blocked_rooms(room_id);

+17

-0

synapse/storage/data_stores/main/schema/delta/43/quarantine_media.sql less more

	0	/* Copyright 2017 Vector Creations Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	ALTER TABLE local_media_repository ADD COLUMN quarantined_by TEXT;
	16	ALTER TABLE remote_media_cache ADD COLUMN quarantined_by TEXT;

+16

-0

synapse/storage/data_stores/main/schema/delta/43/url_cache.sql less more

	0	/* Copyright 2017 Vector Creations Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	ALTER TABLE local_media_repository ADD COLUMN url_cache TEXT;

+33

-0

synapse/storage/data_stores/main/schema/delta/43/user_share.sql less more

	0	/* Copyright 2017 Vector Creations Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- Table keeping track of who shares a room with who. We only keep track
	16	-- of this for local users, so `user_id` is local users only (but we do keep track
	17	-- of which remote users share a room)
	18	CREATE TABLE users_who_share_rooms (
	19	user_id TEXT NOT NULL,
	20	other_user_id TEXT NOT NULL,
	21	room_id TEXT NOT NULL,
	22	share_private BOOLEAN NOT NULL -- is the shared room private? i.e. they share a private room
	23	);
	24
	25
	26	CREATE UNIQUE INDEX users_who_share_rooms_u_idx ON users_who_share_rooms(user_id, other_user_id);
	27	CREATE INDEX users_who_share_rooms_r_idx ON users_who_share_rooms(room_id);
	28	CREATE INDEX users_who_share_rooms_o_idx ON users_who_share_rooms(other_user_id);
	29
	30
	31	-- Make sure that we populate the table initially
	32	UPDATE user_directory_stream_pos SET stream_id = NULL;

+41

-0

synapse/storage/data_stores/main/schema/delta/44/expire_url_cache.sql less more

	0	/* Copyright 2017 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- this didn't work on SQLite 3.7 (because of lack of partial indexes), so was
	16	-- removed and replaced with 46/local_media_repository_url_idx.sql.
	17	--
	18	-- CREATE INDEX local_media_repository_url_idx ON local_media_repository(created_ts) WHERE url_cache IS NOT NULL;
	19
	20	-- we need to change `expires` to `expires_ts` so that we can index on it. SQLite doesn't support
	21	-- indices on expressions until 3.9.
	22	CREATE TABLE local_media_repository_url_cache_new(
	23	url TEXT,
	24	response_code INTEGER,
	25	etag TEXT,
	26	expires_ts BIGINT,
	27	og TEXT,
	28	media_id TEXT,
	29	download_ts BIGINT
	30	);
	31
	32	INSERT INTO local_media_repository_url_cache_new
	33	SELECT url, response_code, etag, expires + download_ts, og, media_id, download_ts FROM local_media_repository_url_cache;
	34
	35	DROP TABLE local_media_repository_url_cache;
	36	ALTER TABLE local_media_repository_url_cache_new RENAME TO local_media_repository_url_cache;
	37
	38	CREATE INDEX local_media_repository_url_cache_expires_idx ON local_media_repository_url_cache(expires_ts);
	39	CREATE INDEX local_media_repository_url_cache_by_url_download_ts ON local_media_repository_url_cache(url, download_ts);
	40	CREATE INDEX local_media_repository_url_cache_media_idx ON local_media_repository_url_cache(media_id);

+167

-0

synapse/storage/data_stores/main/schema/delta/45/group_server.sql less more

	0	/* Copyright 2017 Vector Creations Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	CREATE TABLE groups (
	16	group_id TEXT NOT NULL,
	17	name TEXT, -- the display name of the room
	18	avatar_url TEXT,
	19	short_description TEXT,
	20	long_description TEXT
	21	);
	22
	23	CREATE UNIQUE INDEX groups_idx ON groups(group_id);
	24
	25
	26	-- list of users the group server thinks are joined
	27	CREATE TABLE group_users (
	28	group_id TEXT NOT NULL,
	29	user_id TEXT NOT NULL,
	30	is_admin BOOLEAN NOT NULL,
	31	is_public BOOLEAN NOT NULL -- whether the users membership can be seen by everyone
	32	);
	33
	34
	35	CREATE INDEX groups_users_g_idx ON group_users(group_id, user_id);
	36	CREATE INDEX groups_users_u_idx ON group_users(user_id);
	37
	38	-- list of users the group server thinks are invited
	39	CREATE TABLE group_invites (
	40	group_id TEXT NOT NULL,
	41	user_id TEXT NOT NULL
	42	);
	43
	44	CREATE INDEX groups_invites_g_idx ON group_invites(group_id, user_id);
	45	CREATE INDEX groups_invites_u_idx ON group_invites(user_id);
	46
	47
	48	CREATE TABLE group_rooms (
	49	group_id TEXT NOT NULL,
	50	room_id TEXT NOT NULL,
	51	is_public BOOLEAN NOT NULL -- whether the room can be seen by everyone
	52	);
	53
	54	CREATE UNIQUE INDEX groups_rooms_g_idx ON group_rooms(group_id, room_id);
	55	CREATE INDEX groups_rooms_r_idx ON group_rooms(room_id);
	56
	57
	58	-- Rooms to include in the summary
	59	CREATE TABLE group_summary_rooms (
	60	group_id TEXT NOT NULL,
	61	room_id TEXT NOT NULL,
	62	category_id TEXT NOT NULL,
	63	room_order BIGINT NOT NULL,
	64	is_public BOOLEAN NOT NULL, -- whether the room should be show to everyone
	65	UNIQUE (group_id, category_id, room_id, room_order),
	66	CHECK (room_order > 0)
	67	);
	68
	69	CREATE UNIQUE INDEX group_summary_rooms_g_idx ON group_summary_rooms(group_id, room_id, category_id);
	70
	71
	72	-- Categories to include in the summary
	73	CREATE TABLE group_summary_room_categories (
	74	group_id TEXT NOT NULL,
	75	category_id TEXT NOT NULL,
	76	cat_order BIGINT NOT NULL,
	77	UNIQUE (group_id, category_id, cat_order),
	78	CHECK (cat_order > 0)
	79	);
	80
	81	-- The categories in the group
	82	CREATE TABLE group_room_categories (
	83	group_id TEXT NOT NULL,
	84	category_id TEXT NOT NULL,
	85	profile TEXT NOT NULL,
	86	is_public BOOLEAN NOT NULL, -- whether the category should be show to everyone
	87	UNIQUE (group_id, category_id)
	88	);
	89
	90	-- The users to include in the group summary
	91	CREATE TABLE group_summary_users (
	92	group_id TEXT NOT NULL,
	93	user_id TEXT NOT NULL,
	94	role_id TEXT NOT NULL,
	95	user_order BIGINT NOT NULL,
	96	is_public BOOLEAN NOT NULL -- whether the user should be show to everyone
	97	);
	98
	99	CREATE INDEX group_summary_users_g_idx ON group_summary_users(group_id);
	100
	101	-- The roles to include in the group summary
	102	CREATE TABLE group_summary_roles (
	103	group_id TEXT NOT NULL,
	104	role_id TEXT NOT NULL,
	105	role_order BIGINT NOT NULL,
	106	UNIQUE (group_id, role_id, role_order),
	107	CHECK (role_order > 0)
	108	);
	109
	110
	111	-- The roles in a groups
	112	CREATE TABLE group_roles (
	113	group_id TEXT NOT NULL,
	114	role_id TEXT NOT NULL,
	115	profile TEXT NOT NULL,
	116	is_public BOOLEAN NOT NULL, -- whether the role should be show to everyone
	117	UNIQUE (group_id, role_id)
	118	);
	119
	120
	121	-- List of attestations we've given out and need to renew
	122	CREATE TABLE group_attestations_renewals (
	123	group_id TEXT NOT NULL,
	124	user_id TEXT NOT NULL,
	125	valid_until_ms BIGINT NOT NULL
	126	);
	127
	128	CREATE INDEX group_attestations_renewals_g_idx ON group_attestations_renewals(group_id, user_id);
	129	CREATE INDEX group_attestations_renewals_u_idx ON group_attestations_renewals(user_id);
	130	CREATE INDEX group_attestations_renewals_v_idx ON group_attestations_renewals(valid_until_ms);
	131
	132
	133	-- List of attestations we've received from remotes and are interested in.
	134	CREATE TABLE group_attestations_remote (
	135	group_id TEXT NOT NULL,
	136	user_id TEXT NOT NULL,
	137	valid_until_ms BIGINT NOT NULL,
	138	attestation_json TEXT NOT NULL
	139	);
	140
	141	CREATE INDEX group_attestations_remote_g_idx ON group_attestations_remote(group_id, user_id);
	142	CREATE INDEX group_attestations_remote_u_idx ON group_attestations_remote(user_id);
	143	CREATE INDEX group_attestations_remote_v_idx ON group_attestations_remote(valid_until_ms);
	144
	145
	146	-- The group membership for the HS's users
	147	CREATE TABLE local_group_membership (
	148	group_id TEXT NOT NULL,
	149	user_id TEXT NOT NULL,
	150	is_admin BOOLEAN NOT NULL,
	151	membership TEXT NOT NULL,
	152	is_publicised BOOLEAN NOT NULL, -- if the user is publicising their membership
	153	content TEXT NOT NULL
	154	);
	155
	156	CREATE INDEX local_group_membership_u_idx ON local_group_membership(user_id, group_id);
	157	CREATE INDEX local_group_membership_g_idx ON local_group_membership(group_id);
	158
	159
	160	CREATE TABLE local_group_updates (
	161	stream_id BIGINT NOT NULL,
	162	group_id TEXT NOT NULL,
	163	user_id TEXT NOT NULL,
	164	type TEXT NOT NULL,
	165	content TEXT NOT NULL
	166	);

+28

-0

synapse/storage/data_stores/main/schema/delta/45/profile_cache.sql less more

	0	/* Copyright 2017 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15
	16	-- A subset of remote users whose profiles we have cached.
	17	-- Whether a user is in this table or not is defined by the storage function
	18	-- `is_subscribed_remote_profile_for_user`
	19	CREATE TABLE remote_profile_cache (
	20	user_id TEXT NOT NULL,
	21	displayname TEXT,
	22	avatar_url TEXT,
	23	last_check BIGINT NOT NULL
	24	);
	25
	26	CREATE UNIQUE INDEX remote_profile_cache_user_id ON remote_profile_cache(user_id);
	27	CREATE INDEX remote_profile_cache_time ON remote_profile_cache(last_check);

+17

-0

synapse/storage/data_stores/main/schema/delta/46/drop_refresh_tokens.sql less more

	0	/* Copyright 2017 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	/* we no longer use (or create) the refresh_tokens table */
	16	DROP TABLE IF EXISTS refresh_tokens;

+35

-0

synapse/storage/data_stores/main/schema/delta/46/drop_unique_deleted_pushers.sql less more

	0	/* Copyright 2017 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- drop the unique constraint on deleted_pushers so that we can just insert
	16	-- into it rather than upserting.
	17
	18	CREATE TABLE deleted_pushers2 (
	19	stream_id BIGINT NOT NULL,
	20	app_id TEXT NOT NULL,
	21	pushkey TEXT NOT NULL,
	22	user_id TEXT NOT NULL
	23	);
	24
	25	INSERT INTO deleted_pushers2 (stream_id, app_id, pushkey, user_id)
	26	SELECT stream_id, app_id, pushkey, user_id from deleted_pushers;
	27
	28	DROP TABLE deleted_pushers;
	29	ALTER TABLE deleted_pushers2 RENAME TO deleted_pushers;
	30
	31	-- create the index after doing the inserts because that's more efficient.
	32	-- it also means we can give it the same name as the old one without renaming.
	33	CREATE INDEX deleted_pushers_stream_id ON deleted_pushers (stream_id);
	34

+32

-0

synapse/storage/data_stores/main/schema/delta/46/group_server.sql less more

	0	/* Copyright 2017 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	CREATE TABLE groups_new (
	16	group_id TEXT NOT NULL,
	17	name TEXT, -- the display name of the room
	18	avatar_url TEXT,
	19	short_description TEXT,
	20	long_description TEXT,
	21	is_public BOOL NOT NULL -- whether non-members can access group APIs
	22	);
	23
	24	-- NB: awful hack to get the default to be true on postgres and 1 on sqlite
	25	INSERT INTO groups_new
	26	SELECT group_id, name, avatar_url, short_description, long_description, (1=1) FROM groups;
	27
	28	DROP TABLE groups;
	29	ALTER TABLE groups_new RENAME TO groups;
	30
	31	CREATE UNIQUE INDEX groups_idx ON groups(group_id);

+24

-0

synapse/storage/data_stores/main/schema/delta/46/local_media_repository_url_idx.sql less more

	0	/* Copyright 2017 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- register a background update which will recreate the
	16	-- local_media_repository_url_idx index.
	17	--
	18	-- We do this as a bg update not because it is a particularly onerous
	19	-- operation, but because we'd like it to be a partial index if possible, and
	20	-- the background_index_update code will understand whether we are on
	21	-- postgres or sqlite and behave accordingly.
	22	INSERT INTO background_updates (update_name, progress_json) VALUES
	23	('local_media_repository_url_idx', '{}');

+35

-0

synapse/storage/data_stores/main/schema/delta/46/user_dir_null_room_ids.sql less more

	0	/* Copyright 2017 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- change the user_directory table to also cover global local user profiles
	16	-- rather than just profiles within specific rooms.
	17
	18	CREATE TABLE user_directory2 (
	19	user_id TEXT NOT NULL,
	20	room_id TEXT,
	21	display_name TEXT,
	22	avatar_url TEXT
	23	);
	24
	25	INSERT INTO user_directory2(user_id, room_id, display_name, avatar_url)
	26	SELECT user_id, room_id, display_name, avatar_url from user_directory;
	27
	28	DROP TABLE user_directory;
	29	ALTER TABLE user_directory2 RENAME TO user_directory;
	30
	31	-- create indexes after doing the inserts because that's more efficient.
	32	-- it also means we can give it the same name as the old one without renaming.
	33	CREATE INDEX user_directory_room_idx ON user_directory(room_id);
	34	CREATE UNIQUE INDEX user_directory_user_idx ON user_directory(user_id);

+24

-0

synapse/storage/data_stores/main/schema/delta/46/user_dir_typos.sql less more

	0	/* Copyright 2017 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- this is just embarassing :\|
	16	ALTER TABLE users_in_pubic_room RENAME TO users_in_public_rooms;
	17
	18	-- this is only 300K rows on matrix.org and takes ~3s to generate the index,
	19	-- so is hopefully not going to block anyone else for that long...
	20	CREATE INDEX users_in_public_rooms_room_idx ON users_in_public_rooms(room_id);
	21	CREATE UNIQUE INDEX users_in_public_rooms_user_idx ON users_in_public_rooms(user_id);
	22	DROP INDEX users_in_pubic_room_room_idx;
	23	DROP INDEX users_in_pubic_room_user_idx;

+16

-0

synapse/storage/data_stores/main/schema/delta/47/last_access_media.sql less more

	0	/* Copyright 2018 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	ALTER TABLE local_media_repository ADD COLUMN last_access_ts BIGINT;

+17

-0

synapse/storage/data_stores/main/schema/delta/47/postgres_fts_gin.sql less more

	0	/* Copyright 2018 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	INSERT into background_updates (update_name, progress_json)
	16	VALUES ('event_search_postgres_gin', '{}');

+28

-0

synapse/storage/data_stores/main/schema/delta/47/push_actions_staging.sql less more

	0	/* Copyright 2018 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- Temporary staging area for push actions that have been calculated for an
	16	-- event, but the event hasn't yet been persisted.
	17	-- When the event is persisted the rows are moved over to the
	18	-- event_push_actions table.
	19	CREATE TABLE event_push_actions_staging (
	20	event_id TEXT NOT NULL,
	21	user_id TEXT NOT NULL,
	22	actions TEXT NOT NULL,
	23	notif SMALLINT NOT NULL,
	24	highlight SMALLINT NOT NULL
	25	);
	26
	27	CREATE INDEX event_push_actions_staging_id ON event_push_actions_staging(event_id);

+34

-0

synapse/storage/data_stores/main/schema/delta/47/state_group_seq.py less more

	0	# Copyright 2018 New Vector Ltd
	1	#
	2	# Licensed under the Apache License, Version 2.0 (the "License");
	3	# you may not use this file except in compliance with the License.
	4	# You may obtain a copy of the License at
	5	#
	6	# http://www.apache.org/licenses/LICENSE-2.0
	7	#
	8	# Unless required by applicable law or agreed to in writing, software
	9	# distributed under the License is distributed on an "AS IS" BASIS,
	10	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	# See the License for the specific language governing permissions and
	12	# limitations under the License.
	13
	14	from synapse.storage.engines import PostgresEngine
	15
	16
	17	def run_create(cur, database_engine, args, *kwargs):
	18	if isinstance(database_engine, PostgresEngine):
	19	# if we already have some state groups, we want to start making new
	20	# ones with a higher id.
	21	cur.execute("SELECT max(id) FROM state_groups")
	22	row = cur.fetchone()
	23
	24	if row[0] is None:
	25	start_val = 1
	26	else:
	27	start_val = row[0] + 1
	28
	29	cur.execute("CREATE SEQUENCE state_group_id_seq START WITH %s", (start_val,))
	30
	31
	32	def run_upgrade(args, *kwargs):
	33	pass

+18

-0

synapse/storage/data_stores/main/schema/delta/48/add_user_consent.sql less more

	0	/* Copyright 2018 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	/* record the version of the privacy policy the user has consented to
	16	*/
	17	ALTER TABLE users ADD COLUMN consent_version TEXT;

+17

-0

synapse/storage/data_stores/main/schema/delta/48/add_user_ips_last_seen_index.sql less more

	0	/* Copyright 2018 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	INSERT into background_updates (update_name, progress_json)
	16	VALUES ('user_ips_last_seen_index', '{}');

+25

-0

synapse/storage/data_stores/main/schema/delta/48/deactivated_users.sql less more

	0	/* Copyright 2018 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	/*
	16	* Store any accounts that have been requested to be deactivated.
	17	* We part the account from all the rooms its in when its
	18	* deactivated. This can take some time and synapse may be restarted
	19	* before it completes, so store the user IDs here until the process
	20	* is complete.
	21	*/
	22	CREATE TABLE users_pending_deactivation (
	23	user_id TEXT NOT NULL
	24	);

+63

-0

synapse/storage/data_stores/main/schema/delta/48/group_unique_indexes.py less more

	0	# Copyright 2018 New Vector Ltd
	1	#
	2	# Licensed under the Apache License, Version 2.0 (the "License");
	3	# you may not use this file except in compliance with the License.
	4	# You may obtain a copy of the License at
	5	#
	6	# http://www.apache.org/licenses/LICENSE-2.0
	7	#
	8	# Unless required by applicable law or agreed to in writing, software
	9	# distributed under the License is distributed on an "AS IS" BASIS,
	10	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	# See the License for the specific language governing permissions and
	12	# limitations under the License.
	13
	14	from synapse.storage.engines import PostgresEngine
	15	from synapse.storage.prepare_database import get_statements
	16
	17	FIX_INDEXES = """
	18	-- rebuild indexes as uniques
	19	DROP INDEX groups_invites_g_idx;
	20	CREATE UNIQUE INDEX group_invites_g_idx ON group_invites(group_id, user_id);
	21	DROP INDEX groups_users_g_idx;
	22	CREATE UNIQUE INDEX group_users_g_idx ON group_users(group_id, user_id);
	23
	24	-- rename other indexes to actually match their table names..
	25	DROP INDEX groups_users_u_idx;
	26	CREATE INDEX group_users_u_idx ON group_users(user_id);
	27	DROP INDEX groups_invites_u_idx;
	28	CREATE INDEX group_invites_u_idx ON group_invites(user_id);
	29	DROP INDEX groups_rooms_g_idx;
	30	CREATE UNIQUE INDEX group_rooms_g_idx ON group_rooms(group_id, room_id);
	31	DROP INDEX groups_rooms_r_idx;
	32	CREATE INDEX group_rooms_r_idx ON group_rooms(room_id);
	33	"""
	34
	35
	36	def run_create(cur, database_engine, args, *kwargs):
	37	rowid = "ctid" if isinstance(database_engine, PostgresEngine) else "rowid"
	38
	39	# remove duplicates from group_users & group_invites tables
	40	cur.execute(
	41	"""
	42	DELETE FROM group_users WHERE %s NOT IN (
	43	SELECT min(%s) FROM group_users GROUP BY group_id, user_id
	44	);
	45	"""
	46	% (rowid, rowid)
	47	)
	48	cur.execute(
	49	"""
	50	DELETE FROM group_invites WHERE %s NOT IN (
	51	SELECT min(%s) FROM group_invites GROUP BY group_id, user_id
	52	);
	53	"""
	54	% (rowid, rowid)
	55	)
	56
	57	for statement in get_statements(FIX_INDEXES.splitlines()):
	58	cur.execute(statement)
	59
	60
	61	def run_upgrade(args, *kwargs):
	62	pass

+22

-0

synapse/storage/data_stores/main/schema/delta/48/groups_joinable.sql less more

	0	/* Copyright 2018 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	/*
	16	* This isn't a real ENUM because sqlite doesn't support it
	17	* and we use a default of NULL for inserted rows and interpret
	18	* NULL at the python store level as necessary so that existing
	19	* rows are given the correct default policy.
	20	*/
	21	ALTER TABLE groups ADD COLUMN join_policy TEXT NOT NULL DEFAULT 'invite';

+20

-0

synapse/storage/data_stores/main/schema/delta/49/add_user_consent_server_notice_sent.sql less more

	0	/* Copyright 2018 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	/* record whether we have sent a server notice about consenting to the
	16	* privacy policy. Specifically records the version of the policy we sent
	17	* a message about.
	18	*/
	19	ALTER TABLE users ADD COLUMN consent_server_notice_sent TEXT;

+21

-0

synapse/storage/data_stores/main/schema/delta/49/add_user_daily_visits.sql less more

	0	/* Copyright 2018 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15
	16	CREATE TABLE user_daily_visits ( user_id TEXT NOT NULL,
	17	device_id TEXT,
	18	timestamp BIGINT NOT NULL );
	19	CREATE INDEX user_daily_visits_uts_idx ON user_daily_visits(user_id, timestamp);
	20	CREATE INDEX user_daily_visits_ts_idx ON user_daily_visits(timestamp);

+17

-0

synapse/storage/data_stores/main/schema/delta/49/add_user_ips_last_seen_only_index.sql less more

	0	/* Copyright 2018 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	INSERT into background_updates (update_name, progress_json)
	16	VALUES ('user_ips_last_seen_only_index', '{}');

+19

-0

synapse/storage/data_stores/main/schema/delta/50/add_creation_ts_users_index.sql less more

	0	/* Copyright 2018 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15
	16
	17	INSERT into background_updates (update_name, progress_json)
	18	VALUES ('users_creation_ts', '{}');

+21

-0

synapse/storage/data_stores/main/schema/delta/50/erasure_store.sql less more

	0	/* Copyright 2018 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- a table of users who have requested that their details be erased
	16	CREATE TABLE erased_users (
	17	user_id TEXT NOT NULL
	18	);
	19
	20	CREATE UNIQUE INDEX erased_users_user ON erased_users(user_id);

+96

-0

synapse/storage/data_stores/main/schema/delta/50/make_event_content_nullable.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2018 New Vector Ltd
	2	#
	3	# Licensed under the Apache License, Version 2.0 (the "License");
	4	# you may not use this file except in compliance with the License.
	5	# You may obtain a copy of the License at
	6	#
	7	# http://www.apache.org/licenses/LICENSE-2.0
	8	#
	9	# Unless required by applicable law or agreed to in writing, software
	10	# distributed under the License is distributed on an "AS IS" BASIS,
	11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	# See the License for the specific language governing permissions and
	13	# limitations under the License.
	14
	15	"""
	16	We want to stop populating 'event.content', so we need to make it nullable.
	17
	18	If this has to be rolled back, then the following should populate the missing data:
	19
	20	Postgres:
	21
	22	UPDATE events SET content=(ej.json::json)->'content' FROM event_json ej
	23	WHERE ej.event_id = events.event_id AND
	24	stream_ordering < (
	25	SELECT stream_ordering FROM events WHERE content IS NOT NULL
	26	ORDER BY stream_ordering LIMIT 1
	27	);
	28
	29	UPDATE events SET content=(ej.json::json)->'content' FROM event_json ej
	30	WHERE ej.event_id = events.event_id AND
	31	stream_ordering > (
	32	SELECT stream_ordering FROM events WHERE content IS NOT NULL
	33	ORDER BY stream_ordering DESC LIMIT 1
	34	);
	35
	36	SQLite:
	37
	38	UPDATE events SET content=(
	39	SELECT json_extract(json,'$.content') FROM event_json ej
	40	WHERE ej.event_id = events.event_id
	41	)
	42	WHERE
	43	stream_ordering < (
	44	SELECT stream_ordering FROM events WHERE content IS NOT NULL
	45	ORDER BY stream_ordering LIMIT 1
	46	)
	47	OR stream_ordering > (
	48	SELECT stream_ordering FROM events WHERE content IS NOT NULL
	49	ORDER BY stream_ordering DESC LIMIT 1
	50	);
	51
	52	"""
	53
	54	import logging
	55
	56	from synapse.storage.engines import PostgresEngine
	57
	58	logger = logging.getLogger(__name__)
	59
	60
	61	def run_create(cur, database_engine, args, *kwargs):
	62	pass
	63
	64
	65	def run_upgrade(cur, database_engine, args, *kwargs):
	66	if isinstance(database_engine, PostgresEngine):
	67	cur.execute(
	68	"""
	69	ALTER TABLE events ALTER COLUMN content DROP NOT NULL;
	70	"""
	71	)
	72	return
	73
	74	# sqlite is an arse about this. ref: https://www.sqlite.org/lang_altertable.html
	75
	76	cur.execute(
	77	"SELECT sql FROM sqlite_master WHERE tbl_name='events' AND type='table'"
	78	)
	79	(oldsql,) = cur.fetchone()
	80
	81	sql = oldsql.replace("content TEXT NOT NULL", "content TEXT")
	82	if sql == oldsql:
	83	raise Exception("Couldn't find null constraint to drop in %s" % oldsql)
	84
	85	logger.info("Replacing definition of 'events' with: %s", sql)
	86
	87	cur.execute("PRAGMA schema_version")
	88	(oldver,) = cur.fetchone()
	89	cur.execute("PRAGMA writable_schema=ON")
	90	cur.execute(
	91	"UPDATE sqlite_master SET sql=? WHERE tbl_name='events' AND type='table'",
	92	(sql,),
	93	)
	94	cur.execute("PRAGMA schema_version=%i" % (oldver + 1,))
	95	cur.execute("PRAGMA writable_schema=OFF")

+39

-0

synapse/storage/data_stores/main/schema/delta/51/e2e_room_keys.sql less more

	0	/* Copyright 2017 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- users' optionally backed up encrypted e2e sessions
	16	CREATE TABLE e2e_room_keys (
	17	user_id TEXT NOT NULL,
	18	room_id TEXT NOT NULL,
	19	session_id TEXT NOT NULL,
	20	version TEXT NOT NULL,
	21	first_message_index INT,
	22	forwarded_count INT,
	23	is_verified BOOLEAN,
	24	session_data TEXT NOT NULL
	25	);
	26
	27	CREATE UNIQUE INDEX e2e_room_keys_idx ON e2e_room_keys(user_id, room_id, session_id);
	28
	29	-- the metadata for each generation of encrypted e2e session backups
	30	CREATE TABLE e2e_room_keys_versions (
	31	user_id TEXT NOT NULL,
	32	version TEXT NOT NULL,
	33	algorithm TEXT NOT NULL,
	34	auth_data TEXT NOT NULL,
	35	deleted SMALLINT DEFAULT 0 NOT NULL
	36	);
	37
	38	CREATE UNIQUE INDEX e2e_room_keys_versions_idx ON e2e_room_keys_versions(user_id, version);

+27

-0

synapse/storage/data_stores/main/schema/delta/51/monthly_active_users.sql less more

	0	/* Copyright 2018 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- a table of monthly active users, for use where blocking based on mau limits
	16	CREATE TABLE monthly_active_users (
	17	user_id TEXT NOT NULL,
	18	-- Last time we saw the user. Not guaranteed to be accurate due to rate limiting
	19	-- on updates, Granularity of updates governed by
	20	-- synapse.storage.monthly_active_users.LAST_SEEN_GRANULARITY
	21	-- Measured in ms since epoch.
	22	timestamp BIGINT NOT NULL
	23	);
	24
	25	CREATE UNIQUE INDEX monthly_active_users_users ON monthly_active_users(user_id);
	26	CREATE INDEX monthly_active_users_time_stamp ON monthly_active_users(timestamp);

+19

-0

synapse/storage/data_stores/main/schema/delta/52/add_event_to_state_group_index.sql less more

	0	/* Copyright 2018 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- This is needed to efficiently check for unreferenced state groups during
	16	-- purge. Added events_to_state_group(state_group) index
	17	INSERT into background_updates (update_name, progress_json)
	18	VALUES ('event_to_state_groups_sg_index', '{}');

+36

-0

synapse/storage/data_stores/main/schema/delta/52/device_list_streams_unique_idx.sql less more

	0	/* Copyright 2018 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- register a background update which will create a unique index on
	16	-- device_lists_remote_cache
	17	INSERT into background_updates (update_name, progress_json)
	18	VALUES ('device_lists_remote_cache_unique_idx', '{}');
	19
	20	-- and one on device_lists_remote_extremeties
	21	INSERT into background_updates (update_name, progress_json, depends_on)
	22	VALUES (
	23	'device_lists_remote_extremeties_unique_idx', '{}',
	24
	25	-- doesn't really depend on this, but we need to make sure both happen
	26	-- before we drop the old indexes.
	27	'device_lists_remote_cache_unique_idx'
	28	);
	29
	30	-- once they complete, we can drop the old indexes.
	31	INSERT into background_updates (update_name, progress_json, depends_on)
	32	VALUES (
	33	'drop_device_list_streams_non_unique_indexes', '{}',
	34	'device_lists_remote_extremeties_unique_idx'
	35	);

+53

-0

synapse/storage/data_stores/main/schema/delta/52/e2e_room_keys.sql less more

	0	/* Copyright 2018 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	/* Change version column to an integer so we can do MAX() sensibly
	16	*/
	17	CREATE TABLE e2e_room_keys_versions_new (
	18	user_id TEXT NOT NULL,
	19	version BIGINT NOT NULL,
	20	algorithm TEXT NOT NULL,
	21	auth_data TEXT NOT NULL,
	22	deleted SMALLINT DEFAULT 0 NOT NULL
	23	);
	24
	25	INSERT INTO e2e_room_keys_versions_new
	26	SELECT user_id, CAST(version as BIGINT), algorithm, auth_data, deleted FROM e2e_room_keys_versions;
	27
	28	DROP TABLE e2e_room_keys_versions;
	29	ALTER TABLE e2e_room_keys_versions_new RENAME TO e2e_room_keys_versions;
	30
	31	CREATE UNIQUE INDEX e2e_room_keys_versions_idx ON e2e_room_keys_versions(user_id, version);
	32
	33	/* Change e2e_rooms_keys to match
	34	*/
	35	CREATE TABLE e2e_room_keys_new (
	36	user_id TEXT NOT NULL,
	37	room_id TEXT NOT NULL,
	38	session_id TEXT NOT NULL,
	39	version BIGINT NOT NULL,
	40	first_message_index INT,
	41	forwarded_count INT,
	42	is_verified BOOLEAN,
	43	session_data TEXT NOT NULL
	44	);
	45
	46	INSERT INTO e2e_room_keys_new
	47	SELECT user_id, room_id, session_id, CAST(version as BIGINT), first_message_index, forwarded_count, is_verified, session_data FROM e2e_room_keys;
	48
	49	DROP TABLE e2e_room_keys;
	50	ALTER TABLE e2e_room_keys_new RENAME TO e2e_room_keys;
	51
	52	CREATE UNIQUE INDEX e2e_room_keys_idx ON e2e_room_keys(user_id, room_id, session_id);

+19

-0

synapse/storage/data_stores/main/schema/delta/53/add_user_type_to_users.sql less more

	0	/* Copyright 2018 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	/* The type of the user: NULL for a regular user, or one of the constants in
	16	* synapse.api.constants.UserTypes
	17	*/
	18	ALTER TABLE users ADD COLUMN user_type TEXT DEFAULT NULL;

+16

-0

synapse/storage/data_stores/main/schema/delta/53/drop_sent_transactions.sql less more

	0	/* Copyright 2018 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	DROP TABLE IF EXISTS sent_transactions;

+16

-0

synapse/storage/data_stores/main/schema/delta/53/event_format_version.sql less more

	0	/* Copyright 2019 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	ALTER TABLE event_json ADD COLUMN format_version INTEGER;

+30

-0

synapse/storage/data_stores/main/schema/delta/53/user_dir_populate.sql less more

	0	/* Copyright 2019 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- Set up staging tables
	16	INSERT INTO background_updates (update_name, progress_json) VALUES
	17	('populate_user_directory_createtables', '{}');
	18
	19	-- Run through each room and update the user directory according to who is in it
	20	INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES
	21	('populate_user_directory_process_rooms', '{}', 'populate_user_directory_createtables');
	22
	23	-- Insert all users, if search_all_users is on
	24	INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES
	25	('populate_user_directory_process_users', '{}', 'populate_user_directory_process_rooms');
	26
	27	-- Clean up staging tables
	28	INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES
	29	('populate_user_directory_cleanup', '{}', 'populate_user_directory_process_users');

+30

-0

synapse/storage/data_stores/main/schema/delta/53/user_ips_index.sql less more

	0	/* Copyright 2018 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- analyze user_ips, to help ensure the correct indices are used
	16	INSERT INTO background_updates (update_name, progress_json) VALUES
	17	('user_ips_analyze', '{}');
	18
	19	-- delete duplicates
	20	INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES
	21	('user_ips_remove_dupes', '{}', 'user_ips_analyze');
	22
	23	-- add a new unique index to user_ips table
	24	INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES
	25	('user_ips_device_unique_index', '{}', 'user_ips_remove_dupes');
	26
	27	-- drop the old original index
	28	INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES
	29	('user_ips_drop_nonunique_index', '{}', 'user_ips_device_unique_index');

+44

-0

synapse/storage/data_stores/main/schema/delta/53/user_share.sql less more

	0	/* Copyright 2017 Vector Creations Ltd, 2019 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- Old disused version of the tables below.
	16	DROP TABLE IF EXISTS users_who_share_rooms;
	17
	18	-- Tables keeping track of what users share rooms. This is a map of local users
	19	-- to local or remote users, per room. Remote users cannot be in the user_id
	20	-- column, only the other_user_id column. There are two tables, one for public
	21	-- rooms and those for private rooms.
	22	CREATE TABLE IF NOT EXISTS users_who_share_public_rooms (
	23	user_id TEXT NOT NULL,
	24	other_user_id TEXT NOT NULL,
	25	room_id TEXT NOT NULL
	26	);
	27
	28	CREATE TABLE IF NOT EXISTS users_who_share_private_rooms (
	29	user_id TEXT NOT NULL,
	30	other_user_id TEXT NOT NULL,
	31	room_id TEXT NOT NULL
	32	);
	33
	34	CREATE UNIQUE INDEX users_who_share_public_rooms_u_idx ON users_who_share_public_rooms(user_id, other_user_id, room_id);
	35	CREATE INDEX users_who_share_public_rooms_r_idx ON users_who_share_public_rooms(room_id);
	36	CREATE INDEX users_who_share_public_rooms_o_idx ON users_who_share_public_rooms(other_user_id);
	37
	38	CREATE UNIQUE INDEX users_who_share_private_rooms_u_idx ON users_who_share_private_rooms(user_id, other_user_id, room_id);
	39	CREATE INDEX users_who_share_private_rooms_r_idx ON users_who_share_private_rooms(room_id);
	40	CREATE INDEX users_who_share_private_rooms_o_idx ON users_who_share_private_rooms(other_user_id);
	41
	42	-- Make sure that we populate the tables initially by resetting the stream ID
	43	UPDATE user_directory_stream_pos SET stream_id = NULL;

+29

-0

synapse/storage/data_stores/main/schema/delta/53/user_threepid_id.sql less more

	0	/* Copyright 2019 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- Tracks which identity server a user bound their threepid via.
	16	CREATE TABLE user_threepid_id_server (
	17	user_id TEXT NOT NULL,
	18	medium TEXT NOT NULL,
	19	address TEXT NOT NULL,
	20	id_server TEXT NOT NULL
	21	);
	22
	23	CREATE UNIQUE INDEX user_threepid_id_server_idx ON user_threepid_id_server(
	24	user_id, medium, address, id_server
	25	);
	26
	27	INSERT INTO background_updates (update_name, progress_json) VALUES
	28	('user_threepids_grandfather', '{}');

+28

-0

synapse/storage/data_stores/main/schema/delta/53/users_in_public_rooms.sql less more

	0	/* Copyright 2019 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- We don't need the old version of this table.
	16	DROP TABLE IF EXISTS users_in_public_rooms;
	17
	18	-- Old version of users_in_public_rooms
	19	DROP TABLE IF EXISTS users_who_share_public_rooms;
	20
	21	-- Track what users are in public rooms.
	22	CREATE TABLE IF NOT EXISTS users_in_public_rooms (
	23	user_id TEXT NOT NULL,
	24	room_id TEXT NOT NULL
	25	);
	26
	27	CREATE UNIQUE INDEX users_in_public_rooms_u_idx ON users_in_public_rooms(user_id, room_id);

+30

-0

synapse/storage/data_stores/main/schema/delta/54/account_validity_with_renewal.sql less more

	0	/* Copyright 2019 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- We previously changed the schema for this table without renaming the file, which means
	16	-- that some databases might still be using the old schema. This ensures Synapse uses the
	17	-- right schema for the table.
	18	DROP TABLE IF EXISTS account_validity;
	19
	20	-- Track what users are in public rooms.
	21	CREATE TABLE IF NOT EXISTS account_validity (
	22	user_id TEXT PRIMARY KEY,
	23	expiration_ts_ms BIGINT NOT NULL,
	24	email_sent BOOLEAN NOT NULL,
	25	renewal_token TEXT
	26	);
	27
	28	CREATE INDEX account_validity_email_sent_idx ON account_validity(email_sent, expiration_ts_ms)
	29	CREATE UNIQUE INDEX account_validity_renewal_string_idx ON account_validity(renewal_token)

+23

-0

synapse/storage/data_stores/main/schema/delta/54/add_validity_to_server_keys.sql less more

	0	/* Copyright 2019 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	/* When we can use this key until, before we have to refresh it. */
	16	ALTER TABLE server_signature_keys ADD COLUMN ts_valid_until_ms BIGINT;
	17
	18	UPDATE server_signature_keys SET ts_valid_until_ms = (
	19	SELECT MAX(ts_valid_until_ms) FROM server_keys_json skj WHERE
	20	skj.server_name = server_signature_keys.server_name AND
	21	skj.key_id = server_signature_keys.key_id
	22	);

+23

-0

synapse/storage/data_stores/main/schema/delta/54/delete_forward_extremities.sql less more

	0	/* Copyright 2019 The Matrix.org Foundation C.I.C.
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- Start a background job to cleanup extremities that were incorrectly added
	16	-- by bug #5269.
	17	INSERT INTO background_updates (update_name, progress_json) VALUES
	18	('delete_soft_failed_extremities', '{}');
	19
	20	DROP TABLE IF EXISTS _extremities_to_check; -- To make this delta schema file idempotent.
	21	CREATE TABLE _extremities_to_check AS SELECT event_id FROM event_forward_extremities;
	22	CREATE INDEX _extremities_to_check_id ON _extremities_to_check(event_id);

+30

-0

synapse/storage/data_stores/main/schema/delta/54/drop_legacy_tables.sql less more

	0	/* Copyright 2019 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- we need to do this first due to foreign constraints
	16	DROP TABLE IF EXISTS application_services_regex;
	17
	18	DROP TABLE IF EXISTS application_services;
	19	DROP TABLE IF EXISTS transaction_id_to_pdu;
	20	DROP TABLE IF EXISTS stats_reporting;
	21	DROP TABLE IF EXISTS current_state_resets;
	22	DROP TABLE IF EXISTS event_content_hashes;
	23	DROP TABLE IF EXISTS event_destinations;
	24	DROP TABLE IF EXISTS event_edge_hashes;
	25	DROP TABLE IF EXISTS event_signatures;
	26	DROP TABLE IF EXISTS feedback;
	27	DROP TABLE IF EXISTS room_hosts;
	28	DROP TABLE IF EXISTS server_tls_certificates;
	29	DROP TABLE IF EXISTS state_forward_extremities;

+16

-0

synapse/storage/data_stores/main/schema/delta/54/drop_presence_list.sql less more

	0	/* Copyright 2019 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	DROP TABLE IF EXISTS presence_list;

+27

-0

synapse/storage/data_stores/main/schema/delta/54/relations.sql less more

	0	/* Copyright 2019 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- Tracks related events, like reactions, replies, edits, etc. Note that things
	16	-- in this table are not necessarily "valid", e.g. it may contain edits from
	17	-- people who don't have power to edit other peoples events.
	18	CREATE TABLE IF NOT EXISTS event_relations (
	19	event_id TEXT NOT NULL,
	20	relates_to_id TEXT NOT NULL,
	21	relation_type TEXT NOT NULL,
	22	aggregation_key TEXT
	23	);
	24
	25	CREATE UNIQUE INDEX event_relations_id ON event_relations(event_id);
	26	CREATE INDEX event_relations_relates ON event_relations(relates_to_id, relation_type, aggregation_key);

+80

-0

synapse/storage/data_stores/main/schema/delta/54/stats.sql less more

	0	/* Copyright 2018 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	CREATE TABLE stats_stream_pos (
	16	Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, -- Makes sure this table only has one row.
	17	stream_id BIGINT,
	18	CHECK (Lock='X')
	19	);
	20
	21	INSERT INTO stats_stream_pos (stream_id) VALUES (null);
	22
	23	CREATE TABLE user_stats (
	24	user_id TEXT NOT NULL,
	25	ts BIGINT NOT NULL,
	26	bucket_size INT NOT NULL,
	27	public_rooms INT NOT NULL,
	28	private_rooms INT NOT NULL
	29	);
	30
	31	CREATE UNIQUE INDEX user_stats_user_ts ON user_stats(user_id, ts);
	32
	33	CREATE TABLE room_stats (
	34	room_id TEXT NOT NULL,
	35	ts BIGINT NOT NULL,
	36	bucket_size INT NOT NULL,
	37	current_state_events INT NOT NULL,
	38	joined_members INT NOT NULL,
	39	invited_members INT NOT NULL,
	40	left_members INT NOT NULL,
	41	banned_members INT NOT NULL,
	42	state_events INT NOT NULL
	43	);
	44
	45	CREATE UNIQUE INDEX room_stats_room_ts ON room_stats(room_id, ts);
	46
	47	-- cache of current room state; useful for the publicRooms list
	48	CREATE TABLE room_state (
	49	room_id TEXT NOT NULL,
	50	join_rules TEXT,
	51	history_visibility TEXT,
	52	encryption TEXT,
	53	name TEXT,
	54	topic TEXT,
	55	avatar TEXT,
	56	canonical_alias TEXT
	57	-- get aliases straight from the right table
	58	);
	59
	60	CREATE UNIQUE INDEX room_state_room ON room_state(room_id);
	61
	62	CREATE TABLE room_stats_earliest_token (
	63	room_id TEXT NOT NULL,
	64	token BIGINT NOT NULL
	65	);
	66
	67	CREATE UNIQUE INDEX room_stats_earliest_token_idx ON room_stats_earliest_token(room_id);
	68
	69	-- Set up staging tables
	70	INSERT INTO background_updates (update_name, progress_json) VALUES
	71	('populate_stats_createtables', '{}');
	72
	73	-- Run through each room and update stats
	74	INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES
	75	('populate_stats_process_rooms', '{}', 'populate_stats_createtables');
	76
	77	-- Clean up staging tables
	78	INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES
	79	('populate_stats_cleanup', '{}', 'populate_stats_process_rooms');

+28

-0

synapse/storage/data_stores/main/schema/delta/54/stats2.sql less more

	0	/* Copyright 2019 The Matrix.org Foundation C.I.C.
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- This delta file gets run after `54/stats.sql` delta.
	16
	17	-- We want to add some indices to the temporary stats table, so we re-insert
	18	-- 'populate_stats_createtables' if we are still processing the rooms update.
	19	INSERT INTO background_updates (update_name, progress_json)
	20	SELECT 'populate_stats_createtables', '{}'
	21	WHERE
	22	'populate_stats_process_rooms' IN (
	23	SELECT update_name FROM background_updates
	24	)
	25	AND 'populate_stats_createtables' NOT IN ( -- don't insert if already exists
	26	SELECT update_name FROM background_updates
	27	);

+18

-0

synapse/storage/data_stores/main/schema/delta/55/access_token_expiry.sql less more

	0	/* Copyright 2019 The Matrix.org Foundation C.I.C.
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- when this access token can be used until, in ms since the epoch. NULL means the token
	16	-- never expires.
	17	ALTER TABLE access_tokens ADD COLUMN valid_until_ms BIGINT;

+31

-0

synapse/storage/data_stores/main/schema/delta/55/track_threepid_validations.sql less more

	0	/* Copyright 2019 The Matrix.org Foundation C.I.C.
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14	CREATE TABLE IF NOT EXISTS threepid_validation_session (
	15	session_id TEXT PRIMARY KEY,
	16	medium TEXT NOT NULL,
	17	address TEXT NOT NULL,
	18	client_secret TEXT NOT NULL,
	19	last_send_attempt BIGINT NOT NULL,
	20	validated_at BIGINT
	21	);
	22
	23	CREATE TABLE IF NOT EXISTS threepid_validation_token (
	24	token TEXT PRIMARY KEY,
	25	session_id TEXT NOT NULL,
	26	next_link TEXT,
	27	expires BIGINT NOT NULL
	28	);
	29
	30	CREATE INDEX threepid_validation_token_session_id ON threepid_validation_token(session_id);

+19

-0

synapse/storage/data_stores/main/schema/delta/55/users_alter_deactivated.sql less more

	0	/* Copyright 2019 The Matrix.org Foundation C.I.C.
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	ALTER TABLE users ADD deactivated SMALLINT DEFAULT 0 NOT NULL;
	16
	17	INSERT INTO background_updates (update_name, progress_json) VALUES
	18	('users_set_deactivated_flag', '{}');

+20

-0

synapse/storage/data_stores/main/schema/delta/56/add_spans_to_device_lists.sql less more

	0	/* Copyright 2019 The Matrix.org Foundation C.I.C
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	/*
	16	* Opentracing context data for inclusion in the device_list_update EDUs, as a
	17	* json-encoded dictionary. NULL if opentracing is disabled (or not enabled for this destination).
	18	*/
	19	ALTER TABLE device_lists_outbound_pokes ADD opentracing_context TEXT;

+22

-0

synapse/storage/data_stores/main/schema/delta/56/current_state_events_membership.sql less more

	0	/* Copyright 2019 The Matrix.org Foundation C.I.C.
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- We add membership to current state so that we don't need to join against
	16	-- room_memberships, which can be surprisingly costly (we do such queries
	17	-- very frequently).
	18	-- This will be null for non-membership events and the content.membership key
	19	-- for membership events. (Will also be null for membership events until the
	20	-- background update job has finished).
	21	ALTER TABLE current_state_events ADD membership TEXT;

+24

-0

synapse/storage/data_stores/main/schema/delta/56/current_state_events_membership_mk2.sql less more

	0	/* Copyright 2019 The Matrix.org Foundation C.I.C.
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- We add membership to current state so that we don't need to join against
	16	-- room_memberships, which can be surprisingly costly (we do such queries
	17	-- very frequently).
	18	-- This will be null for non-membership events and the content.membership key
	19	-- for membership events. (Will also be null for membership events until the
	20	-- background update job has finished).
	21
	22	INSERT INTO background_updates (update_name, progress_json) VALUES
	23	('current_state_events_membership', '{}');

+25

-0

synapse/storage/data_stores/main/schema/delta/56/destinations_failure_ts.sql less more

	0	/* Copyright 2019 The Matrix.org Foundation C.I.C
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	/*
	16	* Record the timestamp when a given server started failing
	17	*/
	18	ALTER TABLE destinations ADD failure_ts BIGINT;
	19
	20	/* as a rough approximation, we assume that the server started failing at
	21	* retry_interval before the last retry
	22	*/
	23	UPDATE destinations SET failure_ts = retry_last_ts - retry_interval
	24	WHERE retry_last_ts > 0;

+18

-0

synapse/storage/data_stores/main/schema/delta/56/destinations_retry_interval_type.sql.postgres less more

	0	/* Copyright 2019 The Matrix.org Foundation C.I.C
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- We want to store large retry intervals so we upgrade the column from INT
	16	-- to BIGINT. We don't need to do this on SQLite.
	17	ALTER TABLE destinations ALTER retry_interval SET DATA TYPE BIGINT;

+24

-0

synapse/storage/data_stores/main/schema/delta/56/devices_last_seen.sql less more

	0	/* Copyright 2019 Matrix.org Foundation CIC
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- Track last seen information for a device in the devices table, rather
	16	-- than relying on it being in the user_ips table (which we want to be able
	17	-- to purge old entries from)
	18	ALTER TABLE devices ADD COLUMN last_seen BIGINT;
	19	ALTER TABLE devices ADD COLUMN ip TEXT;
	20	ALTER TABLE devices ADD COLUMN user_agent TEXT;
	21
	22	INSERT INTO background_updates (update_name, progress_json) VALUES
	23	('devices_last_seen', '{}');

+20

-0

synapse/storage/data_stores/main/schema/delta/56/drop_unused_event_tables.sql less more

	0	/* Copyright 2019 The Matrix.org Foundation C.I.C.
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- these tables are never used.
	16	DROP TABLE IF EXISTS room_names;
	17	DROP TABLE IF EXISTS topics;
	18	DROP TABLE IF EXISTS history_visibility;
	19	DROP TABLE IF EXISTS guest_access;

+18

-0

synapse/storage/data_stores/main/schema/delta/56/fix_room_keys_index.sql less more

	0	/* Copyright 2019 Matrix.org Foundation CIC
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- version is supposed to be part of the room keys index
	16	CREATE UNIQUE INDEX e2e_room_keys_with_version_idx ON e2e_room_keys(user_id, version, room_id, session_id);
	17	DROP INDEX IF EXISTS e2e_room_keys_idx;

+16

-0

synapse/storage/data_stores/main/schema/delta/56/public_room_list_idx.sql less more

	0	/* Copyright 2019 The Matrix.org Foundation C.I.C.
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	CREATE INDEX public_room_list_stream_network ON public_room_list_stream (appservice_id, network_id, room_id);

+17

-0

synapse/storage/data_stores/main/schema/delta/56/redaction_censor.sql less more

	0	/* Copyright 2019 The Matrix.org Foundation C.I.C.
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	ALTER TABLE redactions ADD COLUMN have_censored BOOL NOT NULL DEFAULT false;
	16	CREATE INDEX redactions_have_censored ON redactions(event_id) WHERE not have_censored;

+20

-0

synapse/storage/data_stores/main/schema/delta/56/redaction_censor2.sql less more

	0	/* Copyright 2019 The Matrix.org Foundation C.I.C.
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	ALTER TABLE redactions ADD COLUMN received_ts BIGINT;
	16	CREATE INDEX redactions_have_censored_ts ON redactions(received_ts) WHERE not have_censored;
	17
	18	INSERT INTO background_updates (update_name, progress_json) VALUES
	19	('redactions_received_ts', '{}');

+25

-0

synapse/storage/data_stores/main/schema/delta/56/redaction_censor3_fix_update.sql.postgres less more

	0	/* Copyright 2019 The Matrix.org Foundation C.I.C.
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15
	16	-- There was a bug where we may have updated censored redactions as bytes,
	17	-- which can (somehow) cause json to be inserted hex encoded. These updates go
	18	-- and undoes any such hex encoded JSON.
	19
	20	INSERT into background_updates (update_name, progress_json)
	21	VALUES ('event_fix_redactions_bytes_create_index', '{}');
	22
	23	INSERT into background_updates (update_name, progress_json, depends_on)
	24	VALUES ('event_fix_redactions_bytes', '{}', 'event_fix_redactions_bytes_create_index');

+18

-0

synapse/storage/data_stores/main/schema/delta/56/room_membership_idx.sql less more

	0	/* Copyright 2019 The Matrix.org Foundation C.I.C.
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- Adds an index on room_memberships for fetching all forgotten rooms for a user
	16	INSERT INTO background_updates (update_name, progress_json) VALUES
	17	('room_membership_forgotten_idx', '{}');

+152

-0

synapse/storage/data_stores/main/schema/delta/56/stats_separated.sql less more

	0	/* Copyright 2018 New Vector Ltd
	1	* Copyright 2019 The Matrix.org Foundation C.I.C.
	2	*
	3	* Licensed under the Apache License, Version 2.0 (the "License");
	4	* you may not use this file except in compliance with the License.
	5	* You may obtain a copy of the License at
	6	*
	7	* http://www.apache.org/licenses/LICENSE-2.0
	8	*
	9	* Unless required by applicable law or agreed to in writing, software
	10	* distributed under the License is distributed on an "AS IS" BASIS,
	11	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	* See the License for the specific language governing permissions and
	13	* limitations under the License.
	14	*/
	15
	16
	17	----- First clean up from previous versions of room stats.
	18
	19	-- First remove old stats stuff
	20	DROP TABLE IF EXISTS room_stats;
	21	DROP TABLE IF EXISTS room_state;
	22	DROP TABLE IF EXISTS room_stats_state;
	23	DROP TABLE IF EXISTS user_stats;
	24	DROP TABLE IF EXISTS room_stats_earliest_tokens;
	25	DROP TABLE IF EXISTS _temp_populate_stats_position;
	26	DROP TABLE IF EXISTS _temp_populate_stats_rooms;
	27	DROP TABLE IF EXISTS stats_stream_pos;
	28
	29	-- Unschedule old background updates if they're still scheduled
	30	DELETE FROM background_updates WHERE update_name IN (
	31	'populate_stats_createtables',
	32	'populate_stats_process_rooms',
	33	'populate_stats_process_users',
	34	'populate_stats_cleanup'
	35	);
	36
	37	INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES
	38	('populate_stats_process_rooms', '{}', '');
	39
	40	INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES
	41	('populate_stats_process_users', '{}', 'populate_stats_process_rooms');
	42
	43	----- Create tables for our version of room stats.
	44
	45	-- single-row table to track position of incremental updates
	46	DROP TABLE IF EXISTS stats_incremental_position;
	47	CREATE TABLE stats_incremental_position (
	48	Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, -- Makes sure this table only has one row.
	49	stream_id BIGINT NOT NULL,
	50	CHECK (Lock='X')
	51	);
	52
	53	-- insert a null row and make sure it is the only one.
	54	INSERT INTO stats_incremental_position (
	55	stream_id
	56	) SELECT COALESCE(MAX(stream_ordering), 0) from events;
	57
	58	-- represents PRESENT room statistics for a room
	59	-- only holds absolute fields
	60	DROP TABLE IF EXISTS room_stats_current;
	61	CREATE TABLE room_stats_current (
	62	room_id TEXT NOT NULL PRIMARY KEY,
	63
	64	-- These are absolute counts
	65	current_state_events INT NOT NULL,
	66	joined_members INT NOT NULL,
	67	invited_members INT NOT NULL,
	68	left_members INT NOT NULL,
	69	banned_members INT NOT NULL,
	70
	71	local_users_in_room INT NOT NULL,
	72
	73	-- The maximum delta stream position that this row takes into account.
	74	completed_delta_stream_id BIGINT NOT NULL
	75	);
	76
	77
	78	-- represents HISTORICAL room statistics for a room
	79	DROP TABLE IF EXISTS room_stats_historical;
	80	CREATE TABLE room_stats_historical (
	81	room_id TEXT NOT NULL,
	82	-- These stats cover the time from (end_ts - bucket_size)...end_ts (in ms).
	83	-- Note that end_ts is quantised.
	84	end_ts BIGINT NOT NULL,
	85	bucket_size BIGINT NOT NULL,
	86
	87	-- These stats are absolute counts
	88	current_state_events BIGINT NOT NULL,
	89	joined_members BIGINT NOT NULL,
	90	invited_members BIGINT NOT NULL,
	91	left_members BIGINT NOT NULL,
	92	banned_members BIGINT NOT NULL,
	93	local_users_in_room BIGINT NOT NULL,
	94
	95	-- These stats are per time slice
	96	total_events BIGINT NOT NULL,
	97	total_event_bytes BIGINT NOT NULL,
	98
	99	PRIMARY KEY (room_id, end_ts)
	100	);
	101
	102	-- We use this index to speed up deletion of ancient room stats.
	103	CREATE INDEX room_stats_historical_end_ts ON room_stats_historical (end_ts);
	104
	105	-- represents PRESENT statistics for a user
	106	-- only holds absolute fields
	107	DROP TABLE IF EXISTS user_stats_current;
	108	CREATE TABLE user_stats_current (
	109	user_id TEXT NOT NULL PRIMARY KEY,
	110
	111	joined_rooms BIGINT NOT NULL,
	112
	113	-- The maximum delta stream position that this row takes into account.
	114	completed_delta_stream_id BIGINT NOT NULL
	115	);
	116
	117	-- represents HISTORICAL statistics for a user
	118	DROP TABLE IF EXISTS user_stats_historical;
	119	CREATE TABLE user_stats_historical (
	120	user_id TEXT NOT NULL,
	121	end_ts BIGINT NOT NULL,
	122	bucket_size BIGINT NOT NULL,
	123
	124	joined_rooms BIGINT NOT NULL,
	125
	126	invites_sent BIGINT NOT NULL,
	127	rooms_created BIGINT NOT NULL,
	128	total_events BIGINT NOT NULL,
	129	total_event_bytes BIGINT NOT NULL,
	130
	131	PRIMARY KEY (user_id, end_ts)
	132	);
	133
	134	-- We use this index to speed up deletion of ancient user stats.
	135	CREATE INDEX user_stats_historical_end_ts ON user_stats_historical (end_ts);
	136
	137
	138	CREATE TABLE room_stats_state (
	139	room_id TEXT NOT NULL,
	140	name TEXT,
	141	canonical_alias TEXT,
	142	join_rules TEXT,
	143	history_visibility TEXT,
	144	encryption TEXT,
	145	avatar TEXT,
	146	guest_access TEXT,
	147	is_federatable BOOLEAN,
	148	topic TEXT
	149	);
	150
	151	CREATE UNIQUE INDEX room_stats_state_room ON room_stats_state(room_id);

+52

-0

synapse/storage/data_stores/main/schema/delta/56/unique_user_filter_index.py less more

	0	import logging
	1
	2	from synapse.storage.engines import PostgresEngine
	3
	4	logger = logging.getLogger(__name__)
	5
	6
	7	"""
	8	This migration updates the user_filters table as follows:
	9
	10	- drops any (user_id, filter_id) duplicates
	11	- makes the columns NON-NULLable
	12	- turns the index into a UNIQUE index
	13	"""
	14
	15
	16	def run_upgrade(cur, database_engine, args, *kwargs):
	17	pass
	18
	19
	20	def run_create(cur, database_engine, args, *kwargs):
	21	if isinstance(database_engine, PostgresEngine):
	22	select_clause = """
	23	SELECT DISTINCT ON (user_id, filter_id) user_id, filter_id, filter_json
	24	FROM user_filters
	25	"""
	26	else:
	27	select_clause = """
	28	SELECT * FROM user_filters GROUP BY user_id, filter_id
	29	"""
	30	sql = """
	31	DROP TABLE IF EXISTS user_filters_migration;
	32	DROP INDEX IF EXISTS user_filters_unique;
	33	CREATE TABLE user_filters_migration (
	34	user_id TEXT NOT NULL,
	35	filter_id BIGINT NOT NULL,
	36	filter_json BYTEA NOT NULL
	37	);
	38	INSERT INTO user_filters_migration (user_id, filter_id, filter_json)
	39	%s;
	40	CREATE UNIQUE INDEX user_filters_unique ON user_filters_migration
	41	(user_id, filter_id);
	42	DROP TABLE user_filters;
	43	ALTER TABLE user_filters_migration RENAME TO user_filters;
	44	""" % (
	45	select_clause,
	46	)
	47
	48	if isinstance(database_engine, PostgresEngine):
	49	cur.execute(sql)
	50	else:
	51	cur.executescript(sql)

+24

-0

synapse/storage/data_stores/main/schema/delta/56/user_external_ids.sql less more

	0	/* Copyright 2019 The Matrix.org Foundation C.I.C.
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	/*
	16	* a table which records mappings from external auth providers to mxids
	17	*/
	18	CREATE TABLE IF NOT EXISTS user_external_ids (
	19	auth_provider TEXT NOT NULL,
	20	external_id TEXT NOT NULL,
	21	user_id TEXT NOT NULL,
	22	UNIQUE (auth_provider, external_id)
	23	);

+17

-0

synapse/storage/data_stores/main/schema/delta/56/users_in_public_rooms_idx.sql less more

	0	/* Copyright 2019 Matrix.org Foundation CIC
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- this was apparently forgotten when the table was created back in delta 53.
	16	CREATE INDEX users_in_public_rooms_r_idx ON users_in_public_rooms(room_id);

+37

-0

synapse/storage/data_stores/main/schema/full_schemas/16/application_services.sql less more

	0	/* Copyright 2015, 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	/* We used to create tables called application_services and
	16	* application_services_regex, but these are no longer used and are removed in
	17	* delta 54.
	18	*/
	19
	20
	21	CREATE TABLE IF NOT EXISTS application_services_state(
	22	as_id TEXT PRIMARY KEY,
	23	state VARCHAR(5),
	24	last_txn INTEGER
	25	);
	26
	27	CREATE TABLE IF NOT EXISTS application_services_txns(
	28	as_id TEXT NOT NULL,
	29	txn_id INTEGER NOT NULL,
	30	event_ids TEXT NOT NULL,
	31	UNIQUE(as_id, txn_id)
	32	);
	33
	34	CREATE INDEX application_services_txns_id ON application_services_txns (
	35	as_id
	36	);

+70

-0

synapse/storage/data_stores/main/schema/full_schemas/16/event_edges.sql less more

	0	/* Copyright 2014-2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	/* We used to create tables called event_destinations and
	16	* state_forward_extremities, but these are no longer used and are removed in
	17	* delta 54.
	18	*/
	19
	20	CREATE TABLE IF NOT EXISTS event_forward_extremities(
	21	event_id TEXT NOT NULL,
	22	room_id TEXT NOT NULL,
	23	UNIQUE (event_id, room_id)
	24	);
	25
	26	CREATE INDEX ev_extrem_room ON event_forward_extremities(room_id);
	27	CREATE INDEX ev_extrem_id ON event_forward_extremities(event_id);
	28
	29
	30	CREATE TABLE IF NOT EXISTS event_backward_extremities(
	31	event_id TEXT NOT NULL,
	32	room_id TEXT NOT NULL,
	33	UNIQUE (event_id, room_id)
	34	);
	35
	36	CREATE INDEX ev_b_extrem_room ON event_backward_extremities(room_id);
	37	CREATE INDEX ev_b_extrem_id ON event_backward_extremities(event_id);
	38
	39
	40	CREATE TABLE IF NOT EXISTS event_edges(
	41	event_id TEXT NOT NULL,
	42	prev_event_id TEXT NOT NULL,
	43	room_id TEXT NOT NULL,
	44	is_state BOOL NOT NULL, -- true if this is a prev_state edge rather than a regular
	45	-- event dag edge.
	46	UNIQUE (event_id, prev_event_id, room_id, is_state)
	47	);
	48
	49	CREATE INDEX ev_edges_id ON event_edges(event_id);
	50	CREATE INDEX ev_edges_prev_id ON event_edges(prev_event_id);
	51
	52
	53	CREATE TABLE IF NOT EXISTS room_depth(
	54	room_id TEXT NOT NULL,
	55	min_depth INTEGER NOT NULL,
	56	UNIQUE (room_id)
	57	);
	58
	59	CREATE INDEX room_depth_room ON room_depth(room_id);
	60
	61	CREATE TABLE IF NOT EXISTS event_auth(
	62	event_id TEXT NOT NULL,
	63	auth_id TEXT NOT NULL,
	64	room_id TEXT NOT NULL,
	65	UNIQUE (event_id, auth_id, room_id)
	66	);
	67
	68	CREATE INDEX evauth_edges_id ON event_auth(event_id);
	69	CREATE INDEX evauth_edges_auth_id ON event_auth(auth_id);

+38

-0

synapse/storage/data_stores/main/schema/full_schemas/16/event_signatures.sql less more

	0	/* Copyright 2014-2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	/* We used to create tables called event_content_hashes and event_edge_hashes,
	16	* but these are no longer used and are removed in delta 54.
	17	*/
	18
	19	CREATE TABLE IF NOT EXISTS event_reference_hashes (
	20	event_id TEXT,
	21	algorithm TEXT,
	22	hash bytea,
	23	UNIQUE (event_id, algorithm)
	24	);
	25
	26	CREATE INDEX event_reference_hashes_id ON event_reference_hashes(event_id);
	27
	28
	29	CREATE TABLE IF NOT EXISTS event_signatures (
	30	event_id TEXT,
	31	signature_name TEXT,
	32	key_id TEXT,
	33	signature bytea,
	34	UNIQUE (event_id, signature_name, key_id)
	35	);
	36
	37	CREATE INDEX event_signatures_id ON event_signatures(event_id);

+120

-0

synapse/storage/data_stores/main/schema/full_schemas/16/im.sql less more

	0	/* Copyright 2014-2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	/* We used to create tables called room_hosts and feedback,
	16	* but these are no longer used and are removed in delta 54.
	17	*/
	18
	19	CREATE TABLE IF NOT EXISTS events(
	20	stream_ordering INTEGER PRIMARY KEY,
	21	topological_ordering BIGINT NOT NULL,
	22	event_id TEXT NOT NULL,
	23	type TEXT NOT NULL,
	24	room_id TEXT NOT NULL,
	25
	26	-- 'content' used to be created NULLable, but as of delta 50 we drop that constraint.
	27	-- the hack we use to drop the constraint doesn't work for an in-memory sqlite
	28	-- database, which breaks the sytests. Hence, we no longer make it nullable.
	29	content TEXT,
	30
	31	unrecognized_keys TEXT,
	32	processed BOOL NOT NULL,
	33	outlier BOOL NOT NULL,
	34	depth BIGINT DEFAULT 0 NOT NULL,
	35	UNIQUE (event_id)
	36	);
	37
	38	CREATE INDEX events_stream_ordering ON events (stream_ordering);
	39	CREATE INDEX events_topological_ordering ON events (topological_ordering);
	40	CREATE INDEX events_order ON events (topological_ordering, stream_ordering);
	41	CREATE INDEX events_room_id ON events (room_id);
	42	CREATE INDEX events_order_room ON events (
	43	room_id, topological_ordering, stream_ordering
	44	);
	45
	46
	47	CREATE TABLE IF NOT EXISTS event_json(
	48	event_id TEXT NOT NULL,
	49	room_id TEXT NOT NULL,
	50	internal_metadata TEXT NOT NULL,
	51	json TEXT NOT NULL,
	52	UNIQUE (event_id)
	53	);
	54
	55	CREATE INDEX event_json_room_id ON event_json(room_id);
	56
	57
	58	CREATE TABLE IF NOT EXISTS state_events(
	59	event_id TEXT NOT NULL,
	60	room_id TEXT NOT NULL,
	61	type TEXT NOT NULL,
	62	state_key TEXT NOT NULL,
	63	prev_state TEXT,
	64	UNIQUE (event_id)
	65	);
	66
	67	CREATE INDEX state_events_room_id ON state_events (room_id);
	68	CREATE INDEX state_events_type ON state_events (type);
	69	CREATE INDEX state_events_state_key ON state_events (state_key);
	70
	71
	72	CREATE TABLE IF NOT EXISTS current_state_events(
	73	event_id TEXT NOT NULL,
	74	room_id TEXT NOT NULL,
	75	type TEXT NOT NULL,
	76	state_key TEXT NOT NULL,
	77	UNIQUE (event_id),
	78	UNIQUE (room_id, type, state_key)
	79	);
	80
	81	CREATE INDEX current_state_events_room_id ON current_state_events (room_id);
	82	CREATE INDEX current_state_events_type ON current_state_events (type);
	83	CREATE INDEX current_state_events_state_key ON current_state_events (state_key);
	84
	85	CREATE TABLE IF NOT EXISTS room_memberships(
	86	event_id TEXT NOT NULL,
	87	user_id TEXT NOT NULL,
	88	sender TEXT NOT NULL,
	89	room_id TEXT NOT NULL,
	90	membership TEXT NOT NULL,
	91	UNIQUE (event_id)
	92	);
	93
	94	CREATE INDEX room_memberships_room_id ON room_memberships (room_id);
	95	CREATE INDEX room_memberships_user_id ON room_memberships (user_id);
	96
	97	CREATE TABLE IF NOT EXISTS topics(
	98	event_id TEXT NOT NULL,
	99	room_id TEXT NOT NULL,
	100	topic TEXT NOT NULL,
	101	UNIQUE (event_id)
	102	);
	103
	104	CREATE INDEX topics_room_id ON topics(room_id);
	105
	106	CREATE TABLE IF NOT EXISTS room_names(
	107	event_id TEXT NOT NULL,
	108	room_id TEXT NOT NULL,
	109	name TEXT NOT NULL,
	110	UNIQUE (event_id)
	111	);
	112
	113	CREATE INDEX room_names_room_id ON room_names(room_id);
	114
	115	CREATE TABLE IF NOT EXISTS rooms(
	116	room_id TEXT PRIMARY KEY NOT NULL,
	117	is_public BOOL,
	118	creator TEXT
	119	);

+26

-0

synapse/storage/data_stores/main/schema/full_schemas/16/keys.sql less more

	0	/* Copyright 2014-2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- we used to create a table called server_tls_certificates, but this is no
	16	-- longer used, and is removed in delta 54.
	17
	18	CREATE TABLE IF NOT EXISTS server_signature_keys(
	19	server_name TEXT, -- Server name.
	20	key_id TEXT, -- Key version.
	21	from_server TEXT, -- Which key server the key was fetched form.
	22	ts_added_ms BIGINT, -- When the key was added.
	23	verify_key bytea, -- NACL verification key.
	24	UNIQUE (server_name, key_id)
	25	);

+68

-0

synapse/storage/data_stores/main/schema/full_schemas/16/media_repository.sql less more

	0	/* Copyright 2014-2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	CREATE TABLE IF NOT EXISTS local_media_repository (
	16	media_id TEXT, -- The id used to refer to the media.
	17	media_type TEXT, -- The MIME-type of the media.
	18	media_length INTEGER, -- Length of the media in bytes.
	19	created_ts BIGINT, -- When the content was uploaded in ms.
	20	upload_name TEXT, -- The name the media was uploaded with.
	21	user_id TEXT, -- The user who uploaded the file.
	22	UNIQUE (media_id)
	23	);
	24
	25	CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails (
	26	media_id TEXT, -- The id used to refer to the media.
	27	thumbnail_width INTEGER, -- The width of the thumbnail in pixels.
	28	thumbnail_height INTEGER, -- The height of the thumbnail in pixels.
	29	thumbnail_type TEXT, -- The MIME-type of the thumbnail.
	30	thumbnail_method TEXT, -- The method used to make the thumbnail.
	31	thumbnail_length INTEGER, -- The length of the thumbnail in bytes.
	32	UNIQUE (
	33	media_id, thumbnail_width, thumbnail_height, thumbnail_type
	34	)
	35	);
	36
	37	CREATE INDEX local_media_repository_thumbnails_media_id
	38	ON local_media_repository_thumbnails (media_id);
	39
	40	CREATE TABLE IF NOT EXISTS remote_media_cache (
	41	media_origin TEXT, -- The remote HS the media came from.
	42	media_id TEXT, -- The id used to refer to the media on that server.
	43	media_type TEXT, -- The MIME-type of the media.
	44	created_ts BIGINT, -- When the content was uploaded in ms.
	45	upload_name TEXT, -- The name the media was uploaded with.
	46	media_length INTEGER, -- Length of the media in bytes.
	47	filesystem_id TEXT, -- The name used to store the media on disk.
	48	UNIQUE (media_origin, media_id)
	49	);
	50
	51	CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails (
	52	media_origin TEXT, -- The remote HS the media came from.
	53	media_id TEXT, -- The id used to refer to the media.
	54	thumbnail_width INTEGER, -- The width of the thumbnail in pixels.
	55	thumbnail_height INTEGER, -- The height of the thumbnail in pixels.
	56	thumbnail_method TEXT, -- The method used to make the thumbnail
	57	thumbnail_type TEXT, -- The MIME-type of the thumbnail.
	58	thumbnail_length INTEGER, -- The length of the thumbnail in bytes.
	59	filesystem_id TEXT, -- The name used to store the media on disk.
	60	UNIQUE (
	61	media_origin, media_id, thumbnail_width, thumbnail_height,
	62	thumbnail_type
	63	)
	64	);
	65
	66	CREATE INDEX remote_media_cache_thumbnails_media_id
	67	ON remote_media_cache_thumbnails (media_id);

+32

-0

synapse/storage/data_stores/main/schema/full_schemas/16/presence.sql less more

	0	/* Copyright 2014-2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14	CREATE TABLE IF NOT EXISTS presence(
	15	user_id TEXT NOT NULL,
	16	state VARCHAR(20),
	17	status_msg TEXT,
	18	mtime BIGINT, -- miliseconds since last state change
	19	UNIQUE (user_id)
	20	);
	21
	22	-- For each of /my/ users which possibly-remote users are allowed to see their
	23	-- presence state
	24	CREATE TABLE IF NOT EXISTS presence_allow_inbound(
	25	observed_user_id TEXT NOT NULL,
	26	observer_user_id TEXT NOT NULL, -- a UserID,
	27	UNIQUE (observed_user_id, observer_user_id)
	28	);
	29
	30	-- We used to create a table called presence_list, but this is no longer used
	31	-- and is removed in delta 54.⏎

+20

-0

synapse/storage/data_stores/main/schema/full_schemas/16/profiles.sql less more

	0	/* Copyright 2014-2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14	CREATE TABLE IF NOT EXISTS profiles(
	15	user_id TEXT NOT NULL,
	16	displayname TEXT,
	17	avatar_url TEXT,
	18	UNIQUE(user_id)
	19	);

+74

-0

synapse/storage/data_stores/main/schema/full_schemas/16/push.sql less more

	0	/* Copyright 2015, 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	CREATE TABLE IF NOT EXISTS rejections(
	16	event_id TEXT NOT NULL,
	17	reason TEXT NOT NULL,
	18	last_check TEXT NOT NULL,
	19	UNIQUE (event_id)
	20	);
	21
	22	-- Push notification endpoints that users have configured
	23	CREATE TABLE IF NOT EXISTS pushers (
	24	id BIGINT PRIMARY KEY,
	25	user_name TEXT NOT NULL,
	26	access_token BIGINT DEFAULT NULL,
	27	profile_tag VARCHAR(32) NOT NULL,
	28	kind VARCHAR(8) NOT NULL,
	29	app_id VARCHAR(64) NOT NULL,
	30	app_display_name VARCHAR(64) NOT NULL,
	31	device_display_name VARCHAR(128) NOT NULL,
	32	pushkey bytea NOT NULL,
	33	ts BIGINT NOT NULL,
	34	lang VARCHAR(8),
	35	data bytea,
	36	last_token TEXT,
	37	last_success BIGINT,
	38	failing_since BIGINT,
	39	UNIQUE (app_id, pushkey)
	40	);
	41
	42	CREATE TABLE IF NOT EXISTS push_rules (
	43	id BIGINT PRIMARY KEY,
	44	user_name TEXT NOT NULL,
	45	rule_id TEXT NOT NULL,
	46	priority_class SMALLINT NOT NULL,
	47	priority INTEGER NOT NULL DEFAULT 0,
	48	conditions TEXT NOT NULL,
	49	actions TEXT NOT NULL,
	50	UNIQUE(user_name, rule_id)
	51	);
	52
	53	CREATE INDEX push_rules_user_name on push_rules (user_name);
	54
	55	CREATE TABLE IF NOT EXISTS user_filters(
	56	user_id TEXT,
	57	filter_id BIGINT,
	58	filter_json bytea
	59	);
	60
	61	CREATE INDEX user_filters_by_user_id_filter_id ON user_filters(
	62	user_id, filter_id
	63	);
	64
	65	CREATE TABLE IF NOT EXISTS push_rules_enable (
	66	id BIGINT PRIMARY KEY,
	67	user_name TEXT NOT NULL,
	68	rule_id TEXT NOT NULL,
	69	enabled SMALLINT,
	70	UNIQUE(user_name, rule_id)
	71	);
	72
	73	CREATE INDEX push_rules_enable_user_name on push_rules_enable (user_name);

+22

-0

synapse/storage/data_stores/main/schema/full_schemas/16/redactions.sql less more

	0	/* Copyright 2014-2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14	CREATE TABLE IF NOT EXISTS redactions (
	15	event_id TEXT NOT NULL,
	16	redacts TEXT NOT NULL,
	17	UNIQUE (event_id)
	18	);
	19
	20	CREATE INDEX redactions_event_id ON redactions (event_id);
	21	CREATE INDEX redactions_redacts ON redactions (redacts);

+29

-0

synapse/storage/data_stores/main/schema/full_schemas/16/room_aliases.sql less more

	0	/* Copyright 2014-2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	CREATE TABLE IF NOT EXISTS room_aliases(
	16	room_alias TEXT NOT NULL,
	17	room_id TEXT NOT NULL,
	18	UNIQUE (room_alias)
	19	);
	20
	21	CREATE INDEX room_aliases_id ON room_aliases(room_id);
	22
	23	CREATE TABLE IF NOT EXISTS room_alias_servers(
	24	room_alias TEXT NOT NULL,
	25	server TEXT NOT NULL
	26	);
	27
	28	CREATE INDEX room_alias_servers_alias ON room_alias_servers(room_alias);

+40

-0

synapse/storage/data_stores/main/schema/full_schemas/16/state.sql less more

	0	/* Copyright 2014-2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	CREATE TABLE IF NOT EXISTS state_groups(
	16	id BIGINT PRIMARY KEY,
	17	room_id TEXT NOT NULL,
	18	event_id TEXT NOT NULL
	19	);
	20
	21	CREATE TABLE IF NOT EXISTS state_groups_state(
	22	state_group BIGINT NOT NULL,
	23	room_id TEXT NOT NULL,
	24	type TEXT NOT NULL,
	25	state_key TEXT NOT NULL,
	26	event_id TEXT NOT NULL
	27	);
	28
	29	CREATE TABLE IF NOT EXISTS event_to_state_groups(
	30	event_id TEXT NOT NULL,
	31	state_group BIGINT NOT NULL,
	32	UNIQUE (event_id)
	33	);
	34
	35	CREATE INDEX state_groups_id ON state_groups(id);
	36
	37	CREATE INDEX state_groups_state_id ON state_groups_state(state_group);
	38	CREATE INDEX state_groups_state_tuple ON state_groups_state(room_id, type, state_key);
	39	CREATE INDEX event_to_state_groups_id ON event_to_state_groups(event_id);

+44

-0

synapse/storage/data_stores/main/schema/full_schemas/16/transactions.sql less more

	0	/* Copyright 2014-2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14	-- Stores what transaction ids we have received and what our response was
	15	CREATE TABLE IF NOT EXISTS received_transactions(
	16	transaction_id TEXT,
	17	origin TEXT,
	18	ts BIGINT,
	19	response_code INTEGER,
	20	response_json bytea,
	21	has_been_referenced smallint default 0, -- Whether thishas been referenced by a prev_tx
	22	UNIQUE (transaction_id, origin)
	23	);
	24
	25	CREATE INDEX transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0;
	26
	27	-- For sent transactions only.
	28	CREATE TABLE IF NOT EXISTS transaction_id_to_pdu(
	29	transaction_id INTEGER,
	30	destination TEXT,
	31	pdu_id TEXT,
	32	pdu_origin TEXT,
	33	UNIQUE (transaction_id, destination)
	34	);
	35
	36	CREATE INDEX transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination);
	37
	38	-- To track destination health
	39	CREATE TABLE IF NOT EXISTS destinations(
	40	destination TEXT PRIMARY KEY,
	41	retry_last_ts BIGINT,
	42	retry_interval INTEGER
	43	);

+42

-0

synapse/storage/data_stores/main/schema/full_schemas/16/users.sql less more

	0	/* Copyright 2014-2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14	CREATE TABLE IF NOT EXISTS users(
	15	name TEXT,
	16	password_hash TEXT,
	17	creation_ts BIGINT,
	18	admin SMALLINT DEFAULT 0 NOT NULL,
	19	UNIQUE(name)
	20	);
	21
	22	CREATE TABLE IF NOT EXISTS access_tokens(
	23	id BIGINT PRIMARY KEY,
	24	user_id TEXT NOT NULL,
	25	device_id TEXT,
	26	token TEXT NOT NULL,
	27	last_used BIGINT,
	28	UNIQUE(token)
	29	);
	30
	31	CREATE TABLE IF NOT EXISTS user_ips (
	32	user_id TEXT NOT NULL,
	33	access_token TEXT NOT NULL,
	34	device_id TEXT,
	35	ip TEXT NOT NULL,
	36	user_agent TEXT NOT NULL,
	37	last_seen BIGINT NOT NULL
	38	);
	39
	40	CREATE INDEX user_ips_user ON user_ips(user_id);
	41	CREATE INDEX user_ips_user_ip ON user_ips(user_id, access_token, ip);

+2035

-0

synapse/storage/data_stores/main/schema/full_schemas/54/full.sql.postgres less more

	0
	1
	2
	3
	4
	5	CREATE TABLE access_tokens (
	6	id bigint NOT NULL,
	7	user_id text NOT NULL,
	8	device_id text,
	9	token text NOT NULL,
	10	last_used bigint
	11	);
	12
	13
	14
	15	CREATE TABLE account_data (
	16	user_id text NOT NULL,
	17	account_data_type text NOT NULL,
	18	stream_id bigint NOT NULL,
	19	content text NOT NULL
	20	);
	21
	22
	23
	24	CREATE TABLE account_data_max_stream_id (
	25	lock character(1) DEFAULT 'X'::bpchar NOT NULL,
	26	stream_id bigint NOT NULL,
	27	CONSTRAINT private_user_data_max_stream_id_lock_check CHECK ((lock = 'X'::bpchar))
	28	);
	29
	30
	31
	32	CREATE TABLE account_validity (
	33	user_id text NOT NULL,
	34	expiration_ts_ms bigint NOT NULL,
	35	email_sent boolean NOT NULL,
	36	renewal_token text
	37	);
	38
	39
	40
	41	CREATE TABLE application_services_state (
	42	as_id text NOT NULL,
	43	state character varying(5),
	44	last_txn integer
	45	);
	46
	47
	48
	49	CREATE TABLE application_services_txns (
	50	as_id text NOT NULL,
	51	txn_id integer NOT NULL,
	52	event_ids text NOT NULL
	53	);
	54
	55
	56
	57	CREATE TABLE appservice_room_list (
	58	appservice_id text NOT NULL,
	59	network_id text NOT NULL,
	60	room_id text NOT NULL
	61	);
	62
	63
	64
	65	CREATE TABLE appservice_stream_position (
	66	lock character(1) DEFAULT 'X'::bpchar NOT NULL,
	67	stream_ordering bigint,
	68	CONSTRAINT appservice_stream_position_lock_check CHECK ((lock = 'X'::bpchar))
	69	);
	70
	71
	72	CREATE TABLE blocked_rooms (
	73	room_id text NOT NULL,
	74	user_id text NOT NULL
	75	);
	76
	77
	78
	79	CREATE TABLE cache_invalidation_stream (
	80	stream_id bigint,
	81	cache_func text,
	82	keys text[],
	83	invalidation_ts bigint
	84	);
	85
	86
	87
	88	CREATE TABLE current_state_delta_stream (
	89	stream_id bigint NOT NULL,
	90	room_id text NOT NULL,
	91	type text NOT NULL,
	92	state_key text NOT NULL,
	93	event_id text,
	94	prev_event_id text
	95	);
	96
	97
	98
	99	CREATE TABLE current_state_events (
	100	event_id text NOT NULL,
	101	room_id text NOT NULL,
	102	type text NOT NULL,
	103	state_key text NOT NULL
	104	);
	105
	106
	107
	108	CREATE TABLE deleted_pushers (
	109	stream_id bigint NOT NULL,
	110	app_id text NOT NULL,
	111	pushkey text NOT NULL,
	112	user_id text NOT NULL
	113	);
	114
	115
	116
	117	CREATE TABLE destinations (
	118	destination text NOT NULL,
	119	retry_last_ts bigint,
	120	retry_interval integer
	121	);
	122
	123
	124
	125	CREATE TABLE device_federation_inbox (
	126	origin text NOT NULL,
	127	message_id text NOT NULL,
	128	received_ts bigint NOT NULL
	129	);
	130
	131
	132
	133	CREATE TABLE device_federation_outbox (
	134	destination text NOT NULL,
	135	stream_id bigint NOT NULL,
	136	queued_ts bigint NOT NULL,
	137	messages_json text NOT NULL
	138	);
	139
	140
	141
	142	CREATE TABLE device_inbox (
	143	user_id text NOT NULL,
	144	device_id text NOT NULL,
	145	stream_id bigint NOT NULL,
	146	message_json text NOT NULL
	147	);
	148
	149
	150
	151	CREATE TABLE device_lists_outbound_last_success (
	152	destination text NOT NULL,
	153	user_id text NOT NULL,
	154	stream_id bigint NOT NULL
	155	);
	156
	157
	158
	159	CREATE TABLE device_lists_outbound_pokes (
	160	destination text NOT NULL,
	161	stream_id bigint NOT NULL,
	162	user_id text NOT NULL,
	163	device_id text NOT NULL,
	164	sent boolean NOT NULL,
	165	ts bigint NOT NULL
	166	);
	167
	168
	169
	170	CREATE TABLE device_lists_remote_cache (
	171	user_id text NOT NULL,
	172	device_id text NOT NULL,
	173	content text NOT NULL
	174	);
	175
	176
	177
	178	CREATE TABLE device_lists_remote_extremeties (
	179	user_id text NOT NULL,
	180	stream_id text NOT NULL
	181	);
	182
	183
	184
	185	CREATE TABLE device_lists_stream (
	186	stream_id bigint NOT NULL,
	187	user_id text NOT NULL,
	188	device_id text NOT NULL
	189	);
	190
	191
	192
	193	CREATE TABLE device_max_stream_id (
	194	stream_id bigint NOT NULL
	195	);
	196
	197
	198
	199	CREATE TABLE devices (
	200	user_id text NOT NULL,
	201	device_id text NOT NULL,
	202	display_name text
	203	);
	204
	205
	206
	207	CREATE TABLE e2e_device_keys_json (
	208	user_id text NOT NULL,
	209	device_id text NOT NULL,
	210	ts_added_ms bigint NOT NULL,
	211	key_json text NOT NULL
	212	);
	213
	214
	215
	216	CREATE TABLE e2e_one_time_keys_json (
	217	user_id text NOT NULL,
	218	device_id text NOT NULL,
	219	algorithm text NOT NULL,
	220	key_id text NOT NULL,
	221	ts_added_ms bigint NOT NULL,
	222	key_json text NOT NULL
	223	);
	224
	225
	226
	227	CREATE TABLE e2e_room_keys (
	228	user_id text NOT NULL,
	229	room_id text NOT NULL,
	230	session_id text NOT NULL,
	231	version bigint NOT NULL,
	232	first_message_index integer,
	233	forwarded_count integer,
	234	is_verified boolean,
	235	session_data text NOT NULL
	236	);
	237
	238
	239
	240	CREATE TABLE e2e_room_keys_versions (
	241	user_id text NOT NULL,
	242	version bigint NOT NULL,
	243	algorithm text NOT NULL,
	244	auth_data text NOT NULL,
	245	deleted smallint DEFAULT 0 NOT NULL
	246	);
	247
	248
	249
	250	CREATE TABLE erased_users (
	251	user_id text NOT NULL
	252	);
	253
	254
	255
	256	CREATE TABLE event_auth (
	257	event_id text NOT NULL,
	258	auth_id text NOT NULL,
	259	room_id text NOT NULL
	260	);
	261
	262
	263
	264	CREATE TABLE event_backward_extremities (
	265	event_id text NOT NULL,
	266	room_id text NOT NULL
	267	);
	268
	269
	270
	271	CREATE TABLE event_edges (
	272	event_id text NOT NULL,
	273	prev_event_id text NOT NULL,
	274	room_id text NOT NULL,
	275	is_state boolean NOT NULL
	276	);
	277
	278
	279
	280	CREATE TABLE event_forward_extremities (
	281	event_id text NOT NULL,
	282	room_id text NOT NULL
	283	);
	284
	285
	286
	287	CREATE TABLE event_json (
	288	event_id text NOT NULL,
	289	room_id text NOT NULL,
	290	internal_metadata text NOT NULL,
	291	json text NOT NULL,
	292	format_version integer
	293	);
	294
	295
	296
	297	CREATE TABLE event_push_actions (
	298	room_id text NOT NULL,
	299	event_id text NOT NULL,
	300	user_id text NOT NULL,
	301	profile_tag character varying(32),
	302	actions text NOT NULL,
	303	topological_ordering bigint,
	304	stream_ordering bigint,
	305	notif smallint,
	306	highlight smallint
	307	);
	308
	309
	310
	311	CREATE TABLE event_push_actions_staging (
	312	event_id text NOT NULL,
	313	user_id text NOT NULL,
	314	actions text NOT NULL,
	315	notif smallint NOT NULL,
	316	highlight smallint NOT NULL
	317	);
	318
	319
	320
	321	CREATE TABLE event_push_summary (
	322	user_id text NOT NULL,
	323	room_id text NOT NULL,
	324	notif_count bigint NOT NULL,
	325	stream_ordering bigint NOT NULL
	326	);
	327
	328
	329
	330	CREATE TABLE event_push_summary_stream_ordering (
	331	lock character(1) DEFAULT 'X'::bpchar NOT NULL,
	332	stream_ordering bigint NOT NULL,
	333	CONSTRAINT event_push_summary_stream_ordering_lock_check CHECK ((lock = 'X'::bpchar))
	334	);
	335
	336
	337
	338	CREATE TABLE event_reference_hashes (
	339	event_id text,
	340	algorithm text,
	341	hash bytea
	342	);
	343
	344
	345
	346	CREATE TABLE event_relations (
	347	event_id text NOT NULL,
	348	relates_to_id text NOT NULL,
	349	relation_type text NOT NULL,
	350	aggregation_key text
	351	);
	352
	353
	354
	355	CREATE TABLE event_reports (
	356	id bigint NOT NULL,
	357	received_ts bigint NOT NULL,
	358	room_id text NOT NULL,
	359	event_id text NOT NULL,
	360	user_id text NOT NULL,
	361	reason text,
	362	content text
	363	);
	364
	365
	366
	367	CREATE TABLE event_search (
	368	event_id text,
	369	room_id text,
	370	sender text,
	371	key text,
	372	vector tsvector,
	373	origin_server_ts bigint,
	374	stream_ordering bigint
	375	);
	376
	377
	378
	379	CREATE TABLE event_to_state_groups (
	380	event_id text NOT NULL,
	381	state_group bigint NOT NULL
	382	);
	383
	384
	385
	386	CREATE TABLE events (
	387	stream_ordering integer NOT NULL,
	388	topological_ordering bigint NOT NULL,
	389	event_id text NOT NULL,
	390	type text NOT NULL,
	391	room_id text NOT NULL,
	392	content text,
	393	unrecognized_keys text,
	394	processed boolean NOT NULL,
	395	outlier boolean NOT NULL,
	396	depth bigint DEFAULT 0 NOT NULL,
	397	origin_server_ts bigint,
	398	received_ts bigint,
	399	sender text,
	400	contains_url boolean
	401	);
	402
	403
	404
	405	CREATE TABLE ex_outlier_stream (
	406	event_stream_ordering bigint NOT NULL,
	407	event_id text NOT NULL,
	408	state_group bigint NOT NULL
	409	);
	410
	411
	412
	413	CREATE TABLE federation_stream_position (
	414	type text NOT NULL,
	415	stream_id integer NOT NULL
	416	);
	417
	418
	419
	420	CREATE TABLE group_attestations_remote (
	421	group_id text NOT NULL,
	422	user_id text NOT NULL,
	423	valid_until_ms bigint NOT NULL,
	424	attestation_json text NOT NULL
	425	);
	426
	427
	428
	429	CREATE TABLE group_attestations_renewals (
	430	group_id text NOT NULL,
	431	user_id text NOT NULL,
	432	valid_until_ms bigint NOT NULL
	433	);
	434
	435
	436
	437	CREATE TABLE group_invites (
	438	group_id text NOT NULL,
	439	user_id text NOT NULL
	440	);
	441
	442
	443
	444	CREATE TABLE group_roles (
	445	group_id text NOT NULL,
	446	role_id text NOT NULL,
	447	profile text NOT NULL,
	448	is_public boolean NOT NULL
	449	);
	450
	451
	452
	453	CREATE TABLE group_room_categories (
	454	group_id text NOT NULL,
	455	category_id text NOT NULL,
	456	profile text NOT NULL,
	457	is_public boolean NOT NULL
	458	);
	459
	460
	461
	462	CREATE TABLE group_rooms (
	463	group_id text NOT NULL,
	464	room_id text NOT NULL,
	465	is_public boolean NOT NULL
	466	);
	467
	468
	469
	470	CREATE TABLE group_summary_roles (
	471	group_id text NOT NULL,
	472	role_id text NOT NULL,
	473	role_order bigint NOT NULL,
	474	CONSTRAINT group_summary_roles_role_order_check CHECK ((role_order > 0))
	475	);
	476
	477
	478
	479	CREATE TABLE group_summary_room_categories (
	480	group_id text NOT NULL,
	481	category_id text NOT NULL,
	482	cat_order bigint NOT NULL,
	483	CONSTRAINT group_summary_room_categories_cat_order_check CHECK ((cat_order > 0))
	484	);
	485
	486
	487
	488	CREATE TABLE group_summary_rooms (
	489	group_id text NOT NULL,
	490	room_id text NOT NULL,
	491	category_id text NOT NULL,
	492	room_order bigint NOT NULL,
	493	is_public boolean NOT NULL,
	494	CONSTRAINT group_summary_rooms_room_order_check CHECK ((room_order > 0))
	495	);
	496
	497
	498
	499	CREATE TABLE group_summary_users (
	500	group_id text NOT NULL,
	501	user_id text NOT NULL,
	502	role_id text NOT NULL,
	503	user_order bigint NOT NULL,
	504	is_public boolean NOT NULL
	505	);
	506
	507
	508
	509	CREATE TABLE group_users (
	510	group_id text NOT NULL,
	511	user_id text NOT NULL,
	512	is_admin boolean NOT NULL,
	513	is_public boolean NOT NULL
	514	);
	515
	516
	517
	518	CREATE TABLE groups (
	519	group_id text NOT NULL,
	520	name text,
	521	avatar_url text,
	522	short_description text,
	523	long_description text,
	524	is_public boolean NOT NULL,
	525	join_policy text DEFAULT 'invite'::text NOT NULL
	526	);
	527
	528
	529
	530	CREATE TABLE guest_access (
	531	event_id text NOT NULL,
	532	room_id text NOT NULL,
	533	guest_access text NOT NULL
	534	);
	535
	536
	537
	538	CREATE TABLE history_visibility (
	539	event_id text NOT NULL,
	540	room_id text NOT NULL,
	541	history_visibility text NOT NULL
	542	);
	543
	544
	545
	546	CREATE TABLE local_group_membership (
	547	group_id text NOT NULL,
	548	user_id text NOT NULL,
	549	is_admin boolean NOT NULL,
	550	membership text NOT NULL,
	551	is_publicised boolean NOT NULL,
	552	content text NOT NULL
	553	);
	554
	555
	556
	557	CREATE TABLE local_group_updates (
	558	stream_id bigint NOT NULL,
	559	group_id text NOT NULL,
	560	user_id text NOT NULL,
	561	type text NOT NULL,
	562	content text NOT NULL
	563	);
	564
	565
	566
	567	CREATE TABLE local_invites (
	568	stream_id bigint NOT NULL,
	569	inviter text NOT NULL,
	570	invitee text NOT NULL,
	571	event_id text NOT NULL,
	572	room_id text NOT NULL,
	573	locally_rejected text,
	574	replaced_by text
	575	);
	576
	577
	578
	579	CREATE TABLE local_media_repository (
	580	media_id text,
	581	media_type text,
	582	media_length integer,
	583	created_ts bigint,
	584	upload_name text,
	585	user_id text,
	586	quarantined_by text,
	587	url_cache text,
	588	last_access_ts bigint
	589	);
	590
	591
	592
	593	CREATE TABLE local_media_repository_thumbnails (
	594	media_id text,
	595	thumbnail_width integer,
	596	thumbnail_height integer,
	597	thumbnail_type text,
	598	thumbnail_method text,
	599	thumbnail_length integer
	600	);
	601
	602
	603
	604	CREATE TABLE local_media_repository_url_cache (
	605	url text,
	606	response_code integer,
	607	etag text,
	608	expires_ts bigint,
	609	og text,
	610	media_id text,
	611	download_ts bigint
	612	);
	613
	614
	615
	616	CREATE TABLE monthly_active_users (
	617	user_id text NOT NULL,
	618	"timestamp" bigint NOT NULL
	619	);
	620
	621
	622
	623	CREATE TABLE open_id_tokens (
	624	token text NOT NULL,
	625	ts_valid_until_ms bigint NOT NULL,
	626	user_id text NOT NULL
	627	);
	628
	629
	630
	631	CREATE TABLE presence (
	632	user_id text NOT NULL,
	633	state character varying(20),
	634	status_msg text,
	635	mtime bigint
	636	);
	637
	638
	639
	640	CREATE TABLE presence_allow_inbound (
	641	observed_user_id text NOT NULL,
	642	observer_user_id text NOT NULL
	643	);
	644
	645
	646
	647	CREATE TABLE presence_stream (
	648	stream_id bigint,
	649	user_id text,
	650	state text,
	651	last_active_ts bigint,
	652	last_federation_update_ts bigint,
	653	last_user_sync_ts bigint,
	654	status_msg text,
	655	currently_active boolean
	656	);
	657
	658
	659
	660	CREATE TABLE profiles (
	661	user_id text NOT NULL,
	662	displayname text,
	663	avatar_url text
	664	);
	665
	666
	667
	668	CREATE TABLE public_room_list_stream (
	669	stream_id bigint NOT NULL,
	670	room_id text NOT NULL,
	671	visibility boolean NOT NULL,
	672	appservice_id text,
	673	network_id text
	674	);
	675
	676
	677
	678	CREATE TABLE push_rules (
	679	id bigint NOT NULL,
	680	user_name text NOT NULL,
	681	rule_id text NOT NULL,
	682	priority_class smallint NOT NULL,
	683	priority integer DEFAULT 0 NOT NULL,
	684	conditions text NOT NULL,
	685	actions text NOT NULL
	686	);
	687
	688
	689
	690	CREATE TABLE push_rules_enable (
	691	id bigint NOT NULL,
	692	user_name text NOT NULL,
	693	rule_id text NOT NULL,
	694	enabled smallint
	695	);
	696
	697
	698
	699	CREATE TABLE push_rules_stream (
	700	stream_id bigint NOT NULL,
	701	event_stream_ordering bigint NOT NULL,
	702	user_id text NOT NULL,
	703	rule_id text NOT NULL,
	704	op text NOT NULL,
	705	priority_class smallint,
	706	priority integer,
	707	conditions text,
	708	actions text
	709	);
	710
	711
	712
	713	CREATE TABLE pusher_throttle (
	714	pusher bigint NOT NULL,
	715	room_id text NOT NULL,
	716	last_sent_ts bigint,
	717	throttle_ms bigint
	718	);
	719
	720
	721
	722	CREATE TABLE pushers (
	723	id bigint NOT NULL,
	724	user_name text NOT NULL,
	725	access_token bigint,
	726	profile_tag text NOT NULL,
	727	kind text NOT NULL,
	728	app_id text NOT NULL,
	729	app_display_name text NOT NULL,
	730	device_display_name text NOT NULL,
	731	pushkey text NOT NULL,
	732	ts bigint NOT NULL,
	733	lang text,
	734	data text,
	735	last_stream_ordering integer,
	736	last_success bigint,
	737	failing_since bigint
	738	);
	739
	740
	741
	742	CREATE TABLE ratelimit_override (
	743	user_id text NOT NULL,
	744	messages_per_second bigint,
	745	burst_count bigint
	746	);
	747
	748
	749
	750	CREATE TABLE receipts_graph (
	751	room_id text NOT NULL,
	752	receipt_type text NOT NULL,
	753	user_id text NOT NULL,
	754	event_ids text NOT NULL,
	755	data text NOT NULL
	756	);
	757
	758
	759
	760	CREATE TABLE receipts_linearized (
	761	stream_id bigint NOT NULL,
	762	room_id text NOT NULL,
	763	receipt_type text NOT NULL,
	764	user_id text NOT NULL,
	765	event_id text NOT NULL,
	766	data text NOT NULL
	767	);
	768
	769
	770
	771	CREATE TABLE received_transactions (
	772	transaction_id text,
	773	origin text,
	774	ts bigint,
	775	response_code integer,
	776	response_json bytea,
	777	has_been_referenced smallint DEFAULT 0
	778	);
	779
	780
	781
	782	CREATE TABLE redactions (
	783	event_id text NOT NULL,
	784	redacts text NOT NULL
	785	);
	786
	787
	788
	789	CREATE TABLE rejections (
	790	event_id text NOT NULL,
	791	reason text NOT NULL,
	792	last_check text NOT NULL
	793	);
	794
	795
	796
	797	CREATE TABLE remote_media_cache (
	798	media_origin text,
	799	media_id text,
	800	media_type text,
	801	created_ts bigint,
	802	upload_name text,
	803	media_length integer,
	804	filesystem_id text,
	805	last_access_ts bigint,
	806	quarantined_by text
	807	);
	808
	809
	810
	811	CREATE TABLE remote_media_cache_thumbnails (
	812	media_origin text,
	813	media_id text,
	814	thumbnail_width integer,
	815	thumbnail_height integer,
	816	thumbnail_method text,
	817	thumbnail_type text,
	818	thumbnail_length integer,
	819	filesystem_id text
	820	);
	821
	822
	823
	824	CREATE TABLE remote_profile_cache (
	825	user_id text NOT NULL,
	826	displayname text,
	827	avatar_url text,
	828	last_check bigint NOT NULL
	829	);
	830
	831
	832
	833	CREATE TABLE room_account_data (
	834	user_id text NOT NULL,
	835	room_id text NOT NULL,
	836	account_data_type text NOT NULL,
	837	stream_id bigint NOT NULL,
	838	content text NOT NULL
	839	);
	840
	841
	842
	843	CREATE TABLE room_alias_servers (
	844	room_alias text NOT NULL,
	845	server text NOT NULL
	846	);
	847
	848
	849
	850	CREATE TABLE room_aliases (
	851	room_alias text NOT NULL,
	852	room_id text NOT NULL,
	853	creator text
	854	);
	855
	856
	857
	858	CREATE TABLE room_depth (
	859	room_id text NOT NULL,
	860	min_depth integer NOT NULL
	861	);
	862
	863
	864
	865	CREATE TABLE room_memberships (
	866	event_id text NOT NULL,
	867	user_id text NOT NULL,
	868	sender text NOT NULL,
	869	room_id text NOT NULL,
	870	membership text NOT NULL,
	871	forgotten integer DEFAULT 0,
	872	display_name text,
	873	avatar_url text
	874	);
	875
	876
	877
	878	CREATE TABLE room_names (
	879	event_id text NOT NULL,
	880	room_id text NOT NULL,
	881	name text NOT NULL
	882	);
	883
	884
	885
	886	CREATE TABLE room_state (
	887	room_id text NOT NULL,
	888	join_rules text,
	889	history_visibility text,
	890	encryption text,
	891	name text,
	892	topic text,
	893	avatar text,
	894	canonical_alias text
	895	);
	896
	897
	898
	899	CREATE TABLE room_stats (
	900	room_id text NOT NULL,
	901	ts bigint NOT NULL,
	902	bucket_size integer NOT NULL,
	903	current_state_events integer NOT NULL,
	904	joined_members integer NOT NULL,
	905	invited_members integer NOT NULL,
	906	left_members integer NOT NULL,
	907	banned_members integer NOT NULL,
	908	state_events integer NOT NULL
	909	);
	910
	911
	912
	913	CREATE TABLE room_stats_earliest_token (
	914	room_id text NOT NULL,
	915	token bigint NOT NULL
	916	);
	917
	918
	919
	920	CREATE TABLE room_tags (
	921	user_id text NOT NULL,
	922	room_id text NOT NULL,
	923	tag text NOT NULL,
	924	content text NOT NULL
	925	);
	926
	927
	928
	929	CREATE TABLE room_tags_revisions (
	930	user_id text NOT NULL,
	931	room_id text NOT NULL,
	932	stream_id bigint NOT NULL
	933	);
	934
	935
	936
	937	CREATE TABLE rooms (
	938	room_id text NOT NULL,
	939	is_public boolean,
	940	creator text
	941	);
	942
	943
	944
	945	CREATE TABLE server_keys_json (
	946	server_name text NOT NULL,
	947	key_id text NOT NULL,
	948	from_server text NOT NULL,
	949	ts_added_ms bigint NOT NULL,
	950	ts_valid_until_ms bigint NOT NULL,
	951	key_json bytea NOT NULL
	952	);
	953
	954
	955
	956	CREATE TABLE server_signature_keys (
	957	server_name text,
	958	key_id text,
	959	from_server text,
	960	ts_added_ms bigint,
	961	verify_key bytea,
	962	ts_valid_until_ms bigint
	963	);
	964
	965
	966
	967	CREATE TABLE state_events (
	968	event_id text NOT NULL,
	969	room_id text NOT NULL,
	970	type text NOT NULL,
	971	state_key text NOT NULL,
	972	prev_state text
	973	);
	974
	975
	976
	977	CREATE TABLE state_group_edges (
	978	state_group bigint NOT NULL,
	979	prev_state_group bigint NOT NULL
	980	);
	981
	982
	983
	984	CREATE SEQUENCE state_group_id_seq
	985	START WITH 1
	986	INCREMENT BY 1
	987	NO MINVALUE
	988	NO MAXVALUE
	989	CACHE 1;
	990
	991
	992
	993	CREATE TABLE state_groups (
	994	id bigint NOT NULL,
	995	room_id text NOT NULL,
	996	event_id text NOT NULL
	997	);
	998
	999
	1000
	1001	CREATE TABLE state_groups_state (
	1002	state_group bigint NOT NULL,
	1003	room_id text NOT NULL,
	1004	type text NOT NULL,
	1005	state_key text NOT NULL,
	1006	event_id text NOT NULL
	1007	);
	1008
	1009
	1010
	1011	CREATE TABLE stats_stream_pos (
	1012	lock character(1) DEFAULT 'X'::bpchar NOT NULL,
	1013	stream_id bigint,
	1014	CONSTRAINT stats_stream_pos_lock_check CHECK ((lock = 'X'::bpchar))
	1015	);
	1016
	1017
	1018
	1019	CREATE TABLE stream_ordering_to_exterm (
	1020	stream_ordering bigint NOT NULL,
	1021	room_id text NOT NULL,
	1022	event_id text NOT NULL
	1023	);
	1024
	1025
	1026
	1027	CREATE TABLE threepid_guest_access_tokens (
	1028	medium text,
	1029	address text,
	1030	guest_access_token text,
	1031	first_inviter text
	1032	);
	1033
	1034
	1035
	1036	CREATE TABLE topics (
	1037	event_id text NOT NULL,
	1038	room_id text NOT NULL,
	1039	topic text NOT NULL
	1040	);
	1041
	1042
	1043
	1044	CREATE TABLE user_daily_visits (
	1045	user_id text NOT NULL,
	1046	device_id text,
	1047	"timestamp" bigint NOT NULL
	1048	);
	1049
	1050
	1051
	1052	CREATE TABLE user_directory (
	1053	user_id text NOT NULL,
	1054	room_id text,
	1055	display_name text,
	1056	avatar_url text
	1057	);
	1058
	1059
	1060
	1061	CREATE TABLE user_directory_search (
	1062	user_id text NOT NULL,
	1063	vector tsvector
	1064	);
	1065
	1066
	1067
	1068	CREATE TABLE user_directory_stream_pos (
	1069	lock character(1) DEFAULT 'X'::bpchar NOT NULL,
	1070	stream_id bigint,
	1071	CONSTRAINT user_directory_stream_pos_lock_check CHECK ((lock = 'X'::bpchar))
	1072	);
	1073
	1074
	1075
	1076	CREATE TABLE user_filters (
	1077	user_id text,
	1078	filter_id bigint,
	1079	filter_json bytea
	1080	);
	1081
	1082
	1083
	1084	CREATE TABLE user_ips (
	1085	user_id text NOT NULL,
	1086	access_token text NOT NULL,
	1087	device_id text,
	1088	ip text NOT NULL,
	1089	user_agent text NOT NULL,
	1090	last_seen bigint NOT NULL
	1091	);
	1092
	1093
	1094
	1095	CREATE TABLE user_stats (
	1096	user_id text NOT NULL,
	1097	ts bigint NOT NULL,
	1098	bucket_size integer NOT NULL,
	1099	public_rooms integer NOT NULL,
	1100	private_rooms integer NOT NULL
	1101	);
	1102
	1103
	1104
	1105	CREATE TABLE user_threepid_id_server (
	1106	user_id text NOT NULL,
	1107	medium text NOT NULL,
	1108	address text NOT NULL,
	1109	id_server text NOT NULL
	1110	);
	1111
	1112
	1113
	1114	CREATE TABLE user_threepids (
	1115	user_id text NOT NULL,
	1116	medium text NOT NULL,
	1117	address text NOT NULL,
	1118	validated_at bigint NOT NULL,
	1119	added_at bigint NOT NULL
	1120	);
	1121
	1122
	1123
	1124	CREATE TABLE users (
	1125	name text,
	1126	password_hash text,
	1127	creation_ts bigint,
	1128	admin smallint DEFAULT 0 NOT NULL,
	1129	upgrade_ts bigint,
	1130	is_guest smallint DEFAULT 0 NOT NULL,
	1131	appservice_id text,
	1132	consent_version text,
	1133	consent_server_notice_sent text,
	1134	user_type text
	1135	);
	1136
	1137
	1138
	1139	CREATE TABLE users_in_public_rooms (
	1140	user_id text NOT NULL,
	1141	room_id text NOT NULL
	1142	);
	1143
	1144
	1145
	1146	CREATE TABLE users_pending_deactivation (
	1147	user_id text NOT NULL
	1148	);
	1149
	1150
	1151
	1152	CREATE TABLE users_who_share_private_rooms (
	1153	user_id text NOT NULL,
	1154	other_user_id text NOT NULL,
	1155	room_id text NOT NULL
	1156	);
	1157
	1158
	1159
	1160	ALTER TABLE ONLY access_tokens
	1161	ADD CONSTRAINT access_tokens_pkey PRIMARY KEY (id);
	1162
	1163
	1164
	1165	ALTER TABLE ONLY access_tokens
	1166	ADD CONSTRAINT access_tokens_token_key UNIQUE (token);
	1167
	1168
	1169
	1170	ALTER TABLE ONLY account_data
	1171	ADD CONSTRAINT account_data_uniqueness UNIQUE (user_id, account_data_type);
	1172
	1173
	1174
	1175	ALTER TABLE ONLY account_validity
	1176	ADD CONSTRAINT account_validity_pkey PRIMARY KEY (user_id);
	1177
	1178
	1179
	1180	ALTER TABLE ONLY application_services_state
	1181	ADD CONSTRAINT application_services_state_pkey PRIMARY KEY (as_id);
	1182
	1183
	1184
	1185	ALTER TABLE ONLY application_services_txns
	1186	ADD CONSTRAINT application_services_txns_as_id_txn_id_key UNIQUE (as_id, txn_id);
	1187
	1188
	1189
	1190	ALTER TABLE ONLY appservice_stream_position
	1191	ADD CONSTRAINT appservice_stream_position_lock_key UNIQUE (lock);
	1192
	1193
	1194
	1195	ALTER TABLE ONLY current_state_events
	1196	ADD CONSTRAINT current_state_events_event_id_key UNIQUE (event_id);
	1197
	1198
	1199
	1200	ALTER TABLE ONLY current_state_events
	1201	ADD CONSTRAINT current_state_events_room_id_type_state_key_key UNIQUE (room_id, type, state_key);
	1202
	1203
	1204
	1205	ALTER TABLE ONLY destinations
	1206	ADD CONSTRAINT destinations_pkey PRIMARY KEY (destination);
	1207
	1208
	1209
	1210	ALTER TABLE ONLY devices
	1211	ADD CONSTRAINT device_uniqueness UNIQUE (user_id, device_id);
	1212
	1213
	1214
	1215	ALTER TABLE ONLY e2e_device_keys_json
	1216	ADD CONSTRAINT e2e_device_keys_json_uniqueness UNIQUE (user_id, device_id);
	1217
	1218
	1219
	1220	ALTER TABLE ONLY e2e_one_time_keys_json
	1221	ADD CONSTRAINT e2e_one_time_keys_json_uniqueness UNIQUE (user_id, device_id, algorithm, key_id);
	1222
	1223
	1224
	1225	ALTER TABLE ONLY event_backward_extremities
	1226	ADD CONSTRAINT event_backward_extremities_event_id_room_id_key UNIQUE (event_id, room_id);
	1227
	1228
	1229
	1230	ALTER TABLE ONLY event_edges
	1231	ADD CONSTRAINT event_edges_event_id_prev_event_id_room_id_is_state_key UNIQUE (event_id, prev_event_id, room_id, is_state);
	1232
	1233
	1234
	1235	ALTER TABLE ONLY event_forward_extremities
	1236	ADD CONSTRAINT event_forward_extremities_event_id_room_id_key UNIQUE (event_id, room_id);
	1237
	1238
	1239
	1240	ALTER TABLE ONLY event_push_actions
	1241	ADD CONSTRAINT event_id_user_id_profile_tag_uniqueness UNIQUE (room_id, event_id, user_id, profile_tag);
	1242
	1243
	1244
	1245	ALTER TABLE ONLY event_json
	1246	ADD CONSTRAINT event_json_event_id_key UNIQUE (event_id);
	1247
	1248
	1249
	1250	ALTER TABLE ONLY event_push_summary_stream_ordering
	1251	ADD CONSTRAINT event_push_summary_stream_ordering_lock_key UNIQUE (lock);
	1252
	1253
	1254
	1255	ALTER TABLE ONLY event_reference_hashes
	1256	ADD CONSTRAINT event_reference_hashes_event_id_algorithm_key UNIQUE (event_id, algorithm);
	1257
	1258
	1259
	1260	ALTER TABLE ONLY event_reports
	1261	ADD CONSTRAINT event_reports_pkey PRIMARY KEY (id);
	1262
	1263
	1264
	1265	ALTER TABLE ONLY event_to_state_groups
	1266	ADD CONSTRAINT event_to_state_groups_event_id_key UNIQUE (event_id);
	1267
	1268
	1269
	1270	ALTER TABLE ONLY events
	1271	ADD CONSTRAINT events_event_id_key UNIQUE (event_id);
	1272
	1273
	1274
	1275	ALTER TABLE ONLY events
	1276	ADD CONSTRAINT events_pkey PRIMARY KEY (stream_ordering);
	1277
	1278
	1279
	1280	ALTER TABLE ONLY ex_outlier_stream
	1281	ADD CONSTRAINT ex_outlier_stream_pkey PRIMARY KEY (event_stream_ordering);
	1282
	1283
	1284
	1285	ALTER TABLE ONLY group_roles
	1286	ADD CONSTRAINT group_roles_group_id_role_id_key UNIQUE (group_id, role_id);
	1287
	1288
	1289
	1290	ALTER TABLE ONLY group_room_categories
	1291	ADD CONSTRAINT group_room_categories_group_id_category_id_key UNIQUE (group_id, category_id);
	1292
	1293
	1294
	1295	ALTER TABLE ONLY group_summary_roles
	1296	ADD CONSTRAINT group_summary_roles_group_id_role_id_role_order_key UNIQUE (group_id, role_id, role_order);
	1297
	1298
	1299
	1300	ALTER TABLE ONLY group_summary_room_categories
	1301	ADD CONSTRAINT group_summary_room_categories_group_id_category_id_cat_orde_key UNIQUE (group_id, category_id, cat_order);
	1302
	1303
	1304
	1305	ALTER TABLE ONLY group_summary_rooms
	1306	ADD CONSTRAINT group_summary_rooms_group_id_category_id_room_id_room_order_key UNIQUE (group_id, category_id, room_id, room_order);
	1307
	1308
	1309
	1310	ALTER TABLE ONLY guest_access
	1311	ADD CONSTRAINT guest_access_event_id_key UNIQUE (event_id);
	1312
	1313
	1314
	1315	ALTER TABLE ONLY history_visibility
	1316	ADD CONSTRAINT history_visibility_event_id_key UNIQUE (event_id);
	1317
	1318
	1319
	1320	ALTER TABLE ONLY local_media_repository
	1321	ADD CONSTRAINT local_media_repository_media_id_key UNIQUE (media_id);
	1322
	1323
	1324
	1325	ALTER TABLE ONLY local_media_repository_thumbnails
	1326	ADD CONSTRAINT local_media_repository_thumbn_media_id_thumbnail_width_thum_key UNIQUE (media_id, thumbnail_width, thumbnail_height, thumbnail_type);
	1327
	1328
	1329
	1330	ALTER TABLE ONLY user_threepids
	1331	ADD CONSTRAINT medium_address UNIQUE (medium, address);
	1332
	1333
	1334
	1335	ALTER TABLE ONLY open_id_tokens
	1336	ADD CONSTRAINT open_id_tokens_pkey PRIMARY KEY (token);
	1337
	1338
	1339
	1340	ALTER TABLE ONLY presence_allow_inbound
	1341	ADD CONSTRAINT presence_allow_inbound_observed_user_id_observer_user_id_key UNIQUE (observed_user_id, observer_user_id);
	1342
	1343
	1344
	1345	ALTER TABLE ONLY presence
	1346	ADD CONSTRAINT presence_user_id_key UNIQUE (user_id);
	1347
	1348
	1349
	1350	ALTER TABLE ONLY account_data_max_stream_id
	1351	ADD CONSTRAINT private_user_data_max_stream_id_lock_key UNIQUE (lock);
	1352
	1353
	1354
	1355	ALTER TABLE ONLY profiles
	1356	ADD CONSTRAINT profiles_user_id_key UNIQUE (user_id);
	1357
	1358
	1359
	1360	ALTER TABLE ONLY push_rules_enable
	1361	ADD CONSTRAINT push_rules_enable_pkey PRIMARY KEY (id);
	1362
	1363
	1364
	1365	ALTER TABLE ONLY push_rules_enable
	1366	ADD CONSTRAINT push_rules_enable_user_name_rule_id_key UNIQUE (user_name, rule_id);
	1367
	1368
	1369
	1370	ALTER TABLE ONLY push_rules
	1371	ADD CONSTRAINT push_rules_pkey PRIMARY KEY (id);
	1372
	1373
	1374
	1375	ALTER TABLE ONLY push_rules
	1376	ADD CONSTRAINT push_rules_user_name_rule_id_key UNIQUE (user_name, rule_id);
	1377
	1378
	1379
	1380	ALTER TABLE ONLY pusher_throttle
	1381	ADD CONSTRAINT pusher_throttle_pkey PRIMARY KEY (pusher, room_id);
	1382
	1383
	1384
	1385	ALTER TABLE ONLY pushers
	1386	ADD CONSTRAINT pushers2_app_id_pushkey_user_name_key UNIQUE (app_id, pushkey, user_name);
	1387
	1388
	1389
	1390	ALTER TABLE ONLY pushers
	1391	ADD CONSTRAINT pushers2_pkey PRIMARY KEY (id);
	1392
	1393
	1394
	1395	ALTER TABLE ONLY receipts_graph
	1396	ADD CONSTRAINT receipts_graph_uniqueness UNIQUE (room_id, receipt_type, user_id);
	1397
	1398
	1399
	1400	ALTER TABLE ONLY receipts_linearized
	1401	ADD CONSTRAINT receipts_linearized_uniqueness UNIQUE (room_id, receipt_type, user_id);
	1402
	1403
	1404
	1405	ALTER TABLE ONLY received_transactions
	1406	ADD CONSTRAINT received_transactions_transaction_id_origin_key UNIQUE (transaction_id, origin);
	1407
	1408
	1409
	1410	ALTER TABLE ONLY redactions
	1411	ADD CONSTRAINT redactions_event_id_key UNIQUE (event_id);
	1412
	1413
	1414
	1415	ALTER TABLE ONLY rejections
	1416	ADD CONSTRAINT rejections_event_id_key UNIQUE (event_id);
	1417
	1418
	1419
	1420	ALTER TABLE ONLY remote_media_cache
	1421	ADD CONSTRAINT remote_media_cache_media_origin_media_id_key UNIQUE (media_origin, media_id);
	1422
	1423
	1424
	1425	ALTER TABLE ONLY remote_media_cache_thumbnails
	1426	ADD CONSTRAINT remote_media_cache_thumbnails_media_origin_media_id_thumbna_key UNIQUE (media_origin, media_id, thumbnail_width, thumbnail_height, thumbnail_type);
	1427
	1428
	1429
	1430	ALTER TABLE ONLY room_account_data
	1431	ADD CONSTRAINT room_account_data_uniqueness UNIQUE (user_id, room_id, account_data_type);
	1432
	1433
	1434
	1435	ALTER TABLE ONLY room_aliases
	1436	ADD CONSTRAINT room_aliases_room_alias_key UNIQUE (room_alias);
	1437
	1438
	1439
	1440	ALTER TABLE ONLY room_depth
	1441	ADD CONSTRAINT room_depth_room_id_key UNIQUE (room_id);
	1442
	1443
	1444
	1445	ALTER TABLE ONLY room_memberships
	1446	ADD CONSTRAINT room_memberships_event_id_key UNIQUE (event_id);
	1447
	1448
	1449
	1450	ALTER TABLE ONLY room_names
	1451	ADD CONSTRAINT room_names_event_id_key UNIQUE (event_id);
	1452
	1453
	1454
	1455	ALTER TABLE ONLY room_tags_revisions
	1456	ADD CONSTRAINT room_tag_revisions_uniqueness UNIQUE (user_id, room_id);
	1457
	1458
	1459
	1460	ALTER TABLE ONLY room_tags
	1461	ADD CONSTRAINT room_tag_uniqueness UNIQUE (user_id, room_id, tag);
	1462
	1463
	1464
	1465	ALTER TABLE ONLY rooms
	1466	ADD CONSTRAINT rooms_pkey PRIMARY KEY (room_id);
	1467
	1468
	1469
	1470	ALTER TABLE ONLY server_keys_json
	1471	ADD CONSTRAINT server_keys_json_uniqueness UNIQUE (server_name, key_id, from_server);
	1472
	1473
	1474
	1475	ALTER TABLE ONLY server_signature_keys
	1476	ADD CONSTRAINT server_signature_keys_server_name_key_id_key UNIQUE (server_name, key_id);
	1477
	1478
	1479
	1480	ALTER TABLE ONLY state_events
	1481	ADD CONSTRAINT state_events_event_id_key UNIQUE (event_id);
	1482
	1483
	1484
	1485	ALTER TABLE ONLY state_groups
	1486	ADD CONSTRAINT state_groups_pkey PRIMARY KEY (id);
	1487
	1488
	1489
	1490	ALTER TABLE ONLY stats_stream_pos
	1491	ADD CONSTRAINT stats_stream_pos_lock_key UNIQUE (lock);
	1492
	1493
	1494
	1495	ALTER TABLE ONLY topics
	1496	ADD CONSTRAINT topics_event_id_key UNIQUE (event_id);
	1497
	1498
	1499
	1500	ALTER TABLE ONLY user_directory_stream_pos
	1501	ADD CONSTRAINT user_directory_stream_pos_lock_key UNIQUE (lock);
	1502
	1503
	1504
	1505	ALTER TABLE ONLY users
	1506	ADD CONSTRAINT users_name_key UNIQUE (name);
	1507
	1508
	1509
	1510	CREATE INDEX access_tokens_device_id ON access_tokens USING btree (user_id, device_id);
	1511
	1512
	1513
	1514	CREATE INDEX account_data_stream_id ON account_data USING btree (user_id, stream_id);
	1515
	1516
	1517
	1518	CREATE INDEX application_services_txns_id ON application_services_txns USING btree (as_id);
	1519
	1520
	1521
	1522	CREATE UNIQUE INDEX appservice_room_list_idx ON appservice_room_list USING btree (appservice_id, network_id, room_id);
	1523
	1524
	1525
	1526	CREATE UNIQUE INDEX blocked_rooms_idx ON blocked_rooms USING btree (room_id);
	1527
	1528
	1529
	1530	CREATE INDEX cache_invalidation_stream_id ON cache_invalidation_stream USING btree (stream_id);
	1531
	1532
	1533
	1534	CREATE INDEX current_state_delta_stream_idx ON current_state_delta_stream USING btree (stream_id);
	1535
	1536
	1537
	1538	CREATE INDEX current_state_events_member_index ON current_state_events USING btree (state_key) WHERE (type = 'm.room.member'::text);
	1539
	1540
	1541
	1542	CREATE INDEX deleted_pushers_stream_id ON deleted_pushers USING btree (stream_id);
	1543
	1544
	1545
	1546	CREATE INDEX device_federation_inbox_sender_id ON device_federation_inbox USING btree (origin, message_id);
	1547
	1548
	1549
	1550	CREATE INDEX device_federation_outbox_destination_id ON device_federation_outbox USING btree (destination, stream_id);
	1551
	1552
	1553
	1554	CREATE INDEX device_federation_outbox_id ON device_federation_outbox USING btree (stream_id);
	1555
	1556
	1557
	1558	CREATE INDEX device_inbox_stream_id_user_id ON device_inbox USING btree (stream_id, user_id);
	1559
	1560
	1561
	1562	CREATE INDEX device_inbox_user_stream_id ON device_inbox USING btree (user_id, device_id, stream_id);
	1563
	1564
	1565
	1566	CREATE INDEX device_lists_outbound_last_success_idx ON device_lists_outbound_last_success USING btree (destination, user_id, stream_id);
	1567
	1568
	1569
	1570	CREATE INDEX device_lists_outbound_pokes_id ON device_lists_outbound_pokes USING btree (destination, stream_id);
	1571
	1572
	1573
	1574	CREATE INDEX device_lists_outbound_pokes_stream ON device_lists_outbound_pokes USING btree (stream_id);
	1575
	1576
	1577
	1578	CREATE INDEX device_lists_outbound_pokes_user ON device_lists_outbound_pokes USING btree (destination, user_id);
	1579
	1580
	1581
	1582	CREATE UNIQUE INDEX device_lists_remote_cache_unique_id ON device_lists_remote_cache USING btree (user_id, device_id);
	1583
	1584
	1585
	1586	CREATE UNIQUE INDEX device_lists_remote_extremeties_unique_idx ON device_lists_remote_extremeties USING btree (user_id);
	1587
	1588
	1589
	1590	CREATE INDEX device_lists_stream_id ON device_lists_stream USING btree (stream_id, user_id);
	1591
	1592
	1593
	1594	CREATE INDEX device_lists_stream_user_id ON device_lists_stream USING btree (user_id, device_id);
	1595
	1596
	1597
	1598	CREATE UNIQUE INDEX e2e_room_keys_idx ON e2e_room_keys USING btree (user_id, room_id, session_id);
	1599
	1600
	1601
	1602	CREATE UNIQUE INDEX e2e_room_keys_versions_idx ON e2e_room_keys_versions USING btree (user_id, version);
	1603
	1604
	1605
	1606	CREATE UNIQUE INDEX erased_users_user ON erased_users USING btree (user_id);
	1607
	1608
	1609
	1610	CREATE INDEX ev_b_extrem_id ON event_backward_extremities USING btree (event_id);
	1611
	1612
	1613
	1614	CREATE INDEX ev_b_extrem_room ON event_backward_extremities USING btree (room_id);
	1615
	1616
	1617
	1618	CREATE INDEX ev_edges_id ON event_edges USING btree (event_id);
	1619
	1620
	1621
	1622	CREATE INDEX ev_edges_prev_id ON event_edges USING btree (prev_event_id);
	1623
	1624
	1625
	1626	CREATE INDEX ev_extrem_id ON event_forward_extremities USING btree (event_id);
	1627
	1628
	1629
	1630	CREATE INDEX ev_extrem_room ON event_forward_extremities USING btree (room_id);
	1631
	1632
	1633
	1634	CREATE INDEX evauth_edges_id ON event_auth USING btree (event_id);
	1635
	1636
	1637
	1638	CREATE INDEX event_contains_url_index ON events USING btree (room_id, topological_ordering, stream_ordering) WHERE ((contains_url = true) AND (outlier = false));
	1639
	1640
	1641
	1642	CREATE INDEX event_json_room_id ON event_json USING btree (room_id);
	1643
	1644
	1645
	1646	CREATE INDEX event_push_actions_highlights_index ON event_push_actions USING btree (user_id, room_id, topological_ordering, stream_ordering) WHERE (highlight = 1);
	1647
	1648
	1649
	1650	CREATE INDEX event_push_actions_rm_tokens ON event_push_actions USING btree (user_id, room_id, topological_ordering, stream_ordering);
	1651
	1652
	1653
	1654	CREATE INDEX event_push_actions_room_id_user_id ON event_push_actions USING btree (room_id, user_id);
	1655
	1656
	1657
	1658	CREATE INDEX event_push_actions_staging_id ON event_push_actions_staging USING btree (event_id);
	1659
	1660
	1661
	1662	CREATE INDEX event_push_actions_stream_ordering ON event_push_actions USING btree (stream_ordering, user_id);
	1663
	1664
	1665
	1666	CREATE INDEX event_push_actions_u_highlight ON event_push_actions USING btree (user_id, stream_ordering);
	1667
	1668
	1669
	1670	CREATE INDEX event_push_summary_user_rm ON event_push_summary USING btree (user_id, room_id);
	1671
	1672
	1673
	1674	CREATE INDEX event_reference_hashes_id ON event_reference_hashes USING btree (event_id);
	1675
	1676
	1677
	1678	CREATE UNIQUE INDEX event_relations_id ON event_relations USING btree (event_id);
	1679
	1680
	1681
	1682	CREATE INDEX event_relations_relates ON event_relations USING btree (relates_to_id, relation_type, aggregation_key);
	1683
	1684
	1685
	1686	CREATE INDEX event_search_ev_ridx ON event_search USING btree (room_id);
	1687
	1688
	1689
	1690	CREATE UNIQUE INDEX event_search_event_id_idx ON event_search USING btree (event_id);
	1691
	1692
	1693
	1694	CREATE INDEX event_search_fts_idx ON event_search USING gin (vector);
	1695
	1696
	1697
	1698	CREATE INDEX event_to_state_groups_sg_index ON event_to_state_groups USING btree (state_group);
	1699
	1700
	1701
	1702	CREATE INDEX events_order_room ON events USING btree (room_id, topological_ordering, stream_ordering);
	1703
	1704
	1705
	1706	CREATE INDEX events_room_stream ON events USING btree (room_id, stream_ordering);
	1707
	1708
	1709
	1710	CREATE INDEX events_ts ON events USING btree (origin_server_ts, stream_ordering);
	1711
	1712
	1713
	1714	CREATE INDEX group_attestations_remote_g_idx ON group_attestations_remote USING btree (group_id, user_id);
	1715
	1716
	1717
	1718	CREATE INDEX group_attestations_remote_u_idx ON group_attestations_remote USING btree (user_id);
	1719
	1720
	1721
	1722	CREATE INDEX group_attestations_remote_v_idx ON group_attestations_remote USING btree (valid_until_ms);
	1723
	1724
	1725
	1726	CREATE INDEX group_attestations_renewals_g_idx ON group_attestations_renewals USING btree (group_id, user_id);
	1727
	1728
	1729
	1730	CREATE INDEX group_attestations_renewals_u_idx ON group_attestations_renewals USING btree (user_id);
	1731
	1732
	1733
	1734	CREATE INDEX group_attestations_renewals_v_idx ON group_attestations_renewals USING btree (valid_until_ms);
	1735
	1736
	1737
	1738	CREATE UNIQUE INDEX group_invites_g_idx ON group_invites USING btree (group_id, user_id);
	1739
	1740
	1741
	1742	CREATE INDEX group_invites_u_idx ON group_invites USING btree (user_id);
	1743
	1744
	1745
	1746	CREATE UNIQUE INDEX group_rooms_g_idx ON group_rooms USING btree (group_id, room_id);
	1747
	1748
	1749
	1750	CREATE INDEX group_rooms_r_idx ON group_rooms USING btree (room_id);
	1751
	1752
	1753
	1754	CREATE UNIQUE INDEX group_summary_rooms_g_idx ON group_summary_rooms USING btree (group_id, room_id, category_id);
	1755
	1756
	1757
	1758	CREATE INDEX group_summary_users_g_idx ON group_summary_users USING btree (group_id);
	1759
	1760
	1761
	1762	CREATE UNIQUE INDEX group_users_g_idx ON group_users USING btree (group_id, user_id);
	1763
	1764
	1765
	1766	CREATE INDEX group_users_u_idx ON group_users USING btree (user_id);
	1767
	1768
	1769
	1770	CREATE UNIQUE INDEX groups_idx ON groups USING btree (group_id);
	1771
	1772
	1773
	1774	CREATE INDEX local_group_membership_g_idx ON local_group_membership USING btree (group_id);
	1775
	1776
	1777
	1778	CREATE INDEX local_group_membership_u_idx ON local_group_membership USING btree (user_id, group_id);
	1779
	1780
	1781
	1782	CREATE INDEX local_invites_for_user_idx ON local_invites USING btree (invitee, locally_rejected, replaced_by, room_id);
	1783
	1784
	1785
	1786	CREATE INDEX local_invites_id ON local_invites USING btree (stream_id);
	1787
	1788
	1789
	1790	CREATE INDEX local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails USING btree (media_id);
	1791
	1792
	1793
	1794	CREATE INDEX local_media_repository_url_cache_by_url_download_ts ON local_media_repository_url_cache USING btree (url, download_ts);
	1795
	1796
	1797
	1798	CREATE INDEX local_media_repository_url_cache_expires_idx ON local_media_repository_url_cache USING btree (expires_ts);
	1799
	1800
	1801
	1802	CREATE INDEX local_media_repository_url_cache_media_idx ON local_media_repository_url_cache USING btree (media_id);
	1803
	1804
	1805
	1806	CREATE INDEX local_media_repository_url_idx ON local_media_repository USING btree (created_ts) WHERE (url_cache IS NOT NULL);
	1807
	1808
	1809
	1810	CREATE INDEX monthly_active_users_time_stamp ON monthly_active_users USING btree ("timestamp");
	1811
	1812
	1813
	1814	CREATE UNIQUE INDEX monthly_active_users_users ON monthly_active_users USING btree (user_id);
	1815
	1816
	1817
	1818	CREATE INDEX open_id_tokens_ts_valid_until_ms ON open_id_tokens USING btree (ts_valid_until_ms);
	1819
	1820
	1821
	1822	CREATE INDEX presence_stream_id ON presence_stream USING btree (stream_id, user_id);
	1823
	1824
	1825
	1826	CREATE INDEX presence_stream_user_id ON presence_stream USING btree (user_id);
	1827
	1828
	1829
	1830	CREATE INDEX public_room_index ON rooms USING btree (is_public);
	1831
	1832
	1833
	1834	CREATE INDEX public_room_list_stream_idx ON public_room_list_stream USING btree (stream_id);
	1835
	1836
	1837
	1838	CREATE INDEX public_room_list_stream_rm_idx ON public_room_list_stream USING btree (room_id, stream_id);
	1839
	1840
	1841
	1842	CREATE INDEX push_rules_enable_user_name ON push_rules_enable USING btree (user_name);
	1843
	1844
	1845
	1846	CREATE INDEX push_rules_stream_id ON push_rules_stream USING btree (stream_id);
	1847
	1848
	1849
	1850	CREATE INDEX push_rules_stream_user_stream_id ON push_rules_stream USING btree (user_id, stream_id);
	1851
	1852
	1853
	1854	CREATE INDEX push_rules_user_name ON push_rules USING btree (user_name);
	1855
	1856
	1857
	1858	CREATE UNIQUE INDEX ratelimit_override_idx ON ratelimit_override USING btree (user_id);
	1859
	1860
	1861
	1862	CREATE INDEX receipts_linearized_id ON receipts_linearized USING btree (stream_id);
	1863
	1864
	1865
	1866	CREATE INDEX receipts_linearized_room_stream ON receipts_linearized USING btree (room_id, stream_id);
	1867
	1868
	1869
	1870	CREATE INDEX receipts_linearized_user ON receipts_linearized USING btree (user_id);
	1871
	1872
	1873
	1874	CREATE INDEX received_transactions_ts ON received_transactions USING btree (ts);
	1875
	1876
	1877
	1878	CREATE INDEX redactions_redacts ON redactions USING btree (redacts);
	1879
	1880
	1881
	1882	CREATE INDEX remote_profile_cache_time ON remote_profile_cache USING btree (last_check);
	1883
	1884
	1885
	1886	CREATE UNIQUE INDEX remote_profile_cache_user_id ON remote_profile_cache USING btree (user_id);
	1887
	1888
	1889
	1890	CREATE INDEX room_account_data_stream_id ON room_account_data USING btree (user_id, stream_id);
	1891
	1892
	1893
	1894	CREATE INDEX room_alias_servers_alias ON room_alias_servers USING btree (room_alias);
	1895
	1896
	1897
	1898	CREATE INDEX room_aliases_id ON room_aliases USING btree (room_id);
	1899
	1900
	1901
	1902	CREATE INDEX room_depth_room ON room_depth USING btree (room_id);
	1903
	1904
	1905
	1906	CREATE INDEX room_memberships_room_id ON room_memberships USING btree (room_id);
	1907
	1908
	1909
	1910	CREATE INDEX room_memberships_user_id ON room_memberships USING btree (user_id);
	1911
	1912
	1913
	1914	CREATE INDEX room_names_room_id ON room_names USING btree (room_id);
	1915
	1916
	1917
	1918	CREATE UNIQUE INDEX room_state_room ON room_state USING btree (room_id);
	1919
	1920
	1921
	1922	CREATE UNIQUE INDEX room_stats_earliest_token_idx ON room_stats_earliest_token USING btree (room_id);
	1923
	1924
	1925
	1926	CREATE UNIQUE INDEX room_stats_room_ts ON room_stats USING btree (room_id, ts);
	1927
	1928
	1929
	1930	CREATE INDEX state_group_edges_idx ON state_group_edges USING btree (state_group);
	1931
	1932
	1933
	1934	CREATE INDEX state_group_edges_prev_idx ON state_group_edges USING btree (prev_state_group);
	1935
	1936
	1937
	1938	CREATE INDEX state_groups_state_type_idx ON state_groups_state USING btree (state_group, type, state_key);
	1939
	1940
	1941
	1942	CREATE INDEX stream_ordering_to_exterm_idx ON stream_ordering_to_exterm USING btree (stream_ordering);
	1943
	1944
	1945
	1946	CREATE INDEX stream_ordering_to_exterm_rm_idx ON stream_ordering_to_exterm USING btree (room_id, stream_ordering);
	1947
	1948
	1949
	1950	CREATE UNIQUE INDEX threepid_guest_access_tokens_index ON threepid_guest_access_tokens USING btree (medium, address);
	1951
	1952
	1953
	1954	CREATE INDEX topics_room_id ON topics USING btree (room_id);
	1955
	1956
	1957
	1958	CREATE INDEX user_daily_visits_ts_idx ON user_daily_visits USING btree ("timestamp");
	1959
	1960
	1961
	1962	CREATE INDEX user_daily_visits_uts_idx ON user_daily_visits USING btree (user_id, "timestamp");
	1963
	1964
	1965
	1966	CREATE INDEX user_directory_room_idx ON user_directory USING btree (room_id);
	1967
	1968
	1969
	1970	CREATE INDEX user_directory_search_fts_idx ON user_directory_search USING gin (vector);
	1971
	1972
	1973
	1974	CREATE UNIQUE INDEX user_directory_search_user_idx ON user_directory_search USING btree (user_id);
	1975
	1976
	1977
	1978	CREATE UNIQUE INDEX user_directory_user_idx ON user_directory USING btree (user_id);
	1979
	1980
	1981
	1982	CREATE INDEX user_filters_by_user_id_filter_id ON user_filters USING btree (user_id, filter_id);
	1983
	1984
	1985
	1986	CREATE INDEX user_ips_device_id ON user_ips USING btree (user_id, device_id, last_seen);
	1987
	1988
	1989
	1990	CREATE INDEX user_ips_last_seen ON user_ips USING btree (user_id, last_seen);
	1991
	1992
	1993
	1994	CREATE INDEX user_ips_last_seen_only ON user_ips USING btree (last_seen);
	1995
	1996
	1997
	1998	CREATE UNIQUE INDEX user_ips_user_token_ip_unique_index ON user_ips USING btree (user_id, access_token, ip);
	1999
	2000
	2001
	2002	CREATE UNIQUE INDEX user_stats_user_ts ON user_stats USING btree (user_id, ts);
	2003
	2004
	2005
	2006	CREATE UNIQUE INDEX user_threepid_id_server_idx ON user_threepid_id_server USING btree (user_id, medium, address, id_server);
	2007
	2008
	2009
	2010	CREATE INDEX user_threepids_medium_address ON user_threepids USING btree (medium, address);
	2011
	2012
	2013
	2014	CREATE INDEX user_threepids_user_id ON user_threepids USING btree (user_id);
	2015
	2016
	2017
	2018	CREATE INDEX users_creation_ts ON users USING btree (creation_ts);
	2019
	2020
	2021
	2022	CREATE UNIQUE INDEX users_in_public_rooms_u_idx ON users_in_public_rooms USING btree (user_id, room_id);
	2023
	2024
	2025
	2026	CREATE INDEX users_who_share_private_rooms_o_idx ON users_who_share_private_rooms USING btree (other_user_id);
	2027
	2028
	2029
	2030	CREATE INDEX users_who_share_private_rooms_r_idx ON users_who_share_private_rooms USING btree (room_id);
	2031
	2032
	2033
	2034	CREATE UNIQUE INDEX users_who_share_private_rooms_u_idx ON users_who_share_private_rooms USING btree (user_id, other_user_id, room_id);

+259

-0

synapse/storage/data_stores/main/schema/full_schemas/54/full.sql.sqlite less more

	0	CREATE TABLE application_services_state( as_id TEXT PRIMARY KEY, state VARCHAR(5), last_txn INTEGER );
	1	CREATE TABLE application_services_txns( as_id TEXT NOT NULL, txn_id INTEGER NOT NULL, event_ids TEXT NOT NULL, UNIQUE(as_id, txn_id) );
	2	CREATE INDEX application_services_txns_id ON application_services_txns ( as_id );
	3	CREATE TABLE presence( user_id TEXT NOT NULL, state VARCHAR(20), status_msg TEXT, mtime BIGINT, UNIQUE (user_id) );
	4	CREATE TABLE presence_allow_inbound( observed_user_id TEXT NOT NULL, observer_user_id TEXT NOT NULL, UNIQUE (observed_user_id, observer_user_id) );
	5	CREATE TABLE users( name TEXT, password_hash TEXT, creation_ts BIGINT, admin SMALLINT DEFAULT 0 NOT NULL, upgrade_ts BIGINT, is_guest SMALLINT DEFAULT 0 NOT NULL, appservice_id TEXT, consent_version TEXT, consent_server_notice_sent TEXT, user_type TEXT DEFAULT NULL, UNIQUE(name) );
	6	CREATE TABLE access_tokens( id BIGINT PRIMARY KEY, user_id TEXT NOT NULL, device_id TEXT, token TEXT NOT NULL, last_used BIGINT, UNIQUE(token) );
	7	CREATE TABLE user_ips ( user_id TEXT NOT NULL, access_token TEXT NOT NULL, device_id TEXT, ip TEXT NOT NULL, user_agent TEXT NOT NULL, last_seen BIGINT NOT NULL );
	8	CREATE TABLE profiles( user_id TEXT NOT NULL, displayname TEXT, avatar_url TEXT, UNIQUE(user_id) );
	9	CREATE TABLE received_transactions( transaction_id TEXT, origin TEXT, ts BIGINT, response_code INTEGER, response_json bytea, has_been_referenced smallint default 0, UNIQUE (transaction_id, origin) );
	10	CREATE TABLE destinations( destination TEXT PRIMARY KEY, retry_last_ts BIGINT, retry_interval INTEGER );
	11	CREATE TABLE events( stream_ordering INTEGER PRIMARY KEY, topological_ordering BIGINT NOT NULL, event_id TEXT NOT NULL, type TEXT NOT NULL, room_id TEXT NOT NULL, content TEXT, unrecognized_keys TEXT, processed BOOL NOT NULL, outlier BOOL NOT NULL, depth BIGINT DEFAULT 0 NOT NULL, origin_server_ts BIGINT, received_ts BIGINT, sender TEXT, contains_url BOOLEAN, UNIQUE (event_id) );
	12	CREATE INDEX events_order_room ON events ( room_id, topological_ordering, stream_ordering );
	13	CREATE TABLE event_json( event_id TEXT NOT NULL, room_id TEXT NOT NULL, internal_metadata TEXT NOT NULL, json TEXT NOT NULL, format_version INTEGER, UNIQUE (event_id) );
	14	CREATE INDEX event_json_room_id ON event_json(room_id);
	15	CREATE TABLE state_events( event_id TEXT NOT NULL, room_id TEXT NOT NULL, type TEXT NOT NULL, state_key TEXT NOT NULL, prev_state TEXT, UNIQUE (event_id) );
	16	CREATE TABLE current_state_events( event_id TEXT NOT NULL, room_id TEXT NOT NULL, type TEXT NOT NULL, state_key TEXT NOT NULL, UNIQUE (event_id), UNIQUE (room_id, type, state_key) );
	17	CREATE TABLE room_memberships( event_id TEXT NOT NULL, user_id TEXT NOT NULL, sender TEXT NOT NULL, room_id TEXT NOT NULL, membership TEXT NOT NULL, forgotten INTEGER DEFAULT 0, display_name TEXT, avatar_url TEXT, UNIQUE (event_id) );
	18	CREATE INDEX room_memberships_room_id ON room_memberships (room_id);
	19	CREATE INDEX room_memberships_user_id ON room_memberships (user_id);
	20	CREATE TABLE topics( event_id TEXT NOT NULL, room_id TEXT NOT NULL, topic TEXT NOT NULL, UNIQUE (event_id) );
	21	CREATE INDEX topics_room_id ON topics(room_id);
	22	CREATE TABLE room_names( event_id TEXT NOT NULL, room_id TEXT NOT NULL, name TEXT NOT NULL, UNIQUE (event_id) );
	23	CREATE INDEX room_names_room_id ON room_names(room_id);
	24	CREATE TABLE rooms( room_id TEXT PRIMARY KEY NOT NULL, is_public BOOL, creator TEXT );
	25	CREATE TABLE server_signature_keys( server_name TEXT, key_id TEXT, from_server TEXT, ts_added_ms BIGINT, verify_key bytea, ts_valid_until_ms BIGINT, UNIQUE (server_name, key_id) );
	26	CREATE TABLE rejections( event_id TEXT NOT NULL, reason TEXT NOT NULL, last_check TEXT NOT NULL, UNIQUE (event_id) );
	27	CREATE TABLE push_rules ( id BIGINT PRIMARY KEY, user_name TEXT NOT NULL, rule_id TEXT NOT NULL, priority_class SMALLINT NOT NULL, priority INTEGER NOT NULL DEFAULT 0, conditions TEXT NOT NULL, actions TEXT NOT NULL, UNIQUE(user_name, rule_id) );
	28	CREATE INDEX push_rules_user_name on push_rules (user_name);
	29	CREATE TABLE user_filters( user_id TEXT, filter_id BIGINT, filter_json bytea );
	30	CREATE INDEX user_filters_by_user_id_filter_id ON user_filters( user_id, filter_id );
	31	CREATE TABLE push_rules_enable ( id BIGINT PRIMARY KEY, user_name TEXT NOT NULL, rule_id TEXT NOT NULL, enabled SMALLINT, UNIQUE(user_name, rule_id) );
	32	CREATE INDEX push_rules_enable_user_name on push_rules_enable (user_name);
	33	CREATE TABLE event_forward_extremities( event_id TEXT NOT NULL, room_id TEXT NOT NULL, UNIQUE (event_id, room_id) );
	34	CREATE INDEX ev_extrem_room ON event_forward_extremities(room_id);
	35	CREATE INDEX ev_extrem_id ON event_forward_extremities(event_id);
	36	CREATE TABLE event_backward_extremities( event_id TEXT NOT NULL, room_id TEXT NOT NULL, UNIQUE (event_id, room_id) );
	37	CREATE INDEX ev_b_extrem_room ON event_backward_extremities(room_id);
	38	CREATE INDEX ev_b_extrem_id ON event_backward_extremities(event_id);
	39	CREATE TABLE event_edges( event_id TEXT NOT NULL, prev_event_id TEXT NOT NULL, room_id TEXT NOT NULL, is_state BOOL NOT NULL, UNIQUE (event_id, prev_event_id, room_id, is_state) );
	40	CREATE INDEX ev_edges_id ON event_edges(event_id);
	41	CREATE INDEX ev_edges_prev_id ON event_edges(prev_event_id);
	42	CREATE TABLE room_depth( room_id TEXT NOT NULL, min_depth INTEGER NOT NULL, UNIQUE (room_id) );
	43	CREATE INDEX room_depth_room ON room_depth(room_id);
	44	CREATE TABLE state_groups( id BIGINT PRIMARY KEY, room_id TEXT NOT NULL, event_id TEXT NOT NULL );
	45	CREATE TABLE state_groups_state( state_group BIGINT NOT NULL, room_id TEXT NOT NULL, type TEXT NOT NULL, state_key TEXT NOT NULL, event_id TEXT NOT NULL );
	46	CREATE TABLE event_to_state_groups( event_id TEXT NOT NULL, state_group BIGINT NOT NULL, UNIQUE (event_id) );
	47	CREATE TABLE local_media_repository ( media_id TEXT, media_type TEXT, media_length INTEGER, created_ts BIGINT, upload_name TEXT, user_id TEXT, quarantined_by TEXT, url_cache TEXT, last_access_ts BIGINT, UNIQUE (media_id) );
	48	CREATE TABLE local_media_repository_thumbnails ( media_id TEXT, thumbnail_width INTEGER, thumbnail_height INTEGER, thumbnail_type TEXT, thumbnail_method TEXT, thumbnail_length INTEGER, UNIQUE ( media_id, thumbnail_width, thumbnail_height, thumbnail_type ) );
	49	CREATE INDEX local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails (media_id);
	50	CREATE TABLE remote_media_cache ( media_origin TEXT, media_id TEXT, media_type TEXT, created_ts BIGINT, upload_name TEXT, media_length INTEGER, filesystem_id TEXT, last_access_ts BIGINT, quarantined_by TEXT, UNIQUE (media_origin, media_id) );
	51	CREATE TABLE remote_media_cache_thumbnails ( media_origin TEXT, media_id TEXT, thumbnail_width INTEGER, thumbnail_height INTEGER, thumbnail_method TEXT, thumbnail_type TEXT, thumbnail_length INTEGER, filesystem_id TEXT, UNIQUE ( media_origin, media_id, thumbnail_width, thumbnail_height, thumbnail_type ) );
	52	CREATE TABLE redactions ( event_id TEXT NOT NULL, redacts TEXT NOT NULL, UNIQUE (event_id) );
	53	CREATE INDEX redactions_redacts ON redactions (redacts);
	54	CREATE TABLE room_aliases( room_alias TEXT NOT NULL, room_id TEXT NOT NULL, creator TEXT, UNIQUE (room_alias) );
	55	CREATE INDEX room_aliases_id ON room_aliases(room_id);
	56	CREATE TABLE room_alias_servers( room_alias TEXT NOT NULL, server TEXT NOT NULL );
	57	CREATE INDEX room_alias_servers_alias ON room_alias_servers(room_alias);
	58	CREATE TABLE event_reference_hashes ( event_id TEXT, algorithm TEXT, hash bytea, UNIQUE (event_id, algorithm) );
	59	CREATE INDEX event_reference_hashes_id ON event_reference_hashes(event_id);
	60	CREATE TABLE IF NOT EXISTS "server_keys_json" ( server_name TEXT NOT NULL, key_id TEXT NOT NULL, from_server TEXT NOT NULL, ts_added_ms BIGINT NOT NULL, ts_valid_until_ms BIGINT NOT NULL, key_json bytea NOT NULL, CONSTRAINT server_keys_json_uniqueness UNIQUE (server_name, key_id, from_server) );
	61	CREATE TABLE e2e_device_keys_json ( user_id TEXT NOT NULL, device_id TEXT NOT NULL, ts_added_ms BIGINT NOT NULL, key_json TEXT NOT NULL, CONSTRAINT e2e_device_keys_json_uniqueness UNIQUE (user_id, device_id) );
	62	CREATE TABLE e2e_one_time_keys_json ( user_id TEXT NOT NULL, device_id TEXT NOT NULL, algorithm TEXT NOT NULL, key_id TEXT NOT NULL, ts_added_ms BIGINT NOT NULL, key_json TEXT NOT NULL, CONSTRAINT e2e_one_time_keys_json_uniqueness UNIQUE (user_id, device_id, algorithm, key_id) );
	63	CREATE TABLE receipts_graph( room_id TEXT NOT NULL, receipt_type TEXT NOT NULL, user_id TEXT NOT NULL, event_ids TEXT NOT NULL, data TEXT NOT NULL, CONSTRAINT receipts_graph_uniqueness UNIQUE (room_id, receipt_type, user_id) );
	64	CREATE TABLE receipts_linearized ( stream_id BIGINT NOT NULL, room_id TEXT NOT NULL, receipt_type TEXT NOT NULL, user_id TEXT NOT NULL, event_id TEXT NOT NULL, data TEXT NOT NULL, CONSTRAINT receipts_linearized_uniqueness UNIQUE (room_id, receipt_type, user_id) );
	65	CREATE INDEX receipts_linearized_id ON receipts_linearized( stream_id );
	66	CREATE INDEX receipts_linearized_room_stream ON receipts_linearized( room_id, stream_id );
	67	CREATE TABLE IF NOT EXISTS "user_threepids" ( user_id TEXT NOT NULL, medium TEXT NOT NULL, address TEXT NOT NULL, validated_at BIGINT NOT NULL, added_at BIGINT NOT NULL, CONSTRAINT medium_address UNIQUE (medium, address) );
	68	CREATE INDEX user_threepids_user_id ON user_threepids(user_id);
	69	CREATE VIRTUAL TABLE event_search USING fts4 ( event_id, room_id, sender, key, value )
	70	/* event_search(event_id,room_id,sender,"key",value) */;
	71	CREATE TABLE IF NOT EXISTS 'event_search_content'(docid INTEGER PRIMARY KEY, 'c0event_id', 'c1room_id', 'c2sender', 'c3key', 'c4value');
	72	CREATE TABLE IF NOT EXISTS 'event_search_segments'(blockid INTEGER PRIMARY KEY, block BLOB);
	73	CREATE TABLE IF NOT EXISTS 'event_search_segdir'(level INTEGER,idx INTEGER,start_block INTEGER,leaves_end_block INTEGER,end_block INTEGER,root BLOB,PRIMARY KEY(level, idx));
	74	CREATE TABLE IF NOT EXISTS 'event_search_docsize'(docid INTEGER PRIMARY KEY, size BLOB);
	75	CREATE TABLE IF NOT EXISTS 'event_search_stat'(id INTEGER PRIMARY KEY, value BLOB);
	76	CREATE TABLE guest_access( event_id TEXT NOT NULL, room_id TEXT NOT NULL, guest_access TEXT NOT NULL, UNIQUE (event_id) );
	77	CREATE TABLE history_visibility( event_id TEXT NOT NULL, room_id TEXT NOT NULL, history_visibility TEXT NOT NULL, UNIQUE (event_id) );
	78	CREATE TABLE room_tags( user_id TEXT NOT NULL, room_id TEXT NOT NULL, tag TEXT NOT NULL, content TEXT NOT NULL, CONSTRAINT room_tag_uniqueness UNIQUE (user_id, room_id, tag) );
	79	CREATE TABLE room_tags_revisions ( user_id TEXT NOT NULL, room_id TEXT NOT NULL, stream_id BIGINT NOT NULL, CONSTRAINT room_tag_revisions_uniqueness UNIQUE (user_id, room_id) );
	80	CREATE TABLE IF NOT EXISTS "account_data_max_stream_id"( Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, stream_id BIGINT NOT NULL, CHECK (Lock='X') );
	81	CREATE TABLE account_data( user_id TEXT NOT NULL, account_data_type TEXT NOT NULL, stream_id BIGINT NOT NULL, content TEXT NOT NULL, CONSTRAINT account_data_uniqueness UNIQUE (user_id, account_data_type) );
	82	CREATE TABLE room_account_data( user_id TEXT NOT NULL, room_id TEXT NOT NULL, account_data_type TEXT NOT NULL, stream_id BIGINT NOT NULL, content TEXT NOT NULL, CONSTRAINT room_account_data_uniqueness UNIQUE (user_id, room_id, account_data_type) );
	83	CREATE INDEX account_data_stream_id on account_data(user_id, stream_id);
	84	CREATE INDEX room_account_data_stream_id on room_account_data(user_id, stream_id);
	85	CREATE INDEX events_ts ON events(origin_server_ts, stream_ordering);
	86	CREATE TABLE event_push_actions( room_id TEXT NOT NULL, event_id TEXT NOT NULL, user_id TEXT NOT NULL, profile_tag VARCHAR(32), actions TEXT NOT NULL, topological_ordering BIGINT, stream_ordering BIGINT, notif SMALLINT, highlight SMALLINT, CONSTRAINT event_id_user_id_profile_tag_uniqueness UNIQUE (room_id, event_id, user_id, profile_tag) );
	87	CREATE INDEX event_push_actions_room_id_user_id on event_push_actions(room_id, user_id);
	88	CREATE INDEX events_room_stream on events(room_id, stream_ordering);
	89	CREATE INDEX public_room_index on rooms(is_public);
	90	CREATE INDEX receipts_linearized_user ON receipts_linearized( user_id );
	91	CREATE INDEX event_push_actions_rm_tokens on event_push_actions( user_id, room_id, topological_ordering, stream_ordering );
	92	CREATE TABLE presence_stream( stream_id BIGINT, user_id TEXT, state TEXT, last_active_ts BIGINT, last_federation_update_ts BIGINT, last_user_sync_ts BIGINT, status_msg TEXT, currently_active BOOLEAN );
	93	CREATE INDEX presence_stream_id ON presence_stream(stream_id, user_id);
	94	CREATE INDEX presence_stream_user_id ON presence_stream(user_id);
	95	CREATE TABLE push_rules_stream( stream_id BIGINT NOT NULL, event_stream_ordering BIGINT NOT NULL, user_id TEXT NOT NULL, rule_id TEXT NOT NULL, op TEXT NOT NULL, priority_class SMALLINT, priority INTEGER, conditions TEXT, actions TEXT );
	96	CREATE INDEX push_rules_stream_id ON push_rules_stream(stream_id);
	97	CREATE INDEX push_rules_stream_user_stream_id on push_rules_stream(user_id, stream_id);
	98	CREATE TABLE ex_outlier_stream( event_stream_ordering BIGINT PRIMARY KEY NOT NULL, event_id TEXT NOT NULL, state_group BIGINT NOT NULL );
	99	CREATE TABLE threepid_guest_access_tokens( medium TEXT, address TEXT, guest_access_token TEXT, first_inviter TEXT );
	100	CREATE UNIQUE INDEX threepid_guest_access_tokens_index ON threepid_guest_access_tokens(medium, address);
	101	CREATE TABLE local_invites( stream_id BIGINT NOT NULL, inviter TEXT NOT NULL, invitee TEXT NOT NULL, event_id TEXT NOT NULL, room_id TEXT NOT NULL, locally_rejected TEXT, replaced_by TEXT );
	102	CREATE INDEX local_invites_id ON local_invites(stream_id);
	103	CREATE INDEX local_invites_for_user_idx ON local_invites(invitee, locally_rejected, replaced_by, room_id);
	104	CREATE INDEX event_push_actions_stream_ordering on event_push_actions( stream_ordering, user_id );
	105	CREATE TABLE open_id_tokens ( token TEXT NOT NULL PRIMARY KEY, ts_valid_until_ms bigint NOT NULL, user_id TEXT NOT NULL, UNIQUE (token) );
	106	CREATE INDEX open_id_tokens_ts_valid_until_ms ON open_id_tokens(ts_valid_until_ms);
	107	CREATE TABLE pusher_throttle( pusher BIGINT NOT NULL, room_id TEXT NOT NULL, last_sent_ts BIGINT, throttle_ms BIGINT, PRIMARY KEY (pusher, room_id) );
	108	CREATE TABLE event_reports( id BIGINT NOT NULL PRIMARY KEY, received_ts BIGINT NOT NULL, room_id TEXT NOT NULL, event_id TEXT NOT NULL, user_id TEXT NOT NULL, reason TEXT, content TEXT );
	109	CREATE TABLE devices ( user_id TEXT NOT NULL, device_id TEXT NOT NULL, display_name TEXT, CONSTRAINT device_uniqueness UNIQUE (user_id, device_id) );
	110	CREATE TABLE appservice_stream_position( Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, stream_ordering BIGINT, CHECK (Lock='X') );
	111	CREATE TABLE device_inbox ( user_id TEXT NOT NULL, device_id TEXT NOT NULL, stream_id BIGINT NOT NULL, message_json TEXT NOT NULL );
	112	CREATE INDEX device_inbox_user_stream_id ON device_inbox(user_id, device_id, stream_id);
	113	CREATE INDEX received_transactions_ts ON received_transactions(ts);
	114	CREATE TABLE device_federation_outbox ( destination TEXT NOT NULL, stream_id BIGINT NOT NULL, queued_ts BIGINT NOT NULL, messages_json TEXT NOT NULL );
	115	CREATE INDEX device_federation_outbox_destination_id ON device_federation_outbox(destination, stream_id);
	116	CREATE TABLE device_federation_inbox ( origin TEXT NOT NULL, message_id TEXT NOT NULL, received_ts BIGINT NOT NULL );
	117	CREATE INDEX device_federation_inbox_sender_id ON device_federation_inbox(origin, message_id);
	118	CREATE TABLE device_max_stream_id ( stream_id BIGINT NOT NULL );
	119	CREATE TABLE public_room_list_stream ( stream_id BIGINT NOT NULL, room_id TEXT NOT NULL, visibility BOOLEAN NOT NULL , appservice_id TEXT, network_id TEXT);
	120	CREATE INDEX public_room_list_stream_idx on public_room_list_stream( stream_id );
	121	CREATE INDEX public_room_list_stream_rm_idx on public_room_list_stream( room_id, stream_id );
	122	CREATE TABLE state_group_edges( state_group BIGINT NOT NULL, prev_state_group BIGINT NOT NULL );
	123	CREATE INDEX state_group_edges_idx ON state_group_edges(state_group);
	124	CREATE INDEX state_group_edges_prev_idx ON state_group_edges(prev_state_group);
	125	CREATE TABLE stream_ordering_to_exterm ( stream_ordering BIGINT NOT NULL, room_id TEXT NOT NULL, event_id TEXT NOT NULL );
	126	CREATE INDEX stream_ordering_to_exterm_idx on stream_ordering_to_exterm( stream_ordering );
	127	CREATE INDEX stream_ordering_to_exterm_rm_idx on stream_ordering_to_exterm( room_id, stream_ordering );
	128	CREATE TABLE IF NOT EXISTS "event_auth"( event_id TEXT NOT NULL, auth_id TEXT NOT NULL, room_id TEXT NOT NULL );
	129	CREATE INDEX evauth_edges_id ON event_auth(event_id);
	130	CREATE INDEX user_threepids_medium_address on user_threepids (medium, address);
	131	CREATE TABLE appservice_room_list( appservice_id TEXT NOT NULL, network_id TEXT NOT NULL, room_id TEXT NOT NULL );
	132	CREATE UNIQUE INDEX appservice_room_list_idx ON appservice_room_list( appservice_id, network_id, room_id );
	133	CREATE INDEX device_federation_outbox_id ON device_federation_outbox(stream_id);
	134	CREATE TABLE federation_stream_position( type TEXT NOT NULL, stream_id INTEGER NOT NULL );
	135	CREATE TABLE device_lists_remote_cache ( user_id TEXT NOT NULL, device_id TEXT NOT NULL, content TEXT NOT NULL );
	136	CREATE TABLE device_lists_remote_extremeties ( user_id TEXT NOT NULL, stream_id TEXT NOT NULL );
	137	CREATE TABLE device_lists_stream ( stream_id BIGINT NOT NULL, user_id TEXT NOT NULL, device_id TEXT NOT NULL );
	138	CREATE INDEX device_lists_stream_id ON device_lists_stream(stream_id, user_id);
	139	CREATE TABLE device_lists_outbound_pokes ( destination TEXT NOT NULL, stream_id BIGINT NOT NULL, user_id TEXT NOT NULL, device_id TEXT NOT NULL, sent BOOLEAN NOT NULL, ts BIGINT NOT NULL );
	140	CREATE INDEX device_lists_outbound_pokes_id ON device_lists_outbound_pokes(destination, stream_id);
	141	CREATE INDEX device_lists_outbound_pokes_user ON device_lists_outbound_pokes(destination, user_id);
	142	CREATE TABLE event_push_summary ( user_id TEXT NOT NULL, room_id TEXT NOT NULL, notif_count BIGINT NOT NULL, stream_ordering BIGINT NOT NULL );
	143	CREATE INDEX event_push_summary_user_rm ON event_push_summary(user_id, room_id);
	144	CREATE TABLE event_push_summary_stream_ordering ( Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, stream_ordering BIGINT NOT NULL, CHECK (Lock='X') );
	145	CREATE TABLE IF NOT EXISTS "pushers" ( id BIGINT PRIMARY KEY, user_name TEXT NOT NULL, access_token BIGINT DEFAULT NULL, profile_tag TEXT NOT NULL, kind TEXT NOT NULL, app_id TEXT NOT NULL, app_display_name TEXT NOT NULL, device_display_name TEXT NOT NULL, pushkey TEXT NOT NULL, ts BIGINT NOT NULL, lang TEXT, data TEXT, last_stream_ordering INTEGER, last_success BIGINT, failing_since BIGINT, UNIQUE (app_id, pushkey, user_name) );
	146	CREATE INDEX device_lists_outbound_pokes_stream ON device_lists_outbound_pokes(stream_id);
	147	CREATE TABLE ratelimit_override ( user_id TEXT NOT NULL, messages_per_second BIGINT, burst_count BIGINT );
	148	CREATE UNIQUE INDEX ratelimit_override_idx ON ratelimit_override(user_id);
	149	CREATE TABLE current_state_delta_stream ( stream_id BIGINT NOT NULL, room_id TEXT NOT NULL, type TEXT NOT NULL, state_key TEXT NOT NULL, event_id TEXT, prev_event_id TEXT );
	150	CREATE INDEX current_state_delta_stream_idx ON current_state_delta_stream(stream_id);
	151	CREATE TABLE device_lists_outbound_last_success ( destination TEXT NOT NULL, user_id TEXT NOT NULL, stream_id BIGINT NOT NULL );
	152	CREATE INDEX device_lists_outbound_last_success_idx ON device_lists_outbound_last_success( destination, user_id, stream_id );
	153	CREATE TABLE user_directory_stream_pos ( Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, stream_id BIGINT, CHECK (Lock='X') );
	154	CREATE VIRTUAL TABLE user_directory_search USING fts4 ( user_id, value )
	155	/* user_directory_search(user_id,value) */;
	156	CREATE TABLE IF NOT EXISTS 'user_directory_search_content'(docid INTEGER PRIMARY KEY, 'c0user_id', 'c1value');
	157	CREATE TABLE IF NOT EXISTS 'user_directory_search_segments'(blockid INTEGER PRIMARY KEY, block BLOB);
	158	CREATE TABLE IF NOT EXISTS 'user_directory_search_segdir'(level INTEGER,idx INTEGER,start_block INTEGER,leaves_end_block INTEGER,end_block INTEGER,root BLOB,PRIMARY KEY(level, idx));
	159	CREATE TABLE IF NOT EXISTS 'user_directory_search_docsize'(docid INTEGER PRIMARY KEY, size BLOB);
	160	CREATE TABLE IF NOT EXISTS 'user_directory_search_stat'(id INTEGER PRIMARY KEY, value BLOB);
	161	CREATE TABLE blocked_rooms ( room_id TEXT NOT NULL, user_id TEXT NOT NULL );
	162	CREATE UNIQUE INDEX blocked_rooms_idx ON blocked_rooms(room_id);
	163	CREATE TABLE IF NOT EXISTS "local_media_repository_url_cache"( url TEXT, response_code INTEGER, etag TEXT, expires_ts BIGINT, og TEXT, media_id TEXT, download_ts BIGINT );
	164	CREATE INDEX local_media_repository_url_cache_expires_idx ON local_media_repository_url_cache(expires_ts);
	165	CREATE INDEX local_media_repository_url_cache_by_url_download_ts ON local_media_repository_url_cache(url, download_ts);
	166	CREATE INDEX local_media_repository_url_cache_media_idx ON local_media_repository_url_cache(media_id);
	167	CREATE TABLE group_users ( group_id TEXT NOT NULL, user_id TEXT NOT NULL, is_admin BOOLEAN NOT NULL, is_public BOOLEAN NOT NULL );
	168	CREATE TABLE group_invites ( group_id TEXT NOT NULL, user_id TEXT NOT NULL );
	169	CREATE TABLE group_rooms ( group_id TEXT NOT NULL, room_id TEXT NOT NULL, is_public BOOLEAN NOT NULL );
	170	CREATE TABLE group_summary_rooms ( group_id TEXT NOT NULL, room_id TEXT NOT NULL, category_id TEXT NOT NULL, room_order BIGINT NOT NULL, is_public BOOLEAN NOT NULL, UNIQUE (group_id, category_id, room_id, room_order), CHECK (room_order > 0) );
	171	CREATE UNIQUE INDEX group_summary_rooms_g_idx ON group_summary_rooms(group_id, room_id, category_id);
	172	CREATE TABLE group_summary_room_categories ( group_id TEXT NOT NULL, category_id TEXT NOT NULL, cat_order BIGINT NOT NULL, UNIQUE (group_id, category_id, cat_order), CHECK (cat_order > 0) );
	173	CREATE TABLE group_room_categories ( group_id TEXT NOT NULL, category_id TEXT NOT NULL, profile TEXT NOT NULL, is_public BOOLEAN NOT NULL, UNIQUE (group_id, category_id) );
	174	CREATE TABLE group_summary_users ( group_id TEXT NOT NULL, user_id TEXT NOT NULL, role_id TEXT NOT NULL, user_order BIGINT NOT NULL, is_public BOOLEAN NOT NULL );
	175	CREATE INDEX group_summary_users_g_idx ON group_summary_users(group_id);
	176	CREATE TABLE group_summary_roles ( group_id TEXT NOT NULL, role_id TEXT NOT NULL, role_order BIGINT NOT NULL, UNIQUE (group_id, role_id, role_order), CHECK (role_order > 0) );
	177	CREATE TABLE group_roles ( group_id TEXT NOT NULL, role_id TEXT NOT NULL, profile TEXT NOT NULL, is_public BOOLEAN NOT NULL, UNIQUE (group_id, role_id) );
	178	CREATE TABLE group_attestations_renewals ( group_id TEXT NOT NULL, user_id TEXT NOT NULL, valid_until_ms BIGINT NOT NULL );
	179	CREATE INDEX group_attestations_renewals_g_idx ON group_attestations_renewals(group_id, user_id);
	180	CREATE INDEX group_attestations_renewals_u_idx ON group_attestations_renewals(user_id);
	181	CREATE INDEX group_attestations_renewals_v_idx ON group_attestations_renewals(valid_until_ms);
	182	CREATE TABLE group_attestations_remote ( group_id TEXT NOT NULL, user_id TEXT NOT NULL, valid_until_ms BIGINT NOT NULL, attestation_json TEXT NOT NULL );
	183	CREATE INDEX group_attestations_remote_g_idx ON group_attestations_remote(group_id, user_id);
	184	CREATE INDEX group_attestations_remote_u_idx ON group_attestations_remote(user_id);
	185	CREATE INDEX group_attestations_remote_v_idx ON group_attestations_remote(valid_until_ms);
	186	CREATE TABLE local_group_membership ( group_id TEXT NOT NULL, user_id TEXT NOT NULL, is_admin BOOLEAN NOT NULL, membership TEXT NOT NULL, is_publicised BOOLEAN NOT NULL, content TEXT NOT NULL );
	187	CREATE INDEX local_group_membership_u_idx ON local_group_membership(user_id, group_id);
	188	CREATE INDEX local_group_membership_g_idx ON local_group_membership(group_id);
	189	CREATE TABLE local_group_updates ( stream_id BIGINT NOT NULL, group_id TEXT NOT NULL, user_id TEXT NOT NULL, type TEXT NOT NULL, content TEXT NOT NULL );
	190	CREATE TABLE remote_profile_cache ( user_id TEXT NOT NULL, displayname TEXT, avatar_url TEXT, last_check BIGINT NOT NULL );
	191	CREATE UNIQUE INDEX remote_profile_cache_user_id ON remote_profile_cache(user_id);
	192	CREATE INDEX remote_profile_cache_time ON remote_profile_cache(last_check);
	193	CREATE TABLE IF NOT EXISTS "deleted_pushers" ( stream_id BIGINT NOT NULL, app_id TEXT NOT NULL, pushkey TEXT NOT NULL, user_id TEXT NOT NULL );
	194	CREATE INDEX deleted_pushers_stream_id ON deleted_pushers (stream_id);
	195	CREATE TABLE IF NOT EXISTS "groups" ( group_id TEXT NOT NULL, name TEXT, avatar_url TEXT, short_description TEXT, long_description TEXT, is_public BOOL NOT NULL , join_policy TEXT NOT NULL DEFAULT 'invite');
	196	CREATE UNIQUE INDEX groups_idx ON groups(group_id);
	197	CREATE TABLE IF NOT EXISTS "user_directory" ( user_id TEXT NOT NULL, room_id TEXT, display_name TEXT, avatar_url TEXT );
	198	CREATE INDEX user_directory_room_idx ON user_directory(room_id);
	199	CREATE UNIQUE INDEX user_directory_user_idx ON user_directory(user_id);
	200	CREATE TABLE event_push_actions_staging ( event_id TEXT NOT NULL, user_id TEXT NOT NULL, actions TEXT NOT NULL, notif SMALLINT NOT NULL, highlight SMALLINT NOT NULL );
	201	CREATE INDEX event_push_actions_staging_id ON event_push_actions_staging(event_id);
	202	CREATE TABLE users_pending_deactivation ( user_id TEXT NOT NULL );
	203	CREATE UNIQUE INDEX group_invites_g_idx ON group_invites(group_id, user_id);
	204	CREATE UNIQUE INDEX group_users_g_idx ON group_users(group_id, user_id);
	205	CREATE INDEX group_users_u_idx ON group_users(user_id);
	206	CREATE INDEX group_invites_u_idx ON group_invites(user_id);
	207	CREATE UNIQUE INDEX group_rooms_g_idx ON group_rooms(group_id, room_id);
	208	CREATE INDEX group_rooms_r_idx ON group_rooms(room_id);
	209	CREATE TABLE user_daily_visits ( user_id TEXT NOT NULL, device_id TEXT, timestamp BIGINT NOT NULL );
	210	CREATE INDEX user_daily_visits_uts_idx ON user_daily_visits(user_id, timestamp);
	211	CREATE INDEX user_daily_visits_ts_idx ON user_daily_visits(timestamp);
	212	CREATE TABLE erased_users ( user_id TEXT NOT NULL );
	213	CREATE UNIQUE INDEX erased_users_user ON erased_users(user_id);
	214	CREATE TABLE monthly_active_users ( user_id TEXT NOT NULL, timestamp BIGINT NOT NULL );
	215	CREATE UNIQUE INDEX monthly_active_users_users ON monthly_active_users(user_id);
	216	CREATE INDEX monthly_active_users_time_stamp ON monthly_active_users(timestamp);
	217	CREATE TABLE IF NOT EXISTS "e2e_room_keys_versions" ( user_id TEXT NOT NULL, version BIGINT NOT NULL, algorithm TEXT NOT NULL, auth_data TEXT NOT NULL, deleted SMALLINT DEFAULT 0 NOT NULL );
	218	CREATE UNIQUE INDEX e2e_room_keys_versions_idx ON e2e_room_keys_versions(user_id, version);
	219	CREATE TABLE IF NOT EXISTS "e2e_room_keys" ( user_id TEXT NOT NULL, room_id TEXT NOT NULL, session_id TEXT NOT NULL, version BIGINT NOT NULL, first_message_index INT, forwarded_count INT, is_verified BOOLEAN, session_data TEXT NOT NULL );
	220	CREATE UNIQUE INDEX e2e_room_keys_idx ON e2e_room_keys(user_id, room_id, session_id);
	221	CREATE TABLE users_who_share_private_rooms ( user_id TEXT NOT NULL, other_user_id TEXT NOT NULL, room_id TEXT NOT NULL );
	222	CREATE UNIQUE INDEX users_who_share_private_rooms_u_idx ON users_who_share_private_rooms(user_id, other_user_id, room_id);
	223	CREATE INDEX users_who_share_private_rooms_r_idx ON users_who_share_private_rooms(room_id);
	224	CREATE INDEX users_who_share_private_rooms_o_idx ON users_who_share_private_rooms(other_user_id);
	225	CREATE TABLE user_threepid_id_server ( user_id TEXT NOT NULL, medium TEXT NOT NULL, address TEXT NOT NULL, id_server TEXT NOT NULL );
	226	CREATE UNIQUE INDEX user_threepid_id_server_idx ON user_threepid_id_server( user_id, medium, address, id_server );
	227	CREATE TABLE users_in_public_rooms ( user_id TEXT NOT NULL, room_id TEXT NOT NULL );
	228	CREATE UNIQUE INDEX users_in_public_rooms_u_idx ON users_in_public_rooms(user_id, room_id);
	229	CREATE TABLE account_validity ( user_id TEXT PRIMARY KEY, expiration_ts_ms BIGINT NOT NULL, email_sent BOOLEAN NOT NULL, renewal_token TEXT );
	230	CREATE TABLE event_relations ( event_id TEXT NOT NULL, relates_to_id TEXT NOT NULL, relation_type TEXT NOT NULL, aggregation_key TEXT );
	231	CREATE UNIQUE INDEX event_relations_id ON event_relations(event_id);
	232	CREATE INDEX event_relations_relates ON event_relations(relates_to_id, relation_type, aggregation_key);
	233	CREATE TABLE stats_stream_pos ( Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, stream_id BIGINT, CHECK (Lock='X') );
	234	CREATE TABLE user_stats ( user_id TEXT NOT NULL, ts BIGINT NOT NULL, bucket_size INT NOT NULL, public_rooms INT NOT NULL, private_rooms INT NOT NULL );
	235	CREATE UNIQUE INDEX user_stats_user_ts ON user_stats(user_id, ts);
	236	CREATE TABLE room_stats ( room_id TEXT NOT NULL, ts BIGINT NOT NULL, bucket_size INT NOT NULL, current_state_events INT NOT NULL, joined_members INT NOT NULL, invited_members INT NOT NULL, left_members INT NOT NULL, banned_members INT NOT NULL, state_events INT NOT NULL );
	237	CREATE UNIQUE INDEX room_stats_room_ts ON room_stats(room_id, ts);
	238	CREATE TABLE room_state ( room_id TEXT NOT NULL, join_rules TEXT, history_visibility TEXT, encryption TEXT, name TEXT, topic TEXT, avatar TEXT, canonical_alias TEXT );
	239	CREATE UNIQUE INDEX room_state_room ON room_state(room_id);
	240	CREATE TABLE room_stats_earliest_token ( room_id TEXT NOT NULL, token BIGINT NOT NULL );
	241	CREATE UNIQUE INDEX room_stats_earliest_token_idx ON room_stats_earliest_token(room_id);
	242	CREATE INDEX access_tokens_device_id ON access_tokens (user_id, device_id);
	243	CREATE INDEX user_ips_device_id ON user_ips (user_id, device_id, last_seen);
	244	CREATE INDEX event_contains_url_index ON events (room_id, topological_ordering, stream_ordering);
	245	CREATE INDEX event_push_actions_u_highlight ON event_push_actions (user_id, stream_ordering);
	246	CREATE INDEX event_push_actions_highlights_index ON event_push_actions (user_id, room_id, topological_ordering, stream_ordering);
	247	CREATE INDEX current_state_events_member_index ON current_state_events (state_key);
	248	CREATE INDEX device_inbox_stream_id_user_id ON device_inbox (stream_id, user_id);
	249	CREATE INDEX device_lists_stream_user_id ON device_lists_stream (user_id, device_id);
	250	CREATE INDEX local_media_repository_url_idx ON local_media_repository (created_ts);
	251	CREATE INDEX user_ips_last_seen ON user_ips (user_id, last_seen);
	252	CREATE INDEX user_ips_last_seen_only ON user_ips (last_seen);
	253	CREATE INDEX users_creation_ts ON users (creation_ts);
	254	CREATE INDEX event_to_state_groups_sg_index ON event_to_state_groups (state_group);
	255	CREATE UNIQUE INDEX device_lists_remote_cache_unique_id ON device_lists_remote_cache (user_id, device_id);
	256	CREATE INDEX state_groups_state_type_idx ON state_groups_state(state_group, type, state_key);
	257	CREATE UNIQUE INDEX device_lists_remote_extremeties_unique_idx ON device_lists_remote_extremeties (user_id);
	258	CREATE UNIQUE INDEX user_ips_user_token_ip_unique_index ON user_ips (user_id, access_token, ip);

+7

-0

synapse/storage/data_stores/main/schema/full_schemas/54/stream_positions.sql less more

	0
	1	INSERT INTO appservice_stream_position (stream_ordering) SELECT COALESCE(MAX(stream_ordering), 0) FROM events;
	2	INSERT INTO federation_stream_position (type, stream_id) VALUES ('federation', -1);
	3	INSERT INTO federation_stream_position (type, stream_id) SELECT 'events', coalesce(max(stream_ordering), -1) FROM events;
	4	INSERT INTO user_directory_stream_pos (stream_id) VALUES (0);
	5	INSERT INTO stats_stream_pos (stream_id) VALUES (0);
	6	INSERT INTO event_push_summary_stream_ordering (stream_ordering) VALUES (0);

+19

-0

synapse/storage/data_stores/main/schema/full_schemas/README.txt less more

	0	Building full schema dumps
	1	==========================
	2
	3	These schemas need to be made from a database that has had all background updates run.
	4
	5	Postgres
	6	--------
	7
	8	$ pg_dump --format=plain --schema-only --no-tablespaces --no-acl --no-owner $DATABASE_NAME\| sed -e '/^--/d' -e 's/public\.//g' -e '/^SET /d' -e '/^SELECT /d' > full.sql.postgres
	9
	10	SQLite
	11	------
	12
	13	$ sqlite3 $DATABASE_FILE ".schema" > full.sql.sqlite
	14
	15	After
	16	-----
	17
	18	Delete the CREATE statements for "sqlite_stat1", "schema_version", "applied_schema_deltas", and "applied_module_schemas".⏎

+712

-0

synapse/storage/data_stores/main/search.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2015, 2016 OpenMarket Ltd
	2	#
	3	# Licensed under the Apache License, Version 2.0 (the "License");
	4	# you may not use this file except in compliance with the License.
	5	# You may obtain a copy of the License at
	6	#
	7	# http://www.apache.org/licenses/LICENSE-2.0
	8	#
	9	# Unless required by applicable law or agreed to in writing, software
	10	# distributed under the License is distributed on an "AS IS" BASIS,
	11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	# See the License for the specific language governing permissions and
	13	# limitations under the License.
	14
	15	import logging
	16	import re
	17	from collections import namedtuple
	18
	19	from six import string_types
	20
	21	from canonicaljson import json
	22
	23	from twisted.internet import defer
	24
	25	from synapse.api.errors import SynapseError
	26	from synapse.storage._base import make_in_list_sql_clause
	27	from synapse.storage.background_updates import BackgroundUpdateStore
	28	from synapse.storage.engines import PostgresEngine, Sqlite3Engine
	29
	30	logger = logging.getLogger(__name__)
	31
	32	SearchEntry = namedtuple(
	33	"SearchEntry",
	34	["key", "value", "event_id", "room_id", "stream_ordering", "origin_server_ts"],
	35	)
	36
	37
	38	class SearchBackgroundUpdateStore(BackgroundUpdateStore):
	39
	40	EVENT_SEARCH_UPDATE_NAME = "event_search"
	41	EVENT_SEARCH_ORDER_UPDATE_NAME = "event_search_order"
	42	EVENT_SEARCH_USE_GIST_POSTGRES_NAME = "event_search_postgres_gist"
	43	EVENT_SEARCH_USE_GIN_POSTGRES_NAME = "event_search_postgres_gin"
	44
	45	def __init__(self, db_conn, hs):
	46	super(SearchBackgroundUpdateStore, self).__init__(db_conn, hs)
	47
	48	if not hs.config.enable_search:
	49	return
	50
	51	self.register_background_update_handler(
	52	self.EVENT_SEARCH_UPDATE_NAME, self._background_reindex_search
	53	)
	54	self.register_background_update_handler(
	55	self.EVENT_SEARCH_ORDER_UPDATE_NAME, self._background_reindex_search_order
	56	)
	57
	58	# we used to have a background update to turn the GIN index into a
	59	# GIST one; we no longer do that (obviously) because we actually want
	60	# a GIN index. However, it's possible that some people might still have
	61	# the background update queued, so we register a handler to clear the
	62	# background update.
	63	self.register_noop_background_update(self.EVENT_SEARCH_USE_GIST_POSTGRES_NAME)
	64
	65	self.register_background_update_handler(
	66	self.EVENT_SEARCH_USE_GIN_POSTGRES_NAME, self._background_reindex_gin_search
	67	)
	68
	69	@defer.inlineCallbacks
	70	def _background_reindex_search(self, progress, batch_size):
	71	# we work through the events table from highest stream id to lowest
	72	target_min_stream_id = progress["target_min_stream_id_inclusive"]
	73	max_stream_id = progress["max_stream_id_exclusive"]
	74	rows_inserted = progress.get("rows_inserted", 0)
	75
	76	TYPES = ["m.room.name", "m.room.message", "m.room.topic"]
	77
	78	def reindex_search_txn(txn):
	79	sql = (
	80	"SELECT stream_ordering, event_id, room_id, type, json, "
	81	" origin_server_ts FROM events"
	82	" JOIN event_json USING (room_id, event_id)"
	83	" WHERE ? <= stream_ordering AND stream_ordering < ?"
	84	" AND (%s)"
	85	" ORDER BY stream_ordering DESC"
	86	" LIMIT ?"
	87	) % (" OR ".join("type = '%s'" % (t,) for t in TYPES),)
	88
	89	txn.execute(sql, (target_min_stream_id, max_stream_id, batch_size))
	90
	91	# we could stream straight from the results into
	92	# store_search_entries_txn with a generator function, but that
	93	# would mean having two cursors open on the database at once.
	94	# Instead we just build a list of results.
	95	rows = self.cursor_to_dict(txn)
	96	if not rows:
	97	return 0
	98
	99	min_stream_id = rows[-1]["stream_ordering"]
	100
	101	event_search_rows = []
	102	for row in rows:
	103	try:
	104	event_id = row["event_id"]
	105	room_id = row["room_id"]
	106	etype = row["type"]
	107	stream_ordering = row["stream_ordering"]
	108	origin_server_ts = row["origin_server_ts"]
	109	try:
	110	event_json = json.loads(row["json"])
	111	content = event_json["content"]
	112	except Exception:
	113	continue
	114
	115	if etype == "m.room.message":
	116	key = "content.body"
	117	value = content["body"]
	118	elif etype == "m.room.topic":
	119	key = "content.topic"
	120	value = content["topic"]
	121	elif etype == "m.room.name":
	122	key = "content.name"
	123	value = content["name"]
	124	else:
	125	raise Exception("unexpected event type %s" % etype)
	126	except (KeyError, AttributeError):
	127	# If the event is missing a necessary field then
	128	# skip over it.
	129	continue
	130
	131	if not isinstance(value, string_types):
	132	# If the event body, name or topic isn't a string
	133	# then skip over it
	134	continue
	135
	136	event_search_rows.append(
	137	SearchEntry(
	138	key=key,
	139	value=value,
	140	event_id=event_id,
	141	room_id=room_id,
	142	stream_ordering=stream_ordering,
	143	origin_server_ts=origin_server_ts,
	144	)
	145	)
	146
	147	self.store_search_entries_txn(txn, event_search_rows)
	148
	149	progress = {
	150	"target_min_stream_id_inclusive": target_min_stream_id,
	151	"max_stream_id_exclusive": min_stream_id,
	152	"rows_inserted": rows_inserted + len(event_search_rows),
	153	}
	154
	155	self._background_update_progress_txn(
	156	txn, self.EVENT_SEARCH_UPDATE_NAME, progress
	157	)
	158
	159	return len(event_search_rows)
	160
	161	result = yield self.runInteraction(
	162	self.EVENT_SEARCH_UPDATE_NAME, reindex_search_txn
	163	)
	164
	165	if not result:
	166	yield self._end_background_update(self.EVENT_SEARCH_UPDATE_NAME)
	167
	168	return result
	169
	170	@defer.inlineCallbacks
	171	def _background_reindex_gin_search(self, progress, batch_size):
	172	"""This handles old synapses which used GIST indexes, if any;
	173	converting them back to be GIN as per the actual schema.
	174	"""
	175
	176	def create_index(conn):
	177	conn.rollback()
	178
	179	# we have to set autocommit, because postgres refuses to
	180	# CREATE INDEX CONCURRENTLY without it.
	181	conn.set_session(autocommit=True)
	182
	183	try:
	184	c = conn.cursor()
	185
	186	# if we skipped the conversion to GIST, we may already/still
	187	# have an event_search_fts_idx; unfortunately postgres 9.4
	188	# doesn't support CREATE INDEX IF EXISTS so we just catch the
	189	# exception and ignore it.
	190	import psycopg2
	191
	192	try:
	193	c.execute(
	194	"CREATE INDEX CONCURRENTLY event_search_fts_idx"
	195	" ON event_search USING GIN (vector)"
	196	)
	197	except psycopg2.ProgrammingError as e:
	198	logger.warn(
	199	"Ignoring error %r when trying to switch from GIST to GIN", e
	200	)
	201
	202	# we should now be able to delete the GIST index.
	203	c.execute("DROP INDEX IF EXISTS event_search_fts_idx_gist")
	204	finally:
	205	conn.set_session(autocommit=False)
	206
	207	if isinstance(self.database_engine, PostgresEngine):
	208	yield self.runWithConnection(create_index)
	209
	210	yield self._end_background_update(self.EVENT_SEARCH_USE_GIN_POSTGRES_NAME)
	211	return 1
	212
	213	@defer.inlineCallbacks
	214	def _background_reindex_search_order(self, progress, batch_size):
	215	target_min_stream_id = progress["target_min_stream_id_inclusive"]
	216	max_stream_id = progress["max_stream_id_exclusive"]
	217	rows_inserted = progress.get("rows_inserted", 0)
	218	have_added_index = progress["have_added_indexes"]
	219
	220	if not have_added_index:
	221
	222	def create_index(conn):
	223	conn.rollback()
	224	conn.set_session(autocommit=True)
	225	c = conn.cursor()
	226
	227	# We create with NULLS FIRST so that when we search backwards
	228	# we get the ones with non null origin_server_ts first
	229	c.execute(
	230	"CREATE INDEX CONCURRENTLY event_search_room_order ON event_search("
	231	"room_id, origin_server_ts NULLS FIRST, stream_ordering NULLS FIRST)"
	232	)
	233	c.execute(
	234	"CREATE INDEX CONCURRENTLY event_search_order ON event_search("
	235	"origin_server_ts NULLS FIRST, stream_ordering NULLS FIRST)"
	236	)
	237	conn.set_session(autocommit=False)
	238
	239	yield self.runWithConnection(create_index)
	240
	241	pg = dict(progress)
	242	pg["have_added_indexes"] = True
	243
	244	yield self.runInteraction(
	245	self.EVENT_SEARCH_ORDER_UPDATE_NAME,
	246	self._background_update_progress_txn,
	247	self.EVENT_SEARCH_ORDER_UPDATE_NAME,
	248	pg,
	249	)
	250
	251	def reindex_search_txn(txn):
	252	sql = (
	253	"UPDATE event_search AS es SET stream_ordering = e.stream_ordering,"
	254	" origin_server_ts = e.origin_server_ts"
	255	" FROM events AS e"
	256	" WHERE e.event_id = es.event_id"
	257	" AND ? <= e.stream_ordering AND e.stream_ordering < ?"
	258	" RETURNING es.stream_ordering"
	259	)
	260
	261	min_stream_id = max_stream_id - batch_size
	262	txn.execute(sql, (min_stream_id, max_stream_id))
	263	rows = txn.fetchall()
	264
	265	if min_stream_id < target_min_stream_id:
	266	# We've recached the end.
	267	return len(rows), False
	268
	269	progress = {
	270	"target_min_stream_id_inclusive": target_min_stream_id,
	271	"max_stream_id_exclusive": min_stream_id,
	272	"rows_inserted": rows_inserted + len(rows),
	273	"have_added_indexes": True,
	274	}
	275
	276	self._background_update_progress_txn(
	277	txn, self.EVENT_SEARCH_ORDER_UPDATE_NAME, progress
	278	)
	279
	280	return len(rows), True
	281
	282	num_rows, finished = yield self.runInteraction(
	283	self.EVENT_SEARCH_ORDER_UPDATE_NAME, reindex_search_txn
	284	)
	285
	286	if not finished:
	287	yield self._end_background_update(self.EVENT_SEARCH_ORDER_UPDATE_NAME)
	288
	289	return num_rows
	290
	291	def store_search_entries_txn(self, txn, entries):
	292	"""Add entries to the search table
	293
	294	Args:
	295	txn (cursor):
	296	entries (iterable[SearchEntry]):
	297	entries to be added to the table
	298	"""
	299	if not self.hs.config.enable_search:
	300	return
	301	if isinstance(self.database_engine, PostgresEngine):
	302	sql = (
	303	"INSERT INTO event_search"
	304	" (event_id, room_id, key, vector, stream_ordering, origin_server_ts)"
	305	" VALUES (?,?,?,to_tsvector('english', ?),?,?)"
	306	)
	307
	308	args = (
	309	(
	310	entry.event_id,
	311	entry.room_id,
	312	entry.key,
	313	entry.value,
	314	entry.stream_ordering,
	315	entry.origin_server_ts,
	316	)
	317	for entry in entries
	318	)
	319
	320	txn.executemany(sql, args)
	321
	322	elif isinstance(self.database_engine, Sqlite3Engine):
	323	sql = (
	324	"INSERT INTO event_search (event_id, room_id, key, value)"
	325	" VALUES (?,?,?,?)"
	326	)
	327	args = (
	328	(entry.event_id, entry.room_id, entry.key, entry.value)
	329	for entry in entries
	330	)
	331
	332	txn.executemany(sql, args)
	333	else:
	334	# This should be unreachable.
	335	raise Exception("Unrecognized database engine")
	336
	337
	338	class SearchStore(SearchBackgroundUpdateStore):
	339	def __init__(self, db_conn, hs):
	340	super(SearchStore, self).__init__(db_conn, hs)
	341
	342	def store_event_search_txn(self, txn, event, key, value):
	343	"""Add event to the search table
	344
	345	Args:
	346	txn (cursor):
	347	event (EventBase):
	348	key (str):
	349	value (str):
	350	"""
	351	self.store_search_entries_txn(
	352	txn,
	353	(
	354	SearchEntry(
	355	key=key,
	356	value=value,
	357	event_id=event.event_id,
	358	room_id=event.room_id,
	359	stream_ordering=event.internal_metadata.stream_ordering,
	360	origin_server_ts=event.origin_server_ts,
	361	),
	362	),
	363	)
	364
	365	@defer.inlineCallbacks
	366	def search_msgs(self, room_ids, search_term, keys):
	367	"""Performs a full text search over events with given keys.
	368
	369	Args:
	370	room_ids (list): List of room ids to search in
	371	search_term (str): Search term to search for
	372	keys (list): List of keys to search in, currently supports
	373	"content.body", "content.name", "content.topic"
	374
	375	Returns:
	376	list of dicts
	377	"""
	378	clauses = []
	379
	380	search_query = search_query = _parse_query(self.database_engine, search_term)
	381
	382	args = []
	383
	384	# Make sure we don't explode because the person is in too many rooms.
	385	# We filter the results below regardless.
	386	if len(room_ids) < 500:
	387	clause, args = make_in_list_sql_clause(
	388	self.database_engine, "room_id", room_ids
	389	)
	390	clauses = [clause]
	391
	392	local_clauses = []
	393	for key in keys:
	394	local_clauses.append("key = ?")
	395	args.append(key)
	396
	397	clauses.append("(%s)" % (" OR ".join(local_clauses),))
	398
	399	count_args = args
	400	count_clauses = clauses
	401
	402	if isinstance(self.database_engine, PostgresEngine):
	403	sql = (
	404	"SELECT ts_rank_cd(vector, to_tsquery('english', ?)) AS rank,"
	405	" room_id, event_id"
	406	" FROM event_search"
	407	" WHERE vector @@ to_tsquery('english', ?)"
	408	)
	409	args = [search_query, search_query] + args
	410
	411	count_sql = (
	412	"SELECT room_id, count(*) as count FROM event_search"
	413	" WHERE vector @@ to_tsquery('english', ?)"
	414	)
	415	count_args = [search_query] + count_args
	416	elif isinstance(self.database_engine, Sqlite3Engine):
	417	sql = (
	418	"SELECT rank(matchinfo(event_search)) as rank, room_id, event_id"
	419	" FROM event_search"
	420	" WHERE value MATCH ?"
	421	)
	422	args = [search_query] + args
	423
	424	count_sql = (
	425	"SELECT room_id, count(*) as count FROM event_search"
	426	" WHERE value MATCH ?"
	427	)
	428	count_args = [search_term] + count_args
	429	else:
	430	# This should be unreachable.
	431	raise Exception("Unrecognized database engine")
	432
	433	for clause in clauses:
	434	sql += " AND " + clause
	435
	436	for clause in count_clauses:
	437	count_sql += " AND " + clause
	438
	439	# We add an arbitrary limit here to ensure we don't try to pull the
	440	# entire table from the database.
	441	sql += " ORDER BY rank DESC LIMIT 500"
	442
	443	results = yield self._execute("search_msgs", self.cursor_to_dict, sql, *args)
	444
	445	results = list(filter(lambda row: row["room_id"] in room_ids, results))
	446
	447	events = yield self.get_events_as_list([r["event_id"] for r in results])
	448
	449	event_map = {ev.event_id: ev for ev in events}
	450
	451	highlights = None
	452	if isinstance(self.database_engine, PostgresEngine):
	453	highlights = yield self._find_highlights_in_postgres(search_query, events)
	454
	455	count_sql += " GROUP BY room_id"
	456
	457	count_results = yield self._execute(
	458	"search_rooms_count", self.cursor_to_dict, count_sql, *count_args
	459	)
	460
	461	count = sum(row["count"] for row in count_results if row["room_id"] in room_ids)
	462
	463	return {
	464	"results": [
	465	{"event": event_map[r["event_id"]], "rank": r["rank"]}
	466	for r in results
	467	if r["event_id"] in event_map
	468	],
	469	"highlights": highlights,
	470	"count": count,
	471	}
	472
	473	@defer.inlineCallbacks
	474	def search_rooms(self, room_ids, search_term, keys, limit, pagination_token=None):
	475	"""Performs a full text search over events with given keys.
	476
	477	Args:
	478	room_id (list): The room_ids to search in
	479	search_term (str): Search term to search for
	480	keys (list): List of keys to search in, currently supports
	481	"content.body", "content.name", "content.topic"
	482	pagination_token (str): A pagination token previously returned
	483
	484	Returns:
	485	list of dicts
	486	"""
	487	clauses = []
	488
	489	search_query = search_query = _parse_query(self.database_engine, search_term)
	490
	491	args = []
	492
	493	# Make sure we don't explode because the person is in too many rooms.
	494	# We filter the results below regardless.
	495	if len(room_ids) < 500:
	496	clause, args = make_in_list_sql_clause(
	497	self.database_engine, "room_id", room_ids
	498	)
	499	clauses = [clause]
	500
	501	local_clauses = []
	502	for key in keys:
	503	local_clauses.append("key = ?")
	504	args.append(key)
	505
	506	clauses.append("(%s)" % (" OR ".join(local_clauses),))
	507
	508	# take copies of the current args and clauses lists, before adding
	509	# pagination clauses to main query.
	510	count_args = list(args)
	511	count_clauses = list(clauses)
	512
	513	if pagination_token:
	514	try:
	515	origin_server_ts, stream = pagination_token.split(",")
	516	origin_server_ts = int(origin_server_ts)
	517	stream = int(stream)
	518	except Exception:
	519	raise SynapseError(400, "Invalid pagination token")
	520
	521	clauses.append(
	522	"(origin_server_ts < ?"
	523	" OR (origin_server_ts = ? AND stream_ordering < ?))"
	524	)
	525	args.extend([origin_server_ts, origin_server_ts, stream])
	526
	527	if isinstance(self.database_engine, PostgresEngine):
	528	sql = (
	529	"SELECT ts_rank_cd(vector, to_tsquery('english', ?)) as rank,"
	530	" origin_server_ts, stream_ordering, room_id, event_id"
	531	" FROM event_search"
	532	" WHERE vector @@ to_tsquery('english', ?) AND "
	533	)
	534	args = [search_query, search_query] + args
	535
	536	count_sql = (
	537	"SELECT room_id, count(*) as count FROM event_search"
	538	" WHERE vector @@ to_tsquery('english', ?) AND "
	539	)
	540	count_args = [search_query] + count_args
	541	elif isinstance(self.database_engine, Sqlite3Engine):
	542	# We use CROSS JOIN here to ensure we use the right indexes.
	543	# https://sqlite.org/optoverview.html#crossjoin
	544	#
	545	# We want to use the full text search index on event_search to
	546	# extract all possible matches first, then lookup those matches
	547	# in the events table to get the topological ordering. We need
	548	# to use the indexes in this order because sqlite refuses to
	549	# MATCH unless it uses the full text search index
	550	sql = (
	551	"SELECT rank(matchinfo) as rank, room_id, event_id,"
	552	" origin_server_ts, stream_ordering"
	553	" FROM (SELECT key, event_id, matchinfo(event_search) as matchinfo"
	554	" FROM event_search"
	555	" WHERE value MATCH ?"
	556	" )"
	557	" CROSS JOIN events USING (event_id)"
	558	" WHERE "
	559	)
	560	args = [search_query] + args
	561
	562	count_sql = (
	563	"SELECT room_id, count(*) as count FROM event_search"
	564	" WHERE value MATCH ? AND "
	565	)
	566	count_args = [search_term] + count_args
	567	else:
	568	# This should be unreachable.
	569	raise Exception("Unrecognized database engine")
	570
	571	sql += " AND ".join(clauses)
	572	count_sql += " AND ".join(count_clauses)
	573
	574	# We add an arbitrary limit here to ensure we don't try to pull the
	575	# entire table from the database.
	576	if isinstance(self.database_engine, PostgresEngine):
	577	sql += (
	578	" ORDER BY origin_server_ts DESC NULLS LAST,"
	579	" stream_ordering DESC NULLS LAST LIMIT ?"
	580	)
	581	elif isinstance(self.database_engine, Sqlite3Engine):
	582	sql += " ORDER BY origin_server_ts DESC, stream_ordering DESC LIMIT ?"
	583	else:
	584	raise Exception("Unrecognized database engine")
	585
	586	args.append(limit)
	587
	588	results = yield self._execute("search_rooms", self.cursor_to_dict, sql, *args)
	589
	590	results = list(filter(lambda row: row["room_id"] in room_ids, results))
	591
	592	events = yield self.get_events_as_list([r["event_id"] for r in results])
	593
	594	event_map = {ev.event_id: ev for ev in events}
	595
	596	highlights = None
	597	if isinstance(self.database_engine, PostgresEngine):
	598	highlights = yield self._find_highlights_in_postgres(search_query, events)
	599
	600	count_sql += " GROUP BY room_id"
	601
	602	count_results = yield self._execute(
	603	"search_rooms_count", self.cursor_to_dict, count_sql, *count_args
	604	)
	605
	606	count = sum(row["count"] for row in count_results if row["room_id"] in room_ids)
	607
	608	return {
	609	"results": [
	610	{
	611	"event": event_map[r["event_id"]],
	612	"rank": r["rank"],
	613	"pagination_token": "%s,%s"
	614	% (r["origin_server_ts"], r["stream_ordering"]),
	615	}
	616	for r in results
	617	if r["event_id"] in event_map
	618	],
	619	"highlights": highlights,
	620	"count": count,
	621	}
	622
	623	def _find_highlights_in_postgres(self, search_query, events):
	624	"""Given a list of events and a search term, return a list of words
	625	that match from the content of the event.
	626
	627	This is used to give a list of words that clients can match against to
	628	highlight the matching parts.
	629
	630	Args:
	631	search_query (str)
	632	events (list): A list of events
	633
	634	Returns:
	635	deferred : A set of strings.
	636	"""
	637
	638	def f(txn):
	639	highlight_words = set()
	640	for event in events:
	641	# As a hack we simply join values of all possible keys. This is
	642	# fine since we're only using them to find possible highlights.
	643	values = []
	644	for key in ("body", "name", "topic"):
	645	v = event.content.get(key, None)
	646	if v:
	647	values.append(v)
	648
	649	if not values:
	650	continue
	651
	652	value = " ".join(values)
	653
	654	# We need to find some values for StartSel and StopSel that
	655	# aren't in the value so that we can pick results out.
	656	start_sel = "<"
	657	stop_sel = ">"
	658
	659	while start_sel in value:
	660	start_sel += "<"
	661	while stop_sel in value:
	662	stop_sel += ">"
	663
	664	query = "SELECT ts_headline(?, to_tsquery('english', ?), %s)" % (
	665	_to_postgres_options(
	666	{
	667	"StartSel": start_sel,
	668	"StopSel": stop_sel,
	669	"MaxFragments": "50",
	670	}
	671	)
	672	)
	673	txn.execute(query, (value, search_query))
	674	headline, = txn.fetchall()[0]
	675
	676	# Now we need to pick the possible highlights out of the haedline
	677	# result.
	678	matcher_regex = "%s(.*?)%s" % (
	679	re.escape(start_sel),
	680	re.escape(stop_sel),
	681	)
	682
	683	res = re.findall(matcher_regex, headline)
	684	highlight_words.update([r.lower() for r in res])
	685
	686	return highlight_words
	687
	688	return self.runInteraction("_find_highlights", f)
	689
	690
	691	def _to_postgres_options(options_dict):
	692	return "'%s'" % (",".join("%s=%s" % (k, v) for k, v in options_dict.items()),)
	693
	694
	695	def _parse_query(database_engine, search_term):
	696	"""Takes a plain unicode string from the user and converts it into a form
	697	that can be passed to database.
	698	We use this so that we can add prefix matching, which isn't something
	699	that is supported by default.
	700	"""
	701
	702	# Pull out the individual words, discarding any non-word characters.
	703	results = re.findall(r"([\w\-]+)", search_term, re.UNICODE)
	704
	705	if isinstance(database_engine, PostgresEngine):
	706	return " & ".join(result + ":*" for result in results)
	707	elif isinstance(database_engine, Sqlite3Engine):
	708	return " & ".join(result + "*" for result in results)
	709	else:
	710	# This should be unreachable.
	711	raise Exception("Unrecognized database engine")

+101

-0

synapse/storage/data_stores/main/signatures.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2014-2016 OpenMarket Ltd
	2	#
	3	# Licensed under the Apache License, Version 2.0 (the "License");
	4	# you may not use this file except in compliance with the License.
	5	# You may obtain a copy of the License at
	6	#
	7	# http://www.apache.org/licenses/LICENSE-2.0
	8	#
	9	# Unless required by applicable law or agreed to in writing, software
	10	# distributed under the License is distributed on an "AS IS" BASIS,
	11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	# See the License for the specific language governing permissions and
	13	# limitations under the License.
	14
	15	import six
	16
	17	from unpaddedbase64 import encode_base64
	18
	19	from twisted.internet import defer
	20
	21	from synapse.crypto.event_signing import compute_event_reference_hash
	22	from synapse.storage._base import SQLBaseStore
	23	from synapse.util.caches.descriptors import cached, cachedList
	24
	25	# py2 sqlite has buffer hardcoded as only binary type, so we must use it,
	26	# despite being deprecated and removed in favor of memoryview
	27	if six.PY2:
	28	db_binary_type = six.moves.builtins.buffer
	29	else:
	30	db_binary_type = memoryview
	31
	32
	33	class SignatureWorkerStore(SQLBaseStore):
	34	@cached()
	35	def get_event_reference_hash(self, event_id):
	36	# This is a dummy function to allow get_event_reference_hashes
	37	# to use its cache
	38	raise NotImplementedError()
	39
	40	@cachedList(
	41	cached_method_name="get_event_reference_hash", list_name="event_ids", num_args=1
	42	)
	43	def get_event_reference_hashes(self, event_ids):
	44	def f(txn):
	45	return {
	46	event_id: self._get_event_reference_hashes_txn(txn, event_id)
	47	for event_id in event_ids
	48	}
	49
	50	return self.runInteraction("get_event_reference_hashes", f)
	51
	52	@defer.inlineCallbacks
	53	def add_event_hashes(self, event_ids):
	54	hashes = yield self.get_event_reference_hashes(event_ids)
	55	hashes = {
	56	e_id: {k: encode_base64(v) for k, v in h.items() if k == "sha256"}
	57	for e_id, h in hashes.items()
	58	}
	59
	60	return list(hashes.items())
	61
	62	def _get_event_reference_hashes_txn(self, txn, event_id):
	63	"""Get all the hashes for a given PDU.
	64	Args:
	65	txn (cursor):
	66	event_id (str): Id for the Event.
	67	Returns:
	68	A dict[unicode, bytes] of algorithm -> hash.
	69	"""
	70	query = (
	71	"SELECT algorithm, hash"
	72	" FROM event_reference_hashes"
	73	" WHERE event_id = ?"
	74	)
	75	txn.execute(query, (event_id,))
	76	return {k: v for k, v in txn}
	77
	78
	79	class SignatureStore(SignatureWorkerStore):
	80	"""Persistence for event signatures and hashes"""
	81
	82	def _store_event_reference_hashes_txn(self, txn, events):
	83	"""Store a hash for a PDU
	84	Args:
	85	txn (cursor):
	86	events (list): list of Events.
	87	"""
	88
	89	vals = []
	90	for event in events:
	91	ref_alg, ref_hash_bytes = compute_event_reference_hash(event)
	92	vals.append(
	93	{
	94	"event_id": event.event_id,
	95	"algorithm": ref_alg,
	96	"hash": db_binary_type(ref_hash_bytes),
	97	}
	98	)
	99
	100	self._simple_insert_many_txn(txn, table="event_reference_hashes", values=vals)

+1244

-0

synapse/storage/data_stores/main/state.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2014-2016 OpenMarket Ltd
	2	#
	3	# Licensed under the Apache License, Version 2.0 (the "License");
	4	# you may not use this file except in compliance with the License.
	5	# You may obtain a copy of the License at
	6	#
	7	# http://www.apache.org/licenses/LICENSE-2.0
	8	#
	9	# Unless required by applicable law or agreed to in writing, software
	10	# distributed under the License is distributed on an "AS IS" BASIS,
	11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	# See the License for the specific language governing permissions and
	13	# limitations under the License.
	14
	15	import logging
	16	from collections import namedtuple
	17
	18	from six import iteritems, itervalues
	19	from six.moves import range
	20
	21	from twisted.internet import defer
	22
	23	from synapse.api.constants import EventTypes
	24	from synapse.api.errors import NotFoundError
	25	from synapse.storage._base import SQLBaseStore
	26	from synapse.storage.background_updates import BackgroundUpdateStore
	27	from synapse.storage.data_stores.main.events_worker import EventsWorkerStore
	28	from synapse.storage.engines import PostgresEngine
	29	from synapse.storage.state import StateFilter
	30	from synapse.util.caches import get_cache_factor_for, intern_string
	31	from synapse.util.caches.descriptors import cached, cachedList
	32	from synapse.util.caches.dictionary_cache import DictionaryCache
	33	from synapse.util.stringutils import to_ascii
	34
	35	logger = logging.getLogger(__name__)
	36
	37
	38	MAX_STATE_DELTA_HOPS = 100
	39
	40
	41	class _GetStateGroupDelta(
	42	namedtuple("_GetStateGroupDelta", ("prev_group", "delta_ids"))
	43	):
	44	"""Return type of get_state_group_delta that implements __len__, which lets
	45	us use the itrable flag when caching
	46	"""
	47
	48	__slots__ = []
	49
	50	def __len__(self):
	51	return len(self.delta_ids) if self.delta_ids else 0
	52
	53
	54	class StateGroupBackgroundUpdateStore(SQLBaseStore):
	55	"""Defines functions related to state groups needed to run the state backgroud
	56	updates.
	57	"""
	58
	59	def _count_state_group_hops_txn(self, txn, state_group):
	60	"""Given a state group, count how many hops there are in the tree.
	61
	62	This is used to ensure the delta chains don't get too long.
	63	"""
	64	if isinstance(self.database_engine, PostgresEngine):
	65	sql = """
	66	WITH RECURSIVE state(state_group) AS (
	67	VALUES(?::bigint)
	68	UNION ALL
	69	SELECT prev_state_group FROM state_group_edges e, state s
	70	WHERE s.state_group = e.state_group
	71	)
	72	SELECT count(*) FROM state;
	73	"""
	74
	75	txn.execute(sql, (state_group,))
	76	row = txn.fetchone()
	77	if row and row[0]:
	78	return row[0]
	79	else:
	80	return 0
	81	else:
	82	# We don't use WITH RECURSIVE on sqlite3 as there are distributions
	83	# that ship with an sqlite3 version that doesn't support it (e.g. wheezy)
	84	next_group = state_group
	85	count = 0
	86
	87	while next_group:
	88	next_group = self._simple_select_one_onecol_txn(
	89	txn,
	90	table="state_group_edges",
	91	keyvalues={"state_group": next_group},
	92	retcol="prev_state_group",
	93	allow_none=True,
	94	)
	95	if next_group:
	96	count += 1
	97
	98	return count
	99
	100	def _get_state_groups_from_groups_txn(
	101	self, txn, groups, state_filter=StateFilter.all()
	102	):
	103	results = {group: {} for group in groups}
	104
	105	where_clause, where_args = state_filter.make_sql_filter_clause()
	106
	107	# Unless the filter clause is empty, we're going to append it after an
	108	# existing where clause
	109	if where_clause:
	110	where_clause = " AND (%s)" % (where_clause,)
	111
	112	if isinstance(self.database_engine, PostgresEngine):
	113	# Temporarily disable sequential scans in this transaction. This is
	114	# a temporary hack until we can add the right indices in
	115	txn.execute("SET LOCAL enable_seqscan=off")
	116
	117	# The below query walks the state_group tree so that the "state"
	118	# table includes all state_groups in the tree. It then joins
	119	# against `state_groups_state` to fetch the latest state.
	120	# It assumes that previous state groups are always numerically
	121	# lesser.
	122	# The PARTITION is used to get the event_id in the greatest state
	123	# group for the given type, state_key.
	124	# This may return multiple rows per (type, state_key), but last_value
	125	# should be the same.
	126	sql = """
	127	WITH RECURSIVE state(state_group) AS (
	128	VALUES(?::bigint)
	129	UNION ALL
	130	SELECT prev_state_group FROM state_group_edges e, state s
	131	WHERE s.state_group = e.state_group
	132	)
	133	SELECT DISTINCT type, state_key, last_value(event_id) OVER (
	134	PARTITION BY type, state_key ORDER BY state_group ASC
	135	ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
	136	) AS event_id FROM state_groups_state
	137	WHERE state_group IN (
	138	SELECT state_group FROM state
	139	)
	140	"""
	141
	142	for group in groups:
	143	args = [group]
	144	args.extend(where_args)
	145
	146	txn.execute(sql + where_clause, args)
	147	for row in txn:
	148	typ, state_key, event_id = row
	149	key = (typ, state_key)
	150	results[group][key] = event_id
	151	else:
	152	max_entries_returned = state_filter.max_entries_returned()
	153
	154	# We don't use WITH RECURSIVE on sqlite3 as there are distributions
	155	# that ship with an sqlite3 version that doesn't support it (e.g. wheezy)
	156	for group in groups:
	157	next_group = group
	158
	159	while next_group:
	160	# We did this before by getting the list of group ids, and
	161	# then passing that list to sqlite to get latest event for
	162	# each (type, state_key). However, that was terribly slow
	163	# without the right indices (which we can't add until
	164	# after we finish deduping state, which requires this func)
	165	args = [next_group]
	166	args.extend(where_args)
	167
	168	txn.execute(
	169	"SELECT type, state_key, event_id FROM state_groups_state"
	170	" WHERE state_group = ? " + where_clause,
	171	args,
	172	)
	173	results[group].update(
	174	((typ, state_key), event_id)
	175	for typ, state_key, event_id in txn
	176	if (typ, state_key) not in results[group]
	177	)
	178
	179	# If the number of entries in the (type,state_key)->event_id dict
	180	# matches the number of (type,state_keys) types we were searching
	181	# for, then we must have found them all, so no need to go walk
	182	# further down the tree... UNLESS our types filter contained
	183	# wildcards (i.e. Nones) in which case we have to do an exhaustive
	184	# search
	185	if (
	186	max_entries_returned is not None
	187	and len(results[group]) == max_entries_returned
	188	):
	189	break
	190
	191	next_group = self._simple_select_one_onecol_txn(
	192	txn,
	193	table="state_group_edges",
	194	keyvalues={"state_group": next_group},
	195	retcol="prev_state_group",
	196	allow_none=True,
	197	)
	198
	199	return results
	200
	201
	202	# this inherits from EventsWorkerStore because it calls self.get_events
	203	class StateGroupWorkerStore(
	204	EventsWorkerStore, StateGroupBackgroundUpdateStore, SQLBaseStore
	205	):
	206	"""The parts of StateGroupStore that can be called from workers.
	207	"""
	208
	209	STATE_GROUP_DEDUPLICATION_UPDATE_NAME = "state_group_state_deduplication"
	210	STATE_GROUP_INDEX_UPDATE_NAME = "state_group_state_type_index"
	211	CURRENT_STATE_INDEX_UPDATE_NAME = "current_state_members_idx"
	212
	213	def __init__(self, db_conn, hs):
	214	super(StateGroupWorkerStore, self).__init__(db_conn, hs)
	215
	216	# Originally the state store used a single DictionaryCache to cache the
	217	# event IDs for the state types in a given state group to avoid hammering
	218	# on the state_group* tables.
	219	#
	220	# The point of using a DictionaryCache is that it can cache a subset
	221	# of the state events for a given state group (i.e. a subset of the keys for a
	222	# given dict which is an entry in the cache for a given state group ID).
	223	#
	224	# However, this poses problems when performing complicated queries
	225	# on the store - for instance: "give me all the state for this group, but
	226	# limit members to this subset of users", as DictionaryCache's API isn't
	227	# rich enough to say "please cache any of these fields, apart from this subset".
	228	# This is problematic when lazy loading members, which requires this behaviour,
	229	# as without it the cache has no choice but to speculatively load all
	230	# state events for the group, which negates the efficiency being sought.
	231	#
	232	# Rather than overcomplicating DictionaryCache's API, we instead split the
	233	# state_group_cache into two halves - one for tracking non-member events,
	234	# and the other for tracking member_events. This means that lazy loading
	235	# queries can be made in a cache-friendly manner by querying both caches
	236	# separately and then merging the result. So for the example above, you
	237	# would query the members cache for a specific subset of state keys
	238	# (which DictionaryCache will handle efficiently and fine) and the non-members
	239	# cache for all state (which DictionaryCache will similarly handle fine)
	240	# and then just merge the results together.
	241	#
	242	# We size the non-members cache to be smaller than the members cache as the
	243	# vast majority of state in Matrix (today) is member events.
	244
	245	self._state_group_cache = DictionaryCache(
	246	"stateGroupCache",
	247	# TODO: this hasn't been tuned yet
	248	50000 * get_cache_factor_for("stateGroupCache"),
	249	)
	250	self._state_group_members_cache = DictionaryCache(
	251	"stateGroupMembersCache",
	252	500000 * get_cache_factor_for("stateGroupMembersCache"),
	253	)
	254
	255	@defer.inlineCallbacks
	256	def get_room_version(self, room_id):
	257	"""Get the room_version of a given room
	258
	259	Args:
	260	room_id (str)
	261
	262	Returns:
	263	Deferred[str]
	264
	265	Raises:
	266	NotFoundError if the room is unknown
	267	"""
	268	# for now we do this by looking at the create event. We may want to cache this
	269	# more intelligently in future.
	270
	271	# Retrieve the room's create event
	272	create_event = yield self.get_create_event_for_room(room_id)
	273	return create_event.content.get("room_version", "1")
	274
	275	@defer.inlineCallbacks
	276	def get_room_predecessor(self, room_id):
	277	"""Get the predecessor room of an upgraded room if one exists.
	278	Otherwise return None.
	279
	280	Args:
	281	room_id (str)
	282
	283	Returns:
	284	Deferred[unicode\|None]: predecessor room id
	285
	286	Raises:
	287	NotFoundError if the room is unknown
	288	"""
	289	# Retrieve the room's create event
	290	create_event = yield self.get_create_event_for_room(room_id)
	291
	292	# Return predecessor if present
	293	return create_event.content.get("predecessor", None)
	294
	295	@defer.inlineCallbacks
	296	def get_create_event_for_room(self, room_id):
	297	"""Get the create state event for a room.
	298
	299	Args:
	300	room_id (str)
	301
	302	Returns:
	303	Deferred[EventBase]: The room creation event.
	304
	305	Raises:
	306	NotFoundError if the room is unknown
	307	"""
	308	state_ids = yield self.get_current_state_ids(room_id)
	309	create_id = state_ids.get((EventTypes.Create, ""))
	310
	311	# If we can't find the create event, assume we've hit a dead end
	312	if not create_id:
	313	raise NotFoundError("Unknown room %s" % (room_id))
	314
	315	# Retrieve the room's create event and return
	316	create_event = yield self.get_event(create_id)
	317	return create_event
	318
	319	@cached(max_entries=100000, iterable=True)
	320	def get_current_state_ids(self, room_id):
	321	"""Get the current state event ids for a room based on the
	322	current_state_events table.
	323
	324	Args:
	325	room_id (str)
	326
	327	Returns:
	328	deferred: dict of (type, state_key) -> event_id
	329	"""
	330
	331	def _get_current_state_ids_txn(txn):
	332	txn.execute(
	333	"""SELECT type, state_key, event_id FROM current_state_events
	334	WHERE room_id = ?
	335	""",
	336	(room_id,),
	337	)
	338
	339	return {
	340	(intern_string(r[0]), intern_string(r[1])): to_ascii(r[2]) for r in txn
	341	}
	342
	343	return self.runInteraction("get_current_state_ids", _get_current_state_ids_txn)
	344
	345	# FIXME: how should this be cached?
	346	def get_filtered_current_state_ids(self, room_id, state_filter=StateFilter.all()):
	347	"""Get the current state event of a given type for a room based on the
	348	current_state_events table. This may not be as up-to-date as the result
	349	of doing a fresh state resolution as per state_handler.get_current_state
	350
	351	Args:
	352	room_id (str)
	353	state_filter (StateFilter): The state filter used to fetch state
	354	from the database.
	355
	356	Returns:
	357	Deferred[dict[tuple[str, str], str]]: Map from type/state_key to
	358	event ID.
	359	"""
	360
	361	where_clause, where_args = state_filter.make_sql_filter_clause()
	362
	363	if not where_clause:
	364	# We delegate to the cached version
	365	return self.get_current_state_ids(room_id)
	366
	367	def _get_filtered_current_state_ids_txn(txn):
	368	results = {}
	369	sql = """
	370	SELECT type, state_key, event_id FROM current_state_events
	371	WHERE room_id = ?
	372	"""
	373
	374	if where_clause:
	375	sql += " AND (%s)" % (where_clause,)
	376
	377	args = [room_id]
	378	args.extend(where_args)
	379	txn.execute(sql, args)
	380	for row in txn:
	381	typ, state_key, event_id = row
	382	key = (intern_string(typ), intern_string(state_key))
	383	results[key] = event_id
	384
	385	return results
	386
	387	return self.runInteraction(
	388	"get_filtered_current_state_ids", _get_filtered_current_state_ids_txn
	389	)
	390
	391	@defer.inlineCallbacks
	392	def get_canonical_alias_for_room(self, room_id):
	393	"""Get canonical alias for room, if any
	394
	395	Args:
	396	room_id (str)
	397
	398	Returns:
	399	Deferred[str\|None]: The canonical alias, if any
	400	"""
	401
	402	state = yield self.get_filtered_current_state_ids(
	403	room_id, StateFilter.from_types([(EventTypes.CanonicalAlias, "")])
	404	)
	405
	406	event_id = state.get((EventTypes.CanonicalAlias, ""))
	407	if not event_id:
	408	return
	409
	410	event = yield self.get_event(event_id, allow_none=True)
	411	if not event:
	412	return
	413
	414	return event.content.get("canonical_alias")
	415
	416	@cached(max_entries=10000, iterable=True)
	417	def get_state_group_delta(self, state_group):
	418	"""Given a state group try to return a previous group and a delta between
	419	the old and the new.
	420
	421	Returns:
	422	(prev_group, delta_ids), where both may be None.
	423	"""
	424
	425	def _get_state_group_delta_txn(txn):
	426	prev_group = self._simple_select_one_onecol_txn(
	427	txn,
	428	table="state_group_edges",
	429	keyvalues={"state_group": state_group},
	430	retcol="prev_state_group",
	431	allow_none=True,
	432	)
	433
	434	if not prev_group:
	435	return _GetStateGroupDelta(None, None)
	436
	437	delta_ids = self._simple_select_list_txn(
	438	txn,
	439	table="state_groups_state",
	440	keyvalues={"state_group": state_group},
	441	retcols=("type", "state_key", "event_id"),
	442	)
	443
	444	return _GetStateGroupDelta(
	445	prev_group,
	446	{(row["type"], row["state_key"]): row["event_id"] for row in delta_ids},
	447	)
	448
	449	return self.runInteraction("get_state_group_delta", _get_state_group_delta_txn)
	450
	451	@defer.inlineCallbacks
	452	def get_state_groups_ids(self, _room_id, event_ids):
	453	"""Get the event IDs of all the state for the state groups for the given events
	454
	455	Args:
	456	_room_id (str): id of the room for these events
	457	event_ids (iterable[str]): ids of the events
	458
	459	Returns:
	460	Deferred[dict[int, dict[tuple[str, str], str]]]:
	461	dict of state_group_id -> (dict of (type, state_key) -> event id)
	462	"""
	463	if not event_ids:
	464	return {}
	465
	466	event_to_groups = yield self._get_state_group_for_events(event_ids)
	467
	468	groups = set(itervalues(event_to_groups))
	469	group_to_state = yield self._get_state_for_groups(groups)
	470
	471	return group_to_state
	472
	473	@defer.inlineCallbacks
	474	def get_state_ids_for_group(self, state_group):
	475	"""Get the event IDs of all the state in the given state group
	476
	477	Args:
	478	state_group (int)
	479
	480	Returns:
	481	Deferred[dict]: Resolves to a map of (type, state_key) -> event_id
	482	"""
	483	group_to_state = yield self._get_state_for_groups((state_group,))
	484
	485	return group_to_state[state_group]
	486
	487	@defer.inlineCallbacks
	488	def get_state_groups(self, room_id, event_ids):
	489	""" Get the state groups for the given list of event_ids
	490
	491	Returns:
	492	Deferred[dict[int, list[EventBase]]]:
	493	dict of state_group_id -> list of state events.
	494	"""
	495	if not event_ids:
	496	return {}
	497
	498	group_to_ids = yield self.get_state_groups_ids(room_id, event_ids)
	499
	500	state_event_map = yield self.get_events(
	501	[
	502	ev_id
	503	for group_ids in itervalues(group_to_ids)
	504	for ev_id in itervalues(group_ids)
	505	],
	506	get_prev_content=False,
	507	)
	508
	509	return {
	510	group: [
	511	state_event_map[v]
	512	for v in itervalues(event_id_map)
	513	if v in state_event_map
	514	]
	515	for group, event_id_map in iteritems(group_to_ids)
	516	}
	517
	518	@defer.inlineCallbacks
	519	def _get_state_groups_from_groups(self, groups, state_filter):
	520	"""Returns the state groups for a given set of groups, filtering on
	521	types of state events.
	522
	523	Args:
	524	groups(list[int]): list of state group IDs to query
	525	state_filter (StateFilter): The state filter used to fetch state
	526	from the database.
	527	Returns:
	528	Deferred[dict[int, dict[tuple[str, str], str]]]:
	529	dict of state_group_id -> (dict of (type, state_key) -> event id)
	530	"""
	531	results = {}
	532
	533	chunks = [groups[i : i + 100] for i in range(0, len(groups), 100)]
	534	for chunk in chunks:
	535	res = yield self.runInteraction(
	536	"_get_state_groups_from_groups",
	537	self._get_state_groups_from_groups_txn,
	538	chunk,
	539	state_filter,
	540	)
	541	results.update(res)
	542
	543	return results
	544
	545	@defer.inlineCallbacks
	546	def get_state_for_events(self, event_ids, state_filter=StateFilter.all()):
	547	"""Given a list of event_ids and type tuples, return a list of state
	548	dicts for each event.
	549
	550	Args:
	551	event_ids (list[string])
	552	state_filter (StateFilter): The state filter used to fetch state
	553	from the database.
	554
	555	Returns:
	556	deferred: A dict of (event_id) -> (type, state_key) -> [state_events]
	557	"""
	558	event_to_groups = yield self._get_state_group_for_events(event_ids)
	559
	560	groups = set(itervalues(event_to_groups))
	561	group_to_state = yield self._get_state_for_groups(groups, state_filter)
	562
	563	state_event_map = yield self.get_events(
	564	[ev_id for sd in itervalues(group_to_state) for ev_id in itervalues(sd)],
	565	get_prev_content=False,
	566	)
	567
	568	event_to_state = {
	569	event_id: {
	570	k: state_event_map[v]
	571	for k, v in iteritems(group_to_state[group])
	572	if v in state_event_map
	573	}
	574	for event_id, group in iteritems(event_to_groups)
	575	}
	576
	577	return {event: event_to_state[event] for event in event_ids}
	578
	579	@defer.inlineCallbacks
	580	def get_state_ids_for_events(self, event_ids, state_filter=StateFilter.all()):
	581	"""
	582	Get the state dicts corresponding to a list of events, containing the event_ids
	583	of the state events (as opposed to the events themselves)
	584
	585	Args:
	586	event_ids(list(str)): events whose state should be returned
	587	state_filter (StateFilter): The state filter used to fetch state
	588	from the database.
	589
	590	Returns:
	591	A deferred dict from event_id -> (type, state_key) -> event_id
	592	"""
	593	event_to_groups = yield self._get_state_group_for_events(event_ids)
	594
	595	groups = set(itervalues(event_to_groups))
	596	group_to_state = yield self._get_state_for_groups(groups, state_filter)
	597
	598	event_to_state = {
	599	event_id: group_to_state[group]
	600	for event_id, group in iteritems(event_to_groups)
	601	}
	602
	603	return {event: event_to_state[event] for event in event_ids}
	604
	605	@defer.inlineCallbacks
	606	def get_state_for_event(self, event_id, state_filter=StateFilter.all()):
	607	"""
	608	Get the state dict corresponding to a particular event
	609
	610	Args:
	611	event_id(str): event whose state should be returned
	612	state_filter (StateFilter): The state filter used to fetch state
	613	from the database.
	614
	615	Returns:
	616	A deferred dict from (type, state_key) -> state_event
	617	"""
	618	state_map = yield self.get_state_for_events([event_id], state_filter)
	619	return state_map[event_id]
	620
	621	@defer.inlineCallbacks
	622	def get_state_ids_for_event(self, event_id, state_filter=StateFilter.all()):
	623	"""
	624	Get the state dict corresponding to a particular event
	625
	626	Args:
	627	event_id(str): event whose state should be returned
	628	state_filter (StateFilter): The state filter used to fetch state
	629	from the database.
	630
	631	Returns:
	632	A deferred dict from (type, state_key) -> state_event
	633	"""
	634	state_map = yield self.get_state_ids_for_events([event_id], state_filter)
	635	return state_map[event_id]
	636
	637	@cached(max_entries=50000)
	638	def _get_state_group_for_event(self, event_id):
	639	return self._simple_select_one_onecol(
	640	table="event_to_state_groups",
	641	keyvalues={"event_id": event_id},
	642	retcol="state_group",
	643	allow_none=True,
	644	desc="_get_state_group_for_event",
	645	)
	646
	647	@cachedList(
	648	cached_method_name="_get_state_group_for_event",
	649	list_name="event_ids",
	650	num_args=1,
	651	inlineCallbacks=True,
	652	)
	653	def _get_state_group_for_events(self, event_ids):
	654	"""Returns mapping event_id -> state_group
	655	"""
	656	rows = yield self._simple_select_many_batch(
	657	table="event_to_state_groups",
	658	column="event_id",
	659	iterable=event_ids,
	660	keyvalues={},
	661	retcols=("event_id", "state_group"),
	662	desc="_get_state_group_for_events",
	663	)
	664
	665	return {row["event_id"]: row["state_group"] for row in rows}
	666
	667	def _get_state_for_group_using_cache(self, cache, group, state_filter):
	668	"""Checks if group is in cache. See `_get_state_for_groups`
	669
	670	Args:
	671	cache(DictionaryCache): the state group cache to use
	672	group(int): The state group to lookup
	673	state_filter (StateFilter): The state filter used to fetch state
	674	from the database.
	675
	676	Returns 2-tuple (`state_dict`, `got_all`).
	677	`got_all` is a bool indicating if we successfully retrieved all
	678	requests state from the cache, if False we need to query the DB for the
	679	missing state.
	680	"""
	681	is_all, known_absent, state_dict_ids = cache.get(group)
	682
	683	if is_all or state_filter.is_full():
	684	# Either we have everything or want everything, either way
	685	# `is_all` tells us whether we've gotten everything.
	686	return state_filter.filter_state(state_dict_ids), is_all
	687
	688	# tracks whether any of our requested types are missing from the cache
	689	missing_types = False
	690
	691	if state_filter.has_wildcards():
	692	# We don't know if we fetched all the state keys for the types in
	693	# the filter that are wildcards, so we have to assume that we may
	694	# have missed some.
	695	missing_types = True
	696	else:
	697	# There aren't any wild cards, so `concrete_types()` returns the
	698	# complete list of event types we're wanting.
	699	for key in state_filter.concrete_types():
	700	if key not in state_dict_ids and key not in known_absent:
	701	missing_types = True
	702	break
	703
	704	return state_filter.filter_state(state_dict_ids), not missing_types
	705
	706	@defer.inlineCallbacks
	707	def _get_state_for_groups(self, groups, state_filter=StateFilter.all()):
	708	"""Gets the state at each of a list of state groups, optionally
	709	filtering by type/state_key
	710
	711	Args:
	712	groups (iterable[int]): list of state groups for which we want
	713	to get the state.
	714	state_filter (StateFilter): The state filter used to fetch state
	715	from the database.
	716	Returns:
	717	Deferred[dict[int, dict[tuple[str, str], str]]]:
	718	dict of state_group_id -> (dict of (type, state_key) -> event id)
	719	"""
	720
	721	member_filter, non_member_filter = state_filter.get_member_split()
	722
	723	# Now we look them up in the member and non-member caches
	724	non_member_state, incomplete_groups_nm, = (
	725	yield self._get_state_for_groups_using_cache(
	726	groups, self._state_group_cache, state_filter=non_member_filter
	727	)
	728	)
	729
	730	member_state, incomplete_groups_m, = (
	731	yield self._get_state_for_groups_using_cache(
	732	groups, self._state_group_members_cache, state_filter=member_filter
	733	)
	734	)
	735
	736	state = dict(non_member_state)
	737	for group in groups:
	738	state[group].update(member_state[group])
	739
	740	# Now fetch any missing groups from the database
	741
	742	incomplete_groups = incomplete_groups_m \| incomplete_groups_nm
	743
	744	if not incomplete_groups:
	745	return state
	746
	747	cache_sequence_nm = self._state_group_cache.sequence
	748	cache_sequence_m = self._state_group_members_cache.sequence
	749
	750	# Help the cache hit ratio by expanding the filter a bit
	751	db_state_filter = state_filter.return_expanded()
	752
	753	group_to_state_dict = yield self._get_state_groups_from_groups(
	754	list(incomplete_groups), state_filter=db_state_filter
	755	)
	756
	757	# Now lets update the caches
	758	self._insert_into_cache(
	759	group_to_state_dict,
	760	db_state_filter,
	761	cache_seq_num_members=cache_sequence_m,
	762	cache_seq_num_non_members=cache_sequence_nm,
	763	)
	764
	765	# And finally update the result dict, by filtering out any extra
	766	# stuff we pulled out of the database.
	767	for group, group_state_dict in iteritems(group_to_state_dict):
	768	# We just replace any existing entries, as we will have loaded
	769	# everything we need from the database anyway.
	770	state[group] = state_filter.filter_state(group_state_dict)
	771
	772	return state
	773
	774	def _get_state_for_groups_using_cache(self, groups, cache, state_filter):
	775	"""Gets the state at each of a list of state groups, optionally
	776	filtering by type/state_key, querying from a specific cache.
	777
	778	Args:
	779	groups (iterable[int]): list of state groups for which we want
	780	to get the state.
	781	cache (DictionaryCache): the cache of group ids to state dicts which
	782	we will pass through - either the normal state cache or the specific
	783	members state cache.
	784	state_filter (StateFilter): The state filter used to fetch state
	785	from the database.
	786
	787	Returns:
	788	tuple[dict[int, dict[tuple[str, str], str]], set[int]]: Tuple of
	789	dict of state_group_id -> (dict of (type, state_key) -> event id)
	790	of entries in the cache, and the state group ids either missing
	791	from the cache or incomplete.
	792	"""
	793	results = {}
	794	incomplete_groups = set()
	795	for group in set(groups):
	796	state_dict_ids, got_all = self._get_state_for_group_using_cache(
	797	cache, group, state_filter
	798	)
	799	results[group] = state_dict_ids
	800
	801	if not got_all:
	802	incomplete_groups.add(group)
	803
	804	return results, incomplete_groups
	805
	806	def _insert_into_cache(
	807	self,
	808	group_to_state_dict,
	809	state_filter,
	810	cache_seq_num_members,
	811	cache_seq_num_non_members,
	812	):
	813	"""Inserts results from querying the database into the relevant cache.
	814
	815	Args:
	816	group_to_state_dict (dict): The new entries pulled from database.
	817	Map from state group to state dict
	818	state_filter (StateFilter): The state filter used to fetch state
	819	from the database.
	820	cache_seq_num_members (int): Sequence number of member cache since
	821	last lookup in cache
	822	cache_seq_num_non_members (int): Sequence number of member cache since
	823	last lookup in cache
	824	"""
	825
	826	# We need to work out which types we've fetched from the DB for the
	827	# member vs non-member caches. This should be as accurate as possible,
	828	# but can be an underestimate (e.g. when we have wild cards)
	829
	830	member_filter, non_member_filter = state_filter.get_member_split()
	831	if member_filter.is_full():
	832	# We fetched all member events
	833	member_types = None
	834	else:
	835	# `concrete_types()` will only return a subset when there are wild
	836	# cards in the filter, but that's fine.
	837	member_types = member_filter.concrete_types()
	838
	839	if non_member_filter.is_full():
	840	# We fetched all non member events
	841	non_member_types = None
	842	else:
	843	non_member_types = non_member_filter.concrete_types()
	844
	845	for group, group_state_dict in iteritems(group_to_state_dict):
	846	state_dict_members = {}
	847	state_dict_non_members = {}
	848
	849	for k, v in iteritems(group_state_dict):
	850	if k[0] == EventTypes.Member:
	851	state_dict_members[k] = v
	852	else:
	853	state_dict_non_members[k] = v
	854
	855	self._state_group_members_cache.update(
	856	cache_seq_num_members,
	857	key=group,
	858	value=state_dict_members,
	859	fetched_keys=member_types,
	860	)
	861
	862	self._state_group_cache.update(
	863	cache_seq_num_non_members,
	864	key=group,
	865	value=state_dict_non_members,
	866	fetched_keys=non_member_types,
	867	)
	868
	869	def store_state_group(
	870	self, event_id, room_id, prev_group, delta_ids, current_state_ids
	871	):
	872	"""Store a new set of state, returning a newly assigned state group.
	873
	874	Args:
	875	event_id (str): The event ID for which the state was calculated
	876	room_id (str)
	877	prev_group (int\|None): A previous state group for the room, optional.
	878	delta_ids (dict\|None): The delta between state at `prev_group` and
	879	`current_state_ids`, if `prev_group` was given. Same format as
	880	`current_state_ids`.
	881	current_state_ids (dict): The state to store. Map of (type, state_key)
	882	to event_id.
	883
	884	Returns:
	885	Deferred[int]: The state group ID
	886	"""
	887
	888	def _store_state_group_txn(txn):
	889	if current_state_ids is None:
	890	# AFAIK, this can never happen
	891	raise Exception("current_state_ids cannot be None")
	892
	893	state_group = self.database_engine.get_next_state_group_id(txn)
	894
	895	self._simple_insert_txn(
	896	txn,
	897	table="state_groups",
	898	values={"id": state_group, "room_id": room_id, "event_id": event_id},
	899	)
	900
	901	# We persist as a delta if we can, while also ensuring the chain
	902	# of deltas isn't tooo long, as otherwise read performance degrades.
	903	if prev_group:
	904	is_in_db = self._simple_select_one_onecol_txn(
	905	txn,
	906	table="state_groups",
	907	keyvalues={"id": prev_group},
	908	retcol="id",
	909	allow_none=True,
	910	)
	911	if not is_in_db:
	912	raise Exception(
	913	"Trying to persist state with unpersisted prev_group: %r"
	914	% (prev_group,)
	915	)
	916
	917	potential_hops = self._count_state_group_hops_txn(txn, prev_group)
	918	if prev_group and potential_hops < MAX_STATE_DELTA_HOPS:
	919	self._simple_insert_txn(
	920	txn,
	921	table="state_group_edges",
	922	values={"state_group": state_group, "prev_state_group": prev_group},
	923	)
	924
	925	self._simple_insert_many_txn(
	926	txn,
	927	table="state_groups_state",
	928	values=[
	929	{
	930	"state_group": state_group,
	931	"room_id": room_id,
	932	"type": key[0],
	933	"state_key": key[1],
	934	"event_id": state_id,
	935	}
	936	for key, state_id in iteritems(delta_ids)
	937	],
	938	)
	939	else:
	940	self._simple_insert_many_txn(
	941	txn,
	942	table="state_groups_state",
	943	values=[
	944	{
	945	"state_group": state_group,
	946	"room_id": room_id,
	947	"type": key[0],
	948	"state_key": key[1],
	949	"event_id": state_id,
	950	}
	951	for key, state_id in iteritems(current_state_ids)
	952	],
	953	)
	954
	955	# Prefill the state group caches with this group.
	956	# It's fine to use the sequence like this as the state group map
	957	# is immutable. (If the map wasn't immutable then this prefill could
	958	# race with another update)
	959
	960	current_member_state_ids = {
	961	s: ev
	962	for (s, ev) in iteritems(current_state_ids)
	963	if s[0] == EventTypes.Member
	964	}
	965	txn.call_after(
	966	self._state_group_members_cache.update,
	967	self._state_group_members_cache.sequence,
	968	key=state_group,
	969	value=dict(current_member_state_ids),
	970	)
	971
	972	current_non_member_state_ids = {
	973	s: ev
	974	for (s, ev) in iteritems(current_state_ids)
	975	if s[0] != EventTypes.Member
	976	}
	977	txn.call_after(
	978	self._state_group_cache.update,
	979	self._state_group_cache.sequence,
	980	key=state_group,
	981	value=dict(current_non_member_state_ids),
	982	)
	983
	984	return state_group
	985
	986	return self.runInteraction("store_state_group", _store_state_group_txn)
	987
	988
	989	class StateBackgroundUpdateStore(
	990	StateGroupBackgroundUpdateStore, BackgroundUpdateStore
	991	):
	992
	993	STATE_GROUP_DEDUPLICATION_UPDATE_NAME = "state_group_state_deduplication"
	994	STATE_GROUP_INDEX_UPDATE_NAME = "state_group_state_type_index"
	995	CURRENT_STATE_INDEX_UPDATE_NAME = "current_state_members_idx"
	996	EVENT_STATE_GROUP_INDEX_UPDATE_NAME = "event_to_state_groups_sg_index"
	997
	998	def __init__(self, db_conn, hs):
	999	super(StateBackgroundUpdateStore, self).__init__(db_conn, hs)
	1000	self.register_background_update_handler(
	1001	self.STATE_GROUP_DEDUPLICATION_UPDATE_NAME,
	1002	self._background_deduplicate_state,
	1003	)
	1004	self.register_background_update_handler(
	1005	self.STATE_GROUP_INDEX_UPDATE_NAME, self._background_index_state
	1006	)
	1007	self.register_background_index_update(
	1008	self.CURRENT_STATE_INDEX_UPDATE_NAME,
	1009	index_name="current_state_events_member_index",
	1010	table="current_state_events",
	1011	columns=["state_key"],
	1012	where_clause="type='m.room.member'",
	1013	)
	1014	self.register_background_index_update(
	1015	self.EVENT_STATE_GROUP_INDEX_UPDATE_NAME,
	1016	index_name="event_to_state_groups_sg_index",
	1017	table="event_to_state_groups",
	1018	columns=["state_group"],
	1019	)
	1020
	1021	@defer.inlineCallbacks
	1022	def _background_deduplicate_state(self, progress, batch_size):
	1023	"""This background update will slowly deduplicate state by reencoding
	1024	them as deltas.
	1025	"""
	1026	last_state_group = progress.get("last_state_group", 0)
	1027	rows_inserted = progress.get("rows_inserted", 0)
	1028	max_group = progress.get("max_group", None)
	1029
	1030	BATCH_SIZE_SCALE_FACTOR = 100
	1031
	1032	batch_size = max(1, int(batch_size / BATCH_SIZE_SCALE_FACTOR))
	1033
	1034	if max_group is None:
	1035	rows = yield self._execute(
	1036	"_background_deduplicate_state",
	1037	None,
	1038	"SELECT coalesce(max(id), 0) FROM state_groups",
	1039	)
	1040	max_group = rows[0][0]
	1041
	1042	def reindex_txn(txn):
	1043	new_last_state_group = last_state_group
	1044	for count in range(batch_size):
	1045	txn.execute(
	1046	"SELECT id, room_id FROM state_groups"
	1047	" WHERE ? < id AND id <= ?"
	1048	" ORDER BY id ASC"
	1049	" LIMIT 1",
	1050	(new_last_state_group, max_group),
	1051	)
	1052	row = txn.fetchone()
	1053	if row:
	1054	state_group, room_id = row
	1055
	1056	if not row or not state_group:
	1057	return True, count
	1058
	1059	txn.execute(
	1060	"SELECT state_group FROM state_group_edges"
	1061	" WHERE state_group = ?",
	1062	(state_group,),
	1063	)
	1064
	1065	# If we reach a point where we've already started inserting
	1066	# edges we should stop.
	1067	if txn.fetchall():
	1068	return True, count
	1069
	1070	txn.execute(
	1071	"SELECT coalesce(max(id), 0) FROM state_groups"
	1072	" WHERE id < ? AND room_id = ?",
	1073	(state_group, room_id),
	1074	)
	1075	prev_group, = txn.fetchone()
	1076	new_last_state_group = state_group
	1077
	1078	if prev_group:
	1079	potential_hops = self._count_state_group_hops_txn(txn, prev_group)
	1080	if potential_hops >= MAX_STATE_DELTA_HOPS:
	1081	# We want to ensure chains are at most this long,#
	1082	# otherwise read performance degrades.
	1083	continue
	1084
	1085	prev_state = self._get_state_groups_from_groups_txn(
	1086	txn, [prev_group]
	1087	)
	1088	prev_state = prev_state[prev_group]
	1089
	1090	curr_state = self._get_state_groups_from_groups_txn(
	1091	txn, [state_group]
	1092	)
	1093	curr_state = curr_state[state_group]
	1094
	1095	if not set(prev_state.keys()) - set(curr_state.keys()):
	1096	# We can only do a delta if the current has a strict super set
	1097	# of keys
	1098
	1099	delta_state = {
	1100	key: value
	1101	for key, value in iteritems(curr_state)
	1102	if prev_state.get(key, None) != value
	1103	}
	1104
	1105	self._simple_delete_txn(
	1106	txn,
	1107	table="state_group_edges",
	1108	keyvalues={"state_group": state_group},
	1109	)
	1110
	1111	self._simple_insert_txn(
	1112	txn,
	1113	table="state_group_edges",
	1114	values={
	1115	"state_group": state_group,
	1116	"prev_state_group": prev_group,
	1117	},
	1118	)
	1119
	1120	self._simple_delete_txn(
	1121	txn,
	1122	table="state_groups_state",
	1123	keyvalues={"state_group": state_group},
	1124	)
	1125
	1126	self._simple_insert_many_txn(
	1127	txn,
	1128	table="state_groups_state",
	1129	values=[
	1130	{
	1131	"state_group": state_group,
	1132	"room_id": room_id,
	1133	"type": key[0],
	1134	"state_key": key[1],
	1135	"event_id": state_id,
	1136	}
	1137	for key, state_id in iteritems(delta_state)
	1138	],
	1139	)
	1140
	1141	progress = {
	1142	"last_state_group": state_group,
	1143	"rows_inserted": rows_inserted + batch_size,
	1144	"max_group": max_group,
	1145	}
	1146
	1147	self._background_update_progress_txn(
	1148	txn, self.STATE_GROUP_DEDUPLICATION_UPDATE_NAME, progress
	1149	)
	1150
	1151	return False, batch_size
	1152
	1153	finished, result = yield self.runInteraction(
	1154	self.STATE_GROUP_DEDUPLICATION_UPDATE_NAME, reindex_txn
	1155	)
	1156
	1157	if finished:
	1158	yield self._end_background_update(
	1159	self.STATE_GROUP_DEDUPLICATION_UPDATE_NAME
	1160	)
	1161
	1162	return result * BATCH_SIZE_SCALE_FACTOR
	1163
	1164	@defer.inlineCallbacks
	1165	def _background_index_state(self, progress, batch_size):
	1166	def reindex_txn(conn):
	1167	conn.rollback()
	1168	if isinstance(self.database_engine, PostgresEngine):
	1169	# postgres insists on autocommit for the index
	1170	conn.set_session(autocommit=True)
	1171	try:
	1172	txn = conn.cursor()
	1173	txn.execute(
	1174	"CREATE INDEX CONCURRENTLY state_groups_state_type_idx"
	1175	" ON state_groups_state(state_group, type, state_key)"
	1176	)
	1177	txn.execute("DROP INDEX IF EXISTS state_groups_state_id")
	1178	finally:
	1179	conn.set_session(autocommit=False)
	1180	else:
	1181	txn = conn.cursor()
	1182	txn.execute(
	1183	"CREATE INDEX state_groups_state_type_idx"
	1184	" ON state_groups_state(state_group, type, state_key)"
	1185	)
	1186	txn.execute("DROP INDEX IF EXISTS state_groups_state_id")
	1187
	1188	yield self.runWithConnection(reindex_txn)
	1189
	1190	yield self._end_background_update(self.STATE_GROUP_INDEX_UPDATE_NAME)
	1191
	1192	return 1
	1193
	1194
	1195	class StateStore(StateGroupWorkerStore, StateBackgroundUpdateStore):
	1196	""" Keeps track of the state at a given event.
	1197
	1198	This is done by the concept of `state groups`. Every event is a assigned
	1199	a state group (identified by an arbitrary string), which references a
	1200	collection of state events. The current state of an event is then the
	1201	collection of state events referenced by the event's state group.
	1202
	1203	Hence, every change in the current state causes a new state group to be
	1204	generated. However, if no change happens (e.g., if we get a message event
	1205	with only one parent it inherits the state group from its parent.)
	1206
	1207	There are three tables:
	1208	* `state_groups`: Stores group name, first event with in the group and
	1209	room id.
	1210	* `event_to_state_groups`: Maps events to state groups.
	1211	* `state_groups_state`: Maps state group to state events.
	1212	"""
	1213
	1214	def __init__(self, db_conn, hs):
	1215	super(StateStore, self).__init__(db_conn, hs)
	1216
	1217	def _store_event_state_mappings_txn(self, txn, events_and_contexts):
	1218	state_groups = {}
	1219	for event, context in events_and_contexts:
	1220	if event.internal_metadata.is_outlier():
	1221	continue
	1222
	1223	# if the event was rejected, just give it the same state as its
	1224	# predecessor.
	1225	if context.rejected:
	1226	state_groups[event.event_id] = context.prev_group
	1227	continue
	1228
	1229	state_groups[event.event_id] = context.state_group
	1230
	1231	self._simple_insert_many_txn(
	1232	txn,
	1233	table="event_to_state_groups",
	1234	values=[
	1235	{"state_group": state_group_id, "event_id": event_id}
	1236	for event_id, state_group_id in iteritems(state_groups)
	1237	],
	1238	)
	1239
	1240	for event_id, state_group_id in iteritems(state_groups):
	1241	txn.call_after(
	1242	self._get_state_group_for_event.prefill, (event_id,), state_group_id
	1243	)

+119

-0

synapse/storage/data_stores/main/state_deltas.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2018 Vector Creations Ltd
	2	#
	3	# Licensed under the Apache License, Version 2.0 (the "License");
	4	# you may not use this file except in compliance with the License.
	5	# You may obtain a copy of the License at
	6	#
	7	# http://www.apache.org/licenses/LICENSE-2.0
	8	#
	9	# Unless required by applicable law or agreed to in writing, software
	10	# distributed under the License is distributed on an "AS IS" BASIS,
	11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	# See the License for the specific language governing permissions and
	13	# limitations under the License.
	14
	15	import logging
	16
	17	from synapse.storage._base import SQLBaseStore
	18
	19	logger = logging.getLogger(__name__)
	20
	21
	22	class StateDeltasStore(SQLBaseStore):
	23	def get_current_state_deltas(self, prev_stream_id: int, max_stream_id: int):
	24	"""Fetch a list of room state changes since the given stream id
	25
	26	Each entry in the result contains the following fields:
	27	- stream_id (int)
	28	- room_id (str)
	29	- type (str): event type
	30	- state_key (str):
	31	- event_id (str\|None): new event_id for this state key. None if the
	32	state has been deleted.
	33	- prev_event_id (str\|None): previous event_id for this state key. None
	34	if it's new state.
	35
	36	Args:
	37	prev_stream_id (int): point to get changes since (exclusive)
	38	max_stream_id (int): the point that we know has been correctly persisted
	39	- ie, an upper limit to return changes from.
	40
	41	Returns:
	42	Deferred[tuple[int, list[dict]]: A tuple consisting of:
	43	- the stream id which these results go up to
	44	- list of current_state_delta_stream rows. If it is empty, we are
	45	up to date.
	46	"""
	47	prev_stream_id = int(prev_stream_id)
	48
	49	# check we're not going backwards
	50	assert prev_stream_id <= max_stream_id
	51
	52	if not self._curr_state_delta_stream_cache.has_any_entity_changed(
	53	prev_stream_id
	54	):
	55	# if the CSDs haven't changed between prev_stream_id and now, we
	56	# know for certain that they haven't changed between prev_stream_id and
	57	# max_stream_id.
	58	return max_stream_id, []
	59
	60	def get_current_state_deltas_txn(txn):
	61	# First we calculate the max stream id that will give us less than
	62	# N results.
	63	# We arbitarily limit to 100 stream_id entries to ensure we don't
	64	# select toooo many.
	65	sql = """
	66	SELECT stream_id, count(*)
	67	FROM current_state_delta_stream
	68	WHERE stream_id > ? AND stream_id <= ?
	69	GROUP BY stream_id
	70	ORDER BY stream_id ASC
	71	LIMIT 100
	72	"""
	73	txn.execute(sql, (prev_stream_id, max_stream_id))
	74
	75	total = 0
	76
	77	for stream_id, count in txn:
	78	total += count
	79	if total > 100:
	80	# We arbitarily limit to 100 entries to ensure we don't
	81	# select toooo many.
	82	logger.debug(
	83	"Clipping current_state_delta_stream rows to stream_id %i",
	84	stream_id,
	85	)
	86	clipped_stream_id = stream_id
	87	break
	88	else:
	89	# if there's no problem, we may as well go right up to the max_stream_id
	90	clipped_stream_id = max_stream_id
	91
	92	# Now actually get the deltas
	93	sql = """
	94	SELECT stream_id, room_id, type, state_key, event_id, prev_event_id
	95	FROM current_state_delta_stream
	96	WHERE ? < stream_id AND stream_id <= ?
	97	ORDER BY stream_id ASC
	98	"""
	99	txn.execute(sql, (prev_stream_id, clipped_stream_id))
	100	return clipped_stream_id, self.cursor_to_dict(txn)
	101
	102	return self.runInteraction(
	103	"get_current_state_deltas", get_current_state_deltas_txn
	104	)
	105
	106	def _get_max_stream_id_in_current_state_deltas_txn(self, txn):
	107	return self._simple_select_one_onecol_txn(
	108	txn,
	109	table="current_state_delta_stream",
	110	keyvalues={},
	111	retcol="COALESCE(MAX(stream_id), -1)",
	112	)
	113
	114	def get_max_stream_id_in_current_state_deltas(self):
	115	return self.runInteraction(
	116	"get_max_stream_id_in_current_state_deltas",
	117	self._get_max_stream_id_in_current_state_deltas_txn,
	118	)

+881

-0

synapse/storage/data_stores/main/stats.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2018, 2019 New Vector Ltd
	2	# Copyright 2019 The Matrix.org Foundation C.I.C.
	3	#
	4	# Licensed under the Apache License, Version 2.0 (the "License");
	5	# you may not use this file except in compliance with the License.
	6	# You may obtain a copy of the License at
	7	#
	8	# http://www.apache.org/licenses/LICENSE-2.0
	9	#
	10	# Unless required by applicable law or agreed to in writing, software
	11	# distributed under the License is distributed on an "AS IS" BASIS,
	12	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	# See the License for the specific language governing permissions and
	14	# limitations under the License.
	15
	16	import logging
	17	from itertools import chain
	18
	19	from twisted.internet import defer
	20	from twisted.internet.defer import DeferredLock
	21
	22	from synapse.api.constants import EventTypes, Membership
	23	from synapse.storage.data_stores.main.state_deltas import StateDeltasStore
	24	from synapse.storage.engines import PostgresEngine
	25	from synapse.util.caches.descriptors import cached
	26
	27	logger = logging.getLogger(__name__)
	28
	29	# these fields track absolutes (e.g. total number of rooms on the server)
	30	# You can think of these as Prometheus Gauges.
	31	# You can draw these stats on a line graph.
	32	# Example: number of users in a room
	33	ABSOLUTE_STATS_FIELDS = {
	34	"room": (
	35	"current_state_events",
	36	"joined_members",
	37	"invited_members",
	38	"left_members",
	39	"banned_members",
	40	"local_users_in_room",
	41	),
	42	"user": ("joined_rooms",),
	43	}
	44
	45	# these fields are per-timeslice and so should be reset to 0 upon a new slice
	46	# You can draw these stats on a histogram.
	47	# Example: number of events sent locally during a time slice
	48	PER_SLICE_FIELDS = {
	49	"room": ("total_events", "total_event_bytes"),
	50	"user": ("invites_sent", "rooms_created", "total_events", "total_event_bytes"),
	51	}
	52
	53	TYPE_TO_TABLE = {"room": ("room_stats", "room_id"), "user": ("user_stats", "user_id")}
	54
	55	# these are the tables (& ID columns) which contain our actual subjects
	56	TYPE_TO_ORIGIN_TABLE = {"room": ("rooms", "room_id"), "user": ("users", "name")}
	57
	58
	59	class StatsStore(StateDeltasStore):
	60	def __init__(self, db_conn, hs):
	61	super(StatsStore, self).__init__(db_conn, hs)
	62
	63	self.server_name = hs.hostname
	64	self.clock = self.hs.get_clock()
	65	self.stats_enabled = hs.config.stats_enabled
	66	self.stats_bucket_size = hs.config.stats_bucket_size
	67
	68	self.stats_delta_processing_lock = DeferredLock()
	69
	70	self.register_background_update_handler(
	71	"populate_stats_process_rooms", self._populate_stats_process_rooms
	72	)
	73	self.register_background_update_handler(
	74	"populate_stats_process_users", self._populate_stats_process_users
	75	)
	76	# we no longer need to perform clean-up, but we will give ourselves
	77	# the potential to reintroduce it in the future – so documentation
	78	# will still encourage the use of this no-op handler.
	79	self.register_noop_background_update("populate_stats_cleanup")
	80	self.register_noop_background_update("populate_stats_prepare")
	81
	82	def quantise_stats_time(self, ts):
	83	"""
	84	Quantises a timestamp to be a multiple of the bucket size.
	85
	86	Args:
	87	ts (int): the timestamp to quantise, in milliseconds since the Unix
	88	Epoch
	89
	90	Returns:
	91	int: a timestamp which
	92	- is divisible by the bucket size;
	93	- is no later than `ts`; and
	94	- is the largest such timestamp.
	95	"""
	96	return (ts // self.stats_bucket_size) * self.stats_bucket_size
	97
	98	@defer.inlineCallbacks
	99	def _populate_stats_process_users(self, progress, batch_size):
	100	"""
	101	This is a background update which regenerates statistics for users.
	102	"""
	103	if not self.stats_enabled:
	104	yield self._end_background_update("populate_stats_process_users")
	105	return 1
	106
	107	last_user_id = progress.get("last_user_id", "")
	108
	109	def _get_next_batch(txn):
	110	sql = """
	111	SELECT DISTINCT name FROM users
	112	WHERE name > ?
	113	ORDER BY name ASC
	114	LIMIT ?
	115	"""
	116	txn.execute(sql, (last_user_id, batch_size))
	117	return [r for r, in txn]
	118
	119	users_to_work_on = yield self.runInteraction(
	120	"_populate_stats_process_users", _get_next_batch
	121	)
	122
	123	# No more rooms -- complete the transaction.
	124	if not users_to_work_on:
	125	yield self._end_background_update("populate_stats_process_users")
	126	return 1
	127
	128	for user_id in users_to_work_on:
	129	yield self._calculate_and_set_initial_state_for_user(user_id)
	130	progress["last_user_id"] = user_id
	131
	132	yield self.runInteraction(
	133	"populate_stats_process_users",
	134	self._background_update_progress_txn,
	135	"populate_stats_process_users",
	136	progress,
	137	)
	138
	139	return len(users_to_work_on)
	140
	141	@defer.inlineCallbacks
	142	def _populate_stats_process_rooms(self, progress, batch_size):
	143	"""
	144	This is a background update which regenerates statistics for rooms.
	145	"""
	146	if not self.stats_enabled:
	147	yield self._end_background_update("populate_stats_process_rooms")
	148	return 1
	149
	150	last_room_id = progress.get("last_room_id", "")
	151
	152	def _get_next_batch(txn):
	153	sql = """
	154	SELECT DISTINCT room_id FROM current_state_events
	155	WHERE room_id > ?
	156	ORDER BY room_id ASC
	157	LIMIT ?
	158	"""
	159	txn.execute(sql, (last_room_id, batch_size))
	160	return [r for r, in txn]
	161
	162	rooms_to_work_on = yield self.runInteraction(
	163	"populate_stats_rooms_get_batch", _get_next_batch
	164	)
	165
	166	# No more rooms -- complete the transaction.
	167	if not rooms_to_work_on:
	168	yield self._end_background_update("populate_stats_process_rooms")
	169	return 1
	170
	171	for room_id in rooms_to_work_on:
	172	yield self._calculate_and_set_initial_state_for_room(room_id)
	173	progress["last_room_id"] = room_id
	174
	175	yield self.runInteraction(
	176	"_populate_stats_process_rooms",
	177	self._background_update_progress_txn,
	178	"populate_stats_process_rooms",
	179	progress,
	180	)
	181
	182	return len(rooms_to_work_on)
	183
	184	def get_stats_positions(self):
	185	"""
	186	Returns the stats processor positions.
	187	"""
	188	return self._simple_select_one_onecol(
	189	table="stats_incremental_position",
	190	keyvalues={},
	191	retcol="stream_id",
	192	desc="stats_incremental_position",
	193	)
	194
	195	def update_room_state(self, room_id, fields):
	196	"""
	197	Args:
	198	room_id (str)
	199	fields (dict[str:Any])
	200	"""
	201
	202	# For whatever reason some of the fields may contain null bytes, which
	203	# postgres isn't a fan of, so we replace those fields with null.
	204	for col in (
	205	"join_rules",
	206	"history_visibility",
	207	"encryption",
	208	"name",
	209	"topic",
	210	"avatar",
	211	"canonical_alias",
	212	):
	213	field = fields.get(col)
	214	if field and "\0" in field:
	215	fields[col] = None
	216
	217	return self._simple_upsert(
	218	table="room_stats_state",
	219	keyvalues={"room_id": room_id},
	220	values=fields,
	221	desc="update_room_state",
	222	)
	223
	224	def get_statistics_for_subject(self, stats_type, stats_id, start, size=100):
	225	"""
	226	Get statistics for a given subject.
	227
	228	Args:
	229	stats_type (str): The type of subject
	230	stats_id (str): The ID of the subject (e.g. room_id or user_id)
	231	start (int): Pagination start. Number of entries, not timestamp.
	232	size (int): How many entries to return.
	233
	234	Returns:
	235	Deferred[list[dict]], where the dict has the keys of
	236	ABSOLUTE_STATS_FIELDS[stats_type], and "bucket_size" and "end_ts".
	237	"""
	238	return self.runInteraction(
	239	"get_statistics_for_subject",
	240	self._get_statistics_for_subject_txn,
	241	stats_type,
	242	stats_id,
	243	start,
	244	size,
	245	)
	246
	247	def _get_statistics_for_subject_txn(
	248	self, txn, stats_type, stats_id, start, size=100
	249	):
	250	"""
	251	Transaction-bound version of L{get_statistics_for_subject}.
	252	"""
	253
	254	table, id_col = TYPE_TO_TABLE[stats_type]
	255	selected_columns = list(
	256	ABSOLUTE_STATS_FIELDS[stats_type] + PER_SLICE_FIELDS[stats_type]
	257	)
	258
	259	slice_list = self._simple_select_list_paginate_txn(
	260	txn,
	261	table + "_historical",
	262	{id_col: stats_id},
	263	"end_ts",
	264	start,
	265	size,
	266	retcols=selected_columns + ["bucket_size", "end_ts"],
	267	order_direction="DESC",
	268	)
	269
	270	return slice_list
	271
	272	def get_room_stats_state(self, room_id):
	273	"""
	274	Returns the current room_stats_state for a room.
	275
	276	Args:
	277	room_id (str): The ID of the room to return state for.
	278
	279	Returns (dict):
	280	Dictionary containing these keys:
	281	"name", "topic", "canonical_alias", "avatar", "join_rules",
	282	"history_visibility"
	283	"""
	284	return self._simple_select_one(
	285	"room_stats_state",
	286	{"room_id": room_id},
	287	retcols=(
	288	"name",
	289	"topic",
	290	"canonical_alias",
	291	"avatar",
	292	"join_rules",
	293	"history_visibility",
	294	),
	295	)
	296
	297	@cached()
	298	def get_earliest_token_for_stats(self, stats_type, id):
	299	"""
	300	Fetch the "earliest token". This is used by the room stats delta
	301	processor to ignore deltas that have been processed between the
	302	start of the background task and any particular room's stats
	303	being calculated.
	304
	305	Returns:
	306	Deferred[int]
	307	"""
	308	table, id_col = TYPE_TO_TABLE[stats_type]
	309
	310	return self._simple_select_one_onecol(
	311	"%s_current" % (table,),
	312	keyvalues={id_col: id},
	313	retcol="completed_delta_stream_id",
	314	allow_none=True,
	315	)
	316
	317	def bulk_update_stats_delta(self, ts, updates, stream_id):
	318	"""Bulk update stats tables for a given stream_id and updates the stats
	319	incremental position.
	320
	321	Args:
	322	ts (int): Current timestamp in ms
	323	updates(dict[str, dict[str, dict[str, Counter]]]): The updates to
	324	commit as a mapping stats_type -> stats_id -> field -> delta.
	325	stream_id (int): Current position.
	326
	327	Returns:
	328	Deferred
	329	"""
	330
	331	def _bulk_update_stats_delta_txn(txn):
	332	for stats_type, stats_updates in updates.items():
	333	for stats_id, fields in stats_updates.items():
	334	logger.info(
	335	"Updating %s stats for %s: %s", stats_type, stats_id, fields
	336	)
	337	self._update_stats_delta_txn(
	338	txn,
	339	ts=ts,
	340	stats_type=stats_type,
	341	stats_id=stats_id,
	342	fields=fields,
	343	complete_with_stream_id=stream_id,
	344	)
	345
	346	self._simple_update_one_txn(
	347	txn,
	348	table="stats_incremental_position",
	349	keyvalues={},
	350	updatevalues={"stream_id": stream_id},
	351	)
	352
	353	return self.runInteraction(
	354	"bulk_update_stats_delta", _bulk_update_stats_delta_txn
	355	)
	356
	357	def update_stats_delta(
	358	self,
	359	ts,
	360	stats_type,
	361	stats_id,
	362	fields,
	363	complete_with_stream_id,
	364	absolute_field_overrides=None,
	365	):
	366	"""
	367	Updates the statistics for a subject, with a delta (difference/relative
	368	change).
	369
	370	Args:
	371	ts (int): timestamp of the change
	372	stats_type (str): "room" or "user" – the kind of subject
	373	stats_id (str): the subject's ID (room ID or user ID)
	374	fields (dict[str, int]): Deltas of stats values.
	375	complete_with_stream_id (int, optional):
	376	If supplied, converts an incomplete row into a complete row,
	377	with the supplied stream_id marked as the stream_id where the
	378	row was completed.
	379	absolute_field_overrides (dict[str, int]): Current stats values
	380	(i.e. not deltas) of absolute fields.
	381	Does not work with per-slice fields.
	382	"""
	383
	384	return self.runInteraction(
	385	"update_stats_delta",
	386	self._update_stats_delta_txn,
	387	ts,
	388	stats_type,
	389	stats_id,
	390	fields,
	391	complete_with_stream_id=complete_with_stream_id,
	392	absolute_field_overrides=absolute_field_overrides,
	393	)
	394
	395	def _update_stats_delta_txn(
	396	self,
	397	txn,
	398	ts,
	399	stats_type,
	400	stats_id,
	401	fields,
	402	complete_with_stream_id,
	403	absolute_field_overrides=None,
	404	):
	405	if absolute_field_overrides is None:
	406	absolute_field_overrides = {}
	407
	408	table, id_col = TYPE_TO_TABLE[stats_type]
	409
	410	quantised_ts = self.quantise_stats_time(int(ts))
	411	end_ts = quantised_ts + self.stats_bucket_size
	412
	413	# Lets be paranoid and check that all the given field names are known
	414	abs_field_names = ABSOLUTE_STATS_FIELDS[stats_type]
	415	slice_field_names = PER_SLICE_FIELDS[stats_type]
	416	for field in chain(fields.keys(), absolute_field_overrides.keys()):
	417	if field not in abs_field_names and field not in slice_field_names:
	418	# guard against potential SQL injection dodginess
	419	raise ValueError(
	420	"%s is not a recognised field"
	421	" for stats type %s" % (field, stats_type)
	422	)
	423
	424	# Per slice fields do not get added to the _current table
	425
	426	# This calculates the deltas (`field = field + ?` values)
	427	# for absolute fields,
	428	# * defaulting to 0 if not specified
	429	# (required for the INSERT part of upserting to work)
	430	# * omitting overrides specified in `absolute_field_overrides`
	431	deltas_of_absolute_fields = {
	432	key: fields.get(key, 0)
	433	for key in abs_field_names
	434	if key not in absolute_field_overrides
	435	}
	436
	437	# Keep the delta stream ID field up to date
	438	absolute_field_overrides = absolute_field_overrides.copy()
	439	absolute_field_overrides["completed_delta_stream_id"] = complete_with_stream_id
	440
	441	# first upsert the `_current` table
	442	self._upsert_with_additive_relatives_txn(
	443	txn=txn,
	444	table=table + "_current",
	445	keyvalues={id_col: stats_id},
	446	absolutes=absolute_field_overrides,
	447	additive_relatives=deltas_of_absolute_fields,
	448	)
	449
	450	per_slice_additive_relatives = {
	451	key: fields.get(key, 0) for key in slice_field_names
	452	}
	453	self._upsert_copy_from_table_with_additive_relatives_txn(
	454	txn=txn,
	455	into_table=table + "_historical",
	456	keyvalues={id_col: stats_id},
	457	extra_dst_insvalues={"bucket_size": self.stats_bucket_size},
	458	extra_dst_keyvalues={"end_ts": end_ts},
	459	additive_relatives=per_slice_additive_relatives,
	460	src_table=table + "_current",
	461	copy_columns=abs_field_names,
	462	)
	463
	464	def _upsert_with_additive_relatives_txn(
	465	self, txn, table, keyvalues, absolutes, additive_relatives
	466	):
	467	"""Used to update values in the stats tables.
	468
	469	This is basically a slightly convoluted upsert that adds to any
	470	existing rows.
	471
	472	Args:
	473	txn
	474	table (str): Table name
	475	keyvalues (dict[str, any]): Row-identifying key values
	476	absolutes (dict[str, any]): Absolute (set) fields
	477	additive_relatives (dict[str, int]): Fields that will be added onto
	478	if existing row present.
	479	"""
	480	if self.database_engine.can_native_upsert:
	481	absolute_updates = [
	482	"%(field)s = EXCLUDED.%(field)s" % {"field": field}
	483	for field in absolutes.keys()
	484	]
	485
	486	relative_updates = [
	487	"%(field)s = EXCLUDED.%(field)s + %(table)s.%(field)s"
	488	% {"table": table, "field": field}
	489	for field in additive_relatives.keys()
	490	]
	491
	492	insert_cols = []
	493	qargs = []
	494
	495	for (key, val) in chain(
	496	keyvalues.items(), absolutes.items(), additive_relatives.items()
	497	):
	498	insert_cols.append(key)
	499	qargs.append(val)
	500
	501	sql = """
	502	INSERT INTO %(table)s (%(insert_cols_cs)s)
	503	VALUES (%(insert_vals_qs)s)
	504	ON CONFLICT (%(key_columns)s) DO UPDATE SET %(updates)s
	505	""" % {
	506	"table": table,
	507	"insert_cols_cs": ", ".join(insert_cols),
	508	"insert_vals_qs": ", ".join(
	509	["?"] * (len(keyvalues) + len(absolutes) + len(additive_relatives))
	510	),
	511	"key_columns": ", ".join(keyvalues),
	512	"updates": ", ".join(chain(absolute_updates, relative_updates)),
	513	}
	514
	515	txn.execute(sql, qargs)
	516	else:
	517	self.database_engine.lock_table(txn, table)
	518	retcols = list(chain(absolutes.keys(), additive_relatives.keys()))
	519	current_row = self._simple_select_one_txn(
	520	txn, table, keyvalues, retcols, allow_none=True
	521	)
	522	if current_row is None:
	523	merged_dict = {keyvalues, absolutes, **additive_relatives}
	524	self._simple_insert_txn(txn, table, merged_dict)
	525	else:
	526	for (key, val) in additive_relatives.items():
	527	current_row[key] += val
	528	current_row.update(absolutes)
	529	self._simple_update_one_txn(txn, table, keyvalues, current_row)
	530
	531	def _upsert_copy_from_table_with_additive_relatives_txn(
	532	self,
	533	txn,
	534	into_table,
	535	keyvalues,
	536	extra_dst_keyvalues,
	537	extra_dst_insvalues,
	538	additive_relatives,
	539	src_table,
	540	copy_columns,
	541	):
	542	"""Updates the historic stats table with latest updates.
	543
	544	This involves copying "absolute" fields from the `_current` table, and
	545	adding relative fields to any existing values.
	546
	547	Args:
	548	txn: Transaction
	549	into_table (str): The destination table to UPSERT the row into
	550	keyvalues (dict[str, any]): Row-identifying key values
	551	extra_dst_keyvalues (dict[str, any]): Additional keyvalues
	552	for `into_table`.
	553	extra_dst_insvalues (dict[str, any]): Additional values to insert
	554	on new row creation for `into_table`.
	555	additive_relatives (dict[str, any]): Fields that will be added onto
	556	if existing row present. (Must be disjoint from copy_columns.)
	557	src_table (str): The source table to copy from
	558	copy_columns (iterable[str]): The list of columns to copy
	559	"""
	560	if self.database_engine.can_native_upsert:
	561	ins_columns = chain(
	562	keyvalues,
	563	copy_columns,
	564	additive_relatives,
	565	extra_dst_keyvalues,
	566	extra_dst_insvalues,
	567	)
	568	sel_exprs = chain(
	569	keyvalues,
	570	copy_columns,
	571	(
	572	"?"
	573	for _ in chain(
	574	additive_relatives, extra_dst_keyvalues, extra_dst_insvalues
	575	)
	576	),
	577	)
	578	keyvalues_where = ("%s = ?" % f for f in keyvalues)
	579
	580	sets_cc = ("%s = EXCLUDED.%s" % (f, f) for f in copy_columns)
	581	sets_ar = (
	582	"%s = EXCLUDED.%s + %s.%s" % (f, f, into_table, f)
	583	for f in additive_relatives
	584	)
	585
	586	sql = """
	587	INSERT INTO %(into_table)s (%(ins_columns)s)
	588	SELECT %(sel_exprs)s
	589	FROM %(src_table)s
	590	WHERE %(keyvalues_where)s
	591	ON CONFLICT (%(keyvalues)s)
	592	DO UPDATE SET %(sets)s
	593	""" % {
	594	"into_table": into_table,
	595	"ins_columns": ", ".join(ins_columns),
	596	"sel_exprs": ", ".join(sel_exprs),
	597	"keyvalues_where": " AND ".join(keyvalues_where),
	598	"src_table": src_table,
	599	"keyvalues": ", ".join(
	600	chain(keyvalues.keys(), extra_dst_keyvalues.keys())
	601	),
	602	"sets": ", ".join(chain(sets_cc, sets_ar)),
	603	}
	604
	605	qargs = list(
	606	chain(
	607	additive_relatives.values(),
	608	extra_dst_keyvalues.values(),
	609	extra_dst_insvalues.values(),
	610	keyvalues.values(),
	611	)
	612	)
	613	txn.execute(sql, qargs)
	614	else:
	615	self.database_engine.lock_table(txn, into_table)
	616	src_row = self._simple_select_one_txn(
	617	txn, src_table, keyvalues, copy_columns
	618	)
	619	all_dest_keyvalues = {keyvalues, extra_dst_keyvalues}
	620	dest_current_row = self._simple_select_one_txn(
	621	txn,
	622	into_table,
	623	keyvalues=all_dest_keyvalues,
	624	retcols=list(chain(additive_relatives.keys(), copy_columns)),
	625	allow_none=True,
	626	)
	627
	628	if dest_current_row is None:
	629	merged_dict = {
	630	**keyvalues,
	631	**extra_dst_keyvalues,
	632	**extra_dst_insvalues,
	633	**src_row,
	634	**additive_relatives,
	635	}
	636	self._simple_insert_txn(txn, into_table, merged_dict)
	637	else:
	638	for (key, val) in additive_relatives.items():
	639	src_row[key] = dest_current_row[key] + val
	640	self._simple_update_txn(txn, into_table, all_dest_keyvalues, src_row)
	641
	642	def get_changes_room_total_events_and_bytes(self, min_pos, max_pos):
	643	"""Fetches the counts of events in the given range of stream IDs.
	644
	645	Args:
	646	min_pos (int)
	647	max_pos (int)
	648
	649	Returns:
	650	Deferred[dict[str, dict[str, int]]]: Mapping of room ID to field
	651	changes.
	652	"""
	653
	654	return self.runInteraction(
	655	"stats_incremental_total_events_and_bytes",
	656	self.get_changes_room_total_events_and_bytes_txn,
	657	min_pos,
	658	max_pos,
	659	)
	660
	661	def get_changes_room_total_events_and_bytes_txn(self, txn, low_pos, high_pos):
	662	"""Gets the total_events and total_event_bytes counts for rooms and
	663	senders, in a range of stream_orderings (including backfilled events).
	664
	665	Args:
	666	txn
	667	low_pos (int): Low stream ordering
	668	high_pos (int): High stream ordering
	669
	670	Returns:
	671	tuple[dict[str, dict[str, int]], dict[str, dict[str, int]]]: The
	672	room and user deltas for total_events/total_event_bytes in the
	673	format of `stats_id` -> fields
	674	"""
	675
	676	if low_pos >= high_pos:
	677	# nothing to do here.
	678	return {}, {}
	679
	680	if isinstance(self.database_engine, PostgresEngine):
	681	new_bytes_expression = "OCTET_LENGTH(json)"
	682	else:
	683	new_bytes_expression = "LENGTH(CAST(json AS BLOB))"
	684
	685	sql = """
	686	SELECT events.room_id, COUNT(*) AS new_events, SUM(%s) AS new_bytes
	687	FROM events INNER JOIN event_json USING (event_id)
	688	WHERE (? < stream_ordering AND stream_ordering <= ?)
	689	OR (? <= stream_ordering AND stream_ordering <= ?)
	690	GROUP BY events.room_id
	691	""" % (
	692	new_bytes_expression,
	693	)
	694
	695	txn.execute(sql, (low_pos, high_pos, -high_pos, -low_pos))
	696
	697	room_deltas = {
	698	room_id: {"total_events": new_events, "total_event_bytes": new_bytes}
	699	for room_id, new_events, new_bytes in txn
	700	}
	701
	702	sql = """
	703	SELECT events.sender, COUNT(*) AS new_events, SUM(%s) AS new_bytes
	704	FROM events INNER JOIN event_json USING (event_id)
	705	WHERE (? < stream_ordering AND stream_ordering <= ?)
	706	OR (? <= stream_ordering AND stream_ordering <= ?)
	707	GROUP BY events.sender
	708	""" % (
	709	new_bytes_expression,
	710	)
	711
	712	txn.execute(sql, (low_pos, high_pos, -high_pos, -low_pos))
	713
	714	user_deltas = {
	715	user_id: {"total_events": new_events, "total_event_bytes": new_bytes}
	716	for user_id, new_events, new_bytes in txn
	717	if self.hs.is_mine_id(user_id)
	718	}
	719
	720	return room_deltas, user_deltas
	721
	722	@defer.inlineCallbacks
	723	def _calculate_and_set_initial_state_for_room(self, room_id):
	724	"""Calculate and insert an entry into room_stats_current.
	725
	726	Args:
	727	room_id (str)
	728
	729	Returns:
	730	Deferred[tuple[dict, dict, int]]: A tuple of room state, membership
	731	counts and stream position.
	732	"""
	733
	734	def _fetch_current_state_stats(txn):
	735	pos = self.get_room_max_stream_ordering()
	736
	737	rows = self._simple_select_many_txn(
	738	txn,
	739	table="current_state_events",
	740	column="type",
	741	iterable=[
	742	EventTypes.Create,
	743	EventTypes.JoinRules,
	744	EventTypes.RoomHistoryVisibility,
	745	EventTypes.Encryption,
	746	EventTypes.Name,
	747	EventTypes.Topic,
	748	EventTypes.RoomAvatar,
	749	EventTypes.CanonicalAlias,
	750	],
	751	keyvalues={"room_id": room_id, "state_key": ""},
	752	retcols=["event_id"],
	753	)
	754
	755	event_ids = [row["event_id"] for row in rows]
	756
	757	txn.execute(
	758	"""
	759	SELECT membership, count(*) FROM current_state_events
	760	WHERE room_id = ? AND type = 'm.room.member'
	761	GROUP BY membership
	762	""",
	763	(room_id,),
	764	)
	765	membership_counts = {membership: cnt for membership, cnt in txn}
	766
	767	txn.execute(
	768	"""
	769	SELECT COALESCE(count(*), 0) FROM current_state_events
	770	WHERE room_id = ?
	771	""",
	772	(room_id,),
	773	)
	774
	775	current_state_events_count, = txn.fetchone()
	776
	777	users_in_room = self.get_users_in_room_txn(txn, room_id)
	778
	779	return (
	780	event_ids,
	781	membership_counts,
	782	current_state_events_count,
	783	users_in_room,
	784	pos,
	785	)
	786
	787	(
	788	event_ids,
	789	membership_counts,
	790	current_state_events_count,
	791	users_in_room,
	792	pos,
	793	) = yield self.runInteraction(
	794	"get_initial_state_for_room", _fetch_current_state_stats
	795	)
	796
	797	state_event_map = yield self.get_events(event_ids, get_prev_content=False)
	798
	799	room_state = {
	800	"join_rules": None,
	801	"history_visibility": None,
	802	"encryption": None,
	803	"name": None,
	804	"topic": None,
	805	"avatar": None,
	806	"canonical_alias": None,
	807	"is_federatable": True,
	808	}
	809
	810	for event in state_event_map.values():
	811	if event.type == EventTypes.JoinRules:
	812	room_state["join_rules"] = event.content.get("join_rule")
	813	elif event.type == EventTypes.RoomHistoryVisibility:
	814	room_state["history_visibility"] = event.content.get(
	815	"history_visibility"
	816	)
	817	elif event.type == EventTypes.Encryption:
	818	room_state["encryption"] = event.content.get("algorithm")
	819	elif event.type == EventTypes.Name:
	820	room_state["name"] = event.content.get("name")
	821	elif event.type == EventTypes.Topic:
	822	room_state["topic"] = event.content.get("topic")
	823	elif event.type == EventTypes.RoomAvatar:
	824	room_state["avatar"] = event.content.get("url")
	825	elif event.type == EventTypes.CanonicalAlias:
	826	room_state["canonical_alias"] = event.content.get("alias")
	827	elif event.type == EventTypes.Create:
	828	room_state["is_federatable"] = (
	829	event.content.get("m.federate", True) is True
	830	)
	831
	832	yield self.update_room_state(room_id, room_state)
	833
	834	local_users_in_room = [u for u in users_in_room if self.hs.is_mine_id(u)]
	835
	836	yield self.update_stats_delta(
	837	ts=self.clock.time_msec(),
	838	stats_type="room",
	839	stats_id=room_id,
	840	fields={},
	841	complete_with_stream_id=pos,
	842	absolute_field_overrides={
	843	"current_state_events": current_state_events_count,
	844	"joined_members": membership_counts.get(Membership.JOIN, 0),
	845	"invited_members": membership_counts.get(Membership.INVITE, 0),
	846	"left_members": membership_counts.get(Membership.LEAVE, 0),
	847	"banned_members": membership_counts.get(Membership.BAN, 0),
	848	"local_users_in_room": len(local_users_in_room),
	849	},
	850	)
	851
	852	@defer.inlineCallbacks
	853	def _calculate_and_set_initial_state_for_user(self, user_id):
	854	def _calculate_and_set_initial_state_for_user_txn(txn):
	855	pos = self._get_max_stream_id_in_current_state_deltas_txn(txn)
	856
	857	txn.execute(
	858	"""
	859	SELECT COUNT(distinct room_id) FROM current_state_events
	860	WHERE type = 'm.room.member' AND state_key = ?
	861	AND membership = 'join'
	862	""",
	863	(user_id,),
	864	)
	865	count, = txn.fetchone()
	866	return count, pos
	867
	868	joined_rooms, pos = yield self.runInteraction(
	869	"calculate_and_set_initial_state_for_user",
	870	_calculate_and_set_initial_state_for_user_txn,
	871	)
	872
	873	yield self.update_stats_delta(
	874	ts=self.clock.time_msec(),
	875	stats_type="user",
	876	stats_id=user_id,
	877	fields={},
	878	complete_with_stream_id=pos,
	879	absolute_field_overrides={"joined_rooms": joined_rooms},
	880	)

+948

-0

synapse/storage/data_stores/main/stream.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2014-2016 OpenMarket Ltd
	2	#
	3	# Licensed under the Apache License, Version 2.0 (the "License");
	4	# you may not use this file except in compliance with the License.
	5	# You may obtain a copy of the License at
	6	#
	7	# http://www.apache.org/licenses/LICENSE-2.0
	8	#
	9	# Unless required by applicable law or agreed to in writing, software
	10	# distributed under the License is distributed on an "AS IS" BASIS,
	11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	# See the License for the specific language governing permissions and
	13	# limitations under the License.
	14
	15	""" This module is responsible for getting events from the DB for pagination
	16	and event streaming.
	17
	18	The order it returns events in depend on whether we are streaming forwards or
	19	are paginating backwards. We do this because we want to handle out of order
	20	messages nicely, while still returning them in the correct order when we
	21	paginate bacwards.
	22
	23	This is implemented by keeping two ordering columns: stream_ordering and
	24	topological_ordering. Stream ordering is basically insertion/received order
	25	(except for events from backfill requests). The topological_ordering is a
	26	weak ordering of events based on the pdu graph.
	27
	28	This means that we have to have two different types of tokens, depending on
	29	what sort order was used:
	30	- stream tokens are of the form: "s%d", which maps directly to the column
	31	- topological tokems: "t%d-%d", where the integers map to the topological
	32	and stream ordering columns respectively.
	33	"""
	34
	35	import abc
	36	import logging
	37	from collections import namedtuple
	38
	39	from six.moves import range
	40
	41	from twisted.internet import defer
	42
	43	from synapse.logging.context import make_deferred_yieldable, run_in_background
	44	from synapse.storage._base import SQLBaseStore
	45	from synapse.storage.data_stores.main.events_worker import EventsWorkerStore
	46	from synapse.storage.engines import PostgresEngine
	47	from synapse.types import RoomStreamToken
	48	from synapse.util.caches.stream_change_cache import StreamChangeCache
	49
	50	logger = logging.getLogger(__name__)
	51
	52
	53	MAX_STREAM_SIZE = 1000
	54
	55
	56	_STREAM_TOKEN = "stream"
	57	_TOPOLOGICAL_TOKEN = "topological"
	58
	59
	60	# Used as return values for pagination APIs
	61	_EventDictReturn = namedtuple(
	62	"_EventDictReturn", ("event_id", "topological_ordering", "stream_ordering")
	63	)
	64
	65
	66	def generate_pagination_where_clause(
	67	direction, column_names, from_token, to_token, engine
	68	):
	69	"""Creates an SQL expression to bound the columns by the pagination
	70	tokens.
	71
	72	For example creates an SQL expression like:
	73
	74	(6, 7) >= (topological_ordering, stream_ordering)
	75	AND (5, 3) < (topological_ordering, stream_ordering)
	76
	77	would be generated for dir=b, from_token=(6, 7) and to_token=(5, 3).
	78
	79	Note that tokens are considered to be after the row they are in, e.g. if
	80	a row A has a token T, then we consider A to be before T. This convention
	81	is important when figuring out inequalities for the generated SQL, and
	82	produces the following result:
	83	- If paginating forwards then we exclude any rows matching the from
	84	token, but include those that match the to token.
	85	- If paginating backwards then we include any rows matching the from
	86	token, but include those that match the to token.
	87
	88	Args:
	89	direction (str): Whether we're paginating backwards("b") or
	90	forwards ("f").
	91	column_names (tuple[str, str]): The column names to bound. Must not
	92	be user defined as these get inserted directly into the SQL
	93	statement without escapes.
	94	from_token (tuple[int, int]\|None): The start point for the pagination.
	95	This is an exclusive minimum bound if direction is "f", and an
	96	inclusive maximum bound if direction is "b".
	97	to_token (tuple[int, int]\|None): The endpoint point for the pagination.
	98	This is an inclusive maximum bound if direction is "f", and an
	99	exclusive minimum bound if direction is "b".
	100	engine: The database engine to generate the clauses for
	101
	102	Returns:
	103	str: The sql expression
	104	"""
	105	assert direction in ("b", "f")
	106
	107	where_clause = []
	108	if from_token:
	109	where_clause.append(
	110	_make_generic_sql_bound(
	111	bound=">=" if direction == "b" else "<",
	112	column_names=column_names,
	113	values=from_token,
	114	engine=engine,
	115	)
	116	)
	117
	118	if to_token:
	119	where_clause.append(
	120	_make_generic_sql_bound(
	121	bound="<" if direction == "b" else ">=",
	122	column_names=column_names,
	123	values=to_token,
	124	engine=engine,
	125	)
	126	)
	127
	128	return " AND ".join(where_clause)
	129
	130
	131	def _make_generic_sql_bound(bound, column_names, values, engine):
	132	"""Create an SQL expression that bounds the given column names by the
	133	values, e.g. create the equivalent of `(1, 2) < (col1, col2)`.
	134
	135	Only works with two columns.
	136
	137	Older versions of SQLite don't support that syntax so we have to expand it
	138	out manually.
	139
	140	Args:
	141	bound (str): The comparison operator to use. One of ">", "<", ">=",
	142	"<=", where the values are on the left and columns on the right.
	143	names (tuple[str, str]): The column names. Must not be user defined
	144	as these get inserted directly into the SQL statement without
	145	escapes.
	146	values (tuple[int\|None, int]): The values to bound the columns by. If
	147	the first value is None then only creates a bound on the second
	148	column.
	149	engine: The database engine to generate the SQL for
	150
	151	Returns:
	152	str
	153	"""
	154
	155	assert bound in (">", "<", ">=", "<=")
	156
	157	name1, name2 = column_names
	158	val1, val2 = values
	159
	160	if val1 is None:
	161	val2 = int(val2)
	162	return "(%d %s %s)" % (val2, bound, name2)
	163
	164	val1 = int(val1)
	165	val2 = int(val2)
	166
	167	if isinstance(engine, PostgresEngine):
	168	# Postgres doesn't optimise ``(x < a) OR (x=a AND y<b)`` as well
	169	# as it optimises ``(x,y) < (a,b)`` on multicolumn indexes. So we
	170	# use the later form when running against postgres.
	171	return "((%d,%d) %s (%s,%s))" % (val1, val2, bound, name1, name2)
	172
	173	# We want to generate queries of e.g. the form:
	174	#
	175	# (val1 < name1 OR (val1 = name1 AND val2 <= name2))
	176	#
	177	# which is equivalent to (val1, val2) < (name1, name2)
	178
	179	return """(
	180	{val1:d} {strict_bound} {name1}
	181	OR ({val1:d} = {name1} AND {val2:d} {bound} {name2})
	182	)""".format(
	183	name1=name1,
	184	val1=val1,
	185	name2=name2,
	186	val2=val2,
	187	strict_bound=bound[0], # The first bound must always be strict equality here
	188	bound=bound,
	189	)
	190
	191
	192	def filter_to_clause(event_filter):
	193	# NB: This may create SQL clauses that don't optimise well (and we don't
	194	# have indices on all possible clauses). E.g. it may create
	195	# "room_id == X AND room_id != X", which postgres doesn't optimise.
	196
	197	if not event_filter:
	198	return "", []
	199
	200	clauses = []
	201	args = []
	202
	203	if event_filter.types:
	204	clauses.append("(%s)" % " OR ".join("type = ?" for _ in event_filter.types))
	205	args.extend(event_filter.types)
	206
	207	for typ in event_filter.not_types:
	208	clauses.append("type != ?")
	209	args.append(typ)
	210
	211	if event_filter.senders:
	212	clauses.append("(%s)" % " OR ".join("sender = ?" for _ in event_filter.senders))
	213	args.extend(event_filter.senders)
	214
	215	for sender in event_filter.not_senders:
	216	clauses.append("sender != ?")
	217	args.append(sender)
	218
	219	if event_filter.rooms:
	220	clauses.append("(%s)" % " OR ".join("room_id = ?" for _ in event_filter.rooms))
	221	args.extend(event_filter.rooms)
	222
	223	for room_id in event_filter.not_rooms:
	224	clauses.append("room_id != ?")
	225	args.append(room_id)
	226
	227	if event_filter.contains_url:
	228	clauses.append("contains_url = ?")
	229	args.append(event_filter.contains_url)
	230
	231	return " AND ".join(clauses), args
	232
	233
	234	class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
	235	"""This is an abstract base class where subclasses must implement
	236	`get_room_max_stream_ordering` and `get_room_min_stream_ordering`
	237	which can be called in the initializer.
	238	"""
	239
	240	__metaclass__ = abc.ABCMeta
	241
	242	def __init__(self, db_conn, hs):
	243	super(StreamWorkerStore, self).__init__(db_conn, hs)
	244
	245	events_max = self.get_room_max_stream_ordering()
	246	event_cache_prefill, min_event_val = self._get_cache_dict(
	247	db_conn,
	248	"events",
	249	entity_column="room_id",
	250	stream_column="stream_ordering",
	251	max_value=events_max,
	252	)
	253	self._events_stream_cache = StreamChangeCache(
	254	"EventsRoomStreamChangeCache",
	255	min_event_val,
	256	prefilled_cache=event_cache_prefill,
	257	)
	258	self._membership_stream_cache = StreamChangeCache(
	259	"MembershipStreamChangeCache", events_max
	260	)
	261
	262	self._stream_order_on_start = self.get_room_max_stream_ordering()
	263
	264	@abc.abstractmethod
	265	def get_room_max_stream_ordering(self):
	266	raise NotImplementedError()
	267
	268	@abc.abstractmethod
	269	def get_room_min_stream_ordering(self):
	270	raise NotImplementedError()
	271
	272	@defer.inlineCallbacks
	273	def get_room_events_stream_for_rooms(
	274	self, room_ids, from_key, to_key, limit=0, order="DESC"
	275	):
	276	"""Get new room events in stream ordering since `from_key`.
	277
	278	Args:
	279	room_id (str)
	280	from_key (str): Token from which no events are returned before
	281	to_key (str): Token from which no events are returned after. (This
	282	is typically the current stream token)
	283	limit (int): Maximum number of events to return
	284	order (str): Either "DESC" or "ASC". Determines which events are
	285	returned when the result is limited. If "DESC" then the most
	286	recent `limit` events are returned, otherwise returns the
	287	oldest `limit` events.
	288
	289	Returns:
	290	Deferred[dict[str,tuple[list[FrozenEvent], str]]]
	291	A map from room id to a tuple containing:
	292	- list of recent events in the room
	293	- stream ordering key for the start of the chunk of events returned.
	294	"""
	295	from_id = RoomStreamToken.parse_stream_token(from_key).stream
	296
	297	room_ids = yield self._events_stream_cache.get_entities_changed(
	298	room_ids, from_id
	299	)
	300
	301	if not room_ids:
	302	return {}
	303
	304	results = {}
	305	room_ids = list(room_ids)
	306	for rm_ids in (room_ids[i : i + 20] for i in range(0, len(room_ids), 20)):
	307	res = yield make_deferred_yieldable(
	308	defer.gatherResults(
	309	[
	310	run_in_background(
	311	self.get_room_events_stream_for_room,
	312	room_id,
	313	from_key,
	314	to_key,
	315	limit,
	316	order=order,
	317	)
	318	for room_id in rm_ids
	319	],
	320	consumeErrors=True,
	321	)
	322	)
	323	results.update(dict(zip(rm_ids, res)))
	324
	325	return results
	326
	327	def get_rooms_that_changed(self, room_ids, from_key):
	328	"""Given a list of rooms and a token, return rooms where there may have
	329	been changes.
	330
	331	Args:
	332	room_ids (list)
	333	from_key (str): The room_key portion of a StreamToken
	334	"""
	335	from_key = RoomStreamToken.parse_stream_token(from_key).stream
	336	return set(
	337	room_id
	338	for room_id in room_ids
	339	if self._events_stream_cache.has_entity_changed(room_id, from_key)
	340	)
	341
	342	@defer.inlineCallbacks
	343	def get_room_events_stream_for_room(
	344	self, room_id, from_key, to_key, limit=0, order="DESC"
	345	):
	346
	347	"""Get new room events in stream ordering since `from_key`.
	348
	349	Args:
	350	room_id (str)
	351	from_key (str): Token from which no events are returned before
	352	to_key (str): Token from which no events are returned after. (This
	353	is typically the current stream token)
	354	limit (int): Maximum number of events to return
	355	order (str): Either "DESC" or "ASC". Determines which events are
	356	returned when the result is limited. If "DESC" then the most
	357	recent `limit` events are returned, otherwise returns the
	358	oldest `limit` events.
	359
	360	Returns:
	361	Deferred[tuple[list[FrozenEvent], str]]: Returns the list of
	362	events (in ascending order) and the token from the start of
	363	the chunk of events returned.
	364	"""
	365	if from_key == to_key:
	366	return [], from_key
	367
	368	from_id = RoomStreamToken.parse_stream_token(from_key).stream
	369	to_id = RoomStreamToken.parse_stream_token(to_key).stream
	370
	371	has_changed = yield self._events_stream_cache.has_entity_changed(
	372	room_id, from_id
	373	)
	374
	375	if not has_changed:
	376	return [], from_key
	377
	378	def f(txn):
	379	sql = (
	380	"SELECT event_id, stream_ordering FROM events WHERE"
	381	" room_id = ?"
	382	" AND not outlier"
	383	" AND stream_ordering > ? AND stream_ordering <= ?"
	384	" ORDER BY stream_ordering %s LIMIT ?"
	385	) % (order,)
	386	txn.execute(sql, (room_id, from_id, to_id, limit))
	387
	388	rows = [_EventDictReturn(row[0], None, row[1]) for row in txn]
	389	return rows
	390
	391	rows = yield self.runInteraction("get_room_events_stream_for_room", f)
	392
	393	ret = yield self.get_events_as_list(
	394	[r.event_id for r in rows], get_prev_content=True
	395	)
	396
	397	self._set_before_and_after(ret, rows, topo_order=from_id is None)
	398
	399	if order.lower() == "desc":
	400	ret.reverse()
	401
	402	if rows:
	403	key = "s%d" % min(r.stream_ordering for r in rows)
	404	else:
	405	# Assume we didn't get anything because there was nothing to
	406	# get.
	407	key = from_key
	408
	409	return ret, key
	410
	411	@defer.inlineCallbacks
	412	def get_membership_changes_for_user(self, user_id, from_key, to_key):
	413	from_id = RoomStreamToken.parse_stream_token(from_key).stream
	414	to_id = RoomStreamToken.parse_stream_token(to_key).stream
	415
	416	if from_key == to_key:
	417	return []
	418
	419	if from_id:
	420	has_changed = self._membership_stream_cache.has_entity_changed(
	421	user_id, int(from_id)
	422	)
	423	if not has_changed:
	424	return []
	425
	426	def f(txn):
	427	sql = (
	428	"SELECT m.event_id, stream_ordering FROM events AS e,"
	429	" room_memberships AS m"
	430	" WHERE e.event_id = m.event_id"
	431	" AND m.user_id = ?"
	432	" AND e.stream_ordering > ? AND e.stream_ordering <= ?"
	433	" ORDER BY e.stream_ordering ASC"
	434	)
	435	txn.execute(sql, (user_id, from_id, to_id))
	436
	437	rows = [_EventDictReturn(row[0], None, row[1]) for row in txn]
	438
	439	return rows
	440
	441	rows = yield self.runInteraction("get_membership_changes_for_user", f)
	442
	443	ret = yield self.get_events_as_list(
	444	[r.event_id for r in rows], get_prev_content=True
	445	)
	446
	447	self._set_before_and_after(ret, rows, topo_order=False)
	448
	449	return ret
	450
	451	@defer.inlineCallbacks
	452	def get_recent_events_for_room(self, room_id, limit, end_token):
	453	"""Get the most recent events in the room in topological ordering.
	454
	455	Args:
	456	room_id (str)
	457	limit (int)
	458	end_token (str): The stream token representing now.
	459
	460	Returns:
	461	Deferred[tuple[list[FrozenEvent], str]]: Returns a list of
	462	events and a token pointing to the start of the returned
	463	events.
	464	The events returned are in ascending order.
	465	"""
	466
	467	rows, token = yield self.get_recent_event_ids_for_room(
	468	room_id, limit, end_token
	469	)
	470
	471	logger.debug("stream before")
	472	events = yield self.get_events_as_list(
	473	[r.event_id for r in rows], get_prev_content=True
	474	)
	475	logger.debug("stream after")
	476
	477	self._set_before_and_after(events, rows)
	478
	479	return (events, token)
	480
	481	@defer.inlineCallbacks
	482	def get_recent_event_ids_for_room(self, room_id, limit, end_token):
	483	"""Get the most recent events in the room in topological ordering.
	484
	485	Args:
	486	room_id (str)
	487	limit (int)
	488	end_token (str): The stream token representing now.
	489
	490	Returns:
	491	Deferred[tuple[list[_EventDictReturn], str]]: Returns a list of
	492	_EventDictReturn and a token pointing to the start of the returned
	493	events.
	494	The events returned are in ascending order.
	495	"""
	496	# Allow a zero limit here, and no-op.
	497	if limit == 0:
	498	return [], end_token
	499
	500	end_token = RoomStreamToken.parse(end_token)
	501
	502	rows, token = yield self.runInteraction(
	503	"get_recent_event_ids_for_room",
	504	self._paginate_room_events_txn,
	505	room_id,
	506	from_token=end_token,
	507	limit=limit,
	508	)
	509
	510	# We want to return the results in ascending order.
	511	rows.reverse()
	512
	513	return rows, token
	514
	515	def get_room_event_after_stream_ordering(self, room_id, stream_ordering):
	516	"""Gets details of the first event in a room at or after a stream ordering
	517
	518	Args:
	519	room_id (str):
	520	stream_ordering (int):
	521
	522	Returns:
	523	Deferred[(int, int, str)]:
	524	(stream ordering, topological ordering, event_id)
	525	"""
	526
	527	def _f(txn):
	528	sql = (
	529	"SELECT stream_ordering, topological_ordering, event_id"
	530	" FROM events"
	531	" WHERE room_id = ? AND stream_ordering >= ?"
	532	" AND NOT outlier"
	533	" ORDER BY stream_ordering"
	534	" LIMIT 1"
	535	)
	536	txn.execute(sql, (room_id, stream_ordering))
	537	return txn.fetchone()
	538
	539	return self.runInteraction("get_room_event_after_stream_ordering", _f)
	540
	541	@defer.inlineCallbacks
	542	def get_room_events_max_id(self, room_id=None):
	543	"""Returns the current token for rooms stream.
	544
	545	By default, it returns the current global stream token. Specifying a
	546	`room_id` causes it to return the current room specific topological
	547	token.
	548	"""
	549	token = yield self.get_room_max_stream_ordering()
	550	if room_id is None:
	551	return "s%d" % (token,)
	552	else:
	553	topo = yield self.runInteraction(
	554	"_get_max_topological_txn", self._get_max_topological_txn, room_id
	555	)
	556	return "t%d-%d" % (topo, token)
	557
	558	def get_stream_token_for_event(self, event_id):
	559	"""The stream token for an event
	560	Args:
	561	event_id(str): The id of the event to look up a stream token for.
	562	Raises:
	563	StoreError if the event wasn't in the database.
	564	Returns:
	565	A deferred "s%d" stream token.
	566	"""
	567	return self._simple_select_one_onecol(
	568	table="events", keyvalues={"event_id": event_id}, retcol="stream_ordering"
	569	).addCallback(lambda row: "s%d" % (row,))
	570
	571	def get_topological_token_for_event(self, event_id):
	572	"""The stream token for an event
	573	Args:
	574	event_id(str): The id of the event to look up a stream token for.
	575	Raises:
	576	StoreError if the event wasn't in the database.
	577	Returns:
	578	A deferred "t%d-%d" topological token.
	579	"""
	580	return self._simple_select_one(
	581	table="events",
	582	keyvalues={"event_id": event_id},
	583	retcols=("stream_ordering", "topological_ordering"),
	584	desc="get_topological_token_for_event",
	585	).addCallback(
	586	lambda row: "t%d-%d" % (row["topological_ordering"], row["stream_ordering"])
	587	)
	588
	589	def get_max_topological_token(self, room_id, stream_key):
	590	"""Get the max topological token in a room before the given stream
	591	ordering.
	592
	593	Args:
	594	room_id (str)
	595	stream_key (int)
	596
	597	Returns:
	598	Deferred[int]
	599	"""
	600	sql = (
	601	"SELECT coalesce(max(topological_ordering), 0) FROM events"
	602	" WHERE room_id = ? AND stream_ordering < ?"
	603	)
	604	return self._execute(
	605	"get_max_topological_token", None, sql, room_id, stream_key
	606	).addCallback(lambda r: r[0][0] if r else 0)
	607
	608	def _get_max_topological_txn(self, txn, room_id):
	609	txn.execute(
	610	"SELECT MAX(topological_ordering) FROM events" " WHERE room_id = ?",
	611	(room_id,),
	612	)
	613
	614	rows = txn.fetchall()
	615	return rows[0][0] if rows else 0
	616
	617	@staticmethod
	618	def _set_before_and_after(events, rows, topo_order=True):
	619	"""Inserts ordering information to events' internal metadata from
	620	the DB rows.
	621
	622	Args:
	623	events (list[FrozenEvent])
	624	rows (list[_EventDictReturn])
	625	topo_order (bool): Whether the events were ordered topologically
	626	or by stream ordering. If true then all rows should have a non
	627	null topological_ordering.
	628	"""
	629	for event, row in zip(events, rows):
	630	stream = row.stream_ordering
	631	if topo_order and row.topological_ordering:
	632	topo = row.topological_ordering
	633	else:
	634	topo = None
	635	internal = event.internal_metadata
	636	internal.before = str(RoomStreamToken(topo, stream - 1))
	637	internal.after = str(RoomStreamToken(topo, stream))
	638	internal.order = (int(topo) if topo else 0, int(stream))
	639
	640	@defer.inlineCallbacks
	641	def get_events_around(
	642	self, room_id, event_id, before_limit, after_limit, event_filter=None
	643	):
	644	"""Retrieve events and pagination tokens around a given event in a
	645	room.
	646
	647	Args:
	648	room_id (str)
	649	event_id (str)
	650	before_limit (int)
	651	after_limit (int)
	652	event_filter (Filter\|None)
	653
	654	Returns:
	655	dict
	656	"""
	657
	658	results = yield self.runInteraction(
	659	"get_events_around",
	660	self._get_events_around_txn,
	661	room_id,
	662	event_id,
	663	before_limit,
	664	after_limit,
	665	event_filter,
	666	)
	667
	668	events_before = yield self.get_events_as_list(
	669	[e for e in results["before"]["event_ids"]], get_prev_content=True
	670	)
	671
	672	events_after = yield self.get_events_as_list(
	673	[e for e in results["after"]["event_ids"]], get_prev_content=True
	674	)
	675
	676	return {
	677	"events_before": events_before,
	678	"events_after": events_after,
	679	"start": results["before"]["token"],
	680	"end": results["after"]["token"],
	681	}
	682
	683	def _get_events_around_txn(
	684	self, txn, room_id, event_id, before_limit, after_limit, event_filter
	685	):
	686	"""Retrieves event_ids and pagination tokens around a given event in a
	687	room.
	688
	689	Args:
	690	room_id (str)
	691	event_id (str)
	692	before_limit (int)
	693	after_limit (int)
	694	event_filter (Filter\|None)
	695
	696	Returns:
	697	dict
	698	"""
	699
	700	results = self._simple_select_one_txn(
	701	txn,
	702	"events",
	703	keyvalues={"event_id": event_id, "room_id": room_id},
	704	retcols=["stream_ordering", "topological_ordering"],
	705	)
	706
	707	# Paginating backwards includes the event at the token, but paginating
	708	# forward doesn't.
	709	before_token = RoomStreamToken(
	710	results["topological_ordering"] - 1, results["stream_ordering"]
	711	)
	712
	713	after_token = RoomStreamToken(
	714	results["topological_ordering"], results["stream_ordering"]
	715	)
	716
	717	rows, start_token = self._paginate_room_events_txn(
	718	txn,
	719	room_id,
	720	before_token,
	721	direction="b",
	722	limit=before_limit,
	723	event_filter=event_filter,
	724	)
	725	events_before = [r.event_id for r in rows]
	726
	727	rows, end_token = self._paginate_room_events_txn(
	728	txn,
	729	room_id,
	730	after_token,
	731	direction="f",
	732	limit=after_limit,
	733	event_filter=event_filter,
	734	)
	735	events_after = [r.event_id for r in rows]
	736
	737	return {
	738	"before": {"event_ids": events_before, "token": start_token},
	739	"after": {"event_ids": events_after, "token": end_token},
	740	}
	741
	742	@defer.inlineCallbacks
	743	def get_all_new_events_stream(self, from_id, current_id, limit):
	744	"""Get all new events
	745
	746	Returns all events with from_id < stream_ordering <= current_id.
	747
	748	Args:
	749	from_id (int): the stream_ordering of the last event we processed
	750	current_id (int): the stream_ordering of the most recently processed event
	751	limit (int): the maximum number of events to return
	752
	753	Returns:
	754	Deferred[Tuple[int, list[FrozenEvent]]]: A tuple of (next_id, events), where
	755	`next_id` is the next value to pass as `from_id` (it will either be the
	756	stream_ordering of the last returned event, or, if fewer than `limit` events
	757	were found, `current_id`.
	758	"""
	759
	760	def get_all_new_events_stream_txn(txn):
	761	sql = (
	762	"SELECT e.stream_ordering, e.event_id"
	763	" FROM events AS e"
	764	" WHERE"
	765	" ? < e.stream_ordering AND e.stream_ordering <= ?"
	766	" ORDER BY e.stream_ordering ASC"
	767	" LIMIT ?"
	768	)
	769
	770	txn.execute(sql, (from_id, current_id, limit))
	771	rows = txn.fetchall()
	772
	773	upper_bound = current_id
	774	if len(rows) == limit:
	775	upper_bound = rows[-1][0]
	776
	777	return upper_bound, [row[1] for row in rows]
	778
	779	upper_bound, event_ids = yield self.runInteraction(
	780	"get_all_new_events_stream", get_all_new_events_stream_txn
	781	)
	782
	783	events = yield self.get_events_as_list(event_ids)
	784
	785	return upper_bound, events
	786
	787	def get_federation_out_pos(self, typ):
	788	return self._simple_select_one_onecol(
	789	table="federation_stream_position",
	790	retcol="stream_id",
	791	keyvalues={"type": typ},
	792	desc="get_federation_out_pos",
	793	)
	794
	795	def update_federation_out_pos(self, typ, stream_id):
	796	return self._simple_update_one(
	797	table="federation_stream_position",
	798	keyvalues={"type": typ},
	799	updatevalues={"stream_id": stream_id},
	800	desc="update_federation_out_pos",
	801	)
	802
	803	def has_room_changed_since(self, room_id, stream_id):
	804	return self._events_stream_cache.has_entity_changed(room_id, stream_id)
	805
	806	def _paginate_room_events_txn(
	807	self,
	808	txn,
	809	room_id,
	810	from_token,
	811	to_token=None,
	812	direction="b",
	813	limit=-1,
	814	event_filter=None,
	815	):
	816	"""Returns list of events before or after a given token.
	817
	818	Args:
	819	txn
	820	room_id (str)
	821	from_token (RoomStreamToken): The token used to stream from
	822	to_token (RoomStreamToken\|None): A token which if given limits the
	823	results to only those before
	824	direction(char): Either 'b' or 'f' to indicate whether we are
	825	paginating forwards or backwards from `from_key`.
	826	limit (int): The maximum number of events to return.
	827	event_filter (Filter\|None): If provided filters the events to
	828	those that match the filter.
	829
	830	Returns:
	831	Deferred[tuple[list[_EventDictReturn], str]]: Returns the results
	832	as a list of _EventDictReturn and a token that points to the end
	833	of the result set. If no events are returned then the end of the
	834	stream has been reached (i.e. there are no events between
	835	`from_token` and `to_token`), or `limit` is zero.
	836	"""
	837
	838	assert int(limit) >= 0
	839
	840	# Tokens really represent positions between elements, but we use
	841	# the convention of pointing to the event before the gap. Hence
	842	# we have a bit of asymmetry when it comes to equalities.
	843	args = [False, room_id]
	844	if direction == "b":
	845	order = "DESC"
	846	else:
	847	order = "ASC"
	848
	849	bounds = generate_pagination_where_clause(
	850	direction=direction,
	851	column_names=("topological_ordering", "stream_ordering"),
	852	from_token=from_token,
	853	to_token=to_token,
	854	engine=self.database_engine,
	855	)
	856
	857	filter_clause, filter_args = filter_to_clause(event_filter)
	858
	859	if filter_clause:
	860	bounds += " AND " + filter_clause
	861	args.extend(filter_args)
	862
	863	args.append(int(limit))
	864
	865	sql = (
	866	"SELECT event_id, topological_ordering, stream_ordering"
	867	" FROM events"
	868	" WHERE outlier = ? AND room_id = ? AND %(bounds)s"
	869	" ORDER BY topological_ordering %(order)s,"
	870	" stream_ordering %(order)s LIMIT ?"
	871	) % {"bounds": bounds, "order": order}
	872
	873	txn.execute(sql, args)
	874
	875	rows = [_EventDictReturn(row[0], row[1], row[2]) for row in txn]
	876
	877	if rows:
	878	topo = rows[-1].topological_ordering
	879	toke = rows[-1].stream_ordering
	880	if direction == "b":
	881	# Tokens are positions between events.
	882	# This token points after the last event in the chunk.
	883	# We need it to point to the event before it in the chunk
	884	# when we are going backwards so we subtract one from the
	885	# stream part.
	886	toke -= 1
	887	next_token = RoomStreamToken(topo, toke)
	888	else:
	889	# TODO (erikj): We should work out what to do here instead.
	890	next_token = to_token if to_token else from_token
	891
	892	return rows, str(next_token)
	893
	894	@defer.inlineCallbacks
	895	def paginate_room_events(
	896	self, room_id, from_key, to_key=None, direction="b", limit=-1, event_filter=None
	897	):
	898	"""Returns list of events before or after a given token.
	899
	900	Args:
	901	room_id (str)
	902	from_key (str): The token used to stream from
	903	to_key (str\|None): A token which if given limits the results to
	904	only those before
	905	direction(char): Either 'b' or 'f' to indicate whether we are
	906	paginating forwards or backwards from `from_key`.
	907	limit (int): The maximum number of events to return.
	908	event_filter (Filter\|None): If provided filters the events to
	909	those that match the filter.
	910
	911	Returns:
	912	tuple[list[FrozenEvent], str]: Returns the results as a list of
	913	events and a token that points to the end of the result set. If no
	914	events are returned then the end of the stream has been reached
	915	(i.e. there are no events between `from_key` and `to_key`).
	916	"""
	917
	918	from_key = RoomStreamToken.parse(from_key)
	919	if to_key:
	920	to_key = RoomStreamToken.parse(to_key)
	921
	922	rows, token = yield self.runInteraction(
	923	"paginate_room_events",
	924	self._paginate_room_events_txn,
	925	room_id,
	926	from_key,
	927	to_key,
	928	direction,
	929	limit,
	930	event_filter,
	931	)
	932
	933	events = yield self.get_events_as_list(
	934	[r.event_id for r in rows], get_prev_content=True
	935	)
	936
	937	self._set_before_and_after(events, rows)
	938
	939	return (events, token)
	940
	941
	942	class StreamStore(StreamWorkerStore):
	943	def get_room_max_stream_ordering(self):
	944	return self._stream_id_gen.get_current_token()
	945
	946	def get_room_min_stream_ordering(self):
	947	return self._backfill_id_gen.get_current_token()

+265

-0

synapse/storage/data_stores/main/tags.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2014-2016 OpenMarket Ltd
	2	# Copyright 2018 New Vector Ltd
	3	#
	4	# Licensed under the Apache License, Version 2.0 (the "License");
	5	# you may not use this file except in compliance with the License.
	6	# You may obtain a copy of the License at
	7	#
	8	# http://www.apache.org/licenses/LICENSE-2.0
	9	#
	10	# Unless required by applicable law or agreed to in writing, software
	11	# distributed under the License is distributed on an "AS IS" BASIS,
	12	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	# See the License for the specific language governing permissions and
	14	# limitations under the License.
	15
	16	import logging
	17
	18	from six.moves import range
	19
	20	from canonicaljson import json
	21
	22	from twisted.internet import defer
	23
	24	from synapse.storage.data_stores.main.account_data import AccountDataWorkerStore
	25	from synapse.util.caches.descriptors import cached
	26
	27	logger = logging.getLogger(__name__)
	28
	29
	30	class TagsWorkerStore(AccountDataWorkerStore):
	31	@cached()
	32	def get_tags_for_user(self, user_id):
	33	"""Get all the tags for a user.
	34
	35
	36	Args:
	37	user_id(str): The user to get the tags for.
	38	Returns:
	39	A deferred dict mapping from room_id strings to dicts mapping from
	40	tag strings to tag content.
	41	"""
	42
	43	deferred = self._simple_select_list(
	44	"room_tags", {"user_id": user_id}, ["room_id", "tag", "content"]
	45	)
	46
	47	@deferred.addCallback
	48	def tags_by_room(rows):
	49	tags_by_room = {}
	50	for row in rows:
	51	room_tags = tags_by_room.setdefault(row["room_id"], {})
	52	room_tags[row["tag"]] = json.loads(row["content"])
	53	return tags_by_room
	54
	55	return deferred
	56
	57	@defer.inlineCallbacks
	58	def get_all_updated_tags(self, last_id, current_id, limit):
	59	"""Get all the client tags that have changed on the server
	60	Args:
	61	last_id(int): The position to fetch from.
	62	current_id(int): The position to fetch up to.
	63	Returns:
	64	A deferred list of tuples of stream_id int, user_id string,
	65	room_id string, tag string and content string.
	66	"""
	67	if last_id == current_id:
	68	return []
	69
	70	def get_all_updated_tags_txn(txn):
	71	sql = (
	72	"SELECT stream_id, user_id, room_id"
	73	" FROM room_tags_revisions as r"
	74	" WHERE ? < stream_id AND stream_id <= ?"
	75	" ORDER BY stream_id ASC LIMIT ?"
	76	)
	77	txn.execute(sql, (last_id, current_id, limit))
	78	return txn.fetchall()
	79
	80	tag_ids = yield self.runInteraction(
	81	"get_all_updated_tags", get_all_updated_tags_txn
	82	)
	83
	84	def get_tag_content(txn, tag_ids):
	85	sql = (
	86	"SELECT tag, content" " FROM room_tags" " WHERE user_id=? AND room_id=?"
	87	)
	88	results = []
	89	for stream_id, user_id, room_id in tag_ids:
	90	txn.execute(sql, (user_id, room_id))
	91	tags = []
	92	for tag, content in txn:
	93	tags.append(json.dumps(tag) + ":" + content)
	94	tag_json = "{" + ",".join(tags) + "}"
	95	results.append((stream_id, user_id, room_id, tag_json))
	96
	97	return results
	98
	99	batch_size = 50
	100	results = []
	101	for i in range(0, len(tag_ids), batch_size):
	102	tags = yield self.runInteraction(
	103	"get_all_updated_tag_content",
	104	get_tag_content,
	105	tag_ids[i : i + batch_size],
	106	)
	107	results.extend(tags)
	108
	109	return results
	110
	111	@defer.inlineCallbacks
	112	def get_updated_tags(self, user_id, stream_id):
	113	"""Get all the tags for the rooms where the tags have changed since the
	114	given version
	115
	116	Args:
	117	user_id(str): The user to get the tags for.
	118	stream_id(int): The earliest update to get for the user.
	119	Returns:
	120	A deferred dict mapping from room_id strings to lists of tag
	121	strings for all the rooms that changed since the stream_id token.
	122	"""
	123
	124	def get_updated_tags_txn(txn):
	125	sql = (
	126	"SELECT room_id from room_tags_revisions"
	127	" WHERE user_id = ? AND stream_id > ?"
	128	)
	129	txn.execute(sql, (user_id, stream_id))
	130	room_ids = [row[0] for row in txn]
	131	return room_ids
	132
	133	changed = self._account_data_stream_cache.has_entity_changed(
	134	user_id, int(stream_id)
	135	)
	136	if not changed:
	137	return {}
	138
	139	room_ids = yield self.runInteraction("get_updated_tags", get_updated_tags_txn)
	140
	141	results = {}
	142	if room_ids:
	143	tags_by_room = yield self.get_tags_for_user(user_id)
	144	for room_id in room_ids:
	145	results[room_id] = tags_by_room.get(room_id, {})
	146
	147	return results
	148
	149	def get_tags_for_room(self, user_id, room_id):
	150	"""Get all the tags for the given room
	151	Args:
	152	user_id(str): The user to get tags for
	153	room_id(str): The room to get tags for
	154	Returns:
	155	A deferred list of string tags.
	156	"""
	157	return self._simple_select_list(
	158	table="room_tags",
	159	keyvalues={"user_id": user_id, "room_id": room_id},
	160	retcols=("tag", "content"),
	161	desc="get_tags_for_room",
	162	).addCallback(
	163	lambda rows: {row["tag"]: json.loads(row["content"]) for row in rows}
	164	)
	165
	166
	167	class TagsStore(TagsWorkerStore):
	168	@defer.inlineCallbacks
	169	def add_tag_to_room(self, user_id, room_id, tag, content):
	170	"""Add a tag to a room for a user.
	171	Args:
	172	user_id(str): The user to add a tag for.
	173	room_id(str): The room to add a tag for.
	174	tag(str): The tag name to add.
	175	content(dict): A json object to associate with the tag.
	176	Returns:
	177	A deferred that completes once the tag has been added.
	178	"""
	179	content_json = json.dumps(content)
	180
	181	def add_tag_txn(txn, next_id):
	182	self._simple_upsert_txn(
	183	txn,
	184	table="room_tags",
	185	keyvalues={"user_id": user_id, "room_id": room_id, "tag": tag},
	186	values={"content": content_json},
	187	)
	188	self._update_revision_txn(txn, user_id, room_id, next_id)
	189
	190	with self._account_data_id_gen.get_next() as next_id:
	191	yield self.runInteraction("add_tag", add_tag_txn, next_id)
	192
	193	self.get_tags_for_user.invalidate((user_id,))
	194
	195	result = self._account_data_id_gen.get_current_token()
	196	return result
	197
	198	@defer.inlineCallbacks
	199	def remove_tag_from_room(self, user_id, room_id, tag):
	200	"""Remove a tag from a room for a user.
	201	Returns:
	202	A deferred that completes once the tag has been removed
	203	"""
	204
	205	def remove_tag_txn(txn, next_id):
	206	sql = (
	207	"DELETE FROM room_tags "
	208	" WHERE user_id = ? AND room_id = ? AND tag = ?"
	209	)
	210	txn.execute(sql, (user_id, room_id, tag))
	211	self._update_revision_txn(txn, user_id, room_id, next_id)
	212
	213	with self._account_data_id_gen.get_next() as next_id:
	214	yield self.runInteraction("remove_tag", remove_tag_txn, next_id)
	215
	216	self.get_tags_for_user.invalidate((user_id,))
	217
	218	result = self._account_data_id_gen.get_current_token()
	219	return result
	220
	221	def _update_revision_txn(self, txn, user_id, room_id, next_id):
	222	"""Update the latest revision of the tags for the given user and room.
	223
	224	Args:
	225	txn: The database cursor
	226	user_id(str): The ID of the user.
	227	room_id(str): The ID of the room.
	228	next_id(int): The the revision to advance to.
	229	"""
	230
	231	txn.call_after(
	232	self._account_data_stream_cache.entity_has_changed, user_id, next_id
	233	)
	234
	235	update_max_id_sql = (
	236	"UPDATE account_data_max_stream_id"
	237	" SET stream_id = ?"
	238	" WHERE stream_id < ?"
	239	)
	240	txn.execute(update_max_id_sql, (next_id, next_id))
	241
	242	update_sql = (
	243	"UPDATE room_tags_revisions"
	244	" SET stream_id = ?"
	245	" WHERE user_id = ?"
	246	" AND room_id = ?"
	247	)
	248	txn.execute(update_sql, (next_id, user_id, room_id))
	249
	250	if txn.rowcount == 0:
	251	insert_sql = (
	252	"INSERT INTO room_tags_revisions (user_id, room_id, stream_id)"
	253	" VALUES (?, ?, ?)"
	254	)
	255	try:
	256	txn.execute(insert_sql, (user_id, room_id, next_id))
	257	except self.database_engine.module.IntegrityError:
	258	# Ignore insertion errors. It doesn't matter if the row wasn't
	259	# inserted because if two updates happend concurrently the one
	260	# with the higher stream_id will not be reported to a client
	261	# unless the previous update has completed. It doesn't matter
	262	# which stream_id ends up in the table, as long as it is higher
	263	# than the id that the client has.
	264	pass

+273

-0

synapse/storage/data_stores/main/transactions.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2014-2016 OpenMarket Ltd
	2	#
	3	# Licensed under the Apache License, Version 2.0 (the "License");
	4	# you may not use this file except in compliance with the License.
	5	# You may obtain a copy of the License at
	6	#
	7	# http://www.apache.org/licenses/LICENSE-2.0
	8	#
	9	# Unless required by applicable law or agreed to in writing, software
	10	# distributed under the License is distributed on an "AS IS" BASIS,
	11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	# See the License for the specific language governing permissions and
	13	# limitations under the License.
	14
	15	import logging
	16	from collections import namedtuple
	17
	18	import six
	19
	20	from canonicaljson import encode_canonical_json
	21
	22	from twisted.internet import defer
	23
	24	from synapse.metrics.background_process_metrics import run_as_background_process
	25	from synapse.storage._base import SQLBaseStore, db_to_json
	26	from synapse.util.caches.expiringcache import ExpiringCache
	27
	28	# py2 sqlite has buffer hardcoded as only binary type, so we must use it,
	29	# despite being deprecated and removed in favor of memoryview
	30	if six.PY2:
	31	db_binary_type = six.moves.builtins.buffer
	32	else:
	33	db_binary_type = memoryview
	34
	35	logger = logging.getLogger(__name__)
	36
	37
	38	_TransactionRow = namedtuple(
	39	"_TransactionRow",
	40	("id", "transaction_id", "destination", "ts", "response_code", "response_json"),
	41	)
	42
	43	_UpdateTransactionRow = namedtuple(
	44	"_TransactionRow", ("response_code", "response_json")
	45	)
	46
	47	SENTINEL = object()
	48
	49
	50	class TransactionStore(SQLBaseStore):
	51	"""A collection of queries for handling PDUs.
	52	"""
	53
	54	def __init__(self, db_conn, hs):
	55	super(TransactionStore, self).__init__(db_conn, hs)
	56
	57	self._clock.looping_call(self._start_cleanup_transactions, 30 * 60 * 1000)
	58
	59	self._destination_retry_cache = ExpiringCache(
	60	cache_name="get_destination_retry_timings",
	61	clock=self._clock,
	62	expiry_ms=5 * 60 * 1000,
	63	)
	64
	65	def get_received_txn_response(self, transaction_id, origin):
	66	"""For an incoming transaction from a given origin, check if we have
	67	already responded to it. If so, return the response code and response
	68	body (as a dict).
	69
	70	Args:
	71	transaction_id (str)
	72	origin(str)
	73
	74	Returns:
	75	tuple: None if we have not previously responded to
	76	this transaction or a 2-tuple of (int, dict)
	77	"""
	78
	79	return self.runInteraction(
	80	"get_received_txn_response",
	81	self._get_received_txn_response,
	82	transaction_id,
	83	origin,
	84	)
	85
	86	def _get_received_txn_response(self, txn, transaction_id, origin):
	87	result = self._simple_select_one_txn(
	88	txn,
	89	table="received_transactions",
	90	keyvalues={"transaction_id": transaction_id, "origin": origin},
	91	retcols=(
	92	"transaction_id",
	93	"origin",
	94	"ts",
	95	"response_code",
	96	"response_json",
	97	"has_been_referenced",
	98	),
	99	allow_none=True,
	100	)
	101
	102	if result and result["response_code"]:
	103	return result["response_code"], db_to_json(result["response_json"])
	104
	105	else:
	106	return None
	107
	108	def set_received_txn_response(self, transaction_id, origin, code, response_dict):
	109	"""Persist the response we returened for an incoming transaction, and
	110	should return for subsequent transactions with the same transaction_id
	111	and origin.
	112
	113	Args:
	114	txn
	115	transaction_id (str)
	116	origin (str)
	117	code (int)
	118	response_json (str)
	119	"""
	120
	121	return self._simple_insert(
	122	table="received_transactions",
	123	values={
	124	"transaction_id": transaction_id,
	125	"origin": origin,
	126	"response_code": code,
	127	"response_json": db_binary_type(encode_canonical_json(response_dict)),
	128	"ts": self._clock.time_msec(),
	129	},
	130	or_ignore=True,
	131	desc="set_received_txn_response",
	132	)
	133
	134	@defer.inlineCallbacks
	135	def get_destination_retry_timings(self, destination):
	136	"""Gets the current retry timings (if any) for a given destination.
	137
	138	Args:
	139	destination (str)
	140
	141	Returns:
	142	None if not retrying
	143	Otherwise a dict for the retry scheme
	144	"""
	145
	146	result = self._destination_retry_cache.get(destination, SENTINEL)
	147	if result is not SENTINEL:
	148	return result
	149
	150	result = yield self.runInteraction(
	151	"get_destination_retry_timings",
	152	self._get_destination_retry_timings,
	153	destination,
	154	)
	155
	156	# We don't hugely care about race conditions between getting and
	157	# invalidating the cache, since we time out fairly quickly anyway.
	158	self._destination_retry_cache[destination] = result
	159	return result
	160
	161	def _get_destination_retry_timings(self, txn, destination):
	162	result = self._simple_select_one_txn(
	163	txn,
	164	table="destinations",
	165	keyvalues={"destination": destination},
	166	retcols=("destination", "failure_ts", "retry_last_ts", "retry_interval"),
	167	allow_none=True,
	168	)
	169
	170	if result and result["retry_last_ts"] > 0:
	171	return result
	172	else:
	173	return None
	174
	175	def set_destination_retry_timings(
	176	self, destination, failure_ts, retry_last_ts, retry_interval
	177	):
	178	"""Sets the current retry timings for a given destination.
	179	Both timings should be zero if retrying is no longer occuring.
	180
	181	Args:
	182	destination (str)
	183	failure_ts (int\|None) - when the server started failing (ms since epoch)
	184	retry_last_ts (int) - time of last retry attempt in unix epoch ms
	185	retry_interval (int) - how long until next retry in ms
	186	"""
	187
	188	self._destination_retry_cache.pop(destination, None)
	189	return self.runInteraction(
	190	"set_destination_retry_timings",
	191	self._set_destination_retry_timings,
	192	destination,
	193	failure_ts,
	194	retry_last_ts,
	195	retry_interval,
	196	)
	197
	198	def _set_destination_retry_timings(
	199	self, txn, destination, failure_ts, retry_last_ts, retry_interval
	200	):
	201
	202	if self.database_engine.can_native_upsert:
	203	# Upsert retry time interval if retry_interval is zero (i.e. we're
	204	# resetting it) or greater than the existing retry interval.
	205
	206	sql = """
	207	INSERT INTO destinations (
	208	destination, failure_ts, retry_last_ts, retry_interval
	209	)
	210	VALUES (?, ?, ?, ?)
	211	ON CONFLICT (destination) DO UPDATE SET
	212	failure_ts = EXCLUDED.failure_ts,
	213	retry_last_ts = EXCLUDED.retry_last_ts,
	214	retry_interval = EXCLUDED.retry_interval
	215	WHERE
	216	EXCLUDED.retry_interval = 0
	217	OR destinations.retry_interval < EXCLUDED.retry_interval
	218	"""
	219
	220	txn.execute(sql, (destination, failure_ts, retry_last_ts, retry_interval))
	221
	222	return
	223
	224	self.database_engine.lock_table(txn, "destinations")
	225
	226	# We need to be careful here as the data may have changed from under us
	227	# due to a worker setting the timings.
	228
	229	prev_row = self._simple_select_one_txn(
	230	txn,
	231	table="destinations",
	232	keyvalues={"destination": destination},
	233	retcols=("failure_ts", "retry_last_ts", "retry_interval"),
	234	allow_none=True,
	235	)
	236
	237	if not prev_row:
	238	self._simple_insert_txn(
	239	txn,
	240	table="destinations",
	241	values={
	242	"destination": destination,
	243	"failure_ts": failure_ts,
	244	"retry_last_ts": retry_last_ts,
	245	"retry_interval": retry_interval,
	246	},
	247	)
	248	elif retry_interval == 0 or prev_row["retry_interval"] < retry_interval:
	249	self._simple_update_one_txn(
	250	txn,
	251	"destinations",
	252	keyvalues={"destination": destination},
	253	updatevalues={
	254	"failure_ts": failure_ts,
	255	"retry_last_ts": retry_last_ts,
	256	"retry_interval": retry_interval,
	257	},
	258	)
	259
	260	def _start_cleanup_transactions(self):
	261	return run_as_background_process(
	262	"cleanup_transactions", self._cleanup_transactions
	263	)
	264
	265	def _cleanup_transactions(self):
	266	now = self._clock.time_msec()
	267	month_ago = now - 30 * 24 * 60 * 60 * 1000
	268
	269	def _cleanup_transactions_txn(txn):
	270	txn.execute("DELETE FROM received_transactions WHERE ts < ?", (month_ago,))
	271
	272	return self.runInteraction("_cleanup_transactions", _cleanup_transactions_txn)

+827

-0

synapse/storage/data_stores/main/user_directory.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2017 Vector Creations Ltd
	2	#
	3	# Licensed under the Apache License, Version 2.0 (the "License");
	4	# you may not use this file except in compliance with the License.
	5	# You may obtain a copy of the License at
	6	#
	7	# http://www.apache.org/licenses/LICENSE-2.0
	8	#
	9	# Unless required by applicable law or agreed to in writing, software
	10	# distributed under the License is distributed on an "AS IS" BASIS,
	11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	# See the License for the specific language governing permissions and
	13	# limitations under the License.
	14
	15	import logging
	16	import re
	17
	18	from twisted.internet import defer
	19
	20	from synapse.api.constants import EventTypes, JoinRules
	21	from synapse.storage.background_updates import BackgroundUpdateStore
	22	from synapse.storage.data_stores.main.state import StateFilter
	23	from synapse.storage.data_stores.main.state_deltas import StateDeltasStore
	24	from synapse.storage.engines import PostgresEngine, Sqlite3Engine
	25	from synapse.types import get_domain_from_id, get_localpart_from_id
	26	from synapse.util.caches.descriptors import cached
	27
	28	logger = logging.getLogger(__name__)
	29
	30
	31	TEMP_TABLE = "_temp_populate_user_directory"
	32
	33
	34	class UserDirectoryBackgroundUpdateStore(StateDeltasStore, BackgroundUpdateStore):
	35
	36	# How many records do we calculate before sending it to
	37	# add_users_who_share_private_rooms?
	38	SHARE_PRIVATE_WORKING_SET = 500
	39
	40	def __init__(self, db_conn, hs):
	41	super(UserDirectoryBackgroundUpdateStore, self).__init__(db_conn, hs)
	42
	43	self.server_name = hs.hostname
	44
	45	self.register_background_update_handler(
	46	"populate_user_directory_createtables",
	47	self._populate_user_directory_createtables,
	48	)
	49	self.register_background_update_handler(
	50	"populate_user_directory_process_rooms",
	51	self._populate_user_directory_process_rooms,
	52	)
	53	self.register_background_update_handler(
	54	"populate_user_directory_process_users",
	55	self._populate_user_directory_process_users,
	56	)
	57	self.register_background_update_handler(
	58	"populate_user_directory_cleanup", self._populate_user_directory_cleanup
	59	)
	60
	61	@defer.inlineCallbacks
	62	def _populate_user_directory_createtables(self, progress, batch_size):
	63
	64	# Get all the rooms that we want to process.
	65	def _make_staging_area(txn):
	66	sql = (
	67	"CREATE TABLE IF NOT EXISTS "
	68	+ TEMP_TABLE
	69	+ "_rooms(room_id TEXT NOT NULL, events BIGINT NOT NULL)"
	70	)
	71	txn.execute(sql)
	72
	73	sql = (
	74	"CREATE TABLE IF NOT EXISTS "
	75	+ TEMP_TABLE
	76	+ "_position(position TEXT NOT NULL)"
	77	)
	78	txn.execute(sql)
	79
	80	# Get rooms we want to process from the database
	81	sql = """
	82	SELECT room_id, count(*) FROM current_state_events
	83	GROUP BY room_id
	84	"""
	85	txn.execute(sql)
	86	rooms = [{"room_id": x[0], "events": x[1]} for x in txn.fetchall()]
	87	self._simple_insert_many_txn(txn, TEMP_TABLE + "_rooms", rooms)
	88	del rooms
	89
	90	# If search all users is on, get all the users we want to add.
	91	if self.hs.config.user_directory_search_all_users:
	92	sql = (
	93	"CREATE TABLE IF NOT EXISTS "
	94	+ TEMP_TABLE
	95	+ "_users(user_id TEXT NOT NULL)"
	96	)
	97	txn.execute(sql)
	98
	99	txn.execute("SELECT name FROM users")
	100	users = [{"user_id": x[0]} for x in txn.fetchall()]
	101
	102	self._simple_insert_many_txn(txn, TEMP_TABLE + "_users", users)
	103
	104	new_pos = yield self.get_max_stream_id_in_current_state_deltas()
	105	yield self.runInteraction(
	106	"populate_user_directory_temp_build", _make_staging_area
	107	)
	108	yield self._simple_insert(TEMP_TABLE + "_position", {"position": new_pos})
	109
	110	yield self._end_background_update("populate_user_directory_createtables")
	111	return 1
	112
	113	@defer.inlineCallbacks
	114	def _populate_user_directory_cleanup(self, progress, batch_size):
	115	"""
	116	Update the user directory stream position, then clean up the old tables.
	117	"""
	118	position = yield self._simple_select_one_onecol(
	119	TEMP_TABLE + "_position", None, "position"
	120	)
	121	yield self.update_user_directory_stream_pos(position)
	122
	123	def _delete_staging_area(txn):
	124	txn.execute("DROP TABLE IF EXISTS " + TEMP_TABLE + "_rooms")
	125	txn.execute("DROP TABLE IF EXISTS " + TEMP_TABLE + "_users")
	126	txn.execute("DROP TABLE IF EXISTS " + TEMP_TABLE + "_position")
	127
	128	yield self.runInteraction(
	129	"populate_user_directory_cleanup", _delete_staging_area
	130	)
	131
	132	yield self._end_background_update("populate_user_directory_cleanup")
	133	return 1
	134
	135	@defer.inlineCallbacks
	136	def _populate_user_directory_process_rooms(self, progress, batch_size):
	137	"""
	138	Args:
	139	progress (dict)
	140	batch_size (int): Maximum number of state events to process
	141	per cycle.
	142	"""
	143	state = self.hs.get_state_handler()
	144
	145	# If we don't have progress filed, delete everything.
	146	if not progress:
	147	yield self.delete_all_from_user_dir()
	148
	149	def _get_next_batch(txn):
	150	# Only fetch 250 rooms, so we don't fetch too many at once, even
	151	# if those 250 rooms have less than batch_size state events.
	152	sql = """
	153	SELECT room_id, events FROM %s
	154	ORDER BY events DESC
	155	LIMIT 250
	156	""" % (
	157	TEMP_TABLE + "_rooms",
	158	)
	159	txn.execute(sql)
	160	rooms_to_work_on = txn.fetchall()
	161
	162	if not rooms_to_work_on:
	163	return None
	164
	165	# Get how many are left to process, so we can give status on how
	166	# far we are in processing
	167	txn.execute("SELECT COUNT(*) FROM " + TEMP_TABLE + "_rooms")
	168	progress["remaining"] = txn.fetchone()[0]
	169
	170	return rooms_to_work_on
	171
	172	rooms_to_work_on = yield self.runInteraction(
	173	"populate_user_directory_temp_read", _get_next_batch
	174	)
	175
	176	# No more rooms -- complete the transaction.
	177	if not rooms_to_work_on:
	178	yield self._end_background_update("populate_user_directory_process_rooms")
	179	return 1
	180
	181	logger.info(
	182	"Processing the next %d rooms of %d remaining"
	183	% (len(rooms_to_work_on), progress["remaining"])
	184	)
	185
	186	processed_event_count = 0
	187
	188	for room_id, event_count in rooms_to_work_on:
	189	is_in_room = yield self.is_host_joined(room_id, self.server_name)
	190
	191	if is_in_room:
	192	is_public = yield self.is_room_world_readable_or_publicly_joinable(
	193	room_id
	194	)
	195
	196	users_with_profile = yield state.get_current_users_in_room(room_id)
	197	user_ids = set(users_with_profile)
	198
	199	# Update each user in the user directory.
	200	for user_id, profile in users_with_profile.items():
	201	yield self.update_profile_in_user_dir(
	202	user_id, profile.display_name, profile.avatar_url
	203	)
	204
	205	to_insert = set()
	206
	207	if is_public:
	208	for user_id in user_ids:
	209	if self.get_if_app_services_interested_in_user(user_id):
	210	continue
	211
	212	to_insert.add(user_id)
	213
	214	if to_insert:
	215	yield self.add_users_in_public_rooms(room_id, to_insert)
	216	to_insert.clear()
	217	else:
	218	for user_id in user_ids:
	219	if not self.hs.is_mine_id(user_id):
	220	continue
	221
	222	if self.get_if_app_services_interested_in_user(user_id):
	223	continue
	224
	225	for other_user_id in user_ids:
	226	if user_id == other_user_id:
	227	continue
	228
	229	user_set = (user_id, other_user_id)
	230	to_insert.add(user_set)
	231
	232	# If it gets too big, stop and write to the database
	233	# to prevent storing too much in RAM.
	234	if len(to_insert) >= self.SHARE_PRIVATE_WORKING_SET:
	235	yield self.add_users_who_share_private_room(
	236	room_id, to_insert
	237	)
	238	to_insert.clear()
	239
	240	if to_insert:
	241	yield self.add_users_who_share_private_room(room_id, to_insert)
	242	to_insert.clear()
	243
	244	# We've finished a room. Delete it from the table.
	245	yield self._simple_delete_one(TEMP_TABLE + "_rooms", {"room_id": room_id})
	246	# Update the remaining counter.
	247	progress["remaining"] -= 1
	248	yield self.runInteraction(
	249	"populate_user_directory",
	250	self._background_update_progress_txn,
	251	"populate_user_directory_process_rooms",
	252	progress,
	253	)
	254
	255	processed_event_count += event_count
	256
	257	if processed_event_count > batch_size:
	258	# Don't process any more rooms, we've hit our batch size.
	259	return processed_event_count
	260
	261	return processed_event_count
	262
	263	@defer.inlineCallbacks
	264	def _populate_user_directory_process_users(self, progress, batch_size):
	265	"""
	266	If search_all_users is enabled, add all of the users to the user directory.
	267	"""
	268	if not self.hs.config.user_directory_search_all_users:
	269	yield self._end_background_update("populate_user_directory_process_users")
	270	return 1
	271
	272	def _get_next_batch(txn):
	273	sql = "SELECT user_id FROM %s LIMIT %s" % (
	274	TEMP_TABLE + "_users",
	275	str(batch_size),
	276	)
	277	txn.execute(sql)
	278	users_to_work_on = txn.fetchall()
	279
	280	if not users_to_work_on:
	281	return None
	282
	283	users_to_work_on = [x[0] for x in users_to_work_on]
	284
	285	# Get how many are left to process, so we can give status on how
	286	# far we are in processing
	287	sql = "SELECT COUNT(*) FROM " + TEMP_TABLE + "_users"
	288	txn.execute(sql)
	289	progress["remaining"] = txn.fetchone()[0]
	290
	291	return users_to_work_on
	292
	293	users_to_work_on = yield self.runInteraction(
	294	"populate_user_directory_temp_read", _get_next_batch
	295	)
	296
	297	# No more users -- complete the transaction.
	298	if not users_to_work_on:
	299	yield self._end_background_update("populate_user_directory_process_users")
	300	return 1
	301
	302	logger.info(
	303	"Processing the next %d users of %d remaining"
	304	% (len(users_to_work_on), progress["remaining"])
	305	)
	306
	307	for user_id in users_to_work_on:
	308	profile = yield self.get_profileinfo(get_localpart_from_id(user_id))
	309	yield self.update_profile_in_user_dir(
	310	user_id, profile.display_name, profile.avatar_url
	311	)
	312
	313	# We've finished processing a user. Delete it from the table.
	314	yield self._simple_delete_one(TEMP_TABLE + "_users", {"user_id": user_id})
	315	# Update the remaining counter.
	316	progress["remaining"] -= 1
	317	yield self.runInteraction(
	318	"populate_user_directory",
	319	self._background_update_progress_txn,
	320	"populate_user_directory_process_users",
	321	progress,
	322	)
	323
	324	return len(users_to_work_on)
	325
	326	@defer.inlineCallbacks
	327	def is_room_world_readable_or_publicly_joinable(self, room_id):
	328	"""Check if the room is either world_readable or publically joinable
	329	"""
	330
	331	# Create a state filter that only queries join and history state event
	332	types_to_filter = (
	333	(EventTypes.JoinRules, ""),
	334	(EventTypes.RoomHistoryVisibility, ""),
	335	)
	336
	337	current_state_ids = yield self.get_filtered_current_state_ids(
	338	room_id, StateFilter.from_types(types_to_filter)
	339	)
	340
	341	join_rules_id = current_state_ids.get((EventTypes.JoinRules, ""))
	342	if join_rules_id:
	343	join_rule_ev = yield self.get_event(join_rules_id, allow_none=True)
	344	if join_rule_ev:
	345	if join_rule_ev.content.get("join_rule") == JoinRules.PUBLIC:
	346	return True
	347
	348	hist_vis_id = current_state_ids.get((EventTypes.RoomHistoryVisibility, ""))
	349	if hist_vis_id:
	350	hist_vis_ev = yield self.get_event(hist_vis_id, allow_none=True)
	351	if hist_vis_ev:
	352	if hist_vis_ev.content.get("history_visibility") == "world_readable":
	353	return True
	354
	355	return False
	356
	357	def update_profile_in_user_dir(self, user_id, display_name, avatar_url):
	358	"""
	359	Update or add a user's profile in the user directory.
	360	"""
	361
	362	def _update_profile_in_user_dir_txn(txn):
	363	new_entry = self._simple_upsert_txn(
	364	txn,
	365	table="user_directory",
	366	keyvalues={"user_id": user_id},
	367	values={"display_name": display_name, "avatar_url": avatar_url},
	368	lock=False, # We're only inserter
	369	)
	370
	371	if isinstance(self.database_engine, PostgresEngine):
	372	# We weight the localpart most highly, then display name and finally
	373	# server name
	374	if self.database_engine.can_native_upsert:
	375	sql = """
	376	INSERT INTO user_directory_search(user_id, vector)
	377	VALUES (?,
	378	setweight(to_tsvector('english', ?), 'A')
	379	\|\| setweight(to_tsvector('english', ?), 'D')
	380	\|\| setweight(to_tsvector('english', COALESCE(?, '')), 'B')
	381	) ON CONFLICT (user_id) DO UPDATE SET vector=EXCLUDED.vector
	382	"""
	383	txn.execute(
	384	sql,
	385	(
	386	user_id,
	387	get_localpart_from_id(user_id),
	388	get_domain_from_id(user_id),
	389	display_name,
	390	),
	391	)
	392	else:
	393	# TODO: Remove this code after we've bumped the minimum version
	394	# of postgres to always support upserts, so we can get rid of
	395	# `new_entry` usage
	396	if new_entry is True:
	397	sql = """
	398	INSERT INTO user_directory_search(user_id, vector)
	399	VALUES (?,
	400	setweight(to_tsvector('english', ?), 'A')
	401	\|\| setweight(to_tsvector('english', ?), 'D')
	402	\|\| setweight(to_tsvector('english', COALESCE(?, '')), 'B')
	403	)
	404	"""
	405	txn.execute(
	406	sql,
	407	(
	408	user_id,
	409	get_localpart_from_id(user_id),
	410	get_domain_from_id(user_id),
	411	display_name,
	412	),
	413	)
	414	elif new_entry is False:
	415	sql = """
	416	UPDATE user_directory_search
	417	SET vector = setweight(to_tsvector('english', ?), 'A')
	418	\|\| setweight(to_tsvector('english', ?), 'D')
	419	\|\| setweight(to_tsvector('english', COALESCE(?, '')), 'B')
	420	WHERE user_id = ?
	421	"""
	422	txn.execute(
	423	sql,
	424	(
	425	get_localpart_from_id(user_id),
	426	get_domain_from_id(user_id),
	427	display_name,
	428	user_id,
	429	),
	430	)
	431	else:
	432	raise RuntimeError(
	433	"upsert returned None when 'can_native_upsert' is False"
	434	)
	435	elif isinstance(self.database_engine, Sqlite3Engine):
	436	value = "%s %s" % (user_id, display_name) if display_name else user_id
	437	self._simple_upsert_txn(
	438	txn,
	439	table="user_directory_search",
	440	keyvalues={"user_id": user_id},
	441	values={"value": value},
	442	lock=False, # We're only inserter
	443	)
	444	else:
	445	# This should be unreachable.
	446	raise Exception("Unrecognized database engine")
	447
	448	txn.call_after(self.get_user_in_directory.invalidate, (user_id,))
	449
	450	return self.runInteraction(
	451	"update_profile_in_user_dir", _update_profile_in_user_dir_txn
	452	)
	453
	454	def add_users_who_share_private_room(self, room_id, user_id_tuples):
	455	"""Insert entries into the users_who_share_private_rooms table. The first
	456	user should be a local user.
	457
	458	Args:
	459	room_id (str)
	460	user_id_tuples([(str, str)]): iterable of 2-tuple of user IDs.
	461	"""
	462
	463	def _add_users_who_share_room_txn(txn):
	464	self._simple_upsert_many_txn(
	465	txn,
	466	table="users_who_share_private_rooms",
	467	key_names=["user_id", "other_user_id", "room_id"],
	468	key_values=[
	469	(user_id, other_user_id, room_id)
	470	for user_id, other_user_id in user_id_tuples
	471	],
	472	value_names=(),
	473	value_values=None,
	474	)
	475
	476	return self.runInteraction(
	477	"add_users_who_share_room", _add_users_who_share_room_txn
	478	)
	479
	480	def add_users_in_public_rooms(self, room_id, user_ids):
	481	"""Insert entries into the users_who_share_private_rooms table. The first
	482	user should be a local user.
	483
	484	Args:
	485	room_id (str)
	486	user_ids (list[str])
	487	"""
	488
	489	def _add_users_in_public_rooms_txn(txn):
	490
	491	self._simple_upsert_many_txn(
	492	txn,
	493	table="users_in_public_rooms",
	494	key_names=["user_id", "room_id"],
	495	key_values=[(user_id, room_id) for user_id in user_ids],
	496	value_names=(),
	497	value_values=None,
	498	)
	499
	500	return self.runInteraction(
	501	"add_users_in_public_rooms", _add_users_in_public_rooms_txn
	502	)
	503
	504	def delete_all_from_user_dir(self):
	505	"""Delete the entire user directory
	506	"""
	507
	508	def _delete_all_from_user_dir_txn(txn):
	509	txn.execute("DELETE FROM user_directory")
	510	txn.execute("DELETE FROM user_directory_search")
	511	txn.execute("DELETE FROM users_in_public_rooms")
	512	txn.execute("DELETE FROM users_who_share_private_rooms")
	513	txn.call_after(self.get_user_in_directory.invalidate_all)
	514
	515	return self.runInteraction(
	516	"delete_all_from_user_dir", _delete_all_from_user_dir_txn
	517	)
	518
	519	@cached()
	520	def get_user_in_directory(self, user_id):
	521	return self._simple_select_one(
	522	table="user_directory",
	523	keyvalues={"user_id": user_id},
	524	retcols=("display_name", "avatar_url"),
	525	allow_none=True,
	526	desc="get_user_in_directory",
	527	)
	528
	529	def update_user_directory_stream_pos(self, stream_id):
	530	return self._simple_update_one(
	531	table="user_directory_stream_pos",
	532	keyvalues={},
	533	updatevalues={"stream_id": stream_id},
	534	desc="update_user_directory_stream_pos",
	535	)
	536
	537
	538	class UserDirectoryStore(UserDirectoryBackgroundUpdateStore):
	539
	540	# How many records do we calculate before sending it to
	541	# add_users_who_share_private_rooms?
	542	SHARE_PRIVATE_WORKING_SET = 500
	543
	544	def __init__(self, db_conn, hs):
	545	super(UserDirectoryStore, self).__init__(db_conn, hs)
	546
	547	def remove_from_user_dir(self, user_id):
	548	def _remove_from_user_dir_txn(txn):
	549	self._simple_delete_txn(
	550	txn, table="user_directory", keyvalues={"user_id": user_id}
	551	)
	552	self._simple_delete_txn(
	553	txn, table="user_directory_search", keyvalues={"user_id": user_id}
	554	)
	555	self._simple_delete_txn(
	556	txn, table="users_in_public_rooms", keyvalues={"user_id": user_id}
	557	)
	558	self._simple_delete_txn(
	559	txn,
	560	table="users_who_share_private_rooms",
	561	keyvalues={"user_id": user_id},
	562	)
	563	self._simple_delete_txn(
	564	txn,
	565	table="users_who_share_private_rooms",
	566	keyvalues={"other_user_id": user_id},
	567	)
	568	txn.call_after(self.get_user_in_directory.invalidate, (user_id,))
	569
	570	return self.runInteraction("remove_from_user_dir", _remove_from_user_dir_txn)
	571
	572	@defer.inlineCallbacks
	573	def get_users_in_dir_due_to_room(self, room_id):
	574	"""Get all user_ids that are in the room directory because they're
	575	in the given room_id
	576	"""
	577	user_ids_share_pub = yield self._simple_select_onecol(
	578	table="users_in_public_rooms",
	579	keyvalues={"room_id": room_id},
	580	retcol="user_id",
	581	desc="get_users_in_dir_due_to_room",
	582	)
	583
	584	user_ids_share_priv = yield self._simple_select_onecol(
	585	table="users_who_share_private_rooms",
	586	keyvalues={"room_id": room_id},
	587	retcol="other_user_id",
	588	desc="get_users_in_dir_due_to_room",
	589	)
	590
	591	user_ids = set(user_ids_share_pub)
	592	user_ids.update(user_ids_share_priv)
	593
	594	return user_ids
	595
	596	def remove_user_who_share_room(self, user_id, room_id):
	597	"""
	598	Deletes entries in the users_who_share_*_rooms table. The first
	599	user should be a local user.
	600
	601	Args:
	602	user_id (str)
	603	room_id (str)
	604	"""
	605
	606	def _remove_user_who_share_room_txn(txn):
	607	self._simple_delete_txn(
	608	txn,
	609	table="users_who_share_private_rooms",
	610	keyvalues={"user_id": user_id, "room_id": room_id},
	611	)
	612	self._simple_delete_txn(
	613	txn,
	614	table="users_who_share_private_rooms",
	615	keyvalues={"other_user_id": user_id, "room_id": room_id},
	616	)
	617	self._simple_delete_txn(
	618	txn,
	619	table="users_in_public_rooms",
	620	keyvalues={"user_id": user_id, "room_id": room_id},
	621	)
	622
	623	return self.runInteraction(
	624	"remove_user_who_share_room", _remove_user_who_share_room_txn
	625	)
	626
	627	@defer.inlineCallbacks
	628	def get_user_dir_rooms_user_is_in(self, user_id):
	629	"""
	630	Returns the rooms that a user is in.
	631
	632	Args:
	633	user_id(str): Must be a local user
	634
	635	Returns:
	636	list: user_id
	637	"""
	638	rows = yield self._simple_select_onecol(
	639	table="users_who_share_private_rooms",
	640	keyvalues={"user_id": user_id},
	641	retcol="room_id",
	642	desc="get_rooms_user_is_in",
	643	)
	644
	645	pub_rows = yield self._simple_select_onecol(
	646	table="users_in_public_rooms",
	647	keyvalues={"user_id": user_id},
	648	retcol="room_id",
	649	desc="get_rooms_user_is_in",
	650	)
	651
	652	users = set(pub_rows)
	653	users.update(rows)
	654	return list(users)
	655
	656	@defer.inlineCallbacks
	657	def get_rooms_in_common_for_users(self, user_id, other_user_id):
	658	"""Given two user_ids find out the list of rooms they share.
	659	"""
	660	sql = """
	661	SELECT room_id FROM (
	662	SELECT c.room_id FROM current_state_events AS c
	663	INNER JOIN room_memberships AS m USING (event_id)
	664	WHERE type = 'm.room.member'
	665	AND m.membership = 'join'
	666	AND state_key = ?
	667	) AS f1 INNER JOIN (
	668	SELECT c.room_id FROM current_state_events AS c
	669	INNER JOIN room_memberships AS m USING (event_id)
	670	WHERE type = 'm.room.member'
	671	AND m.membership = 'join'
	672	AND state_key = ?
	673	) f2 USING (room_id)
	674	"""
	675
	676	rows = yield self._execute(
	677	"get_rooms_in_common_for_users", None, sql, user_id, other_user_id
	678	)
	679
	680	return [room_id for room_id, in rows]
	681
	682	def get_user_directory_stream_pos(self):
	683	return self._simple_select_one_onecol(
	684	table="user_directory_stream_pos",
	685	keyvalues={},
	686	retcol="stream_id",
	687	desc="get_user_directory_stream_pos",
	688	)
	689
	690	@defer.inlineCallbacks
	691	def search_user_dir(self, user_id, search_term, limit):
	692	"""Searches for users in directory
	693
	694	Returns:
	695	dict of the form::
	696
	697	{
	698	"limited": <bool>, # whether there were more results or not
	699	"results": [ # Ordered by best match first
	700	{
	701	"user_id": <user_id>,
	702	"display_name": <display_name>,
	703	"avatar_url": <avatar_url>
	704	}
	705	]
	706	}
	707	"""
	708
	709	if self.hs.config.user_directory_search_all_users:
	710	join_args = (user_id,)
	711	where_clause = "user_id != ?"
	712	else:
	713	join_args = (user_id,)
	714	where_clause = """
	715	(
	716	EXISTS (select 1 from users_in_public_rooms WHERE user_id = t.user_id)
	717	OR EXISTS (
	718	SELECT 1 FROM users_who_share_private_rooms
	719	WHERE user_id = ? AND other_user_id = t.user_id
	720	)
	721	)
	722	"""
	723
	724	if isinstance(self.database_engine, PostgresEngine):
	725	full_query, exact_query, prefix_query = _parse_query_postgres(search_term)
	726
	727	# We order by rank and then if they have profile info
	728	# The ranking algorithm is hand tweaked for "best" results. Broadly
	729	# the idea is we give a higher weight to exact matches.
	730	# The array of numbers are the weights for the various part of the
	731	# search: (domain, _, display name, localpart)
	732	sql = """
	733	SELECT d.user_id AS user_id, display_name, avatar_url
	734	FROM user_directory_search as t
	735	INNER JOIN user_directory AS d USING (user_id)
	736	WHERE
	737	%s
	738	AND vector @@ to_tsquery('english', ?)
	739	ORDER BY
	740	(CASE WHEN d.user_id IS NOT NULL THEN 4.0 ELSE 1.0 END)
	741	* (CASE WHEN display_name IS NOT NULL THEN 1.2 ELSE 1.0 END)
	742	* (CASE WHEN avatar_url IS NOT NULL THEN 1.2 ELSE 1.0 END)
	743	* (
	744	3 * ts_rank_cd(
	745	'{0.1, 0.1, 0.9, 1.0}',
	746	vector,
	747	to_tsquery('english', ?),
	748	8
	749	)
	750	+ ts_rank_cd(
	751	'{0.1, 0.1, 0.9, 1.0}',
	752	vector,
	753	to_tsquery('english', ?),
	754	8
	755	)
	756	)
	757	DESC,
	758	display_name IS NULL,
	759	avatar_url IS NULL
	760	LIMIT ?
	761	""" % (
	762	where_clause,
	763	)
	764	args = join_args + (full_query, exact_query, prefix_query, limit + 1)
	765	elif isinstance(self.database_engine, Sqlite3Engine):
	766	search_query = _parse_query_sqlite(search_term)
	767
	768	sql = """
	769	SELECT d.user_id AS user_id, display_name, avatar_url
	770	FROM user_directory_search as t
	771	INNER JOIN user_directory AS d USING (user_id)
	772	WHERE
	773	%s
	774	AND value MATCH ?
	775	ORDER BY
	776	rank(matchinfo(user_directory_search)) DESC,
	777	display_name IS NULL,
	778	avatar_url IS NULL
	779	LIMIT ?
	780	""" % (
	781	where_clause,
	782	)
	783	args = join_args + (search_query, limit + 1)
	784	else:
	785	# This should be unreachable.
	786	raise Exception("Unrecognized database engine")
	787
	788	results = yield self._execute(
	789	"search_user_dir", self.cursor_to_dict, sql, *args
	790	)
	791
	792	limited = len(results) > limit
	793
	794	return {"limited": limited, "results": results}
	795
	796
	797	def _parse_query_sqlite(search_term):
	798	"""Takes a plain unicode string from the user and converts it into a form
	799	that can be passed to database.
	800	We use this so that we can add prefix matching, which isn't something
	801	that is supported by default.
	802
	803	We specifically add both a prefix and non prefix matching term so that
	804	exact matches get ranked higher.
	805	"""
	806
	807	# Pull out the individual words, discarding any non-word characters.
	808	results = re.findall(r"([\w\-]+)", search_term, re.UNICODE)
	809	return " & ".join("(%s* OR %s)" % (result, result) for result in results)
	810
	811
	812	def _parse_query_postgres(search_term):
	813	"""Takes a plain unicode string from the user and converts it into a form
	814	that can be passed to database.
	815	We use this so that we can add prefix matching, which isn't something
	816	that is supported by default.
	817	"""
	818
	819	# Pull out the individual words, discarding any non-word characters.
	820	results = re.findall(r"([\w\-]+)", search_term, re.UNICODE)
	821
	822	both = " & ".join("(%s:* \| %s)" % (result, result) for result in results)
	823	exact = " & ".join("%s" % (result,) for result in results)
	824	prefix = " & ".join("%s:*" % (result,) for result in results)
	825
	826	return both, exact, prefix

+91

-0

synapse/storage/data_stores/main/user_erasure_store.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2018 New Vector Ltd
	2	#
	3	# Licensed under the Apache License, Version 2.0 (the "License");
	4	# you may not use this file except in compliance with the License.
	5	# You may obtain a copy of the License at
	6	#
	7	# http://www.apache.org/licenses/LICENSE-2.0
	8	#
	9	# Unless required by applicable law or agreed to in writing, software
	10	# distributed under the License is distributed on an "AS IS" BASIS,
	11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	# See the License for the specific language governing permissions and
	13	# limitations under the License.
	14
	15	import operator
	16
	17	from synapse.storage._base import SQLBaseStore
	18	from synapse.util.caches.descriptors import cached, cachedList
	19
	20
	21	class UserErasureWorkerStore(SQLBaseStore):
	22	@cached()
	23	def is_user_erased(self, user_id):
	24	"""
	25	Check if the given user id has requested erasure
	26
	27	Args:
	28	user_id (str): full user id to check
	29
	30	Returns:
	31	Deferred[bool]: True if the user has requested erasure
	32	"""
	33	return self._simple_select_onecol(
	34	table="erased_users",
	35	keyvalues={"user_id": user_id},
	36	retcol="1",
	37	desc="is_user_erased",
	38	).addCallback(operator.truth)
	39
	40	@cachedList(
	41	cached_method_name="is_user_erased", list_name="user_ids", inlineCallbacks=True
	42	)
	43	def are_users_erased(self, user_ids):
	44	"""
	45	Checks which users in a list have requested erasure
	46
	47	Args:
	48	user_ids (iterable[str]): full user id to check
	49
	50	Returns:
	51	Deferred[dict[str, bool]]:
	52	for each user, whether the user has requested erasure.
	53	"""
	54	# this serves the dual purpose of (a) making sure we can do len and
	55	# iterate it multiple times, and (b) avoiding duplicates.
	56	user_ids = tuple(set(user_ids))
	57
	58	rows = yield self._simple_select_many_batch(
	59	table="erased_users",
	60	column="user_id",
	61	iterable=user_ids,
	62	retcols=("user_id",),
	63	desc="are_users_erased",
	64	)
	65	erased_users = set(row["user_id"] for row in rows)
	66
	67	res = dict((u, u in erased_users) for u in user_ids)
	68	return res
	69
	70
	71	class UserErasureStore(UserErasureWorkerStore):
	72	def mark_user_erased(self, user_id):
	73	"""Indicate that user_id wishes their message history to be erased.
	74
	75	Args:
	76	user_id (str): full user_id to be erased
	77	"""
	78
	79	def f(txn):
	80	# first check if they are already in the list
	81	txn.execute("SELECT 1 FROM erased_users WHERE user_id = ?", (user_id,))
	82	if txn.fetchone():
	83	return
	84
	85	# they are not already there: do the insert.
	86	txn.execute("INSERT INTO erased_users (user_id) VALUES (?)", (user_id,))
	87
	88	self._invalidate_cache_and_stream(txn, self.is_user_erased, (user_id,))
	89
	90	return self.runInteraction("mark_user_erased", f)

+0

-450

~~synapse/storage/deviceinbox.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2016 OpenMarket Ltd
2		#
3		# Licensed under the Apache License, Version 2.0 (the "License");
4		# you may not use this file except in compliance with the License.
5		# You may obtain a copy of the License at
6		#
7		# http://www.apache.org/licenses/LICENSE-2.0
8		#
9		# Unless required by applicable law or agreed to in writing, software
10		# distributed under the License is distributed on an "AS IS" BASIS,
11		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12		# See the License for the specific language governing permissions and
13		# limitations under the License.
14
15		import logging
16
17		from canonicaljson import json
18
19		from twisted.internet import defer
20
21		from synapse.logging.opentracing import log_kv, set_tag, trace
22		from synapse.storage._base import SQLBaseStore
23		from synapse.storage.background_updates import BackgroundUpdateStore
24		from synapse.util.caches.expiringcache import ExpiringCache
25
26		logger = logging.getLogger(__name__)
27
28
29		class DeviceInboxWorkerStore(SQLBaseStore):
30		def get_to_device_stream_token(self):
31		return self._device_inbox_id_gen.get_current_token()
32
33		def get_new_messages_for_device(
34		self, user_id, device_id, last_stream_id, current_stream_id, limit=100
35		):
36		"""
37		Args:
38		user_id(str): The recipient user_id.
39		device_id(str): The recipient device_id.
40		current_stream_id(int): The current position of the to device
41		message stream.
42		Returns:
43		Deferred ([dict], int): List of messages for the device and where
44		in the stream the messages got to.
45		"""
46		has_changed = self._device_inbox_stream_cache.has_entity_changed(
47		user_id, last_stream_id
48		)
49		if not has_changed:
50		return defer.succeed(([], current_stream_id))
51
52		def get_new_messages_for_device_txn(txn):
53		sql = (
54		"SELECT stream_id, message_json FROM device_inbox"
55		" WHERE user_id = ? AND device_id = ?"
56		" AND ? < stream_id AND stream_id <= ?"
57		" ORDER BY stream_id ASC"
58		" LIMIT ?"
59		)
60		txn.execute(
61		sql, (user_id, device_id, last_stream_id, current_stream_id, limit)
62		)
63		messages = []
64		for row in txn:
65		stream_pos = row[0]
66		messages.append(json.loads(row[1]))
67		if len(messages) < limit:
68		stream_pos = current_stream_id
69		return messages, stream_pos
70
71		return self.runInteraction(
72		"get_new_messages_for_device", get_new_messages_for_device_txn
73		)
74
75		@trace
76		@defer.inlineCallbacks
77		def delete_messages_for_device(self, user_id, device_id, up_to_stream_id):
78		"""
79		Args:
80		user_id(str): The recipient user_id.
81		device_id(str): The recipient device_id.
82		up_to_stream_id(int): Where to delete messages up to.
83		Returns:
84		A deferred that resolves to the number of messages deleted.
85		"""
86		# If we have cached the last stream id we've deleted up to, we can
87		# check if there is likely to be anything that needs deleting
88		last_deleted_stream_id = self._last_device_delete_cache.get(
89		(user_id, device_id), None
90		)
91
92		set_tag("last_deleted_stream_id", last_deleted_stream_id)
93
94		if last_deleted_stream_id:
95		has_changed = self._device_inbox_stream_cache.has_entity_changed(
96		user_id, last_deleted_stream_id
97		)
98		if not has_changed:
99		log_kv({"message": "No changes in cache since last check"})
100		return 0
101
102		def delete_messages_for_device_txn(txn):
103		sql = (
104		"DELETE FROM device_inbox"
105		" WHERE user_id = ? AND device_id = ?"
106		" AND stream_id <= ?"
107		)
108		txn.execute(sql, (user_id, device_id, up_to_stream_id))
109		return txn.rowcount
110
111		count = yield self.runInteraction(
112		"delete_messages_for_device", delete_messages_for_device_txn
113		)
114
115		log_kv(
116		{"message": "deleted {} messages for device".format(count), "count": count}
117		)
118
119		# Update the cache, ensuring that we only ever increase the value
120		last_deleted_stream_id = self._last_device_delete_cache.get(
121		(user_id, device_id), 0
122		)
123		self._last_device_delete_cache[(user_id, device_id)] = max(
124		last_deleted_stream_id, up_to_stream_id
125		)
126
127		return count
128
129		@trace
130		def get_new_device_msgs_for_remote(
131		self, destination, last_stream_id, current_stream_id, limit
132		):
133		"""
134		Args:
135		destination(str): The name of the remote server.
136		last_stream_id(int\|long): The last position of the device message stream
137		that the server sent up to.
138		current_stream_id(int\|long): The current position of the device
139		message stream.
140		Returns:
141		Deferred ([dict], int\|long): List of messages for the device and where
142		in the stream the messages got to.
143		"""
144
145		set_tag("destination", destination)
146		set_tag("last_stream_id", last_stream_id)
147		set_tag("current_stream_id", current_stream_id)
148		set_tag("limit", limit)
149
150		has_changed = self._device_federation_outbox_stream_cache.has_entity_changed(
151		destination, last_stream_id
152		)
153		if not has_changed or last_stream_id == current_stream_id:
154		log_kv({"message": "No new messages in stream"})
155		return defer.succeed(([], current_stream_id))
156
157		if limit <= 0:
158		# This can happen if we run out of room for EDUs in the transaction.
159		return defer.succeed(([], last_stream_id))
160
161		@trace
162		def get_new_messages_for_remote_destination_txn(txn):
163		sql = (
164		"SELECT stream_id, messages_json FROM device_federation_outbox"
165		" WHERE destination = ?"
166		" AND ? < stream_id AND stream_id <= ?"
167		" ORDER BY stream_id ASC"
168		" LIMIT ?"
169		)
170		txn.execute(sql, (destination, last_stream_id, current_stream_id, limit))
171		messages = []
172		for row in txn:
173		stream_pos = row[0]
174		messages.append(json.loads(row[1]))
175		if len(messages) < limit:
176		log_kv({"message": "Set stream position to current position"})
177		stream_pos = current_stream_id
178		return messages, stream_pos
179
180		return self.runInteraction(
181		"get_new_device_msgs_for_remote",
182		get_new_messages_for_remote_destination_txn,
183		)
184
185		@trace
186		def delete_device_msgs_for_remote(self, destination, up_to_stream_id):
187		"""Used to delete messages when the remote destination acknowledges
188		their receipt.
189
190		Args:
191		destination(str): The destination server_name
192		up_to_stream_id(int): Where to delete messages up to.
193		Returns:
194		A deferred that resolves when the messages have been deleted.
195		"""
196
197		def delete_messages_for_remote_destination_txn(txn):
198		sql = (
199		"DELETE FROM device_federation_outbox"
200		" WHERE destination = ?"
201		" AND stream_id <= ?"
202		)
203		txn.execute(sql, (destination, up_to_stream_id))
204
205		return self.runInteraction(
206		"delete_device_msgs_for_remote", delete_messages_for_remote_destination_txn
207		)
208
209
210		class DeviceInboxStore(DeviceInboxWorkerStore, BackgroundUpdateStore):
211		DEVICE_INBOX_STREAM_ID = "device_inbox_stream_drop"
212
213		def __init__(self, db_conn, hs):
214		super(DeviceInboxStore, self).__init__(db_conn, hs)
215
216		self.register_background_index_update(
217		"device_inbox_stream_index",
218		index_name="device_inbox_stream_id_user_id",
219		table="device_inbox",
220		columns=["stream_id", "user_id"],
221		)
222
223		self.register_background_update_handler(
224		self.DEVICE_INBOX_STREAM_ID, self._background_drop_index_device_inbox
225		)
226
227		# Map of (user_id, device_id) to the last stream_id that has been
228		# deleted up to. This is so that we can no op deletions.
229		self._last_device_delete_cache = ExpiringCache(
230		cache_name="last_device_delete_cache",
231		clock=self._clock,
232		max_len=10000,
233		expiry_ms=30 * 60 * 1000,
234		)
235
236		@trace
237		@defer.inlineCallbacks
238		def add_messages_to_device_inbox(
239		self, local_messages_by_user_then_device, remote_messages_by_destination
240		):
241		"""Used to send messages from this server.
242
243		Args:
244		sender_user_id(str): The ID of the user sending these messages.
245		local_messages_by_user_and_device(dict):
246		Dictionary of user_id to device_id to message.
247		remote_messages_by_destination(dict):
248		Dictionary of destination server_name to the EDU JSON to send.
249		Returns:
250		A deferred stream_id that resolves when the messages have been
251		inserted.
252		"""
253
254		def add_messages_txn(txn, now_ms, stream_id):
255		# Add the local messages directly to the local inbox.
256		self._add_messages_to_local_device_inbox_txn(
257		txn, stream_id, local_messages_by_user_then_device
258		)
259
260		# Add the remote messages to the federation outbox.
261		# We'll send them to a remote server when we next send a
262		# federation transaction to that destination.
263		sql = (
264		"INSERT INTO device_federation_outbox"
265		" (destination, stream_id, queued_ts, messages_json)"
266		" VALUES (?,?,?,?)"
267		)
268		rows = []
269		for destination, edu in remote_messages_by_destination.items():
270		edu_json = json.dumps(edu)
271		rows.append((destination, stream_id, now_ms, edu_json))
272		txn.executemany(sql, rows)
273
274		with self._device_inbox_id_gen.get_next() as stream_id:
275		now_ms = self.clock.time_msec()
276		yield self.runInteraction(
277		"add_messages_to_device_inbox", add_messages_txn, now_ms, stream_id
278		)
279		for user_id in local_messages_by_user_then_device.keys():
280		self._device_inbox_stream_cache.entity_has_changed(user_id, stream_id)
281		for destination in remote_messages_by_destination.keys():
282		self._device_federation_outbox_stream_cache.entity_has_changed(
283		destination, stream_id
284		)
285
286		return self._device_inbox_id_gen.get_current_token()
287
288		@defer.inlineCallbacks
289		def add_messages_from_remote_to_device_inbox(
290		self, origin, message_id, local_messages_by_user_then_device
291		):
292		def add_messages_txn(txn, now_ms, stream_id):
293		# Check if we've already inserted a matching message_id for that
294		# origin. This can happen if the origin doesn't receive our
295		# acknowledgement from the first time we received the message.
296		already_inserted = self._simple_select_one_txn(
297		txn,
298		table="device_federation_inbox",
299		keyvalues={"origin": origin, "message_id": message_id},
300		retcols=("message_id",),
301		allow_none=True,
302		)
303		if already_inserted is not None:
304		return
305
306		# Add an entry for this message_id so that we know we've processed
307		# it.
308		self._simple_insert_txn(
309		txn,
310		table="device_federation_inbox",
311		values={
312		"origin": origin,
313		"message_id": message_id,
314		"received_ts": now_ms,
315		},
316		)
317
318		# Add the messages to the approriate local device inboxes so that
319		# they'll be sent to the devices when they next sync.
320		self._add_messages_to_local_device_inbox_txn(
321		txn, stream_id, local_messages_by_user_then_device
322		)
323
324		with self._device_inbox_id_gen.get_next() as stream_id:
325		now_ms = self.clock.time_msec()
326		yield self.runInteraction(
327		"add_messages_from_remote_to_device_inbox",
328		add_messages_txn,
329		now_ms,
330		stream_id,
331		)
332		for user_id in local_messages_by_user_then_device.keys():
333		self._device_inbox_stream_cache.entity_has_changed(user_id, stream_id)
334
335		return stream_id
336
337		def _add_messages_to_local_device_inbox_txn(
338		self, txn, stream_id, messages_by_user_then_device
339		):
340		sql = "UPDATE device_max_stream_id" " SET stream_id = ?" " WHERE stream_id < ?"
341		txn.execute(sql, (stream_id, stream_id))
342
343		local_by_user_then_device = {}
344		for user_id, messages_by_device in messages_by_user_then_device.items():
345		messages_json_for_user = {}
346		devices = list(messages_by_device.keys())
347		if len(devices) == 1 and devices[0] == "*":
348		# Handle wildcard device_ids.
349		sql = "SELECT device_id FROM devices" " WHERE user_id = ?"
350		txn.execute(sql, (user_id,))
351		message_json = json.dumps(messages_by_device["*"])
352		for row in txn:
353		# Add the message for all devices for this user on this
354		# server.
355		device = row[0]
356		messages_json_for_user[device] = message_json
357		else:
358		if not devices:
359		continue
360		sql = (
361		"SELECT device_id FROM devices"
362		" WHERE user_id = ? AND device_id IN ("
363		+ ",".join("?" * len(devices))
364		+ ")"
365		)
366		# TODO: Maybe this needs to be done in batches if there are
367		# too many local devices for a given user.
368		txn.execute(sql, [user_id] + devices)
369		for row in txn:
370		# Only insert into the local inbox if the device exists on
371		# this server
372		device = row[0]
373		message_json = json.dumps(messages_by_device[device])
374		messages_json_for_user[device] = message_json
375
376		if messages_json_for_user:
377		local_by_user_then_device[user_id] = messages_json_for_user
378
379		if not local_by_user_then_device:
380		return
381
382		sql = (
383		"INSERT INTO device_inbox"
384		" (user_id, device_id, stream_id, message_json)"
385		" VALUES (?,?,?,?)"
386		)
387		rows = []
388		for user_id, messages_by_device in local_by_user_then_device.items():
389		for device_id, message_json in messages_by_device.items():
390		rows.append((user_id, device_id, stream_id, message_json))
391
392		txn.executemany(sql, rows)
393
394		def get_all_new_device_messages(self, last_pos, current_pos, limit):
395		"""
396		Args:
397		last_pos(int):
398		current_pos(int):
399		limit(int):
400		Returns:
401		A deferred list of rows from the device inbox
402		"""
403		if last_pos == current_pos:
404		return defer.succeed([])
405
406		def get_all_new_device_messages_txn(txn):
407		# We limit like this as we might have multiple rows per stream_id, and
408		# we want to make sure we always get all entries for any stream_id
409		# we return.
410		upper_pos = min(current_pos, last_pos + limit)
411		sql = (
412		"SELECT max(stream_id), user_id"
413		" FROM device_inbox"
414		" WHERE ? < stream_id AND stream_id <= ?"
415		" GROUP BY user_id"
416		)
417		txn.execute(sql, (last_pos, upper_pos))
418		rows = txn.fetchall()
419
420		sql = (
421		"SELECT max(stream_id), destination"
422		" FROM device_federation_outbox"
423		" WHERE ? < stream_id AND stream_id <= ?"
424		" GROUP BY destination"
425		)
426		txn.execute(sql, (last_pos, upper_pos))
427		rows.extend(txn)
428
429		# Order by ascending stream ordering
430		rows.sort()
431
432		return rows
433
434		return self.runInteraction(
435		"get_all_new_device_messages", get_all_new_device_messages_txn
436		)
437
438		@defer.inlineCallbacks
439		def _background_drop_index_device_inbox(self, progress, batch_size):
440		def reindex_txn(conn):
441		txn = conn.cursor()
442		txn.execute("DROP INDEX IF EXISTS device_inbox_stream_id")
443		txn.close()
444
445		yield self.runWithConnection(reindex_txn)
446
447		yield self._end_background_update(self.DEVICE_INBOX_STREAM_ID)
448
449		return 1

+0

-924

~~synapse/storage/devices.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2016 OpenMarket Ltd
2		#
3		# Licensed under the Apache License, Version 2.0 (the "License");
4		# you may not use this file except in compliance with the License.
5		# You may obtain a copy of the License at
6		#
7		# http://www.apache.org/licenses/LICENSE-2.0
8		#
9		# Unless required by applicable law or agreed to in writing, software
10		# distributed under the License is distributed on an "AS IS" BASIS,
11		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12		# See the License for the specific language governing permissions and
13		# limitations under the License.
14		import logging
15
16		from six import iteritems
17
18		from canonicaljson import json
19
20		from twisted.internet import defer
21
22		from synapse.api.errors import StoreError
23		from synapse.logging.opentracing import (
24		get_active_span_text_map,
25		set_tag,
26		trace,
27		whitelisted_homeserver,
28		)
29		from synapse.metrics.background_process_metrics import run_as_background_process
30		from synapse.storage._base import Cache, SQLBaseStore, db_to_json
31		from synapse.storage.background_updates import BackgroundUpdateStore
32		from synapse.util import batch_iter
33		from synapse.util.caches.descriptors import cached, cachedInlineCallbacks, cachedList
34
35		logger = logging.getLogger(__name__)
36
37		DROP_DEVICE_LIST_STREAMS_NON_UNIQUE_INDEXES = (
38		"drop_device_list_streams_non_unique_indexes"
39		)
40
41
42		class DeviceWorkerStore(SQLBaseStore):
43		def get_device(self, user_id, device_id):
44		"""Retrieve a device.
45
46		Args:
47		user_id (str): The ID of the user which owns the device
48		device_id (str): The ID of the device to retrieve
49		Returns:
50		defer.Deferred for a dict containing the device information
51		Raises:
52		StoreError: if the device is not found
53		"""
54		return self._simple_select_one(
55		table="devices",
56		keyvalues={"user_id": user_id, "device_id": device_id},
57		retcols=("user_id", "device_id", "display_name"),
58		desc="get_device",
59		)
60
61		@defer.inlineCallbacks
62		def get_devices_by_user(self, user_id):
63		"""Retrieve all of a user's registered devices.
64
65		Args:
66		user_id (str):
67		Returns:
68		defer.Deferred: resolves to a dict from device_id to a dict
69		containing "device_id", "user_id" and "display_name" for each
70		device.
71		"""
72		devices = yield self._simple_select_list(
73		table="devices",
74		keyvalues={"user_id": user_id},
75		retcols=("user_id", "device_id", "display_name"),
76		desc="get_devices_by_user",
77		)
78
79		return {d["device_id"]: d for d in devices}
80
81		@trace
82		@defer.inlineCallbacks
83		def get_devices_by_remote(self, destination, from_stream_id, limit):
84		"""Get stream of updates to send to remote servers
85
86		Returns:
87		Deferred[tuple[int, list[dict]]]:
88		current stream id (ie, the stream id of the last update included in the
89		response), and the list of updates
90		"""
91		now_stream_id = self._device_list_id_gen.get_current_token()
92
93		has_changed = self._device_list_federation_stream_cache.has_entity_changed(
94		destination, int(from_stream_id)
95		)
96		if not has_changed:
97		return now_stream_id, []
98
99		# We retrieve n+1 devices from the list of outbound pokes where n is
100		# our outbound device update limit. We then check if the very last
101		# device has the same stream_id as the second-to-last device. If so,
102		# then we ignore all devices with that stream_id and only send the
103		# devices with a lower stream_id.
104		#
105		# If when culling the list we end up with no devices afterwards, we
106		# consider the device update to be too large, and simply skip the
107		# stream_id; the rationale being that such a large device list update
108		# is likely an error.
109		updates = yield self.runInteraction(
110		"get_devices_by_remote",
111		self._get_devices_by_remote_txn,
112		destination,
113		from_stream_id,
114		now_stream_id,
115		limit + 1,
116		)
117
118		# Return an empty list if there are no updates
119		if not updates:
120		return now_stream_id, []
121
122		# if we have exceeded the limit, we need to exclude any results with the
123		# same stream_id as the last row.
124		if len(updates) > limit:
125		stream_id_cutoff = updates[-1][2]
126		now_stream_id = stream_id_cutoff - 1
127		else:
128		stream_id_cutoff = None
129
130		# Perform the equivalent of a GROUP BY
131		#
132		# Iterate through the updates list and copy non-duplicate
133		# (user_id, device_id) entries into a map, with the value being
134		# the max stream_id across each set of duplicate entries
135		#
136		# maps (user_id, device_id) -> (stream_id, opentracing_context)
137		# as long as their stream_id does not match that of the last row
138		#
139		# opentracing_context contains the opentracing metadata for the request
140		# that created the poke
141		#
142		# The most recent request's opentracing_context is used as the
143		# context which created the Edu.
144
145		query_map = {}
146		for update in updates:
147		if stream_id_cutoff is not None and update[2] >= stream_id_cutoff:
148		# Stop processing updates
149		break
150
151		key = (update[0], update[1])
152
153		update_context = update[3]
154		update_stream_id = update[2]
155
156		previous_update_stream_id, _ = query_map.get(key, (0, None))
157
158		if update_stream_id > previous_update_stream_id:
159		query_map[key] = (update_stream_id, update_context)
160
161		# If we didn't find any updates with a stream_id lower than the cutoff, it
162		# means that there are more than limit updates all of which have the same
163		# steam_id.
164
165		# That should only happen if a client is spamming the server with new
166		# devices, in which case E2E isn't going to work well anyway. We'll just
167		# skip that stream_id and return an empty list, and continue with the next
168		# stream_id next time.
169		if not query_map:
170		return stream_id_cutoff, []
171
172		results = yield self._get_device_update_edus_by_remote(
173		destination, from_stream_id, query_map
174		)
175
176		return now_stream_id, results
177
178		def _get_devices_by_remote_txn(
179		self, txn, destination, from_stream_id, now_stream_id, limit
180		):
181		"""Return device update information for a given remote destination
182
183		Args:
184		txn (LoggingTransaction): The transaction to execute
185		destination (str): The host the device updates are intended for
186		from_stream_id (int): The minimum stream_id to filter updates by, exclusive
187		now_stream_id (int): The maximum stream_id to filter updates by, inclusive
188		limit (int): Maximum number of device updates to return
189
190		Returns:
191		List: List of device updates
192		"""
193		sql = """
194		SELECT user_id, device_id, stream_id, opentracing_context FROM device_lists_outbound_pokes
195		WHERE destination = ? AND ? < stream_id AND stream_id <= ? AND sent = ?
196		ORDER BY stream_id
197		LIMIT ?
198		"""
199		txn.execute(sql, (destination, from_stream_id, now_stream_id, False, limit))
200
201		return list(txn)
202
203		@defer.inlineCallbacks
204		def _get_device_update_edus_by_remote(self, destination, from_stream_id, query_map):
205		"""Returns a list of device update EDUs as well as E2EE keys
206
207		Args:
208		destination (str): The host the device updates are intended for
209		from_stream_id (int): The minimum stream_id to filter updates by, exclusive
210		query_map (Dict[(str, str): (int, str\|None)]): Dictionary mapping
211		user_id/device_id to update stream_id and the relevent json-encoded
212		opentracing context
213
214		Returns:
215		List[Dict]: List of objects representing an device update EDU
216
217		"""
218		devices = yield self.runInteraction(
219		"_get_e2e_device_keys_txn",
220		self._get_e2e_device_keys_txn,
221		query_map.keys(),
222		include_all_devices=True,
223		include_deleted_devices=True,
224		)
225
226		results = []
227		for user_id, user_devices in iteritems(devices):
228		# The prev_id for the first row is always the last row before
229		# `from_stream_id`
230		prev_id = yield self._get_last_device_update_for_remote_user(
231		destination, user_id, from_stream_id
232		)
233		for device_id, device in iteritems(user_devices):
234		stream_id, opentracing_context = query_map[(user_id, device_id)]
235		result = {
236		"user_id": user_id,
237		"device_id": device_id,
238		"prev_id": [prev_id] if prev_id else [],
239		"stream_id": stream_id,
240		"org.matrix.opentracing_context": opentracing_context,
241		}
242
243		prev_id = stream_id
244
245		if device is not None:
246		key_json = device.get("key_json", None)
247		if key_json:
248		result["keys"] = db_to_json(key_json)
249		device_display_name = device.get("device_display_name", None)
250		if device_display_name:
251		result["device_display_name"] = device_display_name
252		else:
253		result["deleted"] = True
254
255		results.append(result)
256
257		return results
258
259		def _get_last_device_update_for_remote_user(
260		self, destination, user_id, from_stream_id
261		):
262		def f(txn):
263		prev_sent_id_sql = """
264		SELECT coalesce(max(stream_id), 0) as stream_id
265		FROM device_lists_outbound_last_success
266		WHERE destination = ? AND user_id = ? AND stream_id <= ?
267		"""
268		txn.execute(prev_sent_id_sql, (destination, user_id, from_stream_id))
269		rows = txn.fetchall()
270		return rows[0][0]
271
272		return self.runInteraction("get_last_device_update_for_remote_user", f)
273
274		def mark_as_sent_devices_by_remote(self, destination, stream_id):
275		"""Mark that updates have successfully been sent to the destination.
276		"""
277		return self.runInteraction(
278		"mark_as_sent_devices_by_remote",
279		self._mark_as_sent_devices_by_remote_txn,
280		destination,
281		stream_id,
282		)
283
284		def _mark_as_sent_devices_by_remote_txn(self, txn, destination, stream_id):
285		# We update the device_lists_outbound_last_success with the successfully
286		# poked users. We do the join to see which users need to be inserted and
287		# which updated.
288		sql = """
289		SELECT user_id, coalesce(max(o.stream_id), 0), (max(s.stream_id) IS NOT NULL)
290		FROM device_lists_outbound_pokes as o
291		LEFT JOIN device_lists_outbound_last_success as s
292		USING (destination, user_id)
293		WHERE destination = ? AND o.stream_id <= ?
294		GROUP BY user_id
295		"""
296		txn.execute(sql, (destination, stream_id))
297		rows = txn.fetchall()
298
299		sql = """
300		UPDATE device_lists_outbound_last_success
301		SET stream_id = ?
302		WHERE destination = ? AND user_id = ?
303		"""
304		txn.executemany(sql, ((row[1], destination, row[0]) for row in rows if row[2]))
305
306		sql = """
307		INSERT INTO device_lists_outbound_last_success
308		(destination, user_id, stream_id) VALUES (?, ?, ?)
309		"""
310		txn.executemany(
311		sql, ((destination, row[0], row[1]) for row in rows if not row[2])
312		)
313
314		# Delete all sent outbound pokes
315		sql = """
316		DELETE FROM device_lists_outbound_pokes
317		WHERE destination = ? AND stream_id <= ?
318		"""
319		txn.execute(sql, (destination, stream_id))
320
321		def get_device_stream_token(self):
322		return self._device_list_id_gen.get_current_token()
323
324		@trace
325		@defer.inlineCallbacks
326		def get_user_devices_from_cache(self, query_list):
327		"""Get the devices (and keys if any) for remote users from the cache.
328
329		Args:
330		query_list(list): List of (user_id, device_ids), if device_ids is
331		falsey then return all device ids for that user.
332
333		Returns:
334		(user_ids_not_in_cache, results_map), where user_ids_not_in_cache is
335		a set of user_ids and results_map is a mapping of
336		user_id -> device_id -> device_info
337		"""
338		user_ids = set(user_id for user_id, _ in query_list)
339		user_map = yield self.get_device_list_last_stream_id_for_remotes(list(user_ids))
340		user_ids_in_cache = set(
341		user_id for user_id, stream_id in user_map.items() if stream_id
342		)
343		user_ids_not_in_cache = user_ids - user_ids_in_cache
344
345		results = {}
346		for user_id, device_id in query_list:
347		if user_id not in user_ids_in_cache:
348		continue
349
350		if device_id:
351		device = yield self._get_cached_user_device(user_id, device_id)
352		results.setdefault(user_id, {})[device_id] = device
353		else:
354		results[user_id] = yield self._get_cached_devices_for_user(user_id)
355
356		set_tag("in_cache", results)
357		set_tag("not_in_cache", user_ids_not_in_cache)
358
359		return user_ids_not_in_cache, results
360
361		@cachedInlineCallbacks(num_args=2, tree=True)
362		def _get_cached_user_device(self, user_id, device_id):
363		content = yield self._simple_select_one_onecol(
364		table="device_lists_remote_cache",
365		keyvalues={"user_id": user_id, "device_id": device_id},
366		retcol="content",
367		desc="_get_cached_user_device",
368		)
369		return db_to_json(content)
370
371		@cachedInlineCallbacks()
372		def _get_cached_devices_for_user(self, user_id):
373		devices = yield self._simple_select_list(
374		table="device_lists_remote_cache",
375		keyvalues={"user_id": user_id},
376		retcols=("device_id", "content"),
377		desc="_get_cached_devices_for_user",
378		)
379		return {
380		device["device_id"]: db_to_json(device["content"]) for device in devices
381		}
382
383		def get_devices_with_keys_by_user(self, user_id):
384		"""Get all devices (with any device keys) for a user
385
386		Returns:
387		(stream_id, devices)
388		"""
389		return self.runInteraction(
390		"get_devices_with_keys_by_user",
391		self._get_devices_with_keys_by_user_txn,
392		user_id,
393		)
394
395		def _get_devices_with_keys_by_user_txn(self, txn, user_id):
396		now_stream_id = self._device_list_id_gen.get_current_token()
397
398		devices = self._get_e2e_device_keys_txn(
399		txn, [(user_id, None)], include_all_devices=True
400		)
401
402		if devices:
403		user_devices = devices[user_id]
404		results = []
405		for device_id, device in iteritems(user_devices):
406		result = {"device_id": device_id}
407
408		key_json = device.get("key_json", None)
409		if key_json:
410		result["keys"] = db_to_json(key_json)
411		device_display_name = device.get("device_display_name", None)
412		if device_display_name:
413		result["device_display_name"] = device_display_name
414
415		results.append(result)
416
417		return now_stream_id, results
418
419		return now_stream_id, []
420
421		def get_users_whose_devices_changed(self, from_key, user_ids):
422		"""Get set of users whose devices have changed since `from_key` that
423		are in the given list of user_ids.
424
425		Args:
426		from_key (str): The device lists stream token
427		user_ids (Iterable[str])
428
429		Returns:
430		Deferred[set[str]]: The set of user_ids whose devices have changed
431		since `from_key`
432		"""
433		from_key = int(from_key)
434
435		# Get set of users who may have changed. Users not in the returned
436		# list have definitely not changed.
437		to_check = list(
438		self._device_list_stream_cache.get_entities_changed(user_ids, from_key)
439		)
440
441		if not to_check:
442		return defer.succeed(set())
443
444		def _get_users_whose_devices_changed_txn(txn):
445		changes = set()
446
447		sql = """
448		SELECT DISTINCT user_id FROM device_lists_stream
449		WHERE stream_id > ?
450		AND user_id IN (%s)
451		"""
452
453		for chunk in batch_iter(to_check, 100):
454		txn.execute(sql % (",".join("?" for _ in chunk),), (from_key,) + chunk)
455		changes.update(user_id for user_id, in txn)
456
457		return changes
458
459		return self.runInteraction(
460		"get_users_whose_devices_changed", _get_users_whose_devices_changed_txn
461		)
462
463		def get_all_device_list_changes_for_remotes(self, from_key, to_key):
464		"""Return a list of `(stream_id, user_id, destination)` which is the
465		combined list of changes to devices, and which destinations need to be
466		poked. `destination` may be None if no destinations need to be poked.
467		"""
468		# We do a group by here as there can be a large number of duplicate
469		# entries, since we throw away device IDs.
470		sql = """
471		SELECT MAX(stream_id) AS stream_id, user_id, destination
472		FROM device_lists_stream
473		LEFT JOIN device_lists_outbound_pokes USING (stream_id, user_id, device_id)
474		WHERE ? < stream_id AND stream_id <= ?
475		GROUP BY user_id, destination
476		"""
477		return self._execute(
478		"get_all_device_list_changes_for_remotes", None, sql, from_key, to_key
479		)
480
481		@cached(max_entries=10000)
482		def get_device_list_last_stream_id_for_remote(self, user_id):
483		"""Get the last stream_id we got for a user. May be None if we haven't
484		got any information for them.
485		"""
486		return self._simple_select_one_onecol(
487		table="device_lists_remote_extremeties",
488		keyvalues={"user_id": user_id},
489		retcol="stream_id",
490		desc="get_device_list_last_stream_id_for_remote",
491		allow_none=True,
492		)
493
494		@cachedList(
495		cached_method_name="get_device_list_last_stream_id_for_remote",
496		list_name="user_ids",
497		inlineCallbacks=True,
498		)
499		def get_device_list_last_stream_id_for_remotes(self, user_ids):
500		rows = yield self._simple_select_many_batch(
501		table="device_lists_remote_extremeties",
502		column="user_id",
503		iterable=user_ids,
504		retcols=("user_id", "stream_id"),
505		desc="get_device_list_last_stream_id_for_remotes",
506		)
507
508		results = {user_id: None for user_id in user_ids}
509		results.update({row["user_id"]: row["stream_id"] for row in rows})
510
511		return results
512
513
514		class DeviceStore(DeviceWorkerStore, BackgroundUpdateStore):
515		def __init__(self, db_conn, hs):
516		super(DeviceStore, self).__init__(db_conn, hs)
517
518		# Map of (user_id, device_id) -> bool. If there is an entry that implies
519		# the device exists.
520		self.device_id_exists_cache = Cache(
521		name="device_id_exists", keylen=2, max_entries=10000
522		)
523
524		self._clock.looping_call(self._prune_old_outbound_device_pokes, 60 * 60 * 1000)
525
526		self.register_background_index_update(
527		"device_lists_stream_idx",
528		index_name="device_lists_stream_user_id",
529		table="device_lists_stream",
530		columns=["user_id", "device_id"],
531		)
532
533		# create a unique index on device_lists_remote_cache
534		self.register_background_index_update(
535		"device_lists_remote_cache_unique_idx",
536		index_name="device_lists_remote_cache_unique_id",
537		table="device_lists_remote_cache",
538		columns=["user_id", "device_id"],
539		unique=True,
540		)
541
542		# And one on device_lists_remote_extremeties
543		self.register_background_index_update(
544		"device_lists_remote_extremeties_unique_idx",
545		index_name="device_lists_remote_extremeties_unique_idx",
546		table="device_lists_remote_extremeties",
547		columns=["user_id"],
548		unique=True,
549		)
550
551		# once they complete, we can remove the old non-unique indexes.
552		self.register_background_update_handler(
553		DROP_DEVICE_LIST_STREAMS_NON_UNIQUE_INDEXES,
554		self._drop_device_list_streams_non_unique_indexes,
555		)
556
557		@defer.inlineCallbacks
558		def store_device(self, user_id, device_id, initial_device_display_name):
559		"""Ensure the given device is known; add it to the store if not
560
561		Args:
562		user_id (str): id of user associated with the device
563		device_id (str): id of device
564		initial_device_display_name (str): initial displayname of the
565		device. Ignored if device exists.
566		Returns:
567		defer.Deferred: boolean whether the device was inserted or an
568		existing device existed with that ID.
569		"""
570		key = (user_id, device_id)
571		if self.device_id_exists_cache.get(key, None):
572		return False
573
574		try:
575		inserted = yield self._simple_insert(
576		"devices",
577		values={
578		"user_id": user_id,
579		"device_id": device_id,
580		"display_name": initial_device_display_name,
581		},
582		desc="store_device",
583		or_ignore=True,
584		)
585		self.device_id_exists_cache.prefill(key, True)
586		return inserted
587		except Exception as e:
588		logger.error(
589		"store_device with device_id=%s(%r) user_id=%s(%r)"
590		" display_name=%s(%r) failed: %s",
591		type(device_id).__name__,
592		device_id,
593		type(user_id).__name__,
594		user_id,
595		type(initial_device_display_name).__name__,
596		initial_device_display_name,
597		e,
598		)
599		raise StoreError(500, "Problem storing device.")
600
601		@defer.inlineCallbacks
602		def delete_device(self, user_id, device_id):
603		"""Delete a device.
604
605		Args:
606		user_id (str): The ID of the user which owns the device
607		device_id (str): The ID of the device to delete
608		Returns:
609		defer.Deferred
610		"""
611		yield self._simple_delete_one(
612		table="devices",
613		keyvalues={"user_id": user_id, "device_id": device_id},
614		desc="delete_device",
615		)
616
617		self.device_id_exists_cache.invalidate((user_id, device_id))
618
619		@defer.inlineCallbacks
620		def delete_devices(self, user_id, device_ids):
621		"""Deletes several devices.
622
623		Args:
624		user_id (str): The ID of the user which owns the devices
625		device_ids (list): The IDs of the devices to delete
626		Returns:
627		defer.Deferred
628		"""
629		yield self._simple_delete_many(
630		table="devices",
631		column="device_id",
632		iterable=device_ids,
633		keyvalues={"user_id": user_id},
634		desc="delete_devices",
635		)
636		for device_id in device_ids:
637		self.device_id_exists_cache.invalidate((user_id, device_id))
638
639		def update_device(self, user_id, device_id, new_display_name=None):
640		"""Update a device.
641
642		Args:
643		user_id (str): The ID of the user which owns the device
644		device_id (str): The ID of the device to update
645		new_display_name (str\|None): new displayname for device; None
646		to leave unchanged
647		Raises:
648		StoreError: if the device is not found
649		Returns:
650		defer.Deferred
651		"""
652		updates = {}
653		if new_display_name is not None:
654		updates["display_name"] = new_display_name
655		if not updates:
656		return defer.succeed(None)
657		return self._simple_update_one(
658		table="devices",
659		keyvalues={"user_id": user_id, "device_id": device_id},
660		updatevalues=updates,
661		desc="update_device",
662		)
663
664		@defer.inlineCallbacks
665		def mark_remote_user_device_list_as_unsubscribed(self, user_id):
666		"""Mark that we no longer track device lists for remote user.
667		"""
668		yield self._simple_delete(
669		table="device_lists_remote_extremeties",
670		keyvalues={"user_id": user_id},
671		desc="mark_remote_user_device_list_as_unsubscribed",
672		)
673		self.get_device_list_last_stream_id_for_remote.invalidate((user_id,))
674
675		def update_remote_device_list_cache_entry(
676		self, user_id, device_id, content, stream_id
677		):
678		"""Updates a single device in the cache of a remote user's devicelist.
679
680		Note: assumes that we are the only thread that can be updating this user's
681		device list.
682
683		Args:
684		user_id (str): User to update device list for
685		device_id (str): ID of decivice being updated
686		content (dict): new data on this device
687		stream_id (int): the version of the device list
688
689		Returns:
690		Deferred[None]
691		"""
692		return self.runInteraction(
693		"update_remote_device_list_cache_entry",
694		self._update_remote_device_list_cache_entry_txn,
695		user_id,
696		device_id,
697		content,
698		stream_id,
699		)
700
701		def _update_remote_device_list_cache_entry_txn(
702		self, txn, user_id, device_id, content, stream_id
703		):
704		if content.get("deleted"):
705		self._simple_delete_txn(
706		txn,
707		table="device_lists_remote_cache",
708		keyvalues={"user_id": user_id, "device_id": device_id},
709		)
710
711		txn.call_after(self.device_id_exists_cache.invalidate, (user_id, device_id))
712		else:
713		self._simple_upsert_txn(
714		txn,
715		table="device_lists_remote_cache",
716		keyvalues={"user_id": user_id, "device_id": device_id},
717		values={"content": json.dumps(content)},
718		# we don't need to lock, because we assume we are the only thread
719		# updating this user's devices.
720		lock=False,
721		)
722
723		txn.call_after(self._get_cached_user_device.invalidate, (user_id, device_id))
724		txn.call_after(self._get_cached_devices_for_user.invalidate, (user_id,))
725		txn.call_after(
726		self.get_device_list_last_stream_id_for_remote.invalidate, (user_id,)
727		)
728
729		self._simple_upsert_txn(
730		txn,
731		table="device_lists_remote_extremeties",
732		keyvalues={"user_id": user_id},
733		values={"stream_id": stream_id},
734		# again, we can assume we are the only thread updating this user's
735		# extremity.
736		lock=False,
737		)
738
739		def update_remote_device_list_cache(self, user_id, devices, stream_id):
740		"""Replace the entire cache of the remote user's devices.
741
742		Note: assumes that we are the only thread that can be updating this user's
743		device list.
744
745		Args:
746		user_id (str): User to update device list for
747		devices (list[dict]): list of device objects supplied over federation
748		stream_id (int): the version of the device list
749
750		Returns:
751		Deferred[None]
752		"""
753		return self.runInteraction(
754		"update_remote_device_list_cache",
755		self._update_remote_device_list_cache_txn,
756		user_id,
757		devices,
758		stream_id,
759		)
760
761		def _update_remote_device_list_cache_txn(self, txn, user_id, devices, stream_id):
762		self._simple_delete_txn(
763		txn, table="device_lists_remote_cache", keyvalues={"user_id": user_id}
764		)
765
766		self._simple_insert_many_txn(
767		txn,
768		table="device_lists_remote_cache",
769		values=[
770		{
771		"user_id": user_id,
772		"device_id": content["device_id"],
773		"content": json.dumps(content),
774		}
775		for content in devices
776		],
777		)
778
779		txn.call_after(self._get_cached_devices_for_user.invalidate, (user_id,))
780		txn.call_after(self._get_cached_user_device.invalidate_many, (user_id,))
781		txn.call_after(
782		self.get_device_list_last_stream_id_for_remote.invalidate, (user_id,)
783		)
784
785		self._simple_upsert_txn(
786		txn,
787		table="device_lists_remote_extremeties",
788		keyvalues={"user_id": user_id},
789		values={"stream_id": stream_id},
790		# we don't need to lock, because we can assume we are the only thread
791		# updating this user's extremity.
792		lock=False,
793		)
794
795		@defer.inlineCallbacks
796		def add_device_change_to_streams(self, user_id, device_ids, hosts):
797		"""Persist that a user's devices have been updated, and which hosts
798		(if any) should be poked.
799		"""
800		with self._device_list_id_gen.get_next() as stream_id:
801		yield self.runInteraction(
802		"add_device_change_to_streams",
803		self._add_device_change_txn,
804		user_id,
805		device_ids,
806		hosts,
807		stream_id,
808		)
809		return stream_id
810
811		def _add_device_change_txn(self, txn, user_id, device_ids, hosts, stream_id):
812		now = self._clock.time_msec()
813
814		txn.call_after(
815		self._device_list_stream_cache.entity_has_changed, user_id, stream_id
816		)
817		for host in hosts:
818		txn.call_after(
819		self._device_list_federation_stream_cache.entity_has_changed,
820		host,
821		stream_id,
822		)
823
824		# Delete older entries in the table, as we really only care about
825		# when the latest change happened.
826		txn.executemany(
827		"""
828		DELETE FROM device_lists_stream
829		WHERE user_id = ? AND device_id = ? AND stream_id < ?
830		""",
831		[(user_id, device_id, stream_id) for device_id in device_ids],
832		)
833
834		self._simple_insert_many_txn(
835		txn,
836		table="device_lists_stream",
837		values=[
838		{"stream_id": stream_id, "user_id": user_id, "device_id": device_id}
839		for device_id in device_ids
840		],
841		)
842
843		context = get_active_span_text_map()
844
845		self._simple_insert_many_txn(
846		txn,
847		table="device_lists_outbound_pokes",
848		values=[
849		{
850		"destination": destination,
851		"stream_id": stream_id,
852		"user_id": user_id,
853		"device_id": device_id,
854		"sent": False,
855		"ts": now,
856		"opentracing_context": json.dumps(context)
857		if whitelisted_homeserver(destination)
858		else "{}",
859		}
860		for destination in hosts
861		for device_id in device_ids
862		],
863		)
864
865		def _prune_old_outbound_device_pokes(self):
866		"""Delete old entries out of the device_lists_outbound_pokes to ensure
867		that we don't fill up due to dead servers. We keep one entry per
868		(destination, user_id) tuple to ensure that the prev_ids remain correct
869		if the server does come back.
870		"""
871		yesterday = self._clock.time_msec() - 24 * 60 * 60 * 1000
872
873		def _prune_txn(txn):
874		select_sql = """
875		SELECT destination, user_id, max(stream_id) as stream_id
876		FROM device_lists_outbound_pokes
877		GROUP BY destination, user_id
878		HAVING min(ts) < ? AND count(*) > 1
879		"""
880
881		txn.execute(select_sql, (yesterday,))
882		rows = txn.fetchall()
883
884		if not rows:
885		return
886
887		delete_sql = """
888		DELETE FROM device_lists_outbound_pokes
889		WHERE ts < ? AND destination = ? AND user_id = ? AND stream_id < ?
890		"""
891
892		txn.executemany(
893		delete_sql, ((yesterday, row[0], row[1], row[2]) for row in rows)
894		)
895
896		# Since we've deleted unsent deltas, we need to remove the entry
897		# of last successful sent so that the prev_ids are correctly set.
898		sql = """
899		DELETE FROM device_lists_outbound_last_success
900		WHERE destination = ? AND user_id = ?
901		"""
902		txn.executemany(sql, ((row[0], row[1]) for row in rows))
903
904		logger.info("Pruned %d device list outbound pokes", txn.rowcount)
905
906		return run_as_background_process(
907		"prune_old_outbound_device_pokes",
908		self.runInteraction,
909		"_prune_old_outbound_device_pokes",
910		_prune_txn,
911		)
912
913		@defer.inlineCallbacks
914		def _drop_device_list_streams_non_unique_indexes(self, progress, batch_size):
915		def f(conn):
916		txn = conn.cursor()
917		txn.execute("DROP INDEX IF EXISTS device_lists_remote_cache_id")
918		txn.execute("DROP INDEX IF EXISTS device_lists_remote_extremeties_id")
919		txn.close()
920
921		yield self.runWithConnection(f)
922		yield self._end_background_update(DROP_DEVICE_LIST_STREAMS_NON_UNIQUE_INDEXES)
923		return 1

+0

-174

~~synapse/storage/directory.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2014-2016 OpenMarket Ltd
2		#
3		# Licensed under the Apache License, Version 2.0 (the "License");
4		# you may not use this file except in compliance with the License.
5		# You may obtain a copy of the License at
6		#
7		# http://www.apache.org/licenses/LICENSE-2.0
8		#
9		# Unless required by applicable law or agreed to in writing, software
10		# distributed under the License is distributed on an "AS IS" BASIS,
11		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12		# See the License for the specific language governing permissions and
13		# limitations under the License.
14
15		from collections import namedtuple
16
17		from twisted.internet import defer
18
19		from synapse.api.errors import SynapseError
20		from synapse.util.caches.descriptors import cached
21
22		from ._base import SQLBaseStore
23
24		RoomAliasMapping = namedtuple("RoomAliasMapping", ("room_id", "room_alias", "servers"))
25
26
27		class DirectoryWorkerStore(SQLBaseStore):
28		@defer.inlineCallbacks
29		def get_association_from_room_alias(self, room_alias):
30		""" Get's the room_id and server list for a given room_alias
31
32		Args:
33		room_alias (RoomAlias)
34
35		Returns:
36		Deferred: results in namedtuple with keys "room_id" and
37		"servers" or None if no association can be found
38		"""
39		room_id = yield self._simple_select_one_onecol(
40		"room_aliases",
41		{"room_alias": room_alias.to_string()},
42		"room_id",
43		allow_none=True,
44		desc="get_association_from_room_alias",
45		)
46
47		if not room_id:
48		return None
49
50		servers = yield self._simple_select_onecol(
51		"room_alias_servers",
52		{"room_alias": room_alias.to_string()},
53		"server",
54		desc="get_association_from_room_alias",
55		)
56
57		if not servers:
58		return None
59
60		return RoomAliasMapping(room_id, room_alias.to_string(), servers)
61
62		def get_room_alias_creator(self, room_alias):
63		return self._simple_select_one_onecol(
64		table="room_aliases",
65		keyvalues={"room_alias": room_alias},
66		retcol="creator",
67		desc="get_room_alias_creator",
68		)
69
70		@cached(max_entries=5000)
71		def get_aliases_for_room(self, room_id):
72		return self._simple_select_onecol(
73		"room_aliases",
74		{"room_id": room_id},
75		"room_alias",
76		desc="get_aliases_for_room",
77		)
78
79
80		class DirectoryStore(DirectoryWorkerStore):
81		@defer.inlineCallbacks
82		def create_room_alias_association(self, room_alias, room_id, servers, creator=None):
83		""" Creates an association between a room alias and room_id/servers
84
85		Args:
86		room_alias (RoomAlias)
87		room_id (str)
88		servers (list)
89		creator (str): Optional user_id of creator.
90
91		Returns:
92		Deferred
93		"""
94
95		def alias_txn(txn):
96		self._simple_insert_txn(
97		txn,
98		"room_aliases",
99		{
100		"room_alias": room_alias.to_string(),
101		"room_id": room_id,
102		"creator": creator,
103		},
104		)
105
106		self._simple_insert_many_txn(
107		txn,
108		table="room_alias_servers",
109		values=[
110		{"room_alias": room_alias.to_string(), "server": server}
111		for server in servers
112		],
113		)
114
115		self._invalidate_cache_and_stream(
116		txn, self.get_aliases_for_room, (room_id,)
117		)
118
119		try:
120		ret = yield self.runInteraction("create_room_alias_association", alias_txn)
121		except self.database_engine.module.IntegrityError:
122		raise SynapseError(
123		409, "Room alias %s already exists" % room_alias.to_string()
124		)
125		return ret
126
127		@defer.inlineCallbacks
128		def delete_room_alias(self, room_alias):
129		room_id = yield self.runInteraction(
130		"delete_room_alias", self._delete_room_alias_txn, room_alias
131		)
132
133		return room_id
134
135		def _delete_room_alias_txn(self, txn, room_alias):
136		txn.execute(
137		"SELECT room_id FROM room_aliases WHERE room_alias = ?",
138		(room_alias.to_string(),),
139		)
140
141		res = txn.fetchone()
142		if res:
143		room_id = res[0]
144		else:
145		return None
146
147		txn.execute(
148		"DELETE FROM room_aliases WHERE room_alias = ?", (room_alias.to_string(),)
149		)
150
151		txn.execute(
152		"DELETE FROM room_alias_servers WHERE room_alias = ?",
153		(room_alias.to_string(),),
154		)
155
156		self._invalidate_cache_and_stream(txn, self.get_aliases_for_room, (room_id,))
157
158		return room_id
159
160		def update_aliases_for_room(self, old_room_id, new_room_id, creator):
161		def _update_aliases_for_room_txn(txn):
162		sql = "UPDATE room_aliases SET room_id = ?, creator = ? WHERE room_id = ?"
163		txn.execute(sql, (new_room_id, creator, old_room_id))
164		self._invalidate_cache_and_stream(
165		txn, self.get_aliases_for_room, (old_room_id,)
166		)
167		self._invalidate_cache_and_stream(
168		txn, self.get_aliases_for_room, (new_room_id,)
169		)
170
171		return self.runInteraction(
172		"_update_aliases_for_room_txn", _update_aliases_for_room_txn
173		)

+0

-337

~~synapse/storage/e2e_room_keys.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2017 New Vector Ltd
2		#
3		# Licensed under the Apache License, Version 2.0 (the "License");
4		# you may not use this file except in compliance with the License.
5		# You may obtain a copy of the License at
6		#
7		# http://www.apache.org/licenses/LICENSE-2.0
8		#
9		# Unless required by applicable law or agreed to in writing, software
10		# distributed under the License is distributed on an "AS IS" BASIS,
11		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12		# See the License for the specific language governing permissions and
13		# limitations under the License.
14
15		import json
16
17		from twisted.internet import defer
18
19		from synapse.api.errors import StoreError
20		from synapse.logging.opentracing import log_kv, trace
21
22		from ._base import SQLBaseStore
23
24
25		class EndToEndRoomKeyStore(SQLBaseStore):
26		@defer.inlineCallbacks
27		def get_e2e_room_key(self, user_id, version, room_id, session_id):
28		"""Get the encrypted E2E room key for a given session from a given
29		backup version of room_keys. We only store the 'best' room key for a given
30		session at a given time, as determined by the handler.
31
32		Args:
33		user_id(str): the user whose backup we're querying
34		version(str): the version ID of the backup for the set of keys we're querying
35		room_id(str): the ID of the room whose keys we're querying.
36		This is a bit redundant as it's implied by the session_id, but
37		we include for consistency with the rest of the API.
38		session_id(str): the session whose room_key we're querying.
39
40		Returns:
41		A deferred dict giving the session_data and message metadata for
42		this room key.
43		"""
44
45		row = yield self._simple_select_one(
46		table="e2e_room_keys",
47		keyvalues={
48		"user_id": user_id,
49		"version": version,
50		"room_id": room_id,
51		"session_id": session_id,
52		},
53		retcols=(
54		"first_message_index",
55		"forwarded_count",
56		"is_verified",
57		"session_data",
58		),
59		desc="get_e2e_room_key",
60		)
61
62		row["session_data"] = json.loads(row["session_data"])
63
64		return row
65
66		@defer.inlineCallbacks
67		def set_e2e_room_key(self, user_id, version, room_id, session_id, room_key):
68		"""Replaces or inserts the encrypted E2E room key for a given session in
69		a given backup
70
71		Args:
72		user_id(str): the user whose backup we're setting
73		version(str): the version ID of the backup we're updating
74		room_id(str): the ID of the room whose keys we're setting
75		session_id(str): the session whose room_key we're setting
76		room_key(dict): the room_key being set
77		Raises:
78		StoreError
79		"""
80
81		yield self._simple_upsert(
82		table="e2e_room_keys",
83		keyvalues={
84		"user_id": user_id,
85		"version": version,
86		"room_id": room_id,
87		"session_id": session_id,
88		},
89		values={
90		"first_message_index": room_key["first_message_index"],
91		"forwarded_count": room_key["forwarded_count"],
92		"is_verified": room_key["is_verified"],
93		"session_data": json.dumps(room_key["session_data"]),
94		},
95		lock=False,
96		)
97		log_kv(
98		{
99		"message": "Set room key",
100		"room_id": room_id,
101		"session_id": session_id,
102		"room_key": room_key,
103		}
104		)
105
106		@trace
107		@defer.inlineCallbacks
108		def get_e2e_room_keys(self, user_id, version, room_id=None, session_id=None):
109		"""Bulk get the E2E room keys for a given backup, optionally filtered to a given
110		room, or a given session.
111
112		Args:
113		user_id(str): the user whose backup we're querying
114		version(str): the version ID of the backup for the set of keys we're querying
115		room_id(str): Optional. the ID of the room whose keys we're querying, if any.
116		If not specified, we return the keys for all the rooms in the backup.
117		session_id(str): Optional. the session whose room_key we're querying, if any.
118		If specified, we also require the room_id to be specified.
119		If not specified, we return all the keys in this version of
120		the backup (or for the specified room)
121
122		Returns:
123		A deferred list of dicts giving the session_data and message metadata for
124		these room keys.
125		"""
126
127		try:
128		version = int(version)
129		except ValueError:
130		return {"rooms": {}}
131
132		keyvalues = {"user_id": user_id, "version": version}
133		if room_id:
134		keyvalues["room_id"] = room_id
135		if session_id:
136		keyvalues["session_id"] = session_id
137
138		rows = yield self._simple_select_list(
139		table="e2e_room_keys",
140		keyvalues=keyvalues,
141		retcols=(
142		"user_id",
143		"room_id",
144		"session_id",
145		"first_message_index",
146		"forwarded_count",
147		"is_verified",
148		"session_data",
149		),
150		desc="get_e2e_room_keys",
151		)
152
153		sessions = {"rooms": {}}
154		for row in rows:
155		room_entry = sessions["rooms"].setdefault(row["room_id"], {"sessions": {}})
156		room_entry["sessions"][row["session_id"]] = {
157		"first_message_index": row["first_message_index"],
158		"forwarded_count": row["forwarded_count"],
159		"is_verified": row["is_verified"],
160		"session_data": json.loads(row["session_data"]),
161		}
162
163		return sessions
164
165		@trace
166		@defer.inlineCallbacks
167		def delete_e2e_room_keys(self, user_id, version, room_id=None, session_id=None):
168		"""Bulk delete the E2E room keys for a given backup, optionally filtered to a given
169		room or a given session.
170
171		Args:
172		user_id(str): the user whose backup we're deleting from
173		version(str): the version ID of the backup for the set of keys we're deleting
174		room_id(str): Optional. the ID of the room whose keys we're deleting, if any.
175		If not specified, we delete the keys for all the rooms in the backup.
176		session_id(str): Optional. the session whose room_key we're querying, if any.
177		If specified, we also require the room_id to be specified.
178		If not specified, we delete all the keys in this version of
179		the backup (or for the specified room)
180
181		Returns:
182		A deferred of the deletion transaction
183		"""
184
185		keyvalues = {"user_id": user_id, "version": int(version)}
186		if room_id:
187		keyvalues["room_id"] = room_id
188		if session_id:
189		keyvalues["session_id"] = session_id
190
191		yield self._simple_delete(
192		table="e2e_room_keys", keyvalues=keyvalues, desc="delete_e2e_room_keys"
193		)
194
195		@staticmethod
196		def _get_current_version(txn, user_id):
197		txn.execute(
198		"SELECT MAX(version) FROM e2e_room_keys_versions "
199		"WHERE user_id=? AND deleted=0",
200		(user_id,),
201		)
202		row = txn.fetchone()
203		if not row:
204		raise StoreError(404, "No current backup version")
205		return row[0]
206
207		def get_e2e_room_keys_version_info(self, user_id, version=None):
208		"""Get info metadata about a version of our room_keys backup.
209
210		Args:
211		user_id(str): the user whose backup we're querying
212		version(str): Optional. the version ID of the backup we're querying about
213		If missing, we return the information about the current version.
214		Raises:
215		StoreError: with code 404 if there are no e2e_room_keys_versions present
216		Returns:
217		A deferred dict giving the info metadata for this backup version, with
218		fields including:
219		version(str)
220		algorithm(str)
221		auth_data(object): opaque dict supplied by the client
222		"""
223
224		def _get_e2e_room_keys_version_info_txn(txn):
225		if version is None:
226		this_version = self._get_current_version(txn, user_id)
227		else:
228		try:
229		this_version = int(version)
230		except ValueError:
231		# Our versions are all ints so if we can't convert it to an integer,
232		# it isn't there.
233		raise StoreError(404, "No row found")
234
235		result = self._simple_select_one_txn(
236		txn,
237		table="e2e_room_keys_versions",
238		keyvalues={"user_id": user_id, "version": this_version, "deleted": 0},
239		retcols=("version", "algorithm", "auth_data"),
240		)
241		result["auth_data"] = json.loads(result["auth_data"])
242		result["version"] = str(result["version"])
243		return result
244
245		return self.runInteraction(
246		"get_e2e_room_keys_version_info", _get_e2e_room_keys_version_info_txn
247		)
248
249		@trace
250		def create_e2e_room_keys_version(self, user_id, info):
251		"""Atomically creates a new version of this user's e2e_room_keys store
252		with the given version info.
253
254		Args:
255		user_id(str): the user whose backup we're creating a version
256		info(dict): the info about the backup version to be created
257
258		Returns:
259		A deferred string for the newly created version ID
260		"""
261
262		def _create_e2e_room_keys_version_txn(txn):
263		txn.execute(
264		"SELECT MAX(version) FROM e2e_room_keys_versions WHERE user_id=?",
265		(user_id,),
266		)
267		current_version = txn.fetchone()[0]
268		if current_version is None:
269		current_version = "0"
270
271		new_version = str(int(current_version) + 1)
272
273		self._simple_insert_txn(
274		txn,
275		table="e2e_room_keys_versions",
276		values={
277		"user_id": user_id,
278		"version": new_version,
279		"algorithm": info["algorithm"],
280		"auth_data": json.dumps(info["auth_data"]),
281		},
282		)
283
284		return new_version
285
286		return self.runInteraction(
287		"create_e2e_room_keys_version_txn", _create_e2e_room_keys_version_txn
288		)
289
290		@trace
291		def update_e2e_room_keys_version(self, user_id, version, info):
292		"""Update a given backup version
293
294		Args:
295		user_id(str): the user whose backup version we're updating
296		version(str): the version ID of the backup version we're updating
297		info(dict): the new backup version info to store
298		"""
299
300		return self._simple_update(
301		table="e2e_room_keys_versions",
302		keyvalues={"user_id": user_id, "version": version},
303		updatevalues={"auth_data": json.dumps(info["auth_data"])},
304		desc="update_e2e_room_keys_version",
305		)
306
307		@trace
308		def delete_e2e_room_keys_version(self, user_id, version=None):
309		"""Delete a given backup version of the user's room keys.
310		Doesn't delete their actual key data.
311
312		Args:
313		user_id(str): the user whose backup version we're deleting
314		version(str): Optional. the version ID of the backup version we're deleting
315		If missing, we delete the current backup version info.
316		Raises:
317		StoreError: with code 404 if there are no e2e_room_keys_versions present,
318		or if the version requested doesn't exist.
319		"""
320
321		def _delete_e2e_room_keys_version_txn(txn):
322		if version is None:
323		this_version = self._get_current_version(txn, user_id)
324		else:
325		this_version = version
326
327		return self._simple_update_one_txn(
328		txn,
329		table="e2e_room_keys_versions",
330		keyvalues={"user_id": user_id, "version": this_version},
331		updatevalues={"deleted": 1},
332		)
333
334		return self.runInteraction(
335		"delete_e2e_room_keys_version", _delete_e2e_room_keys_version_txn
336		)

+0

-313

~~synapse/storage/end_to_end_keys.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2015, 2016 OpenMarket Ltd
2		#
3		# Licensed under the Apache License, Version 2.0 (the "License");
4		# you may not use this file except in compliance with the License.
5		# You may obtain a copy of the License at
6		#
7		# http://www.apache.org/licenses/LICENSE-2.0
8		#
9		# Unless required by applicable law or agreed to in writing, software
10		# distributed under the License is distributed on an "AS IS" BASIS,
11		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12		# See the License for the specific language governing permissions and
13		# limitations under the License.
14		from six import iteritems
15
16		from canonicaljson import encode_canonical_json
17
18		from twisted.internet import defer
19
20		from synapse.logging.opentracing import log_kv, set_tag, trace
21		from synapse.util.caches.descriptors import cached
22
23		from ._base import SQLBaseStore, db_to_json
24
25
26		class EndToEndKeyWorkerStore(SQLBaseStore):
27		@trace
28		@defer.inlineCallbacks
29		def get_e2e_device_keys(
30		self, query_list, include_all_devices=False, include_deleted_devices=False
31		):
32		"""Fetch a list of device keys.
33		Args:
34		query_list(list): List of pairs of user_ids and device_ids.
35		include_all_devices (bool): whether to include entries for devices
36		that don't have device keys
37		include_deleted_devices (bool): whether to include null entries for
38		devices which no longer exist (but were in the query_list).
39		This option only takes effect if include_all_devices is true.
40		Returns:
41		Dict mapping from user-id to dict mapping from device_id to
42		dict containing "key_json", "device_display_name".
43		"""
44		set_tag("query_list", query_list)
45		if not query_list:
46		return {}
47
48		results = yield self.runInteraction(
49		"get_e2e_device_keys",
50		self._get_e2e_device_keys_txn,
51		query_list,
52		include_all_devices,
53		include_deleted_devices,
54		)
55
56		for user_id, device_keys in iteritems(results):
57		for device_id, device_info in iteritems(device_keys):
58		device_info["keys"] = db_to_json(device_info.pop("key_json"))
59
60		return results
61
62		@trace
63		def _get_e2e_device_keys_txn(
64		self, txn, query_list, include_all_devices=False, include_deleted_devices=False
65		):
66		set_tag("include_all_devices", include_all_devices)
67		set_tag("include_deleted_devices", include_deleted_devices)
68
69		query_clauses = []
70		query_params = []
71
72		if include_all_devices is False:
73		include_deleted_devices = False
74
75		if include_deleted_devices:
76		deleted_devices = set(query_list)
77
78		for (user_id, device_id) in query_list:
79		query_clause = "user_id = ?"
80		query_params.append(user_id)
81
82		if device_id is not None:
83		query_clause += " AND device_id = ?"
84		query_params.append(device_id)
85
86		query_clauses.append(query_clause)
87
88		sql = (
89		"SELECT user_id, device_id, "
90		" d.display_name AS device_display_name, "
91		" k.key_json"
92		" FROM devices d"
93		" %s JOIN e2e_device_keys_json k USING (user_id, device_id)"
94		" WHERE %s"
95		) % (
96		"LEFT" if include_all_devices else "INNER",
97		" OR ".join("(" + q + ")" for q in query_clauses),
98		)
99
100		txn.execute(sql, query_params)
101		rows = self.cursor_to_dict(txn)
102
103		result = {}
104		for row in rows:
105		if include_deleted_devices:
106		deleted_devices.remove((row["user_id"], row["device_id"]))
107		result.setdefault(row["user_id"], {})[row["device_id"]] = row
108
109		if include_deleted_devices:
110		for user_id, device_id in deleted_devices:
111		result.setdefault(user_id, {})[device_id] = None
112
113		log_kv(result)
114		return result
115
116		@defer.inlineCallbacks
117		def get_e2e_one_time_keys(self, user_id, device_id, key_ids):
118		"""Retrieve a number of one-time keys for a user
119
120		Args:
121		user_id(str): id of user to get keys for
122		device_id(str): id of device to get keys for
123		key_ids(list[str]): list of key ids (excluding algorithm) to
124		retrieve
125
126		Returns:
127		deferred resolving to Dict[(str, str), str]: map from (algorithm,
128		key_id) to json string for key
129		"""
130
131		rows = yield self._simple_select_many_batch(
132		table="e2e_one_time_keys_json",
133		column="key_id",
134		iterable=key_ids,
135		retcols=("algorithm", "key_id", "key_json"),
136		keyvalues={"user_id": user_id, "device_id": device_id},
137		desc="add_e2e_one_time_keys_check",
138		)
139		result = {(row["algorithm"], row["key_id"]): row["key_json"] for row in rows}
140		log_kv({"message": "Fetched one time keys for user", "one_time_keys": result})
141		return result
142
143		@defer.inlineCallbacks
144		def add_e2e_one_time_keys(self, user_id, device_id, time_now, new_keys):
145		"""Insert some new one time keys for a device. Errors if any of the
146		keys already exist.
147
148		Args:
149		user_id(str): id of user to get keys for
150		device_id(str): id of device to get keys for
151		time_now(long): insertion time to record (ms since epoch)
152		new_keys(iterable[(str, str, str)]: keys to add - each a tuple of
153		(algorithm, key_id, key json)
154		"""
155
156		def _add_e2e_one_time_keys(txn):
157		set_tag("user_id", user_id)
158		set_tag("device_id", device_id)
159		set_tag("new_keys", new_keys)
160		# We are protected from race between lookup and insertion due to
161		# a unique constraint. If there is a race of two calls to
162		# `add_e2e_one_time_keys` then they'll conflict and we will only
163		# insert one set.
164		self._simple_insert_many_txn(
165		txn,
166		table="e2e_one_time_keys_json",
167		values=[
168		{
169		"user_id": user_id,
170		"device_id": device_id,
171		"algorithm": algorithm,
172		"key_id": key_id,
173		"ts_added_ms": time_now,
174		"key_json": json_bytes,
175		}
176		for algorithm, key_id, json_bytes in new_keys
177		],
178		)
179		self._invalidate_cache_and_stream(
180		txn, self.count_e2e_one_time_keys, (user_id, device_id)
181		)
182
183		yield self.runInteraction(
184		"add_e2e_one_time_keys_insert", _add_e2e_one_time_keys
185		)
186
187		@cached(max_entries=10000)
188		def count_e2e_one_time_keys(self, user_id, device_id):
189		""" Count the number of one time keys the server has for a device
190		Returns:
191		Dict mapping from algorithm to number of keys for that algorithm.
192		"""
193
194		def _count_e2e_one_time_keys(txn):
195		sql = (
196		"SELECT algorithm, COUNT(key_id) FROM e2e_one_time_keys_json"
197		" WHERE user_id = ? AND device_id = ?"
198		" GROUP BY algorithm"
199		)
200		txn.execute(sql, (user_id, device_id))
201		result = {}
202		for algorithm, key_count in txn:
203		result[algorithm] = key_count
204		return result
205
206		return self.runInteraction("count_e2e_one_time_keys", _count_e2e_one_time_keys)
207
208
209		class EndToEndKeyStore(EndToEndKeyWorkerStore, SQLBaseStore):
210		def set_e2e_device_keys(self, user_id, device_id, time_now, device_keys):
211		"""Stores device keys for a device. Returns whether there was a change
212		or the keys were already in the database.
213		"""
214
215		def _set_e2e_device_keys_txn(txn):
216		set_tag("user_id", user_id)
217		set_tag("device_id", device_id)
218		set_tag("time_now", time_now)
219		set_tag("device_keys", device_keys)
220
221		old_key_json = self._simple_select_one_onecol_txn(
222		txn,
223		table="e2e_device_keys_json",
224		keyvalues={"user_id": user_id, "device_id": device_id},
225		retcol="key_json",
226		allow_none=True,
227		)
228
229		# In py3 we need old_key_json to match new_key_json type. The DB
230		# returns unicode while encode_canonical_json returns bytes.
231		new_key_json = encode_canonical_json(device_keys).decode("utf-8")
232
233		if old_key_json == new_key_json:
234		log_kv({"Message": "Device key already stored."})
235		return False
236
237		self._simple_upsert_txn(
238		txn,
239		table="e2e_device_keys_json",
240		keyvalues={"user_id": user_id, "device_id": device_id},
241		values={"ts_added_ms": time_now, "key_json": new_key_json},
242		)
243		log_kv({"message": "Device keys stored."})
244		return True
245
246		return self.runInteraction("set_e2e_device_keys", _set_e2e_device_keys_txn)
247
248		def claim_e2e_one_time_keys(self, query_list):
249		"""Take a list of one time keys out of the database"""
250
251		@trace
252		def _claim_e2e_one_time_keys(txn):
253		sql = (
254		"SELECT key_id, key_json FROM e2e_one_time_keys_json"
255		" WHERE user_id = ? AND device_id = ? AND algorithm = ?"
256		" LIMIT 1"
257		)
258		result = {}
259		delete = []
260		for user_id, device_id, algorithm in query_list:
261		user_result = result.setdefault(user_id, {})
262		device_result = user_result.setdefault(device_id, {})
263		txn.execute(sql, (user_id, device_id, algorithm))
264		for key_id, key_json in txn:
265		device_result[algorithm + ":" + key_id] = key_json
266		delete.append((user_id, device_id, algorithm, key_id))
267		sql = (
268		"DELETE FROM e2e_one_time_keys_json"
269		" WHERE user_id = ? AND device_id = ? AND algorithm = ?"
270		" AND key_id = ?"
271		)
272		for user_id, device_id, algorithm, key_id in delete:
273		log_kv(
274		{
275		"message": "Executing claim e2e_one_time_keys transaction on database."
276		}
277		)
278		txn.execute(sql, (user_id, device_id, algorithm, key_id))
279		log_kv({"message": "finished executing and invalidating cache"})
280		self._invalidate_cache_and_stream(
281		txn, self.count_e2e_one_time_keys, (user_id, device_id)
282		)
283		return result
284
285		return self.runInteraction("claim_e2e_one_time_keys", _claim_e2e_one_time_keys)
286
287		def delete_e2e_keys_by_device(self, user_id, device_id):
288		def delete_e2e_keys_by_device_txn(txn):
289		log_kv(
290		{
291		"message": "Deleting keys for device",
292		"device_id": device_id,
293		"user_id": user_id,
294		}
295		)
296		self._simple_delete_txn(
297		txn,
298		table="e2e_device_keys_json",
299		keyvalues={"user_id": user_id, "device_id": device_id},
300		)
301		self._simple_delete_txn(
302		txn,
303		table="e2e_one_time_keys_json",
304		keyvalues={"user_id": user_id, "device_id": device_id},
305		)
306		self._invalidate_cache_and_stream(
307		txn, self.count_e2e_one_time_keys, (user_id, device_id)
308		)
309
310		return self.runInteraction(
311		"delete_e2e_keys_by_device", delete_e2e_keys_by_device_txn
312		)

+13

-0

synapse/storage/engines/postgres.py less more

21	21	def __init__(self, database_module, database_config):
22	22	self.module = database_module
23	23	self.module.extensions.register_type(self.module.extensions.UNICODE)
	24
	25	# Disables passing `bytes` to txn.execute, c.f. #6186. If you do
	26	# actually want to use bytes than wrap it in `bytearray`.
	27	def _disable_bytes_adapter(_):
	28	raise Exception("Passing bytes to DB is disabled.")
	29
	30	self.module.extensions.register_adapter(bytes, _disable_bytes_adapter)
24	31	self.synchronous_commit = database_config.get("synchronous_commit", True)
25	32	self._version = None # unknown as yet
26	33

78	85	"""
79	86	return True
80	87
	88	@property
	89	def supports_using_any_list(self):
	90	"""Do we support using `a = ANY(?)` and passing a list
	91	"""
	92	return True
	93
81	94	def is_deadlock(self, error):
82	95	if isinstance(error, self.module.DatabaseError):
83	96	# https://www.postgresql.org/docs/current/static/errcodes-appendix.html

+6

-0

synapse/storage/engines/sqlite.py less more

44	44	SQLite 3.15+.
45	45	"""
46	46	return self.module.sqlite_version_info >= (3, 15, 0)
	47
	48	@property
	49	def supports_using_any_list(self):
	50	"""Do we support using `a = ANY(?)` and passing a list
	51	"""
	52	return False
47	53
48	54	def check_database(self, txn):
49	55	pass

+0

-669

~~synapse/storage/event_federation.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2014-2016 OpenMarket Ltd
2		#
3		# Licensed under the Apache License, Version 2.0 (the "License");
4		# you may not use this file except in compliance with the License.
5		# You may obtain a copy of the License at
6		#
7		# http://www.apache.org/licenses/LICENSE-2.0
8		#
9		# Unless required by applicable law or agreed to in writing, software
10		# distributed under the License is distributed on an "AS IS" BASIS,
11		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12		# See the License for the specific language governing permissions and
13		# limitations under the License.
14		import itertools
15		import logging
16		import random
17
18		from six.moves import range
19		from six.moves.queue import Empty, PriorityQueue
20
21		from unpaddedbase64 import encode_base64
22
23		from twisted.internet import defer
24
25		from synapse.api.errors import StoreError
26		from synapse.metrics.background_process_metrics import run_as_background_process
27		from synapse.storage._base import SQLBaseStore
28		from synapse.storage.events_worker import EventsWorkerStore
29		from synapse.storage.signatures import SignatureWorkerStore
30		from synapse.util.caches.descriptors import cached
31
32		logger = logging.getLogger(__name__)
33
34
35		class EventFederationWorkerStore(EventsWorkerStore, SignatureWorkerStore, SQLBaseStore):
36		def get_auth_chain(self, event_ids, include_given=False):
37		"""Get auth events for given event_ids. The events must be state events.
38
39		Args:
40		event_ids (list): state events
41		include_given (bool): include the given events in result
42
43		Returns:
44		list of events
45		"""
46		return self.get_auth_chain_ids(
47		event_ids, include_given=include_given
48		).addCallback(self.get_events_as_list)
49
50		def get_auth_chain_ids(self, event_ids, include_given=False):
51		"""Get auth events for given event_ids. The events must be state events.
52
53		Args:
54		event_ids (list): state events
55		include_given (bool): include the given events in result
56
57		Returns:
58		list of event_ids
59		"""
60		return self.runInteraction(
61		"get_auth_chain_ids", self._get_auth_chain_ids_txn, event_ids, include_given
62		)
63
64		def _get_auth_chain_ids_txn(self, txn, event_ids, include_given):
65		if include_given:
66		results = set(event_ids)
67		else:
68		results = set()
69
70		base_sql = "SELECT auth_id FROM event_auth WHERE event_id IN (%s)"
71
72		front = set(event_ids)
73		while front:
74		new_front = set()
75		front_list = list(front)
76		chunks = [front_list[x : x + 100] for x in range(0, len(front), 100)]
77		for chunk in chunks:
78		txn.execute(base_sql % (",".join(["?"] * len(chunk)),), chunk)
79		new_front.update([r[0] for r in txn])
80
81		new_front -= results
82
83		front = new_front
84		results.update(front)
85
86		return list(results)
87
88		def get_oldest_events_in_room(self, room_id):
89		return self.runInteraction(
90		"get_oldest_events_in_room", self._get_oldest_events_in_room_txn, room_id
91		)
92
93		def get_oldest_events_with_depth_in_room(self, room_id):
94		return self.runInteraction(
95		"get_oldest_events_with_depth_in_room",
96		self.get_oldest_events_with_depth_in_room_txn,
97		room_id,
98		)
99
100		def get_oldest_events_with_depth_in_room_txn(self, txn, room_id):
101		sql = (
102		"SELECT b.event_id, MAX(e.depth) FROM events as e"
103		" INNER JOIN event_edges as g"
104		" ON g.event_id = e.event_id"
105		" INNER JOIN event_backward_extremities as b"
106		" ON g.prev_event_id = b.event_id"
107		" WHERE b.room_id = ? AND g.is_state is ?"
108		" GROUP BY b.event_id"
109		)
110
111		txn.execute(sql, (room_id, False))
112
113		return dict(txn)
114
115		@defer.inlineCallbacks
116		def get_max_depth_of(self, event_ids):
117		"""Returns the max depth of a set of event IDs
118
119		Args:
120		event_ids (list[str])
121
122		Returns
123		Deferred[int]
124		"""
125		rows = yield self._simple_select_many_batch(
126		table="events",
127		column="event_id",
128		iterable=event_ids,
129		retcols=("depth",),
130		desc="get_max_depth_of",
131		)
132
133		if not rows:
134		return 0
135		else:
136		return max(row["depth"] for row in rows)
137
138		def _get_oldest_events_in_room_txn(self, txn, room_id):
139		return self._simple_select_onecol_txn(
140		txn,
141		table="event_backward_extremities",
142		keyvalues={"room_id": room_id},
143		retcol="event_id",
144		)
145
146		@defer.inlineCallbacks
147		def get_prev_events_for_room(self, room_id):
148		"""
149		Gets a subset of the current forward extremities in the given room.
150
151		Limits the result to 10 extremities, so that we can avoid creating
152		events which refer to hundreds of prev_events.
153
154		Args:
155		room_id (str): room_id
156
157		Returns:
158		Deferred[list[(str, dict[str, str], int)]]
159		for each event, a tuple of (event_id, hashes, depth)
160		where hashes is a map from algorithm to hash.
161		"""
162		res = yield self.get_latest_event_ids_and_hashes_in_room(room_id)
163		if len(res) > 10:
164		# Sort by reverse depth, so we point to the most recent.
165		res.sort(key=lambda a: -a[2])
166
167		# we use half of the limit for the actual most recent events, and
168		# the other half to randomly point to some of the older events, to
169		# make sure that we don't completely ignore the older events.
170		res = res[0:5] + random.sample(res[5:], 5)
171
172		return res
173
174		def get_latest_event_ids_and_hashes_in_room(self, room_id):
175		"""
176		Gets the current forward extremities in the given room
177
178		Args:
179		room_id (str): room_id
180
181		Returns:
182		Deferred[list[(str, dict[str, str], int)]]
183		for each event, a tuple of (event_id, hashes, depth)
184		where hashes is a map from algorithm to hash.
185		"""
186
187		return self.runInteraction(
188		"get_latest_event_ids_and_hashes_in_room",
189		self._get_latest_event_ids_and_hashes_in_room,
190		room_id,
191		)
192
193		def get_rooms_with_many_extremities(self, min_count, limit, room_id_filter):
194		"""Get the top rooms with at least N extremities.
195
196		Args:
197		min_count (int): The minimum number of extremities
198		limit (int): The maximum number of rooms to return.
199		room_id_filter (iterable[str]): room_ids to exclude from the results
200
201		Returns:
202		Deferred[list]: At most `limit` room IDs that have at least
203		`min_count` extremities, sorted by extremity count.
204		"""
205
206		def _get_rooms_with_many_extremities_txn(txn):
207		where_clause = "1=1"
208		if room_id_filter:
209		where_clause = "room_id NOT IN (%s)" % (
210		",".join("?" for _ in room_id_filter),
211		)
212
213		sql = """
214		SELECT room_id FROM event_forward_extremities
215		WHERE %s
216		GROUP BY room_id
217		HAVING count(*) > ?
218		ORDER BY count(*) DESC
219		LIMIT ?
220		""" % (
221		where_clause,
222		)
223
224		query_args = list(itertools.chain(room_id_filter, [min_count, limit]))
225		txn.execute(sql, query_args)
226		return [room_id for room_id, in txn]
227
228		return self.runInteraction(
229		"get_rooms_with_many_extremities", _get_rooms_with_many_extremities_txn
230		)
231
232		@cached(max_entries=5000, iterable=True)
233		def get_latest_event_ids_in_room(self, room_id):
234		return self._simple_select_onecol(
235		table="event_forward_extremities",
236		keyvalues={"room_id": room_id},
237		retcol="event_id",
238		desc="get_latest_event_ids_in_room",
239		)
240
241		def _get_latest_event_ids_and_hashes_in_room(self, txn, room_id):
242		sql = (
243		"SELECT e.event_id, e.depth FROM events as e "
244		"INNER JOIN event_forward_extremities as f "
245		"ON e.event_id = f.event_id "
246		"AND e.room_id = f.room_id "
247		"WHERE f.room_id = ?"
248		)
249
250		txn.execute(sql, (room_id,))
251
252		results = []
253		for event_id, depth in txn.fetchall():
254		hashes = self._get_event_reference_hashes_txn(txn, event_id)
255		prev_hashes = {
256		k: encode_base64(v) for k, v in hashes.items() if k == "sha256"
257		}
258		results.append((event_id, prev_hashes, depth))
259
260		return results
261
262		def get_min_depth(self, room_id):
263		""" For hte given room, get the minimum depth we have seen for it.
264		"""
265		return self.runInteraction(
266		"get_min_depth", self._get_min_depth_interaction, room_id
267		)
268
269		def _get_min_depth_interaction(self, txn, room_id):
270		min_depth = self._simple_select_one_onecol_txn(
271		txn,
272		table="room_depth",
273		keyvalues={"room_id": room_id},
274		retcol="min_depth",
275		allow_none=True,
276		)
277
278		return int(min_depth) if min_depth is not None else None
279
280		def get_forward_extremeties_for_room(self, room_id, stream_ordering):
281		"""For a given room_id and stream_ordering, return the forward
282		extremeties of the room at that point in "time".
283
284		Throws a StoreError if we have since purged the index for
285		stream_orderings from that point.
286
287		Args:
288		room_id (str):
289		stream_ordering (int):
290
291		Returns:
292		deferred, which resolves to a list of event_ids
293		"""
294		# We want to make the cache more effective, so we clamp to the last
295		# change before the given ordering.
296		last_change = self._events_stream_cache.get_max_pos_of_last_change(room_id)
297
298		# We don't always have a full stream_to_exterm_id table, e.g. after
299		# the upgrade that introduced it, so we make sure we never ask for a
300		# stream_ordering from before a restart
301		last_change = max(self._stream_order_on_start, last_change)
302
303		# provided the last_change is recent enough, we now clamp the requested
304		# stream_ordering to it.
305		if last_change > self.stream_ordering_month_ago:
306		stream_ordering = min(last_change, stream_ordering)
307
308		return self._get_forward_extremeties_for_room(room_id, stream_ordering)
309
310		@cached(max_entries=5000, num_args=2)
311		def _get_forward_extremeties_for_room(self, room_id, stream_ordering):
312		"""For a given room_id and stream_ordering, return the forward
313		extremeties of the room at that point in "time".
314
315		Throws a StoreError if we have since purged the index for
316		stream_orderings from that point.
317		"""
318
319		if stream_ordering <= self.stream_ordering_month_ago:
320		raise StoreError(400, "stream_ordering too old")
321
322		sql = """
323		SELECT event_id FROM stream_ordering_to_exterm
324		INNER JOIN (
325		SELECT room_id, MAX(stream_ordering) AS stream_ordering
326		FROM stream_ordering_to_exterm
327		WHERE stream_ordering <= ? GROUP BY room_id
328		) AS rms USING (room_id, stream_ordering)
329		WHERE room_id = ?
330		"""
331
332		def get_forward_extremeties_for_room_txn(txn):
333		txn.execute(sql, (stream_ordering, room_id))
334		return [event_id for event_id, in txn]
335
336		return self.runInteraction(
337		"get_forward_extremeties_for_room", get_forward_extremeties_for_room_txn
338		)
339
340		def get_backfill_events(self, room_id, event_list, limit):
341		"""Get a list of Events for a given topic that occurred before (and
342		including) the events in event_list. Return a list of max size `limit`
343
344		Args:
345		txn
346		room_id (str)
347		event_list (list)
348		limit (int)
349		"""
350		return (
351		self.runInteraction(
352		"get_backfill_events",
353		self._get_backfill_events,
354		room_id,
355		event_list,
356		limit,
357		)
358		.addCallback(self.get_events_as_list)
359		.addCallback(lambda l: sorted(l, key=lambda e: -e.depth))
360		)
361
362		def _get_backfill_events(self, txn, room_id, event_list, limit):
363		logger.debug(
364		"_get_backfill_events: %s, %s, %s", room_id, repr(event_list), limit
365		)
366
367		event_results = set()
368
369		# We want to make sure that we do a breadth-first, "depth" ordered
370		# search.
371
372		query = (
373		"SELECT depth, prev_event_id FROM event_edges"
374		" INNER JOIN events"
375		" ON prev_event_id = events.event_id"
376		" WHERE event_edges.event_id = ?"
377		" AND event_edges.is_state = ?"
378		" LIMIT ?"
379		)
380
381		queue = PriorityQueue()
382
383		for event_id in event_list:
384		depth = self._simple_select_one_onecol_txn(
385		txn,
386		table="events",
387		keyvalues={"event_id": event_id, "room_id": room_id},
388		retcol="depth",
389		allow_none=True,
390		)
391
392		if depth:
393		queue.put((-depth, event_id))
394
395		while not queue.empty() and len(event_results) < limit:
396		try:
397		_, event_id = queue.get_nowait()
398		except Empty:
399		break
400
401		if event_id in event_results:
402		continue
403
404		event_results.add(event_id)
405
406		txn.execute(query, (event_id, False, limit - len(event_results)))
407
408		for row in txn:
409		if row[1] not in event_results:
410		queue.put((-row[0], row[1]))
411
412		return event_results
413
414		@defer.inlineCallbacks
415		def get_missing_events(self, room_id, earliest_events, latest_events, limit):
416		ids = yield self.runInteraction(
417		"get_missing_events",
418		self._get_missing_events,
419		room_id,
420		earliest_events,
421		latest_events,
422		limit,
423		)
424		events = yield self.get_events_as_list(ids)
425		return events
426
427		def _get_missing_events(self, txn, room_id, earliest_events, latest_events, limit):
428
429		seen_events = set(earliest_events)
430		front = set(latest_events) - seen_events
431		event_results = []
432
433		query = (
434		"SELECT prev_event_id FROM event_edges "
435		"WHERE room_id = ? AND event_id = ? AND is_state = ? "
436		"LIMIT ?"
437		)
438
439		while front and len(event_results) < limit:
440		new_front = set()
441		for event_id in front:
442		txn.execute(
443		query, (room_id, event_id, False, limit - len(event_results))
444		)
445
446		new_results = set(t[0] for t in txn) - seen_events
447
448		new_front \|= new_results
449		seen_events \|= new_results
450		event_results.extend(new_results)
451
452		front = new_front
453
454		# we built the list working backwards from latest_events; we now need to
455		# reverse it so that the events are approximately chronological.
456		event_results.reverse()
457		return event_results
458
459		@defer.inlineCallbacks
460		def get_successor_events(self, event_ids):
461		"""Fetch all events that have the given events as a prev event
462
463		Args:
464		event_ids (iterable[str])
465
466		Returns:
467		Deferred[list[str]]
468		"""
469		rows = yield self._simple_select_many_batch(
470		table="event_edges",
471		column="prev_event_id",
472		iterable=event_ids,
473		retcols=("event_id",),
474		desc="get_successor_events",
475		)
476
477		return [row["event_id"] for row in rows]
478
479
480		class EventFederationStore(EventFederationWorkerStore):
481		""" Responsible for storing and serving up the various graphs associated
482		with an event. Including the main event graph and the auth chains for an
483		event.
484
485		Also has methods for getting the front (latest) and back (oldest) edges
486		of the event graphs. These are used to generate the parents for new events
487		and backfilling from another server respectively.
488		"""
489
490		EVENT_AUTH_STATE_ONLY = "event_auth_state_only"
491
492		def __init__(self, db_conn, hs):
493		super(EventFederationStore, self).__init__(db_conn, hs)
494
495		self.register_background_update_handler(
496		self.EVENT_AUTH_STATE_ONLY, self._background_delete_non_state_event_auth
497		)
498
499		hs.get_clock().looping_call(
500		self._delete_old_forward_extrem_cache, 60 * 60 * 1000
501		)
502
503		def _update_min_depth_for_room_txn(self, txn, room_id, depth):
504		min_depth = self._get_min_depth_interaction(txn, room_id)
505
506		if min_depth and depth >= min_depth:
507		return
508
509		self._simple_upsert_txn(
510		txn,
511		table="room_depth",
512		keyvalues={"room_id": room_id},
513		values={"min_depth": depth},
514		)
515
516		def _handle_mult_prev_events(self, txn, events):
517		"""
518		For the given event, update the event edges table and forward and
519		backward extremities tables.
520		"""
521		self._simple_insert_many_txn(
522		txn,
523		table="event_edges",
524		values=[
525		{
526		"event_id": ev.event_id,
527		"prev_event_id": e_id,
528		"room_id": ev.room_id,
529		"is_state": False,
530		}
531		for ev in events
532		for e_id in ev.prev_event_ids()
533		],
534		)
535
536		self._update_backward_extremeties(txn, events)
537
538		def _update_backward_extremeties(self, txn, events):
539		"""Updates the event_backward_extremities tables based on the new/updated
540		events being persisted.
541
542		This is called for new events and for events that were outliers, but
543		are now being persisted as non-outliers.
544
545		Forward extremities are handled when we first start persisting the events.
546		"""
547		events_by_room = {}
548		for ev in events:
549		events_by_room.setdefault(ev.room_id, []).append(ev)
550
551		query = (
552		"INSERT INTO event_backward_extremities (event_id, room_id)"
553		" SELECT ?, ? WHERE NOT EXISTS ("
554		" SELECT 1 FROM event_backward_extremities"
555		" WHERE event_id = ? AND room_id = ?"
556		" )"
557		" AND NOT EXISTS ("
558		" SELECT 1 FROM events WHERE event_id = ? AND room_id = ? "
559		" AND outlier = ?"
560		" )"
561		)
562
563		txn.executemany(
564		query,
565		[
566		(e_id, ev.room_id, e_id, ev.room_id, e_id, ev.room_id, False)
567		for ev in events
568		for e_id in ev.prev_event_ids()
569		if not ev.internal_metadata.is_outlier()
570		],
571		)
572
573		query = (
574		"DELETE FROM event_backward_extremities"
575		" WHERE event_id = ? AND room_id = ?"
576		)
577		txn.executemany(
578		query,
579		[
580		(ev.event_id, ev.room_id)
581		for ev in events
582		if not ev.internal_metadata.is_outlier()
583		],
584		)
585
586		def _delete_old_forward_extrem_cache(self):
587		def _delete_old_forward_extrem_cache_txn(txn):
588		# Delete entries older than a month, while making sure we don't delete
589		# the only entries for a room.
590		sql = """
591		DELETE FROM stream_ordering_to_exterm
592		WHERE
593		room_id IN (
594		SELECT room_id
595		FROM stream_ordering_to_exterm
596		WHERE stream_ordering > ?
597		) AND stream_ordering < ?
598		"""
599		txn.execute(
600		sql, (self.stream_ordering_month_ago, self.stream_ordering_month_ago)
601		)
602
603		return run_as_background_process(
604		"delete_old_forward_extrem_cache",
605		self.runInteraction,
606		"_delete_old_forward_extrem_cache",
607		_delete_old_forward_extrem_cache_txn,
608		)
609
610		def clean_room_for_join(self, room_id):
611		return self.runInteraction(
612		"clean_room_for_join", self._clean_room_for_join_txn, room_id
613		)
614
615		def _clean_room_for_join_txn(self, txn, room_id):
616		query = "DELETE FROM event_forward_extremities WHERE room_id = ?"
617
618		txn.execute(query, (room_id,))
619		txn.call_after(self.get_latest_event_ids_in_room.invalidate, (room_id,))
620
621		@defer.inlineCallbacks
622		def _background_delete_non_state_event_auth(self, progress, batch_size):
623		def delete_event_auth(txn):
624		target_min_stream_id = progress.get("target_min_stream_id_inclusive")
625		max_stream_id = progress.get("max_stream_id_exclusive")
626
627		if not target_min_stream_id or not max_stream_id:
628		txn.execute("SELECT COALESCE(MIN(stream_ordering), 0) FROM events")
629		rows = txn.fetchall()
630		target_min_stream_id = rows[0][0]
631
632		txn.execute("SELECT COALESCE(MAX(stream_ordering), 0) FROM events")
633		rows = txn.fetchall()
634		max_stream_id = rows[0][0]
635
636		min_stream_id = max_stream_id - batch_size
637
638		sql = """
639		DELETE FROM event_auth
640		WHERE event_id IN (
641		SELECT event_id FROM events
642		LEFT JOIN state_events USING (room_id, event_id)
643		WHERE ? <= stream_ordering AND stream_ordering < ?
644		AND state_key IS null
645		)
646		"""
647
648		txn.execute(sql, (min_stream_id, max_stream_id))
649
650		new_progress = {
651		"target_min_stream_id_inclusive": target_min_stream_id,
652		"max_stream_id_exclusive": min_stream_id,
653		}
654
655		self._background_update_progress_txn(
656		txn, self.EVENT_AUTH_STATE_ONLY, new_progress
657		)
658
659		return min_stream_id >= target_min_stream_id
660
661		result = yield self.runInteraction(
662		self.EVENT_AUTH_STATE_ONLY, delete_event_auth
663		)
664
665		if not result:
666		yield self._end_background_update(self.EVENT_AUTH_STATE_ONLY)
667
668		return batch_size

+0

-960

~~synapse/storage/event_push_actions.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2015 OpenMarket Ltd
2		# Copyright 2018 New Vector Ltd
3		#
4		# Licensed under the Apache License, Version 2.0 (the "License");
5		# you may not use this file except in compliance with the License.
6		# You may obtain a copy of the License at
7		#
8		# http://www.apache.org/licenses/LICENSE-2.0
9		#
10		# Unless required by applicable law or agreed to in writing, software
11		# distributed under the License is distributed on an "AS IS" BASIS,
12		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13		# See the License for the specific language governing permissions and
14		# limitations under the License.
15
16		import logging
17
18		from six import iteritems
19
20		from canonicaljson import json
21
22		from twisted.internet import defer
23
24		from synapse.metrics.background_process_metrics import run_as_background_process
25		from synapse.storage._base import LoggingTransaction, SQLBaseStore
26		from synapse.util.caches.descriptors import cachedInlineCallbacks
27
28		logger = logging.getLogger(__name__)
29
30
31		DEFAULT_NOTIF_ACTION = ["notify", {"set_tweak": "highlight", "value": False}]
32		DEFAULT_HIGHLIGHT_ACTION = [
33		"notify",
34		{"set_tweak": "sound", "value": "default"},
35		{"set_tweak": "highlight"},
36		]
37
38
39		def _serialize_action(actions, is_highlight):
40		"""Custom serializer for actions. This allows us to "compress" common actions.
41
42		We use the fact that most users have the same actions for notifs (and for
43		highlights).
44		We store these default actions as the empty string rather than the full JSON.
45		Since the empty string isn't valid JSON there is no risk of this clashing with
46		any real JSON actions
47		"""
48		if is_highlight:
49		if actions == DEFAULT_HIGHLIGHT_ACTION:
50		return "" # We use empty string as the column is non-NULL
51		else:
52		if actions == DEFAULT_NOTIF_ACTION:
53		return ""
54		return json.dumps(actions)
55
56
57		def _deserialize_action(actions, is_highlight):
58		"""Custom deserializer for actions. This allows us to "compress" common actions
59		"""
60		if actions:
61		return json.loads(actions)
62
63		if is_highlight:
64		return DEFAULT_HIGHLIGHT_ACTION
65		else:
66		return DEFAULT_NOTIF_ACTION
67
68
69		class EventPushActionsWorkerStore(SQLBaseStore):
70		def __init__(self, db_conn, hs):
71		super(EventPushActionsWorkerStore, self).__init__(db_conn, hs)
72
73		# These get correctly set by _find_stream_orderings_for_times_txn
74		self.stream_ordering_month_ago = None
75		self.stream_ordering_day_ago = None
76
77		cur = LoggingTransaction(
78		db_conn.cursor(),
79		name="_find_stream_orderings_for_times_txn",
80		database_engine=self.database_engine,
81		)
82		self._find_stream_orderings_for_times_txn(cur)
83		cur.close()
84
85		self.find_stream_orderings_looping_call = self._clock.looping_call(
86		self._find_stream_orderings_for_times, 10 * 60 * 1000
87		)
88		self._rotate_delay = 3
89		self._rotate_count = 10000
90
91		@cachedInlineCallbacks(num_args=3, tree=True, max_entries=5000)
92		def get_unread_event_push_actions_by_room_for_user(
93		self, room_id, user_id, last_read_event_id
94		):
95		ret = yield self.runInteraction(
96		"get_unread_event_push_actions_by_room",
97		self._get_unread_counts_by_receipt_txn,
98		room_id,
99		user_id,
100		last_read_event_id,
101		)
102		return ret
103
104		def _get_unread_counts_by_receipt_txn(
105		self, txn, room_id, user_id, last_read_event_id
106		):
107		sql = (
108		"SELECT stream_ordering"
109		" FROM events"
110		" WHERE room_id = ? AND event_id = ?"
111		)
112		txn.execute(sql, (room_id, last_read_event_id))
113		results = txn.fetchall()
114		if len(results) == 0:
115		return {"notify_count": 0, "highlight_count": 0}
116
117		stream_ordering = results[0][0]
118
119		return self._get_unread_counts_by_pos_txn(
120		txn, room_id, user_id, stream_ordering
121		)
122
123		def _get_unread_counts_by_pos_txn(self, txn, room_id, user_id, stream_ordering):
124
125		# First get number of notifications.
126		# We don't need to put a notif=1 clause as all rows always have
127		# notif=1
128		sql = (
129		"SELECT count(*)"
130		" FROM event_push_actions ea"
131		" WHERE"
132		" user_id = ?"
133		" AND room_id = ?"
134		" AND stream_ordering > ?"
135		)
136
137		txn.execute(sql, (user_id, room_id, stream_ordering))
138		row = txn.fetchone()
139		notify_count = row[0] if row else 0
140
141		txn.execute(
142		"""
143		SELECT notif_count FROM event_push_summary
144		WHERE room_id = ? AND user_id = ? AND stream_ordering > ?
145		""",
146		(room_id, user_id, stream_ordering),
147		)
148		rows = txn.fetchall()
149		if rows:
150		notify_count += rows[0][0]
151
152		# Now get the number of highlights
153		sql = (
154		"SELECT count(*)"
155		" FROM event_push_actions ea"
156		" WHERE"
157		" highlight = 1"
158		" AND user_id = ?"
159		" AND room_id = ?"
160		" AND stream_ordering > ?"
161		)
162
163		txn.execute(sql, (user_id, room_id, stream_ordering))
164		row = txn.fetchone()
165		highlight_count = row[0] if row else 0
166
167		return {"notify_count": notify_count, "highlight_count": highlight_count}
168
169		@defer.inlineCallbacks
170		def get_push_action_users_in_range(self, min_stream_ordering, max_stream_ordering):
171		def f(txn):
172		sql = (
173		"SELECT DISTINCT(user_id) FROM event_push_actions WHERE"
174		" stream_ordering >= ? AND stream_ordering <= ?"
175		)
176		txn.execute(sql, (min_stream_ordering, max_stream_ordering))
177		return [r[0] for r in txn]
178
179		ret = yield self.runInteraction("get_push_action_users_in_range", f)
180		return ret
181
182		@defer.inlineCallbacks
183		def get_unread_push_actions_for_user_in_range_for_http(
184		self, user_id, min_stream_ordering, max_stream_ordering, limit=20
185		):
186		"""Get a list of the most recent unread push actions for a given user,
187		within the given stream ordering range. Called by the httppusher.
188
189		Args:
190		user_id (str): The user to fetch push actions for.
191		min_stream_ordering(int): The exclusive lower bound on the
192		stream ordering of event push actions to fetch.
193		max_stream_ordering(int): The inclusive upper bound on the
194		stream ordering of event push actions to fetch.
195		limit (int): The maximum number of rows to return.
196		Returns:
197		A promise which resolves to a list of dicts with the keys "event_id",
198		"room_id", "stream_ordering", "actions".
199		The list will be ordered by ascending stream_ordering.
200		The list will have between 0~limit entries.
201		"""
202		# find rooms that have a read receipt in them and return the next
203		# push actions
204		def get_after_receipt(txn):
205		# find rooms that have a read receipt in them and return the next
206		# push actions
207		sql = (
208		"SELECT ep.event_id, ep.room_id, ep.stream_ordering, ep.actions,"
209		" ep.highlight "
210		" FROM ("
211		" SELECT room_id,"
212		" MAX(stream_ordering) as stream_ordering"
213		" FROM events"
214		" INNER JOIN receipts_linearized USING (room_id, event_id)"
215		" WHERE receipt_type = 'm.read' AND user_id = ?"
216		" GROUP BY room_id"
217		") AS rl,"
218		" event_push_actions AS ep"
219		" WHERE"
220		" ep.room_id = rl.room_id"
221		" AND ep.stream_ordering > rl.stream_ordering"
222		" AND ep.user_id = ?"
223		" AND ep.stream_ordering > ?"
224		" AND ep.stream_ordering <= ?"
225		" ORDER BY ep.stream_ordering ASC LIMIT ?"
226		)
227		args = [user_id, user_id, min_stream_ordering, max_stream_ordering, limit]
228		txn.execute(sql, args)
229		return txn.fetchall()
230
231		after_read_receipt = yield self.runInteraction(
232		"get_unread_push_actions_for_user_in_range_http_arr", get_after_receipt
233		)
234
235		# There are rooms with push actions in them but you don't have a read receipt in
236		# them e.g. rooms you've been invited to, so get push actions for rooms which do
237		# not have read receipts in them too.
238		def get_no_receipt(txn):
239		sql = (
240		"SELECT ep.event_id, ep.room_id, ep.stream_ordering, ep.actions,"
241		" ep.highlight "
242		" FROM event_push_actions AS ep"
243		" INNER JOIN events AS e USING (room_id, event_id)"
244		" WHERE"
245		" ep.room_id NOT IN ("
246		" SELECT room_id FROM receipts_linearized"
247		" WHERE receipt_type = 'm.read' AND user_id = ?"
248		" GROUP BY room_id"
249		" )"
250		" AND ep.user_id = ?"
251		" AND ep.stream_ordering > ?"
252		" AND ep.stream_ordering <= ?"
253		" ORDER BY ep.stream_ordering ASC LIMIT ?"
254		)
255		args = [user_id, user_id, min_stream_ordering, max_stream_ordering, limit]
256		txn.execute(sql, args)
257		return txn.fetchall()
258
259		no_read_receipt = yield self.runInteraction(
260		"get_unread_push_actions_for_user_in_range_http_nrr", get_no_receipt
261		)
262
263		notifs = [
264		{
265		"event_id": row[0],
266		"room_id": row[1],
267		"stream_ordering": row[2],
268		"actions": _deserialize_action(row[3], row[4]),
269		}
270		for row in after_read_receipt + no_read_receipt
271		]
272
273		# Now sort it so it's ordered correctly, since currently it will
274		# contain results from the first query, correctly ordered, followed
275		# by results from the second query, but we want them all ordered
276		# by stream_ordering, oldest first.
277		notifs.sort(key=lambda r: r["stream_ordering"])
278
279		# Take only up to the limit. We have to stop at the limit because
280		# one of the subqueries may have hit the limit.
281		return notifs[:limit]
282
283		@defer.inlineCallbacks
284		def get_unread_push_actions_for_user_in_range_for_email(
285		self, user_id, min_stream_ordering, max_stream_ordering, limit=20
286		):
287		"""Get a list of the most recent unread push actions for a given user,
288		within the given stream ordering range. Called by the emailpusher
289
290		Args:
291		user_id (str): The user to fetch push actions for.
292		min_stream_ordering(int): The exclusive lower bound on the
293		stream ordering of event push actions to fetch.
294		max_stream_ordering(int): The inclusive upper bound on the
295		stream ordering of event push actions to fetch.
296		limit (int): The maximum number of rows to return.
297		Returns:
298		A promise which resolves to a list of dicts with the keys "event_id",
299		"room_id", "stream_ordering", "actions", "received_ts".
300		The list will be ordered by descending received_ts.
301		The list will have between 0~limit entries.
302		"""
303		# find rooms that have a read receipt in them and return the most recent
304		# push actions
305		def get_after_receipt(txn):
306		sql = (
307		"SELECT ep.event_id, ep.room_id, ep.stream_ordering, ep.actions,"
308		" ep.highlight, e.received_ts"
309		" FROM ("
310		" SELECT room_id,"
311		" MAX(stream_ordering) as stream_ordering"
312		" FROM events"
313		" INNER JOIN receipts_linearized USING (room_id, event_id)"
314		" WHERE receipt_type = 'm.read' AND user_id = ?"
315		" GROUP BY room_id"
316		") AS rl,"
317		" event_push_actions AS ep"
318		" INNER JOIN events AS e USING (room_id, event_id)"
319		" WHERE"
320		" ep.room_id = rl.room_id"
321		" AND ep.stream_ordering > rl.stream_ordering"
322		" AND ep.user_id = ?"
323		" AND ep.stream_ordering > ?"
324		" AND ep.stream_ordering <= ?"
325		" ORDER BY ep.stream_ordering DESC LIMIT ?"
326		)
327		args = [user_id, user_id, min_stream_ordering, max_stream_ordering, limit]
328		txn.execute(sql, args)
329		return txn.fetchall()
330
331		after_read_receipt = yield self.runInteraction(
332		"get_unread_push_actions_for_user_in_range_email_arr", get_after_receipt
333		)
334
335		# There are rooms with push actions in them but you don't have a read receipt in
336		# them e.g. rooms you've been invited to, so get push actions for rooms which do
337		# not have read receipts in them too.
338		def get_no_receipt(txn):
339		sql = (
340		"SELECT ep.event_id, ep.room_id, ep.stream_ordering, ep.actions,"
341		" ep.highlight, e.received_ts"
342		" FROM event_push_actions AS ep"
343		" INNER JOIN events AS e USING (room_id, event_id)"
344		" WHERE"
345		" ep.room_id NOT IN ("
346		" SELECT room_id FROM receipts_linearized"
347		" WHERE receipt_type = 'm.read' AND user_id = ?"
348		" GROUP BY room_id"
349		" )"
350		" AND ep.user_id = ?"
351		" AND ep.stream_ordering > ?"
352		" AND ep.stream_ordering <= ?"
353		" ORDER BY ep.stream_ordering DESC LIMIT ?"
354		)
355		args = [user_id, user_id, min_stream_ordering, max_stream_ordering, limit]
356		txn.execute(sql, args)
357		return txn.fetchall()
358
359		no_read_receipt = yield self.runInteraction(
360		"get_unread_push_actions_for_user_in_range_email_nrr", get_no_receipt
361		)
362
363		# Make a list of dicts from the two sets of results.
364		notifs = [
365		{
366		"event_id": row[0],
367		"room_id": row[1],
368		"stream_ordering": row[2],
369		"actions": _deserialize_action(row[3], row[4]),
370		"received_ts": row[5],
371		}
372		for row in after_read_receipt + no_read_receipt
373		]
374
375		# Now sort it so it's ordered correctly, since currently it will
376		# contain results from the first query, correctly ordered, followed
377		# by results from the second query, but we want them all ordered
378		# by received_ts (most recent first)
379		notifs.sort(key=lambda r: -(r["received_ts"] or 0))
380
381		# Now return the first `limit`
382		return notifs[:limit]
383
384		def get_if_maybe_push_in_range_for_user(self, user_id, min_stream_ordering):
385		"""A fast check to see if there might be something to push for the
386		user since the given stream ordering. May return false positives.
387
388		Useful to know whether to bother starting a pusher on start up or not.
389
390		Args:
391		user_id (str)
392		min_stream_ordering (int)
393
394		Returns:
395		Deferred[bool]: True if there may be push to process, False if
396		there definitely isn't.
397		"""
398
399		def _get_if_maybe_push_in_range_for_user_txn(txn):
400		sql = """
401		SELECT 1 FROM event_push_actions
402		WHERE user_id = ? AND stream_ordering > ?
403		LIMIT 1
404		"""
405
406		txn.execute(sql, (user_id, min_stream_ordering))
407		return bool(txn.fetchone())
408
409		return self.runInteraction(
410		"get_if_maybe_push_in_range_for_user",
411		_get_if_maybe_push_in_range_for_user_txn,
412		)
413
414		def add_push_actions_to_staging(self, event_id, user_id_actions):
415		"""Add the push actions for the event to the push action staging area.
416
417		Args:
418		event_id (str)
419		user_id_actions (dict[str, list[dict\|str])]): A dictionary mapping
420		user_id to list of push actions, where an action can either be
421		a string or dict.
422
423		Returns:
424		Deferred
425		"""
426
427		if not user_id_actions:
428		return
429
430		# This is a helper function for generating the necessary tuple that
431		# can be used to inert into the `event_push_actions_staging` table.
432		def _gen_entry(user_id, actions):
433		is_highlight = 1 if _action_has_highlight(actions) else 0
434		return (
435		event_id, # event_id column
436		user_id, # user_id column
437		_serialize_action(actions, is_highlight), # actions column
438		1, # notif column
439		is_highlight, # highlight column
440		)
441
442		def _add_push_actions_to_staging_txn(txn):
443		# We don't use _simple_insert_many here to avoid the overhead
444		# of generating lists of dicts.
445
446		sql = """
447		INSERT INTO event_push_actions_staging
448		(event_id, user_id, actions, notif, highlight)
449		VALUES (?, ?, ?, ?, ?)
450		"""
451
452		txn.executemany(
453		sql,
454		(
455		_gen_entry(user_id, actions)
456		for user_id, actions in iteritems(user_id_actions)
457		),
458		)
459
460		return self.runInteraction(
461		"add_push_actions_to_staging", _add_push_actions_to_staging_txn
462		)
463
464		@defer.inlineCallbacks
465		def remove_push_actions_from_staging(self, event_id):
466		"""Called if we failed to persist the event to ensure that stale push
467		actions don't build up in the DB
468
469		Args:
470		event_id (str)
471		"""
472
473		try:
474		res = yield self._simple_delete(
475		table="event_push_actions_staging",
476		keyvalues={"event_id": event_id},
477		desc="remove_push_actions_from_staging",
478		)
479		return res
480		except Exception:
481		# this method is called from an exception handler, so propagating
482		# another exception here really isn't helpful - there's nothing
483		# the caller can do about it. Just log the exception and move on.
484		logger.exception(
485		"Error removing push actions after event persistence failure"
486		)
487
488		def _find_stream_orderings_for_times(self):
489		return run_as_background_process(
490		"event_push_action_stream_orderings",
491		self.runInteraction,
492		"_find_stream_orderings_for_times",
493		self._find_stream_orderings_for_times_txn,
494		)
495
496		def _find_stream_orderings_for_times_txn(self, txn):
497		logger.info("Searching for stream ordering 1 month ago")
498		self.stream_ordering_month_ago = self._find_first_stream_ordering_after_ts_txn(
499		txn, self._clock.time_msec() - 30 * 24 * 60 * 60 * 1000
500		)
501		logger.info(
502		"Found stream ordering 1 month ago: it's %d", self.stream_ordering_month_ago
503		)
504		logger.info("Searching for stream ordering 1 day ago")
505		self.stream_ordering_day_ago = self._find_first_stream_ordering_after_ts_txn(
506		txn, self._clock.time_msec() - 24 * 60 * 60 * 1000
507		)
508		logger.info(
509		"Found stream ordering 1 day ago: it's %d", self.stream_ordering_day_ago
510		)
511
512		def find_first_stream_ordering_after_ts(self, ts):
513		"""Gets the stream ordering corresponding to a given timestamp.
514
515		Specifically, finds the stream_ordering of the first event that was
516		received on or after the timestamp. This is done by a binary search on
517		the events table, since there is no index on received_ts, so is
518		relatively slow.
519
520		Args:
521		ts (int): timestamp in millis
522
523		Returns:
524		Deferred[int]: stream ordering of the first event received on/after
525		the timestamp
526		"""
527		return self.runInteraction(
528		"_find_first_stream_ordering_after_ts_txn",
529		self._find_first_stream_ordering_after_ts_txn,
530		ts,
531		)
532
533		@staticmethod
534		def _find_first_stream_ordering_after_ts_txn(txn, ts):
535		"""
536		Find the stream_ordering of the first event that was received on or
537		after a given timestamp. This is relatively slow as there is no index
538		on received_ts but we can then use this to delete push actions before
539		this.
540
541		received_ts must necessarily be in the same order as stream_ordering
542		and stream_ordering is indexed, so we manually binary search using
543		stream_ordering
544
545		Args:
546		txn (twisted.enterprise.adbapi.Transaction):
547		ts (int): timestamp to search for
548
549		Returns:
550		int: stream ordering
551		"""
552		txn.execute("SELECT MAX(stream_ordering) FROM events")
553		max_stream_ordering = txn.fetchone()[0]
554
555		if max_stream_ordering is None:
556		return 0
557
558		# We want the first stream_ordering in which received_ts is greater
559		# than or equal to ts. Call this point X.
560		#
561		# We maintain the invariants:
562		#
563		# range_start <= X <= range_end
564		#
565		range_start = 0
566		range_end = max_stream_ordering + 1
567
568		# Given a stream_ordering, look up the timestamp at that
569		# stream_ordering.
570		#
571		# The array may be sparse (we may be missing some stream_orderings).
572		# We treat the gaps as the same as having the same value as the
573		# preceding entry, because we will pick the lowest stream_ordering
574		# which satisfies our requirement of received_ts >= ts.
575		#
576		# For example, if our array of events indexed by stream_ordering is
577		# [10, <none>, 20], we should treat this as being equivalent to
578		# [10, 10, 20].
579		#
580		sql = (
581		"SELECT received_ts FROM events"
582		" WHERE stream_ordering <= ?"
583		" ORDER BY stream_ordering DESC"
584		" LIMIT 1"
585		)
586
587		while range_end - range_start > 0:
588		middle = (range_end + range_start) // 2
589		txn.execute(sql, (middle,))
590		row = txn.fetchone()
591		if row is None:
592		# no rows with stream_ordering<=middle
593		range_start = middle + 1
594		continue
595
596		middle_ts = row[0]
597		if ts > middle_ts:
598		# we got a timestamp lower than the one we were looking for.
599		# definitely need to look higher: X > middle.
600		range_start = middle + 1
601		else:
602		# we got a timestamp higher than (or the same as) the one we
603		# were looking for. We aren't yet sure about the point we
604		# looked up, but we can be sure that X <= middle.
605		range_end = middle
606
607		return range_end
608
609
610		class EventPushActionsStore(EventPushActionsWorkerStore):
611		EPA_HIGHLIGHT_INDEX = "epa_highlight_index"
612
613		def __init__(self, db_conn, hs):
614		super(EventPushActionsStore, self).__init__(db_conn, hs)
615
616		self.register_background_index_update(
617		self.EPA_HIGHLIGHT_INDEX,
618		index_name="event_push_actions_u_highlight",
619		table="event_push_actions",
620		columns=["user_id", "stream_ordering"],
621		)
622
623		self.register_background_index_update(
624		"event_push_actions_highlights_index",
625		index_name="event_push_actions_highlights_index",
626		table="event_push_actions",
627		columns=["user_id", "room_id", "topological_ordering", "stream_ordering"],
628		where_clause="highlight=1",
629		)
630
631		self._doing_notif_rotation = False
632		self._rotate_notif_loop = self._clock.looping_call(
633		self._start_rotate_notifs, 30 * 60 * 1000
634		)
635
636		def _set_push_actions_for_event_and_users_txn(
637		self, txn, events_and_contexts, all_events_and_contexts
638		):
639		"""Handles moving push actions from staging table to main
640		event_push_actions table for all events in `events_and_contexts`.
641
642		Also ensures that all events in `all_events_and_contexts` are removed
643		from the push action staging area.
644
645		Args:
646		events_and_contexts (list[(EventBase, EventContext)]): events
647		we are persisting
648		all_events_and_contexts (list[(EventBase, EventContext)]): all
649		events that we were going to persist. This includes events
650		we've already persisted, etc, that wouldn't appear in
651		events_and_context.
652		"""
653
654		sql = """
655		INSERT INTO event_push_actions (
656		room_id, event_id, user_id, actions, stream_ordering,
657		topological_ordering, notif, highlight
658		)
659		SELECT ?, event_id, user_id, actions, ?, ?, notif, highlight
660		FROM event_push_actions_staging
661		WHERE event_id = ?
662		"""
663
664		if events_and_contexts:
665		txn.executemany(
666		sql,
667		(
668		(
669		event.room_id,
670		event.internal_metadata.stream_ordering,
671		event.depth,
672		event.event_id,
673		)
674		for event, _ in events_and_contexts
675		),
676		)
677
678		for event, _ in events_and_contexts:
679		user_ids = self._simple_select_onecol_txn(
680		txn,
681		table="event_push_actions_staging",
682		keyvalues={"event_id": event.event_id},
683		retcol="user_id",
684		)
685
686		for uid in user_ids:
687		txn.call_after(
688		self.get_unread_event_push_actions_by_room_for_user.invalidate_many,
689		(event.room_id, uid),
690		)
691
692		# Now we delete the staging area for all events that were being
693		# persisted.
694		txn.executemany(
695		"DELETE FROM event_push_actions_staging WHERE event_id = ?",
696		((event.event_id,) for event, _ in all_events_and_contexts),
697		)
698
699		@defer.inlineCallbacks
700		def get_push_actions_for_user(
701		self, user_id, before=None, limit=50, only_highlight=False
702		):
703		def f(txn):
704		before_clause = ""
705		if before:
706		before_clause = "AND epa.stream_ordering < ?"
707		args = [user_id, before, limit]
708		else:
709		args = [user_id, limit]
710
711		if only_highlight:
712		if len(before_clause) > 0:
713		before_clause += " "
714		before_clause += "AND epa.highlight = 1"
715
716		# NB. This assumes event_ids are globally unique since
717		# it makes the query easier to index
718		sql = (
719		"SELECT epa.event_id, epa.room_id,"
720		" epa.stream_ordering, epa.topological_ordering,"
721		" epa.actions, epa.highlight, epa.profile_tag, e.received_ts"
722		" FROM event_push_actions epa, events e"
723		" WHERE epa.event_id = e.event_id"
724		" AND epa.user_id = ? %s"
725		" ORDER BY epa.stream_ordering DESC"
726		" LIMIT ?" % (before_clause,)
727		)
728		txn.execute(sql, args)
729		return self.cursor_to_dict(txn)
730
731		push_actions = yield self.runInteraction("get_push_actions_for_user", f)
732		for pa in push_actions:
733		pa["actions"] = _deserialize_action(pa["actions"], pa["highlight"])
734		return push_actions
735
736		@defer.inlineCallbacks
737		def get_time_of_last_push_action_before(self, stream_ordering):
738		def f(txn):
739		sql = (
740		"SELECT e.received_ts"
741		" FROM event_push_actions AS ep"
742		" JOIN events e ON ep.room_id = e.room_id AND ep.event_id = e.event_id"
743		" WHERE ep.stream_ordering > ?"
744		" ORDER BY ep.stream_ordering ASC"
745		" LIMIT 1"
746		)
747		txn.execute(sql, (stream_ordering,))
748		return txn.fetchone()
749
750		result = yield self.runInteraction("get_time_of_last_push_action_before", f)
751		return result[0] if result else None
752
753		@defer.inlineCallbacks
754		def get_latest_push_action_stream_ordering(self):
755		def f(txn):
756		txn.execute("SELECT MAX(stream_ordering) FROM event_push_actions")
757		return txn.fetchone()
758
759		result = yield self.runInteraction("get_latest_push_action_stream_ordering", f)
760		return result[0] or 0
761
762		def _remove_push_actions_for_event_id_txn(self, txn, room_id, event_id):
763		# Sad that we have to blow away the cache for the whole room here
764		txn.call_after(
765		self.get_unread_event_push_actions_by_room_for_user.invalidate_many,
766		(room_id,),
767		)
768		txn.execute(
769		"DELETE FROM event_push_actions WHERE room_id = ? AND event_id = ?",
770		(room_id, event_id),
771		)
772
773		def _remove_old_push_actions_before_txn(
774		self, txn, room_id, user_id, stream_ordering
775		):
776		"""
777		Purges old push actions for a user and room before a given
778		stream_ordering.
779
780		We however keep a months worth of highlighted notifications, so that
781		users can still get a list of recent highlights.
782
783		Args:
784		txn: The transcation
785		room_id: Room ID to delete from
786		user_id: user ID to delete for
787		stream_ordering: The lowest stream ordering which will
788		not be deleted.
789		"""
790		txn.call_after(
791		self.get_unread_event_push_actions_by_room_for_user.invalidate_many,
792		(room_id, user_id),
793		)
794
795		# We need to join on the events table to get the received_ts for
796		# event_push_actions and sqlite won't let us use a join in a delete so
797		# we can't just delete where received_ts < x. Furthermore we can
798		# only identify event_push_actions by a tuple of room_id, event_id
799		# we we can't use a subquery.
800		# Instead, we look up the stream ordering for the last event in that
801		# room received before the threshold time and delete event_push_actions
802		# in the room with a stream_odering before that.
803		txn.execute(
804		"DELETE FROM event_push_actions "
805		" WHERE user_id = ? AND room_id = ? AND "
806		" stream_ordering <= ?"
807		" AND ((stream_ordering < ? AND highlight = 1) or highlight = 0)",
808		(user_id, room_id, stream_ordering, self.stream_ordering_month_ago),
809		)
810
811		txn.execute(
812		"""
813		DELETE FROM event_push_summary
814		WHERE room_id = ? AND user_id = ? AND stream_ordering <= ?
815		""",
816		(room_id, user_id, stream_ordering),
817		)
818
819		def _start_rotate_notifs(self):
820		return run_as_background_process("rotate_notifs", self._rotate_notifs)
821
822		@defer.inlineCallbacks
823		def _rotate_notifs(self):
824		if self._doing_notif_rotation or self.stream_ordering_day_ago is None:
825		return
826		self._doing_notif_rotation = True
827
828		try:
829		while True:
830		logger.info("Rotating notifications")
831
832		caught_up = yield self.runInteraction(
833		"_rotate_notifs", self._rotate_notifs_txn
834		)
835		if caught_up:
836		break
837		yield self.hs.get_clock().sleep(self._rotate_delay)
838		finally:
839		self._doing_notif_rotation = False
840
841		def _rotate_notifs_txn(self, txn):
842		"""Archives older notifications into event_push_summary. Returns whether
843		the archiving process has caught up or not.
844		"""
845
846		old_rotate_stream_ordering = self._simple_select_one_onecol_txn(
847		txn,
848		table="event_push_summary_stream_ordering",
849		keyvalues={},
850		retcol="stream_ordering",
851		)
852
853		# We don't to try and rotate millions of rows at once, so we cap the
854		# maximum stream ordering we'll rotate before.
855		txn.execute(
856		"""
857		SELECT stream_ordering FROM event_push_actions
858		WHERE stream_ordering > ?
859		ORDER BY stream_ordering ASC LIMIT 1 OFFSET ?
860		""",
861		(old_rotate_stream_ordering, self._rotate_count),
862		)
863		stream_row = txn.fetchone()
864		if stream_row:
865		offset_stream_ordering, = stream_row
866		rotate_to_stream_ordering = min(
867		self.stream_ordering_day_ago, offset_stream_ordering
868		)
869		caught_up = offset_stream_ordering >= self.stream_ordering_day_ago
870		else:
871		rotate_to_stream_ordering = self.stream_ordering_day_ago
872		caught_up = True
873
874		logger.info("Rotating notifications up to: %s", rotate_to_stream_ordering)
875
876		self._rotate_notifs_before_txn(txn, rotate_to_stream_ordering)
877
878		# We have caught up iff we were limited by `stream_ordering_day_ago`
879		return caught_up
880
881		def _rotate_notifs_before_txn(self, txn, rotate_to_stream_ordering):
882		old_rotate_stream_ordering = self._simple_select_one_onecol_txn(
883		txn,
884		table="event_push_summary_stream_ordering",
885		keyvalues={},
886		retcol="stream_ordering",
887		)
888
889		# Calculate the new counts that should be upserted into event_push_summary
890		sql = """
891		SELECT user_id, room_id,
892		coalesce(old.notif_count, 0) + upd.notif_count,
893		upd.stream_ordering,
894		old.user_id
895		FROM (
896		SELECT user_id, room_id, count(*) as notif_count,
897		max(stream_ordering) as stream_ordering
898		FROM event_push_actions
899		WHERE ? <= stream_ordering AND stream_ordering < ?
900		AND highlight = 0
901		GROUP BY user_id, room_id
902		) AS upd
903		LEFT JOIN event_push_summary AS old USING (user_id, room_id)
904		"""
905
906		txn.execute(sql, (old_rotate_stream_ordering, rotate_to_stream_ordering))
907		rows = txn.fetchall()
908
909		logger.info("Rotating notifications, handling %d rows", len(rows))
910
911		# If the `old.user_id` above is NULL then we know there isn't already an
912		# entry in the table, so we simply insert it. Otherwise we update the
913		# existing table.
914		self._simple_insert_many_txn(
915		txn,
916		table="event_push_summary",
917		values=[
918		{
919		"user_id": row[0],
920		"room_id": row[1],
921		"notif_count": row[2],
922		"stream_ordering": row[3],
923		}
924		for row in rows
925		if row[4] is None
926		],
927		)
928
929		txn.executemany(
930		"""
931		UPDATE event_push_summary SET notif_count = ?, stream_ordering = ?
932		WHERE user_id = ? AND room_id = ?
933		""",
934		((row[2], row[3], row[0], row[1]) for row in rows if row[4] is not None),
935		)
936
937		txn.execute(
938		"DELETE FROM event_push_actions"
939		" WHERE ? <= stream_ordering AND stream_ordering < ? AND highlight = 0",
940		(old_rotate_stream_ordering, rotate_to_stream_ordering),
941		)
942
943		logger.info("Rotating notifications, deleted %s push actions", txn.rowcount)
944
945		txn.execute(
946		"UPDATE event_push_summary_stream_ordering SET stream_ordering = ?",
947		(rotate_to_stream_ordering,),
948		)
949
950
951		def _action_has_highlight(actions):
952		for action in actions:
953		try:
954		if action.get("set_tweak", None) == "highlight":
955		return action.get("value", True)
956		except AttributeError:
957		pass
958
959		return False

+0

-2481

~~synapse/storage/events.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2014-2016 OpenMarket Ltd
2		# Copyright 2018-2019 New Vector Ltd
3		# Copyright 2019 The Matrix.org Foundation C.I.C.
4		#
5		# Licensed under the Apache License, Version 2.0 (the "License");
6		# you may not use this file except in compliance with the License.
7		# You may obtain a copy of the License at
8		#
9		# http://www.apache.org/licenses/LICENSE-2.0
10		#
11		# Unless required by applicable law or agreed to in writing, software
12		# distributed under the License is distributed on an "AS IS" BASIS,
13		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14		# See the License for the specific language governing permissions and
15		# limitations under the License.
16
17		import itertools
18		import logging
19		from collections import Counter as c_counter, OrderedDict, deque, namedtuple
20		from functools import wraps
21
22		from six import iteritems, text_type
23		from six.moves import range
24
25		from canonicaljson import encode_canonical_json, json
26		from prometheus_client import Counter, Histogram
27
28		from twisted.internet import defer
29
30		import synapse.metrics
31		from synapse.api.constants import EventTypes
32		from synapse.api.errors import SynapseError
33		from synapse.events import EventBase # noqa: F401
34		from synapse.events.snapshot import EventContext # noqa: F401
35		from synapse.events.utils import prune_event_dict
36		from synapse.logging.context import PreserveLoggingContext, make_deferred_yieldable
37		from synapse.logging.utils import log_function
38		from synapse.metrics import BucketCollector
39		from synapse.metrics.background_process_metrics import run_as_background_process
40		from synapse.state import StateResolutionStore
41		from synapse.storage.background_updates import BackgroundUpdateStore
42		from synapse.storage.event_federation import EventFederationStore
43		from synapse.storage.events_worker import EventsWorkerStore
44		from synapse.storage.state import StateGroupWorkerStore
45		from synapse.types import RoomStreamToken, get_domain_from_id
46		from synapse.util import batch_iter
47		from synapse.util.async_helpers import ObservableDeferred
48		from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
49		from synapse.util.frozenutils import frozendict_json_encoder
50		from synapse.util.metrics import Measure
51
52		logger = logging.getLogger(__name__)
53
54		persist_event_counter = Counter("synapse_storage_events_persisted_events", "")
55		event_counter = Counter(
56		"synapse_storage_events_persisted_events_sep",
57		"",
58		["type", "origin_type", "origin_entity"],
59		)
60
61		# The number of times we are recalculating the current state
62		state_delta_counter = Counter("synapse_storage_events_state_delta", "")
63
64		# The number of times we are recalculating state when there is only a
65		# single forward extremity
66		state_delta_single_event_counter = Counter(
67		"synapse_storage_events_state_delta_single_event", ""
68		)
69
70		# The number of times we are reculating state when we could have resonably
71		# calculated the delta when we calculated the state for an event we were
72		# persisting.
73		state_delta_reuse_delta_counter = Counter(
74		"synapse_storage_events_state_delta_reuse_delta", ""
75		)
76
77		# The number of forward extremities for each new event.
78		forward_extremities_counter = Histogram(
79		"synapse_storage_events_forward_extremities_persisted",
80		"Number of forward extremities for each new event",
81		buckets=(1, 2, 3, 5, 7, 10, 15, 20, 50, 100, 200, 500, "+Inf"),
82		)
83
84		# The number of stale forward extremities for each new event. Stale extremities
85		# are those that were in the previous set of extremities as well as the new.
86		stale_forward_extremities_counter = Histogram(
87		"synapse_storage_events_stale_forward_extremities_persisted",
88		"Number of unchanged forward extremities for each new event",
89		buckets=(0, 1, 2, 3, 5, 7, 10, 15, 20, 50, 100, 200, 500, "+Inf"),
90		)
91
92
93		def encode_json(json_object):
94		"""
95		Encode a Python object as JSON and return it in a Unicode string.
96		"""
97		out = frozendict_json_encoder.encode(json_object)
98		if isinstance(out, bytes):
99		out = out.decode("utf8")
100		return out
101
102
103		class _EventPeristenceQueue(object):
104		"""Queues up events so that they can be persisted in bulk with only one
105		concurrent transaction per room.
106		"""
107
108		_EventPersistQueueItem = namedtuple(
109		"_EventPersistQueueItem", ("events_and_contexts", "backfilled", "deferred")
110		)
111
112		def __init__(self):
113		self._event_persist_queues = {}
114		self._currently_persisting_rooms = set()
115
116		def add_to_queue(self, room_id, events_and_contexts, backfilled):
117		"""Add events to the queue, with the given persist_event options.
118
119		NB: due to the normal usage pattern of this method, it does not
120		follow the synapse logcontext rules, and leaves the logcontext in
121		place whether or not the returned deferred is ready.
122
123		Args:
124		room_id (str):
125		events_and_contexts (list[(EventBase, EventContext)]):
126		backfilled (bool):
127
128		Returns:
129		defer.Deferred: a deferred which will resolve once the events are
130		persisted. Runs its callbacks without a logcontext.
131		"""
132		queue = self._event_persist_queues.setdefault(room_id, deque())
133		if queue:
134		# if the last item in the queue has the same `backfilled` setting,
135		# we can just add these new events to that item.
136		end_item = queue[-1]
137		if end_item.backfilled == backfilled:
138		end_item.events_and_contexts.extend(events_and_contexts)
139		return end_item.deferred.observe()
140
141		deferred = ObservableDeferred(defer.Deferred(), consumeErrors=True)
142
143		queue.append(
144		self._EventPersistQueueItem(
145		events_and_contexts=events_and_contexts,
146		backfilled=backfilled,
147		deferred=deferred,
148		)
149		)
150
151		return deferred.observe()
152
153		def handle_queue(self, room_id, per_item_callback):
154		"""Attempts to handle the queue for a room if not already being handled.
155
156		The given callback will be invoked with for each item in the queue,
157		of type _EventPersistQueueItem. The per_item_callback will continuously
158		be called with new items, unless the queue becomnes empty. The return
159		value of the function will be given to the deferreds waiting on the item,
160		exceptions will be passed to the deferreds as well.
161
162		This function should therefore be called whenever anything is added
163		to the queue.
164
165		If another callback is currently handling the queue then it will not be
166		invoked.
167		"""
168
169		if room_id in self._currently_persisting_rooms:
170		return
171
172		self._currently_persisting_rooms.add(room_id)
173
174		@defer.inlineCallbacks
175		def handle_queue_loop():
176		try:
177		queue = self._get_drainining_queue(room_id)
178		for item in queue:
179		try:
180		ret = yield per_item_callback(item)
181		except Exception:
182		with PreserveLoggingContext():
183		item.deferred.errback()
184		else:
185		with PreserveLoggingContext():
186		item.deferred.callback(ret)
187		finally:
188		queue = self._event_persist_queues.pop(room_id, None)
189		if queue:
190		self._event_persist_queues[room_id] = queue
191		self._currently_persisting_rooms.discard(room_id)
192
193		# set handle_queue_loop off in the background
194		run_as_background_process("persist_events", handle_queue_loop)
195
196		def _get_drainining_queue(self, room_id):
197		queue = self._event_persist_queues.setdefault(room_id, deque())
198
199		try:
200		while True:
201		yield queue.popleft()
202		except IndexError:
203		# Queue has been drained.
204		pass
205
206
207		_EventCacheEntry = namedtuple("_EventCacheEntry", ("event", "redacted_event"))
208
209
210		def _retry_on_integrity_error(func):
211		"""Wraps a database function so that it gets retried on IntegrityError,
212		with `delete_existing=True` passed in.
213
214		Args:
215		func: function that returns a Deferred and accepts a `delete_existing` arg
216		"""
217
218		@wraps(func)
219		@defer.inlineCallbacks
220		def f(self, args, *kwargs):
221		try:
222		res = yield func(self, args, *kwargs)
223		except self.database_engine.module.IntegrityError:
224		logger.exception("IntegrityError, retrying.")
225		res = yield func(self, args, delete_existing=True, *kwargs)
226		return res
227
228		return f
229
230
231		# inherits from EventFederationStore so that we can call _update_backward_extremities
232		# and _handle_mult_prev_events (though arguably those could both be moved in here)
233		class EventsStore(
234		StateGroupWorkerStore,
235		EventFederationStore,
236		EventsWorkerStore,
237		BackgroundUpdateStore,
238		):
239		def __init__(self, db_conn, hs):
240		super(EventsStore, self).__init__(db_conn, hs)
241
242		self._event_persist_queue = _EventPeristenceQueue()
243		self._state_resolution_handler = hs.get_state_resolution_handler()
244
245		# Collect metrics on the number of forward extremities that exist.
246		# Counter of number of extremities to count
247		self._current_forward_extremities_amount = c_counter()
248
249		BucketCollector(
250		"synapse_forward_extremities",
251		lambda: self._current_forward_extremities_amount,
252		buckets=[1, 2, 3, 5, 7, 10, 15, 20, 50, 100, 200, 500, "+Inf"],
253		)
254
255		# Read the extrems every 60 minutes
256		def read_forward_extremities():
257		# run as a background process to make sure that the database transactions
258		# have a logcontext to report to
259		return run_as_background_process(
260		"read_forward_extremities", self._read_forward_extremities
261		)
262
263		hs.get_clock().looping_call(read_forward_extremities, 60 * 60 * 1000)
264
265		def _censor_redactions():
266		return run_as_background_process(
267		"_censor_redactions", self._censor_redactions
268		)
269
270		if self.hs.config.redaction_retention_period is not None:
271		hs.get_clock().looping_call(_censor_redactions, 5 * 60 * 1000)
272
273		@defer.inlineCallbacks
274		def _read_forward_extremities(self):
275		def fetch(txn):
276		txn.execute(
277		"""
278		select count(*) c from event_forward_extremities
279		group by room_id
280		"""
281		)
282		return txn.fetchall()
283
284		res = yield self.runInteraction("read_forward_extremities", fetch)
285		self._current_forward_extremities_amount = c_counter(list(x[0] for x in res))
286
287		@defer.inlineCallbacks
288		def persist_events(self, events_and_contexts, backfilled=False):
289		"""
290		Write events to the database
291		Args:
292		events_and_contexts: list of tuples of (event, context)
293		backfilled (bool): Whether the results are retrieved from federation
294		via backfill or not. Used to determine if they're "new" events
295		which might update the current state etc.
296
297		Returns:
298		Deferred[int]: the stream ordering of the latest persisted event
299		"""
300		partitioned = {}
301		for event, ctx in events_and_contexts:
302		partitioned.setdefault(event.room_id, []).append((event, ctx))
303
304		deferreds = []
305		for room_id, evs_ctxs in iteritems(partitioned):
306		d = self._event_persist_queue.add_to_queue(
307		room_id, evs_ctxs, backfilled=backfilled
308		)
309		deferreds.append(d)
310
311		for room_id in partitioned:
312		self._maybe_start_persisting(room_id)
313
314		yield make_deferred_yieldable(
315		defer.gatherResults(deferreds, consumeErrors=True)
316		)
317
318		max_persisted_id = yield self._stream_id_gen.get_current_token()
319
320		return max_persisted_id
321
322		@defer.inlineCallbacks
323		@log_function
324		def persist_event(self, event, context, backfilled=False):
325		"""
326
327		Args:
328		event (EventBase):
329		context (EventContext):
330		backfilled (bool):
331
332		Returns:
333		Deferred: resolves to (int, int): the stream ordering of ``event``,
334		and the stream ordering of the latest persisted event
335		"""
336		deferred = self._event_persist_queue.add_to_queue(
337		event.room_id, [(event, context)], backfilled=backfilled
338		)
339
340		self._maybe_start_persisting(event.room_id)
341
342		yield make_deferred_yieldable(deferred)
343
344		max_persisted_id = yield self._stream_id_gen.get_current_token()
345		return (event.internal_metadata.stream_ordering, max_persisted_id)
346
347		def _maybe_start_persisting(self, room_id):
348		@defer.inlineCallbacks
349		def persisting_queue(item):
350		with Measure(self._clock, "persist_events"):
351		yield self._persist_events(
352		item.events_and_contexts, backfilled=item.backfilled
353		)
354
355		self._event_persist_queue.handle_queue(room_id, persisting_queue)
356
357		@_retry_on_integrity_error
358		@defer.inlineCallbacks
359		def _persist_events(
360		self, events_and_contexts, backfilled=False, delete_existing=False
361		):
362		"""Persist events to db
363
364		Args:
365		events_and_contexts (list[(EventBase, EventContext)]):
366		backfilled (bool):
367		delete_existing (bool):
368
369		Returns:
370		Deferred: resolves when the events have been persisted
371		"""
372		if not events_and_contexts:
373		return
374
375		chunks = [
376		events_and_contexts[x : x + 100]
377		for x in range(0, len(events_and_contexts), 100)
378		]
379
380		for chunk in chunks:
381		# We can't easily parallelize these since different chunks
382		# might contain the same event. :(
383
384		# NB: Assumes that we are only persisting events for one room
385		# at a time.
386
387		# map room_id->list[event_ids] giving the new forward
388		# extremities in each room
389		new_forward_extremeties = {}
390
391		# map room_id->(type,state_key)->event_id tracking the full
392		# state in each room after adding these events.
393		# This is simply used to prefill the get_current_state_ids
394		# cache
395		current_state_for_room = {}
396
397		# map room_id->(to_delete, to_insert) where to_delete is a list
398		# of type/state keys to remove from current state, and to_insert
399		# is a map (type,key)->event_id giving the state delta in each
400		# room
401		state_delta_for_room = {}
402
403		if not backfilled:
404		with Measure(self._clock, "_calculate_state_and_extrem"):
405		# Work out the new "current state" for each room.
406		# We do this by working out what the new extremities are and then
407		# calculating the state from that.
408		events_by_room = {}
409		for event, context in chunk:
410		events_by_room.setdefault(event.room_id, []).append(
411		(event, context)
412		)
413
414		for room_id, ev_ctx_rm in iteritems(events_by_room):
415		latest_event_ids = yield self.get_latest_event_ids_in_room(
416		room_id
417		)
418		new_latest_event_ids = yield self._calculate_new_extremities(
419		room_id, ev_ctx_rm, latest_event_ids
420		)
421
422		latest_event_ids = set(latest_event_ids)
423		if new_latest_event_ids == latest_event_ids:
424		# No change in extremities, so no change in state
425		continue
426
427		# there should always be at least one forward extremity.
428		# (except during the initial persistence of the send_join
429		# results, in which case there will be no existing
430		# extremities, so we'll `continue` above and skip this bit.)
431		assert new_latest_event_ids, "No forward extremities left!"
432
433		new_forward_extremeties[room_id] = new_latest_event_ids
434
435		len_1 = (
436		len(latest_event_ids) == 1
437		and len(new_latest_event_ids) == 1
438		)
439		if len_1:
440		all_single_prev_not_state = all(
441		len(event.prev_event_ids()) == 1
442		and not event.is_state()
443		for event, ctx in ev_ctx_rm
444		)
445		# Don't bother calculating state if they're just
446		# a long chain of single ancestor non-state events.
447		if all_single_prev_not_state:
448		continue
449
450		state_delta_counter.inc()
451		if len(new_latest_event_ids) == 1:
452		state_delta_single_event_counter.inc()
453
454		# This is a fairly handwavey check to see if we could
455		# have guessed what the delta would have been when
456		# processing one of these events.
457		# What we're interested in is if the latest extremities
458		# were the same when we created the event as they are
459		# now. When this server creates a new event (as opposed
460		# to receiving it over federation) it will use the
461		# forward extremities as the prev_events, so we can
462		# guess this by looking at the prev_events and checking
463		# if they match the current forward extremities.
464		for ev, _ in ev_ctx_rm:
465		prev_event_ids = set(ev.prev_event_ids())
466		if latest_event_ids == prev_event_ids:
467		state_delta_reuse_delta_counter.inc()
468		break
469
470		logger.info("Calculating state delta for room %s", room_id)
471		with Measure(
472		self._clock, "persist_events.get_new_state_after_events"
473		):
474		res = yield self._get_new_state_after_events(
475		room_id,
476		ev_ctx_rm,
477		latest_event_ids,
478		new_latest_event_ids,
479		)
480		current_state, delta_ids = res
481
482		# If either are not None then there has been a change,
483		# and we need to work out the delta (or use that
484		# given)
485		if delta_ids is not None:
486		# If there is a delta we know that we've
487		# only added or replaced state, never
488		# removed keys entirely.
489		state_delta_for_room[room_id] = ([], delta_ids)
490		elif current_state is not None:
491		with Measure(
492		self._clock, "persist_events.calculate_state_delta"
493		):
494		delta = yield self._calculate_state_delta(
495		room_id, current_state
496		)
497		state_delta_for_room[room_id] = delta
498
499		# If we have the current_state then lets prefill
500		# the cache with it.
501		if current_state is not None:
502		current_state_for_room[room_id] = current_state
503
504		# We want to calculate the stream orderings as late as possible, as
505		# we only notify after all events with a lesser stream ordering have
506		# been persisted. I.e. if we spend 10s inside the with block then
507		# that will delay all subsequent events from being notified about.
508		# Hence why we do it down here rather than wrapping the entire
509		# function.
510		#
511		# Its safe to do this after calculating the state deltas etc as we
512		# only need to protect the persistence of the events. This is to
513		# ensure that queries of the form "fetch events since X" don't
514		# return events and stream positions after events that are still in
515		# flight, as otherwise subsequent requests "fetch event since Y"
516		# will not return those events.
517		#
518		# Note: Multiple instances of this function cannot be in flight at
519		# the same time for the same room.
520		if backfilled:
521		stream_ordering_manager = self._backfill_id_gen.get_next_mult(
522		len(chunk)
523		)
524		else:
525		stream_ordering_manager = self._stream_id_gen.get_next_mult(len(chunk))
526
527		with stream_ordering_manager as stream_orderings:
528		for (event, context), stream in zip(chunk, stream_orderings):
529		event.internal_metadata.stream_ordering = stream
530
531		yield self.runInteraction(
532		"persist_events",
533		self._persist_events_txn,
534		events_and_contexts=chunk,
535		backfilled=backfilled,
536		delete_existing=delete_existing,
537		state_delta_for_room=state_delta_for_room,
538		new_forward_extremeties=new_forward_extremeties,
539		)
540		persist_event_counter.inc(len(chunk))
541
542		if not backfilled:
543		# backfilled events have negative stream orderings, so we don't
544		# want to set the event_persisted_position to that.
545		synapse.metrics.event_persisted_position.set(
546		chunk[-1][0].internal_metadata.stream_ordering
547		)
548
549		for event, context in chunk:
550		if context.app_service:
551		origin_type = "local"
552		origin_entity = context.app_service.id
553		elif self.hs.is_mine_id(event.sender):
554		origin_type = "local"
555		origin_entity = "client"
556		else:
557		origin_type = "remote"
558		origin_entity = get_domain_from_id(event.sender)
559
560		event_counter.labels(event.type, origin_type, origin_entity).inc()
561
562		for room_id, new_state in iteritems(current_state_for_room):
563		self.get_current_state_ids.prefill((room_id,), new_state)
564
565		for room_id, latest_event_ids in iteritems(new_forward_extremeties):
566		self.get_latest_event_ids_in_room.prefill(
567		(room_id,), list(latest_event_ids)
568		)
569
570		@defer.inlineCallbacks
571		def _calculate_new_extremities(self, room_id, event_contexts, latest_event_ids):
572		"""Calculates the new forward extremities for a room given events to
573		persist.
574
575		Assumes that we are only persisting events for one room at a time.
576		"""
577
578		# we're only interested in new events which aren't outliers and which aren't
579		# being rejected.
580		new_events = [
581		event
582		for event, ctx in event_contexts
583		if not event.internal_metadata.is_outlier()
584		and not ctx.rejected
585		and not event.internal_metadata.is_soft_failed()
586		]
587
588		latest_event_ids = set(latest_event_ids)
589
590		# start with the existing forward extremities
591		result = set(latest_event_ids)
592
593		# add all the new events to the list
594		result.update(event.event_id for event in new_events)
595
596		# Now remove all events which are prev_events of any of the new events
597		result.difference_update(
598		e_id for event in new_events for e_id in event.prev_event_ids()
599		)
600
601		# Remove any events which are prev_events of any existing events.
602		existing_prevs = yield self._get_events_which_are_prevs(result)
603		result.difference_update(existing_prevs)
604
605		# Finally handle the case where the new events have soft-failed prev
606		# events. If they do we need to remove them and their prev events,
607		# otherwise we end up with dangling extremities.
608		existing_prevs = yield self._get_prevs_before_rejected(
609		e_id for event in new_events for e_id in event.prev_event_ids()
610		)
611		result.difference_update(existing_prevs)
612
613		# We only update metrics for events that change forward extremities
614		# (e.g. we ignore backfill/outliers/etc)
615		if result != latest_event_ids:
616		forward_extremities_counter.observe(len(result))
617		stale = latest_event_ids & result
618		stale_forward_extremities_counter.observe(len(stale))
619
620		return result
621
622		@defer.inlineCallbacks
623		def _get_events_which_are_prevs(self, event_ids):
624		"""Filter the supplied list of event_ids to get those which are prev_events of
625		existing (non-outlier/rejected) events.
626
627		Args:
628		event_ids (Iterable[str]): event ids to filter
629
630		Returns:
631		Deferred[List[str]]: filtered event ids
632		"""
633		results = []
634
635		def _get_events_which_are_prevs_txn(txn, batch):
636		sql = """
637		SELECT prev_event_id, internal_metadata
638		FROM event_edges
639		INNER JOIN events USING (event_id)
640		LEFT JOIN rejections USING (event_id)
641		LEFT JOIN event_json USING (event_id)
642		WHERE
643		prev_event_id IN (%s)
644		AND NOT events.outlier
645		AND rejections.event_id IS NULL
646		""" % (
647		",".join("?" for _ in batch),
648		)
649
650		txn.execute(sql, batch)
651		results.extend(r[0] for r in txn if not json.loads(r[1]).get("soft_failed"))
652
653		for chunk in batch_iter(event_ids, 100):
654		yield self.runInteraction(
655		"_get_events_which_are_prevs", _get_events_which_are_prevs_txn, chunk
656		)
657
658		return results
659
660		@defer.inlineCallbacks
661		def _get_prevs_before_rejected(self, event_ids):
662		"""Get soft-failed ancestors to remove from the extremities.
663
664		Given a set of events, find all those that have been soft-failed or
665		rejected. Returns those soft failed/rejected events and their prev
666		events (whether soft-failed/rejected or not), and recurses up the
667		prev-event graph until it finds no more soft-failed/rejected events.
668
669		This is used to find extremities that are ancestors of new events, but
670		are separated by soft failed events.
671
672		Args:
673		event_ids (Iterable[str]): Events to find prev events for. Note
674		that these must have already been persisted.
675
676		Returns:
677		Deferred[set[str]]
678		"""
679
680		# The set of event_ids to return. This includes all soft-failed events
681		# and their prev events.
682		existing_prevs = set()
683
684		def _get_prevs_before_rejected_txn(txn, batch):
685		to_recursively_check = batch
686
687		while to_recursively_check:
688		sql = """
689		SELECT
690		event_id, prev_event_id, internal_metadata,
691		rejections.event_id IS NOT NULL
692		FROM event_edges
693		INNER JOIN events USING (event_id)
694		LEFT JOIN rejections USING (event_id)
695		LEFT JOIN event_json USING (event_id)
696		WHERE
697		event_id IN (%s)
698		AND NOT events.outlier
699		""" % (
700		",".join("?" for _ in to_recursively_check),
701		)
702
703		txn.execute(sql, to_recursively_check)
704		to_recursively_check = []
705
706		for event_id, prev_event_id, metadata, rejected in txn:
707		if prev_event_id in existing_prevs:
708		continue
709
710		soft_failed = json.loads(metadata).get("soft_failed")
711		if soft_failed or rejected:
712		to_recursively_check.append(prev_event_id)
713		existing_prevs.add(prev_event_id)
714
715		for chunk in batch_iter(event_ids, 100):
716		yield self.runInteraction(
717		"_get_prevs_before_rejected", _get_prevs_before_rejected_txn, chunk
718		)
719
720		return existing_prevs
721
722		@defer.inlineCallbacks
723		def _get_new_state_after_events(
724		self, room_id, events_context, old_latest_event_ids, new_latest_event_ids
725		):
726		"""Calculate the current state dict after adding some new events to
727		a room
728
729		Args:
730		room_id (str):
731		room to which the events are being added. Used for logging etc
732
733		events_context (list[(EventBase, EventContext)]):
734		events and contexts which are being added to the room
735
736		old_latest_event_ids (iterable[str]):
737		the old forward extremities for the room.
738
739		new_latest_event_ids (iterable[str]):
740		the new forward extremities for the room.
741
742		Returns:
743		Deferred[tuple[dict[(str,str), str]\|None, dict[(str,str), str]\|None]]:
744		Returns a tuple of two state maps, the first being the full new current
745		state and the second being the delta to the existing current state.
746		If both are None then there has been no change.
747
748		If there has been a change then we only return the delta if its
749		already been calculated. Conversely if we do know the delta then
750		the new current state is only returned if we've already calculated
751		it.
752		"""
753		# map from state_group to ((type, key) -> event_id) state map
754		state_groups_map = {}
755
756		# Map from (prev state group, new state group) -> delta state dict
757		state_group_deltas = {}
758
759		for ev, ctx in events_context:
760		if ctx.state_group is None:
761		# This should only happen for outlier events.
762		if not ev.internal_metadata.is_outlier():
763		raise Exception(
764		"Context for new event %s has no state "
765		"group" % (ev.event_id,)
766		)
767		continue
768
769		if ctx.state_group in state_groups_map:
770		continue
771
772		# We're only interested in pulling out state that has already
773		# been cached in the context. We'll pull stuff out of the DB later
774		# if necessary.
775		current_state_ids = ctx.get_cached_current_state_ids()
776		if current_state_ids is not None:
777		state_groups_map[ctx.state_group] = current_state_ids
778
779		if ctx.prev_group:
780		state_group_deltas[(ctx.prev_group, ctx.state_group)] = ctx.delta_ids
781
782		# We need to map the event_ids to their state groups. First, let's
783		# check if the event is one we're persisting, in which case we can
784		# pull the state group from its context.
785		# Otherwise we need to pull the state group from the database.
786
787		# Set of events we need to fetch groups for. (We know none of the old
788		# extremities are going to be in events_context).
789		missing_event_ids = set(old_latest_event_ids)
790
791		event_id_to_state_group = {}
792		for event_id in new_latest_event_ids:
793		# First search in the list of new events we're adding.
794		for ev, ctx in events_context:
795		if event_id == ev.event_id and ctx.state_group is not None:
796		event_id_to_state_group[event_id] = ctx.state_group
797		break
798		else:
799		# If we couldn't find it, then we'll need to pull
800		# the state from the database
801		missing_event_ids.add(event_id)
802
803		if missing_event_ids:
804		# Now pull out the state groups for any missing events from DB
805		event_to_groups = yield self._get_state_group_for_events(missing_event_ids)
806		event_id_to_state_group.update(event_to_groups)
807
808		# State groups of old_latest_event_ids
809		old_state_groups = set(
810		event_id_to_state_group[evid] for evid in old_latest_event_ids
811		)
812
813		# State groups of new_latest_event_ids
814		new_state_groups = set(
815		event_id_to_state_group[evid] for evid in new_latest_event_ids
816		)
817
818		# If they old and new groups are the same then we don't need to do
819		# anything.
820		if old_state_groups == new_state_groups:
821		return None, None
822
823		if len(new_state_groups) == 1 and len(old_state_groups) == 1:
824		# If we're going from one state group to another, lets check if
825		# we have a delta for that transition. If we do then we can just
826		# return that.
827
828		new_state_group = next(iter(new_state_groups))
829		old_state_group = next(iter(old_state_groups))
830
831		delta_ids = state_group_deltas.get((old_state_group, new_state_group), None)
832		if delta_ids is not None:
833		# We have a delta from the existing to new current state,
834		# so lets just return that. If we happen to already have
835		# the current state in memory then lets also return that,
836		# but it doesn't matter if we don't.
837		new_state = state_groups_map.get(new_state_group)
838		return new_state, delta_ids
839
840		# Now that we have calculated new_state_groups we need to get
841		# their state IDs so we can resolve to a single state set.
842		missing_state = new_state_groups - set(state_groups_map)
843		if missing_state:
844		group_to_state = yield self._get_state_for_groups(missing_state)
845		state_groups_map.update(group_to_state)
846
847		if len(new_state_groups) == 1:
848		# If there is only one state group, then we know what the current
849		# state is.
850		return state_groups_map[new_state_groups.pop()], None
851
852		# Ok, we need to defer to the state handler to resolve our state sets.
853
854		state_groups = {sg: state_groups_map[sg] for sg in new_state_groups}
855
856		events_map = {ev.event_id: ev for ev, _ in events_context}
857
858		# We need to get the room version, which is in the create event.
859		# Normally that'd be in the database, but its also possible that we're
860		# currently trying to persist it.
861		room_version = None
862		for ev, _ in events_context:
863		if ev.type == EventTypes.Create and ev.state_key == "":
864		room_version = ev.content.get("room_version", "1")
865		break
866
867		if not room_version:
868		room_version = yield self.get_room_version(room_id)
869
870		logger.debug("calling resolve_state_groups from preserve_events")
871		res = yield self._state_resolution_handler.resolve_state_groups(
872		room_id,
873		room_version,
874		state_groups,
875		events_map,
876		state_res_store=StateResolutionStore(self),
877		)
878
879		return res.state, None
880
881		@defer.inlineCallbacks
882		def _calculate_state_delta(self, room_id, current_state):
883		"""Calculate the new state deltas for a room.
884
885		Assumes that we are only persisting events for one room at a time.
886
887		Returns:
888		tuple[list, dict] (to_delete, to_insert): where to_delete are the
889		type/state_keys to remove from current_state_events and `to_insert`
890		are the updates to current_state_events.
891		"""
892		existing_state = yield self.get_current_state_ids(room_id)
893
894		to_delete = [key for key in existing_state if key not in current_state]
895
896		to_insert = {
897		key: ev_id
898		for key, ev_id in iteritems(current_state)
899		if ev_id != existing_state.get(key)
900		}
901
902		return to_delete, to_insert
903
904		@log_function
905		def _persist_events_txn(
906		self,
907		txn,
908		events_and_contexts,
909		backfilled,
910		delete_existing=False,
911		state_delta_for_room={},
912		new_forward_extremeties={},
913		):
914		"""Insert some number of room events into the necessary database tables.
915
916		Rejected events are only inserted into the events table, the events_json table,
917		and the rejections table. Things reading from those table will need to check
918		whether the event was rejected.
919
920		Args:
921		txn (twisted.enterprise.adbapi.Connection): db connection
922		events_and_contexts (list[(EventBase, EventContext)]):
923		events to persist
924		backfilled (bool): True if the events were backfilled
925		delete_existing (bool): True to purge existing table rows for the
926		events from the database. This is useful when retrying due to
927		IntegrityError.
928		state_delta_for_room (dict[str, (list, dict)]):
929		The current-state delta for each room. For each room, a tuple
930		(to_delete, to_insert), being a list of type/state keys to be
931		removed from the current state, and a state set to be added to
932		the current state.
933		new_forward_extremeties (dict[str, list[str]]):
934		The new forward extremities for each room. For each room, a
935		list of the event ids which are the forward extremities.
936
937		"""
938		all_events_and_contexts = events_and_contexts
939
940		min_stream_order = events_and_contexts[0][0].internal_metadata.stream_ordering
941		max_stream_order = events_and_contexts[-1][0].internal_metadata.stream_ordering
942
943		self._update_forward_extremities_txn(
944		txn,
945		new_forward_extremities=new_forward_extremeties,
946		max_stream_order=max_stream_order,
947		)
948
949		# Ensure that we don't have the same event twice.
950		events_and_contexts = self._filter_events_and_contexts_for_duplicates(
951		events_and_contexts
952		)
953
954		self._update_room_depths_txn(
955		txn, events_and_contexts=events_and_contexts, backfilled=backfilled
956		)
957
958		# _update_outliers_txn filters out any events which have already been
959		# persisted, and returns the filtered list.
960		events_and_contexts = self._update_outliers_txn(
961		txn, events_and_contexts=events_and_contexts
962		)
963
964		# From this point onwards the events are only events that we haven't
965		# seen before.
966
967		if delete_existing:
968		# For paranoia reasons, we go and delete all the existing entries
969		# for these events so we can reinsert them.
970		# This gets around any problems with some tables already having
971		# entries.
972		self._delete_existing_rows_txn(txn, events_and_contexts=events_and_contexts)
973
974		self._store_event_txn(txn, events_and_contexts=events_and_contexts)
975
976		# Insert into event_to_state_groups.
977		self._store_event_state_mappings_txn(txn, events_and_contexts)
978
979		# We want to store event_auth mappings for rejected events, as they're
980		# used in state res v2.
981		# This is only necessary if the rejected event appears in an accepted
982		# event's auth chain, but its easier for now just to store them (and
983		# it doesn't take much storage compared to storing the entire event
984		# anyway).
985		self._simple_insert_many_txn(
986		txn,
987		table="event_auth",
988		values=[
989		{
990		"event_id": event.event_id,
991		"room_id": event.room_id,
992		"auth_id": auth_id,
993		}
994		for event, _ in events_and_contexts
995		for auth_id in event.auth_event_ids()
996		if event.is_state()
997		],
998		)
999
1000		# _store_rejected_events_txn filters out any events which were
1001		# rejected, and returns the filtered list.
1002		events_and_contexts = self._store_rejected_events_txn(
1003		txn, events_and_contexts=events_and_contexts
1004		)
1005
1006		# From this point onwards the events are only ones that weren't
1007		# rejected.
1008
1009		self._update_metadata_tables_txn(
1010		txn,
1011		events_and_contexts=events_and_contexts,
1012		all_events_and_contexts=all_events_and_contexts,
1013		backfilled=backfilled,
1014		)
1015
1016		# We call this last as it assumes we've inserted the events into
1017		# room_memberships, where applicable.
1018		self._update_current_state_txn(txn, state_delta_for_room, min_stream_order)
1019
1020		def _update_current_state_txn(self, txn, state_delta_by_room, stream_id):
1021		for room_id, current_state_tuple in iteritems(state_delta_by_room):
1022		to_delete, to_insert = current_state_tuple
1023
1024		# First we add entries to the current_state_delta_stream. We
1025		# do this before updating the current_state_events table so
1026		# that we can use it to calculate the `prev_event_id`. (This
1027		# allows us to not have to pull out the existing state
1028		# unnecessarily).
1029		#
1030		# The stream_id for the update is chosen to be the minimum of the stream_ids
1031		# for the batch of the events that we are persisting; that means we do not
1032		# end up in a situation where workers see events before the
1033		# current_state_delta updates.
1034		#
1035		sql = """
1036		INSERT INTO current_state_delta_stream
1037		(stream_id, room_id, type, state_key, event_id, prev_event_id)
1038		SELECT ?, ?, ?, ?, ?, (
1039		SELECT event_id FROM current_state_events
1040		WHERE room_id = ? AND type = ? AND state_key = ?
1041		)
1042		"""
1043		txn.executemany(
1044		sql,
1045		(
1046		(
1047		stream_id,
1048		room_id,
1049		etype,
1050		state_key,
1051		None,
1052		room_id,
1053		etype,
1054		state_key,
1055		)
1056		for etype, state_key in to_delete
1057		# We sanity check that we're deleting rather than updating
1058		if (etype, state_key) not in to_insert
1059		),
1060		)
1061		txn.executemany(
1062		sql,
1063		(
1064		(
1065		stream_id,
1066		room_id,
1067		etype,
1068		state_key,
1069		ev_id,
1070		room_id,
1071		etype,
1072		state_key,
1073		)
1074		for (etype, state_key), ev_id in iteritems(to_insert)
1075		),
1076		)
1077
1078		# Now we actually update the current_state_events table
1079
1080		txn.executemany(
1081		"DELETE FROM current_state_events"
1082		" WHERE room_id = ? AND type = ? AND state_key = ?",
1083		(
1084		(room_id, etype, state_key)
1085		for etype, state_key in itertools.chain(to_delete, to_insert)
1086		),
1087		)
1088
1089		# We include the membership in the current state table, hence we do
1090		# a lookup when we insert. This assumes that all events have already
1091		# been inserted into room_memberships.
1092		txn.executemany(
1093		"""INSERT INTO current_state_events
1094		(room_id, type, state_key, event_id, membership)
1095		VALUES (?, ?, ?, ?, (SELECT membership FROM room_memberships WHERE event_id = ?))
1096		""",
1097		[
1098		(room_id, key[0], key[1], ev_id, ev_id)
1099		for key, ev_id in iteritems(to_insert)
1100		],
1101		)
1102
1103		txn.call_after(
1104		self._curr_state_delta_stream_cache.entity_has_changed,
1105		room_id,
1106		stream_id,
1107		)
1108
1109		# Invalidate the various caches
1110
1111		# Figure out the changes of membership to invalidate the
1112		# `get_rooms_for_user` cache.
1113		# We find out which membership events we may have deleted
1114		# and which we have added, then we invlidate the caches for all
1115		# those users.
1116		members_changed = set(
1117		state_key
1118		for ev_type, state_key in itertools.chain(to_delete, to_insert)
1119		if ev_type == EventTypes.Member
1120		)
1121
1122		for member in members_changed:
1123		txn.call_after(
1124		self.get_rooms_for_user_with_stream_ordering.invalidate, (member,)
1125		)
1126
1127		self._invalidate_state_caches_and_stream(txn, room_id, members_changed)
1128
1129		def _update_forward_extremities_txn(
1130		self, txn, new_forward_extremities, max_stream_order
1131		):
1132		for room_id, new_extrem in iteritems(new_forward_extremities):
1133		self._simple_delete_txn(
1134		txn, table="event_forward_extremities", keyvalues={"room_id": room_id}
1135		)
1136		txn.call_after(self.get_latest_event_ids_in_room.invalidate, (room_id,))
1137
1138		self._simple_insert_many_txn(
1139		txn,
1140		table="event_forward_extremities",
1141		values=[
1142		{"event_id": ev_id, "room_id": room_id}
1143		for room_id, new_extrem in iteritems(new_forward_extremities)
1144		for ev_id in new_extrem
1145		],
1146		)
1147		# We now insert into stream_ordering_to_exterm a mapping from room_id,
1148		# new stream_ordering to new forward extremeties in the room.
1149		# This allows us to later efficiently look up the forward extremeties
1150		# for a room before a given stream_ordering
1151		self._simple_insert_many_txn(
1152		txn,
1153		table="stream_ordering_to_exterm",
1154		values=[
1155		{
1156		"room_id": room_id,
1157		"event_id": event_id,
1158		"stream_ordering": max_stream_order,
1159		}
1160		for room_id, new_extrem in iteritems(new_forward_extremities)
1161		for event_id in new_extrem
1162		],
1163		)
1164
1165		@classmethod
1166		def _filter_events_and_contexts_for_duplicates(cls, events_and_contexts):
1167		"""Ensure that we don't have the same event twice.
1168
1169		Pick the earliest non-outlier if there is one, else the earliest one.
1170
1171		Args:
1172		events_and_contexts (list[(EventBase, EventContext)]):
1173		Returns:
1174		list[(EventBase, EventContext)]: filtered list
1175		"""
1176		new_events_and_contexts = OrderedDict()
1177		for event, context in events_and_contexts:
1178		prev_event_context = new_events_and_contexts.get(event.event_id)
1179		if prev_event_context:
1180		if not event.internal_metadata.is_outlier():
1181		if prev_event_context[0].internal_metadata.is_outlier():
1182		# To ensure correct ordering we pop, as OrderedDict is
1183		# ordered by first insertion.
1184		new_events_and_contexts.pop(event.event_id, None)
1185		new_events_and_contexts[event.event_id] = (event, context)
1186		else:
1187		new_events_and_contexts[event.event_id] = (event, context)
1188		return list(new_events_and_contexts.values())
1189
1190		def _update_room_depths_txn(self, txn, events_and_contexts, backfilled):
1191		"""Update min_depth for each room
1192
1193		Args:
1194		txn (twisted.enterprise.adbapi.Connection): db connection
1195		events_and_contexts (list[(EventBase, EventContext)]): events
1196		we are persisting
1197		backfilled (bool): True if the events were backfilled
1198		"""
1199		depth_updates = {}
1200		for event, context in events_and_contexts:
1201		# Remove the any existing cache entries for the event_ids
1202		txn.call_after(self._invalidate_get_event_cache, event.event_id)
1203		if not backfilled:
1204		txn.call_after(
1205		self._events_stream_cache.entity_has_changed,
1206		event.room_id,
1207		event.internal_metadata.stream_ordering,
1208		)
1209
1210		if not event.internal_metadata.is_outlier() and not context.rejected:
1211		depth_updates[event.room_id] = max(
1212		event.depth, depth_updates.get(event.room_id, event.depth)
1213		)
1214
1215		for room_id, depth in iteritems(depth_updates):
1216		self._update_min_depth_for_room_txn(txn, room_id, depth)
1217
1218		def _update_outliers_txn(self, txn, events_and_contexts):
1219		"""Update any outliers with new event info.
1220
1221		This turns outliers into ex-outliers (unless the new event was
1222		rejected).
1223
1224		Args:
1225		txn (twisted.enterprise.adbapi.Connection): db connection
1226		events_and_contexts (list[(EventBase, EventContext)]): events
1227		we are persisting
1228
1229		Returns:
1230		list[(EventBase, EventContext)] new list, without events which
1231		are already in the events table.
1232		"""
1233		txn.execute(
1234		"SELECT event_id, outlier FROM events WHERE event_id in (%s)"
1235		% (",".join(["?"] * len(events_and_contexts)),),
1236		[event.event_id for event, _ in events_and_contexts],
1237		)
1238
1239		have_persisted = {event_id: outlier for event_id, outlier in txn}
1240
1241		to_remove = set()
1242		for event, context in events_and_contexts:
1243		if event.event_id not in have_persisted:
1244		continue
1245
1246		to_remove.add(event)
1247
1248		if context.rejected:
1249		# If the event is rejected then we don't care if the event
1250		# was an outlier or not.
1251		continue
1252
1253		outlier_persisted = have_persisted[event.event_id]
1254		if not event.internal_metadata.is_outlier() and outlier_persisted:
1255		# We received a copy of an event that we had already stored as
1256		# an outlier in the database. We now have some state at that
1257		# so we need to update the state_groups table with that state.
1258
1259		# insert into event_to_state_groups.
1260		try:
1261		self._store_event_state_mappings_txn(txn, ((event, context),))
1262		except Exception:
1263		logger.exception("")
1264		raise
1265
1266		metadata_json = encode_json(event.internal_metadata.get_dict())
1267
1268		sql = (
1269		"UPDATE event_json SET internal_metadata = ?" " WHERE event_id = ?"
1270		)
1271		txn.execute(sql, (metadata_json, event.event_id))
1272
1273		# Add an entry to the ex_outlier_stream table to replicate the
1274		# change in outlier status to our workers.
1275		stream_order = event.internal_metadata.stream_ordering
1276		state_group_id = context.state_group
1277		self._simple_insert_txn(
1278		txn,
1279		table="ex_outlier_stream",
1280		values={
1281		"event_stream_ordering": stream_order,
1282		"event_id": event.event_id,
1283		"state_group": state_group_id,
1284		},
1285		)
1286
1287		sql = "UPDATE events SET outlier = ?" " WHERE event_id = ?"
1288		txn.execute(sql, (False, event.event_id))
1289
1290		# Update the event_backward_extremities table now that this
1291		# event isn't an outlier any more.
1292		self._update_backward_extremeties(txn, [event])
1293
1294		return [ec for ec in events_and_contexts if ec[0] not in to_remove]
1295
1296		@classmethod
1297		def _delete_existing_rows_txn(cls, txn, events_and_contexts):
1298		if not events_and_contexts:
1299		# nothing to do here
1300		return
1301
1302		logger.info("Deleting existing")
1303
1304		for table in (
1305		"events",
1306		"event_auth",
1307		"event_json",
1308		"event_edges",
1309		"event_forward_extremities",
1310		"event_reference_hashes",
1311		"event_search",
1312		"event_to_state_groups",
1313		"local_invites",
1314		"state_events",
1315		"rejections",
1316		"redactions",
1317		"room_memberships",
1318		):
1319		txn.executemany(
1320		"DELETE FROM %s WHERE event_id = ?" % (table,),
1321		[(ev.event_id,) for ev, _ in events_and_contexts],
1322		)
1323
1324		for table in ("event_push_actions",):
1325		txn.executemany(
1326		"DELETE FROM %s WHERE room_id = ? AND event_id = ?" % (table,),
1327		[(ev.room_id, ev.event_id) for ev, _ in events_and_contexts],
1328		)
1329
1330		def _store_event_txn(self, txn, events_and_contexts):
1331		"""Insert new events into the event and event_json tables
1332
1333		Args:
1334		txn (twisted.enterprise.adbapi.Connection): db connection
1335		events_and_contexts (list[(EventBase, EventContext)]): events
1336		we are persisting
1337		"""
1338
1339		if not events_and_contexts:
1340		# nothing to do here
1341		return
1342
1343		def event_dict(event):
1344		d = event.get_dict()
1345		d.pop("redacted", None)
1346		d.pop("redacted_because", None)
1347		return d
1348
1349		self._simple_insert_many_txn(
1350		txn,
1351		table="event_json",
1352		values=[
1353		{
1354		"event_id": event.event_id,
1355		"room_id": event.room_id,
1356		"internal_metadata": encode_json(
1357		event.internal_metadata.get_dict()
1358		),
1359		"json": encode_json(event_dict(event)),
1360		"format_version": event.format_version,
1361		}
1362		for event, _ in events_and_contexts
1363		],
1364		)
1365
1366		self._simple_insert_many_txn(
1367		txn,
1368		table="events",
1369		values=[
1370		{
1371		"stream_ordering": event.internal_metadata.stream_ordering,
1372		"topological_ordering": event.depth,
1373		"depth": event.depth,
1374		"event_id": event.event_id,
1375		"room_id": event.room_id,
1376		"type": event.type,
1377		"processed": True,
1378		"outlier": event.internal_metadata.is_outlier(),
1379		"origin_server_ts": int(event.origin_server_ts),
1380		"received_ts": self._clock.time_msec(),
1381		"sender": event.sender,
1382		"contains_url": (
1383		"url" in event.content
1384		and isinstance(event.content["url"], text_type)
1385		),
1386		}
1387		for event, _ in events_and_contexts
1388		],
1389		)
1390
1391		for event, _ in events_and_contexts:
1392		if not event.internal_metadata.is_redacted():
1393		# If we're persisting an unredacted event we go and ensure
1394		# that we mark any redactions that reference this event as
1395		# requiring censoring.
1396		self._simple_update_txn(
1397		txn,
1398		table="redactions",
1399		keyvalues={"redacts": event.event_id},
1400		updatevalues={"have_censored": False},
1401		)
1402
1403		def _store_rejected_events_txn(self, txn, events_and_contexts):
1404		"""Add rows to the 'rejections' table for received events which were
1405		rejected
1406
1407		Args:
1408		txn (twisted.enterprise.adbapi.Connection): db connection
1409		events_and_contexts (list[(EventBase, EventContext)]): events
1410		we are persisting
1411
1412		Returns:
1413		list[(EventBase, EventContext)] new list, without the rejected
1414		events.
1415		"""
1416		# Remove the rejected events from the list now that we've added them
1417		# to the events table and the events_json table.
1418		to_remove = set()
1419		for event, context in events_and_contexts:
1420		if context.rejected:
1421		# Insert the event_id into the rejections table
1422		self._store_rejections_txn(txn, event.event_id, context.rejected)
1423		to_remove.add(event)
1424
1425		return [ec for ec in events_and_contexts if ec[0] not in to_remove]
1426
1427		def _update_metadata_tables_txn(
1428		self, txn, events_and_contexts, all_events_and_contexts, backfilled
1429		):
1430		"""Update all the miscellaneous tables for new events
1431
1432		Args:
1433		txn (twisted.enterprise.adbapi.Connection): db connection
1434		events_and_contexts (list[(EventBase, EventContext)]): events
1435		we are persisting
1436		all_events_and_contexts (list[(EventBase, EventContext)]): all
1437		events that we were going to persist. This includes events
1438		we've already persisted, etc, that wouldn't appear in
1439		events_and_context.
1440		backfilled (bool): True if the events were backfilled
1441		"""
1442
1443		# Insert all the push actions into the event_push_actions table.
1444		self._set_push_actions_for_event_and_users_txn(
1445		txn,
1446		events_and_contexts=events_and_contexts,
1447		all_events_and_contexts=all_events_and_contexts,
1448		)
1449
1450		if not events_and_contexts:
1451		# nothing to do here
1452		return
1453
1454		for event, context in events_and_contexts:
1455		if event.type == EventTypes.Redaction and event.redacts is not None:
1456		# Remove the entries in the event_push_actions table for the
1457		# redacted event.
1458		self._remove_push_actions_for_event_id_txn(
1459		txn, event.room_id, event.redacts
1460		)
1461
1462		# Remove from relations table.
1463		self._handle_redaction(txn, event.redacts)
1464
1465		# Update the event_forward_extremities, event_backward_extremities and
1466		# event_edges tables.
1467		self._handle_mult_prev_events(
1468		txn, events=[event for event, _ in events_and_contexts]
1469		)
1470
1471		for event, _ in events_and_contexts:
1472		if event.type == EventTypes.Name:
1473		# Insert into the event_search table.
1474		self._store_room_name_txn(txn, event)
1475		elif event.type == EventTypes.Topic:
1476		# Insert into the event_search table.
1477		self._store_room_topic_txn(txn, event)
1478		elif event.type == EventTypes.Message:
1479		# Insert into the event_search table.
1480		self._store_room_message_txn(txn, event)
1481		elif event.type == EventTypes.Redaction:
1482		# Insert into the redactions table.
1483		self._store_redaction(txn, event)
1484
1485		self._handle_event_relations(txn, event)
1486
1487		# Insert into the room_memberships table.
1488		self._store_room_members_txn(
1489		txn,
1490		[
1491		event
1492		for event, _ in events_and_contexts
1493		if event.type == EventTypes.Member
1494		],
1495		backfilled=backfilled,
1496		)
1497
1498		# Insert event_reference_hashes table.
1499		self._store_event_reference_hashes_txn(
1500		txn, [event for event, _ in events_and_contexts]
1501		)
1502
1503		state_events_and_contexts = [
1504		ec for ec in events_and_contexts if ec[0].is_state()
1505		]
1506
1507		state_values = []
1508		for event, context in state_events_and_contexts:
1509		vals = {
1510		"event_id": event.event_id,
1511		"room_id": event.room_id,
1512		"type": event.type,
1513		"state_key": event.state_key,
1514		}
1515
1516		# TODO: How does this work with backfilling?
1517		if hasattr(event, "replaces_state"):
1518		vals["prev_state"] = event.replaces_state
1519
1520		state_values.append(vals)
1521
1522		self._simple_insert_many_txn(txn, table="state_events", values=state_values)
1523
1524		# Prefill the event cache
1525		self._add_to_cache(txn, events_and_contexts)
1526
1527		def _add_to_cache(self, txn, events_and_contexts):
1528		to_prefill = []
1529
1530		rows = []
1531		N = 200
1532		for i in range(0, len(events_and_contexts), N):
1533		ev_map = {e[0].event_id: e[0] for e in events_and_contexts[i : i + N]}
1534		if not ev_map:
1535		break
1536
1537		sql = (
1538		"SELECT "
1539		" e.event_id as event_id, "
1540		" r.redacts as redacts,"
1541		" rej.event_id as rejects "
1542		" FROM events as e"
1543		" LEFT JOIN rejections as rej USING (event_id)"
1544		" LEFT JOIN redactions as r ON e.event_id = r.redacts"
1545		" WHERE e.event_id IN (%s)"
1546		) % (",".join(["?"] * len(ev_map)),)
1547
1548		txn.execute(sql, list(ev_map))
1549		rows = self.cursor_to_dict(txn)
1550		for row in rows:
1551		event = ev_map[row["event_id"]]
1552		if not row["rejects"] and not row["redacts"]:
1553		to_prefill.append(
1554		_EventCacheEntry(event=event, redacted_event=None)
1555		)
1556
1557		def prefill():
1558		for cache_entry in to_prefill:
1559		self._get_event_cache.prefill((cache_entry[0].event_id,), cache_entry)
1560
1561		txn.call_after(prefill)
1562
1563		def _store_redaction(self, txn, event):
1564		# invalidate the cache for the redacted event
1565		txn.call_after(self._invalidate_get_event_cache, event.redacts)
1566
1567		self._simple_insert_txn(
1568		txn,
1569		table="redactions",
1570		values={
1571		"event_id": event.event_id,
1572		"redacts": event.redacts,
1573		"received_ts": self._clock.time_msec(),
1574		},
1575		)
1576
1577		@defer.inlineCallbacks
1578		def _censor_redactions(self):
1579		"""Censors all redactions older than the configured period that haven't
1580		been censored yet.
1581
1582		By censor we mean update the event_json table with the redacted event.
1583
1584		Returns:
1585		Deferred
1586		"""
1587
1588		if self.hs.config.redaction_retention_period is None:
1589		return
1590
1591		before_ts = self._clock.time_msec() - self.hs.config.redaction_retention_period
1592
1593		# We fetch all redactions that:
1594		# 1. point to an event we have,
1595		# 2. has a received_ts from before the cut off, and
1596		# 3. we haven't yet censored.
1597		#
1598		# This is limited to 100 events to ensure that we don't try and do too
1599		# much at once. We'll get called again so this should eventually catch
1600		# up.
1601		sql = """
1602		SELECT redactions.event_id, redacts FROM redactions
1603		LEFT JOIN events AS original_event ON (
1604		redacts = original_event.event_id
1605		)
1606		WHERE NOT have_censored
1607		AND redactions.received_ts <= ?
1608		ORDER BY redactions.received_ts ASC
1609		LIMIT ?
1610		"""
1611
1612		rows = yield self._execute(
1613		"_censor_redactions_fetch", None, sql, before_ts, 100
1614		)
1615
1616		updates = []
1617
1618		for redaction_id, event_id in rows:
1619		redaction_event = yield self.get_event(redaction_id, allow_none=True)
1620		original_event = yield self.get_event(
1621		event_id, allow_rejected=True, allow_none=True
1622		)
1623
1624		# The SQL above ensures that we have both the redaction and
1625		# original event, so if the `get_event` calls return None it
1626		# means that the redaction wasn't allowed. Either way we know that
1627		# the result won't change so we mark the fact that we've checked.
1628		if (
1629		redaction_event
1630		and original_event
1631		and original_event.internal_metadata.is_redacted()
1632		):
1633		# Redaction was allowed
1634		pruned_json = encode_canonical_json(
1635		prune_event_dict(original_event.get_dict())
1636		)
1637		else:
1638		# Redaction wasn't allowed
1639		pruned_json = None
1640
1641		updates.append((redaction_id, event_id, pruned_json))
1642
1643		def _update_censor_txn(txn):
1644		for redaction_id, event_id, pruned_json in updates:
1645		if pruned_json:
1646		self._simple_update_one_txn(
1647		txn,
1648		table="event_json",
1649		keyvalues={"event_id": event_id},
1650		updatevalues={"json": pruned_json},
1651		)
1652
1653		self._simple_update_one_txn(
1654		txn,
1655		table="redactions",
1656		keyvalues={"event_id": redaction_id},
1657		updatevalues={"have_censored": True},
1658		)
1659
1660		yield self.runInteraction("_update_censor_txn", _update_censor_txn)
1661
1662		@defer.inlineCallbacks
1663		def count_daily_messages(self):
1664		"""
1665		Returns an estimate of the number of messages sent in the last day.
1666
1667		If it has been significantly less or more than one day since the last
1668		call to this function, it will return None.
1669		"""
1670
1671		def _count_messages(txn):
1672		sql = """
1673		SELECT COALESCE(COUNT(*), 0) FROM events
1674		WHERE type = 'm.room.message'
1675		AND stream_ordering > ?
1676		"""
1677		txn.execute(sql, (self.stream_ordering_day_ago,))
1678		count, = txn.fetchone()
1679		return count
1680
1681		ret = yield self.runInteraction("count_messages", _count_messages)
1682		return ret
1683
1684		@defer.inlineCallbacks
1685		def count_daily_sent_messages(self):
1686		def _count_messages(txn):
1687		# This is good enough as if you have silly characters in your own
1688		# hostname then thats your own fault.
1689		like_clause = "%:" + self.hs.hostname
1690
1691		sql = """
1692		SELECT COALESCE(COUNT(*), 0) FROM events
1693		WHERE type = 'm.room.message'
1694		AND sender LIKE ?
1695		AND stream_ordering > ?
1696		"""
1697
1698		txn.execute(sql, (like_clause, self.stream_ordering_day_ago))
1699		count, = txn.fetchone()
1700		return count
1701
1702		ret = yield self.runInteraction("count_daily_sent_messages", _count_messages)
1703		return ret
1704
1705		@defer.inlineCallbacks
1706		def count_daily_active_rooms(self):
1707		def _count(txn):
1708		sql = """
1709		SELECT COALESCE(COUNT(DISTINCT room_id), 0) FROM events
1710		WHERE type = 'm.room.message'
1711		AND stream_ordering > ?
1712		"""
1713		txn.execute(sql, (self.stream_ordering_day_ago,))
1714		count, = txn.fetchone()
1715		return count
1716
1717		ret = yield self.runInteraction("count_daily_active_rooms", _count)
1718		return ret
1719
1720		def get_current_backfill_token(self):
1721		"""The current minimum token that backfilled events have reached"""
1722		return -self._backfill_id_gen.get_current_token()
1723
1724		def get_current_events_token(self):
1725		"""The current maximum token that events have reached"""
1726		return self._stream_id_gen.get_current_token()
1727
1728		def get_all_new_forward_event_rows(self, last_id, current_id, limit):
1729		if last_id == current_id:
1730		return defer.succeed([])
1731
1732		def get_all_new_forward_event_rows(txn):
1733		sql = (
1734		"SELECT e.stream_ordering, e.event_id, e.room_id, e.type,"
1735		" state_key, redacts, relates_to_id"
1736		" FROM events AS e"
1737		" LEFT JOIN redactions USING (event_id)"
1738		" LEFT JOIN state_events USING (event_id)"
1739		" LEFT JOIN event_relations USING (event_id)"
1740		" WHERE ? < stream_ordering AND stream_ordering <= ?"
1741		" ORDER BY stream_ordering ASC"
1742		" LIMIT ?"
1743		)
1744		txn.execute(sql, (last_id, current_id, limit))
1745		new_event_updates = txn.fetchall()
1746
1747		if len(new_event_updates) == limit:
1748		upper_bound = new_event_updates[-1][0]
1749		else:
1750		upper_bound = current_id
1751
1752		sql = (
1753		"SELECT event_stream_ordering, e.event_id, e.room_id, e.type,"
1754		" state_key, redacts, relates_to_id"
1755		" FROM events AS e"
1756		" INNER JOIN ex_outlier_stream USING (event_id)"
1757		" LEFT JOIN redactions USING (event_id)"
1758		" LEFT JOIN state_events USING (event_id)"
1759		" LEFT JOIN event_relations USING (event_id)"
1760		" WHERE ? < event_stream_ordering"
1761		" AND event_stream_ordering <= ?"
1762		" ORDER BY event_stream_ordering DESC"
1763		)
1764		txn.execute(sql, (last_id, upper_bound))
1765		new_event_updates.extend(txn)
1766
1767		return new_event_updates
1768
1769		return self.runInteraction(
1770		"get_all_new_forward_event_rows", get_all_new_forward_event_rows
1771		)
1772
1773		def get_all_new_backfill_event_rows(self, last_id, current_id, limit):
1774		if last_id == current_id:
1775		return defer.succeed([])
1776
1777		def get_all_new_backfill_event_rows(txn):
1778		sql = (
1779		"SELECT -e.stream_ordering, e.event_id, e.room_id, e.type,"
1780		" state_key, redacts, relates_to_id"
1781		" FROM events AS e"
1782		" LEFT JOIN redactions USING (event_id)"
1783		" LEFT JOIN state_events USING (event_id)"
1784		" LEFT JOIN event_relations USING (event_id)"
1785		" WHERE ? > stream_ordering AND stream_ordering >= ?"
1786		" ORDER BY stream_ordering ASC"
1787		" LIMIT ?"
1788		)
1789		txn.execute(sql, (-last_id, -current_id, limit))
1790		new_event_updates = txn.fetchall()
1791
1792		if len(new_event_updates) == limit:
1793		upper_bound = new_event_updates[-1][0]
1794		else:
1795		upper_bound = current_id
1796
1797		sql = (
1798		"SELECT -event_stream_ordering, e.event_id, e.room_id, e.type,"
1799		" state_key, redacts, relates_to_id"
1800		" FROM events AS e"
1801		" INNER JOIN ex_outlier_stream USING (event_id)"
1802		" LEFT JOIN redactions USING (event_id)"
1803		" LEFT JOIN state_events USING (event_id)"
1804		" LEFT JOIN event_relations USING (event_id)"
1805		" WHERE ? > event_stream_ordering"
1806		" AND event_stream_ordering >= ?"
1807		" ORDER BY event_stream_ordering DESC"
1808		)
1809		txn.execute(sql, (-last_id, -upper_bound))
1810		new_event_updates.extend(txn.fetchall())
1811
1812		return new_event_updates
1813
1814		return self.runInteraction(
1815		"get_all_new_backfill_event_rows", get_all_new_backfill_event_rows
1816		)
1817
1818		@cached(num_args=5, max_entries=10)
1819		def get_all_new_events(
1820		self,
1821		last_backfill_id,
1822		last_forward_id,
1823		current_backfill_id,
1824		current_forward_id,
1825		limit,
1826		):
1827		"""Get all the new events that have arrived at the server either as
1828		new events or as backfilled events"""
1829		have_backfill_events = last_backfill_id != current_backfill_id
1830		have_forward_events = last_forward_id != current_forward_id
1831
1832		if not have_backfill_events and not have_forward_events:
1833		return defer.succeed(AllNewEventsResult([], [], [], [], []))
1834
1835		def get_all_new_events_txn(txn):
1836		sql = (
1837		"SELECT e.stream_ordering, e.event_id, e.room_id, e.type,"
1838		" state_key, redacts"
1839		" FROM events AS e"
1840		" LEFT JOIN redactions USING (event_id)"
1841		" LEFT JOIN state_events USING (event_id)"
1842		" WHERE ? < stream_ordering AND stream_ordering <= ?"
1843		" ORDER BY stream_ordering ASC"
1844		" LIMIT ?"
1845		)
1846		if have_forward_events:
1847		txn.execute(sql, (last_forward_id, current_forward_id, limit))
1848		new_forward_events = txn.fetchall()
1849
1850		if len(new_forward_events) == limit:
1851		upper_bound = new_forward_events[-1][0]
1852		else:
1853		upper_bound = current_forward_id
1854
1855		sql = (
1856		"SELECT event_stream_ordering, event_id, state_group"
1857		" FROM ex_outlier_stream"
1858		" WHERE ? > event_stream_ordering"
1859		" AND event_stream_ordering >= ?"
1860		" ORDER BY event_stream_ordering DESC"
1861		)
1862		txn.execute(sql, (last_forward_id, upper_bound))
1863		forward_ex_outliers = txn.fetchall()
1864		else:
1865		new_forward_events = []
1866		forward_ex_outliers = []
1867
1868		sql = (
1869		"SELECT -e.stream_ordering, e.event_id, e.room_id, e.type,"
1870		" state_key, redacts"
1871		" FROM events AS e"
1872		" LEFT JOIN redactions USING (event_id)"
1873		" LEFT JOIN state_events USING (event_id)"
1874		" WHERE ? > stream_ordering AND stream_ordering >= ?"
1875		" ORDER BY stream_ordering DESC"
1876		" LIMIT ?"
1877		)
1878		if have_backfill_events:
1879		txn.execute(sql, (-last_backfill_id, -current_backfill_id, limit))
1880		new_backfill_events = txn.fetchall()
1881
1882		if len(new_backfill_events) == limit:
1883		upper_bound = new_backfill_events[-1][0]
1884		else:
1885		upper_bound = current_backfill_id
1886
1887		sql = (
1888		"SELECT -event_stream_ordering, event_id, state_group"
1889		" FROM ex_outlier_stream"
1890		" WHERE ? > event_stream_ordering"
1891		" AND event_stream_ordering >= ?"
1892		" ORDER BY event_stream_ordering DESC"
1893		)
1894		txn.execute(sql, (-last_backfill_id, -upper_bound))
1895		backward_ex_outliers = txn.fetchall()
1896		else:
1897		new_backfill_events = []
1898		backward_ex_outliers = []
1899
1900		return AllNewEventsResult(
1901		new_forward_events,
1902		new_backfill_events,
1903		forward_ex_outliers,
1904		backward_ex_outliers,
1905		)
1906
1907		return self.runInteraction("get_all_new_events", get_all_new_events_txn)
1908
1909		def purge_history(self, room_id, token, delete_local_events):
1910		"""Deletes room history before a certain point
1911
1912		Args:
1913		room_id (str):
1914
1915		token (str): A topological token to delete events before
1916
1917		delete_local_events (bool):
1918		if True, we will delete local events as well as remote ones
1919		(instead of just marking them as outliers and deleting their
1920		state groups).
1921		"""
1922
1923		return self.runInteraction(
1924		"purge_history",
1925		self._purge_history_txn,
1926		room_id,
1927		token,
1928		delete_local_events,
1929		)
1930
1931		def _purge_history_txn(self, txn, room_id, token_str, delete_local_events):
1932		token = RoomStreamToken.parse(token_str)
1933
1934		# Tables that should be pruned:
1935		# event_auth
1936		# event_backward_extremities
1937		# event_edges
1938		# event_forward_extremities
1939		# event_json
1940		# event_push_actions
1941		# event_reference_hashes
1942		# event_search
1943		# event_to_state_groups
1944		# events
1945		# rejections
1946		# room_depth
1947		# state_groups
1948		# state_groups_state
1949
1950		# we will build a temporary table listing the events so that we don't
1951		# have to keep shovelling the list back and forth across the
1952		# connection. Annoyingly the python sqlite driver commits the
1953		# transaction on CREATE, so let's do this first.
1954		#
1955		# furthermore, we might already have the table from a previous (failed)
1956		# purge attempt, so let's drop the table first.
1957
1958		txn.execute("DROP TABLE IF EXISTS events_to_purge")
1959
1960		txn.execute(
1961		"CREATE TEMPORARY TABLE events_to_purge ("
1962		" event_id TEXT NOT NULL,"
1963		" should_delete BOOLEAN NOT NULL"
1964		")"
1965		)
1966
1967		# First ensure that we're not about to delete all the forward extremeties
1968		txn.execute(
1969		"SELECT e.event_id, e.depth FROM events as e "
1970		"INNER JOIN event_forward_extremities as f "
1971		"ON e.event_id = f.event_id "
1972		"AND e.room_id = f.room_id "
1973		"WHERE f.room_id = ?",
1974		(room_id,),
1975		)
1976		rows = txn.fetchall()
1977		max_depth = max(row[1] for row in rows)
1978
1979		if max_depth < token.topological:
1980		# We need to ensure we don't delete all the events from the database
1981		# otherwise we wouldn't be able to send any events (due to not
1982		# having any backwards extremeties)
1983		raise SynapseError(
1984		400, "topological_ordering is greater than forward extremeties"
1985		)
1986
1987		logger.info("[purge] looking for events to delete")
1988
1989		should_delete_expr = "state_key IS NULL"
1990		should_delete_params = ()
1991		if not delete_local_events:
1992		should_delete_expr += " AND event_id NOT LIKE ?"
1993
1994		# We include the parameter twice since we use the expression twice
1995		should_delete_params += ("%:" + self.hs.hostname, "%:" + self.hs.hostname)
1996
1997		should_delete_params += (room_id, token.topological)
1998
1999		# Note that we insert events that are outliers and aren't going to be
2000		# deleted, as nothing will happen to them.
2001		txn.execute(
2002		"INSERT INTO events_to_purge"
2003		" SELECT event_id, %s"
2004		" FROM events AS e LEFT JOIN state_events USING (event_id)"
2005		" WHERE (NOT outlier OR (%s)) AND e.room_id = ? AND topological_ordering < ?"
2006		% (should_delete_expr, should_delete_expr),
2007		should_delete_params,
2008		)
2009
2010		# We create the indices after insertion as that's a lot faster.
2011
2012		# create an index on should_delete because later we'll be looking for
2013		# the should_delete / shouldn't_delete subsets
2014		txn.execute(
2015		"CREATE INDEX events_to_purge_should_delete"
2016		" ON events_to_purge(should_delete)"
2017		)
2018
2019		# We do joins against events_to_purge for e.g. calculating state
2020		# groups to purge, etc., so lets make an index.
2021		txn.execute("CREATE INDEX events_to_purge_id" " ON events_to_purge(event_id)")
2022
2023		txn.execute("SELECT event_id, should_delete FROM events_to_purge")
2024		event_rows = txn.fetchall()
2025		logger.info(
2026		"[purge] found %i events before cutoff, of which %i can be deleted",
2027		len(event_rows),
2028		sum(1 for e in event_rows if e[1]),
2029		)
2030
2031		logger.info("[purge] Finding new backward extremities")
2032
2033		# We calculate the new entries for the backward extremeties by finding
2034		# events to be purged that are pointed to by events we're not going to
2035		# purge.
2036		txn.execute(
2037		"SELECT DISTINCT e.event_id FROM events_to_purge AS e"
2038		" INNER JOIN event_edges AS ed ON e.event_id = ed.prev_event_id"
2039		" LEFT JOIN events_to_purge AS ep2 ON ed.event_id = ep2.event_id"
2040		" WHERE ep2.event_id IS NULL"
2041		)
2042		new_backwards_extrems = txn.fetchall()
2043
2044		logger.info("[purge] replacing backward extremities: %r", new_backwards_extrems)
2045
2046		txn.execute(
2047		"DELETE FROM event_backward_extremities WHERE room_id = ?", (room_id,)
2048		)
2049
2050		# Update backward extremeties
2051		txn.executemany(
2052		"INSERT INTO event_backward_extremities (room_id, event_id)"
2053		" VALUES (?, ?)",
2054		[(room_id, event_id) for event_id, in new_backwards_extrems],
2055		)
2056
2057		logger.info("[purge] finding redundant state groups")
2058
2059		# Get all state groups that are referenced by events that are to be
2060		# deleted. We then go and check if they are referenced by other events
2061		# or state groups, and if not we delete them.
2062		txn.execute(
2063		"""
2064		SELECT DISTINCT state_group FROM events_to_purge
2065		INNER JOIN event_to_state_groups USING (event_id)
2066		"""
2067		)
2068
2069		referenced_state_groups = set(sg for sg, in txn)
2070		logger.info(
2071		"[purge] found %i referenced state groups", len(referenced_state_groups)
2072		)
2073
2074		logger.info("[purge] finding state groups that can be deleted")
2075
2076		_ = self._find_unreferenced_groups_during_purge(txn, referenced_state_groups)
2077		state_groups_to_delete, remaining_state_groups = _
2078
2079		logger.info(
2080		"[purge] found %i state groups to delete", len(state_groups_to_delete)
2081		)
2082
2083		logger.info(
2084		"[purge] de-delta-ing %i remaining state groups",
2085		len(remaining_state_groups),
2086		)
2087
2088		# Now we turn the state groups that reference to-be-deleted state
2089		# groups to non delta versions.
2090		for sg in remaining_state_groups:
2091		logger.info("[purge] de-delta-ing remaining state group %s", sg)
2092		curr_state = self._get_state_groups_from_groups_txn(txn, [sg])
2093		curr_state = curr_state[sg]
2094
2095		self._simple_delete_txn(
2096		txn, table="state_groups_state", keyvalues={"state_group": sg}
2097		)
2098
2099		self._simple_delete_txn(
2100		txn, table="state_group_edges", keyvalues={"state_group": sg}
2101		)
2102
2103		self._simple_insert_many_txn(
2104		txn,
2105		table="state_groups_state",
2106		values=[
2107		{
2108		"state_group": sg,
2109		"room_id": room_id,
2110		"type": key[0],
2111		"state_key": key[1],
2112		"event_id": state_id,
2113		}
2114		for key, state_id in iteritems(curr_state)
2115		],
2116		)
2117
2118		logger.info("[purge] removing redundant state groups")
2119		txn.executemany(
2120		"DELETE FROM state_groups_state WHERE state_group = ?",
2121		((sg,) for sg in state_groups_to_delete),
2122		)
2123		txn.executemany(
2124		"DELETE FROM state_groups WHERE id = ?",
2125		((sg,) for sg in state_groups_to_delete),
2126		)
2127
2128		logger.info("[purge] removing events from event_to_state_groups")
2129		txn.execute(
2130		"DELETE FROM event_to_state_groups "
2131		"WHERE event_id IN (SELECT event_id from events_to_purge)"
2132		)
2133		for event_id, _ in event_rows:
2134		txn.call_after(self._get_state_group_for_event.invalidate, (event_id,))
2135
2136		# Delete all remote non-state events
2137		for table in (
2138		"events",
2139		"event_json",
2140		"event_auth",
2141		"event_edges",
2142		"event_forward_extremities",
2143		"event_reference_hashes",
2144		"event_search",
2145		"rejections",
2146		):
2147		logger.info("[purge] removing events from %s", table)
2148
2149		txn.execute(
2150		"DELETE FROM %s WHERE event_id IN ("
2151		" SELECT event_id FROM events_to_purge WHERE should_delete"
2152		")" % (table,)
2153		)
2154
2155		# event_push_actions lacks an index on event_id, and has one on
2156		# (room_id, event_id) instead.
2157		for table in ("event_push_actions",):
2158		logger.info("[purge] removing events from %s", table)
2159
2160		txn.execute(
2161		"DELETE FROM %s WHERE room_id = ? AND event_id IN ("
2162		" SELECT event_id FROM events_to_purge WHERE should_delete"
2163		")" % (table,),
2164		(room_id,),
2165		)
2166
2167		# Mark all state and own events as outliers
2168		logger.info("[purge] marking remaining events as outliers")
2169		txn.execute(
2170		"UPDATE events SET outlier = ?"
2171		" WHERE event_id IN ("
2172		" SELECT event_id FROM events_to_purge "
2173		" WHERE NOT should_delete"
2174		")",
2175		(True,),
2176		)
2177
2178		# synapse tries to take out an exclusive lock on room_depth whenever it
2179		# persists events (because upsert), and once we run this update, we
2180		# will block that for the rest of our transaction.
2181		#
2182		# So, let's stick it at the end so that we don't block event
2183		# persistence.
2184		#
2185		# We do this by calculating the minimum depth of the backwards
2186		# extremities. However, the events in event_backward_extremities
2187		# are ones we don't have yet so we need to look at the events that
2188		# point to it via event_edges table.
2189		txn.execute(
2190		"""
2191		SELECT COALESCE(MIN(depth), 0)
2192		FROM event_backward_extremities AS eb
2193		INNER JOIN event_edges AS eg ON eg.prev_event_id = eb.event_id
2194		INNER JOIN events AS e ON e.event_id = eg.event_id
2195		WHERE eb.room_id = ?
2196		""",
2197		(room_id,),
2198		)
2199		min_depth, = txn.fetchone()
2200
2201		logger.info("[purge] updating room_depth to %d", min_depth)
2202
2203		txn.execute(
2204		"UPDATE room_depth SET min_depth = ? WHERE room_id = ?",
2205		(min_depth, room_id),
2206		)
2207
2208		# finally, drop the temp table. this will commit the txn in sqlite,
2209		# so make sure to keep this actually last.
2210		txn.execute("DROP TABLE events_to_purge")
2211
2212		logger.info("[purge] done")
2213
2214		def _find_unreferenced_groups_during_purge(self, txn, state_groups):
2215		"""Used when purging history to figure out which state groups can be
2216		deleted and which need to be de-delta'ed (due to one of its prev groups
2217		being scheduled for deletion).
2218
2219		Args:
2220		txn
2221		state_groups (set[int]): Set of state groups referenced by events
2222		that are going to be deleted.
2223
2224		Returns:
2225		tuple[set[int], set[int]]: The set of state groups that can be
2226		deleted and the set of state groups that need to be de-delta'ed
2227		"""
2228		# Graph of state group -> previous group
2229		graph = {}
2230
2231		# Set of events that we have found to be referenced by events
2232		referenced_groups = set()
2233
2234		# Set of state groups we've already seen
2235		state_groups_seen = set(state_groups)
2236
2237		# Set of state groups to handle next.
2238		next_to_search = set(state_groups)
2239		while next_to_search:
2240		# We bound size of groups we're looking up at once, to stop the
2241		# SQL query getting too big
2242		if len(next_to_search) < 100:
2243		current_search = next_to_search
2244		next_to_search = set()
2245		else:
2246		current_search = set(itertools.islice(next_to_search, 100))
2247		next_to_search -= current_search
2248
2249		# Check if state groups are referenced
2250		sql = """
2251		SELECT DISTINCT state_group FROM event_to_state_groups
2252		LEFT JOIN events_to_purge AS ep USING (event_id)
2253		WHERE state_group IN (%s) AND ep.event_id IS NULL
2254		""" % (
2255		",".join("?" for _ in current_search),
2256		)
2257		txn.execute(sql, list(current_search))
2258
2259		referenced = set(sg for sg, in txn)
2260		referenced_groups \|= referenced
2261
2262		# We don't continue iterating up the state group graphs for state
2263		# groups that are referenced.
2264		current_search -= referenced
2265
2266		rows = self._simple_select_many_txn(
2267		txn,
2268		table="state_group_edges",
2269		column="prev_state_group",
2270		iterable=current_search,
2271		keyvalues={},
2272		retcols=("prev_state_group", "state_group"),
2273		)
2274
2275		prevs = set(row["state_group"] for row in rows)
2276		# We don't bother re-handling groups we've already seen
2277		prevs -= state_groups_seen
2278		next_to_search \|= prevs
2279		state_groups_seen \|= prevs
2280
2281		for row in rows:
2282		# Note: Each state group can have at most one prev group
2283		graph[row["state_group"]] = row["prev_state_group"]
2284
2285		to_delete = state_groups_seen - referenced_groups
2286
2287		to_dedelta = set()
2288		for sg in referenced_groups:
2289		prev_sg = graph.get(sg)
2290		if prev_sg and prev_sg in to_delete:
2291		to_dedelta.add(sg)
2292
2293		return to_delete, to_dedelta
2294
2295		def purge_room(self, room_id):
2296		"""Deletes all record of a room
2297
2298		Args:
2299		room_id (str):
2300		"""
2301
2302		return self.runInteraction("purge_room", self._purge_room_txn, room_id)
2303
2304		def _purge_room_txn(self, txn, room_id):
2305		# first we have to delete the state groups states
2306		logger.info("[purge] removing %s from state_groups_state", room_id)
2307
2308		txn.execute(
2309		"""
2310		DELETE FROM state_groups_state WHERE state_group IN (
2311		SELECT state_group FROM events JOIN event_to_state_groups USING(event_id)
2312		WHERE events.room_id=?
2313		)
2314		""",
2315		(room_id,),
2316		)
2317
2318		# ... and the state group edges
2319		logger.info("[purge] removing %s from state_group_edges", room_id)
2320
2321		txn.execute(
2322		"""
2323		DELETE FROM state_group_edges WHERE state_group IN (
2324		SELECT state_group FROM events JOIN event_to_state_groups USING(event_id)
2325		WHERE events.room_id=?
2326		)
2327		""",
2328		(room_id,),
2329		)
2330
2331		# ... and the state groups
2332		logger.info("[purge] removing %s from state_groups", room_id)
2333
2334		txn.execute(
2335		"""
2336		DELETE FROM state_groups WHERE id IN (
2337		SELECT state_group FROM events JOIN event_to_state_groups USING(event_id)
2338		WHERE events.room_id=?
2339		)
2340		""",
2341		(room_id,),
2342		)
2343
2344		# and then tables which lack an index on room_id but have one on event_id
2345		for table in (
2346		"event_auth",
2347		"event_edges",
2348		"event_push_actions_staging",
2349		"event_reference_hashes",
2350		"event_relations",
2351		"event_to_state_groups",
2352		"redactions",
2353		"rejections",
2354		"state_events",
2355		):
2356		logger.info("[purge] removing %s from %s", room_id, table)
2357
2358		txn.execute(
2359		"""
2360		DELETE FROM %s WHERE event_id IN (
2361		SELECT event_id FROM events WHERE room_id=?
2362		)
2363		"""
2364		% (table,),
2365		(room_id,),
2366		)
2367
2368		# and finally, the tables with an index on room_id (or no useful index)
2369		for table in (
2370		"current_state_events",
2371		"event_backward_extremities",
2372		"event_forward_extremities",
2373		"event_json",
2374		"event_push_actions",
2375		"event_search",
2376		"events",
2377		"group_rooms",
2378		"public_room_list_stream",
2379		"receipts_graph",
2380		"receipts_linearized",
2381		"room_aliases",
2382		"room_depth",
2383		"room_memberships",
2384		"room_stats_state",
2385		"room_stats_current",
2386		"room_stats_historical",
2387		"room_stats_earliest_token",
2388		"rooms",
2389		"stream_ordering_to_exterm",
2390		"topics",
2391		"users_in_public_rooms",
2392		"users_who_share_private_rooms",
2393		# no useful index, but let's clear them anyway
2394		"appservice_room_list",
2395		"e2e_room_keys",
2396		"event_push_summary",
2397		"pusher_throttle",
2398		"group_summary_rooms",
2399		"local_invites",
2400		"room_account_data",
2401		"room_tags",
2402		):
2403		logger.info("[purge] removing %s from %s", room_id, table)
2404		txn.execute("DELETE FROM %s WHERE room_id=?" % (table,), (room_id,))
2405
2406		# Other tables we do NOT need to clear out:
2407		#
2408		# - blocked_rooms
2409		# This is important, to make sure that we don't accidentally rejoin a blocked
2410		# room after it was purged
2411		#
2412		# - user_directory
2413		# This has a room_id column, but it is unused
2414		#
2415
2416		# Other tables that we might want to consider clearing out include:
2417		#
2418		# - event_reports
2419		# Given that these are intended for abuse management my initial
2420		# inclination is to leave them in place.
2421		#
2422		# - current_state_delta_stream
2423		# - ex_outlier_stream
2424		# - room_tags_revisions
2425		# The problem with these is that they are largeish and there is no room_id
2426		# index on them. In any case we should be clearing out 'stream' tables
2427		# periodically anyway (#5888)
2428
2429		# TODO: we could probably usefully do a bunch of cache invalidation here
2430
2431		logger.info("[purge] done")
2432
2433		@defer.inlineCallbacks
2434		def is_event_after(self, event_id1, event_id2):
2435		"""Returns True if event_id1 is after event_id2 in the stream
2436		"""
2437		to_1, so_1 = yield self._get_event_ordering(event_id1)
2438		to_2, so_2 = yield self._get_event_ordering(event_id2)
2439		return (to_1, so_1) > (to_2, so_2)
2440
2441		@cachedInlineCallbacks(max_entries=5000)
2442		def _get_event_ordering(self, event_id):
2443		res = yield self._simple_select_one(
2444		table="events",
2445		retcols=["topological_ordering", "stream_ordering"],
2446		keyvalues={"event_id": event_id},
2447		allow_none=True,
2448		)
2449
2450		if not res:
2451		raise SynapseError(404, "Could not find event %s" % (event_id,))
2452
2453		return (int(res["topological_ordering"]), int(res["stream_ordering"]))
2454
2455		def get_all_updated_current_state_deltas(self, from_token, to_token, limit):
2456		def get_all_updated_current_state_deltas_txn(txn):
2457		sql = """
2458		SELECT stream_id, room_id, type, state_key, event_id
2459		FROM current_state_delta_stream
2460		WHERE ? < stream_id AND stream_id <= ?
2461		ORDER BY stream_id ASC LIMIT ?
2462		"""
2463		txn.execute(sql, (from_token, to_token, limit))
2464		return txn.fetchall()
2465
2466		return self.runInteraction(
2467		"get_all_updated_current_state_deltas",
2468		get_all_updated_current_state_deltas_txn,
2469		)
2470
2471
2472		AllNewEventsResult = namedtuple(
2473		"AllNewEventsResult",
2474		[
2475		"new_forward_events",
2476		"new_backfill_events",
2477		"forward_ex_outliers",
2478		"backward_ex_outliers",
2479		],
2480		)

+0

-460

~~synapse/storage/events_bg_updates.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2019 The Matrix.org Foundation C.I.C.
2		#
3		# Licensed under the Apache License, Version 2.0 (the "License");
4		# you may not use this file except in compliance with the License.
5		# You may obtain a copy of the License at
6		#
7		# http://www.apache.org/licenses/LICENSE-2.0
8		#
9		# Unless required by applicable law or agreed to in writing, software
10		# distributed under the License is distributed on an "AS IS" BASIS,
11		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12		# See the License for the specific language governing permissions and
13		# limitations under the License.
14
15		import logging
16
17		from six import text_type
18
19		from canonicaljson import json
20
21		from twisted.internet import defer
22
23		from synapse.storage.background_updates import BackgroundUpdateStore
24
25		logger = logging.getLogger(__name__)
26
27
28		class EventsBackgroundUpdatesStore(BackgroundUpdateStore):
29
30		EVENT_ORIGIN_SERVER_TS_NAME = "event_origin_server_ts"
31		EVENT_FIELDS_SENDER_URL_UPDATE_NAME = "event_fields_sender_url"
32		DELETE_SOFT_FAILED_EXTREMITIES = "delete_soft_failed_extremities"
33
34		def __init__(self, db_conn, hs):
35		super(EventsBackgroundUpdatesStore, self).__init__(db_conn, hs)
36
37		self.register_background_update_handler(
38		self.EVENT_ORIGIN_SERVER_TS_NAME, self._background_reindex_origin_server_ts
39		)
40		self.register_background_update_handler(
41		self.EVENT_FIELDS_SENDER_URL_UPDATE_NAME,
42		self._background_reindex_fields_sender,
43		)
44
45		self.register_background_index_update(
46		"event_contains_url_index",
47		index_name="event_contains_url_index",
48		table="events",
49		columns=["room_id", "topological_ordering", "stream_ordering"],
50		where_clause="contains_url = true AND outlier = false",
51		)
52
53		# an event_id index on event_search is useful for the purge_history
54		# api. Plus it means we get to enforce some integrity with a UNIQUE
55		# clause
56		self.register_background_index_update(
57		"event_search_event_id_idx",
58		index_name="event_search_event_id_idx",
59		table="event_search",
60		columns=["event_id"],
61		unique=True,
62		psql_only=True,
63		)
64
65		self.register_background_update_handler(
66		self.DELETE_SOFT_FAILED_EXTREMITIES, self._cleanup_extremities_bg_update
67		)
68
69		self.register_background_update_handler(
70		"redactions_received_ts", self._redactions_received_ts
71		)
72
73		@defer.inlineCallbacks
74		def _background_reindex_fields_sender(self, progress, batch_size):
75		target_min_stream_id = progress["target_min_stream_id_inclusive"]
76		max_stream_id = progress["max_stream_id_exclusive"]
77		rows_inserted = progress.get("rows_inserted", 0)
78
79		INSERT_CLUMP_SIZE = 1000
80
81		def reindex_txn(txn):
82		sql = (
83		"SELECT stream_ordering, event_id, json FROM events"
84		" INNER JOIN event_json USING (event_id)"
85		" WHERE ? <= stream_ordering AND stream_ordering < ?"
86		" ORDER BY stream_ordering DESC"
87		" LIMIT ?"
88		)
89
90		txn.execute(sql, (target_min_stream_id, max_stream_id, batch_size))
91
92		rows = txn.fetchall()
93		if not rows:
94		return 0
95
96		min_stream_id = rows[-1][0]
97
98		update_rows = []
99		for row in rows:
100		try:
101		event_id = row[1]
102		event_json = json.loads(row[2])
103		sender = event_json["sender"]
104		content = event_json["content"]
105
106		contains_url = "url" in content
107		if contains_url:
108		contains_url &= isinstance(content["url"], text_type)
109		except (KeyError, AttributeError):
110		# If the event is missing a necessary field then
111		# skip over it.
112		continue
113
114		update_rows.append((sender, contains_url, event_id))
115
116		sql = "UPDATE events SET sender = ?, contains_url = ? WHERE event_id = ?"
117
118		for index in range(0, len(update_rows), INSERT_CLUMP_SIZE):
119		clump = update_rows[index : index + INSERT_CLUMP_SIZE]
120		txn.executemany(sql, clump)
121
122		progress = {
123		"target_min_stream_id_inclusive": target_min_stream_id,
124		"max_stream_id_exclusive": min_stream_id,
125		"rows_inserted": rows_inserted + len(rows),
126		}
127
128		self._background_update_progress_txn(
129		txn, self.EVENT_FIELDS_SENDER_URL_UPDATE_NAME, progress
130		)
131
132		return len(rows)
133
134		result = yield self.runInteraction(
135		self.EVENT_FIELDS_SENDER_URL_UPDATE_NAME, reindex_txn
136		)
137
138		if not result:
139		yield self._end_background_update(self.EVENT_FIELDS_SENDER_URL_UPDATE_NAME)
140
141		return result
142
143		@defer.inlineCallbacks
144		def _background_reindex_origin_server_ts(self, progress, batch_size):
145		target_min_stream_id = progress["target_min_stream_id_inclusive"]
146		max_stream_id = progress["max_stream_id_exclusive"]
147		rows_inserted = progress.get("rows_inserted", 0)
148
149		INSERT_CLUMP_SIZE = 1000
150
151		def reindex_search_txn(txn):
152		sql = (
153		"SELECT stream_ordering, event_id FROM events"
154		" WHERE ? <= stream_ordering AND stream_ordering < ?"
155		" ORDER BY stream_ordering DESC"
156		" LIMIT ?"
157		)
158
159		txn.execute(sql, (target_min_stream_id, max_stream_id, batch_size))
160
161		rows = txn.fetchall()
162		if not rows:
163		return 0
164
165		min_stream_id = rows[-1][0]
166		event_ids = [row[1] for row in rows]
167
168		rows_to_update = []
169
170		chunks = [event_ids[i : i + 100] for i in range(0, len(event_ids), 100)]
171		for chunk in chunks:
172		ev_rows = self._simple_select_many_txn(
173		txn,
174		table="event_json",
175		column="event_id",
176		iterable=chunk,
177		retcols=["event_id", "json"],
178		keyvalues={},
179		)
180
181		for row in ev_rows:
182		event_id = row["event_id"]
183		event_json = json.loads(row["json"])
184		try:
185		origin_server_ts = event_json["origin_server_ts"]
186		except (KeyError, AttributeError):
187		# If the event is missing a necessary field then
188		# skip over it.
189		continue
190
191		rows_to_update.append((origin_server_ts, event_id))
192
193		sql = "UPDATE events SET origin_server_ts = ? WHERE event_id = ?"
194
195		for index in range(0, len(rows_to_update), INSERT_CLUMP_SIZE):
196		clump = rows_to_update[index : index + INSERT_CLUMP_SIZE]
197		txn.executemany(sql, clump)
198
199		progress = {
200		"target_min_stream_id_inclusive": target_min_stream_id,
201		"max_stream_id_exclusive": min_stream_id,
202		"rows_inserted": rows_inserted + len(rows_to_update),
203		}
204
205		self._background_update_progress_txn(
206		txn, self.EVENT_ORIGIN_SERVER_TS_NAME, progress
207		)
208
209		return len(rows_to_update)
210
211		result = yield self.runInteraction(
212		self.EVENT_ORIGIN_SERVER_TS_NAME, reindex_search_txn
213		)
214
215		if not result:
216		yield self._end_background_update(self.EVENT_ORIGIN_SERVER_TS_NAME)
217
218		return result
219
220		@defer.inlineCallbacks
221		def _cleanup_extremities_bg_update(self, progress, batch_size):
222		"""Background update to clean out extremities that should have been
223		deleted previously.
224
225		Mainly used to deal with the aftermath of #5269.
226		"""
227
228		# This works by first copying all existing forward extremities into the
229		# `_extremities_to_check` table at start up, and then checking each
230		# event in that table whether we have any descendants that are not
231		# soft-failed/rejected. If that is the case then we delete that event
232		# from the forward extremities table.
233		#
234		# For efficiency, we do this in batches by recursively pulling out all
235		# descendants of a batch until we find the non soft-failed/rejected
236		# events, i.e. the set of descendants whose chain of prev events back
237		# to the batch of extremities are all soft-failed or rejected.
238		# Typically, we won't find any such events as extremities will rarely
239		# have any descendants, but if they do then we should delete those
240		# extremities.
241
242		def _cleanup_extremities_bg_update_txn(txn):
243		# The set of extremity event IDs that we're checking this round
244		original_set = set()
245
246		# A dict[str, set[str]] of event ID to their prev events.
247		graph = {}
248
249		# The set of descendants of the original set that are not rejected
250		# nor soft-failed. Ancestors of these events should be removed
251		# from the forward extremities table.
252		non_rejected_leaves = set()
253
254		# Set of event IDs that have been soft failed, and for which we
255		# should check if they have descendants which haven't been soft
256		# failed.
257		soft_failed_events_to_lookup = set()
258
259		# First, we get `batch_size` events from the table, pulling out
260		# their successor events, if any, and the successor events'
261		# rejection status.
262		txn.execute(
263		"""SELECT prev_event_id, event_id, internal_metadata,
264		rejections.event_id IS NOT NULL, events.outlier
265		FROM (
266		SELECT event_id AS prev_event_id
267		FROM _extremities_to_check
268		LIMIT ?
269		) AS f
270		LEFT JOIN event_edges USING (prev_event_id)
271		LEFT JOIN events USING (event_id)
272		LEFT JOIN event_json USING (event_id)
273		LEFT JOIN rejections USING (event_id)
274		""",
275		(batch_size,),
276		)
277
278		for prev_event_id, event_id, metadata, rejected, outlier in txn:
279		original_set.add(prev_event_id)
280
281		if not event_id or outlier:
282		# Common case where the forward extremity doesn't have any
283		# descendants.
284		continue
285
286		graph.setdefault(event_id, set()).add(prev_event_id)
287
288		soft_failed = False
289		if metadata:
290		soft_failed = json.loads(metadata).get("soft_failed")
291
292		if soft_failed or rejected:
293		soft_failed_events_to_lookup.add(event_id)
294		else:
295		non_rejected_leaves.add(event_id)
296
297		# Now we recursively check all the soft-failed descendants we
298		# found above in the same way, until we have nothing left to
299		# check.
300		while soft_failed_events_to_lookup:
301		# We only want to do 100 at a time, so we split given list
302		# into two.
303		batch = list(soft_failed_events_to_lookup)
304		to_check, to_defer = batch[:100], batch[100:]
305		soft_failed_events_to_lookup = set(to_defer)
306
307		sql = """SELECT prev_event_id, event_id, internal_metadata,
308		rejections.event_id IS NOT NULL
309		FROM event_edges
310		INNER JOIN events USING (event_id)
311		INNER JOIN event_json USING (event_id)
312		LEFT JOIN rejections USING (event_id)
313		WHERE
314		prev_event_id IN (%s)
315		AND NOT events.outlier
316		""" % (
317		",".join("?" for _ in to_check),
318		)
319		txn.execute(sql, to_check)
320
321		for prev_event_id, event_id, metadata, rejected in txn:
322		if event_id in graph:
323		# Already handled this event previously, but we still
324		# want to record the edge.
325		graph[event_id].add(prev_event_id)
326		continue
327
328		graph[event_id] = {prev_event_id}
329
330		soft_failed = json.loads(metadata).get("soft_failed")
331		if soft_failed or rejected:
332		soft_failed_events_to_lookup.add(event_id)
333		else:
334		non_rejected_leaves.add(event_id)
335
336		# We have a set of non-soft-failed descendants, so we recurse up
337		# the graph to find all ancestors and add them to the set of event
338		# IDs that we can delete from forward extremities table.
339		to_delete = set()
340		while non_rejected_leaves:
341		event_id = non_rejected_leaves.pop()
342		prev_event_ids = graph.get(event_id, set())
343		non_rejected_leaves.update(prev_event_ids)
344		to_delete.update(prev_event_ids)
345
346		to_delete.intersection_update(original_set)
347
348		deleted = self._simple_delete_many_txn(
349		txn=txn,
350		table="event_forward_extremities",
351		column="event_id",
352		iterable=to_delete,
353		keyvalues={},
354		)
355
356		logger.info(
357		"Deleted %d forward extremities of %d checked, to clean up #5269",
358		deleted,
359		len(original_set),
360		)
361
362		if deleted:
363		# We now need to invalidate the caches of these rooms
364		rows = self._simple_select_many_txn(
365		txn,
366		table="events",
367		column="event_id",
368		iterable=to_delete,
369		keyvalues={},
370		retcols=("room_id",),
371		)
372		room_ids = set(row["room_id"] for row in rows)
373		for room_id in room_ids:
374		txn.call_after(
375		self.get_latest_event_ids_in_room.invalidate, (room_id,)
376		)
377
378		self._simple_delete_many_txn(
379		txn=txn,
380		table="_extremities_to_check",
381		column="event_id",
382		iterable=original_set,
383		keyvalues={},
384		)
385
386		return len(original_set)
387
388		num_handled = yield self.runInteraction(
389		"_cleanup_extremities_bg_update", _cleanup_extremities_bg_update_txn
390		)
391
392		if not num_handled:
393		yield self._end_background_update(self.DELETE_SOFT_FAILED_EXTREMITIES)
394
395		def _drop_table_txn(txn):
396		txn.execute("DROP TABLE _extremities_to_check")
397
398		yield self.runInteraction(
399		"_cleanup_extremities_bg_update_drop_table", _drop_table_txn
400		)
401
402		return num_handled
403
404		@defer.inlineCallbacks
405		def _redactions_received_ts(self, progress, batch_size):
406		"""Handles filling out the `received_ts` column in redactions.
407		"""
408		last_event_id = progress.get("last_event_id", "")
409
410		def _redactions_received_ts_txn(txn):
411		# Fetch the set of event IDs that we want to update
412		sql = """
413		SELECT event_id FROM redactions
414		WHERE event_id > ?
415		ORDER BY event_id ASC
416		LIMIT ?
417		"""
418
419		txn.execute(sql, (last_event_id, batch_size))
420
421		rows = txn.fetchall()
422		if not rows:
423		return 0
424
425		upper_event_id, = rows[-1]
426
427		# Update the redactions with the received_ts.
428		#
429		# Note: Not all events have an associated received_ts, so we
430		# fallback to using origin_server_ts. If we for some reason don't
431		# have an origin_server_ts, lets just use the current timestamp.
432		#
433		# We don't want to leave it null, as then we'll never try and
434		# censor those redactions.
435		sql = """
436		UPDATE redactions
437		SET received_ts = (
438		SELECT COALESCE(received_ts, origin_server_ts, ?) FROM events
439		WHERE events.event_id = redactions.event_id
440		)
441		WHERE ? <= event_id AND event_id <= ?
442		"""
443
444		txn.execute(sql, (self._clock.time_msec(), last_event_id, upper_event_id))
445
446		self._background_update_progress_txn(
447		txn, "redactions_received_ts", {"last_event_id": upper_event_id}
448		)
449
450		return len(rows)
451
452		count = yield self.runInteraction(
453		"_redactions_received_ts", _redactions_received_ts_txn
454		)
455
456		if not count:
457		yield self._end_background_update("redactions_received_ts")
458
459		return count

+0

-878

~~synapse/storage/events_worker.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2018 New Vector Ltd
2		#
3		# Licensed under the Apache License, Version 2.0 (the "License");
4		# you may not use this file except in compliance with the License.
5		# You may obtain a copy of the License at
6		#
7		# http://www.apache.org/licenses/LICENSE-2.0
8		#
9		# Unless required by applicable law or agreed to in writing, software
10		# distributed under the License is distributed on an "AS IS" BASIS,
11		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12		# See the License for the specific language governing permissions and
13		# limitations under the License.
14
15		from __future__ import division
16
17		import itertools
18		import logging
19		from collections import namedtuple
20
21		from canonicaljson import json
22
23		from twisted.internet import defer
24
25		from synapse.api.constants import EventTypes
26		from synapse.api.errors import NotFoundError
27		from synapse.api.room_versions import EventFormatVersions
28		from synapse.events import FrozenEvent, event_type_from_format_version # noqa: F401
29		from synapse.events.snapshot import EventContext # noqa: F401
30		from synapse.events.utils import prune_event
31		from synapse.logging.context import LoggingContext, PreserveLoggingContext
32		from synapse.metrics.background_process_metrics import run_as_background_process
33		from synapse.types import get_domain_from_id
34		from synapse.util import batch_iter
35		from synapse.util.metrics import Measure
36
37		from ._base import SQLBaseStore
38
39		logger = logging.getLogger(__name__)
40
41
42		# These values are used in the `enqueus_event` and `_do_fetch` methods to
43		# control how we batch/bulk fetch events from the database.
44		# The values are plucked out of thing air to make initial sync run faster
45		# on jki.re
46		# TODO: Make these configurable.
47		EVENT_QUEUE_THREADS = 3 # Max number of threads that will fetch events
48		EVENT_QUEUE_ITERATIONS = 3 # No. times we block waiting for requests for events
49		EVENT_QUEUE_TIMEOUT_S = 0.1 # Timeout when waiting for requests for events
50
51
52		_EventCacheEntry = namedtuple("_EventCacheEntry", ("event", "redacted_event"))
53
54
55		class EventsWorkerStore(SQLBaseStore):
56		def get_received_ts(self, event_id):
57		"""Get received_ts (when it was persisted) for the event.
58
59		Raises an exception for unknown events.
60
61		Args:
62		event_id (str)
63
64		Returns:
65		Deferred[int\|None]: Timestamp in milliseconds, or None for events
66		that were persisted before received_ts was implemented.
67		"""
68		return self._simple_select_one_onecol(
69		table="events",
70		keyvalues={"event_id": event_id},
71		retcol="received_ts",
72		desc="get_received_ts",
73		)
74
75		def get_received_ts_by_stream_pos(self, stream_ordering):
76		"""Given a stream ordering get an approximate timestamp of when it
77		happened.
78
79		This is done by simply taking the received ts of the first event that
80		has a stream ordering greater than or equal to the given stream pos.
81		If none exists returns the current time, on the assumption that it must
82		have happened recently.
83
84		Args:
85		stream_ordering (int)
86
87		Returns:
88		Deferred[int]
89		"""
90
91		def _get_approximate_received_ts_txn(txn):
92		sql = """
93		SELECT received_ts FROM events
94		WHERE stream_ordering >= ?
95		LIMIT 1
96		"""
97
98		txn.execute(sql, (stream_ordering,))
99		row = txn.fetchone()
100		if row and row[0]:
101		ts = row[0]
102		else:
103		ts = self.clock.time_msec()
104
105		return ts
106
107		return self.runInteraction(
108		"get_approximate_received_ts", _get_approximate_received_ts_txn
109		)
110
111		@defer.inlineCallbacks
112		def get_event(
113		self,
114		event_id,
115		check_redacted=True,
116		get_prev_content=False,
117		allow_rejected=False,
118		allow_none=False,
119		check_room_id=None,
120		):
121		"""Get an event from the database by event_id.
122
123		Args:
124		event_id (str): The event_id of the event to fetch
125		check_redacted (bool): If True, check if event has been redacted
126		and redact it.
127		get_prev_content (bool): If True and event is a state event,
128		include the previous states content in the unsigned field.
129		allow_rejected (bool): If True return rejected events.
130		allow_none (bool): If True, return None if no event found, if
131		False throw a NotFoundError
132		check_room_id (str\|None): if not None, check the room of the found event.
133		If there is a mismatch, behave as per allow_none.
134
135		Returns:
136		Deferred[EventBase\|None]
137		"""
138		if not isinstance(event_id, str):
139		raise TypeError("Invalid event event_id %r" % (event_id,))
140
141		events = yield self.get_events_as_list(
142		[event_id],
143		check_redacted=check_redacted,
144		get_prev_content=get_prev_content,
145		allow_rejected=allow_rejected,
146		)
147
148		event = events[0] if events else None
149
150		if event is not None and check_room_id is not None:
151		if event.room_id != check_room_id:
152		event = None
153
154		if event is None and not allow_none:
155		raise NotFoundError("Could not find event %s" % (event_id,))
156
157		return event
158
159		@defer.inlineCallbacks
160		def get_events(
161		self,
162		event_ids,
163		check_redacted=True,
164		get_prev_content=False,
165		allow_rejected=False,
166		):
167		"""Get events from the database
168
169		Args:
170		event_ids (list): The event_ids of the events to fetch
171		check_redacted (bool): If True, check if event has been redacted
172		and redact it.
173		get_prev_content (bool): If True and event is a state event,
174		include the previous states content in the unsigned field.
175		allow_rejected (bool): If True return rejected events.
176
177		Returns:
178		Deferred : Dict from event_id to event.
179		"""
180		events = yield self.get_events_as_list(
181		event_ids,
182		check_redacted=check_redacted,
183		get_prev_content=get_prev_content,
184		allow_rejected=allow_rejected,
185		)
186
187		return {e.event_id: e for e in events}
188
189		@defer.inlineCallbacks
190		def get_events_as_list(
191		self,
192		event_ids,
193		check_redacted=True,
194		get_prev_content=False,
195		allow_rejected=False,
196		):
197		"""Get events from the database and return in a list in the same order
198		as given by `event_ids` arg.
199
200		Args:
201		event_ids (list): The event_ids of the events to fetch
202		check_redacted (bool): If True, check if event has been redacted
203		and redact it.
204		get_prev_content (bool): If True and event is a state event,
205		include the previous states content in the unsigned field.
206		allow_rejected (bool): If True return rejected events.
207
208		Returns:
209		Deferred[list[EventBase]]: List of events fetched from the database. The
210		events are in the same order as `event_ids` arg.
211
212		Note that the returned list may be smaller than the list of event
213		IDs if not all events could be fetched.
214		"""
215
216		if not event_ids:
217		return []
218
219		# there may be duplicates so we cast the list to a set
220		event_entry_map = yield self._get_events_from_cache_or_db(
221		set(event_ids), allow_rejected=allow_rejected
222		)
223
224		events = []
225		for event_id in event_ids:
226		entry = event_entry_map.get(event_id, None)
227		if not entry:
228		continue
229
230		if not allow_rejected:
231		assert not entry.event.rejected_reason, (
232		"rejected event returned from _get_events_from_cache_or_db despite "
233		"allow_rejected=False"
234		)
235
236		# We may not have had the original event when we received a redaction, so
237		# we have to recheck auth now.
238
239		if not allow_rejected and entry.event.type == EventTypes.Redaction:
240		if not hasattr(entry.event, "redacts"):
241		# A redacted redaction doesn't have a `redacts` key, in
242		# which case lets just withhold the event.
243		#
244		# Note: Most of the time if the redactions has been
245		# redacted we still have the un-redacted event in the DB
246		# and so we'll still see the `redacts` key. However, this
247		# isn't always true e.g. if we have censored the event.
248		logger.debug(
249		"Withholding redaction event %s as we don't have redacts key",
250		event_id,
251		)
252		continue
253
254		redacted_event_id = entry.event.redacts
255		event_map = yield self._get_events_from_cache_or_db([redacted_event_id])
256		original_event_entry = event_map.get(redacted_event_id)
257		if not original_event_entry:
258		# we don't have the redacted event (or it was rejected).
259		#
260		# We assume that the redaction isn't authorized for now; if the
261		# redacted event later turns up, the redaction will be re-checked,
262		# and if it is found valid, the original will get redacted before it
263		# is served to the client.
264		logger.debug(
265		"Withholding redaction event %s since we don't (yet) have the "
266		"original %s",
267		event_id,
268		redacted_event_id,
269		)
270		continue
271
272		original_event = original_event_entry.event
273		if original_event.type == EventTypes.Create:
274		# we never serve redactions of Creates to clients.
275		logger.info(
276		"Withholding redaction %s of create event %s",
277		event_id,
278		redacted_event_id,
279		)
280		continue
281
282		if original_event.room_id != entry.event.room_id:
283		logger.info(
284		"Withholding redaction %s of event %s from a different room",
285		event_id,
286		redacted_event_id,
287		)
288		continue
289
290		if entry.event.internal_metadata.need_to_check_redaction():
291		original_domain = get_domain_from_id(original_event.sender)
292		redaction_domain = get_domain_from_id(entry.event.sender)
293		if original_domain != redaction_domain:
294		# the senders don't match, so this is forbidden
295		logger.info(
296		"Withholding redaction %s whose sender domain %s doesn't "
297		"match that of redacted event %s %s",
298		event_id,
299		redaction_domain,
300		redacted_event_id,
301		original_domain,
302		)
303		continue
304
305		# Update the cache to save doing the checks again.
306		entry.event.internal_metadata.recheck_redaction = False
307
308		if check_redacted and entry.redacted_event:
309		event = entry.redacted_event
310		else:
311		event = entry.event
312
313		events.append(event)
314
315		if get_prev_content:
316		if "replaces_state" in event.unsigned:
317		prev = yield self.get_event(
318		event.unsigned["replaces_state"],
319		get_prev_content=False,
320		allow_none=True,
321		)
322		if prev:
323		event.unsigned = dict(event.unsigned)
324		event.unsigned["prev_content"] = prev.content
325		event.unsigned["prev_sender"] = prev.sender
326
327		return events
328
329		@defer.inlineCallbacks
330		def _get_events_from_cache_or_db(self, event_ids, allow_rejected=False):
331		"""Fetch a bunch of events from the cache or the database.
332
333		If events are pulled from the database, they will be cached for future lookups.
334
335		Args:
336		event_ids (Iterable[str]): The event_ids of the events to fetch
337		allow_rejected (bool): Whether to include rejected events
338
339		Returns:
340		Deferred[Dict[str, _EventCacheEntry]]:
341		map from event id to result
342		"""
343		event_entry_map = self._get_events_from_cache(
344		event_ids, allow_rejected=allow_rejected
345		)
346
347		missing_events_ids = [e for e in event_ids if e not in event_entry_map]
348
349		if missing_events_ids:
350		log_ctx = LoggingContext.current_context()
351		log_ctx.record_event_fetch(len(missing_events_ids))
352
353		# Note that _get_events_from_db is also responsible for turning db rows
354		# into FrozenEvents (via _get_event_from_row), which involves seeing if
355		# the events have been redacted, and if so pulling the redaction event out
356		# of the database to check it.
357		#
358		missing_events = yield self._get_events_from_db(
359		missing_events_ids, allow_rejected=allow_rejected
360		)
361
362		event_entry_map.update(missing_events)
363
364		return event_entry_map
365
366		def _invalidate_get_event_cache(self, event_id):
367		self._get_event_cache.invalidate((event_id,))
368
369		def _get_events_from_cache(self, events, allow_rejected, update_metrics=True):
370		"""Fetch events from the caches
371
372		Args:
373		events (Iterable[str]): list of event_ids to fetch
374		allow_rejected (bool): Whether to return events that were rejected
375		update_metrics (bool): Whether to update the cache hit ratio metrics
376
377		Returns:
378		dict of event_id -> _EventCacheEntry for each event_id in cache. If
379		allow_rejected is `False` then there will still be an entry but it
380		will be `None`
381		"""
382		event_map = {}
383
384		for event_id in events:
385		ret = self._get_event_cache.get(
386		(event_id,), None, update_metrics=update_metrics
387		)
388		if not ret:
389		continue
390
391		if allow_rejected or not ret.event.rejected_reason:
392		event_map[event_id] = ret
393		else:
394		event_map[event_id] = None
395
396		return event_map
397
398		def _do_fetch(self, conn):
399		"""Takes a database connection and waits for requests for events from
400		the _event_fetch_list queue.
401		"""
402		i = 0
403		while True:
404		with self._event_fetch_lock:
405		event_list = self._event_fetch_list
406		self._event_fetch_list = []
407
408		if not event_list:
409		single_threaded = self.database_engine.single_threaded
410		if single_threaded or i > EVENT_QUEUE_ITERATIONS:
411		self._event_fetch_ongoing -= 1
412		return
413		else:
414		self._event_fetch_lock.wait(EVENT_QUEUE_TIMEOUT_S)
415		i += 1
416		continue
417		i = 0
418
419		self._fetch_event_list(conn, event_list)
420
421		def _fetch_event_list(self, conn, event_list):
422		"""Handle a load of requests from the _event_fetch_list queue
423
424		Args:
425		conn (twisted.enterprise.adbapi.Connection): database connection
426
427		event_list (list[Tuple[list[str], Deferred]]):
428		The fetch requests. Each entry consists of a list of event
429		ids to be fetched, and a deferred to be completed once the
430		events have been fetched.
431
432		The deferreds are callbacked with a dictionary mapping from event id
433		to event row. Note that it may well contain additional events that
434		were not part of this request.
435		"""
436		with Measure(self._clock, "_fetch_event_list"):
437		try:
438		events_to_fetch = set(
439		event_id for events, _ in event_list for event_id in events
440		)
441
442		row_dict = self._new_transaction(
443		conn, "do_fetch", [], [], self._fetch_event_rows, events_to_fetch
444		)
445
446		# We only want to resolve deferreds from the main thread
447		def fire():
448		for _, d in event_list:
449		d.callback(row_dict)
450
451		with PreserveLoggingContext():
452		self.hs.get_reactor().callFromThread(fire)
453		except Exception as e:
454		logger.exception("do_fetch")
455
456		# We only want to resolve deferreds from the main thread
457		def fire(evs, exc):
458		for _, d in evs:
459		if not d.called:
460		with PreserveLoggingContext():
461		d.errback(exc)
462
463		with PreserveLoggingContext():
464		self.hs.get_reactor().callFromThread(fire, event_list, e)
465
466		@defer.inlineCallbacks
467		def _get_events_from_db(self, event_ids, allow_rejected=False):
468		"""Fetch a bunch of events from the database.
469
470		Returned events will be added to the cache for future lookups.
471
472		Args:
473		event_ids (Iterable[str]): The event_ids of the events to fetch
474		allow_rejected (bool): Whether to include rejected events
475
476		Returns:
477		Deferred[Dict[str, _EventCacheEntry]]:
478		map from event id to result. May return extra events which
479		weren't asked for.
480		"""
481		fetched_events = {}
482		events_to_fetch = event_ids
483
484		while events_to_fetch:
485		row_map = yield self._enqueue_events(events_to_fetch)
486
487		# we need to recursively fetch any redactions of those events
488		redaction_ids = set()
489		for event_id in events_to_fetch:
490		row = row_map.get(event_id)
491		fetched_events[event_id] = row
492		if row:
493		redaction_ids.update(row["redactions"])
494
495		events_to_fetch = redaction_ids.difference(fetched_events.keys())
496		if events_to_fetch:
497		logger.debug("Also fetching redaction events %s", events_to_fetch)
498
499		# build a map from event_id to EventBase
500		event_map = {}
501		for event_id, row in fetched_events.items():
502		if not row:
503		continue
504		assert row["event_id"] == event_id
505
506		rejected_reason = row["rejected_reason"]
507
508		if not allow_rejected and rejected_reason:
509		continue
510
511		d = json.loads(row["json"])
512		internal_metadata = json.loads(row["internal_metadata"])
513
514		format_version = row["format_version"]
515		if format_version is None:
516		# This means that we stored the event before we had the concept
517		# of a event format version, so it must be a V1 event.
518		format_version = EventFormatVersions.V1
519
520		original_ev = event_type_from_format_version(format_version)(
521		event_dict=d,
522		internal_metadata_dict=internal_metadata,
523		rejected_reason=rejected_reason,
524		)
525
526		event_map[event_id] = original_ev
527
528		# finally, we can decide whether each one nededs redacting, and build
529		# the cache entries.
530		result_map = {}
531		for event_id, original_ev in event_map.items():
532		redactions = fetched_events[event_id]["redactions"]
533		redacted_event = self._maybe_redact_event_row(
534		original_ev, redactions, event_map
535		)
536
537		cache_entry = _EventCacheEntry(
538		event=original_ev, redacted_event=redacted_event
539		)
540
541		self._get_event_cache.prefill((event_id,), cache_entry)
542		result_map[event_id] = cache_entry
543
544		return result_map
545
546		@defer.inlineCallbacks
547		def _enqueue_events(self, events):
548		"""Fetches events from the database using the _event_fetch_list. This
549		allows batch and bulk fetching of events - it allows us to fetch events
550		without having to create a new transaction for each request for events.
551
552		Args:
553		events (Iterable[str]): events to be fetched.
554
555		Returns:
556		Deferred[Dict[str, Dict]]: map from event id to row data from the database.
557		May contain events that weren't requested.
558		"""
559
560		events_d = defer.Deferred()
561		with self._event_fetch_lock:
562		self._event_fetch_list.append((events, events_d))
563
564		self._event_fetch_lock.notify()
565
566		if self._event_fetch_ongoing < EVENT_QUEUE_THREADS:
567		self._event_fetch_ongoing += 1
568		should_start = True
569		else:
570		should_start = False
571
572		if should_start:
573		run_as_background_process(
574		"fetch_events", self.runWithConnection, self._do_fetch
575		)
576
577		logger.debug("Loading %d events: %s", len(events), events)
578		with PreserveLoggingContext():
579		row_map = yield events_d
580		logger.debug("Loaded %d events (%d rows)", len(events), len(row_map))
581
582		return row_map
583
584		def _fetch_event_rows(self, txn, event_ids):
585		"""Fetch event rows from the database
586
587		Events which are not found are omitted from the result.
588
589		The returned per-event dicts contain the following keys:
590
591		* event_id (str)
592
593		* json (str): json-encoded event structure
594
595		* internal_metadata (str): json-encoded internal metadata dict
596
597		* format_version (int\|None): The format of the event. Hopefully one
598		of EventFormatVersions. 'None' means the event predates
599		EventFormatVersions (so the event is format V1).
600
601		* rejected_reason (str\|None): if the event was rejected, the reason
602		why.
603
604		* redactions (List[str]): a list of event-ids which (claim to) redact
605		this event.
606
607		Args:
608		txn (twisted.enterprise.adbapi.Connection):
609		event_ids (Iterable[str]): event IDs to fetch
610
611		Returns:
612		Dict[str, Dict]: a map from event id to event info.
613		"""
614		event_dict = {}
615		for evs in batch_iter(event_ids, 200):
616		sql = (
617		"SELECT "
618		" e.event_id, "
619		" e.internal_metadata,"
620		" e.json,"
621		" e.format_version, "
622		" rej.reason "
623		" FROM event_json as e"
624		" LEFT JOIN rejections as rej USING (event_id)"
625		" WHERE e.event_id IN (%s)"
626		) % (",".join(["?"] * len(evs)),)
627
628		txn.execute(sql, evs)
629
630		for row in txn:
631		event_id = row[0]
632		event_dict[event_id] = {
633		"event_id": event_id,
634		"internal_metadata": row[1],
635		"json": row[2],
636		"format_version": row[3],
637		"rejected_reason": row[4],
638		"redactions": [],
639		}
640
641		# check for redactions
642		redactions_sql = (
643		"SELECT event_id, redacts FROM redactions WHERE redacts IN (%s)"
644		) % (",".join(["?"] * len(evs)),)
645
646		txn.execute(redactions_sql, evs)
647
648		for (redacter, redacted) in txn:
649		d = event_dict.get(redacted)
650		if d:
651		d["redactions"].append(redacter)
652
653		return event_dict
654
655		def _maybe_redact_event_row(self, original_ev, redactions, event_map):
656		"""Given an event object and a list of possible redacting event ids,
657		determine whether to honour any of those redactions and if so return a redacted
658		event.
659
660		Args:
661		original_ev (EventBase):
662		redactions (iterable[str]): list of event ids of potential redaction events
663		event_map (dict[str, EventBase]): other events which have been fetched, in
664		which we can look up the redaaction events. Map from event id to event.
665
666		Returns:
667		Deferred[EventBase\|None]: if the event should be redacted, a pruned
668		event object. Otherwise, None.
669		"""
670		if original_ev.type == "m.room.create":
671		# we choose to ignore redactions of m.room.create events.
672		return None
673
674		for redaction_id in redactions:
675		redaction_event = event_map.get(redaction_id)
676		if not redaction_event or redaction_event.rejected_reason:
677		# we don't have the redaction event, or the redaction event was not
678		# authorized.
679		logger.debug(
680		"%s was redacted by %s but redaction not found/authed",
681		original_ev.event_id,
682		redaction_id,
683		)
684		continue
685
686		if redaction_event.room_id != original_ev.room_id:
687		logger.debug(
688		"%s was redacted by %s but redaction was in a different room!",
689		original_ev.event_id,
690		redaction_id,
691		)
692		continue
693
694		# Starting in room version v3, some redactions need to be
695		# rechecked if we didn't have the redacted event at the
696		# time, so we recheck on read instead.
697		if redaction_event.internal_metadata.need_to_check_redaction():
698		expected_domain = get_domain_from_id(original_ev.sender)
699		if get_domain_from_id(redaction_event.sender) == expected_domain:
700		# This redaction event is allowed. Mark as not needing a recheck.
701		redaction_event.internal_metadata.recheck_redaction = False
702		else:
703		# Senders don't match, so the event isn't actually redacted
704		logger.debug(
705		"%s was redacted by %s but the senders don't match",
706		original_ev.event_id,
707		redaction_id,
708		)
709		continue
710
711		logger.debug("Redacting %s due to %s", original_ev.event_id, redaction_id)
712
713		# we found a good redaction event. Redact!
714		redacted_event = prune_event(original_ev)
715		redacted_event.unsigned["redacted_by"] = redaction_id
716
717		# It's fine to add the event directly, since get_pdu_json
718		# will serialise this field correctly
719		redacted_event.unsigned["redacted_because"] = redaction_event
720
721		return redacted_event
722
723		# no valid redaction found for this event
724		return None
725
726		@defer.inlineCallbacks
727		def have_events_in_timeline(self, event_ids):
728		"""Given a list of event ids, check if we have already processed and
729		stored them as non outliers.
730		"""
731		rows = yield self._simple_select_many_batch(
732		table="events",
733		retcols=("event_id",),
734		column="event_id",
735		iterable=list(event_ids),
736		keyvalues={"outlier": False},
737		desc="have_events_in_timeline",
738		)
739
740		return set(r["event_id"] for r in rows)
741
742		@defer.inlineCallbacks
743		def have_seen_events(self, event_ids):
744		"""Given a list of event ids, check if we have already processed them.
745
746		Args:
747		event_ids (iterable[str]):
748
749		Returns:
750		Deferred[set[str]]: The events we have already seen.
751		"""
752		results = set()
753
754		def have_seen_events_txn(txn, chunk):
755		sql = "SELECT event_id FROM events as e WHERE e.event_id IN (%s)" % (
756		",".join("?" * len(chunk)),
757		)
758		txn.execute(sql, chunk)
759		for (event_id,) in txn:
760		results.add(event_id)
761
762		# break the input up into chunks of 100
763		input_iterator = iter(event_ids)
764		for chunk in iter(lambda: list(itertools.islice(input_iterator, 100)), []):
765		yield self.runInteraction("have_seen_events", have_seen_events_txn, chunk)
766		return results
767
768		def get_seen_events_with_rejections(self, event_ids):
769		"""Given a list of event ids, check if we rejected them.
770
771		Args:
772		event_ids (list[str])
773
774		Returns:
775		Deferred[dict[str, str\|None):
776		Has an entry for each event id we already have seen. Maps to
777		the rejected reason string if we rejected the event, else maps
778		to None.
779		"""
780		if not event_ids:
781		return defer.succeed({})
782
783		def f(txn):
784		sql = (
785		"SELECT e.event_id, reason FROM events as e "
786		"LEFT JOIN rejections as r ON e.event_id = r.event_id "
787		"WHERE e.event_id = ?"
788		)
789
790		res = {}
791		for event_id in event_ids:
792		txn.execute(sql, (event_id,))
793		row = txn.fetchone()
794		if row:
795		_, rejected = row
796		res[event_id] = rejected
797
798		return res
799
800		return self.runInteraction("get_seen_events_with_rejections", f)
801
802		def _get_total_state_event_counts_txn(self, txn, room_id):
803		"""
804		See get_total_state_event_counts.
805		"""
806		# We join against the events table as that has an index on room_id
807		sql = """
808		SELECT COUNT(*) FROM state_events
809		INNER JOIN events USING (room_id, event_id)
810		WHERE room_id=?
811		"""
812		txn.execute(sql, (room_id,))
813		row = txn.fetchone()
814		return row[0] if row else 0
815
816		def get_total_state_event_counts(self, room_id):
817		"""
818		Gets the total number of state events in a room.
819
820		Args:
821		room_id (str)
822
823		Returns:
824		Deferred[int]
825		"""
826		return self.runInteraction(
827		"get_total_state_event_counts",
828		self._get_total_state_event_counts_txn,
829		room_id,
830		)
831
832		def _get_current_state_event_counts_txn(self, txn, room_id):
833		"""
834		See get_current_state_event_counts.
835		"""
836		sql = "SELECT COUNT(*) FROM current_state_events WHERE room_id=?"
837		txn.execute(sql, (room_id,))
838		row = txn.fetchone()
839		return row[0] if row else 0
840
841		def get_current_state_event_counts(self, room_id):
842		"""
843		Gets the current number of state events in a room.
844
845		Args:
846		room_id (str)
847
848		Returns:
849		Deferred[int]
850		"""
851		return self.runInteraction(
852		"get_current_state_event_counts",
853		self._get_current_state_event_counts_txn,
854		room_id,
855		)
856
857		@defer.inlineCallbacks
858		def get_room_complexity(self, room_id):
859		"""
860		Get a rough approximation of the complexity of the room. This is used by
861		remote servers to decide whether they wish to join the room or not.
862		Higher complexity value indicates that being in the room will consume
863		more resources.
864
865		Args:
866		room_id (str)
867
868		Returns:
869		Deferred[dict[str:int]] of complexity version to complexity.
870		"""
871		state_events = yield self.get_current_state_event_counts(room_id)
872
873		# Call this one "v1", so we can introduce new ones as we want to develop
874		# it.
875		complexity_v1 = round(state_events / 500, 2)
876
877		return {"v1": complexity_v1}

+0

-75

~~synapse/storage/filtering.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2015, 2016 OpenMarket Ltd
2		#
3		# Licensed under the Apache License, Version 2.0 (the "License");
4		# you may not use this file except in compliance with the License.
5		# You may obtain a copy of the License at
6		#
7		# http://www.apache.org/licenses/LICENSE-2.0
8		#
9		# Unless required by applicable law or agreed to in writing, software
10		# distributed under the License is distributed on an "AS IS" BASIS,
11		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12		# See the License for the specific language governing permissions and
13		# limitations under the License.
14
15		from canonicaljson import encode_canonical_json
16
17		from synapse.api.errors import Codes, SynapseError
18		from synapse.util.caches.descriptors import cachedInlineCallbacks
19
20		from ._base import SQLBaseStore, db_to_json
21
22
23		class FilteringStore(SQLBaseStore):
24		@cachedInlineCallbacks(num_args=2)
25		def get_user_filter(self, user_localpart, filter_id):
26		# filter_id is BIGINT UNSIGNED, so if it isn't a number, fail
27		# with a coherent error message rather than 500 M_UNKNOWN.
28		try:
29		int(filter_id)
30		except ValueError:
31		raise SynapseError(400, "Invalid filter ID", Codes.INVALID_PARAM)
32
33		def_json = yield self._simple_select_one_onecol(
34		table="user_filters",
35		keyvalues={"user_id": user_localpart, "filter_id": filter_id},
36		retcol="filter_json",
37		allow_none=False,
38		desc="get_user_filter",
39		)
40
41		return db_to_json(def_json)
42
43		def add_user_filter(self, user_localpart, user_filter):
44		def_json = encode_canonical_json(user_filter)
45
46		# Need an atomic transaction to SELECT the maximal ID so far then
47		# INSERT a new one
48		def _do_txn(txn):
49		sql = (
50		"SELECT filter_id FROM user_filters "
51		"WHERE user_id = ? AND filter_json = ?"
52		)
53		txn.execute(sql, (user_localpart, def_json))
54		filter_id_response = txn.fetchone()
55		if filter_id_response is not None:
56		return filter_id_response[0]
57
58		sql = "SELECT MAX(filter_id) FROM user_filters " "WHERE user_id = ?"
59		txn.execute(sql, (user_localpart,))
60		max_id = txn.fetchone()[0]
61		if max_id is None:
62		filter_id = 0
63		else:
64		filter_id = max_id + 1
65
66		sql = (
67		"INSERT INTO user_filters (user_id, filter_id, filter_json)"
68		"VALUES(?, ?, ?)"
69		)
70		txn.execute(sql, (user_localpart, filter_id, def_json))
71
72		return filter_id
73
74		return self.runInteraction("add_user_filter", _do_txn)

+0

-1181

~~synapse/storage/group_server.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2017 Vector Creations Ltd
2		# Copyright 2018 New Vector Ltd
3		#
4		# Licensed under the Apache License, Version 2.0 (the "License");
5		# you may not use this file except in compliance with the License.
6		# You may obtain a copy of the License at
7		#
8		# http://www.apache.org/licenses/LICENSE-2.0
9		#
10		# Unless required by applicable law or agreed to in writing, software
11		# distributed under the License is distributed on an "AS IS" BASIS,
12		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13		# See the License for the specific language governing permissions and
14		# limitations under the License.
15
16		from canonicaljson import json
17
18		from twisted.internet import defer
19
20		from synapse.api.errors import SynapseError
21
22		from ._base import SQLBaseStore
23
24		# The category ID for the "default" category. We don't store as null in the
25		# database to avoid the fun of null != null
26		_DEFAULT_CATEGORY_ID = ""
27		_DEFAULT_ROLE_ID = ""
28
29
30		class GroupServerStore(SQLBaseStore):
31		def set_group_join_policy(self, group_id, join_policy):
32		"""Set the join policy of a group.
33
34		join_policy can be one of:
35		* "invite"
36		* "open"
37		"""
38		return self._simple_update_one(
39		table="groups",
40		keyvalues={"group_id": group_id},
41		updatevalues={"join_policy": join_policy},
42		desc="set_group_join_policy",
43		)
44
45		def get_group(self, group_id):
46		return self._simple_select_one(
47		table="groups",
48		keyvalues={"group_id": group_id},
49		retcols=(
50		"name",
51		"short_description",
52		"long_description",
53		"avatar_url",
54		"is_public",
55		"join_policy",
56		),
57		allow_none=True,
58		desc="get_group",
59		)
60
61		def get_users_in_group(self, group_id, include_private=False):
62		# TODO: Pagination
63
64		keyvalues = {"group_id": group_id}
65		if not include_private:
66		keyvalues["is_public"] = True
67
68		return self._simple_select_list(
69		table="group_users",
70		keyvalues=keyvalues,
71		retcols=("user_id", "is_public", "is_admin"),
72		desc="get_users_in_group",
73		)
74
75		def get_invited_users_in_group(self, group_id):
76		# TODO: Pagination
77
78		return self._simple_select_onecol(
79		table="group_invites",
80		keyvalues={"group_id": group_id},
81		retcol="user_id",
82		desc="get_invited_users_in_group",
83		)
84
85		def get_rooms_in_group(self, group_id, include_private=False):
86		# TODO: Pagination
87
88		keyvalues = {"group_id": group_id}
89		if not include_private:
90		keyvalues["is_public"] = True
91
92		return self._simple_select_list(
93		table="group_rooms",
94		keyvalues=keyvalues,
95		retcols=("room_id", "is_public"),
96		desc="get_rooms_in_group",
97		)
98
99		def get_rooms_for_summary_by_category(self, group_id, include_private=False):
100		"""Get the rooms and categories that should be included in a summary request
101
102		Returns ([rooms], [categories])
103		"""
104
105		def _get_rooms_for_summary_txn(txn):
106		keyvalues = {"group_id": group_id}
107		if not include_private:
108		keyvalues["is_public"] = True
109
110		sql = """
111		SELECT room_id, is_public, category_id, room_order
112		FROM group_summary_rooms
113		WHERE group_id = ?
114		"""
115
116		if not include_private:
117		sql += " AND is_public = ?"
118		txn.execute(sql, (group_id, True))
119		else:
120		txn.execute(sql, (group_id,))
121
122		rooms = [
123		{
124		"room_id": row[0],
125		"is_public": row[1],
126		"category_id": row[2] if row[2] != _DEFAULT_CATEGORY_ID else None,
127		"order": row[3],
128		}
129		for row in txn
130		]
131
132		sql = """
133		SELECT category_id, is_public, profile, cat_order
134		FROM group_summary_room_categories
135		INNER JOIN group_room_categories USING (group_id, category_id)
136		WHERE group_id = ?
137		"""
138
139		if not include_private:
140		sql += " AND is_public = ?"
141		txn.execute(sql, (group_id, True))
142		else:
143		txn.execute(sql, (group_id,))
144
145		categories = {
146		row[0]: {
147		"is_public": row[1],
148		"profile": json.loads(row[2]),
149		"order": row[3],
150		}
151		for row in txn
152		}
153
154		return rooms, categories
155
156		return self.runInteraction("get_rooms_for_summary", _get_rooms_for_summary_txn)
157
158		def add_room_to_summary(self, group_id, room_id, category_id, order, is_public):
159		return self.runInteraction(
160		"add_room_to_summary",
161		self._add_room_to_summary_txn,
162		group_id,
163		room_id,
164		category_id,
165		order,
166		is_public,
167		)
168
169		def _add_room_to_summary_txn(
170		self, txn, group_id, room_id, category_id, order, is_public
171		):
172		"""Add (or update) room's entry in summary.
173
174		Args:
175		group_id (str)
176		room_id (str)
177		category_id (str): If not None then adds the category to the end of
178		the summary if its not already there. [Optional]
179		order (int): If not None inserts the room at that position, e.g.
180		an order of 1 will put the room first. Otherwise, the room gets
181		added to the end.
182		"""
183		room_in_group = self._simple_select_one_onecol_txn(
184		txn,
185		table="group_rooms",
186		keyvalues={"group_id": group_id, "room_id": room_id},
187		retcol="room_id",
188		allow_none=True,
189		)
190		if not room_in_group:
191		raise SynapseError(400, "room not in group")
192
193		if category_id is None:
194		category_id = _DEFAULT_CATEGORY_ID
195		else:
196		cat_exists = self._simple_select_one_onecol_txn(
197		txn,
198		table="group_room_categories",
199		keyvalues={"group_id": group_id, "category_id": category_id},
200		retcol="group_id",
201		allow_none=True,
202		)
203		if not cat_exists:
204		raise SynapseError(400, "Category doesn't exist")
205
206		# TODO: Check category is part of summary already
207		cat_exists = self._simple_select_one_onecol_txn(
208		txn,
209		table="group_summary_room_categories",
210		keyvalues={"group_id": group_id, "category_id": category_id},
211		retcol="group_id",
212		allow_none=True,
213		)
214		if not cat_exists:
215		# If not, add it with an order larger than all others
216		txn.execute(
217		"""
218		INSERT INTO group_summary_room_categories
219		(group_id, category_id, cat_order)
220		SELECT ?, ?, COALESCE(MAX(cat_order), 0) + 1
221		FROM group_summary_room_categories
222		WHERE group_id = ? AND category_id = ?
223		""",
224		(group_id, category_id, group_id, category_id),
225		)
226
227		existing = self._simple_select_one_txn(
228		txn,
229		table="group_summary_rooms",
230		keyvalues={
231		"group_id": group_id,
232		"room_id": room_id,
233		"category_id": category_id,
234		},
235		retcols=("room_order", "is_public"),
236		allow_none=True,
237		)
238
239		if order is not None:
240		# Shuffle other room orders that come after the given order
241		sql = """
242		UPDATE group_summary_rooms SET room_order = room_order + 1
243		WHERE group_id = ? AND category_id = ? AND room_order >= ?
244		"""
245		txn.execute(sql, (group_id, category_id, order))
246		elif not existing:
247		sql = """
248		SELECT COALESCE(MAX(room_order), 0) + 1 FROM group_summary_rooms
249		WHERE group_id = ? AND category_id = ?
250		"""
251		txn.execute(sql, (group_id, category_id))
252		order, = txn.fetchone()
253
254		if existing:
255		to_update = {}
256		if order is not None:
257		to_update["room_order"] = order
258		if is_public is not None:
259		to_update["is_public"] = is_public
260		self._simple_update_txn(
261		txn,
262		table="group_summary_rooms",
263		keyvalues={
264		"group_id": group_id,
265		"category_id": category_id,
266		"room_id": room_id,
267		},
268		values=to_update,
269		)
270		else:
271		if is_public is None:
272		is_public = True
273
274		self._simple_insert_txn(
275		txn,
276		table="group_summary_rooms",
277		values={
278		"group_id": group_id,
279		"category_id": category_id,
280		"room_id": room_id,
281		"room_order": order,
282		"is_public": is_public,
283		},
284		)
285
286		def remove_room_from_summary(self, group_id, room_id, category_id):
287		if category_id is None:
288		category_id = _DEFAULT_CATEGORY_ID
289
290		return self._simple_delete(
291		table="group_summary_rooms",
292		keyvalues={
293		"group_id": group_id,
294		"category_id": category_id,
295		"room_id": room_id,
296		},
297		desc="remove_room_from_summary",
298		)
299
300		@defer.inlineCallbacks
301		def get_group_categories(self, group_id):
302		rows = yield self._simple_select_list(
303		table="group_room_categories",
304		keyvalues={"group_id": group_id},
305		retcols=("category_id", "is_public", "profile"),
306		desc="get_group_categories",
307		)
308
309		return {
310		row["category_id"]: {
311		"is_public": row["is_public"],
312		"profile": json.loads(row["profile"]),
313		}
314		for row in rows
315		}
316
317		@defer.inlineCallbacks
318		def get_group_category(self, group_id, category_id):
319		category = yield self._simple_select_one(
320		table="group_room_categories",
321		keyvalues={"group_id": group_id, "category_id": category_id},
322		retcols=("is_public", "profile"),
323		desc="get_group_category",
324		)
325
326		category["profile"] = json.loads(category["profile"])
327
328		return category
329
330		def upsert_group_category(self, group_id, category_id, profile, is_public):
331		"""Add/update room category for group
332		"""
333		insertion_values = {}
334		update_values = {"category_id": category_id} # This cannot be empty
335
336		if profile is None:
337		insertion_values["profile"] = "{}"
338		else:
339		update_values["profile"] = json.dumps(profile)
340
341		if is_public is None:
342		insertion_values["is_public"] = True
343		else:
344		update_values["is_public"] = is_public
345
346		return self._simple_upsert(
347		table="group_room_categories",
348		keyvalues={"group_id": group_id, "category_id": category_id},
349		values=update_values,
350		insertion_values=insertion_values,
351		desc="upsert_group_category",
352		)
353
354		def remove_group_category(self, group_id, category_id):
355		return self._simple_delete(
356		table="group_room_categories",
357		keyvalues={"group_id": group_id, "category_id": category_id},
358		desc="remove_group_category",
359		)
360
361		@defer.inlineCallbacks
362		def get_group_roles(self, group_id):
363		rows = yield self._simple_select_list(
364		table="group_roles",
365		keyvalues={"group_id": group_id},
366		retcols=("role_id", "is_public", "profile"),
367		desc="get_group_roles",
368		)
369
370		return {
371		row["role_id"]: {
372		"is_public": row["is_public"],
373		"profile": json.loads(row["profile"]),
374		}
375		for row in rows
376		}
377
378		@defer.inlineCallbacks
379		def get_group_role(self, group_id, role_id):
380		role = yield self._simple_select_one(
381		table="group_roles",
382		keyvalues={"group_id": group_id, "role_id": role_id},
383		retcols=("is_public", "profile"),
384		desc="get_group_role",
385		)
386
387		role["profile"] = json.loads(role["profile"])
388
389		return role
390
391		def upsert_group_role(self, group_id, role_id, profile, is_public):
392		"""Add/remove user role
393		"""
394		insertion_values = {}
395		update_values = {"role_id": role_id} # This cannot be empty
396
397		if profile is None:
398		insertion_values["profile"] = "{}"
399		else:
400		update_values["profile"] = json.dumps(profile)
401
402		if is_public is None:
403		insertion_values["is_public"] = True
404		else:
405		update_values["is_public"] = is_public
406
407		return self._simple_upsert(
408		table="group_roles",
409		keyvalues={"group_id": group_id, "role_id": role_id},
410		values=update_values,
411		insertion_values=insertion_values,
412		desc="upsert_group_role",
413		)
414
415		def remove_group_role(self, group_id, role_id):
416		return self._simple_delete(
417		table="group_roles",
418		keyvalues={"group_id": group_id, "role_id": role_id},
419		desc="remove_group_role",
420		)
421
422		def add_user_to_summary(self, group_id, user_id, role_id, order, is_public):
423		return self.runInteraction(
424		"add_user_to_summary",
425		self._add_user_to_summary_txn,
426		group_id,
427		user_id,
428		role_id,
429		order,
430		is_public,
431		)
432
433		def _add_user_to_summary_txn(
434		self, txn, group_id, user_id, role_id, order, is_public
435		):
436		"""Add (or update) user's entry in summary.
437
438		Args:
439		group_id (str)
440		user_id (str)
441		role_id (str): If not None then adds the role to the end of
442		the summary if its not already there. [Optional]
443		order (int): If not None inserts the user at that position, e.g.
444		an order of 1 will put the user first. Otherwise, the user gets
445		added to the end.
446		"""
447		user_in_group = self._simple_select_one_onecol_txn(
448		txn,
449		table="group_users",
450		keyvalues={"group_id": group_id, "user_id": user_id},
451		retcol="user_id",
452		allow_none=True,
453		)
454		if not user_in_group:
455		raise SynapseError(400, "user not in group")
456
457		if role_id is None:
458		role_id = _DEFAULT_ROLE_ID
459		else:
460		role_exists = self._simple_select_one_onecol_txn(
461		txn,
462		table="group_roles",
463		keyvalues={"group_id": group_id, "role_id": role_id},
464		retcol="group_id",
465		allow_none=True,
466		)
467		if not role_exists:
468		raise SynapseError(400, "Role doesn't exist")
469
470		# TODO: Check role is part of the summary already
471		role_exists = self._simple_select_one_onecol_txn(
472		txn,
473		table="group_summary_roles",
474		keyvalues={"group_id": group_id, "role_id": role_id},
475		retcol="group_id",
476		allow_none=True,
477		)
478		if not role_exists:
479		# If not, add it with an order larger than all others
480		txn.execute(
481		"""
482		INSERT INTO group_summary_roles
483		(group_id, role_id, role_order)
484		SELECT ?, ?, COALESCE(MAX(role_order), 0) + 1
485		FROM group_summary_roles
486		WHERE group_id = ? AND role_id = ?
487		""",
488		(group_id, role_id, group_id, role_id),
489		)
490
491		existing = self._simple_select_one_txn(
492		txn,
493		table="group_summary_users",
494		keyvalues={"group_id": group_id, "user_id": user_id, "role_id": role_id},
495		retcols=("user_order", "is_public"),
496		allow_none=True,
497		)
498
499		if order is not None:
500		# Shuffle other users orders that come after the given order
501		sql = """
502		UPDATE group_summary_users SET user_order = user_order + 1
503		WHERE group_id = ? AND role_id = ? AND user_order >= ?
504		"""
505		txn.execute(sql, (group_id, role_id, order))
506		elif not existing:
507		sql = """
508		SELECT COALESCE(MAX(user_order), 0) + 1 FROM group_summary_users
509		WHERE group_id = ? AND role_id = ?
510		"""
511		txn.execute(sql, (group_id, role_id))
512		order, = txn.fetchone()
513
514		if existing:
515		to_update = {}
516		if order is not None:
517		to_update["user_order"] = order
518		if is_public is not None:
519		to_update["is_public"] = is_public
520		self._simple_update_txn(
521		txn,
522		table="group_summary_users",
523		keyvalues={
524		"group_id": group_id,
525		"role_id": role_id,
526		"user_id": user_id,
527		},
528		values=to_update,
529		)
530		else:
531		if is_public is None:
532		is_public = True
533
534		self._simple_insert_txn(
535		txn,
536		table="group_summary_users",
537		values={
538		"group_id": group_id,
539		"role_id": role_id,
540		"user_id": user_id,
541		"user_order": order,
542		"is_public": is_public,
543		},
544		)
545
546		def remove_user_from_summary(self, group_id, user_id, role_id):
547		if role_id is None:
548		role_id = _DEFAULT_ROLE_ID
549
550		return self._simple_delete(
551		table="group_summary_users",
552		keyvalues={"group_id": group_id, "role_id": role_id, "user_id": user_id},
553		desc="remove_user_from_summary",
554		)
555
556		def get_users_for_summary_by_role(self, group_id, include_private=False):
557		"""Get the users and roles that should be included in a summary request
558
559		Returns ([users], [roles])
560		"""
561
562		def _get_users_for_summary_txn(txn):
563		keyvalues = {"group_id": group_id}
564		if not include_private:
565		keyvalues["is_public"] = True
566
567		sql = """
568		SELECT user_id, is_public, role_id, user_order
569		FROM group_summary_users
570		WHERE group_id = ?
571		"""
572
573		if not include_private:
574		sql += " AND is_public = ?"
575		txn.execute(sql, (group_id, True))
576		else:
577		txn.execute(sql, (group_id,))
578
579		users = [
580		{
581		"user_id": row[0],
582		"is_public": row[1],
583		"role_id": row[2] if row[2] != _DEFAULT_ROLE_ID else None,
584		"order": row[3],
585		}
586		for row in txn
587		]
588
589		sql = """
590		SELECT role_id, is_public, profile, role_order
591		FROM group_summary_roles
592		INNER JOIN group_roles USING (group_id, role_id)
593		WHERE group_id = ?
594		"""
595
596		if not include_private:
597		sql += " AND is_public = ?"
598		txn.execute(sql, (group_id, True))
599		else:
600		txn.execute(sql, (group_id,))
601
602		roles = {
603		row[0]: {
604		"is_public": row[1],
605		"profile": json.loads(row[2]),
606		"order": row[3],
607		}
608		for row in txn
609		}
610
611		return users, roles
612
613		return self.runInteraction(
614		"get_users_for_summary_by_role", _get_users_for_summary_txn
615		)
616
617		def is_user_in_group(self, user_id, group_id):
618		return self._simple_select_one_onecol(
619		table="group_users",
620		keyvalues={"group_id": group_id, "user_id": user_id},
621		retcol="user_id",
622		allow_none=True,
623		desc="is_user_in_group",
624		).addCallback(lambda r: bool(r))
625
626		def is_user_admin_in_group(self, group_id, user_id):
627		return self._simple_select_one_onecol(
628		table="group_users",
629		keyvalues={"group_id": group_id, "user_id": user_id},
630		retcol="is_admin",
631		allow_none=True,
632		desc="is_user_admin_in_group",
633		)
634
635		def add_group_invite(self, group_id, user_id):
636		"""Record that the group server has invited a user
637		"""
638		return self._simple_insert(
639		table="group_invites",
640		values={"group_id": group_id, "user_id": user_id},
641		desc="add_group_invite",
642		)
643
644		def is_user_invited_to_local_group(self, group_id, user_id):
645		"""Has the group server invited a user?
646		"""
647		return self._simple_select_one_onecol(
648		table="group_invites",
649		keyvalues={"group_id": group_id, "user_id": user_id},
650		retcol="user_id",
651		desc="is_user_invited_to_local_group",
652		allow_none=True,
653		)
654
655		def get_users_membership_info_in_group(self, group_id, user_id):
656		"""Get a dict describing the membership of a user in a group.
657
658		Example if joined:
659
660		{
661		"membership": "join",
662		"is_public": True,
663		"is_privileged": False,
664		}
665
666		Returns an empty dict if the user is not join/invite/etc
667		"""
668
669		def _get_users_membership_in_group_txn(txn):
670		row = self._simple_select_one_txn(
671		txn,
672		table="group_users",
673		keyvalues={"group_id": group_id, "user_id": user_id},
674		retcols=("is_admin", "is_public"),
675		allow_none=True,
676		)
677
678		if row:
679		return {
680		"membership": "join",
681		"is_public": row["is_public"],
682		"is_privileged": row["is_admin"],
683		}
684
685		row = self._simple_select_one_onecol_txn(
686		txn,
687		table="group_invites",
688		keyvalues={"group_id": group_id, "user_id": user_id},
689		retcol="user_id",
690		allow_none=True,
691		)
692
693		if row:
694		return {"membership": "invite"}
695
696		return {}
697
698		return self.runInteraction(
699		"get_users_membership_info_in_group", _get_users_membership_in_group_txn
700		)
701
702		def add_user_to_group(
703		self,
704		group_id,
705		user_id,
706		is_admin=False,
707		is_public=True,
708		local_attestation=None,
709		remote_attestation=None,
710		):
711		"""Add a user to the group server.
712
713		Args:
714		group_id (str)
715		user_id (str)
716		is_admin (bool)
717		is_public (bool)
718		local_attestation (dict): The attestation the GS created to give
719		to the remote server. Optional if the user and group are on the
720		same server
721		remote_attestation (dict): The attestation given to GS by remote
722		server. Optional if the user and group are on the same server
723		"""
724
725		def _add_user_to_group_txn(txn):
726		self._simple_insert_txn(
727		txn,
728		table="group_users",
729		values={
730		"group_id": group_id,
731		"user_id": user_id,
732		"is_admin": is_admin,
733		"is_public": is_public,
734		},
735		)
736
737		self._simple_delete_txn(
738		txn,
739		table="group_invites",
740		keyvalues={"group_id": group_id, "user_id": user_id},
741		)
742
743		if local_attestation:
744		self._simple_insert_txn(
745		txn,
746		table="group_attestations_renewals",
747		values={
748		"group_id": group_id,
749		"user_id": user_id,
750		"valid_until_ms": local_attestation["valid_until_ms"],
751		},
752		)
753		if remote_attestation:
754		self._simple_insert_txn(
755		txn,
756		table="group_attestations_remote",
757		values={
758		"group_id": group_id,
759		"user_id": user_id,
760		"valid_until_ms": remote_attestation["valid_until_ms"],
761		"attestation_json": json.dumps(remote_attestation),
762		},
763		)
764
765		return self.runInteraction("add_user_to_group", _add_user_to_group_txn)
766
767		def remove_user_from_group(self, group_id, user_id):
768		def _remove_user_from_group_txn(txn):
769		self._simple_delete_txn(
770		txn,
771		table="group_users",
772		keyvalues={"group_id": group_id, "user_id": user_id},
773		)
774		self._simple_delete_txn(
775		txn,
776		table="group_invites",
777		keyvalues={"group_id": group_id, "user_id": user_id},
778		)
779		self._simple_delete_txn(
780		txn,
781		table="group_attestations_renewals",
782		keyvalues={"group_id": group_id, "user_id": user_id},
783		)
784		self._simple_delete_txn(
785		txn,
786		table="group_attestations_remote",
787		keyvalues={"group_id": group_id, "user_id": user_id},
788		)
789		self._simple_delete_txn(
790		txn,
791		table="group_summary_users",
792		keyvalues={"group_id": group_id, "user_id": user_id},
793		)
794
795		return self.runInteraction(
796		"remove_user_from_group", _remove_user_from_group_txn
797		)
798
799		def add_room_to_group(self, group_id, room_id, is_public):
800		return self._simple_insert(
801		table="group_rooms",
802		values={"group_id": group_id, "room_id": room_id, "is_public": is_public},
803		desc="add_room_to_group",
804		)
805
806		def update_room_in_group_visibility(self, group_id, room_id, is_public):
807		return self._simple_update(
808		table="group_rooms",
809		keyvalues={"group_id": group_id, "room_id": room_id},
810		updatevalues={"is_public": is_public},
811		desc="update_room_in_group_visibility",
812		)
813
814		def remove_room_from_group(self, group_id, room_id):
815		def _remove_room_from_group_txn(txn):
816		self._simple_delete_txn(
817		txn,
818		table="group_rooms",
819		keyvalues={"group_id": group_id, "room_id": room_id},
820		)
821
822		self._simple_delete_txn(
823		txn,
824		table="group_summary_rooms",
825		keyvalues={"group_id": group_id, "room_id": room_id},
826		)
827
828		return self.runInteraction(
829		"remove_room_from_group", _remove_room_from_group_txn
830		)
831
832		def get_publicised_groups_for_user(self, user_id):
833		"""Get all groups a user is publicising
834		"""
835		return self._simple_select_onecol(
836		table="local_group_membership",
837		keyvalues={"user_id": user_id, "membership": "join", "is_publicised": True},
838		retcol="group_id",
839		desc="get_publicised_groups_for_user",
840		)
841
842		def update_group_publicity(self, group_id, user_id, publicise):
843		"""Update whether the user is publicising their membership of the group
844		"""
845		return self._simple_update_one(
846		table="local_group_membership",
847		keyvalues={"group_id": group_id, "user_id": user_id},
848		updatevalues={"is_publicised": publicise},
849		desc="update_group_publicity",
850		)
851
852		@defer.inlineCallbacks
853		def register_user_group_membership(
854		self,
855		group_id,
856		user_id,
857		membership,
858		is_admin=False,
859		content={},
860		local_attestation=None,
861		remote_attestation=None,
862		is_publicised=False,
863		):
864		"""Registers that a local user is a member of a (local or remote) group.
865
866		Args:
867		group_id (str)
868		user_id (str)
869		membership (str)
870		is_admin (bool)
871		content (dict): Content of the membership, e.g. includes the inviter
872		if the user has been invited.
873		local_attestation (dict): If remote group then store the fact that we
874		have given out an attestation, else None.
875		remote_attestation (dict): If remote group then store the remote
876		attestation from the group, else None.
877		"""
878
879		def _register_user_group_membership_txn(txn, next_id):
880		# TODO: Upsert?
881		self._simple_delete_txn(
882		txn,
883		table="local_group_membership",
884		keyvalues={"group_id": group_id, "user_id": user_id},
885		)
886		self._simple_insert_txn(
887		txn,
888		table="local_group_membership",
889		values={
890		"group_id": group_id,
891		"user_id": user_id,
892		"is_admin": is_admin,
893		"membership": membership,
894		"is_publicised": is_publicised,
895		"content": json.dumps(content),
896		},
897		)
898
899		self._simple_insert_txn(
900		txn,
901		table="local_group_updates",
902		values={
903		"stream_id": next_id,
904		"group_id": group_id,
905		"user_id": user_id,
906		"type": "membership",
907		"content": json.dumps(
908		{"membership": membership, "content": content}
909		),
910		},
911		)
912		self._group_updates_stream_cache.entity_has_changed(user_id, next_id)
913
914		# TODO: Insert profile to ensure it comes down stream if its a join.
915
916		if membership == "join":
917		if local_attestation:
918		self._simple_insert_txn(
919		txn,
920		table="group_attestations_renewals",
921		values={
922		"group_id": group_id,
923		"user_id": user_id,
924		"valid_until_ms": local_attestation["valid_until_ms"],
925		},
926		)
927		if remote_attestation:
928		self._simple_insert_txn(
929		txn,
930		table="group_attestations_remote",
931		values={
932		"group_id": group_id,
933		"user_id": user_id,
934		"valid_until_ms": remote_attestation["valid_until_ms"],
935		"attestation_json": json.dumps(remote_attestation),
936		},
937		)
938		else:
939		self._simple_delete_txn(
940		txn,
941		table="group_attestations_renewals",
942		keyvalues={"group_id": group_id, "user_id": user_id},
943		)
944		self._simple_delete_txn(
945		txn,
946		table="group_attestations_remote",
947		keyvalues={"group_id": group_id, "user_id": user_id},
948		)
949
950		return next_id
951
952		with self._group_updates_id_gen.get_next() as next_id:
953		res = yield self.runInteraction(
954		"register_user_group_membership",
955		_register_user_group_membership_txn,
956		next_id,
957		)
958		return res
959
960		@defer.inlineCallbacks
961		def create_group(
962		self, group_id, user_id, name, avatar_url, short_description, long_description
963		):
964		yield self._simple_insert(
965		table="groups",
966		values={
967		"group_id": group_id,
968		"name": name,
969		"avatar_url": avatar_url,
970		"short_description": short_description,
971		"long_description": long_description,
972		"is_public": True,
973		},
974		desc="create_group",
975		)
976
977		@defer.inlineCallbacks
978		def update_group_profile(self, group_id, profile):
979		yield self._simple_update_one(
980		table="groups",
981		keyvalues={"group_id": group_id},
982		updatevalues=profile,
983		desc="update_group_profile",
984		)
985
986		def get_attestations_need_renewals(self, valid_until_ms):
987		"""Get all attestations that need to be renewed until givent time
988		"""
989
990		def _get_attestations_need_renewals_txn(txn):
991		sql = """
992		SELECT group_id, user_id FROM group_attestations_renewals
993		WHERE valid_until_ms <= ?
994		"""
995		txn.execute(sql, (valid_until_ms,))
996		return self.cursor_to_dict(txn)
997
998		return self.runInteraction(
999		"get_attestations_need_renewals", _get_attestations_need_renewals_txn
1000		)
1001
1002		def update_attestation_renewal(self, group_id, user_id, attestation):
1003		"""Update an attestation that we have renewed
1004		"""
1005		return self._simple_update_one(
1006		table="group_attestations_renewals",
1007		keyvalues={"group_id": group_id, "user_id": user_id},
1008		updatevalues={"valid_until_ms": attestation["valid_until_ms"]},
1009		desc="update_attestation_renewal",
1010		)
1011
1012		def update_remote_attestion(self, group_id, user_id, attestation):
1013		"""Update an attestation that a remote has renewed
1014		"""
1015		return self._simple_update_one(
1016		table="group_attestations_remote",
1017		keyvalues={"group_id": group_id, "user_id": user_id},
1018		updatevalues={
1019		"valid_until_ms": attestation["valid_until_ms"],
1020		"attestation_json": json.dumps(attestation),
1021		},
1022		desc="update_remote_attestion",
1023		)
1024
1025		def remove_attestation_renewal(self, group_id, user_id):
1026		"""Remove an attestation that we thought we should renew, but actually
1027		shouldn't. Ideally this would never get called as we would never
1028		incorrectly try and do attestations for local users on local groups.
1029
1030		Args:
1031		group_id (str)
1032		user_id (str)
1033		"""
1034		return self._simple_delete(
1035		table="group_attestations_renewals",
1036		keyvalues={"group_id": group_id, "user_id": user_id},
1037		desc="remove_attestation_renewal",
1038		)
1039
1040		@defer.inlineCallbacks
1041		def get_remote_attestation(self, group_id, user_id):
1042		"""Get the attestation that proves the remote agrees that the user is
1043		in the group.
1044		"""
1045		row = yield self._simple_select_one(
1046		table="group_attestations_remote",
1047		keyvalues={"group_id": group_id, "user_id": user_id},
1048		retcols=("valid_until_ms", "attestation_json"),
1049		desc="get_remote_attestation",
1050		allow_none=True,
1051		)
1052
1053		now = int(self._clock.time_msec())
1054		if row and now < row["valid_until_ms"]:
1055		return json.loads(row["attestation_json"])
1056
1057		return None
1058
1059		def get_joined_groups(self, user_id):
1060		return self._simple_select_onecol(
1061		table="local_group_membership",
1062		keyvalues={"user_id": user_id, "membership": "join"},
1063		retcol="group_id",
1064		desc="get_joined_groups",
1065		)
1066
1067		def get_all_groups_for_user(self, user_id, now_token):
1068		def _get_all_groups_for_user_txn(txn):
1069		sql = """
1070		SELECT group_id, type, membership, u.content
1071		FROM local_group_updates AS u
1072		INNER JOIN local_group_membership USING (group_id, user_id)
1073		WHERE user_id = ? AND membership != 'leave'
1074		AND stream_id <= ?
1075		"""
1076		txn.execute(sql, (user_id, now_token))
1077		return [
1078		{
1079		"group_id": row[0],
1080		"type": row[1],
1081		"membership": row[2],
1082		"content": json.loads(row[3]),
1083		}
1084		for row in txn
1085		]
1086
1087		return self.runInteraction(
1088		"get_all_groups_for_user", _get_all_groups_for_user_txn
1089		)
1090
1091		def get_groups_changes_for_user(self, user_id, from_token, to_token):
1092		from_token = int(from_token)
1093		has_changed = self._group_updates_stream_cache.has_entity_changed(
1094		user_id, from_token
1095		)
1096		if not has_changed:
1097		return []
1098
1099		def _get_groups_changes_for_user_txn(txn):
1100		sql = """
1101		SELECT group_id, membership, type, u.content
1102		FROM local_group_updates AS u
1103		INNER JOIN local_group_membership USING (group_id, user_id)
1104		WHERE user_id = ? AND ? < stream_id AND stream_id <= ?
1105		"""
1106		txn.execute(sql, (user_id, from_token, to_token))
1107		return [
1108		{
1109		"group_id": group_id,
1110		"membership": membership,
1111		"type": gtype,
1112		"content": json.loads(content_json),
1113		}
1114		for group_id, membership, gtype, content_json in txn
1115		]
1116
1117		return self.runInteraction(
1118		"get_groups_changes_for_user", _get_groups_changes_for_user_txn
1119		)
1120
1121		def get_all_groups_changes(self, from_token, to_token, limit):
1122		from_token = int(from_token)
1123		has_changed = self._group_updates_stream_cache.has_any_entity_changed(
1124		from_token
1125		)
1126		if not has_changed:
1127		return []
1128
1129		def _get_all_groups_changes_txn(txn):
1130		sql = """
1131		SELECT stream_id, group_id, user_id, type, content
1132		FROM local_group_updates
1133		WHERE ? < stream_id AND stream_id <= ?
1134		LIMIT ?
1135		"""
1136		txn.execute(sql, (from_token, to_token, limit))
1137		return [
1138		(stream_id, group_id, user_id, gtype, json.loads(content_json))
1139		for stream_id, group_id, user_id, gtype, content_json in txn
1140		]
1141
1142		return self.runInteraction(
1143		"get_all_groups_changes", _get_all_groups_changes_txn
1144		)
1145
1146		def get_group_stream_token(self):
1147		return self._group_updates_id_gen.get_current_token()
1148
1149		def delete_group(self, group_id):
1150		"""Deletes a group fully from the database.
1151
1152		Args:
1153		group_id (str)
1154
1155		Returns:
1156		Deferred
1157		"""
1158
1159		def _delete_group_txn(txn):
1160		tables = [
1161		"groups",
1162		"group_users",
1163		"group_invites",
1164		"group_rooms",
1165		"group_summary_rooms",
1166		"group_summary_room_categories",
1167		"group_room_categories",
1168		"group_summary_users",
1169		"group_summary_roles",
1170		"group_roles",
1171		"group_attestations_renewals",
1172		"group_attestations_remote",
1173		]
1174
1175		for table in tables:
1176		self._simple_delete_txn(
1177		txn, table=table, keyvalues={"group_id": group_id}
1178		)
1179
1180		return self.runInteraction("delete_group", _delete_group_txn)

+0

-194

synapse/storage/keys.py less more

13	13	# See the License for the specific language governing permissions and
14	14	# limitations under the License.
15	15
16		import itertools
17	16	import logging
18	17
19		import six
20
21	18	import attr
22		from signedjson.key import decode_verify_key_bytes
23
24		from synapse.util import batch_iter
25		from synapse.util.caches.descriptors import cached, cachedList
26
27		from ._base import SQLBaseStore
28	19
29	20	logger = logging.getLogger(__name__)
30
31		# py2 sqlite has buffer hardcoded as only binary type, so we must use it,
32		# despite being deprecated and removed in favor of memoryview
33		if six.PY2:
34		db_binary_type = six.moves.builtins.buffer
35		else:
36		db_binary_type = memoryview
37	21
38	22
39	23	@attr.s(slots=True, frozen=True)
40	24	class FetchKeyResult(object):
41	25	verify_key = attr.ib() # VerifyKey: the key itself
42	26	valid_until_ts = attr.ib() # int: how long we can use this key for
43
44
45		class KeyStore(SQLBaseStore):
46		"""Persistence for signature verification keys
47		"""
48
49		@cached()
50		def _get_server_verify_key(self, server_name_and_key_id):
51		raise NotImplementedError()
52
53		@cachedList(
54		cached_method_name="_get_server_verify_key", list_name="server_name_and_key_ids"
55		)
56		def get_server_verify_keys(self, server_name_and_key_ids):
57		"""
58		Args:
59		server_name_and_key_ids (iterable[Tuple[str, str]]):
60		iterable of (server_name, key-id) tuples to fetch keys for
61
62		Returns:
63		Deferred: resolves to dict[Tuple[str, str], FetchKeyResult\|None]:
64		map from (server_name, key_id) -> FetchKeyResult, or None if the key is
65		unknown
66		"""
67		keys = {}
68
69		def _get_keys(txn, batch):
70		"""Processes a batch of keys to fetch, and adds the result to `keys`."""
71
72		# batch_iter always returns tuples so it's safe to do len(batch)
73		sql = (
74		"SELECT server_name, key_id, verify_key, ts_valid_until_ms "
75		"FROM server_signature_keys WHERE 1=0"
76		) + " OR (server_name=? AND key_id=?)" * len(batch)
77
78		txn.execute(sql, tuple(itertools.chain.from_iterable(batch)))
79
80		for row in txn:
81		server_name, key_id, key_bytes, ts_valid_until_ms = row
82
83		if ts_valid_until_ms is None:
84		# Old keys may be stored with a ts_valid_until_ms of null,
85		# in which case we treat this as if it was set to `0`, i.e.
86		# it won't match key requests that define a minimum
87		# `ts_valid_until_ms`.
88		ts_valid_until_ms = 0
89
90		res = FetchKeyResult(
91		verify_key=decode_verify_key_bytes(key_id, bytes(key_bytes)),
92		valid_until_ts=ts_valid_until_ms,
93		)
94		keys[(server_name, key_id)] = res
95
96		def _txn(txn):
97		for batch in batch_iter(server_name_and_key_ids, 50):
98		_get_keys(txn, batch)
99		return keys
100
101		return self.runInteraction("get_server_verify_keys", _txn)
102
103		def store_server_verify_keys(self, from_server, ts_added_ms, verify_keys):
104		"""Stores NACL verification keys for remote servers.
105		Args:
106		from_server (str): Where the verification keys were looked up
107		ts_added_ms (int): The time to record that the key was added
108		verify_keys (iterable[tuple[str, str, FetchKeyResult]]):
109		keys to be stored. Each entry is a triplet of
110		(server_name, key_id, key).
111		"""
112		key_values = []
113		value_values = []
114		invalidations = []
115		for server_name, key_id, fetch_result in verify_keys:
116		key_values.append((server_name, key_id))
117		value_values.append(
118		(
119		from_server,
120		ts_added_ms,
121		fetch_result.valid_until_ts,
122		db_binary_type(fetch_result.verify_key.encode()),
123		)
124		)
125		# invalidate takes a tuple corresponding to the params of
126		# _get_server_verify_key. _get_server_verify_key only takes one
127		# param, which is itself the 2-tuple (server_name, key_id).
128		invalidations.append((server_name, key_id))
129
130		def _invalidate(res):
131		f = self._get_server_verify_key.invalidate
132		for i in invalidations:
133		f((i,))
134		return res
135
136		return self.runInteraction(
137		"store_server_verify_keys",
138		self._simple_upsert_many_txn,
139		table="server_signature_keys",
140		key_names=("server_name", "key_id"),
141		key_values=key_values,
142		value_names=(
143		"from_server",
144		"ts_added_ms",
145		"ts_valid_until_ms",
146		"verify_key",
147		),
148		value_values=value_values,
149		).addCallback(_invalidate)
150
151		def store_server_keys_json(
152		self, server_name, key_id, from_server, ts_now_ms, ts_expires_ms, key_json_bytes
153		):
154		"""Stores the JSON bytes for a set of keys from a server
155		The JSON should be signed by the originating server, the intermediate
156		server, and by this server. Updates the value for the
157		(server_name, key_id, from_server) triplet if one already existed.
158		Args:
159		server_name (str): The name of the server.
160		key_id (str): The identifer of the key this JSON is for.
161		from_server (str): The server this JSON was fetched from.
162		ts_now_ms (int): The time now in milliseconds.
163		ts_valid_until_ms (int): The time when this json stops being valid.
164		key_json (bytes): The encoded JSON.
165		"""
166		return self._simple_upsert(
167		table="server_keys_json",
168		keyvalues={
169		"server_name": server_name,
170		"key_id": key_id,
171		"from_server": from_server,
172		},
173		values={
174		"server_name": server_name,
175		"key_id": key_id,
176		"from_server": from_server,
177		"ts_added_ms": ts_now_ms,
178		"ts_valid_until_ms": ts_expires_ms,
179		"key_json": db_binary_type(key_json_bytes),
180		},
181		desc="store_server_keys_json",
182		)
183
184		def get_server_keys_json(self, server_keys):
185		"""Retrive the key json for a list of server_keys and key ids.
186		If no keys are found for a given server, key_id and source then
187		that server, key_id, and source triplet entry will be an empty list.
188		The JSON is returned as a byte array so that it can be efficiently
189		used in an HTTP response.
190		Args:
191		server_keys (list): List of (server_name, key_id, source) triplets.
192		Returns:
193		Deferred[dict[Tuple[str, str, str\|None], list[dict]]]:
194		Dict mapping (server_name, key_id, source) triplets to lists of dicts
195		"""
196
197		def _get_server_keys_json_txn(txn):
198		results = {}
199		for server_name, key_id, from_server in server_keys:
200		keyvalues = {"server_name": server_name}
201		if key_id is not None:
202		keyvalues["key_id"] = key_id
203		if from_server is not None:
204		keyvalues["from_server"] = from_server
205		rows = self._simple_select_list_txn(
206		txn,
207		"server_keys_json",
208		keyvalues=keyvalues,
209		retcols=(
210		"key_id",
211		"from_server",
212		"ts_added_ms",
213		"ts_valid_until_ms",
214		"key_json",
215		),
216		)
217		results[(server_name, key_id, from_server)] = rows
218		return results
219
220		return self.runInteraction("get_server_keys_json", _get_server_keys_json_txn)

+0

-373

~~synapse/storage/media_repository.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2014-2016 OpenMarket Ltd
2		#
3		# Licensed under the Apache License, Version 2.0 (the "License");
4		# you may not use this file except in compliance with the License.
5		# You may obtain a copy of the License at
6		#
7		# http://www.apache.org/licenses/LICENSE-2.0
8		#
9		# Unless required by applicable law or agreed to in writing, software
10		# distributed under the License is distributed on an "AS IS" BASIS,
11		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12		# See the License for the specific language governing permissions and
13		# limitations under the License.
14		from synapse.storage.background_updates import BackgroundUpdateStore
15
16
17		class MediaRepositoryStore(BackgroundUpdateStore):
18		"""Persistence for attachments and avatars"""
19
20		def __init__(self, db_conn, hs):
21		super(MediaRepositoryStore, self).__init__(db_conn, hs)
22
23		self.register_background_index_update(
24		update_name="local_media_repository_url_idx",
25		index_name="local_media_repository_url_idx",
26		table="local_media_repository",
27		columns=["created_ts"],
28		where_clause="url_cache IS NOT NULL",
29		)
30
31		def get_local_media(self, media_id):
32		"""Get the metadata for a local piece of media
33		Returns:
34		None if the media_id doesn't exist.
35		"""
36		return self._simple_select_one(
37		"local_media_repository",
38		{"media_id": media_id},
39		(
40		"media_type",
41		"media_length",
42		"upload_name",
43		"created_ts",
44		"quarantined_by",
45		"url_cache",
46		),
47		allow_none=True,
48		desc="get_local_media",
49		)
50
51		def store_local_media(
52		self,
53		media_id,
54		media_type,
55		time_now_ms,
56		upload_name,
57		media_length,
58		user_id,
59		url_cache=None,
60		):
61		return self._simple_insert(
62		"local_media_repository",
63		{
64		"media_id": media_id,
65		"media_type": media_type,
66		"created_ts": time_now_ms,
67		"upload_name": upload_name,
68		"media_length": media_length,
69		"user_id": user_id.to_string(),
70		"url_cache": url_cache,
71		},
72		desc="store_local_media",
73		)
74
75		def get_url_cache(self, url, ts):
76		"""Get the media_id and ts for a cached URL as of the given timestamp
77		Returns:
78		None if the URL isn't cached.
79		"""
80
81		def get_url_cache_txn(txn):
82		# get the most recently cached result (relative to the given ts)
83		sql = (
84		"SELECT response_code, etag, expires_ts, og, media_id, download_ts"
85		" FROM local_media_repository_url_cache"
86		" WHERE url = ? AND download_ts <= ?"
87		" ORDER BY download_ts DESC LIMIT 1"
88		)
89		txn.execute(sql, (url, ts))
90		row = txn.fetchone()
91
92		if not row:
93		# ...or if we've requested a timestamp older than the oldest
94		# copy in the cache, return the oldest copy (if any)
95		sql = (
96		"SELECT response_code, etag, expires_ts, og, media_id, download_ts"
97		" FROM local_media_repository_url_cache"
98		" WHERE url = ? AND download_ts > ?"
99		" ORDER BY download_ts ASC LIMIT 1"
100		)
101		txn.execute(sql, (url, ts))
102		row = txn.fetchone()
103
104		if not row:
105		return None
106
107		return dict(
108		zip(
109		(
110		"response_code",
111		"etag",
112		"expires_ts",
113		"og",
114		"media_id",
115		"download_ts",
116		),
117		row,
118		)
119		)
120
121		return self.runInteraction("get_url_cache", get_url_cache_txn)
122
123		def store_url_cache(
124		self, url, response_code, etag, expires_ts, og, media_id, download_ts
125		):
126		return self._simple_insert(
127		"local_media_repository_url_cache",
128		{
129		"url": url,
130		"response_code": response_code,
131		"etag": etag,
132		"expires_ts": expires_ts,
133		"og": og,
134		"media_id": media_id,
135		"download_ts": download_ts,
136		},
137		desc="store_url_cache",
138		)
139
140		def get_local_media_thumbnails(self, media_id):
141		return self._simple_select_list(
142		"local_media_repository_thumbnails",
143		{"media_id": media_id},
144		(
145		"thumbnail_width",
146		"thumbnail_height",
147		"thumbnail_method",
148		"thumbnail_type",
149		"thumbnail_length",
150		),
151		desc="get_local_media_thumbnails",
152		)
153
154		def store_local_thumbnail(
155		self,
156		media_id,
157		thumbnail_width,
158		thumbnail_height,
159		thumbnail_type,
160		thumbnail_method,
161		thumbnail_length,
162		):
163		return self._simple_insert(
164		"local_media_repository_thumbnails",
165		{
166		"media_id": media_id,
167		"thumbnail_width": thumbnail_width,
168		"thumbnail_height": thumbnail_height,
169		"thumbnail_method": thumbnail_method,
170		"thumbnail_type": thumbnail_type,
171		"thumbnail_length": thumbnail_length,
172		},
173		desc="store_local_thumbnail",
174		)
175
176		def get_cached_remote_media(self, origin, media_id):
177		return self._simple_select_one(
178		"remote_media_cache",
179		{"media_origin": origin, "media_id": media_id},
180		(
181		"media_type",
182		"media_length",
183		"upload_name",
184		"created_ts",
185		"filesystem_id",
186		"quarantined_by",
187		),
188		allow_none=True,
189		desc="get_cached_remote_media",
190		)
191
192		def store_cached_remote_media(
193		self,
194		origin,
195		media_id,
196		media_type,
197		media_length,
198		time_now_ms,
199		upload_name,
200		filesystem_id,
201		):
202		return self._simple_insert(
203		"remote_media_cache",
204		{
205		"media_origin": origin,
206		"media_id": media_id,
207		"media_type": media_type,
208		"media_length": media_length,
209		"created_ts": time_now_ms,
210		"upload_name": upload_name,
211		"filesystem_id": filesystem_id,
212		"last_access_ts": time_now_ms,
213		},
214		desc="store_cached_remote_media",
215		)
216
217		def update_cached_last_access_time(self, local_media, remote_media, time_ms):
218		"""Updates the last access time of the given media
219
220		Args:
221		local_media (iterable[str]): Set of media_ids
222		remote_media (iterable[(str, str)]): Set of (server_name, media_id)
223		time_ms: Current time in milliseconds
224		"""
225
226		def update_cache_txn(txn):
227		sql = (
228		"UPDATE remote_media_cache SET last_access_ts = ?"
229		" WHERE media_origin = ? AND media_id = ?"
230		)
231
232		txn.executemany(
233		sql,
234		(
235		(time_ms, media_origin, media_id)
236		for media_origin, media_id in remote_media
237		),
238		)
239
240		sql = (
241		"UPDATE local_media_repository SET last_access_ts = ?"
242		" WHERE media_id = ?"
243		)
244
245		txn.executemany(sql, ((time_ms, media_id) for media_id in local_media))
246
247		return self.runInteraction("update_cached_last_access_time", update_cache_txn)
248
249		def get_remote_media_thumbnails(self, origin, media_id):
250		return self._simple_select_list(
251		"remote_media_cache_thumbnails",
252		{"media_origin": origin, "media_id": media_id},
253		(
254		"thumbnail_width",
255		"thumbnail_height",
256		"thumbnail_method",
257		"thumbnail_type",
258		"thumbnail_length",
259		"filesystem_id",
260		),
261		desc="get_remote_media_thumbnails",
262		)
263
264		def store_remote_media_thumbnail(
265		self,
266		origin,
267		media_id,
268		filesystem_id,
269		thumbnail_width,
270		thumbnail_height,
271		thumbnail_type,
272		thumbnail_method,
273		thumbnail_length,
274		):
275		return self._simple_insert(
276		"remote_media_cache_thumbnails",
277		{
278		"media_origin": origin,
279		"media_id": media_id,
280		"thumbnail_width": thumbnail_width,
281		"thumbnail_height": thumbnail_height,
282		"thumbnail_method": thumbnail_method,
283		"thumbnail_type": thumbnail_type,
284		"thumbnail_length": thumbnail_length,
285		"filesystem_id": filesystem_id,
286		},
287		desc="store_remote_media_thumbnail",
288		)
289
290		def get_remote_media_before(self, before_ts):
291		sql = (
292		"SELECT media_origin, media_id, filesystem_id"
293		" FROM remote_media_cache"
294		" WHERE last_access_ts < ?"
295		)
296
297		return self._execute(
298		"get_remote_media_before", self.cursor_to_dict, sql, before_ts
299		)
300
301		def delete_remote_media(self, media_origin, media_id):
302		def delete_remote_media_txn(txn):
303		self._simple_delete_txn(
304		txn,
305		"remote_media_cache",
306		keyvalues={"media_origin": media_origin, "media_id": media_id},
307		)
308		self._simple_delete_txn(
309		txn,
310		"remote_media_cache_thumbnails",
311		keyvalues={"media_origin": media_origin, "media_id": media_id},
312		)
313
314		return self.runInteraction("delete_remote_media", delete_remote_media_txn)
315
316		def get_expired_url_cache(self, now_ts):
317		sql = (
318		"SELECT media_id FROM local_media_repository_url_cache"
319		" WHERE expires_ts < ?"
320		" ORDER BY expires_ts ASC"
321		" LIMIT 500"
322		)
323
324		def _get_expired_url_cache_txn(txn):
325		txn.execute(sql, (now_ts,))
326		return [row[0] for row in txn]
327
328		return self.runInteraction("get_expired_url_cache", _get_expired_url_cache_txn)
329
330		def delete_url_cache(self, media_ids):
331		if len(media_ids) == 0:
332		return
333
334		sql = "DELETE FROM local_media_repository_url_cache" " WHERE media_id = ?"
335
336		def _delete_url_cache_txn(txn):
337		txn.executemany(sql, [(media_id,) for media_id in media_ids])
338
339		return self.runInteraction("delete_url_cache", _delete_url_cache_txn)
340
341		def get_url_cache_media_before(self, before_ts):
342		sql = (
343		"SELECT media_id FROM local_media_repository"
344		" WHERE created_ts < ? AND url_cache IS NOT NULL"
345		" ORDER BY created_ts ASC"
346		" LIMIT 500"
347		)
348
349		def _get_url_cache_media_before_txn(txn):
350		txn.execute(sql, (before_ts,))
351		return [row[0] for row in txn]
352
353		return self.runInteraction(
354		"get_url_cache_media_before", _get_url_cache_media_before_txn
355		)
356
357		def delete_url_cache_media(self, media_ids):
358		if len(media_ids) == 0:
359		return
360
361		def _delete_url_cache_media_txn(txn):
362		sql = "DELETE FROM local_media_repository" " WHERE media_id = ?"
363
364		txn.executemany(sql, [(media_id,) for media_id in media_ids])
365
366		sql = "DELETE FROM local_media_repository_thumbnails" " WHERE media_id = ?"
367
368		txn.executemany(sql, [(media_id,) for media_id in media_ids])
369
370		return self.runInteraction(
371		"delete_url_cache_media", _delete_url_cache_media_txn
372		)

+0

-306

~~synapse/storage/monthly_active_users.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2018 New Vector
2		#
3		# Licensed under the Apache License, Version 2.0 (the "License");
4		# you may not use this file except in compliance with the License.
5		# You may obtain a copy of the License at
6		#
7		# http://www.apache.org/licenses/LICENSE-2.0
8		#
9		# Unless required by applicable law or agreed to in writing, software
10		# distributed under the License is distributed on an "AS IS" BASIS,
11		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12		# See the License for the specific language governing permissions and
13		# limitations under the License.
14		import logging
15
16		from twisted.internet import defer
17
18		from synapse.util.caches.descriptors import cached
19
20		from ._base import SQLBaseStore
21
22		logger = logging.getLogger(__name__)
23
24		# Number of msec of granularity to store the monthly_active_user timestamp
25		# This means it is not necessary to update the table on every request
26		LAST_SEEN_GRANULARITY = 60 * 60 * 1000
27
28
29		class MonthlyActiveUsersStore(SQLBaseStore):
30		def __init__(self, dbconn, hs):
31		super(MonthlyActiveUsersStore, self).__init__(None, hs)
32		self._clock = hs.get_clock()
33		self.hs = hs
34		self.reserved_users = ()
35		# Do not add more reserved users than the total allowable number
36		self._new_transaction(
37		dbconn,
38		"initialise_mau_threepids",
39		[],
40		[],
41		self._initialise_reserved_users,
42		hs.config.mau_limits_reserved_threepids[: self.hs.config.max_mau_value],
43		)
44
45		def _initialise_reserved_users(self, txn, threepids):
46		"""Ensures that reserved threepids are accounted for in the MAU table, should
47		be called on start up.
48
49		Args:
50		txn (cursor):
51		threepids (list[dict]): List of threepid dicts to reserve
52		"""
53		reserved_user_list = []
54
55		for tp in threepids:
56		user_id = self.get_user_id_by_threepid_txn(txn, tp["medium"], tp["address"])
57
58		if user_id:
59		is_support = self.is_support_user_txn(txn, user_id)
60		if not is_support:
61		self.upsert_monthly_active_user_txn(txn, user_id)
62		reserved_user_list.append(user_id)
63		else:
64		logger.warning("mau limit reserved threepid %s not found in db" % tp)
65		self.reserved_users = tuple(reserved_user_list)
66
67		@defer.inlineCallbacks
68		def reap_monthly_active_users(self):
69		"""Cleans out monthly active user table to ensure that no stale
70		entries exist.
71
72		Returns:
73		Deferred[]
74		"""
75
76		def _reap_users(txn):
77		# Purge stale users
78
79		thirty_days_ago = int(self._clock.time_msec()) - (1000 * 60 * 60 * 24 * 30)
80		query_args = [thirty_days_ago]
81		base_sql = "DELETE FROM monthly_active_users WHERE timestamp < ?"
82
83		# Need if/else since 'AND user_id NOT IN ({})' fails on Postgres
84		# when len(reserved_users) == 0. Works fine on sqlite.
85		if len(self.reserved_users) > 0:
86		# questionmarks is a hack to overcome sqlite not supporting
87		# tuples in 'WHERE IN %s'
88		questionmarks = "?" * len(self.reserved_users)
89
90		query_args.extend(self.reserved_users)
91		sql = base_sql + """ AND user_id NOT IN ({})""".format(
92		",".join(questionmarks)
93		)
94		else:
95		sql = base_sql
96
97		txn.execute(sql, query_args)
98
99		if self.hs.config.limit_usage_by_mau:
100		# If MAU user count still exceeds the MAU threshold, then delete on
101		# a least recently active basis.
102		# Note it is not possible to write this query using OFFSET due to
103		# incompatibilities in how sqlite and postgres support the feature.
104		# sqlite requires 'LIMIT -1 OFFSET ?', the LIMIT must be present
105		# While Postgres does not require 'LIMIT', but also does not support
106		# negative LIMIT values. So there is no way to write it that both can
107		# support
108		safe_guard = self.hs.config.max_mau_value - len(self.reserved_users)
109		# Must be greater than zero for postgres
110		safe_guard = safe_guard if safe_guard > 0 else 0
111		query_args = [safe_guard]
112
113		base_sql = """
114		DELETE FROM monthly_active_users
115		WHERE user_id NOT IN (
116		SELECT user_id FROM monthly_active_users
117		ORDER BY timestamp DESC
118		LIMIT ?
119		)
120		"""
121		# Need if/else since 'AND user_id NOT IN ({})' fails on Postgres
122		# when len(reserved_users) == 0. Works fine on sqlite.
123		if len(self.reserved_users) > 0:
124		query_args.extend(self.reserved_users)
125		sql = base_sql + """ AND user_id NOT IN ({})""".format(
126		",".join(questionmarks)
127		)
128		else:
129		sql = base_sql
130		txn.execute(sql, query_args)
131
132		yield self.runInteraction("reap_monthly_active_users", _reap_users)
133		# It seems poor to invalidate the whole cache, Postgres supports
134		# 'Returning' which would allow me to invalidate only the
135		# specific users, but sqlite has no way to do this and instead
136		# I would need to SELECT and the DELETE which without locking
137		# is racy.
138		# Have resolved to invalidate the whole cache for now and do
139		# something about it if and when the perf becomes significant
140		self.user_last_seen_monthly_active.invalidate_all()
141		self.get_monthly_active_count.invalidate_all()
142
143		@cached(num_args=0)
144		def get_monthly_active_count(self):
145		"""Generates current count of monthly active users
146
147		Returns:
148		Defered[int]: Number of current monthly active users
149		"""
150
151		def _count_users(txn):
152		sql = "SELECT COALESCE(count(*), 0) FROM monthly_active_users"
153
154		txn.execute(sql)
155		count, = txn.fetchone()
156		return count
157
158		return self.runInteraction("count_users", _count_users)
159
160		@defer.inlineCallbacks
161		def get_registered_reserved_users_count(self):
162		"""Of the reserved threepids defined in config, how many are associated
163		with registered users?
164
165		Returns:
166		Defered[int]: Number of real reserved users
167		"""
168		count = 0
169		for tp in self.hs.config.mau_limits_reserved_threepids:
170		user_id = yield self.hs.get_datastore().get_user_id_by_threepid(
171		tp["medium"], tp["address"]
172		)
173		if user_id:
174		count = count + 1
175		return count
176
177		@defer.inlineCallbacks
178		def upsert_monthly_active_user(self, user_id):
179		"""Updates or inserts the user into the monthly active user table, which
180		is used to track the current MAU usage of the server
181
182		Args:
183		user_id (str): user to add/update
184		"""
185		# Support user never to be included in MAU stats. Note I can't easily call this
186		# from upsert_monthly_active_user_txn because then I need a _txn form of
187		# is_support_user which is complicated because I want to cache the result.
188		# Therefore I call it here and ignore the case where
189		# upsert_monthly_active_user_txn is called directly from
190		# _initialise_reserved_users reasoning that it would be very strange to
191		# include a support user in this context.
192
193		is_support = yield self.is_support_user(user_id)
194		if is_support:
195		return
196
197		yield self.runInteraction(
198		"upsert_monthly_active_user", self.upsert_monthly_active_user_txn, user_id
199		)
200
201		user_in_mau = self.user_last_seen_monthly_active.cache.get(
202		(user_id,), None, update_metrics=False
203		)
204		if user_in_mau is None:
205		self.get_monthly_active_count.invalidate(())
206
207		self.user_last_seen_monthly_active.invalidate((user_id,))
208
209		def upsert_monthly_active_user_txn(self, txn, user_id):
210		"""Updates or inserts monthly active user member
211
212		Note that, after calling this method, it will generally be necessary
213		to invalidate the caches on user_last_seen_monthly_active and
214		get_monthly_active_count. We can't do that here, because we are running
215		in a database thread rather than the main thread, and we can't call
216		txn.call_after because txn may not be a LoggingTransaction.
217
218		We consciously do not call is_support_txn from this method because it
219		is not possible to cache the response. is_support_txn will be false in
220		almost all cases, so it seems reasonable to call it only for
221		upsert_monthly_active_user and to call is_support_txn manually
222		for cases where upsert_monthly_active_user_txn is called directly,
223		like _initialise_reserved_users
224
225		In short, don't call this method with support users. (Support users
226		should not appear in the MAU stats).
227
228		Args:
229		txn (cursor):
230		user_id (str): user to add/update
231
232		Returns:
233		bool: True if a new entry was created, False if an
234		existing one was updated.
235		"""
236
237		# Am consciously deciding to lock the table on the basis that is ought
238		# never be a big table and alternative approaches (batching multiple
239		# upserts into a single txn) introduced a lot of extra complexity.
240		# See https://github.com/matrix-org/synapse/issues/3854 for more
241		is_insert = self._simple_upsert_txn(
242		txn,
243		table="monthly_active_users",
244		keyvalues={"user_id": user_id},
245		values={"timestamp": int(self._clock.time_msec())},
246		)
247
248		return is_insert
249
250		@cached(num_args=1)
251		def user_last_seen_monthly_active(self, user_id):
252		"""
253		Checks if a given user is part of the monthly active user group
254		Arguments:
255		user_id (str): user to add/update
256		Return:
257		Deferred[int] : timestamp since last seen, None if never seen
258
259		"""
260
261		return self._simple_select_one_onecol(
262		table="monthly_active_users",
263		keyvalues={"user_id": user_id},
264		retcol="timestamp",
265		allow_none=True,
266		desc="user_last_seen_monthly_active",
267		)
268
269		@defer.inlineCallbacks
270		def populate_monthly_active_users(self, user_id):
271		"""Checks on the state of monthly active user limits and optionally
272		add the user to the monthly active tables
273
274		Args:
275		user_id(str): the user_id to query
276		"""
277		if self.hs.config.limit_usage_by_mau or self.hs.config.mau_stats_only:
278		# Trial users and guests should not be included as part of MAU group
279		is_guest = yield self.is_guest(user_id)
280		if is_guest:
281		return
282		is_trial = yield self.is_trial_user(user_id)
283		if is_trial:
284		return
285
286		last_seen_timestamp = yield self.user_last_seen_monthly_active(user_id)
287		now = self.hs.get_clock().time_msec()
288
289		# We want to reduce to the total number of db writes, and are happy
290		# to trade accuracy of timestamp in order to lighten load. This means
291		# We always insert new users (where MAU threshold has not been reached),
292		# but only update if we have not previously seen the user for
293		# LAST_SEEN_GRANULARITY ms
294		if last_seen_timestamp is None:
295		# In the case where mau_stats_only is True and limit_usage_by_mau is
296		# False, there is no point in checking get_monthly_active_count - it
297		# adds no value and will break the logic if max_mau_value is exceeded.
298		if not self.hs.config.limit_usage_by_mau:
299		yield self.upsert_monthly_active_user(user_id)
300		else:
301		count = yield self.get_monthly_active_count()
302		if count < self.hs.config.max_mau_value:
303		yield self.upsert_monthly_active_user(user_id)
304		elif now - last_seen_timestamp > LAST_SEEN_GRANULARITY:
305		yield self.upsert_monthly_active_user(user_id)

+0

-31

~~synapse/storage/openid.py~~ less more

0		from ._base import SQLBaseStore
1
2
3		class OpenIdStore(SQLBaseStore):
4		def insert_open_id_token(self, token, ts_valid_until_ms, user_id):
5		return self._simple_insert(
6		table="open_id_tokens",
7		values={
8		"token": token,
9		"ts_valid_until_ms": ts_valid_until_ms,
10		"user_id": user_id,
11		},
12		desc="insert_open_id_token",
13		)
14
15		def get_user_id_for_open_id_token(self, token, ts_now_ms):
16		def get_user_id_for_token_txn(txn):
17		sql = (
18		"SELECT user_id FROM open_id_tokens"
19		" WHERE token = ? AND ? <= ts_valid_until_ms"
20		)
21
22		txn.execute(sql, (token, ts_now_ms))
23
24		rows = txn.fetchall()
25		if not rows:
26		return None
27		else:
28		return rows[0][0]
29
30		return self.runInteraction("get_user_id_for_token", get_user_id_for_token_txn)

+138

-48

synapse/storage/prepare_database.py less more

13	13	# See the License for the specific language governing permissions and
14	14	# limitations under the License.
15	15
16		import fnmatch
17	16	import imp
18	17	import logging
19	18	import os
20	19	import re
	20
	21	import attr
21	22
22	23	from synapse.storage.engines.postgres import PostgresEngine
23	24

53	54	application config, or None if we are connecting to an existing
54	55	database which we expect to be configured already
55	56	"""
	57
	58	# For now we only have the one datastore.
	59	data_stores = ["main"]
	60
56	61	try:
57	62	cur = db_conn.cursor()
58	63	version_info = _get_or_create_schema_state(cur, database_engine)

67	72	raise UpgradeDatabaseException("Database needs to be upgraded")
68	73	else:
69	74	_upgrade_existing_database(
70		cur, user_version, delta_files, upgraded, database_engine, config
	75	cur,
	76	user_version,
	77	delta_files,
	78	upgraded,
	79	database_engine,
	80	config,
	81	data_stores=data_stores,
71	82	)
72	83	else:
73		_setup_new_database(cur, database_engine)
	84	_setup_new_database(cur, database_engine, data_stores=data_stores)
74	85
75	86	# check if any of our configured dynamic modules want a database
76	87	if config is not None:

83	94	raise
84	95
85	96
86		def _setup_new_database(cur, database_engine):
	97	def _setup_new_database(cur, database_engine, data_stores):
87	98	"""Sets up the database by finding a base set of "full schemas" and then
88		applying any necessary deltas.
	99	applying any necessary deltas, including schemas from the given data
	100	stores.
89	101
90	102	The "full_schemas" directory has subdirectories named after versions. This
91	103	function searches for the highest version less than or equal to

110	122
111	123	In the example foo.sql and bar.sql would be run, and then any delta files
112	124	for versions strictly greater than 11.
	125
	126	Note: we apply the full schemas and deltas from the top level `schema/`
	127	folder as well those in the data stores specified.
	128
	129	Args:
	130	cur (Cursor): a database cursor
	131	database_engine (DatabaseEngine)
	132	data_stores (list[str]): The names of the data stores to instantiate
	133	on the given database.
113	134	"""
114	135	current_dir = os.path.join(dir_path, "schema", "full_schemas")
115	136	directory_entries = os.listdir(current_dir)
116	137
117		valid_dirs = []
118		pattern = re.compile(r"^\d+(\.sql)?$")
	138	# First we find the highest full schema version we have
	139	valid_versions = []
	140
	141	for filename in directory_entries:
	142	try:
	143	ver = int(filename)
	144	except ValueError:
	145	continue
	146
	147	if ver <= SCHEMA_VERSION:
	148	valid_versions.append(ver)
	149
	150	if not valid_versions:
	151	raise PrepareDatabaseException(
	152	"Could not find a suitable base set of full schemas"
	153	)
	154
	155	max_current_ver = max(valid_versions)
	156
	157	logger.debug("Initialising schema v%d", max_current_ver)
	158
	159	# Now lets find all the full schema files, both in the global schema and
	160	# in data store schemas.
	161	directories = [os.path.join(current_dir, str(max_current_ver))]
	162	directories.extend(
	163	os.path.join(
	164	dir_path,
	165	"data_stores",
	166	data_store,
	167	"schema",
	168	"full_schemas",
	169	str(max_current_ver),
	170	)
	171	for data_store in data_stores
	172	)
	173
	174	directory_entries = []
	175	for directory in directories:
	176	directory_entries.extend(
	177	_DirectoryListing(file_name, os.path.join(directory, file_name))
	178	for file_name in os.listdir(directory)
	179	)
119	180
120	181	if isinstance(database_engine, PostgresEngine):
121	182	specific = "postgres"
122	183	else:
123	184	specific = "sqlite"
124	185
125		specific_pattern = re.compile(r"^\d+(\.sql." + specific + r")?$")
126
127		for filename in directory_entries:
128		match = pattern.match(filename) or specific_pattern.match(filename)
129		abs_path = os.path.join(current_dir, filename)
130		if match and os.path.isdir(abs_path):
131		ver = int(match.group(0))
132		if ver <= SCHEMA_VERSION:
133		valid_dirs.append((ver, abs_path))
134		else:
135		logger.debug("Ignoring entry '%s' in 'full_schemas'", filename)
136
137		if not valid_dirs:
138		raise PrepareDatabaseException(
139		"Could not find a suitable base set of full schemas"
140		)
141
142		max_current_ver, sql_dir = max(valid_dirs, key=lambda x: x[0])
143
144		logger.debug("Initialising schema v%d", max_current_ver)
145
146		directory_entries = os.listdir(sql_dir)
147
148		for filename in sorted(
149		fnmatch.filter(directory_entries, "*.sql")
150		+ fnmatch.filter(directory_entries, "*.sql." + specific)
151		):
152		sql_loc = os.path.join(sql_dir, filename)
153		logger.debug("Applying schema %s", sql_loc)
154		executescript(cur, sql_loc)
	186	directory_entries.sort()
	187	for entry in directory_entries:
	188	if entry.file_name.endswith(".sql") or entry.file_name.endswith(
	189	".sql." + specific
	190	):
	191	logger.debug("Applying schema %s", entry.absolute_path)
	192	executescript(cur, entry.absolute_path)
155	193
156	194	cur.execute(
157	195	database_engine.convert_param_style(
158		"INSERT INTO schema_version (version, upgraded)" " VALUES (?,?)"
	196	"INSERT INTO schema_version (version, upgraded) VALUES (?,?)"
159	197	),
160	198	(max_current_ver, False),
161	199	)

167	205	upgraded=False,
168	206	database_engine=database_engine,
169	207	config=None,
	208	data_stores=data_stores,
170	209	is_empty=True,
171	210	)
172	211

178	217	upgraded,
179	218	database_engine,
180	219	config,
	220	data_stores,
181	221	is_empty=False,
182	222	):
183	223	"""Upgrades an existing database.

214	254	only if `upgraded` is True. Then `foo.sql` and `bar.py` would be run in
215	255	some arbitrary order.
216	256
	257	Note: we apply the delta files from the specified data stores as well as
	258	those in the top-level schema. We apply all delta files across data stores
	259	for a version before applying those in the next version.
	260
217	261	Args:
218	262	cur (Cursor)
219	263	current_version (int): The current version of the schema.

223	267	applied deltas or from full schema file. If `True` the function
224	268	will never apply delta files for the given `current_version`, since
225	269	the current_version wasn't generated by applying those delta files.
	270	database_engine (DatabaseEngine)
	271	config (synapse.config.homeserver.HomeServerConfig\|None):
	272	application config, or None if we are connecting to an existing
	273	database which we expect to be configured already
	274	data_stores (list[str]): The names of the data stores to instantiate
	275	on the given database.
	276	is_empty (bool): Is this a blank database? I.e. do we need to run the
	277	upgrade portions of the delta scripts.
226	278	"""
227	279
228	280	if current_version > SCHEMA_VERSION:

247	299	for v in range(start_ver, SCHEMA_VERSION + 1):
248	300	logger.info("Upgrading schema to v%d", v)
249	301
	302	# We need to search both the global and per data store schema
	303	# directories for schema updates.
	304
	305	# First we find the directories to search in
250	306	delta_dir = os.path.join(dir_path, "schema", "delta", str(v))
251
252		try:
253		directory_entries = os.listdir(delta_dir)
254		except OSError:
255		logger.exception("Could not open delta dir for version %d", v)
256		raise UpgradeDatabaseException(
257		"Could not open delta dir for version %d" % (v,)
	307	directories = [delta_dir]
	308	for data_store in data_stores:
	309	directories.append(
	310	os.path.join(
	311	dir_path, "data_stores", data_store, "schema", "delta", str(v)
	312	)
258	313	)
259	314
	315	# Now find which directories have anything of interest.
	316	directory_entries = []
	317	for directory in directories:
	318	logger.debug("Looking for schema deltas in %s", directory)
	319	try:
	320	file_names = os.listdir(directory)
	321	directory_entries.extend(
	322	_DirectoryListing(file_name, os.path.join(directory, file_name))
	323	for file_name in file_names
	324	)
	325	except FileNotFoundError:
	326	# Data stores can have empty entries for a given version delta.
	327	pass
	328	except OSError:
	329	raise UpgradeDatabaseException(
	330	"Could not open delta dir for version %d: %s" % (v, directory)
	331	)
	332
	333	# We sort to ensure that we apply the delta files in a consistent
	334	# order (to avoid bugs caused by inconsistent directory listing order)
260	335	directory_entries.sort()
261		for file_name in directory_entries:
	336	for entry in directory_entries:
	337	file_name = entry.file_name
262	338	relative_path = os.path.join(str(v), file_name)
263		logger.debug("Found file: %s", relative_path)
	339	absolute_path = entry.absolute_path
	340
	341	logger.debug("Found file: %s (%s)", relative_path, absolute_path)
264	342	if relative_path in applied_delta_files:
265	343	continue
266	344
267		absolute_path = os.path.join(dir_path, "schema", "delta", relative_path)
268	345	root_name, ext = os.path.splitext(file_name)
269	346	if ext == ".py":
270	347	# This is a python upgrade module. We need to import into some

447	524	return current_version, applied_deltas, upgraded
448	525
449	526	return None
	527
	528
	529	@attr.s()
	530	class _DirectoryListing(object):
	531	"""Helper class to store schema file name and the
	532	absolute path to it.
	533
	534	These entries get sorted, so for consistency we want to ensure that
	535	`file_name` attr is kept first.
	536	"""
	537
	538	file_name = attr.ib()
	539	absolute_path = attr.ib()

+0

-136

synapse/storage/presence.py less more

14	14
15	15	from collections import namedtuple
16	16
17		from twisted.internet import defer
18
19	17	from synapse.api.constants import PresenceState
20		from synapse.util import batch_iter
21		from synapse.util.caches.descriptors import cached, cachedList
22
23		from ._base import SQLBaseStore
24	18
25	19
26	20	class UserPresenceState(

72	66	status_msg=None,
73	67	currently_active=False,
74	68	)
75
76
77		class PresenceStore(SQLBaseStore):
78		@defer.inlineCallbacks
79		def update_presence(self, presence_states):
80		stream_ordering_manager = self._presence_id_gen.get_next_mult(
81		len(presence_states)
82		)
83
84		with stream_ordering_manager as stream_orderings:
85		yield self.runInteraction(
86		"update_presence",
87		self._update_presence_txn,
88		stream_orderings,
89		presence_states,
90		)
91
92		return stream_orderings[-1], self._presence_id_gen.get_current_token()
93
94		def _update_presence_txn(self, txn, stream_orderings, presence_states):
95		for stream_id, state in zip(stream_orderings, presence_states):
96		txn.call_after(
97		self.presence_stream_cache.entity_has_changed, state.user_id, stream_id
98		)
99		txn.call_after(self._get_presence_for_user.invalidate, (state.user_id,))
100
101		# Actually insert new rows
102		self._simple_insert_many_txn(
103		txn,
104		table="presence_stream",
105		values=[
106		{
107		"stream_id": stream_id,
108		"user_id": state.user_id,
109		"state": state.state,
110		"last_active_ts": state.last_active_ts,
111		"last_federation_update_ts": state.last_federation_update_ts,
112		"last_user_sync_ts": state.last_user_sync_ts,
113		"status_msg": state.status_msg,
114		"currently_active": state.currently_active,
115		}
116		for state in presence_states
117		],
118		)
119
120		# Delete old rows to stop database from getting really big
121		sql = (
122		"DELETE FROM presence_stream WHERE" " stream_id < ?" " AND user_id IN (%s)"
123		)
124
125		for states in batch_iter(presence_states, 50):
126		args = [stream_id]
127		args.extend(s.user_id for s in states)
128		txn.execute(sql % (",".join("?" for _ in states),), args)
129
130		def get_all_presence_updates(self, last_id, current_id):
131		if last_id == current_id:
132		return defer.succeed([])
133
134		def get_all_presence_updates_txn(txn):
135		sql = (
136		"SELECT stream_id, user_id, state, last_active_ts,"
137		" last_federation_update_ts, last_user_sync_ts, status_msg,"
138		" currently_active"
139		" FROM presence_stream"
140		" WHERE ? < stream_id AND stream_id <= ?"
141		)
142		txn.execute(sql, (last_id, current_id))
143		return txn.fetchall()
144
145		return self.runInteraction(
146		"get_all_presence_updates", get_all_presence_updates_txn
147		)
148
149		@cached()
150		def _get_presence_for_user(self, user_id):
151		raise NotImplementedError()
152
153		@cachedList(
154		cached_method_name="_get_presence_for_user",
155		list_name="user_ids",
156		num_args=1,
157		inlineCallbacks=True,
158		)
159		def get_presence_for_users(self, user_ids):
160		rows = yield self._simple_select_many_batch(
161		table="presence_stream",
162		column="user_id",
163		iterable=user_ids,
164		keyvalues={},
165		retcols=(
166		"user_id",
167		"state",
168		"last_active_ts",
169		"last_federation_update_ts",
170		"last_user_sync_ts",
171		"status_msg",
172		"currently_active",
173		),
174		desc="get_presence_for_users",
175		)
176
177		for row in rows:
178		row["currently_active"] = bool(row["currently_active"])
179
180		return {row["user_id"]: UserPresenceState(**row) for row in rows}
181
182		def get_current_presence_token(self):
183		return self._presence_id_gen.get_current_token()
184
185		def allow_presence_visible(self, observed_localpart, observer_userid):
186		return self._simple_insert(
187		table="presence_allow_inbound",
188		values={
189		"observed_user_id": observed_localpart,
190		"observer_user_id": observer_userid,
191		},
192		desc="allow_presence_visible",
193		or_ignore=True,
194		)
195
196		def disallow_presence_visible(self, observed_localpart, observer_userid):
197		return self._simple_delete_one(
198		table="presence_allow_inbound",
199		keyvalues={
200		"observed_user_id": observed_localpart,
201		"observer_user_id": observer_userid,
202		},
203		desc="disallow_presence_visible",
204		)

+0

-179

~~synapse/storage/profile.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2014-2016 OpenMarket Ltd
2		#
3		# Licensed under the Apache License, Version 2.0 (the "License");
4		# you may not use this file except in compliance with the License.
5		# You may obtain a copy of the License at
6		#
7		# http://www.apache.org/licenses/LICENSE-2.0
8		#
9		# Unless required by applicable law or agreed to in writing, software
10		# distributed under the License is distributed on an "AS IS" BASIS,
11		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12		# See the License for the specific language governing permissions and
13		# limitations under the License.
14
15		from twisted.internet import defer
16
17		from synapse.api.errors import StoreError
18		from synapse.storage.roommember import ProfileInfo
19
20		from ._base import SQLBaseStore
21
22
23		class ProfileWorkerStore(SQLBaseStore):
24		@defer.inlineCallbacks
25		def get_profileinfo(self, user_localpart):
26		try:
27		profile = yield self._simple_select_one(
28		table="profiles",
29		keyvalues={"user_id": user_localpart},
30		retcols=("displayname", "avatar_url"),
31		desc="get_profileinfo",
32		)
33		except StoreError as e:
34		if e.code == 404:
35		# no match
36		return ProfileInfo(None, None)
37		else:
38		raise
39
40		return ProfileInfo(
41		avatar_url=profile["avatar_url"], display_name=profile["displayname"]
42		)
43
44		def get_profile_displayname(self, user_localpart):
45		return self._simple_select_one_onecol(
46		table="profiles",
47		keyvalues={"user_id": user_localpart},
48		retcol="displayname",
49		desc="get_profile_displayname",
50		)
51
52		def get_profile_avatar_url(self, user_localpart):
53		return self._simple_select_one_onecol(
54		table="profiles",
55		keyvalues={"user_id": user_localpart},
56		retcol="avatar_url",
57		desc="get_profile_avatar_url",
58		)
59
60		def get_from_remote_profile_cache(self, user_id):
61		return self._simple_select_one(
62		table="remote_profile_cache",
63		keyvalues={"user_id": user_id},
64		retcols=("displayname", "avatar_url"),
65		allow_none=True,
66		desc="get_from_remote_profile_cache",
67		)
68
69		def create_profile(self, user_localpart):
70		return self._simple_insert(
71		table="profiles", values={"user_id": user_localpart}, desc="create_profile"
72		)
73
74		def set_profile_displayname(self, user_localpart, new_displayname):
75		return self._simple_update_one(
76		table="profiles",
77		keyvalues={"user_id": user_localpart},
78		updatevalues={"displayname": new_displayname},
79		desc="set_profile_displayname",
80		)
81
82		def set_profile_avatar_url(self, user_localpart, new_avatar_url):
83		return self._simple_update_one(
84		table="profiles",
85		keyvalues={"user_id": user_localpart},
86		updatevalues={"avatar_url": new_avatar_url},
87		desc="set_profile_avatar_url",
88		)
89
90
91		class ProfileStore(ProfileWorkerStore):
92		def add_remote_profile_cache(self, user_id, displayname, avatar_url):
93		"""Ensure we are caching the remote user's profiles.
94
95		This should only be called when `is_subscribed_remote_profile_for_user`
96		would return true for the user.
97		"""
98		return self._simple_upsert(
99		table="remote_profile_cache",
100		keyvalues={"user_id": user_id},
101		values={
102		"displayname": displayname,
103		"avatar_url": avatar_url,
104		"last_check": self._clock.time_msec(),
105		},
106		desc="add_remote_profile_cache",
107		)
108
109		def update_remote_profile_cache(self, user_id, displayname, avatar_url):
110		return self._simple_update(
111		table="remote_profile_cache",
112		keyvalues={"user_id": user_id},
113		values={
114		"displayname": displayname,
115		"avatar_url": avatar_url,
116		"last_check": self._clock.time_msec(),
117		},
118		desc="update_remote_profile_cache",
119		)
120
121		@defer.inlineCallbacks
122		def maybe_delete_remote_profile_cache(self, user_id):
123		"""Check if we still care about the remote user's profile, and if we
124		don't then remove their profile from the cache
125		"""
126		subscribed = yield self.is_subscribed_remote_profile_for_user(user_id)
127		if not subscribed:
128		yield self._simple_delete(
129		table="remote_profile_cache",
130		keyvalues={"user_id": user_id},
131		desc="delete_remote_profile_cache",
132		)
133
134		def get_remote_profile_cache_entries_that_expire(self, last_checked):
135		"""Get all users who haven't been checked since `last_checked`
136		"""
137
138		def _get_remote_profile_cache_entries_that_expire_txn(txn):
139		sql = """
140		SELECT user_id, displayname, avatar_url
141		FROM remote_profile_cache
142		WHERE last_check < ?
143		"""
144
145		txn.execute(sql, (last_checked,))
146
147		return self.cursor_to_dict(txn)
148
149		return self.runInteraction(
150		"get_remote_profile_cache_entries_that_expire",
151		_get_remote_profile_cache_entries_that_expire_txn,
152		)
153
154		@defer.inlineCallbacks
155		def is_subscribed_remote_profile_for_user(self, user_id):
156		"""Check whether we are interested in a remote user's profile.
157		"""
158		res = yield self._simple_select_one_onecol(
159		table="group_users",
160		keyvalues={"user_id": user_id},
161		retcol="user_id",
162		allow_none=True,
163		desc="should_update_remote_profile_cache_for_user",
164		)
165
166		if res:
167		return True
168
169		res = yield self._simple_select_one_onecol(
170		table="group_invites",
171		keyvalues={"user_id": user_id},
172		retcol="user_id",
173		allow_none=True,
174		desc="should_update_remote_profile_cache_for_user",
175		)
176
177		if res:
178		return True

+0

-702

synapse/storage/push_rule.py less more

13	13	# See the License for the specific language governing permissions and
14	14	# limitations under the License.
15	15
16		import abc
17		import logging
18
19		from canonicaljson import json
20
21		from twisted.internet import defer
22
23		from synapse.push.baserules import list_with_base_rules
24		from synapse.storage.appservice import ApplicationServiceWorkerStore
25		from synapse.storage.pusher import PusherWorkerStore
26		from synapse.storage.receipts import ReceiptsWorkerStore
27		from synapse.storage.roommember import RoomMemberWorkerStore
28		from synapse.util.caches.descriptors import cachedInlineCallbacks, cachedList
29		from synapse.util.caches.stream_change_cache import StreamChangeCache
30
31		from ._base import SQLBaseStore
32
33		logger = logging.getLogger(__name__)
34
35
36		def _load_rules(rawrules, enabled_map):
37		ruleslist = []
38		for rawrule in rawrules:
39		rule = dict(rawrule)
40		rule["conditions"] = json.loads(rawrule["conditions"])
41		rule["actions"] = json.loads(rawrule["actions"])
42		ruleslist.append(rule)
43
44		# We're going to be mutating this a lot, so do a deep copy
45		rules = list(list_with_base_rules(ruleslist))
46
47		for i, rule in enumerate(rules):
48		rule_id = rule["rule_id"]
49		if rule_id in enabled_map:
50		if rule.get("enabled", True) != bool(enabled_map[rule_id]):
51		# Rules are cached across users.
52		rule = dict(rule)
53		rule["enabled"] = bool(enabled_map[rule_id])
54		rules[i] = rule
55
56		return rules
57
58
59		class PushRulesWorkerStore(
60		ApplicationServiceWorkerStore,
61		ReceiptsWorkerStore,
62		PusherWorkerStore,
63		RoomMemberWorkerStore,
64		SQLBaseStore,
65		):
66		"""This is an abstract base class where subclasses must implement
67		`get_max_push_rules_stream_id` which can be called in the initializer.
68		"""
69
70		# This ABCMeta metaclass ensures that we cannot be instantiated without
71		# the abstract methods being implemented.
72		__metaclass__ = abc.ABCMeta
73
74		def __init__(self, db_conn, hs):
75		super(PushRulesWorkerStore, self).__init__(db_conn, hs)
76
77		push_rules_prefill, push_rules_id = self._get_cache_dict(
78		db_conn,
79		"push_rules_stream",
80		entity_column="user_id",
81		stream_column="stream_id",
82		max_value=self.get_max_push_rules_stream_id(),
83		)
84
85		self.push_rules_stream_cache = StreamChangeCache(
86		"PushRulesStreamChangeCache",
87		push_rules_id,
88		prefilled_cache=push_rules_prefill,
89		)
90
91		@abc.abstractmethod
92		def get_max_push_rules_stream_id(self):
93		"""Get the position of the push rules stream.
94
95		Returns:
96		int
97		"""
98		raise NotImplementedError()
99
100		@cachedInlineCallbacks(max_entries=5000)
101		def get_push_rules_for_user(self, user_id):
102		rows = yield self._simple_select_list(
103		table="push_rules",
104		keyvalues={"user_name": user_id},
105		retcols=(
106		"user_name",
107		"rule_id",
108		"priority_class",
109		"priority",
110		"conditions",
111		"actions",
112		),
113		desc="get_push_rules_enabled_for_user",
114		)
115
116		rows.sort(key=lambda row: (-int(row["priority_class"]), -int(row["priority"])))
117
118		enabled_map = yield self.get_push_rules_enabled_for_user(user_id)
119
120		rules = _load_rules(rows, enabled_map)
121
122		return rules
123
124		@cachedInlineCallbacks(max_entries=5000)
125		def get_push_rules_enabled_for_user(self, user_id):
126		results = yield self._simple_select_list(
127		table="push_rules_enable",
128		keyvalues={"user_name": user_id},
129		retcols=("user_name", "rule_id", "enabled"),
130		desc="get_push_rules_enabled_for_user",
131		)
132		return {r["rule_id"]: False if r["enabled"] == 0 else True for r in results}
133
134		def have_push_rules_changed_for_user(self, user_id, last_id):
135		if not self.push_rules_stream_cache.has_entity_changed(user_id, last_id):
136		return defer.succeed(False)
137		else:
138
139		def have_push_rules_changed_txn(txn):
140		sql = (
141		"SELECT COUNT(stream_id) FROM push_rules_stream"
142		" WHERE user_id = ? AND ? < stream_id"
143		)
144		txn.execute(sql, (user_id, last_id))
145		count, = txn.fetchone()
146		return bool(count)
147
148		return self.runInteraction(
149		"have_push_rules_changed", have_push_rules_changed_txn
150		)
151
152		@cachedList(
153		cached_method_name="get_push_rules_for_user",
154		list_name="user_ids",
155		num_args=1,
156		inlineCallbacks=True,
157		)
158		def bulk_get_push_rules(self, user_ids):
159		if not user_ids:
160		return {}
161
162		results = {user_id: [] for user_id in user_ids}
163
164		rows = yield self._simple_select_many_batch(
165		table="push_rules",
166		column="user_name",
167		iterable=user_ids,
168		retcols=("*",),
169		desc="bulk_get_push_rules",
170		)
171
172		rows.sort(key=lambda row: (-int(row["priority_class"]), -int(row["priority"])))
173
174		for row in rows:
175		results.setdefault(row["user_name"], []).append(row)
176
177		enabled_map_by_user = yield self.bulk_get_push_rules_enabled(user_ids)
178
179		for user_id, rules in results.items():
180		results[user_id] = _load_rules(rules, enabled_map_by_user.get(user_id, {}))
181
182		return results
183
184		@defer.inlineCallbacks
185		def move_push_rule_from_room_to_room(self, new_room_id, user_id, rule):
186		"""Move a single push rule from one room to another for a specific user.
187
188		Args:
189		new_room_id (str): ID of the new room.
190		user_id (str): ID of user the push rule belongs to.
191		rule (Dict): A push rule.
192		"""
193		# Create new rule id
194		rule_id_scope = "/".join(rule["rule_id"].split("/")[:-1])
195		new_rule_id = rule_id_scope + "/" + new_room_id
196
197		# Change room id in each condition
198		for condition in rule.get("conditions", []):
199		if condition.get("key") == "room_id":
200		condition["pattern"] = new_room_id
201
202		# Add the rule for the new room
203		yield self.add_push_rule(
204		user_id=user_id,
205		rule_id=new_rule_id,
206		priority_class=rule["priority_class"],
207		conditions=rule["conditions"],
208		actions=rule["actions"],
209		)
210
211		# Delete push rule for the old room
212		yield self.delete_push_rule(user_id, rule["rule_id"])
213
214		@defer.inlineCallbacks
215		def move_push_rules_from_room_to_room_for_user(
216		self, old_room_id, new_room_id, user_id
217		):
218		"""Move all of the push rules from one room to another for a specific
219		user.
220
221		Args:
222		old_room_id (str): ID of the old room.
223		new_room_id (str): ID of the new room.
224		user_id (str): ID of user to copy push rules for.
225		"""
226		# Retrieve push rules for this user
227		user_push_rules = yield self.get_push_rules_for_user(user_id)
228
229		# Get rules relating to the old room, move them to the new room, then
230		# delete them from the old room
231		for rule in user_push_rules:
232		conditions = rule.get("conditions", [])
233		if any(
234		(c.get("key") == "room_id" and c.get("pattern") == old_room_id)
235		for c in conditions
236		):
237		self.move_push_rule_from_room_to_room(new_room_id, user_id, rule)
238
239		@defer.inlineCallbacks
240		def bulk_get_push_rules_for_room(self, event, context):
241		state_group = context.state_group
242		if not state_group:
243		# If state_group is None it means it has yet to be assigned a
244		# state group, i.e. we need to make sure that calls with a state_group
245		# of None don't hit previous cached calls with a None state_group.
246		# To do this we set the state_group to a new object as object() != object()
247		state_group = object()
248
249		current_state_ids = yield context.get_current_state_ids(self)
250		result = yield self._bulk_get_push_rules_for_room(
251		event.room_id, state_group, current_state_ids, event=event
252		)
253		return result
254
255		@cachedInlineCallbacks(num_args=2, cache_context=True)
256		def _bulk_get_push_rules_for_room(
257		self, room_id, state_group, current_state_ids, cache_context, event=None
258		):
259		# We don't use `state_group`, its there so that we can cache based
260		# on it. However, its important that its never None, since two current_state's
261		# with a state_group of None are likely to be different.
262		# See bulk_get_push_rules_for_room for how we work around this.
263		assert state_group is not None
264
265		# We also will want to generate notifs for other people in the room so
266		# their unread countss are correct in the event stream, but to avoid
267		# generating them for bot / AS users etc, we only do so for people who've
268		# sent a read receipt into the room.
269
270		users_in_room = yield self._get_joined_users_from_context(
271		room_id,
272		state_group,
273		current_state_ids,
274		on_invalidate=cache_context.invalidate,
275		event=event,
276		)
277
278		# We ignore app service users for now. This is so that we don't fill
279		# up the `get_if_users_have_pushers` cache with AS entries that we
280		# know don't have pushers, nor even read receipts.
281		local_users_in_room = set(
282		u
283		for u in users_in_room
284		if self.hs.is_mine_id(u)
285		and not self.get_if_app_services_interested_in_user(u)
286		)
287
288		# users in the room who have pushers need to get push rules run because
289		# that's how their pushers work
290		if_users_with_pushers = yield self.get_if_users_have_pushers(
291		local_users_in_room, on_invalidate=cache_context.invalidate
292		)
293		user_ids = set(
294		uid for uid, have_pusher in if_users_with_pushers.items() if have_pusher
295		)
296
297		users_with_receipts = yield self.get_users_with_read_receipts_in_room(
298		room_id, on_invalidate=cache_context.invalidate
299		)
300
301		# any users with pushers must be ours: they have pushers
302		for uid in users_with_receipts:
303		if uid in local_users_in_room:
304		user_ids.add(uid)
305
306		rules_by_user = yield self.bulk_get_push_rules(
307		user_ids, on_invalidate=cache_context.invalidate
308		)
309
310		rules_by_user = {k: v for k, v in rules_by_user.items() if v is not None}
311
312		return rules_by_user
313
314		@cachedList(
315		cached_method_name="get_push_rules_enabled_for_user",
316		list_name="user_ids",
317		num_args=1,
318		inlineCallbacks=True,
319		)
320		def bulk_get_push_rules_enabled(self, user_ids):
321		if not user_ids:
322		return {}
323
324		results = {user_id: {} for user_id in user_ids}
325
326		rows = yield self._simple_select_many_batch(
327		table="push_rules_enable",
328		column="user_name",
329		iterable=user_ids,
330		retcols=("user_name", "rule_id", "enabled"),
331		desc="bulk_get_push_rules_enabled",
332		)
333		for row in rows:
334		enabled = bool(row["enabled"])
335		results.setdefault(row["user_name"], {})[row["rule_id"]] = enabled
336		return results
337
338
339		class PushRuleStore(PushRulesWorkerStore):
340		@defer.inlineCallbacks
341		def add_push_rule(
342		self,
343		user_id,
344		rule_id,
345		priority_class,
346		conditions,
347		actions,
348		before=None,
349		after=None,
350		):
351		conditions_json = json.dumps(conditions)
352		actions_json = json.dumps(actions)
353		with self._push_rules_stream_id_gen.get_next() as ids:
354		stream_id, event_stream_ordering = ids
355		if before or after:
356		yield self.runInteraction(
357		"_add_push_rule_relative_txn",
358		self._add_push_rule_relative_txn,
359		stream_id,
360		event_stream_ordering,
361		user_id,
362		rule_id,
363		priority_class,
364		conditions_json,
365		actions_json,
366		before,
367		after,
368		)
369		else:
370		yield self.runInteraction(
371		"_add_push_rule_highest_priority_txn",
372		self._add_push_rule_highest_priority_txn,
373		stream_id,
374		event_stream_ordering,
375		user_id,
376		rule_id,
377		priority_class,
378		conditions_json,
379		actions_json,
380		)
381
382		def _add_push_rule_relative_txn(
383		self,
384		txn,
385		stream_id,
386		event_stream_ordering,
387		user_id,
388		rule_id,
389		priority_class,
390		conditions_json,
391		actions_json,
392		before,
393		after,
394		):
395		# Lock the table since otherwise we'll have annoying races between the
396		# SELECT here and the UPSERT below.
397		self.database_engine.lock_table(txn, "push_rules")
398
399		relative_to_rule = before or after
400
401		res = self._simple_select_one_txn(
402		txn,
403		table="push_rules",
404		keyvalues={"user_name": user_id, "rule_id": relative_to_rule},
405		retcols=["priority_class", "priority"],
406		allow_none=True,
407		)
408
409		if not res:
410		raise RuleNotFoundException(
411		"before/after rule not found: %s" % (relative_to_rule,)
412		)
413
414		base_priority_class = res["priority_class"]
415		base_rule_priority = res["priority"]
416
417		if base_priority_class != priority_class:
418		raise InconsistentRuleException(
419		"Given priority class does not match class of relative rule"
420		)
421
422		if before:
423		# Higher priority rules are executed first, So adding a rule before
424		# a rule means giving it a higher priority than that rule.
425		new_rule_priority = base_rule_priority + 1
426		else:
427		# We increment the priority of the existing rules to make space for
428		# the new rule. Therefore if we want this rule to appear after
429		# an existing rule we give it the priority of the existing rule,
430		# and then increment the priority of the existing rule.
431		new_rule_priority = base_rule_priority
432
433		sql = (
434		"UPDATE push_rules SET priority = priority + 1"
435		" WHERE user_name = ? AND priority_class = ? AND priority >= ?"
436		)
437
438		txn.execute(sql, (user_id, priority_class, new_rule_priority))
439
440		self._upsert_push_rule_txn(
441		txn,
442		stream_id,
443		event_stream_ordering,
444		user_id,
445		rule_id,
446		priority_class,
447		new_rule_priority,
448		conditions_json,
449		actions_json,
450		)
451
452		def _add_push_rule_highest_priority_txn(
453		self,
454		txn,
455		stream_id,
456		event_stream_ordering,
457		user_id,
458		rule_id,
459		priority_class,
460		conditions_json,
461		actions_json,
462		):
463		# Lock the table since otherwise we'll have annoying races between the
464		# SELECT here and the UPSERT below.
465		self.database_engine.lock_table(txn, "push_rules")
466
467		# find the highest priority rule in that class
468		sql = (
469		"SELECT COUNT(*), MAX(priority) FROM push_rules"
470		" WHERE user_name = ? and priority_class = ?"
471		)
472		txn.execute(sql, (user_id, priority_class))
473		res = txn.fetchall()
474		(how_many, highest_prio) = res[0]
475
476		new_prio = 0
477		if how_many > 0:
478		new_prio = highest_prio + 1
479
480		self._upsert_push_rule_txn(
481		txn,
482		stream_id,
483		event_stream_ordering,
484		user_id,
485		rule_id,
486		priority_class,
487		new_prio,
488		conditions_json,
489		actions_json,
490		)
491
492		def _upsert_push_rule_txn(
493		self,
494		txn,
495		stream_id,
496		event_stream_ordering,
497		user_id,
498		rule_id,
499		priority_class,
500		priority,
501		conditions_json,
502		actions_json,
503		update_stream=True,
504		):
505		"""Specialised version of _simple_upsert_txn that picks a push_rule_id
506		using the _push_rule_id_gen if it needs to insert the rule. It assumes
507		that the "push_rules" table is locked"""
508
509		sql = (
510		"UPDATE push_rules"
511		" SET priority_class = ?, priority = ?, conditions = ?, actions = ?"
512		" WHERE user_name = ? AND rule_id = ?"
513		)
514
515		txn.execute(
516		sql,
517		(priority_class, priority, conditions_json, actions_json, user_id, rule_id),
518		)
519
520		if txn.rowcount == 0:
521		# We didn't update a row with the given rule_id so insert one
522		push_rule_id = self._push_rule_id_gen.get_next()
523
524		self._simple_insert_txn(
525		txn,
526		table="push_rules",
527		values={
528		"id": push_rule_id,
529		"user_name": user_id,
530		"rule_id": rule_id,
531		"priority_class": priority_class,
532		"priority": priority,
533		"conditions": conditions_json,
534		"actions": actions_json,
535		},
536		)
537
538		if update_stream:
539		self._insert_push_rules_update_txn(
540		txn,
541		stream_id,
542		event_stream_ordering,
543		user_id,
544		rule_id,
545		op="ADD",
546		data={
547		"priority_class": priority_class,
548		"priority": priority,
549		"conditions": conditions_json,
550		"actions": actions_json,
551		},
552		)
553
554		@defer.inlineCallbacks
555		def delete_push_rule(self, user_id, rule_id):
556		"""
557		Delete a push rule. Args specify the row to be deleted and can be
558		any of the columns in the push_rule table, but below are the
559		standard ones
560
561		Args:
562		user_id (str): The matrix ID of the push rule owner
563		rule_id (str): The rule_id of the rule to be deleted
564		"""
565
566		def delete_push_rule_txn(txn, stream_id, event_stream_ordering):
567		self._simple_delete_one_txn(
568		txn, "push_rules", {"user_name": user_id, "rule_id": rule_id}
569		)
570
571		self._insert_push_rules_update_txn(
572		txn, stream_id, event_stream_ordering, user_id, rule_id, op="DELETE"
573		)
574
575		with self._push_rules_stream_id_gen.get_next() as ids:
576		stream_id, event_stream_ordering = ids
577		yield self.runInteraction(
578		"delete_push_rule",
579		delete_push_rule_txn,
580		stream_id,
581		event_stream_ordering,
582		)
583
584		@defer.inlineCallbacks
585		def set_push_rule_enabled(self, user_id, rule_id, enabled):
586		with self._push_rules_stream_id_gen.get_next() as ids:
587		stream_id, event_stream_ordering = ids
588		yield self.runInteraction(
589		"_set_push_rule_enabled_txn",
590		self._set_push_rule_enabled_txn,
591		stream_id,
592		event_stream_ordering,
593		user_id,
594		rule_id,
595		enabled,
596		)
597
598		def _set_push_rule_enabled_txn(
599		self, txn, stream_id, event_stream_ordering, user_id, rule_id, enabled
600		):
601		new_id = self._push_rules_enable_id_gen.get_next()
602		self._simple_upsert_txn(
603		txn,
604		"push_rules_enable",
605		{"user_name": user_id, "rule_id": rule_id},
606		{"enabled": 1 if enabled else 0},
607		{"id": new_id},
608		)
609
610		self._insert_push_rules_update_txn(
611		txn,
612		stream_id,
613		event_stream_ordering,
614		user_id,
615		rule_id,
616		op="ENABLE" if enabled else "DISABLE",
617		)
618
619		@defer.inlineCallbacks
620		def set_push_rule_actions(self, user_id, rule_id, actions, is_default_rule):
621		actions_json = json.dumps(actions)
622
623		def set_push_rule_actions_txn(txn, stream_id, event_stream_ordering):
624		if is_default_rule:
625		# Add a dummy rule to the rules table with the user specified
626		# actions.
627		priority_class = -1
628		priority = 1
629		self._upsert_push_rule_txn(
630		txn,
631		stream_id,
632		event_stream_ordering,
633		user_id,
634		rule_id,
635		priority_class,
636		priority,
637		"[]",
638		actions_json,
639		update_stream=False,
640		)
641		else:
642		self._simple_update_one_txn(
643		txn,
644		"push_rules",
645		{"user_name": user_id, "rule_id": rule_id},
646		{"actions": actions_json},
647		)
648
649		self._insert_push_rules_update_txn(
650		txn,
651		stream_id,
652		event_stream_ordering,
653		user_id,
654		rule_id,
655		op="ACTIONS",
656		data={"actions": actions_json},
657		)
658
659		with self._push_rules_stream_id_gen.get_next() as ids:
660		stream_id, event_stream_ordering = ids
661		yield self.runInteraction(
662		"set_push_rule_actions",
663		set_push_rule_actions_txn,
664		stream_id,
665		event_stream_ordering,
666		)
667
668		def _insert_push_rules_update_txn(
669		self, txn, stream_id, event_stream_ordering, user_id, rule_id, op, data=None
670		):
671		values = {
672		"stream_id": stream_id,
673		"event_stream_ordering": event_stream_ordering,
674		"user_id": user_id,
675		"rule_id": rule_id,
676		"op": op,
677		}
678		if data is not None:
679		values.update(data)
680
681		self._simple_insert_txn(txn, "push_rules_stream", values=values)
682
683		txn.call_after(self.get_push_rules_for_user.invalidate, (user_id,))
684		txn.call_after(self.get_push_rules_enabled_for_user.invalidate, (user_id,))
685		txn.call_after(
686		self.push_rules_stream_cache.entity_has_changed, user_id, stream_id
687		)
688
689		def get_all_push_rule_updates(self, last_id, current_id, limit):
690		"""Get all the push rules changes that have happend on the server"""
691		if last_id == current_id:
692		return defer.succeed([])
693
694		def get_all_push_rule_updates_txn(txn):
695		sql = (
696		"SELECT stream_id, event_stream_ordering, user_id, rule_id,"
697		" op, priority_class, priority, conditions, actions"
698		" FROM push_rules_stream"
699		" WHERE ? < stream_id AND stream_id <= ?"
700		" ORDER BY stream_id ASC LIMIT ?"
701		)
702		txn.execute(sql, (last_id, current_id, limit))
703		return txn.fetchall()
704
705		return self.runInteraction(
706		"get_all_push_rule_updates", get_all_push_rule_updates_txn
707		)
708
709		def get_push_rules_stream_token(self):
710		"""Get the position of the push rules stream.
711		Returns a pair of a stream id for the push_rules stream and the
712		room stream ordering it corresponds to."""
713		return self._push_rules_stream_id_gen.get_current_token()
714
715		def get_max_push_rules_stream_id(self):
716		return self.get_push_rules_stream_token()[0]
717
718	16
719	17	class RuleNotFoundException(Exception):
720	18	pass

+0

-372

~~synapse/storage/pusher.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2014-2016 OpenMarket Ltd
2		# Copyright 2018 New Vector Ltd
3		#
4		# Licensed under the Apache License, Version 2.0 (the "License");
5		# you may not use this file except in compliance with the License.
6		# You may obtain a copy of the License at
7		#
8		# http://www.apache.org/licenses/LICENSE-2.0
9		#
10		# Unless required by applicable law or agreed to in writing, software
11		# distributed under the License is distributed on an "AS IS" BASIS,
12		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13		# See the License for the specific language governing permissions and
14		# limitations under the License.
15
16		import logging
17
18		import six
19
20		from canonicaljson import encode_canonical_json, json
21
22		from twisted.internet import defer
23
24		from synapse.util.caches.descriptors import cachedInlineCallbacks, cachedList
25
26		from ._base import SQLBaseStore
27
28		logger = logging.getLogger(__name__)
29
30		if six.PY2:
31		db_binary_type = six.moves.builtins.buffer
32		else:
33		db_binary_type = memoryview
34
35
36		class PusherWorkerStore(SQLBaseStore):
37		def _decode_pushers_rows(self, rows):
38		for r in rows:
39		dataJson = r["data"]
40		r["data"] = None
41		try:
42		if isinstance(dataJson, db_binary_type):
43		dataJson = str(dataJson).decode("UTF8")
44
45		r["data"] = json.loads(dataJson)
46		except Exception as e:
47		logger.warn(
48		"Invalid JSON in data for pusher %d: %s, %s",
49		r["id"],
50		dataJson,
51		e.args[0],
52		)
53		pass
54
55		if isinstance(r["pushkey"], db_binary_type):
56		r["pushkey"] = str(r["pushkey"]).decode("UTF8")
57
58		return rows
59
60		@defer.inlineCallbacks
61		def user_has_pusher(self, user_id):
62		ret = yield self._simple_select_one_onecol(
63		"pushers", {"user_name": user_id}, "id", allow_none=True
64		)
65		return ret is not None
66
67		def get_pushers_by_app_id_and_pushkey(self, app_id, pushkey):
68		return self.get_pushers_by({"app_id": app_id, "pushkey": pushkey})
69
70		def get_pushers_by_user_id(self, user_id):
71		return self.get_pushers_by({"user_name": user_id})
72
73		@defer.inlineCallbacks
74		def get_pushers_by(self, keyvalues):
75		ret = yield self._simple_select_list(
76		"pushers",
77		keyvalues,
78		[
79		"id",
80		"user_name",
81		"access_token",
82		"profile_tag",
83		"kind",
84		"app_id",
85		"app_display_name",
86		"device_display_name",
87		"pushkey",
88		"ts",
89		"lang",
90		"data",
91		"last_stream_ordering",
92		"last_success",
93		"failing_since",
94		],
95		desc="get_pushers_by",
96		)
97		return self._decode_pushers_rows(ret)
98
99		@defer.inlineCallbacks
100		def get_all_pushers(self):
101		def get_pushers(txn):
102		txn.execute("SELECT * FROM pushers")
103		rows = self.cursor_to_dict(txn)
104
105		return self._decode_pushers_rows(rows)
106
107		rows = yield self.runInteraction("get_all_pushers", get_pushers)
108		return rows
109
110		def get_all_updated_pushers(self, last_id, current_id, limit):
111		if last_id == current_id:
112		return defer.succeed(([], []))
113
114		def get_all_updated_pushers_txn(txn):
115		sql = (
116		"SELECT id, user_name, access_token, profile_tag, kind,"
117		" app_id, app_display_name, device_display_name, pushkey, ts,"
118		" lang, data"
119		" FROM pushers"
120		" WHERE ? < id AND id <= ?"
121		" ORDER BY id ASC LIMIT ?"
122		)
123		txn.execute(sql, (last_id, current_id, limit))
124		updated = txn.fetchall()
125
126		sql = (
127		"SELECT stream_id, user_id, app_id, pushkey"
128		" FROM deleted_pushers"
129		" WHERE ? < stream_id AND stream_id <= ?"
130		" ORDER BY stream_id ASC LIMIT ?"
131		)
132		txn.execute(sql, (last_id, current_id, limit))
133		deleted = txn.fetchall()
134
135		return updated, deleted
136
137		return self.runInteraction(
138		"get_all_updated_pushers", get_all_updated_pushers_txn
139		)
140
141		def get_all_updated_pushers_rows(self, last_id, current_id, limit):
142		"""Get all the pushers that have changed between the given tokens.
143
144		Returns:
145		Deferred(list(tuple)): each tuple consists of:
146		stream_id (str)
147		user_id (str)
148		app_id (str)
149		pushkey (str)
150		was_deleted (bool): whether the pusher was added/updated (False)
151		or deleted (True)
152		"""
153
154		if last_id == current_id:
155		return defer.succeed([])
156
157		def get_all_updated_pushers_rows_txn(txn):
158		sql = (
159		"SELECT id, user_name, app_id, pushkey"
160		" FROM pushers"
161		" WHERE ? < id AND id <= ?"
162		" ORDER BY id ASC LIMIT ?"
163		)
164		txn.execute(sql, (last_id, current_id, limit))
165		results = [list(row) + [False] for row in txn]
166
167		sql = (
168		"SELECT stream_id, user_id, app_id, pushkey"
169		" FROM deleted_pushers"
170		" WHERE ? < stream_id AND stream_id <= ?"
171		" ORDER BY stream_id ASC LIMIT ?"
172		)
173		txn.execute(sql, (last_id, current_id, limit))
174
175		results.extend(list(row) + [True] for row in txn)
176		results.sort() # Sort so that they're ordered by stream id
177
178		return results
179
180		return self.runInteraction(
181		"get_all_updated_pushers_rows", get_all_updated_pushers_rows_txn
182		)
183
184		@cachedInlineCallbacks(num_args=1, max_entries=15000)
185		def get_if_user_has_pusher(self, user_id):
186		# This only exists for the cachedList decorator
187		raise NotImplementedError()
188
189		@cachedList(
190		cached_method_name="get_if_user_has_pusher",
191		list_name="user_ids",
192		num_args=1,
193		inlineCallbacks=True,
194		)
195		def get_if_users_have_pushers(self, user_ids):
196		rows = yield self._simple_select_many_batch(
197		table="pushers",
198		column="user_name",
199		iterable=user_ids,
200		retcols=["user_name"],
201		desc="get_if_users_have_pushers",
202		)
203
204		result = {user_id: False for user_id in user_ids}
205		result.update({r["user_name"]: True for r in rows})
206
207		return result
208
209
210		class PusherStore(PusherWorkerStore):
211		def get_pushers_stream_token(self):
212		return self._pushers_id_gen.get_current_token()
213
214		@defer.inlineCallbacks
215		def add_pusher(
216		self,
217		user_id,
218		access_token,
219		kind,
220		app_id,
221		app_display_name,
222		device_display_name,
223		pushkey,
224		pushkey_ts,
225		lang,
226		data,
227		last_stream_ordering,
228		profile_tag="",
229		):
230		with self._pushers_id_gen.get_next() as stream_id:
231		# no need to lock because `pushers` has a unique key on
232		# (app_id, pushkey, user_name) so _simple_upsert will retry
233		yield self._simple_upsert(
234		table="pushers",
235		keyvalues={"app_id": app_id, "pushkey": pushkey, "user_name": user_id},
236		values={
237		"access_token": access_token,
238		"kind": kind,
239		"app_display_name": app_display_name,
240		"device_display_name": device_display_name,
241		"ts": pushkey_ts,
242		"lang": lang,
243		"data": encode_canonical_json(data),
244		"last_stream_ordering": last_stream_ordering,
245		"profile_tag": profile_tag,
246		"id": stream_id,
247		},
248		desc="add_pusher",
249		lock=False,
250		)
251
252		user_has_pusher = self.get_if_user_has_pusher.cache.get(
253		(user_id,), None, update_metrics=False
254		)
255
256		if user_has_pusher is not True:
257		# invalidate, since we the user might not have had a pusher before
258		yield self.runInteraction(
259		"add_pusher",
260		self._invalidate_cache_and_stream,
261		self.get_if_user_has_pusher,
262		(user_id,),
263		)
264
265		@defer.inlineCallbacks
266		def delete_pusher_by_app_id_pushkey_user_id(self, app_id, pushkey, user_id):
267		def delete_pusher_txn(txn, stream_id):
268		self._invalidate_cache_and_stream(
269		txn, self.get_if_user_has_pusher, (user_id,)
270		)
271
272		self._simple_delete_one_txn(
273		txn,
274		"pushers",
275		{"app_id": app_id, "pushkey": pushkey, "user_name": user_id},
276		)
277
278		# it's possible for us to end up with duplicate rows for
279		# (app_id, pushkey, user_id) at different stream_ids, but that
280		# doesn't really matter.
281		self._simple_insert_txn(
282		txn,
283		table="deleted_pushers",
284		values={
285		"stream_id": stream_id,
286		"app_id": app_id,
287		"pushkey": pushkey,
288		"user_id": user_id,
289		},
290		)
291
292		with self._pushers_id_gen.get_next() as stream_id:
293		yield self.runInteraction("delete_pusher", delete_pusher_txn, stream_id)
294
295		@defer.inlineCallbacks
296		def update_pusher_last_stream_ordering(
297		self, app_id, pushkey, user_id, last_stream_ordering
298		):
299		yield self._simple_update_one(
300		"pushers",
301		{"app_id": app_id, "pushkey": pushkey, "user_name": user_id},
302		{"last_stream_ordering": last_stream_ordering},
303		desc="update_pusher_last_stream_ordering",
304		)
305
306		@defer.inlineCallbacks
307		def update_pusher_last_stream_ordering_and_success(
308		self, app_id, pushkey, user_id, last_stream_ordering, last_success
309		):
310		"""Update the last stream ordering position we've processed up to for
311		the given pusher.
312
313		Args:
314		app_id (str)
315		pushkey (str)
316		last_stream_ordering (int)
317		last_success (int)
318
319		Returns:
320		Deferred[bool]: True if the pusher still exists; False if it has been deleted.
321		"""
322		updated = yield self._simple_update(
323		table="pushers",
324		keyvalues={"app_id": app_id, "pushkey": pushkey, "user_name": user_id},
325		updatevalues={
326		"last_stream_ordering": last_stream_ordering,
327		"last_success": last_success,
328		},
329		desc="update_pusher_last_stream_ordering_and_success",
330		)
331
332		return bool(updated)
333
334		@defer.inlineCallbacks
335		def update_pusher_failing_since(self, app_id, pushkey, user_id, failing_since):
336		yield self._simple_update(
337		table="pushers",
338		keyvalues={"app_id": app_id, "pushkey": pushkey, "user_name": user_id},
339		updatevalues={"failing_since": failing_since},
340		desc="update_pusher_failing_since",
341		)
342
343		@defer.inlineCallbacks
344		def get_throttle_params_by_room(self, pusher_id):
345		res = yield self._simple_select_list(
346		"pusher_throttle",
347		{"pusher": pusher_id},
348		["room_id", "last_sent_ts", "throttle_ms"],
349		desc="get_throttle_params_by_room",
350		)
351
352		params_by_room = {}
353		for row in res:
354		params_by_room[row["room_id"]] = {
355		"last_sent_ts": row["last_sent_ts"],
356		"throttle_ms": row["throttle_ms"],
357		}
358
359		return params_by_room
360
361		@defer.inlineCallbacks
362		def set_throttle_params(self, pusher_id, room_id, params):
363		# no need to lock because `pusher_throttle` has a primary key on
364		# (pusher, room_id) so _simple_upsert will retry
365		yield self._simple_upsert(
366		"pusher_throttle",
367		{"pusher": pusher_id, "room_id": room_id},
368		params,
369		desc="set_throttle_params",
370		lock=False,
371		)

+0

-529

~~synapse/storage/receipts.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2014-2016 OpenMarket Ltd
2		# Copyright 2018 New Vector Ltd
3		#
4		# Licensed under the Apache License, Version 2.0 (the "License");
5		# you may not use this file except in compliance with the License.
6		# You may obtain a copy of the License at
7		#
8		# http://www.apache.org/licenses/LICENSE-2.0
9		#
10		# Unless required by applicable law or agreed to in writing, software
11		# distributed under the License is distributed on an "AS IS" BASIS,
12		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13		# See the License for the specific language governing permissions and
14		# limitations under the License.
15
16		import abc
17		import logging
18
19		from canonicaljson import json
20
21		from twisted.internet import defer
22
23		from synapse.util.caches.descriptors import cached, cachedInlineCallbacks, cachedList
24		from synapse.util.caches.stream_change_cache import StreamChangeCache
25
26		from ._base import SQLBaseStore
27		from .util.id_generators import StreamIdGenerator
28
29		logger = logging.getLogger(__name__)
30
31
32		class ReceiptsWorkerStore(SQLBaseStore):
33		"""This is an abstract base class where subclasses must implement
34		`get_max_receipt_stream_id` which can be called in the initializer.
35		"""
36
37		# This ABCMeta metaclass ensures that we cannot be instantiated without
38		# the abstract methods being implemented.
39		__metaclass__ = abc.ABCMeta
40
41		def __init__(self, db_conn, hs):
42		super(ReceiptsWorkerStore, self).__init__(db_conn, hs)
43
44		self._receipts_stream_cache = StreamChangeCache(
45		"ReceiptsRoomChangeCache", self.get_max_receipt_stream_id()
46		)
47
48		@abc.abstractmethod
49		def get_max_receipt_stream_id(self):
50		"""Get the current max stream ID for receipts stream
51
52		Returns:
53		int
54		"""
55		raise NotImplementedError()
56
57		@cachedInlineCallbacks()
58		def get_users_with_read_receipts_in_room(self, room_id):
59		receipts = yield self.get_receipts_for_room(room_id, "m.read")
60		return set(r["user_id"] for r in receipts)
61
62		@cached(num_args=2)
63		def get_receipts_for_room(self, room_id, receipt_type):
64		return self._simple_select_list(
65		table="receipts_linearized",
66		keyvalues={"room_id": room_id, "receipt_type": receipt_type},
67		retcols=("user_id", "event_id"),
68		desc="get_receipts_for_room",
69		)
70
71		@cached(num_args=3)
72		def get_last_receipt_event_id_for_user(self, user_id, room_id, receipt_type):
73		return self._simple_select_one_onecol(
74		table="receipts_linearized",
75		keyvalues={
76		"room_id": room_id,
77		"receipt_type": receipt_type,
78		"user_id": user_id,
79		},
80		retcol="event_id",
81		desc="get_own_receipt_for_user",
82		allow_none=True,
83		)
84
85		@cachedInlineCallbacks(num_args=2)
86		def get_receipts_for_user(self, user_id, receipt_type):
87		rows = yield self._simple_select_list(
88		table="receipts_linearized",
89		keyvalues={"user_id": user_id, "receipt_type": receipt_type},
90		retcols=("room_id", "event_id"),
91		desc="get_receipts_for_user",
92		)
93
94		return {row["room_id"]: row["event_id"] for row in rows}
95
96		@defer.inlineCallbacks
97		def get_receipts_for_user_with_orderings(self, user_id, receipt_type):
98		def f(txn):
99		sql = (
100		"SELECT rl.room_id, rl.event_id,"
101		" e.topological_ordering, e.stream_ordering"
102		" FROM receipts_linearized AS rl"
103		" INNER JOIN events AS e USING (room_id, event_id)"
104		" WHERE rl.room_id = e.room_id"
105		" AND rl.event_id = e.event_id"
106		" AND user_id = ?"
107		)
108		txn.execute(sql, (user_id,))
109		return txn.fetchall()
110
111		rows = yield self.runInteraction("get_receipts_for_user_with_orderings", f)
112		return {
113		row[0]: {
114		"event_id": row[1],
115		"topological_ordering": row[2],
116		"stream_ordering": row[3],
117		}
118		for row in rows
119		}
120
121		@defer.inlineCallbacks
122		def get_linearized_receipts_for_rooms(self, room_ids, to_key, from_key=None):
123		"""Get receipts for multiple rooms for sending to clients.
124
125		Args:
126		room_ids (list): List of room_ids.
127		to_key (int): Max stream id to fetch receipts upto.
128		from_key (int): Min stream id to fetch receipts from. None fetches
129		from the start.
130
131		Returns:
132		list: A list of receipts.
133		"""
134		room_ids = set(room_ids)
135
136		if from_key is not None:
137		# Only ask the database about rooms where there have been new
138		# receipts added since `from_key`
139		room_ids = yield self._receipts_stream_cache.get_entities_changed(
140		room_ids, from_key
141		)
142
143		results = yield self._get_linearized_receipts_for_rooms(
144		room_ids, to_key, from_key=from_key
145		)
146
147		return [ev for res in results.values() for ev in res]
148
149		def get_linearized_receipts_for_room(self, room_id, to_key, from_key=None):
150		"""Get receipts for a single room for sending to clients.
151
152		Args:
153		room_ids (str): The room id.
154		to_key (int): Max stream id to fetch receipts upto.
155		from_key (int): Min stream id to fetch receipts from. None fetches
156		from the start.
157
158		Returns:
159		Deferred[list]: A list of receipts.
160		"""
161		if from_key is not None:
162		# Check the cache first to see if any new receipts have been added
163		# since`from_key`. If not we can no-op.
164		if not self._receipts_stream_cache.has_entity_changed(room_id, from_key):
165		defer.succeed([])
166
167		return self._get_linearized_receipts_for_room(room_id, to_key, from_key)
168
169		@cachedInlineCallbacks(num_args=3, tree=True)
170		def _get_linearized_receipts_for_room(self, room_id, to_key, from_key=None):
171		"""See get_linearized_receipts_for_room
172		"""
173
174		def f(txn):
175		if from_key:
176		sql = (
177		"SELECT * FROM receipts_linearized WHERE"
178		" room_id = ? AND stream_id > ? AND stream_id <= ?"
179		)
180
181		txn.execute(sql, (room_id, from_key, to_key))
182		else:
183		sql = (
184		"SELECT * FROM receipts_linearized WHERE"
185		" room_id = ? AND stream_id <= ?"
186		)
187
188		txn.execute(sql, (room_id, to_key))
189
190		rows = self.cursor_to_dict(txn)
191
192		return rows
193
194		rows = yield self.runInteraction("get_linearized_receipts_for_room", f)
195
196		if not rows:
197		return []
198
199		content = {}
200		for row in rows:
201		content.setdefault(row["event_id"], {}).setdefault(row["receipt_type"], {})[
202		row["user_id"]
203		] = json.loads(row["data"])
204
205		return [{"type": "m.receipt", "room_id": room_id, "content": content}]
206
207		@cachedList(
208		cached_method_name="_get_linearized_receipts_for_room",
209		list_name="room_ids",
210		num_args=3,
211		inlineCallbacks=True,
212		)
213		def _get_linearized_receipts_for_rooms(self, room_ids, to_key, from_key=None):
214		if not room_ids:
215		return {}
216
217		def f(txn):
218		if from_key:
219		sql = (
220		"SELECT * FROM receipts_linearized WHERE"
221		" room_id IN (%s) AND stream_id > ? AND stream_id <= ?"
222		) % (",".join(["?"] * len(room_ids)))
223		args = list(room_ids)
224		args.extend([from_key, to_key])
225
226		txn.execute(sql, args)
227		else:
228		sql = (
229		"SELECT * FROM receipts_linearized WHERE"
230		" room_id IN (%s) AND stream_id <= ?"
231		) % (",".join(["?"] * len(room_ids)))
232
233		args = list(room_ids)
234		args.append(to_key)
235
236		txn.execute(sql, args)
237
238		return self.cursor_to_dict(txn)
239
240		txn_results = yield self.runInteraction("_get_linearized_receipts_for_rooms", f)
241
242		results = {}
243		for row in txn_results:
244		# We want a single event per room, since we want to batch the
245		# receipts by room, event and type.
246		room_event = results.setdefault(
247		row["room_id"],
248		{"type": "m.receipt", "room_id": row["room_id"], "content": {}},
249		)
250
251		# The content is of the form:
252		# {"$foo:bar": { "read": { "@user:host": <receipt> }, .. }, .. }
253		event_entry = room_event["content"].setdefault(row["event_id"], {})
254		receipt_type = event_entry.setdefault(row["receipt_type"], {})
255
256		receipt_type[row["user_id"]] = json.loads(row["data"])
257
258		results = {
259		room_id: [results[room_id]] if room_id in results else []
260		for room_id in room_ids
261		}
262		return results
263
264		def get_all_updated_receipts(self, last_id, current_id, limit=None):
265		if last_id == current_id:
266		return defer.succeed([])
267
268		def get_all_updated_receipts_txn(txn):
269		sql = (
270		"SELECT stream_id, room_id, receipt_type, user_id, event_id, data"
271		" FROM receipts_linearized"
272		" WHERE ? < stream_id AND stream_id <= ?"
273		" ORDER BY stream_id ASC"
274		)
275		args = [last_id, current_id]
276		if limit is not None:
277		sql += " LIMIT ?"
278		args.append(limit)
279		txn.execute(sql, args)
280
281		return (r[0:5] + (json.loads(r[5]),) for r in txn)
282
283		return self.runInteraction(
284		"get_all_updated_receipts", get_all_updated_receipts_txn
285		)
286
287		def _invalidate_get_users_with_receipts_in_room(
288		self, room_id, receipt_type, user_id
289		):
290		if receipt_type != "m.read":
291		return
292
293		# Returns either an ObservableDeferred or the raw result
294		res = self.get_users_with_read_receipts_in_room.cache.get(
295		room_id, None, update_metrics=False
296		)
297
298		# first handle the Deferred case
299		if isinstance(res, defer.Deferred):
300		if res.called:
301		res = res.result
302		else:
303		res = None
304
305		if res and user_id in res:
306		# We'd only be adding to the set, so no point invalidating if the
307		# user is already there
308		return
309
310		self.get_users_with_read_receipts_in_room.invalidate((room_id,))
311
312
313		class ReceiptsStore(ReceiptsWorkerStore):
314		def __init__(self, db_conn, hs):
315		# We instantiate this first as the ReceiptsWorkerStore constructor
316		# needs to be able to call get_max_receipt_stream_id
317		self._receipts_id_gen = StreamIdGenerator(
318		db_conn, "receipts_linearized", "stream_id"
319		)
320
321		super(ReceiptsStore, self).__init__(db_conn, hs)
322
323		def get_max_receipt_stream_id(self):
324		return self._receipts_id_gen.get_current_token()
325
326		def insert_linearized_receipt_txn(
327		self, txn, room_id, receipt_type, user_id, event_id, data, stream_id
328		):
329		"""Inserts a read-receipt into the database if it's newer than the current RR
330
331		Returns: int\|None
332		None if the RR is older than the current RR
333		otherwise, the rx timestamp of the event that the RR corresponds to
334		(or 0 if the event is unknown)
335		"""
336		res = self._simple_select_one_txn(
337		txn,
338		table="events",
339		retcols=["stream_ordering", "received_ts"],
340		keyvalues={"event_id": event_id},
341		allow_none=True,
342		)
343
344		stream_ordering = int(res["stream_ordering"]) if res else None
345		rx_ts = res["received_ts"] if res else 0
346
347		# We don't want to clobber receipts for more recent events, so we
348		# have to compare orderings of existing receipts
349		if stream_ordering is not None:
350		sql = (
351		"SELECT stream_ordering, event_id FROM events"
352		" INNER JOIN receipts_linearized as r USING (event_id, room_id)"
353		" WHERE r.room_id = ? AND r.receipt_type = ? AND r.user_id = ?"
354		)
355		txn.execute(sql, (room_id, receipt_type, user_id))
356
357		for so, eid in txn:
358		if int(so) >= stream_ordering:
359		logger.debug(
360		"Ignoring new receipt for %s in favour of existing "
361		"one for later event %s",
362		event_id,
363		eid,
364		)
365		return None
366
367		txn.call_after(self.get_receipts_for_room.invalidate, (room_id, receipt_type))
368		txn.call_after(
369		self._invalidate_get_users_with_receipts_in_room,
370		room_id,
371		receipt_type,
372		user_id,
373		)
374		txn.call_after(self.get_receipts_for_user.invalidate, (user_id, receipt_type))
375		# FIXME: This shouldn't invalidate the whole cache
376		txn.call_after(
377		self._get_linearized_receipts_for_room.invalidate_many, (room_id,)
378		)
379
380		txn.call_after(
381		self._receipts_stream_cache.entity_has_changed, room_id, stream_id
382		)
383
384		txn.call_after(
385		self.get_last_receipt_event_id_for_user.invalidate,
386		(user_id, room_id, receipt_type),
387		)
388
389		self._simple_delete_txn(
390		txn,
391		table="receipts_linearized",
392		keyvalues={
393		"room_id": room_id,
394		"receipt_type": receipt_type,
395		"user_id": user_id,
396		},
397		)
398
399		self._simple_insert_txn(
400		txn,
401		table="receipts_linearized",
402		values={
403		"stream_id": stream_id,
404		"room_id": room_id,
405		"receipt_type": receipt_type,
406		"user_id": user_id,
407		"event_id": event_id,
408		"data": json.dumps(data),
409		},
410		)
411
412		if receipt_type == "m.read" and stream_ordering is not None:
413		self._remove_old_push_actions_before_txn(
414		txn, room_id=room_id, user_id=user_id, stream_ordering=stream_ordering
415		)
416
417		return rx_ts
418
419		@defer.inlineCallbacks
420		def insert_receipt(self, room_id, receipt_type, user_id, event_ids, data):
421		"""Insert a receipt, either from local client or remote server.
422
423		Automatically does conversion between linearized and graph
424		representations.
425		"""
426		if not event_ids:
427		return
428
429		if len(event_ids) == 1:
430		linearized_event_id = event_ids[0]
431		else:
432		# we need to points in graph -> linearized form.
433		# TODO: Make this better.
434		def graph_to_linear(txn):
435		query = (
436		"SELECT event_id WHERE room_id = ? AND stream_ordering IN ("
437		" SELECT max(stream_ordering) WHERE event_id IN (%s)"
438		")"
439		) % (",".join(["?"] * len(event_ids)))
440
441		txn.execute(query, [room_id] + event_ids)
442		rows = txn.fetchall()
443		if rows:
444		return rows[0][0]
445		else:
446		raise RuntimeError("Unrecognized event_ids: %r" % (event_ids,))
447
448		linearized_event_id = yield self.runInteraction(
449		"insert_receipt_conv", graph_to_linear
450		)
451
452		stream_id_manager = self._receipts_id_gen.get_next()
453		with stream_id_manager as stream_id:
454		event_ts = yield self.runInteraction(
455		"insert_linearized_receipt",
456		self.insert_linearized_receipt_txn,
457		room_id,
458		receipt_type,
459		user_id,
460		linearized_event_id,
461		data,
462		stream_id=stream_id,
463		)
464
465		if event_ts is None:
466		return None
467
468		now = self._clock.time_msec()
469		logger.debug(
470		"RR for event %s in %s (%i ms old)",
471		linearized_event_id,
472		room_id,
473		now - event_ts,
474		)
475
476		yield self.insert_graph_receipt(room_id, receipt_type, user_id, event_ids, data)
477
478		max_persisted_id = self._receipts_id_gen.get_current_token()
479
480		return stream_id, max_persisted_id
481
482		def insert_graph_receipt(self, room_id, receipt_type, user_id, event_ids, data):
483		return self.runInteraction(
484		"insert_graph_receipt",
485		self.insert_graph_receipt_txn,
486		room_id,
487		receipt_type,
488		user_id,
489		event_ids,
490		data,
491		)
492
493		def insert_graph_receipt_txn(
494		self, txn, room_id, receipt_type, user_id, event_ids, data
495		):
496		txn.call_after(self.get_receipts_for_room.invalidate, (room_id, receipt_type))
497		txn.call_after(
498		self._invalidate_get_users_with_receipts_in_room,
499		room_id,
500		receipt_type,
501		user_id,
502		)
503		txn.call_after(self.get_receipts_for_user.invalidate, (user_id, receipt_type))
504		# FIXME: This shouldn't invalidate the whole cache
505		txn.call_after(
506		self._get_linearized_receipts_for_room.invalidate_many, (room_id,)
507		)
508
509		self._simple_delete_txn(
510		txn,
511		table="receipts_graph",
512		keyvalues={
513		"room_id": room_id,
514		"receipt_type": receipt_type,
515		"user_id": user_id,
516		},
517		)
518		self._simple_insert_txn(
519		txn,
520		table="receipts_graph",
521		values={
522		"room_id": room_id,
523		"receipt_type": receipt_type,
524		"user_id": user_id,
525		"event_ids": json.dumps(event_ids),
526		"data": json.dumps(data),
527		},
528		)

+0

-1491

~~synapse/storage/registration.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2014-2016 OpenMarket Ltd
2		# Copyright 2017-2018 New Vector Ltd
3		# Copyright 2019 The Matrix.org Foundation C.I.C.
4		#
5		# Licensed under the Apache License, Version 2.0 (the "License");
6		# you may not use this file except in compliance with the License.
7		# You may obtain a copy of the License at
8		#
9		# http://www.apache.org/licenses/LICENSE-2.0
10		#
11		# Unless required by applicable law or agreed to in writing, software
12		# distributed under the License is distributed on an "AS IS" BASIS,
13		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14		# See the License for the specific language governing permissions and
15		# limitations under the License.
16
17		import logging
18		import re
19
20		from six import iterkeys
21		from six.moves import range
22
23		from twisted.internet import defer
24		from twisted.internet.defer import Deferred
25
26		from synapse.api.constants import UserTypes
27		from synapse.api.errors import Codes, StoreError, SynapseError, ThreepidValidationError
28		from synapse.metrics.background_process_metrics import run_as_background_process
29		from synapse.storage import background_updates
30		from synapse.storage._base import SQLBaseStore
31		from synapse.types import UserID
32		from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
33
34		THIRTY_MINUTES_IN_MS = 30 * 60 * 1000
35
36		logger = logging.getLogger(__name__)
37
38
39		class RegistrationWorkerStore(SQLBaseStore):
40		def __init__(self, db_conn, hs):
41		super(RegistrationWorkerStore, self).__init__(db_conn, hs)
42
43		self.config = hs.config
44		self.clock = hs.get_clock()
45
46		@cached()
47		def get_user_by_id(self, user_id):
48		return self._simple_select_one(
49		table="users",
50		keyvalues={"name": user_id},
51		retcols=[
52		"name",
53		"password_hash",
54		"is_guest",
55		"consent_version",
56		"consent_server_notice_sent",
57		"appservice_id",
58		"creation_ts",
59		"user_type",
60		],
61		allow_none=True,
62		desc="get_user_by_id",
63		)
64
65		@defer.inlineCallbacks
66		def is_trial_user(self, user_id):
67		"""Checks if user is in the "trial" period, i.e. within the first
68		N days of registration defined by `mau_trial_days` config
69
70		Args:
71		user_id (str)
72
73		Returns:
74		Deferred[bool]
75		"""
76
77		info = yield self.get_user_by_id(user_id)
78		if not info:
79		return False
80
81		now = self.clock.time_msec()
82		trial_duration_ms = self.config.mau_trial_days * 24 * 60 * 60 * 1000
83		is_trial = (now - info["creation_ts"] * 1000) < trial_duration_ms
84		return is_trial
85
86		@cached()
87		def get_user_by_access_token(self, token):
88		"""Get a user from the given access token.
89
90		Args:
91		token (str): The access token of a user.
92		Returns:
93		defer.Deferred: None, if the token did not match, otherwise dict
94		including the keys `name`, `is_guest`, `device_id`, `token_id`,
95		`valid_until_ms`.
96		"""
97		return self.runInteraction(
98		"get_user_by_access_token", self._query_for_auth, token
99		)
100
101		@cachedInlineCallbacks()
102		def get_expiration_ts_for_user(self, user_id):
103		"""Get the expiration timestamp for the account bearing a given user ID.
104
105		Args:
106		user_id (str): The ID of the user.
107		Returns:
108		defer.Deferred: None, if the account has no expiration timestamp,
109		otherwise int representation of the timestamp (as a number of
110		milliseconds since epoch).
111		"""
112		res = yield self._simple_select_one_onecol(
113		table="account_validity",
114		keyvalues={"user_id": user_id},
115		retcol="expiration_ts_ms",
116		allow_none=True,
117		desc="get_expiration_ts_for_user",
118		)
119		return res
120
121		@defer.inlineCallbacks
122		def set_account_validity_for_user(
123		self, user_id, expiration_ts, email_sent, renewal_token=None
124		):
125		"""Updates the account validity properties of the given account, with the
126		given values.
127
128		Args:
129		user_id (str): ID of the account to update properties for.
130		expiration_ts (int): New expiration date, as a timestamp in milliseconds
131		since epoch.
132		email_sent (bool): True means a renewal email has been sent for this
133		account and there's no need to send another one for the current validity
134		period.
135		renewal_token (str): Renewal token the user can use to extend the validity
136		of their account. Defaults to no token.
137		"""
138
139		def set_account_validity_for_user_txn(txn):
140		self._simple_update_txn(
141		txn=txn,
142		table="account_validity",
143		keyvalues={"user_id": user_id},
144		updatevalues={
145		"expiration_ts_ms": expiration_ts,
146		"email_sent": email_sent,
147		"renewal_token": renewal_token,
148		},
149		)
150		self._invalidate_cache_and_stream(
151		txn, self.get_expiration_ts_for_user, (user_id,)
152		)
153
154		yield self.runInteraction(
155		"set_account_validity_for_user", set_account_validity_for_user_txn
156		)
157
158		@defer.inlineCallbacks
159		def set_renewal_token_for_user(self, user_id, renewal_token):
160		"""Defines a renewal token for a given user.
161
162		Args:
163		user_id (str): ID of the user to set the renewal token for.
164		renewal_token (str): Random unique string that will be used to renew the
165		user's account.
166
167		Raises:
168		StoreError: The provided token is already set for another user.
169		"""
170		yield self._simple_update_one(
171		table="account_validity",
172		keyvalues={"user_id": user_id},
173		updatevalues={"renewal_token": renewal_token},
174		desc="set_renewal_token_for_user",
175		)
176
177		@defer.inlineCallbacks
178		def get_user_from_renewal_token(self, renewal_token):
179		"""Get a user ID from a renewal token.
180
181		Args:
182		renewal_token (str): The renewal token to perform the lookup with.
183
184		Returns:
185		defer.Deferred[str]: The ID of the user to which the token belongs.
186		"""
187		res = yield self._simple_select_one_onecol(
188		table="account_validity",
189		keyvalues={"renewal_token": renewal_token},
190		retcol="user_id",
191		desc="get_user_from_renewal_token",
192		)
193
194		return res
195
196		@defer.inlineCallbacks
197		def get_renewal_token_for_user(self, user_id):
198		"""Get the renewal token associated with a given user ID.
199
200		Args:
201		user_id (str): The user ID to lookup a token for.
202
203		Returns:
204		defer.Deferred[str]: The renewal token associated with this user ID.
205		"""
206		res = yield self._simple_select_one_onecol(
207		table="account_validity",
208		keyvalues={"user_id": user_id},
209		retcol="renewal_token",
210		desc="get_renewal_token_for_user",
211		)
212
213		return res
214
215		@defer.inlineCallbacks
216		def get_users_expiring_soon(self):
217		"""Selects users whose account will expire in the [now, now + renew_at] time
218		window (see configuration for account_validity for information on what renew_at
219		refers to).
220
221		Returns:
222		Deferred: Resolves to a list[dict[user_id (str), expiration_ts_ms (int)]]
223		"""
224
225		def select_users_txn(txn, now_ms, renew_at):
226		sql = (
227		"SELECT user_id, expiration_ts_ms FROM account_validity"
228		" WHERE email_sent = ? AND (expiration_ts_ms - ?) <= ?"
229		)
230		values = [False, now_ms, renew_at]
231		txn.execute(sql, values)
232		return self.cursor_to_dict(txn)
233
234		res = yield self.runInteraction(
235		"get_users_expiring_soon",
236		select_users_txn,
237		self.clock.time_msec(),
238		self.config.account_validity.renew_at,
239		)
240
241		return res
242
243		@defer.inlineCallbacks
244		def set_renewal_mail_status(self, user_id, email_sent):
245		"""Sets or unsets the flag that indicates whether a renewal email has been sent
246		to the user (and the user hasn't renewed their account yet).
247
248		Args:
249		user_id (str): ID of the user to set/unset the flag for.
250		email_sent (bool): Flag which indicates whether a renewal email has been sent
251		to this user.
252		"""
253		yield self._simple_update_one(
254		table="account_validity",
255		keyvalues={"user_id": user_id},
256		updatevalues={"email_sent": email_sent},
257		desc="set_renewal_mail_status",
258		)
259
260		@defer.inlineCallbacks
261		def delete_account_validity_for_user(self, user_id):
262		"""Deletes the entry for the given user in the account validity table, removing
263		their expiration date and renewal token.
264
265		Args:
266		user_id (str): ID of the user to remove from the account validity table.
267		"""
268		yield self._simple_delete_one(
269		table="account_validity",
270		keyvalues={"user_id": user_id},
271		desc="delete_account_validity_for_user",
272		)
273
274		@defer.inlineCallbacks
275		def is_server_admin(self, user):
276		"""Determines if a user is an admin of this homeserver.
277
278		Args:
279		user (UserID): user ID of the user to test
280
281		Returns (bool):
282		true iff the user is a server admin, false otherwise.
283		"""
284		res = yield self._simple_select_one_onecol(
285		table="users",
286		keyvalues={"name": user.to_string()},
287		retcol="admin",
288		allow_none=True,
289		desc="is_server_admin",
290		)
291
292		return res if res else False
293
294		def set_server_admin(self, user, admin):
295		"""Sets whether a user is an admin of this homeserver.
296
297		Args:
298		user (UserID): user ID of the user to test
299		admin (bool): true iff the user is to be a server admin,
300		false otherwise.
301		"""
302		return self._simple_update_one(
303		table="users",
304		keyvalues={"name": user.to_string()},
305		updatevalues={"admin": 1 if admin else 0},
306		desc="set_server_admin",
307		)
308
309		def _query_for_auth(self, txn, token):
310		sql = (
311		"SELECT users.name, users.is_guest, access_tokens.id as token_id,"
312		" access_tokens.device_id, access_tokens.valid_until_ms"
313		" FROM users"
314		" INNER JOIN access_tokens on users.name = access_tokens.user_id"
315		" WHERE token = ?"
316		)
317
318		txn.execute(sql, (token,))
319		rows = self.cursor_to_dict(txn)
320		if rows:
321		return rows[0]
322
323		return None
324
325		@cachedInlineCallbacks()
326		def is_real_user(self, user_id):
327		"""Determines if the user is a real user, ie does not have a 'user_type'.
328
329		Args:
330		user_id (str): user id to test
331
332		Returns:
333		Deferred[bool]: True if user 'user_type' is null or empty string
334		"""
335		res = yield self.runInteraction("is_real_user", self.is_real_user_txn, user_id)
336		return res
337
338		@cachedInlineCallbacks()
339		def is_support_user(self, user_id):
340		"""Determines if the user is of type UserTypes.SUPPORT
341
342		Args:
343		user_id (str): user id to test
344
345		Returns:
346		Deferred[bool]: True if user is of type UserTypes.SUPPORT
347		"""
348		res = yield self.runInteraction(
349		"is_support_user", self.is_support_user_txn, user_id
350		)
351		return res
352
353		def is_real_user_txn(self, txn, user_id):
354		res = self._simple_select_one_onecol_txn(
355		txn=txn,
356		table="users",
357		keyvalues={"name": user_id},
358		retcol="user_type",
359		allow_none=True,
360		)
361		return res is None
362
363		def is_support_user_txn(self, txn, user_id):
364		res = self._simple_select_one_onecol_txn(
365		txn=txn,
366		table="users",
367		keyvalues={"name": user_id},
368		retcol="user_type",
369		allow_none=True,
370		)
371		return True if res == UserTypes.SUPPORT else False
372
373		def get_users_by_id_case_insensitive(self, user_id):
374		"""Gets users that match user_id case insensitively.
375		Returns a mapping of user_id -> password_hash.
376		"""
377
378		def f(txn):
379		sql = (
380		"SELECT name, password_hash FROM users" " WHERE lower(name) = lower(?)"
381		)
382		txn.execute(sql, (user_id,))
383		return dict(txn)
384
385		return self.runInteraction("get_users_by_id_case_insensitive", f)
386
387		async def get_user_by_external_id(
388		self, auth_provider: str, external_id: str
389		) -> str:
390		"""Look up a user by their external auth id
391
392		Args:
393		auth_provider: identifier for the remote auth provider
394		external_id: id on that system
395
396		Returns:
397		str\|None: the mxid of the user, or None if they are not known
398		"""
399		return await self._simple_select_one_onecol(
400		table="user_external_ids",
401		keyvalues={"auth_provider": auth_provider, "external_id": external_id},
402		retcol="user_id",
403		allow_none=True,
404		desc="get_user_by_external_id",
405		)
406
407		@defer.inlineCallbacks
408		def count_all_users(self):
409		"""Counts all users registered on the homeserver."""
410
411		def _count_users(txn):
412		txn.execute("SELECT COUNT(*) AS users FROM users")
413		rows = self.cursor_to_dict(txn)
414		if rows:
415		return rows[0]["users"]
416		return 0
417
418		ret = yield self.runInteraction("count_users", _count_users)
419		return ret
420
421		def count_daily_user_type(self):
422		"""
423		Counts 1) native non guest users
424		2) native guests users
425		3) bridged users
426		who registered on the homeserver in the past 24 hours
427		"""
428
429		def _count_daily_user_type(txn):
430		yesterday = int(self._clock.time()) - (60 * 60 * 24)
431
432		sql = """
433		SELECT user_type, COALESCE(count(*), 0) AS count FROM (
434		SELECT
435		CASE
436		WHEN is_guest=0 AND appservice_id IS NULL THEN 'native'
437		WHEN is_guest=1 AND appservice_id IS NULL THEN 'guest'
438		WHEN is_guest=0 AND appservice_id IS NOT NULL THEN 'bridged'
439		END AS user_type
440		FROM users
441		WHERE creation_ts > ?
442		) AS t GROUP BY user_type
443		"""
444		results = {"native": 0, "guest": 0, "bridged": 0}
445		txn.execute(sql, (yesterday,))
446		for row in txn:
447		results[row[0]] = row[1]
448		return results
449
450		return self.runInteraction("count_daily_user_type", _count_daily_user_type)
451
452		@defer.inlineCallbacks
453		def count_nonbridged_users(self):
454		def _count_users(txn):
455		txn.execute(
456		"""
457		SELECT COALESCE(COUNT(*), 0) FROM users
458		WHERE appservice_id IS NULL
459		"""
460		)
461		count, = txn.fetchone()
462		return count
463
464		ret = yield self.runInteraction("count_users", _count_users)
465		return ret
466
467		@defer.inlineCallbacks
468		def count_real_users(self):
469		"""Counts all users without a special user_type registered on the homeserver."""
470
471		def _count_users(txn):
472		txn.execute("SELECT COUNT(*) AS users FROM users where user_type is null")
473		rows = self.cursor_to_dict(txn)
474		if rows:
475		return rows[0]["users"]
476		return 0
477
478		ret = yield self.runInteraction("count_real_users", _count_users)
479		return ret
480
481		@defer.inlineCallbacks
482		def find_next_generated_user_id_localpart(self):
483		"""
484		Gets the localpart of the next generated user ID.
485
486		Generated user IDs are integers, and we aim for them to be as small as
487		we can. Unfortunately, it's possible some of them are already taken by
488		existing users, and there may be gaps in the already taken range. This
489		function returns the start of the first allocatable gap. This is to
490		avoid the case of ID 10000000 being pre-allocated, so us wasting the
491		first (and shortest) many generated user IDs.
492		"""
493
494		def _find_next_generated_user_id(txn):
495		txn.execute("SELECT name FROM users")
496
497		regex = re.compile(r"^@(\d+):")
498
499		found = set()
500
501		for (user_id,) in txn:
502		match = regex.search(user_id)
503		if match:
504		found.add(int(match.group(1)))
505		for i in range(len(found) + 1):
506		if i not in found:
507		return i
508
509		return (
510		(
511		yield self.runInteraction(
512		"find_next_generated_user_id", _find_next_generated_user_id
513		)
514		)
515		)
516
517		@defer.inlineCallbacks
518		def get_user_id_by_threepid(self, medium, address):
519		"""Returns user id from threepid
520
521		Args:
522		medium (str): threepid medium e.g. email
523		address (str): threepid address e.g. me@example.com
524
525		Returns:
526		Deferred[str\|None]: user id or None if no user id/threepid mapping exists
527		"""
528		user_id = yield self.runInteraction(
529		"get_user_id_by_threepid", self.get_user_id_by_threepid_txn, medium, address
530		)
531		return user_id
532
533		def get_user_id_by_threepid_txn(self, txn, medium, address):
534		"""Returns user id from threepid
535
536		Args:
537		txn (cursor):
538		medium (str): threepid medium e.g. email
539		address (str): threepid address e.g. me@example.com
540
541		Returns:
542		str\|None: user id or None if no user id/threepid mapping exists
543		"""
544		ret = self._simple_select_one_txn(
545		txn,
546		"user_threepids",
547		{"medium": medium, "address": address},
548		["user_id"],
549		True,
550		)
551		if ret:
552		return ret["user_id"]
553		return None
554
555		@defer.inlineCallbacks
556		def user_add_threepid(self, user_id, medium, address, validated_at, added_at):
557		yield self._simple_upsert(
558		"user_threepids",
559		{"medium": medium, "address": address},
560		{"user_id": user_id, "validated_at": validated_at, "added_at": added_at},
561		)
562
563		@defer.inlineCallbacks
564		def user_get_threepids(self, user_id):
565		ret = yield self._simple_select_list(
566		"user_threepids",
567		{"user_id": user_id},
568		["medium", "address", "validated_at", "added_at"],
569		"user_get_threepids",
570		)
571		return ret
572
573		def user_delete_threepid(self, user_id, medium, address):
574		return self._simple_delete(
575		"user_threepids",
576		keyvalues={"user_id": user_id, "medium": medium, "address": address},
577		desc="user_delete_threepids",
578		)
579
580		def add_user_bound_threepid(self, user_id, medium, address, id_server):
581		"""The server proxied a bind request to the given identity server on
582		behalf of the given user. We need to remember this in case the user
583		asks us to unbind the threepid.
584
585		Args:
586		user_id (str)
587		medium (str)
588		address (str)
589		id_server (str)
590
591		Returns:
592		Deferred
593		"""
594		# We need to use an upsert, in case they user had already bound the
595		# threepid
596		return self._simple_upsert(
597		table="user_threepid_id_server",
598		keyvalues={
599		"user_id": user_id,
600		"medium": medium,
601		"address": address,
602		"id_server": id_server,
603		},
604		values={},
605		insertion_values={},
606		desc="add_user_bound_threepid",
607		)
608
609		def user_get_bound_threepids(self, user_id):
610		"""Get the threepids that a user has bound to an identity server through the homeserver
611		The homeserver remembers where binds to an identity server occurred. Using this
612		method can retrieve those threepids.
613
614		Args:
615		user_id (str): The ID of the user to retrieve threepids for
616
617		Returns:
618		Deferred[list[dict]]: List of dictionaries containing the following:
619		medium (str): The medium of the threepid (e.g "email")
620		address (str): The address of the threepid (e.g "bob@example.com")
621		"""
622		return self._simple_select_list(
623		table="user_threepid_id_server",
624		keyvalues={"user_id": user_id},
625		retcols=["medium", "address"],
626		desc="user_get_bound_threepids",
627		)
628
629		def remove_user_bound_threepid(self, user_id, medium, address, id_server):
630		"""The server proxied an unbind request to the given identity server on
631		behalf of the given user, so we remove the mapping of threepid to
632		identity server.
633
634		Args:
635		user_id (str)
636		medium (str)
637		address (str)
638		id_server (str)
639
640		Returns:
641		Deferred
642		"""
643		return self._simple_delete(
644		table="user_threepid_id_server",
645		keyvalues={
646		"user_id": user_id,
647		"medium": medium,
648		"address": address,
649		"id_server": id_server,
650		},
651		desc="remove_user_bound_threepid",
652		)
653
654		def get_id_servers_user_bound(self, user_id, medium, address):
655		"""Get the list of identity servers that the server proxied bind
656		requests to for given user and threepid
657
658		Args:
659		user_id (str)
660		medium (str)
661		address (str)
662
663		Returns:
664		Deferred[list[str]]: Resolves to a list of identity servers
665		"""
666		return self._simple_select_onecol(
667		table="user_threepid_id_server",
668		keyvalues={"user_id": user_id, "medium": medium, "address": address},
669		retcol="id_server",
670		desc="get_id_servers_user_bound",
671		)
672
673		@cachedInlineCallbacks()
674		def get_user_deactivated_status(self, user_id):
675		"""Retrieve the value for the `deactivated` property for the provided user.
676
677		Args:
678		user_id (str): The ID of the user to retrieve the status for.
679
680		Returns:
681		defer.Deferred(bool): The requested value.
682		"""
683
684		res = yield self._simple_select_one_onecol(
685		table="users",
686		keyvalues={"name": user_id},
687		retcol="deactivated",
688		desc="get_user_deactivated_status",
689		)
690
691		# Convert the integer into a boolean.
692		return res == 1
693
694		def get_threepid_validation_session(
695		self, medium, client_secret, address=None, sid=None, validated=True
696		):
697		"""Gets a session_id and last_send_attempt (if available) for a
698		combination of validation metadata
699
700		Args:
701		medium (str\|None): The medium of the 3PID
702		address (str\|None): The address of the 3PID
703		sid (str\|None): The ID of the validation session
704		client_secret (str): A unique string provided by the client to help identify this
705		validation attempt
706		validated (bool\|None): Whether sessions should be filtered by
707		whether they have been validated already or not. None to
708		perform no filtering
709
710		Returns:
711		Deferred[dict\|None]: A dict containing the following:
712		* address - address of the 3pid
713		* medium - medium of the 3pid
714		* client_secret - a secret provided by the client for this validation session
715		* session_id - ID of the validation session
716		* send_attempt - a number serving to dedupe send attempts for this session
717		* validated_at - timestamp of when this session was validated if so
718
719		Otherwise None if a validation session is not found
720		"""
721		if not client_secret:
722		raise SynapseError(
723		400, "Missing parameter: client_secret", errcode=Codes.MISSING_PARAM
724		)
725
726		keyvalues = {"client_secret": client_secret}
727		if medium:
728		keyvalues["medium"] = medium
729		if address:
730		keyvalues["address"] = address
731		if sid:
732		keyvalues["session_id"] = sid
733
734		assert address or sid
735
736		def get_threepid_validation_session_txn(txn):
737		sql = """
738		SELECT address, session_id, medium, client_secret,
739		last_send_attempt, validated_at
740		FROM threepid_validation_session WHERE %s
741		""" % (
742		" AND ".join("%s = ?" % k for k in iterkeys(keyvalues)),
743		)
744
745		if validated is not None:
746		sql += " AND validated_at IS " + ("NOT NULL" if validated else "NULL")
747
748		sql += " LIMIT 1"
749
750		txn.execute(sql, list(keyvalues.values()))
751		rows = self.cursor_to_dict(txn)
752		if not rows:
753		return None
754
755		return rows[0]
756
757		return self.runInteraction(
758		"get_threepid_validation_session", get_threepid_validation_session_txn
759		)
760
761		def delete_threepid_session(self, session_id):
762		"""Removes a threepid validation session from the database. This can
763		be done after validation has been performed and whatever action was
764		waiting on it has been carried out
765
766		Args:
767		session_id (str): The ID of the session to delete
768		"""
769
770		def delete_threepid_session_txn(txn):
771		self._simple_delete_txn(
772		txn,
773		table="threepid_validation_token",
774		keyvalues={"session_id": session_id},
775		)
776		self._simple_delete_txn(
777		txn,
778		table="threepid_validation_session",
779		keyvalues={"session_id": session_id},
780		)
781
782		return self.runInteraction(
783		"delete_threepid_session", delete_threepid_session_txn
784		)
785
786
787		class RegistrationStore(
788		RegistrationWorkerStore, background_updates.BackgroundUpdateStore
789		):
790		def __init__(self, db_conn, hs):
791		super(RegistrationStore, self).__init__(db_conn, hs)
792
793		self.clock = hs.get_clock()
794
795		self.register_background_index_update(
796		"access_tokens_device_index",
797		index_name="access_tokens_device_id",
798		table="access_tokens",
799		columns=["user_id", "device_id"],
800		)
801
802		self.register_background_index_update(
803		"users_creation_ts",
804		index_name="users_creation_ts",
805		table="users",
806		columns=["creation_ts"],
807		)
808
809		self._account_validity = hs.config.account_validity
810
811		# we no longer use refresh tokens, but it's possible that some people
812		# might have a background update queued to build this index. Just
813		# clear the background update.
814		self.register_noop_background_update("refresh_tokens_device_index")
815
816		self.register_background_update_handler(
817		"user_threepids_grandfather", self._bg_user_threepids_grandfather
818		)
819
820		self.register_background_update_handler(
821		"users_set_deactivated_flag", self._background_update_set_deactivated_flag
822		)
823
824		# Create a background job for culling expired 3PID validity tokens
825		def start_cull():
826		# run as a background process to make sure that the database transactions
827		# have a logcontext to report to
828		return run_as_background_process(
829		"cull_expired_threepid_validation_tokens",
830		self.cull_expired_threepid_validation_tokens,
831		)
832
833		hs.get_clock().looping_call(start_cull, THIRTY_MINUTES_IN_MS)
834
835		@defer.inlineCallbacks
836		def _background_update_set_deactivated_flag(self, progress, batch_size):
837		"""Retrieves a list of all deactivated users and sets the 'deactivated' flag to 1
838		for each of them.
839		"""
840
841		last_user = progress.get("user_id", "")
842
843		def _background_update_set_deactivated_flag_txn(txn):
844		txn.execute(
845		"""
846		SELECT
847		users.name,
848		COUNT(access_tokens.token) AS count_tokens,
849		COUNT(user_threepids.address) AS count_threepids
850		FROM users
851		LEFT JOIN access_tokens ON (access_tokens.user_id = users.name)
852		LEFT JOIN user_threepids ON (user_threepids.user_id = users.name)
853		WHERE (users.password_hash IS NULL OR users.password_hash = '')
854		AND (users.appservice_id IS NULL OR users.appservice_id = '')
855		AND users.is_guest = 0
856		AND users.name > ?
857		GROUP BY users.name
858		ORDER BY users.name ASC
859		LIMIT ?;
860		""",
861		(last_user, batch_size),
862		)
863
864		rows = self.cursor_to_dict(txn)
865
866		if not rows:
867		return True, 0
868
869		rows_processed_nb = 0
870
871		for user in rows:
872		if not user["count_tokens"] and not user["count_threepids"]:
873		self.set_user_deactivated_status_txn(txn, user["name"], True)
874		rows_processed_nb += 1
875
876		logger.info("Marked %d rows as deactivated", rows_processed_nb)
877
878		self._background_update_progress_txn(
879		txn, "users_set_deactivated_flag", {"user_id": rows[-1]["name"]}
880		)
881
882		if batch_size > len(rows):
883		return True, len(rows)
884		else:
885		return False, len(rows)
886
887		end, nb_processed = yield self.runInteraction(
888		"users_set_deactivated_flag", _background_update_set_deactivated_flag_txn
889		)
890
891		if end:
892		yield self._end_background_update("users_set_deactivated_flag")
893
894		return nb_processed
895
896		@defer.inlineCallbacks
897		def add_access_token_to_user(self, user_id, token, device_id, valid_until_ms):
898		"""Adds an access token for the given user.
899
900		Args:
901		user_id (str): The user ID.
902		token (str): The new access token to add.
903		device_id (str): ID of the device to associate with the access
904		token
905		valid_until_ms (int\|None): when the token is valid until. None for
906		no expiry.
907		Raises:
908		StoreError if there was a problem adding this.
909		"""
910		next_id = self._access_tokens_id_gen.get_next()
911
912		yield self._simple_insert(
913		"access_tokens",
914		{
915		"id": next_id,
916		"user_id": user_id,
917		"token": token,
918		"device_id": device_id,
919		"valid_until_ms": valid_until_ms,
920		},
921		desc="add_access_token_to_user",
922		)
923
924		def register_user(
925		self,
926		user_id,
927		password_hash=None,
928		was_guest=False,
929		make_guest=False,
930		appservice_id=None,
931		create_profile_with_displayname=None,
932		admin=False,
933		user_type=None,
934		):
935		"""Attempts to register an account.
936
937		Args:
938		user_id (str): The desired user ID to register.
939		password_hash (str): Optional. The password hash for this user.
940		was_guest (bool): Optional. Whether this is a guest account being
941		upgraded to a non-guest account.
942		make_guest (boolean): True if the the new user should be guest,
943		false to add a regular user account.
944		appservice_id (str): The ID of the appservice registering the user.
945		create_profile_with_displayname (unicode): Optionally create a profile for
946		the user, setting their displayname to the given value
947		admin (boolean): is an admin user?
948		user_type (str\|None): type of user. One of the values from
949		api.constants.UserTypes, or None for a normal user.
950
951		Raises:
952		StoreError if the user_id could not be registered.
953		"""
954		return self.runInteraction(
955		"register_user",
956		self._register_user,
957		user_id,
958		password_hash,
959		was_guest,
960		make_guest,
961		appservice_id,
962		create_profile_with_displayname,
963		admin,
964		user_type,
965		)
966
967		def _register_user(
968		self,
969		txn,
970		user_id,
971		password_hash,
972		was_guest,
973		make_guest,
974		appservice_id,
975		create_profile_with_displayname,
976		admin,
977		user_type,
978		):
979		user_id_obj = UserID.from_string(user_id)
980
981		now = int(self.clock.time())
982
983		try:
984		if was_guest:
985		# Ensure that the guest user actually exists
986		# ``allow_none=False`` makes this raise an exception
987		# if the row isn't in the database.
988		self._simple_select_one_txn(
989		txn,
990		"users",
991		keyvalues={"name": user_id, "is_guest": 1},
992		retcols=("name",),
993		allow_none=False,
994		)
995
996		self._simple_update_one_txn(
997		txn,
998		"users",
999		keyvalues={"name": user_id, "is_guest": 1},
1000		updatevalues={
1001		"password_hash": password_hash,
1002		"upgrade_ts": now,
1003		"is_guest": 1 if make_guest else 0,
1004		"appservice_id": appservice_id,
1005		"admin": 1 if admin else 0,
1006		"user_type": user_type,
1007		},
1008		)
1009		else:
1010		self._simple_insert_txn(
1011		txn,
1012		"users",
1013		values={
1014		"name": user_id,
1015		"password_hash": password_hash,
1016		"creation_ts": now,
1017		"is_guest": 1 if make_guest else 0,
1018		"appservice_id": appservice_id,
1019		"admin": 1 if admin else 0,
1020		"user_type": user_type,
1021		},
1022		)
1023
1024		except self.database_engine.module.IntegrityError:
1025		raise StoreError(400, "User ID already taken.", errcode=Codes.USER_IN_USE)
1026
1027		if self._account_validity.enabled:
1028		self.set_expiration_date_for_user_txn(txn, user_id)
1029
1030		if create_profile_with_displayname:
1031		# set a default displayname serverside to avoid ugly race
1032		# between auto-joins and clients trying to set displaynames
1033		#
1034		# obviously the 'profiles' table uses localpart for user_id
1035		# while everything else uses the full mxid.
1036		txn.execute(
1037		"INSERT INTO profiles(user_id, displayname) VALUES (?,?)",
1038		(user_id_obj.localpart, create_profile_with_displayname),
1039		)
1040
1041		if self.hs.config.stats_enabled:
1042		# we create a new completed user statistics row
1043
1044		# we don't strictly need current_token since this user really can't
1045		# have any state deltas before now (as it is a new user), but still,
1046		# we include it for completeness.
1047		current_token = self._get_max_stream_id_in_current_state_deltas_txn(txn)
1048		self._update_stats_delta_txn(
1049		txn, now, "user", user_id, {}, complete_with_stream_id=current_token
1050		)
1051
1052		self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,))
1053		txn.call_after(self.is_guest.invalidate, (user_id,))
1054
1055		def record_user_external_id(
1056		self, auth_provider: str, external_id: str, user_id: str
1057		) -> Deferred:
1058		"""Record a mapping from an external user id to a mxid
1059
1060		Args:
1061		auth_provider: identifier for the remote auth provider
1062		external_id: id on that system
1063		user_id: complete mxid that it is mapped to
1064		"""
1065		return self._simple_insert(
1066		table="user_external_ids",
1067		values={
1068		"auth_provider": auth_provider,
1069		"external_id": external_id,
1070		"user_id": user_id,
1071		},
1072		desc="record_user_external_id",
1073		)
1074
1075		def user_set_password_hash(self, user_id, password_hash):
1076		"""
1077		NB. This does not evict any cache because the one use for this
1078		removes most of the entries subsequently anyway so it would be
1079		pointless. Use flush_user separately.
1080		"""
1081
1082		def user_set_password_hash_txn(txn):
1083		self._simple_update_one_txn(
1084		txn, "users", {"name": user_id}, {"password_hash": password_hash}
1085		)
1086		self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,))
1087
1088		return self.runInteraction("user_set_password_hash", user_set_password_hash_txn)
1089
1090		def user_set_consent_version(self, user_id, consent_version):
1091		"""Updates the user table to record privacy policy consent
1092
1093		Args:
1094		user_id (str): full mxid of the user to update
1095		consent_version (str): version of the policy the user has consented
1096		to
1097
1098		Raises:
1099		StoreError(404) if user not found
1100		"""
1101
1102		def f(txn):
1103		self._simple_update_one_txn(
1104		txn,
1105		table="users",
1106		keyvalues={"name": user_id},
1107		updatevalues={"consent_version": consent_version},
1108		)
1109		self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,))
1110
1111		return self.runInteraction("user_set_consent_version", f)
1112
1113		def user_set_consent_server_notice_sent(self, user_id, consent_version):
1114		"""Updates the user table to record that we have sent the user a server
1115		notice about privacy policy consent
1116
1117		Args:
1118		user_id (str): full mxid of the user to update
1119		consent_version (str): version of the policy we have notified the
1120		user about
1121
1122		Raises:
1123		StoreError(404) if user not found
1124		"""
1125
1126		def f(txn):
1127		self._simple_update_one_txn(
1128		txn,
1129		table="users",
1130		keyvalues={"name": user_id},
1131		updatevalues={"consent_server_notice_sent": consent_version},
1132		)
1133		self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,))
1134
1135		return self.runInteraction("user_set_consent_server_notice_sent", f)
1136
1137		def user_delete_access_tokens(self, user_id, except_token_id=None, device_id=None):
1138		"""
1139		Invalidate access tokens belonging to a user
1140
1141		Args:
1142		user_id (str): ID of user the tokens belong to
1143		except_token_id (str): list of access_tokens IDs which should
1144		not be deleted
1145		device_id (str\|None): ID of device the tokens are associated with.
1146		If None, tokens associated with any device (or no device) will
1147		be deleted
1148		Returns:
1149		defer.Deferred[list[str, int, str\|None, int]]: a list of
1150		(token, token id, device id) for each of the deleted tokens
1151		"""
1152
1153		def f(txn):
1154		keyvalues = {"user_id": user_id}
1155		if device_id is not None:
1156		keyvalues["device_id"] = device_id
1157
1158		items = keyvalues.items()
1159		where_clause = " AND ".join(k + " = ?" for k, _ in items)
1160		values = [v for _, v in items]
1161		if except_token_id:
1162		where_clause += " AND id != ?"
1163		values.append(except_token_id)
1164
1165		txn.execute(
1166		"SELECT token, id, device_id FROM access_tokens WHERE %s"
1167		% where_clause,
1168		values,
1169		)
1170		tokens_and_devices = [(r[0], r[1], r[2]) for r in txn]
1171
1172		for token, _, _ in tokens_and_devices:
1173		self._invalidate_cache_and_stream(
1174		txn, self.get_user_by_access_token, (token,)
1175		)
1176
1177		txn.execute("DELETE FROM access_tokens WHERE %s" % where_clause, values)
1178
1179		return tokens_and_devices
1180
1181		return self.runInteraction("user_delete_access_tokens", f)
1182
1183		def delete_access_token(self, access_token):
1184		def f(txn):
1185		self._simple_delete_one_txn(
1186		txn, table="access_tokens", keyvalues={"token": access_token}
1187		)
1188
1189		self._invalidate_cache_and_stream(
1190		txn, self.get_user_by_access_token, (access_token,)
1191		)
1192
1193		return self.runInteraction("delete_access_token", f)
1194
1195		@cachedInlineCallbacks()
1196		def is_guest(self, user_id):
1197		res = yield self._simple_select_one_onecol(
1198		table="users",
1199		keyvalues={"name": user_id},
1200		retcol="is_guest",
1201		allow_none=True,
1202		desc="is_guest",
1203		)
1204
1205		return res if res else False
1206
1207		def add_user_pending_deactivation(self, user_id):
1208		"""
1209		Adds a user to the table of users who need to be parted from all the rooms they're
1210		in
1211		"""
1212		return self._simple_insert(
1213		"users_pending_deactivation",
1214		values={"user_id": user_id},
1215		desc="add_user_pending_deactivation",
1216		)
1217
1218		def del_user_pending_deactivation(self, user_id):
1219		"""
1220		Removes the given user to the table of users who need to be parted from all the
1221		rooms they're in, effectively marking that user as fully deactivated.
1222		"""
1223		# XXX: This should be simple_delete_one but we failed to put a unique index on
1224		# the table, so somehow duplicate entries have ended up in it.
1225		return self._simple_delete(
1226		"users_pending_deactivation",
1227		keyvalues={"user_id": user_id},
1228		desc="del_user_pending_deactivation",
1229		)
1230
1231		def get_user_pending_deactivation(self):
1232		"""
1233		Gets one user from the table of users waiting to be parted from all the rooms
1234		they're in.
1235		"""
1236		return self._simple_select_one_onecol(
1237		"users_pending_deactivation",
1238		keyvalues={},
1239		retcol="user_id",
1240		allow_none=True,
1241		desc="get_users_pending_deactivation",
1242		)
1243
1244		@defer.inlineCallbacks
1245		def _bg_user_threepids_grandfather(self, progress, batch_size):
1246		"""We now track which identity servers a user binds their 3PID to, so
1247		we need to handle the case of existing bindings where we didn't track
1248		this.
1249
1250		We do this by grandfathering in existing user threepids assuming that
1251		they used one of the server configured trusted identity servers.
1252		"""
1253		id_servers = set(self.config.trusted_third_party_id_servers)
1254
1255		def _bg_user_threepids_grandfather_txn(txn):
1256		sql = """
1257		INSERT INTO user_threepid_id_server
1258		(user_id, medium, address, id_server)
1259		SELECT user_id, medium, address, ?
1260		FROM user_threepids
1261		"""
1262
1263		txn.executemany(sql, [(id_server,) for id_server in id_servers])
1264
1265		if id_servers:
1266		yield self.runInteraction(
1267		"_bg_user_threepids_grandfather", _bg_user_threepids_grandfather_txn
1268		)
1269
1270		yield self._end_background_update("user_threepids_grandfather")
1271
1272		return 1
1273
1274		def validate_threepid_session(self, session_id, client_secret, token, current_ts):
1275		"""Attempt to validate a threepid session using a token
1276
1277		Args:
1278		session_id (str): The id of a validation session
1279		client_secret (str): A unique string provided by the client to
1280		help identify this validation attempt
1281		token (str): A validation token
1282		current_ts (int): The current unix time in milliseconds. Used for
1283		checking token expiry status
1284
1285		Raises:
1286		ThreepidValidationError: if a matching validation token was not found or has
1287		expired
1288
1289		Returns:
1290		deferred str\|None: A str representing a link to redirect the user
1291		to if there is one.
1292		"""
1293
1294		# Insert everything into a transaction in order to run atomically
1295		def validate_threepid_session_txn(txn):
1296		row = self._simple_select_one_txn(
1297		txn,
1298		table="threepid_validation_session",
1299		keyvalues={"session_id": session_id},
1300		retcols=["client_secret", "validated_at"],
1301		allow_none=True,
1302		)
1303
1304		if not row:
1305		raise ThreepidValidationError(400, "Unknown session_id")
1306		retrieved_client_secret = row["client_secret"]
1307		validated_at = row["validated_at"]
1308
1309		if retrieved_client_secret != client_secret:
1310		raise ThreepidValidationError(
1311		400, "This client_secret does not match the provided session_id"
1312		)
1313
1314		row = self._simple_select_one_txn(
1315		txn,
1316		table="threepid_validation_token",
1317		keyvalues={"session_id": session_id, "token": token},
1318		retcols=["expires", "next_link"],
1319		allow_none=True,
1320		)
1321
1322		if not row:
1323		raise ThreepidValidationError(
1324		400, "Validation token not found or has expired"
1325		)
1326		expires = row["expires"]
1327		next_link = row["next_link"]
1328
1329		# If the session is already validated, no need to revalidate
1330		if validated_at:
1331		return next_link
1332
1333		if expires <= current_ts:
1334		raise ThreepidValidationError(
1335		400, "This token has expired. Please request a new one"
1336		)
1337
1338		# Looks good. Validate the session
1339		self._simple_update_txn(
1340		txn,
1341		table="threepid_validation_session",
1342		keyvalues={"session_id": session_id},
1343		updatevalues={"validated_at": self.clock.time_msec()},
1344		)
1345
1346		return next_link
1347
1348		# Return next_link if it exists
1349		return self.runInteraction(
1350		"validate_threepid_session_txn", validate_threepid_session_txn
1351		)
1352
1353		def upsert_threepid_validation_session(
1354		self,
1355		medium,
1356		address,
1357		client_secret,
1358		send_attempt,
1359		session_id,
1360		validated_at=None,
1361		):
1362		"""Upsert a threepid validation session
1363		Args:
1364		medium (str): The medium of the 3PID
1365		address (str): The address of the 3PID
1366		client_secret (str): A unique string provided by the client to
1367		help identify this validation attempt
1368		send_attempt (int): The latest send_attempt on this session
1369		session_id (str): The id of this validation session
1370		validated_at (int\|None): The unix timestamp in milliseconds of
1371		when the session was marked as valid
1372		"""
1373		insertion_values = {
1374		"medium": medium,
1375		"address": address,
1376		"client_secret": client_secret,
1377		}
1378
1379		if validated_at:
1380		insertion_values["validated_at"] = validated_at
1381
1382		return self._simple_upsert(
1383		table="threepid_validation_session",
1384		keyvalues={"session_id": session_id},
1385		values={"last_send_attempt": send_attempt},
1386		insertion_values=insertion_values,
1387		desc="upsert_threepid_validation_session",
1388		)
1389
1390		def start_or_continue_validation_session(
1391		self,
1392		medium,
1393		address,
1394		session_id,
1395		client_secret,
1396		send_attempt,
1397		next_link,
1398		token,
1399		token_expires,
1400		):
1401		"""Creates a new threepid validation session if it does not already
1402		exist and associates a new validation token with it
1403
1404		Args:
1405		medium (str): The medium of the 3PID
1406		address (str): The address of the 3PID
1407		session_id (str): The id of this validation session
1408		client_secret (str): A unique string provided by the client to
1409		help identify this validation attempt
1410		send_attempt (int): The latest send_attempt on this session
1411		next_link (str\|None): The link to redirect the user to upon
1412		successful validation
1413		token (str): The validation token
1414		token_expires (int): The timestamp for which after the token
1415		will no longer be valid
1416		"""
1417
1418		def start_or_continue_validation_session_txn(txn):
1419		# Create or update a validation session
1420		self._simple_upsert_txn(
1421		txn,
1422		table="threepid_validation_session",
1423		keyvalues={"session_id": session_id},
1424		values={"last_send_attempt": send_attempt},
1425		insertion_values={
1426		"medium": medium,
1427		"address": address,
1428		"client_secret": client_secret,
1429		},
1430		)
1431
1432		# Create a new validation token with this session ID
1433		self._simple_insert_txn(
1434		txn,
1435		table="threepid_validation_token",
1436		values={
1437		"session_id": session_id,
1438		"token": token,
1439		"next_link": next_link,
1440		"expires": token_expires,
1441		},
1442		)
1443
1444		return self.runInteraction(
1445		"start_or_continue_validation_session",
1446		start_or_continue_validation_session_txn,
1447		)
1448
1449		def cull_expired_threepid_validation_tokens(self):
1450		"""Remove threepid validation tokens with expiry dates that have passed"""
1451
1452		def cull_expired_threepid_validation_tokens_txn(txn, ts):
1453		sql = """
1454		DELETE FROM threepid_validation_token WHERE
1455		expires < ?
1456		"""
1457		return txn.execute(sql, (ts,))
1458
1459		return self.runInteraction(
1460		"cull_expired_threepid_validation_tokens",
1461		cull_expired_threepid_validation_tokens_txn,
1462		self.clock.time_msec(),
1463		)
1464
1465		def set_user_deactivated_status_txn(self, txn, user_id, deactivated):
1466		self._simple_update_one_txn(
1467		txn=txn,
1468		table="users",
1469		keyvalues={"name": user_id},
1470		updatevalues={"deactivated": 1 if deactivated else 0},
1471		)
1472		self._invalidate_cache_and_stream(
1473		txn, self.get_user_deactivated_status, (user_id,)
1474		)
1475
1476		@defer.inlineCallbacks
1477		def set_user_deactivated_status(self, user_id, deactivated):
1478		"""Set the `deactivated` property for the provided user to the provided value.
1479
1480		Args:
1481		user_id (str): The ID of the user to set the status for.
1482		deactivated (bool): The value to set for `deactivated`.
1483		"""
1484
1485		yield self.runInteraction(
1486		"set_user_deactivated_status",
1487		self.set_user_deactivated_status_txn,
1488		user_id,
1489		deactivated,
1490		)

+0

-42

~~synapse/storage/rejections.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2014-2016 OpenMarket Ltd
2		#
3		# Licensed under the Apache License, Version 2.0 (the "License");
4		# you may not use this file except in compliance with the License.
5		# You may obtain a copy of the License at
6		#
7		# http://www.apache.org/licenses/LICENSE-2.0
8		#
9		# Unless required by applicable law or agreed to in writing, software
10		# distributed under the License is distributed on an "AS IS" BASIS,
11		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12		# See the License for the specific language governing permissions and
13		# limitations under the License.
14
15		import logging
16
17		from ._base import SQLBaseStore
18
19		logger = logging.getLogger(__name__)
20
21
22		class RejectionsStore(SQLBaseStore):
23		def _store_rejections_txn(self, txn, event_id, reason):
24		self._simple_insert_txn(
25		txn,
26		table="rejections",
27		values={
28		"event_id": event_id,
29		"reason": reason,
30		"last_check": self._clock.time_msec(),
31		},
32		)
33
34		def get_rejection_reason(self, event_id):
35		return self._simple_select_one_onecol(
36		table="rejections",
37		retcol="reason",
38		keyvalues={"event_id": event_id},
39		allow_none=True,
40		desc="get_rejection_reason",
41		)

+0

-359

synapse/storage/relations.py less more

16	16
17	17	import attr
18	18
19		from synapse.api.constants import RelationTypes
20	19	from synapse.api.errors import SynapseError
21		from synapse.storage._base import SQLBaseStore
22		from synapse.storage.stream import generate_pagination_where_clause
23		from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
24	20
25	21	logger = logging.getLogger(__name__)
26	22

112	108
113	109	def as_tuple(self):
114	110	return attr.astuple(self)
115
116
117		class RelationsWorkerStore(SQLBaseStore):
118		@cached(tree=True)
119		def get_relations_for_event(
120		self,
121		event_id,
122		relation_type=None,
123		event_type=None,
124		aggregation_key=None,
125		limit=5,
126		direction="b",
127		from_token=None,
128		to_token=None,
129		):
130		"""Get a list of relations for an event, ordered by topological ordering.
131
132		Args:
133		event_id (str): Fetch events that relate to this event ID.
134		relation_type (str\|None): Only fetch events with this relation
135		type, if given.
136		event_type (str\|None): Only fetch events with this event type, if
137		given.
138		aggregation_key (str\|None): Only fetch events with this aggregation
139		key, if given.
140		limit (int): Only fetch the most recent `limit` events.
141		direction (str): Whether to fetch the most recent first (`"b"`) or
142		the oldest first (`"f"`).
143		from_token (RelationPaginationToken\|None): Fetch rows from the given
144		token, or from the start if None.
145		to_token (RelationPaginationToken\|None): Fetch rows up to the given
146		token, or up to the end if None.
147
148		Returns:
149		Deferred[PaginationChunk]: List of event IDs that match relations
150		requested. The rows are of the form `{"event_id": "..."}`.
151		"""
152
153		where_clause = ["relates_to_id = ?"]
154		where_args = [event_id]
155
156		if relation_type is not None:
157		where_clause.append("relation_type = ?")
158		where_args.append(relation_type)
159
160		if event_type is not None:
161		where_clause.append("type = ?")
162		where_args.append(event_type)
163
164		if aggregation_key:
165		where_clause.append("aggregation_key = ?")
166		where_args.append(aggregation_key)
167
168		pagination_clause = generate_pagination_where_clause(
169		direction=direction,
170		column_names=("topological_ordering", "stream_ordering"),
171		from_token=attr.astuple(from_token) if from_token else None,
172		to_token=attr.astuple(to_token) if to_token else None,
173		engine=self.database_engine,
174		)
175
176		if pagination_clause:
177		where_clause.append(pagination_clause)
178
179		if direction == "b":
180		order = "DESC"
181		else:
182		order = "ASC"
183
184		sql = """
185		SELECT event_id, topological_ordering, stream_ordering
186		FROM event_relations
187		INNER JOIN events USING (event_id)
188		WHERE %s
189		ORDER BY topological_ordering %s, stream_ordering %s
190		LIMIT ?
191		""" % (
192		" AND ".join(where_clause),
193		order,
194		order,
195		)
196
197		def _get_recent_references_for_event_txn(txn):
198		txn.execute(sql, where_args + [limit + 1])
199
200		last_topo_id = None
201		last_stream_id = None
202		events = []
203		for row in txn:
204		events.append({"event_id": row[0]})
205		last_topo_id = row[1]
206		last_stream_id = row[2]
207
208		next_batch = None
209		if len(events) > limit and last_topo_id and last_stream_id:
210		next_batch = RelationPaginationToken(last_topo_id, last_stream_id)
211
212		return PaginationChunk(
213		chunk=list(events[:limit]), next_batch=next_batch, prev_batch=from_token
214		)
215
216		return self.runInteraction(
217		"get_recent_references_for_event", _get_recent_references_for_event_txn
218		)
219
220		@cached(tree=True)
221		def get_aggregation_groups_for_event(
222		self,
223		event_id,
224		event_type=None,
225		limit=5,
226		direction="b",
227		from_token=None,
228		to_token=None,
229		):
230		"""Get a list of annotations on the event, grouped by event type and
231		aggregation key, sorted by count.
232
233		This is used e.g. to get the what and how many reactions have happend
234		on an event.
235
236		Args:
237		event_id (str): Fetch events that relate to this event ID.
238		event_type (str\|None): Only fetch events with this event type, if
239		given.
240		limit (int): Only fetch the `limit` groups.
241		direction (str): Whether to fetch the highest count first (`"b"`) or
242		the lowest count first (`"f"`).
243		from_token (AggregationPaginationToken\|None): Fetch rows from the
244		given token, or from the start if None.
245		to_token (AggregationPaginationToken\|None): Fetch rows up to the
246		given token, or up to the end if None.
247
248
249		Returns:
250		Deferred[PaginationChunk]: List of groups of annotations that
251		match. Each row is a dict with `type`, `key` and `count` fields.
252		"""
253
254		where_clause = ["relates_to_id = ?", "relation_type = ?"]
255		where_args = [event_id, RelationTypes.ANNOTATION]
256
257		if event_type:
258		where_clause.append("type = ?")
259		where_args.append(event_type)
260
261		having_clause = generate_pagination_where_clause(
262		direction=direction,
263		column_names=("COUNT(*)", "MAX(stream_ordering)"),
264		from_token=attr.astuple(from_token) if from_token else None,
265		to_token=attr.astuple(to_token) if to_token else None,
266		engine=self.database_engine,
267		)
268
269		if direction == "b":
270		order = "DESC"
271		else:
272		order = "ASC"
273
274		if having_clause:
275		having_clause = "HAVING " + having_clause
276		else:
277		having_clause = ""
278
279		sql = """
280		SELECT type, aggregation_key, COUNT(DISTINCT sender), MAX(stream_ordering)
281		FROM event_relations
282		INNER JOIN events USING (event_id)
283		WHERE {where_clause}
284		GROUP BY relation_type, type, aggregation_key
285		{having_clause}
286		ORDER BY COUNT(*) {order}, MAX(stream_ordering) {order}
287		LIMIT ?
288		""".format(
289		where_clause=" AND ".join(where_clause),
290		order=order,
291		having_clause=having_clause,
292		)
293
294		def _get_aggregation_groups_for_event_txn(txn):
295		txn.execute(sql, where_args + [limit + 1])
296
297		next_batch = None
298		events = []
299		for row in txn:
300		events.append({"type": row[0], "key": row[1], "count": row[2]})
301		next_batch = AggregationPaginationToken(row[2], row[3])
302
303		if len(events) <= limit:
304		next_batch = None
305
306		return PaginationChunk(
307		chunk=list(events[:limit]), next_batch=next_batch, prev_batch=from_token
308		)
309
310		return self.runInteraction(
311		"get_aggregation_groups_for_event", _get_aggregation_groups_for_event_txn
312		)
313
314		@cachedInlineCallbacks()
315		def get_applicable_edit(self, event_id):
316		"""Get the most recent edit (if any) that has happened for the given
317		event.
318
319		Correctly handles checking whether edits were allowed to happen.
320
321		Args:
322		event_id (str): The original event ID
323
324		Returns:
325		Deferred[EventBase\|None]: Returns the most recent edit, if any.
326		"""
327
328		# We only allow edits for `m.room.message` events that have the same sender
329		# and event type. We can't assert these things during regular event auth so
330		# we have to do the checks post hoc.
331
332		# Fetches latest edit that has the same type and sender as the
333		# original, and is an `m.room.message`.
334		sql = """
335		SELECT edit.event_id FROM events AS edit
336		INNER JOIN event_relations USING (event_id)
337		INNER JOIN events AS original ON
338		original.event_id = relates_to_id
339		AND edit.type = original.type
340		AND edit.sender = original.sender
341		WHERE
342		relates_to_id = ?
343		AND relation_type = ?
344		AND edit.type = 'm.room.message'
345		ORDER by edit.origin_server_ts DESC, edit.event_id DESC
346		LIMIT 1
347		"""
348
349		def _get_applicable_edit_txn(txn):
350		txn.execute(sql, (event_id, RelationTypes.REPLACE))
351		row = txn.fetchone()
352		if row:
353		return row[0]
354
355		edit_id = yield self.runInteraction(
356		"get_applicable_edit", _get_applicable_edit_txn
357		)
358
359		if not edit_id:
360		return
361
362		edit_event = yield self.get_event(edit_id, allow_none=True)
363		return edit_event
364
365		def has_user_annotated_event(self, parent_id, event_type, aggregation_key, sender):
366		"""Check if a user has already annotated an event with the same key
367		(e.g. already liked an event).
368
369		Args:
370		parent_id (str): The event being annotated
371		event_type (str): The event type of the annotation
372		aggregation_key (str): The aggregation key of the annotation
373		sender (str): The sender of the annotation
374
375		Returns:
376		Deferred[bool]
377		"""
378
379		sql = """
380		SELECT 1 FROM event_relations
381		INNER JOIN events USING (event_id)
382		WHERE
383		relates_to_id = ?
384		AND relation_type = ?
385		AND type = ?
386		AND sender = ?
387		AND aggregation_key = ?
388		LIMIT 1;
389		"""
390
391		def _get_if_user_has_annotated_event(txn):
392		txn.execute(
393		sql,
394		(
395		parent_id,
396		RelationTypes.ANNOTATION,
397		event_type,
398		sender,
399		aggregation_key,
400		),
401		)
402
403		return bool(txn.fetchone())
404
405		return self.runInteraction(
406		"get_if_user_has_annotated_event", _get_if_user_has_annotated_event
407		)
408
409
410		class RelationsStore(RelationsWorkerStore):
411		def _handle_event_relations(self, txn, event):
412		"""Handles inserting relation data during peristence of events
413
414		Args:
415		txn
416		event (EventBase)
417		"""
418		relation = event.content.get("m.relates_to")
419		if not relation:
420		# No relations
421		return
422
423		rel_type = relation.get("rel_type")
424		if rel_type not in (
425		RelationTypes.ANNOTATION,
426		RelationTypes.REFERENCE,
427		RelationTypes.REPLACE,
428		):
429		# Unknown relation type
430		return
431
432		parent_id = relation.get("event_id")
433		if not parent_id:
434		# Invalid relation
435		return
436
437		aggregation_key = relation.get("key")
438
439		self._simple_insert_txn(
440		txn,
441		table="event_relations",
442		values={
443		"event_id": event.event_id,
444		"relates_to_id": parent_id,
445		"relation_type": rel_type,
446		"aggregation_key": aggregation_key,
447		},
448		)
449
450		txn.call_after(self.get_relations_for_event.invalidate_many, (parent_id,))
451		txn.call_after(
452		self.get_aggregation_groups_for_event.invalidate_many, (parent_id,)
453		)
454
455		if rel_type == RelationTypes.REPLACE:
456		txn.call_after(self.get_applicable_edit.invalidate, (parent_id,))
457
458		def _handle_redaction(self, txn, redacted_event_id):
459		"""Handles receiving a redaction and checking whether we need to remove
460		any redacted relations from the database.
461
462		Args:
463		txn
464		redacted_event_id (str): The event that was redacted.
465		"""
466
467		self._simple_delete_txn(
468		txn, table="event_relations", keyvalues={"event_id": redacted_event_id}
469		)

+0

-585

~~synapse/storage/room.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2014-2016 OpenMarket Ltd
2		#
3		# Licensed under the Apache License, Version 2.0 (the "License");
4		# you may not use this file except in compliance with the License.
5		# You may obtain a copy of the License at
6		#
7		# http://www.apache.org/licenses/LICENSE-2.0
8		#
9		# Unless required by applicable law or agreed to in writing, software
10		# distributed under the License is distributed on an "AS IS" BASIS,
11		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12		# See the License for the specific language governing permissions and
13		# limitations under the License.
14
15		import collections
16		import logging
17		import re
18
19		from canonicaljson import json
20
21		from twisted.internet import defer
22
23		from synapse.api.errors import StoreError
24		from synapse.storage._base import SQLBaseStore
25		from synapse.storage.search import SearchStore
26		from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
27
28		logger = logging.getLogger(__name__)
29
30
31		OpsLevel = collections.namedtuple(
32		"OpsLevel", ("ban_level", "kick_level", "redact_level")
33		)
34
35		RatelimitOverride = collections.namedtuple(
36		"RatelimitOverride", ("messages_per_second", "burst_count")
37		)
38
39
40		class RoomWorkerStore(SQLBaseStore):
41		def get_room(self, room_id):
42		"""Retrieve a room.
43
44		Args:
45		room_id (str): The ID of the room to retrieve.
46		Returns:
47		A dict containing the room information, or None if the room is unknown.
48		"""
49		return self._simple_select_one(
50		table="rooms",
51		keyvalues={"room_id": room_id},
52		retcols=("room_id", "is_public", "creator"),
53		desc="get_room",
54		allow_none=True,
55		)
56
57		def get_public_room_ids(self):
58		return self._simple_select_onecol(
59		table="rooms",
60		keyvalues={"is_public": True},
61		retcol="room_id",
62		desc="get_public_room_ids",
63		)
64
65		@cached(num_args=2, max_entries=100)
66		def get_public_room_ids_at_stream_id(self, stream_id, network_tuple):
67		"""Get pulbic rooms for a particular list, or across all lists.
68
69		Args:
70		stream_id (int)
71		network_tuple (ThirdPartyInstanceID): The list to use (None, None)
72		means the main list, None means all lsits.
73		"""
74		return self.runInteraction(
75		"get_public_room_ids_at_stream_id",
76		self.get_public_room_ids_at_stream_id_txn,
77		stream_id,
78		network_tuple=network_tuple,
79		)
80
81		def get_public_room_ids_at_stream_id_txn(self, txn, stream_id, network_tuple):
82		return {
83		rm
84		for rm, vis in self.get_published_at_stream_id_txn(
85		txn, stream_id, network_tuple=network_tuple
86		).items()
87		if vis
88		}
89
90		def get_published_at_stream_id_txn(self, txn, stream_id, network_tuple):
91		if network_tuple:
92		# We want to get from a particular list. No aggregation required.
93
94		sql = """
95		SELECT room_id, visibility FROM public_room_list_stream
96		INNER JOIN (
97		SELECT room_id, max(stream_id) AS stream_id
98		FROM public_room_list_stream
99		WHERE stream_id <= ? %s
100		GROUP BY room_id
101		) grouped USING (room_id, stream_id)
102		"""
103
104		if network_tuple.appservice_id is not None:
105		txn.execute(
106		sql % ("AND appservice_id = ? AND network_id = ?",),
107		(stream_id, network_tuple.appservice_id, network_tuple.network_id),
108		)
109		else:
110		txn.execute(sql % ("AND appservice_id IS NULL",), (stream_id,))
111		return dict(txn)
112		else:
113		# We want to get from all lists, so we need to aggregate the results
114
115		logger.info("Executing full list")
116
117		sql = """
118		SELECT room_id, visibility
119		FROM public_room_list_stream
120		INNER JOIN (
121		SELECT
122		room_id, max(stream_id) AS stream_id, appservice_id,
123		network_id
124		FROM public_room_list_stream
125		WHERE stream_id <= ?
126		GROUP BY room_id, appservice_id, network_id
127		) grouped USING (room_id, stream_id)
128		"""
129
130		txn.execute(sql, (stream_id,))
131
132		results = {}
133		# A room is visible if its visible on any list.
134		for room_id, visibility in txn:
135		results[room_id] = bool(visibility) or results.get(room_id, False)
136
137		return results
138
139		def get_public_room_changes(self, prev_stream_id, new_stream_id, network_tuple):
140		def get_public_room_changes_txn(txn):
141		then_rooms = self.get_public_room_ids_at_stream_id_txn(
142		txn, prev_stream_id, network_tuple
143		)
144
145		now_rooms_dict = self.get_published_at_stream_id_txn(
146		txn, new_stream_id, network_tuple
147		)
148
149		now_rooms_visible = set(rm for rm, vis in now_rooms_dict.items() if vis)
150		now_rooms_not_visible = set(
151		rm for rm, vis in now_rooms_dict.items() if not vis
152		)
153
154		newly_visible = now_rooms_visible - then_rooms
155		newly_unpublished = now_rooms_not_visible & then_rooms
156
157		return newly_visible, newly_unpublished
158
159		return self.runInteraction(
160		"get_public_room_changes", get_public_room_changes_txn
161		)
162
163		@cached(max_entries=10000)
164		def is_room_blocked(self, room_id):
165		return self._simple_select_one_onecol(
166		table="blocked_rooms",
167		keyvalues={"room_id": room_id},
168		retcol="1",
169		allow_none=True,
170		desc="is_room_blocked",
171		)
172
173		@cachedInlineCallbacks(max_entries=10000)
174		def get_ratelimit_for_user(self, user_id):
175		"""Check if there are any overrides for ratelimiting for the given
176		user
177
178		Args:
179		user_id (str)
180
181		Returns:
182		RatelimitOverride if there is an override, else None. If the contents
183		of RatelimitOverride are None or 0 then ratelimitng has been
184		disabled for that user entirely.
185		"""
186		row = yield self._simple_select_one(
187		table="ratelimit_override",
188		keyvalues={"user_id": user_id},
189		retcols=("messages_per_second", "burst_count"),
190		allow_none=True,
191		desc="get_ratelimit_for_user",
192		)
193
194		if row:
195		return RatelimitOverride(
196		messages_per_second=row["messages_per_second"],
197		burst_count=row["burst_count"],
198		)
199		else:
200		return None
201
202
203		class RoomStore(RoomWorkerStore, SearchStore):
204		@defer.inlineCallbacks
205		def store_room(self, room_id, room_creator_user_id, is_public):
206		"""Stores a room.
207
208		Args:
209		room_id (str): The desired room ID, can be None.
210		room_creator_user_id (str): The user ID of the room creator.
211		is_public (bool): True to indicate that this room should appear in
212		public room lists.
213		Raises:
214		StoreError if the room could not be stored.
215		"""
216		try:
217
218		def store_room_txn(txn, next_id):
219		self._simple_insert_txn(
220		txn,
221		"rooms",
222		{
223		"room_id": room_id,
224		"creator": room_creator_user_id,
225		"is_public": is_public,
226		},
227		)
228		if is_public:
229		self._simple_insert_txn(
230		txn,
231		table="public_room_list_stream",
232		values={
233		"stream_id": next_id,
234		"room_id": room_id,
235		"visibility": is_public,
236		},
237		)
238
239		with self._public_room_id_gen.get_next() as next_id:
240		yield self.runInteraction("store_room_txn", store_room_txn, next_id)
241		except Exception as e:
242		logger.error("store_room with room_id=%s failed: %s", room_id, e)
243		raise StoreError(500, "Problem creating room.")
244
245		@defer.inlineCallbacks
246		def set_room_is_public(self, room_id, is_public):
247		def set_room_is_public_txn(txn, next_id):
248		self._simple_update_one_txn(
249		txn,
250		table="rooms",
251		keyvalues={"room_id": room_id},
252		updatevalues={"is_public": is_public},
253		)
254
255		entries = self._simple_select_list_txn(
256		txn,
257		table="public_room_list_stream",
258		keyvalues={
259		"room_id": room_id,
260		"appservice_id": None,
261		"network_id": None,
262		},
263		retcols=("stream_id", "visibility"),
264		)
265
266		entries.sort(key=lambda r: r["stream_id"])
267
268		add_to_stream = True
269		if entries:
270		add_to_stream = bool(entries[-1]["visibility"]) != is_public
271
272		if add_to_stream:
273		self._simple_insert_txn(
274		txn,
275		table="public_room_list_stream",
276		values={
277		"stream_id": next_id,
278		"room_id": room_id,
279		"visibility": is_public,
280		"appservice_id": None,
281		"network_id": None,
282		},
283		)
284
285		with self._public_room_id_gen.get_next() as next_id:
286		yield self.runInteraction(
287		"set_room_is_public", set_room_is_public_txn, next_id
288		)
289		self.hs.get_notifier().on_new_replication_data()
290
291		@defer.inlineCallbacks
292		def set_room_is_public_appservice(
293		self, room_id, appservice_id, network_id, is_public
294		):
295		"""Edit the appservice/network specific public room list.
296
297		Each appservice can have a number of published room lists associated
298		with them, keyed off of an appservice defined `network_id`, which
299		basically represents a single instance of a bridge to a third party
300		network.
301
302		Args:
303		room_id (str)
304		appservice_id (str)
305		network_id (str)
306		is_public (bool): Whether to publish or unpublish the room from the
307		list.
308		"""
309
310		def set_room_is_public_appservice_txn(txn, next_id):
311		if is_public:
312		try:
313		self._simple_insert_txn(
314		txn,
315		table="appservice_room_list",
316		values={
317		"appservice_id": appservice_id,
318		"network_id": network_id,
319		"room_id": room_id,
320		},
321		)
322		except self.database_engine.module.IntegrityError:
323		# We've already inserted, nothing to do.
324		return
325		else:
326		self._simple_delete_txn(
327		txn,
328		table="appservice_room_list",
329		keyvalues={
330		"appservice_id": appservice_id,
331		"network_id": network_id,
332		"room_id": room_id,
333		},
334		)
335
336		entries = self._simple_select_list_txn(
337		txn,
338		table="public_room_list_stream",
339		keyvalues={
340		"room_id": room_id,
341		"appservice_id": appservice_id,
342		"network_id": network_id,
343		},
344		retcols=("stream_id", "visibility"),
345		)
346
347		entries.sort(key=lambda r: r["stream_id"])
348
349		add_to_stream = True
350		if entries:
351		add_to_stream = bool(entries[-1]["visibility"]) != is_public
352
353		if add_to_stream:
354		self._simple_insert_txn(
355		txn,
356		table="public_room_list_stream",
357		values={
358		"stream_id": next_id,
359		"room_id": room_id,
360		"visibility": is_public,
361		"appservice_id": appservice_id,
362		"network_id": network_id,
363		},
364		)
365
366		with self._public_room_id_gen.get_next() as next_id:
367		yield self.runInteraction(
368		"set_room_is_public_appservice",
369		set_room_is_public_appservice_txn,
370		next_id,
371		)
372		self.hs.get_notifier().on_new_replication_data()
373
374		def get_room_count(self):
375		"""Retrieve a list of all rooms
376		"""
377
378		def f(txn):
379		sql = "SELECT count(*) FROM rooms"
380		txn.execute(sql)
381		row = txn.fetchone()
382		return row[0] or 0
383
384		return self.runInteraction("get_rooms", f)
385
386		def _store_room_topic_txn(self, txn, event):
387		if hasattr(event, "content") and "topic" in event.content:
388		self.store_event_search_txn(
389		txn, event, "content.topic", event.content["topic"]
390		)
391
392		def _store_room_name_txn(self, txn, event):
393		if hasattr(event, "content") and "name" in event.content:
394		self.store_event_search_txn(
395		txn, event, "content.name", event.content["name"]
396		)
397
398		def _store_room_message_txn(self, txn, event):
399		if hasattr(event, "content") and "body" in event.content:
400		self.store_event_search_txn(
401		txn, event, "content.body", event.content["body"]
402		)
403
404		def add_event_report(
405		self, room_id, event_id, user_id, reason, content, received_ts
406		):
407		next_id = self._event_reports_id_gen.get_next()
408		return self._simple_insert(
409		table="event_reports",
410		values={
411		"id": next_id,
412		"received_ts": received_ts,
413		"room_id": room_id,
414		"event_id": event_id,
415		"user_id": user_id,
416		"reason": reason,
417		"content": json.dumps(content),
418		},
419		desc="add_event_report",
420		)
421
422		def get_current_public_room_stream_id(self):
423		return self._public_room_id_gen.get_current_token()
424
425		def get_all_new_public_rooms(self, prev_id, current_id, limit):
426		def get_all_new_public_rooms(txn):
427		sql = """
428		SELECT stream_id, room_id, visibility, appservice_id, network_id
429		FROM public_room_list_stream
430		WHERE stream_id > ? AND stream_id <= ?
431		ORDER BY stream_id ASC
432		LIMIT ?
433		"""
434
435		txn.execute(sql, (prev_id, current_id, limit))
436		return txn.fetchall()
437
438		if prev_id == current_id:
439		return defer.succeed([])
440
441		return self.runInteraction("get_all_new_public_rooms", get_all_new_public_rooms)
442
443		@defer.inlineCallbacks
444		def block_room(self, room_id, user_id):
445		"""Marks the room as blocked. Can be called multiple times.
446
447		Args:
448		room_id (str): Room to block
449		user_id (str): Who blocked it
450
451		Returns:
452		Deferred
453		"""
454		yield self._simple_upsert(
455		table="blocked_rooms",
456		keyvalues={"room_id": room_id},
457		values={},
458		insertion_values={"user_id": user_id},
459		desc="block_room",
460		)
461		yield self.runInteraction(
462		"block_room_invalidation",
463		self._invalidate_cache_and_stream,
464		self.is_room_blocked,
465		(room_id,),
466		)
467
468		def get_media_mxcs_in_room(self, room_id):
469		"""Retrieves all the local and remote media MXC URIs in a given room
470
471		Args:
472		room_id (str)
473
474		Returns:
475		The local and remote media as a lists of tuples where the key is
476		the hostname and the value is the media ID.
477		"""
478
479		def _get_media_mxcs_in_room_txn(txn):
480		local_mxcs, remote_mxcs = self._get_media_mxcs_in_room_txn(txn, room_id)
481		local_media_mxcs = []
482		remote_media_mxcs = []
483
484		# Convert the IDs to MXC URIs
485		for media_id in local_mxcs:
486		local_media_mxcs.append("mxc://%s/%s" % (self.hs.hostname, media_id))
487		for hostname, media_id in remote_mxcs:
488		remote_media_mxcs.append("mxc://%s/%s" % (hostname, media_id))
489
490		return local_media_mxcs, remote_media_mxcs
491
492		return self.runInteraction("get_media_ids_in_room", _get_media_mxcs_in_room_txn)
493
494		def quarantine_media_ids_in_room(self, room_id, quarantined_by):
495		"""For a room loops through all events with media and quarantines
496		the associated media
497		"""
498
499		def _quarantine_media_in_room_txn(txn):
500		local_mxcs, remote_mxcs = self._get_media_mxcs_in_room_txn(txn, room_id)
501		total_media_quarantined = 0
502
503		# Now update all the tables to set the quarantined_by flag
504
505		txn.executemany(
506		"""
507		UPDATE local_media_repository
508		SET quarantined_by = ?
509		WHERE media_id = ?
510		""",
511		((quarantined_by, media_id) for media_id in local_mxcs),
512		)
513
514		txn.executemany(
515		"""
516		UPDATE remote_media_cache
517		SET quarantined_by = ?
518		WHERE media_origin = ? AND media_id = ?
519		""",
520		(
521		(quarantined_by, origin, media_id)
522		for origin, media_id in remote_mxcs
523		),
524		)
525
526		total_media_quarantined += len(local_mxcs)
527		total_media_quarantined += len(remote_mxcs)
528
529		return total_media_quarantined
530
531		return self.runInteraction(
532		"quarantine_media_in_room", _quarantine_media_in_room_txn
533		)
534
535		def _get_media_mxcs_in_room_txn(self, txn, room_id):
536		"""Retrieves all the local and remote media MXC URIs in a given room
537
538		Args:
539		txn (cursor)
540		room_id (str)
541
542		Returns:
543		The local and remote media as a lists of tuples where the key is
544		the hostname and the value is the media ID.
545		"""
546		mxc_re = re.compile("^mxc://([^/]+)/([^/#?]+)")
547
548		next_token = self.get_current_events_token() + 1
549		local_media_mxcs = []
550		remote_media_mxcs = []
551
552		while next_token:
553		sql = """
554		SELECT stream_ordering, json FROM events
555		JOIN event_json USING (room_id, event_id)
556		WHERE room_id = ?
557		AND stream_ordering < ?
558		AND contains_url = ? AND outlier = ?
559		ORDER BY stream_ordering DESC
560		LIMIT ?
561		"""
562		txn.execute(sql, (room_id, next_token, True, False, 100))
563
564		next_token = None
565		for stream_ordering, content_json in txn:
566		next_token = stream_ordering
567		event_json = json.loads(content_json)
568		content = event_json["content"]
569		content_url = content.get("url")
570		thumbnail_url = content.get("info", {}).get("thumbnail_url")
571
572		for url in (content_url, thumbnail_url):
573		if not url:
574		continue
575		matches = mxc_re.match(url)
576		if matches:
577		hostname = matches.group(1)
578		media_id = matches.group(2)
579		if hostname == self.hs.hostname:
580		local_media_mxcs.append(media_id)
581		else:
582		remote_media_mxcs.append((hostname, media_id))
583
584		return local_media_mxcs, remote_media_mxcs

+0

-1072

synapse/storage/roommember.py less more

15	15
16	16	import logging
17	17	from collections import namedtuple
18
19		from six import iteritems, itervalues
20
21		from canonicaljson import json
22
23		from twisted.internet import defer
24
25		from synapse.api.constants import EventTypes, Membership
26		from synapse.metrics import LaterGauge
27		from synapse.metrics.background_process_metrics import run_as_background_process
28		from synapse.storage._base import LoggingTransaction
29		from synapse.storage.engines import Sqlite3Engine
30		from synapse.storage.events_worker import EventsWorkerStore
31		from synapse.types import get_domain_from_id
32		from synapse.util.async_helpers import Linearizer
33		from synapse.util.caches import intern_string
34		from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
35		from synapse.util.stringutils import to_ascii
36	18
37	19	logger = logging.getLogger(__name__)
38	20

54	36	# a given membership type, suitable for use in calculating heroes for a room.
55	37	# "count" points to the total numberr of users of a given membership type.
56	38	MemberSummary = namedtuple("MemberSummary", ("members", "count"))
57
58		_MEMBERSHIP_PROFILE_UPDATE_NAME = "room_membership_profile_update"
59		_CURRENT_STATE_MEMBERSHIP_UPDATE_NAME = "current_state_events_membership"
60
61
62		class RoomMemberWorkerStore(EventsWorkerStore):
63		def __init__(self, db_conn, hs):
64		super(RoomMemberWorkerStore, self).__init__(db_conn, hs)
65
66		# Is the current_state_events.membership up to date? Or is the
67		# background update still running?
68		self._current_state_events_membership_up_to_date = False
69
70		txn = LoggingTransaction(
71		db_conn.cursor(),
72		name="_check_safe_current_state_events_membership_updated",
73		database_engine=self.database_engine,
74		)
75		self._check_safe_current_state_events_membership_updated_txn(txn)
76		txn.close()
77
78		if self.hs.config.metrics_flags.known_servers:
79		self._known_servers_count = 1
80		self.hs.get_clock().looping_call(
81		run_as_background_process,
82		60 * 1000,
83		"_count_known_servers",
84		self._count_known_servers,
85		)
86		self.hs.get_clock().call_later(
87		1000,
88		run_as_background_process,
89		"_count_known_servers",
90		self._count_known_servers,
91		)
92		LaterGauge(
93		"synapse_federation_known_servers",
94		"",
95		[],
96		lambda: self._known_servers_count,
97		)
98
99		@defer.inlineCallbacks
100		def _count_known_servers(self):
101		"""
102		Count the servers that this server knows about.
103
104		The statistic is stored on the class for the
105		`synapse_federation_known_servers` LaterGauge to collect.
106		"""
107
108		def _transact(txn):
109		if isinstance(self.database_engine, Sqlite3Engine):
110		query = """
111		SELECT COUNT(DISTINCT substr(out.user_id, pos+1))
112		FROM (
113		SELECT rm.user_id as user_id, instr(rm.user_id, ':')
114		AS pos FROM room_memberships as rm
115		INNER JOIN current_state_events as c ON rm.event_id = c.event_id
116		WHERE c.type = 'm.room.member'
117		) as out
118		"""
119		else:
120		query = """
121		SELECT COUNT(DISTINCT split_part(state_key, ':', 2))
122		FROM current_state_events
123		WHERE type = 'm.room.member' AND membership = 'join';
124		"""
125		txn.execute(query)
126		return list(txn)[0][0]
127
128		count = yield self.runInteraction("get_known_servers", _transact)
129
130		# We always know about ourselves, even if we have nothing in
131		# room_memberships (for example, the server is new).
132		self._known_servers_count = max([count, 1])
133		return self._known_servers_count
134
135		def _check_safe_current_state_events_membership_updated_txn(self, txn):
136		"""Checks if it is safe to assume the new current_state_events
137		membership column is up to date
138		"""
139
140		pending_update = self._simple_select_one_txn(
141		txn,
142		table="background_updates",
143		keyvalues={"update_name": _CURRENT_STATE_MEMBERSHIP_UPDATE_NAME},
144		retcols=["update_name"],
145		allow_none=True,
146		)
147
148		self._current_state_events_membership_up_to_date = not pending_update
149
150		# If the update is still running, reschedule to run.
151		if pending_update:
152		self._clock.call_later(
153		15.0,
154		run_as_background_process,
155		"_check_safe_current_state_events_membership_updated",
156		self.runInteraction,
157		"_check_safe_current_state_events_membership_updated",
158		self._check_safe_current_state_events_membership_updated_txn,
159		)
160
161		@cachedInlineCallbacks(max_entries=100000, iterable=True, cache_context=True)
162		def get_hosts_in_room(self, room_id, cache_context):
163		"""Returns the set of all hosts currently in the room
164		"""
165		user_ids = yield self.get_users_in_room(
166		room_id, on_invalidate=cache_context.invalidate
167		)
168		hosts = frozenset(get_domain_from_id(user_id) for user_id in user_ids)
169		return hosts
170
171		@cached(max_entries=100000, iterable=True)
172		def get_users_in_room(self, room_id):
173		return self.runInteraction(
174		"get_users_in_room", self.get_users_in_room_txn, room_id
175		)
176
177		def get_users_in_room_txn(self, txn, room_id):
178		# If we can assume current_state_events.membership is up to date
179		# then we can avoid a join, which is a Very Good Thing given how
180		# frequently this function gets called.
181		if self._current_state_events_membership_up_to_date:
182		sql = """
183		SELECT state_key FROM current_state_events
184		WHERE type = 'm.room.member' AND room_id = ? AND membership = ?
185		"""
186		else:
187		sql = """
188		SELECT state_key FROM room_memberships as m
189		INNER JOIN current_state_events as c
190		ON m.event_id = c.event_id
191		AND m.room_id = c.room_id
192		AND m.user_id = c.state_key
193		WHERE c.type = 'm.room.member' AND c.room_id = ? AND m.membership = ?
194		"""
195
196		txn.execute(sql, (room_id, Membership.JOIN))
197		return [to_ascii(r[0]) for r in txn]
198
199		@cached(max_entries=100000)
200		def get_room_summary(self, room_id):
201		""" Get the details of a room roughly suitable for use by the room
202		summary extension to /sync. Useful when lazy loading room members.
203		Args:
204		room_id (str): The room ID to query
205		Returns:
206		Deferred[dict[str, MemberSummary]:
207		dict of membership states, pointing to a MemberSummary named tuple.
208		"""
209
210		def _get_room_summary_txn(txn):
211		# first get counts.
212		# We do this all in one transaction to keep the cache small.
213		# FIXME: get rid of this when we have room_stats
214
215		# If we can assume current_state_events.membership is up to date
216		# then we can avoid a join, which is a Very Good Thing given how
217		# frequently this function gets called.
218		if self._current_state_events_membership_up_to_date:
219		# Note, rejected events will have a null membership field, so
220		# we we manually filter them out.
221		sql = """
222		SELECT count(*), membership FROM current_state_events
223		WHERE type = 'm.room.member' AND room_id = ?
224		AND membership IS NOT NULL
225		GROUP BY membership
226		"""
227		else:
228		sql = """
229		SELECT count(*), m.membership FROM room_memberships as m
230		INNER JOIN current_state_events as c
231		ON m.event_id = c.event_id
232		AND m.room_id = c.room_id
233		AND m.user_id = c.state_key
234		WHERE c.type = 'm.room.member' AND c.room_id = ?
235		GROUP BY m.membership
236		"""
237
238		txn.execute(sql, (room_id,))
239		res = {}
240		for count, membership in txn:
241		summary = res.setdefault(to_ascii(membership), MemberSummary([], count))
242
243		# we order by membership and then fairly arbitrarily by event_id so
244		# heroes are consistent
245		if self._current_state_events_membership_up_to_date:
246		# Note, rejected events will have a null membership field, so
247		# we we manually filter them out.
248		sql = """
249		SELECT state_key, membership, event_id
250		FROM current_state_events
251		WHERE type = 'm.room.member' AND room_id = ?
252		AND membership IS NOT NULL
253		ORDER BY
254		CASE membership WHEN ? THEN 1 WHEN ? THEN 2 ELSE 3 END ASC,
255		event_id ASC
256		LIMIT ?
257		"""
258		else:
259		sql = """
260		SELECT c.state_key, m.membership, c.event_id
261		FROM room_memberships as m
262		INNER JOIN current_state_events as c USING (room_id, event_id)
263		WHERE c.type = 'm.room.member' AND c.room_id = ?
264		ORDER BY
265		CASE m.membership WHEN ? THEN 1 WHEN ? THEN 2 ELSE 3 END ASC,
266		c.event_id ASC
267		LIMIT ?
268		"""
269
270		# 6 is 5 (number of heroes) plus 1, in case one of them is the calling user.
271		txn.execute(sql, (room_id, Membership.JOIN, Membership.INVITE, 6))
272		for user_id, membership, event_id in txn:
273		summary = res[to_ascii(membership)]
274		# we will always have a summary for this membership type at this
275		# point given the summary currently contains the counts.
276		members = summary.members
277		members.append((to_ascii(user_id), to_ascii(event_id)))
278
279		return res
280
281		return self.runInteraction("get_room_summary", _get_room_summary_txn)
282
283		def _get_user_counts_in_room_txn(self, txn, room_id):
284		"""
285		Get the user count in a room by membership.
286
287		Args:
288		room_id (str)
289		membership (Membership)
290
291		Returns:
292		Deferred[int]
293		"""
294		sql = """
295		SELECT m.membership, count(*) FROM room_memberships as m
296		INNER JOIN current_state_events as c USING(event_id)
297		WHERE c.type = 'm.room.member' AND c.room_id = ?
298		GROUP BY m.membership
299		"""
300
301		txn.execute(sql, (room_id,))
302		return {row[0]: row[1] for row in txn}
303
304		@cached()
305		def get_invited_rooms_for_user(self, user_id):
306		""" Get all the rooms the user is invited to
307		Args:
308		user_id (str): The user ID.
309		Returns:
310		A deferred list of RoomsForUser.
311		"""
312
313		return self.get_rooms_for_user_where_membership_is(user_id, [Membership.INVITE])
314
315		@defer.inlineCallbacks
316		def get_invite_for_user_in_room(self, user_id, room_id):
317		"""Gets the invite for the given user and room
318
319		Args:
320		user_id (str)
321		room_id (str)
322
323		Returns:
324		Deferred: Resolves to either a RoomsForUser or None if no invite was
325		found.
326		"""
327		invites = yield self.get_invited_rooms_for_user(user_id)
328		for invite in invites:
329		if invite.room_id == room_id:
330		return invite
331		return None
332
333		@defer.inlineCallbacks
334		def get_rooms_for_user_where_membership_is(self, user_id, membership_list):
335		""" Get all the rooms for this user where the membership for this user
336		matches one in the membership list.
337
338		Filters out forgotten rooms.
339
340		Args:
341		user_id (str): The user ID.
342		membership_list (list): A list of synapse.api.constants.Membership
343		values which the user must be in.
344
345		Returns:
346		Deferred[list[RoomsForUser]]
347		"""
348		if not membership_list:
349		return defer.succeed(None)
350
351		rooms = yield self.runInteraction(
352		"get_rooms_for_user_where_membership_is",
353		self._get_rooms_for_user_where_membership_is_txn,
354		user_id,
355		membership_list,
356		)
357
358		# Now we filter out forgotten rooms
359		forgotten_rooms = yield self.get_forgotten_rooms_for_user(user_id)
360		return [room for room in rooms if room.room_id not in forgotten_rooms]
361
362		def _get_rooms_for_user_where_membership_is_txn(
363		self, txn, user_id, membership_list
364		):
365
366		do_invite = Membership.INVITE in membership_list
367		membership_list = [m for m in membership_list if m != Membership.INVITE]
368
369		results = []
370		if membership_list:
371		if self._current_state_events_membership_up_to_date:
372		sql = """
373		SELECT room_id, e.sender, c.membership, event_id, e.stream_ordering
374		FROM current_state_events AS c
375		INNER JOIN events AS e USING (room_id, event_id)
376		WHERE
377		c.type = 'm.room.member'
378		AND state_key = ?
379		AND c.membership IN (%s)
380		""" % (
381		",".join("?" * len(membership_list))
382		)
383		else:
384		sql = """
385		SELECT room_id, e.sender, m.membership, event_id, e.stream_ordering
386		FROM current_state_events AS c
387		INNER JOIN room_memberships AS m USING (room_id, event_id)
388		INNER JOIN events AS e USING (room_id, event_id)
389		WHERE
390		c.type = 'm.room.member'
391		AND state_key = ?
392		AND m.membership IN (%s)
393		""" % (
394		",".join("?" * len(membership_list))
395		)
396
397		txn.execute(sql, (user_id, *membership_list))
398		results = [RoomsForUser(**r) for r in self.cursor_to_dict(txn)]
399
400		if do_invite:
401		sql = (
402		"SELECT i.room_id, inviter, i.event_id, e.stream_ordering"
403		" FROM local_invites as i"
404		" INNER JOIN events as e USING (event_id)"
405		" WHERE invitee = ? AND locally_rejected is NULL"
406		" AND replaced_by is NULL"
407		)
408
409		txn.execute(sql, (user_id,))
410		results.extend(
411		RoomsForUser(
412		room_id=r["room_id"],
413		sender=r["inviter"],
414		event_id=r["event_id"],
415		stream_ordering=r["stream_ordering"],
416		membership=Membership.INVITE,
417		)
418		for r in self.cursor_to_dict(txn)
419		)
420
421		return results
422
423		@cachedInlineCallbacks(max_entries=500000, iterable=True)
424		def get_rooms_for_user_with_stream_ordering(self, user_id):
425		"""Returns a set of room_ids the user is currently joined to
426
427		Args:
428		user_id (str)
429
430		Returns:
431		Deferred[frozenset[GetRoomsForUserWithStreamOrdering]]: Returns
432		the rooms the user is in currently, along with the stream ordering
433		of the most recent join for that user and room.
434		"""
435		rooms = yield self.get_rooms_for_user_where_membership_is(
436		user_id, membership_list=[Membership.JOIN]
437		)
438		return frozenset(
439		GetRoomsForUserWithStreamOrdering(r.room_id, r.stream_ordering)
440		for r in rooms
441		)
442
443		@defer.inlineCallbacks
444		def get_rooms_for_user(self, user_id, on_invalidate=None):
445		"""Returns a set of room_ids the user is currently joined to
446		"""
447		rooms = yield self.get_rooms_for_user_with_stream_ordering(
448		user_id, on_invalidate=on_invalidate
449		)
450		return frozenset(r.room_id for r in rooms)
451
452		@cachedInlineCallbacks(max_entries=500000, cache_context=True, iterable=True)
453		def get_users_who_share_room_with_user(self, user_id, cache_context):
454		"""Returns the set of users who share a room with `user_id`
455		"""
456		room_ids = yield self.get_rooms_for_user(
457		user_id, on_invalidate=cache_context.invalidate
458		)
459
460		user_who_share_room = set()
461		for room_id in room_ids:
462		user_ids = yield self.get_users_in_room(
463		room_id, on_invalidate=cache_context.invalidate
464		)
465		user_who_share_room.update(user_ids)
466
467		return user_who_share_room
468
469		@defer.inlineCallbacks
470		def get_joined_users_from_context(self, event, context):
471		state_group = context.state_group
472		if not state_group:
473		# If state_group is None it means it has yet to be assigned a
474		# state group, i.e. we need to make sure that calls with a state_group
475		# of None don't hit previous cached calls with a None state_group.
476		# To do this we set the state_group to a new object as object() != object()
477		state_group = object()
478
479		current_state_ids = yield context.get_current_state_ids(self)
480		result = yield self._get_joined_users_from_context(
481		event.room_id, state_group, current_state_ids, event=event, context=context
482		)
483		return result
484
485		def get_joined_users_from_state(self, room_id, state_entry):
486		state_group = state_entry.state_group
487		if not state_group:
488		# If state_group is None it means it has yet to be assigned a
489		# state group, i.e. we need to make sure that calls with a state_group
490		# of None don't hit previous cached calls with a None state_group.
491		# To do this we set the state_group to a new object as object() != object()
492		state_group = object()
493
494		return self._get_joined_users_from_context(
495		room_id, state_group, state_entry.state, context=state_entry
496		)
497
498		@cachedInlineCallbacks(
499		num_args=2, cache_context=True, iterable=True, max_entries=100000
500		)
501		def _get_joined_users_from_context(
502		self,
503		room_id,
504		state_group,
505		current_state_ids,
506		cache_context,
507		event=None,
508		context=None,
509		):
510		# We don't use `state_group`, it's there so that we can cache based
511		# on it. However, it's important that it's never None, since two current_states
512		# with a state_group of None are likely to be different.
513		# See bulk_get_push_rules_for_room for how we work around this.
514		assert state_group is not None
515
516		users_in_room = {}
517		member_event_ids = [
518		e_id
519		for key, e_id in iteritems(current_state_ids)
520		if key[0] == EventTypes.Member
521		]
522
523		if context is not None:
524		# If we have a context with a delta from a previous state group,
525		# check if we also have the result from the previous group in cache.
526		# If we do then we can reuse that result and simply update it with
527		# any membership changes in `delta_ids`
528		if context.prev_group and context.delta_ids:
529		prev_res = self._get_joined_users_from_context.cache.get(
530		(room_id, context.prev_group), None
531		)
532		if prev_res and isinstance(prev_res, dict):
533		users_in_room = dict(prev_res)
534		member_event_ids = [
535		e_id
536		for key, e_id in iteritems(context.delta_ids)
537		if key[0] == EventTypes.Member
538		]
539		for etype, state_key in context.delta_ids:
540		users_in_room.pop(state_key, None)
541
542		# We check if we have any of the member event ids in the event cache
543		# before we ask the DB
544
545		# We don't update the event cache hit ratio as it completely throws off
546		# the hit ratio counts. After all, we don't populate the cache if we
547		# miss it here
548		event_map = self._get_events_from_cache(
549		member_event_ids, allow_rejected=False, update_metrics=False
550		)
551
552		missing_member_event_ids = []
553		for event_id in member_event_ids:
554		ev_entry = event_map.get(event_id)
555		if ev_entry:
556		if ev_entry.event.membership == Membership.JOIN:
557		users_in_room[to_ascii(ev_entry.event.state_key)] = ProfileInfo(
558		display_name=to_ascii(
559		ev_entry.event.content.get("displayname", None)
560		),
561		avatar_url=to_ascii(
562		ev_entry.event.content.get("avatar_url", None)
563		),
564		)
565		else:
566		missing_member_event_ids.append(event_id)
567
568		if missing_member_event_ids:
569		rows = yield self._simple_select_many_batch(
570		table="room_memberships",
571		column="event_id",
572		iterable=missing_member_event_ids,
573		retcols=("user_id", "display_name", "avatar_url"),
574		keyvalues={"membership": Membership.JOIN},
575		batch_size=500,
576		desc="_get_joined_users_from_context",
577		)
578
579		users_in_room.update(
580		{
581		to_ascii(row["user_id"]): ProfileInfo(
582		avatar_url=to_ascii(row["avatar_url"]),
583		display_name=to_ascii(row["display_name"]),
584		)
585		for row in rows
586		}
587		)
588
589		if event is not None and event.type == EventTypes.Member:
590		if event.membership == Membership.JOIN:
591		if event.event_id in member_event_ids:
592		users_in_room[to_ascii(event.state_key)] = ProfileInfo(
593		display_name=to_ascii(event.content.get("displayname", None)),
594		avatar_url=to_ascii(event.content.get("avatar_url", None)),
595		)
596
597		return users_in_room
598
599		@cachedInlineCallbacks(max_entries=10000)
600		def is_host_joined(self, room_id, host):
601		if "%" in host or "_" in host:
602		raise Exception("Invalid host name")
603
604		sql = """
605		SELECT state_key FROM current_state_events AS c
606		INNER JOIN room_memberships AS m USING (event_id)
607		WHERE m.membership = 'join'
608		AND type = 'm.room.member'
609		AND c.room_id = ?
610		AND state_key LIKE ?
611		LIMIT 1
612		"""
613
614		# We do need to be careful to ensure that host doesn't have any wild cards
615		# in it, but we checked above for known ones and we'll check below that
616		# the returned user actually has the correct domain.
617		like_clause = "%:" + host
618
619		rows = yield self._execute("is_host_joined", None, sql, room_id, like_clause)
620
621		if not rows:
622		return False
623
624		user_id = rows[0][0]
625		if get_domain_from_id(user_id) != host:
626		# This can only happen if the host name has something funky in it
627		raise Exception("Invalid host name")
628
629		return True
630
631		@cachedInlineCallbacks()
632		def was_host_joined(self, room_id, host):
633		"""Check whether the server is or ever was in the room.
634
635		Args:
636		room_id (str)
637		host (str)
638
639		Returns:
640		Deferred: Resolves to True if the host is/was in the room, otherwise
641		False.
642		"""
643		if "%" in host or "_" in host:
644		raise Exception("Invalid host name")
645
646		sql = """
647		SELECT user_id FROM room_memberships
648		WHERE room_id = ?
649		AND user_id LIKE ?
650		AND membership = 'join'
651		LIMIT 1
652		"""
653
654		# We do need to be careful to ensure that host doesn't have any wild cards
655		# in it, but we checked above for known ones and we'll check below that
656		# the returned user actually has the correct domain.
657		like_clause = "%:" + host
658
659		rows = yield self._execute("was_host_joined", None, sql, room_id, like_clause)
660
661		if not rows:
662		return False
663
664		user_id = rows[0][0]
665		if get_domain_from_id(user_id) != host:
666		# This can only happen if the host name has something funky in it
667		raise Exception("Invalid host name")
668
669		return True
670
671		def get_joined_hosts(self, room_id, state_entry):
672		state_group = state_entry.state_group
673		if not state_group:
674		# If state_group is None it means it has yet to be assigned a
675		# state group, i.e. we need to make sure that calls with a state_group
676		# of None don't hit previous cached calls with a None state_group.
677		# To do this we set the state_group to a new object as object() != object()
678		state_group = object()
679
680		return self._get_joined_hosts(
681		room_id, state_group, state_entry.state, state_entry=state_entry
682		)
683
684		@cachedInlineCallbacks(num_args=2, max_entries=10000, iterable=True)
685		# @defer.inlineCallbacks
686		def _get_joined_hosts(self, room_id, state_group, current_state_ids, state_entry):
687		# We don't use `state_group`, its there so that we can cache based
688		# on it. However, its important that its never None, since two current_state's
689		# with a state_group of None are likely to be different.
690		# See bulk_get_push_rules_for_room for how we work around this.
691		assert state_group is not None
692
693		cache = self._get_joined_hosts_cache(room_id)
694		joined_hosts = yield cache.get_destinations(state_entry)
695
696		return joined_hosts
697
698		@cached(max_entries=10000)
699		def _get_joined_hosts_cache(self, room_id):
700		return _JoinedHostsCache(self, room_id)
701
702		@cachedInlineCallbacks(num_args=2)
703		def did_forget(self, user_id, room_id):
704		"""Returns whether user_id has elected to discard history for room_id.
705
706		Returns False if they have since re-joined."""
707
708		def f(txn):
709		sql = (
710		"SELECT"
711		" COUNT(*)"
712		" FROM"
713		" room_memberships"
714		" WHERE"
715		" user_id = ?"
716		" AND"
717		" room_id = ?"
718		" AND"
719		" forgotten = 0"
720		)
721		txn.execute(sql, (user_id, room_id))
722		rows = txn.fetchall()
723		return rows[0][0]
724
725		count = yield self.runInteraction("did_forget_membership", f)
726		return count == 0
727
728		@cached()
729		def get_forgotten_rooms_for_user(self, user_id):
730		"""Gets all rooms the user has forgotten.
731
732		Args:
733		user_id (str)
734
735		Returns:
736		Deferred[set[str]]
737		"""
738
739		def _get_forgotten_rooms_for_user_txn(txn):
740		# This is a slightly convoluted query that first looks up all rooms
741		# that the user has forgotten in the past, then rechecks that list
742		# to see if any have subsequently been updated. This is done so that
743		# we can use a partial index on `forgotten = 1` on the assumption
744		# that few users will actually forget many rooms.
745		#
746		# Note that a room is considered "forgotten" if all membership
747		# events for that user and room have the forgotten field set (as
748		# when a user forgets a room we update all rows for that user and
749		# room, not just the current one).
750		sql = """
751		SELECT room_id, (
752		SELECT count(*) FROM room_memberships
753		WHERE room_id = m.room_id AND user_id = m.user_id AND forgotten = 0
754		) AS count
755		FROM room_memberships AS m
756		WHERE user_id = ? AND forgotten = 1
757		GROUP BY room_id, user_id;
758		"""
759		txn.execute(sql, (user_id,))
760		return set(row[0] for row in txn if row[1] == 0)
761
762		return self.runInteraction(
763		"get_forgotten_rooms_for_user", _get_forgotten_rooms_for_user_txn
764		)
765
766		@defer.inlineCallbacks
767		def get_rooms_user_has_been_in(self, user_id):
768		"""Get all rooms that the user has ever been in.
769
770		Args:
771		user_id (str)
772
773		Returns:
774		Deferred[set[str]]: Set of room IDs.
775		"""
776
777		room_ids = yield self._simple_select_onecol(
778		table="room_memberships",
779		keyvalues={"membership": Membership.JOIN, "user_id": user_id},
780		retcol="room_id",
781		desc="get_rooms_user_has_been_in",
782		)
783
784		return set(room_ids)
785
786
787		class RoomMemberStore(RoomMemberWorkerStore):
788		def __init__(self, db_conn, hs):
789		super(RoomMemberStore, self).__init__(db_conn, hs)
790		self.register_background_update_handler(
791		_MEMBERSHIP_PROFILE_UPDATE_NAME, self._background_add_membership_profile
792		)
793		self.register_background_update_handler(
794		_CURRENT_STATE_MEMBERSHIP_UPDATE_NAME,
795		self._background_current_state_membership,
796		)
797		self.register_background_index_update(
798		"room_membership_forgotten_idx",
799		index_name="room_memberships_user_room_forgotten",
800		table="room_memberships",
801		columns=["user_id", "room_id"],
802		where_clause="forgotten = 1",
803		)
804
805		def _store_room_members_txn(self, txn, events, backfilled):
806		"""Store a room member in the database.
807		"""
808		self._simple_insert_many_txn(
809		txn,
810		table="room_memberships",
811		values=[
812		{
813		"event_id": event.event_id,
814		"user_id": event.state_key,
815		"sender": event.user_id,
816		"room_id": event.room_id,
817		"membership": event.membership,
818		"display_name": event.content.get("displayname", None),
819		"avatar_url": event.content.get("avatar_url", None),
820		}
821		for event in events
822		],
823		)
824
825		for event in events:
826		txn.call_after(
827		self._membership_stream_cache.entity_has_changed,
828		event.state_key,
829		event.internal_metadata.stream_ordering,
830		)
831		txn.call_after(
832		self.get_invited_rooms_for_user.invalidate, (event.state_key,)
833		)
834
835		# We update the local_invites table only if the event is "current",
836		# i.e., its something that has just happened. If the event is an
837		# outlier it is only current if its an "out of band membership",
838		# like a remote invite or a rejection of a remote invite.
839		is_new_state = not backfilled and (
840		not event.internal_metadata.is_outlier()
841		or event.internal_metadata.is_out_of_band_membership()
842		)
843		is_mine = self.hs.is_mine_id(event.state_key)
844		if is_new_state and is_mine:
845		if event.membership == Membership.INVITE:
846		self._simple_insert_txn(
847		txn,
848		table="local_invites",
849		values={
850		"event_id": event.event_id,
851		"invitee": event.state_key,
852		"inviter": event.sender,
853		"room_id": event.room_id,
854		"stream_id": event.internal_metadata.stream_ordering,
855		},
856		)
857		else:
858		sql = (
859		"UPDATE local_invites SET stream_id = ?, replaced_by = ? WHERE"
860		" room_id = ? AND invitee = ? AND locally_rejected is NULL"
861		" AND replaced_by is NULL"
862		)
863
864		txn.execute(
865		sql,
866		(
867		event.internal_metadata.stream_ordering,
868		event.event_id,
869		event.room_id,
870		event.state_key,
871		),
872		)
873
874		@defer.inlineCallbacks
875		def locally_reject_invite(self, user_id, room_id):
876		sql = (
877		"UPDATE local_invites SET stream_id = ?, locally_rejected = ? WHERE"
878		" room_id = ? AND invitee = ? AND locally_rejected is NULL"
879		" AND replaced_by is NULL"
880		)
881
882		def f(txn, stream_ordering):
883		txn.execute(sql, (stream_ordering, True, room_id, user_id))
884
885		with self._stream_id_gen.get_next() as stream_ordering:
886		yield self.runInteraction("locally_reject_invite", f, stream_ordering)
887
888		def forget(self, user_id, room_id):
889		"""Indicate that user_id wishes to discard history for room_id."""
890
891		def f(txn):
892		sql = (
893		"UPDATE"
894		" room_memberships"
895		" SET"
896		" forgotten = 1"
897		" WHERE"
898		" user_id = ?"
899		" AND"
900		" room_id = ?"
901		)
902		txn.execute(sql, (user_id, room_id))
903
904		self._invalidate_cache_and_stream(txn, self.did_forget, (user_id, room_id))
905		self._invalidate_cache_and_stream(
906		txn, self.get_forgotten_rooms_for_user, (user_id,)
907		)
908
909		return self.runInteraction("forget_membership", f)
910
911		@defer.inlineCallbacks
912		def _background_add_membership_profile(self, progress, batch_size):
913		target_min_stream_id = progress.get(
914		"target_min_stream_id_inclusive", self._min_stream_order_on_start
915		)
916		max_stream_id = progress.get(
917		"max_stream_id_exclusive", self._stream_order_on_start + 1
918		)
919
920		INSERT_CLUMP_SIZE = 1000
921
922		def add_membership_profile_txn(txn):
923		sql = """
924		SELECT stream_ordering, event_id, events.room_id, event_json.json
925		FROM events
926		INNER JOIN event_json USING (event_id)
927		INNER JOIN room_memberships USING (event_id)
928		WHERE ? <= stream_ordering AND stream_ordering < ?
929		AND type = 'm.room.member'
930		ORDER BY stream_ordering DESC
931		LIMIT ?
932		"""
933
934		txn.execute(sql, (target_min_stream_id, max_stream_id, batch_size))
935
936		rows = self.cursor_to_dict(txn)
937		if not rows:
938		return 0
939
940		min_stream_id = rows[-1]["stream_ordering"]
941
942		to_update = []
943		for row in rows:
944		event_id = row["event_id"]
945		room_id = row["room_id"]
946		try:
947		event_json = json.loads(row["json"])
948		content = event_json["content"]
949		except Exception:
950		continue
951
952		display_name = content.get("displayname", None)
953		avatar_url = content.get("avatar_url", None)
954
955		if display_name or avatar_url:
956		to_update.append((display_name, avatar_url, event_id, room_id))
957
958		to_update_sql = """
959		UPDATE room_memberships SET display_name = ?, avatar_url = ?
960		WHERE event_id = ? AND room_id = ?
961		"""
962		for index in range(0, len(to_update), INSERT_CLUMP_SIZE):
963		clump = to_update[index : index + INSERT_CLUMP_SIZE]
964		txn.executemany(to_update_sql, clump)
965
966		progress = {
967		"target_min_stream_id_inclusive": target_min_stream_id,
968		"max_stream_id_exclusive": min_stream_id,
969		}
970
971		self._background_update_progress_txn(
972		txn, _MEMBERSHIP_PROFILE_UPDATE_NAME, progress
973		)
974
975		return len(rows)
976
977		result = yield self.runInteraction(
978		_MEMBERSHIP_PROFILE_UPDATE_NAME, add_membership_profile_txn
979		)
980
981		if not result:
982		yield self._end_background_update(_MEMBERSHIP_PROFILE_UPDATE_NAME)
983
984		return result
985
986		@defer.inlineCallbacks
987		def _background_current_state_membership(self, progress, batch_size):
988		"""Update the new membership column on current_state_events.
989
990		This works by iterating over all rooms in alphebetical order.
991		"""
992
993		def _background_current_state_membership_txn(txn, last_processed_room):
994		processed = 0
995		while processed < batch_size:
996		txn.execute(
997		"""
998		SELECT MIN(room_id) FROM current_state_events WHERE room_id > ?
999		""",
1000		(last_processed_room,),
1001		)
1002		row = txn.fetchone()
1003		if not row or not row[0]:
1004		return processed, True
1005
1006		next_room, = row
1007
1008		sql = """
1009		UPDATE current_state_events
1010		SET membership = (
1011		SELECT membership FROM room_memberships
1012		WHERE event_id = current_state_events.event_id
1013		)
1014		WHERE room_id = ?
1015		"""
1016		txn.execute(sql, (next_room,))
1017		processed += txn.rowcount
1018
1019		last_processed_room = next_room
1020
1021		self._background_update_progress_txn(
1022		txn,
1023		_CURRENT_STATE_MEMBERSHIP_UPDATE_NAME,
1024		{"last_processed_room": last_processed_room},
1025		)
1026
1027		return processed, False
1028
1029		# If we haven't got a last processed room then just use the empty
1030		# string, which will compare before all room IDs correctly.
1031		last_processed_room = progress.get("last_processed_room", "")
1032
1033		row_count, finished = yield self.runInteraction(
1034		"_background_current_state_membership_update",
1035		_background_current_state_membership_txn,
1036		last_processed_room,
1037		)
1038
1039		if finished:
1040		yield self._end_background_update(_CURRENT_STATE_MEMBERSHIP_UPDATE_NAME)
1041
1042		return row_count
1043
1044
1045		class _JoinedHostsCache(object):
1046		"""Cache for joined hosts in a room that is optimised to handle updates
1047		via state deltas.
1048		"""
1049
1050		def __init__(self, store, room_id):
1051		self.store = store
1052		self.room_id = room_id
1053
1054		self.hosts_to_joined_users = {}
1055
1056		self.state_group = object()
1057
1058		self.linearizer = Linearizer("_JoinedHostsCache")
1059
1060		self._len = 0
1061
1062		@defer.inlineCallbacks
1063		def get_destinations(self, state_entry):
1064		"""Get set of destinations for a state entry
1065
1066		Args:
1067		state_entry(synapse.state._StateCacheEntry)
1068		"""
1069		if state_entry.state_group == self.state_group:
1070		return frozenset(self.hosts_to_joined_users)
1071
1072		with (yield self.linearizer.queue(())):
1073		if state_entry.state_group == self.state_group:
1074		pass
1075		elif state_entry.prev_group == self.state_group:
1076		for (typ, state_key), event_id in iteritems(state_entry.delta_ids):
1077		if typ != EventTypes.Member:
1078		continue
1079
1080		host = intern_string(get_domain_from_id(state_key))
1081		user_id = state_key
1082		known_joins = self.hosts_to_joined_users.setdefault(host, set())
1083
1084		event = yield self.store.get_event(event_id)
1085		if event.membership == Membership.JOIN:
1086		known_joins.add(user_id)
1087		else:
1088		known_joins.discard(user_id)
1089
1090		if not known_joins:
1091		self.hosts_to_joined_users.pop(host, None)
1092		else:
1093		joined_users = yield self.store.get_joined_users_from_state(
1094		self.room_id, state_entry
1095		)
1096
1097		self.hosts_to_joined_users = {}
1098		for user_id in joined_users:
1099		host = intern_string(get_domain_from_id(user_id))
1100		self.hosts_to_joined_users.setdefault(host, set()).add(user_id)
1101
1102		if state_entry.state_group:
1103		self.state_group = state_entry.state_group
1104		else:
1105		self.state_group = object()
1106		self._len = sum(len(v) for v in itervalues(self.hosts_to_joined_users))
1107		return frozenset(self.hosts_to_joined_users)
1108
1109		def __len__(self):
1110		return self._len

+0

-63

~~synapse/storage/schema/delta/12/v12.sql~~ less more

0		/* Copyright 2015, 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		CREATE TABLE IF NOT EXISTS rejections(
16		event_id TEXT NOT NULL,
17		reason TEXT NOT NULL,
18		last_check TEXT NOT NULL,
19		UNIQUE (event_id)
20		);
21
22		-- Push notification endpoints that users have configured
23		CREATE TABLE IF NOT EXISTS pushers (
24		id INTEGER PRIMARY KEY AUTOINCREMENT,
25		user_name TEXT NOT NULL,
26		profile_tag VARCHAR(32) NOT NULL,
27		kind VARCHAR(8) NOT NULL,
28		app_id VARCHAR(64) NOT NULL,
29		app_display_name VARCHAR(64) NOT NULL,
30		device_display_name VARCHAR(128) NOT NULL,
31		pushkey VARBINARY(512) NOT NULL,
32		ts BIGINT UNSIGNED NOT NULL,
33		lang VARCHAR(8),
34		data LONGBLOB,
35		last_token TEXT,
36		last_success BIGINT UNSIGNED,
37		failing_since BIGINT UNSIGNED,
38		UNIQUE (app_id, pushkey)
39		);
40
41		CREATE TABLE IF NOT EXISTS push_rules (
42		id INTEGER PRIMARY KEY AUTOINCREMENT,
43		user_name TEXT NOT NULL,
44		rule_id TEXT NOT NULL,
45		priority_class TINYINT NOT NULL,
46		priority INTEGER NOT NULL DEFAULT 0,
47		conditions TEXT NOT NULL,
48		actions TEXT NOT NULL,
49		UNIQUE(user_name, rule_id)
50		);
51
52		CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name);
53
54		CREATE TABLE IF NOT EXISTS user_filters(
55		user_id TEXT,
56		filter_id BIGINT UNSIGNED,
57		filter_json LONGBLOB
58		);
59
60		CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters(
61		user_id, filter_id
62		);

+0

-19

~~synapse/storage/schema/delta/13/v13.sql~~ less more

0		/* Copyright 2015, 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		/* We used to create a tables called application_services and
16		* application_services_regex, but these are no longer used and are removed in
17		* delta 54.
18		*/

+0

-23

~~synapse/storage/schema/delta/14/v14.sql~~ less more

0		/* Copyright 2015, 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14		CREATE TABLE IF NOT EXISTS push_rules_enable (
15		id INTEGER PRIMARY KEY AUTOINCREMENT,
16		user_name TEXT NOT NULL,
17		rule_id TEXT NOT NULL,
18		enabled TINYINT,
19		UNIQUE(user_name, rule_id)
20		);
21
22		CREATE INDEX IF NOT EXISTS push_rules_enable_user_name on push_rules_enable (user_name);

+0

-31

~~synapse/storage/schema/delta/15/appservice_txns.sql~~ less more

0		/* Copyright 2015, 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		CREATE TABLE IF NOT EXISTS application_services_state(
16		as_id TEXT PRIMARY KEY,
17		state VARCHAR(5),
18		last_txn INTEGER
19		);
20
21		CREATE TABLE IF NOT EXISTS application_services_txns(
22		as_id TEXT NOT NULL,
23		txn_id INTEGER NOT NULL,
24		event_ids TEXT NOT NULL,
25		UNIQUE(as_id, txn_id)
26		);
27
28		CREATE INDEX IF NOT EXISTS application_services_txns_id ON application_services_txns (
29		as_id
30		);

+0

-2

~~synapse/storage/schema/delta/15/presence_indices.sql~~ less more

0
1		CREATE INDEX IF NOT EXISTS presence_list_user_id ON presence_list (user_id);

+0

-24

~~synapse/storage/schema/delta/15/v15.sql~~ less more

0		-- Drop, copy & recreate pushers table to change unique key
1		-- Also add access_token column at the same time
2		CREATE TABLE IF NOT EXISTS pushers2 (
3		id BIGINT PRIMARY KEY,
4		user_name TEXT NOT NULL,
5		access_token BIGINT DEFAULT NULL,
6		profile_tag VARCHAR(32) NOT NULL,
7		kind VARCHAR(8) NOT NULL,
8		app_id VARCHAR(64) NOT NULL,
9		app_display_name VARCHAR(64) NOT NULL,
10		device_display_name VARCHAR(128) NOT NULL,
11		pushkey bytea NOT NULL,
12		ts BIGINT NOT NULL,
13		lang VARCHAR(8),
14		data bytea,
15		last_token TEXT,
16		last_success BIGINT,
17		failing_since BIGINT,
18		UNIQUE (app_id, pushkey)
19		);
20		INSERT INTO pushers2 (id, user_name, profile_tag, kind, app_id, app_display_name, device_display_name, pushkey, ts, lang, data, last_token, last_success, failing_since)
21		SELECT id, user_name, profile_tag, kind, app_id, app_display_name, device_display_name, pushkey, ts, lang, data, last_token, last_success, failing_since FROM pushers;
22		DROP TABLE pushers;
23		ALTER TABLE pushers2 RENAME TO pushers;

+0

-4

~~synapse/storage/schema/delta/16/events_order_index.sql~~ less more

0		CREATE INDEX events_order ON events (topological_ordering, stream_ordering);
1		CREATE INDEX events_order_room ON events (
2		room_id, topological_ordering, stream_ordering
3		);

+0

-2

~~synapse/storage/schema/delta/16/remote_media_cache_index.sql~~ less more

0		CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id
1		ON remote_media_cache_thumbnails (media_id);⏎

+0

-9

~~synapse/storage/schema/delta/16/remove_duplicates.sql~~ less more

0
1
2		DELETE FROM event_to_state_groups WHERE state_group not in (
3		SELECT MAX(state_group) FROM event_to_state_groups GROUP BY event_id
4		);
5
6		DELETE FROM event_to_state_groups WHERE rowid not in (
7		SELECT MIN(rowid) FROM event_to_state_groups GROUP BY event_id
8		);

+0

-3

~~synapse/storage/schema/delta/16/room_alias_index.sql~~ less more

0
1		CREATE INDEX IF NOT EXISTS room_aliases_id ON room_aliases(room_id);
2		CREATE INDEX IF NOT EXISTS room_alias_servers_alias ON room_alias_servers(room_alias);

+0

-72

~~synapse/storage/schema/delta/16/unique_constraints.sql~~ less more

0
1		-- We can use SQLite features here, since other db support was only added in v16
2
3		--
4		DELETE FROM current_state_events WHERE rowid not in (
5		SELECT MIN(rowid) FROM current_state_events GROUP BY event_id
6		);
7
8		DROP INDEX IF EXISTS current_state_events_event_id;
9		CREATE UNIQUE INDEX current_state_events_event_id ON current_state_events(event_id);
10
11		--
12		DELETE FROM room_memberships WHERE rowid not in (
13		SELECT MIN(rowid) FROM room_memberships GROUP BY event_id
14		);
15
16		DROP INDEX IF EXISTS room_memberships_event_id;
17		CREATE UNIQUE INDEX room_memberships_event_id ON room_memberships(event_id);
18
19		--
20		DELETE FROM topics WHERE rowid not in (
21		SELECT MIN(rowid) FROM topics GROUP BY event_id
22		);
23
24		DROP INDEX IF EXISTS topics_event_id;
25		CREATE UNIQUE INDEX topics_event_id ON topics(event_id);
26
27		--
28		DELETE FROM room_names WHERE rowid not in (
29		SELECT MIN(rowid) FROM room_names GROUP BY event_id
30		);
31
32		DROP INDEX IF EXISTS room_names_id;
33		CREATE UNIQUE INDEX room_names_id ON room_names(event_id);
34
35		--
36		DELETE FROM presence WHERE rowid not in (
37		SELECT MIN(rowid) FROM presence GROUP BY user_id
38		);
39
40		DROP INDEX IF EXISTS presence_id;
41		CREATE UNIQUE INDEX presence_id ON presence(user_id);
42
43		--
44		DELETE FROM presence_allow_inbound WHERE rowid not in (
45		SELECT MIN(rowid) FROM presence_allow_inbound
46		GROUP BY observed_user_id, observer_user_id
47		);
48
49		DROP INDEX IF EXISTS presence_allow_inbound_observers;
50		CREATE UNIQUE INDEX presence_allow_inbound_observers ON presence_allow_inbound(
51		observed_user_id, observer_user_id
52		);
53
54		--
55		DELETE FROM presence_list WHERE rowid not in (
56		SELECT MIN(rowid) FROM presence_list
57		GROUP BY user_id, observed_user_id
58		);
59
60		DROP INDEX IF EXISTS presence_list_observers;
61		CREATE UNIQUE INDEX presence_list_observers ON presence_list(
62		user_id, observed_user_id
63		);
64
65		--
66		DELETE FROM room_aliases WHERE rowid not in (
67		SELECT MIN(rowid) FROM room_aliases GROUP BY room_alias
68		);
69
70		DROP INDEX IF EXISTS room_aliases_id;
71		CREATE INDEX room_aliases_id ON room_aliases(room_id);

+0

-56

~~synapse/storage/schema/delta/16/users.sql~~ less more

0		-- Convert `access_tokens`.user from rowids to user strings.
1		-- MUST BE DONE BEFORE REMOVING ID COLUMN FROM USERS TABLE BELOW
2		CREATE TABLE IF NOT EXISTS new_access_tokens(
3		id BIGINT UNSIGNED PRIMARY KEY,
4		user_id TEXT NOT NULL,
5		device_id TEXT,
6		token TEXT NOT NULL,
7		last_used BIGINT UNSIGNED,
8		UNIQUE(token)
9		);
10
11		INSERT INTO new_access_tokens
12		SELECT a.id, u.name, a.device_id, a.token, a.last_used
13		FROM access_tokens as a
14		INNER JOIN users as u ON u.id = a.user_id;
15
16		DROP TABLE access_tokens;
17
18		ALTER TABLE new_access_tokens RENAME TO access_tokens;
19
20		-- Remove ID column from `users` table
21		CREATE TABLE IF NOT EXISTS new_users(
22		name TEXT,
23		password_hash TEXT,
24		creation_ts BIGINT UNSIGNED,
25		admin BOOL DEFAULT 0 NOT NULL,
26		UNIQUE(name)
27		);
28
29		INSERT INTO new_users SELECT name, password_hash, creation_ts, admin FROM users;
30
31		DROP TABLE users;
32
33		ALTER TABLE new_users RENAME TO users;
34
35
36		-- Remove UNIQUE constraint from `user_ips` table
37		CREATE TABLE IF NOT EXISTS new_user_ips (
38		user_id TEXT NOT NULL,
39		access_token TEXT NOT NULL,
40		device_id TEXT,
41		ip TEXT NOT NULL,
42		user_agent TEXT NOT NULL,
43		last_seen BIGINT UNSIGNED NOT NULL
44		);
45
46		INSERT INTO new_user_ips
47		SELECT user, access_token, device_id, ip, user_agent, last_seen FROM user_ips;
48
49		DROP TABLE user_ips;
50
51		ALTER TABLE new_user_ips RENAME TO user_ips;
52
53		CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user_id);
54		CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user_id, access_token, ip);
55

+0

-18

~~synapse/storage/schema/delta/17/drop_indexes.sql~~ less more

0		/* Copyright 2015, 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		DROP INDEX IF EXISTS sent_transaction_dest;
16		DROP INDEX IF EXISTS sent_transaction_sent;
17		DROP INDEX IF EXISTS user_ips_user;

+0

-24

~~synapse/storage/schema/delta/17/server_keys.sql~~ less more

0		/* Copyright 2015, 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		CREATE TABLE IF NOT EXISTS server_keys_json (
16		server_name TEXT, -- Server name.
17		key_id TEXT, -- Requested key id.
18		from_server TEXT, -- Which server the keys were fetched from.
19		ts_added_ms INTEGER, -- When the keys were fetched
20		ts_valid_until_ms INTEGER, -- When this version of the keys exipires.
21		key_json bytea, -- JSON certificate for the remote server.
22		CONSTRAINT uniqueness UNIQUE (server_name, key_id, from_server)
23		);

+0

-9

~~synapse/storage/schema/delta/17/user_threepids.sql~~ less more

0		CREATE TABLE user_threepids (
1		user_id TEXT NOT NULL,
2		medium TEXT NOT NULL,
3		address TEXT NOT NULL,
4		validated_at BIGINT NOT NULL,
5		added_at BIGINT NOT NULL,
6		CONSTRAINT user_medium_address UNIQUE (user_id, medium, address)
7		);
8		CREATE INDEX user_threepids_user_id ON user_threepids(user_id);

+0

-32

~~synapse/storage/schema/delta/18/server_keys_bigger_ints.sql~~ less more

0		/* Copyright 2015, 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15
16		CREATE TABLE IF NOT EXISTS new_server_keys_json (
17		server_name TEXT NOT NULL, -- Server name.
18		key_id TEXT NOT NULL, -- Requested key id.
19		from_server TEXT NOT NULL, -- Which server the keys were fetched from.
20		ts_added_ms BIGINT NOT NULL, -- When the keys were fetched
21		ts_valid_until_ms BIGINT NOT NULL, -- When this version of the keys exipires.
22		key_json bytea NOT NULL, -- JSON certificate for the remote server.
23		CONSTRAINT server_keys_json_uniqueness UNIQUE (server_name, key_id, from_server)
24		);
25
26		INSERT INTO new_server_keys_json
27		SELECT server_name, key_id, from_server,ts_added_ms, ts_valid_until_ms, key_json FROM server_keys_json ;
28
29		DROP TABLE server_keys_json;
30
31		ALTER TABLE new_server_keys_json RENAME TO server_keys_json;

+0

-19

~~synapse/storage/schema/delta/19/event_index.sql~~ less more

0		/* Copyright 2015, 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15
16		CREATE INDEX events_order_topo_stream_room ON events(
17		topological_ordering, stream_ordering, room_id
18		);

+0

-1

~~synapse/storage/schema/delta/20/dummy.sql~~ less more

0

SELECT 1;

+0

-88

~~synapse/storage/schema/delta/20/pushers.py~~ less more

0		# Copyright 2015, 2016 OpenMarket Ltd
1		#
2		# Licensed under the Apache License, Version 2.0 (the "License");
3		# you may not use this file except in compliance with the License.
4		# You may obtain a copy of the License at
5		#
6		# http://www.apache.org/licenses/LICENSE-2.0
7		#
8		# Unless required by applicable law or agreed to in writing, software
9		# distributed under the License is distributed on an "AS IS" BASIS,
10		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		# See the License for the specific language governing permissions and
12		# limitations under the License.
13
14
15		"""
16		Main purpose of this upgrade is to change the unique key on the
17		pushers table again (it was missed when the v16 full schema was
18		made) but this also changes the pushkey and data columns to text.
19		When selecting a bytea column into a text column, postgres inserts
20		the hex encoded data, and there's no portable way of getting the
21		UTF-8 bytes, so we have to do it in Python.
22		"""
23
24		import logging
25
26		logger = logging.getLogger(__name__)
27
28
29		def run_create(cur, database_engine, args, *kwargs):
30		logger.info("Porting pushers table...")
31		cur.execute(
32		"""
33		CREATE TABLE IF NOT EXISTS pushers2 (
34		id BIGINT PRIMARY KEY,
35		user_name TEXT NOT NULL,
36		access_token BIGINT DEFAULT NULL,
37		profile_tag VARCHAR(32) NOT NULL,
38		kind VARCHAR(8) NOT NULL,
39		app_id VARCHAR(64) NOT NULL,
40		app_display_name VARCHAR(64) NOT NULL,
41		device_display_name VARCHAR(128) NOT NULL,
42		pushkey TEXT NOT NULL,
43		ts BIGINT NOT NULL,
44		lang VARCHAR(8),
45		data TEXT,
46		last_token TEXT,
47		last_success BIGINT,
48		failing_since BIGINT,
49		UNIQUE (app_id, pushkey, user_name)
50		)
51		"""
52		)
53		cur.execute(
54		"""SELECT
55		id, user_name, access_token, profile_tag, kind,
56		app_id, app_display_name, device_display_name,
57		pushkey, ts, lang, data, last_token, last_success,
58		failing_since
59		FROM pushers
60		"""
61		)
62		count = 0
63		for row in cur.fetchall():
64		row = list(row)
65		row[8] = bytes(row[8]).decode("utf-8")
66		row[11] = bytes(row[11]).decode("utf-8")
67		cur.execute(
68		database_engine.convert_param_style(
69		"""
70		INSERT into pushers2 (
71		id, user_name, access_token, profile_tag, kind,
72		app_id, app_display_name, device_display_name,
73		pushkey, ts, lang, data, last_token, last_success,
74		failing_since
75		) values (%s)"""
76		% (",".join(["?" for _ in range(len(row))]))
77		),
78		row,
79		)
80		count += 1
81		cur.execute("DROP TABLE pushers")
82		cur.execute("ALTER TABLE pushers2 RENAME TO pushers")
83		logger.info("Moved %d pushers to new table", count)
84
85
86		def run_upgrade(args, *kwargs):
87		pass

+0

-34

~~synapse/storage/schema/delta/21/end_to_end_keys.sql~~ less more

0		/* Copyright 2015, 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15
16		CREATE TABLE IF NOT EXISTS e2e_device_keys_json (
17		user_id TEXT NOT NULL, -- The user these keys are for.
18		device_id TEXT NOT NULL, -- Which of the user's devices these keys are for.
19		ts_added_ms BIGINT NOT NULL, -- When the keys were uploaded.
20		key_json TEXT NOT NULL, -- The keys for the device as a JSON blob.
21		CONSTRAINT e2e_device_keys_json_uniqueness UNIQUE (user_id, device_id)
22		);
23
24
25		CREATE TABLE IF NOT EXISTS e2e_one_time_keys_json (
26		user_id TEXT NOT NULL, -- The user this one-time key is for.
27		device_id TEXT NOT NULL, -- The device this one-time key is for.
28		algorithm TEXT NOT NULL, -- Which algorithm this one-time key is for.
29		key_id TEXT NOT NULL, -- An id for suppressing duplicate uploads.
30		ts_added_ms BIGINT NOT NULL, -- When this key was uploaded.
31		key_json TEXT NOT NULL, -- The key as a JSON blob.
32		CONSTRAINT e2e_one_time_keys_json_uniqueness UNIQUE (user_id, device_id, algorithm, key_id)
33		);

+0

-38

~~synapse/storage/schema/delta/21/receipts.sql~~ less more

0		/* Copyright 2015, 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15
16		CREATE TABLE IF NOT EXISTS receipts_graph(
17		room_id TEXT NOT NULL,
18		receipt_type TEXT NOT NULL,
19		user_id TEXT NOT NULL,
20		event_ids TEXT NOT NULL,
21		data TEXT NOT NULL,
22		CONSTRAINT receipts_graph_uniqueness UNIQUE (room_id, receipt_type, user_id)
23		);
24
25		CREATE TABLE IF NOT EXISTS receipts_linearized (
26		stream_id BIGINT NOT NULL,
27		room_id TEXT NOT NULL,
28		receipt_type TEXT NOT NULL,
29		user_id TEXT NOT NULL,
30		event_id TEXT NOT NULL,
31		data TEXT NOT NULL,
32		CONSTRAINT receipts_linearized_uniqueness UNIQUE (room_id, receipt_type, user_id)
33		);
34
35		CREATE INDEX receipts_linearized_id ON receipts_linearized(
36		stream_id
37		);

+0

-22

~~synapse/storage/schema/delta/22/receipts_index.sql~~ less more

0		/* Copyright 2015, 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		/** Using CREATE INDEX directly is deprecated in favour of using background
16		* update see synapse/storage/schema/delta/33/access_tokens_device_index.sql
17		* and synapse/storage/registration.py for an example using
18		* "access_tokens_device_index" **/
19		CREATE INDEX receipts_linearized_room_stream ON receipts_linearized(
20		room_id, stream_id
21		);

+0

-19

~~synapse/storage/schema/delta/22/user_threepids_unique.sql~~ less more

0		CREATE TABLE IF NOT EXISTS user_threepids2 (
1		user_id TEXT NOT NULL,
2		medium TEXT NOT NULL,
3		address TEXT NOT NULL,
4		validated_at BIGINT NOT NULL,
5		added_at BIGINT NOT NULL,
6		CONSTRAINT medium_address UNIQUE (medium, address)
7		);
8
9		INSERT INTO user_threepids2
10		SELECT * FROM user_threepids WHERE added_at IN (
11		SELECT max(added_at) FROM user_threepids GROUP BY medium, address
12		)
13		;
14
15		DROP TABLE user_threepids;
16		ALTER TABLE user_threepids2 RENAME TO user_threepids;
17
18		CREATE INDEX user_threepids_user_id ON user_threepids(user_id);

+0

-16

~~synapse/storage/schema/delta/23/drop_state_index.sql~~ less more

0		/* Copyright 2015, 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		DROP INDEX IF EXISTS state_groups_state_tuple;

+0

-18

~~synapse/storage/schema/delta/24/stats_reporting.sql~~ less more

0		/* Copyright 2019 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		/* We used to create a table called stats_reporting, but this is no longer
16		* used and is removed in delta 54.
17		*/⏎

+0

-82

~~synapse/storage/schema/delta/25/fts.py~~ less more

0		# Copyright 2015, 2016 OpenMarket Ltd
1		#
2		# Licensed under the Apache License, Version 2.0 (the "License");
3		# you may not use this file except in compliance with the License.
4		# You may obtain a copy of the License at
5		#
6		# http://www.apache.org/licenses/LICENSE-2.0
7		#
8		# Unless required by applicable law or agreed to in writing, software
9		# distributed under the License is distributed on an "AS IS" BASIS,
10		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		# See the License for the specific language governing permissions and
12		# limitations under the License.
13
14		import logging
15
16		import simplejson
17
18		from synapse.storage.engines import PostgresEngine, Sqlite3Engine
19		from synapse.storage.prepare_database import get_statements
20
21		logger = logging.getLogger(__name__)
22
23
24		POSTGRES_TABLE = """
25		CREATE TABLE IF NOT EXISTS event_search (
26		event_id TEXT,
27		room_id TEXT,
28		sender TEXT,
29		key TEXT,
30		vector tsvector
31		);
32
33		CREATE INDEX event_search_fts_idx ON event_search USING gin(vector);
34		CREATE INDEX event_search_ev_idx ON event_search(event_id);
35		CREATE INDEX event_search_ev_ridx ON event_search(room_id);
36		"""
37
38
39		SQLITE_TABLE = (
40		"CREATE VIRTUAL TABLE event_search"
41		" USING fts4 ( event_id, room_id, sender, key, value )"
42		)
43
44
45		def run_create(cur, database_engine, args, *kwargs):
46		if isinstance(database_engine, PostgresEngine):
47		for statement in get_statements(POSTGRES_TABLE.splitlines()):
48		cur.execute(statement)
49		elif isinstance(database_engine, Sqlite3Engine):
50		cur.execute(SQLITE_TABLE)
51		else:
52		raise Exception("Unrecognized database engine")
53
54		cur.execute("SELECT MIN(stream_ordering) FROM events")
55		rows = cur.fetchall()
56		min_stream_id = rows[0][0]
57
58		cur.execute("SELECT MAX(stream_ordering) FROM events")
59		rows = cur.fetchall()
60		max_stream_id = rows[0][0]
61
62		if min_stream_id is not None and max_stream_id is not None:
63		progress = {
64		"target_min_stream_id_inclusive": min_stream_id,
65		"max_stream_id_exclusive": max_stream_id + 1,
66		"rows_inserted": 0,
67		}
68		progress_json = simplejson.dumps(progress)
69
70		sql = (
71		"INSERT into background_updates (update_name, progress_json)"
72		" VALUES (?, ?)"
73		)
74
75		sql = database_engine.convert_param_style(sql)
76
77		cur.execute(sql, ("event_search", progress_json))
78
79
80		def run_upgrade(args, *kwargs):
81		pass

+0

-25

~~synapse/storage/schema/delta/25/guest_access.sql~~ less more

0		/* Copyright 2015, 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		/*
16		* This is a manual index of guest_access content of state events,
17		* so that we can join on them in SELECT statements.
18		*/
19		CREATE TABLE IF NOT EXISTS guest_access(
20		event_id TEXT NOT NULL,
21		room_id TEXT NOT NULL,
22		guest_access TEXT NOT NULL,
23		UNIQUE (event_id)
24		);

+0

-25

~~synapse/storage/schema/delta/25/history_visibility.sql~~ less more

0		/* Copyright 2015, 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		/*
16		* This is a manual index of history_visibility content of state events,
17		* so that we can join on them in SELECT statements.
18		*/
19		CREATE TABLE IF NOT EXISTS history_visibility(
20		event_id TEXT NOT NULL,
21		room_id TEXT NOT NULL,
22		history_visibility TEXT NOT NULL,
23		UNIQUE (event_id)
24		);

+0

-38

~~synapse/storage/schema/delta/25/tags.sql~~ less more

0		/* Copyright 2015, 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15
16		CREATE TABLE IF NOT EXISTS room_tags(
17		user_id TEXT NOT NULL,
18		room_id TEXT NOT NULL,
19		tag TEXT NOT NULL, -- The name of the tag.
20		content TEXT NOT NULL, -- The JSON content of the tag.
21		CONSTRAINT room_tag_uniqueness UNIQUE (user_id, room_id, tag)
22		);
23
24		CREATE TABLE IF NOT EXISTS room_tags_revisions (
25		user_id TEXT NOT NULL,
26		room_id TEXT NOT NULL,
27		stream_id BIGINT NOT NULL, -- The current version of the room tags.
28		CONSTRAINT room_tag_revisions_uniqueness UNIQUE (user_id, room_id)
29		);
30
31		CREATE TABLE IF NOT EXISTS private_user_data_max_stream_id(
32		Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, -- Makes sure this table only has one row.
33		stream_id BIGINT NOT NULL,
34		CHECK (Lock='X')
35		);
36
37		INSERT INTO private_user_data_max_stream_id (stream_id) VALUES (0);

+0

-17

~~synapse/storage/schema/delta/26/account_data.sql~~ less more

0		/* Copyright 2015, 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15
16		ALTER TABLE private_user_data_max_stream_id RENAME TO account_data_max_stream_id;

+0

-36

~~synapse/storage/schema/delta/27/account_data.sql~~ less more

0		/* Copyright 2015, 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		CREATE TABLE IF NOT EXISTS account_data(
16		user_id TEXT NOT NULL,
17		account_data_type TEXT NOT NULL, -- The type of the account_data.
18		stream_id BIGINT NOT NULL, -- The version of the account_data.
19		content TEXT NOT NULL, -- The JSON content of the account_data
20		CONSTRAINT account_data_uniqueness UNIQUE (user_id, account_data_type)
21		);
22
23
24		CREATE TABLE IF NOT EXISTS room_account_data(
25		user_id TEXT NOT NULL,
26		room_id TEXT NOT NULL,
27		account_data_type TEXT NOT NULL, -- The type of the account_data.
28		stream_id BIGINT NOT NULL, -- The version of the account_data.
29		content TEXT NOT NULL, -- The JSON content of the account_data
30		CONSTRAINT room_account_data_uniqueness UNIQUE (user_id, room_id, account_data_type)
31		);
32
33
34		CREATE INDEX account_data_stream_id on account_data(user_id, stream_id);
35		CREATE INDEX room_account_data_stream_id on room_account_data(user_id, stream_id);

+0

-26

~~synapse/storage/schema/delta/27/forgotten_memberships.sql~~ less more

0		/* Copyright 2015, 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		/*
16		* Keeps track of what rooms users have left and don't want to be able to
17		* access again.
18		*
19		* If all users on this server have left a room, we can delete the room
20		* entirely.
21		*
22		* This column should always contain either 0 or 1.
23		*/
24
25		ALTER TABLE room_memberships ADD COLUMN forgotten INTEGER DEFAULT 0;

+0

-61

~~synapse/storage/schema/delta/27/ts.py~~ less more

0		# Copyright 2015, 2016 OpenMarket Ltd
1		#
2		# Licensed under the Apache License, Version 2.0 (the "License");
3		# you may not use this file except in compliance with the License.
4		# You may obtain a copy of the License at
5		#
6		# http://www.apache.org/licenses/LICENSE-2.0
7		#
8		# Unless required by applicable law or agreed to in writing, software
9		# distributed under the License is distributed on an "AS IS" BASIS,
10		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		# See the License for the specific language governing permissions and
12		# limitations under the License.
13
14		import logging
15
16		import simplejson
17
18		from synapse.storage.prepare_database import get_statements
19
20		logger = logging.getLogger(__name__)
21
22
23		ALTER_TABLE = (
24		"ALTER TABLE events ADD COLUMN origin_server_ts BIGINT;"
25		"CREATE INDEX events_ts ON events(origin_server_ts, stream_ordering);"
26		)
27
28
29		def run_create(cur, database_engine, args, *kwargs):
30		for statement in get_statements(ALTER_TABLE.splitlines()):
31		cur.execute(statement)
32
33		cur.execute("SELECT MIN(stream_ordering) FROM events")
34		rows = cur.fetchall()
35		min_stream_id = rows[0][0]
36
37		cur.execute("SELECT MAX(stream_ordering) FROM events")
38		rows = cur.fetchall()
39		max_stream_id = rows[0][0]
40
41		if min_stream_id is not None and max_stream_id is not None:
42		progress = {
43		"target_min_stream_id_inclusive": min_stream_id,
44		"max_stream_id_exclusive": max_stream_id + 1,
45		"rows_inserted": 0,
46		}
47		progress_json = simplejson.dumps(progress)
48
49		sql = (
50		"INSERT into background_updates (update_name, progress_json)"
51		" VALUES (?, ?)"
52		)
53
54		sql = database_engine.convert_param_style(sql)
55
56		cur.execute(sql, ("event_origin_server_ts", progress_json))
57
58
59		def run_upgrade(args, *kwargs):
60		pass

+0

-27

~~synapse/storage/schema/delta/28/event_push_actions.sql~~ less more

0		/* Copyright 2015 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		CREATE TABLE IF NOT EXISTS event_push_actions(
16		room_id TEXT NOT NULL,
17		event_id TEXT NOT NULL,
18		user_id TEXT NOT NULL,
19		profile_tag VARCHAR(32),
20		actions TEXT NOT NULL,
21		CONSTRAINT event_id_user_id_profile_tag_uniqueness UNIQUE (room_id, event_id, user_id, profile_tag)
22		);
23
24
25		CREATE INDEX event_push_actions_room_id_event_id_user_id_profile_tag on event_push_actions(room_id, event_id, user_id, profile_tag);
26		CREATE INDEX event_push_actions_room_id_user_id on event_push_actions(room_id, user_id);

+0

-20

~~synapse/storage/schema/delta/28/events_room_stream.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		/** Using CREATE INDEX directly is deprecated in favour of using background
16		* update see synapse/storage/schema/delta/33/access_tokens_device_index.sql
17		* and synapse/storage/registration.py for an example using
18		* "access_tokens_device_index" **/
19		CREATE INDEX events_room_stream on events(room_id, stream_ordering);

+0

-20

~~synapse/storage/schema/delta/28/public_roms_index.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		/** Using CREATE INDEX directly is deprecated in favour of using background
16		* update see synapse/storage/schema/delta/33/access_tokens_device_index.sql
17		* and synapse/storage/registration.py for an example using
18		* "access_tokens_device_index" **/
19		CREATE INDEX public_room_index on rooms(is_public);

+0

-22

~~synapse/storage/schema/delta/28/receipts_user_id_index.sql~~ less more

0		/* Copyright 2015, 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		/** Using CREATE INDEX directly is deprecated in favour of using background
16		* update see synapse/storage/schema/delta/33/access_tokens_device_index.sql
17		* and synapse/storage/registration.py for an example using
18		* "access_tokens_device_index" **/
19		CREATE INDEX receipts_linearized_user ON receipts_linearized(
20		user_id
21		);

+0

-21

~~synapse/storage/schema/delta/28/upgrade_times.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		/*
16		* Stores the timestamp when a user upgraded from a guest to a full user, if
17		* that happened.
18		*/
19
20		ALTER TABLE users ADD COLUMN upgrade_ts BIGINT;

+0

-22

~~synapse/storage/schema/delta/28/users_is_guest.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		ALTER TABLE users ADD is_guest SMALLINT DEFAULT 0 NOT NULL;
16		/*
17		* NB: any guest users created between 27 and 28 will be incorrectly
18		* marked as not guests: we don't bother to fill these in correctly
19		* because guest access is not really complete in 27 anyway so it's
20		* very unlikley there will be any guest users created.
21		*/

+0

-35

~~synapse/storage/schema/delta/29/push_actions.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		ALTER TABLE event_push_actions ADD COLUMN topological_ordering BIGINT;
16		ALTER TABLE event_push_actions ADD COLUMN stream_ordering BIGINT;
17		ALTER TABLE event_push_actions ADD COLUMN notif SMALLINT;
18		ALTER TABLE event_push_actions ADD COLUMN highlight SMALLINT;
19
20		UPDATE event_push_actions SET stream_ordering = (
21		SELECT stream_ordering FROM events WHERE event_id = event_push_actions.event_id
22		), topological_ordering = (
23		SELECT topological_ordering FROM events WHERE event_id = event_push_actions.event_id
24		);
25
26		UPDATE event_push_actions SET notif = 1, highlight = 0;
27
28		/** Using CREATE INDEX directly is deprecated in favour of using background
29		* update see synapse/storage/schema/delta/33/access_tokens_device_index.sql
30		* and synapse/storage/registration.py for an example using
31		* "access_tokens_device_index" **/
32		CREATE INDEX event_push_actions_rm_tokens on event_push_actions(
33		user_id, room_id, topological_ordering, stream_ordering
34		);

+0

-16

~~synapse/storage/schema/delta/30/alias_creator.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		ALTER TABLE room_aliases ADD COLUMN creator TEXT;

+0

-69

~~synapse/storage/schema/delta/30/as_users.py~~ less more

0		# Copyright 2016 OpenMarket Ltd
1		#
2		# Licensed under the Apache License, Version 2.0 (the "License");
3		# you may not use this file except in compliance with the License.
4		# You may obtain a copy of the License at
5		#
6		# http://www.apache.org/licenses/LICENSE-2.0
7		#
8		# Unless required by applicable law or agreed to in writing, software
9		# distributed under the License is distributed on an "AS IS" BASIS,
10		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		# See the License for the specific language governing permissions and
12		# limitations under the License.
13		import logging
14
15		from six.moves import range
16
17		from synapse.config.appservice import load_appservices
18
19		logger = logging.getLogger(__name__)
20
21
22		def run_create(cur, database_engine, args, *kwargs):
23		# NULL indicates user was not registered by an appservice.
24		try:
25		cur.execute("ALTER TABLE users ADD COLUMN appservice_id TEXT")
26		except Exception:
27		# Maybe we already added the column? Hope so...
28		pass
29
30
31		def run_upgrade(cur, database_engine, config, args, *kwargs):
32		cur.execute("SELECT name FROM users")
33		rows = cur.fetchall()
34
35		config_files = []
36		try:
37		config_files = config.app_service_config_files
38		except AttributeError:
39		logger.warning("Could not get app_service_config_files from config")
40		pass
41
42		appservices = load_appservices(config.server_name, config_files)
43
44		owned = {}
45
46		for row in rows:
47		user_id = row[0]
48		for appservice in appservices:
49		if appservice.is_exclusive_user(user_id):
50		if user_id in owned.keys():
51		logger.error(
52		"user_id %s was owned by more than one application"
53		" service (IDs %s and %s); assigning arbitrarily to %s"
54		% (user_id, owned[user_id], appservice.id, owned[user_id])
55		)
56		owned.setdefault(appservice.id, []).append(user_id)
57
58		for as_id, user_ids in owned.items():
59		n = 100
60		user_chunks = (user_ids[i : i + 100] for i in range(0, len(user_ids), n))
61		for chunk in user_chunks:
62		cur.execute(
63		database_engine.convert_param_style(
64		"UPDATE users SET appservice_id = ? WHERE name IN (%s)"
65		% (",".join("?" for _ in chunk),)
66		),
67		[as_id] + chunk,
68		)

+0

-25

~~synapse/storage/schema/delta/30/deleted_pushers.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		CREATE TABLE IF NOT EXISTS deleted_pushers(
16		stream_id BIGINT NOT NULL,
17		app_id TEXT NOT NULL,
18		pushkey TEXT NOT NULL,
19		user_id TEXT NOT NULL,
20		/* We only track the most recent delete for each app_id, pushkey and user_id. */
21		UNIQUE (app_id, pushkey, user_id)
22		);
23
24		CREATE INDEX deleted_pushers_stream_id ON deleted_pushers (stream_id);

+0

-30

~~synapse/storage/schema/delta/30/presence_stream.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15
16		CREATE TABLE presence_stream(
17		stream_id BIGINT,
18		user_id TEXT,
19		state TEXT,
20		last_active_ts BIGINT,
21		last_federation_update_ts BIGINT,
22		last_user_sync_ts BIGINT,
23		status_msg TEXT,
24		currently_active BOOLEAN
25		);
26
27		CREATE INDEX presence_stream_id ON presence_stream(stream_id, user_id);
28		CREATE INDEX presence_stream_user_id ON presence_stream(user_id);
29		CREATE INDEX presence_stream_state ON presence_stream(state);

+0

-23

~~synapse/storage/schema/delta/30/public_rooms.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15
16		/* This release removes the restriction that published rooms must have an alias,
17		* so we go back and ensure the only 'public' rooms are ones with an alias.
18		* We use (1 = 0) and (1 = 1) so that it works in both postgres and sqlite
19		*/
20		UPDATE rooms SET is_public = (1 = 0) WHERE is_public = (1 = 1) AND room_id not in (
21		SELECT room_id FROM room_aliases
22		);

+0

-38

~~synapse/storage/schema/delta/30/push_rule_stream.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15
16
17		CREATE TABLE push_rules_stream(
18		stream_id BIGINT NOT NULL,
19		event_stream_ordering BIGINT NOT NULL,
20		user_id TEXT NOT NULL,
21		rule_id TEXT NOT NULL,
22		op TEXT NOT NULL, -- One of "ENABLE", "DISABLE", "ACTIONS", "ADD", "DELETE"
23		priority_class SMALLINT,
24		priority INTEGER,
25		conditions TEXT,
26		actions TEXT
27		);
28
29		-- The extra data for each operation is:
30		-- * ENABLE, DISABLE, DELETE: []
31		-- * ACTIONS: ["actions"]
32		-- * ADD: ["priority_class", "priority", "actions", "conditions"]
33
34		-- Index for replication queries.
35		CREATE INDEX push_rules_stream_id ON push_rules_stream(stream_id);
36		-- Index for /sync queries.
37		CREATE INDEX push_rules_stream_user_stream_id on push_rules_stream(user_id, stream_id);

+0

-33

~~synapse/storage/schema/delta/30/state_stream.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15
16		/* We used to create a table called current_state_resets, but this is no
17		* longer used and is removed in delta 54.
18		*/
19
20		/* The outlier events that have aquired a state group typically through
21		* backfill. This is tracked separately to the events table, as assigning a
22		* state group change the position of the existing event in the stream
23		* ordering.
24		* However since a stream_ordering is assigned in persist_event for the
25		* (event, state) pair, we can use that stream_ordering to identify when
26		* the new state was assigned for the event.
27		*/
28		CREATE TABLE IF NOT EXISTS ex_outlier_stream(
29		event_stream_ordering BIGINT PRIMARY KEY NOT NULL,
30		event_id TEXT NOT NULL,
31		state_group BIGINT NOT NULL
32		);

+0

-24

~~synapse/storage/schema/delta/30/threepid_guest_access_tokens.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- Stores guest account access tokens generated for unbound 3pids.
16		CREATE TABLE threepid_guest_access_tokens(
17		medium TEXT, -- The medium of the 3pid. Must be "email".
18		address TEXT, -- The 3pid address.
19		guest_access_token TEXT, -- The access token for a guest user for this 3pid.
20		first_inviter TEXT -- User ID of the first user to invite this 3pid to a room.
21		);
22
23		CREATE UNIQUE INDEX threepid_guest_access_tokens_index ON threepid_guest_access_tokens(medium, address);

+0

-42

~~synapse/storage/schema/delta/31/invites.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15
16		CREATE TABLE local_invites(
17		stream_id BIGINT NOT NULL,
18		inviter TEXT NOT NULL,
19		invitee TEXT NOT NULL,
20		event_id TEXT NOT NULL,
21		room_id TEXT NOT NULL,
22		locally_rejected TEXT,
23		replaced_by TEXT
24		);
25
26		-- Insert all invites for local users into new `invites` table
27		INSERT INTO local_invites SELECT
28		stream_ordering as stream_id,
29		sender as inviter,
30		state_key as invitee,
31		event_id,
32		room_id,
33		NULL as locally_rejected,
34		NULL as replaced_by
35		FROM events
36		NATURAL JOIN current_state_events
37		NATURAL JOIN room_memberships
38		WHERE membership = 'invite' AND state_key IN (SELECT name FROM users);
39
40		CREATE INDEX local_invites_id ON local_invites(stream_id);
41		CREATE INDEX local_invites_for_user_idx ON local_invites(invitee, locally_rejected, replaced_by, room_id);

+0

-27

~~synapse/storage/schema/delta/31/local_media_repository_url_cache.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		CREATE TABLE local_media_repository_url_cache(
16		url TEXT, -- the URL being cached
17		response_code INTEGER, -- the HTTP response code of this download attempt
18		etag TEXT, -- the etag header of this response
19		expires INTEGER, -- the number of ms this response was valid for
20		og TEXT, -- cache of the OG metadata of this URL as JSON
21		media_id TEXT, -- the media_id, if any, of the URL's content in the repo
22		download_ts BIGINT -- the timestamp of this download attempt
23		);
24
25		CREATE INDEX local_media_repository_url_cache_by_url_download_ts
26		ON local_media_repository_url_cache(url, download_ts);

+0

-87

~~synapse/storage/schema/delta/31/pushers.py~~ less more

0		# Copyright 2016 OpenMarket Ltd
1		#
2		# Licensed under the Apache License, Version 2.0 (the "License");
3		# you may not use this file except in compliance with the License.
4		# You may obtain a copy of the License at
5		#
6		# http://www.apache.org/licenses/LICENSE-2.0
7		#
8		# Unless required by applicable law or agreed to in writing, software
9		# distributed under the License is distributed on an "AS IS" BASIS,
10		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		# See the License for the specific language governing permissions and
12		# limitations under the License.
13
14
15		# Change the last_token to last_stream_ordering now that pushers no longer
16		# listen on an event stream but instead select out of the event_push_actions
17		# table.
18
19
20		import logging
21
22		logger = logging.getLogger(__name__)
23
24
25		def token_to_stream_ordering(token):
26		return int(token[1:].split("_")[0])
27
28
29		def run_create(cur, database_engine, args, *kwargs):
30		logger.info("Porting pushers table, delta 31...")
31		cur.execute(
32		"""
33		CREATE TABLE IF NOT EXISTS pushers2 (
34		id BIGINT PRIMARY KEY,
35		user_name TEXT NOT NULL,
36		access_token BIGINT DEFAULT NULL,
37		profile_tag VARCHAR(32) NOT NULL,
38		kind VARCHAR(8) NOT NULL,
39		app_id VARCHAR(64) NOT NULL,
40		app_display_name VARCHAR(64) NOT NULL,
41		device_display_name VARCHAR(128) NOT NULL,
42		pushkey TEXT NOT NULL,
43		ts BIGINT NOT NULL,
44		lang VARCHAR(8),
45		data TEXT,
46		last_stream_ordering INTEGER,
47		last_success BIGINT,
48		failing_since BIGINT,
49		UNIQUE (app_id, pushkey, user_name)
50		)
51		"""
52		)
53		cur.execute(
54		"""SELECT
55		id, user_name, access_token, profile_tag, kind,
56		app_id, app_display_name, device_display_name,
57		pushkey, ts, lang, data, last_token, last_success,
58		failing_since
59		FROM pushers
60		"""
61		)
62		count = 0
63		for row in cur.fetchall():
64		row = list(row)
65		row[12] = token_to_stream_ordering(row[12])
66		cur.execute(
67		database_engine.convert_param_style(
68		"""
69		INSERT into pushers2 (
70		id, user_name, access_token, profile_tag, kind,
71		app_id, app_display_name, device_display_name,
72		pushkey, ts, lang, data, last_stream_ordering, last_success,
73		failing_since
74		) values (%s)"""
75		% (",".join(["?" for _ in range(len(row))]))
76		),
77		row,
78		)
79		count += 1
80		cur.execute("DROP TABLE pushers")
81		cur.execute("ALTER TABLE pushers2 RENAME TO pushers")
82		logger.info("Moved %d pushers to new table", count)
83
84
85		def run_upgrade(cur, database_engine, args, *kwargs):
86		pass

+0

-22

~~synapse/storage/schema/delta/31/pushers_index.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		/** Using CREATE INDEX directly is deprecated in favour of using background
16		* update see synapse/storage/schema/delta/33/access_tokens_device_index.sql
17		* and synapse/storage/registration.py for an example using
18		* "access_tokens_device_index" **/
19		CREATE INDEX event_push_actions_stream_ordering on event_push_actions(
20		stream_ordering, user_id
21		);

+0

-66

~~synapse/storage/schema/delta/31/search_update.py~~ less more

0		# Copyright 2016 OpenMarket Ltd
1		#
2		# Licensed under the Apache License, Version 2.0 (the "License");
3		# you may not use this file except in compliance with the License.
4		# You may obtain a copy of the License at
5		#
6		# http://www.apache.org/licenses/LICENSE-2.0
7		#
8		# Unless required by applicable law or agreed to in writing, software
9		# distributed under the License is distributed on an "AS IS" BASIS,
10		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		# See the License for the specific language governing permissions and
12		# limitations under the License.
13
14		import logging
15
16		import simplejson
17
18		from synapse.storage.engines import PostgresEngine
19		from synapse.storage.prepare_database import get_statements
20
21		logger = logging.getLogger(__name__)
22
23
24		ALTER_TABLE = """
25		ALTER TABLE event_search ADD COLUMN origin_server_ts BIGINT;
26		ALTER TABLE event_search ADD COLUMN stream_ordering BIGINT;
27		"""
28
29
30		def run_create(cur, database_engine, args, *kwargs):
31		if not isinstance(database_engine, PostgresEngine):
32		return
33
34		for statement in get_statements(ALTER_TABLE.splitlines()):
35		cur.execute(statement)
36
37		cur.execute("SELECT MIN(stream_ordering) FROM events")
38		rows = cur.fetchall()
39		min_stream_id = rows[0][0]
40
41		cur.execute("SELECT MAX(stream_ordering) FROM events")
42		rows = cur.fetchall()
43		max_stream_id = rows[0][0]
44
45		if min_stream_id is not None and max_stream_id is not None:
46		progress = {
47		"target_min_stream_id_inclusive": min_stream_id,
48		"max_stream_id_exclusive": max_stream_id + 1,
49		"rows_inserted": 0,
50		"have_added_indexes": False,
51		}
52		progress_json = simplejson.dumps(progress)
53
54		sql = (
55		"INSERT into background_updates (update_name, progress_json)"
56		" VALUES (?, ?)"
57		)
58
59		sql = database_engine.convert_param_style(sql)
60
61		cur.execute(sql, ("event_search_order", progress_json))
62
63
64		def run_upgrade(cur, database_engine, args, *kwargs):
65		pass

+0

-16

~~synapse/storage/schema/delta/32/events.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		ALTER TABLE events ADD COLUMN received_ts BIGINT;

+0

-9

~~synapse/storage/schema/delta/32/openid.sql~~ less more

0
1		CREATE TABLE open_id_tokens (
2		token TEXT NOT NULL PRIMARY KEY,
3		ts_valid_until_ms bigint NOT NULL,
4		user_id TEXT NOT NULL,
5		UNIQUE (token)
6		);
7
8		CREATE index open_id_tokens_ts_valid_until_ms ON open_id_tokens(ts_valid_until_ms);

+0

-23

~~synapse/storage/schema/delta/32/pusher_throttle.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15
16		CREATE TABLE pusher_throttle(
17		pusher BIGINT NOT NULL,
18		room_id TEXT NOT NULL,
19		last_sent_ts BIGINT,
20		throttle_ms BIGINT,
21		PRIMARY KEY (pusher, room_id)
22		);

+0

-34

~~synapse/storage/schema/delta/32/remove_indices.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15
16		-- The following indices are redundant, other indices are equivalent or
17		-- supersets
18		DROP INDEX IF EXISTS events_room_id; -- Prefix of events_room_stream
19		DROP INDEX IF EXISTS events_order; -- Prefix of events_order_topo_stream_room
20		DROP INDEX IF EXISTS events_topological_ordering; -- Prefix of events_order_topo_stream_room
21		DROP INDEX IF EXISTS events_stream_ordering; -- Duplicate of PRIMARY KEY
22		DROP INDEX IF EXISTS state_groups_id; -- Duplicate of PRIMARY KEY
23		DROP INDEX IF EXISTS event_to_state_groups_id; -- Duplicate of PRIMARY KEY
24		DROP INDEX IF EXISTS event_push_actions_room_id_event_id_user_id_profile_tag; -- Duplicate of UNIQUE CONSTRAINT
25
26		DROP INDEX IF EXISTS st_extrem_id; -- Prefix of UNIQUE CONSTRAINT
27		DROP INDEX IF EXISTS event_signatures_id; -- Prefix of UNIQUE CONSTRAINT
28		DROP INDEX IF EXISTS redactions_event_id; -- Duplicate of UNIQUE CONSTRAINT
29
30		-- The following indices were unused
31		DROP INDEX IF EXISTS remote_media_cache_thumbnails_media_id;
32		DROP INDEX IF EXISTS evauth_edges_auth_id;
33		DROP INDEX IF EXISTS presence_stream_state;

+0

-25

~~synapse/storage/schema/delta/32/reports.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15
16		CREATE TABLE event_reports(
17		id BIGINT NOT NULL PRIMARY KEY,
18		received_ts BIGINT NOT NULL,
19		room_id TEXT NOT NULL,
20		event_id TEXT NOT NULL,
21		user_id TEXT NOT NULL,
22		reason TEXT,
23		content TEXT
24		);

+0

-17

~~synapse/storage/schema/delta/33/access_tokens_device_index.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		INSERT INTO background_updates (update_name, progress_json) VALUES
16		('access_tokens_device_index', '{}');

+0

-21

~~synapse/storage/schema/delta/33/devices.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		CREATE TABLE devices (
16		user_id TEXT NOT NULL,
17		device_id TEXT NOT NULL,
18		display_name TEXT,
19		CONSTRAINT device_uniqueness UNIQUE (user_id, device_id)
20		);

+0

-19

~~synapse/storage/schema/delta/33/devices_for_e2e_keys.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- make sure that we have a device record for each set of E2E keys, so that the
16		-- user can delete them if they like.
17		INSERT INTO devices
18		SELECT user_id, device_id, NULL FROM e2e_device_keys_json;

+0

-20

~~synapse/storage/schema/delta/33/devices_for_e2e_keys_clear_unknown_device.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- a previous version of the "devices_for_e2e_keys" delta set all the device
16		-- names to "unknown device". This wasn't terribly helpful
17		UPDATE devices
18		SET display_name = NULL
19		WHERE display_name = 'unknown device';

+0

-61

~~synapse/storage/schema/delta/33/event_fields.py~~ less more

0		# Copyright 2016 OpenMarket Ltd
1		#
2		# Licensed under the Apache License, Version 2.0 (the "License");
3		# you may not use this file except in compliance with the License.
4		# You may obtain a copy of the License at
5		#
6		# http://www.apache.org/licenses/LICENSE-2.0
7		#
8		# Unless required by applicable law or agreed to in writing, software
9		# distributed under the License is distributed on an "AS IS" BASIS,
10		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		# See the License for the specific language governing permissions and
12		# limitations under the License.
13
14		import logging
15
16		import simplejson
17
18		from synapse.storage.prepare_database import get_statements
19
20		logger = logging.getLogger(__name__)
21
22
23		ALTER_TABLE = """
24		ALTER TABLE events ADD COLUMN sender TEXT;
25		ALTER TABLE events ADD COLUMN contains_url BOOLEAN;
26		"""
27
28
29		def run_create(cur, database_engine, args, *kwargs):
30		for statement in get_statements(ALTER_TABLE.splitlines()):
31		cur.execute(statement)
32
33		cur.execute("SELECT MIN(stream_ordering) FROM events")
34		rows = cur.fetchall()
35		min_stream_id = rows[0][0]
36
37		cur.execute("SELECT MAX(stream_ordering) FROM events")
38		rows = cur.fetchall()
39		max_stream_id = rows[0][0]
40
41		if min_stream_id is not None and max_stream_id is not None:
42		progress = {
43		"target_min_stream_id_inclusive": min_stream_id,
44		"max_stream_id_exclusive": max_stream_id + 1,
45		"rows_inserted": 0,
46		}
47		progress_json = simplejson.dumps(progress)
48
49		sql = (
50		"INSERT into background_updates (update_name, progress_json)"
51		" VALUES (?, ?)"
52		)
53
54		sql = database_engine.convert_param_style(sql)
55
56		cur.execute(sql, ("event_fields_sender_url", progress_json))
57
58
59		def run_upgrade(cur, database_engine, args, *kwargs):
60		pass

+0

-30

~~synapse/storage/schema/delta/33/remote_media_ts.py~~ less more

0		# Copyright 2016 OpenMarket Ltd
1		#
2		# Licensed under the Apache License, Version 2.0 (the "License");
3		# you may not use this file except in compliance with the License.
4		# You may obtain a copy of the License at
5		#
6		# http://www.apache.org/licenses/LICENSE-2.0
7		#
8		# Unless required by applicable law or agreed to in writing, software
9		# distributed under the License is distributed on an "AS IS" BASIS,
10		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		# See the License for the specific language governing permissions and
12		# limitations under the License.
13
14		import time
15
16		ALTER_TABLE = "ALTER TABLE remote_media_cache ADD COLUMN last_access_ts BIGINT"
17
18
19		def run_create(cur, database_engine, args, *kwargs):
20		cur.execute(ALTER_TABLE)
21
22
23		def run_upgrade(cur, database_engine, args, *kwargs):
24		cur.execute(
25		database_engine.convert_param_style(
26		"UPDATE remote_media_cache SET last_access_ts = ?"
27		),
28		(int(time.time() * 1000),),
29		)

+0

-17

~~synapse/storage/schema/delta/33/user_ips_index.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		INSERT INTO background_updates (update_name, progress_json) VALUES
16		('user_ips_device_index', '{}');

+0

-23

~~synapse/storage/schema/delta/34/appservice_stream.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		CREATE TABLE IF NOT EXISTS appservice_stream_position(
16		Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, -- Makes sure this table only has one row.
17		stream_ordering BIGINT,
18		CHECK (Lock='X')
19		);
20
21		INSERT INTO appservice_stream_position (stream_ordering)
22		SELECT COALESCE(MAX(stream_ordering), 0) FROM events;

+0

-46

~~synapse/storage/schema/delta/34/cache_stream.py~~ less more

0		# Copyright 2016 OpenMarket Ltd
1		#
2		# Licensed under the Apache License, Version 2.0 (the "License");
3		# you may not use this file except in compliance with the License.
4		# You may obtain a copy of the License at
5		#
6		# http://www.apache.org/licenses/LICENSE-2.0
7		#
8		# Unless required by applicable law or agreed to in writing, software
9		# distributed under the License is distributed on an "AS IS" BASIS,
10		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		# See the License for the specific language governing permissions and
12		# limitations under the License.
13
14		import logging
15
16		from synapse.storage.engines import PostgresEngine
17		from synapse.storage.prepare_database import get_statements
18
19		logger = logging.getLogger(__name__)
20
21
22		# This stream is used to notify replication slaves that some caches have
23		# been invalidated that they cannot infer from the other streams.
24		CREATE_TABLE = """
25		CREATE TABLE cache_invalidation_stream (
26		stream_id BIGINT,
27		cache_func TEXT,
28		keys TEXT[],
29		invalidation_ts BIGINT
30		);
31
32		CREATE INDEX cache_invalidation_stream_id ON cache_invalidation_stream(stream_id);
33		"""
34
35
36		def run_create(cur, database_engine, args, *kwargs):
37		if not isinstance(database_engine, PostgresEngine):
38		return
39
40		for statement in get_statements(CREATE_TABLE.splitlines()):
41		cur.execute(statement)
42
43
44		def run_upgrade(cur, database_engine, args, *kwargs):
45		pass

+0

-24

~~synapse/storage/schema/delta/34/device_inbox.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		CREATE TABLE device_inbox (
16		user_id TEXT NOT NULL,
17		device_id TEXT NOT NULL,
18		stream_id BIGINT NOT NULL,
19		message_json TEXT NOT NULL -- {"type":, "sender":, "content",}
20		);
21
22		CREATE INDEX device_inbox_user_stream_id ON device_inbox(user_id, device_id, stream_id);
23		CREATE INDEX device_inbox_stream_id ON device_inbox(stream_id);

+0

-20

~~synapse/storage/schema/delta/34/push_display_name_rename.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		DELETE FROM push_rules WHERE rule_id = 'global/override/.m.rule.contains_display_name';
16		UPDATE push_rules SET rule_id = 'global/override/.m.rule.contains_display_name' WHERE rule_id = 'global/underride/.m.rule.contains_display_name';
17
18		DELETE FROM push_rules_enable WHERE rule_id = 'global/override/.m.rule.contains_display_name';
19		UPDATE push_rules_enable SET rule_id = 'global/override/.m.rule.contains_display_name' WHERE rule_id = 'global/underride/.m.rule.contains_display_name';

+0

-32

~~synapse/storage/schema/delta/34/received_txn_purge.py~~ less more

0		# Copyright 2016 OpenMarket Ltd
1		#
2		# Licensed under the Apache License, Version 2.0 (the "License");
3		# you may not use this file except in compliance with the License.
4		# You may obtain a copy of the License at
5		#
6		# http://www.apache.org/licenses/LICENSE-2.0
7		#
8		# Unless required by applicable law or agreed to in writing, software
9		# distributed under the License is distributed on an "AS IS" BASIS,
10		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		# See the License for the specific language governing permissions and
12		# limitations under the License.
13
14		import logging
15
16		from synapse.storage.engines import PostgresEngine
17
18		logger = logging.getLogger(__name__)
19
20
21		def run_create(cur, database_engine, args, *kwargs):
22		if isinstance(database_engine, PostgresEngine):
23		cur.execute("TRUNCATE received_transactions")
24		else:
25		cur.execute("DELETE FROM received_transactions")
26
27		cur.execute("CREATE INDEX received_transactions_ts ON received_transactions(ts)")
28
29
30		def run_upgrade(cur, database_engine, args, *kwargs):
31		pass

+17

-0

synapse/storage/schema/delta/35/00background_updates_add_col.sql less more

	0	/* Copyright 2016 OpenMarket Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15
	16	ALTER TABLE background_updates ADD COLUMN depends_on TEXT;

+0

-20

~~synapse/storage/schema/delta/35/add_state_index.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15
16		ALTER TABLE background_updates ADD COLUMN depends_on TEXT;
17
18		INSERT into background_updates (update_name, progress_json, depends_on)
19		VALUES ('state_group_state_type_index', '{}', 'state_group_state_deduplication');

+0

-17

~~synapse/storage/schema/delta/35/contains_url.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		INSERT into background_updates (update_name, progress_json)
16		VALUES ('event_contains_url_index', '{}');

+0

-39

~~synapse/storage/schema/delta/35/device_outbox.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		DROP TABLE IF EXISTS device_federation_outbox;
16		CREATE TABLE device_federation_outbox (
17		destination TEXT NOT NULL,
18		stream_id BIGINT NOT NULL,
19		queued_ts BIGINT NOT NULL,
20		messages_json TEXT NOT NULL
21		);
22
23
24		DROP INDEX IF EXISTS device_federation_outbox_destination_id;
25		CREATE INDEX device_federation_outbox_destination_id
26		ON device_federation_outbox(destination, stream_id);
27
28
29		DROP TABLE IF EXISTS device_federation_inbox;
30		CREATE TABLE device_federation_inbox (
31		origin TEXT NOT NULL,
32		message_id TEXT NOT NULL,
33		received_ts BIGINT NOT NULL
34		);
35
36		DROP INDEX IF EXISTS device_federation_inbox_sender_id;
37		CREATE INDEX device_federation_inbox_sender_id
38		ON device_federation_inbox(origin, message_id);

+0

-21

~~synapse/storage/schema/delta/35/device_stream_id.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		CREATE TABLE device_max_stream_id (
16		stream_id BIGINT NOT NULL
17		);
18
19		INSERT INTO device_max_stream_id (stream_id)
20		SELECT COALESCE(MAX(stream_id), 0) FROM device_inbox;

+0

-17

~~synapse/storage/schema/delta/35/event_push_actions_index.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		INSERT into background_updates (update_name, progress_json)
16		VALUES ('epa_highlight_index', '{}');

+0

-33

~~synapse/storage/schema/delta/35/public_room_list_change_stream.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15
16		CREATE TABLE public_room_list_stream (
17		stream_id BIGINT NOT NULL,
18		room_id TEXT NOT NULL,
19		visibility BOOLEAN NOT NULL
20		);
21
22		INSERT INTO public_room_list_stream (stream_id, room_id, visibility)
23		SELECT 1, room_id, is_public FROM rooms
24		WHERE is_public = CAST(1 AS BOOLEAN);
25
26		CREATE INDEX public_room_list_stream_idx on public_room_list_stream(
27		stream_id
28		);
29
30		CREATE INDEX public_room_list_stream_rm_idx on public_room_list_stream(
31		room_id, stream_id
32		);

+0

-22

~~synapse/storage/schema/delta/35/state.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		CREATE TABLE state_group_edges(
16		state_group BIGINT NOT NULL,
17		prev_state_group BIGINT NOT NULL
18		);
19
20		CREATE INDEX state_group_edges_idx ON state_group_edges(state_group);
21		CREATE INDEX state_group_edges_prev_idx ON state_group_edges(prev_state_group);

+0

-17

~~synapse/storage/schema/delta/35/state_dedupe.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		INSERT into background_updates (update_name, progress_json)
16		VALUES ('state_group_state_deduplication', '{}');

+0

-37

~~synapse/storage/schema/delta/35/stream_order_to_extrem.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15
16		CREATE TABLE stream_ordering_to_exterm (
17		stream_ordering BIGINT NOT NULL,
18		room_id TEXT NOT NULL,
19		event_id TEXT NOT NULL
20		);
21
22		INSERT INTO stream_ordering_to_exterm (stream_ordering, room_id, event_id)
23		SELECT stream_ordering, room_id, event_id FROM event_forward_extremities
24		INNER JOIN (
25		SELECT room_id, max(stream_ordering) as stream_ordering FROM events
26		INNER JOIN event_forward_extremities USING (room_id, event_id)
27		GROUP BY room_id
28		) AS rms USING (room_id);
29
30		CREATE INDEX stream_ordering_to_exterm_idx on stream_ordering_to_exterm(
31		stream_ordering
32		);
33
34		CREATE INDEX stream_ordering_to_exterm_rm_idx on stream_ordering_to_exterm(
35		room_id, stream_ordering
36		);

+0

-26

~~synapse/storage/schema/delta/36/readd_public_rooms.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- Re-add some entries to stream_ordering_to_exterm that were incorrectly deleted
16		INSERT INTO stream_ordering_to_exterm (stream_ordering, room_id, event_id)
17		SELECT
18		(SELECT stream_ordering FROM events where event_id = e.event_id) AS stream_ordering,
19		room_id,
20		event_id
21		FROM event_forward_extremities AS e
22		WHERE NOT EXISTS (
23		SELECT room_id FROM stream_ordering_to_exterm AS s
24		WHERE s.room_id = e.room_id
25		);

+0

-85

~~synapse/storage/schema/delta/37/remove_auth_idx.py~~ less more

0		# Copyright 2016 OpenMarket Ltd
1		#
2		# Licensed under the Apache License, Version 2.0 (the "License");
3		# you may not use this file except in compliance with the License.
4		# You may obtain a copy of the License at
5		#
6		# http://www.apache.org/licenses/LICENSE-2.0
7		#
8		# Unless required by applicable law or agreed to in writing, software
9		# distributed under the License is distributed on an "AS IS" BASIS,
10		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		# See the License for the specific language governing permissions and
12		# limitations under the License.
13
14		import logging
15
16		from synapse.storage.engines import PostgresEngine
17		from synapse.storage.prepare_database import get_statements
18
19		logger = logging.getLogger(__name__)
20
21		DROP_INDICES = """
22		-- We only ever query based on event_id
23		DROP INDEX IF EXISTS state_events_room_id;
24		DROP INDEX IF EXISTS state_events_type;
25		DROP INDEX IF EXISTS state_events_state_key;
26
27		-- room_id is indexed elsewhere
28		DROP INDEX IF EXISTS current_state_events_room_id;
29		DROP INDEX IF EXISTS current_state_events_state_key;
30		DROP INDEX IF EXISTS current_state_events_type;
31
32		DROP INDEX IF EXISTS transactions_have_ref;
33
34		-- (topological_ordering, stream_ordering, room_id) seems like a strange index,
35		-- and is used incredibly rarely.
36		DROP INDEX IF EXISTS events_order_topo_stream_room;
37
38		-- an equivalent index to this actually gets re-created in delta 41, because it
39		-- turned out that deleting it wasn't a great plan :/. In any case, let's
40		-- delete it here, and delta 41 will create a new one with an added UNIQUE
41		-- constraint
42		DROP INDEX IF EXISTS event_search_ev_idx;
43		"""
44
45		POSTGRES_DROP_CONSTRAINT = """
46		ALTER TABLE event_auth DROP CONSTRAINT IF EXISTS event_auth_event_id_auth_id_room_id_key;
47		"""
48
49		SQLITE_DROP_CONSTRAINT = """
50		DROP INDEX IF EXISTS evauth_edges_id;
51
52		CREATE TABLE IF NOT EXISTS event_auth_new(
53		event_id TEXT NOT NULL,
54		auth_id TEXT NOT NULL,
55		room_id TEXT NOT NULL
56		);
57
58		INSERT INTO event_auth_new
59		SELECT event_id, auth_id, room_id
60		FROM event_auth;
61
62		DROP TABLE event_auth;
63
64		ALTER TABLE event_auth_new RENAME TO event_auth;
65
66		CREATE INDEX evauth_edges_id ON event_auth(event_id);
67		"""
68
69
70		def run_create(cur, database_engine, args, *kwargs):
71		for statement in get_statements(DROP_INDICES.splitlines()):
72		cur.execute(statement)
73
74		if isinstance(database_engine, PostgresEngine):
75		drop_constraint = POSTGRES_DROP_CONSTRAINT
76		else:
77		drop_constraint = SQLITE_DROP_CONSTRAINT
78
79		for statement in get_statements(drop_constraint.splitlines()):
80		cur.execute(statement)
81
82
83		def run_upgrade(cur, database_engine, args, *kwargs):
84		pass

+0

-52

~~synapse/storage/schema/delta/37/user_threepids.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		/*
16		* Update any email addresses that were stored with mixed case into all
17		* lowercase
18		*/
19
20		-- There may be "duplicate" emails (with different case) already in the table,
21		-- so we find them and move all but the most recently used account.
22		UPDATE user_threepids
23		SET medium = 'email_old'
24		WHERE medium = 'email'
25		AND address IN (
26		-- We select all the addresses that are linked to the user_id that is NOT
27		-- the most recently created.
28		SELECT u.address
29		FROM
30		user_threepids AS u,
31		-- `duplicate_addresses` is a table of all the email addresses that
32		-- appear multiple times and when the binding was created
33		(
34		SELECT lower(u1.address) AS address, max(u1.added_at) AS max_ts
35		FROM user_threepids AS u1
36		INNER JOIN user_threepids AS u2 ON u1.medium = u2.medium AND lower(u1.address) = lower(u2.address) AND u1.address != u2.address
37		WHERE u1.medium = 'email' AND u2.medium = 'email'
38		GROUP BY lower(u1.address)
39		) AS duplicate_addresses
40		WHERE
41		lower(u.address) = duplicate_addresses.address
42		AND u.added_at != max_ts -- NOT the most recently created
43		);
44
45
46		-- This update is now safe since we've removed the duplicate addresses.
47		UPDATE user_threepids SET address = LOWER(address) WHERE medium = 'email';
48
49
50		/* Add an index for the select we do on passwored reset */
51		CREATE INDEX user_threepids_medium_address on user_threepids (medium, address);

+0

-19

~~synapse/storage/schema/delta/38/postgres_fts_gist.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- We no longer do this given we back it out again in schema 47
16
17		-- INSERT into background_updates (update_name, progress_json)
18		-- VALUES ('event_search_postgres_gist', '{}');

+0

-29

~~synapse/storage/schema/delta/39/appservice_room_list.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		CREATE TABLE appservice_room_list(
16		appservice_id TEXT NOT NULL,
17		network_id TEXT NOT NULL,
18		room_id TEXT NOT NULL
19		);
20
21		-- Each appservice can have multiple published room lists associated with them,
22		-- keyed of a particular network_id
23		CREATE UNIQUE INDEX appservice_room_list_idx ON appservice_room_list(
24		appservice_id, network_id, room_id
25		);
26
27		ALTER TABLE public_room_list_stream ADD COLUMN appservice_id TEXT;
28		ALTER TABLE public_room_list_stream ADD COLUMN network_id TEXT;

+0

-16

~~synapse/storage/schema/delta/39/device_federation_stream_idx.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		CREATE INDEX device_federation_outbox_id ON device_federation_outbox(stream_id);

+0

-17

~~synapse/storage/schema/delta/39/event_push_index.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		INSERT INTO background_updates (update_name, progress_json) VALUES
16		('event_push_actions_highlights_index', '{}');

+0

-22

~~synapse/storage/schema/delta/39/federation_out_position.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		CREATE TABLE federation_stream_position(
16		type TEXT NOT NULL,
17		stream_id INTEGER NOT NULL
18		);
19
20		INSERT INTO federation_stream_position (type, stream_id) VALUES ('federation', -1);
21		INSERT INTO federation_stream_position (type, stream_id) SELECT 'events', coalesce(max(stream_ordering), -1) FROM events;

+0

-20

~~synapse/storage/schema/delta/39/membership_profile.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		ALTER TABLE room_memberships ADD COLUMN display_name TEXT;
16		ALTER TABLE room_memberships ADD COLUMN avatar_url TEXT;
17
18		INSERT into background_updates (update_name, progress_json)
19		VALUES ('room_membership_profile_update', '{}');

+0

-17

~~synapse/storage/schema/delta/40/current_state_idx.sql~~ less more

0		/* Copyright 2017 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		INSERT INTO background_updates (update_name, progress_json) VALUES
16		('current_state_members_idx', '{}');

+0

-21

~~synapse/storage/schema/delta/40/device_inbox.sql~~ less more

0		/* Copyright 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- turn the pre-fill startup query into a index-only scan on postgresql.
16		INSERT into background_updates (update_name, progress_json)
17		VALUES ('device_inbox_stream_index', '{}');
18
19		INSERT into background_updates (update_name, progress_json, depends_on)
20		VALUES ('device_inbox_stream_drop', '{}', 'device_inbox_stream_index');

+0

-60

~~synapse/storage/schema/delta/40/device_list_streams.sql~~ less more

0		/* Copyright 2017 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- Cache of remote devices.
16		CREATE TABLE device_lists_remote_cache (
17		user_id TEXT NOT NULL,
18		device_id TEXT NOT NULL,
19		content TEXT NOT NULL
20		);
21
22		-- The last update we got for a user. Empty if we're not receiving updates for
23		-- that user.
24		CREATE TABLE device_lists_remote_extremeties (
25		user_id TEXT NOT NULL,
26		stream_id TEXT NOT NULL
27		);
28
29		-- we used to create non-unique indexes on these tables, but as of update 52 we create
30		-- unique indexes concurrently:
31		--
32		-- CREATE INDEX device_lists_remote_cache_id ON device_lists_remote_cache(user_id, device_id);
33		-- CREATE INDEX device_lists_remote_extremeties_id ON device_lists_remote_extremeties(user_id, stream_id);
34
35
36		-- Stream of device lists updates. Includes both local and remotes
37		CREATE TABLE device_lists_stream (
38		stream_id BIGINT NOT NULL,
39		user_id TEXT NOT NULL,
40		device_id TEXT NOT NULL
41		);
42
43		CREATE INDEX device_lists_stream_id ON device_lists_stream(stream_id, user_id);
44
45
46		-- The stream of updates to send to other servers. We keep at least one row
47		-- per user that was sent so that the prev_id for any new updates can be
48		-- calculated
49		CREATE TABLE device_lists_outbound_pokes (
50		destination TEXT NOT NULL,
51		stream_id BIGINT NOT NULL,
52		user_id TEXT NOT NULL,
53		device_id TEXT NOT NULL,
54		sent BOOLEAN NOT NULL,
55		ts BIGINT NOT NULL -- So that in future we can clear out pokes to dead servers
56		);
57
58		CREATE INDEX device_lists_outbound_pokes_id ON device_lists_outbound_pokes(destination, stream_id);
59		CREATE INDEX device_lists_outbound_pokes_user ON device_lists_outbound_pokes(destination, user_id);

+0

-37

~~synapse/storage/schema/delta/40/event_push_summary.sql~~ less more

0		/* Copyright 2017 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- Aggregate of old notification counts that have been deleted out of the
16		-- main event_push_actions table. This count does not include those that were
17		-- highlights, as they remain in the event_push_actions table.
18		CREATE TABLE event_push_summary (
19		user_id TEXT NOT NULL,
20		room_id TEXT NOT NULL,
21		notif_count BIGINT NOT NULL,
22		stream_ordering BIGINT NOT NULL
23		);
24
25		CREATE INDEX event_push_summary_user_rm ON event_push_summary(user_id, room_id);
26
27
28		-- The stream ordering up to which we have aggregated the event_push_actions
29		-- table into event_push_summary
30		CREATE TABLE event_push_summary_stream_ordering (
31		Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, -- Makes sure this table only has one row.
32		stream_ordering BIGINT NOT NULL,
33		CHECK (Lock='X')
34		);
35
36		INSERT INTO event_push_summary_stream_ordering (stream_ordering) VALUES (0);

+0

-39

~~synapse/storage/schema/delta/40/pushers.sql~~ less more

0		/* Copyright 2017 Vector Creations Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		CREATE TABLE IF NOT EXISTS pushers2 (
16		id BIGINT PRIMARY KEY,
17		user_name TEXT NOT NULL,
18		access_token BIGINT DEFAULT NULL,
19		profile_tag TEXT NOT NULL,
20		kind TEXT NOT NULL,
21		app_id TEXT NOT NULL,
22		app_display_name TEXT NOT NULL,
23		device_display_name TEXT NOT NULL,
24		pushkey TEXT NOT NULL,
25		ts BIGINT NOT NULL,
26		lang TEXT,
27		data TEXT,
28		last_stream_ordering INTEGER,
29		last_success BIGINT,
30		failing_since BIGINT,
31		UNIQUE (app_id, pushkey, user_name)
32		);
33
34		INSERT INTO pushers2 SELECT * FROM PUSHERS;
35
36		DROP TABLE PUSHERS;
37
38		ALTER TABLE pushers2 RENAME TO pushers;

+0

-17

~~synapse/storage/schema/delta/41/device_list_stream_idx.sql~~ less more

0		/* Copyright 2017 Vector Creations Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		INSERT into background_updates (update_name, progress_json)
16		VALUES ('device_lists_stream_idx', '{}');

+0

-16

~~synapse/storage/schema/delta/41/device_outbound_index.sql~~ less more

0		/* Copyright 2017 Vector Creations Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		CREATE INDEX device_lists_outbound_pokes_stream ON device_lists_outbound_pokes(stream_id);

+0

-17

~~synapse/storage/schema/delta/41/event_search_event_id_idx.sql~~ less more

0		/* Copyright 2017 Vector Creations Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		INSERT into background_updates (update_name, progress_json)
16		VALUES ('event_search_event_id_idx', '{}');

+0

-22

~~synapse/storage/schema/delta/41/ratelimit.sql~~ less more

0		/* Copyright 2017 Vector Creations Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		CREATE TABLE ratelimit_override (
16		user_id TEXT NOT NULL,
17		messages_per_second BIGINT,
18		burst_count BIGINT
19		);
20
21		CREATE UNIQUE INDEX ratelimit_override_idx ON ratelimit_override(user_id);

+0

-26

~~synapse/storage/schema/delta/42/current_state_delta.sql~~ less more

0		/* Copyright 2017 Vector Creations Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15
16		CREATE TABLE current_state_delta_stream (
17		stream_id BIGINT NOT NULL,
18		room_id TEXT NOT NULL,
19		type TEXT NOT NULL,
20		state_key TEXT NOT NULL,
21		event_id TEXT, -- Is null if the key was removed
22		prev_event_id TEXT -- Is null if the key was added
23		);
24
25		CREATE INDEX current_state_delta_stream_idx ON current_state_delta_stream(stream_id);

+0

-33

~~synapse/storage/schema/delta/42/device_list_last_id.sql~~ less more

0		/* Copyright 2017 Vector Creations Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15
16		-- Table of last stream_id that we sent to destination for user_id. This is
17		-- used to fill out the `prev_id` fields of outbound device list updates.
18		CREATE TABLE device_lists_outbound_last_success (
19		destination TEXT NOT NULL,
20		user_id TEXT NOT NULL,
21		stream_id BIGINT NOT NULL
22		);
23
24		INSERT INTO device_lists_outbound_last_success
25		SELECT destination, user_id, coalesce(max(stream_id), 0) as stream_id
26		FROM device_lists_outbound_pokes
27		WHERE sent = (1 = 1) -- sqlite doesn't have inbuilt boolean values
28		GROUP BY destination, user_id;
29
30		CREATE INDEX device_lists_outbound_last_success_idx ON device_lists_outbound_last_success(
31		destination, user_id, stream_id
32		);

+0

-17

~~synapse/storage/schema/delta/42/event_auth_state_only.sql~~ less more

0		/* Copyright 2017 Vector Creations Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		INSERT INTO background_updates (update_name, progress_json) VALUES
16		('event_auth_state_only', '{}');

+0

-84

~~synapse/storage/schema/delta/42/user_dir.py~~ less more

0		# Copyright 2017 Vector Creations Ltd
1		#
2		# Licensed under the Apache License, Version 2.0 (the "License");
3		# you may not use this file except in compliance with the License.
4		# You may obtain a copy of the License at
5		#
6		# http://www.apache.org/licenses/LICENSE-2.0
7		#
8		# Unless required by applicable law or agreed to in writing, software
9		# distributed under the License is distributed on an "AS IS" BASIS,
10		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		# See the License for the specific language governing permissions and
12		# limitations under the License.
13
14		import logging
15
16		from synapse.storage.engines import PostgresEngine, Sqlite3Engine
17		from synapse.storage.prepare_database import get_statements
18
19		logger = logging.getLogger(__name__)
20
21
22		BOTH_TABLES = """
23		CREATE TABLE user_directory_stream_pos (
24		Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, -- Makes sure this table only has one row.
25		stream_id BIGINT,
26		CHECK (Lock='X')
27		);
28
29		INSERT INTO user_directory_stream_pos (stream_id) VALUES (null);
30
31		CREATE TABLE user_directory (
32		user_id TEXT NOT NULL,
33		room_id TEXT NOT NULL, -- A room_id that we know the user is joined to
34		display_name TEXT,
35		avatar_url TEXT
36		);
37
38		CREATE INDEX user_directory_room_idx ON user_directory(room_id);
39		CREATE UNIQUE INDEX user_directory_user_idx ON user_directory(user_id);
40
41		CREATE TABLE users_in_pubic_room (
42		user_id TEXT NOT NULL,
43		room_id TEXT NOT NULL -- A room_id that we know is public
44		);
45
46		CREATE INDEX users_in_pubic_room_room_idx ON users_in_pubic_room(room_id);
47		CREATE UNIQUE INDEX users_in_pubic_room_user_idx ON users_in_pubic_room(user_id);
48		"""
49
50
51		POSTGRES_TABLE = """
52		CREATE TABLE user_directory_search (
53		user_id TEXT NOT NULL,
54		vector tsvector
55		);
56
57		CREATE INDEX user_directory_search_fts_idx ON user_directory_search USING gin(vector);
58		CREATE UNIQUE INDEX user_directory_search_user_idx ON user_directory_search(user_id);
59		"""
60
61
62		SQLITE_TABLE = """
63		CREATE VIRTUAL TABLE user_directory_search
64		USING fts4 ( user_id, value );
65		"""
66
67
68		def run_create(cur, database_engine, args, *kwargs):
69		for statement in get_statements(BOTH_TABLES.splitlines()):
70		cur.execute(statement)
71
72		if isinstance(database_engine, PostgresEngine):
73		for statement in get_statements(POSTGRES_TABLE.splitlines()):
74		cur.execute(statement)
75		elif isinstance(database_engine, Sqlite3Engine):
76		for statement in get_statements(SQLITE_TABLE.splitlines()):
77		cur.execute(statement)
78		else:
79		raise Exception("Unrecognized database engine")
80
81
82		def run_upgrade(args, *kwargs):
83		pass

+0

-21

~~synapse/storage/schema/delta/43/blocked_rooms.sql~~ less more

0		/* Copyright 2017 Vector Creations Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		CREATE TABLE blocked_rooms (
16		room_id TEXT NOT NULL,
17		user_id TEXT NOT NULL -- Admin who blocked the room
18		);
19
20		CREATE UNIQUE INDEX blocked_rooms_idx ON blocked_rooms(room_id);

+0

-17

~~synapse/storage/schema/delta/43/quarantine_media.sql~~ less more

0		/* Copyright 2017 Vector Creations Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		ALTER TABLE local_media_repository ADD COLUMN quarantined_by TEXT;
16		ALTER TABLE remote_media_cache ADD COLUMN quarantined_by TEXT;

+0

-16

~~synapse/storage/schema/delta/43/url_cache.sql~~ less more

0		/* Copyright 2017 Vector Creations Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		ALTER TABLE local_media_repository ADD COLUMN url_cache TEXT;

+0

-33

~~synapse/storage/schema/delta/43/user_share.sql~~ less more

0		/* Copyright 2017 Vector Creations Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- Table keeping track of who shares a room with who. We only keep track
16		-- of this for local users, so `user_id` is local users only (but we do keep track
17		-- of which remote users share a room)
18		CREATE TABLE users_who_share_rooms (
19		user_id TEXT NOT NULL,
20		other_user_id TEXT NOT NULL,
21		room_id TEXT NOT NULL,
22		share_private BOOLEAN NOT NULL -- is the shared room private? i.e. they share a private room
23		);
24
25
26		CREATE UNIQUE INDEX users_who_share_rooms_u_idx ON users_who_share_rooms(user_id, other_user_id);
27		CREATE INDEX users_who_share_rooms_r_idx ON users_who_share_rooms(room_id);
28		CREATE INDEX users_who_share_rooms_o_idx ON users_who_share_rooms(other_user_id);
29
30
31		-- Make sure that we populate the table initially
32		UPDATE user_directory_stream_pos SET stream_id = NULL;

+0

-41

~~synapse/storage/schema/delta/44/expire_url_cache.sql~~ less more

0		/* Copyright 2017 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- this didn't work on SQLite 3.7 (because of lack of partial indexes), so was
16		-- removed and replaced with 46/local_media_repository_url_idx.sql.
17		--
18		-- CREATE INDEX local_media_repository_url_idx ON local_media_repository(created_ts) WHERE url_cache IS NOT NULL;
19
20		-- we need to change `expires` to `expires_ts` so that we can index on it. SQLite doesn't support
21		-- indices on expressions until 3.9.
22		CREATE TABLE local_media_repository_url_cache_new(
23		url TEXT,
24		response_code INTEGER,
25		etag TEXT,
26		expires_ts BIGINT,
27		og TEXT,
28		media_id TEXT,
29		download_ts BIGINT
30		);
31
32		INSERT INTO local_media_repository_url_cache_new
33		SELECT url, response_code, etag, expires + download_ts, og, media_id, download_ts FROM local_media_repository_url_cache;
34
35		DROP TABLE local_media_repository_url_cache;
36		ALTER TABLE local_media_repository_url_cache_new RENAME TO local_media_repository_url_cache;
37
38		CREATE INDEX local_media_repository_url_cache_expires_idx ON local_media_repository_url_cache(expires_ts);
39		CREATE INDEX local_media_repository_url_cache_by_url_download_ts ON local_media_repository_url_cache(url, download_ts);
40		CREATE INDEX local_media_repository_url_cache_media_idx ON local_media_repository_url_cache(media_id);

+0

-167

~~synapse/storage/schema/delta/45/group_server.sql~~ less more

0		/* Copyright 2017 Vector Creations Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		CREATE TABLE groups (
16		group_id TEXT NOT NULL,
17		name TEXT, -- the display name of the room
18		avatar_url TEXT,
19		short_description TEXT,
20		long_description TEXT
21		);
22
23		CREATE UNIQUE INDEX groups_idx ON groups(group_id);
24
25
26		-- list of users the group server thinks are joined
27		CREATE TABLE group_users (
28		group_id TEXT NOT NULL,
29		user_id TEXT NOT NULL,
30		is_admin BOOLEAN NOT NULL,
31		is_public BOOLEAN NOT NULL -- whether the users membership can be seen by everyone
32		);
33
34
35		CREATE INDEX groups_users_g_idx ON group_users(group_id, user_id);
36		CREATE INDEX groups_users_u_idx ON group_users(user_id);
37
38		-- list of users the group server thinks are invited
39		CREATE TABLE group_invites (
40		group_id TEXT NOT NULL,
41		user_id TEXT NOT NULL
42		);
43
44		CREATE INDEX groups_invites_g_idx ON group_invites(group_id, user_id);
45		CREATE INDEX groups_invites_u_idx ON group_invites(user_id);
46
47
48		CREATE TABLE group_rooms (
49		group_id TEXT NOT NULL,
50		room_id TEXT NOT NULL,
51		is_public BOOLEAN NOT NULL -- whether the room can be seen by everyone
52		);
53
54		CREATE UNIQUE INDEX groups_rooms_g_idx ON group_rooms(group_id, room_id);
55		CREATE INDEX groups_rooms_r_idx ON group_rooms(room_id);
56
57
58		-- Rooms to include in the summary
59		CREATE TABLE group_summary_rooms (
60		group_id TEXT NOT NULL,
61		room_id TEXT NOT NULL,
62		category_id TEXT NOT NULL,
63		room_order BIGINT NOT NULL,
64		is_public BOOLEAN NOT NULL, -- whether the room should be show to everyone
65		UNIQUE (group_id, category_id, room_id, room_order),
66		CHECK (room_order > 0)
67		);
68
69		CREATE UNIQUE INDEX group_summary_rooms_g_idx ON group_summary_rooms(group_id, room_id, category_id);
70
71
72		-- Categories to include in the summary
73		CREATE TABLE group_summary_room_categories (
74		group_id TEXT NOT NULL,
75		category_id TEXT NOT NULL,
76		cat_order BIGINT NOT NULL,
77		UNIQUE (group_id, category_id, cat_order),
78		CHECK (cat_order > 0)
79		);
80
81		-- The categories in the group
82		CREATE TABLE group_room_categories (
83		group_id TEXT NOT NULL,
84		category_id TEXT NOT NULL,
85		profile TEXT NOT NULL,
86		is_public BOOLEAN NOT NULL, -- whether the category should be show to everyone
87		UNIQUE (group_id, category_id)
88		);
89
90		-- The users to include in the group summary
91		CREATE TABLE group_summary_users (
92		group_id TEXT NOT NULL,
93		user_id TEXT NOT NULL,
94		role_id TEXT NOT NULL,
95		user_order BIGINT NOT NULL,
96		is_public BOOLEAN NOT NULL -- whether the user should be show to everyone
97		);
98
99		CREATE INDEX group_summary_users_g_idx ON group_summary_users(group_id);
100
101		-- The roles to include in the group summary
102		CREATE TABLE group_summary_roles (
103		group_id TEXT NOT NULL,
104		role_id TEXT NOT NULL,
105		role_order BIGINT NOT NULL,
106		UNIQUE (group_id, role_id, role_order),
107		CHECK (role_order > 0)
108		);
109
110
111		-- The roles in a groups
112		CREATE TABLE group_roles (
113		group_id TEXT NOT NULL,
114		role_id TEXT NOT NULL,
115		profile TEXT NOT NULL,
116		is_public BOOLEAN NOT NULL, -- whether the role should be show to everyone
117		UNIQUE (group_id, role_id)
118		);
119
120
121		-- List of attestations we've given out and need to renew
122		CREATE TABLE group_attestations_renewals (
123		group_id TEXT NOT NULL,
124		user_id TEXT NOT NULL,
125		valid_until_ms BIGINT NOT NULL
126		);
127
128		CREATE INDEX group_attestations_renewals_g_idx ON group_attestations_renewals(group_id, user_id);
129		CREATE INDEX group_attestations_renewals_u_idx ON group_attestations_renewals(user_id);
130		CREATE INDEX group_attestations_renewals_v_idx ON group_attestations_renewals(valid_until_ms);
131
132
133		-- List of attestations we've received from remotes and are interested in.
134		CREATE TABLE group_attestations_remote (
135		group_id TEXT NOT NULL,
136		user_id TEXT NOT NULL,
137		valid_until_ms BIGINT NOT NULL,
138		attestation_json TEXT NOT NULL
139		);
140
141		CREATE INDEX group_attestations_remote_g_idx ON group_attestations_remote(group_id, user_id);
142		CREATE INDEX group_attestations_remote_u_idx ON group_attestations_remote(user_id);
143		CREATE INDEX group_attestations_remote_v_idx ON group_attestations_remote(valid_until_ms);
144
145
146		-- The group membership for the HS's users
147		CREATE TABLE local_group_membership (
148		group_id TEXT NOT NULL,
149		user_id TEXT NOT NULL,
150		is_admin BOOLEAN NOT NULL,
151		membership TEXT NOT NULL,
152		is_publicised BOOLEAN NOT NULL, -- if the user is publicising their membership
153		content TEXT NOT NULL
154		);
155
156		CREATE INDEX local_group_membership_u_idx ON local_group_membership(user_id, group_id);
157		CREATE INDEX local_group_membership_g_idx ON local_group_membership(group_id);
158
159
160		CREATE TABLE local_group_updates (
161		stream_id BIGINT NOT NULL,
162		group_id TEXT NOT NULL,
163		user_id TEXT NOT NULL,
164		type TEXT NOT NULL,
165		content TEXT NOT NULL
166		);

+0

-28

~~synapse/storage/schema/delta/45/profile_cache.sql~~ less more

0		/* Copyright 2017 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15
16		-- A subset of remote users whose profiles we have cached.
17		-- Whether a user is in this table or not is defined by the storage function
18		-- `is_subscribed_remote_profile_for_user`
19		CREATE TABLE remote_profile_cache (
20		user_id TEXT NOT NULL,
21		displayname TEXT,
22		avatar_url TEXT,
23		last_check BIGINT NOT NULL
24		);
25
26		CREATE UNIQUE INDEX remote_profile_cache_user_id ON remote_profile_cache(user_id);
27		CREATE INDEX remote_profile_cache_time ON remote_profile_cache(last_check);

+0

-17

~~synapse/storage/schema/delta/46/drop_refresh_tokens.sql~~ less more

0		/* Copyright 2017 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		/* we no longer use (or create) the refresh_tokens table */
16		DROP TABLE IF EXISTS refresh_tokens;

+0

-35

~~synapse/storage/schema/delta/46/drop_unique_deleted_pushers.sql~~ less more

0		/* Copyright 2017 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- drop the unique constraint on deleted_pushers so that we can just insert
16		-- into it rather than upserting.
17
18		CREATE TABLE deleted_pushers2 (
19		stream_id BIGINT NOT NULL,
20		app_id TEXT NOT NULL,
21		pushkey TEXT NOT NULL,
22		user_id TEXT NOT NULL
23		);
24
25		INSERT INTO deleted_pushers2 (stream_id, app_id, pushkey, user_id)
26		SELECT stream_id, app_id, pushkey, user_id from deleted_pushers;
27
28		DROP TABLE deleted_pushers;
29		ALTER TABLE deleted_pushers2 RENAME TO deleted_pushers;
30
31		-- create the index after doing the inserts because that's more efficient.
32		-- it also means we can give it the same name as the old one without renaming.
33		CREATE INDEX deleted_pushers_stream_id ON deleted_pushers (stream_id);
34

+0

-32

~~synapse/storage/schema/delta/46/group_server.sql~~ less more

0		/* Copyright 2017 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		CREATE TABLE groups_new (
16		group_id TEXT NOT NULL,
17		name TEXT, -- the display name of the room
18		avatar_url TEXT,
19		short_description TEXT,
20		long_description TEXT,
21		is_public BOOL NOT NULL -- whether non-members can access group APIs
22		);
23
24		-- NB: awful hack to get the default to be true on postgres and 1 on sqlite
25		INSERT INTO groups_new
26		SELECT group_id, name, avatar_url, short_description, long_description, (1=1) FROM groups;
27
28		DROP TABLE groups;
29		ALTER TABLE groups_new RENAME TO groups;
30
31		CREATE UNIQUE INDEX groups_idx ON groups(group_id);

+0

-24

~~synapse/storage/schema/delta/46/local_media_repository_url_idx.sql~~ less more

0		/* Copyright 2017 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- register a background update which will recreate the
16		-- local_media_repository_url_idx index.
17		--
18		-- We do this as a bg update not because it is a particularly onerous
19		-- operation, but because we'd like it to be a partial index if possible, and
20		-- the background_index_update code will understand whether we are on
21		-- postgres or sqlite and behave accordingly.
22		INSERT INTO background_updates (update_name, progress_json) VALUES
23		('local_media_repository_url_idx', '{}');

+0

-35

~~synapse/storage/schema/delta/46/user_dir_null_room_ids.sql~~ less more

0		/* Copyright 2017 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- change the user_directory table to also cover global local user profiles
16		-- rather than just profiles within specific rooms.
17
18		CREATE TABLE user_directory2 (
19		user_id TEXT NOT NULL,
20		room_id TEXT,
21		display_name TEXT,
22		avatar_url TEXT
23		);
24
25		INSERT INTO user_directory2(user_id, room_id, display_name, avatar_url)
26		SELECT user_id, room_id, display_name, avatar_url from user_directory;
27
28		DROP TABLE user_directory;
29		ALTER TABLE user_directory2 RENAME TO user_directory;
30
31		-- create indexes after doing the inserts because that's more efficient.
32		-- it also means we can give it the same name as the old one without renaming.
33		CREATE INDEX user_directory_room_idx ON user_directory(room_id);
34		CREATE UNIQUE INDEX user_directory_user_idx ON user_directory(user_id);

+0

-24

~~synapse/storage/schema/delta/46/user_dir_typos.sql~~ less more

0		/* Copyright 2017 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- this is just embarassing :\|
16		ALTER TABLE users_in_pubic_room RENAME TO users_in_public_rooms;
17
18		-- this is only 300K rows on matrix.org and takes ~3s to generate the index,
19		-- so is hopefully not going to block anyone else for that long...
20		CREATE INDEX users_in_public_rooms_room_idx ON users_in_public_rooms(room_id);
21		CREATE UNIQUE INDEX users_in_public_rooms_user_idx ON users_in_public_rooms(user_id);
22		DROP INDEX users_in_pubic_room_room_idx;
23		DROP INDEX users_in_pubic_room_user_idx;

+0

-16

~~synapse/storage/schema/delta/47/last_access_media.sql~~ less more

0		/* Copyright 2018 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		ALTER TABLE local_media_repository ADD COLUMN last_access_ts BIGINT;

+0

-17

~~synapse/storage/schema/delta/47/postgres_fts_gin.sql~~ less more

0		/* Copyright 2018 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		INSERT into background_updates (update_name, progress_json)
16		VALUES ('event_search_postgres_gin', '{}');

+0

-28

~~synapse/storage/schema/delta/47/push_actions_staging.sql~~ less more

0		/* Copyright 2018 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- Temporary staging area for push actions that have been calculated for an
16		-- event, but the event hasn't yet been persisted.
17		-- When the event is persisted the rows are moved over to the
18		-- event_push_actions table.
19		CREATE TABLE event_push_actions_staging (
20		event_id TEXT NOT NULL,
21		user_id TEXT NOT NULL,
22		actions TEXT NOT NULL,
23		notif SMALLINT NOT NULL,
24		highlight SMALLINT NOT NULL
25		);
26
27		CREATE INDEX event_push_actions_staging_id ON event_push_actions_staging(event_id);

+0

-34

~~synapse/storage/schema/delta/47/state_group_seq.py~~ less more

0		# Copyright 2018 New Vector Ltd
1		#
2		# Licensed under the Apache License, Version 2.0 (the "License");
3		# you may not use this file except in compliance with the License.
4		# You may obtain a copy of the License at
5		#
6		# http://www.apache.org/licenses/LICENSE-2.0
7		#
8		# Unless required by applicable law or agreed to in writing, software
9		# distributed under the License is distributed on an "AS IS" BASIS,
10		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		# See the License for the specific language governing permissions and
12		# limitations under the License.
13
14		from synapse.storage.engines import PostgresEngine
15
16
17		def run_create(cur, database_engine, args, *kwargs):
18		if isinstance(database_engine, PostgresEngine):
19		# if we already have some state groups, we want to start making new
20		# ones with a higher id.
21		cur.execute("SELECT max(id) FROM state_groups")
22		row = cur.fetchone()
23
24		if row[0] is None:
25		start_val = 1
26		else:
27		start_val = row[0] + 1
28
29		cur.execute("CREATE SEQUENCE state_group_id_seq START WITH %s", (start_val,))
30
31
32		def run_upgrade(args, *kwargs):
33		pass

+0

-18

~~synapse/storage/schema/delta/48/add_user_consent.sql~~ less more

0		/* Copyright 2018 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		/* record the version of the privacy policy the user has consented to
16		*/
17		ALTER TABLE users ADD COLUMN consent_version TEXT;

+0

-17

~~synapse/storage/schema/delta/48/add_user_ips_last_seen_index.sql~~ less more

0		/* Copyright 2018 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		INSERT into background_updates (update_name, progress_json)
16		VALUES ('user_ips_last_seen_index', '{}');

+0

-25

~~synapse/storage/schema/delta/48/deactivated_users.sql~~ less more

0		/* Copyright 2018 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		/*
16		* Store any accounts that have been requested to be deactivated.
17		* We part the account from all the rooms its in when its
18		* deactivated. This can take some time and synapse may be restarted
19		* before it completes, so store the user IDs here until the process
20		* is complete.
21		*/
22		CREATE TABLE users_pending_deactivation (
23		user_id TEXT NOT NULL
24		);

+0

-63

~~synapse/storage/schema/delta/48/group_unique_indexes.py~~ less more

0		# Copyright 2018 New Vector Ltd
1		#
2		# Licensed under the Apache License, Version 2.0 (the "License");
3		# you may not use this file except in compliance with the License.
4		# You may obtain a copy of the License at
5		#
6		# http://www.apache.org/licenses/LICENSE-2.0
7		#
8		# Unless required by applicable law or agreed to in writing, software
9		# distributed under the License is distributed on an "AS IS" BASIS,
10		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		# See the License for the specific language governing permissions and
12		# limitations under the License.
13
14		from synapse.storage.engines import PostgresEngine
15		from synapse.storage.prepare_database import get_statements
16
17		FIX_INDEXES = """
18		-- rebuild indexes as uniques
19		DROP INDEX groups_invites_g_idx;
20		CREATE UNIQUE INDEX group_invites_g_idx ON group_invites(group_id, user_id);
21		DROP INDEX groups_users_g_idx;
22		CREATE UNIQUE INDEX group_users_g_idx ON group_users(group_id, user_id);
23
24		-- rename other indexes to actually match their table names..
25		DROP INDEX groups_users_u_idx;
26		CREATE INDEX group_users_u_idx ON group_users(user_id);
27		DROP INDEX groups_invites_u_idx;
28		CREATE INDEX group_invites_u_idx ON group_invites(user_id);
29		DROP INDEX groups_rooms_g_idx;
30		CREATE UNIQUE INDEX group_rooms_g_idx ON group_rooms(group_id, room_id);
31		DROP INDEX groups_rooms_r_idx;
32		CREATE INDEX group_rooms_r_idx ON group_rooms(room_id);
33		"""
34
35
36		def run_create(cur, database_engine, args, *kwargs):
37		rowid = "ctid" if isinstance(database_engine, PostgresEngine) else "rowid"
38
39		# remove duplicates from group_users & group_invites tables
40		cur.execute(
41		"""
42		DELETE FROM group_users WHERE %s NOT IN (
43		SELECT min(%s) FROM group_users GROUP BY group_id, user_id
44		);
45		"""
46		% (rowid, rowid)
47		)
48		cur.execute(
49		"""
50		DELETE FROM group_invites WHERE %s NOT IN (
51		SELECT min(%s) FROM group_invites GROUP BY group_id, user_id
52		);
53		"""
54		% (rowid, rowid)
55		)
56
57		for statement in get_statements(FIX_INDEXES.splitlines()):
58		cur.execute(statement)
59
60
61		def run_upgrade(args, *kwargs):
62		pass

+0

-22

~~synapse/storage/schema/delta/48/groups_joinable.sql~~ less more

0		/* Copyright 2018 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		/*
16		* This isn't a real ENUM because sqlite doesn't support it
17		* and we use a default of NULL for inserted rows and interpret
18		* NULL at the python store level as necessary so that existing
19		* rows are given the correct default policy.
20		*/
21		ALTER TABLE groups ADD COLUMN join_policy TEXT NOT NULL DEFAULT 'invite';

+0

-20

~~synapse/storage/schema/delta/49/add_user_consent_server_notice_sent.sql~~ less more

0		/* Copyright 2018 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		/* record whether we have sent a server notice about consenting to the
16		* privacy policy. Specifically records the version of the policy we sent
17		* a message about.
18		*/
19		ALTER TABLE users ADD COLUMN consent_server_notice_sent TEXT;

+0

-21

~~synapse/storage/schema/delta/49/add_user_daily_visits.sql~~ less more

0		/* Copyright 2018 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15
16		CREATE TABLE user_daily_visits ( user_id TEXT NOT NULL,
17		device_id TEXT,
18		timestamp BIGINT NOT NULL );
19		CREATE INDEX user_daily_visits_uts_idx ON user_daily_visits(user_id, timestamp);
20		CREATE INDEX user_daily_visits_ts_idx ON user_daily_visits(timestamp);

+0

-17

~~synapse/storage/schema/delta/49/add_user_ips_last_seen_only_index.sql~~ less more

0		/* Copyright 2018 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		INSERT into background_updates (update_name, progress_json)
16		VALUES ('user_ips_last_seen_only_index', '{}');

+0

-19

~~synapse/storage/schema/delta/50/add_creation_ts_users_index.sql~~ less more

0		/* Copyright 2018 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15
16
17		INSERT into background_updates (update_name, progress_json)
18		VALUES ('users_creation_ts', '{}');

+0

-21

~~synapse/storage/schema/delta/50/erasure_store.sql~~ less more

0		/* Copyright 2018 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- a table of users who have requested that their details be erased
16		CREATE TABLE erased_users (
17		user_id TEXT NOT NULL
18		);
19
20		CREATE UNIQUE INDEX erased_users_user ON erased_users(user_id);

+0

-96

~~synapse/storage/schema/delta/50/make_event_content_nullable.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2018 New Vector Ltd
2		#
3		# Licensed under the Apache License, Version 2.0 (the "License");
4		# you may not use this file except in compliance with the License.
5		# You may obtain a copy of the License at
6		#
7		# http://www.apache.org/licenses/LICENSE-2.0
8		#
9		# Unless required by applicable law or agreed to in writing, software
10		# distributed under the License is distributed on an "AS IS" BASIS,
11		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12		# See the License for the specific language governing permissions and
13		# limitations under the License.
14
15		"""
16		We want to stop populating 'event.content', so we need to make it nullable.
17
18		If this has to be rolled back, then the following should populate the missing data:
19
20		Postgres:
21
22		UPDATE events SET content=(ej.json::json)->'content' FROM event_json ej
23		WHERE ej.event_id = events.event_id AND
24		stream_ordering < (
25		SELECT stream_ordering FROM events WHERE content IS NOT NULL
26		ORDER BY stream_ordering LIMIT 1
27		);
28
29		UPDATE events SET content=(ej.json::json)->'content' FROM event_json ej
30		WHERE ej.event_id = events.event_id AND
31		stream_ordering > (
32		SELECT stream_ordering FROM events WHERE content IS NOT NULL
33		ORDER BY stream_ordering DESC LIMIT 1
34		);
35
36		SQLite:
37
38		UPDATE events SET content=(
39		SELECT json_extract(json,'$.content') FROM event_json ej
40		WHERE ej.event_id = events.event_id
41		)
42		WHERE
43		stream_ordering < (
44		SELECT stream_ordering FROM events WHERE content IS NOT NULL
45		ORDER BY stream_ordering LIMIT 1
46		)
47		OR stream_ordering > (
48		SELECT stream_ordering FROM events WHERE content IS NOT NULL
49		ORDER BY stream_ordering DESC LIMIT 1
50		);
51
52		"""
53
54		import logging
55
56		from synapse.storage.engines import PostgresEngine
57
58		logger = logging.getLogger(__name__)
59
60
61		def run_create(cur, database_engine, args, *kwargs):
62		pass
63
64
65		def run_upgrade(cur, database_engine, args, *kwargs):
66		if isinstance(database_engine, PostgresEngine):
67		cur.execute(
68		"""
69		ALTER TABLE events ALTER COLUMN content DROP NOT NULL;
70		"""
71		)
72		return
73
74		# sqlite is an arse about this. ref: https://www.sqlite.org/lang_altertable.html
75
76		cur.execute(
77		"SELECT sql FROM sqlite_master WHERE tbl_name='events' AND type='table'"
78		)
79		(oldsql,) = cur.fetchone()
80
81		sql = oldsql.replace("content TEXT NOT NULL", "content TEXT")
82		if sql == oldsql:
83		raise Exception("Couldn't find null constraint to drop in %s" % oldsql)
84
85		logger.info("Replacing definition of 'events' with: %s", sql)
86
87		cur.execute("PRAGMA schema_version")
88		(oldver,) = cur.fetchone()
89		cur.execute("PRAGMA writable_schema=ON")
90		cur.execute(
91		"UPDATE sqlite_master SET sql=? WHERE tbl_name='events' AND type='table'",
92		(sql,),
93		)
94		cur.execute("PRAGMA schema_version=%i" % (oldver + 1,))
95		cur.execute("PRAGMA writable_schema=OFF")

+0

-39

~~synapse/storage/schema/delta/51/e2e_room_keys.sql~~ less more

0		/* Copyright 2017 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- users' optionally backed up encrypted e2e sessions
16		CREATE TABLE e2e_room_keys (
17		user_id TEXT NOT NULL,
18		room_id TEXT NOT NULL,
19		session_id TEXT NOT NULL,
20		version TEXT NOT NULL,
21		first_message_index INT,
22		forwarded_count INT,
23		is_verified BOOLEAN,
24		session_data TEXT NOT NULL
25		);
26
27		CREATE UNIQUE INDEX e2e_room_keys_idx ON e2e_room_keys(user_id, room_id, session_id);
28
29		-- the metadata for each generation of encrypted e2e session backups
30		CREATE TABLE e2e_room_keys_versions (
31		user_id TEXT NOT NULL,
32		version TEXT NOT NULL,
33		algorithm TEXT NOT NULL,
34		auth_data TEXT NOT NULL,
35		deleted SMALLINT DEFAULT 0 NOT NULL
36		);
37
38		CREATE UNIQUE INDEX e2e_room_keys_versions_idx ON e2e_room_keys_versions(user_id, version);

+0

-27

~~synapse/storage/schema/delta/51/monthly_active_users.sql~~ less more

0		/* Copyright 2018 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- a table of monthly active users, for use where blocking based on mau limits
16		CREATE TABLE monthly_active_users (
17		user_id TEXT NOT NULL,
18		-- Last time we saw the user. Not guaranteed to be accurate due to rate limiting
19		-- on updates, Granularity of updates governed by
20		-- synapse.storage.monthly_active_users.LAST_SEEN_GRANULARITY
21		-- Measured in ms since epoch.
22		timestamp BIGINT NOT NULL
23		);
24
25		CREATE UNIQUE INDEX monthly_active_users_users ON monthly_active_users(user_id);
26		CREATE INDEX monthly_active_users_time_stamp ON monthly_active_users(timestamp);

+0

-19

~~synapse/storage/schema/delta/52/add_event_to_state_group_index.sql~~ less more

0		/* Copyright 2018 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- This is needed to efficiently check for unreferenced state groups during
16		-- purge. Added events_to_state_group(state_group) index
17		INSERT into background_updates (update_name, progress_json)
18		VALUES ('event_to_state_groups_sg_index', '{}');

+0

-36

~~synapse/storage/schema/delta/52/device_list_streams_unique_idx.sql~~ less more

0		/* Copyright 2018 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- register a background update which will create a unique index on
16		-- device_lists_remote_cache
17		INSERT into background_updates (update_name, progress_json)
18		VALUES ('device_lists_remote_cache_unique_idx', '{}');
19
20		-- and one on device_lists_remote_extremeties
21		INSERT into background_updates (update_name, progress_json, depends_on)
22		VALUES (
23		'device_lists_remote_extremeties_unique_idx', '{}',
24
25		-- doesn't really depend on this, but we need to make sure both happen
26		-- before we drop the old indexes.
27		'device_lists_remote_cache_unique_idx'
28		);
29
30		-- once they complete, we can drop the old indexes.
31		INSERT into background_updates (update_name, progress_json, depends_on)
32		VALUES (
33		'drop_device_list_streams_non_unique_indexes', '{}',
34		'device_lists_remote_extremeties_unique_idx'
35		);

+0

-53

~~synapse/storage/schema/delta/52/e2e_room_keys.sql~~ less more

0		/* Copyright 2018 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		/* Change version column to an integer so we can do MAX() sensibly
16		*/
17		CREATE TABLE e2e_room_keys_versions_new (
18		user_id TEXT NOT NULL,
19		version BIGINT NOT NULL,
20		algorithm TEXT NOT NULL,
21		auth_data TEXT NOT NULL,
22		deleted SMALLINT DEFAULT 0 NOT NULL
23		);
24
25		INSERT INTO e2e_room_keys_versions_new
26		SELECT user_id, CAST(version as BIGINT), algorithm, auth_data, deleted FROM e2e_room_keys_versions;
27
28		DROP TABLE e2e_room_keys_versions;
29		ALTER TABLE e2e_room_keys_versions_new RENAME TO e2e_room_keys_versions;
30
31		CREATE UNIQUE INDEX e2e_room_keys_versions_idx ON e2e_room_keys_versions(user_id, version);
32
33		/* Change e2e_rooms_keys to match
34		*/
35		CREATE TABLE e2e_room_keys_new (
36		user_id TEXT NOT NULL,
37		room_id TEXT NOT NULL,
38		session_id TEXT NOT NULL,
39		version BIGINT NOT NULL,
40		first_message_index INT,
41		forwarded_count INT,
42		is_verified BOOLEAN,
43		session_data TEXT NOT NULL
44		);
45
46		INSERT INTO e2e_room_keys_new
47		SELECT user_id, room_id, session_id, CAST(version as BIGINT), first_message_index, forwarded_count, is_verified, session_data FROM e2e_room_keys;
48
49		DROP TABLE e2e_room_keys;
50		ALTER TABLE e2e_room_keys_new RENAME TO e2e_room_keys;
51
52		CREATE UNIQUE INDEX e2e_room_keys_idx ON e2e_room_keys(user_id, room_id, session_id);

+0

-19

~~synapse/storage/schema/delta/53/add_user_type_to_users.sql~~ less more

0		/* Copyright 2018 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		/* The type of the user: NULL for a regular user, or one of the constants in
16		* synapse.api.constants.UserTypes
17		*/
18		ALTER TABLE users ADD COLUMN user_type TEXT DEFAULT NULL;

+0

-16

~~synapse/storage/schema/delta/53/drop_sent_transactions.sql~~ less more

0		/* Copyright 2018 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		DROP TABLE IF EXISTS sent_transactions;

+0

-16

~~synapse/storage/schema/delta/53/event_format_version.sql~~ less more

0		/* Copyright 2019 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		ALTER TABLE event_json ADD COLUMN format_version INTEGER;

+0

-30

~~synapse/storage/schema/delta/53/user_dir_populate.sql~~ less more

0		/* Copyright 2019 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- Set up staging tables
16		INSERT INTO background_updates (update_name, progress_json) VALUES
17		('populate_user_directory_createtables', '{}');
18
19		-- Run through each room and update the user directory according to who is in it
20		INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES
21		('populate_user_directory_process_rooms', '{}', 'populate_user_directory_createtables');
22
23		-- Insert all users, if search_all_users is on
24		INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES
25		('populate_user_directory_process_users', '{}', 'populate_user_directory_process_rooms');
26
27		-- Clean up staging tables
28		INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES
29		('populate_user_directory_cleanup', '{}', 'populate_user_directory_process_users');

+0

-30

~~synapse/storage/schema/delta/53/user_ips_index.sql~~ less more

0		/* Copyright 2018 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- analyze user_ips, to help ensure the correct indices are used
16		INSERT INTO background_updates (update_name, progress_json) VALUES
17		('user_ips_analyze', '{}');
18
19		-- delete duplicates
20		INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES
21		('user_ips_remove_dupes', '{}', 'user_ips_analyze');
22
23		-- add a new unique index to user_ips table
24		INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES
25		('user_ips_device_unique_index', '{}', 'user_ips_remove_dupes');
26
27		-- drop the old original index
28		INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES
29		('user_ips_drop_nonunique_index', '{}', 'user_ips_device_unique_index');

+0

-44

~~synapse/storage/schema/delta/53/user_share.sql~~ less more

0		/* Copyright 2017 Vector Creations Ltd, 2019 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- Old disused version of the tables below.
16		DROP TABLE IF EXISTS users_who_share_rooms;
17
18		-- Tables keeping track of what users share rooms. This is a map of local users
19		-- to local or remote users, per room. Remote users cannot be in the user_id
20		-- column, only the other_user_id column. There are two tables, one for public
21		-- rooms and those for private rooms.
22		CREATE TABLE IF NOT EXISTS users_who_share_public_rooms (
23		user_id TEXT NOT NULL,
24		other_user_id TEXT NOT NULL,
25		room_id TEXT NOT NULL
26		);
27
28		CREATE TABLE IF NOT EXISTS users_who_share_private_rooms (
29		user_id TEXT NOT NULL,
30		other_user_id TEXT NOT NULL,
31		room_id TEXT NOT NULL
32		);
33
34		CREATE UNIQUE INDEX users_who_share_public_rooms_u_idx ON users_who_share_public_rooms(user_id, other_user_id, room_id);
35		CREATE INDEX users_who_share_public_rooms_r_idx ON users_who_share_public_rooms(room_id);
36		CREATE INDEX users_who_share_public_rooms_o_idx ON users_who_share_public_rooms(other_user_id);
37
38		CREATE UNIQUE INDEX users_who_share_private_rooms_u_idx ON users_who_share_private_rooms(user_id, other_user_id, room_id);
39		CREATE INDEX users_who_share_private_rooms_r_idx ON users_who_share_private_rooms(room_id);
40		CREATE INDEX users_who_share_private_rooms_o_idx ON users_who_share_private_rooms(other_user_id);
41
42		-- Make sure that we populate the tables initially by resetting the stream ID
43		UPDATE user_directory_stream_pos SET stream_id = NULL;

+0

-29

~~synapse/storage/schema/delta/53/user_threepid_id.sql~~ less more

0		/* Copyright 2019 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- Tracks which identity server a user bound their threepid via.
16		CREATE TABLE user_threepid_id_server (
17		user_id TEXT NOT NULL,
18		medium TEXT NOT NULL,
19		address TEXT NOT NULL,
20		id_server TEXT NOT NULL
21		);
22
23		CREATE UNIQUE INDEX user_threepid_id_server_idx ON user_threepid_id_server(
24		user_id, medium, address, id_server
25		);
26
27		INSERT INTO background_updates (update_name, progress_json) VALUES
28		('user_threepids_grandfather', '{}');

+0

-28

~~synapse/storage/schema/delta/53/users_in_public_rooms.sql~~ less more

0		/* Copyright 2019 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- We don't need the old version of this table.
16		DROP TABLE IF EXISTS users_in_public_rooms;
17
18		-- Old version of users_in_public_rooms
19		DROP TABLE IF EXISTS users_who_share_public_rooms;
20
21		-- Track what users are in public rooms.
22		CREATE TABLE IF NOT EXISTS users_in_public_rooms (
23		user_id TEXT NOT NULL,
24		room_id TEXT NOT NULL
25		);
26
27		CREATE UNIQUE INDEX users_in_public_rooms_u_idx ON users_in_public_rooms(user_id, room_id);

+0

-30

~~synapse/storage/schema/delta/54/account_validity_with_renewal.sql~~ less more

0		/* Copyright 2019 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- We previously changed the schema for this table without renaming the file, which means
16		-- that some databases might still be using the old schema. This ensures Synapse uses the
17		-- right schema for the table.
18		DROP TABLE IF EXISTS account_validity;
19
20		-- Track what users are in public rooms.
21		CREATE TABLE IF NOT EXISTS account_validity (
22		user_id TEXT PRIMARY KEY,
23		expiration_ts_ms BIGINT NOT NULL,
24		email_sent BOOLEAN NOT NULL,
25		renewal_token TEXT
26		);
27
28		CREATE INDEX account_validity_email_sent_idx ON account_validity(email_sent, expiration_ts_ms)
29		CREATE UNIQUE INDEX account_validity_renewal_string_idx ON account_validity(renewal_token)

+0

-23

~~synapse/storage/schema/delta/54/add_validity_to_server_keys.sql~~ less more

0		/* Copyright 2019 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		/* When we can use this key until, before we have to refresh it. */
16		ALTER TABLE server_signature_keys ADD COLUMN ts_valid_until_ms BIGINT;
17
18		UPDATE server_signature_keys SET ts_valid_until_ms = (
19		SELECT MAX(ts_valid_until_ms) FROM server_keys_json skj WHERE
20		skj.server_name = server_signature_keys.server_name AND
21		skj.key_id = server_signature_keys.key_id
22		);

+0

-23

~~synapse/storage/schema/delta/54/delete_forward_extremities.sql~~ less more

0		/* Copyright 2019 The Matrix.org Foundation C.I.C.
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- Start a background job to cleanup extremities that were incorrectly added
16		-- by bug #5269.
17		INSERT INTO background_updates (update_name, progress_json) VALUES
18		('delete_soft_failed_extremities', '{}');
19
20		DROP TABLE IF EXISTS _extremities_to_check; -- To make this delta schema file idempotent.
21		CREATE TABLE _extremities_to_check AS SELECT event_id FROM event_forward_extremities;
22		CREATE INDEX _extremities_to_check_id ON _extremities_to_check(event_id);

+0

-30

~~synapse/storage/schema/delta/54/drop_legacy_tables.sql~~ less more

0		/* Copyright 2019 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- we need to do this first due to foreign constraints
16		DROP TABLE IF EXISTS application_services_regex;
17
18		DROP TABLE IF EXISTS application_services;
19		DROP TABLE IF EXISTS transaction_id_to_pdu;
20		DROP TABLE IF EXISTS stats_reporting;
21		DROP TABLE IF EXISTS current_state_resets;
22		DROP TABLE IF EXISTS event_content_hashes;
23		DROP TABLE IF EXISTS event_destinations;
24		DROP TABLE IF EXISTS event_edge_hashes;
25		DROP TABLE IF EXISTS event_signatures;
26		DROP TABLE IF EXISTS feedback;
27		DROP TABLE IF EXISTS room_hosts;
28		DROP TABLE IF EXISTS server_tls_certificates;
29		DROP TABLE IF EXISTS state_forward_extremities;

+0

-16

~~synapse/storage/schema/delta/54/drop_presence_list.sql~~ less more

0		/* Copyright 2019 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		DROP TABLE IF EXISTS presence_list;

+0

-27

~~synapse/storage/schema/delta/54/relations.sql~~ less more

0		/* Copyright 2019 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- Tracks related events, like reactions, replies, edits, etc. Note that things
16		-- in this table are not necessarily "valid", e.g. it may contain edits from
17		-- people who don't have power to edit other peoples events.
18		CREATE TABLE IF NOT EXISTS event_relations (
19		event_id TEXT NOT NULL,
20		relates_to_id TEXT NOT NULL,
21		relation_type TEXT NOT NULL,
22		aggregation_key TEXT
23		);
24
25		CREATE UNIQUE INDEX event_relations_id ON event_relations(event_id);
26		CREATE INDEX event_relations_relates ON event_relations(relates_to_id, relation_type, aggregation_key);

+0

-80

~~synapse/storage/schema/delta/54/stats.sql~~ less more

0		/* Copyright 2018 New Vector Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		CREATE TABLE stats_stream_pos (
16		Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, -- Makes sure this table only has one row.
17		stream_id BIGINT,
18		CHECK (Lock='X')
19		);
20
21		INSERT INTO stats_stream_pos (stream_id) VALUES (null);
22
23		CREATE TABLE user_stats (
24		user_id TEXT NOT NULL,
25		ts BIGINT NOT NULL,
26		bucket_size INT NOT NULL,
27		public_rooms INT NOT NULL,
28		private_rooms INT NOT NULL
29		);
30
31		CREATE UNIQUE INDEX user_stats_user_ts ON user_stats(user_id, ts);
32
33		CREATE TABLE room_stats (
34		room_id TEXT NOT NULL,
35		ts BIGINT NOT NULL,
36		bucket_size INT NOT NULL,
37		current_state_events INT NOT NULL,
38		joined_members INT NOT NULL,
39		invited_members INT NOT NULL,
40		left_members INT NOT NULL,
41		banned_members INT NOT NULL,
42		state_events INT NOT NULL
43		);
44
45		CREATE UNIQUE INDEX room_stats_room_ts ON room_stats(room_id, ts);
46
47		-- cache of current room state; useful for the publicRooms list
48		CREATE TABLE room_state (
49		room_id TEXT NOT NULL,
50		join_rules TEXT,
51		history_visibility TEXT,
52		encryption TEXT,
53		name TEXT,
54		topic TEXT,
55		avatar TEXT,
56		canonical_alias TEXT
57		-- get aliases straight from the right table
58		);
59
60		CREATE UNIQUE INDEX room_state_room ON room_state(room_id);
61
62		CREATE TABLE room_stats_earliest_token (
63		room_id TEXT NOT NULL,
64		token BIGINT NOT NULL
65		);
66
67		CREATE UNIQUE INDEX room_stats_earliest_token_idx ON room_stats_earliest_token(room_id);
68
69		-- Set up staging tables
70		INSERT INTO background_updates (update_name, progress_json) VALUES
71		('populate_stats_createtables', '{}');
72
73		-- Run through each room and update stats
74		INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES
75		('populate_stats_process_rooms', '{}', 'populate_stats_createtables');
76
77		-- Clean up staging tables
78		INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES
79		('populate_stats_cleanup', '{}', 'populate_stats_process_rooms');

+0

-28

~~synapse/storage/schema/delta/54/stats2.sql~~ less more

0		/* Copyright 2019 The Matrix.org Foundation C.I.C.
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- This delta file gets run after `54/stats.sql` delta.
16
17		-- We want to add some indices to the temporary stats table, so we re-insert
18		-- 'populate_stats_createtables' if we are still processing the rooms update.
19		INSERT INTO background_updates (update_name, progress_json)
20		SELECT 'populate_stats_createtables', '{}'
21		WHERE
22		'populate_stats_process_rooms' IN (
23		SELECT update_name FROM background_updates
24		)
25		AND 'populate_stats_createtables' NOT IN ( -- don't insert if already exists
26		SELECT update_name FROM background_updates
27		);

+0

-18

~~synapse/storage/schema/delta/55/access_token_expiry.sql~~ less more

0		/* Copyright 2019 The Matrix.org Foundation C.I.C.
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- when this access token can be used until, in ms since the epoch. NULL means the token
16		-- never expires.
17		ALTER TABLE access_tokens ADD COLUMN valid_until_ms BIGINT;

+0

-31

~~synapse/storage/schema/delta/55/track_threepid_validations.sql~~ less more

0		/* Copyright 2019 The Matrix.org Foundation C.I.C.
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14		CREATE TABLE IF NOT EXISTS threepid_validation_session (
15		session_id TEXT PRIMARY KEY,
16		medium TEXT NOT NULL,
17		address TEXT NOT NULL,
18		client_secret TEXT NOT NULL,
19		last_send_attempt BIGINT NOT NULL,
20		validated_at BIGINT
21		);
22
23		CREATE TABLE IF NOT EXISTS threepid_validation_token (
24		token TEXT PRIMARY KEY,
25		session_id TEXT NOT NULL,
26		next_link TEXT,
27		expires BIGINT NOT NULL
28		);
29
30		CREATE INDEX threepid_validation_token_session_id ON threepid_validation_token(session_id);

+0

-19

~~synapse/storage/schema/delta/55/users_alter_deactivated.sql~~ less more

0		/* Copyright 2019 The Matrix.org Foundation C.I.C.
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		ALTER TABLE users ADD deactivated SMALLINT DEFAULT 0 NOT NULL;
16
17		INSERT INTO background_updates (update_name, progress_json) VALUES
18		('users_set_deactivated_flag', '{}');

+0

-20

~~synapse/storage/schema/delta/56/add_spans_to_device_lists.sql~~ less more

0		/* Copyright 2019 The Matrix.org Foundation C.I.C
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		/*
16		* Opentracing context data for inclusion in the device_list_update EDUs, as a
17		* json-encoded dictionary. NULL if opentracing is disabled (or not enabled for this destination).
18		*/
19		ALTER TABLE device_lists_outbound_pokes ADD opentracing_context TEXT;

+0

-22

~~synapse/storage/schema/delta/56/current_state_events_membership.sql~~ less more

0		/* Copyright 2019 The Matrix.org Foundation C.I.C.
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- We add membership to current state so that we don't need to join against
16		-- room_memberships, which can be surprisingly costly (we do such queries
17		-- very frequently).
18		-- This will be null for non-membership events and the content.membership key
19		-- for membership events. (Will also be null for membership events until the
20		-- background update job has finished).
21		ALTER TABLE current_state_events ADD membership TEXT;

+0

-24

~~synapse/storage/schema/delta/56/current_state_events_membership_mk2.sql~~ less more

0		/* Copyright 2019 The Matrix.org Foundation C.I.C.
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- We add membership to current state so that we don't need to join against
16		-- room_memberships, which can be surprisingly costly (we do such queries
17		-- very frequently).
18		-- This will be null for non-membership events and the content.membership key
19		-- for membership events. (Will also be null for membership events until the
20		-- background update job has finished).
21
22		INSERT INTO background_updates (update_name, progress_json) VALUES
23		('current_state_events_membership', '{}');

+0

-25

~~synapse/storage/schema/delta/56/destinations_failure_ts.sql~~ less more

0		/* Copyright 2019 The Matrix.org Foundation C.I.C
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		/*
16		* Record the timestamp when a given server started failing
17		*/
18		ALTER TABLE destinations ADD failure_ts BIGINT;
19
20		/* as a rough approximation, we assume that the server started failing at
21		* retry_interval before the last retry
22		*/
23		UPDATE destinations SET failure_ts = retry_last_ts - retry_interval
24		WHERE retry_last_ts > 0;

+0

-18

~~synapse/storage/schema/delta/56/destinations_retry_interval_type.sql.postgres~~ less more

0		/* Copyright 2019 The Matrix.org Foundation C.I.C
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- We want to store large retry intervals so we upgrade the column from INT
16		-- to BIGINT. We don't need to do this on SQLite.
17		ALTER TABLE destinations ALTER retry_interval SET DATA TYPE BIGINT;

+0

-24

~~synapse/storage/schema/delta/56/devices_last_seen.sql~~ less more

0		/* Copyright 2019 Matrix.org Foundation CIC
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- Track last seen information for a device in the devices table, rather
16		-- than relying on it being in the user_ips table (which we want to be able
17		-- to purge old entries from)
18		ALTER TABLE devices ADD COLUMN last_seen BIGINT;
19		ALTER TABLE devices ADD COLUMN ip TEXT;
20		ALTER TABLE devices ADD COLUMN user_agent TEXT;
21
22		INSERT INTO background_updates (update_name, progress_json) VALUES
23		('devices_last_seen', '{}');

+0

-18

~~synapse/storage/schema/delta/56/fix_room_keys_index.sql~~ less more

0		/* Copyright 2019 Matrix.org Foundation CIC
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- version is supposed to be part of the room keys index
16		CREATE UNIQUE INDEX e2e_room_keys_with_version_idx ON e2e_room_keys(user_id, version, room_id, session_id);
17		DROP INDEX IF EXISTS e2e_room_keys_idx;

+18

-0

synapse/storage/schema/delta/56/hidden_devices.sql less more

	0	/* Copyright 2019 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- device list needs to know which ones are "real" devices, and which ones are
	16	-- just used to avoid collisions
	17	ALTER TABLE devices ADD COLUMN hidden BOOLEAN DEFAULT FALSE;

+0

-17

~~synapse/storage/schema/delta/56/redaction_censor.sql~~ less more

0		/* Copyright 2019 The Matrix.org Foundation C.I.C.
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		ALTER TABLE redactions ADD COLUMN have_censored BOOL NOT NULL DEFAULT false;
16		CREATE INDEX redactions_have_censored ON redactions(event_id) WHERE not have_censored;

+0

-20

~~synapse/storage/schema/delta/56/redaction_censor2.sql~~ less more

0		/* Copyright 2019 The Matrix.org Foundation C.I.C.
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		ALTER TABLE redactions ADD COLUMN received_ts BIGINT;
16		CREATE INDEX redactions_have_censored_ts ON redactions(received_ts) WHERE not have_censored;
17
18		INSERT INTO background_updates (update_name, progress_json) VALUES
19		('redactions_received_ts', '{}');

+0

-18

~~synapse/storage/schema/delta/56/room_membership_idx.sql~~ less more

0		/* Copyright 2019 The Matrix.org Foundation C.I.C.
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- Adds an index on room_memberships for fetching all forgotten rooms for a user
16		INSERT INTO background_updates (update_name, progress_json) VALUES
17		('room_membership_forgotten_idx', '{}');

+55

-0

synapse/storage/schema/delta/56/signing_keys.sql less more

	0	/* Copyright 2019 New Vector Ltd
	1	*
	2	* Licensed under the Apache License, Version 2.0 (the "License");
	3	* you may not use this file except in compliance with the License.
	4	* You may obtain a copy of the License at
	5	*
	6	* http://www.apache.org/licenses/LICENSE-2.0
	7	*
	8	* Unless required by applicable law or agreed to in writing, software
	9	* distributed under the License is distributed on an "AS IS" BASIS,
	10	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	11	* See the License for the specific language governing permissions and
	12	* limitations under the License.
	13	*/
	14
	15	-- cross-signing keys
	16	CREATE TABLE IF NOT EXISTS e2e_cross_signing_keys (
	17	user_id TEXT NOT NULL,
	18	-- the type of cross-signing key (master, user_signing, or self_signing)
	19	keytype TEXT NOT NULL,
	20	-- the full key information, as a json-encoded dict
	21	keydata TEXT NOT NULL,
	22	-- for keeping the keys in order, so that we can fetch the latest one
	23	stream_id BIGINT NOT NULL
	24	);
	25
	26	CREATE UNIQUE INDEX e2e_cross_signing_keys_idx ON e2e_cross_signing_keys(user_id, keytype, stream_id);
	27
	28	-- cross-signing signatures
	29	CREATE TABLE IF NOT EXISTS e2e_cross_signing_signatures (
	30	-- user who did the signing
	31	user_id TEXT NOT NULL,
	32	-- key used to sign
	33	key_id TEXT NOT NULL,
	34	-- user who was signed
	35	target_user_id TEXT NOT NULL,
	36	-- device/key that was signed
	37	target_device_id TEXT NOT NULL,
	38	-- the actual signature
	39	signature TEXT NOT NULL
	40	);
	41
	42	CREATE UNIQUE INDEX e2e_cross_signing_signatures_idx ON e2e_cross_signing_signatures(user_id, target_user_id, target_device_id);
	43
	44	-- stream of user signature updates
	45	CREATE TABLE IF NOT EXISTS user_signature_stream (
	46	-- uses the same stream ID as device list stream
	47	stream_id BIGINT NOT NULL,
	48	-- user who did the signing
	49	from_user_id TEXT NOT NULL,
	50	-- list of users who were signed, as a JSON array
	51	user_ids TEXT NOT NULL
	52	);
	53
	54	CREATE UNIQUE INDEX user_signature_stream_idx ON user_signature_stream(stream_id);

+0

-152

~~synapse/storage/schema/delta/56/stats_separated.sql~~ less more

0		/* Copyright 2018 New Vector Ltd
1		* Copyright 2019 The Matrix.org Foundation C.I.C.
2		*
3		* Licensed under the Apache License, Version 2.0 (the "License");
4		* you may not use this file except in compliance with the License.
5		* You may obtain a copy of the License at
6		*
7		* http://www.apache.org/licenses/LICENSE-2.0
8		*
9		* Unless required by applicable law or agreed to in writing, software
10		* distributed under the License is distributed on an "AS IS" BASIS,
11		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12		* See the License for the specific language governing permissions and
13		* limitations under the License.
14		*/
15
16
17		----- First clean up from previous versions of room stats.
18
19		-- First remove old stats stuff
20		DROP TABLE IF EXISTS room_stats;
21		DROP TABLE IF EXISTS room_state;
22		DROP TABLE IF EXISTS room_stats_state;
23		DROP TABLE IF EXISTS user_stats;
24		DROP TABLE IF EXISTS room_stats_earliest_tokens;
25		DROP TABLE IF EXISTS _temp_populate_stats_position;
26		DROP TABLE IF EXISTS _temp_populate_stats_rooms;
27		DROP TABLE IF EXISTS stats_stream_pos;
28
29		-- Unschedule old background updates if they're still scheduled
30		DELETE FROM background_updates WHERE update_name IN (
31		'populate_stats_createtables',
32		'populate_stats_process_rooms',
33		'populate_stats_process_users',
34		'populate_stats_cleanup'
35		);
36
37		INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES
38		('populate_stats_process_rooms', '{}', '');
39
40		INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES
41		('populate_stats_process_users', '{}', 'populate_stats_process_rooms');
42
43		----- Create tables for our version of room stats.
44
45		-- single-row table to track position of incremental updates
46		DROP TABLE IF EXISTS stats_incremental_position;
47		CREATE TABLE stats_incremental_position (
48		Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, -- Makes sure this table only has one row.
49		stream_id BIGINT NOT NULL,
50		CHECK (Lock='X')
51		);
52
53		-- insert a null row and make sure it is the only one.
54		INSERT INTO stats_incremental_position (
55		stream_id
56		) SELECT COALESCE(MAX(stream_ordering), 0) from events;
57
58		-- represents PRESENT room statistics for a room
59		-- only holds absolute fields
60		DROP TABLE IF EXISTS room_stats_current;
61		CREATE TABLE room_stats_current (
62		room_id TEXT NOT NULL PRIMARY KEY,
63
64		-- These are absolute counts
65		current_state_events INT NOT NULL,
66		joined_members INT NOT NULL,
67		invited_members INT NOT NULL,
68		left_members INT NOT NULL,
69		banned_members INT NOT NULL,
70
71		local_users_in_room INT NOT NULL,
72
73		-- The maximum delta stream position that this row takes into account.
74		completed_delta_stream_id BIGINT NOT NULL
75		);
76
77
78		-- represents HISTORICAL room statistics for a room
79		DROP TABLE IF EXISTS room_stats_historical;
80		CREATE TABLE room_stats_historical (
81		room_id TEXT NOT NULL,
82		-- These stats cover the time from (end_ts - bucket_size)...end_ts (in ms).
83		-- Note that end_ts is quantised.
84		end_ts BIGINT NOT NULL,
85		bucket_size BIGINT NOT NULL,
86
87		-- These stats are absolute counts
88		current_state_events BIGINT NOT NULL,
89		joined_members BIGINT NOT NULL,
90		invited_members BIGINT NOT NULL,
91		left_members BIGINT NOT NULL,
92		banned_members BIGINT NOT NULL,
93		local_users_in_room BIGINT NOT NULL,
94
95		-- These stats are per time slice
96		total_events BIGINT NOT NULL,
97		total_event_bytes BIGINT NOT NULL,
98
99		PRIMARY KEY (room_id, end_ts)
100		);
101
102		-- We use this index to speed up deletion of ancient room stats.
103		CREATE INDEX room_stats_historical_end_ts ON room_stats_historical (end_ts);
104
105		-- represents PRESENT statistics for a user
106		-- only holds absolute fields
107		DROP TABLE IF EXISTS user_stats_current;
108		CREATE TABLE user_stats_current (
109		user_id TEXT NOT NULL PRIMARY KEY,
110
111		joined_rooms BIGINT NOT NULL,
112
113		-- The maximum delta stream position that this row takes into account.
114		completed_delta_stream_id BIGINT NOT NULL
115		);
116
117		-- represents HISTORICAL statistics for a user
118		DROP TABLE IF EXISTS user_stats_historical;
119		CREATE TABLE user_stats_historical (
120		user_id TEXT NOT NULL,
121		end_ts BIGINT NOT NULL,
122		bucket_size BIGINT NOT NULL,
123
124		joined_rooms BIGINT NOT NULL,
125
126		invites_sent BIGINT NOT NULL,
127		rooms_created BIGINT NOT NULL,
128		total_events BIGINT NOT NULL,
129		total_event_bytes BIGINT NOT NULL,
130
131		PRIMARY KEY (user_id, end_ts)
132		);
133
134		-- We use this index to speed up deletion of ancient user stats.
135		CREATE INDEX user_stats_historical_end_ts ON user_stats_historical (end_ts);
136
137
138		CREATE TABLE room_stats_state (
139		room_id TEXT NOT NULL,
140		name TEXT,
141		canonical_alias TEXT,
142		join_rules TEXT,
143		history_visibility TEXT,
144		encryption TEXT,
145		avatar TEXT,
146		guest_access TEXT,
147		is_federatable BOOLEAN,
148		topic TEXT
149		);
150
151		CREATE UNIQUE INDEX room_stats_state_room ON room_stats_state(room_id);

+0

-24

~~synapse/storage/schema/delta/56/user_external_ids.sql~~ less more

0		/* Copyright 2019 The Matrix.org Foundation C.I.C.
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		/*
16		* a table which records mappings from external auth providers to mxids
17		*/
18		CREATE TABLE IF NOT EXISTS user_external_ids (
19		auth_provider TEXT NOT NULL,
20		external_id TEXT NOT NULL,
21		user_id TEXT NOT NULL,
22		UNIQUE (auth_provider, external_id)
23		);

+0

-17

~~synapse/storage/schema/delta/56/users_in_public_rooms_idx.sql~~ less more

0		/* Copyright 2019 Matrix.org Foundation CIC
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- this was apparently forgotten when the table was created back in delta 53.
16		CREATE INDEX users_in_public_rooms_r_idx ON users_in_public_rooms(room_id);

+0

-37

~~synapse/storage/schema/full_schemas/16/application_services.sql~~ less more

0		/* Copyright 2015, 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		/* We used to create tables called application_services and
16		* application_services_regex, but these are no longer used and are removed in
17		* delta 54.
18		*/
19
20
21		CREATE TABLE IF NOT EXISTS application_services_state(
22		as_id TEXT PRIMARY KEY,
23		state VARCHAR(5),
24		last_txn INTEGER
25		);
26
27		CREATE TABLE IF NOT EXISTS application_services_txns(
28		as_id TEXT NOT NULL,
29		txn_id INTEGER NOT NULL,
30		event_ids TEXT NOT NULL,
31		UNIQUE(as_id, txn_id)
32		);
33
34		CREATE INDEX application_services_txns_id ON application_services_txns (
35		as_id
36		);

+0

-70

~~synapse/storage/schema/full_schemas/16/event_edges.sql~~ less more

0		/* Copyright 2014-2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		/* We used to create tables called event_destinations and
16		* state_forward_extremities, but these are no longer used and are removed in
17		* delta 54.
18		*/
19
20		CREATE TABLE IF NOT EXISTS event_forward_extremities(
21		event_id TEXT NOT NULL,
22		room_id TEXT NOT NULL,
23		UNIQUE (event_id, room_id)
24		);
25
26		CREATE INDEX ev_extrem_room ON event_forward_extremities(room_id);
27		CREATE INDEX ev_extrem_id ON event_forward_extremities(event_id);
28
29
30		CREATE TABLE IF NOT EXISTS event_backward_extremities(
31		event_id TEXT NOT NULL,
32		room_id TEXT NOT NULL,
33		UNIQUE (event_id, room_id)
34		);
35
36		CREATE INDEX ev_b_extrem_room ON event_backward_extremities(room_id);
37		CREATE INDEX ev_b_extrem_id ON event_backward_extremities(event_id);
38
39
40		CREATE TABLE IF NOT EXISTS event_edges(
41		event_id TEXT NOT NULL,
42		prev_event_id TEXT NOT NULL,
43		room_id TEXT NOT NULL,
44		is_state BOOL NOT NULL, -- true if this is a prev_state edge rather than a regular
45		-- event dag edge.
46		UNIQUE (event_id, prev_event_id, room_id, is_state)
47		);
48
49		CREATE INDEX ev_edges_id ON event_edges(event_id);
50		CREATE INDEX ev_edges_prev_id ON event_edges(prev_event_id);
51
52
53		CREATE TABLE IF NOT EXISTS room_depth(
54		room_id TEXT NOT NULL,
55		min_depth INTEGER NOT NULL,
56		UNIQUE (room_id)
57		);
58
59		CREATE INDEX room_depth_room ON room_depth(room_id);
60
61		CREATE TABLE IF NOT EXISTS event_auth(
62		event_id TEXT NOT NULL,
63		auth_id TEXT NOT NULL,
64		room_id TEXT NOT NULL,
65		UNIQUE (event_id, auth_id, room_id)
66		);
67
68		CREATE INDEX evauth_edges_id ON event_auth(event_id);
69		CREATE INDEX evauth_edges_auth_id ON event_auth(auth_id);

+0

-38

~~synapse/storage/schema/full_schemas/16/event_signatures.sql~~ less more

0		/* Copyright 2014-2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		/* We used to create tables called event_content_hashes and event_edge_hashes,
16		* but these are no longer used and are removed in delta 54.
17		*/
18
19		CREATE TABLE IF NOT EXISTS event_reference_hashes (
20		event_id TEXT,
21		algorithm TEXT,
22		hash bytea,
23		UNIQUE (event_id, algorithm)
24		);
25
26		CREATE INDEX event_reference_hashes_id ON event_reference_hashes(event_id);
27
28
29		CREATE TABLE IF NOT EXISTS event_signatures (
30		event_id TEXT,
31		signature_name TEXT,
32		key_id TEXT,
33		signature bytea,
34		UNIQUE (event_id, signature_name, key_id)
35		);
36
37		CREATE INDEX event_signatures_id ON event_signatures(event_id);

+0

-120

~~synapse/storage/schema/full_schemas/16/im.sql~~ less more

0		/* Copyright 2014-2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		/* We used to create tables called room_hosts and feedback,
16		* but these are no longer used and are removed in delta 54.
17		*/
18
19		CREATE TABLE IF NOT EXISTS events(
20		stream_ordering INTEGER PRIMARY KEY,
21		topological_ordering BIGINT NOT NULL,
22		event_id TEXT NOT NULL,
23		type TEXT NOT NULL,
24		room_id TEXT NOT NULL,
25
26		-- 'content' used to be created NULLable, but as of delta 50 we drop that constraint.
27		-- the hack we use to drop the constraint doesn't work for an in-memory sqlite
28		-- database, which breaks the sytests. Hence, we no longer make it nullable.
29		content TEXT,
30
31		unrecognized_keys TEXT,
32		processed BOOL NOT NULL,
33		outlier BOOL NOT NULL,
34		depth BIGINT DEFAULT 0 NOT NULL,
35		UNIQUE (event_id)
36		);
37
38		CREATE INDEX events_stream_ordering ON events (stream_ordering);
39		CREATE INDEX events_topological_ordering ON events (topological_ordering);
40		CREATE INDEX events_order ON events (topological_ordering, stream_ordering);
41		CREATE INDEX events_room_id ON events (room_id);
42		CREATE INDEX events_order_room ON events (
43		room_id, topological_ordering, stream_ordering
44		);
45
46
47		CREATE TABLE IF NOT EXISTS event_json(
48		event_id TEXT NOT NULL,
49		room_id TEXT NOT NULL,
50		internal_metadata TEXT NOT NULL,
51		json TEXT NOT NULL,
52		UNIQUE (event_id)
53		);
54
55		CREATE INDEX event_json_room_id ON event_json(room_id);
56
57
58		CREATE TABLE IF NOT EXISTS state_events(
59		event_id TEXT NOT NULL,
60		room_id TEXT NOT NULL,
61		type TEXT NOT NULL,
62		state_key TEXT NOT NULL,
63		prev_state TEXT,
64		UNIQUE (event_id)
65		);
66
67		CREATE INDEX state_events_room_id ON state_events (room_id);
68		CREATE INDEX state_events_type ON state_events (type);
69		CREATE INDEX state_events_state_key ON state_events (state_key);
70
71
72		CREATE TABLE IF NOT EXISTS current_state_events(
73		event_id TEXT NOT NULL,
74		room_id TEXT NOT NULL,
75		type TEXT NOT NULL,
76		state_key TEXT NOT NULL,
77		UNIQUE (event_id),
78		UNIQUE (room_id, type, state_key)
79		);
80
81		CREATE INDEX current_state_events_room_id ON current_state_events (room_id);
82		CREATE INDEX current_state_events_type ON current_state_events (type);
83		CREATE INDEX current_state_events_state_key ON current_state_events (state_key);
84
85		CREATE TABLE IF NOT EXISTS room_memberships(
86		event_id TEXT NOT NULL,
87		user_id TEXT NOT NULL,
88		sender TEXT NOT NULL,
89		room_id TEXT NOT NULL,
90		membership TEXT NOT NULL,
91		UNIQUE (event_id)
92		);
93
94		CREATE INDEX room_memberships_room_id ON room_memberships (room_id);
95		CREATE INDEX room_memberships_user_id ON room_memberships (user_id);
96
97		CREATE TABLE IF NOT EXISTS topics(
98		event_id TEXT NOT NULL,
99		room_id TEXT NOT NULL,
100		topic TEXT NOT NULL,
101		UNIQUE (event_id)
102		);
103
104		CREATE INDEX topics_room_id ON topics(room_id);
105
106		CREATE TABLE IF NOT EXISTS room_names(
107		event_id TEXT NOT NULL,
108		room_id TEXT NOT NULL,
109		name TEXT NOT NULL,
110		UNIQUE (event_id)
111		);
112
113		CREATE INDEX room_names_room_id ON room_names(room_id);
114
115		CREATE TABLE IF NOT EXISTS rooms(
116		room_id TEXT PRIMARY KEY NOT NULL,
117		is_public BOOL,
118		creator TEXT
119		);

+0

-26

~~synapse/storage/schema/full_schemas/16/keys.sql~~ less more

0		/* Copyright 2014-2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		-- we used to create a table called server_tls_certificates, but this is no
16		-- longer used, and is removed in delta 54.
17
18		CREATE TABLE IF NOT EXISTS server_signature_keys(
19		server_name TEXT, -- Server name.
20		key_id TEXT, -- Key version.
21		from_server TEXT, -- Which key server the key was fetched form.
22		ts_added_ms BIGINT, -- When the key was added.
23		verify_key bytea, -- NACL verification key.
24		UNIQUE (server_name, key_id)
25		);

+0

-68

~~synapse/storage/schema/full_schemas/16/media_repository.sql~~ less more

0		/* Copyright 2014-2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		CREATE TABLE IF NOT EXISTS local_media_repository (
16		media_id TEXT, -- The id used to refer to the media.
17		media_type TEXT, -- The MIME-type of the media.
18		media_length INTEGER, -- Length of the media in bytes.
19		created_ts BIGINT, -- When the content was uploaded in ms.
20		upload_name TEXT, -- The name the media was uploaded with.
21		user_id TEXT, -- The user who uploaded the file.
22		UNIQUE (media_id)
23		);
24
25		CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails (
26		media_id TEXT, -- The id used to refer to the media.
27		thumbnail_width INTEGER, -- The width of the thumbnail in pixels.
28		thumbnail_height INTEGER, -- The height of the thumbnail in pixels.
29		thumbnail_type TEXT, -- The MIME-type of the thumbnail.
30		thumbnail_method TEXT, -- The method used to make the thumbnail.
31		thumbnail_length INTEGER, -- The length of the thumbnail in bytes.
32		UNIQUE (
33		media_id, thumbnail_width, thumbnail_height, thumbnail_type
34		)
35		);
36
37		CREATE INDEX local_media_repository_thumbnails_media_id
38		ON local_media_repository_thumbnails (media_id);
39
40		CREATE TABLE IF NOT EXISTS remote_media_cache (
41		media_origin TEXT, -- The remote HS the media came from.
42		media_id TEXT, -- The id used to refer to the media on that server.
43		media_type TEXT, -- The MIME-type of the media.
44		created_ts BIGINT, -- When the content was uploaded in ms.
45		upload_name TEXT, -- The name the media was uploaded with.
46		media_length INTEGER, -- Length of the media in bytes.
47		filesystem_id TEXT, -- The name used to store the media on disk.
48		UNIQUE (media_origin, media_id)
49		);
50
51		CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails (
52		media_origin TEXT, -- The remote HS the media came from.
53		media_id TEXT, -- The id used to refer to the media.
54		thumbnail_width INTEGER, -- The width of the thumbnail in pixels.
55		thumbnail_height INTEGER, -- The height of the thumbnail in pixels.
56		thumbnail_method TEXT, -- The method used to make the thumbnail
57		thumbnail_type TEXT, -- The MIME-type of the thumbnail.
58		thumbnail_length INTEGER, -- The length of the thumbnail in bytes.
59		filesystem_id TEXT, -- The name used to store the media on disk.
60		UNIQUE (
61		media_origin, media_id, thumbnail_width, thumbnail_height,
62		thumbnail_type
63		)
64		);
65
66		CREATE INDEX remote_media_cache_thumbnails_media_id
67		ON remote_media_cache_thumbnails (media_id);

+0

-32

~~synapse/storage/schema/full_schemas/16/presence.sql~~ less more

0		/* Copyright 2014-2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14		CREATE TABLE IF NOT EXISTS presence(
15		user_id TEXT NOT NULL,
16		state VARCHAR(20),
17		status_msg TEXT,
18		mtime BIGINT, -- miliseconds since last state change
19		UNIQUE (user_id)
20		);
21
22		-- For each of /my/ users which possibly-remote users are allowed to see their
23		-- presence state
24		CREATE TABLE IF NOT EXISTS presence_allow_inbound(
25		observed_user_id TEXT NOT NULL,
26		observer_user_id TEXT NOT NULL, -- a UserID,
27		UNIQUE (observed_user_id, observer_user_id)
28		);
29
30		-- We used to create a table called presence_list, but this is no longer used
31		-- and is removed in delta 54.⏎

+0

-20

~~synapse/storage/schema/full_schemas/16/profiles.sql~~ less more

0		/* Copyright 2014-2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14		CREATE TABLE IF NOT EXISTS profiles(
15		user_id TEXT NOT NULL,
16		displayname TEXT,
17		avatar_url TEXT,
18		UNIQUE(user_id)
19		);

+0

-74

~~synapse/storage/schema/full_schemas/16/push.sql~~ less more

0		/* Copyright 2015, 2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		CREATE TABLE IF NOT EXISTS rejections(
16		event_id TEXT NOT NULL,
17		reason TEXT NOT NULL,
18		last_check TEXT NOT NULL,
19		UNIQUE (event_id)
20		);
21
22		-- Push notification endpoints that users have configured
23		CREATE TABLE IF NOT EXISTS pushers (
24		id BIGINT PRIMARY KEY,
25		user_name TEXT NOT NULL,
26		access_token BIGINT DEFAULT NULL,
27		profile_tag VARCHAR(32) NOT NULL,
28		kind VARCHAR(8) NOT NULL,
29		app_id VARCHAR(64) NOT NULL,
30		app_display_name VARCHAR(64) NOT NULL,
31		device_display_name VARCHAR(128) NOT NULL,
32		pushkey bytea NOT NULL,
33		ts BIGINT NOT NULL,
34		lang VARCHAR(8),
35		data bytea,
36		last_token TEXT,
37		last_success BIGINT,
38		failing_since BIGINT,
39		UNIQUE (app_id, pushkey)
40		);
41
42		CREATE TABLE IF NOT EXISTS push_rules (
43		id BIGINT PRIMARY KEY,
44		user_name TEXT NOT NULL,
45		rule_id TEXT NOT NULL,
46		priority_class SMALLINT NOT NULL,
47		priority INTEGER NOT NULL DEFAULT 0,
48		conditions TEXT NOT NULL,
49		actions TEXT NOT NULL,
50		UNIQUE(user_name, rule_id)
51		);
52
53		CREATE INDEX push_rules_user_name on push_rules (user_name);
54
55		CREATE TABLE IF NOT EXISTS user_filters(
56		user_id TEXT,
57		filter_id BIGINT,
58		filter_json bytea
59		);
60
61		CREATE INDEX user_filters_by_user_id_filter_id ON user_filters(
62		user_id, filter_id
63		);
64
65		CREATE TABLE IF NOT EXISTS push_rules_enable (
66		id BIGINT PRIMARY KEY,
67		user_name TEXT NOT NULL,
68		rule_id TEXT NOT NULL,
69		enabled SMALLINT,
70		UNIQUE(user_name, rule_id)
71		);
72
73		CREATE INDEX push_rules_enable_user_name on push_rules_enable (user_name);

+0

-22

~~synapse/storage/schema/full_schemas/16/redactions.sql~~ less more

0		/* Copyright 2014-2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14		CREATE TABLE IF NOT EXISTS redactions (
15		event_id TEXT NOT NULL,
16		redacts TEXT NOT NULL,
17		UNIQUE (event_id)
18		);
19
20		CREATE INDEX redactions_event_id ON redactions (event_id);
21		CREATE INDEX redactions_redacts ON redactions (redacts);

+0

-29

~~synapse/storage/schema/full_schemas/16/room_aliases.sql~~ less more

0		/* Copyright 2014-2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		CREATE TABLE IF NOT EXISTS room_aliases(
16		room_alias TEXT NOT NULL,
17		room_id TEXT NOT NULL,
18		UNIQUE (room_alias)
19		);
20
21		CREATE INDEX room_aliases_id ON room_aliases(room_id);
22
23		CREATE TABLE IF NOT EXISTS room_alias_servers(
24		room_alias TEXT NOT NULL,
25		server TEXT NOT NULL
26		);
27
28		CREATE INDEX room_alias_servers_alias ON room_alias_servers(room_alias);

+0

-40

~~synapse/storage/schema/full_schemas/16/state.sql~~ less more

0		/* Copyright 2014-2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14
15		CREATE TABLE IF NOT EXISTS state_groups(
16		id BIGINT PRIMARY KEY,
17		room_id TEXT NOT NULL,
18		event_id TEXT NOT NULL
19		);
20
21		CREATE TABLE IF NOT EXISTS state_groups_state(
22		state_group BIGINT NOT NULL,
23		room_id TEXT NOT NULL,
24		type TEXT NOT NULL,
25		state_key TEXT NOT NULL,
26		event_id TEXT NOT NULL
27		);
28
29		CREATE TABLE IF NOT EXISTS event_to_state_groups(
30		event_id TEXT NOT NULL,
31		state_group BIGINT NOT NULL,
32		UNIQUE (event_id)
33		);
34
35		CREATE INDEX state_groups_id ON state_groups(id);
36
37		CREATE INDEX state_groups_state_id ON state_groups_state(state_group);
38		CREATE INDEX state_groups_state_tuple ON state_groups_state(room_id, type, state_key);
39		CREATE INDEX event_to_state_groups_id ON event_to_state_groups(event_id);

+0

-44

~~synapse/storage/schema/full_schemas/16/transactions.sql~~ less more

0		/* Copyright 2014-2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14		-- Stores what transaction ids we have received and what our response was
15		CREATE TABLE IF NOT EXISTS received_transactions(
16		transaction_id TEXT,
17		origin TEXT,
18		ts BIGINT,
19		response_code INTEGER,
20		response_json bytea,
21		has_been_referenced smallint default 0, -- Whether thishas been referenced by a prev_tx
22		UNIQUE (transaction_id, origin)
23		);
24
25		CREATE INDEX transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0;
26
27		-- For sent transactions only.
28		CREATE TABLE IF NOT EXISTS transaction_id_to_pdu(
29		transaction_id INTEGER,
30		destination TEXT,
31		pdu_id TEXT,
32		pdu_origin TEXT,
33		UNIQUE (transaction_id, destination)
34		);
35
36		CREATE INDEX transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination);
37
38		-- To track destination health
39		CREATE TABLE IF NOT EXISTS destinations(
40		destination TEXT PRIMARY KEY,
41		retry_last_ts BIGINT,
42		retry_interval INTEGER
43		);

+0

-42

~~synapse/storage/schema/full_schemas/16/users.sql~~ less more

0		/* Copyright 2014-2016 OpenMarket Ltd
1		*
2		* Licensed under the Apache License, Version 2.0 (the "License");
3		* you may not use this file except in compliance with the License.
4		* You may obtain a copy of the License at
5		*
6		* http://www.apache.org/licenses/LICENSE-2.0
7		*
8		* Unless required by applicable law or agreed to in writing, software
9		* distributed under the License is distributed on an "AS IS" BASIS,
10		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11		* See the License for the specific language governing permissions and
12		* limitations under the License.
13		*/
14		CREATE TABLE IF NOT EXISTS users(
15		name TEXT,
16		password_hash TEXT,
17		creation_ts BIGINT,
18		admin SMALLINT DEFAULT 0 NOT NULL,
19		UNIQUE(name)
20		);
21
22		CREATE TABLE IF NOT EXISTS access_tokens(
23		id BIGINT PRIMARY KEY,
24		user_id TEXT NOT NULL,
25		device_id TEXT,
26		token TEXT NOT NULL,
27		last_used BIGINT,
28		UNIQUE(token)
29		);
30
31		CREATE TABLE IF NOT EXISTS user_ips (
32		user_id TEXT NOT NULL,
33		access_token TEXT NOT NULL,
34		device_id TEXT,
35		ip TEXT NOT NULL,
36		user_agent TEXT NOT NULL,
37		last_seen BIGINT NOT NULL
38		);
39
40		CREATE INDEX user_ips_user ON user_ips(user_id);
41		CREATE INDEX user_ips_user_ip ON user_ips(user_id, access_token, ip);

+8

-0

synapse/storage/schema/full_schemas/54/full.sql less more

	0
	1
	2	CREATE TABLE background_updates (
	3	update_name text NOT NULL,
	4	progress_json text NOT NULL,
	5	depends_on text,
	6	CONSTRAINT background_updates_uniqueness UNIQUE (update_name)
	7	);

+0

-2052

~~synapse/storage/schema/full_schemas/54/full.sql.postgres~~ less more

0
1
2
3
4
5		CREATE TABLE access_tokens (
6		id bigint NOT NULL,
7		user_id text NOT NULL,
8		device_id text,
9		token text NOT NULL,
10		last_used bigint
11		);
12
13
14
15		CREATE TABLE account_data (
16		user_id text NOT NULL,
17		account_data_type text NOT NULL,
18		stream_id bigint NOT NULL,
19		content text NOT NULL
20		);
21
22
23
24		CREATE TABLE account_data_max_stream_id (
25		lock character(1) DEFAULT 'X'::bpchar NOT NULL,
26		stream_id bigint NOT NULL,
27		CONSTRAINT private_user_data_max_stream_id_lock_check CHECK ((lock = 'X'::bpchar))
28		);
29
30
31
32		CREATE TABLE account_validity (
33		user_id text NOT NULL,
34		expiration_ts_ms bigint NOT NULL,
35		email_sent boolean NOT NULL,
36		renewal_token text
37		);
38
39
40
41		CREATE TABLE application_services_state (
42		as_id text NOT NULL,
43		state character varying(5),
44		last_txn integer
45		);
46
47
48
49		CREATE TABLE application_services_txns (
50		as_id text NOT NULL,
51		txn_id integer NOT NULL,
52		event_ids text NOT NULL
53		);
54
55
56
57		CREATE TABLE appservice_room_list (
58		appservice_id text NOT NULL,
59		network_id text NOT NULL,
60		room_id text NOT NULL
61		);
62
63
64
65		CREATE TABLE appservice_stream_position (
66		lock character(1) DEFAULT 'X'::bpchar NOT NULL,
67		stream_ordering bigint,
68		CONSTRAINT appservice_stream_position_lock_check CHECK ((lock = 'X'::bpchar))
69		);
70
71
72
73		CREATE TABLE background_updates (
74		update_name text NOT NULL,
75		progress_json text NOT NULL,
76		depends_on text
77		);
78
79
80
81		CREATE TABLE blocked_rooms (
82		room_id text NOT NULL,
83		user_id text NOT NULL
84		);
85
86
87
88		CREATE TABLE cache_invalidation_stream (
89		stream_id bigint,
90		cache_func text,
91		keys text[],
92		invalidation_ts bigint
93		);
94
95
96
97		CREATE TABLE current_state_delta_stream (
98		stream_id bigint NOT NULL,
99		room_id text NOT NULL,
100		type text NOT NULL,
101		state_key text NOT NULL,
102		event_id text,
103		prev_event_id text
104		);
105
106
107
108		CREATE TABLE current_state_events (
109		event_id text NOT NULL,
110		room_id text NOT NULL,
111		type text NOT NULL,
112		state_key text NOT NULL
113		);
114
115
116
117		CREATE TABLE deleted_pushers (
118		stream_id bigint NOT NULL,
119		app_id text NOT NULL,
120		pushkey text NOT NULL,
121		user_id text NOT NULL
122		);
123
124
125
126		CREATE TABLE destinations (
127		destination text NOT NULL,
128		retry_last_ts bigint,
129		retry_interval integer
130		);
131
132
133
134		CREATE TABLE device_federation_inbox (
135		origin text NOT NULL,
136		message_id text NOT NULL,
137		received_ts bigint NOT NULL
138		);
139
140
141
142		CREATE TABLE device_federation_outbox (
143		destination text NOT NULL,
144		stream_id bigint NOT NULL,
145		queued_ts bigint NOT NULL,
146		messages_json text NOT NULL
147		);
148
149
150
151		CREATE TABLE device_inbox (
152		user_id text NOT NULL,
153		device_id text NOT NULL,
154		stream_id bigint NOT NULL,
155		message_json text NOT NULL
156		);
157
158
159
160		CREATE TABLE device_lists_outbound_last_success (
161		destination text NOT NULL,
162		user_id text NOT NULL,
163		stream_id bigint NOT NULL
164		);
165
166
167
168		CREATE TABLE device_lists_outbound_pokes (
169		destination text NOT NULL,
170		stream_id bigint NOT NULL,
171		user_id text NOT NULL,
172		device_id text NOT NULL,
173		sent boolean NOT NULL,
174		ts bigint NOT NULL
175		);
176
177
178
179		CREATE TABLE device_lists_remote_cache (
180		user_id text NOT NULL,
181		device_id text NOT NULL,
182		content text NOT NULL
183		);
184
185
186
187		CREATE TABLE device_lists_remote_extremeties (
188		user_id text NOT NULL,
189		stream_id text NOT NULL
190		);
191
192
193
194		CREATE TABLE device_lists_stream (
195		stream_id bigint NOT NULL,
196		user_id text NOT NULL,
197		device_id text NOT NULL
198		);
199
200
201
202		CREATE TABLE device_max_stream_id (
203		stream_id bigint NOT NULL
204		);
205
206
207
208		CREATE TABLE devices (
209		user_id text NOT NULL,
210		device_id text NOT NULL,
211		display_name text
212		);
213
214
215
216		CREATE TABLE e2e_device_keys_json (
217		user_id text NOT NULL,
218		device_id text NOT NULL,
219		ts_added_ms bigint NOT NULL,
220		key_json text NOT NULL
221		);
222
223
224
225		CREATE TABLE e2e_one_time_keys_json (
226		user_id text NOT NULL,
227		device_id text NOT NULL,
228		algorithm text NOT NULL,
229		key_id text NOT NULL,
230		ts_added_ms bigint NOT NULL,
231		key_json text NOT NULL
232		);
233
234
235
236		CREATE TABLE e2e_room_keys (
237		user_id text NOT NULL,
238		room_id text NOT NULL,
239		session_id text NOT NULL,
240		version bigint NOT NULL,
241		first_message_index integer,
242		forwarded_count integer,
243		is_verified boolean,
244		session_data text NOT NULL
245		);
246
247
248
249		CREATE TABLE e2e_room_keys_versions (
250		user_id text NOT NULL,
251		version bigint NOT NULL,
252		algorithm text NOT NULL,
253		auth_data text NOT NULL,
254		deleted smallint DEFAULT 0 NOT NULL
255		);
256
257
258
259		CREATE TABLE erased_users (
260		user_id text NOT NULL
261		);
262
263
264
265		CREATE TABLE event_auth (
266		event_id text NOT NULL,
267		auth_id text NOT NULL,
268		room_id text NOT NULL
269		);
270
271
272
273		CREATE TABLE event_backward_extremities (
274		event_id text NOT NULL,
275		room_id text NOT NULL
276		);
277
278
279
280		CREATE TABLE event_edges (
281		event_id text NOT NULL,
282		prev_event_id text NOT NULL,
283		room_id text NOT NULL,
284		is_state boolean NOT NULL
285		);
286
287
288
289		CREATE TABLE event_forward_extremities (
290		event_id text NOT NULL,
291		room_id text NOT NULL
292		);
293
294
295
296		CREATE TABLE event_json (
297		event_id text NOT NULL,
298		room_id text NOT NULL,
299		internal_metadata text NOT NULL,
300		json text NOT NULL,
301		format_version integer
302		);
303
304
305
306		CREATE TABLE event_push_actions (
307		room_id text NOT NULL,
308		event_id text NOT NULL,
309		user_id text NOT NULL,
310		profile_tag character varying(32),
311		actions text NOT NULL,
312		topological_ordering bigint,
313		stream_ordering bigint,
314		notif smallint,
315		highlight smallint
316		);
317
318
319
320		CREATE TABLE event_push_actions_staging (
321		event_id text NOT NULL,
322		user_id text NOT NULL,
323		actions text NOT NULL,
324		notif smallint NOT NULL,
325		highlight smallint NOT NULL
326		);
327
328
329
330		CREATE TABLE event_push_summary (
331		user_id text NOT NULL,
332		room_id text NOT NULL,
333		notif_count bigint NOT NULL,
334		stream_ordering bigint NOT NULL
335		);
336
337
338
339		CREATE TABLE event_push_summary_stream_ordering (
340		lock character(1) DEFAULT 'X'::bpchar NOT NULL,
341		stream_ordering bigint NOT NULL,
342		CONSTRAINT event_push_summary_stream_ordering_lock_check CHECK ((lock = 'X'::bpchar))
343		);
344
345
346
347		CREATE TABLE event_reference_hashes (
348		event_id text,
349		algorithm text,
350		hash bytea
351		);
352
353
354
355		CREATE TABLE event_relations (
356		event_id text NOT NULL,
357		relates_to_id text NOT NULL,
358		relation_type text NOT NULL,
359		aggregation_key text
360		);
361
362
363
364		CREATE TABLE event_reports (
365		id bigint NOT NULL,
366		received_ts bigint NOT NULL,
367		room_id text NOT NULL,
368		event_id text NOT NULL,
369		user_id text NOT NULL,
370		reason text,
371		content text
372		);
373
374
375
376		CREATE TABLE event_search (
377		event_id text,
378		room_id text,
379		sender text,
380		key text,
381		vector tsvector,
382		origin_server_ts bigint,
383		stream_ordering bigint
384		);
385
386
387
388		CREATE TABLE event_to_state_groups (
389		event_id text NOT NULL,
390		state_group bigint NOT NULL
391		);
392
393
394
395		CREATE TABLE events (
396		stream_ordering integer NOT NULL,
397		topological_ordering bigint NOT NULL,
398		event_id text NOT NULL,
399		type text NOT NULL,
400		room_id text NOT NULL,
401		content text,
402		unrecognized_keys text,
403		processed boolean NOT NULL,
404		outlier boolean NOT NULL,
405		depth bigint DEFAULT 0 NOT NULL,
406		origin_server_ts bigint,
407		received_ts bigint,
408		sender text,
409		contains_url boolean
410		);
411
412
413
414		CREATE TABLE ex_outlier_stream (
415		event_stream_ordering bigint NOT NULL,
416		event_id text NOT NULL,
417		state_group bigint NOT NULL
418		);
419
420
421
422		CREATE TABLE federation_stream_position (
423		type text NOT NULL,
424		stream_id integer NOT NULL
425		);
426
427
428
429		CREATE TABLE group_attestations_remote (
430		group_id text NOT NULL,
431		user_id text NOT NULL,
432		valid_until_ms bigint NOT NULL,
433		attestation_json text NOT NULL
434		);
435
436
437
438		CREATE TABLE group_attestations_renewals (
439		group_id text NOT NULL,
440		user_id text NOT NULL,
441		valid_until_ms bigint NOT NULL
442		);
443
444
445
446		CREATE TABLE group_invites (
447		group_id text NOT NULL,
448		user_id text NOT NULL
449		);
450
451
452
453		CREATE TABLE group_roles (
454		group_id text NOT NULL,
455		role_id text NOT NULL,
456		profile text NOT NULL,
457		is_public boolean NOT NULL
458		);
459
460
461
462		CREATE TABLE group_room_categories (
463		group_id text NOT NULL,
464		category_id text NOT NULL,
465		profile text NOT NULL,
466		is_public boolean NOT NULL
467		);
468
469
470
471		CREATE TABLE group_rooms (
472		group_id text NOT NULL,
473		room_id text NOT NULL,
474		is_public boolean NOT NULL
475		);
476
477
478
479		CREATE TABLE group_summary_roles (
480		group_id text NOT NULL,
481		role_id text NOT NULL,
482		role_order bigint NOT NULL,
483		CONSTRAINT group_summary_roles_role_order_check CHECK ((role_order > 0))
484		);
485
486
487
488		CREATE TABLE group_summary_room_categories (
489		group_id text NOT NULL,
490		category_id text NOT NULL,
491		cat_order bigint NOT NULL,
492		CONSTRAINT group_summary_room_categories_cat_order_check CHECK ((cat_order > 0))
493		);
494
495
496
497		CREATE TABLE group_summary_rooms (
498		group_id text NOT NULL,
499		room_id text NOT NULL,
500		category_id text NOT NULL,
501		room_order bigint NOT NULL,
502		is_public boolean NOT NULL,
503		CONSTRAINT group_summary_rooms_room_order_check CHECK ((room_order > 0))
504		);
505
506
507
508		CREATE TABLE group_summary_users (
509		group_id text NOT NULL,
510		user_id text NOT NULL,
511		role_id text NOT NULL,
512		user_order bigint NOT NULL,
513		is_public boolean NOT NULL
514		);
515
516
517
518		CREATE TABLE group_users (
519		group_id text NOT NULL,
520		user_id text NOT NULL,
521		is_admin boolean NOT NULL,
522		is_public boolean NOT NULL
523		);
524
525
526
527		CREATE TABLE groups (
528		group_id text NOT NULL,
529		name text,
530		avatar_url text,
531		short_description text,
532		long_description text,
533		is_public boolean NOT NULL,
534		join_policy text DEFAULT 'invite'::text NOT NULL
535		);
536
537
538
539		CREATE TABLE guest_access (
540		event_id text NOT NULL,
541		room_id text NOT NULL,
542		guest_access text NOT NULL
543		);
544
545
546
547		CREATE TABLE history_visibility (
548		event_id text NOT NULL,
549		room_id text NOT NULL,
550		history_visibility text NOT NULL
551		);
552
553
554
555		CREATE TABLE local_group_membership (
556		group_id text NOT NULL,
557		user_id text NOT NULL,
558		is_admin boolean NOT NULL,
559		membership text NOT NULL,
560		is_publicised boolean NOT NULL,
561		content text NOT NULL
562		);
563
564
565
566		CREATE TABLE local_group_updates (
567		stream_id bigint NOT NULL,
568		group_id text NOT NULL,
569		user_id text NOT NULL,
570		type text NOT NULL,
571		content text NOT NULL
572		);
573
574
575
576		CREATE TABLE local_invites (
577		stream_id bigint NOT NULL,
578		inviter text NOT NULL,
579		invitee text NOT NULL,
580		event_id text NOT NULL,
581		room_id text NOT NULL,
582		locally_rejected text,
583		replaced_by text
584		);
585
586
587
588		CREATE TABLE local_media_repository (
589		media_id text,
590		media_type text,
591		media_length integer,
592		created_ts bigint,
593		upload_name text,
594		user_id text,
595		quarantined_by text,
596		url_cache text,
597		last_access_ts bigint
598		);
599
600
601
602		CREATE TABLE local_media_repository_thumbnails (
603		media_id text,
604		thumbnail_width integer,
605		thumbnail_height integer,
606		thumbnail_type text,
607		thumbnail_method text,
608		thumbnail_length integer
609		);
610
611
612
613		CREATE TABLE local_media_repository_url_cache (
614		url text,
615		response_code integer,
616		etag text,
617		expires_ts bigint,
618		og text,
619		media_id text,
620		download_ts bigint
621		);
622
623
624
625		CREATE TABLE monthly_active_users (
626		user_id text NOT NULL,
627		"timestamp" bigint NOT NULL
628		);
629
630
631
632		CREATE TABLE open_id_tokens (
633		token text NOT NULL,
634		ts_valid_until_ms bigint NOT NULL,
635		user_id text NOT NULL
636		);
637
638
639
640		CREATE TABLE presence (
641		user_id text NOT NULL,
642		state character varying(20),
643		status_msg text,
644		mtime bigint
645		);
646
647
648
649		CREATE TABLE presence_allow_inbound (
650		observed_user_id text NOT NULL,
651		observer_user_id text NOT NULL
652		);
653
654
655
656		CREATE TABLE presence_stream (
657		stream_id bigint,
658		user_id text,
659		state text,
660		last_active_ts bigint,
661		last_federation_update_ts bigint,
662		last_user_sync_ts bigint,
663		status_msg text,
664		currently_active boolean
665		);
666
667
668
669		CREATE TABLE profiles (
670		user_id text NOT NULL,
671		displayname text,
672		avatar_url text
673		);
674
675
676
677		CREATE TABLE public_room_list_stream (
678		stream_id bigint NOT NULL,
679		room_id text NOT NULL,
680		visibility boolean NOT NULL,
681		appservice_id text,
682		network_id text
683		);
684
685
686
687		CREATE TABLE push_rules (
688		id bigint NOT NULL,
689		user_name text NOT NULL,
690		rule_id text NOT NULL,
691		priority_class smallint NOT NULL,
692		priority integer DEFAULT 0 NOT NULL,
693		conditions text NOT NULL,
694		actions text NOT NULL
695		);
696
697
698
699		CREATE TABLE push_rules_enable (
700		id bigint NOT NULL,
701		user_name text NOT NULL,
702		rule_id text NOT NULL,
703		enabled smallint
704		);
705
706
707
708		CREATE TABLE push_rules_stream (
709		stream_id bigint NOT NULL,
710		event_stream_ordering bigint NOT NULL,
711		user_id text NOT NULL,
712		rule_id text NOT NULL,
713		op text NOT NULL,
714		priority_class smallint,
715		priority integer,
716		conditions text,
717		actions text
718		);
719
720
721
722		CREATE TABLE pusher_throttle (
723		pusher bigint NOT NULL,
724		room_id text NOT NULL,
725		last_sent_ts bigint,
726		throttle_ms bigint
727		);
728
729
730
731		CREATE TABLE pushers (
732		id bigint NOT NULL,
733		user_name text NOT NULL,
734		access_token bigint,
735		profile_tag text NOT NULL,
736		kind text NOT NULL,
737		app_id text NOT NULL,
738		app_display_name text NOT NULL,
739		device_display_name text NOT NULL,
740		pushkey text NOT NULL,
741		ts bigint NOT NULL,
742		lang text,
743		data text,
744		last_stream_ordering integer,
745		last_success bigint,
746		failing_since bigint
747		);
748
749
750
751		CREATE TABLE ratelimit_override (
752		user_id text NOT NULL,
753		messages_per_second bigint,
754		burst_count bigint
755		);
756
757
758
759		CREATE TABLE receipts_graph (
760		room_id text NOT NULL,
761		receipt_type text NOT NULL,
762		user_id text NOT NULL,
763		event_ids text NOT NULL,
764		data text NOT NULL
765		);
766
767
768
769		CREATE TABLE receipts_linearized (
770		stream_id bigint NOT NULL,
771		room_id text NOT NULL,
772		receipt_type text NOT NULL,
773		user_id text NOT NULL,
774		event_id text NOT NULL,
775		data text NOT NULL
776		);
777
778
779
780		CREATE TABLE received_transactions (
781		transaction_id text,
782		origin text,
783		ts bigint,
784		response_code integer,
785		response_json bytea,
786		has_been_referenced smallint DEFAULT 0
787		);
788
789
790
791		CREATE TABLE redactions (
792		event_id text NOT NULL,
793		redacts text NOT NULL
794		);
795
796
797
798		CREATE TABLE rejections (
799		event_id text NOT NULL,
800		reason text NOT NULL,
801		last_check text NOT NULL
802		);
803
804
805
806		CREATE TABLE remote_media_cache (
807		media_origin text,
808		media_id text,
809		media_type text,
810		created_ts bigint,
811		upload_name text,
812		media_length integer,
813		filesystem_id text,
814		last_access_ts bigint,
815		quarantined_by text
816		);
817
818
819
820		CREATE TABLE remote_media_cache_thumbnails (
821		media_origin text,
822		media_id text,
823		thumbnail_width integer,
824		thumbnail_height integer,
825		thumbnail_method text,
826		thumbnail_type text,
827		thumbnail_length integer,
828		filesystem_id text
829		);
830
831
832
833		CREATE TABLE remote_profile_cache (
834		user_id text NOT NULL,
835		displayname text,
836		avatar_url text,
837		last_check bigint NOT NULL
838		);
839
840
841
842		CREATE TABLE room_account_data (
843		user_id text NOT NULL,
844		room_id text NOT NULL,
845		account_data_type text NOT NULL,
846		stream_id bigint NOT NULL,
847		content text NOT NULL
848		);
849
850
851
852		CREATE TABLE room_alias_servers (
853		room_alias text NOT NULL,
854		server text NOT NULL
855		);
856
857
858
859		CREATE TABLE room_aliases (
860		room_alias text NOT NULL,
861		room_id text NOT NULL,
862		creator text
863		);
864
865
866
867		CREATE TABLE room_depth (
868		room_id text NOT NULL,
869		min_depth integer NOT NULL
870		);
871
872
873
874		CREATE TABLE room_memberships (
875		event_id text NOT NULL,
876		user_id text NOT NULL,
877		sender text NOT NULL,
878		room_id text NOT NULL,
879		membership text NOT NULL,
880		forgotten integer DEFAULT 0,
881		display_name text,
882		avatar_url text
883		);
884
885
886
887		CREATE TABLE room_names (
888		event_id text NOT NULL,
889		room_id text NOT NULL,
890		name text NOT NULL
891		);
892
893
894
895		CREATE TABLE room_state (
896		room_id text NOT NULL,
897		join_rules text,
898		history_visibility text,
899		encryption text,
900		name text,
901		topic text,
902		avatar text,
903		canonical_alias text
904		);
905
906
907
908		CREATE TABLE room_stats (
909		room_id text NOT NULL,
910		ts bigint NOT NULL,
911		bucket_size integer NOT NULL,
912		current_state_events integer NOT NULL,
913		joined_members integer NOT NULL,
914		invited_members integer NOT NULL,
915		left_members integer NOT NULL,
916		banned_members integer NOT NULL,
917		state_events integer NOT NULL
918		);
919
920
921
922		CREATE TABLE room_stats_earliest_token (
923		room_id text NOT NULL,
924		token bigint NOT NULL
925		);
926
927
928
929		CREATE TABLE room_tags (
930		user_id text NOT NULL,
931		room_id text NOT NULL,
932		tag text NOT NULL,
933		content text NOT NULL
934		);
935
936
937
938		CREATE TABLE room_tags_revisions (
939		user_id text NOT NULL,
940		room_id text NOT NULL,
941		stream_id bigint NOT NULL
942		);
943
944
945
946		CREATE TABLE rooms (
947		room_id text NOT NULL,
948		is_public boolean,
949		creator text
950		);
951
952
953
954		CREATE TABLE server_keys_json (
955		server_name text NOT NULL,
956		key_id text NOT NULL,
957		from_server text NOT NULL,
958		ts_added_ms bigint NOT NULL,
959		ts_valid_until_ms bigint NOT NULL,
960		key_json bytea NOT NULL
961		);
962
963
964
965		CREATE TABLE server_signature_keys (
966		server_name text,
967		key_id text,
968		from_server text,
969		ts_added_ms bigint,
970		verify_key bytea,
971		ts_valid_until_ms bigint
972		);
973
974
975
976		CREATE TABLE state_events (
977		event_id text NOT NULL,
978		room_id text NOT NULL,
979		type text NOT NULL,
980		state_key text NOT NULL,
981		prev_state text
982		);
983
984
985
986		CREATE TABLE state_group_edges (
987		state_group bigint NOT NULL,
988		prev_state_group bigint NOT NULL
989		);
990
991
992
993		CREATE SEQUENCE state_group_id_seq
994		START WITH 1
995		INCREMENT BY 1
996		NO MINVALUE
997		NO MAXVALUE
998		CACHE 1;
999
1000
1001
1002		CREATE TABLE state_groups (
1003		id bigint NOT NULL,
1004		room_id text NOT NULL,
1005		event_id text NOT NULL
1006		);
1007
1008
1009
1010		CREATE TABLE state_groups_state (
1011		state_group bigint NOT NULL,
1012		room_id text NOT NULL,
1013		type text NOT NULL,
1014		state_key text NOT NULL,
1015		event_id text NOT NULL
1016		);
1017
1018
1019
1020		CREATE TABLE stats_stream_pos (
1021		lock character(1) DEFAULT 'X'::bpchar NOT NULL,
1022		stream_id bigint,
1023		CONSTRAINT stats_stream_pos_lock_check CHECK ((lock = 'X'::bpchar))
1024		);
1025
1026
1027
1028		CREATE TABLE stream_ordering_to_exterm (
1029		stream_ordering bigint NOT NULL,
1030		room_id text NOT NULL,
1031		event_id text NOT NULL
1032		);
1033
1034
1035
1036		CREATE TABLE threepid_guest_access_tokens (
1037		medium text,
1038		address text,
1039		guest_access_token text,
1040		first_inviter text
1041		);
1042
1043
1044
1045		CREATE TABLE topics (
1046		event_id text NOT NULL,
1047		room_id text NOT NULL,
1048		topic text NOT NULL
1049		);
1050
1051
1052
1053		CREATE TABLE user_daily_visits (
1054		user_id text NOT NULL,
1055		device_id text,
1056		"timestamp" bigint NOT NULL
1057		);
1058
1059
1060
1061		CREATE TABLE user_directory (
1062		user_id text NOT NULL,
1063		room_id text,
1064		display_name text,
1065		avatar_url text
1066		);
1067
1068
1069
1070		CREATE TABLE user_directory_search (
1071		user_id text NOT NULL,
1072		vector tsvector
1073		);
1074
1075
1076
1077		CREATE TABLE user_directory_stream_pos (
1078		lock character(1) DEFAULT 'X'::bpchar NOT NULL,
1079		stream_id bigint,
1080		CONSTRAINT user_directory_stream_pos_lock_check CHECK ((lock = 'X'::bpchar))
1081		);
1082
1083
1084
1085		CREATE TABLE user_filters (
1086		user_id text,
1087		filter_id bigint,
1088		filter_json bytea
1089		);
1090
1091
1092
1093		CREATE TABLE user_ips (
1094		user_id text NOT NULL,
1095		access_token text NOT NULL,
1096		device_id text,
1097		ip text NOT NULL,
1098		user_agent text NOT NULL,
1099		last_seen bigint NOT NULL
1100		);
1101
1102
1103
1104		CREATE TABLE user_stats (
1105		user_id text NOT NULL,
1106		ts bigint NOT NULL,
1107		bucket_size integer NOT NULL,
1108		public_rooms integer NOT NULL,
1109		private_rooms integer NOT NULL
1110		);
1111
1112
1113
1114		CREATE TABLE user_threepid_id_server (
1115		user_id text NOT NULL,
1116		medium text NOT NULL,
1117		address text NOT NULL,
1118		id_server text NOT NULL
1119		);
1120
1121
1122
1123		CREATE TABLE user_threepids (
1124		user_id text NOT NULL,
1125		medium text NOT NULL,
1126		address text NOT NULL,
1127		validated_at bigint NOT NULL,
1128		added_at bigint NOT NULL
1129		);
1130
1131
1132
1133		CREATE TABLE users (
1134		name text,
1135		password_hash text,
1136		creation_ts bigint,
1137		admin smallint DEFAULT 0 NOT NULL,
1138		upgrade_ts bigint,
1139		is_guest smallint DEFAULT 0 NOT NULL,
1140		appservice_id text,
1141		consent_version text,
1142		consent_server_notice_sent text,
1143		user_type text
1144		);
1145
1146
1147
1148		CREATE TABLE users_in_public_rooms (
1149		user_id text NOT NULL,
1150		room_id text NOT NULL
1151		);
1152
1153
1154
1155		CREATE TABLE users_pending_deactivation (
1156		user_id text NOT NULL
1157		);
1158
1159
1160
1161		CREATE TABLE users_who_share_private_rooms (
1162		user_id text NOT NULL,
1163		other_user_id text NOT NULL,
1164		room_id text NOT NULL
1165		);
1166
1167
1168
1169		ALTER TABLE ONLY access_tokens
1170		ADD CONSTRAINT access_tokens_pkey PRIMARY KEY (id);
1171
1172
1173
1174		ALTER TABLE ONLY access_tokens
1175		ADD CONSTRAINT access_tokens_token_key UNIQUE (token);
1176
1177
1178
1179		ALTER TABLE ONLY account_data
1180		ADD CONSTRAINT account_data_uniqueness UNIQUE (user_id, account_data_type);
1181
1182
1183
1184		ALTER TABLE ONLY account_validity
1185		ADD CONSTRAINT account_validity_pkey PRIMARY KEY (user_id);
1186
1187
1188
1189		ALTER TABLE ONLY application_services_state
1190		ADD CONSTRAINT application_services_state_pkey PRIMARY KEY (as_id);
1191
1192
1193
1194		ALTER TABLE ONLY application_services_txns
1195		ADD CONSTRAINT application_services_txns_as_id_txn_id_key UNIQUE (as_id, txn_id);
1196
1197
1198
1199		ALTER TABLE ONLY appservice_stream_position
1200		ADD CONSTRAINT appservice_stream_position_lock_key UNIQUE (lock);
1201
1202
1203
1204		ALTER TABLE ONLY background_updates
1205		ADD CONSTRAINT background_updates_uniqueness UNIQUE (update_name);
1206
1207
1208
1209		ALTER TABLE ONLY current_state_events
1210		ADD CONSTRAINT current_state_events_event_id_key UNIQUE (event_id);
1211
1212
1213
1214		ALTER TABLE ONLY current_state_events
1215		ADD CONSTRAINT current_state_events_room_id_type_state_key_key UNIQUE (room_id, type, state_key);
1216
1217
1218
1219		ALTER TABLE ONLY destinations
1220		ADD CONSTRAINT destinations_pkey PRIMARY KEY (destination);
1221
1222
1223
1224		ALTER TABLE ONLY devices
1225		ADD CONSTRAINT device_uniqueness UNIQUE (user_id, device_id);
1226
1227
1228
1229		ALTER TABLE ONLY e2e_device_keys_json
1230		ADD CONSTRAINT e2e_device_keys_json_uniqueness UNIQUE (user_id, device_id);
1231
1232
1233
1234		ALTER TABLE ONLY e2e_one_time_keys_json
1235		ADD CONSTRAINT e2e_one_time_keys_json_uniqueness UNIQUE (user_id, device_id, algorithm, key_id);
1236
1237
1238
1239		ALTER TABLE ONLY event_backward_extremities
1240		ADD CONSTRAINT event_backward_extremities_event_id_room_id_key UNIQUE (event_id, room_id);
1241
1242
1243
1244		ALTER TABLE ONLY event_edges
1245		ADD CONSTRAINT event_edges_event_id_prev_event_id_room_id_is_state_key UNIQUE (event_id, prev_event_id, room_id, is_state);
1246
1247
1248
1249		ALTER TABLE ONLY event_forward_extremities
1250		ADD CONSTRAINT event_forward_extremities_event_id_room_id_key UNIQUE (event_id, room_id);
1251
1252
1253
1254		ALTER TABLE ONLY event_push_actions
1255		ADD CONSTRAINT event_id_user_id_profile_tag_uniqueness UNIQUE (room_id, event_id, user_id, profile_tag);
1256
1257
1258
1259		ALTER TABLE ONLY event_json
1260		ADD CONSTRAINT event_json_event_id_key UNIQUE (event_id);
1261
1262
1263
1264		ALTER TABLE ONLY event_push_summary_stream_ordering
1265		ADD CONSTRAINT event_push_summary_stream_ordering_lock_key UNIQUE (lock);
1266
1267
1268
1269		ALTER TABLE ONLY event_reference_hashes
1270		ADD CONSTRAINT event_reference_hashes_event_id_algorithm_key UNIQUE (event_id, algorithm);
1271
1272
1273
1274		ALTER TABLE ONLY event_reports
1275		ADD CONSTRAINT event_reports_pkey PRIMARY KEY (id);
1276
1277
1278
1279		ALTER TABLE ONLY event_to_state_groups
1280		ADD CONSTRAINT event_to_state_groups_event_id_key UNIQUE (event_id);
1281
1282
1283
1284		ALTER TABLE ONLY events
1285		ADD CONSTRAINT events_event_id_key UNIQUE (event_id);
1286
1287
1288
1289		ALTER TABLE ONLY events
1290		ADD CONSTRAINT events_pkey PRIMARY KEY (stream_ordering);
1291
1292
1293
1294		ALTER TABLE ONLY ex_outlier_stream
1295		ADD CONSTRAINT ex_outlier_stream_pkey PRIMARY KEY (event_stream_ordering);
1296
1297
1298
1299		ALTER TABLE ONLY group_roles
1300		ADD CONSTRAINT group_roles_group_id_role_id_key UNIQUE (group_id, role_id);
1301
1302
1303
1304		ALTER TABLE ONLY group_room_categories
1305		ADD CONSTRAINT group_room_categories_group_id_category_id_key UNIQUE (group_id, category_id);
1306
1307
1308
1309		ALTER TABLE ONLY group_summary_roles
1310		ADD CONSTRAINT group_summary_roles_group_id_role_id_role_order_key UNIQUE (group_id, role_id, role_order);
1311
1312
1313
1314		ALTER TABLE ONLY group_summary_room_categories
1315		ADD CONSTRAINT group_summary_room_categories_group_id_category_id_cat_orde_key UNIQUE (group_id, category_id, cat_order);
1316
1317
1318
1319		ALTER TABLE ONLY group_summary_rooms
1320		ADD CONSTRAINT group_summary_rooms_group_id_category_id_room_id_room_order_key UNIQUE (group_id, category_id, room_id, room_order);
1321
1322
1323
1324		ALTER TABLE ONLY guest_access
1325		ADD CONSTRAINT guest_access_event_id_key UNIQUE (event_id);
1326
1327
1328
1329		ALTER TABLE ONLY history_visibility
1330		ADD CONSTRAINT history_visibility_event_id_key UNIQUE (event_id);
1331
1332
1333
1334		ALTER TABLE ONLY local_media_repository
1335		ADD CONSTRAINT local_media_repository_media_id_key UNIQUE (media_id);
1336
1337
1338
1339		ALTER TABLE ONLY local_media_repository_thumbnails
1340		ADD CONSTRAINT local_media_repository_thumbn_media_id_thumbnail_width_thum_key UNIQUE (media_id, thumbnail_width, thumbnail_height, thumbnail_type);
1341
1342
1343
1344		ALTER TABLE ONLY user_threepids
1345		ADD CONSTRAINT medium_address UNIQUE (medium, address);
1346
1347
1348
1349		ALTER TABLE ONLY open_id_tokens
1350		ADD CONSTRAINT open_id_tokens_pkey PRIMARY KEY (token);
1351
1352
1353
1354		ALTER TABLE ONLY presence_allow_inbound
1355		ADD CONSTRAINT presence_allow_inbound_observed_user_id_observer_user_id_key UNIQUE (observed_user_id, observer_user_id);
1356
1357
1358
1359		ALTER TABLE ONLY presence
1360		ADD CONSTRAINT presence_user_id_key UNIQUE (user_id);
1361
1362
1363
1364		ALTER TABLE ONLY account_data_max_stream_id
1365		ADD CONSTRAINT private_user_data_max_stream_id_lock_key UNIQUE (lock);
1366
1367
1368
1369		ALTER TABLE ONLY profiles
1370		ADD CONSTRAINT profiles_user_id_key UNIQUE (user_id);
1371
1372
1373
1374		ALTER TABLE ONLY push_rules_enable
1375		ADD CONSTRAINT push_rules_enable_pkey PRIMARY KEY (id);
1376
1377
1378
1379		ALTER TABLE ONLY push_rules_enable
1380		ADD CONSTRAINT push_rules_enable_user_name_rule_id_key UNIQUE (user_name, rule_id);
1381
1382
1383
1384		ALTER TABLE ONLY push_rules
1385		ADD CONSTRAINT push_rules_pkey PRIMARY KEY (id);
1386
1387
1388
1389		ALTER TABLE ONLY push_rules
1390		ADD CONSTRAINT push_rules_user_name_rule_id_key UNIQUE (user_name, rule_id);
1391
1392
1393
1394		ALTER TABLE ONLY pusher_throttle
1395		ADD CONSTRAINT pusher_throttle_pkey PRIMARY KEY (pusher, room_id);
1396
1397
1398
1399		ALTER TABLE ONLY pushers
1400		ADD CONSTRAINT pushers2_app_id_pushkey_user_name_key UNIQUE (app_id, pushkey, user_name);
1401
1402
1403
1404		ALTER TABLE ONLY pushers
1405		ADD CONSTRAINT pushers2_pkey PRIMARY KEY (id);
1406
1407
1408
1409		ALTER TABLE ONLY receipts_graph
1410		ADD CONSTRAINT receipts_graph_uniqueness UNIQUE (room_id, receipt_type, user_id);
1411
1412
1413
1414		ALTER TABLE ONLY receipts_linearized
1415		ADD CONSTRAINT receipts_linearized_uniqueness UNIQUE (room_id, receipt_type, user_id);
1416
1417
1418
1419		ALTER TABLE ONLY received_transactions
1420		ADD CONSTRAINT received_transactions_transaction_id_origin_key UNIQUE (transaction_id, origin);
1421
1422
1423
1424		ALTER TABLE ONLY redactions
1425		ADD CONSTRAINT redactions_event_id_key UNIQUE (event_id);
1426
1427
1428
1429		ALTER TABLE ONLY rejections
1430		ADD CONSTRAINT rejections_event_id_key UNIQUE (event_id);
1431
1432
1433
1434		ALTER TABLE ONLY remote_media_cache
1435		ADD CONSTRAINT remote_media_cache_media_origin_media_id_key UNIQUE (media_origin, media_id);
1436
1437
1438
1439		ALTER TABLE ONLY remote_media_cache_thumbnails
1440		ADD CONSTRAINT remote_media_cache_thumbnails_media_origin_media_id_thumbna_key UNIQUE (media_origin, media_id, thumbnail_width, thumbnail_height, thumbnail_type);
1441
1442
1443
1444		ALTER TABLE ONLY room_account_data
1445		ADD CONSTRAINT room_account_data_uniqueness UNIQUE (user_id, room_id, account_data_type);
1446
1447
1448
1449		ALTER TABLE ONLY room_aliases
1450		ADD CONSTRAINT room_aliases_room_alias_key UNIQUE (room_alias);
1451
1452
1453
1454		ALTER TABLE ONLY room_depth
1455		ADD CONSTRAINT room_depth_room_id_key UNIQUE (room_id);
1456
1457
1458
1459		ALTER TABLE ONLY room_memberships
1460		ADD CONSTRAINT room_memberships_event_id_key UNIQUE (event_id);
1461
1462
1463
1464		ALTER TABLE ONLY room_names
1465		ADD CONSTRAINT room_names_event_id_key UNIQUE (event_id);
1466
1467
1468
1469		ALTER TABLE ONLY room_tags_revisions
1470		ADD CONSTRAINT room_tag_revisions_uniqueness UNIQUE (user_id, room_id);
1471
1472
1473
1474		ALTER TABLE ONLY room_tags
1475		ADD CONSTRAINT room_tag_uniqueness UNIQUE (user_id, room_id, tag);
1476
1477
1478
1479		ALTER TABLE ONLY rooms
1480		ADD CONSTRAINT rooms_pkey PRIMARY KEY (room_id);
1481
1482
1483
1484		ALTER TABLE ONLY server_keys_json
1485		ADD CONSTRAINT server_keys_json_uniqueness UNIQUE (server_name, key_id, from_server);
1486
1487
1488
1489		ALTER TABLE ONLY server_signature_keys
1490		ADD CONSTRAINT server_signature_keys_server_name_key_id_key UNIQUE (server_name, key_id);
1491
1492
1493
1494		ALTER TABLE ONLY state_events
1495		ADD CONSTRAINT state_events_event_id_key UNIQUE (event_id);
1496
1497
1498
1499		ALTER TABLE ONLY state_groups
1500		ADD CONSTRAINT state_groups_pkey PRIMARY KEY (id);
1501
1502
1503
1504		ALTER TABLE ONLY stats_stream_pos
1505		ADD CONSTRAINT stats_stream_pos_lock_key UNIQUE (lock);
1506
1507
1508
1509		ALTER TABLE ONLY topics
1510		ADD CONSTRAINT topics_event_id_key UNIQUE (event_id);
1511
1512
1513
1514		ALTER TABLE ONLY user_directory_stream_pos
1515		ADD CONSTRAINT user_directory_stream_pos_lock_key UNIQUE (lock);
1516
1517
1518
1519		ALTER TABLE ONLY users
1520		ADD CONSTRAINT users_name_key UNIQUE (name);
1521
1522
1523
1524		CREATE INDEX access_tokens_device_id ON access_tokens USING btree (user_id, device_id);
1525
1526
1527
1528		CREATE INDEX account_data_stream_id ON account_data USING btree (user_id, stream_id);
1529
1530
1531
1532		CREATE INDEX application_services_txns_id ON application_services_txns USING btree (as_id);
1533
1534
1535
1536		CREATE UNIQUE INDEX appservice_room_list_idx ON appservice_room_list USING btree (appservice_id, network_id, room_id);
1537
1538
1539
1540		CREATE UNIQUE INDEX blocked_rooms_idx ON blocked_rooms USING btree (room_id);
1541
1542
1543
1544		CREATE INDEX cache_invalidation_stream_id ON cache_invalidation_stream USING btree (stream_id);
1545
1546
1547
1548		CREATE INDEX current_state_delta_stream_idx ON current_state_delta_stream USING btree (stream_id);
1549
1550
1551
1552		CREATE INDEX current_state_events_member_index ON current_state_events USING btree (state_key) WHERE (type = 'm.room.member'::text);
1553
1554
1555
1556		CREATE INDEX deleted_pushers_stream_id ON deleted_pushers USING btree (stream_id);
1557
1558
1559
1560		CREATE INDEX device_federation_inbox_sender_id ON device_federation_inbox USING btree (origin, message_id);
1561
1562
1563
1564		CREATE INDEX device_federation_outbox_destination_id ON device_federation_outbox USING btree (destination, stream_id);
1565
1566
1567
1568		CREATE INDEX device_federation_outbox_id ON device_federation_outbox USING btree (stream_id);
1569
1570
1571
1572		CREATE INDEX device_inbox_stream_id_user_id ON device_inbox USING btree (stream_id, user_id);
1573
1574
1575
1576		CREATE INDEX device_inbox_user_stream_id ON device_inbox USING btree (user_id, device_id, stream_id);
1577
1578
1579
1580		CREATE INDEX device_lists_outbound_last_success_idx ON device_lists_outbound_last_success USING btree (destination, user_id, stream_id);
1581
1582
1583
1584		CREATE INDEX device_lists_outbound_pokes_id ON device_lists_outbound_pokes USING btree (destination, stream_id);
1585
1586
1587
1588		CREATE INDEX device_lists_outbound_pokes_stream ON device_lists_outbound_pokes USING btree (stream_id);
1589
1590
1591
1592		CREATE INDEX device_lists_outbound_pokes_user ON device_lists_outbound_pokes USING btree (destination, user_id);
1593
1594
1595
1596		CREATE UNIQUE INDEX device_lists_remote_cache_unique_id ON device_lists_remote_cache USING btree (user_id, device_id);
1597
1598
1599
1600		CREATE UNIQUE INDEX device_lists_remote_extremeties_unique_idx ON device_lists_remote_extremeties USING btree (user_id);
1601
1602
1603
1604		CREATE INDEX device_lists_stream_id ON device_lists_stream USING btree (stream_id, user_id);
1605
1606
1607
1608		CREATE INDEX device_lists_stream_user_id ON device_lists_stream USING btree (user_id, device_id);
1609
1610
1611
1612		CREATE UNIQUE INDEX e2e_room_keys_idx ON e2e_room_keys USING btree (user_id, room_id, session_id);
1613
1614
1615
1616		CREATE UNIQUE INDEX e2e_room_keys_versions_idx ON e2e_room_keys_versions USING btree (user_id, version);
1617
1618
1619
1620		CREATE UNIQUE INDEX erased_users_user ON erased_users USING btree (user_id);
1621
1622
1623
1624		CREATE INDEX ev_b_extrem_id ON event_backward_extremities USING btree (event_id);
1625
1626
1627
1628		CREATE INDEX ev_b_extrem_room ON event_backward_extremities USING btree (room_id);
1629
1630
1631
1632		CREATE INDEX ev_edges_id ON event_edges USING btree (event_id);
1633
1634
1635
1636		CREATE INDEX ev_edges_prev_id ON event_edges USING btree (prev_event_id);
1637
1638
1639
1640		CREATE INDEX ev_extrem_id ON event_forward_extremities USING btree (event_id);
1641
1642
1643
1644		CREATE INDEX ev_extrem_room ON event_forward_extremities USING btree (room_id);
1645
1646
1647
1648		CREATE INDEX evauth_edges_id ON event_auth USING btree (event_id);
1649
1650
1651
1652		CREATE INDEX event_contains_url_index ON events USING btree (room_id, topological_ordering, stream_ordering) WHERE ((contains_url = true) AND (outlier = false));
1653
1654
1655
1656		CREATE INDEX event_json_room_id ON event_json USING btree (room_id);
1657
1658
1659
1660		CREATE INDEX event_push_actions_highlights_index ON event_push_actions USING btree (user_id, room_id, topological_ordering, stream_ordering) WHERE (highlight = 1);
1661
1662
1663
1664		CREATE INDEX event_push_actions_rm_tokens ON event_push_actions USING btree (user_id, room_id, topological_ordering, stream_ordering);
1665
1666
1667
1668		CREATE INDEX event_push_actions_room_id_user_id ON event_push_actions USING btree (room_id, user_id);
1669
1670
1671
1672		CREATE INDEX event_push_actions_staging_id ON event_push_actions_staging USING btree (event_id);
1673
1674
1675
1676		CREATE INDEX event_push_actions_stream_ordering ON event_push_actions USING btree (stream_ordering, user_id);
1677
1678
1679
1680		CREATE INDEX event_push_actions_u_highlight ON event_push_actions USING btree (user_id, stream_ordering);
1681
1682
1683
1684		CREATE INDEX event_push_summary_user_rm ON event_push_summary USING btree (user_id, room_id);
1685
1686
1687
1688		CREATE INDEX event_reference_hashes_id ON event_reference_hashes USING btree (event_id);
1689
1690
1691
1692		CREATE UNIQUE INDEX event_relations_id ON event_relations USING btree (event_id);
1693
1694
1695
1696		CREATE INDEX event_relations_relates ON event_relations USING btree (relates_to_id, relation_type, aggregation_key);
1697
1698
1699
1700		CREATE INDEX event_search_ev_ridx ON event_search USING btree (room_id);
1701
1702
1703
1704		CREATE UNIQUE INDEX event_search_event_id_idx ON event_search USING btree (event_id);
1705
1706
1707
1708		CREATE INDEX event_search_fts_idx ON event_search USING gin (vector);
1709
1710
1711
1712		CREATE INDEX event_to_state_groups_sg_index ON event_to_state_groups USING btree (state_group);
1713
1714
1715
1716		CREATE INDEX events_order_room ON events USING btree (room_id, topological_ordering, stream_ordering);
1717
1718
1719
1720		CREATE INDEX events_room_stream ON events USING btree (room_id, stream_ordering);
1721
1722
1723
1724		CREATE INDEX events_ts ON events USING btree (origin_server_ts, stream_ordering);
1725
1726
1727
1728		CREATE INDEX group_attestations_remote_g_idx ON group_attestations_remote USING btree (group_id, user_id);
1729
1730
1731
1732		CREATE INDEX group_attestations_remote_u_idx ON group_attestations_remote USING btree (user_id);
1733
1734
1735
1736		CREATE INDEX group_attestations_remote_v_idx ON group_attestations_remote USING btree (valid_until_ms);
1737
1738
1739
1740		CREATE INDEX group_attestations_renewals_g_idx ON group_attestations_renewals USING btree (group_id, user_id);
1741
1742
1743
1744		CREATE INDEX group_attestations_renewals_u_idx ON group_attestations_renewals USING btree (user_id);
1745
1746
1747
1748		CREATE INDEX group_attestations_renewals_v_idx ON group_attestations_renewals USING btree (valid_until_ms);
1749
1750
1751
1752		CREATE UNIQUE INDEX group_invites_g_idx ON group_invites USING btree (group_id, user_id);
1753
1754
1755
1756		CREATE INDEX group_invites_u_idx ON group_invites USING btree (user_id);
1757
1758
1759
1760		CREATE UNIQUE INDEX group_rooms_g_idx ON group_rooms USING btree (group_id, room_id);
1761
1762
1763
1764		CREATE INDEX group_rooms_r_idx ON group_rooms USING btree (room_id);
1765
1766
1767
1768		CREATE UNIQUE INDEX group_summary_rooms_g_idx ON group_summary_rooms USING btree (group_id, room_id, category_id);
1769
1770
1771
1772		CREATE INDEX group_summary_users_g_idx ON group_summary_users USING btree (group_id);
1773
1774
1775
1776		CREATE UNIQUE INDEX group_users_g_idx ON group_users USING btree (group_id, user_id);
1777
1778
1779
1780		CREATE INDEX group_users_u_idx ON group_users USING btree (user_id);
1781
1782
1783
1784		CREATE UNIQUE INDEX groups_idx ON groups USING btree (group_id);
1785
1786
1787
1788		CREATE INDEX local_group_membership_g_idx ON local_group_membership USING btree (group_id);
1789
1790
1791
1792		CREATE INDEX local_group_membership_u_idx ON local_group_membership USING btree (user_id, group_id);
1793
1794
1795
1796		CREATE INDEX local_invites_for_user_idx ON local_invites USING btree (invitee, locally_rejected, replaced_by, room_id);
1797
1798
1799
1800		CREATE INDEX local_invites_id ON local_invites USING btree (stream_id);
1801
1802
1803
1804		CREATE INDEX local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails USING btree (media_id);
1805
1806
1807
1808		CREATE INDEX local_media_repository_url_cache_by_url_download_ts ON local_media_repository_url_cache USING btree (url, download_ts);
1809
1810
1811
1812		CREATE INDEX local_media_repository_url_cache_expires_idx ON local_media_repository_url_cache USING btree (expires_ts);
1813
1814
1815
1816		CREATE INDEX local_media_repository_url_cache_media_idx ON local_media_repository_url_cache USING btree (media_id);
1817
1818
1819
1820		CREATE INDEX local_media_repository_url_idx ON local_media_repository USING btree (created_ts) WHERE (url_cache IS NOT NULL);
1821
1822
1823
1824		CREATE INDEX monthly_active_users_time_stamp ON monthly_active_users USING btree ("timestamp");
1825
1826
1827
1828		CREATE UNIQUE INDEX monthly_active_users_users ON monthly_active_users USING btree (user_id);
1829
1830
1831
1832		CREATE INDEX open_id_tokens_ts_valid_until_ms ON open_id_tokens USING btree (ts_valid_until_ms);
1833
1834
1835
1836		CREATE INDEX presence_stream_id ON presence_stream USING btree (stream_id, user_id);
1837
1838
1839
1840		CREATE INDEX presence_stream_user_id ON presence_stream USING btree (user_id);
1841
1842
1843
1844		CREATE INDEX public_room_index ON rooms USING btree (is_public);
1845
1846
1847
1848		CREATE INDEX public_room_list_stream_idx ON public_room_list_stream USING btree (stream_id);
1849
1850
1851
1852		CREATE INDEX public_room_list_stream_rm_idx ON public_room_list_stream USING btree (room_id, stream_id);
1853
1854
1855
1856		CREATE INDEX push_rules_enable_user_name ON push_rules_enable USING btree (user_name);
1857
1858
1859
1860		CREATE INDEX push_rules_stream_id ON push_rules_stream USING btree (stream_id);
1861
1862
1863
1864		CREATE INDEX push_rules_stream_user_stream_id ON push_rules_stream USING btree (user_id, stream_id);
1865
1866
1867
1868		CREATE INDEX push_rules_user_name ON push_rules USING btree (user_name);
1869
1870
1871
1872		CREATE UNIQUE INDEX ratelimit_override_idx ON ratelimit_override USING btree (user_id);
1873
1874
1875
1876		CREATE INDEX receipts_linearized_id ON receipts_linearized USING btree (stream_id);
1877
1878
1879
1880		CREATE INDEX receipts_linearized_room_stream ON receipts_linearized USING btree (room_id, stream_id);
1881
1882
1883
1884		CREATE INDEX receipts_linearized_user ON receipts_linearized USING btree (user_id);
1885
1886
1887
1888		CREATE INDEX received_transactions_ts ON received_transactions USING btree (ts);
1889
1890
1891
1892		CREATE INDEX redactions_redacts ON redactions USING btree (redacts);
1893
1894
1895
1896		CREATE INDEX remote_profile_cache_time ON remote_profile_cache USING btree (last_check);
1897
1898
1899
1900		CREATE UNIQUE INDEX remote_profile_cache_user_id ON remote_profile_cache USING btree (user_id);
1901
1902
1903
1904		CREATE INDEX room_account_data_stream_id ON room_account_data USING btree (user_id, stream_id);
1905
1906
1907
1908		CREATE INDEX room_alias_servers_alias ON room_alias_servers USING btree (room_alias);
1909
1910
1911
1912		CREATE INDEX room_aliases_id ON room_aliases USING btree (room_id);
1913
1914
1915
1916		CREATE INDEX room_depth_room ON room_depth USING btree (room_id);
1917
1918
1919
1920		CREATE INDEX room_memberships_room_id ON room_memberships USING btree (room_id);
1921
1922
1923
1924		CREATE INDEX room_memberships_user_id ON room_memberships USING btree (user_id);
1925
1926
1927
1928		CREATE INDEX room_names_room_id ON room_names USING btree (room_id);
1929
1930
1931
1932		CREATE UNIQUE INDEX room_state_room ON room_state USING btree (room_id);
1933
1934
1935
1936		CREATE UNIQUE INDEX room_stats_earliest_token_idx ON room_stats_earliest_token USING btree (room_id);
1937
1938
1939
1940		CREATE UNIQUE INDEX room_stats_room_ts ON room_stats USING btree (room_id, ts);
1941
1942
1943
1944		CREATE INDEX state_group_edges_idx ON state_group_edges USING btree (state_group);
1945
1946
1947
1948		CREATE INDEX state_group_edges_prev_idx ON state_group_edges USING btree (prev_state_group);
1949
1950
1951
1952		CREATE INDEX state_groups_state_type_idx ON state_groups_state USING btree (state_group, type, state_key);
1953
1954
1955
1956		CREATE INDEX stream_ordering_to_exterm_idx ON stream_ordering_to_exterm USING btree (stream_ordering);
1957
1958
1959
1960		CREATE INDEX stream_ordering_to_exterm_rm_idx ON stream_ordering_to_exterm USING btree (room_id, stream_ordering);
1961
1962
1963
1964		CREATE UNIQUE INDEX threepid_guest_access_tokens_index ON threepid_guest_access_tokens USING btree (medium, address);
1965
1966
1967
1968		CREATE INDEX topics_room_id ON topics USING btree (room_id);
1969
1970
1971
1972		CREATE INDEX user_daily_visits_ts_idx ON user_daily_visits USING btree ("timestamp");
1973
1974
1975
1976		CREATE INDEX user_daily_visits_uts_idx ON user_daily_visits USING btree (user_id, "timestamp");
1977
1978
1979
1980		CREATE INDEX user_directory_room_idx ON user_directory USING btree (room_id);
1981
1982
1983
1984		CREATE INDEX user_directory_search_fts_idx ON user_directory_search USING gin (vector);
1985
1986
1987
1988		CREATE UNIQUE INDEX user_directory_search_user_idx ON user_directory_search USING btree (user_id);
1989
1990
1991
1992		CREATE UNIQUE INDEX user_directory_user_idx ON user_directory USING btree (user_id);
1993
1994
1995
1996		CREATE INDEX user_filters_by_user_id_filter_id ON user_filters USING btree (user_id, filter_id);
1997
1998
1999
2000		CREATE INDEX user_ips_device_id ON user_ips USING btree (user_id, device_id, last_seen);
2001
2002
2003
2004		CREATE INDEX user_ips_last_seen ON user_ips USING btree (user_id, last_seen);
2005
2006
2007
2008		CREATE INDEX user_ips_last_seen_only ON user_ips USING btree (last_seen);
2009
2010
2011
2012		CREATE UNIQUE INDEX user_ips_user_token_ip_unique_index ON user_ips USING btree (user_id, access_token, ip);
2013
2014
2015
2016		CREATE UNIQUE INDEX user_stats_user_ts ON user_stats USING btree (user_id, ts);
2017
2018
2019
2020		CREATE UNIQUE INDEX user_threepid_id_server_idx ON user_threepid_id_server USING btree (user_id, medium, address, id_server);
2021
2022
2023
2024		CREATE INDEX user_threepids_medium_address ON user_threepids USING btree (medium, address);
2025
2026
2027
2028		CREATE INDEX user_threepids_user_id ON user_threepids USING btree (user_id);
2029
2030
2031
2032		CREATE INDEX users_creation_ts ON users USING btree (creation_ts);
2033
2034
2035
2036		CREATE UNIQUE INDEX users_in_public_rooms_u_idx ON users_in_public_rooms USING btree (user_id, room_id);
2037
2038
2039
2040		CREATE INDEX users_who_share_private_rooms_o_idx ON users_who_share_private_rooms USING btree (other_user_id);
2041
2042
2043
2044		CREATE INDEX users_who_share_private_rooms_r_idx ON users_who_share_private_rooms USING btree (room_id);
2045
2046
2047
2048		CREATE UNIQUE INDEX users_who_share_private_rooms_u_idx ON users_who_share_private_rooms USING btree (user_id, other_user_id, room_id);
2049
2050
2051

+0

-260

~~synapse/storage/schema/full_schemas/54/full.sql.sqlite~~ less more

0		CREATE TABLE application_services_state( as_id TEXT PRIMARY KEY, state VARCHAR(5), last_txn INTEGER );
1		CREATE TABLE application_services_txns( as_id TEXT NOT NULL, txn_id INTEGER NOT NULL, event_ids TEXT NOT NULL, UNIQUE(as_id, txn_id) );
2		CREATE INDEX application_services_txns_id ON application_services_txns ( as_id );
3		CREATE TABLE presence( user_id TEXT NOT NULL, state VARCHAR(20), status_msg TEXT, mtime BIGINT, UNIQUE (user_id) );
4		CREATE TABLE presence_allow_inbound( observed_user_id TEXT NOT NULL, observer_user_id TEXT NOT NULL, UNIQUE (observed_user_id, observer_user_id) );
5		CREATE TABLE users( name TEXT, password_hash TEXT, creation_ts BIGINT, admin SMALLINT DEFAULT 0 NOT NULL, upgrade_ts BIGINT, is_guest SMALLINT DEFAULT 0 NOT NULL, appservice_id TEXT, consent_version TEXT, consent_server_notice_sent TEXT, user_type TEXT DEFAULT NULL, UNIQUE(name) );
6		CREATE TABLE access_tokens( id BIGINT PRIMARY KEY, user_id TEXT NOT NULL, device_id TEXT, token TEXT NOT NULL, last_used BIGINT, UNIQUE(token) );
7		CREATE TABLE user_ips ( user_id TEXT NOT NULL, access_token TEXT NOT NULL, device_id TEXT, ip TEXT NOT NULL, user_agent TEXT NOT NULL, last_seen BIGINT NOT NULL );
8		CREATE TABLE profiles( user_id TEXT NOT NULL, displayname TEXT, avatar_url TEXT, UNIQUE(user_id) );
9		CREATE TABLE received_transactions( transaction_id TEXT, origin TEXT, ts BIGINT, response_code INTEGER, response_json bytea, has_been_referenced smallint default 0, UNIQUE (transaction_id, origin) );
10		CREATE TABLE destinations( destination TEXT PRIMARY KEY, retry_last_ts BIGINT, retry_interval INTEGER );
11		CREATE TABLE events( stream_ordering INTEGER PRIMARY KEY, topological_ordering BIGINT NOT NULL, event_id TEXT NOT NULL, type TEXT NOT NULL, room_id TEXT NOT NULL, content TEXT, unrecognized_keys TEXT, processed BOOL NOT NULL, outlier BOOL NOT NULL, depth BIGINT DEFAULT 0 NOT NULL, origin_server_ts BIGINT, received_ts BIGINT, sender TEXT, contains_url BOOLEAN, UNIQUE (event_id) );
12		CREATE INDEX events_order_room ON events ( room_id, topological_ordering, stream_ordering );
13		CREATE TABLE event_json( event_id TEXT NOT NULL, room_id TEXT NOT NULL, internal_metadata TEXT NOT NULL, json TEXT NOT NULL, format_version INTEGER, UNIQUE (event_id) );
14		CREATE INDEX event_json_room_id ON event_json(room_id);
15		CREATE TABLE state_events( event_id TEXT NOT NULL, room_id TEXT NOT NULL, type TEXT NOT NULL, state_key TEXT NOT NULL, prev_state TEXT, UNIQUE (event_id) );
16		CREATE TABLE current_state_events( event_id TEXT NOT NULL, room_id TEXT NOT NULL, type TEXT NOT NULL, state_key TEXT NOT NULL, UNIQUE (event_id), UNIQUE (room_id, type, state_key) );
17		CREATE TABLE room_memberships( event_id TEXT NOT NULL, user_id TEXT NOT NULL, sender TEXT NOT NULL, room_id TEXT NOT NULL, membership TEXT NOT NULL, forgotten INTEGER DEFAULT 0, display_name TEXT, avatar_url TEXT, UNIQUE (event_id) );
18		CREATE INDEX room_memberships_room_id ON room_memberships (room_id);
19		CREATE INDEX room_memberships_user_id ON room_memberships (user_id);
20		CREATE TABLE topics( event_id TEXT NOT NULL, room_id TEXT NOT NULL, topic TEXT NOT NULL, UNIQUE (event_id) );
21		CREATE INDEX topics_room_id ON topics(room_id);
22		CREATE TABLE room_names( event_id TEXT NOT NULL, room_id TEXT NOT NULL, name TEXT NOT NULL, UNIQUE (event_id) );
23		CREATE INDEX room_names_room_id ON room_names(room_id);
24		CREATE TABLE rooms( room_id TEXT PRIMARY KEY NOT NULL, is_public BOOL, creator TEXT );
25		CREATE TABLE server_signature_keys( server_name TEXT, key_id TEXT, from_server TEXT, ts_added_ms BIGINT, verify_key bytea, ts_valid_until_ms BIGINT, UNIQUE (server_name, key_id) );
26		CREATE TABLE rejections( event_id TEXT NOT NULL, reason TEXT NOT NULL, last_check TEXT NOT NULL, UNIQUE (event_id) );
27		CREATE TABLE push_rules ( id BIGINT PRIMARY KEY, user_name TEXT NOT NULL, rule_id TEXT NOT NULL, priority_class SMALLINT NOT NULL, priority INTEGER NOT NULL DEFAULT 0, conditions TEXT NOT NULL, actions TEXT NOT NULL, UNIQUE(user_name, rule_id) );
28		CREATE INDEX push_rules_user_name on push_rules (user_name);
29		CREATE TABLE user_filters( user_id TEXT, filter_id BIGINT, filter_json bytea );
30		CREATE INDEX user_filters_by_user_id_filter_id ON user_filters( user_id, filter_id );
31		CREATE TABLE push_rules_enable ( id BIGINT PRIMARY KEY, user_name TEXT NOT NULL, rule_id TEXT NOT NULL, enabled SMALLINT, UNIQUE(user_name, rule_id) );
32		CREATE INDEX push_rules_enable_user_name on push_rules_enable (user_name);
33		CREATE TABLE event_forward_extremities( event_id TEXT NOT NULL, room_id TEXT NOT NULL, UNIQUE (event_id, room_id) );
34		CREATE INDEX ev_extrem_room ON event_forward_extremities(room_id);
35		CREATE INDEX ev_extrem_id ON event_forward_extremities(event_id);
36		CREATE TABLE event_backward_extremities( event_id TEXT NOT NULL, room_id TEXT NOT NULL, UNIQUE (event_id, room_id) );
37		CREATE INDEX ev_b_extrem_room ON event_backward_extremities(room_id);
38		CREATE INDEX ev_b_extrem_id ON event_backward_extremities(event_id);
39		CREATE TABLE event_edges( event_id TEXT NOT NULL, prev_event_id TEXT NOT NULL, room_id TEXT NOT NULL, is_state BOOL NOT NULL, UNIQUE (event_id, prev_event_id, room_id, is_state) );
40		CREATE INDEX ev_edges_id ON event_edges(event_id);
41		CREATE INDEX ev_edges_prev_id ON event_edges(prev_event_id);
42		CREATE TABLE room_depth( room_id TEXT NOT NULL, min_depth INTEGER NOT NULL, UNIQUE (room_id) );
43		CREATE INDEX room_depth_room ON room_depth(room_id);
44		CREATE TABLE state_groups( id BIGINT PRIMARY KEY, room_id TEXT NOT NULL, event_id TEXT NOT NULL );
45		CREATE TABLE state_groups_state( state_group BIGINT NOT NULL, room_id TEXT NOT NULL, type TEXT NOT NULL, state_key TEXT NOT NULL, event_id TEXT NOT NULL );
46		CREATE TABLE event_to_state_groups( event_id TEXT NOT NULL, state_group BIGINT NOT NULL, UNIQUE (event_id) );
47		CREATE TABLE local_media_repository ( media_id TEXT, media_type TEXT, media_length INTEGER, created_ts BIGINT, upload_name TEXT, user_id TEXT, quarantined_by TEXT, url_cache TEXT, last_access_ts BIGINT, UNIQUE (media_id) );
48		CREATE TABLE local_media_repository_thumbnails ( media_id TEXT, thumbnail_width INTEGER, thumbnail_height INTEGER, thumbnail_type TEXT, thumbnail_method TEXT, thumbnail_length INTEGER, UNIQUE ( media_id, thumbnail_width, thumbnail_height, thumbnail_type ) );
49		CREATE INDEX local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails (media_id);
50		CREATE TABLE remote_media_cache ( media_origin TEXT, media_id TEXT, media_type TEXT, created_ts BIGINT, upload_name TEXT, media_length INTEGER, filesystem_id TEXT, last_access_ts BIGINT, quarantined_by TEXT, UNIQUE (media_origin, media_id) );
51		CREATE TABLE remote_media_cache_thumbnails ( media_origin TEXT, media_id TEXT, thumbnail_width INTEGER, thumbnail_height INTEGER, thumbnail_method TEXT, thumbnail_type TEXT, thumbnail_length INTEGER, filesystem_id TEXT, UNIQUE ( media_origin, media_id, thumbnail_width, thumbnail_height, thumbnail_type ) );
52		CREATE TABLE redactions ( event_id TEXT NOT NULL, redacts TEXT NOT NULL, UNIQUE (event_id) );
53		CREATE INDEX redactions_redacts ON redactions (redacts);
54		CREATE TABLE room_aliases( room_alias TEXT NOT NULL, room_id TEXT NOT NULL, creator TEXT, UNIQUE (room_alias) );
55		CREATE INDEX room_aliases_id ON room_aliases(room_id);
56		CREATE TABLE room_alias_servers( room_alias TEXT NOT NULL, server TEXT NOT NULL );
57		CREATE INDEX room_alias_servers_alias ON room_alias_servers(room_alias);
58		CREATE TABLE event_reference_hashes ( event_id TEXT, algorithm TEXT, hash bytea, UNIQUE (event_id, algorithm) );
59		CREATE INDEX event_reference_hashes_id ON event_reference_hashes(event_id);
60		CREATE TABLE IF NOT EXISTS "server_keys_json" ( server_name TEXT NOT NULL, key_id TEXT NOT NULL, from_server TEXT NOT NULL, ts_added_ms BIGINT NOT NULL, ts_valid_until_ms BIGINT NOT NULL, key_json bytea NOT NULL, CONSTRAINT server_keys_json_uniqueness UNIQUE (server_name, key_id, from_server) );
61		CREATE TABLE e2e_device_keys_json ( user_id TEXT NOT NULL, device_id TEXT NOT NULL, ts_added_ms BIGINT NOT NULL, key_json TEXT NOT NULL, CONSTRAINT e2e_device_keys_json_uniqueness UNIQUE (user_id, device_id) );
62		CREATE TABLE e2e_one_time_keys_json ( user_id TEXT NOT NULL, device_id TEXT NOT NULL, algorithm TEXT NOT NULL, key_id TEXT NOT NULL, ts_added_ms BIGINT NOT NULL, key_json TEXT NOT NULL, CONSTRAINT e2e_one_time_keys_json_uniqueness UNIQUE (user_id, device_id, algorithm, key_id) );
63		CREATE TABLE receipts_graph( room_id TEXT NOT NULL, receipt_type TEXT NOT NULL, user_id TEXT NOT NULL, event_ids TEXT NOT NULL, data TEXT NOT NULL, CONSTRAINT receipts_graph_uniqueness UNIQUE (room_id, receipt_type, user_id) );
64		CREATE TABLE receipts_linearized ( stream_id BIGINT NOT NULL, room_id TEXT NOT NULL, receipt_type TEXT NOT NULL, user_id TEXT NOT NULL, event_id TEXT NOT NULL, data TEXT NOT NULL, CONSTRAINT receipts_linearized_uniqueness UNIQUE (room_id, receipt_type, user_id) );
65		CREATE INDEX receipts_linearized_id ON receipts_linearized( stream_id );
66		CREATE INDEX receipts_linearized_room_stream ON receipts_linearized( room_id, stream_id );
67		CREATE TABLE IF NOT EXISTS "user_threepids" ( user_id TEXT NOT NULL, medium TEXT NOT NULL, address TEXT NOT NULL, validated_at BIGINT NOT NULL, added_at BIGINT NOT NULL, CONSTRAINT medium_address UNIQUE (medium, address) );
68		CREATE INDEX user_threepids_user_id ON user_threepids(user_id);
69		CREATE TABLE background_updates( update_name TEXT NOT NULL, progress_json TEXT NOT NULL, depends_on TEXT, CONSTRAINT background_updates_uniqueness UNIQUE (update_name) );
70		CREATE VIRTUAL TABLE event_search USING fts4 ( event_id, room_id, sender, key, value )
71		/* event_search(event_id,room_id,sender,"key",value) */;
72		CREATE TABLE IF NOT EXISTS 'event_search_content'(docid INTEGER PRIMARY KEY, 'c0event_id', 'c1room_id', 'c2sender', 'c3key', 'c4value');
73		CREATE TABLE IF NOT EXISTS 'event_search_segments'(blockid INTEGER PRIMARY KEY, block BLOB);
74		CREATE TABLE IF NOT EXISTS 'event_search_segdir'(level INTEGER,idx INTEGER,start_block INTEGER,leaves_end_block INTEGER,end_block INTEGER,root BLOB,PRIMARY KEY(level, idx));
75		CREATE TABLE IF NOT EXISTS 'event_search_docsize'(docid INTEGER PRIMARY KEY, size BLOB);
76		CREATE TABLE IF NOT EXISTS 'event_search_stat'(id INTEGER PRIMARY KEY, value BLOB);
77		CREATE TABLE guest_access( event_id TEXT NOT NULL, room_id TEXT NOT NULL, guest_access TEXT NOT NULL, UNIQUE (event_id) );
78		CREATE TABLE history_visibility( event_id TEXT NOT NULL, room_id TEXT NOT NULL, history_visibility TEXT NOT NULL, UNIQUE (event_id) );
79		CREATE TABLE room_tags( user_id TEXT NOT NULL, room_id TEXT NOT NULL, tag TEXT NOT NULL, content TEXT NOT NULL, CONSTRAINT room_tag_uniqueness UNIQUE (user_id, room_id, tag) );
80		CREATE TABLE room_tags_revisions ( user_id TEXT NOT NULL, room_id TEXT NOT NULL, stream_id BIGINT NOT NULL, CONSTRAINT room_tag_revisions_uniqueness UNIQUE (user_id, room_id) );
81		CREATE TABLE IF NOT EXISTS "account_data_max_stream_id"( Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, stream_id BIGINT NOT NULL, CHECK (Lock='X') );
82		CREATE TABLE account_data( user_id TEXT NOT NULL, account_data_type TEXT NOT NULL, stream_id BIGINT NOT NULL, content TEXT NOT NULL, CONSTRAINT account_data_uniqueness UNIQUE (user_id, account_data_type) );
83		CREATE TABLE room_account_data( user_id TEXT NOT NULL, room_id TEXT NOT NULL, account_data_type TEXT NOT NULL, stream_id BIGINT NOT NULL, content TEXT NOT NULL, CONSTRAINT room_account_data_uniqueness UNIQUE (user_id, room_id, account_data_type) );
84		CREATE INDEX account_data_stream_id on account_data(user_id, stream_id);
85		CREATE INDEX room_account_data_stream_id on room_account_data(user_id, stream_id);
86		CREATE INDEX events_ts ON events(origin_server_ts, stream_ordering);
87		CREATE TABLE event_push_actions( room_id TEXT NOT NULL, event_id TEXT NOT NULL, user_id TEXT NOT NULL, profile_tag VARCHAR(32), actions TEXT NOT NULL, topological_ordering BIGINT, stream_ordering BIGINT, notif SMALLINT, highlight SMALLINT, CONSTRAINT event_id_user_id_profile_tag_uniqueness UNIQUE (room_id, event_id, user_id, profile_tag) );
88		CREATE INDEX event_push_actions_room_id_user_id on event_push_actions(room_id, user_id);
89		CREATE INDEX events_room_stream on events(room_id, stream_ordering);
90		CREATE INDEX public_room_index on rooms(is_public);
91		CREATE INDEX receipts_linearized_user ON receipts_linearized( user_id );
92		CREATE INDEX event_push_actions_rm_tokens on event_push_actions( user_id, room_id, topological_ordering, stream_ordering );
93		CREATE TABLE presence_stream( stream_id BIGINT, user_id TEXT, state TEXT, last_active_ts BIGINT, last_federation_update_ts BIGINT, last_user_sync_ts BIGINT, status_msg TEXT, currently_active BOOLEAN );
94		CREATE INDEX presence_stream_id ON presence_stream(stream_id, user_id);
95		CREATE INDEX presence_stream_user_id ON presence_stream(user_id);
96		CREATE TABLE push_rules_stream( stream_id BIGINT NOT NULL, event_stream_ordering BIGINT NOT NULL, user_id TEXT NOT NULL, rule_id TEXT NOT NULL, op TEXT NOT NULL, priority_class SMALLINT, priority INTEGER, conditions TEXT, actions TEXT );
97		CREATE INDEX push_rules_stream_id ON push_rules_stream(stream_id);
98		CREATE INDEX push_rules_stream_user_stream_id on push_rules_stream(user_id, stream_id);
99		CREATE TABLE ex_outlier_stream( event_stream_ordering BIGINT PRIMARY KEY NOT NULL, event_id TEXT NOT NULL, state_group BIGINT NOT NULL );
100		CREATE TABLE threepid_guest_access_tokens( medium TEXT, address TEXT, guest_access_token TEXT, first_inviter TEXT );
101		CREATE UNIQUE INDEX threepid_guest_access_tokens_index ON threepid_guest_access_tokens(medium, address);
102		CREATE TABLE local_invites( stream_id BIGINT NOT NULL, inviter TEXT NOT NULL, invitee TEXT NOT NULL, event_id TEXT NOT NULL, room_id TEXT NOT NULL, locally_rejected TEXT, replaced_by TEXT );
103		CREATE INDEX local_invites_id ON local_invites(stream_id);
104		CREATE INDEX local_invites_for_user_idx ON local_invites(invitee, locally_rejected, replaced_by, room_id);
105		CREATE INDEX event_push_actions_stream_ordering on event_push_actions( stream_ordering, user_id );
106		CREATE TABLE open_id_tokens ( token TEXT NOT NULL PRIMARY KEY, ts_valid_until_ms bigint NOT NULL, user_id TEXT NOT NULL, UNIQUE (token) );
107		CREATE INDEX open_id_tokens_ts_valid_until_ms ON open_id_tokens(ts_valid_until_ms);
108		CREATE TABLE pusher_throttle( pusher BIGINT NOT NULL, room_id TEXT NOT NULL, last_sent_ts BIGINT, throttle_ms BIGINT, PRIMARY KEY (pusher, room_id) );
109		CREATE TABLE event_reports( id BIGINT NOT NULL PRIMARY KEY, received_ts BIGINT NOT NULL, room_id TEXT NOT NULL, event_id TEXT NOT NULL, user_id TEXT NOT NULL, reason TEXT, content TEXT );
110		CREATE TABLE devices ( user_id TEXT NOT NULL, device_id TEXT NOT NULL, display_name TEXT, CONSTRAINT device_uniqueness UNIQUE (user_id, device_id) );
111		CREATE TABLE appservice_stream_position( Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, stream_ordering BIGINT, CHECK (Lock='X') );
112		CREATE TABLE device_inbox ( user_id TEXT NOT NULL, device_id TEXT NOT NULL, stream_id BIGINT NOT NULL, message_json TEXT NOT NULL );
113		CREATE INDEX device_inbox_user_stream_id ON device_inbox(user_id, device_id, stream_id);
114		CREATE INDEX received_transactions_ts ON received_transactions(ts);
115		CREATE TABLE device_federation_outbox ( destination TEXT NOT NULL, stream_id BIGINT NOT NULL, queued_ts BIGINT NOT NULL, messages_json TEXT NOT NULL );
116		CREATE INDEX device_federation_outbox_destination_id ON device_federation_outbox(destination, stream_id);
117		CREATE TABLE device_federation_inbox ( origin TEXT NOT NULL, message_id TEXT NOT NULL, received_ts BIGINT NOT NULL );
118		CREATE INDEX device_federation_inbox_sender_id ON device_federation_inbox(origin, message_id);
119		CREATE TABLE device_max_stream_id ( stream_id BIGINT NOT NULL );
120		CREATE TABLE public_room_list_stream ( stream_id BIGINT NOT NULL, room_id TEXT NOT NULL, visibility BOOLEAN NOT NULL , appservice_id TEXT, network_id TEXT);
121		CREATE INDEX public_room_list_stream_idx on public_room_list_stream( stream_id );
122		CREATE INDEX public_room_list_stream_rm_idx on public_room_list_stream( room_id, stream_id );
123		CREATE TABLE state_group_edges( state_group BIGINT NOT NULL, prev_state_group BIGINT NOT NULL );
124		CREATE INDEX state_group_edges_idx ON state_group_edges(state_group);
125		CREATE INDEX state_group_edges_prev_idx ON state_group_edges(prev_state_group);
126		CREATE TABLE stream_ordering_to_exterm ( stream_ordering BIGINT NOT NULL, room_id TEXT NOT NULL, event_id TEXT NOT NULL );
127		CREATE INDEX stream_ordering_to_exterm_idx on stream_ordering_to_exterm( stream_ordering );
128		CREATE INDEX stream_ordering_to_exterm_rm_idx on stream_ordering_to_exterm( room_id, stream_ordering );
129		CREATE TABLE IF NOT EXISTS "event_auth"( event_id TEXT NOT NULL, auth_id TEXT NOT NULL, room_id TEXT NOT NULL );
130		CREATE INDEX evauth_edges_id ON event_auth(event_id);
131		CREATE INDEX user_threepids_medium_address on user_threepids (medium, address);
132		CREATE TABLE appservice_room_list( appservice_id TEXT NOT NULL, network_id TEXT NOT NULL, room_id TEXT NOT NULL );
133		CREATE UNIQUE INDEX appservice_room_list_idx ON appservice_room_list( appservice_id, network_id, room_id );
134		CREATE INDEX device_federation_outbox_id ON device_federation_outbox(stream_id);
135		CREATE TABLE federation_stream_position( type TEXT NOT NULL, stream_id INTEGER NOT NULL );
136		CREATE TABLE device_lists_remote_cache ( user_id TEXT NOT NULL, device_id TEXT NOT NULL, content TEXT NOT NULL );
137		CREATE TABLE device_lists_remote_extremeties ( user_id TEXT NOT NULL, stream_id TEXT NOT NULL );
138		CREATE TABLE device_lists_stream ( stream_id BIGINT NOT NULL, user_id TEXT NOT NULL, device_id TEXT NOT NULL );
139		CREATE INDEX device_lists_stream_id ON device_lists_stream(stream_id, user_id);
140		CREATE TABLE device_lists_outbound_pokes ( destination TEXT NOT NULL, stream_id BIGINT NOT NULL, user_id TEXT NOT NULL, device_id TEXT NOT NULL, sent BOOLEAN NOT NULL, ts BIGINT NOT NULL );
141		CREATE INDEX device_lists_outbound_pokes_id ON device_lists_outbound_pokes(destination, stream_id);
142		CREATE INDEX device_lists_outbound_pokes_user ON device_lists_outbound_pokes(destination, user_id);
143		CREATE TABLE event_push_summary ( user_id TEXT NOT NULL, room_id TEXT NOT NULL, notif_count BIGINT NOT NULL, stream_ordering BIGINT NOT NULL );
144		CREATE INDEX event_push_summary_user_rm ON event_push_summary(user_id, room_id);
145		CREATE TABLE event_push_summary_stream_ordering ( Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, stream_ordering BIGINT NOT NULL, CHECK (Lock='X') );
146		CREATE TABLE IF NOT EXISTS "pushers" ( id BIGINT PRIMARY KEY, user_name TEXT NOT NULL, access_token BIGINT DEFAULT NULL, profile_tag TEXT NOT NULL, kind TEXT NOT NULL, app_id TEXT NOT NULL, app_display_name TEXT NOT NULL, device_display_name TEXT NOT NULL, pushkey TEXT NOT NULL, ts BIGINT NOT NULL, lang TEXT, data TEXT, last_stream_ordering INTEGER, last_success BIGINT, failing_since BIGINT, UNIQUE (app_id, pushkey, user_name) );
147		CREATE INDEX device_lists_outbound_pokes_stream ON device_lists_outbound_pokes(stream_id);
148		CREATE TABLE ratelimit_override ( user_id TEXT NOT NULL, messages_per_second BIGINT, burst_count BIGINT );
149		CREATE UNIQUE INDEX ratelimit_override_idx ON ratelimit_override(user_id);
150		CREATE TABLE current_state_delta_stream ( stream_id BIGINT NOT NULL, room_id TEXT NOT NULL, type TEXT NOT NULL, state_key TEXT NOT NULL, event_id TEXT, prev_event_id TEXT );
151		CREATE INDEX current_state_delta_stream_idx ON current_state_delta_stream(stream_id);
152		CREATE TABLE device_lists_outbound_last_success ( destination TEXT NOT NULL, user_id TEXT NOT NULL, stream_id BIGINT NOT NULL );
153		CREATE INDEX device_lists_outbound_last_success_idx ON device_lists_outbound_last_success( destination, user_id, stream_id );
154		CREATE TABLE user_directory_stream_pos ( Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, stream_id BIGINT, CHECK (Lock='X') );
155		CREATE VIRTUAL TABLE user_directory_search USING fts4 ( user_id, value )
156		/* user_directory_search(user_id,value) */;
157		CREATE TABLE IF NOT EXISTS 'user_directory_search_content'(docid INTEGER PRIMARY KEY, 'c0user_id', 'c1value');
158		CREATE TABLE IF NOT EXISTS 'user_directory_search_segments'(blockid INTEGER PRIMARY KEY, block BLOB);
159		CREATE TABLE IF NOT EXISTS 'user_directory_search_segdir'(level INTEGER,idx INTEGER,start_block INTEGER,leaves_end_block INTEGER,end_block INTEGER,root BLOB,PRIMARY KEY(level, idx));
160		CREATE TABLE IF NOT EXISTS 'user_directory_search_docsize'(docid INTEGER PRIMARY KEY, size BLOB);
161		CREATE TABLE IF NOT EXISTS 'user_directory_search_stat'(id INTEGER PRIMARY KEY, value BLOB);
162		CREATE TABLE blocked_rooms ( room_id TEXT NOT NULL, user_id TEXT NOT NULL );
163		CREATE UNIQUE INDEX blocked_rooms_idx ON blocked_rooms(room_id);
164		CREATE TABLE IF NOT EXISTS "local_media_repository_url_cache"( url TEXT, response_code INTEGER, etag TEXT, expires_ts BIGINT, og TEXT, media_id TEXT, download_ts BIGINT );
165		CREATE INDEX local_media_repository_url_cache_expires_idx ON local_media_repository_url_cache(expires_ts);
166		CREATE INDEX local_media_repository_url_cache_by_url_download_ts ON local_media_repository_url_cache(url, download_ts);
167		CREATE INDEX local_media_repository_url_cache_media_idx ON local_media_repository_url_cache(media_id);
168		CREATE TABLE group_users ( group_id TEXT NOT NULL, user_id TEXT NOT NULL, is_admin BOOLEAN NOT NULL, is_public BOOLEAN NOT NULL );
169		CREATE TABLE group_invites ( group_id TEXT NOT NULL, user_id TEXT NOT NULL );
170		CREATE TABLE group_rooms ( group_id TEXT NOT NULL, room_id TEXT NOT NULL, is_public BOOLEAN NOT NULL );
171		CREATE TABLE group_summary_rooms ( group_id TEXT NOT NULL, room_id TEXT NOT NULL, category_id TEXT NOT NULL, room_order BIGINT NOT NULL, is_public BOOLEAN NOT NULL, UNIQUE (group_id, category_id, room_id, room_order), CHECK (room_order > 0) );
172		CREATE UNIQUE INDEX group_summary_rooms_g_idx ON group_summary_rooms(group_id, room_id, category_id);
173		CREATE TABLE group_summary_room_categories ( group_id TEXT NOT NULL, category_id TEXT NOT NULL, cat_order BIGINT NOT NULL, UNIQUE (group_id, category_id, cat_order), CHECK (cat_order > 0) );
174		CREATE TABLE group_room_categories ( group_id TEXT NOT NULL, category_id TEXT NOT NULL, profile TEXT NOT NULL, is_public BOOLEAN NOT NULL, UNIQUE (group_id, category_id) );
175		CREATE TABLE group_summary_users ( group_id TEXT NOT NULL, user_id TEXT NOT NULL, role_id TEXT NOT NULL, user_order BIGINT NOT NULL, is_public BOOLEAN NOT NULL );
176		CREATE INDEX group_summary_users_g_idx ON group_summary_users(group_id);
177		CREATE TABLE group_summary_roles ( group_id TEXT NOT NULL, role_id TEXT NOT NULL, role_order BIGINT NOT NULL, UNIQUE (group_id, role_id, role_order), CHECK (role_order > 0) );
178		CREATE TABLE group_roles ( group_id TEXT NOT NULL, role_id TEXT NOT NULL, profile TEXT NOT NULL, is_public BOOLEAN NOT NULL, UNIQUE (group_id, role_id) );
179		CREATE TABLE group_attestations_renewals ( group_id TEXT NOT NULL, user_id TEXT NOT NULL, valid_until_ms BIGINT NOT NULL );
180		CREATE INDEX group_attestations_renewals_g_idx ON group_attestations_renewals(group_id, user_id);
181		CREATE INDEX group_attestations_renewals_u_idx ON group_attestations_renewals(user_id);
182		CREATE INDEX group_attestations_renewals_v_idx ON group_attestations_renewals(valid_until_ms);
183		CREATE TABLE group_attestations_remote ( group_id TEXT NOT NULL, user_id TEXT NOT NULL, valid_until_ms BIGINT NOT NULL, attestation_json TEXT NOT NULL );
184		CREATE INDEX group_attestations_remote_g_idx ON group_attestations_remote(group_id, user_id);
185		CREATE INDEX group_attestations_remote_u_idx ON group_attestations_remote(user_id);
186		CREATE INDEX group_attestations_remote_v_idx ON group_attestations_remote(valid_until_ms);
187		CREATE TABLE local_group_membership ( group_id TEXT NOT NULL, user_id TEXT NOT NULL, is_admin BOOLEAN NOT NULL, membership TEXT NOT NULL, is_publicised BOOLEAN NOT NULL, content TEXT NOT NULL );
188		CREATE INDEX local_group_membership_u_idx ON local_group_membership(user_id, group_id);
189		CREATE INDEX local_group_membership_g_idx ON local_group_membership(group_id);
190		CREATE TABLE local_group_updates ( stream_id BIGINT NOT NULL, group_id TEXT NOT NULL, user_id TEXT NOT NULL, type TEXT NOT NULL, content TEXT NOT NULL );
191		CREATE TABLE remote_profile_cache ( user_id TEXT NOT NULL, displayname TEXT, avatar_url TEXT, last_check BIGINT NOT NULL );
192		CREATE UNIQUE INDEX remote_profile_cache_user_id ON remote_profile_cache(user_id);
193		CREATE INDEX remote_profile_cache_time ON remote_profile_cache(last_check);
194		CREATE TABLE IF NOT EXISTS "deleted_pushers" ( stream_id BIGINT NOT NULL, app_id TEXT NOT NULL, pushkey TEXT NOT NULL, user_id TEXT NOT NULL );
195		CREATE INDEX deleted_pushers_stream_id ON deleted_pushers (stream_id);
196		CREATE TABLE IF NOT EXISTS "groups" ( group_id TEXT NOT NULL, name TEXT, avatar_url TEXT, short_description TEXT, long_description TEXT, is_public BOOL NOT NULL , join_policy TEXT NOT NULL DEFAULT 'invite');
197		CREATE UNIQUE INDEX groups_idx ON groups(group_id);
198		CREATE TABLE IF NOT EXISTS "user_directory" ( user_id TEXT NOT NULL, room_id TEXT, display_name TEXT, avatar_url TEXT );
199		CREATE INDEX user_directory_room_idx ON user_directory(room_id);
200		CREATE UNIQUE INDEX user_directory_user_idx ON user_directory(user_id);
201		CREATE TABLE event_push_actions_staging ( event_id TEXT NOT NULL, user_id TEXT NOT NULL, actions TEXT NOT NULL, notif SMALLINT NOT NULL, highlight SMALLINT NOT NULL );
202		CREATE INDEX event_push_actions_staging_id ON event_push_actions_staging(event_id);
203		CREATE TABLE users_pending_deactivation ( user_id TEXT NOT NULL );
204		CREATE UNIQUE INDEX group_invites_g_idx ON group_invites(group_id, user_id);
205		CREATE UNIQUE INDEX group_users_g_idx ON group_users(group_id, user_id);
206		CREATE INDEX group_users_u_idx ON group_users(user_id);
207		CREATE INDEX group_invites_u_idx ON group_invites(user_id);
208		CREATE UNIQUE INDEX group_rooms_g_idx ON group_rooms(group_id, room_id);
209		CREATE INDEX group_rooms_r_idx ON group_rooms(room_id);
210		CREATE TABLE user_daily_visits ( user_id TEXT NOT NULL, device_id TEXT, timestamp BIGINT NOT NULL );
211		CREATE INDEX user_daily_visits_uts_idx ON user_daily_visits(user_id, timestamp);
212		CREATE INDEX user_daily_visits_ts_idx ON user_daily_visits(timestamp);
213		CREATE TABLE erased_users ( user_id TEXT NOT NULL );
214		CREATE UNIQUE INDEX erased_users_user ON erased_users(user_id);
215		CREATE TABLE monthly_active_users ( user_id TEXT NOT NULL, timestamp BIGINT NOT NULL );
216		CREATE UNIQUE INDEX monthly_active_users_users ON monthly_active_users(user_id);
217		CREATE INDEX monthly_active_users_time_stamp ON monthly_active_users(timestamp);
218		CREATE TABLE IF NOT EXISTS "e2e_room_keys_versions" ( user_id TEXT NOT NULL, version BIGINT NOT NULL, algorithm TEXT NOT NULL, auth_data TEXT NOT NULL, deleted SMALLINT DEFAULT 0 NOT NULL );
219		CREATE UNIQUE INDEX e2e_room_keys_versions_idx ON e2e_room_keys_versions(user_id, version);
220		CREATE TABLE IF NOT EXISTS "e2e_room_keys" ( user_id TEXT NOT NULL, room_id TEXT NOT NULL, session_id TEXT NOT NULL, version BIGINT NOT NULL, first_message_index INT, forwarded_count INT, is_verified BOOLEAN, session_data TEXT NOT NULL );
221		CREATE UNIQUE INDEX e2e_room_keys_idx ON e2e_room_keys(user_id, room_id, session_id);
222		CREATE TABLE users_who_share_private_rooms ( user_id TEXT NOT NULL, other_user_id TEXT NOT NULL, room_id TEXT NOT NULL );
223		CREATE UNIQUE INDEX users_who_share_private_rooms_u_idx ON users_who_share_private_rooms(user_id, other_user_id, room_id);
224		CREATE INDEX users_who_share_private_rooms_r_idx ON users_who_share_private_rooms(room_id);
225		CREATE INDEX users_who_share_private_rooms_o_idx ON users_who_share_private_rooms(other_user_id);
226		CREATE TABLE user_threepid_id_server ( user_id TEXT NOT NULL, medium TEXT NOT NULL, address TEXT NOT NULL, id_server TEXT NOT NULL );
227		CREATE UNIQUE INDEX user_threepid_id_server_idx ON user_threepid_id_server( user_id, medium, address, id_server );
228		CREATE TABLE users_in_public_rooms ( user_id TEXT NOT NULL, room_id TEXT NOT NULL );
229		CREATE UNIQUE INDEX users_in_public_rooms_u_idx ON users_in_public_rooms(user_id, room_id);
230		CREATE TABLE account_validity ( user_id TEXT PRIMARY KEY, expiration_ts_ms BIGINT NOT NULL, email_sent BOOLEAN NOT NULL, renewal_token TEXT );
231		CREATE TABLE event_relations ( event_id TEXT NOT NULL, relates_to_id TEXT NOT NULL, relation_type TEXT NOT NULL, aggregation_key TEXT );
232		CREATE UNIQUE INDEX event_relations_id ON event_relations(event_id);
233		CREATE INDEX event_relations_relates ON event_relations(relates_to_id, relation_type, aggregation_key);
234		CREATE TABLE stats_stream_pos ( Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, stream_id BIGINT, CHECK (Lock='X') );
235		CREATE TABLE user_stats ( user_id TEXT NOT NULL, ts BIGINT NOT NULL, bucket_size INT NOT NULL, public_rooms INT NOT NULL, private_rooms INT NOT NULL );
236		CREATE UNIQUE INDEX user_stats_user_ts ON user_stats(user_id, ts);
237		CREATE TABLE room_stats ( room_id TEXT NOT NULL, ts BIGINT NOT NULL, bucket_size INT NOT NULL, current_state_events INT NOT NULL, joined_members INT NOT NULL, invited_members INT NOT NULL, left_members INT NOT NULL, banned_members INT NOT NULL, state_events INT NOT NULL );
238		CREATE UNIQUE INDEX room_stats_room_ts ON room_stats(room_id, ts);
239		CREATE TABLE room_state ( room_id TEXT NOT NULL, join_rules TEXT, history_visibility TEXT, encryption TEXT, name TEXT, topic TEXT, avatar TEXT, canonical_alias TEXT );
240		CREATE UNIQUE INDEX room_state_room ON room_state(room_id);
241		CREATE TABLE room_stats_earliest_token ( room_id TEXT NOT NULL, token BIGINT NOT NULL );
242		CREATE UNIQUE INDEX room_stats_earliest_token_idx ON room_stats_earliest_token(room_id);
243		CREATE INDEX access_tokens_device_id ON access_tokens (user_id, device_id);
244		CREATE INDEX user_ips_device_id ON user_ips (user_id, device_id, last_seen);
245		CREATE INDEX event_contains_url_index ON events (room_id, topological_ordering, stream_ordering);
246		CREATE INDEX event_push_actions_u_highlight ON event_push_actions (user_id, stream_ordering);
247		CREATE INDEX event_push_actions_highlights_index ON event_push_actions (user_id, room_id, topological_ordering, stream_ordering);
248		CREATE INDEX current_state_events_member_index ON current_state_events (state_key);
249		CREATE INDEX device_inbox_stream_id_user_id ON device_inbox (stream_id, user_id);
250		CREATE INDEX device_lists_stream_user_id ON device_lists_stream (user_id, device_id);
251		CREATE INDEX local_media_repository_url_idx ON local_media_repository (created_ts);
252		CREATE INDEX user_ips_last_seen ON user_ips (user_id, last_seen);
253		CREATE INDEX user_ips_last_seen_only ON user_ips (last_seen);
254		CREATE INDEX users_creation_ts ON users (creation_ts);
255		CREATE INDEX event_to_state_groups_sg_index ON event_to_state_groups (state_group);
256		CREATE UNIQUE INDEX device_lists_remote_cache_unique_id ON device_lists_remote_cache (user_id, device_id);
257		CREATE INDEX state_groups_state_type_idx ON state_groups_state(state_group, type, state_key);
258		CREATE UNIQUE INDEX device_lists_remote_extremeties_unique_idx ON device_lists_remote_extremeties (user_id);
259		CREATE UNIQUE INDEX user_ips_user_token_ip_unique_index ON user_ips (user_id, access_token, ip);

+0

-7

~~synapse/storage/schema/full_schemas/54/stream_positions.sql~~ less more

0
1		INSERT INTO appservice_stream_position (stream_ordering) SELECT COALESCE(MAX(stream_ordering), 0) FROM events;
2		INSERT INTO federation_stream_position (type, stream_id) VALUES ('federation', -1);
3		INSERT INTO federation_stream_position (type, stream_id) SELECT 'events', coalesce(max(stream_ordering), -1) FROM events;
4		INSERT INTO user_directory_stream_pos (stream_id) VALUES (0);
5		INSERT INTO stats_stream_pos (stream_id) VALUES (0);
6		INSERT INTO event_push_summary_stream_ordering (stream_ordering) VALUES (0);

+0

-19

~~synapse/storage/schema/full_schemas/README.txt~~ less more

0		Building full schema dumps
1		==========================
2
3		These schemas need to be made from a database that has had all background updates run.
4
5		Postgres
6		--------
7
8		$ pg_dump --format=plain --schema-only --no-tablespaces --no-acl --no-owner $DATABASE_NAME\| sed -e '/^--/d' -e 's/public\.//g' -e '/^SET /d' -e '/^SELECT /d' > full.sql.postgres
9
10		SQLite
11		------
12
13		$ sqlite3 $DATABASE_FILE ".schema" > full.sql.sqlite
14
15		After
16		-----
17
18		Delete the CREATE statements for "sqlite_stat1", "schema_version", "applied_schema_deltas", and "applied_module_schemas".⏎

+0

-703

~~synapse/storage/search.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2015, 2016 OpenMarket Ltd
2		#
3		# Licensed under the Apache License, Version 2.0 (the "License");
4		# you may not use this file except in compliance with the License.
5		# You may obtain a copy of the License at
6		#
7		# http://www.apache.org/licenses/LICENSE-2.0
8		#
9		# Unless required by applicable law or agreed to in writing, software
10		# distributed under the License is distributed on an "AS IS" BASIS,
11		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12		# See the License for the specific language governing permissions and
13		# limitations under the License.
14
15		import logging
16		import re
17		from collections import namedtuple
18
19		from six import string_types
20
21		from canonicaljson import json
22
23		from twisted.internet import defer
24
25		from synapse.api.errors import SynapseError
26		from synapse.storage.engines import PostgresEngine, Sqlite3Engine
27
28		from .background_updates import BackgroundUpdateStore
29
30		logger = logging.getLogger(__name__)
31
32		SearchEntry = namedtuple(
33		"SearchEntry",
34		["key", "value", "event_id", "room_id", "stream_ordering", "origin_server_ts"],
35		)
36
37
38		class SearchStore(BackgroundUpdateStore):
39
40		EVENT_SEARCH_UPDATE_NAME = "event_search"
41		EVENT_SEARCH_ORDER_UPDATE_NAME = "event_search_order"
42		EVENT_SEARCH_USE_GIST_POSTGRES_NAME = "event_search_postgres_gist"
43		EVENT_SEARCH_USE_GIN_POSTGRES_NAME = "event_search_postgres_gin"
44
45		def __init__(self, db_conn, hs):
46		super(SearchStore, self).__init__(db_conn, hs)
47
48		if not hs.config.enable_search:
49		return
50
51		self.register_background_update_handler(
52		self.EVENT_SEARCH_UPDATE_NAME, self._background_reindex_search
53		)
54		self.register_background_update_handler(
55		self.EVENT_SEARCH_ORDER_UPDATE_NAME, self._background_reindex_search_order
56		)
57
58		# we used to have a background update to turn the GIN index into a
59		# GIST one; we no longer do that (obviously) because we actually want
60		# a GIN index. However, it's possible that some people might still have
61		# the background update queued, so we register a handler to clear the
62		# background update.
63		self.register_noop_background_update(self.EVENT_SEARCH_USE_GIST_POSTGRES_NAME)
64
65		self.register_background_update_handler(
66		self.EVENT_SEARCH_USE_GIN_POSTGRES_NAME, self._background_reindex_gin_search
67		)
68
69		@defer.inlineCallbacks
70		def _background_reindex_search(self, progress, batch_size):
71		# we work through the events table from highest stream id to lowest
72		target_min_stream_id = progress["target_min_stream_id_inclusive"]
73		max_stream_id = progress["max_stream_id_exclusive"]
74		rows_inserted = progress.get("rows_inserted", 0)
75
76		TYPES = ["m.room.name", "m.room.message", "m.room.topic"]
77
78		def reindex_search_txn(txn):
79		sql = (
80		"SELECT stream_ordering, event_id, room_id, type, json, "
81		" origin_server_ts FROM events"
82		" JOIN event_json USING (room_id, event_id)"
83		" WHERE ? <= stream_ordering AND stream_ordering < ?"
84		" AND (%s)"
85		" ORDER BY stream_ordering DESC"
86		" LIMIT ?"
87		) % (" OR ".join("type = '%s'" % (t,) for t in TYPES),)
88
89		txn.execute(sql, (target_min_stream_id, max_stream_id, batch_size))
90
91		# we could stream straight from the results into
92		# store_search_entries_txn with a generator function, but that
93		# would mean having two cursors open on the database at once.
94		# Instead we just build a list of results.
95		rows = self.cursor_to_dict(txn)
96		if not rows:
97		return 0
98
99		min_stream_id = rows[-1]["stream_ordering"]
100
101		event_search_rows = []
102		for row in rows:
103		try:
104		event_id = row["event_id"]
105		room_id = row["room_id"]
106		etype = row["type"]
107		stream_ordering = row["stream_ordering"]
108		origin_server_ts = row["origin_server_ts"]
109		try:
110		event_json = json.loads(row["json"])
111		content = event_json["content"]
112		except Exception:
113		continue
114
115		if etype == "m.room.message":
116		key = "content.body"
117		value = content["body"]
118		elif etype == "m.room.topic":
119		key = "content.topic"
120		value = content["topic"]
121		elif etype == "m.room.name":
122		key = "content.name"
123		value = content["name"]
124		else:
125		raise Exception("unexpected event type %s" % etype)
126		except (KeyError, AttributeError):
127		# If the event is missing a necessary field then
128		# skip over it.
129		continue
130
131		if not isinstance(value, string_types):
132		# If the event body, name or topic isn't a string
133		# then skip over it
134		continue
135
136		event_search_rows.append(
137		SearchEntry(
138		key=key,
139		value=value,
140		event_id=event_id,
141		room_id=room_id,
142		stream_ordering=stream_ordering,
143		origin_server_ts=origin_server_ts,
144		)
145		)
146
147		self.store_search_entries_txn(txn, event_search_rows)
148
149		progress = {
150		"target_min_stream_id_inclusive": target_min_stream_id,
151		"max_stream_id_exclusive": min_stream_id,
152		"rows_inserted": rows_inserted + len(event_search_rows),
153		}
154
155		self._background_update_progress_txn(
156		txn, self.EVENT_SEARCH_UPDATE_NAME, progress
157		)
158
159		return len(event_search_rows)
160
161		result = yield self.runInteraction(
162		self.EVENT_SEARCH_UPDATE_NAME, reindex_search_txn
163		)
164
165		if not result:
166		yield self._end_background_update(self.EVENT_SEARCH_UPDATE_NAME)
167
168		return result
169
170		@defer.inlineCallbacks
171		def _background_reindex_gin_search(self, progress, batch_size):
172		"""This handles old synapses which used GIST indexes, if any;
173		converting them back to be GIN as per the actual schema.
174		"""
175
176		def create_index(conn):
177		conn.rollback()
178
179		# we have to set autocommit, because postgres refuses to
180		# CREATE INDEX CONCURRENTLY without it.
181		conn.set_session(autocommit=True)
182
183		try:
184		c = conn.cursor()
185
186		# if we skipped the conversion to GIST, we may already/still
187		# have an event_search_fts_idx; unfortunately postgres 9.4
188		# doesn't support CREATE INDEX IF EXISTS so we just catch the
189		# exception and ignore it.
190		import psycopg2
191
192		try:
193		c.execute(
194		"CREATE INDEX CONCURRENTLY event_search_fts_idx"
195		" ON event_search USING GIN (vector)"
196		)
197		except psycopg2.ProgrammingError as e:
198		logger.warn(
199		"Ignoring error %r when trying to switch from GIST to GIN", e
200		)
201
202		# we should now be able to delete the GIST index.
203		c.execute("DROP INDEX IF EXISTS event_search_fts_idx_gist")
204		finally:
205		conn.set_session(autocommit=False)
206
207		if isinstance(self.database_engine, PostgresEngine):
208		yield self.runWithConnection(create_index)
209
210		yield self._end_background_update(self.EVENT_SEARCH_USE_GIN_POSTGRES_NAME)
211		return 1
212
213		@defer.inlineCallbacks
214		def _background_reindex_search_order(self, progress, batch_size):
215		target_min_stream_id = progress["target_min_stream_id_inclusive"]
216		max_stream_id = progress["max_stream_id_exclusive"]
217		rows_inserted = progress.get("rows_inserted", 0)
218		have_added_index = progress["have_added_indexes"]
219
220		if not have_added_index:
221
222		def create_index(conn):
223		conn.rollback()
224		conn.set_session(autocommit=True)
225		c = conn.cursor()
226
227		# We create with NULLS FIRST so that when we search backwards
228		# we get the ones with non null origin_server_ts first
229		c.execute(
230		"CREATE INDEX CONCURRENTLY event_search_room_order ON event_search("
231		"room_id, origin_server_ts NULLS FIRST, stream_ordering NULLS FIRST)"
232		)
233		c.execute(
234		"CREATE INDEX CONCURRENTLY event_search_order ON event_search("
235		"origin_server_ts NULLS FIRST, stream_ordering NULLS FIRST)"
236		)
237		conn.set_session(autocommit=False)
238
239		yield self.runWithConnection(create_index)
240
241		pg = dict(progress)
242		pg["have_added_indexes"] = True
243
244		yield self.runInteraction(
245		self.EVENT_SEARCH_ORDER_UPDATE_NAME,
246		self._background_update_progress_txn,
247		self.EVENT_SEARCH_ORDER_UPDATE_NAME,
248		pg,
249		)
250
251		def reindex_search_txn(txn):
252		sql = (
253		"UPDATE event_search AS es SET stream_ordering = e.stream_ordering,"
254		" origin_server_ts = e.origin_server_ts"
255		" FROM events AS e"
256		" WHERE e.event_id = es.event_id"
257		" AND ? <= e.stream_ordering AND e.stream_ordering < ?"
258		" RETURNING es.stream_ordering"
259		)
260
261		min_stream_id = max_stream_id - batch_size
262		txn.execute(sql, (min_stream_id, max_stream_id))
263		rows = txn.fetchall()
264
265		if min_stream_id < target_min_stream_id:
266		# We've recached the end.
267		return len(rows), False
268
269		progress = {
270		"target_min_stream_id_inclusive": target_min_stream_id,
271		"max_stream_id_exclusive": min_stream_id,
272		"rows_inserted": rows_inserted + len(rows),
273		"have_added_indexes": True,
274		}
275
276		self._background_update_progress_txn(
277		txn, self.EVENT_SEARCH_ORDER_UPDATE_NAME, progress
278		)
279
280		return len(rows), True
281
282		num_rows, finished = yield self.runInteraction(
283		self.EVENT_SEARCH_ORDER_UPDATE_NAME, reindex_search_txn
284		)
285
286		if not finished:
287		yield self._end_background_update(self.EVENT_SEARCH_ORDER_UPDATE_NAME)
288
289		return num_rows
290
291		def store_event_search_txn(self, txn, event, key, value):
292		"""Add event to the search table
293
294		Args:
295		txn (cursor):
296		event (EventBase):
297		key (str):
298		value (str):
299		"""
300		self.store_search_entries_txn(
301		txn,
302		(
303		SearchEntry(
304		key=key,
305		value=value,
306		event_id=event.event_id,
307		room_id=event.room_id,
308		stream_ordering=event.internal_metadata.stream_ordering,
309		origin_server_ts=event.origin_server_ts,
310		),
311		),
312		)
313
314		def store_search_entries_txn(self, txn, entries):
315		"""Add entries to the search table
316
317		Args:
318		txn (cursor):
319		entries (iterable[SearchEntry]):
320		entries to be added to the table
321		"""
322		if not self.hs.config.enable_search:
323		return
324		if isinstance(self.database_engine, PostgresEngine):
325		sql = (
326		"INSERT INTO event_search"
327		" (event_id, room_id, key, vector, stream_ordering, origin_server_ts)"
328		" VALUES (?,?,?,to_tsvector('english', ?),?,?)"
329		)
330
331		args = (
332		(
333		entry.event_id,
334		entry.room_id,
335		entry.key,
336		entry.value,
337		entry.stream_ordering,
338		entry.origin_server_ts,
339		)
340		for entry in entries
341		)
342
343		txn.executemany(sql, args)
344
345		elif isinstance(self.database_engine, Sqlite3Engine):
346		sql = (
347		"INSERT INTO event_search (event_id, room_id, key, value)"
348		" VALUES (?,?,?,?)"
349		)
350		args = (
351		(entry.event_id, entry.room_id, entry.key, entry.value)
352		for entry in entries
353		)
354
355		txn.executemany(sql, args)
356		else:
357		# This should be unreachable.
358		raise Exception("Unrecognized database engine")
359
360		@defer.inlineCallbacks
361		def search_msgs(self, room_ids, search_term, keys):
362		"""Performs a full text search over events with given keys.
363
364		Args:
365		room_ids (list): List of room ids to search in
366		search_term (str): Search term to search for
367		keys (list): List of keys to search in, currently supports
368		"content.body", "content.name", "content.topic"
369
370		Returns:
371		list of dicts
372		"""
373		clauses = []
374
375		search_query = search_query = _parse_query(self.database_engine, search_term)
376
377		args = []
378
379		# Make sure we don't explode because the person is in too many rooms.
380		# We filter the results below regardless.
381		if len(room_ids) < 500:
382		clauses.append("room_id IN (%s)" % (",".join(["?"] * len(room_ids)),))
383		args.extend(room_ids)
384
385		local_clauses = []
386		for key in keys:
387		local_clauses.append("key = ?")
388		args.append(key)
389
390		clauses.append("(%s)" % (" OR ".join(local_clauses),))
391
392		count_args = args
393		count_clauses = clauses
394
395		if isinstance(self.database_engine, PostgresEngine):
396		sql = (
397		"SELECT ts_rank_cd(vector, to_tsquery('english', ?)) AS rank,"
398		" room_id, event_id"
399		" FROM event_search"
400		" WHERE vector @@ to_tsquery('english', ?)"
401		)
402		args = [search_query, search_query] + args
403
404		count_sql = (
405		"SELECT room_id, count(*) as count FROM event_search"
406		" WHERE vector @@ to_tsquery('english', ?)"
407		)
408		count_args = [search_query] + count_args
409		elif isinstance(self.database_engine, Sqlite3Engine):
410		sql = (
411		"SELECT rank(matchinfo(event_search)) as rank, room_id, event_id"
412		" FROM event_search"
413		" WHERE value MATCH ?"
414		)
415		args = [search_query] + args
416
417		count_sql = (
418		"SELECT room_id, count(*) as count FROM event_search"
419		" WHERE value MATCH ?"
420		)
421		count_args = [search_term] + count_args
422		else:
423		# This should be unreachable.
424		raise Exception("Unrecognized database engine")
425
426		for clause in clauses:
427		sql += " AND " + clause
428
429		for clause in count_clauses:
430		count_sql += " AND " + clause
431
432		# We add an arbitrary limit here to ensure we don't try to pull the
433		# entire table from the database.
434		sql += " ORDER BY rank DESC LIMIT 500"
435
436		results = yield self._execute("search_msgs", self.cursor_to_dict, sql, *args)
437
438		results = list(filter(lambda row: row["room_id"] in room_ids, results))
439
440		events = yield self.get_events_as_list([r["event_id"] for r in results])
441
442		event_map = {ev.event_id: ev for ev in events}
443
444		highlights = None
445		if isinstance(self.database_engine, PostgresEngine):
446		highlights = yield self._find_highlights_in_postgres(search_query, events)
447
448		count_sql += " GROUP BY room_id"
449
450		count_results = yield self._execute(
451		"search_rooms_count", self.cursor_to_dict, count_sql, *count_args
452		)
453
454		count = sum(row["count"] for row in count_results if row["room_id"] in room_ids)
455
456		return {
457		"results": [
458		{"event": event_map[r["event_id"]], "rank": r["rank"]}
459		for r in results
460		if r["event_id"] in event_map
461		],
462		"highlights": highlights,
463		"count": count,
464		}
465
466		@defer.inlineCallbacks
467		def search_rooms(self, room_ids, search_term, keys, limit, pagination_token=None):
468		"""Performs a full text search over events with given keys.
469
470		Args:
471		room_id (list): The room_ids to search in
472		search_term (str): Search term to search for
473		keys (list): List of keys to search in, currently supports
474		"content.body", "content.name", "content.topic"
475		pagination_token (str): A pagination token previously returned
476
477		Returns:
478		list of dicts
479		"""
480		clauses = []
481
482		search_query = search_query = _parse_query(self.database_engine, search_term)
483
484		args = []
485
486		# Make sure we don't explode because the person is in too many rooms.
487		# We filter the results below regardless.
488		if len(room_ids) < 500:
489		clauses.append("room_id IN (%s)" % (",".join(["?"] * len(room_ids)),))
490		args.extend(room_ids)
491
492		local_clauses = []
493		for key in keys:
494		local_clauses.append("key = ?")
495		args.append(key)
496
497		clauses.append("(%s)" % (" OR ".join(local_clauses),))
498
499		# take copies of the current args and clauses lists, before adding
500		# pagination clauses to main query.
501		count_args = list(args)
502		count_clauses = list(clauses)
503
504		if pagination_token:
505		try:
506		origin_server_ts, stream = pagination_token.split(",")
507		origin_server_ts = int(origin_server_ts)
508		stream = int(stream)
509		except Exception:
510		raise SynapseError(400, "Invalid pagination token")
511
512		clauses.append(
513		"(origin_server_ts < ?"
514		" OR (origin_server_ts = ? AND stream_ordering < ?))"
515		)
516		args.extend([origin_server_ts, origin_server_ts, stream])
517
518		if isinstance(self.database_engine, PostgresEngine):
519		sql = (
520		"SELECT ts_rank_cd(vector, to_tsquery('english', ?)) as rank,"
521		" origin_server_ts, stream_ordering, room_id, event_id"
522		" FROM event_search"
523		" WHERE vector @@ to_tsquery('english', ?) AND "
524		)
525		args = [search_query, search_query] + args
526
527		count_sql = (
528		"SELECT room_id, count(*) as count FROM event_search"
529		" WHERE vector @@ to_tsquery('english', ?) AND "
530		)
531		count_args = [search_query] + count_args
532		elif isinstance(self.database_engine, Sqlite3Engine):
533		# We use CROSS JOIN here to ensure we use the right indexes.
534		# https://sqlite.org/optoverview.html#crossjoin
535		#
536		# We want to use the full text search index on event_search to
537		# extract all possible matches first, then lookup those matches
538		# in the events table to get the topological ordering. We need
539		# to use the indexes in this order because sqlite refuses to
540		# MATCH unless it uses the full text search index
541		sql = (
542		"SELECT rank(matchinfo) as rank, room_id, event_id,"
543		" origin_server_ts, stream_ordering"
544		" FROM (SELECT key, event_id, matchinfo(event_search) as matchinfo"
545		" FROM event_search"
546		" WHERE value MATCH ?"
547		" )"
548		" CROSS JOIN events USING (event_id)"
549		" WHERE "
550		)
551		args = [search_query] + args
552
553		count_sql = (
554		"SELECT room_id, count(*) as count FROM event_search"
555		" WHERE value MATCH ? AND "
556		)
557		count_args = [search_term] + count_args
558		else:
559		# This should be unreachable.
560		raise Exception("Unrecognized database engine")
561
562		sql += " AND ".join(clauses)
563		count_sql += " AND ".join(count_clauses)
564
565		# We add an arbitrary limit here to ensure we don't try to pull the
566		# entire table from the database.
567		if isinstance(self.database_engine, PostgresEngine):
568		sql += (
569		" ORDER BY origin_server_ts DESC NULLS LAST,"
570		" stream_ordering DESC NULLS LAST LIMIT ?"
571		)
572		elif isinstance(self.database_engine, Sqlite3Engine):
573		sql += " ORDER BY origin_server_ts DESC, stream_ordering DESC LIMIT ?"
574		else:
575		raise Exception("Unrecognized database engine")
576
577		args.append(limit)
578
579		results = yield self._execute("search_rooms", self.cursor_to_dict, sql, *args)
580
581		results = list(filter(lambda row: row["room_id"] in room_ids, results))
582
583		events = yield self.get_events_as_list([r["event_id"] for r in results])
584
585		event_map = {ev.event_id: ev for ev in events}
586
587		highlights = None
588		if isinstance(self.database_engine, PostgresEngine):
589		highlights = yield self._find_highlights_in_postgres(search_query, events)
590
591		count_sql += " GROUP BY room_id"
592
593		count_results = yield self._execute(
594		"search_rooms_count", self.cursor_to_dict, count_sql, *count_args
595		)
596
597		count = sum(row["count"] for row in count_results if row["room_id"] in room_ids)
598
599		return {
600		"results": [
601		{
602		"event": event_map[r["event_id"]],
603		"rank": r["rank"],
604		"pagination_token": "%s,%s"
605		% (r["origin_server_ts"], r["stream_ordering"]),
606		}
607		for r in results
608		if r["event_id"] in event_map
609		],
610		"highlights": highlights,
611		"count": count,
612		}
613
614		def _find_highlights_in_postgres(self, search_query, events):
615		"""Given a list of events and a search term, return a list of words
616		that match from the content of the event.
617
618		This is used to give a list of words that clients can match against to
619		highlight the matching parts.
620
621		Args:
622		search_query (str)
623		events (list): A list of events
624
625		Returns:
626		deferred : A set of strings.
627		"""
628
629		def f(txn):
630		highlight_words = set()
631		for event in events:
632		# As a hack we simply join values of all possible keys. This is
633		# fine since we're only using them to find possible highlights.
634		values = []
635		for key in ("body", "name", "topic"):
636		v = event.content.get(key, None)
637		if v:
638		values.append(v)
639
640		if not values:
641		continue
642
643		value = " ".join(values)
644
645		# We need to find some values for StartSel and StopSel that
646		# aren't in the value so that we can pick results out.
647		start_sel = "<"
648		stop_sel = ">"
649
650		while start_sel in value:
651		start_sel += "<"
652		while stop_sel in value:
653		stop_sel += ">"
654
655		query = "SELECT ts_headline(?, to_tsquery('english', ?), %s)" % (
656		_to_postgres_options(
657		{
658		"StartSel": start_sel,
659		"StopSel": stop_sel,
660		"MaxFragments": "50",
661		}
662		)
663		)
664		txn.execute(query, (value, search_query))
665		headline, = txn.fetchall()[0]
666
667		# Now we need to pick the possible highlights out of the haedline
668		# result.
669		matcher_regex = "%s(.*?)%s" % (
670		re.escape(start_sel),
671		re.escape(stop_sel),
672		)
673
674		res = re.findall(matcher_regex, headline)
675		highlight_words.update([r.lower() for r in res])
676
677		return highlight_words
678
679		return self.runInteraction("_find_highlights", f)
680
681
682		def _to_postgres_options(options_dict):
683		return "'%s'" % (",".join("%s=%s" % (k, v) for k, v in options_dict.items()),)
684
685
686		def _parse_query(database_engine, search_term):
687		"""Takes a plain unicode string from the user and converts it into a form
688		that can be passed to database.
689		We use this so that we can add prefix matching, which isn't something
690		that is supported by default.
691		"""
692
693		# Pull out the individual words, discarding any non-word characters.
694		results = re.findall(r"([\w\-]+)", search_term, re.UNICODE)
695
696		if isinstance(database_engine, PostgresEngine):
697		return " & ".join(result + ":*" for result in results)
698		elif isinstance(database_engine, Sqlite3Engine):
699		return " & ".join(result + "*" for result in results)
700		else:
701		# This should be unreachable.
702		raise Exception("Unrecognized database engine")

+0

-102

~~synapse/storage/signatures.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2014-2016 OpenMarket Ltd
2		#
3		# Licensed under the Apache License, Version 2.0 (the "License");
4		# you may not use this file except in compliance with the License.
5		# You may obtain a copy of the License at
6		#
7		# http://www.apache.org/licenses/LICENSE-2.0
8		#
9		# Unless required by applicable law or agreed to in writing, software
10		# distributed under the License is distributed on an "AS IS" BASIS,
11		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12		# See the License for the specific language governing permissions and
13		# limitations under the License.
14
15		import six
16
17		from unpaddedbase64 import encode_base64
18
19		from twisted.internet import defer
20
21		from synapse.crypto.event_signing import compute_event_reference_hash
22		from synapse.util.caches.descriptors import cached, cachedList
23
24		from ._base import SQLBaseStore
25
26		# py2 sqlite has buffer hardcoded as only binary type, so we must use it,
27		# despite being deprecated and removed in favor of memoryview
28		if six.PY2:
29		db_binary_type = six.moves.builtins.buffer
30		else:
31		db_binary_type = memoryview
32
33
34		class SignatureWorkerStore(SQLBaseStore):
35		@cached()
36		def get_event_reference_hash(self, event_id):
37		# This is a dummy function to allow get_event_reference_hashes
38		# to use its cache
39		raise NotImplementedError()
40
41		@cachedList(
42		cached_method_name="get_event_reference_hash", list_name="event_ids", num_args=1
43		)
44		def get_event_reference_hashes(self, event_ids):
45		def f(txn):
46		return {
47		event_id: self._get_event_reference_hashes_txn(txn, event_id)
48		for event_id in event_ids
49		}
50
51		return self.runInteraction("get_event_reference_hashes", f)
52
53		@defer.inlineCallbacks
54		def add_event_hashes(self, event_ids):
55		hashes = yield self.get_event_reference_hashes(event_ids)
56		hashes = {
57		e_id: {k: encode_base64(v) for k, v in h.items() if k == "sha256"}
58		for e_id, h in hashes.items()
59		}
60
61		return list(hashes.items())
62
63		def _get_event_reference_hashes_txn(self, txn, event_id):
64		"""Get all the hashes for a given PDU.
65		Args:
66		txn (cursor):
67		event_id (str): Id for the Event.
68		Returns:
69		A dict[unicode, bytes] of algorithm -> hash.
70		"""
71		query = (
72		"SELECT algorithm, hash"
73		" FROM event_reference_hashes"
74		" WHERE event_id = ?"
75		)
76		txn.execute(query, (event_id,))
77		return {k: v for k, v in txn}
78
79
80		class SignatureStore(SignatureWorkerStore):
81		"""Persistence for event signatures and hashes"""
82
83		def _store_event_reference_hashes_txn(self, txn, events):
84		"""Store a hash for a PDU
85		Args:
86		txn (cursor):
87		events (list): list of Events.
88		"""
89
90		vals = []
91		for event in events:
92		ref_alg, ref_hash_bytes = compute_event_reference_hash(event)
93		vals.append(
94		{
95		"event_id": event.event_id,
96		"algorithm": ref_alg,
97		"hash": db_binary_type(ref_hash_bytes),
98		}
99		)
100
101		self._simple_insert_many_txn(txn, table="event_reference_hashes", values=vals)

+0

-1205

synapse/storage/state.py less more

13	13	# limitations under the License.
14	14
15	15	import logging
16		from collections import namedtuple
17	16
18	17	from six import iteritems, itervalues
19		from six.moves import range
20	18
21	19	import attr
22	20
23		from twisted.internet import defer
24
25	21	from synapse.api.constants import EventTypes
26		from synapse.api.errors import NotFoundError
27		from synapse.storage._base import SQLBaseStore
28		from synapse.storage.background_updates import BackgroundUpdateStore
29		from synapse.storage.engines import PostgresEngine
30		from synapse.storage.events_worker import EventsWorkerStore
31		from synapse.util.caches import get_cache_factor_for, intern_string
32		from synapse.util.caches.descriptors import cached, cachedList
33		from synapse.util.caches.dictionary_cache import DictionaryCache
34		from synapse.util.stringutils import to_ascii
35	22
36	23	logger = logging.getLogger(__name__)
37
38
39		MAX_STATE_DELTA_HOPS = 100
40
41
42		class _GetStateGroupDelta(
43		namedtuple("_GetStateGroupDelta", ("prev_group", "delta_ids"))
44		):
45		"""Return type of get_state_group_delta that implements __len__, which lets
46		us use the itrable flag when caching
47		"""
48
49		__slots__ = []
50
51		def __len__(self):
52		return len(self.delta_ids) if self.delta_ids else 0
53	24
54	25
55	26	@attr.s(slots=True)

350	321	)
351	322
352	323	return member_filter, non_member_filter
353
354
355		# this inherits from EventsWorkerStore because it calls self.get_events
356		class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore):
357		"""The parts of StateGroupStore that can be called from workers.
358		"""
359
360		STATE_GROUP_DEDUPLICATION_UPDATE_NAME = "state_group_state_deduplication"
361		STATE_GROUP_INDEX_UPDATE_NAME = "state_group_state_type_index"
362		CURRENT_STATE_INDEX_UPDATE_NAME = "current_state_members_idx"
363
364		def __init__(self, db_conn, hs):
365		super(StateGroupWorkerStore, self).__init__(db_conn, hs)
366
367		# Originally the state store used a single DictionaryCache to cache the
368		# event IDs for the state types in a given state group to avoid hammering
369		# on the state_group* tables.
370		#
371		# The point of using a DictionaryCache is that it can cache a subset
372		# of the state events for a given state group (i.e. a subset of the keys for a
373		# given dict which is an entry in the cache for a given state group ID).
374		#
375		# However, this poses problems when performing complicated queries
376		# on the store - for instance: "give me all the state for this group, but
377		# limit members to this subset of users", as DictionaryCache's API isn't
378		# rich enough to say "please cache any of these fields, apart from this subset".
379		# This is problematic when lazy loading members, which requires this behaviour,
380		# as without it the cache has no choice but to speculatively load all
381		# state events for the group, which negates the efficiency being sought.
382		#
383		# Rather than overcomplicating DictionaryCache's API, we instead split the
384		# state_group_cache into two halves - one for tracking non-member events,
385		# and the other for tracking member_events. This means that lazy loading
386		# queries can be made in a cache-friendly manner by querying both caches
387		# separately and then merging the result. So for the example above, you
388		# would query the members cache for a specific subset of state keys
389		# (which DictionaryCache will handle efficiently and fine) and the non-members
390		# cache for all state (which DictionaryCache will similarly handle fine)
391		# and then just merge the results together.
392		#
393		# We size the non-members cache to be smaller than the members cache as the
394		# vast majority of state in Matrix (today) is member events.
395
396		self._state_group_cache = DictionaryCache(
397		"stateGroupCache",
398		# TODO: this hasn't been tuned yet
399		50000 * get_cache_factor_for("stateGroupCache"),
400		)
401		self._state_group_members_cache = DictionaryCache(
402		"stateGroupMembersCache",
403		500000 * get_cache_factor_for("stateGroupMembersCache"),
404		)
405
406		@defer.inlineCallbacks
407		def get_room_version(self, room_id):
408		"""Get the room_version of a given room
409
410		Args:
411		room_id (str)
412
413		Returns:
414		Deferred[str]
415
416		Raises:
417		NotFoundError if the room is unknown
418		"""
419		# for now we do this by looking at the create event. We may want to cache this
420		# more intelligently in future.
421
422		# Retrieve the room's create event
423		create_event = yield self.get_create_event_for_room(room_id)
424		return create_event.content.get("room_version", "1")
425
426		@defer.inlineCallbacks
427		def get_room_predecessor(self, room_id):
428		"""Get the predecessor room of an upgraded room if one exists.
429		Otherwise return None.
430
431		Args:
432		room_id (str)
433
434		Returns:
435		Deferred[unicode\|None]: predecessor room id
436
437		Raises:
438		NotFoundError if the room is unknown
439		"""
440		# Retrieve the room's create event
441		create_event = yield self.get_create_event_for_room(room_id)
442
443		# Return predecessor if present
444		return create_event.content.get("predecessor", None)
445
446		@defer.inlineCallbacks
447		def get_create_event_for_room(self, room_id):
448		"""Get the create state event for a room.
449
450		Args:
451		room_id (str)
452
453		Returns:
454		Deferred[EventBase]: The room creation event.
455
456		Raises:
457		NotFoundError if the room is unknown
458		"""
459		state_ids = yield self.get_current_state_ids(room_id)
460		create_id = state_ids.get((EventTypes.Create, ""))
461
462		# If we can't find the create event, assume we've hit a dead end
463		if not create_id:
464		raise NotFoundError("Unknown room %s" % (room_id))
465
466		# Retrieve the room's create event and return
467		create_event = yield self.get_event(create_id)
468		return create_event
469
470		@cached(max_entries=100000, iterable=True)
471		def get_current_state_ids(self, room_id):
472		"""Get the current state event ids for a room based on the
473		current_state_events table.
474
475		Args:
476		room_id (str)
477
478		Returns:
479		deferred: dict of (type, state_key) -> event_id
480		"""
481
482		def _get_current_state_ids_txn(txn):
483		txn.execute(
484		"""SELECT type, state_key, event_id FROM current_state_events
485		WHERE room_id = ?
486		""",
487		(room_id,),
488		)
489
490		return {
491		(intern_string(r[0]), intern_string(r[1])): to_ascii(r[2]) for r in txn
492		}
493
494		return self.runInteraction("get_current_state_ids", _get_current_state_ids_txn)
495
496		# FIXME: how should this be cached?
497		def get_filtered_current_state_ids(self, room_id, state_filter=StateFilter.all()):
498		"""Get the current state event of a given type for a room based on the
499		current_state_events table. This may not be as up-to-date as the result
500		of doing a fresh state resolution as per state_handler.get_current_state
501
502		Args:
503		room_id (str)
504		state_filter (StateFilter): The state filter used to fetch state
505		from the database.
506
507		Returns:
508		Deferred[dict[tuple[str, str], str]]: Map from type/state_key to
509		event ID.
510		"""
511
512		where_clause, where_args = state_filter.make_sql_filter_clause()
513
514		if not where_clause:
515		# We delegate to the cached version
516		return self.get_current_state_ids(room_id)
517
518		def _get_filtered_current_state_ids_txn(txn):
519		results = {}
520		sql = """
521		SELECT type, state_key, event_id FROM current_state_events
522		WHERE room_id = ?
523		"""
524
525		if where_clause:
526		sql += " AND (%s)" % (where_clause,)
527
528		args = [room_id]
529		args.extend(where_args)
530		txn.execute(sql, args)
531		for row in txn:
532		typ, state_key, event_id = row
533		key = (intern_string(typ), intern_string(state_key))
534		results[key] = event_id
535
536		return results
537
538		return self.runInteraction(
539		"get_filtered_current_state_ids", _get_filtered_current_state_ids_txn
540		)
541
542		@defer.inlineCallbacks
543		def get_canonical_alias_for_room(self, room_id):
544		"""Get canonical alias for room, if any
545
546		Args:
547		room_id (str)
548
549		Returns:
550		Deferred[str\|None]: The canonical alias, if any
551		"""
552
553		state = yield self.get_filtered_current_state_ids(
554		room_id, StateFilter.from_types([(EventTypes.CanonicalAlias, "")])
555		)
556
557		event_id = state.get((EventTypes.CanonicalAlias, ""))
558		if not event_id:
559		return
560
561		event = yield self.get_event(event_id, allow_none=True)
562		if not event:
563		return
564
565		return event.content.get("canonical_alias")
566
567		@cached(max_entries=10000, iterable=True)
568		def get_state_group_delta(self, state_group):
569		"""Given a state group try to return a previous group and a delta between
570		the old and the new.
571
572		Returns:
573		(prev_group, delta_ids), where both may be None.
574		"""
575
576		def _get_state_group_delta_txn(txn):
577		prev_group = self._simple_select_one_onecol_txn(
578		txn,
579		table="state_group_edges",
580		keyvalues={"state_group": state_group},
581		retcol="prev_state_group",
582		allow_none=True,
583		)
584
585		if not prev_group:
586		return _GetStateGroupDelta(None, None)
587
588		delta_ids = self._simple_select_list_txn(
589		txn,
590		table="state_groups_state",
591		keyvalues={"state_group": state_group},
592		retcols=("type", "state_key", "event_id"),
593		)
594
595		return _GetStateGroupDelta(
596		prev_group,
597		{(row["type"], row["state_key"]): row["event_id"] for row in delta_ids},
598		)
599
600		return self.runInteraction("get_state_group_delta", _get_state_group_delta_txn)
601
602		@defer.inlineCallbacks
603		def get_state_groups_ids(self, _room_id, event_ids):
604		"""Get the event IDs of all the state for the state groups for the given events
605
606		Args:
607		_room_id (str): id of the room for these events
608		event_ids (iterable[str]): ids of the events
609
610		Returns:
611		Deferred[dict[int, dict[tuple[str, str], str]]]:
612		dict of state_group_id -> (dict of (type, state_key) -> event id)
613		"""
614		if not event_ids:
615		return {}
616
617		event_to_groups = yield self._get_state_group_for_events(event_ids)
618
619		groups = set(itervalues(event_to_groups))
620		group_to_state = yield self._get_state_for_groups(groups)
621
622		return group_to_state
623
624		@defer.inlineCallbacks
625		def get_state_ids_for_group(self, state_group):
626		"""Get the event IDs of all the state in the given state group
627
628		Args:
629		state_group (int)
630
631		Returns:
632		Deferred[dict]: Resolves to a map of (type, state_key) -> event_id
633		"""
634		group_to_state = yield self._get_state_for_groups((state_group,))
635
636		return group_to_state[state_group]
637
638		@defer.inlineCallbacks
639		def get_state_groups(self, room_id, event_ids):
640		""" Get the state groups for the given list of event_ids
641
642		Returns:
643		Deferred[dict[int, list[EventBase]]]:
644		dict of state_group_id -> list of state events.
645		"""
646		if not event_ids:
647		return {}
648
649		group_to_ids = yield self.get_state_groups_ids(room_id, event_ids)
650
651		state_event_map = yield self.get_events(
652		[
653		ev_id
654		for group_ids in itervalues(group_to_ids)
655		for ev_id in itervalues(group_ids)
656		],
657		get_prev_content=False,
658		)
659
660		return {
661		group: [
662		state_event_map[v]
663		for v in itervalues(event_id_map)
664		if v in state_event_map
665		]
666		for group, event_id_map in iteritems(group_to_ids)
667		}
668
669		@defer.inlineCallbacks
670		def _get_state_groups_from_groups(self, groups, state_filter):
671		"""Returns the state groups for a given set of groups, filtering on
672		types of state events.
673
674		Args:
675		groups(list[int]): list of state group IDs to query
676		state_filter (StateFilter): The state filter used to fetch state
677		from the database.
678		Returns:
679		Deferred[dict[int, dict[tuple[str, str], str]]]:
680		dict of state_group_id -> (dict of (type, state_key) -> event id)
681		"""
682		results = {}
683
684		chunks = [groups[i : i + 100] for i in range(0, len(groups), 100)]
685		for chunk in chunks:
686		res = yield self.runInteraction(
687		"_get_state_groups_from_groups",
688		self._get_state_groups_from_groups_txn,
689		chunk,
690		state_filter,
691		)
692		results.update(res)
693
694		return results
695
696		def _get_state_groups_from_groups_txn(
697		self, txn, groups, state_filter=StateFilter.all()
698		):
699		results = {group: {} for group in groups}
700
701		where_clause, where_args = state_filter.make_sql_filter_clause()
702
703		# Unless the filter clause is empty, we're going to append it after an
704		# existing where clause
705		if where_clause:
706		where_clause = " AND (%s)" % (where_clause,)
707
708		if isinstance(self.database_engine, PostgresEngine):
709		# Temporarily disable sequential scans in this transaction. This is
710		# a temporary hack until we can add the right indices in
711		txn.execute("SET LOCAL enable_seqscan=off")
712
713		# The below query walks the state_group tree so that the "state"
714		# table includes all state_groups in the tree. It then joins
715		# against `state_groups_state` to fetch the latest state.
716		# It assumes that previous state groups are always numerically
717		# lesser.
718		# The PARTITION is used to get the event_id in the greatest state
719		# group for the given type, state_key.
720		# This may return multiple rows per (type, state_key), but last_value
721		# should be the same.
722		sql = """
723		WITH RECURSIVE state(state_group) AS (
724		VALUES(?::bigint)
725		UNION ALL
726		SELECT prev_state_group FROM state_group_edges e, state s
727		WHERE s.state_group = e.state_group
728		)
729		SELECT DISTINCT type, state_key, last_value(event_id) OVER (
730		PARTITION BY type, state_key ORDER BY state_group ASC
731		ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
732		) AS event_id FROM state_groups_state
733		WHERE state_group IN (
734		SELECT state_group FROM state
735		)
736		"""
737
738		for group in groups:
739		args = [group]
740		args.extend(where_args)
741
742		txn.execute(sql + where_clause, args)
743		for row in txn:
744		typ, state_key, event_id = row
745		key = (typ, state_key)
746		results[group][key] = event_id
747		else:
748		max_entries_returned = state_filter.max_entries_returned()
749
750		# We don't use WITH RECURSIVE on sqlite3 as there are distributions
751		# that ship with an sqlite3 version that doesn't support it (e.g. wheezy)
752		for group in groups:
753		next_group = group
754
755		while next_group:
756		# We did this before by getting the list of group ids, and
757		# then passing that list to sqlite to get latest event for
758		# each (type, state_key). However, that was terribly slow
759		# without the right indices (which we can't add until
760		# after we finish deduping state, which requires this func)
761		args = [next_group]
762		args.extend(where_args)
763
764		txn.execute(
765		"SELECT type, state_key, event_id FROM state_groups_state"
766		" WHERE state_group = ? " + where_clause,
767		args,
768		)
769		results[group].update(
770		((typ, state_key), event_id)
771		for typ, state_key, event_id in txn
772		if (typ, state_key) not in results[group]
773		)
774
775		# If the number of entries in the (type,state_key)->event_id dict
776		# matches the number of (type,state_keys) types we were searching
777		# for, then we must have found them all, so no need to go walk
778		# further down the tree... UNLESS our types filter contained
779		# wildcards (i.e. Nones) in which case we have to do an exhaustive
780		# search
781		if (
782		max_entries_returned is not None
783		and len(results[group]) == max_entries_returned
784		):
785		break
786
787		next_group = self._simple_select_one_onecol_txn(
788		txn,
789		table="state_group_edges",
790		keyvalues={"state_group": next_group},
791		retcol="prev_state_group",
792		allow_none=True,
793		)
794
795		return results
796
797		@defer.inlineCallbacks
798		def get_state_for_events(self, event_ids, state_filter=StateFilter.all()):
799		"""Given a list of event_ids and type tuples, return a list of state
800		dicts for each event.
801
802		Args:
803		event_ids (list[string])
804		state_filter (StateFilter): The state filter used to fetch state
805		from the database.
806
807		Returns:
808		deferred: A dict of (event_id) -> (type, state_key) -> [state_events]
809		"""
810		event_to_groups = yield self._get_state_group_for_events(event_ids)
811
812		groups = set(itervalues(event_to_groups))
813		group_to_state = yield self._get_state_for_groups(groups, state_filter)
814
815		state_event_map = yield self.get_events(
816		[ev_id for sd in itervalues(group_to_state) for ev_id in itervalues(sd)],
817		get_prev_content=False,
818		)
819
820		event_to_state = {
821		event_id: {
822		k: state_event_map[v]
823		for k, v in iteritems(group_to_state[group])
824		if v in state_event_map
825		}
826		for event_id, group in iteritems(event_to_groups)
827		}
828
829		return {event: event_to_state[event] for event in event_ids}
830
831		@defer.inlineCallbacks
832		def get_state_ids_for_events(self, event_ids, state_filter=StateFilter.all()):
833		"""
834		Get the state dicts corresponding to a list of events, containing the event_ids
835		of the state events (as opposed to the events themselves)
836
837		Args:
838		event_ids(list(str)): events whose state should be returned
839		state_filter (StateFilter): The state filter used to fetch state
840		from the database.
841
842		Returns:
843		A deferred dict from event_id -> (type, state_key) -> event_id
844		"""
845		event_to_groups = yield self._get_state_group_for_events(event_ids)
846
847		groups = set(itervalues(event_to_groups))
848		group_to_state = yield self._get_state_for_groups(groups, state_filter)
849
850		event_to_state = {
851		event_id: group_to_state[group]
852		for event_id, group in iteritems(event_to_groups)
853		}
854
855		return {event: event_to_state[event] for event in event_ids}
856
857		@defer.inlineCallbacks
858		def get_state_for_event(self, event_id, state_filter=StateFilter.all()):
859		"""
860		Get the state dict corresponding to a particular event
861
862		Args:
863		event_id(str): event whose state should be returned
864		state_filter (StateFilter): The state filter used to fetch state
865		from the database.
866
867		Returns:
868		A deferred dict from (type, state_key) -> state_event
869		"""
870		state_map = yield self.get_state_for_events([event_id], state_filter)
871		return state_map[event_id]
872
873		@defer.inlineCallbacks
874		def get_state_ids_for_event(self, event_id, state_filter=StateFilter.all()):
875		"""
876		Get the state dict corresponding to a particular event
877
878		Args:
879		event_id(str): event whose state should be returned
880		state_filter (StateFilter): The state filter used to fetch state
881		from the database.
882
883		Returns:
884		A deferred dict from (type, state_key) -> state_event
885		"""
886		state_map = yield self.get_state_ids_for_events([event_id], state_filter)
887		return state_map[event_id]
888
889		@cached(max_entries=50000)
890		def _get_state_group_for_event(self, event_id):
891		return self._simple_select_one_onecol(
892		table="event_to_state_groups",
893		keyvalues={"event_id": event_id},
894		retcol="state_group",
895		allow_none=True,
896		desc="_get_state_group_for_event",
897		)
898
899		@cachedList(
900		cached_method_name="_get_state_group_for_event",
901		list_name="event_ids",
902		num_args=1,
903		inlineCallbacks=True,
904		)
905		def _get_state_group_for_events(self, event_ids):
906		"""Returns mapping event_id -> state_group
907		"""
908		rows = yield self._simple_select_many_batch(
909		table="event_to_state_groups",
910		column="event_id",
911		iterable=event_ids,
912		keyvalues={},
913		retcols=("event_id", "state_group"),
914		desc="_get_state_group_for_events",
915		)
916
917		return {row["event_id"]: row["state_group"] for row in rows}
918
919		def _get_state_for_group_using_cache(self, cache, group, state_filter):
920		"""Checks if group is in cache. See `_get_state_for_groups`
921
922		Args:
923		cache(DictionaryCache): the state group cache to use
924		group(int): The state group to lookup
925		state_filter (StateFilter): The state filter used to fetch state
926		from the database.
927
928		Returns 2-tuple (`state_dict`, `got_all`).
929		`got_all` is a bool indicating if we successfully retrieved all
930		requests state from the cache, if False we need to query the DB for the
931		missing state.
932		"""
933		is_all, known_absent, state_dict_ids = cache.get(group)
934
935		if is_all or state_filter.is_full():
936		# Either we have everything or want everything, either way
937		# `is_all` tells us whether we've gotten everything.
938		return state_filter.filter_state(state_dict_ids), is_all
939
940		# tracks whether any of our requested types are missing from the cache
941		missing_types = False
942
943		if state_filter.has_wildcards():
944		# We don't know if we fetched all the state keys for the types in
945		# the filter that are wildcards, so we have to assume that we may
946		# have missed some.
947		missing_types = True
948		else:
949		# There aren't any wild cards, so `concrete_types()` returns the
950		# complete list of event types we're wanting.
951		for key in state_filter.concrete_types():
952		if key not in state_dict_ids and key not in known_absent:
953		missing_types = True
954		break
955
956		return state_filter.filter_state(state_dict_ids), not missing_types
957
958		@defer.inlineCallbacks
959		def _get_state_for_groups(self, groups, state_filter=StateFilter.all()):
960		"""Gets the state at each of a list of state groups, optionally
961		filtering by type/state_key
962
963		Args:
964		groups (iterable[int]): list of state groups for which we want
965		to get the state.
966		state_filter (StateFilter): The state filter used to fetch state
967		from the database.
968		Returns:
969		Deferred[dict[int, dict[tuple[str, str], str]]]:
970		dict of state_group_id -> (dict of (type, state_key) -> event id)
971		"""
972
973		member_filter, non_member_filter = state_filter.get_member_split()
974
975		# Now we look them up in the member and non-member caches
976		non_member_state, incomplete_groups_nm, = (
977		yield self._get_state_for_groups_using_cache(
978		groups, self._state_group_cache, state_filter=non_member_filter
979		)
980		)
981
982		member_state, incomplete_groups_m, = (
983		yield self._get_state_for_groups_using_cache(
984		groups, self._state_group_members_cache, state_filter=member_filter
985		)
986		)
987
988		state = dict(non_member_state)
989		for group in groups:
990		state[group].update(member_state[group])
991
992		# Now fetch any missing groups from the database
993
994		incomplete_groups = incomplete_groups_m \| incomplete_groups_nm
995
996		if not incomplete_groups:
997		return state
998
999		cache_sequence_nm = self._state_group_cache.sequence
1000		cache_sequence_m = self._state_group_members_cache.sequence
1001
1002		# Help the cache hit ratio by expanding the filter a bit
1003		db_state_filter = state_filter.return_expanded()
1004
1005		group_to_state_dict = yield self._get_state_groups_from_groups(
1006		list(incomplete_groups), state_filter=db_state_filter
1007		)
1008
1009		# Now lets update the caches
1010		self._insert_into_cache(
1011		group_to_state_dict,
1012		db_state_filter,
1013		cache_seq_num_members=cache_sequence_m,
1014		cache_seq_num_non_members=cache_sequence_nm,
1015		)
1016
1017		# And finally update the result dict, by filtering out any extra
1018		# stuff we pulled out of the database.
1019		for group, group_state_dict in iteritems(group_to_state_dict):
1020		# We just replace any existing entries, as we will have loaded
1021		# everything we need from the database anyway.
1022		state[group] = state_filter.filter_state(group_state_dict)
1023
1024		return state
1025
1026		def _get_state_for_groups_using_cache(self, groups, cache, state_filter):
1027		"""Gets the state at each of a list of state groups, optionally
1028		filtering by type/state_key, querying from a specific cache.
1029
1030		Args:
1031		groups (iterable[int]): list of state groups for which we want
1032		to get the state.
1033		cache (DictionaryCache): the cache of group ids to state dicts which
1034		we will pass through - either the normal state cache or the specific
1035		members state cache.
1036		state_filter (StateFilter): The state filter used to fetch state
1037		from the database.
1038
1039		Returns:
1040		tuple[dict[int, dict[tuple[str, str], str]], set[int]]: Tuple of
1041		dict of state_group_id -> (dict of (type, state_key) -> event id)
1042		of entries in the cache, and the state group ids either missing
1043		from the cache or incomplete.
1044		"""
1045		results = {}
1046		incomplete_groups = set()
1047		for group in set(groups):
1048		state_dict_ids, got_all = self._get_state_for_group_using_cache(
1049		cache, group, state_filter
1050		)
1051		results[group] = state_dict_ids
1052
1053		if not got_all:
1054		incomplete_groups.add(group)
1055
1056		return results, incomplete_groups
1057
1058		def _insert_into_cache(
1059		self,
1060		group_to_state_dict,
1061		state_filter,
1062		cache_seq_num_members,
1063		cache_seq_num_non_members,
1064		):
1065		"""Inserts results from querying the database into the relevant cache.
1066
1067		Args:
1068		group_to_state_dict (dict): The new entries pulled from database.
1069		Map from state group to state dict
1070		state_filter (StateFilter): The state filter used to fetch state
1071		from the database.
1072		cache_seq_num_members (int): Sequence number of member cache since
1073		last lookup in cache
1074		cache_seq_num_non_members (int): Sequence number of member cache since
1075		last lookup in cache
1076		"""
1077
1078		# We need to work out which types we've fetched from the DB for the
1079		# member vs non-member caches. This should be as accurate as possible,
1080		# but can be an underestimate (e.g. when we have wild cards)
1081
1082		member_filter, non_member_filter = state_filter.get_member_split()
1083		if member_filter.is_full():
1084		# We fetched all member events
1085		member_types = None
1086		else:
1087		# `concrete_types()` will only return a subset when there are wild
1088		# cards in the filter, but that's fine.
1089		member_types = member_filter.concrete_types()
1090
1091		if non_member_filter.is_full():
1092		# We fetched all non member events
1093		non_member_types = None
1094		else:
1095		non_member_types = non_member_filter.concrete_types()
1096
1097		for group, group_state_dict in iteritems(group_to_state_dict):
1098		state_dict_members = {}
1099		state_dict_non_members = {}
1100
1101		for k, v in iteritems(group_state_dict):
1102		if k[0] == EventTypes.Member:
1103		state_dict_members[k] = v
1104		else:
1105		state_dict_non_members[k] = v
1106
1107		self._state_group_members_cache.update(
1108		cache_seq_num_members,
1109		key=group,
1110		value=state_dict_members,
1111		fetched_keys=member_types,
1112		)
1113
1114		self._state_group_cache.update(
1115		cache_seq_num_non_members,
1116		key=group,
1117		value=state_dict_non_members,
1118		fetched_keys=non_member_types,
1119		)
1120
1121		def store_state_group(
1122		self, event_id, room_id, prev_group, delta_ids, current_state_ids
1123		):
1124		"""Store a new set of state, returning a newly assigned state group.
1125
1126		Args:
1127		event_id (str): The event ID for which the state was calculated
1128		room_id (str)
1129		prev_group (int\|None): A previous state group for the room, optional.
1130		delta_ids (dict\|None): The delta between state at `prev_group` and
1131		`current_state_ids`, if `prev_group` was given. Same format as
1132		`current_state_ids`.
1133		current_state_ids (dict): The state to store. Map of (type, state_key)
1134		to event_id.
1135
1136		Returns:
1137		Deferred[int]: The state group ID
1138		"""
1139
1140		def _store_state_group_txn(txn):
1141		if current_state_ids is None:
1142		# AFAIK, this can never happen
1143		raise Exception("current_state_ids cannot be None")
1144
1145		state_group = self.database_engine.get_next_state_group_id(txn)
1146
1147		self._simple_insert_txn(
1148		txn,
1149		table="state_groups",
1150		values={"id": state_group, "room_id": room_id, "event_id": event_id},
1151		)
1152
1153		# We persist as a delta if we can, while also ensuring the chain
1154		# of deltas isn't tooo long, as otherwise read performance degrades.
1155		if prev_group:
1156		is_in_db = self._simple_select_one_onecol_txn(
1157		txn,
1158		table="state_groups",
1159		keyvalues={"id": prev_group},
1160		retcol="id",
1161		allow_none=True,
1162		)
1163		if not is_in_db:
1164		raise Exception(
1165		"Trying to persist state with unpersisted prev_group: %r"
1166		% (prev_group,)
1167		)
1168
1169		potential_hops = self._count_state_group_hops_txn(txn, prev_group)
1170		if prev_group and potential_hops < MAX_STATE_DELTA_HOPS:
1171		self._simple_insert_txn(
1172		txn,
1173		table="state_group_edges",
1174		values={"state_group": state_group, "prev_state_group": prev_group},
1175		)
1176
1177		self._simple_insert_many_txn(
1178		txn,
1179		table="state_groups_state",
1180		values=[
1181		{
1182		"state_group": state_group,
1183		"room_id": room_id,
1184		"type": key[0],
1185		"state_key": key[1],
1186		"event_id": state_id,
1187		}
1188		for key, state_id in iteritems(delta_ids)
1189		],
1190		)
1191		else:
1192		self._simple_insert_many_txn(
1193		txn,
1194		table="state_groups_state",
1195		values=[
1196		{
1197		"state_group": state_group,
1198		"room_id": room_id,
1199		"type": key[0],
1200		"state_key": key[1],
1201		"event_id": state_id,
1202		}
1203		for key, state_id in iteritems(current_state_ids)
1204		],
1205		)
1206
1207		# Prefill the state group caches with this group.
1208		# It's fine to use the sequence like this as the state group map
1209		# is immutable. (If the map wasn't immutable then this prefill could
1210		# race with another update)
1211
1212		current_member_state_ids = {
1213		s: ev
1214		for (s, ev) in iteritems(current_state_ids)
1215		if s[0] == EventTypes.Member
1216		}
1217		txn.call_after(
1218		self._state_group_members_cache.update,
1219		self._state_group_members_cache.sequence,
1220		key=state_group,
1221		value=dict(current_member_state_ids),
1222		)
1223
1224		current_non_member_state_ids = {
1225		s: ev
1226		for (s, ev) in iteritems(current_state_ids)
1227		if s[0] != EventTypes.Member
1228		}
1229		txn.call_after(
1230		self._state_group_cache.update,
1231		self._state_group_cache.sequence,
1232		key=state_group,
1233		value=dict(current_non_member_state_ids),
1234		)
1235
1236		return state_group
1237
1238		return self.runInteraction("store_state_group", _store_state_group_txn)
1239
1240		def _count_state_group_hops_txn(self, txn, state_group):
1241		"""Given a state group, count how many hops there are in the tree.
1242
1243		This is used to ensure the delta chains don't get too long.
1244		"""
1245		if isinstance(self.database_engine, PostgresEngine):
1246		sql = """
1247		WITH RECURSIVE state(state_group) AS (
1248		VALUES(?::bigint)
1249		UNION ALL
1250		SELECT prev_state_group FROM state_group_edges e, state s
1251		WHERE s.state_group = e.state_group
1252		)
1253		SELECT count(*) FROM state;
1254		"""
1255
1256		txn.execute(sql, (state_group,))
1257		row = txn.fetchone()
1258		if row and row[0]:
1259		return row[0]
1260		else:
1261		return 0
1262		else:
1263		# We don't use WITH RECURSIVE on sqlite3 as there are distributions
1264		# that ship with an sqlite3 version that doesn't support it (e.g. wheezy)
1265		next_group = state_group
1266		count = 0
1267
1268		while next_group:
1269		next_group = self._simple_select_one_onecol_txn(
1270		txn,
1271		table="state_group_edges",
1272		keyvalues={"state_group": next_group},
1273		retcol="prev_state_group",
1274		allow_none=True,
1275		)
1276		if next_group:
1277		count += 1
1278
1279		return count
1280
1281
1282		class StateStore(StateGroupWorkerStore, BackgroundUpdateStore):
1283		""" Keeps track of the state at a given event.
1284
1285		This is done by the concept of `state groups`. Every event is a assigned
1286		a state group (identified by an arbitrary string), which references a
1287		collection of state events. The current state of an event is then the
1288		collection of state events referenced by the event's state group.
1289
1290		Hence, every change in the current state causes a new state group to be
1291		generated. However, if no change happens (e.g., if we get a message event
1292		with only one parent it inherits the state group from its parent.)
1293
1294		There are three tables:
1295		* `state_groups`: Stores group name, first event with in the group and
1296		room id.
1297		* `event_to_state_groups`: Maps events to state groups.
1298		* `state_groups_state`: Maps state group to state events.
1299		"""
1300
1301		STATE_GROUP_DEDUPLICATION_UPDATE_NAME = "state_group_state_deduplication"
1302		STATE_GROUP_INDEX_UPDATE_NAME = "state_group_state_type_index"
1303		CURRENT_STATE_INDEX_UPDATE_NAME = "current_state_members_idx"
1304		EVENT_STATE_GROUP_INDEX_UPDATE_NAME = "event_to_state_groups_sg_index"
1305
1306		def __init__(self, db_conn, hs):
1307		super(StateStore, self).__init__(db_conn, hs)
1308		self.register_background_update_handler(
1309		self.STATE_GROUP_DEDUPLICATION_UPDATE_NAME,
1310		self._background_deduplicate_state,
1311		)
1312		self.register_background_update_handler(
1313		self.STATE_GROUP_INDEX_UPDATE_NAME, self._background_index_state
1314		)
1315		self.register_background_index_update(
1316		self.CURRENT_STATE_INDEX_UPDATE_NAME,
1317		index_name="current_state_events_member_index",
1318		table="current_state_events",
1319		columns=["state_key"],
1320		where_clause="type='m.room.member'",
1321		)
1322		self.register_background_index_update(
1323		self.EVENT_STATE_GROUP_INDEX_UPDATE_NAME,
1324		index_name="event_to_state_groups_sg_index",
1325		table="event_to_state_groups",
1326		columns=["state_group"],
1327		)
1328
1329		def _store_event_state_mappings_txn(self, txn, events_and_contexts):
1330		state_groups = {}
1331		for event, context in events_and_contexts:
1332		if event.internal_metadata.is_outlier():
1333		continue
1334
1335		# if the event was rejected, just give it the same state as its
1336		# predecessor.
1337		if context.rejected:
1338		state_groups[event.event_id] = context.prev_group
1339		continue
1340
1341		state_groups[event.event_id] = context.state_group
1342
1343		self._simple_insert_many_txn(
1344		txn,
1345		table="event_to_state_groups",
1346		values=[
1347		{"state_group": state_group_id, "event_id": event_id}
1348		for event_id, state_group_id in iteritems(state_groups)
1349		],
1350		)
1351
1352		for event_id, state_group_id in iteritems(state_groups):
1353		txn.call_after(
1354		self._get_state_group_for_event.prefill, (event_id,), state_group_id
1355		)
1356
1357		@defer.inlineCallbacks
1358		def _background_deduplicate_state(self, progress, batch_size):
1359		"""This background update will slowly deduplicate state by reencoding
1360		them as deltas.
1361		"""
1362		last_state_group = progress.get("last_state_group", 0)
1363		rows_inserted = progress.get("rows_inserted", 0)
1364		max_group = progress.get("max_group", None)
1365
1366		BATCH_SIZE_SCALE_FACTOR = 100
1367
1368		batch_size = max(1, int(batch_size / BATCH_SIZE_SCALE_FACTOR))
1369
1370		if max_group is None:
1371		rows = yield self._execute(
1372		"_background_deduplicate_state",
1373		None,
1374		"SELECT coalesce(max(id), 0) FROM state_groups",
1375		)
1376		max_group = rows[0][0]
1377
1378		def reindex_txn(txn):
1379		new_last_state_group = last_state_group
1380		for count in range(batch_size):
1381		txn.execute(
1382		"SELECT id, room_id FROM state_groups"
1383		" WHERE ? < id AND id <= ?"
1384		" ORDER BY id ASC"
1385		" LIMIT 1",
1386		(new_last_state_group, max_group),
1387		)
1388		row = txn.fetchone()
1389		if row:
1390		state_group, room_id = row
1391
1392		if not row or not state_group:
1393		return True, count
1394
1395		txn.execute(
1396		"SELECT state_group FROM state_group_edges"
1397		" WHERE state_group = ?",
1398		(state_group,),
1399		)
1400
1401		# If we reach a point where we've already started inserting
1402		# edges we should stop.
1403		if txn.fetchall():
1404		return True, count
1405
1406		txn.execute(
1407		"SELECT coalesce(max(id), 0) FROM state_groups"
1408		" WHERE id < ? AND room_id = ?",
1409		(state_group, room_id),
1410		)
1411		prev_group, = txn.fetchone()
1412		new_last_state_group = state_group
1413
1414		if prev_group:
1415		potential_hops = self._count_state_group_hops_txn(txn, prev_group)
1416		if potential_hops >= MAX_STATE_DELTA_HOPS:
1417		# We want to ensure chains are at most this long,#
1418		# otherwise read performance degrades.
1419		continue
1420
1421		prev_state = self._get_state_groups_from_groups_txn(
1422		txn, [prev_group]
1423		)
1424		prev_state = prev_state[prev_group]
1425
1426		curr_state = self._get_state_groups_from_groups_txn(
1427		txn, [state_group]
1428		)
1429		curr_state = curr_state[state_group]
1430
1431		if not set(prev_state.keys()) - set(curr_state.keys()):
1432		# We can only do a delta if the current has a strict super set
1433		# of keys
1434
1435		delta_state = {
1436		key: value
1437		for key, value in iteritems(curr_state)
1438		if prev_state.get(key, None) != value
1439		}
1440
1441		self._simple_delete_txn(
1442		txn,
1443		table="state_group_edges",
1444		keyvalues={"state_group": state_group},
1445		)
1446
1447		self._simple_insert_txn(
1448		txn,
1449		table="state_group_edges",
1450		values={
1451		"state_group": state_group,
1452		"prev_state_group": prev_group,
1453		},
1454		)
1455
1456		self._simple_delete_txn(
1457		txn,
1458		table="state_groups_state",
1459		keyvalues={"state_group": state_group},
1460		)
1461
1462		self._simple_insert_many_txn(
1463		txn,
1464		table="state_groups_state",
1465		values=[
1466		{
1467		"state_group": state_group,
1468		"room_id": room_id,
1469		"type": key[0],
1470		"state_key": key[1],
1471		"event_id": state_id,
1472		}
1473		for key, state_id in iteritems(delta_state)
1474		],
1475		)
1476
1477		progress = {
1478		"last_state_group": state_group,
1479		"rows_inserted": rows_inserted + batch_size,
1480		"max_group": max_group,
1481		}
1482
1483		self._background_update_progress_txn(
1484		txn, self.STATE_GROUP_DEDUPLICATION_UPDATE_NAME, progress
1485		)
1486
1487		return False, batch_size
1488
1489		finished, result = yield self.runInteraction(
1490		self.STATE_GROUP_DEDUPLICATION_UPDATE_NAME, reindex_txn
1491		)
1492
1493		if finished:
1494		yield self._end_background_update(
1495		self.STATE_GROUP_DEDUPLICATION_UPDATE_NAME
1496		)
1497
1498		return result * BATCH_SIZE_SCALE_FACTOR
1499
1500		@defer.inlineCallbacks
1501		def _background_index_state(self, progress, batch_size):
1502		def reindex_txn(conn):
1503		conn.rollback()
1504		if isinstance(self.database_engine, PostgresEngine):
1505		# postgres insists on autocommit for the index
1506		conn.set_session(autocommit=True)
1507		try:
1508		txn = conn.cursor()
1509		txn.execute(
1510		"CREATE INDEX CONCURRENTLY state_groups_state_type_idx"
1511		" ON state_groups_state(state_group, type, state_key)"
1512		)
1513		txn.execute("DROP INDEX IF EXISTS state_groups_state_id")
1514		finally:
1515		conn.set_session(autocommit=False)
1516		else:
1517		txn = conn.cursor()
1518		txn.execute(
1519		"CREATE INDEX state_groups_state_type_idx"
1520		" ON state_groups_state(state_group, type, state_key)"
1521		)
1522		txn.execute("DROP INDEX IF EXISTS state_groups_state_id")
1523
1524		yield self.runWithConnection(reindex_txn)
1525
1526		yield self._end_background_update(self.STATE_GROUP_INDEX_UPDATE_NAME)
1527
1528		return 1

+0

-99

~~synapse/storage/state_deltas.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2018 Vector Creations Ltd
2		#
3		# Licensed under the Apache License, Version 2.0 (the "License");
4		# you may not use this file except in compliance with the License.
5		# You may obtain a copy of the License at
6		#
7		# http://www.apache.org/licenses/LICENSE-2.0
8		#
9		# Unless required by applicable law or agreed to in writing, software
10		# distributed under the License is distributed on an "AS IS" BASIS,
11		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12		# See the License for the specific language governing permissions and
13		# limitations under the License.
14
15		import logging
16
17		from synapse.storage._base import SQLBaseStore
18
19		logger = logging.getLogger(__name__)
20
21
22		class StateDeltasStore(SQLBaseStore):
23		def get_current_state_deltas(self, prev_stream_id):
24		"""Fetch a list of room state changes since the given stream id
25
26		Each entry in the result contains the following fields:
27		- stream_id (int)
28		- room_id (str)
29		- type (str): event type
30		- state_key (str):
31		- event_id (str\|None): new event_id for this state key. None if the
32		state has been deleted.
33		- prev_event_id (str\|None): previous event_id for this state key. None
34		if it's new state.
35
36		Args:
37		prev_stream_id (int): point to get changes since (exclusive)
38
39		Returns:
40		Deferred[list[dict]]: results
41		"""
42		prev_stream_id = int(prev_stream_id)
43		if not self._curr_state_delta_stream_cache.has_any_entity_changed(
44		prev_stream_id
45		):
46		return []
47
48		def get_current_state_deltas_txn(txn):
49		# First we calculate the max stream id that will give us less than
50		# N results.
51		# We arbitarily limit to 100 stream_id entries to ensure we don't
52		# select toooo many.
53		sql = """
54		SELECT stream_id, count(*)
55		FROM current_state_delta_stream
56		WHERE stream_id > ?
57		GROUP BY stream_id
58		ORDER BY stream_id ASC
59		LIMIT 100
60		"""
61		txn.execute(sql, (prev_stream_id,))
62
63		total = 0
64		max_stream_id = prev_stream_id
65		for max_stream_id, count in txn:
66		total += count
67		if total > 100:
68		# We arbitarily limit to 100 entries to ensure we don't
69		# select toooo many.
70		break
71
72		# Now actually get the deltas
73		sql = """
74		SELECT stream_id, room_id, type, state_key, event_id, prev_event_id
75		FROM current_state_delta_stream
76		WHERE ? < stream_id AND stream_id <= ?
77		ORDER BY stream_id ASC
78		"""
79		txn.execute(sql, (prev_stream_id, max_stream_id))
80		return self.cursor_to_dict(txn)
81
82		return self.runInteraction(
83		"get_current_state_deltas", get_current_state_deltas_txn
84		)
85
86		def _get_max_stream_id_in_current_state_deltas_txn(self, txn):
87		return self._simple_select_one_onecol_txn(
88		txn,
89		table="current_state_delta_stream",
90		keyvalues={},
91		retcol="COALESCE(MAX(stream_id), -1)",
92		)
93
94		def get_max_stream_id_in_current_state_deltas(self):
95		return self.runInteraction(
96		"get_max_stream_id_in_current_state_deltas",
97		self._get_max_stream_id_in_current_state_deltas_txn,
98		)

+0

-878

~~synapse/storage/stats.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2018, 2019 New Vector Ltd
2		# Copyright 2019 The Matrix.org Foundation C.I.C.
3		#
4		# Licensed under the Apache License, Version 2.0 (the "License");
5		# you may not use this file except in compliance with the License.
6		# You may obtain a copy of the License at
7		#
8		# http://www.apache.org/licenses/LICENSE-2.0
9		#
10		# Unless required by applicable law or agreed to in writing, software
11		# distributed under the License is distributed on an "AS IS" BASIS,
12		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13		# See the License for the specific language governing permissions and
14		# limitations under the License.
15
16		import logging
17		from itertools import chain
18
19		from twisted.internet import defer
20		from twisted.internet.defer import DeferredLock
21
22		from synapse.api.constants import EventTypes, Membership
23		from synapse.storage import PostgresEngine
24		from synapse.storage.state_deltas import StateDeltasStore
25		from synapse.util.caches.descriptors import cached
26
27		logger = logging.getLogger(__name__)
28
29		# these fields track absolutes (e.g. total number of rooms on the server)
30		# You can think of these as Prometheus Gauges.
31		# You can draw these stats on a line graph.
32		# Example: number of users in a room
33		ABSOLUTE_STATS_FIELDS = {
34		"room": (
35		"current_state_events",
36		"joined_members",
37		"invited_members",
38		"left_members",
39		"banned_members",
40		"local_users_in_room",
41		),
42		"user": ("joined_rooms",),
43		}
44
45		# these fields are per-timeslice and so should be reset to 0 upon a new slice
46		# You can draw these stats on a histogram.
47		# Example: number of events sent locally during a time slice
48		PER_SLICE_FIELDS = {
49		"room": ("total_events", "total_event_bytes"),
50		"user": ("invites_sent", "rooms_created", "total_events", "total_event_bytes"),
51		}
52
53		TYPE_TO_TABLE = {"room": ("room_stats", "room_id"), "user": ("user_stats", "user_id")}
54
55		# these are the tables (& ID columns) which contain our actual subjects
56		TYPE_TO_ORIGIN_TABLE = {"room": ("rooms", "room_id"), "user": ("users", "name")}
57
58
59		class StatsStore(StateDeltasStore):
60		def __init__(self, db_conn, hs):
61		super(StatsStore, self).__init__(db_conn, hs)
62
63		self.server_name = hs.hostname
64		self.clock = self.hs.get_clock()
65		self.stats_enabled = hs.config.stats_enabled
66		self.stats_bucket_size = hs.config.stats_bucket_size
67
68		self.stats_delta_processing_lock = DeferredLock()
69
70		self.register_background_update_handler(
71		"populate_stats_process_rooms", self._populate_stats_process_rooms
72		)
73		self.register_background_update_handler(
74		"populate_stats_process_users", self._populate_stats_process_users
75		)
76		# we no longer need to perform clean-up, but we will give ourselves
77		# the potential to reintroduce it in the future – so documentation
78		# will still encourage the use of this no-op handler.
79		self.register_noop_background_update("populate_stats_cleanup")
80		self.register_noop_background_update("populate_stats_prepare")
81
82		def quantise_stats_time(self, ts):
83		"""
84		Quantises a timestamp to be a multiple of the bucket size.
85
86		Args:
87		ts (int): the timestamp to quantise, in milliseconds since the Unix
88		Epoch
89
90		Returns:
91		int: a timestamp which
92		- is divisible by the bucket size;
93		- is no later than `ts`; and
94		- is the largest such timestamp.
95		"""
96		return (ts // self.stats_bucket_size) * self.stats_bucket_size
97
98		@defer.inlineCallbacks
99		def _populate_stats_process_users(self, progress, batch_size):
100		"""
101		This is a background update which regenerates statistics for users.
102		"""
103		if not self.stats_enabled:
104		yield self._end_background_update("populate_stats_process_users")
105		return 1
106
107		last_user_id = progress.get("last_user_id", "")
108
109		def _get_next_batch(txn):
110		sql = """
111		SELECT DISTINCT name FROM users
112		WHERE name > ?
113		ORDER BY name ASC
114		LIMIT ?
115		"""
116		txn.execute(sql, (last_user_id, batch_size))
117		return [r for r, in txn]
118
119		users_to_work_on = yield self.runInteraction(
120		"_populate_stats_process_users", _get_next_batch
121		)
122
123		# No more rooms -- complete the transaction.
124		if not users_to_work_on:
125		yield self._end_background_update("populate_stats_process_users")
126		return 1
127
128		for user_id in users_to_work_on:
129		yield self._calculate_and_set_initial_state_for_user(user_id)
130		progress["last_user_id"] = user_id
131
132		yield self.runInteraction(
133		"populate_stats_process_users",
134		self._background_update_progress_txn,
135		"populate_stats_process_users",
136		progress,
137		)
138
139		return len(users_to_work_on)
140
141		@defer.inlineCallbacks
142		def _populate_stats_process_rooms(self, progress, batch_size):
143		"""
144		This is a background update which regenerates statistics for rooms.
145		"""
146		if not self.stats_enabled:
147		yield self._end_background_update("populate_stats_process_rooms")
148		return 1
149
150		last_room_id = progress.get("last_room_id", "")
151
152		def _get_next_batch(txn):
153		sql = """
154		SELECT DISTINCT room_id FROM current_state_events
155		WHERE room_id > ?
156		ORDER BY room_id ASC
157		LIMIT ?
158		"""
159		txn.execute(sql, (last_room_id, batch_size))
160		return [r for r, in txn]
161
162		rooms_to_work_on = yield self.runInteraction(
163		"populate_stats_rooms_get_batch", _get_next_batch
164		)
165
166		# No more rooms -- complete the transaction.
167		if not rooms_to_work_on:
168		yield self._end_background_update("populate_stats_process_rooms")
169		return 1
170
171		for room_id in rooms_to_work_on:
172		yield self._calculate_and_set_initial_state_for_room(room_id)
173		progress["last_room_id"] = room_id
174
175		yield self.runInteraction(
176		"_populate_stats_process_rooms",
177		self._background_update_progress_txn,
178		"populate_stats_process_rooms",
179		progress,
180		)
181
182		return len(rooms_to_work_on)
183
184		def get_stats_positions(self):
185		"""
186		Returns the stats processor positions.
187		"""
188		return self._simple_select_one_onecol(
189		table="stats_incremental_position",
190		keyvalues={},
191		retcol="stream_id",
192		desc="stats_incremental_position",
193		)
194
195		def update_room_state(self, room_id, fields):
196		"""
197		Args:
198		room_id (str)
199		fields (dict[str:Any])
200		"""
201
202		# For whatever reason some of the fields may contain null bytes, which
203		# postgres isn't a fan of, so we replace those fields with null.
204		for col in (
205		"join_rules",
206		"history_visibility",
207		"encryption",
208		"name",
209		"topic",
210		"avatar",
211		"canonical_alias",
212		):
213		field = fields.get(col)
214		if field and "\0" in field:
215		fields[col] = None
216
217		return self._simple_upsert(
218		table="room_stats_state",
219		keyvalues={"room_id": room_id},
220		values=fields,
221		desc="update_room_state",
222		)
223
224		def get_statistics_for_subject(self, stats_type, stats_id, start, size=100):
225		"""
226		Get statistics for a given subject.
227
228		Args:
229		stats_type (str): The type of subject
230		stats_id (str): The ID of the subject (e.g. room_id or user_id)
231		start (int): Pagination start. Number of entries, not timestamp.
232		size (int): How many entries to return.
233
234		Returns:
235		Deferred[list[dict]], where the dict has the keys of
236		ABSOLUTE_STATS_FIELDS[stats_type], and "bucket_size" and "end_ts".
237		"""
238		return self.runInteraction(
239		"get_statistics_for_subject",
240		self._get_statistics_for_subject_txn,
241		stats_type,
242		stats_id,
243		start,
244		size,
245		)
246
247		def _get_statistics_for_subject_txn(
248		self, txn, stats_type, stats_id, start, size=100
249		):
250		"""
251		Transaction-bound version of L{get_statistics_for_subject}.
252		"""
253
254		table, id_col = TYPE_TO_TABLE[stats_type]
255		selected_columns = list(
256		ABSOLUTE_STATS_FIELDS[stats_type] + PER_SLICE_FIELDS[stats_type]
257		)
258
259		slice_list = self._simple_select_list_paginate_txn(
260		txn,
261		table + "_historical",
262		{id_col: stats_id},
263		"end_ts",
264		start,
265		size,
266		retcols=selected_columns + ["bucket_size", "end_ts"],
267		order_direction="DESC",
268		)
269
270		return slice_list
271
272		def get_room_stats_state(self, room_id):
273		"""
274		Returns the current room_stats_state for a room.
275
276		Args:
277		room_id (str): The ID of the room to return state for.
278
279		Returns (dict):
280		Dictionary containing these keys:
281		"name", "topic", "canonical_alias", "avatar", "join_rules",
282		"history_visibility"
283		"""
284		return self._simple_select_one(
285		"room_stats_state",
286		{"room_id": room_id},
287		retcols=(
288		"name",
289		"topic",
290		"canonical_alias",
291		"avatar",
292		"join_rules",
293		"history_visibility",
294		),
295		)
296
297		@cached()
298		def get_earliest_token_for_stats(self, stats_type, id):
299		"""
300		Fetch the "earliest token". This is used by the room stats delta
301		processor to ignore deltas that have been processed between the
302		start of the background task and any particular room's stats
303		being calculated.
304
305		Returns:
306		Deferred[int]
307		"""
308		table, id_col = TYPE_TO_TABLE[stats_type]
309
310		return self._simple_select_one_onecol(
311		"%s_current" % (table,),
312		keyvalues={id_col: id},
313		retcol="completed_delta_stream_id",
314		allow_none=True,
315		)
316
317		def bulk_update_stats_delta(self, ts, updates, stream_id):
318		"""Bulk update stats tables for a given stream_id and updates the stats
319		incremental position.
320
321		Args:
322		ts (int): Current timestamp in ms
323		updates(dict[str, dict[str, dict[str, Counter]]]): The updates to
324		commit as a mapping stats_type -> stats_id -> field -> delta.
325		stream_id (int): Current position.
326
327		Returns:
328		Deferred
329		"""
330
331		def _bulk_update_stats_delta_txn(txn):
332		for stats_type, stats_updates in updates.items():
333		for stats_id, fields in stats_updates.items():
334		self._update_stats_delta_txn(
335		txn,
336		ts=ts,
337		stats_type=stats_type,
338		stats_id=stats_id,
339		fields=fields,
340		complete_with_stream_id=stream_id,
341		)
342
343		self._simple_update_one_txn(
344		txn,
345		table="stats_incremental_position",
346		keyvalues={},
347		updatevalues={"stream_id": stream_id},
348		)
349
350		return self.runInteraction(
351		"bulk_update_stats_delta", _bulk_update_stats_delta_txn
352		)
353
354		def update_stats_delta(
355		self,
356		ts,
357		stats_type,
358		stats_id,
359		fields,
360		complete_with_stream_id,
361		absolute_field_overrides=None,
362		):
363		"""
364		Updates the statistics for a subject, with a delta (difference/relative
365		change).
366
367		Args:
368		ts (int): timestamp of the change
369		stats_type (str): "room" or "user" – the kind of subject
370		stats_id (str): the subject's ID (room ID or user ID)
371		fields (dict[str, int]): Deltas of stats values.
372		complete_with_stream_id (int, optional):
373		If supplied, converts an incomplete row into a complete row,
374		with the supplied stream_id marked as the stream_id where the
375		row was completed.
376		absolute_field_overrides (dict[str, int]): Current stats values
377		(i.e. not deltas) of absolute fields.
378		Does not work with per-slice fields.
379		"""
380
381		return self.runInteraction(
382		"update_stats_delta",
383		self._update_stats_delta_txn,
384		ts,
385		stats_type,
386		stats_id,
387		fields,
388		complete_with_stream_id=complete_with_stream_id,
389		absolute_field_overrides=absolute_field_overrides,
390		)
391
392		def _update_stats_delta_txn(
393		self,
394		txn,
395		ts,
396		stats_type,
397		stats_id,
398		fields,
399		complete_with_stream_id,
400		absolute_field_overrides=None,
401		):
402		if absolute_field_overrides is None:
403		absolute_field_overrides = {}
404
405		table, id_col = TYPE_TO_TABLE[stats_type]
406
407		quantised_ts = self.quantise_stats_time(int(ts))
408		end_ts = quantised_ts + self.stats_bucket_size
409
410		# Lets be paranoid and check that all the given field names are known
411		abs_field_names = ABSOLUTE_STATS_FIELDS[stats_type]
412		slice_field_names = PER_SLICE_FIELDS[stats_type]
413		for field in chain(fields.keys(), absolute_field_overrides.keys()):
414		if field not in abs_field_names and field not in slice_field_names:
415		# guard against potential SQL injection dodginess
416		raise ValueError(
417		"%s is not a recognised field"
418		" for stats type %s" % (field, stats_type)
419		)
420
421		# Per slice fields do not get added to the _current table
422
423		# This calculates the deltas (`field = field + ?` values)
424		# for absolute fields,
425		# * defaulting to 0 if not specified
426		# (required for the INSERT part of upserting to work)
427		# * omitting overrides specified in `absolute_field_overrides`
428		deltas_of_absolute_fields = {
429		key: fields.get(key, 0)
430		for key in abs_field_names
431		if key not in absolute_field_overrides
432		}
433
434		# Keep the delta stream ID field up to date
435		absolute_field_overrides = absolute_field_overrides.copy()
436		absolute_field_overrides["completed_delta_stream_id"] = complete_with_stream_id
437
438		# first upsert the `_current` table
439		self._upsert_with_additive_relatives_txn(
440		txn=txn,
441		table=table + "_current",
442		keyvalues={id_col: stats_id},
443		absolutes=absolute_field_overrides,
444		additive_relatives=deltas_of_absolute_fields,
445		)
446
447		per_slice_additive_relatives = {
448		key: fields.get(key, 0) for key in slice_field_names
449		}
450		self._upsert_copy_from_table_with_additive_relatives_txn(
451		txn=txn,
452		into_table=table + "_historical",
453		keyvalues={id_col: stats_id},
454		extra_dst_insvalues={"bucket_size": self.stats_bucket_size},
455		extra_dst_keyvalues={"end_ts": end_ts},
456		additive_relatives=per_slice_additive_relatives,
457		src_table=table + "_current",
458		copy_columns=abs_field_names,
459		)
460
461		def _upsert_with_additive_relatives_txn(
462		self, txn, table, keyvalues, absolutes, additive_relatives
463		):
464		"""Used to update values in the stats tables.
465
466		This is basically a slightly convoluted upsert that adds to any
467		existing rows.
468
469		Args:
470		txn
471		table (str): Table name
472		keyvalues (dict[str, any]): Row-identifying key values
473		absolutes (dict[str, any]): Absolute (set) fields
474		additive_relatives (dict[str, int]): Fields that will be added onto
475		if existing row present.
476		"""
477		if self.database_engine.can_native_upsert:
478		absolute_updates = [
479		"%(field)s = EXCLUDED.%(field)s" % {"field": field}
480		for field in absolutes.keys()
481		]
482
483		relative_updates = [
484		"%(field)s = EXCLUDED.%(field)s + %(table)s.%(field)s"
485		% {"table": table, "field": field}
486		for field in additive_relatives.keys()
487		]
488
489		insert_cols = []
490		qargs = []
491
492		for (key, val) in chain(
493		keyvalues.items(), absolutes.items(), additive_relatives.items()
494		):
495		insert_cols.append(key)
496		qargs.append(val)
497
498		sql = """
499		INSERT INTO %(table)s (%(insert_cols_cs)s)
500		VALUES (%(insert_vals_qs)s)
501		ON CONFLICT (%(key_columns)s) DO UPDATE SET %(updates)s
502		""" % {
503		"table": table,
504		"insert_cols_cs": ", ".join(insert_cols),
505		"insert_vals_qs": ", ".join(
506		["?"] * (len(keyvalues) + len(absolutes) + len(additive_relatives))
507		),
508		"key_columns": ", ".join(keyvalues),
509		"updates": ", ".join(chain(absolute_updates, relative_updates)),
510		}
511
512		txn.execute(sql, qargs)
513		else:
514		self.database_engine.lock_table(txn, table)
515		retcols = list(chain(absolutes.keys(), additive_relatives.keys()))
516		current_row = self._simple_select_one_txn(
517		txn, table, keyvalues, retcols, allow_none=True
518		)
519		if current_row is None:
520		merged_dict = {keyvalues, absolutes, **additive_relatives}
521		self._simple_insert_txn(txn, table, merged_dict)
522		else:
523		for (key, val) in additive_relatives.items():
524		current_row[key] += val
525		current_row.update(absolutes)
526		self._simple_update_one_txn(txn, table, keyvalues, current_row)
527
528		def _upsert_copy_from_table_with_additive_relatives_txn(
529		self,
530		txn,
531		into_table,
532		keyvalues,
533		extra_dst_keyvalues,
534		extra_dst_insvalues,
535		additive_relatives,
536		src_table,
537		copy_columns,
538		):
539		"""Updates the historic stats table with latest updates.
540
541		This involves copying "absolute" fields from the `_current` table, and
542		adding relative fields to any existing values.
543
544		Args:
545		txn: Transaction
546		into_table (str): The destination table to UPSERT the row into
547		keyvalues (dict[str, any]): Row-identifying key values
548		extra_dst_keyvalues (dict[str, any]): Additional keyvalues
549		for `into_table`.
550		extra_dst_insvalues (dict[str, any]): Additional values to insert
551		on new row creation for `into_table`.
552		additive_relatives (dict[str, any]): Fields that will be added onto
553		if existing row present. (Must be disjoint from copy_columns.)
554		src_table (str): The source table to copy from
555		copy_columns (iterable[str]): The list of columns to copy
556		"""
557		if self.database_engine.can_native_upsert:
558		ins_columns = chain(
559		keyvalues,
560		copy_columns,
561		additive_relatives,
562		extra_dst_keyvalues,
563		extra_dst_insvalues,
564		)
565		sel_exprs = chain(
566		keyvalues,
567		copy_columns,
568		(
569		"?"
570		for _ in chain(
571		additive_relatives, extra_dst_keyvalues, extra_dst_insvalues
572		)
573		),
574		)
575		keyvalues_where = ("%s = ?" % f for f in keyvalues)
576
577		sets_cc = ("%s = EXCLUDED.%s" % (f, f) for f in copy_columns)
578		sets_ar = (
579		"%s = EXCLUDED.%s + %s.%s" % (f, f, into_table, f)
580		for f in additive_relatives
581		)
582
583		sql = """
584		INSERT INTO %(into_table)s (%(ins_columns)s)
585		SELECT %(sel_exprs)s
586		FROM %(src_table)s
587		WHERE %(keyvalues_where)s
588		ON CONFLICT (%(keyvalues)s)
589		DO UPDATE SET %(sets)s
590		""" % {
591		"into_table": into_table,
592		"ins_columns": ", ".join(ins_columns),
593		"sel_exprs": ", ".join(sel_exprs),
594		"keyvalues_where": " AND ".join(keyvalues_where),
595		"src_table": src_table,
596		"keyvalues": ", ".join(
597		chain(keyvalues.keys(), extra_dst_keyvalues.keys())
598		),
599		"sets": ", ".join(chain(sets_cc, sets_ar)),
600		}
601
602		qargs = list(
603		chain(
604		additive_relatives.values(),
605		extra_dst_keyvalues.values(),
606		extra_dst_insvalues.values(),
607		keyvalues.values(),
608		)
609		)
610		txn.execute(sql, qargs)
611		else:
612		self.database_engine.lock_table(txn, into_table)
613		src_row = self._simple_select_one_txn(
614		txn, src_table, keyvalues, copy_columns
615		)
616		all_dest_keyvalues = {keyvalues, extra_dst_keyvalues}
617		dest_current_row = self._simple_select_one_txn(
618		txn,
619		into_table,
620		keyvalues=all_dest_keyvalues,
621		retcols=list(chain(additive_relatives.keys(), copy_columns)),
622		allow_none=True,
623		)
624
625		if dest_current_row is None:
626		merged_dict = {
627		**keyvalues,
628		**extra_dst_keyvalues,
629		**extra_dst_insvalues,
630		**src_row,
631		**additive_relatives,
632		}
633		self._simple_insert_txn(txn, into_table, merged_dict)
634		else:
635		for (key, val) in additive_relatives.items():
636		src_row[key] = dest_current_row[key] + val
637		self._simple_update_txn(txn, into_table, all_dest_keyvalues, src_row)
638
639		def get_changes_room_total_events_and_bytes(self, min_pos, max_pos):
640		"""Fetches the counts of events in the given range of stream IDs.
641
642		Args:
643		min_pos (int)
644		max_pos (int)
645
646		Returns:
647		Deferred[dict[str, dict[str, int]]]: Mapping of room ID to field
648		changes.
649		"""
650
651		return self.runInteraction(
652		"stats_incremental_total_events_and_bytes",
653		self.get_changes_room_total_events_and_bytes_txn,
654		min_pos,
655		max_pos,
656		)
657
658		def get_changes_room_total_events_and_bytes_txn(self, txn, low_pos, high_pos):
659		"""Gets the total_events and total_event_bytes counts for rooms and
660		senders, in a range of stream_orderings (including backfilled events).
661
662		Args:
663		txn
664		low_pos (int): Low stream ordering
665		high_pos (int): High stream ordering
666
667		Returns:
668		tuple[dict[str, dict[str, int]], dict[str, dict[str, int]]]: The
669		room and user deltas for total_events/total_event_bytes in the
670		format of `stats_id` -> fields
671		"""
672
673		if low_pos >= high_pos:
674		# nothing to do here.
675		return {}, {}
676
677		if isinstance(self.database_engine, PostgresEngine):
678		new_bytes_expression = "OCTET_LENGTH(json)"
679		else:
680		new_bytes_expression = "LENGTH(CAST(json AS BLOB))"
681
682		sql = """
683		SELECT events.room_id, COUNT(*) AS new_events, SUM(%s) AS new_bytes
684		FROM events INNER JOIN event_json USING (event_id)
685		WHERE (? < stream_ordering AND stream_ordering <= ?)
686		OR (? <= stream_ordering AND stream_ordering <= ?)
687		GROUP BY events.room_id
688		""" % (
689		new_bytes_expression,
690		)
691
692		txn.execute(sql, (low_pos, high_pos, -high_pos, -low_pos))
693
694		room_deltas = {
695		room_id: {"total_events": new_events, "total_event_bytes": new_bytes}
696		for room_id, new_events, new_bytes in txn
697		}
698
699		sql = """
700		SELECT events.sender, COUNT(*) AS new_events, SUM(%s) AS new_bytes
701		FROM events INNER JOIN event_json USING (event_id)
702		WHERE (? < stream_ordering AND stream_ordering <= ?)
703		OR (? <= stream_ordering AND stream_ordering <= ?)
704		GROUP BY events.sender
705		""" % (
706		new_bytes_expression,
707		)
708
709		txn.execute(sql, (low_pos, high_pos, -high_pos, -low_pos))
710
711		user_deltas = {
712		user_id: {"total_events": new_events, "total_event_bytes": new_bytes}
713		for user_id, new_events, new_bytes in txn
714		if self.hs.is_mine_id(user_id)
715		}
716
717		return room_deltas, user_deltas
718
719		@defer.inlineCallbacks
720		def _calculate_and_set_initial_state_for_room(self, room_id):
721		"""Calculate and insert an entry into room_stats_current.
722
723		Args:
724		room_id (str)
725
726		Returns:
727		Deferred[tuple[dict, dict, int]]: A tuple of room state, membership
728		counts and stream position.
729		"""
730
731		def _fetch_current_state_stats(txn):
732		pos = self.get_room_max_stream_ordering()
733
734		rows = self._simple_select_many_txn(
735		txn,
736		table="current_state_events",
737		column="type",
738		iterable=[
739		EventTypes.Create,
740		EventTypes.JoinRules,
741		EventTypes.RoomHistoryVisibility,
742		EventTypes.Encryption,
743		EventTypes.Name,
744		EventTypes.Topic,
745		EventTypes.RoomAvatar,
746		EventTypes.CanonicalAlias,
747		],
748		keyvalues={"room_id": room_id, "state_key": ""},
749		retcols=["event_id"],
750		)
751
752		event_ids = [row["event_id"] for row in rows]
753
754		txn.execute(
755		"""
756		SELECT membership, count(*) FROM current_state_events
757		WHERE room_id = ? AND type = 'm.room.member'
758		GROUP BY membership
759		""",
760		(room_id,),
761		)
762		membership_counts = {membership: cnt for membership, cnt in txn}
763
764		txn.execute(
765		"""
766		SELECT COALESCE(count(*), 0) FROM current_state_events
767		WHERE room_id = ?
768		""",
769		(room_id,),
770		)
771
772		current_state_events_count, = txn.fetchone()
773
774		users_in_room = self.get_users_in_room_txn(txn, room_id)
775
776		return (
777		event_ids,
778		membership_counts,
779		current_state_events_count,
780		users_in_room,
781		pos,
782		)
783
784		(
785		event_ids,
786		membership_counts,
787		current_state_events_count,
788		users_in_room,
789		pos,
790		) = yield self.runInteraction(
791		"get_initial_state_for_room", _fetch_current_state_stats
792		)
793
794		state_event_map = yield self.get_events(event_ids, get_prev_content=False)
795
796		room_state = {
797		"join_rules": None,
798		"history_visibility": None,
799		"encryption": None,
800		"name": None,
801		"topic": None,
802		"avatar": None,
803		"canonical_alias": None,
804		"is_federatable": True,
805		}
806
807		for event in state_event_map.values():
808		if event.type == EventTypes.JoinRules:
809		room_state["join_rules"] = event.content.get("join_rule")
810		elif event.type == EventTypes.RoomHistoryVisibility:
811		room_state["history_visibility"] = event.content.get(
812		"history_visibility"
813		)
814		elif event.type == EventTypes.Encryption:
815		room_state["encryption"] = event.content.get("algorithm")
816		elif event.type == EventTypes.Name:
817		room_state["name"] = event.content.get("name")
818		elif event.type == EventTypes.Topic:
819		room_state["topic"] = event.content.get("topic")
820		elif event.type == EventTypes.RoomAvatar:
821		room_state["avatar"] = event.content.get("url")
822		elif event.type == EventTypes.CanonicalAlias:
823		room_state["canonical_alias"] = event.content.get("alias")
824		elif event.type == EventTypes.Create:
825		room_state["is_federatable"] = (
826		event.content.get("m.federate", True) is True
827		)
828
829		yield self.update_room_state(room_id, room_state)
830
831		local_users_in_room = [u for u in users_in_room if self.hs.is_mine_id(u)]
832
833		yield self.update_stats_delta(
834		ts=self.clock.time_msec(),
835		stats_type="room",
836		stats_id=room_id,
837		fields={},
838		complete_with_stream_id=pos,
839		absolute_field_overrides={
840		"current_state_events": current_state_events_count,
841		"joined_members": membership_counts.get(Membership.JOIN, 0),
842		"invited_members": membership_counts.get(Membership.INVITE, 0),
843		"left_members": membership_counts.get(Membership.LEAVE, 0),
844		"banned_members": membership_counts.get(Membership.BAN, 0),
845		"local_users_in_room": len(local_users_in_room),
846		},
847		)
848
849		@defer.inlineCallbacks
850		def _calculate_and_set_initial_state_for_user(self, user_id):
851		def _calculate_and_set_initial_state_for_user_txn(txn):
852		pos = self._get_max_stream_id_in_current_state_deltas_txn(txn)
853
854		txn.execute(
855		"""
856		SELECT COUNT(distinct room_id) FROM current_state_events
857		WHERE type = 'm.room.member' AND state_key = ?
858		AND membership = 'join'
859		""",
860		(user_id,),
861		)
862		count, = txn.fetchone()
863		return count, pos
864
865		joined_rooms, pos = yield self.runInteraction(
866		"calculate_and_set_initial_state_for_user",
867		_calculate_and_set_initial_state_for_user_txn,
868		)
869
870		yield self.update_stats_delta(
871		ts=self.clock.time_msec(),
872		stats_type="user",
873		stats_id=user_id,
874		fields={},
875		complete_with_stream_id=pos,
876		absolute_field_overrides={"joined_rooms": joined_rooms},
877		)

+0

-948

~~synapse/storage/stream.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2014-2016 OpenMarket Ltd
2		#
3		# Licensed under the Apache License, Version 2.0 (the "License");
4		# you may not use this file except in compliance with the License.
5		# You may obtain a copy of the License at
6		#
7		# http://www.apache.org/licenses/LICENSE-2.0
8		#
9		# Unless required by applicable law or agreed to in writing, software
10		# distributed under the License is distributed on an "AS IS" BASIS,
11		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12		# See the License for the specific language governing permissions and
13		# limitations under the License.
14
15		""" This module is responsible for getting events from the DB for pagination
16		and event streaming.
17
18		The order it returns events in depend on whether we are streaming forwards or
19		are paginating backwards. We do this because we want to handle out of order
20		messages nicely, while still returning them in the correct order when we
21		paginate bacwards.
22
23		This is implemented by keeping two ordering columns: stream_ordering and
24		topological_ordering. Stream ordering is basically insertion/received order
25		(except for events from backfill requests). The topological_ordering is a
26		weak ordering of events based on the pdu graph.
27
28		This means that we have to have two different types of tokens, depending on
29		what sort order was used:
30		- stream tokens are of the form: "s%d", which maps directly to the column
31		- topological tokems: "t%d-%d", where the integers map to the topological
32		and stream ordering columns respectively.
33		"""
34
35		import abc
36		import logging
37		from collections import namedtuple
38
39		from six.moves import range
40
41		from twisted.internet import defer
42
43		from synapse.logging.context import make_deferred_yieldable, run_in_background
44		from synapse.storage._base import SQLBaseStore
45		from synapse.storage.engines import PostgresEngine
46		from synapse.storage.events_worker import EventsWorkerStore
47		from synapse.types import RoomStreamToken
48		from synapse.util.caches.stream_change_cache import StreamChangeCache
49
50		logger = logging.getLogger(__name__)
51
52
53		MAX_STREAM_SIZE = 1000
54
55
56		_STREAM_TOKEN = "stream"
57		_TOPOLOGICAL_TOKEN = "topological"
58
59
60		# Used as return values for pagination APIs
61		_EventDictReturn = namedtuple(
62		"_EventDictReturn", ("event_id", "topological_ordering", "stream_ordering")
63		)
64
65
66		def generate_pagination_where_clause(
67		direction, column_names, from_token, to_token, engine
68		):
69		"""Creates an SQL expression to bound the columns by the pagination
70		tokens.
71
72		For example creates an SQL expression like:
73
74		(6, 7) >= (topological_ordering, stream_ordering)
75		AND (5, 3) < (topological_ordering, stream_ordering)
76
77		would be generated for dir=b, from_token=(6, 7) and to_token=(5, 3).
78
79		Note that tokens are considered to be after the row they are in, e.g. if
80		a row A has a token T, then we consider A to be before T. This convention
81		is important when figuring out inequalities for the generated SQL, and
82		produces the following result:
83		- If paginating forwards then we exclude any rows matching the from
84		token, but include those that match the to token.
85		- If paginating backwards then we include any rows matching the from
86		token, but include those that match the to token.
87
88		Args:
89		direction (str): Whether we're paginating backwards("b") or
90		forwards ("f").
91		column_names (tuple[str, str]): The column names to bound. Must not
92		be user defined as these get inserted directly into the SQL
93		statement without escapes.
94		from_token (tuple[int, int]\|None): The start point for the pagination.
95		This is an exclusive minimum bound if direction is "f", and an
96		inclusive maximum bound if direction is "b".
97		to_token (tuple[int, int]\|None): The endpoint point for the pagination.
98		This is an inclusive maximum bound if direction is "f", and an
99		exclusive minimum bound if direction is "b".
100		engine: The database engine to generate the clauses for
101
102		Returns:
103		str: The sql expression
104		"""
105		assert direction in ("b", "f")
106
107		where_clause = []
108		if from_token:
109		where_clause.append(
110		_make_generic_sql_bound(
111		bound=">=" if direction == "b" else "<",
112		column_names=column_names,
113		values=from_token,
114		engine=engine,
115		)
116		)
117
118		if to_token:
119		where_clause.append(
120		_make_generic_sql_bound(
121		bound="<" if direction == "b" else ">=",
122		column_names=column_names,
123		values=to_token,
124		engine=engine,
125		)
126		)
127
128		return " AND ".join(where_clause)
129
130
131		def _make_generic_sql_bound(bound, column_names, values, engine):
132		"""Create an SQL expression that bounds the given column names by the
133		values, e.g. create the equivalent of `(1, 2) < (col1, col2)`.
134
135		Only works with two columns.
136
137		Older versions of SQLite don't support that syntax so we have to expand it
138		out manually.
139
140		Args:
141		bound (str): The comparison operator to use. One of ">", "<", ">=",
142		"<=", where the values are on the left and columns on the right.
143		names (tuple[str, str]): The column names. Must not be user defined
144		as these get inserted directly into the SQL statement without
145		escapes.
146		values (tuple[int\|None, int]): The values to bound the columns by. If
147		the first value is None then only creates a bound on the second
148		column.
149		engine: The database engine to generate the SQL for
150
151		Returns:
152		str
153		"""
154
155		assert bound in (">", "<", ">=", "<=")
156
157		name1, name2 = column_names
158		val1, val2 = values
159
160		if val1 is None:
161		val2 = int(val2)
162		return "(%d %s %s)" % (val2, bound, name2)
163
164		val1 = int(val1)
165		val2 = int(val2)
166
167		if isinstance(engine, PostgresEngine):
168		# Postgres doesn't optimise ``(x < a) OR (x=a AND y<b)`` as well
169		# as it optimises ``(x,y) < (a,b)`` on multicolumn indexes. So we
170		# use the later form when running against postgres.
171		return "((%d,%d) %s (%s,%s))" % (val1, val2, bound, name1, name2)
172
173		# We want to generate queries of e.g. the form:
174		#
175		# (val1 < name1 OR (val1 = name1 AND val2 <= name2))
176		#
177		# which is equivalent to (val1, val2) < (name1, name2)
178
179		return """(
180		{val1:d} {strict_bound} {name1}
181		OR ({val1:d} = {name1} AND {val2:d} {bound} {name2})
182		)""".format(
183		name1=name1,
184		val1=val1,
185		name2=name2,
186		val2=val2,
187		strict_bound=bound[0], # The first bound must always be strict equality here
188		bound=bound,
189		)
190
191
192		def filter_to_clause(event_filter):
193		# NB: This may create SQL clauses that don't optimise well (and we don't
194		# have indices on all possible clauses). E.g. it may create
195		# "room_id == X AND room_id != X", which postgres doesn't optimise.
196
197		if not event_filter:
198		return "", []
199
200		clauses = []
201		args = []
202
203		if event_filter.types:
204		clauses.append("(%s)" % " OR ".join("type = ?" for _ in event_filter.types))
205		args.extend(event_filter.types)
206
207		for typ in event_filter.not_types:
208		clauses.append("type != ?")
209		args.append(typ)
210
211		if event_filter.senders:
212		clauses.append("(%s)" % " OR ".join("sender = ?" for _ in event_filter.senders))
213		args.extend(event_filter.senders)
214
215		for sender in event_filter.not_senders:
216		clauses.append("sender != ?")
217		args.append(sender)
218
219		if event_filter.rooms:
220		clauses.append("(%s)" % " OR ".join("room_id = ?" for _ in event_filter.rooms))
221		args.extend(event_filter.rooms)
222
223		for room_id in event_filter.not_rooms:
224		clauses.append("room_id != ?")
225		args.append(room_id)
226
227		if event_filter.contains_url:
228		clauses.append("contains_url = ?")
229		args.append(event_filter.contains_url)
230
231		return " AND ".join(clauses), args
232
233
234		class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
235		"""This is an abstract base class where subclasses must implement
236		`get_room_max_stream_ordering` and `get_room_min_stream_ordering`
237		which can be called in the initializer.
238		"""
239
240		__metaclass__ = abc.ABCMeta
241
242		def __init__(self, db_conn, hs):
243		super(StreamWorkerStore, self).__init__(db_conn, hs)
244
245		events_max = self.get_room_max_stream_ordering()
246		event_cache_prefill, min_event_val = self._get_cache_dict(
247		db_conn,
248		"events",
249		entity_column="room_id",
250		stream_column="stream_ordering",
251		max_value=events_max,
252		)
253		self._events_stream_cache = StreamChangeCache(
254		"EventsRoomStreamChangeCache",
255		min_event_val,
256		prefilled_cache=event_cache_prefill,
257		)
258		self._membership_stream_cache = StreamChangeCache(
259		"MembershipStreamChangeCache", events_max
260		)
261
262		self._stream_order_on_start = self.get_room_max_stream_ordering()
263
264		@abc.abstractmethod
265		def get_room_max_stream_ordering(self):
266		raise NotImplementedError()
267
268		@abc.abstractmethod
269		def get_room_min_stream_ordering(self):
270		raise NotImplementedError()
271
272		@defer.inlineCallbacks
273		def get_room_events_stream_for_rooms(
274		self, room_ids, from_key, to_key, limit=0, order="DESC"
275		):
276		"""Get new room events in stream ordering since `from_key`.
277
278		Args:
279		room_id (str)
280		from_key (str): Token from which no events are returned before
281		to_key (str): Token from which no events are returned after. (This
282		is typically the current stream token)
283		limit (int): Maximum number of events to return
284		order (str): Either "DESC" or "ASC". Determines which events are
285		returned when the result is limited. If "DESC" then the most
286		recent `limit` events are returned, otherwise returns the
287		oldest `limit` events.
288
289		Returns:
290		Deferred[dict[str,tuple[list[FrozenEvent], str]]]
291		A map from room id to a tuple containing:
292		- list of recent events in the room
293		- stream ordering key for the start of the chunk of events returned.
294		"""
295		from_id = RoomStreamToken.parse_stream_token(from_key).stream
296
297		room_ids = yield self._events_stream_cache.get_entities_changed(
298		room_ids, from_id
299		)
300
301		if not room_ids:
302		return {}
303
304		results = {}
305		room_ids = list(room_ids)
306		for rm_ids in (room_ids[i : i + 20] for i in range(0, len(room_ids), 20)):
307		res = yield make_deferred_yieldable(
308		defer.gatherResults(
309		[
310		run_in_background(
311		self.get_room_events_stream_for_room,
312		room_id,
313		from_key,
314		to_key,
315		limit,
316		order=order,
317		)
318		for room_id in rm_ids
319		],
320		consumeErrors=True,
321		)
322		)
323		results.update(dict(zip(rm_ids, res)))
324
325		return results
326
327		def get_rooms_that_changed(self, room_ids, from_key):
328		"""Given a list of rooms and a token, return rooms where there may have
329		been changes.
330
331		Args:
332		room_ids (list)
333		from_key (str): The room_key portion of a StreamToken
334		"""
335		from_key = RoomStreamToken.parse_stream_token(from_key).stream
336		return set(
337		room_id
338		for room_id in room_ids
339		if self._events_stream_cache.has_entity_changed(room_id, from_key)
340		)
341
342		@defer.inlineCallbacks
343		def get_room_events_stream_for_room(
344		self, room_id, from_key, to_key, limit=0, order="DESC"
345		):
346
347		"""Get new room events in stream ordering since `from_key`.
348
349		Args:
350		room_id (str)
351		from_key (str): Token from which no events are returned before
352		to_key (str): Token from which no events are returned after. (This
353		is typically the current stream token)
354		limit (int): Maximum number of events to return
355		order (str): Either "DESC" or "ASC". Determines which events are
356		returned when the result is limited. If "DESC" then the most
357		recent `limit` events are returned, otherwise returns the
358		oldest `limit` events.
359
360		Returns:
361		Deferred[tuple[list[FrozenEvent], str]]: Returns the list of
362		events (in ascending order) and the token from the start of
363		the chunk of events returned.
364		"""
365		if from_key == to_key:
366		return [], from_key
367
368		from_id = RoomStreamToken.parse_stream_token(from_key).stream
369		to_id = RoomStreamToken.parse_stream_token(to_key).stream
370
371		has_changed = yield self._events_stream_cache.has_entity_changed(
372		room_id, from_id
373		)
374
375		if not has_changed:
376		return [], from_key
377
378		def f(txn):
379		sql = (
380		"SELECT event_id, stream_ordering FROM events WHERE"
381		" room_id = ?"
382		" AND not outlier"
383		" AND stream_ordering > ? AND stream_ordering <= ?"
384		" ORDER BY stream_ordering %s LIMIT ?"
385		) % (order,)
386		txn.execute(sql, (room_id, from_id, to_id, limit))
387
388		rows = [_EventDictReturn(row[0], None, row[1]) for row in txn]
389		return rows
390
391		rows = yield self.runInteraction("get_room_events_stream_for_room", f)
392
393		ret = yield self.get_events_as_list(
394		[r.event_id for r in rows], get_prev_content=True
395		)
396
397		self._set_before_and_after(ret, rows, topo_order=from_id is None)
398
399		if order.lower() == "desc":
400		ret.reverse()
401
402		if rows:
403		key = "s%d" % min(r.stream_ordering for r in rows)
404		else:
405		# Assume we didn't get anything because there was nothing to
406		# get.
407		key = from_key
408
409		return ret, key
410
411		@defer.inlineCallbacks
412		def get_membership_changes_for_user(self, user_id, from_key, to_key):
413		from_id = RoomStreamToken.parse_stream_token(from_key).stream
414		to_id = RoomStreamToken.parse_stream_token(to_key).stream
415
416		if from_key == to_key:
417		return []
418
419		if from_id:
420		has_changed = self._membership_stream_cache.has_entity_changed(
421		user_id, int(from_id)
422		)
423		if not has_changed:
424		return []
425
426		def f(txn):
427		sql = (
428		"SELECT m.event_id, stream_ordering FROM events AS e,"
429		" room_memberships AS m"
430		" WHERE e.event_id = m.event_id"
431		" AND m.user_id = ?"
432		" AND e.stream_ordering > ? AND e.stream_ordering <= ?"
433		" ORDER BY e.stream_ordering ASC"
434		)
435		txn.execute(sql, (user_id, from_id, to_id))
436
437		rows = [_EventDictReturn(row[0], None, row[1]) for row in txn]
438
439		return rows
440
441		rows = yield self.runInteraction("get_membership_changes_for_user", f)
442
443		ret = yield self.get_events_as_list(
444		[r.event_id for r in rows], get_prev_content=True
445		)
446
447		self._set_before_and_after(ret, rows, topo_order=False)
448
449		return ret
450
451		@defer.inlineCallbacks
452		def get_recent_events_for_room(self, room_id, limit, end_token):
453		"""Get the most recent events in the room in topological ordering.
454
455		Args:
456		room_id (str)
457		limit (int)
458		end_token (str): The stream token representing now.
459
460		Returns:
461		Deferred[tuple[list[FrozenEvent], str]]: Returns a list of
462		events and a token pointing to the start of the returned
463		events.
464		The events returned are in ascending order.
465		"""
466
467		rows, token = yield self.get_recent_event_ids_for_room(
468		room_id, limit, end_token
469		)
470
471		logger.debug("stream before")
472		events = yield self.get_events_as_list(
473		[r.event_id for r in rows], get_prev_content=True
474		)
475		logger.debug("stream after")
476
477		self._set_before_and_after(events, rows)
478
479		return (events, token)
480
481		@defer.inlineCallbacks
482		def get_recent_event_ids_for_room(self, room_id, limit, end_token):
483		"""Get the most recent events in the room in topological ordering.
484
485		Args:
486		room_id (str)
487		limit (int)
488		end_token (str): The stream token representing now.
489
490		Returns:
491		Deferred[tuple[list[_EventDictReturn], str]]: Returns a list of
492		_EventDictReturn and a token pointing to the start of the returned
493		events.
494		The events returned are in ascending order.
495		"""
496		# Allow a zero limit here, and no-op.
497		if limit == 0:
498		return [], end_token
499
500		end_token = RoomStreamToken.parse(end_token)
501
502		rows, token = yield self.runInteraction(
503		"get_recent_event_ids_for_room",
504		self._paginate_room_events_txn,
505		room_id,
506		from_token=end_token,
507		limit=limit,
508		)
509
510		# We want to return the results in ascending order.
511		rows.reverse()
512
513		return rows, token
514
515		def get_room_event_after_stream_ordering(self, room_id, stream_ordering):
516		"""Gets details of the first event in a room at or after a stream ordering
517
518		Args:
519		room_id (str):
520		stream_ordering (int):
521
522		Returns:
523		Deferred[(int, int, str)]:
524		(stream ordering, topological ordering, event_id)
525		"""
526
527		def _f(txn):
528		sql = (
529		"SELECT stream_ordering, topological_ordering, event_id"
530		" FROM events"
531		" WHERE room_id = ? AND stream_ordering >= ?"
532		" AND NOT outlier"
533		" ORDER BY stream_ordering"
534		" LIMIT 1"
535		)
536		txn.execute(sql, (room_id, stream_ordering))
537		return txn.fetchone()
538
539		return self.runInteraction("get_room_event_after_stream_ordering", _f)
540
541		@defer.inlineCallbacks
542		def get_room_events_max_id(self, room_id=None):
543		"""Returns the current token for rooms stream.
544
545		By default, it returns the current global stream token. Specifying a
546		`room_id` causes it to return the current room specific topological
547		token.
548		"""
549		token = yield self.get_room_max_stream_ordering()
550		if room_id is None:
551		return "s%d" % (token,)
552		else:
553		topo = yield self.runInteraction(
554		"_get_max_topological_txn", self._get_max_topological_txn, room_id
555		)
556		return "t%d-%d" % (topo, token)
557
558		def get_stream_token_for_event(self, event_id):
559		"""The stream token for an event
560		Args:
561		event_id(str): The id of the event to look up a stream token for.
562		Raises:
563		StoreError if the event wasn't in the database.
564		Returns:
565		A deferred "s%d" stream token.
566		"""
567		return self._simple_select_one_onecol(
568		table="events", keyvalues={"event_id": event_id}, retcol="stream_ordering"
569		).addCallback(lambda row: "s%d" % (row,))
570
571		def get_topological_token_for_event(self, event_id):
572		"""The stream token for an event
573		Args:
574		event_id(str): The id of the event to look up a stream token for.
575		Raises:
576		StoreError if the event wasn't in the database.
577		Returns:
578		A deferred "t%d-%d" topological token.
579		"""
580		return self._simple_select_one(
581		table="events",
582		keyvalues={"event_id": event_id},
583		retcols=("stream_ordering", "topological_ordering"),
584		desc="get_topological_token_for_event",
585		).addCallback(
586		lambda row: "t%d-%d" % (row["topological_ordering"], row["stream_ordering"])
587		)
588
589		def get_max_topological_token(self, room_id, stream_key):
590		"""Get the max topological token in a room before the given stream
591		ordering.
592
593		Args:
594		room_id (str)
595		stream_key (int)
596
597		Returns:
598		Deferred[int]
599		"""
600		sql = (
601		"SELECT coalesce(max(topological_ordering), 0) FROM events"
602		" WHERE room_id = ? AND stream_ordering < ?"
603		)
604		return self._execute(
605		"get_max_topological_token", None, sql, room_id, stream_key
606		).addCallback(lambda r: r[0][0] if r else 0)
607
608		def _get_max_topological_txn(self, txn, room_id):
609		txn.execute(
610		"SELECT MAX(topological_ordering) FROM events" " WHERE room_id = ?",
611		(room_id,),
612		)
613
614		rows = txn.fetchall()
615		return rows[0][0] if rows else 0
616
617		@staticmethod
618		def _set_before_and_after(events, rows, topo_order=True):
619		"""Inserts ordering information to events' internal metadata from
620		the DB rows.
621
622		Args:
623		events (list[FrozenEvent])
624		rows (list[_EventDictReturn])
625		topo_order (bool): Whether the events were ordered topologically
626		or by stream ordering. If true then all rows should have a non
627		null topological_ordering.
628		"""
629		for event, row in zip(events, rows):
630		stream = row.stream_ordering
631		if topo_order and row.topological_ordering:
632		topo = row.topological_ordering
633		else:
634		topo = None
635		internal = event.internal_metadata
636		internal.before = str(RoomStreamToken(topo, stream - 1))
637		internal.after = str(RoomStreamToken(topo, stream))
638		internal.order = (int(topo) if topo else 0, int(stream))
639
640		@defer.inlineCallbacks
641		def get_events_around(
642		self, room_id, event_id, before_limit, after_limit, event_filter=None
643		):
644		"""Retrieve events and pagination tokens around a given event in a
645		room.
646
647		Args:
648		room_id (str)
649		event_id (str)
650		before_limit (int)
651		after_limit (int)
652		event_filter (Filter\|None)
653
654		Returns:
655		dict
656		"""
657
658		results = yield self.runInteraction(
659		"get_events_around",
660		self._get_events_around_txn,
661		room_id,
662		event_id,
663		before_limit,
664		after_limit,
665		event_filter,
666		)
667
668		events_before = yield self.get_events_as_list(
669		[e for e in results["before"]["event_ids"]], get_prev_content=True
670		)
671
672		events_after = yield self.get_events_as_list(
673		[e for e in results["after"]["event_ids"]], get_prev_content=True
674		)
675
676		return {
677		"events_before": events_before,
678		"events_after": events_after,
679		"start": results["before"]["token"],
680		"end": results["after"]["token"],
681		}
682
683		def _get_events_around_txn(
684		self, txn, room_id, event_id, before_limit, after_limit, event_filter
685		):
686		"""Retrieves event_ids and pagination tokens around a given event in a
687		room.
688
689		Args:
690		room_id (str)
691		event_id (str)
692		before_limit (int)
693		after_limit (int)
694		event_filter (Filter\|None)
695
696		Returns:
697		dict
698		"""
699
700		results = self._simple_select_one_txn(
701		txn,
702		"events",
703		keyvalues={"event_id": event_id, "room_id": room_id},
704		retcols=["stream_ordering", "topological_ordering"],
705		)
706
707		# Paginating backwards includes the event at the token, but paginating
708		# forward doesn't.
709		before_token = RoomStreamToken(
710		results["topological_ordering"] - 1, results["stream_ordering"]
711		)
712
713		after_token = RoomStreamToken(
714		results["topological_ordering"], results["stream_ordering"]
715		)
716
717		rows, start_token = self._paginate_room_events_txn(
718		txn,
719		room_id,
720		before_token,
721		direction="b",
722		limit=before_limit,
723		event_filter=event_filter,
724		)
725		events_before = [r.event_id for r in rows]
726
727		rows, end_token = self._paginate_room_events_txn(
728		txn,
729		room_id,
730		after_token,
731		direction="f",
732		limit=after_limit,
733		event_filter=event_filter,
734		)
735		events_after = [r.event_id for r in rows]
736
737		return {
738		"before": {"event_ids": events_before, "token": start_token},
739		"after": {"event_ids": events_after, "token": end_token},
740		}
741
742		@defer.inlineCallbacks
743		def get_all_new_events_stream(self, from_id, current_id, limit):
744		"""Get all new events
745
746		Returns all events with from_id < stream_ordering <= current_id.
747
748		Args:
749		from_id (int): the stream_ordering of the last event we processed
750		current_id (int): the stream_ordering of the most recently processed event
751		limit (int): the maximum number of events to return
752
753		Returns:
754		Deferred[Tuple[int, list[FrozenEvent]]]: A tuple of (next_id, events), where
755		`next_id` is the next value to pass as `from_id` (it will either be the
756		stream_ordering of the last returned event, or, if fewer than `limit` events
757		were found, `current_id`.
758		"""
759
760		def get_all_new_events_stream_txn(txn):
761		sql = (
762		"SELECT e.stream_ordering, e.event_id"
763		" FROM events AS e"
764		" WHERE"
765		" ? < e.stream_ordering AND e.stream_ordering <= ?"
766		" ORDER BY e.stream_ordering ASC"
767		" LIMIT ?"
768		)
769
770		txn.execute(sql, (from_id, current_id, limit))
771		rows = txn.fetchall()
772
773		upper_bound = current_id
774		if len(rows) == limit:
775		upper_bound = rows[-1][0]
776
777		return upper_bound, [row[1] for row in rows]
778
779		upper_bound, event_ids = yield self.runInteraction(
780		"get_all_new_events_stream", get_all_new_events_stream_txn
781		)
782
783		events = yield self.get_events_as_list(event_ids)
784
785		return upper_bound, events
786
787		def get_federation_out_pos(self, typ):
788		return self._simple_select_one_onecol(
789		table="federation_stream_position",
790		retcol="stream_id",
791		keyvalues={"type": typ},
792		desc="get_federation_out_pos",
793		)
794
795		def update_federation_out_pos(self, typ, stream_id):
796		return self._simple_update_one(
797		table="federation_stream_position",
798		keyvalues={"type": typ},
799		updatevalues={"stream_id": stream_id},
800		desc="update_federation_out_pos",
801		)
802
803		def has_room_changed_since(self, room_id, stream_id):
804		return self._events_stream_cache.has_entity_changed(room_id, stream_id)
805
806		def _paginate_room_events_txn(
807		self,
808		txn,
809		room_id,
810		from_token,
811		to_token=None,
812		direction="b",
813		limit=-1,
814		event_filter=None,
815		):
816		"""Returns list of events before or after a given token.
817
818		Args:
819		txn
820		room_id (str)
821		from_token (RoomStreamToken): The token used to stream from
822		to_token (RoomStreamToken\|None): A token which if given limits the
823		results to only those before
824		direction(char): Either 'b' or 'f' to indicate whether we are
825		paginating forwards or backwards from `from_key`.
826		limit (int): The maximum number of events to return.
827		event_filter (Filter\|None): If provided filters the events to
828		those that match the filter.
829
830		Returns:
831		Deferred[tuple[list[_EventDictReturn], str]]: Returns the results
832		as a list of _EventDictReturn and a token that points to the end
833		of the result set. If no events are returned then the end of the
834		stream has been reached (i.e. there are no events between
835		`from_token` and `to_token`), or `limit` is zero.
836		"""
837
838		assert int(limit) >= 0
839
840		# Tokens really represent positions between elements, but we use
841		# the convention of pointing to the event before the gap. Hence
842		# we have a bit of asymmetry when it comes to equalities.
843		args = [False, room_id]
844		if direction == "b":
845		order = "DESC"
846		else:
847		order = "ASC"
848
849		bounds = generate_pagination_where_clause(
850		direction=direction,
851		column_names=("topological_ordering", "stream_ordering"),
852		from_token=from_token,
853		to_token=to_token,
854		engine=self.database_engine,
855		)
856
857		filter_clause, filter_args = filter_to_clause(event_filter)
858
859		if filter_clause:
860		bounds += " AND " + filter_clause
861		args.extend(filter_args)
862
863		args.append(int(limit))
864
865		sql = (
866		"SELECT event_id, topological_ordering, stream_ordering"
867		" FROM events"
868		" WHERE outlier = ? AND room_id = ? AND %(bounds)s"
869		" ORDER BY topological_ordering %(order)s,"
870		" stream_ordering %(order)s LIMIT ?"
871		) % {"bounds": bounds, "order": order}
872
873		txn.execute(sql, args)
874
875		rows = [_EventDictReturn(row[0], row[1], row[2]) for row in txn]
876
877		if rows:
878		topo = rows[-1].topological_ordering
879		toke = rows[-1].stream_ordering
880		if direction == "b":
881		# Tokens are positions between events.
882		# This token points after the last event in the chunk.
883		# We need it to point to the event before it in the chunk
884		# when we are going backwards so we subtract one from the
885		# stream part.
886		toke -= 1
887		next_token = RoomStreamToken(topo, toke)
888		else:
889		# TODO (erikj): We should work out what to do here instead.
890		next_token = to_token if to_token else from_token
891
892		return rows, str(next_token)
893
894		@defer.inlineCallbacks
895		def paginate_room_events(
896		self, room_id, from_key, to_key=None, direction="b", limit=-1, event_filter=None
897		):
898		"""Returns list of events before or after a given token.
899
900		Args:
901		room_id (str)
902		from_key (str): The token used to stream from
903		to_key (str\|None): A token which if given limits the results to
904		only those before
905		direction(char): Either 'b' or 'f' to indicate whether we are
906		paginating forwards or backwards from `from_key`.
907		limit (int): The maximum number of events to return.
908		event_filter (Filter\|None): If provided filters the events to
909		those that match the filter.
910
911		Returns:
912		tuple[list[FrozenEvent], str]: Returns the results as a list of
913		events and a token that points to the end of the result set. If no
914		events are returned then the end of the stream has been reached
915		(i.e. there are no events between `from_key` and `to_key`).
916		"""
917
918		from_key = RoomStreamToken.parse(from_key)
919		if to_key:
920		to_key = RoomStreamToken.parse(to_key)
921
922		rows, token = yield self.runInteraction(
923		"paginate_room_events",
924		self._paginate_room_events_txn,
925		room_id,
926		from_key,
927		to_key,
928		direction,
929		limit,
930		event_filter,
931		)
932
933		events = yield self.get_events_as_list(
934		[r.event_id for r in rows], get_prev_content=True
935		)
936
937		self._set_before_and_after(events, rows)
938
939		return (events, token)
940
941
942		class StreamStore(StreamWorkerStore):
943		def get_room_max_stream_ordering(self):
944		return self._stream_id_gen.get_current_token()
945
946		def get_room_min_stream_ordering(self):
947		return self._backfill_id_gen.get_current_token()

+0

-265

~~synapse/storage/tags.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2014-2016 OpenMarket Ltd
2		# Copyright 2018 New Vector Ltd
3		#
4		# Licensed under the Apache License, Version 2.0 (the "License");
5		# you may not use this file except in compliance with the License.
6		# You may obtain a copy of the License at
7		#
8		# http://www.apache.org/licenses/LICENSE-2.0
9		#
10		# Unless required by applicable law or agreed to in writing, software
11		# distributed under the License is distributed on an "AS IS" BASIS,
12		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13		# See the License for the specific language governing permissions and
14		# limitations under the License.
15
16		import logging
17
18		from six.moves import range
19
20		from canonicaljson import json
21
22		from twisted.internet import defer
23
24		from synapse.storage.account_data import AccountDataWorkerStore
25		from synapse.util.caches.descriptors import cached
26
27		logger = logging.getLogger(__name__)
28
29
30		class TagsWorkerStore(AccountDataWorkerStore):
31		@cached()
32		def get_tags_for_user(self, user_id):
33		"""Get all the tags for a user.
34
35
36		Args:
37		user_id(str): The user to get the tags for.
38		Returns:
39		A deferred dict mapping from room_id strings to dicts mapping from
40		tag strings to tag content.
41		"""
42
43		deferred = self._simple_select_list(
44		"room_tags", {"user_id": user_id}, ["room_id", "tag", "content"]
45		)
46
47		@deferred.addCallback
48		def tags_by_room(rows):
49		tags_by_room = {}
50		for row in rows:
51		room_tags = tags_by_room.setdefault(row["room_id"], {})
52		room_tags[row["tag"]] = json.loads(row["content"])
53		return tags_by_room
54
55		return deferred
56
57		@defer.inlineCallbacks
58		def get_all_updated_tags(self, last_id, current_id, limit):
59		"""Get all the client tags that have changed on the server
60		Args:
61		last_id(int): The position to fetch from.
62		current_id(int): The position to fetch up to.
63		Returns:
64		A deferred list of tuples of stream_id int, user_id string,
65		room_id string, tag string and content string.
66		"""
67		if last_id == current_id:
68		return []
69
70		def get_all_updated_tags_txn(txn):
71		sql = (
72		"SELECT stream_id, user_id, room_id"
73		" FROM room_tags_revisions as r"
74		" WHERE ? < stream_id AND stream_id <= ?"
75		" ORDER BY stream_id ASC LIMIT ?"
76		)
77		txn.execute(sql, (last_id, current_id, limit))
78		return txn.fetchall()
79
80		tag_ids = yield self.runInteraction(
81		"get_all_updated_tags", get_all_updated_tags_txn
82		)
83
84		def get_tag_content(txn, tag_ids):
85		sql = (
86		"SELECT tag, content" " FROM room_tags" " WHERE user_id=? AND room_id=?"
87		)
88		results = []
89		for stream_id, user_id, room_id in tag_ids:
90		txn.execute(sql, (user_id, room_id))
91		tags = []
92		for tag, content in txn:
93		tags.append(json.dumps(tag) + ":" + content)
94		tag_json = "{" + ",".join(tags) + "}"
95		results.append((stream_id, user_id, room_id, tag_json))
96
97		return results
98
99		batch_size = 50
100		results = []
101		for i in range(0, len(tag_ids), batch_size):
102		tags = yield self.runInteraction(
103		"get_all_updated_tag_content",
104		get_tag_content,
105		tag_ids[i : i + batch_size],
106		)
107		results.extend(tags)
108
109		return results
110
111		@defer.inlineCallbacks
112		def get_updated_tags(self, user_id, stream_id):
113		"""Get all the tags for the rooms where the tags have changed since the
114		given version
115
116		Args:
117		user_id(str): The user to get the tags for.
118		stream_id(int): The earliest update to get for the user.
119		Returns:
120		A deferred dict mapping from room_id strings to lists of tag
121		strings for all the rooms that changed since the stream_id token.
122		"""
123
124		def get_updated_tags_txn(txn):
125		sql = (
126		"SELECT room_id from room_tags_revisions"
127		" WHERE user_id = ? AND stream_id > ?"
128		)
129		txn.execute(sql, (user_id, stream_id))
130		room_ids = [row[0] for row in txn]
131		return room_ids
132
133		changed = self._account_data_stream_cache.has_entity_changed(
134		user_id, int(stream_id)
135		)
136		if not changed:
137		return {}
138
139		room_ids = yield self.runInteraction("get_updated_tags", get_updated_tags_txn)
140
141		results = {}
142		if room_ids:
143		tags_by_room = yield self.get_tags_for_user(user_id)
144		for room_id in room_ids:
145		results[room_id] = tags_by_room.get(room_id, {})
146
147		return results
148
149		def get_tags_for_room(self, user_id, room_id):
150		"""Get all the tags for the given room
151		Args:
152		user_id(str): The user to get tags for
153		room_id(str): The room to get tags for
154		Returns:
155		A deferred list of string tags.
156		"""
157		return self._simple_select_list(
158		table="room_tags",
159		keyvalues={"user_id": user_id, "room_id": room_id},
160		retcols=("tag", "content"),
161		desc="get_tags_for_room",
162		).addCallback(
163		lambda rows: {row["tag"]: json.loads(row["content"]) for row in rows}
164		)
165
166
167		class TagsStore(TagsWorkerStore):
168		@defer.inlineCallbacks
169		def add_tag_to_room(self, user_id, room_id, tag, content):
170		"""Add a tag to a room for a user.
171		Args:
172		user_id(str): The user to add a tag for.
173		room_id(str): The room to add a tag for.
174		tag(str): The tag name to add.
175		content(dict): A json object to associate with the tag.
176		Returns:
177		A deferred that completes once the tag has been added.
178		"""
179		content_json = json.dumps(content)
180
181		def add_tag_txn(txn, next_id):
182		self._simple_upsert_txn(
183		txn,
184		table="room_tags",
185		keyvalues={"user_id": user_id, "room_id": room_id, "tag": tag},
186		values={"content": content_json},
187		)
188		self._update_revision_txn(txn, user_id, room_id, next_id)
189
190		with self._account_data_id_gen.get_next() as next_id:
191		yield self.runInteraction("add_tag", add_tag_txn, next_id)
192
193		self.get_tags_for_user.invalidate((user_id,))
194
195		result = self._account_data_id_gen.get_current_token()
196		return result
197
198		@defer.inlineCallbacks
199		def remove_tag_from_room(self, user_id, room_id, tag):
200		"""Remove a tag from a room for a user.
201		Returns:
202		A deferred that completes once the tag has been removed
203		"""
204
205		def remove_tag_txn(txn, next_id):
206		sql = (
207		"DELETE FROM room_tags "
208		" WHERE user_id = ? AND room_id = ? AND tag = ?"
209		)
210		txn.execute(sql, (user_id, room_id, tag))
211		self._update_revision_txn(txn, user_id, room_id, next_id)
212
213		with self._account_data_id_gen.get_next() as next_id:
214		yield self.runInteraction("remove_tag", remove_tag_txn, next_id)
215
216		self.get_tags_for_user.invalidate((user_id,))
217
218		result = self._account_data_id_gen.get_current_token()
219		return result
220
221		def _update_revision_txn(self, txn, user_id, room_id, next_id):
222		"""Update the latest revision of the tags for the given user and room.
223
224		Args:
225		txn: The database cursor
226		user_id(str): The ID of the user.
227		room_id(str): The ID of the room.
228		next_id(int): The the revision to advance to.
229		"""
230
231		txn.call_after(
232		self._account_data_stream_cache.entity_has_changed, user_id, next_id
233		)
234
235		update_max_id_sql = (
236		"UPDATE account_data_max_stream_id"
237		" SET stream_id = ?"
238		" WHERE stream_id < ?"
239		)
240		txn.execute(update_max_id_sql, (next_id, next_id))
241
242		update_sql = (
243		"UPDATE room_tags_revisions"
244		" SET stream_id = ?"
245		" WHERE user_id = ?"
246		" AND room_id = ?"
247		)
248		txn.execute(update_sql, (next_id, user_id, room_id))
249
250		if txn.rowcount == 0:
251		insert_sql = (
252		"INSERT INTO room_tags_revisions (user_id, room_id, stream_id)"
253		" VALUES (?, ?, ?)"
254		)
255		try:
256		txn.execute(insert_sql, (user_id, room_id, next_id))
257		except self.database_engine.module.IntegrityError:
258		# Ignore insertion errors. It doesn't matter if the row wasn't
259		# inserted because if two updates happend concurrently the one
260		# with the higher stream_id will not be reported to a client
261		# unless the previous update has completed. It doesn't matter
262		# which stream_id ends up in the table, as long as it is higher
263		# than the id that the client has.
264		pass

+0

-274

~~synapse/storage/transactions.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2014-2016 OpenMarket Ltd
2		#
3		# Licensed under the Apache License, Version 2.0 (the "License");
4		# you may not use this file except in compliance with the License.
5		# You may obtain a copy of the License at
6		#
7		# http://www.apache.org/licenses/LICENSE-2.0
8		#
9		# Unless required by applicable law or agreed to in writing, software
10		# distributed under the License is distributed on an "AS IS" BASIS,
11		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12		# See the License for the specific language governing permissions and
13		# limitations under the License.
14
15		import logging
16		from collections import namedtuple
17
18		import six
19
20		from canonicaljson import encode_canonical_json
21
22		from twisted.internet import defer
23
24		from synapse.metrics.background_process_metrics import run_as_background_process
25		from synapse.util.caches.expiringcache import ExpiringCache
26
27		from ._base import SQLBaseStore, db_to_json
28
29		# py2 sqlite has buffer hardcoded as only binary type, so we must use it,
30		# despite being deprecated and removed in favor of memoryview
31		if six.PY2:
32		db_binary_type = six.moves.builtins.buffer
33		else:
34		db_binary_type = memoryview
35
36		logger = logging.getLogger(__name__)
37
38
39		_TransactionRow = namedtuple(
40		"_TransactionRow",
41		("id", "transaction_id", "destination", "ts", "response_code", "response_json"),
42		)
43
44		_UpdateTransactionRow = namedtuple(
45		"_TransactionRow", ("response_code", "response_json")
46		)
47
48		SENTINEL = object()
49
50
51		class TransactionStore(SQLBaseStore):
52		"""A collection of queries for handling PDUs.
53		"""
54
55		def __init__(self, db_conn, hs):
56		super(TransactionStore, self).__init__(db_conn, hs)
57
58		self._clock.looping_call(self._start_cleanup_transactions, 30 * 60 * 1000)
59
60		self._destination_retry_cache = ExpiringCache(
61		cache_name="get_destination_retry_timings",
62		clock=self._clock,
63		expiry_ms=5 * 60 * 1000,
64		)
65
66		def get_received_txn_response(self, transaction_id, origin):
67		"""For an incoming transaction from a given origin, check if we have
68		already responded to it. If so, return the response code and response
69		body (as a dict).
70
71		Args:
72		transaction_id (str)
73		origin(str)
74
75		Returns:
76		tuple: None if we have not previously responded to
77		this transaction or a 2-tuple of (int, dict)
78		"""
79
80		return self.runInteraction(
81		"get_received_txn_response",
82		self._get_received_txn_response,
83		transaction_id,
84		origin,
85		)
86
87		def _get_received_txn_response(self, txn, transaction_id, origin):
88		result = self._simple_select_one_txn(
89		txn,
90		table="received_transactions",
91		keyvalues={"transaction_id": transaction_id, "origin": origin},
92		retcols=(
93		"transaction_id",
94		"origin",
95		"ts",
96		"response_code",
97		"response_json",
98		"has_been_referenced",
99		),
100		allow_none=True,
101		)
102
103		if result and result["response_code"]:
104		return result["response_code"], db_to_json(result["response_json"])
105
106		else:
107		return None
108
109		def set_received_txn_response(self, transaction_id, origin, code, response_dict):
110		"""Persist the response we returened for an incoming transaction, and
111		should return for subsequent transactions with the same transaction_id
112		and origin.
113
114		Args:
115		txn
116		transaction_id (str)
117		origin (str)
118		code (int)
119		response_json (str)
120		"""
121
122		return self._simple_insert(
123		table="received_transactions",
124		values={
125		"transaction_id": transaction_id,
126		"origin": origin,
127		"response_code": code,
128		"response_json": db_binary_type(encode_canonical_json(response_dict)),
129		"ts": self._clock.time_msec(),
130		},
131		or_ignore=True,
132		desc="set_received_txn_response",
133		)
134
135		@defer.inlineCallbacks
136		def get_destination_retry_timings(self, destination):
137		"""Gets the current retry timings (if any) for a given destination.
138
139		Args:
140		destination (str)
141
142		Returns:
143		None if not retrying
144		Otherwise a dict for the retry scheme
145		"""
146
147		result = self._destination_retry_cache.get(destination, SENTINEL)
148		if result is not SENTINEL:
149		return result
150
151		result = yield self.runInteraction(
152		"get_destination_retry_timings",
153		self._get_destination_retry_timings,
154		destination,
155		)
156
157		# We don't hugely care about race conditions between getting and
158		# invalidating the cache, since we time out fairly quickly anyway.
159		self._destination_retry_cache[destination] = result
160		return result
161
162		def _get_destination_retry_timings(self, txn, destination):
163		result = self._simple_select_one_txn(
164		txn,
165		table="destinations",
166		keyvalues={"destination": destination},
167		retcols=("destination", "failure_ts", "retry_last_ts", "retry_interval"),
168		allow_none=True,
169		)
170
171		if result and result["retry_last_ts"] > 0:
172		return result
173		else:
174		return None
175
176		def set_destination_retry_timings(
177		self, destination, failure_ts, retry_last_ts, retry_interval
178		):
179		"""Sets the current retry timings for a given destination.
180		Both timings should be zero if retrying is no longer occuring.
181
182		Args:
183		destination (str)
184		failure_ts (int\|None) - when the server started failing (ms since epoch)
185		retry_last_ts (int) - time of last retry attempt in unix epoch ms
186		retry_interval (int) - how long until next retry in ms
187		"""
188
189		self._destination_retry_cache.pop(destination, None)
190		return self.runInteraction(
191		"set_destination_retry_timings",
192		self._set_destination_retry_timings,
193		destination,
194		failure_ts,
195		retry_last_ts,
196		retry_interval,
197		)
198
199		def _set_destination_retry_timings(
200		self, txn, destination, failure_ts, retry_last_ts, retry_interval
201		):
202
203		if self.database_engine.can_native_upsert:
204		# Upsert retry time interval if retry_interval is zero (i.e. we're
205		# resetting it) or greater than the existing retry interval.
206
207		sql = """
208		INSERT INTO destinations (
209		destination, failure_ts, retry_last_ts, retry_interval
210		)
211		VALUES (?, ?, ?, ?)
212		ON CONFLICT (destination) DO UPDATE SET
213		failure_ts = EXCLUDED.failure_ts,
214		retry_last_ts = EXCLUDED.retry_last_ts,
215		retry_interval = EXCLUDED.retry_interval
216		WHERE
217		EXCLUDED.retry_interval = 0
218		OR destinations.retry_interval < EXCLUDED.retry_interval
219		"""
220
221		txn.execute(sql, (destination, failure_ts, retry_last_ts, retry_interval))
222
223		return
224
225		self.database_engine.lock_table(txn, "destinations")
226
227		# We need to be careful here as the data may have changed from under us
228		# due to a worker setting the timings.
229
230		prev_row = self._simple_select_one_txn(
231		txn,
232		table="destinations",
233		keyvalues={"destination": destination},
234		retcols=("failure_ts", "retry_last_ts", "retry_interval"),
235		allow_none=True,
236		)
237
238		if not prev_row:
239		self._simple_insert_txn(
240		txn,
241		table="destinations",
242		values={
243		"destination": destination,
244		"failure_ts": failure_ts,
245		"retry_last_ts": retry_last_ts,
246		"retry_interval": retry_interval,
247		},
248		)
249		elif retry_interval == 0 or prev_row["retry_interval"] < retry_interval:
250		self._simple_update_one_txn(
251		txn,
252		"destinations",
253		keyvalues={"destination": destination},
254		updatevalues={
255		"failure_ts": failure_ts,
256		"retry_last_ts": retry_last_ts,
257		"retry_interval": retry_interval,
258		},
259		)
260
261		def _start_cleanup_transactions(self):
262		return run_as_background_process(
263		"cleanup_transactions", self._cleanup_transactions
264		)
265
266		def _cleanup_transactions(self):
267		now = self._clock.time_msec()
268		month_ago = now - 30 * 24 * 60 * 60 * 1000
269
270		def _cleanup_transactions_txn(txn):
271		txn.execute("DELETE FROM received_transactions WHERE ts < ?", (month_ago,))
272
273		return self.runInteraction("_cleanup_transactions", _cleanup_transactions_txn)

+0

-817

~~synapse/storage/user_directory.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2017 Vector Creations Ltd
2		#
3		# Licensed under the Apache License, Version 2.0 (the "License");
4		# you may not use this file except in compliance with the License.
5		# You may obtain a copy of the License at
6		#
7		# http://www.apache.org/licenses/LICENSE-2.0
8		#
9		# Unless required by applicable law or agreed to in writing, software
10		# distributed under the License is distributed on an "AS IS" BASIS,
11		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12		# See the License for the specific language governing permissions and
13		# limitations under the License.
14
15		import logging
16		import re
17
18		from twisted.internet import defer
19
20		from synapse.api.constants import EventTypes, JoinRules
21		from synapse.storage.background_updates import BackgroundUpdateStore
22		from synapse.storage.engines import PostgresEngine, Sqlite3Engine
23		from synapse.storage.state import StateFilter
24		from synapse.storage.state_deltas import StateDeltasStore
25		from synapse.types import get_domain_from_id, get_localpart_from_id
26		from synapse.util.caches.descriptors import cached
27
28		logger = logging.getLogger(__name__)
29
30
31		TEMP_TABLE = "_temp_populate_user_directory"
32
33
34		class UserDirectoryStore(StateDeltasStore, BackgroundUpdateStore):
35
36		# How many records do we calculate before sending it to
37		# add_users_who_share_private_rooms?
38		SHARE_PRIVATE_WORKING_SET = 500
39
40		def __init__(self, db_conn, hs):
41		super(UserDirectoryStore, self).__init__(db_conn, hs)
42
43		self.server_name = hs.hostname
44
45		self.register_background_update_handler(
46		"populate_user_directory_createtables",
47		self._populate_user_directory_createtables,
48		)
49		self.register_background_update_handler(
50		"populate_user_directory_process_rooms",
51		self._populate_user_directory_process_rooms,
52		)
53		self.register_background_update_handler(
54		"populate_user_directory_process_users",
55		self._populate_user_directory_process_users,
56		)
57		self.register_background_update_handler(
58		"populate_user_directory_cleanup", self._populate_user_directory_cleanup
59		)
60
61		@defer.inlineCallbacks
62		def _populate_user_directory_createtables(self, progress, batch_size):
63
64		# Get all the rooms that we want to process.
65		def _make_staging_area(txn):
66		sql = (
67		"CREATE TABLE IF NOT EXISTS "
68		+ TEMP_TABLE
69		+ "_rooms(room_id TEXT NOT NULL, events BIGINT NOT NULL)"
70		)
71		txn.execute(sql)
72
73		sql = (
74		"CREATE TABLE IF NOT EXISTS "
75		+ TEMP_TABLE
76		+ "_position(position TEXT NOT NULL)"
77		)
78		txn.execute(sql)
79
80		# Get rooms we want to process from the database
81		sql = """
82		SELECT room_id, count(*) FROM current_state_events
83		GROUP BY room_id
84		"""
85		txn.execute(sql)
86		rooms = [{"room_id": x[0], "events": x[1]} for x in txn.fetchall()]
87		self._simple_insert_many_txn(txn, TEMP_TABLE + "_rooms", rooms)
88		del rooms
89
90		# If search all users is on, get all the users we want to add.
91		if self.hs.config.user_directory_search_all_users:
92		sql = (
93		"CREATE TABLE IF NOT EXISTS "
94		+ TEMP_TABLE
95		+ "_users(user_id TEXT NOT NULL)"
96		)
97		txn.execute(sql)
98
99		txn.execute("SELECT name FROM users")
100		users = [{"user_id": x[0]} for x in txn.fetchall()]
101
102		self._simple_insert_many_txn(txn, TEMP_TABLE + "_users", users)
103
104		new_pos = yield self.get_max_stream_id_in_current_state_deltas()
105		yield self.runInteraction(
106		"populate_user_directory_temp_build", _make_staging_area
107		)
108		yield self._simple_insert(TEMP_TABLE + "_position", {"position": new_pos})
109
110		yield self._end_background_update("populate_user_directory_createtables")
111		return 1
112
113		@defer.inlineCallbacks
114		def _populate_user_directory_cleanup(self, progress, batch_size):
115		"""
116		Update the user directory stream position, then clean up the old tables.
117		"""
118		position = yield self._simple_select_one_onecol(
119		TEMP_TABLE + "_position", None, "position"
120		)
121		yield self.update_user_directory_stream_pos(position)
122
123		def _delete_staging_area(txn):
124		txn.execute("DROP TABLE IF EXISTS " + TEMP_TABLE + "_rooms")
125		txn.execute("DROP TABLE IF EXISTS " + TEMP_TABLE + "_users")
126		txn.execute("DROP TABLE IF EXISTS " + TEMP_TABLE + "_position")
127
128		yield self.runInteraction(
129		"populate_user_directory_cleanup", _delete_staging_area
130		)
131
132		yield self._end_background_update("populate_user_directory_cleanup")
133		return 1
134
135		@defer.inlineCallbacks
136		def _populate_user_directory_process_rooms(self, progress, batch_size):
137		"""
138		Args:
139		progress (dict)
140		batch_size (int): Maximum number of state events to process
141		per cycle.
142		"""
143		state = self.hs.get_state_handler()
144
145		# If we don't have progress filed, delete everything.
146		if not progress:
147		yield self.delete_all_from_user_dir()
148
149		def _get_next_batch(txn):
150		# Only fetch 250 rooms, so we don't fetch too many at once, even
151		# if those 250 rooms have less than batch_size state events.
152		sql = """
153		SELECT room_id, events FROM %s
154		ORDER BY events DESC
155		LIMIT 250
156		""" % (
157		TEMP_TABLE + "_rooms",
158		)
159		txn.execute(sql)
160		rooms_to_work_on = txn.fetchall()
161
162		if not rooms_to_work_on:
163		return None
164
165		# Get how many are left to process, so we can give status on how
166		# far we are in processing
167		txn.execute("SELECT COUNT(*) FROM " + TEMP_TABLE + "_rooms")
168		progress["remaining"] = txn.fetchone()[0]
169
170		return rooms_to_work_on
171
172		rooms_to_work_on = yield self.runInteraction(
173		"populate_user_directory_temp_read", _get_next_batch
174		)
175
176		# No more rooms -- complete the transaction.
177		if not rooms_to_work_on:
178		yield self._end_background_update("populate_user_directory_process_rooms")
179		return 1
180
181		logger.info(
182		"Processing the next %d rooms of %d remaining"
183		% (len(rooms_to_work_on), progress["remaining"])
184		)
185
186		processed_event_count = 0
187
188		for room_id, event_count in rooms_to_work_on:
189		is_in_room = yield self.is_host_joined(room_id, self.server_name)
190
191		if is_in_room:
192		is_public = yield self.is_room_world_readable_or_publicly_joinable(
193		room_id
194		)
195
196		users_with_profile = yield state.get_current_users_in_room(room_id)
197		user_ids = set(users_with_profile)
198
199		# Update each user in the user directory.
200		for user_id, profile in users_with_profile.items():
201		yield self.update_profile_in_user_dir(
202		user_id, profile.display_name, profile.avatar_url
203		)
204
205		to_insert = set()
206
207		if is_public:
208		for user_id in user_ids:
209		if self.get_if_app_services_interested_in_user(user_id):
210		continue
211
212		to_insert.add(user_id)
213
214		if to_insert:
215		yield self.add_users_in_public_rooms(room_id, to_insert)
216		to_insert.clear()
217		else:
218		for user_id in user_ids:
219		if not self.hs.is_mine_id(user_id):
220		continue
221
222		if self.get_if_app_services_interested_in_user(user_id):
223		continue
224
225		for other_user_id in user_ids:
226		if user_id == other_user_id:
227		continue
228
229		user_set = (user_id, other_user_id)
230		to_insert.add(user_set)
231
232		# If it gets too big, stop and write to the database
233		# to prevent storing too much in RAM.
234		if len(to_insert) >= self.SHARE_PRIVATE_WORKING_SET:
235		yield self.add_users_who_share_private_room(
236		room_id, to_insert
237		)
238		to_insert.clear()
239
240		if to_insert:
241		yield self.add_users_who_share_private_room(room_id, to_insert)
242		to_insert.clear()
243
244		# We've finished a room. Delete it from the table.
245		yield self._simple_delete_one(TEMP_TABLE + "_rooms", {"room_id": room_id})
246		# Update the remaining counter.
247		progress["remaining"] -= 1
248		yield self.runInteraction(
249		"populate_user_directory",
250		self._background_update_progress_txn,
251		"populate_user_directory_process_rooms",
252		progress,
253		)
254
255		processed_event_count += event_count
256
257		if processed_event_count > batch_size:
258		# Don't process any more rooms, we've hit our batch size.
259		return processed_event_count
260
261		return processed_event_count
262
263		@defer.inlineCallbacks
264		def _populate_user_directory_process_users(self, progress, batch_size):
265		"""
266		If search_all_users is enabled, add all of the users to the user directory.
267		"""
268		if not self.hs.config.user_directory_search_all_users:
269		yield self._end_background_update("populate_user_directory_process_users")
270		return 1
271
272		def _get_next_batch(txn):
273		sql = "SELECT user_id FROM %s LIMIT %s" % (
274		TEMP_TABLE + "_users",
275		str(batch_size),
276		)
277		txn.execute(sql)
278		users_to_work_on = txn.fetchall()
279
280		if not users_to_work_on:
281		return None
282
283		users_to_work_on = [x[0] for x in users_to_work_on]
284
285		# Get how many are left to process, so we can give status on how
286		# far we are in processing
287		sql = "SELECT COUNT(*) FROM " + TEMP_TABLE + "_users"
288		txn.execute(sql)
289		progress["remaining"] = txn.fetchone()[0]
290
291		return users_to_work_on
292
293		users_to_work_on = yield self.runInteraction(
294		"populate_user_directory_temp_read", _get_next_batch
295		)
296
297		# No more users -- complete the transaction.
298		if not users_to_work_on:
299		yield self._end_background_update("populate_user_directory_process_users")
300		return 1
301
302		logger.info(
303		"Processing the next %d users of %d remaining"
304		% (len(users_to_work_on), progress["remaining"])
305		)
306
307		for user_id in users_to_work_on:
308		profile = yield self.get_profileinfo(get_localpart_from_id(user_id))
309		yield self.update_profile_in_user_dir(
310		user_id, profile.display_name, profile.avatar_url
311		)
312
313		# We've finished processing a user. Delete it from the table.
314		yield self._simple_delete_one(TEMP_TABLE + "_users", {"user_id": user_id})
315		# Update the remaining counter.
316		progress["remaining"] -= 1
317		yield self.runInteraction(
318		"populate_user_directory",
319		self._background_update_progress_txn,
320		"populate_user_directory_process_users",
321		progress,
322		)
323
324		return len(users_to_work_on)
325
326		@defer.inlineCallbacks
327		def is_room_world_readable_or_publicly_joinable(self, room_id):
328		"""Check if the room is either world_readable or publically joinable
329		"""
330
331		# Create a state filter that only queries join and history state event
332		types_to_filter = (
333		(EventTypes.JoinRules, ""),
334		(EventTypes.RoomHistoryVisibility, ""),
335		)
336
337		current_state_ids = yield self.get_filtered_current_state_ids(
338		room_id, StateFilter.from_types(types_to_filter)
339		)
340
341		join_rules_id = current_state_ids.get((EventTypes.JoinRules, ""))
342		if join_rules_id:
343		join_rule_ev = yield self.get_event(join_rules_id, allow_none=True)
344		if join_rule_ev:
345		if join_rule_ev.content.get("join_rule") == JoinRules.PUBLIC:
346		return True
347
348		hist_vis_id = current_state_ids.get((EventTypes.RoomHistoryVisibility, ""))
349		if hist_vis_id:
350		hist_vis_ev = yield self.get_event(hist_vis_id, allow_none=True)
351		if hist_vis_ev:
352		if hist_vis_ev.content.get("history_visibility") == "world_readable":
353		return True
354
355		return False
356
357		def update_profile_in_user_dir(self, user_id, display_name, avatar_url):
358		"""
359		Update or add a user's profile in the user directory.
360		"""
361
362		def _update_profile_in_user_dir_txn(txn):
363		new_entry = self._simple_upsert_txn(
364		txn,
365		table="user_directory",
366		keyvalues={"user_id": user_id},
367		values={"display_name": display_name, "avatar_url": avatar_url},
368		lock=False, # We're only inserter
369		)
370
371		if isinstance(self.database_engine, PostgresEngine):
372		# We weight the localpart most highly, then display name and finally
373		# server name
374		if self.database_engine.can_native_upsert:
375		sql = """
376		INSERT INTO user_directory_search(user_id, vector)
377		VALUES (?,
378		setweight(to_tsvector('english', ?), 'A')
379		\|\| setweight(to_tsvector('english', ?), 'D')
380		\|\| setweight(to_tsvector('english', COALESCE(?, '')), 'B')
381		) ON CONFLICT (user_id) DO UPDATE SET vector=EXCLUDED.vector
382		"""
383		txn.execute(
384		sql,
385		(
386		user_id,
387		get_localpart_from_id(user_id),
388		get_domain_from_id(user_id),
389		display_name,
390		),
391		)
392		else:
393		# TODO: Remove this code after we've bumped the minimum version
394		# of postgres to always support upserts, so we can get rid of
395		# `new_entry` usage
396		if new_entry is True:
397		sql = """
398		INSERT INTO user_directory_search(user_id, vector)
399		VALUES (?,
400		setweight(to_tsvector('english', ?), 'A')
401		\|\| setweight(to_tsvector('english', ?), 'D')
402		\|\| setweight(to_tsvector('english', COALESCE(?, '')), 'B')
403		)
404		"""
405		txn.execute(
406		sql,
407		(
408		user_id,
409		get_localpart_from_id(user_id),
410		get_domain_from_id(user_id),
411		display_name,
412		),
413		)
414		elif new_entry is False:
415		sql = """
416		UPDATE user_directory_search
417		SET vector = setweight(to_tsvector('english', ?), 'A')
418		\|\| setweight(to_tsvector('english', ?), 'D')
419		\|\| setweight(to_tsvector('english', COALESCE(?, '')), 'B')
420		WHERE user_id = ?
421		"""
422		txn.execute(
423		sql,
424		(
425		get_localpart_from_id(user_id),
426		get_domain_from_id(user_id),
427		display_name,
428		user_id,
429		),
430		)
431		else:
432		raise RuntimeError(
433		"upsert returned None when 'can_native_upsert' is False"
434		)
435		elif isinstance(self.database_engine, Sqlite3Engine):
436		value = "%s %s" % (user_id, display_name) if display_name else user_id
437		self._simple_upsert_txn(
438		txn,
439		table="user_directory_search",
440		keyvalues={"user_id": user_id},
441		values={"value": value},
442		lock=False, # We're only inserter
443		)
444		else:
445		# This should be unreachable.
446		raise Exception("Unrecognized database engine")
447
448		txn.call_after(self.get_user_in_directory.invalidate, (user_id,))
449
450		return self.runInteraction(
451		"update_profile_in_user_dir", _update_profile_in_user_dir_txn
452		)
453
454		def remove_from_user_dir(self, user_id):
455		def _remove_from_user_dir_txn(txn):
456		self._simple_delete_txn(
457		txn, table="user_directory", keyvalues={"user_id": user_id}
458		)
459		self._simple_delete_txn(
460		txn, table="user_directory_search", keyvalues={"user_id": user_id}
461		)
462		self._simple_delete_txn(
463		txn, table="users_in_public_rooms", keyvalues={"user_id": user_id}
464		)
465		self._simple_delete_txn(
466		txn,
467		table="users_who_share_private_rooms",
468		keyvalues={"user_id": user_id},
469		)
470		self._simple_delete_txn(
471		txn,
472		table="users_who_share_private_rooms",
473		keyvalues={"other_user_id": user_id},
474		)
475		txn.call_after(self.get_user_in_directory.invalidate, (user_id,))
476
477		return self.runInteraction("remove_from_user_dir", _remove_from_user_dir_txn)
478
479		@defer.inlineCallbacks
480		def get_users_in_dir_due_to_room(self, room_id):
481		"""Get all user_ids that are in the room directory because they're
482		in the given room_id
483		"""
484		user_ids_share_pub = yield self._simple_select_onecol(
485		table="users_in_public_rooms",
486		keyvalues={"room_id": room_id},
487		retcol="user_id",
488		desc="get_users_in_dir_due_to_room",
489		)
490
491		user_ids_share_priv = yield self._simple_select_onecol(
492		table="users_who_share_private_rooms",
493		keyvalues={"room_id": room_id},
494		retcol="other_user_id",
495		desc="get_users_in_dir_due_to_room",
496		)
497
498		user_ids = set(user_ids_share_pub)
499		user_ids.update(user_ids_share_priv)
500
501		return user_ids
502
503		def add_users_who_share_private_room(self, room_id, user_id_tuples):
504		"""Insert entries into the users_who_share_private_rooms table. The first
505		user should be a local user.
506
507		Args:
508		room_id (str)
509		user_id_tuples([(str, str)]): iterable of 2-tuple of user IDs.
510		"""
511
512		def _add_users_who_share_room_txn(txn):
513		self._simple_upsert_many_txn(
514		txn,
515		table="users_who_share_private_rooms",
516		key_names=["user_id", "other_user_id", "room_id"],
517		key_values=[
518		(user_id, other_user_id, room_id)
519		for user_id, other_user_id in user_id_tuples
520		],
521		value_names=(),
522		value_values=None,
523		)
524
525		return self.runInteraction(
526		"add_users_who_share_room", _add_users_who_share_room_txn
527		)
528
529		def add_users_in_public_rooms(self, room_id, user_ids):
530		"""Insert entries into the users_who_share_private_rooms table. The first
531		user should be a local user.
532
533		Args:
534		room_id (str)
535		user_ids (list[str])
536		"""
537
538		def _add_users_in_public_rooms_txn(txn):
539
540		self._simple_upsert_many_txn(
541		txn,
542		table="users_in_public_rooms",
543		key_names=["user_id", "room_id"],
544		key_values=[(user_id, room_id) for user_id in user_ids],
545		value_names=(),
546		value_values=None,
547		)
548
549		return self.runInteraction(
550		"add_users_in_public_rooms", _add_users_in_public_rooms_txn
551		)
552
553		def remove_user_who_share_room(self, user_id, room_id):
554		"""
555		Deletes entries in the users_who_share_*_rooms table. The first
556		user should be a local user.
557
558		Args:
559		user_id (str)
560		room_id (str)
561		"""
562
563		def _remove_user_who_share_room_txn(txn):
564		self._simple_delete_txn(
565		txn,
566		table="users_who_share_private_rooms",
567		keyvalues={"user_id": user_id, "room_id": room_id},
568		)
569		self._simple_delete_txn(
570		txn,
571		table="users_who_share_private_rooms",
572		keyvalues={"other_user_id": user_id, "room_id": room_id},
573		)
574		self._simple_delete_txn(
575		txn,
576		table="users_in_public_rooms",
577		keyvalues={"user_id": user_id, "room_id": room_id},
578		)
579
580		return self.runInteraction(
581		"remove_user_who_share_room", _remove_user_who_share_room_txn
582		)
583
584		@defer.inlineCallbacks
585		def get_user_dir_rooms_user_is_in(self, user_id):
586		"""
587		Returns the rooms that a user is in.
588
589		Args:
590		user_id(str): Must be a local user
591
592		Returns:
593		list: user_id
594		"""
595		rows = yield self._simple_select_onecol(
596		table="users_who_share_private_rooms",
597		keyvalues={"user_id": user_id},
598		retcol="room_id",
599		desc="get_rooms_user_is_in",
600		)
601
602		pub_rows = yield self._simple_select_onecol(
603		table="users_in_public_rooms",
604		keyvalues={"user_id": user_id},
605		retcol="room_id",
606		desc="get_rooms_user_is_in",
607		)
608
609		users = set(pub_rows)
610		users.update(rows)
611		return list(users)
612
613		@defer.inlineCallbacks
614		def get_rooms_in_common_for_users(self, user_id, other_user_id):
615		"""Given two user_ids find out the list of rooms they share.
616		"""
617		sql = """
618		SELECT room_id FROM (
619		SELECT c.room_id FROM current_state_events AS c
620		INNER JOIN room_memberships AS m USING (event_id)
621		WHERE type = 'm.room.member'
622		AND m.membership = 'join'
623		AND state_key = ?
624		) AS f1 INNER JOIN (
625		SELECT c.room_id FROM current_state_events AS c
626		INNER JOIN room_memberships AS m USING (event_id)
627		WHERE type = 'm.room.member'
628		AND m.membership = 'join'
629		AND state_key = ?
630		) f2 USING (room_id)
631		"""
632
633		rows = yield self._execute(
634		"get_rooms_in_common_for_users", None, sql, user_id, other_user_id
635		)
636
637		return [room_id for room_id, in rows]
638
639		def delete_all_from_user_dir(self):
640		"""Delete the entire user directory
641		"""
642
643		def _delete_all_from_user_dir_txn(txn):
644		txn.execute("DELETE FROM user_directory")
645		txn.execute("DELETE FROM user_directory_search")
646		txn.execute("DELETE FROM users_in_public_rooms")
647		txn.execute("DELETE FROM users_who_share_private_rooms")
648		txn.call_after(self.get_user_in_directory.invalidate_all)
649
650		return self.runInteraction(
651		"delete_all_from_user_dir", _delete_all_from_user_dir_txn
652		)
653
654		@cached()
655		def get_user_in_directory(self, user_id):
656		return self._simple_select_one(
657		table="user_directory",
658		keyvalues={"user_id": user_id},
659		retcols=("display_name", "avatar_url"),
660		allow_none=True,
661		desc="get_user_in_directory",
662		)
663
664		def get_user_directory_stream_pos(self):
665		return self._simple_select_one_onecol(
666		table="user_directory_stream_pos",
667		keyvalues={},
668		retcol="stream_id",
669		desc="get_user_directory_stream_pos",
670		)
671
672		def update_user_directory_stream_pos(self, stream_id):
673		return self._simple_update_one(
674		table="user_directory_stream_pos",
675		keyvalues={},
676		updatevalues={"stream_id": stream_id},
677		desc="update_user_directory_stream_pos",
678		)
679
680		@defer.inlineCallbacks
681		def search_user_dir(self, user_id, search_term, limit):
682		"""Searches for users in directory
683
684		Returns:
685		dict of the form::
686
687		{
688		"limited": <bool>, # whether there were more results or not
689		"results": [ # Ordered by best match first
690		{
691		"user_id": <user_id>,
692		"display_name": <display_name>,
693		"avatar_url": <avatar_url>
694		}
695		]
696		}
697		"""
698
699		if self.hs.config.user_directory_search_all_users:
700		join_args = (user_id,)
701		where_clause = "user_id != ?"
702		else:
703		join_args = (user_id,)
704		where_clause = """
705		(
706		EXISTS (select 1 from users_in_public_rooms WHERE user_id = t.user_id)
707		OR EXISTS (
708		SELECT 1 FROM users_who_share_private_rooms
709		WHERE user_id = ? AND other_user_id = t.user_id
710		)
711		)
712		"""
713
714		if isinstance(self.database_engine, PostgresEngine):
715		full_query, exact_query, prefix_query = _parse_query_postgres(search_term)
716
717		# We order by rank and then if they have profile info
718		# The ranking algorithm is hand tweaked for "best" results. Broadly
719		# the idea is we give a higher weight to exact matches.
720		# The array of numbers are the weights for the various part of the
721		# search: (domain, _, display name, localpart)
722		sql = """
723		SELECT d.user_id AS user_id, display_name, avatar_url
724		FROM user_directory_search as t
725		INNER JOIN user_directory AS d USING (user_id)
726		WHERE
727		%s
728		AND vector @@ to_tsquery('english', ?)
729		ORDER BY
730		(CASE WHEN d.user_id IS NOT NULL THEN 4.0 ELSE 1.0 END)
731		* (CASE WHEN display_name IS NOT NULL THEN 1.2 ELSE 1.0 END)
732		* (CASE WHEN avatar_url IS NOT NULL THEN 1.2 ELSE 1.0 END)
733		* (
734		3 * ts_rank_cd(
735		'{0.1, 0.1, 0.9, 1.0}',
736		vector,
737		to_tsquery('english', ?),
738		8
739		)
740		+ ts_rank_cd(
741		'{0.1, 0.1, 0.9, 1.0}',
742		vector,
743		to_tsquery('english', ?),
744		8
745		)
746		)
747		DESC,
748		display_name IS NULL,
749		avatar_url IS NULL
750		LIMIT ?
751		""" % (
752		where_clause,
753		)
754		args = join_args + (full_query, exact_query, prefix_query, limit + 1)
755		elif isinstance(self.database_engine, Sqlite3Engine):
756		search_query = _parse_query_sqlite(search_term)
757
758		sql = """
759		SELECT d.user_id AS user_id, display_name, avatar_url
760		FROM user_directory_search as t
761		INNER JOIN user_directory AS d USING (user_id)
762		WHERE
763		%s
764		AND value MATCH ?
765		ORDER BY
766		rank(matchinfo(user_directory_search)) DESC,
767		display_name IS NULL,
768		avatar_url IS NULL
769		LIMIT ?
770		""" % (
771		where_clause,
772		)
773		args = join_args + (search_query, limit + 1)
774		else:
775		# This should be unreachable.
776		raise Exception("Unrecognized database engine")
777
778		results = yield self._execute(
779		"search_user_dir", self.cursor_to_dict, sql, *args
780		)
781
782		limited = len(results) > limit
783
784		return {"limited": limited, "results": results}
785
786
787		def _parse_query_sqlite(search_term):
788		"""Takes a plain unicode string from the user and converts it into a form
789		that can be passed to database.
790		We use this so that we can add prefix matching, which isn't something
791		that is supported by default.
792
793		We specifically add both a prefix and non prefix matching term so that
794		exact matches get ranked higher.
795		"""
796
797		# Pull out the individual words, discarding any non-word characters.
798		results = re.findall(r"([\w\-]+)", search_term, re.UNICODE)
799		return " & ".join("(%s* OR %s)" % (result, result) for result in results)
800
801
802		def _parse_query_postgres(search_term):
803		"""Takes a plain unicode string from the user and converts it into a form
804		that can be passed to database.
805		We use this so that we can add prefix matching, which isn't something
806		that is supported by default.
807		"""
808
809		# Pull out the individual words, discarding any non-word characters.
810		results = re.findall(r"([\w\-]+)", search_term, re.UNICODE)
811
812		both = " & ".join("(%s:* \| %s)" % (result, result) for result in results)
813		exact = " & ".join("%s" % (result,) for result in results)
814		prefix = " & ".join("%s:*" % (result,) for result in results)
815
816		return both, exact, prefix

+0

-91

~~synapse/storage/user_erasure_store.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2018 New Vector Ltd
2		#
3		# Licensed under the Apache License, Version 2.0 (the "License");
4		# you may not use this file except in compliance with the License.
5		# You may obtain a copy of the License at
6		#
7		# http://www.apache.org/licenses/LICENSE-2.0
8		#
9		# Unless required by applicable law or agreed to in writing, software
10		# distributed under the License is distributed on an "AS IS" BASIS,
11		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12		# See the License for the specific language governing permissions and
13		# limitations under the License.
14
15		import operator
16
17		from synapse.storage._base import SQLBaseStore
18		from synapse.util.caches.descriptors import cached, cachedList
19
20
21		class UserErasureWorkerStore(SQLBaseStore):
22		@cached()
23		def is_user_erased(self, user_id):
24		"""
25		Check if the given user id has requested erasure
26
27		Args:
28		user_id (str): full user id to check
29
30		Returns:
31		Deferred[bool]: True if the user has requested erasure
32		"""
33		return self._simple_select_onecol(
34		table="erased_users",
35		keyvalues={"user_id": user_id},
36		retcol="1",
37		desc="is_user_erased",
38		).addCallback(operator.truth)
39
40		@cachedList(
41		cached_method_name="is_user_erased", list_name="user_ids", inlineCallbacks=True
42		)
43		def are_users_erased(self, user_ids):
44		"""
45		Checks which users in a list have requested erasure
46
47		Args:
48		user_ids (iterable[str]): full user id to check
49
50		Returns:
51		Deferred[dict[str, bool]]:
52		for each user, whether the user has requested erasure.
53		"""
54		# this serves the dual purpose of (a) making sure we can do len and
55		# iterate it multiple times, and (b) avoiding duplicates.
56		user_ids = tuple(set(user_ids))
57
58		def _get_erased_users(txn):
59		txn.execute(
60		"SELECT user_id FROM erased_users WHERE user_id IN (%s)"
61		% (",".join("?" * len(user_ids))),
62		user_ids,
63		)
64		return set(r[0] for r in txn)
65
66		erased_users = yield self.runInteraction("are_users_erased", _get_erased_users)
67		res = dict((u, u in erased_users) for u in user_ids)
68		return res
69
70
71		class UserErasureStore(UserErasureWorkerStore):
72		def mark_user_erased(self, user_id):
73		"""Indicate that user_id wishes their message history to be erased.
74
75		Args:
76		user_id (str): full user_id to be erased
77		"""
78
79		def f(txn):
80		# first check if they are already in the list
81		txn.execute("SELECT 1 FROM erased_users WHERE user_id = ?", (user_id,))
82		if txn.fetchone():
83		return
84
85		# they are not already there: do the insert.
86		txn.execute("INSERT INTO erased_users (user_id) VALUES (?)", (user_id,))
87
88		self._invalidate_cache_and_stream(txn, self.is_user_erased, (user_id,))
89
90		return self.runInteraction("mark_user_erased", f)

+26

-1

synapse/types.py less more

0	0	# -- coding: utf-8 --
1	1	# Copyright 2014-2016 OpenMarket Ltd
	2	# Copyright 2019 The Matrix.org Foundation C.I.C.
2	3	#
3	4	# Licensed under the Apache License, Version 2.0 (the "License");
4	5	# you may not use this file except in compliance with the License.

16	17	from collections import namedtuple
17	18
18	19	import attr
	20	from signedjson.key import decode_verify_key_bytes
	21	from unpaddedbase64 import decode_base64
19	22
20	23	from synapse.api.errors import SynapseError
21	24

317	320	)
318	321	):
319	322	_SEPARATOR = "_"
	323	START = None # type: StreamToken
320	324
321	325	@classmethod
322	326	def from_string(cls, string):

401	405	followed by the "stream_ordering" id of the event it comes after.
402	406	"""
403	407
404		__slots__ = []
	408	__slots__ = [] # type: list
405	409
406	410	@classmethod
407	411	def parse(cls, string):

474	478	user_id = attr.ib()
475	479	event_ids = attr.ib()
476	480	data = attr.ib()
	481
	482
	483	def get_verify_key_from_cross_signing_key(key_info):
	484	"""Get the key ID and signedjson verify key from a cross-signing key dict
	485
	486	Args:
	487	key_info (dict): a cross-signing key dict, which must have a "keys"
	488	property that has exactly one item in it
	489
	490	Returns:
	491	(str, VerifyKey): the key ID and verify key for the cross-signing key
	492	"""
	493	# make sure that exactly one key is provided
	494	if "keys" not in key_info:
	495	raise ValueError("Invalid key")
	496	keys = key_info["keys"]
	497	if len(keys) != 1:
	498	raise ValueError("Invalid key")
	499	# and return that one key
	500	for key_id, key_data in keys.items():
	501	return (key_id, decode_verify_key_bytes(key_id, decode_base64(key_data)))

+36

-3

synapse/util/async_helpers.py less more

12	12	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	13	# See the License for the specific language governing permissions and
14	14	# limitations under the License.
	15
15	16	import collections
16	17	import logging
17	18	from contextlib import contextmanager
	19	from typing import Dict, Sequence, Set, Union
18	20
19	21	from six.moves import range
	22
	23	import attr
20	24
21	25	from twisted.internet import defer
22	26	from twisted.internet.defer import CancelledError

212	216	# the first element is the number of things executing, and
213	217	# the second element is an OrderedDict, where the keys are deferreds for the
214	218	# things blocked from executing.
215		self.key_to_defer = {}
	219	self.key_to_defer = (
	220	{}
	221	) # type: Dict[str, Sequence[Union[int, Dict[defer.Deferred, int]]]]
216	222
217	223	def queue(self, key):
218	224	# we avoid doing defer.inlineCallbacks here, so that cancellation works correctly.

339	345
340	346	def __init__(self):
341	347	# Latest readers queued
342		self.key_to_current_readers = {}
	348	self.key_to_current_readers = {} # type: Dict[str, Set[defer.Deferred]]
343	349
344	350	# Latest writer queued
345		self.key_to_current_writer = {}
	351	self.key_to_current_writer = {} # type: Dict[str, defer.Deferred]
346	352
347	353	@defer.inlineCallbacks
348	354	def read(self, key):

478	484	deferred.addCallbacks(success_cb, failure_cb)
479	485
480	486	return new_d
	487
	488
	489	@attr.s(slots=True, frozen=True)
	490	class DoneAwaitable(object):
	491	"""Simple awaitable that returns the provided value.
	492	"""
	493
	494	value = attr.ib()
	495
	496	def __await__(self):
	497	return self
	498
	499	def __iter__(self):
	500	return self
	501
	502	def __next__(self):
	503	raise StopIteration(self.value)
	504
	505
	506	def maybe_awaitable(value):
	507	"""Convert a value to an awaitable if not already an awaitable.
	508	"""
	509
	510	if hasattr(value, "__await__"):
	511	return value
	512
	513	return DoneAwaitable(value)

+2

-1

synapse/util/caches/__init__.py less more

15	15
16	16	import logging
17	17	import os
	18	from typing import Dict
18	19
19	20	import six
20	21	from six.moves import intern

36	37
37	38
38	39	caches_by_name = {}
39		collectors_by_name = {}
	40	collectors_by_name = {} # type: Dict
40	41
41	42	cache_size = Gauge("synapse_util_caches_cache:size", "", ["name"])
42	43	cache_hits = Gauge("synapse_util_caches_cache:hits", "", ["name"])

+20

-2

synapse/util/caches/descriptors.py less more

17	17	import logging
18	18	import threading
19	19	from collections import namedtuple
	20	from typing import Any, cast
20	21
21	22	from six import itervalues
22	23
23	24	from prometheus_client import Gauge
	25	from typing_extensions import Protocol
24	26
25	27	from twisted.internet import defer
26	28

34	36	from . import register_cache
35	37
36	38	logger = logging.getLogger(__name__)
	39
	40
	41	class _CachedFunction(Protocol):
	42	invalidate = None # type: Any
	43	invalidate_all = None # type: Any
	44	invalidate_many = None # type: Any
	45	prefill = None # type: Any
	46	cache = None # type: Any
	47	num_args = None # type: Any
	48
	49	def __name__(self):
	50	...
37	51
38	52
39	53	cache_pending_metric = Gauge(

244	258
245	259
246	260	class _CacheDescriptorBase(object):
247		def __init__(self, orig, num_args, inlineCallbacks, cache_context=False):
	261	def __init__(
	262	self, orig: _CachedFunction, num_args, inlineCallbacks, cache_context=False
	263	):
248	264	self.orig = orig
249	265
250	266	if inlineCallbacks:

403	419	return tuple(get_cache_key_gen(args, kwargs))
404	420
405	421	@functools.wraps(self.orig)
406		def wrapped(args, *kwargs):
	422	def _wrapped(args, *kwargs):
407	423	# If we're passed a cache_context then we'll want to call its invalidate()
408	424	# whenever we are invalidated
409	425	invalidate_callback = kwargs.pop("on_invalidate", None)

438	454	observer = result_d.observe()
439	455
440	456	return make_deferred_yieldable(observer)
	457
	458	wrapped = cast(_CachedFunction, _wrapped)
441	459
442	460	if self.num_args == 1:
443	461	wrapped.invalidate = lambda key: cache.invalidate(key[0])

+3

-1

synapse/util/caches/treecache.py less more

	0	from typing import Dict
	1
0	2	from six import itervalues
1	3
2	4	SENTINEL = object()

11	13
12	14	def __init__(self):
13	15	self.size = 0
14		self.root = {}
	16	self.root = {} # type: Dict
15	17
16	18	def __setitem__(self, key, value):
17	19	return self.set(key, value)

+4

-2

synapse/util/metrics.py less more

59	59	)
60	60
61	61
62		def measure_func(name):
	62	def measure_func(name=None):
63	63	def wrapper(func):
	64	block_name = func.__name__ if name is None else name
	65
64	66	@wraps(func)
65	67	@defer.inlineCallbacks
66	68	def measured_func(self, args, *kwargs):
67		with Measure(self.clock, name):
	69	with Measure(self.clock, block_name):
68	70	r = yield func(self, args, *kwargs)
69	71	return r
70	72

+1

-1

synapse/util/module_loader.py less more

53	53	if spec is None:
54	54	raise Exception("Unable to load module at %s" % (location,))
55	55	mod = importlib.util.module_from_spec(spec)
56		spec.loader.exec_module(mod)
	56	spec.loader.exec_module(mod) # type: ignore
57	57	return mod

+219

-0

synapse/util/patch_inline_callbacks.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2018 New Vector Ltd
	2	#
	3	# Licensed under the Apache License, Version 2.0 (the "License");
	4	# you may not use this file except in compliance with the License.
	5	# You may obtain a copy of the License at
	6	#
	7	# http://www.apache.org/licenses/LICENSE-2.0
	8	#
	9	# Unless required by applicable law or agreed to in writing, software
	10	# distributed under the License is distributed on an "AS IS" BASIS,
	11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	# See the License for the specific language governing permissions and
	13	# limitations under the License.
	14
	15	from __future__ import print_function
	16
	17	import functools
	18	import sys
	19	from typing import Any, Callable, List
	20
	21	from twisted.internet import defer
	22	from twisted.internet.defer import Deferred
	23	from twisted.python.failure import Failure
	24
	25	# Tracks if we've already patched inlineCallbacks
	26	_already_patched = False
	27
	28
	29	def do_patch():
	30	"""
	31	Patch defer.inlineCallbacks so that it checks the state of the logcontext on exit
	32	"""
	33
	34	from synapse.logging.context import LoggingContext
	35
	36	global _already_patched
	37
	38	orig_inline_callbacks = defer.inlineCallbacks
	39	if _already_patched:
	40	return
	41
	42	def new_inline_callbacks(f):
	43	@functools.wraps(f)
	44	def wrapped(args, *kwargs):
	45	start_context = LoggingContext.current_context()
	46	changes = [] # type: List[str]
	47	orig = orig_inline_callbacks(_check_yield_points(f, changes))
	48
	49	try:
	50	res = orig(args, *kwargs)
	51	except Exception:
	52	if LoggingContext.current_context() != start_context:
	53	for err in changes:
	54	print(err, file=sys.stderr)
	55
	56	err = "%s changed context from %s to %s on exception" % (
	57	f,
	58	start_context,
	59	LoggingContext.current_context(),
	60	)
	61	print(err, file=sys.stderr)
	62	raise Exception(err)
	63	raise
	64
	65	if not isinstance(res, Deferred) or res.called:
	66	if LoggingContext.current_context() != start_context:
	67	for err in changes:
	68	print(err, file=sys.stderr)
	69
	70	err = "Completed %s changed context from %s to %s" % (
	71	f,
	72	start_context,
	73	LoggingContext.current_context(),
	74	)
	75	# print the error to stderr because otherwise all we
	76	# see in travis-ci is the 500 error
	77	print(err, file=sys.stderr)
	78	raise Exception(err)
	79	return res
	80
	81	if LoggingContext.current_context() != LoggingContext.sentinel:
	82	err = (
	83	"%s returned incomplete deferred in non-sentinel context "
	84	"%s (start was %s)"
	85	) % (f, LoggingContext.current_context(), start_context)
	86	print(err, file=sys.stderr)
	87	raise Exception(err)
	88
	89	def check_ctx(r):
	90	if LoggingContext.current_context() != start_context:
	91	for err in changes:
	92	print(err, file=sys.stderr)
	93	err = "%s completion of %s changed context from %s to %s" % (
	94	"Failure" if isinstance(r, Failure) else "Success",
	95	f,
	96	start_context,
	97	LoggingContext.current_context(),
	98	)
	99	print(err, file=sys.stderr)
	100	raise Exception(err)
	101	return r
	102
	103	res.addBoth(check_ctx)
	104	return res
	105
	106	return wrapped
	107
	108	defer.inlineCallbacks = new_inline_callbacks
	109	_already_patched = True
	110
	111
	112	def _check_yield_points(f: Callable, changes: List[str]):
	113	"""Wraps a generator that is about to be passed to defer.inlineCallbacks
	114	checking that after every yield the log contexts are correct.
	115
	116	It's perfectly valid for log contexts to change within a function, e.g. due
	117	to new Measure blocks, so such changes are added to the given `changes`
	118	list instead of triggering an exception.
	119
	120	Args:
	121	f: generator function to wrap
	122	changes: A list of strings detailing how the contexts
	123	changed within a function.
	124
	125	Returns:
	126	function
	127	"""
	128
	129	from synapse.logging.context import LoggingContext
	130
	131	@functools.wraps(f)
	132	def check_yield_points_inner(args, *kwargs):
	133	gen = f(args, *kwargs)
	134
	135	last_yield_line_no = gen.gi_frame.f_lineno
	136	result = None # type: Any
	137	while True:
	138	expected_context = LoggingContext.current_context()
	139
	140	try:
	141	isFailure = isinstance(result, Failure)
	142	if isFailure:
	143	d = result.throwExceptionIntoGenerator(gen)
	144	else:
	145	d = gen.send(result)
	146	except (StopIteration, defer._DefGen_Return) as e:
	147	if LoggingContext.current_context() != expected_context:
	148	# This happens when the context is lost sometime after the
	149	# final yield and returning. E.g. we forgot to yield on a
	150	# function that returns a deferred.
	151	#
	152	# We don't raise here as it's perfectly valid for contexts to
	153	# change in a function, as long as it sets the correct context
	154	# on resolving (which is checked separately).
	155	err = (
	156	"Function %r returned and changed context from %s to %s,"
	157	" in %s between %d and end of func"
	158	% (
	159	f.__qualname__,
	160	expected_context,
	161	LoggingContext.current_context(),
	162	f.__code__.co_filename,
	163	last_yield_line_no,
	164	)
	165	)
	166	changes.append(err)
	167	return getattr(e, "value", None)
	168
	169	frame = gen.gi_frame
	170
	171	if isinstance(d, defer.Deferred) and not d.called:
	172	# This happens if we yield on a deferred that doesn't follow
	173	# the log context rules without wrapping in a `make_deferred_yieldable`.
	174	# We raise here as this should never happen.
	175	if LoggingContext.current_context() is not LoggingContext.sentinel:
	176	err = (
	177	"%s yielded with context %s rather than sentinel,"
	178	" yielded on line %d in %s"
	179	% (
	180	frame.f_code.co_name,
	181	LoggingContext.current_context(),
	182	frame.f_lineno,
	183	frame.f_code.co_filename,
	184	)
	185	)
	186	raise Exception(err)
	187
	188	try:
	189	result = yield d
	190	except Exception as e:
	191	result = Failure(e)
	192
	193	if LoggingContext.current_context() != expected_context:
	194
	195	# This happens because the context is lost sometime after the
	196	# previous yield and after the current yield. E.g. the
	197	# deferred we waited on didn't follow the rules, or we forgot to
	198	# yield on a function between the two yield points.
	199	#
	200	# We don't raise here as its perfectly valid for contexts to
	201	# change in a function, as long as it sets the correct context
	202	# on resolving (which is checked separately).
	203	err = (
	204	"%s changed context from %s to %s, happened between lines %d and %d in %s"
	205	% (
	206	frame.f_code.co_name,
	207	expected_context,
	208	LoggingContext.current_context(),
	209	last_yield_line_no,
	210	frame.f_lineno,
	211	frame.f_code.co_filename,
	212	)
	213	)
	214	changes.append(err)
	215
	216	last_yield_line_no = frame.f_lineno
	217
	218	return check_yield_points_inner

+2

-2

tests/__init__.py less more

15	15
16	16	from twisted.trial import util
17	17
18		import tests.patch_inline_callbacks
	18	from synapse.util.patch_inline_callbacks import do_patch
19	19
20	20	# attempt to do the patch before we load any synapse code
21		tests.patch_inline_callbacks.do_patch()
	21	do_patch()
22	22
23	23	util.DEFAULT_TIMEOUT_DURATION = 20

+16

-9

tests/config/test_tls.py less more

20	20
21	21	from OpenSSL import SSL
22	22
	23	from synapse.config._base import Config, RootConfig
23	24	from synapse.config.tls import ConfigError, TlsConfig
24	25	from synapse.crypto.context_factory import ClientTLSOptionsFactory
25	26
26	27	from tests.unittest import TestCase
27	28
28	29
29		class TestConfig(TlsConfig):
	30	class FakeServer(Config):
	31	section = "server"
	32
30	33	def has_tls_listener(self):
31	34	return False
	35
	36
	37	class TestConfig(RootConfig):
	38	config_classes = [FakeServer, TlsConfig]
32	39
33	40
34	41	class TLSConfigTests(TestCase):

201	208	conf = TestConfig()
202	209	conf.read_config(
203	210	yaml.safe_load(
204		TestConfig().generate_config_section(
	211	TestConfig().generate_config(
205	212	"/config_dir_path",
206	213	"my_super_secure_server",
207	214	"/data_dir_path",
208		"/tls_cert_path",
209		"tls_private_key",
210		None, # This is the acme_domain
	215	tls_certificate_path="/tls_cert_path",
	216	tls_private_key_path="tls_private_key",
	217	acme_domain=None, # This is the acme_domain
211	218	)
212	219	),
213	220	"/config_dir_path",

222	229	conf = TestConfig()
223	230	conf.read_config(
224	231	yaml.safe_load(
225		TestConfig().generate_config_section(
	232	TestConfig().generate_config(
226	233	"/config_dir_path",
227	234	"my_super_secure_server",
228	235	"/data_dir_path",
229		"/tls_cert_path",
230		"tls_private_key",
231		"my_supe_secure_server", # This is the acme_domain
	236	tls_certificate_path="/tls_cert_path",
	237	tls_private_key_path="tls_private_key",
	238	acme_domain="my_supe_secure_server", # This is the acme_domain
232	239	)
233	240	),
234	241	"/config_dir_path",

+359

-1

tests/handlers/test_e2e_keys.py less more

0	0	# -- coding: utf-8 --
1	1	# Copyright 2016 OpenMarket Ltd
	2	# Copyright 2019 New Vector Ltd
	3	# Copyright 2019 The Matrix.org Foundation C.I.C.
2	4	#
3	5	# Licensed under the Apache License, Version 2.0 (the "License");
4	6	# you may not use this file except in compliance with the License.

14	16
15	17	import mock
16	18
	19	import signedjson.key as key
	20	import signedjson.sign as sign
	21
17	22	from twisted.internet import defer
18	23
19		import synapse.api.errors
20	24	import synapse.handlers.e2e_keys
21	25	import synapse.storage
22	26	from synapse.api import errors

144	148	"one_time_keys": {local_user: {device_id: {"alg1:k1": "key1"}}},
145	149	},
146	150	)
	151
	152	@defer.inlineCallbacks
	153	def test_replace_master_key(self):
	154	"""uploading a new signing key should make the old signing key unavailable"""
	155	local_user = "@boris:" + self.hs.hostname
	156	keys1 = {
	157	"master_key": {
	158	# private key: 2lonYOM6xYKdEsO+6KrC766xBcHnYnim1x/4LFGF8B0
	159	"user_id": local_user,
	160	"usage": ["master"],
	161	"keys": {
	162	"ed25519:nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk": "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk"
	163	},
	164	}
	165	}
	166	yield self.handler.upload_signing_keys_for_user(local_user, keys1)
	167
	168	keys2 = {
	169	"master_key": {
	170	# private key: 4TL4AjRYwDVwD3pqQzcor+ez/euOB1/q78aTJ+czDNs
	171	"user_id": local_user,
	172	"usage": ["master"],
	173	"keys": {
	174	"ed25519:Hq6gL+utB4ET+UvD5ci0kgAwsX6qP/zvf8v6OInU5iw": "Hq6gL+utB4ET+UvD5ci0kgAwsX6qP/zvf8v6OInU5iw"
	175	},
	176	}
	177	}
	178	yield self.handler.upload_signing_keys_for_user(local_user, keys2)
	179
	180	devices = yield self.handler.query_devices(
	181	{"device_keys": {local_user: []}}, 0, local_user
	182	)
	183	self.assertDictEqual(devices["master_keys"], {local_user: keys2["master_key"]})
	184
	185	@defer.inlineCallbacks
	186	def test_reupload_signatures(self):
	187	"""re-uploading a signature should not fail"""
	188	local_user = "@boris:" + self.hs.hostname
	189	keys1 = {
	190	"master_key": {
	191	# private key: HvQBbU+hc2Zr+JP1sE0XwBe1pfZZEYtJNPJLZJtS+F8
	192	"user_id": local_user,
	193	"usage": ["master"],
	194	"keys": {
	195	"ed25519:EmkqvokUn8p+vQAGZitOk4PWjp7Ukp3txV2TbMPEiBQ": "EmkqvokUn8p+vQAGZitOk4PWjp7Ukp3txV2TbMPEiBQ"
	196	},
	197	},
	198	"self_signing_key": {
	199	# private key: 2lonYOM6xYKdEsO+6KrC766xBcHnYnim1x/4LFGF8B0
	200	"user_id": local_user,
	201	"usage": ["self_signing"],
	202	"keys": {
	203	"ed25519:nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk": "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk"
	204	},
	205	},
	206	}
	207	master_signing_key = key.decode_signing_key_base64(
	208	"ed25519",
	209	"EmkqvokUn8p+vQAGZitOk4PWjp7Ukp3txV2TbMPEiBQ",
	210	"HvQBbU+hc2Zr+JP1sE0XwBe1pfZZEYtJNPJLZJtS+F8",
	211	)
	212	sign.sign_json(keys1["self_signing_key"], local_user, master_signing_key)
	213	signing_key = key.decode_signing_key_base64(
	214	"ed25519",
	215	"nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk",
	216	"2lonYOM6xYKdEsO+6KrC766xBcHnYnim1x/4LFGF8B0",
	217	)
	218	yield self.handler.upload_signing_keys_for_user(local_user, keys1)
	219
	220	# upload two device keys, which will be signed later by the self-signing key
	221	device_key_1 = {
	222	"user_id": local_user,
	223	"device_id": "abc",
	224	"algorithms": ["m.olm.curve25519-aes-sha256", "m.megolm.v1.aes-sha"],
	225	"keys": {
	226	"ed25519:abc": "base64+ed25519+key",
	227	"curve25519:abc": "base64+curve25519+key",
	228	},
	229	"signatures": {local_user: {"ed25519:abc": "base64+signature"}},
	230	}
	231	device_key_2 = {
	232	"user_id": local_user,
	233	"device_id": "def",
	234	"algorithms": ["m.olm.curve25519-aes-sha256", "m.megolm.v1.aes-sha"],
	235	"keys": {
	236	"ed25519:def": "base64+ed25519+key",
	237	"curve25519:def": "base64+curve25519+key",
	238	},
	239	"signatures": {local_user: {"ed25519:def": "base64+signature"}},
	240	}
	241
	242	yield self.handler.upload_keys_for_user(
	243	local_user, "abc", {"device_keys": device_key_1}
	244	)
	245	yield self.handler.upload_keys_for_user(
	246	local_user, "def", {"device_keys": device_key_2}
	247	)
	248
	249	# sign the first device key and upload it
	250	del device_key_1["signatures"]
	251	sign.sign_json(device_key_1, local_user, signing_key)
	252	yield self.handler.upload_signatures_for_device_keys(
	253	local_user, {local_user: {"abc": device_key_1}}
	254	)
	255
	256	# sign the second device key and upload both device keys. The server
	257	# should ignore the first device key since it already has a valid
	258	# signature for it
	259	del device_key_2["signatures"]
	260	sign.sign_json(device_key_2, local_user, signing_key)
	261	yield self.handler.upload_signatures_for_device_keys(
	262	local_user, {local_user: {"abc": device_key_1, "def": device_key_2}}
	263	)
	264
	265	device_key_1["signatures"][local_user]["ed25519:abc"] = "base64+signature"
	266	device_key_2["signatures"][local_user]["ed25519:def"] = "base64+signature"
	267	devices = yield self.handler.query_devices(
	268	{"device_keys": {local_user: []}}, 0, local_user
	269	)
	270	del devices["device_keys"][local_user]["abc"]["unsigned"]
	271	del devices["device_keys"][local_user]["def"]["unsigned"]
	272	self.assertDictEqual(devices["device_keys"][local_user]["abc"], device_key_1)
	273	self.assertDictEqual(devices["device_keys"][local_user]["def"], device_key_2)
	274
	275	@defer.inlineCallbacks
	276	def test_self_signing_key_doesnt_show_up_as_device(self):
	277	"""signing keys should be hidden when fetching a user's devices"""
	278	local_user = "@boris:" + self.hs.hostname
	279	keys1 = {
	280	"master_key": {
	281	# private key: 2lonYOM6xYKdEsO+6KrC766xBcHnYnim1x/4LFGF8B0
	282	"user_id": local_user,
	283	"usage": ["master"],
	284	"keys": {
	285	"ed25519:nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk": "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk"
	286	},
	287	}
	288	}
	289	yield self.handler.upload_signing_keys_for_user(local_user, keys1)
	290
	291	res = None
	292	try:
	293	yield self.hs.get_device_handler().check_device_registered(
	294	user_id=local_user,
	295	device_id="nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk",
	296	initial_device_display_name="new display name",
	297	)
	298	except errors.SynapseError as e:
	299	res = e.code
	300	self.assertEqual(res, 400)
	301
	302	res = yield self.handler.query_local_devices({local_user: None})
	303	self.assertDictEqual(res, {local_user: {}})
	304
	305	@defer.inlineCallbacks
	306	def test_upload_signatures(self):
	307	"""should check signatures that are uploaded"""
	308	# set up a user with cross-signing keys and a device. This user will
	309	# try uploading signatures
	310	local_user = "@boris:" + self.hs.hostname
	311	device_id = "xyz"
	312	# private key: OMkooTr76ega06xNvXIGPbgvvxAOzmQncN8VObS7aBA
	313	device_pubkey = "NnHhnqiMFQkq969szYkooLaBAXW244ZOxgukCvm2ZeY"
	314	device_key = {
	315	"user_id": local_user,
	316	"device_id": device_id,
	317	"algorithms": ["m.olm.curve25519-aes-sha256", "m.megolm.v1.aes-sha"],
	318	"keys": {"curve25519:xyz": "curve25519+key", "ed25519:xyz": device_pubkey},
	319	"signatures": {local_user: {"ed25519:xyz": "something"}},
	320	}
	321	device_signing_key = key.decode_signing_key_base64(
	322	"ed25519", "xyz", "OMkooTr76ega06xNvXIGPbgvvxAOzmQncN8VObS7aBA"
	323	)
	324
	325	yield self.handler.upload_keys_for_user(
	326	local_user, device_id, {"device_keys": device_key}
	327	)
	328
	329	# private key: 2lonYOM6xYKdEsO+6KrC766xBcHnYnim1x/4LFGF8B0
	330	master_pubkey = "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk"
	331	master_key = {
	332	"user_id": local_user,
	333	"usage": ["master"],
	334	"keys": {"ed25519:" + master_pubkey: master_pubkey},
	335	}
	336	master_signing_key = key.decode_signing_key_base64(
	337	"ed25519", master_pubkey, "2lonYOM6xYKdEsO+6KrC766xBcHnYnim1x/4LFGF8B0"
	338	)
	339	usersigning_pubkey = "Hq6gL+utB4ET+UvD5ci0kgAwsX6qP/zvf8v6OInU5iw"
	340	usersigning_key = {
	341	# private key: 4TL4AjRYwDVwD3pqQzcor+ez/euOB1/q78aTJ+czDNs
	342	"user_id": local_user,
	343	"usage": ["user_signing"],
	344	"keys": {"ed25519:" + usersigning_pubkey: usersigning_pubkey},
	345	}
	346	usersigning_signing_key = key.decode_signing_key_base64(
	347	"ed25519", usersigning_pubkey, "4TL4AjRYwDVwD3pqQzcor+ez/euOB1/q78aTJ+czDNs"
	348	)
	349	sign.sign_json(usersigning_key, local_user, master_signing_key)
	350	# private key: HvQBbU+hc2Zr+JP1sE0XwBe1pfZZEYtJNPJLZJtS+F8
	351	selfsigning_pubkey = "EmkqvokUn8p+vQAGZitOk4PWjp7Ukp3txV2TbMPEiBQ"
	352	selfsigning_key = {
	353	"user_id": local_user,
	354	"usage": ["self_signing"],
	355	"keys": {"ed25519:" + selfsigning_pubkey: selfsigning_pubkey},
	356	}
	357	selfsigning_signing_key = key.decode_signing_key_base64(
	358	"ed25519", selfsigning_pubkey, "HvQBbU+hc2Zr+JP1sE0XwBe1pfZZEYtJNPJLZJtS+F8"
	359	)
	360	sign.sign_json(selfsigning_key, local_user, master_signing_key)
	361	cross_signing_keys = {
	362	"master_key": master_key,
	363	"user_signing_key": usersigning_key,
	364	"self_signing_key": selfsigning_key,
	365	}
	366	yield self.handler.upload_signing_keys_for_user(local_user, cross_signing_keys)
	367
	368	# set up another user with a master key. This user will be signed by
	369	# the first user
	370	other_user = "@otherboris:" + self.hs.hostname
	371	other_master_pubkey = "fHZ3NPiKxoLQm5OoZbKa99SYxprOjNs4TwJUKP+twCM"
	372	other_master_key = {
	373	# private key: oyw2ZUx0O4GifbfFYM0nQvj9CL0b8B7cyN4FprtK8OI
	374	"user_id": other_user,
	375	"usage": ["master"],
	376	"keys": {"ed25519:" + other_master_pubkey: other_master_pubkey},
	377	}
	378	yield self.handler.upload_signing_keys_for_user(
	379	other_user, {"master_key": other_master_key}
	380	)
	381
	382	# test various signature failures (see below)
	383	ret = yield self.handler.upload_signatures_for_device_keys(
	384	local_user,
	385	{
	386	local_user: {
	387	# fails because the signature is invalid
	388	# should fail with INVALID_SIGNATURE
	389	device_id: {
	390	"user_id": local_user,
	391	"device_id": device_id,
	392	"algorithms": [
	393	"m.olm.curve25519-aes-sha256",
	394	"m.megolm.v1.aes-sha",
	395	],
	396	"keys": {
	397	"curve25519:xyz": "curve25519+key",
	398	# private key: OMkooTr76ega06xNvXIGPbgvvxAOzmQncN8VObS7aBA
	399	"ed25519:xyz": device_pubkey,
	400	},
	401	"signatures": {
	402	local_user: {"ed25519:" + selfsigning_pubkey: "something"}
	403	},
	404	},
	405	# fails because device is unknown
	406	# should fail with NOT_FOUND
	407	"unknown": {
	408	"user_id": local_user,
	409	"device_id": "unknown",
	410	"signatures": {
	411	local_user: {"ed25519:" + selfsigning_pubkey: "something"}
	412	},
	413	},
	414	# fails because the signature is invalid
	415	# should fail with INVALID_SIGNATURE
	416	master_pubkey: {
	417	"user_id": local_user,
	418	"usage": ["master"],
	419	"keys": {"ed25519:" + master_pubkey: master_pubkey},
	420	"signatures": {
	421	local_user: {"ed25519:" + device_pubkey: "something"}
	422	},
	423	},
	424	},
	425	other_user: {
	426	# fails because the device is not the user's master-signing key
	427	# should fail with NOT_FOUND
	428	"unknown": {
	429	"user_id": other_user,
	430	"device_id": "unknown",
	431	"signatures": {
	432	local_user: {"ed25519:" + usersigning_pubkey: "something"}
	433	},
	434	},
	435	other_master_pubkey: {
	436	# fails because the key doesn't match what the server has
	437	# should fail with UNKNOWN
	438	"user_id": other_user,
	439	"usage": ["master"],
	440	"keys": {"ed25519:" + other_master_pubkey: other_master_pubkey},
	441	"something": "random",
	442	"signatures": {
	443	local_user: {"ed25519:" + usersigning_pubkey: "something"}
	444	},
	445	},
	446	},
	447	},
	448	)
	449
	450	user_failures = ret["failures"][local_user]
	451	self.assertEqual(
	452	user_failures[device_id]["errcode"], errors.Codes.INVALID_SIGNATURE
	453	)
	454	self.assertEqual(
	455	user_failures[master_pubkey]["errcode"], errors.Codes.INVALID_SIGNATURE
	456	)
	457	self.assertEqual(user_failures["unknown"]["errcode"], errors.Codes.NOT_FOUND)
	458
	459	other_user_failures = ret["failures"][other_user]
	460	self.assertEqual(
	461	other_user_failures["unknown"]["errcode"], errors.Codes.NOT_FOUND
	462	)
	463	self.assertEqual(
	464	other_user_failures[other_master_pubkey]["errcode"], errors.Codes.UNKNOWN
	465	)
	466
	467	# test successful signatures
	468	del device_key["signatures"]
	469	sign.sign_json(device_key, local_user, selfsigning_signing_key)
	470	sign.sign_json(master_key, local_user, device_signing_key)
	471	sign.sign_json(other_master_key, local_user, usersigning_signing_key)
	472	ret = yield self.handler.upload_signatures_for_device_keys(
	473	local_user,
	474	{
	475	local_user: {device_id: device_key, master_pubkey: master_key},
	476	other_user: {other_master_pubkey: other_master_key},
	477	},
	478	)
	479
	480	self.assertEqual(ret["failures"], {})
	481
	482	# fetch the signed keys/devices and make sure that the signatures are there
	483	ret = yield self.handler.query_devices(
	484	{"device_keys": {local_user: [], other_user: []}}, 0, local_user
	485	)
	486
	487	self.assertEqual(
	488	ret["device_keys"][local_user]["xyz"]["signatures"][local_user][
	489	"ed25519:" + selfsigning_pubkey
	490	],
	491	device_key["signatures"][local_user]["ed25519:" + selfsigning_pubkey],
	492	)
	493	self.assertEqual(
	494	ret["master_keys"][local_user]["signatures"][local_user][
	495	"ed25519:" + device_id
	496	],
	497	master_key["signatures"][local_user]["ed25519:" + device_id],
	498	)
	499	self.assertEqual(
	500	ret["master_keys"][other_user]["signatures"][local_user][
	501	"ed25519:" + usersigning_pubkey
	502	],
	503	other_master_key["signatures"][local_user]["ed25519:" + usersigning_pubkey],
	504	)

+37

-22

tests/handlers/test_e2e_room_keys.py less more

186	186	self.assertEqual(res, 404)
187	187
188	188	@defer.inlineCallbacks
	189	def test_update_omitted_version(self):
	190	"""Check that the update succeeds if the version is missing from the body
	191	"""
	192	version = yield self.handler.create_version(
	193	self.local_user,
	194	{"algorithm": "m.megolm_backup.v1", "auth_data": "first_version_auth_data"},
	195	)
	196	self.assertEqual(version, "1")
	197
	198	yield self.handler.update_version(
	199	self.local_user,
	200	version,
	201	{
	202	"algorithm": "m.megolm_backup.v1",
	203	"auth_data": "revised_first_version_auth_data",
	204	},
	205	)
	206
	207	# check we can retrieve it as the current version
	208	res = yield self.handler.get_version_info(self.local_user)
	209	self.assertDictEqual(
	210	res,
	211	{
	212	"algorithm": "m.megolm_backup.v1",
	213	"auth_data": "revised_first_version_auth_data",
	214	"version": version,
	215	},
	216	)
	217
	218	@defer.inlineCallbacks
189	219	def test_update_bad_version(self):
190		"""Check that we get a 400 if the version in the body is missing or
191		doesn't match
192		"""
193		version = yield self.handler.create_version(
194		self.local_user,
195		{"algorithm": "m.megolm_backup.v1", "auth_data": "first_version_auth_data"},
196		)
197		self.assertEqual(version, "1")
198
199		res = None
200		try:
201		yield self.handler.update_version(
202		self.local_user,
203		version,
204		{
205		"algorithm": "m.megolm_backup.v1",
206		"auth_data": "revised_first_version_auth_data",
207		},
208		)
209		except errors.SynapseError as e:
210		res = e.code
211		self.assertEqual(res, 400)
	220	"""Check that we get a 400 if the version in the body doesn't match
	221	"""
	222	version = yield self.handler.create_version(
	223	self.local_user,
	224	{"algorithm": "m.megolm_backup.v1", "auth_data": "first_version_auth_data"},
	225	)
	226	self.assertEqual(version, "1")
212	227
213	228	res = None
214	229	try:

+81

-0

tests/handlers/test_federation.py less more

	0	# -- coding: utf-8 --
	1	# Copyright 2019 The Matrix.org Foundation C.I.C.
	2	#
	3	# Licensed under the Apache License, Version 2.0 (the "License");
	4	# you may not use this file except in compliance with the License.
	5	# You may obtain a copy of the License at
	6	#
	7	# http://www.apache.org/licenses/LICENSE-2.0
	8	#
	9	# Unless required by applicable law or agreed to in writing, software
	10	# distributed under the License is distributed on an "AS IS" BASIS,
	11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	# See the License for the specific language governing permissions and
	13	# limitations under the License.
	14	from synapse.api.constants import EventTypes
	15	from synapse.api.errors import AuthError, Codes
	16	from synapse.rest import admin
	17	from synapse.rest.client.v1 import login, room
	18
	19	from tests import unittest
	20
	21
	22	class FederationTestCase(unittest.HomeserverTestCase):
	23	servlets = [
	24	admin.register_servlets,
	25	login.register_servlets,
	26	room.register_servlets,
	27	]
	28
	29	def make_homeserver(self, reactor, clock):
	30	hs = self.setup_test_homeserver(http_client=None)
	31	self.handler = hs.get_handlers().federation_handler
	32	self.store = hs.get_datastore()
	33	return hs
	34
	35	def test_exchange_revoked_invite(self):
	36	user_id = self.register_user("kermit", "test")
	37	tok = self.login("kermit", "test")
	38
	39	room_id = self.helper.create_room_as(room_creator=user_id, tok=tok)
	40
	41	# Send a 3PID invite event with an empty body so it's considered as a revoked one.
	42	invite_token = "sometoken"
	43	self.helper.send_state(
	44	room_id=room_id,
	45	event_type=EventTypes.ThirdPartyInvite,
	46	state_key=invite_token,
	47	body={},
	48	tok=tok,
	49	)
	50
	51	d = self.handler.on_exchange_third_party_invite_request(
	52	room_id=room_id,
	53	event_dict={
	54	"type": EventTypes.Member,
	55	"room_id": room_id,
	56	"sender": user_id,
	57	"state_key": "@someone:example.org",
	58	"content": {
	59	"membership": "invite",
	60	"third_party_invite": {
	61	"display_name": "alice",
	62	"signed": {
	63	"mxid": "@alice:localhost",
	64	"token": invite_token,
	65	"signatures": {
	66	"magic.forest": {
	67	"ed25519:3": "fQpGIW1Snz+pwLZu6sTy2aHy/DYWWTspTJRPyNp0PKkymfIsNffysMl6ObMMFdIJhk6g6pwlIqZ54rxo8SLmAg"
	68	}
	69	},
	70	},
	71	},
	72	},
	73	},
	74	)
	75
	76	failure = self.get_failure(d, AuthError).value
	77
	78	self.assertEqual(failure.code, 403, failure)
	79	self.assertEqual(failure.errcode, Codes.FORBIDDEN, failure)
	80	self.assertEqual(failure.msg, "You are not invited to this room.")

+39

-0

tests/handlers/test_presence.py less more

21	21	from synapse.events import room_version_to_event_format
22	22	from synapse.events.builder import EventBuilder
23	23	from synapse.handlers.presence import (
	24	EXTERNAL_PROCESS_EXPIRY,
24	25	FEDERATION_PING_INTERVAL,
25	26	FEDERATION_TIMEOUT,
26	27	IDLE_TIMER,

410	411
411	412	self.assertIsNotNone(new_state)
412	413	self.assertEquals(state, new_state)
	414
	415
	416	class PresenceHandlerTestCase(unittest.HomeserverTestCase):
	417	def prepare(self, reactor, clock, hs):
	418	self.presence_handler = hs.get_presence_handler()
	419	self.clock = hs.get_clock()
	420
	421	def test_external_process_timeout(self):
	422	"""Test that if an external process doesn't update the records for a while
	423	we time out their syncing users presence.
	424	"""
	425	process_id = 1
	426	user_id = "@test:server"
	427
	428	# Notify handler that a user is now syncing.
	429	self.get_success(
	430	self.presence_handler.update_external_syncs_row(
	431	process_id, user_id, True, self.clock.time_msec()
	432	)
	433	)
	434
	435	# Check that if we wait a while without telling the handler the user has
	436	# stopped syncing that their presence state doesn't get timed out.
	437	self.reactor.advance(EXTERNAL_PROCESS_EXPIRY / 2)
	438
	439	state = self.get_success(
	440	self.presence_handler.get_state(UserID.from_string(user_id))
	441	)
	442	self.assertEqual(state.state, PresenceState.ONLINE)
	443
	444	# Check that if the external process timeout fires, then the syncing
	445	# user gets timed out
	446	self.reactor.advance(EXTERNAL_PROCESS_EXPIRY)
	447
	448	state = self.get_success(
	449	self.presence_handler.get_state(UserID.from_string(user_id))
	450	)
	451	self.assertEqual(state.state, PresenceState.OFFLINE)
413	452
414	453
415	454	class PresenceJoinTestCase(unittest.HomeserverTestCase):

+0

-39

~~tests/handlers/test_roomlist.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2018 New Vector Ltd
2		#
3		# Licensed under the Apache License, Version 2.0 (the "License");
4		# you may not use this file except in compliance with the License.
5		# You may obtain a copy of the License at
6		#
7		# http://www.apache.org/licenses/LICENSE-2.0
8		#
9		# Unless required by applicable law or agreed to in writing, software
10		# distributed under the License is distributed on an "AS IS" BASIS,
11		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12		# See the License for the specific language governing permissions and
13		# limitations under the License.
14
15		from synapse.handlers.room_list import RoomListNextBatch
16
17		import tests.unittest
18		import tests.utils
19
20
21		class RoomListTestCase(tests.unittest.TestCase):
22		""" Tests RoomList's RoomListNextBatch. """
23
24		def setUp(self):
25		pass
26
27		def test_check_read_batch_tokens(self):
28		batch_token = RoomListNextBatch(
29		stream_ordering="abcdef",
30		public_room_stream_id="123",
31		current_limit=20,
32		direction_is_forward=True,
33		).to_token()
34		next_batch = RoomListNextBatch.from_token(batch_token)
35		self.assertEquals(next_batch.stream_ordering, "abcdef")
36		self.assertEquals(next_batch.public_room_stream_id, "123")
37		self.assertEquals(next_batch.current_limit, 20)
38		self.assertEquals(next_batch.direction_is_forward, True)

+5

-5

tests/handlers/test_stats.py less more

12	12	# See the License for the specific language governing permissions and
13	13	# limitations under the License.
14	14
15		from synapse import storage
16	15	from synapse.rest import admin
17	16	from synapse.rest.client.v1 import login, room
	17	from synapse.storage.data_stores.main import stats
18	18
19	19	from tests import unittest
20	20

86	86	)
87	87
88	88	def _get_current_stats(self, stats_type, stat_id):
89		table, id_col = storage.stats.TYPE_TO_TABLE[stats_type]
90
91		cols = list(storage.stats.ABSOLUTE_STATS_FIELDS[stats_type]) + list(
92		storage.stats.PER_SLICE_FIELDS[stats_type]
	89	table, id_col = stats.TYPE_TO_TABLE[stats_type]
	90
	91	cols = list(stats.ABSOLUTE_STATS_FIELDS[stats_type]) + list(
	92	stats.PER_SLICE_FIELDS[stats_type]
93	93	)
94	94
95	95	end_ts = self.store.quantise_stats_time(self.reactor.seconds() * 1000)

+1

-1

tests/handlers/test_typing.py less more

138	138	defer.succeed(1)
139	139	)
140	140
141		self.datastore.get_current_state_deltas.return_value = None
	141	self.datastore.get_current_state_deltas.return_value = (0, None)
142	142
143	143	self.datastore.get_to_device_stream_token = lambda: 0
144	144	self.datastore.get_new_device_msgs_for_remote = lambda args, *kargs: ([], 0)

+0

-94

~~tests/patch_inline_callbacks.py~~ less more

0		# -- coding: utf-8 --
1		# Copyright 2018 New Vector Ltd
2		#
3		# Licensed under the Apache License, Version 2.0 (the "License");
4		# you may not use this file except in compliance with the License.
5		# You may obtain a copy of the License at
6		#
7		# http://www.apache.org/licenses/LICENSE-2.0
8		#
9		# Unless required by applicable law or agreed to in writing, software
10		# distributed under the License is distributed on an "AS IS" BASIS,
11		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12		# See the License for the specific language governing permissions and
13		# limitations under the License.
14
15		from __future__ import print_function
16
17		import functools
18		import sys
19
20		from twisted.internet import defer
21		from twisted.internet.defer import Deferred
22		from twisted.python.failure import Failure
23
24
25		def do_patch():
26		"""
27		Patch defer.inlineCallbacks so that it checks the state of the logcontext on exit
28		"""
29
30		from synapse.logging.context import LoggingContext
31
32		orig_inline_callbacks = defer.inlineCallbacks
33
34		def new_inline_callbacks(f):
35
36		orig = orig_inline_callbacks(f)
37
38		@functools.wraps(f)
39		def wrapped(args, *kwargs):
40		start_context = LoggingContext.current_context()
41
42		try:
43		res = orig(args, *kwargs)
44		except Exception:
45		if LoggingContext.current_context() != start_context:
46		err = "%s changed context from %s to %s on exception" % (
47		f,
48		start_context,
49		LoggingContext.current_context(),
50		)
51		print(err, file=sys.stderr)
52		raise Exception(err)
53		raise
54
55		if not isinstance(res, Deferred) or res.called:
56		if LoggingContext.current_context() != start_context:
57		err = "%s changed context from %s to %s" % (
58		f,
59		start_context,
60		LoggingContext.current_context(),
61		)
62		# print the error to stderr because otherwise all we
63		# see in travis-ci is the 500 error
64		print(err, file=sys.stderr)
65		raise Exception(err)
66		return res
67
68		if LoggingContext.current_context() != LoggingContext.sentinel:
69		err = (
70		"%s returned incomplete deferred in non-sentinel context "
71		"%s (start was %s)"
72		) % (f, LoggingContext.current_context(), start_context)
73		print(err, file=sys.stderr)
74		raise Exception(err)
75
76		def check_ctx(r):
77		if LoggingContext.current_context() != start_context:
78		err = "%s completion of %s changed context from %s to %s" % (
79		"Failure" if isinstance(r, Failure) else "Success",
80		f,
81		start_context,
82		LoggingContext.current_context(),
83		)
84		print(err, file=sys.stderr)
85		raise Exception(err)
86		return r
87
88		res.addBoth(check_ctx)
89		return res
90
91		return wrapped
92
93		defer.inlineCallbacks = new_inline_callbacks

+1

-1

tests/rest/admin/test_admin.py less more

61	61	self.device_handler.check_device_registered = Mock(return_value="FAKE")
62	62
63	63	self.datastore = Mock(return_value=Mock())
64		self.datastore.get_current_state_deltas = Mock(return_value=[])
	64	self.datastore.get_current_state_deltas = Mock(return_value=(0, []))
65	65
66	66	self.secrets = Mock()
67	67

+9

-0

tests/rest/client/v1/test_rooms.py less more

483	483	self.render(request)
484	484	self.assertEquals(400, channel.code)
485	485
	486	def test_post_room_invitees_invalid_mxid(self):
	487	# POST with invalid invitee, see https://github.com/matrix-org/synapse/issues/4088
	488	# Note the trailing space in the MXID here!
	489	request, channel = self.make_request(
	490	"POST", "/createRoom", b'{"invite":["@alice:example.com "]}'
	491	)
	492	self.render(request)
	493	self.assertEquals(400, channel.code)
	494
486	495
487	496	class RoomTopicTestCase(RoomBase):
488	497	""" Tests /rooms/$room_id/topic REST events. """

+54

-14

tests/rest/client/v2_alpha/test_account.py less more

22	22	import pkg_resources
23	23
24	24	import synapse.rest.admin
25		from synapse.api.constants import LoginType
26		from synapse.rest.client.v1 import login
	25	from synapse.api.constants import LoginType, Membership
	26	from synapse.rest.client.v1 import login, room
27	27	from synapse.rest.client.v2_alpha import account, register
28	28
29	29	from tests import unittest

243	243	synapse.rest.admin.register_servlets_for_client_rest_resource,
244	244	login.register_servlets,
245	245	account.register_servlets,
	246	room.register_servlets,
246	247	]
247	248
248	249	def make_homeserver(self, reactor, clock):
249		hs = self.setup_test_homeserver()
250		return hs
	250	self.hs = self.setup_test_homeserver()
	251	return self.hs
251	252
252	253	def test_deactivate_account(self):
253	254	user_id = self.register_user("kermit", "test")
254	255	tok = self.login("kermit", "test")
255	256
	257	self.deactivate(user_id, tok)
	258
	259	store = self.hs.get_datastore()
	260
	261	# Check that the user has been marked as deactivated.
	262	self.assertTrue(self.get_success(store.get_user_deactivated_status(user_id)))
	263
	264	# Check that this access token has been invalidated.
	265	request, channel = self.make_request("GET", "account/whoami")
	266	self.render(request)
	267	self.assertEqual(request.code, 401)
	268
	269	@unittest.INFO
	270	def test_pending_invites(self):
	271	"""Tests that deactivating a user rejects every pending invite for them."""
	272	store = self.hs.get_datastore()
	273
	274	inviter_id = self.register_user("inviter", "test")
	275	inviter_tok = self.login("inviter", "test")
	276
	277	invitee_id = self.register_user("invitee", "test")
	278	invitee_tok = self.login("invitee", "test")
	279
	280	# Make @inviter:test invite @invitee:test in a new room.
	281	room_id = self.helper.create_room_as(inviter_id, tok=inviter_tok)
	282	self.helper.invite(
	283	room=room_id, src=inviter_id, targ=invitee_id, tok=inviter_tok
	284	)
	285
	286	# Make sure the invite is here.
	287	pending_invites = self.get_success(store.get_invited_rooms_for_user(invitee_id))
	288	self.assertEqual(len(pending_invites), 1, pending_invites)
	289	self.assertEqual(pending_invites[0].room_id, room_id, pending_invites)
	290
	291	# Deactivate @invitee:test.
	292	self.deactivate(invitee_id, invitee_tok)
	293
	294	# Check that the invite isn't there anymore.
	295	pending_invites = self.get_success(store.get_invited_rooms_for_user(invitee_id))
	296	self.assertEqual(len(pending_invites), 0, pending_invites)
	297
	298	# Check that the membership of @invitee:test in the room is now "leave".
	299	memberships = self.get_success(
	300	store.get_rooms_for_user_where_membership_is(invitee_id, [Membership.LEAVE])
	301	)
	302	self.assertEqual(len(memberships), 1, memberships)
	303	self.assertEqual(memberships[0].room_id, room_id, memberships)
	304
	305	def deactivate(self, user_id, tok):
256	306	request_data = json.dumps(
257	307	{
258	308	"auth": {

268	318	)
269	319	self.render(request)
270	320	self.assertEqual(request.code, 200)
271
272		store = self.hs.get_datastore()
273
274		# Check that the user has been marked as deactivated.
275		self.assertTrue(self.get_success(store.get_user_deactivated_status(user_id)))
276
277		# Check that this access token has been invalidated.
278		request, channel = self.make_request("GET", "account/whoami")
279		self.render(request)
280		self.assertEqual(request.code, 401)

+1

-1

tests/rest/client/v2_alpha/test_filter.py less more

91	91	)
92	92	self.render(request)
93	93
94		self.assertEqual(channel.result["code"], b"400")
	94	self.assertEqual(channel.result["code"], b"404")
95	95	self.assertEquals(channel.json_body["errcode"], Codes.NOT_FOUND)
96	96
97	97	# Currently invalid params do not have an appropriate errcode

+57

-2

tests/server_notices/test_resource_limits_server_notices.py less more

16	16
17	17	from twisted.internet import defer
18	18
19		from synapse.api.constants import EventTypes, ServerNoticeMsgType
	19	from synapse.api.constants import EventTypes, LimitBlockingTypes, ServerNoticeMsgType
20	20	from synapse.api.errors import ResourceLimitError
21	21	from synapse.server_notices.resource_limits_server_notices import (
22	22	ResourceLimitsServerNotices,

132	132	self.get_success(self._rlsn.maybe_send_server_notice_to_user(self.user_id))
133	133
134	134	# Would be better to check contents, but 2 calls == set blocking event
135		self.assertTrue(self._send_notice.call_count == 2)
	135	self.assertEqual(self._send_notice.call_count, 2)
136	136
137	137	def test_maybe_send_server_notice_to_user_add_blocked_notice_noop(self):
138	138	"""

156	156	self.get_success(self._rlsn.maybe_send_server_notice_to_user(self.user_id))
157	157
158	158	self._send_notice.assert_not_called()
	159
	160	def test_maybe_send_server_notice_when_alerting_suppressed_room_unblocked(self):
	161	"""
	162	Test that when server is over MAU limit and alerting is suppressed, then
	163	an alert message is not sent into the room
	164	"""
	165	self.hs.config.mau_limit_alerting = False
	166	self._rlsn._auth.check_auth_blocking = Mock(
	167	side_effect=ResourceLimitError(
	168	403, "foo", limit_type=LimitBlockingTypes.MONTHLY_ACTIVE_USER
	169	)
	170	)
	171	self.get_success(self._rlsn.maybe_send_server_notice_to_user(self.user_id))
	172
	173	self.assertTrue(self._send_notice.call_count == 0)
	174
	175	def test_check_hs_disabled_unaffected_by_mau_alert_suppression(self):
	176	"""
	177	Test that when a server is disabled, that MAU limit alerting is ignored.
	178	"""
	179	self.hs.config.mau_limit_alerting = False
	180	self._rlsn._auth.check_auth_blocking = Mock(
	181	side_effect=ResourceLimitError(
	182	403, "foo", limit_type=LimitBlockingTypes.HS_DISABLED
	183	)
	184	)
	185	self.get_success(self._rlsn.maybe_send_server_notice_to_user(self.user_id))
	186
	187	# Would be better to check contents, but 2 calls == set blocking event
	188	self.assertEqual(self._send_notice.call_count, 2)
	189
	190	def test_maybe_send_server_notice_when_alerting_suppressed_room_blocked(self):
	191	"""
	192	When the room is already in a blocked state, test that when alerting
	193	is suppressed that the room is returned to an unblocked state.
	194	"""
	195	self.hs.config.mau_limit_alerting = False
	196	self._rlsn._auth.check_auth_blocking = Mock(
	197	side_effect=ResourceLimitError(
	198	403, "foo", limit_type=LimitBlockingTypes.MONTHLY_ACTIVE_USER
	199	)
	200	)
	201	self._rlsn._server_notices_manager.__is_room_currently_blocked = Mock(
	202	return_value=defer.succeed((True, []))
	203	)
	204
	205	mock_event = Mock(
	206	type=EventTypes.Message, content={"msgtype": ServerNoticeMsgType}
	207	)
	208	self._rlsn._store.get_events = Mock(
	209	return_value=defer.succeed({"123": mock_event})
	210	)
	211	self.get_success(self._rlsn.maybe_send_server_notice_to_user(self.user_id))
	212
	213	self._send_notice.assert_called_once()
159	214
160	215
161	216	class TestResourceLimitsServerNoticesWithRealRooms(unittest.HomeserverTestCase):

+1

-1

tests/storage/test_appservice.py less more

23	23
24	24	from synapse.appservice import ApplicationService, ApplicationServiceState
25	25	from synapse.config._base import ConfigError
26		from synapse.storage.appservice import (
	26	from synapse.storage.data_stores.main.appservice import (
27	27	ApplicationServiceStore,
28	28	ApplicationServiceTransactionStore,
29	29	)

+2

-0

tests/storage/test_cleanup_extrems.py less more

49	49
50	50	schema_path = os.path.join(
51	51	prepare_database.dir_path,
	52	"data_stores",
	53	"main",
52	54	"schema",
53	55	"delta",
54	56	"54",

+6

-6

tests/storage/test_end_to_end_keys.py less more

37	37	self.assertIn("user", res)
38	38	self.assertIn("device", res["user"])
39	39	dev = res["user"]["device"]
40		self.assertDictContainsSubset({"keys": json, "device_display_name": None}, dev)
	40	self.assertDictContainsSubset(json, dev)
41	41
42	42	@defer.inlineCallbacks
43	43	def test_reupload_key(self):

67	67	self.assertIn("device", res["user"])
68	68	dev = res["user"]["device"]
69	69	self.assertDictContainsSubset(
70		{"keys": json, "device_display_name": "display_name"}, dev
	70	{"key": "value", "unsigned": {"device_display_name": "display_name"}}, dev
71	71	)
72	72
73	73	@defer.inlineCallbacks

79	79	yield self.store.store_device("user2", "device1", None)
80	80	yield self.store.store_device("user2", "device2", None)
81	81
82		yield self.store.set_e2e_device_keys("user1", "device1", now, "json11")
83		yield self.store.set_e2e_device_keys("user1", "device2", now, "json12")
84		yield self.store.set_e2e_device_keys("user2", "device1", now, "json21")
85		yield self.store.set_e2e_device_keys("user2", "device2", now, "json22")
	82	yield self.store.set_e2e_device_keys("user1", "device1", now, {"key": "json11"})
	83	yield self.store.set_e2e_device_keys("user1", "device2", now, {"key": "json12"})
	84	yield self.store.set_e2e_device_keys("user2", "device1", now, {"key": "json21"})
	85	yield self.store.set_e2e_device_keys("user2", "device2", now, {"key": "json22"})
86	86
87	87	res = yield self.store.get_e2e_device_keys(
88	88	(("user1", "device1"), ("user2", "device2"))

+1

-1

tests/storage/test_event_federation.py less more

56	56	"(event_id, algorithm, hash) "
57	57	"VALUES (?, 'sha256', ?)"
58	58	),
59		(event_id, b"ffff"),
	59	(event_id, bytearray(b"ffff")),
60	60	)
61	61
62	62	for i in range(0, 11):

+49

-9

tests/storage/test_monthly_active_users.py less more

49	49	{"medium": "email", "address": user2_email},
50	50	{"medium": "email", "address": user3_email},
51	51	]
	52	self.hs.config.mau_limits_reserved_threepids = threepids
52	53	# -1 because user3 is a support user and does not count
53	54	user_num = len(threepids) - 1
54	55

83	84	self.hs.config.max_mau_value = 0
84	85
85	86	self.reactor.advance(FORTY_DAYS)
	87	self.hs.config.max_mau_value = 5
86	88
87	89	self.store.reap_monthly_active_users()
88	90	self.pump()

146	148	self.store.reap_monthly_active_users()
147	149	self.pump()
148	150	count = self.store.get_monthly_active_count()
149		self.assertEquals(
150		self.get_success(count), initial_users - self.hs.config.max_mau_value
151		)
	151	self.assertEquals(self.get_success(count), self.hs.config.max_mau_value)
152	152
153	153	self.reactor.advance(FORTY_DAYS)
154	154	self.store.reap_monthly_active_users()

156	156
157	157	count = self.store.get_monthly_active_count()
158	158	self.assertEquals(self.get_success(count), 0)
	159
	160	def test_reap_monthly_active_users_reserved_users(self):
	161	""" Tests that reaping correctly handles reaping where reserved users are
	162	present"""
	163
	164	self.hs.config.max_mau_value = 5
	165	initial_users = 5
	166	reserved_user_number = initial_users - 1
	167	threepids = []
	168	for i in range(initial_users):
	169	user = "@user%d:server" % i
	170	email = "user%d@example.com" % i
	171	self.get_success(self.store.upsert_monthly_active_user(user))
	172	threepids.append({"medium": "email", "address": email})
	173	# Need to ensure that the most recent entries in the
	174	# monthly_active_users table are reserved
	175	now = int(self.hs.get_clock().time_msec())
	176	if i != 0:
	177	self.get_success(
	178	self.store.register_user(user_id=user, password_hash=None)
	179	)
	180	self.get_success(
	181	self.store.user_add_threepid(user, "email", email, now, now)
	182	)
	183
	184	self.hs.config.mau_limits_reserved_threepids = threepids
	185	self.store.runInteraction(
	186	"initialise", self.store._initialise_reserved_users, threepids
	187	)
	188	count = self.store.get_monthly_active_count()
	189	self.assertTrue(self.get_success(count), initial_users)
	190
	191	users = self.store.get_registered_reserved_users()
	192	self.assertEquals(len(self.get_success(users)), reserved_user_number)
	193
	194	self.get_success(self.store.reap_monthly_active_users())
	195	count = self.store.get_monthly_active_count()
	196	self.assertEquals(self.get_success(count), self.hs.config.max_mau_value)
159	197
160	198	def test_populate_monthly_users_is_guest(self):
161	199	# Test that guest users are not added to mau list

191	229
192	230	def test_get_reserved_real_user_account(self):
193	231	# Test no reserved users, or reserved threepids
194		count = self.store.get_registered_reserved_users_count()
195		self.assertEquals(self.get_success(count), 0)
	232	users = self.get_success(self.store.get_registered_reserved_users())
	233	self.assertEquals(len(users), 0)
196	234	# Test reserved users but no registered users
197	235
198	236	user1 = "@user1:example.com"
199	237	user2 = "@user2:example.com"
	238
200	239	user1_email = "user1@example.com"
201	240	user2_email = "user2@example.com"
202	241	threepids = [

209	248	)
210	249
211	250	self.pump()
212		count = self.store.get_registered_reserved_users_count()
213		self.assertEquals(self.get_success(count), 0)
	251	users = self.get_success(self.store.get_registered_reserved_users())
	252	self.assertEquals(len(users), 0)
214	253
215	254	# Test reserved registed users
216	255	self.store.register_user(user_id=user1, password_hash=None)

220	259	now = int(self.hs.get_clock().time_msec())
221	260	self.store.user_add_threepid(user1, "email", user1_email, now, now)
222	261	self.store.user_add_threepid(user2, "email", user2_email, now, now)
223		count = self.store.get_registered_reserved_users_count()
224		self.assertEquals(self.get_success(count), len(threepids))
	262
	263	users = self.get_success(self.store.get_registered_reserved_users())
	264	self.assertEquals(len(users), len(threepids))
225	265
226	266	def test_support_user_not_add_to_mau_limits(self):
227	267	support_user_id = "@support:test"

+1

-1

tests/storage/test_profile.py less more

15	15
16	16	from twisted.internet import defer
17	17
18		from synapse.storage.profile import ProfileStore
	18	from synapse.storage.data_stores.main.profile import ProfileStore
19	19	from synapse.types import UserID
20	20
21	21	from tests import unittest

+1

-1

tests/storage/test_user_directory.py less more

14	14
15	15	from twisted.internet import defer
16	16
17		from synapse.storage import UserDirectoryStore
	17	from synapse.storage.data_stores.main.user_directory import UserDirectoryStore
18	18
19	19	from tests import unittest
20	20	from tests.utils import setup_test_homeserver

+2

-11

tests/utils.py less more

37	37	from synapse.server import HomeServer
38	38	from synapse.storage import DataStore
39	39	from synapse.storage.engines import PostgresEngine, create_engine
40		from synapse.storage.prepare_database import (
41		_get_or_create_schema_state,
42		_setup_new_database,
43		prepare_database,
44		)
	40	from synapse.storage.prepare_database import prepare_database
45	41	from synapse.util.ratelimitutils import FederationRateLimiter
46	42
47	43	# set this to True to run the tests against postgres instead of sqlite.

87	83	host=POSTGRES_HOST,
88	84	password=POSTGRES_PASSWORD,
89	85	)
90		cur = db_conn.cursor()
91		_get_or_create_schema_state(cur, db_engine)
92		_setup_new_database(cur, db_engine)
93		db_conn.commit()
94		cur.close()
	86	prepare_database(db_conn, db_engine, None)
95	87	db_conn.close()
96	88
97	89	def _cleanup():

144	136	"limit_usage_by_mau": False,
145	137	"hs_disabled": False,
146	138	"hs_disabled_message": "",
147		"hs_disabled_limit_type": "",
148	139	"max_mau_value": 50,
149	140	"mau_trial_days": 0,
150	141	"mau_stats_only": False,

+3

-3

tox.ini less more

117	117	commands =
118	118	python -m black --check --diff .
119	119	/bin/sh -c "flake8 synapse tests scripts scripts-dev scripts/hash_password scripts/register_new_matrix_user scripts/synapse_port_db synctl {env:PEP8SUFFIX:}"
	120	{toxinidir}/scripts-dev/config-lint.sh
120	121
121	122	[testenv:check_isort]
122	123	skip_install = True

160	161	skip_install = True
161	162	deps =
162	163	{[base]deps}
163		mypy
	164	mypy==0.730
164	165	mypy-zope
165		typeshed
166	166	env =
167	167	MYPYPATH = stubs/
168	168	extras = all
169		commands = mypy --show-traceback \
	169	commands = mypy --show-traceback --check-untyped-defs --show-error-codes --follow-imports=normal \
170	170	synapse/logging/ \
171	171	synapse/config/