matrix: Update synapse to 0.18.0

Add federation_reader as a another worker.

roles/matrix/defaults/main.yml

+---
+
+matrix_nginx_config:
+  federation_reader:
+    port: 8011
+    locations:
+      - "~ ^/_matrix/federation/v1/event/"
+      - "~ ^/_matrix/federation/v1/state/"
+      - "~ ^/_matrix/federation/v1/state_ids/"
+      - "~ ^/_matrix/federation/v1/backfill/"
+      - "~ ^/_matrix/federation/v1/get_missing_events/"
+      - "~ ^/_matrix/federation/v1/publicRooms"
+      - "~ ^/_matrix/federation/v1/query/"
+      - "~ ^/_matrix/federation/v1/make_join/"
+      - "~ ^/_matrix/federation/v1/make_leave/"
+      - "~ ^/_matrix/federation/v1/send_join/"
+      - "~ ^/_matrix/federation/v2/send_join/"
+      - "~ ^/_matrix/federation/v1/send_leave/"
+      - "~ ^/_matrix/federation/v2/send_leave/"
+      - "~ ^/_matrix/federation/v1/invite/"
+      - "~ ^/_matrix/federation/v2/invite/"
+      - "~ ^/_matrix/federation/v1/query_auth/"
+      - "~ ^/_matrix/federation/v1/event_auth/"
+      - "~ ^/_matrix/federation/v1/exchange_third_party_invite/"
+      - "~ ^/_matrix/federation/v1/user/devices/"
+      - "~ ^/_matrix/federation/v1/get_groups_publicised$"
+      - "~ ^/_matrix/key/v2/query"
+      - "~ ^/_matrix/federation/v1/send/"  # Don't load balance
+  media_repository:
+    port: 8013
+    locations:
+      - "~ ^/_matrix/media/"
+      - "~ ^/_synapse/admin/v1/purge_media_cache$"
+      - "~ ^/_synapse/admin/v1/room/.*/media.*$"
+      - "~ ^/_synapse/admin/v1/user/.*/media.*$"
+      - "~ ^/_synapse/admin/v1/media/.*$"
+      - "~ ^/_synapse/admin/v1/quarantine_media/.*$"
+  homeserver:
+    port: 8008
+    locations:
+      - "/_matrix"
+      - "/_synapse"

roles/matrix/files/synapse-worker@.service

@@ -2,6 +2,7 @@
 Description=Synapse Matrix worker (%i)
 BindsTo=synapse.service
 After=synapse.service
+AssertPathExists=/etc/synapse/worker-%i.yaml
 
 [Service]
 Type=notify
@@ -12,8 +13,8 @@ Restart=on-abort
 User=synapse
 
 WorkingDirectory=~
-ExecStart=/var/lib/synapse/venv/bin/python -m synapse.app.%i --config-path=/etc/synapse/homeserver.yaml --config-path=/etc/synapse/worker-%i.yaml
-SyslogIdentifier=synapse-worker-%i
+ExecStart=/var/lib/synapse/venv/bin/python -m synapse.app.generic_worker --config-path=/etc/synapse/homeserver.yaml --config-path=/etc/synapse/worker-%i.yaml
+SyslogIdentifier=synapse
 
 Environment=LD_PRELOAD=/usr/lib/libjemalloc.so
 

roles/matrix/files/synapse.service

 [Unit]
 Description=Synapse Matrix homeserver
-Requires=postgresql.service
+Requires=postgresql.service redis.service
 After=network.target postgresql.service
 
 [Service]

roles/matrix/files/worker-appservice.yaml

+worker_name: appservice
 worker_app: synapse.app.appservice
 
 # vim:ft=yaml sw=2 et:

roles/matrix/files/worker-federation_reader.yaml

+worker_name: federation_reader
+worker_app: synapse.app.generic_worker
+worker_listeners:
+  - port: 8011
+    type: http
+    x_forwarded: true
+    bind_addresses: ['::1', '127.0.0.1']
+    resources:
+      - names: [federation]
+        compress: false
+
+# vim:ft=yaml sw=2 et:

roles/matrix/files/worker-federation_sender.yaml

+worker_name: federation_sender
 worker_app: synapse.app.federation_sender
 
 # vim:ft=yaml sw=2 et:

roles/matrix/files/worker-media_repository.yaml

+worker_name: media_repository
 worker_app: synapse.app.media_repository
 worker_listeners:
   - port: 8013

roles/matrix/tasks/main.yml

@@ -24,6 +24,7 @@
       - postgresql-libs
       - python
       - python2
+      - redis
       - tcl
       - tk
       - zlib
@@ -51,7 +52,7 @@
 - name: install synapse
   pip:
     name:
-      - 'matrix-synapse[postgres,systemd,url_preview]'
+      - 'matrix-synapse[postgres,systemd,url_preview,redis]'
       - pip
     state: latest
     extra_args: '-U --upgrade-strategy=eager'
@@ -118,6 +119,7 @@
   with_items:
     - log_config.yaml
     - worker-appservice.yaml
+    - worker-federation_reader.yaml
     - worker-federation_sender.yaml
     - worker-media_repository.yaml
   notify:
@@ -179,6 +181,7 @@
   with_items:
     - synapse.service
     - synapse-worker@appservice.service
+    - synapse-worker@federation_reader.service
     - synapse-worker@federation_sender.service
     - synapse-worker@media_repository.service
   notify:

roles/matrix/templates/homeserver.yaml.j2

 # vim:ft=yaml sw=2 et:
 
-# Worker config
-worker_app: synapse.app.homeserver
-worker_log_config: "/etc/synapse/log_config.yaml"
-worker_replication_host: 127.0.0.1
-worker_replication_port: 9092
-worker_replication_http_port: 9093
-
-# Delegate to federation_sender worker
-send_federation: false
-
-# Delegate to appservice worker
-notify_appservices: false
-
 ################################################################################
 
 # Configuration file for Synapse.
@@ -105,7 +92,9 @@ allow_public_rooms_over_federation: true
 #gc_thresholds: [700, 10, 10]
 
 # Set the limit on the returned events in the timeline in the get
-# and sync operations. The default value is -1, means no upper limit.
+# and sync operations. The default value is 100. -1 means no upper limit.
+#
+# Uncomment the following to increase the limit to 5000.
 #
 #filter_timeline_limit: 5000
 
@@ -121,38 +110,6 @@ allow_public_rooms_over_federation: true
 #
 #enable_search: false
 
-# Restrict federation to the following whitelist of domains.
-# N.B. we recommend also firewalling your federation listener to limit
-# inbound federation traffic as early as possible, rather than relying
-# purely on this application-layer restriction.  If not specified, the
-# default is to whitelist everything.
-#
-#federation_domain_whitelist:
-#  - lon.example.com
-#  - nyc.example.com
-#  - syd.example.com
-
-# Prevent federation requests from being sent to the following
-# blacklist IP address CIDR ranges. If this option is not specified, or
-# specified with an empty list, no ip range blacklist will be enforced.
-#
-# As of Synapse v1.4.0 this option also affects any outbound requests to identity
-# servers provided by user input.
-#
-# (0.0.0.0 and :: are always blacklisted, whether or not they are explicitly
-# listed here, since they correspond to unroutable addresses.)
-#
-federation_ip_range_blacklist:
-  - '127.0.0.0/8'
-  - '10.0.0.0/8'
-  - '172.16.0.0/12'
-  - '192.168.0.0/16'
-  - '100.64.0.0/10'
-  - '169.254.0.0/16'
-  - '::1/128'
-  - 'fe80::/64'
-  - 'fc00::/7'
-
 # List of ports that Synapse should listen on, their purpose and their
 # configuration.
 #
@@ -181,7 +138,7 @@ federation_ip_range_blacklist:
 #       names: a list of names of HTTP resources. See below for a list of
 #           valid resource names.
 #
-#       compress: set to true to enable HTTP comression for this resource.
+#       compress: set to true to enable HTTP compression for this resource.
 #
 #   additional_resources: Only valid for an 'http' listener. A map of
 #        additional endpoints which should be loaded via dynamic modules.
@@ -249,10 +206,6 @@ listeners:
     #    module: my_module.CustomRequestHandler
     #    config: {}
 
-  - port: 9092
-    type: replication
-    bind_addresses: ['::1', '127.0.0.1']
-
   - port: 9093
     type: http
     bind_addresses: ['::1', '127.0.0.1']
@@ -622,6 +575,39 @@ acme:
 
 
 
+# Restrict federation to the following whitelist of domains.
+# N.B. we recommend also firewalling your federation listener to limit
+# inbound federation traffic as early as possible, rather than relying
+# purely on this application-layer restriction.  If not specified, the
+# default is to whitelist everything.
+#
+#federation_domain_whitelist:
+#  - lon.example.com
+#  - nyc.example.com
+#  - syd.example.com
+
+# Prevent federation requests from being sent to the following
+# blacklist IP address CIDR ranges. If this option is not specified, or
+# specified with an empty list, no ip range blacklist will be enforced.
+#
+# As of Synapse v1.4.0 this option also affects any outbound requests to identity
+# servers provided by user input.
+#
+# (0.0.0.0 and :: are always blacklisted, whether or not they are explicitly
+# listed here, since they correspond to unroutable addresses.)
+#
+federation_ip_range_blacklist:
+  - '127.0.0.0/8'
+  - '10.0.0.0/8'
+  - '172.16.0.0/12'
+  - '192.168.0.0/16'
+  - '100.64.0.0/10'
+  - '169.254.0.0/16'
+  - '::1/128'
+  - 'fe80::/64'
+  - 'fc00::/7'
+
+
 ## Caching ##
 
 # Caching can be configured through the following options.
@@ -696,7 +682,7 @@ caches:
 #database:
 #  name: psycopg2
 #  args:
-#    user: synapse
+#    user: synapse_user
 #    password: secretpassword
 #    database: synapse
 #    host: localhost
@@ -721,6 +707,7 @@ database:
 # https://docs.python.org/3.7/library/logging.config.html#configuration-dictionary-schema
 #
 log_config: "/etc/synapse/log_config.yaml"
+worker_log_config: "/etc/synapse/log_config.yaml"
 
 
 ## Ratelimiting ##
@@ -1828,6 +1815,9 @@ sso:
 # Each JSON Web Token needs to contain a "sub" (subject) claim, which is
 # used as the localpart of the mxid.
 #
+# Additionally, the expiration time ("exp"), not before time ("nbf"),
+# and issued at ("iat") claims are validated if present.
+#
 # Note that this is a non-standard login type and client support is
 # expected to be non-existant.
 #
@@ -1855,6 +1845,24 @@ sso:
     #
     #algorithm: "provided-by-your-issuer"
 
+    # The issuer to validate the "iss" claim against.
+    #
+    # Optional, if provided the "iss" claim will be required and
+    # validated for all JSON web tokens.
+    #
+    #issuer: "provided-by-your-issuer"
+
+    # A list of audiences to validate the "aud" claim against.
+    #
+    # Optional, if provided the "aud" claim will be required and
+    # validated for all JSON web tokens.
+    #
+    # Note that if the "aud" claim is included in a JSON web token then
+    # validation will fail without configuring audiences.
+    #
+    #audiences:
+    #    - "provided-by-your-issuer"
+
 
 password_config:
    # Uncomment to disable password login
@@ -1944,8 +1952,8 @@ email:
   #
   notif_from: "Arch Linux %(app)s server <noreply@archlinux.org>"
 
-  # app_name defines the default value for '%(app)s' in notif_from. It
-  # defaults to 'Matrix'.
+  # app_name defines the default value for '%(app)s' in notif_from and email
+  # subjects. It defaults to 'Matrix'.
   #
   #app_name: my_branded_matrix_server
 
@@ -2014,6 +2022,73 @@ email:
   #
   #template_dir: "res/templates"
 
+  # Subjects to use when sending emails from Synapse.
+  #
+  # The placeholder '%(app)s' will be replaced with the value of the 'app_name'
+  # setting above, or by a value dictated by the Matrix client application.
+  #
+  # If a subject isn't overridden in this configuration file, the value used as
+  # its example will be used.
+  #
+  #subjects:
+
+    # Subjects for notification emails.
+    #
+    # On top of the '%(app)s' placeholder, these can use the following
+    # placeholders:
+    #
+    #   * '%(person)s', which will be replaced by the display name of the user(s)
+    #      that sent the message(s), e.g. "Alice and Bob".
+    #   * '%(room)s', which will be replaced by the name of the room the
+    #      message(s) have been sent to, e.g. "My super room".
+    #
+    # See the example provided for each setting to see which placeholder can be
+    # used and how to use them.
+    #
+    # Subject to use to notify about one message from one or more user(s) in a
+    # room which has a name.
+    #message_from_person_in_room: "[%(app)s] You have a message on %(app)s from %(person)s in the %(room)s room..."
+    #
+    # Subject to use to notify about one message from one or more user(s) in a
+    # room which doesn't have a name.
+    #message_from_person: "[%(app)s] You have a message on %(app)s from %(person)s..."
+    #
+    # Subject to use to notify about multiple messages from one or more users in
+    # a room which doesn't have a name.
+    #messages_from_person: "[%(app)s] You have messages on %(app)s from %(person)s..."
+    #
+    # Subject to use to notify about multiple messages in a room which has a
+    # name.
+    #messages_in_room: "[%(app)s] You have messages on %(app)s in the %(room)s room..."
+    #
+    # Subject to use to notify about multiple messages in multiple rooms.
+    #messages_in_room_and_others: "[%(app)s] You have messages on %(app)s in the %(room)s room and others..."
+    #
+    # Subject to use to notify about multiple messages from multiple persons in
+    # multiple rooms. This is similar to the setting above except it's used when
+    # the room in which the notification was triggered has no name.
+    #messages_from_person_and_others: "[%(app)s] You have messages on %(app)s from %(person)s and others..."
+    #
+    # Subject to use to notify about an invite to a room which has a name.
+    #invite_from_person_to_room: "[%(app)s] %(person)s has invited you to join the %(room)s room on %(app)s..."
+    #
+    # Subject to use to notify about an invite to a room which doesn't have a
+    # name.
+    #invite_from_person: "[%(app)s] %(person)s has invited you to chat on %(app)s..."
+
+    # Subject for emails related to account administration.
+    #
+    # On top of the '%(app)s' placeholder, these one can use the
+    # '%(server_name)s' placeholder, which will be replaced by the value of the
+    # 'server_name' setting in your Synapse configuration.
+    #
+    # Subject to use when sending a password reset email.
+    #password_reset: "[%(server_name)s] Password reset"
+    #
+    # Subject to use when sending a verification email to assert an address's
+    # ownership.
+    #email_validation: "[%(server_name)s] Validate your email"
+
 
 # Password providers allow homeserver administrators to integrate
 # their Synapse installation with existing authentication methods
@@ -2324,3 +2399,62 @@ opentracing:
     #
     #  logging:
     #    false
+
+
+## Workers ##
+
+worker_app: synapse.app.homeserver
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
+
+# Disables sending of outbound federation transactions on the main process.
+# Uncomment if using a federation sender worker.
+#
+send_federation: false
+notify_appservices: false
+
+# It is possible to run multiple federation sender workers, in which case the
+# work is balanced across them.
+#
+# This configuration must be shared between all federation sender workers, and if
+# changed all federation sender workers must be stopped at the same time and then
+# started, to ensure that all instances are running with the same config (otherwise
+# events may be dropped).
+#
+federation_sender_instances:
+  - federation_sender
+
+# When using workers this should be a map from `worker_name` to the
+# HTTP replication listener of the worker, if configured.
+#
+#instance_map:
+#  worker1:
+#    host: localhost
+#    port: 8034
+
+# Experimental: When using workers you can define which workers should
+# handle event persistence and typing notifications. Any worker
+# specified here must also be in the `instance_map`.
+#
+#stream_writers:
+#  events: worker1
+#  typing: worker1
+
+
+# Configuration for Redis when using workers. This *must* be enabled when
+# using workers (unless using old style direct TCP configuration).
+#
+redis:
+  # Uncomment the below to enable Redis support.
+  #
+  enabled: true
+
+  # Optional host and port to use to connect to redis. Defaults to
+  # localhost and 6379
+  #
+  #host: localhost
+  #port: 6379
+
+  # Optional password if configured on the Redis instance
+  #
+  #password: <secret_password>

roles/matrix/templates/nginx.d.conf.j2

-upstream matrix {
-    server 127.0.0.1:8008;
-}
-
-upstream matrix_media_repository {
-    server 127.0.0.1:8013;
+{% for name, config in matrix_nginx_config.items() %}
+upstream matrix_{{ name }} {
+    server 127.0.0.1:{{ config.port }};
 }
 
+{% endfor %}
 server {
     listen       80;
     listen       [::]:80;
@@ -34,20 +32,18 @@ server {
     ssl_certificate_key  /etc/letsencrypt/live/{{ matrix_domain }}/privkey.pem;
     ssl_trusted_certificate /etc/letsencrypt/live/{{ matrix_domain }}/chain.pem;
 
-    location /_matrix/media {
-        access_log   /var/log/nginx/{{ matrix_domain }}/access.log main;
-        proxy_pass http://matrix_media_repository;
-        proxy_set_header X-Forwarded-For $remote_addr;
-        proxy_read_timeout 900s;
-    }
-
-    location /_matrix {
+{% for name, config in matrix_nginx_config.items() %}
+    # {{ name }}
+{% for location in config.locations %}
+    location {{ location }} {
         access_log   /var/log/nginx/{{ matrix_domain }}/access.log main;
-        proxy_pass http://matrix;
+        proxy_pass http://matrix_{{ name }};
         proxy_set_header X-Forwarded-For $remote_addr;
         proxy_read_timeout 900s;
     }
+{% endfor %}
 
+{% endfor %}
     location = / {
         add_header Content-Type text/plain;
         return 200 "Nothing to see here.";