Rework Synapse into multiple workers and Redis

Signed-off-by: Magic_RB <magic_rb@redalder.org>

containers/conduit.nix

@@ -1,4 +1,106 @@
 { nglib, nixpkgs }:
+let
+  logConfig = pkgs: (pkgs.formats.yaml {}).generate "log.yaml"
+    {
+      version = 1;
+
+      formatters.precise.format = "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s";
+      handlers.console =
+        {
+          class = "logging.StreamHandler";
+          formatter = "precise";
+        };
+      loggers."synapse.storage.SQL" =
+        {
+          level = "INFO";
+        };
+      root =
+        {
+          level = "INFO";
+          handlers = [ "console" ];
+        };
+
+      disable_existing_loggers = false;
+    };
+
+  commonConfig = pkgs: (pkgs.formats.yaml {}).generate "common.yaml"
+    {
+      server_name = "matrix.redalder.org";
+      report_stats = "yes";
+      pid_file = "/homeserver.pid";
+
+      log_config = logConfig pkgs;
+
+      trusted_key_servers =
+        [
+          {
+            server_name = "matrix.org";
+          }
+        ];
+      media_store_path = "/var/lib/synapse/media_store";
+      signing_key_path = "/var/lib/synapse/signing.key";
+
+      enable_registration = false;
+      enable_registration_without_verification = false;
+
+      federation_sender_instances = [
+        "worker-federation-sender-0"
+      ];
+    };
+
+  genericWorker = { listener_resources, name }:
+    nglib.makeSystem {
+      system = "x86_64-linux";
+      name = "synapse-worker-${name}";
+      inherit nixpkgs;
+      config = ({ pkgs, ... }:
+        {
+          dumb-init = {
+            enable = true;
+            type.services = { };
+          };
+
+          services.synapse.workers.${name} = {
+            settings = {
+              worker_app = "synapse.app.generic_worker";
+
+              # The replication listener on the main synapse process.
+              worker_replication_host = "127.0.0.1";
+              worker_replication_http_port = 9093;
+
+              worker_listeners = [
+                {
+                  port = 6167;
+                  tls = false;
+                  type = "http";
+                  x_forwarded = true;
+                  bind_adrresses = [ "0.0.0.0" ];
+                  resources =
+                    [
+                      {
+                        names = listener_resources;
+                        compress = false;
+                      }
+                    ];
+                }
+              ];
+
+              worker_log_config = logConfig pkgs;
+            };
+            arguments = {
+              config-path = [
+                (commonConfig pkgs)
+                "/secrets/extra.yaml"
+                "/var/lib/registrations/extra.yaml"
+              ];
+              keys-directory = [
+                "/var/lib/synapse/keys"
+              ];
+            };
+          };
+        });
+    };
+in
 {
   postgresql = nglib.makeSystem {
     system = "x86_64-linux";
@@ -33,6 +135,56 @@
       };
   };
 
+  redis = nglib.makeSystem {
+    system = "x86_64-linux";
+    name = "redis";
+    inherit nixpkgs;
+    config = ({ pkgs, ... }:
+      {
+        dumb-init = {
+          enable = true;
+          type.services = { };
+        };
+
+        users.users."redis" = {
+          home = "/var/empty";
+          uid = 9001;
+          group = "redis";
+        };
+
+        users.groups."redis" = {
+          gid = 9001;
+        };
+
+        init.services.redis = {
+          enabled = true;
+          shutdownOnExit = true;
+          script =
+            pkgs.writeShellScript "redis-run" ''
+              cd /var/lib/redis
+              chpst -U redis:redis ${pkgs.redis}/bin/redis-server ${./redis.conf}
+            '';
+        };
+
+        init.services.redis-setup = {
+          enabled = true;
+          script =
+            pkgs.writeShellScript "redis-run" ''
+              export PATH="${pkgs.redis}/bin:$PATH"
+              nc -z 127.0.0.1 6379 -w 10 -v || exit 1
+
+              redis-cli acl setuser default on '>'"$(cat /secrets/redis_password)" allcommands allkeys
+              sleep 86400
+            '';
+        };
+      });
+  };
+
+  synapseFederationSender = genericWorker { name = "generic"; listener_resources = [ "health" ]; };
+  synapseFederationReceiver = genericWorker { name = "generic"; listener_resources = [ "health" "federation" ]; };
+  synapseClient = genericWorker { name = "generic"; listener_resources = [ "client" "health" ]; };
+  synapseSync = genericWorker { name = "generic"; listener_resources = [ "client" "health" ]; };
+
   synapse = nglib.makeSystem {
     system = "x86_64-linux";
     name = "synapse";
@@ -49,56 +201,27 @@
           shutdownOnExit = true;
           script =
             let
-              logConfig = (pkgs.formats.yaml {}).generate "log.yaml"
-                {
-                  # Log configuration for Synapse.
-                  #
-                  # This is a YAML file containing a standard Python logging configuration
-                  # dictionary. See [1] for details on the valid settings.
-                  #
-                  # Synapse also supports structured logging for machine readable logs which can
-                  # be ingested by ELK stacks. See [2] for details.
-                  #
-                  # [1]: https://docs.python.org/3.7/library/logging.config.html#configuration-dictionary-schema
-                  # [2]: https://matrix-org.github.io/synapse/latest/structured_logging.html
-
-                  version = 1;
-
-                  formatters.precise.format = "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s";
-                  handlers.console =
-                    {
-                      class = "logging.StreamHandler";
-                      formatter = "precise";
-                    };
-                  loggers."synapse.storage.SQL" =
-                    {
-                      level = "INFO";
-                    };
-                  root =
-                    {
-                      level = "INFO";
-                      handlers = [ "console" ];
-                    };
-
-                  disable_existing_loggers = false;
-                };
               synapseConfig = (pkgs.formats.yaml {}).generate "synapse.yaml"
                 {
-                  server_name = "matrix.redalder.org";
-                  report_stats = "yes";
-                  pid_file = "/homeserver.pid";
-
-                  enable_registration = false;
-                  enable_registration_without_verification = false;
-
                   listeners =
                     [
+                      # The HTTP replication port
+                      {
+                        port = 9093;
+                        bind_addresses = [ "0.0.0.0" ];
+                        type = "http";
+                        resources = [
+                          {
+                            names = [ "replication" ];
+                          }
+                        ];
+                      }
                       {
                         port = 6167;
                         tls = false;
                         type = "http";
                         x_forwarded = true;
-                        bind_adrresses = [ "127.0.0.1" ];
+                        bind_adrresses = [ "0.0.0.0" ];
                         resources =
                           [
                             {
@@ -108,63 +231,26 @@
                           ];
                       }
                     ];
-                  log_config = logConfig;
+
-                  trusted_key_servers =
+                  public_baseurl = "https://matrix.redalder.org/";
-                    [
+
-                      {
+                  # Add a random shared secret to authenticate traffic.
-                        server_name = "matrix.org";
+                  worker_replication_secret = "";
-                      }
-                    ];
-                  media_store_path = "/var/lib/synapse/media_store";
-                  signing_key_path = "/var/lib/synapse/signing.key";
                 };
             in
               pkgs.writeShellScript "synapse"
                 ''
-                  if [ -f "/var/lib/synapse/sqlite.db" ] && ! [ -f "/var/lib/synapse/migration_done" ]
-                  then
-                    echo "Beginning migration from SQLite to PostgeSQL!"
-
-                    cat > /tmp/synapse_postgres.yaml <<EOF
-                  database:
-                    name: psycopg2
-                    args:
-                      user: synapse
-                      password: ''${PSQL_PASSWORD}
-                      database: synapse
-                      host: 127.0.0.1
-                      cp_min: 5
-                      cp_max: 10
-                  $(cat ${synapseConfig})
-                  EOF
-
-                    for ((i=0; i<5; i++))
-                    do
-                      ${pkgs.matrix-synapse.python.withPackages (_: [ (pkgs.python3.pkgs.toPythonModule pkgs.matrix-synapse) ] ++ pkgs.matrix-synapse.propagatedBuildInputs)}/bin/python ${pkgs.matrix-synapse}/lib/python*/site-packages/synapse/_scripts/synapse_port_db.py \
-                        --sqlite-database "/var/lib/synapse/sqlite.db" \
-                        --postgres-config "/tmp/synapse_postgres.yaml"
-                      [ $? -eq 0 ] && touch /var/lib/synapse/migration_done && break
-
-                      echo "Migration attempt ''${i}/5 failed! Retrying in 30 seconds..."
-                      sleep 30
-
-                      if ((5 == i))
-                      then
-                        echo "Migration failed!"
-                        exit 1
-                      fi
-                    done
-                  fi
-
                   [ -e /var/lib/synapse/signing.key ] || \
                     ${pkgs.matrix-synapse}/bin/synapse_homeserver \
                       --config-path ${synapseConfig} \
+                      --config-path ${commonConfig pkgs} \
                       --config-path /secrets/extra.yaml \
                       --config-path /var/lib/registrations/extra.yaml \
                       --keys-directory /var/lib/synapse/keys \
                       --generate-keys
                   ${pkgs.matrix-synapse}/bin/synapse_homeserver \
                     --config-path ${synapseConfig} \
+                    --config-path ${commonConfig pkgs} \
                     --config-path /secrets/extra.yaml \
                     --config-path /var/lib/registrations/extra.yaml \
                     --keys-directory /var/lib/synapse/keys

flake.lock

@@ -75,11 +75,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1667135915,
+        "lastModified": 1682262484,
-        "narHash": "sha256-maknqbSp2HbPze+XyAn0DTXCNArNyspCC3xOSNefhaw=",
+        "narHash": "sha256-2ep7UAYzvgGQ+uuwDgRqXksDCgqT3dEOCvOgS4hcOMs=",
         "owner": "nix-community",
         "repo": "NixNG",
-        "rev": "a958d9efc442f63d3363767d74c2df28b4e595ac",
+        "rev": "69019125a41249605fde6fca2acc51933725e848",
         "type": "github"
       },
       "original": {
@@ -107,11 +107,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1667231093,
+        "lastModified": 1681920287,
-        "narHash": "sha256-RERXruzBEBuf0c7OfZeX1hxEKB+PTCUNxWeB6C1jd8Y=",
+        "narHash": "sha256-+/d6XQQfhhXVfqfLROJoqj3TuG38CAeoT6jO1g9r1k0=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "d40fea9aeb8840fea0d377baa4b38e39b9582458",
+        "rev": "645bc49f34fa8eff95479f0345ff57e55b53437e",
         "type": "github"
       },
       "original": {

flake.nix

@@ -28,7 +28,7 @@
     in
       {
         nixngSystems =
-          let base = { nglib = nixng.nglib nixpkgs.lib; inherit nixpkgs; };
+          let base = { nglib = nixng.nglib; inherit nixpkgs; };
           in
             { hydra = (import ./containers/hydra.nix base).hydra;
               hydraPostgresql = (import ./containers/hydra.nix base).postgresql;
@@ -48,7 +48,12 @@
               reicio = import ./containers/reicio.nix base;
               baikal = import ./containers/baikal.nix base;
               conduit = (import ./containers/conduit.nix base).synapse;
+              synapseFederationSender = (import ./containers/conduit.nix base).synapseFederationSender;
+              synapseFederationReceiver = (import ./containers/conduit.nix base).synapseFederationReceiver;
+              synapseClient = (import ./containers/conduit.nix base).synapseClient;
+              synapseSync = (import ./containers/conduit.nix base).synapseSync;
               conduitPostgresql = (import ./containers/conduit.nix base).postgresql;
+              conduitRedis = (import ./containers/conduit.nix base).redis;
               mautrix-facebook = import ./containers/mautrix-facebook.nix base;
               heisenbridge = import ./containers/heisenbridge.nix base;
             };

nomad/regions/do-1/ingress-upstreams.conf

@@ -22,6 +22,18 @@ upstream matrix-synapse {
   server {{ env "NOMAD_UPSTREAM_ADDR_matrix_synapse" }};
 }
 
+upstream matrix-synapse-federation-receiver {
+  server {{ env "NOMAD_UPSTREAM_ADDR_matrix_synapse_federation_receiver" }};
+}
+
+upstream matrix-synapse-client {
+  server {{ env "NOMAD_UPSTREAM_ADDR_matrix_synapse_client" }};
+}
+
+upstream matrix-synapse-sync {
+  server {{ env "NOMAD_UPSTREAM_ADDR_matrix_synapse_sync" }};
+}
+
 upstream matrix-mautrix-facebook {
   server {{ env "NOMAD_UPSTREAM_ADDR_matrix-mautrix-facebook" }};
 }
@@ -97,10 +109,220 @@ server {
   server_name matrix.redalder.org;
   merge_slashes off;
 
-  location /_matrix/ {
+  # sync
-      proxy_pass http://matrix-synapse$request_uri;
+  location ~ ^/_matrix/client/(r0|v3)/sync$|^/_matrix/client/(api/v1|r0|v3)/events$|^/_matrix/client/(api/v1|r0|v3)/initialSync$|^/_matrix/client/(api/v1|r0|v3)/rooms/[^/]+/initialSync$ {
-      proxy_set_header Host $http_host;
+      # note: do not add a path (even a single /) after the port in `proxy_pass`,
-      proxy_buffering off;
+      # otherwise nginx will canonicalise the URI and cause signature verification
+      # errors.
+      proxy_pass http://matrix-synapse-sync;
+      proxy_set_header X-Forwarded-For $remote_addr;
+      proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_set_header Host $host;
+
+      # Nginx by default only allows file uploads up to 1M in size
+      # Increase client_max_body_size to match max_upload_size defined in homeserver.yaml
+      client_max_body_size 50M;
+
+      # Synapse responses may be chunked, which is an HTTP/1.1 feature.
+      proxy_http_version 1.1;
+  }
+
+  # federation
+  location ~ ^/_matrix/federation/v1/event/|^/_matrix/federation/v1/state/|^/_matrix/federation/v1/state_ids/|^/_matrix/federation/v1/backfill/|^/_matrix/federation/v1/get_missing_events/|^/_matrix/federation/v1/publicRooms|^/_matrix/federation/v1/query/|^/_matrix/federation/v1/make_join/|^/_matrix/federation/v1/make_leave/|^/_matrix/federation/(v1|v2)/send_join/|^/_matrix/federation/(v1|v2)/send_leave/|^/_matrix/federation/(v1|v2)/invite/|^/_matrix/federation/v1/event_auth/|^/_matrix/federation/v1/timestamp_to_event/|^/_matrix/federation/v1/exchange_third_party_invite/|^/_matrix/federation/v1/user/devices/|^/_matrix/key/v2/query|^/_matrix/federation/v1/hierarchy/ {
+      # note: do not add a path (even a single /) after the port in `proxy_pass`,
+      # otherwise nginx will canonicalise the URI and cause signature verification
+      # errors.
+      proxy_pass http://matrix-synapse-federation-receiver;
+      proxy_set_header X-Forwarded-For $remote_addr;
+      proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_set_header Host $host;
+
+      # Nginx by default only allows file uploads up to 1M in size
+      # Increase client_max_body_size to match max_upload_size defined in homeserver.yaml
+      client_max_body_size 50M;
+
+      # Synapse responses may be chunked, which is an HTTP/1.1 feature.
+      proxy_http_version 1.1;
+  }
+
+  # inbound federation requests
+  location ~ ^/_matrix/federation/v1/send/ {
+      # note: do not add a path (even a single /) after the port in `proxy_pass`,
+      # otherwise nginx will canonicalise the URI and cause signature verification
+      # errors.
+      proxy_pass http://matrix-synapse-federation-receiver;
+      proxy_set_header X-Forwarded-For $remote_addr;
+      proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_set_header Host $host;
+
+      # Nginx by default only allows file uploads up to 1M in size
+      # Increase client_max_body_size to match max_upload_size defined in homeserver.yaml
+      client_max_body_size 50M;
+
+      # Synapse responses may be chunked, which is an HTTP/1.1 feature.
+      proxy_http_version 1.1;
+  }
+
+  # client api
+  location ~ ^/_matrix/client/(api/v1|r0|v3|unstable)/createRoom$|^/_matrix/client/(api/v1|r0|v3|unstable)/publicRooms$|^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/joined_members$|^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/context/.*$|^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/members$|^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/state$|^/_matrix/client/v1/rooms/.*/hierarchy$|^/_matrix/client/(v1|unstable)/rooms/.*/relations/|^/_matrix/client/v1/rooms/.*/threads$|^/_matrix/client/unstable/org.matrix.msc2716/rooms/.*/batch_send$|^/_matrix/client/unstable/im.nheko.summary/rooms/.*/summary$|^/_matrix/client/(r0|v3|unstable)/account/3pid$|^/_matrix/client/(r0|v3|unstable)/account/whoami$|^/_matrix/client/(r0|v3|unstable)/devices$|^/_matrix/client/versions$|^/_matrix/client/(api/v1|r0|v3|unstable)/voip/turnServer$|^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/event/|^/_matrix/client/(api/v1|r0|v3|unstable)/joined_rooms$|^/_matrix/client/v1/rooms/.*/timestamp_to_event$|^/_matrix/client/(api/v1|r0|v3|unstable/.*)/rooms/.*/aliases|^/_matrix/client/(api/v1|r0|v3|unstable)/search$|^/_matrix/client/(r0|v3|unstable)/user/.*/filter(/|$) {
+      # note: do not add a path (even a single /) after the port in `proxy_pass`,
+      # otherwise nginx will canonicalise the URI and cause signature verification
+      # errors.
+      proxy_pass http://matrix-synapse-client;
+      proxy_set_header X-Forwarded-For $remote_addr;
+      proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_set_header Host $host;
+
+      # Nginx by default only allows file uploads up to 1M in size
+      # Increase client_max_body_size to match max_upload_size defined in homeserver.yaml
+      client_max_body_size 50M;
+
+      # Synapse responses may be chunked, which is an HTTP/1.1 feature.
+      proxy_http_version 1.1;
+  }
+
+  # encryption
+  location ~ ^/_matrix/client/(r0|v3|unstable)/keys/query$|^/_matrix/client/(r0|v3|unstable)/keys/changes$|^/_matrix/client/(r0|v3|unstable)/keys/claim$|^/_matrix/client/(r0|v3|unstable)/room_keys/|^/_matrix/client/(r0|v3|unstable)/keys/upload/ {
+      # note: do not add a path (even a single /) after the port in `proxy_pass`,
+      # otherwise nginx will canonicalise the URI and cause signature verification
+      # errors.
+      proxy_pass http://matrix-synapse-client;
+      proxy_set_header X-Forwarded-For $remote_addr;
+      proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_set_header Host $host;
+
+      # Nginx by default only allows file uploads up to 1M in size
+      # Increase client_max_body_size to match max_upload_size defined in homeserver.yaml
+      client_max_body_size 50M;
+
+      # Synapse responses may be chunked, which is an HTTP/1.1 feature.
+      proxy_http_version 1.1;
+  }
+
+  # registration login
+  location ~ ^/_matrix/client/(api/v1|r0|v3|unstable)/login$|^/_matrix/client/(r0|v3|unstable)/register$|^/_matrix/client/(r0|v3|unstable)/register/available$|^/_matrix/client/v1/register/m.login.registration_token/validity$|^/_matrix/client/(r0|v3|unstable)/password_policy$ {
+      # note: do not add a path (even a single /) after the port in `proxy_pass`,
+      # otherwise nginx will canonicalise the URI and cause signature verification
+      # errors.
+      proxy_pass http://matrix-synapse-client;
+      proxy_set_header X-Forwarded-For $remote_addr;
+      proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_set_header Host $host;
+
+      # Nginx by default only allows file uploads up to 1M in size
+      # Increase client_max_body_size to match max_upload_size defined in homeserver.yaml
+      client_max_body_size 50M;
+
+      # Synapse responses may be chunked, which is an HTTP/1.1 feature.
+      proxy_http_version 1.1;
+  }
+
+  # event sending
+  location ~ ^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/redact|^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/send|^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/state/|^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$|^/_matrix/client/(api/v1|r0|v3|unstable)/join/|^/_matrix/client/(api/v1|r0|v3|unstable)/knock/|^/_matrix/client/(api/v1|r0|v3|unstable)/profile/ {
+      # note: do not add a path (even a single /) after the port in `proxy_pass`,
+      # otherwise nginx will canonicalise the URI and cause signature verification
+      # errors.
+      proxy_pass http://matrix-synapse-client;
+      proxy_set_header X-Forwarded-For $remote_addr;
+      proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_set_header Host $host;
+
+      # Nginx by default only allows file uploads up to 1M in size
+      # Increase client_max_body_size to match max_upload_size defined in homeserver.yaml
+      client_max_body_size 50M;
+
+      # Synapse responses may be chunked, which is an HTTP/1.1 feature.
+      proxy_http_version 1.1;
+  }
+
+  # account data
+  location ~ ^/_matrix/client/(r0|v3|unstable)/.*/tags|^/_matrix/client/(r0|v3|unstable)/.*/account_data {
+      # note: do not add a path (even a single /) after the port in `proxy_pass`,
+      # otherwise nginx will canonicalise the URI and cause signature verification
+      # errors.
+      proxy_pass http://matrix-synapse;
+      proxy_set_header X-Forwarded-For $remote_addr;
+      proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_set_header Host $host;
+
+      # Nginx by default only allows file uploads up to 1M in size
+      # Increase client_max_body_size to match max_upload_size defined in homeserver.yaml
+      client_max_body_size 50M;
+
+      # Synapse responses may be chunked, which is an HTTP/1.1 feature.
+      proxy_http_version 1.1;
+  }
+
+  # receipts requests
+  location ~ ^/_matrix/client/(r0|v3|unstable)/rooms/.*/receipt|^/_matrix/client/(r0|v3|unstable)/rooms/.*/read_markers {
+      # note: do not add a path (even a single /) after the port in `proxy_pass`,
+      # otherwise nginx will canonicalise the URI and cause signature verification
+      # errors.
+      proxy_pass http://matrix-synapse;
+      proxy_set_header X-Forwarded-For $remote_addr;
+      proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_set_header Host $host;
+
+      # Nginx by default only allows file uploads up to 1M in size
+      # Increase client_max_body_size to match max_upload_size defined in homeserver.yaml
+      client_max_body_size 50M;
+
+      # Synapse responses may be chunked, which is an HTTP/1.1 feature.
+      proxy_http_version 1.1;
+  }
+
+  # presence requests
+  location ~ ^/_matrix/client/(api/v1|r0|v3|unstable)/presence/ {
+      # note: do not add a path (even a single /) after the port in `proxy_pass`,
+      # otherwise nginx will canonicalise the URI and cause signature verification
+      # errors.
+      proxy_pass http://matrix-synapse;
+      proxy_set_header X-Forwarded-For $remote_addr;
+      proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_set_header Host $host;
+
+      # Nginx by default only allows file uploads up to 1M in size
+      # Increase client_max_body_size to match max_upload_size defined in homeserver.yaml
+      client_max_body_size 50M;
+
+      # Synapse responses may be chunked, which is an HTTP/1.1 feature.
+      proxy_http_version 1.1;
+  }
+
+  # user directory search
+  location ~ ^/_matrix/client/(r0|v3|unstable)/user_directory/search$ {
+      # note: do not add a path (even a single /) after the port in `proxy_pass`,
+      # otherwise nginx will canonicalise the URI and cause signature verification
+      # errors.
+      proxy_pass http://matrix-synapse-client;
+      proxy_set_header X-Forwarded-For $remote_addr;
+      proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_set_header Host $host;
+
+      # Nginx by default only allows file uploads up to 1M in size
+      # Increase client_max_body_size to match max_upload_size defined in homeserver.yaml
+      client_max_body_size 50M;
+
+      # Synapse responses may be chunked, which is an HTTP/1.1 feature.
+      proxy_http_version 1.1;
+  }
+
+  # the rest
+  location ~ ^(/.well-known|/_matrix|/_synapse/client) {
+      # note: do not add a path (even a single /) after the port in `proxy_pass`,
+      # otherwise nginx will canonicalise the URI and cause signature verification
+      # errors.
+      proxy_pass http://matrix-synapse;
+      proxy_set_header X-Forwarded-For $remote_addr;
+      proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_set_header Host $host;
+
+      # Nginx by default only allows file uploads up to 1M in size
+      # Increase client_max_body_size to match max_upload_size defined in homeserver.yaml
+      client_max_body_size 50M;
+
+      # Synapse responses may be chunked, which is an HTTP/1.1 feature.
+      proxy_http_version 1.1;
   }
 
   location /mufb/ {

nomad/regions/do-1/ingress.hcl

@@ -136,6 +136,36 @@ job "ingress" {
               }
             }
 
+            upstreams {
+              destination_name = "matrix-synapse-client"
+              local_bind_port  = 6168
+              datacenter = "homelab-1"
+
+              mesh_gateway {
+                mode = "local"
+              }
+            }
+
+            upstreams {
+              destination_name = "matrix-synapse-federation-receiver"
+              local_bind_port  = 6169
+              datacenter = "homelab-1"
+
+              mesh_gateway {
+                mode = "local"
+              }
+            }
+
+            upstreams {
+              destination_name = "matrix-synapse-sync"
+              local_bind_port  = 6170
+              datacenter = "homelab-1"
+
+              mesh_gateway {
+                mode = "local"
+              }
+            }
+
             upstreams {
               destination_name = "matrix-mautrix-facebook"
               local_bind_port  = 29319

nomad/regions/homelab-1/conduit.tf

@@ -6,7 +6,7 @@ resource "nomad_volume" "matrix-synapse" {
   external_id = "matrix-synapse"
 
   capability {
-    access_mode = "single-node-writer"
+    access_mode = "multi-node-multi-writer"
     attachment_mode = "file-system"
   }
 
@@ -98,10 +98,35 @@ resource "nomad_volume" "matrix-mautrix-facebook" {
   }
 }
 
+resource "nomad_volume" "matrix-redis" {
+  type        = "csi"
+  plugin_id   = "org.democratic-csi.nfs"
+  volume_id   = "matrix-redis"
+  name        = "matrix-redis"
+  external_id = "matrix-redis"
+
+  capability {
+    access_mode = "single-node-writer"
+    attachment_mode = "file-system"
+  }
+
+  context = {
+    server = "blowhole.hosts.in.redalder.org"
+    share = "/var/nfs/matrix/redis"
+    node_attach_driver = "nfs"
+    provisioner_driver = "node-manual"
+  }
+
+  mount_options {
+    fs_type = "nfs"
+    mount_flags = [ "nfsvers=3", "hard", "async" ]
+  }
+}
+
 resource "vault_policy" "matrix-mautrix-facebook-policy" {
   name = "matrix-mautrix-facebook-policy"
   policy = <<EOF
-path "kv/data/matrix/mautrix-facebook" {
+path "kv/data/cluster/matrix/mautrix-facebook" {
   capabilities = ["read"]
 }
 EOF
@@ -110,7 +135,7 @@ EOF
 resource "vault_policy" "matrix-synapse-policy" {
   name = "matrix-synapse-policy"
   policy = <<EOF
-path "kv/data/matrix/synapse" {
+path "kv/data/cluster/matrix/synapse" {
   capabilities = ["read"]
 }
 EOF

nomad/regions/homelab-1/job/conduit.hcl

@@ -11,7 +11,7 @@ job "matrix" {
   type = "service"
 
   group "mautrix-facebook" {
-    count = 1
+    count = 0
 
     volume "matrix-mautrix-facebook" {
       type = "csi"
@@ -85,7 +85,7 @@ job "matrix" {
 
       template {
         data = <<EOF
-{{ with secret "kv/data/matrix/mautrix-facebook" }}
+{{ with secret "kv/data/cluster/matrix/mautrix-facebook" }}
 MAUTRIX_FACEBOOK_APPSERVICE_AS_TOKEN={{ .Data.data.as_token }}
 MAUTRIX_FACEBOOK_APPSERVICE_HS_TOKEN={{ .Data.data.hs_token }}
 {{ end }}
@@ -153,6 +153,667 @@ EOF
     }
   }
 
+  group "redis" {
+    count = 1
+
+    volume "matrix-redis" {
+      type = "csi"
+      source = "matrix-redis"
+      read_only = false
+
+      attachment_mode = "file-system"
+      access_mode     = "single-node-writer"
+    }
+
+    restart {
+      attempts = 5
+      delay = "5s"
+    }
+
+    network {
+      mode = "bridge"
+    }
+
+    vault {
+      policies = ["matrix-synapse-policy"]
+    }
+
+    service {
+      name = "matrix-redis"
+      port = "6379"
+
+      # check {
+      #   type = "http"
+      #   address_mode = "alloc"
+      #   path = "/health"
+      #   port = "6167"
+      #   interval = "2s"
+      #   timeout = "2s"
+      # }
+
+      connect {
+        sidecar_service {}
+      }
+    }
+
+    task "redis" {
+      driver = "docker"
+
+      volume_mount {
+        volume = "matrix-redis"
+        destination = "/var/lib/redis"
+        read_only = false
+      }
+
+      config {
+        nix_flake_ref = "${var.flake_ref}#nixngSystems.conduitRedis.config.system.build.toplevel"
+        nix_flake_sha = var.flake_sha
+        entrypoint = [ "init" ]
+      }
+
+      template {
+        data = <<EOF
+{{ with secret "kv/data/cluster/matrix/synapse" -}}
+{{ .Data.data.redis_password }}
+{{ end -}}
+EOF
+        destination = "/secrets/redis_password"
+      }
+    }
+  }
+
+  group "synapse-client" {
+    count = 1
+
+    restart {
+      attempts = 5
+      delay = "5s"
+    }
+
+    network {
+      mode = "bridge"
+    }
+
+    vault {
+      policies = ["matrix-synapse-policy"]
+    }
+
+    volume "matrix-synapse" {
+      type = "csi"
+      source = "matrix-synapse"
+      read_only = false
+
+      attachment_mode = "file-system"
+      access_mode     = "multi-node-multi-writer"
+    }
+
+    volume "matrix-registrations" {
+      type = "csi"
+      source = "matrix-registrations"
+      read_only = false
+
+      attachment_mode = "file-system"
+      access_mode     = "multi-node-multi-writer"
+    }
+
+    service {
+      name = "matrix-synapse-client"
+      port = "6167"
+
+      check {
+        type = "http"
+        address_mode = "alloc"
+        path = "/health"
+        port = "6167"
+        interval = "2s"
+        timeout = "2s"
+      }
+
+      connect {
+        sidecar_service {
+          proxy {
+            upstreams {
+              destination_name = "matrix-mautrix-facebook"
+              local_bind_port  = 29319
+            }
+
+            upstreams {
+              destination_name = "matrix-heisenbridge"
+              local_bind_port  = 9898
+            }
+
+            upstreams {
+              destination_name = "matrix-postgresql"
+              local_bind_port = 5432
+            }
+
+            upstreams {
+              destination_name = "matrix-redis"
+              local_bind_port = 6379
+            }
+
+            upstreams {
+              destination_name = "matrix-synapse-replication"
+              local_bind_port = 9093
+            }
+          }
+        }
+      }
+    }
+
+    task "synapse-client" {
+      driver = "docker"
+
+      volume_mount {
+        volume = "matrix-synapse"
+        destination = "/var/lib/synapse"
+        read_only = false
+      }
+
+      volume_mount {
+        volume = "matrix-registrations"
+        destination = "/var/lib/registrations"
+        read_only = false
+      }
+
+      config {
+        nix_flake_ref = "${var.flake_ref}#nixngSystems.synapseClient.config.system.build.toplevel"
+        nix_flake_sha = var.flake_sha
+        entrypoint = [ "init" ]
+      }
+
+      resources {
+        cpu = 1024
+        memory = 2048
+      }
+
+      template {
+        data = <<EOF
+{{ with secret "kv/data/cluster/matrix/synapse" }}
+worker_name: "worker-client-{{ env "NOMAD_ALLOC_INDEX" }}"
+registration_shared_secret: "{{ .Data.data.registration_shared_secret }}"
+macaroon_secret_key: "{{ .Data.data.macaroon_secret_key }}"
+form_secret: "{{ .Data.data.form_secret }}"
+database:
+  name: "psycopg2"
+  args:
+    user: "synapse"
+    password:  "{{ .Data.data.pgpass }}"
+    database: "synapse"
+    host: "127.0.0.1"
+    cp_min: 5
+    cp_max: 10
+redis:
+  enabled: true
+  password: "{{ .Data.data.redis_password }}"
+{{ end }}
+EOF
+        destination = "/secrets/extra.yaml"
+      }
+    }
+  }
+
+  group "synapse-sync" {
+    count = 1
+
+    restart {
+      attempts = 5
+      delay = "5s"
+    }
+
+    network {
+      mode = "bridge"
+    }
+
+    vault {
+      policies = ["matrix-synapse-policy"]
+    }
+
+    volume "matrix-synapse" {
+      type = "csi"
+      source = "matrix-synapse"
+      read_only = false
+
+      attachment_mode = "file-system"
+      access_mode     = "multi-node-multi-writer"
+    }
+
+    volume "matrix-registrations" {
+      type = "csi"
+      source = "matrix-registrations"
+      read_only = false
+
+      attachment_mode = "file-system"
+      access_mode     = "multi-node-multi-writer"
+    }
+
+    service {
+      name = "matrix-synapse-sync"
+      port = "6167"
+
+      check {
+        type = "http"
+        address_mode = "alloc"
+        path = "/health"
+        port = "6167"
+        interval = "2s"
+        timeout = "2s"
+      }
+
+      connect {
+        sidecar_service {
+          proxy {
+            upstreams {
+              destination_name = "matrix-mautrix-facebook"
+              local_bind_port  = 29319
+            }
+
+            upstreams {
+              destination_name = "matrix-heisenbridge"
+              local_bind_port  = 9898
+            }
+
+            upstreams {
+              destination_name = "matrix-postgresql"
+              local_bind_port = 5432
+            }
+
+            upstreams {
+              destination_name = "matrix-redis"
+              local_bind_port = 6379
+            }
+
+            upstreams {
+              destination_name = "matrix-synapse-replication"
+              local_bind_port = 9093
+            }
+          }
+        }
+      }
+    }
+
+    task "synapse-sync" {
+      driver = "docker"
+
+      volume_mount {
+        volume = "matrix-synapse"
+        destination = "/var/lib/synapse"
+        read_only = false
+      }
+
+      volume_mount {
+        volume = "matrix-registrations"
+        destination = "/var/lib/registrations"
+        read_only = false
+      }
+
+      config {
+        nix_flake_ref = "${var.flake_ref}#nixngSystems.synapseSync.config.system.build.toplevel"
+        nix_flake_sha = var.flake_sha
+        entrypoint = [ "init" ]
+      }
+
+      resources {
+        cpu = 1024
+        memory = 512
+      }
+
+      template {
+        data = <<EOF
+{{ with secret "kv/data/cluster/matrix/synapse" }}
+worker_name: "worker-sync-{{ env "NOMAD_ALLOC_INDEX" }}"
+registration_shared_secret: "{{ .Data.data.registration_shared_secret }}"
+macaroon_secret_key: "{{ .Data.data.macaroon_secret_key }}"
+form_secret: "{{ .Data.data.form_secret }}"
+database:
+  name: "psycopg2"
+  args:
+    user: "synapse"
+    password:  "{{ .Data.data.pgpass }}"
+    database: "synapse"
+    host: "127.0.0.1"
+    cp_min: 5
+    cp_max: 10
+redis:
+  enabled: true
+  password: "{{ .Data.data.redis_password }}"
+{{ end }}
+EOF
+        destination = "/secrets/extra.yaml"
+      }
+    }
+  }
+
+  group "synapse-federation-receiver" {
+    count = 1
+
+    restart {
+      attempts = 5
+      delay = "5s"
+    }
+
+    network {
+      mode = "bridge"
+    }
+
+    vault {
+      policies = ["matrix-synapse-policy"]
+    }
+
+    volume "matrix-synapse" {
+      type = "csi"
+      source = "matrix-synapse"
+      read_only = false
+
+      attachment_mode = "file-system"
+      access_mode     = "multi-node-multi-writer"
+    }
+
+    volume "matrix-registrations" {
+      type = "csi"
+      source = "matrix-registrations"
+      read_only = false
+
+      attachment_mode = "file-system"
+      access_mode     = "multi-node-multi-writer"
+    }
+
+    service {
+      name = "matrix-synapse-federation-receiver"
+      port = "6167"
+
+      check {
+        type = "http"
+        address_mode = "alloc"
+        path = "/health"
+        port = "6167"
+        interval = "2s"
+        timeout = "2s"
+      }
+
+      connect {
+        sidecar_service {
+          proxy {
+            upstreams {
+              destination_name = "matrix-mautrix-facebook"
+              local_bind_port  = 29319
+            }
+
+            upstreams {
+              destination_name = "matrix-heisenbridge"
+              local_bind_port  = 9898
+            }
+
+            upstreams {
+              destination_name = "matrix-postgresql"
+              local_bind_port = 5432
+            }
+
+            upstreams {
+              destination_name = "matrix-redis"
+              local_bind_port = 6379
+            }
+
+            upstreams {
+              destination_name = "matrix-synapse-replication"
+              local_bind_port = 9093
+            }
+          }
+        }
+      }
+    }
+
+    task "synapse-federation-receiver" {
+      driver = "docker"
+
+      volume_mount {
+        volume = "matrix-synapse"
+        destination = "/var/lib/synapse"
+        read_only = false
+      }
+
+      volume_mount {
+        volume = "matrix-registrations"
+        destination = "/var/lib/registrations"
+        read_only = false
+      }
+
+      config {
+        nix_flake_ref = "${var.flake_ref}#nixngSystems.synapseFederationReceiver.config.system.build.toplevel"
+        nix_flake_sha = var.flake_sha
+        entrypoint = [ "init" ]
+      }
+
+      resources {
+        cpu = 1024
+        memory = 512
+      }
+
+      template {
+        data = <<EOF
+{{ with secret "kv/data/cluster/matrix/synapse" }}
+worker_name: "worker-federation-receiver-{{ env "NOMAD_ALLOC_INDEX" }}"
+registration_shared_secret: "{{ .Data.data.registration_shared_secret }}"
+macaroon_secret_key: "{{ .Data.data.macaroon_secret_key }}"
+form_secret: "{{ .Data.data.form_secret }}"
+database:
+  name: "psycopg2"
+  args:
+    user: "synapse"
+    password:  "{{ .Data.data.pgpass }}"
+    database: "synapse"
+    host: "127.0.0.1"
+    cp_min: 5
+    cp_max: 10
+redis:
+  enabled: true
+  password: "{{ .Data.data.redis_password }}"
+{{ end }}
+EOF
+        destination = "/secrets/extra.yaml"
+      }
+    }
+  }
+
+  group "synapse-federation-sender" {
+    count = 1
+
+    restart {
+      attempts = 5
+      delay = "5s"
+    }
+
+    network {
+      mode = "bridge"
+    }
+
+    vault {
+      policies = ["matrix-synapse-policy"]
+    }
+
+    volume "matrix-synapse" {
+      type = "csi"
+      source = "matrix-synapse"
+      read_only = false
+
+      attachment_mode = "file-system"
+      access_mode     = "multi-node-multi-writer"
+    }
+
+    volume "matrix-registrations" {
+      type = "csi"
+      source = "matrix-registrations"
+      read_only = false
+
+      attachment_mode = "file-system"
+      access_mode     = "multi-node-multi-writer"
+    }
+
+    service {
+      name = "matrix-synapse-federation-sender"
+      port = "6167"
+
+      check {
+        type = "http"
+        address_mode = "alloc"
+        path = "/health"
+        port = "6167"
+        interval = "2s"
+        timeout = "2s"
+      }
+
+      connect {
+        sidecar_service {
+          proxy {
+            upstreams {
+              destination_name = "matrix-mautrix-facebook"
+              local_bind_port  = 29319
+            }
+
+            upstreams {
+              destination_name = "matrix-heisenbridge"
+              local_bind_port  = 9898
+            }
+
+            upstreams {
+              destination_name = "matrix-postgresql"
+              local_bind_port = 5432
+            }
+
+            upstreams {
+              destination_name = "matrix-redis"
+              local_bind_port = 6379
+            }
+
+            upstreams {
+              destination_name = "matrix-synapse-replication"
+              local_bind_port = 9093
+            }
+          }
+        }
+      }
+    }
+
+    task "synapse-federation-sender" {
+      driver = "docker"
+
+      volume_mount {
+        volume = "matrix-synapse"
+        destination = "/var/lib/synapse"
+        read_only = false
+      }
+
+      volume_mount {
+        volume = "matrix-registrations"
+        destination = "/var/lib/registrations"
+        read_only = false
+      }
+
+      config {
+        nix_flake_ref = "${var.flake_ref}#nixngSystems.synapseFederationSender.config.system.build.toplevel"
+        nix_flake_sha = var.flake_sha
+        entrypoint = [ "init" ]
+      }
+
+      resources {
+        cpu = 1024
+        memory = 512
+      }
+
+      template {
+        data = <<EOF
+{{ with secret "kv/data/cluster/matrix/synapse" }}
+worker_name: "worker-federation-sender-{{ env "NOMAD_ALLOC_INDEX" }}"
+registration_shared_secret: "{{ .Data.data.registration_shared_secret }}"
+macaroon_secret_key: "{{ .Data.data.macaroon_secret_key }}"
+form_secret: "{{ .Data.data.form_secret }}"
+database:
+  name: "psycopg2"
+  args:
+    user: "synapse"
+    password:  "{{ .Data.data.pgpass }}"
+    database: "synapse"
+    host: "127.0.0.1"
+    cp_min: 5
+    cp_max: 10
+redis:
+  enabled: true
+  password: "{{ .Data.data.redis_password }}"
+{{ end }}
+EOF
+        destination = "/secrets/extra.yaml"
+      }
+    }
+  }
+
+  group "postgresql" {
+    count = 1
+
+    volume "matrix-postgresql" {
+      type = "csi"
+      source = "matrix-postgresql"
+      read_only = false
+
+      attachment_mode = "file-system"
+      access_mode     = "single-node-writer"
+    }
+
+    restart {
+      attempts = 5
+      delay = "5s"
+    }
+
+    network {
+      mode = "bridge"
+    }
+
+    service {
+      name = "matrix-postgresql"
+      port = "5432"
+
+      # check {
+
+      # }
+
+      connect {
+        sidecar_service {}
+      }
+    }
+
+    task "postgresql" {
+      driver = "docker"
+
+      volume_mount {
+        volume = "matrix-postgresql"
+        destination = "/var/lib/postgresql"
+        read_only = false
+      }
+
+      config {
+        nix_flake_ref = "${var.flake_ref}#nixngSystems.conduitPostgresql.config.system.build.toplevel"
+        nix_flake_sha = var.flake_sha
+        entrypoint = [ "init" ]
+      }
+
+      resources {
+        cpu = 500
+        memory = 1024
+        memory_max = 1536
+      }
+
+      template {
+        data = <<EOF
+alter user synapse with password '{{ with secret "kv/data/cluster/matrix/synapse" }}{{ .Data.data.pgpass }}{{ end }}';
+EOF
+        destination = "secrets/init.sql"
+      }
+
+      vault {
+        policies = ["matrix-synapse-policy"]
+      }
+    }
+  }
+
   group "synapse" {
     count = 1
 
@@ -162,16 +823,7 @@ EOF
       read_only = false
 
       attachment_mode = "file-system"
-      access_mode     = "single-node-writer"
+      access_mode     = "multi-node-multi-writer"
-    }
-
-    volume "matrix-postgresql" {
-      type = "csi"
-      source = "matrix-postgresql"
-      read_only = false
-
-      attachment_mode = "file-system"
-      access_mode     = "single-node-writer"
     }
 
     volume "matrix-registrations" {
@@ -199,7 +851,7 @@ EOF
       check {
         type = "http"
         address_mode = "alloc"
-        path = "/_matrix/client/versions"
+        path = "/health"
         port = "6167"
         interval = "2s"
         timeout = "2s"
@@ -217,45 +869,36 @@ EOF
               destination_name = "matrix-heisenbridge"
               local_bind_port  = 9898
             }
+
+            upstreams {
+              destination_name = "matrix-postgresql"
+              local_bind_port = 5432
+            }
+
+            upstreams {
+              destination_name = "matrix-redis"
+              local_bind_port = 6379
+            }
           }
         }
       }
     }
 
-    task "postgresql" {
+    service {
-      driver = "docker"
+      name = "matrix-synapse-replication"
+      port = "9093"
 
-      volume_mount {
+      # check {
-        volume = "matrix-postgresql"
+      #   type = "http"
-        destination = "/var/lib/postgresql"
+      #   address_mode = "alloc"
-        read_only = false
+      #   path = "/"
-      }
+      #   port = "9093"
+      #   interval = "2s"
+      #   timeout = "2s"
+      # }
 
-      config {
+      connect {
-        nix_flake_ref = "${var.flake_ref}#nixngSystems.conduitPostgresql.config.system.build.toplevel"
+        sidecar_service {}
-        nix_flake_sha = var.flake_sha
-        entrypoint = [ "init" ]
-      }
-
-      resources {
-        cpu = 500
-        memory = 1024
-        memory_max = 1536
-      }
-
-      template {
-        data = <<EOF
-alter user synapse with encrypted password '{{ with secret "kv/data/matrix/synapse" }}{{ .Data.data.pgpass }}{{ end }}';
-\c synapse;
-SELECT setval('application_services_txn_id_seq', (
-    SELECT GREATEST(MAX(txn_id), 0) FROM application_services_txns
-));
-EOF
-        destination = "secrets/init.sql"
-      }
-
-      vault {
-        policies = ["matrix-synapse-policy"]
       }
     }
 
@@ -282,7 +925,7 @@ EOF
 
       resources {
         cpu = 2048
-        memory = 2048
+        memory = 768
       }
 
       vault {
@@ -291,18 +934,7 @@ EOF
 
       template {
         data = <<EOF
-{{ with secret "kv/data/matrix/synapse" }}
+{{ with secret "kv/data/cluster/matrix/synapse" }}
-PSQL_PASSWORD={{ .Data.data.pgpass }}
-{{ end }}
-EOF
-        destination = "secrets/environment"
-        env = true
-        perms = "400"
-      }
-
-      template {
-        data = <<EOF
-{{ with secret "kv/data/matrix/synapse" }}
 registration_shared_secret: "{{ .Data.data.registration_shared_secret }}"
 macaroon_secret_key: "{{ .Data.data.macaroon_secret_key }}"
 form_secret: "{{ .Data.data.form_secret }}"
@@ -315,6 +947,9 @@ database:
     host: "127.0.0.1"
     cp_min: 5
     cp_max: 10
+redis:
+  enabled: true
+  password: "{{ .Data.data.redis_password }}"
 {{ end }}
 EOF
         destination = "/secrets/extra.yaml"