Add redis to synapse

Signed-off-by: Magic_RB <magic_rb@redalder.org>

containers/conduit.nix

@@ -1,4 +1,106 @@
 { nglib, nixpkgs }:
+let
+  logConfig = pkgs: (pkgs.formats.yaml {}).generate "log.yaml"
+    {
+      version = 1;
+
+      formatters.precise.format = "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s";
+      handlers.console =
+        {
+          class = "logging.StreamHandler";
+          formatter = "precise";
+        };
+      loggers."synapse.storage.SQL" =
+        {
+          level = "INFO";
+        };
+      root =
+        {
+          level = "INFO";
+          handlers = [ "console" ];
+        };
+
+      disable_existing_loggers = false;
+    };
+
+  commonConfig = pkgs: (pkgs.formats.yaml {}).generate "common.yaml"
+    {
+      server_name = "matrix.redalder.org";
+      report_stats = "yes";
+      pid_file = "/homeserver.pid";
+
+      log_config = logConfig pkgs;
+
+      trusted_key_servers =
+        [
+          {
+            server_name = "matrix.org";
+          }
+        ];
+      media_store_path = "/var/lib/synapse/media_store";
+      signing_key_path = "/var/lib/synapse/signing.key";
+
+      enable_registration = false;
+      enable_registration_without_verification = false;
+
+      federation_sender_instances = [
+        "worker-federation-sender-0"
+      ];
+    };
+
+  genericWorker = { listener_resources, name }:
+    nglib.makeSystem {
+      system = "x86_64-linux";
+      name = "synapse-worker-${name}";
+      inherit nixpkgs;
+      config = ({ pkgs, ... }:
+        {
+          dumb-init = {
+            enable = true;
+            type.services = { };
+          };
+
+          services.synapse.workers.${name} = {
+            settings = {
+              worker_app = "synapse.app.generic_worker";
+
+              # The replication listener on the main synapse process.
+              worker_replication_host = "127.0.0.1";
+              worker_replication_http_port = 9093;
+
+              worker_listeners = [
+                {
+                  port = 6167;
+                  tls = false;
+                  type = "http";
+                  x_forwarded = true;
+                  bind_adrresses = [ "0.0.0.0" ];
+                  resources =
+                    [
+                      {
+                        names = listener_resources;
+                        compress = false;
+                      }
+                    ];
+                }
+              ];
+
+              worker_log_config = logConfig pkgs;
+            };
+            arguments = {
+              config-path = [
+                (commonConfig pkgs)
+                "/secrets/extra.yaml"
+                "/var/lib/registrations/extra.yaml"
+              ];
+              keys-directory = [
+                "/var/lib/synapse/keys"
+              ];
+            };
+          };
+        });
+    };
+in
 {
   postgresql = nglib.makeSystem {
     system = "x86_64-linux";
@@ -33,6 +135,56 @@
       };
   };
 
+  redis = nglib.makeSystem {
+    system = "x86_64-linux";
+    name = "redis";
+    inherit nixpkgs;
+    config = ({ pkgs, ... }:
+      {
+        dumb-init = {
+          enable = true;
+          type.services = { };
+        };
+
+        users.users."redis" = {
+          home = "/var/empty";
+          uid = 9001;
+          group = "redis";
+        };
+
+        users.groups."redis" = {
+          gid = 9001;
+        };
+
+        init.services.redis = {
+          enabled = true;
+          shutdownOnExit = true;
+          script =
+            pkgs.writeShellScript "redis-run" ''
+              cd /var/lib/redis
+              chpst -U redis:redis ${pkgs.redis}/bin/redis-server ${./redis.conf}
+            '';
+        };
+
+        init.services.redis-setup = {
+          enabled = true;
+          script =
+            pkgs.writeShellScript "redis-run" ''
+              export PATH="${pkgs.redis}/bin:$PATH"
+              nc -z 127.0.0.1 6379 -w 10 -v || exit 1
+
+              redis-cli acl setuser default on '>'"$(cat /secrets/redis_password)" allcommands allkeys
+              sleep 86400
+            '';
+        };
+      });
+  };
+
+  synapseFederationSender = genericWorker { name = "generic"; listener_resources = [ "health" ]; };
+  synapseFederationReceiver = genericWorker { name = "generic"; listener_resources = [ "health" "federation" ]; };
+  synapseClient = genericWorker { name = "generic"; listener_resources = [ "client" "health" ]; };
+  synapseSync = genericWorker { name = "generic"; listener_resources = [ "client" "health" ]; };
+
   synapse = nglib.makeSystem {
     system = "x86_64-linux";
     name = "synapse";
@@ -49,56 +201,27 @@
           shutdownOnExit = true;
           script =
             let
-              logConfig = (pkgs.formats.yaml {}).generate "log.yaml"
-                {
-                  # Log configuration for Synapse.
-                  #
-                  # This is a YAML file containing a standard Python logging configuration
-                  # dictionary. See [1] for details on the valid settings.
-                  #
-                  # Synapse also supports structured logging for machine readable logs which can
-                  # be ingested by ELK stacks. See [2] for details.
-                  #
-                  # [1]: https://docs.python.org/3.7/library/logging.config.html#configuration-dictionary-schema
-                  # [2]: https://matrix-org.github.io/synapse/latest/structured_logging.html
-
-                  version = 1;
-
-                  formatters.precise.format = "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s";
-                  handlers.console =
-                    {
-                      class = "logging.StreamHandler";
-                      formatter = "precise";
-                    };
-                  loggers."synapse.storage.SQL" =
-                    {
-                      level = "INFO";
-                    };
-                  root =
-                    {
-                      level = "INFO";
-                      handlers = [ "console" ];
-                    };
-
-                  disable_existing_loggers = false;
-                };
               synapseConfig = (pkgs.formats.yaml {}).generate "synapse.yaml"
                 {
-                  server_name = "matrix.redalder.org";
-                  report_stats = "yes";
-                  pid_file = "/homeserver.pid";
-
-                  enable_registration = false;
-                  enable_registration_without_verification = false;
-
                   listeners =
                     [
+                      # The HTTP replication port
+                      {
+                        port = 9093;
+                        bind_addresses = [ "0.0.0.0" ];
+                        type = "http";
+                        resources = [
+                          {
+                            names = [ "replication" ];
+                          }
+                        ];
+                      }
                       {
                         port = 6167;
                         tls = false;
                         type = "http";
                         x_forwarded = true;
-                        bind_adrresses = [ "127.0.0.1" ];
+                        bind_adrresses = [ "0.0.0.0" ];
                         resources =
                           [
                             {
@@ -108,63 +231,26 @@
                           ];
                       }
                     ];
-                  log_config = logConfig;
+
-                  trusted_key_servers =
+                  public_baseurl = "https://matrix.redalder.org/";
-                    [
+
-                      {
+                  # Add a random shared secret to authenticate traffic.
-                        server_name = "matrix.org";
+                  worker_replication_secret = "";
-                      }
-                    ];
-                  media_store_path = "/var/lib/synapse/media_store";
-                  signing_key_path = "/var/lib/synapse/signing.key";
                 };
             in
               pkgs.writeShellScript "synapse"
                 ''
-                  if [ -f "/var/lib/synapse/sqlite.db" ] && ! [ -f "/var/lib/synapse/migration_done" ]
-                  then
-                    echo "Beginning migration from SQLite to PostgeSQL!"
-
-                    cat > /tmp/synapse_postgres.yaml <<EOF
-                  database:
-                    name: psycopg2
-                    args:
-                      user: synapse
-                      password: ''${PSQL_PASSWORD}
-                      database: synapse
-                      host: 127.0.0.1
-                      cp_min: 5
-                      cp_max: 10
-                  $(cat ${synapseConfig})
-                  EOF
-
-                    for ((i=0; i<5; i++))
-                    do
-                      ${pkgs.matrix-synapse.python.withPackages (_: [ (pkgs.python3.pkgs.toPythonModule pkgs.matrix-synapse) ] ++ pkgs.matrix-synapse.propagatedBuildInputs)}/bin/python ${pkgs.matrix-synapse}/lib/python*/site-packages/synapse/_scripts/synapse_port_db.py \
-                        --sqlite-database "/var/lib/synapse/sqlite.db" \
-                        --postgres-config "/tmp/synapse_postgres.yaml"
-                      [ $? -eq 0 ] && touch /var/lib/synapse/migration_done && break
-
-                      echo "Migration attempt ''${i}/5 failed! Retrying in 30 seconds..."
-                      sleep 30
-
-                      if ((5 == i))
-                      then
-                        echo "Migration failed!"
-                        exit 1
-                      fi
-                    done
-                  fi
-
                   [ -e /var/lib/synapse/signing.key ] || \
                     ${pkgs.matrix-synapse}/bin/synapse_homeserver \
                       --config-path ${synapseConfig} \
+                      --config-path ${commonConfig pkgs} \
                       --config-path /secrets/extra.yaml \
                       --config-path /var/lib/registrations/extra.yaml \
                       --keys-directory /var/lib/synapse/keys \
                       --generate-keys
                   ${pkgs.matrix-synapse}/bin/synapse_homeserver \
                     --config-path ${synapseConfig} \
+                    --config-path ${commonConfig pkgs} \
                     --config-path /secrets/extra.yaml \
                     --config-path /var/lib/registrations/extra.yaml \
                     --keys-directory /var/lib/synapse/keys

flake.lock

@@ -75,11 +75,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1667135915,
+        "lastModified": 1682262484,
-        "narHash": "sha256-maknqbSp2HbPze+XyAn0DTXCNArNyspCC3xOSNefhaw=",
+        "narHash": "sha256-2ep7UAYzvgGQ+uuwDgRqXksDCgqT3dEOCvOgS4hcOMs=",
         "owner": "nix-community",
         "repo": "NixNG",
-        "rev": "a958d9efc442f63d3363767d74c2df28b4e595ac",
+        "rev": "69019125a41249605fde6fca2acc51933725e848",
         "type": "github"
       },
       "original": {
@@ -107,11 +107,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1667231093,
+        "lastModified": 1681920287,
-        "narHash": "sha256-RERXruzBEBuf0c7OfZeX1hxEKB+PTCUNxWeB6C1jd8Y=",
+        "narHash": "sha256-+/d6XQQfhhXVfqfLROJoqj3TuG38CAeoT6jO1g9r1k0=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "d40fea9aeb8840fea0d377baa4b38e39b9582458",
+        "rev": "645bc49f34fa8eff95479f0345ff57e55b53437e",
         "type": "github"
       },
       "original": {

flake.nix

@@ -28,7 +28,7 @@
     in
       {
         nixngSystems =
-          let base = { nglib = nixng.nglib nixpkgs.lib; inherit nixpkgs; };
+          let base = { nglib = nixng.nglib; inherit nixpkgs; };
           in
             { hydra = (import ./containers/hydra.nix base).hydra;
               hydraPostgresql = (import ./containers/hydra.nix base).postgresql;
@@ -48,7 +48,12 @@
               reicio = import ./containers/reicio.nix base;
               baikal = import ./containers/baikal.nix base;
               conduit = (import ./containers/conduit.nix base).synapse;
+              synapseFederationSender = (import ./containers/conduit.nix base).synapseFederationSender;
+              synapseFederationReceiver = (import ./containers/conduit.nix base).synapseFederationReceiver;
+              synapseClient = (import ./containers/conduit.nix base).synapseClient;
+              synapseSync = (import ./containers/conduit.nix base).synapseSync;
               conduitPostgresql = (import ./containers/conduit.nix base).postgresql;
+              conduitRedis = (import ./containers/conduit.nix base).redis;
               mautrix-facebook = import ./containers/mautrix-facebook.nix base;
               heisenbridge = import ./containers/heisenbridge.nix base;
             };

nomad/regions/homelab-1/conduit.tf

@@ -98,10 +98,35 @@ resource "nomad_volume" "matrix-mautrix-facebook" {
   }
 }
 
+resource "nomad_volume" "matrix-redis" {
+  type        = "csi"
+  plugin_id   = "org.democratic-csi.nfs"
+  volume_id   = "matrix-redis"
+  name        = "matrix-redis"
+  external_id = "matrix-redis"
+
+  capability {
+    access_mode = "single-node-writer"
+    attachment_mode = "file-system"
+  }
+
+  context = {
+    server = "blowhole.hosts.in.redalder.org"
+    share = "/var/nfs/matrix/redis"
+    node_attach_driver = "nfs"
+    provisioner_driver = "node-manual"
+  }
+
+  mount_options {
+    fs_type = "nfs"
+    mount_flags = [ "nfsvers=3", "hard", "async" ]
+  }
+}
+
 resource "vault_policy" "matrix-mautrix-facebook-policy" {
   name = "matrix-mautrix-facebook-policy"
   policy = <<EOF
-path "kv/data/matrix/mautrix-facebook" {
+path "kv/data/cluster/matrix/mautrix-facebook" {
   capabilities = ["read"]
 }
 EOF
@@ -110,7 +135,7 @@ EOF
 resource "vault_policy" "matrix-synapse-policy" {
   name = "matrix-synapse-policy"
   policy = <<EOF
-path "kv/data/matrix/synapse" {
+path "kv/data/cluster/matrix/synapse" {
   capabilities = ["read"]
 }
 EOF