Transfer Matrix Synapse to PostgreSQL from SQLite

Signed-off-by: main <magic_rb@redalder.org>

containers/conduit.nix

@@ -1,111 +1,175 @@
 { nglib, nixpkgs }:
-nglib.makeSystem {
+{
-  system = "x86_64-linux";
+  postgresql = nglib.makeSystem {
-  name = "synapse";
+    system = "x86_64-linux";
-  inherit nixpkgs;
+    name = "nixng-synapse-postgresql";
-  config = ({ pkgs, ... }:
+    inherit nixpkgs;
-    {
+    config = { pkgs, config, ... }:
-      dumb-init = {
+      {
-        enable = true;
+        config = {
-        type.services = { };
+          dumb-init = {
+            enable = true;
+            type.services = {};
+          };
+          services.postgresql = {
+            enable = true;
+            package = pkgs.postgresql_12;
+
+            initialScript = "/secrets/init.sql";
+            enableTCPIP = true;
+
+            authentication = "host  all all all          md5";
+
+            ensureDatabases = { "synapse" = { ENCODING = "UTF8"; TEMPLATE = "template0"; }; };
+            ensureExtensions = {};
+            ensureUsers = [
+              { name = "synapse"; ensurePermissions = {
+                  "DATABASE \"synapse\"" = "ALL PRIVILEGES";
+                };
+              }
+            ];
+          };
+        };
       };
+  };
 
-      init.services.synapse = {
+  synapse = nglib.makeSystem {
-        enabled = true;
+    system = "x86_64-linux";
-        shutdownOnExit = true;
+    name = "synapse";
-        script =
+    inherit nixpkgs;
-          let
+    config = ({ pkgs, ... }:
-            logConfig = (pkgs.formats.yaml {}).generate "log.yaml"
+      {
-              {
+        dumb-init = {
-                # Log configuration for Synapse.
+          enable = true;
-                #
+          type.services = { };
-                # This is a YAML file containing a standard Python logging configuration
+        };
-                # dictionary. See [1] for details on the valid settings.
-                #
-                # Synapse also supports structured logging for machine readable logs which can
-                # be ingested by ELK stacks. See [2] for details.
-                #
-                # [1]: https://docs.python.org/3.7/library/logging.config.html#configuration-dictionary-schema
-                # [2]: https://matrix-org.github.io/synapse/latest/structured_logging.html
 
-                version = 1;
+        init.services.synapse = {
+          enabled = true;
+          shutdownOnExit = true;
+          script =
+            let
+              logConfig = (pkgs.formats.yaml {}).generate "log.yaml"
+                {
+                  # Log configuration for Synapse.
+                  #
+                  # This is a YAML file containing a standard Python logging configuration
+                  # dictionary. See [1] for details on the valid settings.
+                  #
+                  # Synapse also supports structured logging for machine readable logs which can
+                  # be ingested by ELK stacks. See [2] for details.
+                  #
+                  # [1]: https://docs.python.org/3.7/library/logging.config.html#configuration-dictionary-schema
+                  # [2]: https://matrix-org.github.io/synapse/latest/structured_logging.html
 
-                formatters.precise.format = "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s";
+                  version = 1;
-                handlers.console =
-                  {
-                    class = "logging.StreamHandler";
-                    formatter = "precise";
-                  };
-                loggers."synapse.storage.SQL" =
-                  {
-                    level = "INFO";
-                  };
-                root =
-                  {
-                    level = "INFO";
-                    handlers = [ "console" ];
-                  };
 
-                disable_existing_loggers = false;
+                  formatters.precise.format = "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s";
-              };
+                  handlers.console =
-            synapseConfig = (pkgs.formats.yaml {}).generate "conduit.yaml"
-              {
-                server_name = "matrix.redalder.org";
-                report_stats = "yes";
-                pid_file = "/homeserver.pid";
-
-                enable_registration = false;
-                enable_registration_without_verification = false;
-
-                listeners =
-                  [
                     {
-                      port = 6167;
+                      class = "logging.StreamHandler";
-                      tls = false;
+                      formatter = "precise";
-                      type = "http";
+                    };
-                      x_forwarded = true;
+                  loggers."synapse.storage.SQL" =
-                      bind_adrresses = [ "127.0.0.1" ];
-                      resources =
-                        [
-                          {
-                            names = [ "client" "federation" ];
-                            compress = false;
-                          }
-                        ];
-                    }
-                  ];
-                database =
-                  {
-                    name = "sqlite3";
-                    compress = false;
-                    args.database = "/var/lib/synapse/sqlite.db";
-                  };
-                log_config = logConfig;
-                trusted_key_servers =
-                  [
                     {
-                      server_name = "matrix.org";
+                      level = "INFO";
-                    }
+                    };
-                  ];
+                  root =
-                media_store_path = "/var/lib/synapse/media_store";
+                    {
-                signing_key_path = "/var/lib/synapse/signing.key";
+                      level = "INFO";
-              };
+                      handlers = [ "console" ];
-          in
+                    };
-            pkgs.writeShellScript "conduit"
+
-              ''
+                  disable_existing_loggers = false;
-                [ -e /var/lib/synapse/signing.key ] || \
+                };
+              synapseConfig = (pkgs.formats.yaml {}).generate "synapse.yaml"
+                {
+                  server_name = "matrix.redalder.org";
+                  report_stats = "yes";
+                  pid_file = "/homeserver.pid";
+
+                  enable_registration = false;
+                  enable_registration_without_verification = false;
+
+                  listeners =
+                    [
+                      {
+                        port = 6167;
+                        tls = false;
+                        type = "http";
+                        x_forwarded = true;
+                        bind_adrresses = [ "127.0.0.1" ];
+                        resources =
+                          [
+                            {
+                              names = [ "client" "federation" ];
+                              compress = false;
+                            }
+                          ];
+                      }
+                    ];
+                  log_config = logConfig;
+                  trusted_key_servers =
+                    [
+                      {
+                        server_name = "matrix.org";
+                      }
+                    ];
+                  media_store_path = "/var/lib/synapse/media_store";
+                  signing_key_path = "/var/lib/synapse/signing.key";
+                };
+            in
+              pkgs.writeShellScript "synapse"
+                ''
+                  if [ -f "/var/lib/synapse/sqlite.db" ] && ! [ -f "/var/lib/synapse/migration_done" ]
+                  then
+                    echo "Beginning migration from SQLite to PostgeSQL!"
+
+                    cat > /tmp/synapse_postgres.yaml <<EOF
+                  database:
+                    name: psycopg2
+                    args:
+                      user: synapse
+                      password: ''${PSQL_PASSWORD}
+                      database: synapse
+                      host: 127.0.0.1
+                      cp_min: 5
+                      cp_max: 10
+                  $(cat ${synapseConfig})
+                  EOF
+
+                    for ((i=0; i<5; i++))
+                    do
+                      ${pkgs.matrix-synapse.python.withPackages (_: [ (pkgs.python3.pkgs.toPythonModule pkgs.matrix-synapse) ] ++ pkgs.matrix-synapse.propagatedBuildInputs)}/bin/python ${pkgs.matrix-synapse}/lib/python*/site-packages/synapse/_scripts/synapse_port_db.py \
+                        --sqlite-database "/var/lib/synapse/sqlite.db" \
+                        --postgres-config "/tmp/synapse_postgres.yaml"
+                      [ $? -eq 0 ] && touch /var/lib/synapse/migration_done && break
+
+                      echo "Migration attempt ''${i}/5 failed! Retrying in 30 seconds..."
+                      sleep 30
+
+                      if ((5 == i))
+                      then
+                        echo "Migration failed!"
+                        exit 1
+                      fi
+                    done
+                  fi
+
+                  [ -e /var/lib/synapse/signing.key ] || \
+                    ${pkgs.matrix-synapse}/bin/synapse_homeserver \
+                      --config-path ${synapseConfig} \
+                      --config-path /secrets/extra.yaml \
+                      --config-path /var/lib/registrations/extra.yaml \
+                      --keys-directory /var/lib/synapse/keys \
+                      --generate-keys
                   ${pkgs.matrix-synapse}/bin/synapse_homeserver \
                     --config-path ${synapseConfig} \
                     --config-path /secrets/extra.yaml \
                     --config-path /var/lib/registrations/extra.yaml \
-                    --keys-directory /var/lib/synapse/keys \
+                    --keys-directory /var/lib/synapse/keys
-                    --generate-keys
+                '';
-                ${pkgs.matrix-synapse}/bin/synapse_homeserver \
+        };
-                  --config-path ${synapseConfig} \
+      });
-                  --config-path /secrets/extra.yaml \
+  };
-                  --config-path /var/lib/registrations/extra.yaml \
-                  --keys-directory /var/lib/synapse/keys
-              '';
-      };
-    });
 }

flake.lock

@@ -75,15 +75,16 @@
         ]
       },
       "locked": {
-        "lastModified": 1659247306,
+        "lastModified": 1667091930,
-        "narHash": "sha256-7HnMAj+OzChKVZWp5bqnTTXfKsGSIymryLOHek0bDBE=",
+        "narHash": "sha256-GLefEUOPtBb0Xj4MS6NkGFceThzJ0uMd3VVUJHyI390=",
         "owner": "nix-community",
         "repo": "NixNG",
-        "rev": "043a9a5353501b017adaa3730f09074d2f4c9514",
+        "rev": "6e0bcffbbf814dea4ae91892f255b5c65d21b23e",
         "type": "github"
       },
       "original": {
         "owner": "nix-community",
+        "ref": "postgres-db-settings",
         "repo": "NixNG",
         "type": "github"
       }

flake.nix

@@ -47,7 +47,8 @@
               home-assistantPostgresql = (import ./containers/home-assistant.nix base).postgresql;
               reicio = import ./containers/reicio.nix base;
               baikal = import ./containers/baikal.nix base;
-              conduit = import ./containers/conduit.nix base;
+              conduit = (import ./containers/conduit.nix base).synapse;
+              conduitPostgresql = (import ./containers/conduit.nix base).postgresql;
               mautrix-facebook = import ./containers/mautrix-facebook.nix base;
               heisenbridge = import ./containers/heisenbridge.nix base;
             };

nomad/regions/homelab-1/conduit.tf

@@ -23,6 +23,31 @@ resource "nomad_volume" "matrix-synapse" {
   }
 }
 
+resource "nomad_volume" "matrix-postgresql" {
+  type        = "csi"
+  plugin_id   = "org.democratic-csi.nfs"
+  volume_id   = "matrix-postgresql"
+  name        = "matrix-postgresql"
+  external_id = "matrix-postgresql"
+
+  capability {
+    access_mode = "single-node-writer"
+    attachment_mode = "file-system"
+  }
+
+  context = {
+    server = "blowhole.hosts.in.redalder.org"
+    share = "/var/nfs/matrix/postgresql"
+    node_attach_driver = "nfs"
+    provisioner_driver = "node-manual"
+  }
+
+  mount_options {
+    fs_type = "nfs"
+    mount_flags = [ "nfsvers=3", "hard", "async" ]
+  }
+}
+
 resource "nomad_volume" "matrix-registrations" {
   type        = "csi"
   plugin_id   = "org.democratic-csi.nfs"

nomad/regions/homelab-1/job/conduit.hcl

@@ -165,6 +165,15 @@ EOF
       access_mode     = "single-node-writer"
     }
 
+    volume "matrix-postgresql" {
+      type = "csi"
+      source = "matrix-postgresql"
+      read_only = false
+
+      attachment_mode = "file-system"
+      access_mode     = "single-node-writer"
+    }
+
     volume "matrix-registrations" {
       type = "csi"
       source = "matrix-registrations"
@@ -213,6 +222,43 @@ EOF
       }
     }
 
+    task "postgresql" {
+      driver = "docker"
+
+      volume_mount {
+        volume = "matrix-postgresql"
+        destination = "/var/lib/postgresql"
+        read_only = false
+      }
+
+      config {
+        nix_flake_ref = "${var.flake_ref}#nixngSystems.conduitPostgresql.config.system.build.toplevel"
+        nix_flake_sha = var.flake_sha
+        entrypoint = [ "init" ]
+      }
+
+      resources {
+        cpu = 500
+        memory = 128
+        memory_max = 256
+      }
+
+      template {
+        data = <<EOF
+alter user synapse with encrypted password '{{ with secret "kv/data/matrix/synapse" }}{{ .Data.data.pgpass }}{{ end }}';
+\c synapse;
+SELECT setval('application_services_txn_id_seq', (
+         SELECT GREATEST(MAX(txn_id), 0) FROM application_services_txns
+     ));
+EOF
+        destination = "secrets/init.sql"
+      }
+
+      vault {
+        policies = ["matrix-synapse-policy"]
+      }
+    }
+
     task "synapse" {
       driver = "docker"
 
@@ -246,9 +292,29 @@ EOF
       template {
         data = <<EOF
 {{ with secret "kv/data/matrix/synapse" }}
+PSQL_PASSWORD={{ .Data.data.pgpass }}
+{{ end }}
+EOF
+        destination = "secrets/environment"
+        env = true
+        perms = "400"
+      }
+
+      template {
+        data = <<EOF
+{{ with secret "kv/data/matrix/synapse" }}
 registration_shared_secret: "{{ .Data.data.registration_shared_secret }}"
 macaroon_secret_key: "{{ .Data.data.macaroon_secret_key }}"
 form_secret: "{{ .Data.data.form_secret }}"
+database:
+  name: "psycopg2"
+  args:
+    user: "synapse"
+    password:  "{{ .Data.data.pgpass }}"
+    database: "synapse"
+    host: "127.0.0.1"
+    cp_min: 5
+    cp_max: 10
 {{ end }}
 EOF
         destination = "/secrets/extra.yaml"

nomad/regions/homelab-1/job/home-assistant.hcl

@@ -236,7 +236,7 @@ EOF
       }
 
       config {
-        nix_flake_ref = "${var.flake_ref}#nixngSystems.home-assistant.postgresql.config.system.build.toplevel"
+        nix_flake_ref = "${var.flake_ref}#nixngSystems.home-assistantPostgresql.config.system.build.toplevel"
         nix_flake_sha = var.flake_sha
         entrypoint = [ "init" ]
       }