feat(server/multi region config): mount file and read config from file (#3407)

* feat(multi region config): mount file and read config from file
* feat(helm): allow multi region config to be mounted from a secret
* Allow the file name to be amended

packages/server/modules/multiregion/domain/operations.ts

@@ -1,5 +1,3 @@
-import { RegionServerConfig } from '@/modules/multiregion/domain/types'
+import { MultiRegionConfig } from '@/modules/multiregion/domain/types'
 
-export type GetAvailableRegionConfigs = () => Promise<{
-  [key: string]: RegionServerConfig
-}>
+export type GetAvailableRegionConfigs = () => Promise<MultiRegionConfig>

packages/server/modules/multiregion/domain/types.ts

@@ -1,12 +1,4 @@
-export type RegionServerConfig = {
-  postgres: {
-    /**
-     * Full Postgres connection URI (e.g. "postgres://user:password@host:port/dbname")
-     */
-    connectionUri: string
-    /**
-     * SSL cert, if any
-     */
-    publicTlsCertificate?: string
-  }
-}
+import { z } from 'zod'
+import { multiRegionConfigSchema } from '@/modules/multiregion/helpers/validation'
+
+export type MultiRegionConfig = z.infer<typeof multiRegionConfigSchema>

packages/server/modules/multiregion/helpers/validation.ts

@@ -0,0 +1,25 @@
+import { z } from 'zod'
+
+export const regionServerConfigSchema = z.object({
+  postgres: z.object({
+    connectionUri: z
+      .string()
+      .url()
+      .describe(
+        'Full Postgres connection URI (e.g. "postgres://user:password@host:port/dbname")'
+      ),
+    publicTlsCertificate: z
+      .string()
+      .describe('Public TLS ("CA") certificate for the Postgres server')
+  })
+  //TODO - add the rest of the config when blob storage is implemented
+  // blobStorage: z
+  //   .object({
+  //     endpoint: z.string().url(),
+  //     accessKey: z.string(),
+  //     secretKey: z.string(),
+  //     bucket: z.string()
+  //   })
+})
+
+export const multiRegionConfigSchema = z.record(z.string(), regionServerConfigSchema)

packages/server/modules/multiregion/services/config.ts

@@ -1,14 +1,26 @@
-import { GetAvailableRegionConfigs } from '@/modules/multiregion/domain/operations'
+import { packageRoot } from '@/bootstrap'
+import path from 'node:path'
+import fs from 'node:fs/promises'
+import { getMultiRegionConfigPath } from '@/modules/shared/helpers/envHelper'
+import type { Optional } from '@speckle/shared'
+import type { GetAvailableRegionConfigs } from '@/modules/multiregion/domain/operations'
+import { type MultiRegionConfig } from '@/modules/multiregion/domain/types'
+import { multiRegionConfigSchema } from '@/modules/multiregion/helpers/validation'
+
+let multiRegionConfig: Optional<MultiRegionConfig> = undefined
 
 export const getAvailableRegionConfigsFactory =
   (): GetAvailableRegionConfigs => async () => {
-    // TODO: Hardcoded for now, should be fetched from a config file
-    return {
-      eu: {
-        postgres: {
-          connectionUri: 'postgresql://speckle:speckle@localhost/speckle_eu',
-          publicTlsCertificate: undefined
-        }
-      }
-    }
+    if (multiRegionConfig) return multiRegionConfig
+
+    const relativePath = getMultiRegionConfigPath() // This will throw if the path is not set
+    const fullPath = path.resolve(packageRoot, relativePath)
+    const file = await fs.readFile(fullPath, 'utf-8')
+
+    const parsedJson = JSON.parse(file) // This will throw if the file is not valid JSON
+
+    const multiRegionConfigFileContents = multiRegionConfigSchema.parse(parsedJson) // This will throw if the config is invalid
+
+    multiRegionConfig = multiRegionConfigFileContents
+    return multiRegionConfig
   }

packages/server/modules/shared/helpers/envHelper.ts

@@ -412,3 +412,7 @@ export function getOtelTraceKey() {
 export function getOtelHeaderValue() {
   return getStringFromEnv('OTEL_TRACE_VALUE')
 }
+
+export function getMultiRegionConfigPath() {
+  return getStringFromEnv('MULTI_REGION_CONFIG_PATH')
+}

utils/helm/speckle-server/templates/_helpers.tpl

@@ -1060,4 +1060,8 @@ Generate the environment variables for Speckle server and Speckle objects deploy
 - name: OTEL_TRACE_VALUE
   value: {{ .Values.openTelemetry.tracing.value | quote }}
 {{- end }}
+{{- if .Values.featureFlags.workspacesMultiRegionEnabled }}
+- name: MULTI_REGION_CONFIG_PATH
+  value: {{ (printf "/%s" .Values.multiRegion.config.secretKey) | quote}}
+{{- end }}
 {{- end }}

utils/helm/speckle-server/templates/objects/deployment.yml

@@ -57,6 +57,15 @@ spec:
           - name: postgres-certificate
             mountPath: /postgres-certificate
         {{- end }}
+        {{- if .Values.featureFlags.automateModuleEnabled }}
+          - name: encryption-keys
+            readOnly: true
+            mountPath: /encryption-keys
+        {{- end }}
+        {{- if .Values.featureFlags.workspacesMultiRegionEnabled }}
+          - name: multi-region-config
+            mountPath: /multi-region-config
+        {{- end }}
 
         # Allow for k8s to remove the pod from the service endpoints to stop receive traffic
         lifecycle:
@@ -128,3 +137,13 @@ spec:
           configMap:
             name: postgres-certificate
       {{- end }}
+      {{- if .Values.featureFlags.automateModuleEnabled }}
+        - name: encryption-keys
+          secret:
+            secretName: encryption-keys
+      {{- end }}
+      {{- if .Values.featureFlags.workspacesMultiRegionEnabled }}
+        - name: multi-region-config
+          secret:
+            secretName: {{ .Values.multiRegion.config.secretName }}
+      {{- end }}

utils/helm/speckle-server/templates/server/deployment.yml

@@ -62,6 +62,11 @@ spec:
             readOnly: true
             mountPath: /encryption-keys
         {{- end }}
+        {{- if .Values.featureFlags.workspacesMultiRegionEnabled }}
+          - name: multi-region-config
+            mountPath: /
+            readOnly: true
+        {{- end }}
 
         # Allow for k8s to remove the pod from the service endpoints to stop receive traffic
         lifecycle:
@@ -137,3 +142,11 @@ spec:
           secret:
             secretName: encryption-keys
       {{- end }}
+      {{- if .Values.featureFlags.workspacesMultiRegionEnabled }}
+        - name: multi-region-config
+          secret:
+            secretName: {{ .Values.multiRegion.config.secretName }}
+            items:
+            - key: {{ .Values.multiRegion.config.secretKey }}
+              path: "/"
+      {{- end }}

utils/helm/speckle-server/values.schema.json

@@ -74,6 +74,11 @@
           "type": "boolean",
           "description": "High level flag that enables the billing integration",
           "default": false
+        },
+        "workspacesMultiRegionEnabled": {
+          "type": "boolean",
+          "description": "Toggles whether multi-region is available within workspaces. workspaceModuleEnabled must also be enabled.",
+          "default": false
         }
       }
     },
@@ -523,6 +528,26 @@
         }
       }
     },
+    "multiRegion": {
+      "type": "object",
+      "properties": {
+        "config": {
+          "type": "object",
+          "properties": {
+            "secretName": {
+              "type": "string",
+              "description": "If workspacesMultiRegionEnabled is enabled, the server will be deployed in a multi-region configuration based on the values in a secret. This allows the default secret name to be overridden.",
+              "default": "multi-region-config"
+            },
+            "secretKey": {
+              "type": "string",
+              "description": "If workspacesMultiRegionEnabled is enabled, the server will be deployed in a multi-region configuration based on the values in a secret. This allows the default secret key and filename to be overridden.",
+              "default": "multi-region-config.json"
+            }
+          }
+        }
+      }
+    },
     "server": {
       "type": "object",
       "properties": {

utils/helm/speckle-server/values.yaml

@@ -51,6 +51,8 @@ featureFlags:
   gatekeeperModuleEnabled: false
   ## @param featureFlags.billingIntegrationEnabled High level flag that enables the billing integration
   billingIntegrationEnabled: false
+  ## @param featureFlags.workspacesMultiRegionEnabled Toggles whether multi-region is available within workspaces. workspaceModuleEnabled must also be enabled.
+  workspacesMultiRegionEnabled: false
 
 analytics:
   ## @param analytics.enabled Enable or disable analytics
@@ -403,6 +405,13 @@ openTelemetry:
     ##
     value: ''
 
+multiRegion:
+  config:
+    ## @param multiRegion.config.secretName If workspacesMultiRegionEnabled is enabled, the server will be deployed in a multi-region configuration based on the values in a secret. This allows the default secret name to be overridden.
+    secretName: 'multi-region-config'
+    ## @param multiRegion.config.secretKey If workspacesMultiRegionEnabled is enabled, the server will be deployed in a multi-region configuration based on the values in a secret. This allows the default secret key and filename to be overridden.
+    secretKey: 'multi-region-config.json'
+
 ## @section Server
 ## @descriptionStart
 ## Defines parameters related to the backend server component of Speckle.