feat(fileimport service): allow dedicated Redis server for queue (#4858)

docker-compose-speckle.yml

@@ -73,6 +73,7 @@ services:
       REDIS_URL: 'redis://redis'
       PREVIEW_SERVICE_USE_PRIVATE_OBJECTS_SERVER_URL: 'true'
       PREVIEW_SERVICE_REDIS_URL: 'redis://redis'
+      FILEIMPORT_SERVICE_REDIS_URL: 'redis://redis'
 
       S3_ENDPOINT: 'http://minio:9000'
       S3_ACCESS_KEY: 'minioadmin'

packages/fileimport-service/src/nextGen/main.ts

@@ -17,6 +17,7 @@ let appState: AppState = AppState.STARTING
 let currentJob: { logger: Logger; done: Bull.DoneCallback } | undefined = undefined
 
 export const main = async () => {
+  logger.info('Starting FileUploads Service (nextGen 🚀)...')
   // we discussed doing push based metrics from here
   // await initMetrics({ app, registry: initPrometheusRegistry() })
 

packages/server/modules/fileuploads/queues/fileimports.ts

@@ -1,5 +1,6 @@
 import { UninitializedResourceAccessError } from '@/modules/shared/errors'
 import {
+  getFileimportServiceRedisUrl,
   getFileUploadTimeLimitMinutes,
   getRedisUrl,
   isProdEnv,
@@ -45,7 +46,7 @@ const defaultJobOptions = {
 export const initializeQueue = async () => {
   queue = await setupQueue({
     queueName: FILE_IMPORT_SERVICE_QUEUE_NAME,
-    redisUrl: getRedisUrl(),
+    redisUrl: getFileimportServiceRedisUrl() ?? getRedisUrl(),
     options: {
       ...(!isTestEnv() ? { limiter } : {}),
       defaultJobOptions

packages/server/modules/shared/helpers/envHelper.ts

@@ -128,6 +128,10 @@ export const previewServiceShouldUsePrivateObjectsServerUrl = (): boolean => {
   return getBooleanFromEnv('PREVIEW_SERVICE_USE_PRIVATE_OBJECTS_SERVER_URL')
 }
 
+export const getFileimportServiceRedisUrl = (): string | undefined => {
+  return process.env['FILEIMPORT_SERVICE_REDIS_URL']
+}
+
 export const getPreviewServiceRedisUrl = (): string | undefined => {
   return process.env['PREVIEW_SERVICE_REDIS_URL']
 }

utils/helm/speckle-server/templates/_helpers.tpl

@@ -779,6 +779,14 @@ Generate the environment variables for Speckle server and Speckle objects deploy
       key: {{ default "preview_service_redis_url" .Values.redis.previewServiceConnectionString.secretKey }}
 {{- end }}
 
+{{- if .Values.featureFlags.nextGenFileImporterEnabled }}
+- name: FILEIMPORT_SERVICE_REDIS_URL
+  valueFrom:
+    secretKeyRef:
+      name: {{ default .Values.secretName .Values.redis.fileimportServiceConnectionString.secretName }}
+      key: {{ default "fileimport_service_redis_url" .Values.redis.fileimportServiceConnectionString.secretKey }}
+{{- end }}
+
 # *** PostgreSQL Database ***
 - name: POSTGRES_URL
   valueFrom:

utils/helm/speckle-server/templates/fileimport_service/deployment.yml

@@ -85,11 +85,13 @@ spec:
           - name: SPECKLE_SERVER_URL
             value: {{ printf "http://%s:%s" ( include "objects.service.fqdn" $ ) ( include "objects.port" $ ) }}
 
+          {{- if .Values.featureFlags.nextGenFileImporterEnabled }}
           - name: REDIS_URL
             valueFrom:
               secretKeyRef:
-                name: {{ default .Values.secretName .Values.redis.connectionString.secretName }}
-                key: {{ default "redis_url" .Values.redis.connectionString.secretKey }}
+                name: {{ default .Values.secretName .Values.redis.fileimportServiceConnectionString.secretName }}
+                key: {{ default "fileimport_service_redis_url" .Values.redis.fileimportServiceConnectionString.secretKey }}
+          {{- end }}
 
           - name: PG_CONNECTION_STRING
             valueFrom:

utils/helm/speckle-server/templates/fileimport_service/serviceaccount.yml

@@ -20,5 +20,8 @@ secrets:
 {{- if .Values.featureFlags.workspacesMultiRegionEnabled }}
   - name: {{ default .Values.secretName .Values.multiRegion.config.secretName }}
 {{- end }}
+{{- if .Values.featureFlags.nextGenFileImporterEnabled }}
+  - name: {{ default .Values.secretName .Values.redis.fileimportServiceConnectionString.secretName }}
+{{- end }}
 {{- end -}}
 {{- end }}

utils/helm/speckle-server/templates/objects/serviceaccount.yml

@@ -52,5 +52,8 @@ secrets:
 {{- if .Values.preview_service.dedicatedPreviewsQueue }}
   - name: {{ default .Values.secretName .Values.redis.previewServiceConnectionString.secretName }}
 {{- end }}
+{{- if .Values.featureFlags.nextGenFileImporterEnabled }}
+  - name: {{ default .Values.secretName .Values.redis.fileimportServiceConnectionString.secretName }}
+{{- end }}
 {{- end -}}
 {{- end }}

utils/helm/speckle-server/templates/server/serviceaccount.yml

@@ -52,5 +52,8 @@ secrets:
 {{- if .Values.preview_service.dedicatedPreviewsQueue }}
   - name: {{ default .Values.secretName .Values.redis.previewServiceConnectionString.secretName }}
 {{- end }}
+{{- if .Values.featureFlags.nextGenFileImporterEnabled }}
+  - name: {{ default .Values.secretName .Values.redis.fileimportServiceConnectionString.secretName }}
+{{- end }}
 {{- end -}}
 {{- end }}

utils/helm/speckle-server/values.schema.json

@@ -521,6 +521,21 @@
             }
           }
         },
+        "fileimportServiceConnectionString": {
+          "type": "object",
+          "properties": {
+            "secretName": {
+              "type": "string",
+              "description": "Required. A secret containing the full connection string to the Redis store (e.g. in format of `protocol://username:password@host:port/database`) stored within the Kubernetes cluster as an opaque Kubernetes Secret. Ref: https://kubernetes.io/docs/concepts/configuration/secret/#opaque-secrets",
+              "default": ""
+            },
+            "secretKey": {
+              "type": "string",
+              "description": "Required. The key within the Kubernetes Secret holding the Redis connection string.",
+              "default": ""
+            }
+          }
+        },
         "networkPolicy": {
           "type": "object",
           "properties": {

utils/helm/speckle-server/values.yaml

@@ -390,6 +390,14 @@ redis:
     ##
     secretKey: ''
 
+  fileimportServiceConnectionString:
+    ## @param redis.fileimportServiceConnectionString.secretName Required. A secret containing the full connection string to the Redis store (e.g. in format of `protocol://username:password@host:port/database`) stored within the Kubernetes cluster as an opaque Kubernetes Secret. Ref: https://kubernetes.io/docs/concepts/configuration/secret/#opaque-secrets
+    ##
+    secretName: ''
+    ## @param redis.fileimportServiceConnectionString.secretKey Required. The key within the Kubernetes Secret holding the Redis connection string.
+    ##
+    secretKey: ''
+
   ## @extra redis.networkPolicy If networkPolicy is enabled for Speckle server, this provides the NetworkPolicy with the necessary details to allow egress connections to the Redis store
   ##
   networkPolicy: