From dae238f705ea7343c312f57365eca9cac31fb24a Mon Sep 17 00:00:00 2001 From: Iain Sproat <68657+iainsproat@users.noreply.github.com> Date: Tue, 3 Jun 2025 08:55:54 +0100 Subject: [PATCH] feat(fileimport service): allow dedicated Redis server for queue (#4858) --- docker-compose-speckle.yml | 1 + packages/fileimport-service/src/nextGen/main.ts | 1 + .../modules/fileuploads/queues/fileimports.ts | 3 ++- .../server/modules/shared/helpers/envHelper.ts | 4 ++++ utils/helm/speckle-server/templates/_helpers.tpl | 8 ++++++++ .../templates/fileimport_service/deployment.yml | 6 ++++-- .../fileimport_service/serviceaccount.yml | 3 +++ .../templates/objects/serviceaccount.yml | 3 +++ .../templates/server/serviceaccount.yml | 3 +++ utils/helm/speckle-server/values.schema.json | 15 +++++++++++++++ utils/helm/speckle-server/values.yaml | 8 ++++++++ 11 files changed, 52 insertions(+), 3 deletions(-) diff --git a/docker-compose-speckle.yml b/docker-compose-speckle.yml index 658d2ccc2..4bf1b2f8a 100644 --- a/docker-compose-speckle.yml +++ b/docker-compose-speckle.yml @@ -73,6 +73,7 @@ services: REDIS_URL: 'redis://redis' PREVIEW_SERVICE_USE_PRIVATE_OBJECTS_SERVER_URL: 'true' PREVIEW_SERVICE_REDIS_URL: 'redis://redis' + FILEIMPORT_SERVICE_REDIS_URL: 'redis://redis' S3_ENDPOINT: 'http://minio:9000' S3_ACCESS_KEY: 'minioadmin' diff --git a/packages/fileimport-service/src/nextGen/main.ts b/packages/fileimport-service/src/nextGen/main.ts index 453ccbd32..e7bf5424c 100644 --- a/packages/fileimport-service/src/nextGen/main.ts +++ b/packages/fileimport-service/src/nextGen/main.ts @@ -17,6 +17,7 @@ let appState: AppState = AppState.STARTING let currentJob: { logger: Logger; done: Bull.DoneCallback } | undefined = undefined export const main = async () => { + logger.info('Starting FileUploads Service (nextGen 🚀)...') // we discussed doing push based metrics from here // await initMetrics({ app, registry: initPrometheusRegistry() }) diff --git a/packages/server/modules/fileuploads/queues/fileimports.ts b/packages/server/modules/fileuploads/queues/fileimports.ts index dda5d4707..ab56a7a93 100644 --- a/packages/server/modules/fileuploads/queues/fileimports.ts +++ b/packages/server/modules/fileuploads/queues/fileimports.ts @@ -1,5 +1,6 @@ import { UninitializedResourceAccessError } from '@/modules/shared/errors' import { + getFileimportServiceRedisUrl, getFileUploadTimeLimitMinutes, getRedisUrl, isProdEnv, @@ -45,7 +46,7 @@ const defaultJobOptions = { export const initializeQueue = async () => { queue = await setupQueue({ queueName: FILE_IMPORT_SERVICE_QUEUE_NAME, - redisUrl: getRedisUrl(), + redisUrl: getFileimportServiceRedisUrl() ?? getRedisUrl(), options: { ...(!isTestEnv() ? { limiter } : {}), defaultJobOptions diff --git a/packages/server/modules/shared/helpers/envHelper.ts b/packages/server/modules/shared/helpers/envHelper.ts index 621aa365e..862a5eb8c 100644 --- a/packages/server/modules/shared/helpers/envHelper.ts +++ b/packages/server/modules/shared/helpers/envHelper.ts @@ -128,6 +128,10 @@ export const previewServiceShouldUsePrivateObjectsServerUrl = (): boolean => { return getBooleanFromEnv('PREVIEW_SERVICE_USE_PRIVATE_OBJECTS_SERVER_URL') } +export const getFileimportServiceRedisUrl = (): string | undefined => { + return process.env['FILEIMPORT_SERVICE_REDIS_URL'] +} + export const getPreviewServiceRedisUrl = (): string | undefined => { return process.env['PREVIEW_SERVICE_REDIS_URL'] } diff --git a/utils/helm/speckle-server/templates/_helpers.tpl b/utils/helm/speckle-server/templates/_helpers.tpl index b7d9c1902..9adc81872 100644 --- a/utils/helm/speckle-server/templates/_helpers.tpl +++ b/utils/helm/speckle-server/templates/_helpers.tpl @@ -779,6 +779,14 @@ Generate the environment variables for Speckle server and Speckle objects deploy key: {{ default "preview_service_redis_url" .Values.redis.previewServiceConnectionString.secretKey }} {{- end }} +{{- if .Values.featureFlags.nextGenFileImporterEnabled }} +- name: FILEIMPORT_SERVICE_REDIS_URL + valueFrom: + secretKeyRef: + name: {{ default .Values.secretName .Values.redis.fileimportServiceConnectionString.secretName }} + key: {{ default "fileimport_service_redis_url" .Values.redis.fileimportServiceConnectionString.secretKey }} +{{- end }} + # *** PostgreSQL Database *** - name: POSTGRES_URL valueFrom: diff --git a/utils/helm/speckle-server/templates/fileimport_service/deployment.yml b/utils/helm/speckle-server/templates/fileimport_service/deployment.yml index 84fea7cbc..88e96a554 100644 --- a/utils/helm/speckle-server/templates/fileimport_service/deployment.yml +++ b/utils/helm/speckle-server/templates/fileimport_service/deployment.yml @@ -85,11 +85,13 @@ spec: - name: SPECKLE_SERVER_URL value: {{ printf "http://%s:%s" ( include "objects.service.fqdn" $ ) ( include "objects.port" $ ) }} + {{- if .Values.featureFlags.nextGenFileImporterEnabled }} - name: REDIS_URL valueFrom: secretKeyRef: - name: {{ default .Values.secretName .Values.redis.connectionString.secretName }} - key: {{ default "redis_url" .Values.redis.connectionString.secretKey }} + name: {{ default .Values.secretName .Values.redis.fileimportServiceConnectionString.secretName }} + key: {{ default "fileimport_service_redis_url" .Values.redis.fileimportServiceConnectionString.secretKey }} + {{- end }} - name: PG_CONNECTION_STRING valueFrom: diff --git a/utils/helm/speckle-server/templates/fileimport_service/serviceaccount.yml b/utils/helm/speckle-server/templates/fileimport_service/serviceaccount.yml index e2178c989..fcf364f7d 100644 --- a/utils/helm/speckle-server/templates/fileimport_service/serviceaccount.yml +++ b/utils/helm/speckle-server/templates/fileimport_service/serviceaccount.yml @@ -20,5 +20,8 @@ secrets: {{- if .Values.featureFlags.workspacesMultiRegionEnabled }} - name: {{ default .Values.secretName .Values.multiRegion.config.secretName }} {{- end }} +{{- if .Values.featureFlags.nextGenFileImporterEnabled }} + - name: {{ default .Values.secretName .Values.redis.fileimportServiceConnectionString.secretName }} +{{- end }} {{- end -}} {{- end }} diff --git a/utils/helm/speckle-server/templates/objects/serviceaccount.yml b/utils/helm/speckle-server/templates/objects/serviceaccount.yml index fe7303b2c..f18f03824 100644 --- a/utils/helm/speckle-server/templates/objects/serviceaccount.yml +++ b/utils/helm/speckle-server/templates/objects/serviceaccount.yml @@ -52,5 +52,8 @@ secrets: {{- if .Values.preview_service.dedicatedPreviewsQueue }} - name: {{ default .Values.secretName .Values.redis.previewServiceConnectionString.secretName }} {{- end }} +{{- if .Values.featureFlags.nextGenFileImporterEnabled }} + - name: {{ default .Values.secretName .Values.redis.fileimportServiceConnectionString.secretName }} +{{- end }} {{- end -}} {{- end }} diff --git a/utils/helm/speckle-server/templates/server/serviceaccount.yml b/utils/helm/speckle-server/templates/server/serviceaccount.yml index 4359e8558..00adc46a8 100644 --- a/utils/helm/speckle-server/templates/server/serviceaccount.yml +++ b/utils/helm/speckle-server/templates/server/serviceaccount.yml @@ -52,5 +52,8 @@ secrets: {{- if .Values.preview_service.dedicatedPreviewsQueue }} - name: {{ default .Values.secretName .Values.redis.previewServiceConnectionString.secretName }} {{- end }} +{{- if .Values.featureFlags.nextGenFileImporterEnabled }} + - name: {{ default .Values.secretName .Values.redis.fileimportServiceConnectionString.secretName }} +{{- end }} {{- end -}} {{- end }} diff --git a/utils/helm/speckle-server/values.schema.json b/utils/helm/speckle-server/values.schema.json index ac2fd757d..e0808d92b 100644 --- a/utils/helm/speckle-server/values.schema.json +++ b/utils/helm/speckle-server/values.schema.json @@ -521,6 +521,21 @@ } } }, + "fileimportServiceConnectionString": { + "type": "object", + "properties": { + "secretName": { + "type": "string", + "description": "Required. A secret containing the full connection string to the Redis store (e.g. in format of `protocol://username:password@host:port/database`) stored within the Kubernetes cluster as an opaque Kubernetes Secret. Ref: https://kubernetes.io/docs/concepts/configuration/secret/#opaque-secrets", + "default": "" + }, + "secretKey": { + "type": "string", + "description": "Required. The key within the Kubernetes Secret holding the Redis connection string.", + "default": "" + } + } + }, "networkPolicy": { "type": "object", "properties": { diff --git a/utils/helm/speckle-server/values.yaml b/utils/helm/speckle-server/values.yaml index dfbc5c328..0a8b10c31 100644 --- a/utils/helm/speckle-server/values.yaml +++ b/utils/helm/speckle-server/values.yaml @@ -390,6 +390,14 @@ redis: ## secretKey: '' + fileimportServiceConnectionString: + ## @param redis.fileimportServiceConnectionString.secretName Required. A secret containing the full connection string to the Redis store (e.g. in format of `protocol://username:password@host:port/database`) stored within the Kubernetes cluster as an opaque Kubernetes Secret. Ref: https://kubernetes.io/docs/concepts/configuration/secret/#opaque-secrets + ## + secretName: '' + ## @param redis.fileimportServiceConnectionString.secretKey Required. The key within the Kubernetes Secret holding the Redis connection string. + ## + secretKey: '' + ## @extra redis.networkPolicy If networkPolicy is enabled for Speckle server, this provides the NetworkPolicy with the necessary details to allow egress connections to the Redis store ## networkPolicy: