huanld/speckle-server
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

chore(feature flags): merge background jobs with next gen file importer (#5273)

Browse Source 
* chore(feature flags): remove legacyIfcImporterEnabled & experimentalIfcImporterEnabled
- experimentalIfcImporterEnabled is now assumed to be permanently true
- legacyIfcImporterEnabled is now assumed to be permanently false

* chore(feature flags): merge background jobs with next gen file importer

* remove reference to Redis from metrics

* fix README linting

* Update README instructions

* Add port for healthz to definition
This commit is contained in:
Iain Sproat
2025-08-26 12:21:31 +01:00
committed by GitHub
parent e9d06a48ee
commit ea6676238e
26 changed files with 47 additions and 1059 deletions
Download Patch File Download Diff File Expand all files Collapse all files
utils/helm/speckle-server/templates/_helpers.tpl
+1 -24
View File
@@ -578,9 +578,6 @@ Generate the environment variables for Speckle server and Speckle objects deploy
- name: FF_MOVE_PROJECT_REGION_ENABLED
value: {{ .Values.featureFlags.moveProjectRegionEnabled | quote }}
- name: FF_BACKGROUND_JOBS_ENABLED
value: {{ .Values.featureFlags.backgroundJobsEnabled | quote }}
{{- if .Values.featureFlags.gatekeeperModuleEnabled }}
- name: LICENSE_TOKEN
valueFrom:
@@ -812,26 +809,6 @@ Generate the environment variables for Speckle server and Speckle objects deploy
key: {{ default "preview_service_redis_url" .Values.redis.previewServiceConnectionString.secretKey }}
{{- end }}
{{- if (and .Values.featureFlags.nextGenFileImporterEnabled (not .Values.featureFlags.backgroundJobsEnabled)) }}
- name: FILEIMPORT_SERVICE_RHINO_REDIS_URL
valueFrom:
secretKeyRef:
name: {{ default .Values.secretName .Values.redis.fileImportService.rhino.connectionString.secretName }}
key: {{ default "fileimport_service_rhino_redis_url" .Values.redis.fileImportService.rhino.connectionString.secretKey }}
- name: FILEIMPORT_SERVICE_RHINO_QUEUE_NAME
value: {{ .Values.redis.fileImportService.rhino.queueName | quote }}
- name: FILEIMPORT_SERVICE_IFC_REDIS_URL
valueFrom:
secretKeyRef:
name: {{ default .Values.secretName .Values.redis.fileImportService.ifc.connectionString.secretName }}
key: {{ default "fileimport_service_ifc_redis_url" .Values.redis.fileImportService.ifc.connectionString.secretKey }}
- name: FILEIMPORT_SERVICE_IFC_QUEUE_NAME
value: {{ .Values.redis.fileImportService.ifc.queueName | quote }}
{{- end }}
# *** PostgreSQL Database ***
- name: POSTGRES_URL
valueFrom:
@@ -1175,7 +1152,7 @@ Generate the environment variables for Speckle server and Speckle objects deploy
value: {{ .Values.featureFlags.rhinoFileImporterEnabled | quote }}
{{- end }}
{{- if .Values.featureFlags.backgroundJobsEnabled }}
{{- if .Values.featureFlags.nextGenFileImporterEnabled }}
- name: FILEIMPORT_QUEUE_POSTGRES_URL
valueFrom:
secretKeyRef:
utils/helm/speckle-server/templates/fileimport_service/deployment.yml
+1 -44
View File
@@ -41,49 +41,31 @@ spec:
periodSeconds: 10
failureThreshold: 18 # 10 * 18 = 180s to startup
timeoutSeconds: 3
{{- if .Values.featureFlags.nextGenFileImporterEnabled }}
httpGet:
path: /healthz
port: 9080
{{- else }}
exec:
command:
- /usr/bin/node
- -e
- "process.exit((Date.now() - require('fs').readFileSync('/tmp/last_successful_query', 'utf8') > 25 * 60 * 1000) ? 1 : 0)"
{{- end }}
livenessProbe:
initialDelaySeconds: 60
periodSeconds: 60
timeoutSeconds: 3
{{- if .Values.featureFlags.nextGenFileImporterEnabled }}
httpGet:
path: /healthz
port: 9080
{{- else }}
exec:
command:
- /usr/bin/node
- -e
- "process.exit((Date.now() - require('fs').readFileSync('/tmp/last_successful_query', 'utf8') > 25 * 60 * 1000) ? 1 : 0)"
{{- end }}
readinessProbe:
initialDelaySeconds: 60
periodSeconds: 10
timeoutSeconds: 3
{{- if .Values.featureFlags.nextGenFileImporterEnabled }}
httpGet:
path: /healthz
port: 9080
{{- else }}
exec:
command:
- /usr/bin/node
- -e
- "process.exit((Date.now() - require('fs').readFileSync('/tmp/last_successful_query', 'utf8') > 25 * 60 * 1000) ? 1 : 0)"
{{- end }}
resources:
{{- with .Values.fileimport_service.requests }}
@@ -121,26 +103,9 @@ spec:
{{- end }}
env:
{{- if (not .Values.featureFlags.nextGenFileImporterEnabled) }}
- name: SPECKLE_SERVER_URL
value: {{ printf "http://%s:%s" ( include "objects.service.fqdn" $ ) ( include "objects.port" $ ) }}
{{- end }}
{{- if .Values.featureFlags.nextGenFileImporterEnabled }}
- name: REDIS_URL
valueFrom:
secretKeyRef:
name: {{ default .Values.secretName .Values.redis.fileImportService.ifc.connectionString.secretName }}
key: {{ default "fileimport_service_ifc_redis_url" .Values.redis.fileImportService.ifc.connectionString.secretKey }}
- name: QUEUE_NAME
value: {{ .Values.redis.fileImportService.ifc.queueName | quote }}
{{- else }}
# Dummy value for backwards compatibility
- name: REDIS_URL
value: "redis://redis"
{{- end }}
{{- if (not .Values.featureFlags.nextGenFileImporterEnabled) }}
- name: PG_CONNECTION_STRING
valueFrom:
secretKeyRef:
@@ -157,7 +122,6 @@ spec:
- name: LOG_LEVEL
value: {{ .Values.fileimport_service.logLevel | quote }}
- name: LOG_PRETTY
value: {{ .Values.fileimport_service.logPretty | quote }}
@@ -165,25 +129,18 @@ spec:
- name: NODE_EXTRA_CA_CERTS
value: "/postgres-certificate/ca-certificate.crt"
{{- end }}
{{- end }}
- name: FILE_IMPORT_TIME_LIMIT_MIN
value: {{ (or .Values.file_import_time_limit_min .Values.fileimport_service.time_limit_min) | quote }}
{{- if (not .Values.featureFlags.nextGenFileImporterEnabled) }}
# DEPRECATED
- name: MAX_OBJECT_SIZE_MB
value: {{ .Values.server.max_object_size_mb | quote }}
{{- end }}
{{- if (and .Values.featureFlags.workspacesMultiRegionEnabled (not .Values.featureFlags.nextGenFileImporterEnabled)) }}
{{- if .Values.featureFlags.workspacesMultiRegionEnabled }}
- name: FF_WORKSPACES_MULTI_REGION_ENABLED
value: {{ .Values.featureFlags.workspacesMultiRegionEnabled | quote }}
- name: MULTI_REGION_CONFIG_PATH
value: "/multi-region-config/multi-region-config.json"
{{- end }}
{{- if .Values.featureFlags.nextGenFileImporterEnabled }}
- name: FF_NEXT_GEN_FILE_IMPORTER_ENABLED
value: {{ .Values.featureFlags.nextGenFileImporterEnabled | quote }}
{{- end }}
{{- with .Values.fileimport_service.additionalEnvVars }}
{{- toYaml . | nindent 10}}
{{- end }}
utils/helm/speckle-server/templates/fileimport_service/serviceaccount.yml
-4
View File
@@ -20,9 +20,5 @@ secrets:
{{- if .Values.featureFlags.workspacesMultiRegionEnabled }}
- name: {{ default .Values.secretName .Values.multiRegion.config.secretName }}
{{- end }}
{{- if .Values.featureFlags.nextGenFileImporterEnabled }}
- name: {{ default .Values.secretName .Values.redis.fileImportService.rhino.connectionString.secretName }}
- name: {{ default .Values.secretName .Values.redis.fileImportService.ifc.connectionString.secretName }}
{{- end }}
{{- end -}}
{{- end }}
utils/helm/speckle-server/templates/ifc_import_service/configmap-db-certificate.yml
+1 -1
View File
@@ -1,4 +1,4 @@
{{ if ( and .Values.featureFlags.backgroundJobsEnabled .Values.ifc_import_service.db.certificate.enabled .Values.ifc_import_service.db.certificate.data ) }}
{{ if ( and .Values.ifc_import_service.enabled .Values.ifc_import_service.db.certificate.enabled .Values.ifc_import_service.db.certificate.data ) }}
apiVersion: v1
kind: ConfigMap
utils/helm/speckle-server/templates/ifc_import_service/deployment.yml
+4 -1
View File
@@ -1,4 +1,4 @@
{{- if (and .Values.ifc_import_service.enabled .Values.featureFlags.backgroundJobsEnabled) }}
{{- if .Values.ifc_import_service.enabled }}
apiVersion: apps/v1
kind: Deployment
metadata:
@@ -30,6 +30,9 @@ spec:
- name: metrics
containerPort: 9093
protocol: TCP
- name: healthz
containerPort: 9080
protocol: TCP
startupProbe:
periodSeconds: 10
utils/helm/speckle-server/templates/ifc_import_service/service.yml
+1 -1
View File
@@ -1,4 +1,4 @@
{{- if (and .Values.ifc_import_service.enabled .Values.featureFlags.backgroundJobsEnabled) }}
{{- if .Values.ifc_import_service.enabled }}
apiVersion: v1
kind: Service
metadata:
utils/helm/speckle-server/templates/ifc_import_service/serviceaccount.yml
+1 -1
View File
@@ -1,4 +1,4 @@
{{- if (and .Values.ifc_import_service.enabled .Values.featureFlags.backgroundJobsEnabled .Values.ifc_import_service.serviceAccount.create) }}
{{- if (and .Values.ifc_import_service.enabled .Values.ifc_import_service.serviceAccount.create) }}
apiVersion: v1
kind: ServiceAccount
metadata:
utils/helm/speckle-server/templates/objects/serviceaccount.yml
-4
View File
@@ -52,10 +52,6 @@ secrets:
{{- if .Values.preview_service.dedicatedPreviewsQueue }}
- name: {{ default .Values.secretName .Values.redis.previewServiceConnectionString.secretName }}
{{- end }}
{{- if .Values.featureFlags.nextGenFileImporterEnabled }}
- name: {{ default .Values.secretName .Values.redis.fileImportService.rhino.connectionString.secretName }}
- name: {{ default .Values.secretName .Values.redis.fileImportService.ifc.connectionString.secretName }}
{{- end }}
{{- if .Values.featureFlags.accIntegrationEnabled }}
- name: {{ default .Values.secretName .Values.server.accIntegration.clientSecret.secretName }}
{{- end }}
utils/helm/speckle-server/templates/server/serviceaccount.yml
-4
View File
@@ -52,10 +52,6 @@ secrets:
{{- if .Values.preview_service.dedicatedPreviewsQueue }}
- name: {{ default .Values.secretName .Values.redis.previewServiceConnectionString.secretName }}
{{- end }}
{{- if .Values.featureFlags.nextGenFileImporterEnabled }}
- name: {{ default .Values.secretName .Values.redis.fileImportService.rhino.connectionString.secretName }}
- name: {{ default .Values.secretName .Values.redis.fileImportService.ifc.connectionString.secretName }}
{{- end }}
{{- if .Values.featureFlags.accIntegrationEnabled }}
- name: {{ default .Values.secretName .Values.server.accIntegration.clientSecret.secretName }}
{{- end }}
utils/helm/speckle-server/values.schema.json
+1 -61
View File
@@ -107,7 +107,7 @@
},
"nextGenFileImporterEnabled": {
"type": "boolean",
"description": "Enables the next generation file importer",
"description": "Enables the next generation file importer. Requires ifc_import_service to be enabled and configured.",
"default": false
},
"legacyFileImportsEnabled": {
@@ -115,11 +115,6 @@
"description": "Enables the legacy file upload mechanism, using REST API to proxy file uploads via the server",
"default": false
},
"backgroundJobsEnabled": {
"type": "boolean",
"description": "Enables the ability to run background jobs (such as the IFC importer) in Speckle",
"default": false
},
"accIntegrationEnabled": {
"type": "boolean",
"description": "Enables the ability to import data from ACC",
@@ -571,61 +566,6 @@
}
}
},
"fileImportService": {
"type": "object",
"properties": {
"rhino": {
"type": "object",
"properties": {
"connectionString": {
"type": "object",
"properties": {
"secretName": {
"type": "string",
"description": "Required. A secret containing the full connection string to the Redis store (e.g. in format of `protocol://username:password@host:port/database`) stored within the Kubernetes cluster as an opaque Kubernetes Secret. Ref: https://kubernetes.io/docs/concepts/configuration/secret/#opaque-secrets",
"default": ""
},
"secretKey": {
"type": "string",
"description": "Required. The key within the Kubernetes Secret holding the Redis connection string.",
"default": ""
}
}
},
"queueName": {
"type": "string",
"description": "Optional. The name of the queue created within the Redis cluster.",
"default": "fileimport-service-jobs"
}
}
},
"ifc": {
"type": "object",
"properties": {
"connectionString": {
"type": "object",
"properties": {
"secretName": {
"type": "string",
"description": "Required. A secret containing the full connection string to the Redis store (e.g. in format of `protocol://username:password@host:port/database`) stored within the Kubernetes cluster as an opaque Kubernetes Secret. Ref: https://kubernetes.io/docs/concepts/configuration/secret/#opaque-secrets",
"default": ""
},
"secretKey": {
"type": "string",
"description": "Required. The key within the Kubernetes Secret holding the Redis connection string.",
"default": ""
}
}
},
"queueName": {
"type": "string",
"description": "Optional. The name of the queue created within the Redis cluster.",
"default": "fileimport-service-jobs"
}
}
}
}
},
"networkPolicy": {
"type": "object",
"properties": {
utils/helm/speckle-server/values.yaml
+1 -27
View File
@@ -63,12 +63,10 @@ featureFlags:
moveProjectRegionEnabled: false
## @param featureFlags.retryErroredPreviewsEnabled Toggles whether the system periodically retries preview requests which have previously errored.
retryErroredPreviewsEnabled: false
## @param featureFlags.nextGenFileImporterEnabled Enables the next generation file importer
## @param featureFlags.nextGenFileImporterEnabled Enables the next generation file importer. Requires ifc_import_service to be enabled and configured.
nextGenFileImporterEnabled: false
## @param featureFlags.legacyFileImportsEnabled Enables the legacy file upload mechanism, using REST API to proxy file uploads via the server
legacyFileImportsEnabled: false
## @param featureFlags.backgroundJobsEnabled Enables the ability to run background jobs (such as the IFC importer) in Speckle
backgroundJobsEnabled: false
## @param featureFlags.accIntegrationEnabled Enables the ability to import data from ACC
accIntegrationEnabled: false
## @param featureFlags.rhinoFileImporterEnabled Enables the dedicated Rhino based file importer. This is not part of the deployment.
@@ -421,30 +419,6 @@ redis:
##
secretKey: ''
fileImportService:
rhino:
connectionString:
## @param redis.fileImportService.rhino.connectionString.secretName Required. A secret containing the full connection string to the Redis store (e.g. in format of `protocol://username:password@host:port/database`) stored within the Kubernetes cluster as an opaque Kubernetes Secret. Ref: https://kubernetes.io/docs/concepts/configuration/secret/#opaque-secrets
##
secretName: ''
## @param redis.fileImportService.rhino.connectionString.secretKey Required. The key within the Kubernetes Secret holding the Redis connection string.
##
secretKey: ''
## @param redis.fileImportService.rhino.queueName Optional. The name of the queue created within the Redis cluster.
##
queueName: 'fileimport-service-jobs'
ifc:
connectionString:
## @param redis.fileImportService.ifc.connectionString.secretName Required. A secret containing the full connection string to the Redis store (e.g. in format of `protocol://username:password@host:port/database`) stored within the Kubernetes cluster as an opaque Kubernetes Secret. Ref: https://kubernetes.io/docs/concepts/configuration/secret/#opaque-secrets
##
secretName: ''
## @param redis.fileImportService.ifc.connectionString.secretKey Required. The key within the Kubernetes Secret holding the Redis connection string.
##
secretKey: ''
## @param redis.fileImportService.ifc.queueName Optional. The name of the queue created within the Redis cluster.
##
queueName: 'fileimport-service-jobs'
## @extra redis.networkPolicy If networkPolicy is enabled for Speckle server, this provides the NetworkPolicy with the necessary details to allow egress connections to the Redis store
##
networkPolicy: