From 6b9a0e78d5155dcd05f58d2baf7e214cf772ddbd Mon Sep 17 00:00:00 2001 From: Iain Sproat <68657+iainsproat@users.noreply.github.com> Date: Thu, 17 Jul 2025 12:22:53 +0100 Subject: [PATCH] feat(helm chart): optionally deploy ifc importer service (#5077) --- .github/workflows/builds.yml | 22 +++ packages/ifc-import-service/.env.example | 1 + .../.gitignore | 0 .../.python-version | 0 .../Dockerfile | 6 +- packages/ifc-import-service/README.md | 7 + .../main.py | 0 .../pyproject.toml | 0 .../src/ifc_importer/__init__.py | 0 .../src/ifc_importer/config.py | 0 .../src/ifc_importer/domain.py | 0 .../src/ifc_importer/job_processor.py | 0 .../src/ifc_importer/repository.py | 0 .../uv.lock | 0 packages/ifc-importer/README.md | 0 setup/db/10-docker_postgres_init.sql | 12 -- tests/deployment/docker-compose/Tiltfile | 20 ++- tests/deployment/helm/Tiltfile | 3 + .../helm/manifests/speckle-server.secret.yaml | 1 + .../helm/values/speckle-server.values.yaml | 8 +- utils/eslint-projectwide.mjs | 2 +- .../speckle-server/templates/_helpers.tpl | 39 ++-- .../templates/ifc_import_service/_helpers.tpl | 53 ++++++ .../configmap-db-certificate.yml | 14 ++ .../ifc_import_service/deployment.yml | 138 ++++++++++++++ .../templates/ifc_import_service/service.yml | 18 ++ .../ifc_import_service/serviceaccount.yml | 18 ++ utils/helm/speckle-server/values.schema.json | 168 ++++++++++++++++++ utils/helm/speckle-server/values.yaml | 112 +++++++++++- 29 files changed, 603 insertions(+), 39 deletions(-) create mode 100644 packages/ifc-import-service/.env.example rename packages/{ifc-importer => ifc-import-service}/.gitignore (100%) rename packages/{ifc-importer => ifc-import-service}/.python-version (100%) rename packages/{ifc-importer => ifc-import-service}/Dockerfile (86%) create mode 100644 packages/ifc-import-service/README.md rename packages/{ifc-importer => ifc-import-service}/main.py (100%) rename packages/{ifc-importer => ifc-import-service}/pyproject.toml (100%) rename packages/{ifc-importer => ifc-import-service}/src/ifc_importer/__init__.py (100%) rename packages/{ifc-importer => ifc-import-service}/src/ifc_importer/config.py (100%) rename packages/{ifc-importer => ifc-import-service}/src/ifc_importer/domain.py (100%) rename packages/{ifc-importer => ifc-import-service}/src/ifc_importer/job_processor.py (100%) rename packages/{ifc-importer => ifc-import-service}/src/ifc_importer/repository.py (100%) rename packages/{ifc-importer => ifc-import-service}/uv.lock (100%) delete mode 100644 packages/ifc-importer/README.md create mode 100644 utils/helm/speckle-server/templates/ifc_import_service/_helpers.tpl create mode 100644 utils/helm/speckle-server/templates/ifc_import_service/configmap-db-certificate.yml create mode 100644 utils/helm/speckle-server/templates/ifc_import_service/deployment.yml create mode 100644 utils/helm/speckle-server/templates/ifc_import_service/service.yml create mode 100644 utils/helm/speckle-server/templates/ifc_import_service/serviceaccount.yml diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index ff726bb52..b13e9996d 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -147,6 +147,28 @@ jobs: tags: ${{ inputs.IMAGE_PREFIX }}/speckle-fileimport-service:${{ inputs.IMAGE_VERSION_TAG }} file: ./packages/fileimport-service/Dockerfile + docker-build-ifc-import-service: + runs-on: blacksmith + name: IFC import service + steps: + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ inputs.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push + uses: useblacksmith/build-push-action@v1 + with: + push: true + tags: ${{ inputs.IMAGE_PREFIX }}/speckle-ifc-import-service:${{ inputs.IMAGE_VERSION_TAG }} + file: ./packages/ifc-import-service/Dockerfile + docker-build-test-deploy: runs-on: blacksmith name: Test deploy util diff --git a/packages/ifc-import-service/.env.example b/packages/ifc-import-service/.env.example new file mode 100644 index 000000000..d4137e20b --- /dev/null +++ b/packages/ifc-import-service/.env.example @@ -0,0 +1 @@ +FILEIMPORT_QUEUE_POSTGRES_URL=postgres://speckle:speckle@127.0.0.1/speckle2_test diff --git a/packages/ifc-importer/.gitignore b/packages/ifc-import-service/.gitignore similarity index 100% rename from packages/ifc-importer/.gitignore rename to packages/ifc-import-service/.gitignore diff --git a/packages/ifc-importer/.python-version b/packages/ifc-import-service/.python-version similarity index 100% rename from packages/ifc-importer/.python-version rename to packages/ifc-import-service/.python-version diff --git a/packages/ifc-importer/Dockerfile b/packages/ifc-import-service/Dockerfile similarity index 86% rename from packages/ifc-importer/Dockerfile rename to packages/ifc-import-service/Dockerfile index 5ae57f3ea..4ceb2b64d 100644 --- a/packages/ifc-importer/Dockerfile +++ b/packages/ifc-import-service/Dockerfile @@ -14,12 +14,12 @@ ENV UV_LINK_MODE=copy \ UV_PROJECT_ENVIRONMENT=/app -COPY pyproject.toml uv.lock / +COPY ./packages/ifc-import-service/pyproject.toml ./packages/ifc-import-service/uv.lock / # Install dependencies using uv RUN uv sync --frozen --no-dev --no-install-project -COPY . /src +COPY ./packages/ifc-import-service /src WORKDIR /src RUN --mount=type=cache,target=/root/.cache \ @@ -43,7 +43,7 @@ EOT STOPSIGNAL SIGINT COPY --from=build --chown=app:app /app /app -COPY --chown=app:app main.py /app +COPY --chown=app:app ./packages/ifc-import-service/main.py /app # COPY --chown=appuser:appuser . . # # Switch to non-root user diff --git a/packages/ifc-import-service/README.md b/packages/ifc-import-service/README.md new file mode 100644 index 000000000..1e48f7ba5 --- /dev/null +++ b/packages/ifc-import-service/README.md @@ -0,0 +1,7 @@ +# IFC File Importer + +This package provides a microservice for importing IFC files in to Speckle. + +It is a worker for the queuing system, built on Postgres. + +It is intended to eventually replace the File Import Service. diff --git a/packages/ifc-importer/main.py b/packages/ifc-import-service/main.py similarity index 100% rename from packages/ifc-importer/main.py rename to packages/ifc-import-service/main.py diff --git a/packages/ifc-importer/pyproject.toml b/packages/ifc-import-service/pyproject.toml similarity index 100% rename from packages/ifc-importer/pyproject.toml rename to packages/ifc-import-service/pyproject.toml diff --git a/packages/ifc-importer/src/ifc_importer/__init__.py b/packages/ifc-import-service/src/ifc_importer/__init__.py similarity index 100% rename from packages/ifc-importer/src/ifc_importer/__init__.py rename to packages/ifc-import-service/src/ifc_importer/__init__.py diff --git a/packages/ifc-importer/src/ifc_importer/config.py b/packages/ifc-import-service/src/ifc_importer/config.py similarity index 100% rename from packages/ifc-importer/src/ifc_importer/config.py rename to packages/ifc-import-service/src/ifc_importer/config.py diff --git a/packages/ifc-importer/src/ifc_importer/domain.py b/packages/ifc-import-service/src/ifc_importer/domain.py similarity index 100% rename from packages/ifc-importer/src/ifc_importer/domain.py rename to packages/ifc-import-service/src/ifc_importer/domain.py diff --git a/packages/ifc-importer/src/ifc_importer/job_processor.py b/packages/ifc-import-service/src/ifc_importer/job_processor.py similarity index 100% rename from packages/ifc-importer/src/ifc_importer/job_processor.py rename to packages/ifc-import-service/src/ifc_importer/job_processor.py diff --git a/packages/ifc-importer/src/ifc_importer/repository.py b/packages/ifc-import-service/src/ifc_importer/repository.py similarity index 100% rename from packages/ifc-importer/src/ifc_importer/repository.py rename to packages/ifc-import-service/src/ifc_importer/repository.py diff --git a/packages/ifc-importer/uv.lock b/packages/ifc-import-service/uv.lock similarity index 100% rename from packages/ifc-importer/uv.lock rename to packages/ifc-import-service/uv.lock diff --git a/packages/ifc-importer/README.md b/packages/ifc-importer/README.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/setup/db/10-docker_postgres_init.sql b/setup/db/10-docker_postgres_init.sql index adaf78e15..3240d81b5 100644 --- a/setup/db/10-docker_postgres_init.sql +++ b/setup/db/10-docker_postgres_init.sql @@ -9,15 +9,3 @@ CREATE DATABASE speckle2_test LC_CTYPE = 'en_US.utf8' TABLESPACE = pg_default CONNECTION LIMIT = -1; -CREATE USER preview_service_test WITH PASSWORD 'preview_service_test'; -CREATE DATABASE preview_service_test - WITH - OWNER = preview_service_test - ENCODING = 'UTF8' - LC_COLLATE = 'en_US.utf8' - LC_CTYPE = 'en_US.utf8' - TABLESPACE = pg_default - CONNECTION LIMIT = -1; -GRANT CREATE ON TABLESPACE pg_default TO preview_service_test; -- required to create databases -ALTER USER preview_service_test CREATEDB; -- Allow user to create databases -GRANT pg_write_all_data TO preview_service_test; diff --git a/tests/deployment/docker-compose/Tiltfile b/tests/deployment/docker-compose/Tiltfile index 8557b52be..744d8d543 100644 --- a/tests/deployment/docker-compose/Tiltfile +++ b/tests/deployment/docker-compose/Tiltfile @@ -7,17 +7,20 @@ if IS_LOAD_DOCKER: else: symbols = load_dynamic('./../build-images.tiltfile') +IMAGE_PREFIX = os.getenv('IMAGE_PREFIX', 'speckle') + # Prepare the Speckle images # (loads the images from tgz file if CI, otherwise builds them or pulls them from remote as a fallback on non-amd64 machines) speckle_image = symbols['speckle_image'] -speckle_image('fileimport-service') -speckle_image('frontend-2') -speckle_image('docker-compose-ingress') -# speckle_image('monitor-deployment') -speckle_image('preview-service') -speckle_image('server') -speckle_image('test-deployment') -speckle_image('webhook-service') +speckle_image('fileimport-service', None, IMAGE_PREFIX) +speckle_image('frontend-2', None, IMAGE_PREFIX) +speckle_image('ifc-import-service', None, IMAGE_PREFIX) +speckle_image('docker-compose-ingress', None, IMAGE_PREFIX) +# speckle_image('monitor-deployment', None, IMAGE_PREFIX) +speckle_image('preview-service', None, IMAGE_PREFIX) +speckle_image('server', None, IMAGE_PREFIX) +speckle_image('test-deployment', None, IMAGE_PREFIX) +speckle_image('webhook-service', None, IMAGE_PREFIX) # Prepare the dependencies docker_compose('../../docker-compose-deps.yml') @@ -34,6 +37,7 @@ dc_resource('speckle-frontend-2', resource_deps=[], labels=['speckle']) dc_resource('speckle-ingress', resource_deps=[], labels=['speckle']) dc_resource('preview-service', resource_deps=['postgres'], labels=['speckle']) dc_resource('fileimport-service', resource_deps=[], labels=['speckle']) +dc_resource('ifc-import-service', resource_deps=[], labels=['speckle']) dc_resource('webhook-service', resource_deps=['postgres'], labels=['speckle']) # Test the Speckle resources diff --git a/tests/deployment/helm/Tiltfile b/tests/deployment/helm/Tiltfile index 5bbbd038e..53754cdd8 100644 --- a/tests/deployment/helm/Tiltfile +++ b/tests/deployment/helm/Tiltfile @@ -23,6 +23,7 @@ IMAGE_PREFIX = os.getenv('IMAGE_PREFIX', 'speckle') speckle_image = symbols['speckle_image'] speckle_image('fileimport-service', None, IMAGE_PREFIX) speckle_image('frontend-2', None, IMAGE_PREFIX) +speckle_image('ifc-import-service', None, IMAGE_PREFIX) speckle_image('monitor-deployment', None, IMAGE_PREFIX) speckle_image('preview-service', None, IMAGE_PREFIX) speckle_image('server', None, IMAGE_PREFIX) @@ -126,6 +127,7 @@ helm_resource('speckle-server', image_deps=[ 'speckle/speckle-fileimport-service', 'speckle/speckle-frontend-2', + 'speckle/speckle-ifc-import-service', 'speckle/speckle-monitor-deployment', 'speckle/speckle-preview-service', 'speckle/speckle-server', @@ -136,6 +138,7 @@ helm_resource('speckle-server', image_keys=[ 'fileimport_service.image', 'frontend_2.image', + 'ifc_import_service.image', 'monitor.image', 'preview_service.image', 'server.image', diff --git a/tests/deployment/helm/manifests/speckle-server.secret.yaml b/tests/deployment/helm/manifests/speckle-server.secret.yaml index 53d6bcf98..5991f3615 100644 --- a/tests/deployment/helm/manifests/speckle-server.secret.yaml +++ b/tests/deployment/helm/manifests/speckle-server.secret.yaml @@ -4,6 +4,7 @@ data: session_secret: 'c3BvcmtsZXNzcHJlY2tsZXNzcGVrbGU=' redis_url: 'cmVkaXM6Ly86dmFsa2V5QHZhbGtleS1wcmltYXJ5LnZhbGtleS5zdmMuY2x1c3Rlci5sb2NhbDo2Mzc5Cg==' postgres_url: 'cG9zdGdyZXNxbDovL3NwZWNrbGU6c3BlY2tsZUBwb3N0Z3Jlc3FsLnBvc3RncmVzLnN2Yy5jbHVzdGVyLmxvY2FsOjU0MzIvc3BlY2tsZQo=' + fileimport_queue_postgres_url: 'cG9zdGdyZXNxbDovL3NwZWNrbGU6c3BlY2tsZUBwb3N0Z3Jlc3FsLnBvc3RncmVzLnN2Yy5jbHVzdGVyLmxvY2FsOjU0MzIvc3BlY2tsZQo=' kind: Secret metadata: name: server-vars diff --git a/tests/deployment/helm/values/speckle-server.values.yaml b/tests/deployment/helm/values/speckle-server.values.yaml index 135003584..733aa273e 100644 --- a/tests/deployment/helm/values/speckle-server.values.yaml +++ b/tests/deployment/helm/values/speckle-server.values.yaml @@ -19,7 +19,7 @@ server: email: enabled: false - speckleAutomateUrl: http://automate.speckle.internal # value is overridden when deployed with by ./scripts/local_deployment.sh + speckleAutomateUrl: http://automate.speckle.internal monitoring: mp: enabled: false @@ -60,11 +60,17 @@ fileimport_service: replicas: 0 logPretty: true +ifc_import_service: + enabled: false + replicas: 1 + logPretty: true + monitoring: replicas: 0 logPretty: true s3: + publicEndpoint: 'http://speckle.internal:9002' endpoint: 'http://minio.minio.svc.cluster.local:9000' bucket: 'speckle-server' access_key: minioadmin diff --git a/utils/eslint-projectwide.mjs b/utils/eslint-projectwide.mjs index 47ce65f00..824529b5d 100644 --- a/utils/eslint-projectwide.mjs +++ b/utils/eslint-projectwide.mjs @@ -24,7 +24,7 @@ const getFileNames = () => { } const resolvePackageContexts = async (absoluteFileNames) => { - const ignoredPackageNames = ['ifc-importer'] + const ignoredPackageNames = ['ifc-import-service'] const allPackages = [ ...(await fs.readdir(path.join(rootDir, './packages'), { withFileTypes: true })) .filter((d) => { diff --git a/utils/helm/speckle-server/templates/_helpers.tpl b/utils/helm/speckle-server/templates/_helpers.tpl index 7053c066e..447c87f96 100644 --- a/utils/helm/speckle-server/templates/_helpers.tpl +++ b/utils/helm/speckle-server/templates/_helpers.tpl @@ -523,7 +523,7 @@ Retrieve the s3 parameters from ConfigMap if enabled, or default to retrieving t {{- $configMap := (lookup "v1" "ConfigMap" .Values.namespace .Values.s3.configMap.name ) -}} {{- printf "%s" ( $configMap.data | toJson ) }} {{- else }} - {{- $result := dict "endpoint" .Values.s3.endpoint "bucket" .Values.s3.bucket "access_key" .Values.s3.access_key }} + {{- $result := dict "endpoint" .Values.s3.endpoint "bucket" .Values.s3.bucket "access_key" .Values.s3.access_key "publicEndpoint" .Values.s3.publicEndpoint }} {{- $result | toJson }} {{- end }} {{- end }} @@ -578,6 +578,9 @@ Generate the environment variables for Speckle server and Speckle objects deploy - name: FF_MOVE_PROJECT_REGION_ENABLED value: {{ .Values.featureFlags.moveProjectRegionEnabled | quote }} +- name: FF_BACKGROUND_JOBS_ENABLED + value: {{ .Values.featureFlags.backgroundJobsEnabled | quote }} + {{- if .Values.featureFlags.gatekeeperModuleEnabled }} - name: LICENSE_TOKEN valueFrom: @@ -786,7 +789,7 @@ Generate the environment variables for Speckle server and Speckle objects deploy key: {{ default "preview_service_redis_url" .Values.redis.previewServiceConnectionString.secretKey }} {{- end }} -{{- if .Values.featureFlags.nextGenFileImporterEnabled }} +{{- if (and .Values.featureFlags.nextGenFileImporterEnabled (not .Values.featureFlags.backgroundJobsEnabled)) }} - name: FILEIMPORT_SERVICE_RHINO_REDIS_URL valueFrom: secretKeyRef: @@ -861,11 +864,15 @@ Generate the environment variables for Speckle server and Speckle objects deploy - name: NODE_TLS_REJECT_UNAUTHORIZED value: {{ .Values.tlsRejectUnauthorized | quote }} -# *** S3 Object Storage *** {{- if (or .Values.s3.configMap.enabled .Values.s3.endpoint) }} +# *** S3 Object Storage *** {{- $s3values := ((include "server.s3Values" .) | fromJson ) }} - name: S3_ENDPOINT value: {{ $s3values.endpoint }} +{{- if $s3values.publicEndpoint }} +- name: S3_PUBLIC_ENDPOINT + value: {{ $s3values.publicEndpoint }} +{{- end }} - name: S3_ACCESS_KEY value: {{ $s3values.access_key }} - name: S3_BUCKET @@ -888,8 +895,8 @@ Generate the environment variables for Speckle server and Speckle objects deploy - name: STRATEGY_LOCAL value: "{{ .Values.server.auth.local.enabled }}" -# Google Auth {{- if .Values.server.auth.google.enabled }} +# Google Auth - name: STRATEGY_GOOGLE value: "true" - name: GOOGLE_CLIENT_ID @@ -901,8 +908,8 @@ Generate the environment variables for Speckle server and Speckle objects deploy key: {{ default "google_client_secret" .Values.server.auth.google.clientSecret.secretKey }} {{- end }} -# Github Auth {{- if .Values.server.auth.github.enabled }} +# Github Auth - name: STRATEGY_GITHUB value: "true" - name: GITHUB_CLIENT_ID @@ -914,8 +921,8 @@ Generate the environment variables for Speckle server and Speckle objects deploy key: {{ default "github_client_secret" .Values.server.auth.github.clientSecret.secretKey }} {{- end }} -# AzureAD Auth {{- if .Values.server.auth.azure_ad.enabled }} +# AzureAD Auth - name: STRATEGY_AZURE_AD value: "true" - name: AZURE_AD_ORG_NAME @@ -934,8 +941,8 @@ Generate the environment variables for Speckle server and Speckle objects deploy {{- end }} -# OpenID Connect Auth {{- if .Values.server.auth.oidc.enabled }} +# OpenID Connect Auth - name: STRATEGY_OIDC value: "true" - name: OIDC_NAME @@ -952,9 +959,9 @@ Generate the environment variables for Speckle server and Speckle objects deploy {{- end }} -# *** Email *** {{- if .Values.server.email.enabled }} +# *** Email *** - name: EMAIL value: "true" - name: EMAIL_HOST @@ -974,8 +981,8 @@ Generate the environment variables for Speckle server and Speckle objects deploy value: {{ .Values.server.email.verificationTimeoutMinutes | quote }} {{- end }} -# *** Newsletter *** {{- if .Values.server.mailchimp.enabled }} +# *** Newsletter *** - name: MAILCHIMP_ENABLED value: "true" - name: MAILCHIMP_API_KEY @@ -993,8 +1000,8 @@ Generate the environment variables for Speckle server and Speckle objects deploy value: "{{ .Values.server.mailchimp.onboardingListId}}" {{- end }} -# Monitoring - Apollo {{- if .Values.server.monitoring.apollo.enabled }} +# Monitoring - Apollo - name: APOLLO_GRAPH_ID value: {{ .Values.server.monitoring.apollo.graph_id }} - name: APOLLO_SCHEMA_REPORTING @@ -1017,7 +1024,6 @@ Generate the environment variables for Speckle server and Speckle objects deploy {{- end }} # Rate Limiting - - name: RATELIMITER_ENABLED value: "{{ .Values.server.ratelimiting.enabled }}" @@ -1110,9 +1116,9 @@ Generate the environment variables for Speckle server and Speckle objects deploy value: "{{ .Values.server.ratelimiting.burst_get_auth }}" {{- end }} -# OpenTelemetry {{- if .Values.openTelemetry.tracing.url }} +# OpenTelemetry - name: OTEL_TRACE_URL value: {{ .Values.openTelemetry.tracing.url | quote }} {{- end }} @@ -1125,9 +1131,9 @@ Generate the environment variables for Speckle server and Speckle objects deploy value: {{ .Values.openTelemetry.tracing.value | quote }} {{- end }} -# Multi-region {{- if .Values.featureFlags.workspacesMultiRegionEnabled }} +# Multi-region - name: MULTI_REGION_CONFIG_PATH value: "/multi-region-config/multi-region-config.json" {{- end }} @@ -1136,6 +1142,13 @@ Generate the environment variables for Speckle server and Speckle objects deploy - name: FF_NEXT_GEN_FILE_IMPORTER_ENABLED value: {{ .Values.featureFlags.nextGenFileImporterEnabled | quote }} {{- end }} +{{- if .Values.featureFlags.backgroundJobsEnabled }} +- name: FILEIMPORT_QUEUE_POSTGRES_URL + valueFrom: + secretKeyRef: + name: {{ default .Values.secretName .Values.ifc_import_service.db.connectionString.secretName }} + key: {{ default "fileimport_queue_postgres_url" .Values.ifc_import_service.db.connectionString.secretKey }} +{{- end }} {{- if .Values.featureFlags.largeFileUploadsEnabled }} - name: FF_LARGE_FILE_IMPORTS_ENABLED value: {{ .Values.featureFlags.largeFileUploadsEnabled | quote }} diff --git a/utils/helm/speckle-server/templates/ifc_import_service/_helpers.tpl b/utils/helm/speckle-server/templates/ifc_import_service/_helpers.tpl new file mode 100644 index 000000000..6d0d7579a --- /dev/null +++ b/utils/helm/speckle-server/templates/ifc_import_service/_helpers.tpl @@ -0,0 +1,53 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "ifc_import_service.name" -}} +{{- default "speckle-ifc-import-service" .Values.ifc_import_service.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ifc_import_service.fullname" -}} +{{- if .Values.ifc_import_service.fullnameOverride }} +{{- .Values.ifc_import_service.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default "speckle-ifc-import-service" .Values.ifc_import_service.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "ifc_import_service.labels" -}} +{{ include "speckle.commonLabels" . }} +app.kubernetes.io/component: {{ include "ifc_import_service.name" . }} +{{ include "ifc_import_service.selectorLabels" . }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "ifc_import_service.selectorLabels" -}} +app: {{ include "ifc_import_service.name" . }} +app.kubernetes.io/name: {{ include "ifc_import_service.name" . }} +{{ include "speckle.commonSelectorLabels" . }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "ifc_import_service.serviceAccountName" -}} +{{- if .Values.ifc_import_service.serviceAccount.create }} +{{- default (include "ifc_import_service.fullname" .) .Values.ifc_import_service.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.ifc_import_service.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/utils/helm/speckle-server/templates/ifc_import_service/configmap-db-certificate.yml b/utils/helm/speckle-server/templates/ifc_import_service/configmap-db-certificate.yml new file mode 100644 index 000000000..3b8cc38b6 --- /dev/null +++ b/utils/helm/speckle-server/templates/ifc_import_service/configmap-db-certificate.yml @@ -0,0 +1,14 @@ +{{ if ( and .Values.featureFlags.backgroundJobsEnabled .Values.ifc_import_service.db.certificate.enabled .Values.ifc_import_service.db.certificate.data ) }} + +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.ifc_import_service.db.certificate.configMapName }} + namespace: {{ .Values.namespace }} + labels: +{{ include "speckle.labels" . | indent 4 }} +data: + {{ .Values.ifc_import_service.db.certificate.configMapKey }}: | +{{ .Values.ifc_import_service.db.certificate.data | indent 4 }} + +{{ end }} diff --git a/utils/helm/speckle-server/templates/ifc_import_service/deployment.yml b/utils/helm/speckle-server/templates/ifc_import_service/deployment.yml new file mode 100644 index 000000000..b7c9188df --- /dev/null +++ b/utils/helm/speckle-server/templates/ifc_import_service/deployment.yml @@ -0,0 +1,138 @@ +{{- if (and .Values.ifc_import_service.enabled .Values.featureFlags.backgroundJobsEnabled) }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "ifc_import_service.name" $ }} + namespace: {{ .Values.namespace }} + labels: +{{ include "ifc_import_service.labels" . | indent 4 }} +spec: + replicas: {{ .Values.ifc_import_service.replicas }} + selector: + matchLabels: + app: {{ include "ifc_import_service.name" $ }} + project: speckle-server + {{- with .Values.ifc_import_service.deploymentStrategy }} + strategy: + {{- toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + labels: +{{ include "ifc_import_service.labels" . | indent 8 }} + spec: + containers: + - name: main + image: {{ default (printf "speckle/speckle-ifc-import-service:%s" .Values.docker_image_tag) .Values.ifc_import_service.image }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + + ports: + - name: metrics + containerPort: 9093 + protocol: TCP + + # TODO: Enable health checks + # livenessProbe: + # initialDelaySeconds: 60 + # periodSeconds: 60 + # httpGet: + # path: /healthz + # port: 9080 + + resources: + {{- with .Values.ifc_import_service.requests }} + requests: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.ifc_import_service.limits }} + limits: + {{- toYaml . | nindent 12 }} + {{- end }} + + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 20000 + + volumeMounts: + - mountPath: /tmp + name: tmp + {{- if .Values.ifc_import_service.db.certificate.enabled }} + - name: postgres-certificate + mountPath: /postgres-certificate + {{- end }} + + env: + - name: FILEIMPORT_QUEUE_POSTGRES_URL + valueFrom: + secretKeyRef: + name: {{ default .Values.secretName .Values.ifc_import_service.db.connectionString.secretName }} + key: {{ default "postgres_url" .Values.ifc_import_service.db.connectionString.secretKey }} + + - name: LOG_LEVEL + value: {{ .Values.ifc_import_service.logLevel | quote }} + + - name: LOG_PRETTY + value: {{ .Values.ifc_import_service.logPretty | quote }} + + {{- if .Values.ifc_import_service.db.certificate.enabled }} + - name: EXTRA_CA_CERTS + value: {{ printf "/postgres-certificate/%s" .Values.ifc_import_service.db.certificate.configMapKey }} + {{- end }} + + - name: FILE_IMPORT_TIME_LIMIT_MIN + value: {{ .Values.file_import_time_limit_min | quote }} + {{- if .Values.featureFlags.experimentalIfcImporterEnabled }} + - name: FF_EXPERIMENTAL_IFC_IMPORTER_ENABLED + value: {{ .Values.featureFlags.experimentalIfcImporterEnabled | quote }} + {{- end }} + {{- with .Values.ifc_import_service.additionalEnvVars }} + {{- toYaml . | nindent 10}} + {{- end }} + {{- if .Values.ifc_import_service.affinity }} + affinity: {{- include "speckle.renderTpl" (dict "value" .Values.ifc_import_service.affinity "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.ifc_import_service.nodeSelector }} + nodeSelector: {{- include "speckle.renderTpl" (dict "value" .Values.ifc_import_service.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.ifc_import_service.tolerations }} + tolerations: {{- include "speckle.renderTpl" (dict "value" .Values.ifc_import_service.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.ifc_import_service.topologySpreadConstraints }} + topologySpreadConstraints: {{- include "speckle.renderTpl" (dict "value" .Values.ifc_import_service.topologySpreadConstraints "context" $) | nindent 8 }} + {{- end }} + + securityContext: + runAsNonRoot: true + runAsUser: 20000 + fsGroup: 25000 + fsGroupChangePolicy: OnRootMismatch + runAsGroup: 30000 + seccompProfile: + type: RuntimeDefault + priorityClassName: low-priority + {{- if .Values.ifc_import_service.serviceAccount.create }} + serviceAccountName: {{ include "ifc_import_service.name" $ }} + {{- else }} + {{- /* NOTE: If there is a service account, Kubernetes adds the imagePullSecrets to Pods automatically. */}} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + # Should be > File import timeout to allow finishing up imports + terminationGracePeriodSeconds: 610 + volumes: + - name: tmp + emptyDir: {} + {{- if .Values.ifc_import_service.db.certificate.enabled }} + - name: postgres-certificate + configMap: + name: {{ .Values.ifc_import_service.db.certificate.configMapName }} + {{- end }} +{{- end }} diff --git a/utils/helm/speckle-server/templates/ifc_import_service/service.yml b/utils/helm/speckle-server/templates/ifc_import_service/service.yml new file mode 100644 index 000000000..744575317 --- /dev/null +++ b/utils/helm/speckle-server/templates/ifc_import_service/service.yml @@ -0,0 +1,18 @@ +{{- if (and .Values.ifc_import_service.enabled .Values.featureFlags.backgroundJobsEnabled) }} +apiVersion: v1 +kind: Service +metadata: + name: {{ printf "%s-metrics" (include "ifc_import_service.name" $) }} + namespace: {{ .Values.namespace }} + labels: +{{ include "ifc_import_service.labels" . | indent 4 }} +spec: + selector: + app: {{ include "ifc_import_service.name" $ }} + project: speckle-server + ports: + - protocol: TCP + name: web + port: 9093 + targetPort: metrics +{{- end }} diff --git a/utils/helm/speckle-server/templates/ifc_import_service/serviceaccount.yml b/utils/helm/speckle-server/templates/ifc_import_service/serviceaccount.yml new file mode 100644 index 000000000..f5b7040fe --- /dev/null +++ b/utils/helm/speckle-server/templates/ifc_import_service/serviceaccount.yml @@ -0,0 +1,18 @@ +{{- if (and .Values.ifc_import_service.enabled .Values.featureFlags.backgroundJobsEnabled .Values.ifc_import_service.serviceAccount.create) }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "ifc_import_service.name" $ }} + namespace: {{ .Values.namespace | quote }} + labels: +{{ include "ifc_import_service.labels" $ | indent 4 }} + annotations: + "kubernetes.io/enforce-mountable-secrets": "true" +automountServiceAccountToken: false +{{- with .Values.imagePullSecrets }} +imagePullSecrets: + {{- toYaml . | nindent 2 }} +{{- end }} +secrets: + - name: {{ default .Values.secretName .Values.ifc_import_service.db.connectionString.secretName }} +{{- end }} diff --git a/utils/helm/speckle-server/values.schema.json b/utils/helm/speckle-server/values.schema.json index 0f0997c65..f11c1f79b 100644 --- a/utils/helm/speckle-server/values.schema.json +++ b/utils/helm/speckle-server/values.schema.json @@ -124,6 +124,11 @@ "type": "boolean", "description": "Enables the ability to parse IFC files using the legacy IFC importer.", "default": false + }, + "backgroundJobsEnabled": { + "type": "boolean", + "description": "Enables the ability to run background jobs (such as the IFC importer) in Speckle", + "default": false } } }, @@ -421,6 +426,11 @@ "description": "The URL at which the s3 compatible storage is hosted. If `s3.configMap.enabled` is true, this value is ignored.", "default": "" }, + "publicEndpoint": { + "type": "string", + "description": "The URL at which the s3 compatible storage is accessible from a user's browser.", + "default": "" + }, "bucket": { "type": "string", "description": "The s3 compatible bucket in which Speckle data will be stored. If `s3.configMap.enabled` is true, this value is ignored.", @@ -2434,6 +2444,164 @@ } } }, + "ifc_import_service": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "If enabled, the File Import Service will be deployed within the cluster.", + "default": false + }, + "replicas": { + "type": "number", + "description": "The number of instances of the FileImport Service pod to be deployed within the cluster.", + "default": 1 + }, + "deploymentStrategy": { + "type": "object", + "properties": { + "type": { + "type": "string", + "description": "The deployment strategy type to be used for the FileImport Service. This can be set to either `Recreate` or `RollingUpdate`. The default is `Recreate` as file import service does not require 100% availability.", + "default": "Recreate" + } + } + }, + "logLevel": { + "type": "string", + "description": "The minimum level of logs which will be output. Suitable values are trace, debug, info, warn, error, fatal, or silent", + "default": "info" + }, + "logPretty": { + "type": "boolean", + "description": "If enabled, will output logs in a human-readable format. Otherwise, logs will be output in JSON format.", + "default": false + }, + "image": { + "type": "string", + "description": "The Docker image to be used for the Speckle FileImport Service component. If blank, defaults to speckle/speckle-fileimport-service:{{ .Values.docker_image_tag }}. If provided, this value should be the full path including tag. The docker_image_tag value will be ignored.", + "default": "" + }, + "db": { + "type": "object", + "properties": { + "postgresMaxConnections": { + "type": "number", + "description": "The maximum number of connections that the File Import Service postgres client will make to the Postgres database.", + "default": 1 + }, + "connectionString": { + "type": "object", + "properties": { + "secretName": { + "type": "string", + "description": "The name of the Kubernetes Secret which contains the connection string to the Postgres database. Defaults to value of .Values.secretName if this value is not provided.", + "default": "" + }, + "secretKey": { + "type": "string", + "description": "The name of the key within the Kubernetes Secret which contains the connection string to the Postgres database.", + "default": "fileimport_queue_postgres_url" + } + } + }, + "certificate": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "If true, the certificate in the configmap will be mounted as a file in to the pod.", + "default": false + }, + "configMapName": { + "type": "string", + "description": "The name of the Kubernetes ConfigMap in which the certificate can be found.", + "default": "ifc-importer-postgres-certificate" + }, + "configMapKey": { + "type": "string", + "description": "The key within the Kubernetes ConfigMap in which the certificate can be found.", + "default": "ca-certificate.crt" + }, + "data": { + "type": "string", + "description": "If not empty and the certificate is enabled, this Helm Chart will populate a Kubernetes ConfigMap of the given name & key with this data. This is intended to be contents of a PEM file, and will be mounted as a file in the pod.", + "default": "" + } + } + } + } + }, + "requests": { + "type": "object", + "properties": { + "cpu": { + "type": "string", + "description": "The CPU that should be available on a node when scheduling this pod.", + "default": "100m" + }, + "memory": { + "type": "string", + "description": "The Memory that should be available on a node when scheduling this pod.", + "default": "512Mi" + } + } + }, + "limits": { + "type": "object", + "properties": { + "cpu": { + "type": "string", + "description": "The maximum CPU that will be made available to the FileImport Service Pod in a given period.", + "default": "1000m" + }, + "memory": { + "type": "string", + "description": "The maximum Memory that will be made available to the FileImport Service Pod.", + "default": "2Gi" + } + } + }, + "affinity": { + "type": "object", + "description": "Affinity for Speckle FileImport Service pod scheduling", + "default": {} + }, + "nodeSelector": { + "type": "object", + "description": "Node labels for Speckle FileImport Service pods scheduling", + "default": {} + }, + "tolerations": { + "type": "array", + "description": "Tolerations for Speckle FileImport Service pods scheduling", + "default": [], + "items": {} + }, + "topologySpreadConstraints": { + "type": "array", + "description": "Spread Constraints for Speckle FileImport Service pod scheduling", + "default": [], + "items": {} + }, + "serviceAccount": { + "type": "object", + "properties": { + "create": { + "type": "boolean", + "description": "If enabled, a Kubernetes Service Account will be created for this pod.", + "default": true + } + } + }, + "additionalEnvVars": { + "type": "array", + "description": "Additional environment variables to be passed to the FileImport service pod", + "default": [], + "items": {} + } + } + }, "monitoring": { "type": "object", "properties": { diff --git a/utils/helm/speckle-server/values.yaml b/utils/helm/speckle-server/values.yaml index eff36c415..92070b2cf 100644 --- a/utils/helm/speckle-server/values.yaml +++ b/utils/helm/speckle-server/values.yaml @@ -71,6 +71,8 @@ featureFlags: experimentalIfcImporterEnabled: false ## @param featureFlags.legacyIfcImporterEnabled Enables the ability to parse IFC files using the legacy IFC importer. legacyIfcImporterEnabled: false + ## @param featureFlags.backgroundJobsEnabled Enables the ability to run background jobs (such as the IFC importer) in Speckle + backgroundJobsEnabled: false analytics: ## @param analytics.enabled Enable or disable analytics @@ -312,9 +314,18 @@ s3: name: '' ## @param s3.endpoint The URL at which the s3 compatible storage is hosted. If `s3.configMap.enabled` is true, this value is ignored. ## The url should be prefixed by the protocol (e.g. `https://`) - ## The url may need to include the port if it is not the default (e.g. `443` for `https` protocol) + ## If the s3 compatible storage is accessible from the server via a private network, the private network URL should be used here. + ## If the 'public' URL, accessible from a user's browser, differs it should be used in `s3.publicEndpoint`. + ## The url may need to include the port if it is not the default (e.g. `https://example.org:9000`) ## endpoint: '' + ## @param s3.publicEndpoint The URL at which the s3 compatible storage is accessible from a user's browser. + ## This is only required if the `endpoint` is not accessible from a user's browser (e.g. the endpoint is in a private network), as it will default to the `endpoint` value if not provided. + ## If `s3.configMap.enabled` is true, this value is ignored. + ## The url should be prefixed by the protocol (e.g. `https://`) + ## The url may need to include the port if it is not the default (e.g. `https://example.org:9000`) + ## + publicEndpoint: '' ## @param s3.bucket The s3 compatible bucket in which Speckle data will be stored. If `s3.configMap.enabled` is true, this value is ignored. ## The access key should be granted write permissions to this bucket ## @@ -1474,6 +1485,105 @@ fileimport_service: ## time_limit_min: 30 +## @section IFC Importer Service +## @descriptionStart +## Defines parameters related to the IFC Import Service component of Speckle. The IFC Import Service is intended to eventually replace the File Import Service. +## @descriptionEnd +## +ifc_import_service: + ## @param ifc_import_service.enabled If enabled, the File Import Service will be deployed within the cluster. + enabled: false + ## @param ifc_import_service.replicas The number of instances of the FileImport Service pod to be deployed within the cluster. + ## + replicas: 1 + deploymentStrategy: + ## @param ifc_import_service.deploymentStrategy.type The deployment strategy type to be used for the FileImport Service. This can be set to either `Recreate` or `RollingUpdate`. The default is `Recreate` as file import service does not require 100% availability. + ## + type: 'Recreate' + ## @param ifc_import_service.logLevel The minimum level of logs which will be output. Suitable values are trace, debug, info, warn, error, fatal, or silent + ## + logLevel: 'info' + ## @param ifc_import_service.logPretty If enabled, will output logs in a human-readable format. Otherwise, logs will be output in JSON format. + ## + logPretty: false + + ## @param ifc_import_service.image The Docker image to be used for the Speckle FileImport Service component. If blank, defaults to speckle/speckle-fileimport-service:{{ .Values.docker_image_tag }}. If provided, this value should be the full path including tag. The docker_image_tag value will be ignored. + ## + image: '' + + db: + ## @param ifc_import_service.db.postgresMaxConnections The maximum number of connections that the File Import Service postgres client will make to the Postgres database. + ## + postgresMaxConnections: 1 + connectionString: + ## @param ifc_import_service.db.connectionString.secretName The name of the Kubernetes Secret which contains the connection string to the Postgres database. Defaults to value of .Values.secretName if this value is not provided. + ## + secretName: '' + ## @param ifc_import_service.db.connectionString.secretKey The name of the key within the Kubernetes Secret which contains the connection string to the Postgres database. + ## + secretKey: 'fileimport_queue_postgres_url' + + certificate: + ## @param ifc_import_service.db.certificate.enabled If true, the certificate in the configmap will be mounted as a file in to the pod. + ## + enabled: false + ## @param ifc_import_service.db.certificate.configMapName The name of the Kubernetes ConfigMap in which the certificate can be found. + ## + configMapName: 'ifc-importer-postgres-certificate' + ## @param ifc_import_service.db.certificate.configMapKey The key within the Kubernetes ConfigMap in which the certificate can be found. + ## + configMapKey: 'ca-certificate.crt' + ## @param ifc_import_service.db.certificate.data If not empty and the certificate is enabled, this Helm Chart will populate a Kubernetes ConfigMap of the given name & key with this data. This is intended to be contents of a PEM file, and will be mounted as a file in the pod. + ## + data: '' + + requests: + ## @param ifc_import_service.requests.cpu The CPU that should be available on a node when scheduling this pod. + ## ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + ## + cpu: 100m + ## @param ifc_import_service.requests.memory The Memory that should be available on a node when scheduling this pod. + ## Depending on the Kubernetes cluster's configuration, exceeding this value may result in pod eviction from a node. + ## ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + ## + memory: 512Mi + limits: + ## @param ifc_import_service.limits.cpu The maximum CPU that will be made available to the FileImport Service Pod in a given period. + ## If this limit is exceeded, execution of the Pod will be paused until the next period. + ## ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + ## + cpu: 1000m + ## @param ifc_import_service.limits.memory The maximum Memory that will be made available to the FileImport Service Pod. + ## If this limit is exceeded, processes within the pod that request additional memory may be stopped. + ## ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + ## + memory: 2Gi + ## @param ifc_import_service.affinity Affinity for Speckle FileImport Service pod scheduling + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## + affinity: {} + ## @param ifc_import_service.nodeSelector Node labels for Speckle FileImport Service pods scheduling + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + ## @param ifc_import_service.tolerations Tolerations for Speckle FileImport Service pods scheduling + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + ## @param ifc_import_service.topologySpreadConstraints Spread Constraints for Speckle FileImport Service pod scheduling + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## + topologySpreadConstraints: [] + serviceAccount: + ## @param ifc_import_service.serviceAccount.create If enabled, a Kubernetes Service Account will be created for this pod. + ## This provides additional security by limiting this pod's access to the Kubernetes API and to Secrets on the Kubernetes cluster. + ## If disabled, the default Service Account will be used which in most Kubernetes configurations will grant this pod + ## access to most secrets on the cluster and access to the Kubernetes API. + ## + create: true + ## @param ifc_import_service.additionalEnvVars Additional environment variables to be passed to the FileImport service pod + additionalEnvVars: [] + ## @section Monitoring ## @descriptionStart ## Provides Speckle with metrics related to the Postgres database.