Merge branch 'main' into andrew/web-2567-update-members-and-guests-settings-pages

2025-03-18 15:20:08 +00:00
parent 81b46c6ae0 e663b857b8
commit 79beb79d2e
13 changed files with 127 additions and 52 deletions
@@ -1,4 +1,3 @@
-version: '2.4'
 services:
  speckle-ingress:
    build:
@@ -22,12 +21,16 @@ services:
    restart: always
    environment:
      NUXT_PUBLIC_SERVER_NAME: 'local'
+      #TODO: Change this to the URL of your server. This is the URL of the server as accessed by users.
      NUXT_PUBLIC_API_ORIGIN: 'http://127.0.0.1'
+      #TODO: Change this to the URL of your server. This is the URL of the server as accessed by users.
      NUXT_PUBLIC_BASE_URL: 'http://127.0.0.1'
+      # This is the URL of the server as accessed via this docker compose network.
      NUXT_PUBLIC_BACKEND_API_ORIGIN: 'http://speckle-server:3000'
      NUXT_PUBLIC_LOG_LEVEL: 'warn'
      NUXT_REDIS_URL: 'redis://redis'
      LOG_LEVEL: 'info'
+      LOG_PRETTY: 'true'

  speckle-server:
    build:
@@ -47,14 +50,20 @@ services:
      retries: 3
      start_period: 90s
    environment:
-      # TODO: Change this to the URL of the speckle server, as accessed from the network
+      # TODO. Change this to the url of your server. This is the URL of the server as accessed by users.
      CANONICAL_URL: 'http://127.0.0.1'
+      # This is the URL of the server as accessed by other Speckle services within this docker compose network, such as preview-service.
+      # This will be the same value as NUXT_PUBLIC_BACKEND_API_ORIGIN as defined in the frontend-2 service.
+      PRIVATE_OBJECTS_SERVER_URL: 'http://speckle-server:3000'

      # TODO: Change this to a unique secret for this server
      SESSION_SECRET: 'TODO:Replace'

+      # This is the authentication strategy to use. Local (i.e. username & password) is the default strategy.
      STRATEGY_LOCAL: 'true'
+
      LOG_LEVEL: 'info'
+      LOG_PRETTY: 'true'

      POSTGRES_URL: 'postgres'
      POSTGRES_USER: 'speckle'
@@ -62,6 +71,8 @@ services:
      POSTGRES_DB: 'speckle'

      REDIS_URL: 'redis://redis'
+      PREVIEW_SERVICE_USE_PRIVATE_OBJECTS_SERVER_URL: 'true'
+      PREVIEW_SERVICE_REDIS_URL: 'redis://redis'

      S3_ENDPOINT: 'http://minio:9000'
      S3_ACCESS_KEY: 'minioadmin'
@@ -85,10 +96,11 @@ services:
    mem_limit: '3000m'
    memswap_limit: '3000m'
    environment:
-      HOST: '127.0.0.1' # Only accept connections from localhost, as preview service does not need to be exposed outside the container.
-      METRICS_HOST: '127.0.0.1' # Amend if you want to expose Prometheus metrics outside of the container
+      HOST: '127.0.0.1' # The preview service does not need to be exposed outside the container.
+      PORT: '3001'
      LOG_LEVEL: 'info'
-      PG_CONNECTION_STRING: 'postgres://speckle:speckle@postgres/speckle'
+      LOG_PRETTY: 'true'
+      REDIS_URL: 'redis://redis'

  webhook-service:
    build:
@@ -99,6 +111,7 @@ services:
    restart: always
    environment:
      LOG_LEVEL: 'info'
+      LOG_PRETTY: 'true'
      PG_CONNECTION_STRING: 'postgres://speckle:speckle@postgres/speckle'

  fileimport-service:
@@ -110,6 +123,7 @@ services:
    restart: always
    environment:
      LOG_LEVEL: 'info'
+      LOG_PRETTY: 'true'
      PG_CONNECTION_STRING: 'postgres://speckle:speckle@postgres/speckle'
      SPECKLE_SERVER_URL: 'http://speckle-server:3000'
      FILE_IMPORT_TIME_LIMIT_MIN: 10
@@ -74,4 +74,5 @@ ENV IFC_DOTNET_DLL_PATH='/speckle-server/packages/fileimport-service/src/ifc-dot

 WORKDIR /speckle-server/packages/fileimport-service

-ENTRYPOINT [ "tini", "--", "node", "--loader=./dist/src/aliasLoader.js", "bin/www.js" ]
+ENTRYPOINT [ "tini", "--", "node", "--loader=./dist/src/aliasLoader.js" ]
+CMD ["bin/www.js"]
@@ -37,7 +37,11 @@ import {
  storeTokenScopesFactory,
  storeUserServerAppTokenFactory
 } from '@/modules/core/repositories/tokens'
-import { getServerOrigin } from '@/modules/shared/helpers/envHelper'
+import {
+  getPrivateObjectsServerOrigin,
+  getServerOrigin,
+  previewServiceShouldUsePrivateObjectsServerUrl
+} from '@/modules/shared/helpers/envHelper'
 import { requestObjectPreviewFactory } from '@/modules/previews/queues/previews'
 import type { Queue } from 'bull'
 import type { Knex } from 'knex'
@@ -61,7 +65,10 @@ const buildCreateObjectPreviewFunction = ({
      queue: previewRequestQueue,
      responseQueue: responseQueueName
    }),
-    serverOrigin: getServerOrigin(),
+    // use the private server origin if defined, otherwise use the public server origin
+    serverOrigin: previewServiceShouldUsePrivateObjectsServerUrl()
+      ? getPrivateObjectsServerOrigin()
+      : getServerOrigin(),
    storeObjectPreview: storeObjectPreviewFactory({ db: projectDb }),
    getStreamCollaborators: getStreamCollaboratorsFactory({ db }),
    createAppToken: createAppTokenFactory({
@@ -1,7 +1,7 @@
 import { MisconfiguredEnvironmentError } from '@/modules/shared/errors'
 import { trimEnd } from 'lodash'
 import * as Environment from '@speckle/shared/dist/commonjs/environment/index.js'
-import { ensureError } from '@speckle/shared'
+import { ensureError, Nullable } from '@speckle/shared'

 export function getStringFromEnv(
  envVarKey: string,
@@ -28,6 +28,32 @@ export function getBooleanFromEnv(envVarKey: string, aDefault = false): boolean
  return ['1', 'true', true].includes(process.env[envVarKey] || aDefault.toString())
 }

+function mustGetUrlFromEnv(name: string, trimTrailingSlash: boolean = false): URL {
+  const url = getUrlFromEnv(name, trimTrailingSlash)
+  if (!url) throw new MisconfiguredEnvironmentError(`${name} env var not configured`)
+  return url
+}
+
+function getUrlFromEnv(
+  name: string,
+  trimTrailingSlash: boolean = false
+): Nullable<URL> {
+  const value = process.env[name]
+  if (!value) {
+    return null
+  }
+  try {
+    return new URL(trimTrailingSlash ? trimEnd(value, '/') : value)
+  } catch (e: unknown) {
+    const err = ensureError(e, 'Unknown error parsing URL')
+    if (err instanceof TypeError && err.message === 'Invalid URL')
+      throw new MisconfiguredEnvironmentError(`${name} has to be a valid URL`, {
+        cause: err
+      })
+    throw new MisconfiguredEnvironmentError(`Error parsing ${name} URL`, { cause: err })
+  }
+}
+
 export function getSessionSecret() {
  if (!process.env.SESSION_SECRET) {
    throw new MisconfiguredEnvironmentError('SESSION_SECRET env var not configured')
@@ -86,6 +112,10 @@ export function getRedisUrl() {
  return getStringFromEnv('REDIS_URL')
 }

+export const previewServiceShouldUsePrivateObjectsServerUrl = (): boolean => {
+  return getBooleanFromEnv('PREVIEW_SERVICE_USE_PRIVATE_OBJECTS_SERVER_URL')
+}
+
 export const getPreviewServiceRedisUrl = (): string | undefined => {
  return process.env['PREVIEW_SERVICE_REDIS_URL']
 }
@@ -195,33 +225,19 @@ export function getFrontendOrigin() {
 }

 /**
- * Get server app origin/base URL
+ * Get server app origin/base URL.
+ * This is the public server URL, i.e. 'canonical url', used for external communication.
 */
 export function getServerOrigin() {
-  if (!process.env.CANONICAL_URL) {
-    throw new MisconfiguredEnvironmentError(
-      'Server origin environment variable (CANONICAL_URL) not configured'
-    )
-  }
+  return mustGetUrlFromEnv('CANONICAL_URL', true).origin
+}

-  try {
-    return new URL(trimEnd(process.env.CANONICAL_URL, '/')).origin
-  } catch (e) {
-    const err = ensureError(e)
-    if (e instanceof TypeError && e.message === 'Invalid URL') {
-      throw new MisconfiguredEnvironmentError(
-        `Server origin environment variable (CANONICAL_URL) is not a valid URL: ${process.env.CANONICAL_URL} ${err.message}`,
-        {
-          cause: e,
-          info: {
-            value: process.env.CANONICAL_URL
-          }
-        }
-      )
-    }
-
-    throw err
-  }
+/**
+ *
+ * @returns the private server origin, which is used for internal communication between services
+ */
+export function getPrivateObjectsServerOrigin() {
+  return mustGetUrlFromEnv('PRIVATE_OBJECTS_SERVER_URL', true).origin
 }

 export function getBindAddress(aDefault: string = '127.0.0.1') {
@@ -239,26 +255,12 @@ export function isSSLServer() {
  return /^https:\/\//.test(getServerOrigin())
 }

-function parseUrlVar(value: string, name: string) {
-  try {
-    return new URL(value)
-  } catch (err: unknown) {
-    if (err instanceof TypeError && err.message === 'Invalid URL')
-      throw new MisconfiguredEnvironmentError(`${name} has to be a valid URL`)
-    throw err
-  }
-}
-
 export function getServerMovedFrom() {
-  const value = process.env.MIGRATION_SERVER_MOVED_FROM
-  if (!value) return value
-  return parseUrlVar(value, 'MIGRATION_SERVER_MOVED_FROM')
+  return getUrlFromEnv('MIGRATION_SERVER_MOVED_FROM')
 }

 export function getServerMovedTo() {
-  const value = process.env.MIGRATION_SERVER_MOVED_TO
-  if (!value) return value
-  return parseUrlVar(value, 'MIGRATION_SERVER_MOVED_TO')
+  return getUrlFromEnv('MIGRATION_SERVER_MOVED_TO')
 }

 export function adminOverrideEnabled() {
@@ -528,7 +528,6 @@ Retrieve the s3 parameters from ConfigMap if enabled, or default to retrieving t
 {{- end }}
 {{- end }}

-
 {{/*
 Generate the environment variables for Speckle server and Speckle objects deployments
 */}}
@@ -542,6 +541,10 @@ Generate the environment variables for Speckle server and Speckle objects deploy

 - name: PORT
  value: {{ include "server.port" $ | quote }}
+
+- name: PRIVATE_OBJECTS_SERVER_URL
+  value: {{ printf "http://%s:%s" ( include "objects.service.fqdn" $ ) ( include "objects.port" $ ) }}
+
 - name: LOG_LEVEL
  value: {{ .Values.server.logLevel }}
 - name: LOG_PRETTY
@@ -799,6 +802,12 @@ Generate the environment variables for Speckle server and Speckle objects deploy
  value: {{ .Values.server.gendoAI.ratelimiting.burstRenderRequestPeriodSeconds | quote }}
 {{- end }}

+# *** Preview service ***
+{{- if .Values.preview_service.deployInCluster }}
+- name: PREVIEW_SERVICE_USE_PRIVATE_OBJECTS_SERVER_URL
+  value: "true"
+{{- end }}
+
 # *** Redis ***
 - name: REDIS_URL
  valueFrom:
@@ -23,6 +23,11 @@ spec:
      - name: main
        image: {{ default (printf "speckle/speckle-fileimport-service:%s" .Values.docker_image_tag) .Values.fileimport_service.image }}
        imagePullPolicy: {{ .Values.imagePullPolicy }}
+        args: #overwrites the Dockerfile CMD statement
+        {{- if .Values.fileimport_service.inspect.enabled }}
+          - {{ printf "--inspect=%s" .Values.fileimport_service.inspect.port }}
+        {{- end }}
+          - "bin/www.js"

        ports:
          - name: metrics
@@ -1,3 +1,4 @@
+{{- if .Values.preview_service.deployInCluster }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -121,4 +122,4 @@ spec:

      # Should be > preview generation time ( 1 hour for good measure )
      terminationGracePeriodSeconds: 3600
-
+{{- end }}
@@ -1,3 +1,4 @@
+{{- if .Values.preview_service.deployInCluster }}
 {{- if (and (.Values.preview_service.networkPolicy.enabled) (eq .Values.networkPlugin.type "cilium")) -}}
 apiVersion: cilium.io/v2
 kind: CiliumNetworkPolicy
@@ -38,3 +39,4 @@ spec:
    # postgres
 {{ include "speckle.networkpolicy.egress.postgres.cilium" $ | indent 4 }}
 {{- end }}
+{{- end }}
@@ -1,3 +1,4 @@
+{{- if .Values.preview_service.deployInCluster }}
 {{- if (and (.Values.preview_service.networkPolicy.enabled) (eq .Values.networkPlugin.type "kubernetes")) -}}
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy
@@ -38,4 +39,5 @@ spec:
          protocol: UDP
    # postgres
 {{ include "speckle.networkpolicy.egress.postgres" $ | indent 4 }}
-{{- end -}}
+{{- end }}
+{{- end }}
@@ -1,3 +1,4 @@
+{{- if .Values.preview_service.deployInCluster }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -14,3 +15,4 @@ spec:
      name: web
      port: {{ .Values.preview_service.port }}
      targetPort: metrics
+{{- end }}
@@ -1,3 +1,4 @@
+{{- if .Values.preview_service.deployInCluster }}
 {{- if .Values.preview_service.serviceAccount.create -}}
 apiVersion: v1
 kind: ServiceAccount
@@ -21,3 +22,4 @@ secrets:
  - name: {{ default .Values.secretName .Values.redis.previewServiceConnectionString.secretName }}
 {{- end }}
 {{- end -}}
+{{- end }}
@@ -1884,6 +1884,11 @@
    "preview_service": {
      "type": "object",
      "properties": {
+        "deployInCluster": {
+          "type": "boolean",
+          "description": "If enabled, the Preview Service will be deployed within the cluster and speckle-server will be configured to send the kubernetes service url of the objects server to the Preview Service.",
+          "default": true
+        },
        "dedicatedPreviewsQueue": {
          "type": "boolean",
          "description": "Allows using a dedicated redis url for the preview service job queue",
@@ -2138,6 +2143,21 @@
          "description": "The maximum number of connections that the File Import Service postgres client will make to the Postgres database.",
          "default": 1
        },
+        "inspect": {
+          "type": "object",
+          "properties": {
+            "enabled": {
+              "type": "boolean",
+              "description": "If enabled, indicates that the File Import service should be deployed with the nodejs inspect feature enabled",
+              "default": false
+            },
+            "port": {
+              "type": "string",
+              "description": "The port on which the nodejs inspect feature should be exposed",
+              "default": "7000"
+            }
+          }
+        },
        "requests": {
          "type": "object",
          "properties": {
@@ -1101,6 +1101,8 @@ frontend_2:
 ## @descriptionEnd
 ##
 preview_service:
+  ## @param preview_service.deployInCluster If enabled, the Preview Service will be deployed within the cluster and speckle-server will be configured to send the kubernetes service url of the objects server to the Preview Service.
+  deployInCluster: true
  ## @param preview_service.dedicatedPreviewsQueue Allows using a dedicated redis url for the preview service job queue
  ##
  dedicatedPreviewsQueue: false
@@ -1279,6 +1281,12 @@ fileimport_service:
  ##
  postgresMaxConnections: 1

+  inspect:
+    ## @param fileimport_service.inspect.enabled If enabled, indicates that the File Import service should be deployed with the nodejs inspect feature enabled
+    enabled: false
+    ## @param fileimport_service.inspect.port The port on which the nodejs inspect feature should be exposed
+    port: '7000'
+
  requests:
    ## @param fileimport_service.requests.cpu The CPU that should be available on a node when scheduling this pod.
    ## ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/