gergo/previews (#3765)

* feat(preview-generator): add new preview generator webapp

* wip(preview-service): reworking the preview service backend

* feat(previews): logging

* feat(preview-service): streamline payloads

* fix(preview-service): do not log the full payload

* feat(preview-service): build new preview service

* feat(preview-service): add separate response queue

* feat(previews): integrate preview queues with the server

* feat(previews): use module alias

* chore(previews): remove old preview service code

* feat(previews): log stuff on job statuses

* fix(previews): add missing deps and scripts

* fix(previews): package deps fix

* fix(server): moar typing fixes

* Metrics related to jobs: total count, request failures, response errors & durations

* duration should include unit.
- histogram metric should be summary
- error responses include duration in seconds
- attempt to remove metric before adding it (prevent errors with duplicate metrics)

* fix(server, frontend): some ts fixes

* fixes

* fix(frontend): remove unneeded ts-expect-error

* chore(preview-service): eslint

* TS fix

* feat(previews): more smoal fixes

* fix(preview-service): alias loading

* feat(helm): updates for new preview service queue setup

* feat(preview-service): launch new browser for each job

* feat(preview-service): add timeout, fix liveliness

* fix(helm): add access to new secret in service accounts

* tidy metrics into a separate file

* Remove broken preview service acceptance test

* fix broken import

* Add metrics to test

* feat(preview-service): handle preview service shutdown properly

* fix(previews): merge bork

---------

Co-authored-by: Iain Sproat <68657+iainsproat@users.noreply.github.com>
Co-authored-by: Kristaps Fabians Geikins <fabis94@live.com>

.github/workflows/preview-service-acceptance.yml

@@ -1,125 +0,0 @@
-name: Preview service acceptance test
-
-on:
-  workflow_dispatch:
-  pull_request: # Pushing a new commit to the HEAD ref of a pull request will trigger the “synchronize” event
-    paths:
-      - .yarnrc.yml .
-      - .yarn
-      - package.json
-      - '.github/workflows/preview-service-acceptance.yml'
-      - 'packages/frontend-2/type-augmentations/stubs/**/*'
-      - 'packages/preview-service/**/*'
-      - 'packages/viewer/**/*'
-      - 'packages/objectloader/**/*'
-      - 'packages/shared/**/*'
-
-env:
-  REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository_owner }}/speckle-preview-service
-  OUTPUT_FILE_PATH: 'preview-service-output/${{ github.sha }}.png'
-
-jobs:
-  build-preview-service:
-    name: Build Preview Service
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      pull-requests: read
-      packages: write # publishing container to GitHub registry
-
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      - name: Log in to the Container registry
-        uses: docker/login-action@v3.3.0
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Extract metadata (tags, labels) for Docker
-        id: meta
-        uses: docker/metadata-action@v5.5.1
-        with:
-          tags: type=sha,format=long
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-      - name: Build and load preview-service Docker image
-        uses: docker/build-push-action@v6
-        with:
-          context: .
-          file: ./packages/preview-service/Dockerfile
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-    outputs:
-      tags: ${{ steps.meta.outputs.tags }}
-
-  preview-service-acceptance:
-    name: Preview Service Acceptance test
-    runs-on: ubuntu-latest
-    needs: build-preview-service
-
-    permissions:
-      contents: write # to update the screenshot saved in the branch. This is a HACK as GitHub API does not yet support uploading attachments to a comment.
-      pull-requests: write # to write a comment on the PR
-      packages: read # to download the preview-service image
-
-    services:
-      postgres:
-        # Docker Hub image
-        image: postgres:16.4-bookworm@sha256:e62fbf9d3e2b49816a32c400ed2dba83e3b361e6833e624024309c35d334b412
-        env:
-          POSTGRES_DB: preview_service_test
-          POSTGRES_PASSWORD: preview_service_test
-          POSTGRES_USER: preview_service_test
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 5432:5432
-      preview-service:
-        image: ${{ needs.build-preview-service.outputs.tags }}
-        env:
-          # note that the host is the postgres service name
-          PG_CONNECTION_STRING: postgres://preview_service_test:preview_service_test@postgres:5432/preview_service_test
-
-    steps:
-      - uses: actions/checkout@v4
-      - name: Install Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: 20
-          cache: 'yarn'
-      - name: Install dependencies
-        working-directory: packages/preview-service
-        run: yarn install
-
-      - name: Run the acceptance test
-        working-directory: packages/preview-service
-        run: yarn test:acceptance
-        env:
-          NODE_ENV: test
-          TEST_DB: preview_service_test
-          # note that the host is localhost, but the port is the port mapped to the postgres service
-          PG_CONNECTION_STRING: postgres://preview_service_test:preview_service_test@localhost:5432/preview_service_test
-          OUTPUT_FILE_PATH: ${{ env.OUTPUT_FILE_PATH }}
-          S3_BUCKET: ${{ vars.S3_BUCKET }}
-          S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
-          S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }}
-          S3_ENDPOINT: ${{ vars.S3_ENDPOINT }}
-          S3_REGION: ${{ vars.S3_REGION }}
-
-      - uses: actions/github-script@v7
-        with:
-          script: |
-            github.rest.issues.createComment({
-              issue_number: context.issue.number,
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              body: '📸 Preview service has generated <a href="${{ vars.S3_ENDPOINT }}/${{ vars.S3_BUCKET }}/${{ env.OUTPUT_FILE_PATH}}">an image.</a>'
-            })

.zed/settings.json

@@ -0,0 +1,5 @@
+// Folder-specific settings
+//
+// For a full list of overridable settings, and general information on folder-specific settings,
+// see the documentation: https://zed.dev/docs/configuring-zed#settings-files
+{}

packages/frontend-2/package.json

@@ -83,10 +83,6 @@
     "ws": "^8.17.1"
   },
   "devDependencies": {
-    "@babel/core": "^7.19.6",
-    "@babel/preset-env": "^7.19.4",
-    "@babel/preset-react": "^7.18.6",
-    "@babel/preset-typescript": "^7.18.6",
     "@datadog/datadog-ci": "^2.37.0",
     "@eslint/config-inspector": "^0.4.10",
     "@graphql-codegen/cli": "^5.0.5",

packages/preview-frontend/README.md

@@ -0,0 +1,7 @@
+# preview-frontend
+
+This app is the frontend of the preview service.
+
+It is built into static assets and bundled into the final preview service.
+
+To test the app locally, run `yarn dev` and call the functions available on the global `window` object.

packages/preview-frontend/eslint.config.mjs

@@ -0,0 +1,61 @@
+import { baseConfigs, globals, getESMDirname } from '../../eslint.config.mjs'
+import tseslint from 'typescript-eslint'
+
+/**
+ * @type {Array<import('eslint').Linter.FlatConfig>}
+ */
+const configs = [
+  ...baseConfigs,
+  {
+    files: ['**/*.js'],
+    languageOptions: {
+      sourceType: 'module'
+    }
+  },
+  {
+    files: ['*.{js,cjs,mjs,ts}'],
+    languageOptions: {
+      globals: {
+        ...globals.node
+      }
+    }
+  },
+  {
+    files: ['**/*.src'],
+    languageOptions: {
+      globals: {
+        ...globals.browser
+      }
+    }
+  },
+  ...tseslint.configs.recommendedTypeChecked.map((c) => ({
+    ...c,
+    files: [...(c.files || []), '**/*.ts', '**/*.d.ts']
+  })),
+  {
+    files: ['**/*.ts', '**/*.d.ts'],
+    languageOptions: {
+      parserOptions: {
+        tsconfigRootDir: getESMDirname(import.meta.url),
+        project: './tsconfig.json'
+      }
+    },
+    rules: {
+      '@typescript-eslint/no-unused-vars': 'off',
+      '@typescript-eslint/no-non-null-assertion': 'error',
+      '@typescript-eslint/no-base-to-string': 'off', // too restrictive
+      '@typescript-eslint/restrict-template-expressions': 'off', // too restrictive
+      '@typescript-eslint/no-unsafe-enum-comparison': 'off', // too restrictive
+      '@typescript-eslint/require-await': 'off', // too restrictive
+      '@typescript-eslint/unbound-method': 'off', // too restrictive
+      '@typescript-eslint/no-misused-promises': 'off'
+    }
+  },
+  {
+    rules: {
+      'no-console': 'off'
+    }
+  }
+]
+
+export default configs

packages/preview-frontend/index.html

@@ -0,0 +1,25 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width,initial-scale=1.0" />
+    <title>Speckle Viewer</title>
+    <style>
+      body {
+        font-family: 'Space Mono', monospace !important;
+        margin: 0px;
+      }
+      button {
+        font-family: 'Space Mono', monospace !important;
+        border-color: #0a66ff;
+      }
+    </style>
+  </head>
+  <body>
+    <div
+      id="renderer"
+      style="width: 700px; height: 400px; left: 0px; top: 0px; position: absolute"
+    ></div>
+    <script type="module" defer="defer" src="/src/main.ts"></script>
+  </body>
+</html>

packages/preview-frontend/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "@speckle/preview-frontend",
+  "description": "Webapp to Generate PNG previews of Speckle objects",
+  "private": true,
+  "homepage": "https://speckle.systems",
+  "type": "module",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/specklesystems/speckle-server.git",
+    "directory": "packages/preview-frontend"
+  },
+  "scripts": {
+    "dev": "vite",
+    "build": "tsc && vite build",
+    "preview": "vite preview",
+    "lint": "yarn lint:tsc && yarn lint:eslint",
+    "lint:tsc": "tsc --noEmit",
+    "lint:eslint": "eslint ."
+  },
+  "dependencies": {
+    "@speckle/shared": "workspace:^",
+    "@speckle/viewer": "workspace:^"
+  },
+  "devDependencies": {
+    "eslint": "^9.4.0",
+    "eslint-config-prettier": "^9.1.0",
+    "typescript": "^5.7.2",
+    "typescript-eslint": "^7.12.0",
+    "vite": "^6.0.0"
+  }
+}

packages/preview-frontend/src/main.ts

@@ -0,0 +1,85 @@
+import {
+  Load,
+  LoadArgs,
+  PreviewGenerator,
+  PreviewResult,
+  TakeScreenshot
+} from '@speckle/shared/dist/esm/previews/interface.js'
+import {
+  Viewer,
+  DefaultViewerParams,
+  SpeckleLoader,
+  UrlHelper,
+  UpdateFlags
+} from '@speckle/viewer'
+import { CameraController } from '@speckle/viewer'
+
+declare global {
+  // eslint-disable-next-line @typescript-eslint/no-empty-object-type
+  interface Window extends PreviewGenerator {}
+}
+
+let viewer: Viewer | undefined = undefined
+
+const init = async (): Promise<Viewer> => {
+  /** Get the HTML container */
+  const container = document.getElementById('renderer') as HTMLElement
+
+  /** Configure the viewer params */
+  const params = DefaultViewerParams
+  params.showStats = false
+  params.verbose = false
+
+  /** Create Viewer instance */
+  const viewer = new Viewer(container, params)
+  /** Initialise the viewer */
+  await viewer.init()
+
+  /** Add the stock camera controller extension */
+  viewer.createExtension(CameraController)
+  return viewer
+}
+
+const load: Load = async ({ url, token }: LoadArgs) => {
+  if (!viewer) viewer = await init()
+  /** Create a loader for the speckle stream */
+  const resourceUrls = await UrlHelper.getResourceUrls(url, token)
+  for (const resourceUrl of resourceUrls) {
+    const loader = new SpeckleLoader(viewer.getWorldTree(), resourceUrl, token)
+    /** Load the speckle data */
+    await viewer.loadObject(loader, true)
+  }
+}
+
+window.load = load
+
+// TODO: replace with sleep from speckle/shared
+const waitForAnimation = async (ms = 70) =>
+  await new Promise((resolve) => {
+    setTimeout(resolve, ms)
+  })
+
+const takeScreenshot: TakeScreenshot = async () => {
+  if (!viewer) viewer = await init()
+  const ret: PreviewResult = {
+    durationSeconds: 0,
+    screenshots: {}
+  }
+
+  const t0 = Date.now()
+
+  viewer.resize()
+  const cameraController = viewer.getExtension(CameraController)
+  cameraController.setCameraView([], false, 0.95)
+  await waitForAnimation(100)
+
+  for (let i = 0; i < 24; i++) {
+    cameraController.setCameraView({ azimuth: Math.PI / 12, polar: 0 }, false)
+    viewer.requestRender(UpdateFlags.RENDER_RESET)
+    await waitForAnimation(10)
+    ret.screenshots[i + ''] = await viewer.screenshot()
+  }
+  ret.durationSeconds = (Date.now() - t0) / 1000
+  return ret
+}
+window.takeScreenshot = takeScreenshot

packages/preview-frontend/src/vite-env.d.ts

@@ -0,0 +1 @@
+/// <reference types="vite/client" />

packages/preview-frontend/tsconfig.json

@@ -0,0 +1,23 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "useDefineForClassFields": true,
+    "module": "ESNext",
+    "lib": ["ES2020", "DOM", "DOM.Iterable"],
+    "skipLibCheck": true,
+
+    /* Bundler mode */
+    "moduleResolution": "Bundler",
+    "allowImportingTsExtensions": true,
+    "isolatedModules": true,
+    "moduleDetection": "force",
+    "noEmit": true,
+
+    /* Linting */
+    "strict": true,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "noFallthroughCasesInSwitch": true
+  },
+  "include": ["src"]
+}

packages/preview-service/.env.example

@@ -1,8 +1,7 @@
 PREVIEWS_HEADED='true'
 CHROMIUM_EXECUTABLE_PATH='/usr/bin/google-chrome-stable'
 USER_DATA_DIR='/tmp/puppeteer'
-PG_CONNECTION_STRING='postgres://speckle:speckle@127.0.0.1/speckle'
-POSTGRES_MAX_CONNECTIONS_PREVIEW_SERVICE='2'
+REDIS_URL='redis://localhost'
 PROMETHEUS_METRICS_PORT='9094'
 PORT='3001'
 LOG_LEVEL='info'

packages/preview-service/.gitignore

@@ -1 +1 @@
-public/
+public

packages/preview-service/.vscode/launch.json

@@ -0,0 +1,18 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Launch via YARN",
+      "request": "launch",
+      "console": "integratedTerminal",
+      "runtimeArgs": ["dev"],
+      "runtimeExecutable": "yarn",
+      "skipFiles": ["<node_internals>/**"],
+      "type": "node",
+      "envFile": "${workspaceFolder}/.env"
+    }
+  ]
+}

packages/preview-service/Dockerfile

@@ -29,6 +29,7 @@ COPY package.json yarn.lock ./
 
 # Only copy in the relevant package.json files for the dependencies
 COPY packages/frontend-2/type-augmentations/stubs ./packages/frontend-2/type-augmentations/stubs/
+COPY packages/preview-frontend/package.json ./packages/preview-frontend/
 COPY packages/preview-service/package.json ./packages/preview-service/
 COPY packages/viewer/package.json ./packages/viewer/
 COPY packages/objectloader/package.json ./packages/objectloader/
@@ -40,6 +41,7 @@ RUN yarn workspaces focus -A && yarn
 COPY packages/shared ./packages/shared/
 COPY packages/objectloader ./packages/objectloader/
 COPY packages/viewer ./packages/viewer/
+COPY packages/preview-frontend ./packages/preview-frontend/
 COPY packages/preview-service ./packages/preview-service/
 
 # This way the foreach only builds the frontend and its deps
@@ -86,15 +88,13 @@ COPY .yarn ./.yarn
 COPY package.json yarn.lock ./
 
 # Only copy in the relevant package.json files for the dependencies
-COPY packages/frontend-2/type-augmentations/stubs ./packages/frontend-2/type-augmentations/stubs/
 COPY packages/preview-service/package.json ./packages/preview-service/
 
 WORKDIR /speckle-server/packages
 
 COPY --link --from=build-stage /speckle-server/packages/shared ./shared
-COPY --link --from=build-stage /speckle-server/packages/objectloader ./objectloader
-COPY --link --from=build-stage /speckle-server/packages/viewer ./viewer
 COPY --link --from=build-stage /speckle-server/packages/preview-service ./preview-service
+COPY --link --from=build-stage /speckle-server/packages/preview-frontend/dist ./preview-service/public
 
 WORKDIR /speckle-server/packages/preview-service
 
@@ -131,7 +131,10 @@ RUN apt-get update && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
+ENV CHROMIUM_EXECUTABLE_PATH="/usr/bin/google-chrome"
+ENV USER_DATA_DIR='/tmp/puppeteer'
+
 # Run everything after as non-privileged user.
 USER pptruser
 
-ENTRYPOINT [ "tini", "--", "node", "--loader=./dist/src/aliasLoader.js", "bin/www.js" ]
+CMD [ "tini", "--", "node", "--loader=./dist/bootstrap.js", "dist/main.js" ]

packages/preview-service/bin/www.js

@@ -1,2 +0,0 @@
-#!/usr/bin/env node
-import '../dist/src/bin.js'

packages/preview-service/eslint.config.mjs

@@ -1,54 +1,27 @@
+import { baseConfigs, globals, getESMDirname } from '../../eslint.config.mjs'
 import tseslint from 'typescript-eslint'
-import {
-  baseConfigs,
-  getESMDirname,
-  globals,
-  prettierConfig
-} from '../../eslint.config.mjs'
 
+/**
+ * @type {Array<import('eslint').Linter.FlatConfig>}
+ */
 const configs = [
   ...baseConfigs,
-  {
-    ignores: ['dist', 'public', 'docs']
-  },
-  {
-    files: ['webpack.config.renderPage.cjs'],
-    languageOptions: {
-      globals: {
-        ...globals.node
-      }
-    }
-  },
   {
     files: ['**/*.js'],
-    ignores: ['renderPage', '**/*.mjs', 'src/scripts/puppeteerDriver.js'],
     languageOptions: {
-      sourceType: 'module',
+      sourceType: 'module'
+    }
+  },
+  {
+    files: ['*.{js,cjs,mjs,ts}'],
+    languageOptions: {
       globals: {
         ...globals.node
       }
     }
   },
   {
-    files: ['bin/www'],
-    languageOptions: {
-      sourceType: 'module',
-      globals: {
-        ...globals.node
-      }
-    }
-  },
-  {
-    files: ['renderPage/**/*.js'],
-    languageOptions: {
-      sourceType: 'module',
-      globals: {
-        ...globals.browser
-      }
-    }
-  },
-  {
-    files: ['src/scripts/puppeteerDriver.js'],
+    files: ['**/*.src'],
     languageOptions: {
       globals: {
         ...globals.browser
@@ -68,19 +41,21 @@ const configs = [
       }
     },
     rules: {
-      '@typescript-eslint/no-explicit-any': 'error',
-      '@typescript-eslint/no-unsafe-return': 'error'
+      '@typescript-eslint/no-unused-vars': 'off',
+      '@typescript-eslint/no-non-null-assertion': 'error',
+      '@typescript-eslint/no-base-to-string': 'off', // too restrictive
+      '@typescript-eslint/restrict-template-expressions': 'off', // too restrictive
+      '@typescript-eslint/no-unsafe-enum-comparison': 'off', // too restrictive
+      '@typescript-eslint/require-await': 'off', // too restrictive
+      '@typescript-eslint/unbound-method': 'off', // too restrictive
+      '@typescript-eslint/no-misused-promises': 'off'
     }
   },
   {
-    files: ['**/*.spec.{js,ts}'],
-    languageOptions: {
-      globals: {
-        ...globals.node
-      }
+    rules: {
+      'no-console': 'off'
     }
-  },
-  prettierConfig
+  }
 ]
 
 export default configs

packages/preview-service/package.json

@@ -15,78 +15,43 @@
     "node": "^18.19.0"
   },
   "scripts": {
-    "build:tsc:watch": "tsc -p ./tsconfig.build.json --watch",
-    "build:webpack:watch": "webpack --env build --config webpack.config.renderPage.cjs --watch",
-    "run:watch": "NODE_ENV=development LOG_PRETTY=true LOG_LEVEL=debug nodemon --exec \"yarn start\" --trace-deprecation --watch ./bin/www.js --watch ./dist",
-    "dev": "concurrently \"npm:build:tsc:watch\" \"npm:build:webpack:watch\"  \"npm:run:watch\"",
-    "dev:headed": "PREVIEWS_HEADED=true yarn dev",
-    "build:tsc": "rimraf ./dist/src && tsc -p ./tsconfig.build.json",
-    "build:webpack": "webpack --env build --config webpack.config.renderPage.cjs",
-    "build": "yarn build:tsc && yarn build:webpack",
+    "build:frontend": "yarn workspace @speckle/preview-frontend build",
+    "link:frontend": "yarn build:frontend && rimraf ./public && ln -s ../preview-frontend/dist ./public",
+    "dev": "tsx --env-file=.env --watch src/main.ts",
+    "publishTask": "tsx --env-file=.env scripts/publishTask.ts",
+    "test": "echo 'no tests configured'",
     "lint": "yarn lint:tsc && yarn lint:eslint",
     "lint:ci": "yarn lint:tsc",
     "lint:tsc": "tsc --noEmit",
     "lint:eslint": "eslint .",
-    "start": "node --loader=./dist/src/aliasLoader.js ./bin/www.js",
-    "test": "NODE_ENV=test LOG_LEVEL=silent LOG_PRETTY=true vitest run --sequence.shuffle --exclude 'tests/acceptance/**/*.spec.ts'",
-    "test:acceptance": "NODE_ENV=test LOG_LEVEL=debug LOG_PRETTY=true vitest run 'tests/acceptance/acceptance.spec.ts' --sequence.shuffle --hookTimeout 60000 --testNamePattern 'Acceptance'"
+    "build": "tsc -p ./tsconfig.build.json"
   },
   "dependencies": {
-    "@speckle/objectloader": "workspace:^",
     "@speckle/shared": "workspace:^",
-    "@speckle/viewer": "workspace:^",
-    "axios": "^1.7.7",
-    "cookie-parser": "~1.4.4",
-    "crypto": "^1.0.1",
-    "dotenv": "^16.4.5",
-    "esm-module-alias": "^2.2.0",
+    "bull": "^4.16.4",
+    "dotenv": "^16.4.7",
+    "esm-module-alias": "^2.2.1",
     "express": "^4.19.2",
-    "file-type": "^16.5.4",
-    "http-errors": "~1.6.3",
-    "join-images": "^1.1.3",
-    "knex": "^2.5.1",
-    "lodash": "^4.17.21",
-    "lodash-es": "^4.17.21",
-    "node-fetch": "^2.6.1",
-    "pg": "^8.7.3",
-    "pg-query-stream": "^4.2.3",
     "pino": "^8.7.0",
-    "pino-http": "^8.2.1",
+    "pino-http": "^8.6.1",
     "pino-pretty": "^9.1.1",
-    "prom-client": "^14.0.1",
-    "puppeteer": "^22.11.1",
-    "sharp": "^0.32.6",
-    "tarn": "^3.0.2",
-    "yargs": "^17.3.0",
-    "zlib": "^1.0.5",
+    "puppeteer": "^23.9.0",
     "znv": "^0.4.0",
-    "zod": "^3.22.4"
+    "zod": "^3.23.8"
   },
   "devDependencies": {
-    "@aws-sdk/client-s3": "^3.645.0",
-    "@babel/core": "^7.17.5",
+    "@swc/cli": "^0.5.1",
+    "@swc/core": "^1.9.3",
     "@types/express": "^4.17.13",
-    "@types/lodash-es": "^4.17.6",
     "@types/node": "^18.19.38",
-    "@vitest/coverage-istanbul": "^1.6.0",
-    "babel-loader": "^8.2.2",
-    "clean-webpack-plugin": "^4.0.0-alpha.0",
-    "concurrently": "^8.2.2",
-    "crypto-random-string": "^5.0.0",
-    "csv-parse": "^5.5.6",
     "eslint": "^9.4.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-plugin-vitest": "^0.5.4",
-    "html-webpack-plugin": "^5.3.1",
-    "iconv-lite": "^0.6.3",
-    "nodemon": "^2.0.20",
     "prettier": "^2.5.1",
-    "rimraf": "^5.0.7",
+    "rimraf": "^6.0.1",
+    "ts-node": "^10.9.2",
+    "tsx": "^4.19.2",
     "typescript": "^4.6.4",
-    "typescript-eslint": "^7.12.0",
-    "vitest": "^1.6.0",
-    "webpack": "^5.76.0",
-    "webpack-cli": "^4.6.0",
-    "webpack-dev-server": "^4.6.0"
+    "typescript-eslint": "^7.12.0"
   }
 }

packages/preview-service/readme.md

@@ -2,7 +2,7 @@
 
 This component generates object previews for Speckle Objects.
 
-It reads preview tasks from the DB and uses Puppeteer and an internal Viewer to generate previews, which are currently stored in the DB.
+It reads preview tasks from a Redis backed Bull queue and uses Puppeteer and an internal Viewer to generate previews, which are sent back to a response queue.
 
 This is an overview of this service:
 
@@ -38,7 +38,7 @@ Finally, you can run the preview service with:
 yarn dev
 ```
 
-This will use the default dev DB connection of `postgres://speckle:speckle@127.0.0.1/speckle`. You can pass the environment variable `PG_CONNECTION_STRING` to change this to a different DB.
+This will use the default dev DB connection. You can pass the environment variable `REDIS_URL` to change this to a different DB.
 
 ### In a docker image
 

packages/preview-service/renderPage/README.md

@@ -1,3 +0,0 @@
-# Render Page
-
-This is the page that is rendered by Puppeteer within Chromium. It is packaged by Webpack to run in the browser context.

packages/preview-service/renderPage/src/app.js

@@ -1,19 +0,0 @@
-import { DefaultViewerParams, LegacyViewer } from '@speckle/viewer'
-
-console.log('Initialising Viewer')
-const v = new LegacyViewer(document.getElementById('renderer'), DefaultViewerParams)
-window.v = v
-
-// v.on( ViewerEvent.LoadProgress, args => logger.debug( args ) )
-
-window.LoadData = async function LoadData(url) {
-  // token is not used in this context, since the preview service talks directly to the DB
-  await v.loadObject(url, undefined)
-}
-
-window.onload = () => {
-  const testUrl = window.location.hash.substring(1)
-  if (testUrl) {
-    window.LoadData(testUrl)
-  }
-}

packages/preview-service/renderPage/src/example.html

@@ -1,31 +0,0 @@
-<!DOCTYPE html>
-<html>
-  <head>
-    <meta charset="utf-8" />
-    <title>Speckle Viewer</title>
-    <meta name="viewport" content="width=device-width, initial-scale=1" />
-    <!--
-     <link rel="preconnect" href="https://fonts.gstatic.com">
-      <link href="https://fonts.googleapis.com/css2?family=Space+Mono:ital,wght@0,400;0,700;1,700&display=swap" rel="stylesheet">
-     <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/skeleton/2.0.4/skeleton.min.css" integrity="sha512-EZLkOqwILORob+p0BXZc+Vm3RgJBOe1Iq/0fiI7r/wJgzOFZMlsqTa29UEl6v6U6gsV4uIpsNZoV32YZqrCRCQ==" crossorigin="anonymous" />
--->
-    <link href="{%=o.htmlWebpackPlugin.files.favicon%}" rel="shortcut icon" />
-    <style type="text/css">
-      body {
-        font-family: 'Space Mono', monospace !important;
-        margin: 0px;
-      }
-      button {
-        font-family: 'Space Mono', monospace !important;
-        border-color: #0a66ff;
-      }
-      #renderer {
-        height: 400px;
-        width: 700px;
-      }
-    </style>
-  </head>
-  <body>
-    <div id="renderer"></div>
-  </body>
-</html>

packages/preview-service/scripts/publishTask.ts

@@ -0,0 +1,15 @@
+import Bull from 'bull'
+import { REDIS_URL } from '../src/config.js'
+
+const jobQueue = new Bull('preview-service-jobs', REDIS_URL)
+
+await jobQueue.add({
+  url: 'https://latest.speckle.systems/projects/8b94a55ee5/models/7f98c5b62e',
+  token: '',
+  jobId: '1',
+  responseQueue: 'preview-service-results'
+})
+
+console.log('published')
+
+process.exit()

packages/preview-service/src/aliasLoader.ts

@@ -1,8 +0,0 @@
-import generateAliasesResolver from 'esm-module-alias'
-import { packageRoot, srcRoot } from './root.js'
-import path from 'node:path'
-
-export const resolve = generateAliasesResolver({
-  '@': srcRoot,
-  '#': path.resolve(packageRoot, './tests')
-})

packages/preview-service/src/bin.ts

@@ -1,15 +0,0 @@
-import '@/bootstrap.js' // This has side-effects and has to be imported first
-
-import { startServer } from '@/server/server.js'
-import { startPreviewService } from '@/server/background.js'
-
-const start = async () => {
-  await startServer()
-  await startPreviewService()
-}
-
-start()
-  .then()
-  .catch((err) => {
-    throw err
-  })

packages/preview-service/src/bootstrap.ts

@@ -1,2 +1,28 @@
+import generateAliasesResolver from 'esm-module-alias'
+import path from 'node:path'
+import fs from 'node:fs'
+import { fileURLToPath } from 'url'
 import dotenv from 'dotenv'
 dotenv.config()
+
+/**
+ * Singleton module for src root and package root directory resolution
+ */
+const __filename = fileURLToPath(import.meta.url)
+const srcRoot = path.dirname(__filename)
+
+// Recursively walk back from __dirname till we find our package.json
+let packageRoot = srcRoot
+while (packageRoot !== '/') {
+  if (fs.readdirSync(packageRoot).includes('package.json')) {
+    break
+  }
+  packageRoot = path.resolve(packageRoot, '..')
+}
+
+export { srcRoot, packageRoot }
+
+export const resolve = generateAliasesResolver({
+  '@': srcRoot,
+  '#': path.resolve(packageRoot, './tests')
+})

packages/preview-service/src/clients/execHealthcheck.ts

@@ -1,6 +0,0 @@
-import fs from 'fs'
-
-export type UpdateHealthcheckData = () => void
-export const updateHealthcheckDataFactory =
-  (deps: { healthCheckFilePath: string }) => () =>
-    fs.writeFile(deps.healthCheckFilePath, Date.now().toLocaleString(), () => {})

packages/preview-service/src/clients/knex.ts

@@ -1,85 +0,0 @@
-import { knexLogger as logger } from '@/observability/logging.js'
-import {
-  getConnectionAcquireTimeoutMillis,
-  getConnectionCreateTimeoutMillis,
-  getPostgresConnectionString,
-  getPostgresMaxConnections,
-  isDevOrTestEnv,
-  isTest
-} from '@/utils/env.js'
-import Environment from '@speckle/shared/dist/commonjs/environment/index.js'
-import {
-  loadMultiRegionsConfig,
-  configureKnexClient
-} from '@speckle/shared/dist/commonjs/environment/multiRegionConfig.js'
-import { Knex } from 'knex'
-
-const { FF_WORKSPACES_MULTI_REGION_ENABLED } = Environment.getFeatureFlags()
-
-type ConfiguredKnexClient = ReturnType<typeof configureKnexClient>
-export type DbClients = Record<'main', ConfiguredKnexClient> &
-  Record<string, ConfiguredKnexClient>
-let dbClients: DbClients
-
-export const getDbClients = async () => {
-  if (dbClients) return dbClients
-  const maxConnections = getPostgresMaxConnections()
-  const connectionAcquireTimeoutMillis = getConnectionAcquireTimeoutMillis()
-  const connectionCreateTimeoutMillis = getConnectionCreateTimeoutMillis()
-
-  const configArgs = {
-    migrationDirs: [],
-    isTestEnv: isTest(),
-    isDevOrTestEnv: isDevOrTestEnv(),
-    logger,
-    maxConnections,
-    applicationName: 'speckle_preview_service',
-    connectionAcquireTimeoutMillis,
-    connectionCreateTimeoutMillis
-  }
-  if (!FF_WORKSPACES_MULTI_REGION_ENABLED) {
-    const mainClient = configureKnexClient(
-      {
-        postgres: {
-          connectionUri: getPostgresConnectionString()
-        }
-      },
-      configArgs
-    )
-    dbClients = { main: mainClient }
-  } else {
-    const configPath = process.env.MULTI_REGION_CONFIG_PATH || 'multiregion.json'
-    const config = await loadMultiRegionsConfig({ path: configPath })
-    const clients = [['main', configureKnexClient(config.main, configArgs)]]
-    Object.entries(config.regions).map(([key, config]) => {
-      clients.push([key, configureKnexClient(config, configArgs)])
-    })
-    dbClients = Object.fromEntries(clients) as DbClients
-  }
-  return dbClients
-}
-
-export const getProjectDbClient = async ({
-  projectId
-}: {
-  projectId: string
-}): Promise<Knex> => {
-  const dbClients = await getDbClients()
-  const mainDb = dbClients.main.public
-  if (!FF_WORKSPACES_MULTI_REGION_ENABLED) return mainDb
-
-  const projectRegion = await mainDb<{ id: string; regionKey: string | null }>(
-    'streams'
-  )
-    .select('id', 'regionKey')
-    .where({ id: projectId })
-    .first()
-
-  if (!projectRegion?.regionKey) return mainDb
-
-  const regionDb = dbClients[projectRegion.regionKey]
-  if (!regionDb)
-    throw new Error(`Project region client not found for ${projectRegion.regionKey}`)
-
-  return regionDb.public
-}

packages/preview-service/src/clients/previewService.ts

@@ -1,28 +0,0 @@
-import type { Angle, ObjectIdentifier } from '@/domain/domain.js'
-import { isCastableToBrand } from '@/utils/brand.js'
-import axios from 'axios'
-import { z } from 'zod'
-
-const previewResponseSchema = z.record(
-  z.string().refine((value): value is Angle => isCastableToBrand<Angle>(value)),
-  z.string()
-)
-
-export type GeneratePreview = (
-  task: ObjectIdentifier
-) => Promise<Record<Angle, string | undefined>>
-
-export const generatePreviewFactory =
-  ({
-    serviceOrigin,
-    timeout
-  }: {
-    serviceOrigin: string
-    timeout: number
-  }): GeneratePreview =>
-  async (task: ObjectIdentifier) => {
-    const previewUrl = `${serviceOrigin}/preview/${task.streamId}/${task.objectId}`
-    const res = await axios.get(previewUrl, { timeout })
-    const previewResponse = previewResponseSchema.parse(res.data)
-    return previewResponse
-  }

packages/preview-service/src/clients/puppeteer.ts

@@ -1,91 +0,0 @@
-import { extendLoggerComponent } from '@/observability/logging.js'
-import { isDevelopment } from '@/utils/env.js'
-import type { Logger } from 'pino'
-import puppeteer, { type EvaluateFunc, type PuppeteerLaunchOptions } from 'puppeteer'
-
-export type LoadPageAndEvaluateScript = (...args: unknown[]) => Promise<unknown>
-
-export type PuppeteerClient = {
-  loadPageAndEvaluateScript: LoadPageAndEvaluateScript
-  dispose: () => Promise<void>
-}
-
-export const puppeteerClientFactory = async (deps: {
-  logger: Logger
-  url: string
-  script: EvaluateFunc<[unknown[]]>
-  launchParams?: PuppeteerLaunchOptions
-  timeoutMilliseconds: number
-}): Promise<PuppeteerClient> => {
-  const logger = extendLoggerComponent(
-    deps.logger.child({ renderPageUrl: deps.url }),
-    'puppeteer'
-  )
-  const { url, script, launchParams } = deps
-  const browser = await puppeteer.launch({ ...launchParams, dumpio: isDevelopment() })
-  return {
-    loadPageAndEvaluateScript: async (...args: unknown[]) => {
-      if (!browser) {
-        const errorMessage = 'Browser must be initialized using init() before use.'
-        logger.error(errorMessage)
-        throw new Error(errorMessage)
-      }
-      logger.info('Loading page from {renderPageUrl}')
-      const page = await browser.newPage()
-
-      page.setDefaultTimeout(deps.timeoutMilliseconds)
-
-      await page.goto(url)
-
-      logger.info('Page loaded from {renderPageUrl}')
-
-      // Handle page crash (oom?)
-      page
-        .on('error', (err) => {
-          logger.error(err, 'Page crashed')
-          throw err
-        })
-        .on('console', (message) => {
-          let messageText = message.text()
-          if (messageText.startsWith('data:image'))
-            messageText = messageText.substring(0, 200).concat('...')
-          logger.debug(
-            {
-              puppeteerMessageType: message.type().substring(0, 3).toUpperCase()
-            },
-            `{puppeteerMessageType} ${messageText}`
-          )
-        })
-        .on('pageerror', (error) => {
-          logger.error(error, 'Puppeteer page encountered an error.')
-        })
-        .on('response', (response) =>
-          logger.info(
-            {
-              response: {
-                headers: response.headers(),
-                status: response.status(),
-                statusText: response.statusText()
-              }
-            },
-            'Response received from puppeteer page'
-          )
-        )
-        .on('requestfailed', (request) =>
-          logger.error(
-            { err: request.failure()?.errorText, puppeteerPageUrl: request.url() },
-            'Request sent to puppeteer page failed'
-          )
-        )
-
-      const evaluationResult: unknown = await page.evaluate(script, args)
-
-      logger.info('Page evaluated with Puppeteer script.')
-      return evaluationResult
-    },
-    dispose: async () => {
-      if (!browser) return
-      await browser.close()
-    }
-  }
-}

packages/preview-service/src/config.ts

@@ -0,0 +1,22 @@
+import { z } from 'zod'
+import { parseEnv } from 'znv'
+
+export const {
+  REDIS_URL,
+  PORT,
+  PREVIEW_TIMEOUT,
+  PREVIEWS_HEADED,
+  CHROMIUM_EXECUTABLE_PATH,
+  USER_DATA_DIR,
+  LOG_LEVEL,
+  LOG_PRETTY
+} = parseEnv(process.env, {
+  REDIS_URL: z.string().url(),
+  PORT: z.number(),
+  PREVIEW_TIMEOUT: z.number().default(3600000),
+  PREVIEWS_HEADED: z.boolean().default(false),
+  CHROMIUM_EXECUTABLE_PATH: z.string(),
+  USER_DATA_DIR: z.string(),
+  LOG_LEVEL: z.string().default('info'),
+  LOG_PRETTY: z.boolean().default(false)
+})

packages/preview-service/src/domain/backgroundWorker.ts

@@ -1,7 +0,0 @@
-export enum WorkStatus {
-  SUCCESS = 'SUCCESS',
-  NOWORKFOUND = 'NOWORKFOUND',
-  FAILED = 'FAILED'
-}
-
-export type WorkToBeDone = () => Promise<WorkStatus>

packages/preview-service/src/domain/const.ts

@@ -1 +0,0 @@
-export const REQUEST_ID_HEADER = 'x-request-id'

packages/preview-service/src/domain/domain.ts

@@ -1,14 +0,0 @@
-import { Brand } from '@/utils/brand.js'
-
-export type ObjectIdentifier = {
-  streamId: string
-  objectId: string
-}
-
-export type Preview = {
-  previewId: string
-  imgBuffer: Buffer
-}
-
-export type Angle = Brand<string, 'Angle'>
-export type PreviewId = Brand<string, 'PreviewId'>

packages/preview-service/src/jobProcessor.ts

@@ -0,0 +1,103 @@
+import { Page, Browser } from 'puppeteer'
+import { PreviewGenerator } from '@speckle/shared/dist/esm/previews/interface.js'
+import {
+  JobPayload,
+  PreviewResultPayload,
+  PreviewSuccessPayload
+} from '@speckle/shared/dist/esm/previews/job.js'
+import { Logger } from 'pino'
+import { timeoutAt } from '@speckle/shared'
+
+declare global {
+  // eslint-disable-next-line @typescript-eslint/no-empty-object-type
+  interface Window extends PreviewGenerator {}
+}
+
+type SharedArgs = {
+  job: JobPayload
+  port: number
+  timeout: number
+  logger: Logger
+}
+
+type JobArgs = SharedArgs & {
+  browser: Browser
+}
+
+type PageArgs = SharedArgs & {
+  page: Page
+}
+
+export const jobProcessor = async ({
+  logger,
+  browser,
+  job,
+  port,
+  timeout
+}: JobArgs): Promise<PreviewResultPayload> => {
+  const jobId = job.jobId
+  const jobLogger = logger.child({ jobId, serverUrl: job.url })
+  const start = new Date()
+  jobLogger.info('Picked up job {jobId} for {serverUrl}')
+
+  let page: Page | undefined = undefined
+  try {
+    page = await browser.newPage()
+
+    const result = await pageFunction({ page, job, logger: jobLogger, port, timeout })
+    const elapsed = (new Date().getTime() - start.getTime()) / 1000
+    jobLogger.info(
+      { status: result.status, elapsed },
+      'Processes job {jobId} with result {status}. It took {elapsed} seconds.'
+    )
+    return result
+  } catch (err) {
+    const elapsed = (new Date().getTime() - start.getTime()) / 1000
+    jobLogger.error(
+      { err, elapsed },
+      'Failed to process {jobId} job. It took {elapsed} seconds'
+    )
+    const reason =
+      err instanceof Error
+        ? err.stack ?? err.toString()
+        : err instanceof Object
+        ? err.toString()
+        : 'unknown error'
+
+    return {
+      jobId: job.jobId,
+      status: 'error',
+      result: {
+        durationSeconds: elapsed
+      },
+      reason
+    }
+  } finally {
+    await page?.close()
+  }
+}
+
+const pageFunction = async ({
+  page,
+  job,
+  port,
+  timeout,
+  logger
+}: PageArgs): Promise<PreviewSuccessPayload> => {
+  page.on('error', (err) => {
+    logger.error({ err }, 'Page crashed')
+    throw err
+  })
+  await page.goto(`http://127.0.0.1:${port}/index.html`)
+  page.setDefaultTimeout(timeout)
+  // page.setDefaultTimeout(deps.timeoutMilliseconds)
+  const previewResult = await Promise.race([
+    page.evaluate(async (job: JobPayload) => {
+      await window.load(job)
+      return await window.takeScreenshot()
+    }, job),
+    timeoutAt(timeout)
+  ])
+
+  return { jobId: job.jobId, status: 'success', result: previewResult }
+}

packages/preview-service/src/logging.ts

@@ -0,0 +1,10 @@
+import {
+  extendLoggerComponent,
+  getLogger
+} from '@speckle/shared/dist/commonjs/observability/index.js'
+import { LOG_LEVEL, LOG_PRETTY } from '@/config.js'
+
+export const logger = extendLoggerComponent(
+  getLogger(LOG_LEVEL, LOG_PRETTY),
+  'preview-service'
+)

packages/preview-service/src/main.ts

@@ -0,0 +1,140 @@
+import express from 'express'
+import puppeteer, { Browser } from 'puppeteer'
+import {
+  REDIS_URL,
+  PORT,
+  CHROMIUM_EXECUTABLE_PATH,
+  PREVIEWS_HEADED,
+  USER_DATA_DIR,
+  PREVIEW_TIMEOUT
+} from '@/config.js'
+import Bull from 'bull'
+import { logger } from '@/logging.js'
+import { jobProcessor } from '@/jobProcessor.js'
+import { Redis, RedisOptions } from 'ioredis'
+import { jobPayload } from '@speckle/shared/dist/esm/previews/job.js'
+import { wait } from '@speckle/shared'
+
+const app = express()
+const port = PORT
+
+// serve the preview-frontend
+app.use(express.static('public'))
+
+let client: Redis
+let subscriber: Redis
+
+const opts = {
+  // redisOpts here will contain at least a property of connectionName which will identify the queue based on its name
+  createClient(type: string, redisOpts: RedisOptions) {
+    switch (type) {
+      case 'client':
+        if (!client) {
+          client = new Redis(REDIS_URL, redisOpts)
+        }
+        return client
+      case 'subscriber':
+        if (!subscriber) {
+          subscriber = new Redis(REDIS_URL, {
+            ...redisOpts,
+            maxRetriesPerRequest: null,
+            enableReadyCheck: false
+          })
+        }
+        return subscriber
+      case 'bclient':
+        return new Redis(REDIS_URL, {
+          ...redisOpts,
+          maxRetriesPerRequest: null,
+          enableReadyCheck: false
+        })
+      default:
+        throw new Error('Unexpected connection type: ' + type)
+    }
+  }
+}
+const jobQueue = new Bull('preview-service-jobs', opts)
+
+// store this callback, so on shutdown we can error the job
+let jobDoneCallback: Bull.DoneCallback | undefined = undefined
+
+const server = app.listen(port, async () => {
+  logger.info({ port }, '📡 Started Preview Service server, listening on {port}')
+
+  const launchBrowser = async (): Promise<Browser> => {
+    logger.debug('Starting browser')
+    return await puppeteer.launch({
+      headless: !PREVIEWS_HEADED,
+      executablePath: CHROMIUM_EXECUTABLE_PATH,
+      userDataDir: USER_DATA_DIR,
+      // we trust the web content that is running, so can disable the sandbox
+      // disabling the sandbox allows us to run the docker image without linux kernel privileges
+      args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage']
+    })
+  }
+  logger.debug('Starting message queues')
+
+  // nothing after this line is getting called, this blocks
+  await jobQueue.process(async (payload, done) => {
+    try {
+      console.log('starting')
+      jobDoneCallback = done
+
+      await wait(30000)
+      const browser = await launchBrowser()
+      const parseResult = jobPayload.safeParse(payload.data)
+      if (!parseResult.success) {
+        logger.error({ parseError: parseResult.error }, 'Invalid job payload')
+        return done(parseResult.error)
+      }
+      const job = parseResult.data
+      const result = await jobProcessor({
+        logger,
+        browser,
+        job: parseResult.data,
+        port: PORT,
+        timeout: PREVIEW_TIMEOUT
+      })
+
+      const resultsQueue = new Bull(job.responseQueue, opts)
+      // with removeOnComplete, the job response potentially containing a large images,
+      // is cleared from the response queue
+      await resultsQueue.add(result, { removeOnComplete: true })
+      await browser.close()
+      done()
+    } catch (err) {
+      logger.error({ err }, 'Job processing failed')
+      if (err instanceof Error) {
+        done(err)
+      } else {
+        throw err
+      }
+    }
+    jobDoneCallback = undefined
+  })
+})
+
+const shutdown = async () => {
+  // stop accepting new jobs
+  await jobQueue.pause(
+    true, // just pausing this local worker of the queue
+    true // do not wait for active jobs to finish
+  )
+
+  // if there is a job currently running, cancell it with an error
+  if (jobDoneCallback) {
+    console.log('canceling job')
+    jobDoneCallback(new Error('Job cancelled due to perview-service shutdown'))
+    console.log('should be canceled')
+  }
+
+  logger.info('Received signal to shut down')
+  server.close(() => {
+    logger.debug('Exiting the express server')
+    process.exit()
+  })
+}
+
+process.on('SIGINT', async () => await shutdown())
+process.on('SIGQUIT', async () => await shutdown())
+process.on('SIGABRT', async () => await shutdown())

packages/preview-service/src/observability/expressLogging.ts

@@ -1,46 +0,0 @@
-import { REQUEST_ID_HEADER } from '@/domain/const.js'
-import { logger } from '@/observability/logging.js'
-import { randomUUID } from 'crypto'
-import type { Request } from 'express'
-import type { IncomingHttpHeaders, IncomingMessage } from 'http'
-import { get } from 'lodash-es'
-import { pinoHttp } from 'pino-http'
-
-function determineRequestId(headers: IncomingHttpHeaders, uuidGenerator = randomUUID) {
-  const idHeader = headers[REQUEST_ID_HEADER]
-  if (!idHeader) return uuidGenerator()
-  if (Array.isArray(idHeader)) return idHeader[0] ?? uuidGenerator()
-  return idHeader
-}
-
-const generateReqId = (req: IncomingMessage) => determineRequestId(req.headers)
-
-export const getRequestPath = (req: IncomingMessage | Request) => {
-  const path = ((get(req, 'originalUrl') || get(req, 'url') || '') as string).split(
-    '?'
-  )[0]
-  return path?.length ? path : null
-}
-
-export const loggingExpressMiddleware = pinoHttp({
-  genReqId: generateReqId,
-  logger,
-  autoLogging: true,
-  // this is here, to force logging 500 responses as errors in the final log
-  // and we don't really care about 3xx stuff
-  // all the user related 4xx responses are treated as info
-  customLogLevel: (req, res, error) => {
-    const path = getRequestPath(req)
-    const shouldBeDebug = ['/metrics'].includes(path || '') ?? false
-
-    if (res.statusCode >= 400 && res.statusCode < 500) {
-      return 'info'
-    } else if (res.statusCode >= 500 || error) {
-      return 'error'
-    } else if (res.statusCode >= 300 && res.statusCode < 400) {
-      return 'silent'
-    }
-
-    return shouldBeDebug ? 'debug' : 'info'
-  }
-})

packages/preview-service/src/observability/logging.ts

@@ -1,14 +0,0 @@
-import { getLogLevel, isLogPretty } from '@/utils/env.js'
-import {
-  extendLoggerComponent as elc,
-  getLogger
-} from '@speckle/shared/dist/commonjs/observability/index.js'
-export const extendLoggerComponent = elc
-
-export const logger = extendLoggerComponent(
-  getLogger(getLogLevel(), isLogPretty()),
-  'preview-service'
-)
-export const serverLogger = extendLoggerComponent(logger, 'server')
-export const testLogger = getLogger(getLogLevel(), isLogPretty())
-export const knexLogger = extendLoggerComponent(logger, 'knex')

packages/preview-service/src/observability/metricsApp.ts

@@ -1,28 +0,0 @@
-import { getDbClients } from '@/clients/knex.js'
-import { loggingExpressMiddleware } from '@/observability/expressLogging.js'
-import { metricsRouterFactory } from '@/observability/metricsRoute.js'
-import { initPrometheusMetrics } from '@/observability/prometheusMetrics.js'
-import { errorHandler } from '@/utils/errorHandler.js'
-import express from 'express'
-import createError from 'http-errors'
-
-export const appFactory = async () => {
-  const db = (await getDbClients()).main.public
-  initPrometheusMetrics({ db })
-  const app = express()
-
-  app.use(loggingExpressMiddleware)
-  app.use(express.json({ limit: '100mb' }))
-  app.use(express.urlencoded({ limit: '100mb', extended: false }))
-
-  app.use('/metrics', metricsRouterFactory())
-
-  // catch 404 and forward to error handler
-  app.use(function (req, _res, next) {
-    next(createError(404, `Not Found: ${req.url}`))
-  })
-  app.set('json spaces', 2) // pretty print json
-
-  app.use(errorHandler)
-  return app
-}

packages/preview-service/src/observability/metricsRoute.ts

@@ -1,15 +0,0 @@
-import express, { RequestHandler } from 'express'
-import prometheusClient from 'prom-client'
-
-export const metricsRouterFactory = () => {
-  const metricsRouter = express.Router()
-
-  metricsRouter.get(
-    '/', //root path of the sub-path to which this router is attached (should be `/metrics`)
-    (async (_req, res) => {
-      res.setHeader('Content-Type', prometheusClient.register.contentType)
-      res.end(await prometheusClient.register.metrics())
-    }) as RequestHandler //FIXME: this works around a type error with async, which is resolved in express 5
-  )
-  return metricsRouter
-}

packages/preview-service/src/observability/prometheusMetrics.ts

@@ -1,170 +0,0 @@
-import { logger } from '@/observability/logging.js'
-import { getPostgresMaxConnections } from '@/utils/env.js'
-import type { Knex } from 'knex'
-import { isObject } from 'lodash-es'
-import type { Counter, Histogram, Summary } from 'prom-client'
-import prometheusClient from 'prom-client'
-import { Pool } from 'tarn'
-
-// let metricFree: Gauge<string> | null = null
-// let metricUsed: Gauge<string> = null
-// let metricPendingAquires: Gauge<string> | null = null
-let metricQueryDuration: Summary<string> | null = null
-let metricQueryErrors: Counter<string> | null = null
-export let metricDuration: Histogram<string> | null = null
-export let metricOperationErrors: Counter<string> | null = null
-
-let prometheusInitialized = false
-
-function isPrometheusInitialized() {
-  return prometheusInitialized
-}
-
-function initKnexPrometheusMetrics(params: { db: Knex }) {
-  const queryStartTime: Record<string, number> = {}
-  const { db } = params
-  if (!('pool' in db.client)) {
-    throw new Error(
-      'DB client does not have a pool. Skipping knex metrics initialization.'
-    )
-  }
-  // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
-  const dbConnectionPool = db.client.pool as Pool<unknown>
-  //metricFree =
-  new prometheusClient.Gauge({
-    name: 'speckle_server_knex_free',
-    help: 'Number of free DB connections',
-    collect() {
-      this.set(dbConnectionPool.numFree())
-    }
-  })
-
-  //metricUsed =
-  new prometheusClient.Gauge({
-    name: 'speckle_server_knex_used',
-    help: 'Number of used DB connections',
-    collect() {
-      this.set(dbConnectionPool.numUsed())
-    }
-  })
-
-  //metricPendingAquires =
-  new prometheusClient.Gauge({
-    name: 'speckle_server_knex_pending',
-    help: 'Number of pending DB connection aquires',
-    collect() {
-      this.set(dbConnectionPool.numPendingAcquires())
-    }
-  })
-
-  //metricPendingCreates =
-  new prometheusClient.Gauge({
-    name: 'speckle_server_knex_pending_creates',
-    help: 'Number of pending DB connection creates',
-    collect() {
-      this.set(dbConnectionPool.numPendingCreates())
-    }
-  })
-
-  //metricPendingValidations =
-  new prometheusClient.Gauge({
-    name: 'speckle_server_knex_pending_validations',
-    help: 'Number of pending DB connection validations. This is a state between pending acquisition and acquiring a connection.',
-    collect() {
-      this.set(dbConnectionPool.numPendingValidations())
-    }
-  })
-
-  //metricRemainingCapacity =
-  new prometheusClient.Gauge({
-    name: 'speckle_server_knex_remaining_capacity',
-    help: 'Remaining capacity of the DB connection pool',
-    collect() {
-      const postgresMaxConnections = getPostgresMaxConnections()
-      const demand =
-        dbConnectionPool.numUsed() +
-        dbConnectionPool.numPendingCreates() +
-        dbConnectionPool.numPendingValidations() +
-        dbConnectionPool.numPendingAcquires()
-
-      this.set(Math.max(postgresMaxConnections - demand, 0))
-    }
-  })
-
-  metricQueryDuration = new prometheusClient.Summary({
-    name: 'speckle_server_knex_query_duration',
-    help: 'Summary of the DB query durations in seconds'
-  })
-
-  metricQueryErrors = new prometheusClient.Counter({
-    name: 'speckle_server_knex_query_errors',
-    help: 'Number of DB queries with errors'
-  })
-
-  db.on('query', (data) => {
-    if (isObject(data) && '__knexQueryUid' in data) {
-      const queryId = String(data.__knexQueryUid)
-      queryStartTime[queryId] = Date.now()
-    }
-  })
-
-  db.on('query-response', (_data, obj) => {
-    if (isObject(obj) && '__knexQueryUid' in obj) {
-      const queryId = String(obj.__knexQueryUid)
-      const durationSec = (Date.now() - queryStartTime[queryId]) / 1000
-      delete queryStartTime[queryId]
-      if (metricQueryDuration && !isNaN(durationSec))
-        metricQueryDuration.observe(durationSec)
-    }
-  })
-
-  db.on('query-error', (_err, querySpec) => {
-    if (isObject(querySpec) && '__knexQueryUid' in querySpec) {
-      const queryId = String(querySpec.__knexQueryUid)
-      const durationSec = (Date.now() - queryStartTime[queryId]) / 1000
-      delete queryStartTime[queryId]
-
-      if (metricQueryDuration && !isNaN(durationSec))
-        metricQueryDuration.observe(durationSec)
-
-      if (metricQueryErrors) metricQueryErrors.inc()
-    }
-  })
-}
-
-export function initPrometheusMetrics(params: { db: Knex }) {
-  logger.info('Initializing Prometheus metrics...')
-  if (isPrometheusInitialized()) {
-    logger.info('Prometheus metrics already initialized')
-    return
-  }
-
-  prometheusInitialized = true
-
-  prometheusClient.register.clear()
-  prometheusClient.register.setDefaultLabels({
-    project: 'speckle-server',
-    app: 'preview-service'
-  })
-
-  try {
-    metricDuration = new prometheusClient.Histogram({
-      name: 'speckle_server_operation_duration',
-      help: 'Summary of the operation durations in seconds',
-      buckets: [0.5, 1, 5, 10, 30, 60, 300, 600, 1200, 1800],
-      labelNames: ['op']
-    })
-
-    metricOperationErrors = new prometheusClient.Counter({
-      name: 'speckle_server_operation_errors',
-      help: 'Number of operations with errors',
-      labelNames: ['op']
-    })
-
-    initKnexPrometheusMetrics(params)
-    prometheusClient.collectDefaultMetrics()
-  } catch (e) {
-    logger.error(e, 'Failed to initialize Prometheus metrics.')
-    prometheusInitialized = false
-  }
-}

packages/preview-service/src/repositories/objectPreview.ts

@@ -1,68 +0,0 @@
-import type { Angle, ObjectIdentifier, PreviewId } from '@/domain/domain.js'
-import type { Knex } from 'knex'
-
-export type ObjectPreviewRow = ObjectIdentifier & {
-  preview: Record<Angle, PreviewId>
-  previewStatus: number
-  lastUpdate: number
-}
-export const ObjectPreview = (deps: { db: Knex }) =>
-  deps.db<ObjectPreviewRow>('object_preview')
-
-export type GetNextUnstartedObjectPreview = () => Promise<ObjectIdentifier>
-export const getNextUnstartedObjectPreviewFactory =
-  (deps: { db: Knex }): GetNextUnstartedObjectPreview =>
-  async () => {
-    const { db } = deps
-    const {
-      rows: [maybeRow]
-    } = await db.raw<{ rows: ObjectIdentifier[] }>(`
-    UPDATE object_preview
-    SET
-      "previewStatus" = 1,
-      "lastUpdate" = NOW()
-    FROM (
-      SELECT "streamId", "objectId" FROM object_preview
-      WHERE "previewStatus" = 0 OR ("previewStatus" = 1 AND "lastUpdate" < NOW() - INTERVAL '1 WEEK')
-      ORDER BY "priority" ASC, "lastUpdate" ASC
-      LIMIT 1
-    ) as task
-    WHERE object_preview."streamId" = task."streamId" AND object_preview."objectId" = task."objectId"
-    RETURNING object_preview."streamId", object_preview."objectId"
-  `)
-    return maybeRow
-  }
-
-export type UpdatePreviewMetadataParams = ObjectIdentifier & {
-  metadata: Record<string, string>
-}
-export type UpdatePreviewMetadata = (
-  params: UpdatePreviewMetadataParams
-) => Promise<void>
-export const updatePreviewMetadataFactory =
-  (deps: { db: Knex }): UpdatePreviewMetadata =>
-  async (params) => {
-    const { db } = deps
-    // Update preview metadata
-    await db.raw<void>(
-      `
-      UPDATE object_preview
-      SET
-        "previewStatus" = 2,
-        "lastUpdate" = NOW(),
-        "preview" = ?
-      WHERE "streamId" = ? AND "objectId" = ?
-    `,
-      [params.metadata, params.streamId, params.objectId]
-    )
-  }
-
-export type NotifyUpdate = (params: ObjectIdentifier) => Promise<void>
-export const notifyUpdateFactory =
-  (deps: { db: Knex }): NotifyUpdate =>
-  async (params) => {
-    const { db } = deps
-    await db.raw<void>(
-      `NOTIFY preview_generation_update, 'finished:${params.streamId}:${params.objectId}'`
-    )
-  }

packages/preview-service/src/repositories/objects.ts

@@ -1,92 +0,0 @@
-import type { ObjectIdentifier } from '@/domain/domain.js'
-import type { Knex } from 'knex'
-import type { PassThrough } from 'stream'
-
-export const Objects = (deps: { db: Knex }) => deps.db<DbObject>('objects')
-
-type DbObject = {
-  id: string
-  streamId: string
-  data: object
-  totalChildrenCount: number
-}
-
-type ReturnedObject = {
-  id: string
-  data: { totalChildrenCount: number } & Record<string, unknown>
-}
-
-export type GetObject = (params: ObjectIdentifier) => Promise<ReturnedObject | null>
-export const getObjectFactory =
-  (deps: { db: Knex }): GetObject =>
-  async ({ streamId, objectId }) => {
-    const { db } = deps
-    const res = await Objects({ db })
-      .where({ streamId, id: objectId })
-      .select('*')
-      .first()
-    if (!res) return null
-    const returned: ReturnedObject = {
-      id: res.id,
-      data: { totalChildrenCount: res.totalChildrenCount, ...res.data }
-    }
-    return returned
-  }
-
-export type GetObjectChildrenStream = (params: ObjectIdentifier) => Promise<PassThrough>
-export const getObjectChildrenStreamFactory =
-  (deps: { db: Knex }): GetObjectChildrenStream =>
-  async ({ streamId, objectId }) => {
-    const { db } = deps
-    const q = db.with(
-      'object_children_closure',
-      db.raw(
-        `SELECT objects.id as parent, d.key as child, d.value as mindepth, ? as "streamId"
-        FROM objects
-        JOIN jsonb_each_text(objects.data->'__closure') d ON true
-        where objects.id = ?`,
-        [streamId, objectId]
-      )
-    )
-    await q.select('id')
-    await q.select(db.raw('data::text as "dataText"'))
-    await q.from('object_children_closure')
-
-    await q
-      .rightJoin('objects', function () {
-        this.on('objects.streamId', '=', 'object_children_closure.streamId').andOn(
-          'objects.id',
-          '=',
-          'object_children_closure.child'
-        )
-      })
-      .where(
-        db.raw('object_children_closure."streamId" = ? AND parent = ?', [
-          streamId,
-          objectId
-        ])
-      )
-      .orderBy('objects.id')
-    return q.stream({ highWaterMark: 500 })
-  }
-
-type BatchObjectIdentifier = {
-  streamId: string
-  objectIds: string[]
-}
-export type GetObjectsStream = (params: BatchObjectIdentifier) => PassThrough
-export const getObjectsStreamFactory =
-  (deps: { db: Knex }): GetObjectsStream =>
-  ({ streamId, objectIds }) => {
-    const { db } = deps
-    const res = Objects({ db })
-      .whereIn('id', objectIds)
-      .andWhere('streamId', streamId)
-      .orderBy('id')
-      .select(
-        db.raw(
-          '"id", "speckleType", "totalChildrenCount", "totalChildrenCountByDepth", "createdAt", data::text as "dataText"'
-        )
-      )
-    return res.stream({ highWaterMark: 500 })
-  }

packages/preview-service/src/repositories/previews.ts

@@ -1,16 +0,0 @@
-import type { Preview } from '@/domain/domain.js'
-import type { Knex } from 'knex'
-
-export type PreviewRow = { id: string; data: Buffer }
-export const Previews = (deps: { db: Knex }) => deps.db<PreviewRow>('previews')
-
-export type InsertPreview = (params: Preview) => Promise<void>
-export const insertPreviewFactory =
-  (deps: { db: Knex }): InsertPreview =>
-  async (params) => {
-    const { db } = deps
-    await db.raw(
-      'INSERT INTO "previews" (id, data) VALUES (?, ?) ON CONFLICT DO NOTHING',
-      [params.previewId, params.imgBuffer]
-    )
-  }

packages/preview-service/src/root.ts

@@ -1,21 +0,0 @@
-import path from 'node:path'
-import fs from 'node:fs'
-import { fileURLToPath } from 'url'
-
-/**
- * Singleton module for src root and package root directory resolution
- */
-
-const __filename = fileURLToPath(import.meta.url)
-const srcRoot = path.dirname(__filename)
-
-// Recursively walk back from __dirname till we find our package.json
-let packageRoot = srcRoot
-while (packageRoot !== '/') {
-  if (fs.readdirSync(packageRoot).includes('package.json')) {
-    break
-  }
-  packageRoot = path.resolve(packageRoot, '..')
-}
-
-export { srcRoot, packageRoot }

packages/preview-service/src/scripts/puppeteerDriver.js

@@ -1,47 +0,0 @@
-export const puppeteerDriver = async (objectUrl) => {
-  const waitForAnimation = async (ms = 70) =>
-    await new Promise((resolve) => {
-      setTimeout(resolve, ms)
-    })
-  const ret = {
-    duration: 0,
-    mem: 0,
-    scr: {}
-  }
-
-  const t0 = Date.now()
-
-  await window.v.init()
-
-  try {
-    await window.v.loadObjectAsync(objectUrl)
-  } catch {
-    // Main call failed. Wait some time for other objects to load inside the viewer and generate the preview anyway
-    await waitForAnimation(1000)
-  }
-  window.v.resize()
-  window.v.zoom(undefined, 0.95, false)
-  await waitForAnimation(100)
-
-  for (let i = 0; i < 24; i++) {
-    window.v.setView({ azimuth: Math.PI / 12, polar: 0 }, false)
-    window.v.getRenderer().resetPipeline(true)
-    /** Not sure what the frame time when running pupeteer is, but it's not 16ms.
-     *  That's why we're allowing more time between frames than probably needed
-     *  In a future update, we'll have the viewer signal when convergence is complete
-     *  regradless of how many frames/time that takes
-     */
-    /** 22.11.2022 Alex: Commenting this out for now */
-    // await waitForAnimation(2500)
-    await waitForAnimation()
-    ret.scr[i + ''] = await window.v.screenshot()
-  }
-
-  ret.duration = (Date.now() - t0) / 1000
-  ret.mem = {
-    total: performance.memory.totalJSHeapSize,
-    used: performance.memory.usedJSHeapSize
-  }
-  ret.userAgent = navigator.userAgent
-  return ret
-}

packages/preview-service/src/server/app.ts

@@ -1,35 +0,0 @@
-import { loggingExpressMiddleware } from '@/observability/expressLogging.js'
-import { srcRoot } from '@/root.js'
-import apiRouterFactory from '@/server/routes/api.js'
-import indexRouterFactory from '@/server/routes/index.js'
-import objectsRouterFactory from '@/server/routes/objects.js'
-import previewRouterFactory from '@/server/routes/preview.js'
-import { errorHandler } from '@/utils/errorHandler.js'
-import express from 'express'
-import createError from 'http-errors'
-import path from 'path'
-
-export const appFactory = () => {
-  const app = express()
-
-  app.use(loggingExpressMiddleware)
-
-  app.use(express.json({ limit: '100mb' }))
-  app.use(express.urlencoded({ limit: '100mb', extended: false }))
-  //webpack will build the renderPage and save it to the packages/preview-service/dist/public directory
-  app.use(express.static(path.join(srcRoot, '../public')))
-
-  app.use('/', indexRouterFactory())
-  app.use('/preview', previewRouterFactory())
-  app.use('/objects', objectsRouterFactory())
-  app.use('/api', apiRouterFactory())
-
-  // catch 404 and forward to error handler
-  app.use(function (req, res, next) {
-    next(createError(404, `Not Found: ${req.url}`))
-  })
-
-  app.set('json spaces', 2) // pretty print json
-  app.use(errorHandler)
-  return app
-}

packages/preview-service/src/server/background.ts

@@ -1,98 +0,0 @@
-/**
- * @fileoverview Background service for preview service. This service is responsible for generating 360 previews for objects.
- */
-//FIXME this doesn't quite fit in the /server directory, but it's not a service either. It's a background worker.
-import { updateHealthcheckDataFactory } from '@/clients/execHealthcheck.js'
-import { generatePreviewFactory } from '@/clients/previewService.js'
-import { WorkStatus } from '@/domain/backgroundWorker.js'
-import { extendLoggerComponent, logger } from '@/observability/logging.js'
-import { initPrometheusMetrics } from '@/observability/prometheusMetrics.js'
-import {
-  getNextUnstartedObjectPreviewFactory,
-  notifyUpdateFactory,
-  updatePreviewMetadataFactory
-} from '@/repositories/objectPreview.js'
-import { insertPreviewFactory } from '@/repositories/previews.js'
-import { generateAndStore360PreviewFactory } from '@/services/360preview.js'
-import { pollForAndCreatePreviewFactory } from '@/services/pollForPreview.js'
-import { throwUncoveredError, wait } from '@speckle/shared'
-import {
-  getHealthCheckFilePath,
-  serviceOrigin,
-  getPreviewTimeout
-} from '@/utils/env.js'
-import { DbClients, getDbClients } from '@/clients/knex.js'
-
-let shouldExit = false
-
-export async function startPreviewService() {
-  const backgroundLogger = extendLoggerComponent(logger, 'backgroundWorker')
-  backgroundLogger.info('📸 Starting Preview Service background worker')
-
-  process.on('SIGTERM', () => {
-    shouldExit = true
-    backgroundLogger.info('Shutting down...')
-  })
-
-  process.on('SIGINT', () => {
-    shouldExit = true
-    backgroundLogger.info('Shutting down...')
-  })
-
-  const dbClients = await getDbClients()
-
-  // TODO, this should also be initialized for all DBs
-  initPrometheusMetrics({ db: dbClients.main.public })
-
-  const clientGenerator = infiniteDbClientsIterator(dbClients)
-
-  while (!shouldExit) {
-    const db = clientGenerator.next().value
-    if (!db) throw new Error('The infinite client generator failed to return a client')
-
-    const status = await pollForAndCreatePreviewFactory({
-      updateHealthcheckData: updateHealthcheckDataFactory({
-        healthCheckFilePath: getHealthCheckFilePath()
-      }),
-      getNextUnstartedObjectPreview: getNextUnstartedObjectPreviewFactory({ db }),
-      generateAndStore360Preview: generateAndStore360PreviewFactory({
-        generatePreview: generatePreviewFactory({
-          serviceOrigin: serviceOrigin(),
-          timeout: getPreviewTimeout()
-        }),
-        insertPreview: insertPreviewFactory({ db })
-      }),
-      updatePreviewMetadata: updatePreviewMetadataFactory({ db }),
-      notifyUpdate: notifyUpdateFactory({ db }),
-      logger: backgroundLogger
-    })()
-
-    switch (status) {
-      case WorkStatus.SUCCESS:
-        await wait(10)
-        break
-      case WorkStatus.NOWORKFOUND:
-        await wait(1000)
-        break
-      case WorkStatus.FAILED:
-        await wait(5000)
-        break
-      default:
-        throwUncoveredError(status)
-    }
-  }
-  process.exit(0)
-}
-
-function* infiniteDbClientsIterator(dbClients: DbClients) {
-  let index = 0
-  const dbClientEntries = Object.values(dbClients)
-  const clientCount = dbClientEntries.length
-  while (true) {
-    // reset index
-    if (index === clientCount) index = 0
-    const dbConnection = dbClientEntries[index]
-    index++
-    yield dbConnection.public
-  }
-}

packages/preview-service/src/server/routes/api.ts

@@ -1,81 +0,0 @@
-import { getProjectDbClient } from '@/clients/knex.js'
-import { getObjectsStreamFactory } from '@/repositories/objects.js'
-import { isSimpleTextRequested, simpleTextOrJsonContentType } from '@/utils/headers.js'
-import { SpeckleObjectsStream } from '@/utils/speckleObjectsStream.js'
-import express from 'express'
-import { PassThrough, pipeline } from 'stream'
-import zlib from 'zlib'
-import { z } from 'zod'
-
-const apiRouterFactory = () => {
-  const apiRouter = express.Router()
-
-  const getObjectsRequestBodySchema = z.object({
-    objects: z.preprocess((objects) => JSON.parse(String(objects)), z.array(z.string()))
-  })
-
-  // This method was copy-pasted from the server method, without authentication/authorization (this web service is an internal one)
-  apiRouter.post(
-    '/getobjects/:streamId',
-    (async (req, res) => {
-      const boundLogger = req.log.child({
-        streamId: req.params.streamId
-      })
-      const getObjectsRequestBody = await getObjectsRequestBodySchema.parseAsync(
-        req.body
-      )
-
-      res.writeHead(200, {
-        'Content-Encoding': 'gzip',
-        'Content-Type': simpleTextOrJsonContentType(req)
-      })
-
-      const projectDb = await getProjectDbClient({ projectId: req.params.streamId })
-
-      const dbStream = getObjectsStreamFactory({ db: projectDb })({
-        streamId: req.params.streamId,
-        objectIds: getObjectsRequestBody.objects
-      })
-
-      // https://knexjs.org/faq/recipes.html#manually-closing-streams
-      // https://github.com/knex/knex/issues/2324
-      const responseCloseHandler = () => {
-        dbStream.end()
-        dbStream.destroy()
-      }
-
-      dbStream.on('close', () => {
-        res.removeListener('close', responseCloseHandler)
-      })
-      res.on('close', responseCloseHandler)
-
-      const speckleObjStream = new SpeckleObjectsStream(isSimpleTextRequested(req))
-
-      const gzipStream = zlib.createGzip()
-
-      pipeline(
-        dbStream,
-        speckleObjStream,
-        gzipStream,
-        new PassThrough({ highWaterMark: 16384 * 31 }),
-        res,
-        (err) => {
-          if (err) {
-            boundLogger.error(err, `Error streaming objects.`)
-          } else {
-            boundLogger.info(
-              {
-                numberOfStreamedObjects: getObjectsRequestBody.objects.length,
-                sizeOfStreamedObjectsMB: gzipStream.bytesWritten / 1000000
-              },
-              'Streamed {numberOfStreamedObjects} objects (size: {sizeOfStreamedObjectsMB} MB)'
-            )
-          }
-        }
-      )
-    }) as express.RequestHandler //FIXME: this works around a type error with async, which is resolved in express 5
-  )
-  return apiRouter
-}
-
-export default apiRouterFactory

packages/preview-service/src/server/routes/index.ts

@@ -1,13 +0,0 @@
-import express from 'express'
-
-const indexRouterFactory = () => {
-  const indexRouter = express.Router()
-
-  indexRouter.get('/', (_req, res) => {
-    res.send('Speckle Object Preview Service')
-  })
-
-  return indexRouter
-}
-
-export default indexRouterFactory

packages/preview-service/src/server/routes/objects.ts

@@ -1,116 +0,0 @@
-import { getProjectDbClient } from '@/clients/knex.js'
-import {
-  getObjectChildrenStreamFactory,
-  getObjectFactory
-} from '@/repositories/objects.js'
-import { isSimpleTextRequested, simpleTextOrJsonContentType } from '@/utils/headers.js'
-import { SpeckleObjectsStream } from '@/utils/speckleObjectsStream.js'
-import express, { RequestHandler } from 'express'
-import { PassThrough, pipeline } from 'stream'
-import zlib from 'zlib'
-
-const objectsRouterFactory = () => {
-  const objectsRouter = express.Router()
-
-  // This method was copy-pasted from the server method, without authentication/authorization (this web service is an internal one)
-  objectsRouter.get(
-    '/:streamId/:objectId',
-    async function (req, res) {
-      const boundLogger = req.log.child({
-        streamId: req.params.streamId,
-        objectId: req.params.objectId
-      })
-
-      const projectDb = await getProjectDbClient({ projectId: req.params.streamId })
-      // Populate first object (the "commit")
-      const obj = await getObjectFactory({ db: projectDb })({
-        streamId: req.params.streamId,
-        objectId: req.params.objectId
-      })
-
-      if (!obj) {
-        return res.status(404).send('Failed to find object.')
-      }
-
-      res.writeHead(200, {
-        'Content-Encoding': 'gzip',
-        'Content-Type': simpleTextOrJsonContentType(req)
-      })
-
-      const dbStream = await getObjectChildrenStreamFactory({ db: projectDb })({
-        streamId: req.params.streamId,
-        objectId: req.params.objectId
-      })
-
-      // https://knexjs.org/faq/recipes.html#manually-closing-streams
-      // https://github.com/knex/knex/issues/2324
-      const responseCloseHandler = () => {
-        dbStream.end()
-        dbStream.destroy()
-      }
-
-      dbStream.on('close', () => {
-        res.removeListener('close', responseCloseHandler)
-      })
-      res.on('close', responseCloseHandler)
-
-      const speckleObjStream = new SpeckleObjectsStream(isSimpleTextRequested(req))
-      const gzipStream = zlib.createGzip()
-
-      speckleObjStream.write(obj)
-
-      pipeline(
-        dbStream,
-        speckleObjStream,
-        gzipStream,
-        new PassThrough({ highWaterMark: 16384 * 31 }),
-        res,
-        (err) => {
-          if (err) {
-            switch (err.code) {
-              case 'ERR_STREAM_PREMATURE_CLOSE':
-                boundLogger.info({ err }, 'Client closed connection early')
-                break
-              default:
-                boundLogger.error({ err }, 'Error downloading object from stream')
-                break
-            }
-          } else {
-            boundLogger.info(
-              { megaBytesWritten: gzipStream.bytesWritten / 1000000 },
-              'Downloaded object from stream (size: {megaBytesWritten} MB)'
-            )
-          }
-        }
-      )
-    } as RequestHandler //FIXME: this works around a type error with async, which is resolved in express 5
-  )
-
-  objectsRouter.get(
-    '/:streamId/:objectId/single',
-    (async (req, res) => {
-      const boundLogger = req.log.child({
-        streamId: req.params.streamId,
-        objectId: req.params.objectId
-      })
-
-      const projectDb = await getProjectDbClient({ projectId: req.params.streamId })
-      const obj = await getObjectFactory({ db: projectDb })({
-        streamId: req.params.streamId,
-        objectId: req.params.objectId
-      })
-
-      if (!obj) {
-        return res.status(404).send('Failed to find object.')
-      }
-
-      boundLogger.info('Downloaded single object.')
-
-      res.send(obj.data)
-    }) as RequestHandler //FIXME: this works around a type error with async, which is resolved in express 5
-  )
-
-  return objectsRouter
-}
-
-export default objectsRouterFactory

packages/preview-service/src/server/routes/preview.ts

@@ -1,71 +0,0 @@
-import { puppeteerClientFactory } from '@/clients/puppeteer.js'
-import { puppeteerDriver } from '@/scripts/puppeteerDriver.js'
-import { getScreenshotFactory } from '@/services/screenshot.js'
-import {
-  getChromiumExecutablePath,
-  getPreviewTimeout,
-  getPuppeteerUserDataDir,
-  serviceOrigin,
-  shouldBeHeadless
-} from '@/utils/env.js'
-import express, { RequestHandler } from 'express'
-
-const previewRouterFactory = () => {
-  const previewRouter = express.Router()
-
-  previewRouter.get(
-    '/:streamId/:objectId',
-    async function (req, res) {
-      const { streamId, objectId } = req.params || {}
-      const safeParamRgx = /^[\w]+$/i
-      if (!safeParamRgx.test(streamId) || !safeParamRgx.test(objectId)) {
-        return res.status(400).json({ error: 'Invalid streamId or objectId!' })
-      }
-      const boundLogger = req.log.child({ streamId, objectId })
-
-      boundLogger.info('Requesting screenshot.')
-
-      //FIXME should we be creating a puppeteer client for every request, or per app instance?
-      const puppeteerClient = await puppeteerClientFactory({
-        logger: boundLogger,
-        url: `${serviceOrigin()}/render/`,
-        script: puppeteerDriver,
-        launchParams: {
-          headless: shouldBeHeadless(),
-          userDataDir: getPuppeteerUserDataDir(),
-          executablePath: getChromiumExecutablePath(),
-          protocolTimeout: getPreviewTimeout(),
-          // we trust the web content that is running, so can disable the sandbox
-          // disabling the sandbox allows us to run the docker image without linux kernel privileges
-          args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage']
-        },
-        timeoutMilliseconds: getPreviewTimeout()
-      })
-
-      let screenshot: { [key: string]: string } | null = null
-      try {
-        screenshot = await getScreenshotFactory({
-          loadPageAndEvaluateScript: puppeteerClient.loadPageAndEvaluateScript,
-          logger: boundLogger,
-          serviceOrigin: serviceOrigin()
-        })({
-          objectId,
-          streamId
-        })
-      } finally {
-        await puppeteerClient.dispose()
-      }
-
-      if (!screenshot) {
-        return res.status(500).end()
-      }
-
-      res.setHeader('content-type', 'image/png')
-      res.send(screenshot)
-    } as RequestHandler //FIXME: this works around a type error with async, which is resolved in express 5
-  )
-
-  return previewRouter
-}
-
-export default previewRouterFactory

packages/preview-service/src/server/server.ts

@@ -1,109 +0,0 @@
-import { serverLogger } from '@/observability/logging.js'
-import { appFactory as metricsAppFactory } from '@/observability/metricsApp.js'
-import { appFactory } from '@/server/app.js'
-import { getAppPort, getHost, getMetricsHost, getMetricsPort } from '@/utils/env.js'
-import http from 'http'
-import { isNaN, isString, toNumber } from 'lodash-es'
-
-export const startServer = async (params?: { serveOnRandomPort?: boolean }) => {
-  /**
-   * Get port from environment and store in Express.
-   */
-  const inputPort = params?.serveOnRandomPort ? 0 : normalizePort(getAppPort())
-  const app = appFactory()
-  app.set('port', inputPort)
-
-  // we place the metrics on a separate port as we wish to expose it to external monitoring tools, but do not wish to expose other routes (for now)
-  const inputMetricsPort = params?.serveOnRandomPort
-    ? 0
-    : normalizePort(getMetricsPort())
-  const metricsApp = await metricsAppFactory()
-  metricsApp.set('port', inputMetricsPort)
-
-  /**
-   * Create HTTP server.
-   */
-
-  const server = http.createServer(app)
-  const metricsServer = http.createServer(metricsApp)
-
-  /**
-   * Listen on provided port, on all network interfaces.
-   */
-  const host = getHost()
-  server.on('error', onErrorFactory(inputPort))
-  server.on('listening', () => {
-    serverLogger.info('📡 Started Preview Service server')
-    onListening(server)
-  })
-  server.listen(inputPort, host)
-
-  const metricsHost = getMetricsHost()
-  metricsServer.on('error', onErrorFactory(inputPort))
-  metricsServer.on('listening', () => {
-    serverLogger.info('📊 Started Preview Service metrics server')
-    onListening(metricsServer)
-  })
-  metricsServer.listen(inputMetricsPort, metricsHost)
-
-  return { app, server, metricsServer }
-}
-
-export const stopServer = (params: { server: http.Server }) => {
-  const { server } = params
-  server.close()
-}
-
-/**
- * Normalize a port into a number, string, or false.
- */
-function normalizePort(val: string | number) {
-  const port = toNumber(val)
-  if (!isNaN(port) && port >= 0) return port
-
-  throw new Error('Invalid port; port must be a positive integer.')
-}
-
-/**
- * Event listener for HTTP server "error" event.
- */
-
-const onErrorFactory = (port: string | number | false) => (error: Error) => {
-  if ('syscall' in error && error.syscall !== 'listen') {
-    throw error
-  }
-
-  const bind = isString(port) ? 'Pipe ' + port : 'Port ' + port
-
-  if (!('code' in error)) throw error
-
-  // handle specific listen errors with friendly messages
-  switch (error.code) {
-    case 'EACCES':
-      serverLogger.error(error, bind + ' requires elevated privileges')
-      process.exit(1)
-    case 'EADDRINUSE':
-      serverLogger.error(error, bind + ' is already in use')
-      process.exit(1)
-    default:
-      throw error
-  }
-}
-
-/**
- * Event listener for HTTP server "listening" event.
- */
-
-function onListening(referenceServer: http.Server) {
-  const addr = referenceServer.address()
-  if (!addr) throw new Error('Server address is not defined')
-
-  switch (typeof addr) {
-    case 'string':
-      serverLogger.info(`Listening on pipe ${addr}`)
-      return addr
-    default:
-      serverLogger.info(`Listening on port ${addr.port}`)
-      return addr.port
-  }
-}

packages/preview-service/src/services/360preview.ts

@@ -1,61 +0,0 @@
-import type { GeneratePreview } from '@/clients/previewService.js'
-import type { Angle, ObjectIdentifier, PreviewId } from '@/domain/domain.js'
-import type { InsertPreview } from '@/repositories/previews.js'
-import crypto from 'crypto'
-import { joinImages } from 'join-images'
-
-export type GenerateAndStore360Preview = (
-  task: ObjectIdentifier
-) => Promise<{ metadata: Record<Angle, PreviewId> }>
-export const generateAndStore360PreviewFactory =
-  (deps: {
-    generatePreview: GeneratePreview
-    insertPreview: InsertPreview
-  }): GenerateAndStore360Preview =>
-  async (task: ObjectIdentifier) => {
-    const responseBody = await deps.generatePreview(task)
-
-    // metadata is key of angle and value of previewId
-    const metadata: Record<Angle, PreviewId> = {}
-    const allImgsArr: Buffer[] = []
-    let i = 0
-    for (const aKey in responseBody) {
-      const angle = aKey as Angle
-      const value = responseBody[angle]
-      if (!value) {
-        continue
-      }
-      const imgBuffer = Buffer.from(
-        value.replace(/^data:image\/\w+;base64,/, ''),
-        'base64'
-      )
-      const previewId = crypto
-        .createHash('md5')
-        .update(imgBuffer)
-        .digest('hex') as PreviewId
-
-      // Save first preview image
-      if (i++ === 0) {
-        await deps.insertPreview({ previewId, imgBuffer })
-        metadata[angle] = previewId
-      }
-
-      allImgsArr.push(imgBuffer)
-    }
-
-    // stitch 360 image
-    const fullImg = await joinImages(allImgsArr, {
-      direction: 'horizontal',
-      offset: 700,
-      margin: '0 700 0 700',
-      color: { alpha: 0, r: 0, g: 0, b: 0 }
-    })
-    const png = fullImg.png({ quality: 95 })
-    const buff = await png.toBuffer()
-    const fullImgId = crypto.createHash('md5').update(buff).digest('hex') as PreviewId
-
-    await deps.insertPreview({ previewId: fullImgId, imgBuffer: buff })
-    metadata['all' as Angle] = fullImgId
-
-    return { metadata }
-  }

packages/preview-service/src/services/pollForPreview.ts

@@ -1,88 +0,0 @@
-import type { UpdateHealthcheckData } from '@/clients/execHealthcheck.js'
-import {
-  metricDuration,
-  metricOperationErrors
-} from '@/observability/prometheusMetrics.js'
-import type {
-  GetNextUnstartedObjectPreview,
-  NotifyUpdate,
-  UpdatePreviewMetadata
-} from '@/repositories/objectPreview.js'
-import type { GenerateAndStore360Preview } from '@/services/360preview.js'
-import type { Logger } from 'pino'
-import type { LabelValues } from 'prom-client'
-import { WorkStatus, type WorkToBeDone } from '@/domain/backgroundWorker.js'
-
-export const pollForAndCreatePreviewFactory =
-  (deps: {
-    updateHealthcheckData: UpdateHealthcheckData
-    getNextUnstartedObjectPreview: GetNextUnstartedObjectPreview
-    generateAndStore360Preview: GenerateAndStore360Preview
-    updatePreviewMetadata: UpdatePreviewMetadata
-    notifyUpdate: NotifyUpdate
-    logger: Logger
-  }): WorkToBeDone =>
-  async () => {
-    try {
-      const task = await deps.getNextUnstartedObjectPreview()
-
-      // notify the healthcheck that we are still alive
-      deps.updateHealthcheckData()
-
-      if (!task) {
-        return WorkStatus.NOWORKFOUND
-      }
-      const logger = deps.logger.child({
-        projectId: task.streamId,
-        objectId: task.objectId
-      })
-
-      logger.info('Found next preview task for {projectId}/{objectId}')
-
-      let metricDurationEnd:
-        | (<T extends string>(labels?: LabelValues<T>) => number)
-        | undefined = undefined
-      if (metricDuration) {
-        metricDurationEnd = metricDuration.startTimer()
-      }
-
-      try {
-        const { metadata } = await deps.generateAndStore360Preview(task)
-
-        await deps.updatePreviewMetadata({
-          metadata,
-          streamId: task.streamId,
-          objectId: task.objectId
-        })
-        logger.info(
-          { previewStatus: 'succeeded' },
-          'Preview generation completed. Status: {previewStatus}'
-        )
-
-        await deps.notifyUpdate({ streamId: task.streamId, objectId: task.objectId })
-      } catch (err) {
-        logger.error(
-          { err, previewStatus: 'failed' },
-          'Preview generation completed. Status: {previewStatus}'
-        )
-        await deps.updatePreviewMetadata({
-          metadata: { err: err instanceof Error ? err.message : JSON.stringify(err) },
-          streamId: task.streamId,
-          objectId: task.objectId
-        })
-        metricOperationErrors?.labels('preview').inc()
-        return WorkStatus.FAILED
-      }
-      if (metricDurationEnd) {
-        metricDurationEnd({ op: 'preview' })
-      }
-
-      return WorkStatus.SUCCESS
-    } catch (err) {
-      if (metricOperationErrors) {
-        metricOperationErrors.labels('main_loop').inc()
-      }
-      deps.logger.error(err, 'Error executing task')
-      return WorkStatus.FAILED
-    }
-  }

packages/preview-service/src/services/screenshot.ts

@@ -1,87 +0,0 @@
-import { LoadPageAndEvaluateScript } from '@/clients/puppeteer.js'
-import type { ObjectIdentifier } from '@/domain/domain.js'
-import { reduce } from 'lodash-es'
-import type { Logger } from 'pino'
-import { z } from 'zod'
-
-export type GetScreenshot = (
-  params: ObjectIdentifier
-) => Promise<{ [key: string]: string } | null>
-
-export const getScreenshotFactory =
-  (deps: {
-    loadPageAndEvaluateScript: LoadPageAndEvaluateScript
-    logger: Logger
-    serviceOrigin: string
-  }): GetScreenshot =>
-  async (params) => {
-    const objectUrl = `${deps.serviceOrigin}/streams/${params.streamId}/objects/${params.objectId}`
-
-    const RenderOutputSchema = z.object({
-      duration: z.number(),
-      mem: z.object({ total: z.number() }),
-      scr: z.record(z.string())
-    })
-    type RenderOutput = z.infer<typeof RenderOutputSchema>
-
-    let renderOutput: RenderOutput
-    try {
-      // assume it is of type RenderOutput, and validate later
-      const rawRenderOutput = await deps.loadPageAndEvaluateScript(objectUrl)
-      renderOutput = await RenderOutputSchema.parseAsync(rawRenderOutput)
-    } catch (err) {
-      if (err instanceof z.ZodError) {
-        deps.logger.error(
-          err,
-          'Error generating preview. Expected output was not returned.'
-        )
-      } else {
-        deps.logger.error(err, 'Error generating preview.')
-      }
-      return null
-    }
-
-    deps.logger.info(
-      {
-        durationSeconds: renderOutput.duration,
-        totalMemoryMB: renderOutput.mem.total / 1000000,
-        resultingImages: {
-          count: Object.keys(renderOutput.scr || {}).length,
-          totalStringSize: reduce(
-            renderOutput.scr || {},
-            (acc: number, val: string) => acc + val.length,
-            0
-          )
-        }
-      },
-      `Generated preview.`
-    )
-    return renderOutput.scr
-
-    // return `
-    // <html><body>
-    // <div>Generated by: ${ret.userAgent}</div>
-    // <div>Duration in seconds: ${ret.duration}</div>
-    // <div>Memory in MB: ${ret.mem.total / 1000000}</div>
-    // <div>Used Memory in MB: ${ret.mem.used / 1000000}</div>
-    // <img height="200px" src="${ret.scr['-2']}" /><br />
-    // <img height="200px" src="${ret.scr['-1']}" /><br />
-    // <img height="200px" src="${ret.scr['0']}" /><br />
-    // <img height="200px" src="${ret.scr['1']}" /><br />
-    // <img height="200px" src="${ret.scr['2']}" /><br />
-    // </body></html>
-    // `
-
-    // const imageBuffer = new Buffer.from(
-    //   b64Image.replace(/^data:image\/\w+;base64,/, ''),
-    //   'base64'
-    // )
-
-    // // await page.waitForTimeout(500);
-    // //var response = await page.screenshot({
-    // //  type: 'png',
-    // //  clip: {x: 0, y: 0, width: 800, height: 800}
-    // //});
-
-    // return imageBuffer
-  }

packages/preview-service/src/utils/brand.ts

@@ -1,9 +0,0 @@
-declare const brand: unique symbol
-
-export type Brand<T, TBrand extends string> = T & { [brand]: TBrand }
-
-export const isCastableToBrand = <TBrand extends string>(
-  val: string | undefined | null
-): val is TBrand => {
-  return !!val
-}

packages/preview-service/src/utils/env.ts

@@ -1,40 +0,0 @@
-function getIntFromEnv(envVarKey: string, aDefault = '0'): number {
-  return parseInt(process.env[envVarKey] || aDefault)
-}
-function getBooleanFromEnv(envVarKey: string, aDefault = false): boolean {
-  return ['1', 'true', true].includes(process.env[envVarKey] || aDefault.toString())
-}
-
-export const getAppPort = () => process.env['PORT'] || '3001'
-export const getChromiumExecutablePath = () => {
-  if (isDevelopment()) return undefined // use default
-  return process.env['CHROMIUM_EXECUTABLE_PATH'] || '/usr/bin/google-chrome-stable'
-}
-export const getHealthCheckFilePath = () =>
-  process.env['HEALTHCHECK_FILE_PATH'] || '/tmp/last_successful_query'
-export const getHost = () => process.env['HOST'] || '127.0.0.1'
-export const getMetricsHost = () => process.env['METRICS_HOST'] || '127.0.0.1'
-export const getLogLevel = () => process.env['LOG_LEVEL'] || 'info'
-export const getMetricsPort = () => process.env['PROMETHEUS_METRICS_PORT'] || '9094'
-export const getNodeEnv = () => process.env['NODE_ENV'] || 'production'
-export const getPostgresConnectionString = () =>
-  process.env['PG_CONNECTION_STRING'] || 'postgres://speckle:speckle@127.0.0.1/speckle'
-export const getPostgresMaxConnections = () =>
-  getIntFromEnv('POSTGRES_MAX_CONNECTIONS_PREVIEW_SERVICE', '2')
-export const getConnectionAcquireTimeoutMillis = () =>
-  getIntFromEnv('POSTGRES_CONNECTION_ACQUIRE_TIMEOUT_MILLIS', '16000')
-export const getConnectionCreateTimeoutMillis = () =>
-  getIntFromEnv('POSTGRES_CONNECTION_CREATE_TIMEOUT_MILLIS', '5000')
-export const getPreviewTimeout = () => getIntFromEnv('PREVIEW_TIMEOUT', '3600000')
-export const getPuppeteerUserDataDir = () => {
-  if (isDevelopment()) return undefined // use default
-  return process.env['USER_DATA_DIR'] || '/tmp/puppeteer'
-}
-export const isDevelopment = () =>
-  getNodeEnv() === 'development' || getNodeEnv() === 'dev'
-export const isLogPretty = () => getBooleanFromEnv('LOG_PRETTY')
-export const isProduction = () => getNodeEnv() === 'production'
-export const isTest = () => getNodeEnv() === 'test'
-export const isDevOrTestEnv = () => isDevelopment() || isTest()
-export const serviceOrigin = () => `http://${getHost()}:${getAppPort()}`
-export const shouldBeHeadless = () => !getBooleanFromEnv('PREVIEWS_HEADED')

packages/preview-service/src/utils/errorHandler.ts

@@ -1,25 +0,0 @@
-import { ErrorRequestHandler } from 'express'
-import { isNaN, isObject, isString } from 'lodash-es'
-
-export const errorHandler: ErrorRequestHandler = (err, req, res) => {
-  if (
-    isObject(err) &&
-    'status' in err &&
-    typeof err.status === 'number' &&
-    !isNaN(err.status)
-  ) {
-    res.status(err?.status)
-  } else {
-    res.status(500)
-  }
-
-  res.setHeader('Content-Type', 'application/json')
-
-  if (req.app.get('env') === 'development') {
-    res.send(JSON.stringify(err, undefined, 2))
-  } else if (isObject(err) && 'message' in err && isString(err.message)) {
-    res.send(JSON.stringify({ message: err.message }))
-  } else {
-    res.send(JSON.stringify({ message: 'Internal Server Error' }))
-  }
-}

packages/preview-service/src/utils/headers.ts

@@ -1,7 +0,0 @@
-import express from 'express'
-
-export const isSimpleTextRequested = (req: express.Request) =>
-  req.headers.accept === 'text/plain'
-
-export const simpleTextOrJsonContentType = (req: express.Request) =>
-  isSimpleTextRequested(req) ? 'text/plain' : 'application/json'

packages/preview-service/src/utils/runtime.ts

@@ -1,9 +0,0 @@
-import path from 'path'
-import { fileURLToPath } from 'url'
-
-export const getDirname = (importMetaUrl: string) => {
-  const __filename = fileURLToPath(importMetaUrl)
-  const __dirname = path.dirname(__filename)
-
-  return __dirname
-}

packages/preview-service/src/utils/speckleObjectsStream.ts

@@ -1,50 +0,0 @@
-import { Transform, type TransformCallback } from 'stream'
-
-// A stream that converts database objects stream to "{id}\t{data_json}\n" stream or a json stream of obj.data fields
-
-export class SpeckleObjectsStream extends Transform {
-  isFirstObject: boolean
-  simpleText: boolean
-
-  constructor(simpleText: boolean) {
-    super({ writableObjectMode: true })
-    this.simpleText = simpleText
-
-    if (!this.simpleText) this.push('[')
-    this.isFirstObject = true
-  }
-
-  _transform(
-    dbObj: { id: string; dataText: unknown; data: unknown },
-    _encoding: BufferEncoding,
-    callback: TransformCallback
-  ) {
-    let objData = dbObj.dataText
-    if (objData === undefined) objData = JSON.stringify(dbObj.data)
-
-    try {
-      if (this.simpleText) {
-        this.push(`${dbObj.id}\t`)
-        this.push(objData)
-        this.push('\n')
-      } else {
-        // JSON output
-        if (!this.isFirstObject) this.push(',')
-        this.push(objData)
-        this.isFirstObject = false
-      }
-      callback()
-    } catch (e) {
-      if (typeof e === 'undefined' || e === null || e instanceof Error) {
-        callback(e)
-      } else {
-        callback(new Error(JSON.stringify(e)))
-      }
-    }
-  }
-
-  _flush(callback: TransformCallback) {
-    if (!this.simpleText) this.push(']')
-    callback()
-  }
-}

packages/preview-service/tests/acceptance/README.md

@@ -1,3 +0,0 @@
-# Acceptance tests
-
-This directory contains acceptance tests for the Preview Service.

packages/preview-service/tests/acceptance/acceptance.spec.ts

@@ -1,195 +0,0 @@
-import { acceptanceTest } from '#/helpers/testExtensions.js'
-import { ObjectPreview, type ObjectPreviewRow } from '@/repositories/objectPreview.js'
-import { Previews } from '@/repositories/previews.js'
-import cryptoRandomString from 'crypto-random-string'
-import { afterEach, beforeEach, describe, expect } from 'vitest'
-import { promises as fs } from 'fs'
-import { OBJECTS_TABLE_NAME } from '#/migrations/migrations.js'
-import type { Angle } from '@/domain/domain.js'
-import { testLogger as logger } from '@/observability/logging.js'
-import { parse } from 'csv-parse'
-import iconv from 'iconv-lite'
-import { createReadStream } from 'fs'
-import { PutObjectCommand, PutObjectCommandInput, S3Client } from '@aws-sdk/client-s3'
-import { finished } from 'stream/promises'
-
-const getS3Config = () => {
-  return {
-    credentials: {
-      accessKeyId: process.env.S3_ACCESS_KEY || '',
-      secretAccessKey: process.env.S3_SECRET_KEY || ''
-    },
-    endpoint: process.env.S3_ENDPOINT || '',
-    forcePathStyle: true,
-    // s3ForcePathStyle: true,
-    // signatureVersion: 'v4',
-    region: process.env.S3_REGION || 'us-east-1'
-  }
-}
-
-describe.sequential('Acceptance', () => {
-  describe.sequential('Trigger and wait for the preview-service', () => {
-    beforeEach(() => {
-      // const dbName = inject('dbName')
-      logger.info('🤜 running acceptance test before-each')
-    })
-    afterEach(() => {
-      logger.info('🤛 running acceptance test after-each')
-    })
-
-    // we use integration test and not e2e test because we don't need the server
-    acceptanceTest(
-      'loads data, waits for completion, extracts rendered image',
-      {
-        timeout: 300000 //5 minutes
-      },
-      async ({ context }) => {
-        const { db } = context
-        // load data
-        const streamId = cryptoRandomString({ length: 10 })
-
-        const processFile = async (csvFileName: string) => {
-          const objectRows: Record<string, unknown>[] = []
-
-          // Initialize the parser
-          const parser = createReadStream(csvFileName)
-            .pipe(iconv.decodeStream('utf8'))
-            .pipe(
-              parse({
-                delimiter: ',',
-                quote: '"',
-                // escape: null,
-                // eslint-disable-next-line camelcase
-                skip_empty_lines: true,
-                // eslint-disable-next-line camelcase
-                auto_parse: true
-              })
-            )
-          parser.on('readable', () => {
-            let record: unknown
-            while ((record = parser.read()) !== null) {
-              if (!Array.isArray(record)) throw new Error('Invalid record')
-              if (record.length !== 7) throw new Error('Invalid record length')
-
-              // "id","speckleType","totalChildrenCount","totalChildrenCountByDepth","createdAt","data","streamId"
-              const row = {
-                id: <unknown>record[0],
-                speckleType: <unknown>record[1],
-                totalChildrenCount: <unknown>record[2],
-                totalChildrenCountByDepth: <unknown>record[3],
-                createdAt: <unknown>record[4],
-                data: <unknown>JSON.parse(record[5] as string),
-                streamId //NOTE we override the stream ID to ensure it's loaded into the test stream
-              }
-
-              objectRows.push(row)
-            }
-          })
-          parser.on('end', () => {
-            //no-op, completion of the stream is handled by the finished(parser) call.
-          })
-          parser.on('error', (err) => {
-            expect(err).not.toBeDefined()
-          })
-
-          await finished(parser)
-          return objectRows
-        }
-
-        const objectId = 'b81d1d9295a995d9479186324b6f145a' //base object in tests/data/deltas.csv
-        try {
-          const objectsToInsert = await processFile('tests/data/deltas.csv')
-          await db.batchInsert(OBJECTS_TABLE_NAME, objectsToInsert)
-        } catch (err) {
-          expect(err, 'Error parsing CSV file.').toBeUndefined()
-        }
-
-        const objectPreviewRow = {
-          streamId,
-          objectId,
-          priority: 0,
-          previewStatus: 0
-        }
-        await ObjectPreview({ db }).insert(objectPreviewRow).onConflict().ignore()
-
-        //poll the database until the preview is ready
-        let objectPreviewResult: Pick<ObjectPreviewRow, 'preview' | 'previewStatus'>[] =
-          []
-        while (
-          objectPreviewResult.length === 0 ||
-          objectPreviewResult[0].previewStatus !== 2
-        ) {
-          objectPreviewResult = await ObjectPreview({ db })
-            .select(['preview', 'previewStatus'])
-            .where('streamId', streamId)
-            .andWhere('objectId', objectId)
-
-          logger.info(
-            { result: objectPreviewResult, streamId, objectId },
-            '🔍 Polled object preview for a result for {streamId} and {objectId}'
-          )
-          // wait a second before polling again
-          await new Promise((resolve) => setTimeout(resolve, 1000))
-        }
-
-        const previewData = await Previews({ db })
-          .select(['data'])
-          .where('id', objectPreviewResult[0].preview['all' as Angle])
-          .first()
-        logger.info({ previewData }, '🔍 Retrieved preview data')
-
-        if (!previewData) {
-          expect(previewData).toBeDefined()
-          expect(previewData).not.toBeNull()
-          return //HACK to appease typescript
-        }
-
-        if (!process.env.OUTPUT_FILE_PATH)
-          throw new Error('OUTPUT_FILE_PATH environment variable not set')
-
-        const outputFilePath = process.env.OUTPUT_FILE_PATH
-
-        const s3Config = getS3Config()
-
-        if (s3Config.credentials.accessKeyId && s3Config.credentials.secretAccessKey) {
-          logger.info(
-            { outputFilePath },
-            'S3 credentials provided, saving to S3 at {outputFilePath}'
-          )
-          const s3Client = new S3Client(s3Config)
-
-          const params: PutObjectCommandInput = {
-            Bucket: 'github-action-speckle-preview-service-acceptance-test',
-            Key: outputFilePath,
-            Body: previewData.data,
-            ACL: 'public-read',
-            Metadata: {
-              // Defines metadata tags.
-              // 'x-amz-meta-my-key': 'your-value'
-            }
-          }
-
-          const uploadObject = async () => {
-            try {
-              const data = await s3Client.send(new PutObjectCommand(params))
-              logger.info(
-                'Successfully uploaded object: ' + params.Bucket + '/' + params.Key
-              )
-              return data
-            } catch (err) {
-              logger.error(err, 'Failed to upload object')
-            }
-          }
-
-          await uploadObject()
-        } else {
-          logger.info(
-            { outputFilePath },
-            'No S3 credentials provided, saving to local file system at {outputFilePath}'
-          )
-          await fs.writeFile(outputFilePath, previewData.data)
-        }
-      }
-    )
-  })
-})

packages/preview-service/tests/e2e/roundtrip.spec.ts

@@ -1,34 +0,0 @@
-// example tests to confirm the servers are running and the API is working
-
-import { getServerPort } from '#/helpers/helpers.js'
-import { e2eTest } from '#/helpers/testExtensions.js'
-import { describe } from 'vitest'
-
-describe.concurrent('E2E', () => {
-  describe.concurrent('Example', () => {
-    e2eTest('should start a server on an unique port', async ({ context }) => {
-      const port = getServerPort(context.server)
-      console.log(`port1 : ${port}`)
-      await Promise.resolve()
-    })
-    e2eTest('should start a server on a different port', async ({ context }) => {
-      const port = getServerPort(context.server)
-      console.log(`port2 : ${port}`)
-      await Promise.resolve()
-    })
-  })
-  describe.concurrent('adding a job in the database', () => {
-    e2eTest('should create a preview', async ({ context }) => {
-      const port = getServerPort(context.server)
-      console.log(`port3 : ${port}`)
-
-      //TODO add an object in the object store
-      //TODO add a job in the database
-      //wait for the job in the database to be updated
-      //wait for the job in the database to be completed
-      //ensure the preview is created
-      //ensure the preview has all the required angles
-      await Promise.resolve()
-    })
-  })
-})

packages/preview-service/tests/helpers/helpers.ts

@@ -1,49 +0,0 @@
-import { startServer } from '@/server/server.js'
-import http from 'http'
-import type { AddressInfo } from 'net'
-import { getPostgresConnectionString } from '@/utils/env.js'
-
-export const startAndWaitOnServers = async () => {
-  let serverAddress: string | AddressInfo | null = null
-  let metricsServerAddress: string | AddressInfo | null = null
-
-  const { app, server, metricsServer } = await startServer({
-    serveOnRandomPort: true
-  })
-  server.on('listening', () => {
-    serverAddress = server.address()
-  })
-  metricsServer.on('listening', () => {
-    metricsServerAddress = metricsServer.address()
-  })
-
-  //HACK wait until both servers are available
-  while (!serverAddress || !metricsServerAddress) {
-    // wait for the servers to start
-    await new Promise((resolve) => setTimeout(resolve, 100))
-  }
-
-  return { app, server, metricsServer }
-}
-
-export const getServerPort = (server: http.Server) => {
-  const address = server.address()
-  if (address && typeof address !== 'string') {
-    return address.port
-  }
-  throw new Error('Server port is not available')
-}
-
-export const customizePostgresConnectionString = (databaseName?: string) => {
-  const originalPostgresConnectionString = getPostgresConnectionString()
-  if (!databaseName) return originalPostgresConnectionString
-
-  const originalPostgresUrl = new URL(originalPostgresConnectionString)
-  const protocol = originalPostgresUrl.protocol
-  const user = originalPostgresUrl.username
-  const pass = originalPostgresUrl.password
-  const host = originalPostgresUrl.hostname
-  const port = originalPostgresUrl.port
-  const origin = `${protocol}//${user}:${pass}@${host}:${port}`
-  return new URL(databaseName, origin).toString()
-}

packages/preview-service/tests/helpers/testExtensions.ts

@@ -1,95 +0,0 @@
-import { stopServer } from '@/server/server.js'
-import { inject, test } from 'vitest'
-import { getTestDb } from '#/helpers/testKnexClient.js'
-import { startAndWaitOnServers } from '#/helpers/helpers.js'
-import type { Knex } from 'knex'
-import { Server } from 'http'
-
-export interface AcceptanceTestContext {
-  context: {
-    db: Knex
-  }
-}
-
-// vitest reference: https://vitest.dev/guide/test-context#fixture-initialization
-export const acceptanceTest = test.extend<AcceptanceTestContext>({
-  // this key has to match the top level key in the interface (i.e. `context`). Some vitest typing magic at work here.
-  context: [
-    // eslint-disable-next-line @typescript-eslint/no-unused-vars
-    async ({ task, onTestFinished }, use) => {
-      const dbName = inject('dbName')
-      // equivalent of beforeEach
-      const db = getTestDb(dbName)
-
-      // schedule the cleanup. Runs regardless of test status, and runs after afterEach.
-      onTestFinished(async () => {
-        //no-op
-      })
-
-      // now run the test
-      await use({ db })
-    },
-    { auto: true } // we want to run this for each databaseIntegrationTest, even if the context is not explicitly requested by the test
-  ]
-})
-
-export interface DatabaseIntegrationTestContext {
-  context: {
-    db: Knex.Transaction
-  }
-}
-
-// vitest reference: https://vitest.dev/guide/test-context#fixture-initialization
-export const databaseIntegrationTest = test.extend<DatabaseIntegrationTestContext>({
-  // this key has to match the top level key in the interface (i.e. `context`). Some vitest typing magic at work here.
-  context: [
-    // eslint-disable-next-line @typescript-eslint/no-unused-vars
-    async ({ task, onTestFinished }, use) => {
-      const dbName = inject('dbName')
-      // equivalent of beforeEach
-      const db = await getTestDb(dbName).transaction()
-
-      // schedule the cleanup. Runs regardless of test status, and runs after afterEach.
-      onTestFinished(async () => {
-        await db.rollback()
-      })
-
-      // now run the test
-      await use({ db })
-    },
-    { auto: true } // we want to run this for each databaseIntegrationTest, even if the context is not explicitly requested by the test
-  ]
-})
-
-export interface E2ETestContext extends DatabaseIntegrationTestContext {
-  context: {
-    db: Knex.Transaction
-    server: Server
-    metricsServer: Server
-  }
-}
-
-// vitest reference: https://vitest.dev/guide/test-context#fixture-initialization
-export const e2eTest = test.extend<E2ETestContext>({
-  // this key has to match the top level key in the interface (i.e. `context`). Some vitest typing magic at work here.
-  context: [
-    // eslint-disable-next-line @typescript-eslint/no-unused-vars
-    async ({ task, onTestFinished }, use) => {
-      const dbName = inject('dbName')
-      // equivalent of beforeEach
-      const db = await getTestDb(dbName).transaction()
-      const { server, metricsServer } = await startAndWaitOnServers()
-
-      // schedule the cleanup. Runs regardless of test status, and runs after afterEach.
-      onTestFinished(async () => {
-        if (server) stopServer({ server })
-        if (metricsServer) stopServer({ server: metricsServer })
-        if (db) await db.rollback()
-      })
-
-      // now run the test
-      await use({ db, server, metricsServer })
-    },
-    { auto: true } // we want to run this for each e2eTest, even if the context is not explicitly requested by the test
-  ]
-})

packages/preview-service/tests/helpers/testKnexClient.ts

@@ -1,20 +0,0 @@
-/* eslint-disable camelcase */
-import { knex } from 'knex'
-import { customizePostgresConnectionString } from '#/helpers/helpers.js'
-
-export const getTestDb = (databaseName?: string) =>
-  knex({
-    client: 'pg',
-    connection: {
-      application_name: 'speckle_preview_service',
-      connectionString: customizePostgresConnectionString(databaseName)
-    },
-    pool: { min: 0, max: 2 }
-    // migrations are managed in the server package for production
-    // for tests, we are creating a new database for each test run so we can't use this default migration functionality
-    // migrations: {
-    //   extension: '.ts',
-    //   directory: path.resolve(__dirname, '../migrations'),
-    //   loadExtensions: ['js', 'ts']
-    // }
-  })

packages/preview-service/tests/hooks/globalSetup.ts

@@ -1,79 +0,0 @@
-/**
- * These hooks are run once, before and after the test suite.
- * It is configured via the vitest.config.ts file.
- */
-import '@/bootstrap.js' // This has side-effects and has to be imported first
-import { getTestDb } from '#/helpers/testKnexClient.js'
-import { down, up } from '#/migrations/migrations.js'
-import { testLogger as logger } from '@/observability/logging.js'
-import cryptoRandomString from 'crypto-random-string'
-import type { GlobalSetupContext } from 'vitest/node'
-
-declare module 'vitest' {
-  export interface ProvidedContext {
-    dbName: string
-  }
-}
-
-const dbName =
-  process.env.TEST_DB || // in the acceptance tests we need to use a database name that is known prior to the test running
-  `preview_service_${cryptoRandomString({
-    length: 10,
-    type: 'alphanumeric'
-  })}`.toLocaleLowerCase() //postgres will automatically lower case new db names
-let isDatabaseCreatedExternally = true
-
-/**
- * Global setup hook
- * This hook is run once before any tests are run
- * Defined in vitest.config.ts under test.globalSetup
- */
-export async function setup({ provide }: GlobalSetupContext) {
-  logger.info('🏃🏻‍♀️‍➡️ Running vitest setup global hook')
-  const superUserDbClient = getTestDb()
-  const dbAlreadyExists = await superUserDbClient('pg_database')
-    .select('datname')
-    .where('datname', dbName)
-  if (!dbAlreadyExists.length) {
-    isDatabaseCreatedExternally = false
-    await superUserDbClient.raw(`CREATE DATABASE ${dbName}
-    WITH
-    OWNER = preview_service_test
-    ENCODING = 'UTF8'
-    TABLESPACE = pg_default
-    CONNECTION LIMIT = -1;`)
-  }
-  await superUserDbClient.destroy() // need to explicitly close the connection in clients to prevent hanging tests
-
-  // this provides the dbName to all tests, and can be accessed via inject('dbName'). NB: The test extensions already implement this, so use a test extension.
-  provide('dbName', dbName)
-
-  const db = getTestDb(dbName)
-  await up(db) //we need the migration to occur in our new database, so cannot use knex's built in migration functionality.
-  await db.destroy() // need to explicitly close the connection in clients to prevent hanging tests
-  logger.info(
-    `💁🏽‍♀️ Completed the vitest setup global hook. Database created at ${dbName}`
-  )
-}
-
-/**
- * Global teardown hook
- * This hook is run once after all tests are run
- * Defined in vitest.config.ts under test.globalTeardown
- */
-export async function teardown() {
-  logger.info('🏃🏻‍♀️ Running vitest teardown global hook')
-  const db = getTestDb(dbName)
-  await down(db) //we need the migration to occur in our named database, so cannot use knex's built in migration functionality.
-  await db.destroy() // need to explicitly close the connection in clients to prevent hanging tests
-
-  if (!isDatabaseCreatedExternally) {
-    //use connection without database to drop the db
-    const superUserDbClient = getTestDb()
-    await superUserDbClient.raw(`DROP DATABASE ${dbName};`)
-    await superUserDbClient.destroy() // need to explicitly close the connection in clients to prevent hanging tests
-  }
-  logger.info(
-    `✅ Completed the vitest teardown global hook. Destroyed database at ${dbName}`
-  )
-}

packages/preview-service/tests/integration/repositories/objectPreview.spec.ts

@@ -1,38 +0,0 @@
-import { databaseIntegrationTest } from '#/helpers/testExtensions.js'
-import {
-  ObjectPreview,
-  getNextUnstartedObjectPreviewFactory
-} from '@/repositories/objectPreview.js'
-import cryptoRandomString from 'crypto-random-string'
-import { describe, expect } from 'vitest'
-
-describe.concurrent('Repositories: ObjectPreview', () => {
-  describe.concurrent('getNextUnstartedObjectPreview', () => {
-    databaseIntegrationTest(
-      'should return the next unstarted object preview',
-      async ({ context }) => {
-        const streamId = cryptoRandomString({ length: 10 })
-        const objectId = cryptoRandomString({ length: 10 })
-        const insertionObject = {
-          streamId,
-          objectId,
-          priority: 0,
-          previewStatus: 0
-        }
-        const sqlQuery = ObjectPreview({ db: context.db })
-          .insert(insertionObject)
-          .onConflict()
-          .ignore()
-        await context.db.raw(sqlQuery.toQuery())
-
-        const getNextUnstartedObjectPreview = getNextUnstartedObjectPreviewFactory({
-          db: context.db
-        })
-        const result = await getNextUnstartedObjectPreview()
-        expect(result).toBeDefined()
-        expect(result.streamId).toEqual(streamId)
-        expect(result.objectId).toEqual(objectId)
-      }
-    )
-  })
-})

packages/preview-service/tests/migrations/README.md

@@ -1,7 +0,0 @@
-# Knex Migrations
-
-This is not your regular knex migrations directory.
-
-Because the test database is expected to be in a clean state before each test, we need to run migrations rollback and up before each run of tests and additionally rollback after each run.
-
-Therefore we can just have one single migration file, and don't need to version it.

packages/preview-service/tests/migrations/migrations.ts

@@ -1,106 +0,0 @@
-import type { Knex } from 'knex'
-
-const OBJECT_PREVIEW_TABLE_NAME = 'object_preview'
-const PREVIEWS_TABLE_NAME = 'previews'
-export const OBJECTS_TABLE_NAME = 'objects'
-const DB_NAME_PREFIX = 'preview_service_'
-
-const getDatabaseName = (deps: { db: Knex }) => {
-  return deps.db.raw<{ rows: { datname: string }[] }>(
-    `SELECT current_database() as datname`
-  )
-}
-
-const getAllTableNames = (deps: { db: Knex }) => {
-  return deps.db.raw<{ rows: { tablename: string }[] }>(
-    `SELECT tablename FROM pg_tables WHERE schemaname='public'`
-  )
-}
-
-const throwIfDbNameDoesNotStartWithPrefix = async (deps: { db: Knex }) => {
-  const { rows: dbNameRows } = await getDatabaseName(deps)
-  const dbName = dbNameRows[0].datname
-  if (!dbName.startsWith(DB_NAME_PREFIX)) {
-    throw new Error(
-      `Database name does not start with "${DB_NAME_PREFIX}", it is unsafe to migrate to test schema. Aborting.`
-    )
-  }
-}
-
-const hasExpectedTableNames = (params: { tableNames: string[] }) => {
-  const { tableNames } = params
-  return (
-    tableNames.length === 3 &&
-    [OBJECT_PREVIEW_TABLE_NAME, OBJECTS_TABLE_NAME, PREVIEWS_TABLE_NAME].every((t) =>
-      tableNames.includes(t)
-    )
-  )
-}
-
-const throwIfNotSafeToMigrateUp = async (deps: { db: Knex }) => {
-  await throwIfDbNameDoesNotStartWithPrefix(deps)
-
-  const { rows } = await getAllTableNames(deps)
-  const tableNames = rows.map((x) => x.tablename)
-  if (tableNames.length > 0 && !hasExpectedTableNames({ tableNames })) {
-    throw new Error(
-      `Database has unexpected tables, it is unsafe to migrate to test schema. Aborting. Tables found: ${tableNames.join(
-        ', '
-      )}`
-    )
-  }
-}
-
-const throwIfNotSafeToMigrateDown = async (deps: { db: Knex }) => {
-  await throwIfDbNameDoesNotStartWithPrefix(deps)
-
-  const { rows } = await getAllTableNames(deps)
-  const tableNames = rows.map((x) => x.tablename)
-  if (!hasExpectedTableNames({ tableNames })) {
-    throw new Error(
-      `Database already has unexpected tables, it is unsafe to migrate to test schema. Aborting. Tables found: ${tableNames.join(
-        ', '
-      )}`
-    )
-  }
-}
-
-export const up = async (db: Knex) => {
-  await throwIfNotSafeToMigrateUp({ db })
-
-  await db.schema.createTable(OBJECT_PREVIEW_TABLE_NAME, (table) => {
-    table.string('streamId', 10) //ignoring fk on streams table for simplicity
-    table.string('objectId').notNullable()
-    table.integer('previewStatus').notNullable().defaultTo(0) //TODO should be an enum
-    table.integer('priority').notNullable().defaultTo(1)
-    table.timestamp('lastUpdate').notNullable().defaultTo(db.fn.now())
-    table.jsonb('preview')
-    table.primary(['streamId', 'objectId'])
-    table.index(['previewStatus', 'priority', 'lastUpdate'])
-  })
-
-  await db.schema.createTable(PREVIEWS_TABLE_NAME, (table) => {
-    table.string('id').primary()
-    table.binary('data')
-  })
-
-  await db.schema.createTable(OBJECTS_TABLE_NAME, (table) => {
-    table.string('id')
-    table.string('streamId', 10) //ignoring fk on streams table for simplicity
-    table.string('speckleType', 1024).defaultTo('Base').notNullable()
-    table.integer('totalChildrenCount')
-    table.jsonb('totalChildrenCountByDepth')
-    table.timestamp('createdAt').defaultTo(db.fn.now())
-    table.jsonb('data')
-    table.index('id')
-    table.index('streamId')
-    table.primary(['streamId', 'id'])
-  })
-}
-
-export const down = async (db: Knex) => {
-  await throwIfNotSafeToMigrateDown({ db })
-  await db.schema.dropTable(OBJECT_PREVIEW_TABLE_NAME)
-  await db.schema.dropTable(PREVIEWS_TABLE_NAME)
-  await db.schema.dropTable(OBJECTS_TABLE_NAME)
-}

packages/preview-service/tests/unit/services/pollForPreview.spec.ts

@@ -1,48 +0,0 @@
-import { logger } from '@/observability/logging.js'
-import { pollForAndCreatePreviewFactory } from '@/services/pollForPreview.js'
-import { describe, expect, it } from 'vitest'
-
-describe.concurrent('Polling for preview', () => {
-  describe.concurrent('pollForAndCreatePreview', () => {
-    it('calls all component functions with expected parameters', async () => {
-      const called: Record<string, number> = {}
-      const pollForAndCreatePreview = pollForAndCreatePreviewFactory({
-        updateHealthcheckData: () => {
-          called['updateHealthcheckData'] = called['updateHealthcheckData']++ || 1
-        },
-        getNextUnstartedObjectPreview: async () =>
-          Promise.resolve({
-            streamId: 'streamId',
-            objectId: 'objectId'
-          }),
-        generateAndStore360Preview: async (task) => {
-          called['generateAndStore360Preview'] =
-            called['generateAndStore360Preview']++ || 1
-          expect(task).toEqual({ streamId: 'streamId', objectId: 'objectId' })
-          return Promise.resolve({ metadata: { all: 'myJoinedUpPreviewId' } })
-        },
-        updatePreviewMetadata: async (params) => {
-          called['updatePreviewMetadata'] = called['updatePreviewMetadata']++ || 1
-          expect(params).toEqual({
-            metadata: { all: 'myJoinedUpPreviewId' },
-            streamId: 'streamId',
-            objectId: 'objectId'
-          })
-          return Promise.resolve()
-        },
-        notifyUpdate: async (task) => {
-          called['notifyUpdate'] = called['notifyUpdate']++ || 1
-          expect(task).toEqual({ streamId: 'streamId', objectId: 'objectId' })
-          return Promise.resolve()
-        },
-        logger
-      })
-
-      await pollForAndCreatePreview()
-      expect(called['updateHealthcheckData']).toBeGreaterThanOrEqual(1)
-      expect(called['generateAndStore360Preview']).toBeGreaterThanOrEqual(1)
-      expect(called['updatePreviewMetadata']).toBeGreaterThanOrEqual(1)
-      expect(called['notifyUpdate']).toBeGreaterThanOrEqual(1)
-    })
-  })
-})

packages/preview-service/tests/unit/services/screenshot.spec.ts

@@ -1,47 +0,0 @@
-import { LoadPageAndEvaluateScript } from '@/clients/puppeteer.js'
-import { logger } from '@/observability/logging.js'
-import { getScreenshotFactory } from '@/services/screenshot.js'
-import { afterEach, describe, expect, it, vi } from 'vitest'
-
-describe('Screenshot', () => {
-  afterEach(() => {
-    vi.restoreAllMocks()
-  })
-  describe('with Puppeteer returning a valid responses', () => {
-    const loadPageAndEvaluateScript: LoadPageAndEvaluateScript = (
-      urlOfObjectToScreenshot
-    ) => {
-      //NOTE if this expectation fails it won't get explicitly captured by vitest. Instead we get null output from getScreenshot.
-      expect(urlOfObjectToScreenshot).toBe(
-        'http://localhost:0000/streams/streamId/objects/objectId'
-      )
-      return Promise.resolve({
-        duration: 1000,
-        mem: { total: 500, used: 400 },
-        userAgent: 'Test Testerson',
-        scr: {
-          '0': 'data:image/png;base64,foobar',
-          '1': 'data:image/png;base64,foobar'
-        }
-      })
-    }
-
-    it('receives the screenshot', async () => {
-      const getScreenshot = getScreenshotFactory({
-        loadPageAndEvaluateScript,
-        logger,
-        serviceOrigin: 'http://localhost:0000'
-      })
-      const screenshot = await getScreenshot({
-        streamId: 'streamId',
-        objectId: 'objectId'
-      })
-      if (!screenshot) {
-        expect(screenshot).not.toBe(null)
-        return //to avoid TS error
-      }
-      expect(screenshot['0']).toBe('data:image/png;base64,foobar')
-      expect(screenshot['1']).toBe('data:image/png;base64,foobar')
-    })
-  })
-})

packages/preview-service/tsconfig.json

@@ -1,12 +1,16 @@
 {
   "compilerOptions": {
-    /* Visit https://aka.ms/tsconfig.json to read more about this file */
+    "baseUrl": "./" /* Specify the base directory to resolve non-relative module names. */,
+    "paths": {
+      "@/*": ["./src/*"]
+    },
+    /* Visit https://aka.ms/tsconfig to read more about this file */
 
     /* Projects */
-    // "incremental": true,                              /* Enable incremental compilation */
+    // "incremental": true,                              /* Save .tsbuildinfo files to allow for incremental compilation of projects. */
     // "composite": true,                                /* Enable constraints that allow a TypeScript project to be used with project references. */
-    // "tsBuildInfoFile": "./",                          /* Specify the folder for .tsbuildinfo incremental compilation files. */
-    // "disableSourceOfProjectReferenceRedirect": true,  /* Disable preferring source files instead of declaration files when referencing composite projects */
+    // "tsBuildInfoFile": "./.tsbuildinfo",              /* Specify the path to .tsbuildinfo incremental compilation file. */
+    // "disableSourceOfProjectReferenceRedirect": true,  /* Disable preferring source files instead of declaration files when referencing composite projects. */
     // "disableSolutionSearching": true,                 /* Opt a project out of multi-project reference checking when editing. */
     // "disableReferencedProjectLoad": true,             /* Reduce the number of projects loaded automatically by TypeScript. */
 
@@ -14,96 +18,99 @@
     "target": "ES2022" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
     // "lib": [],                                        /* Specify a set of bundled library declaration files that describe the target runtime environment. */
     // "jsx": "preserve",                                /* Specify what JSX code is generated. */
-    // "experimentalDecorators": true,                   /* Enable experimental support for TC39 stage 2 draft decorators. */
+    // "experimentalDecorators": true,                   /* Enable experimental support for legacy experimental decorators. */
     // "emitDecoratorMetadata": true,                    /* Emit design-type metadata for decorated declarations in source files. */
-    // "jsxFactory": "",                                 /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h' */
+    // "jsxFactory": "",                                 /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */
     // "jsxFragmentFactory": "",                         /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
-    // "jsxImportSource": "",                            /* Specify module specifier used to import the JSX factory functions when using `jsx: react-jsx*`.` */
-    // "reactNamespace": "",                             /* Specify the object invoked for `createElement`. This only applies when targeting `react` JSX emit. */
+    // "jsxImportSource": "",                            /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */
+    // "reactNamespace": "",                             /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */
     // "noLib": true,                                    /* Disable including any library files, including the default lib.d.ts. */
     // "useDefineForClassFields": true,                  /* Emit ECMAScript-standard-compliant class fields. */
+    // "moduleDetection": "auto",                        /* Control what method is used to detect module-format JS files. */
 
     /* Modules */
-    "module": "node16" /* Specify what module code is generated. */,
-    "rootDir": "./" /* Specify the root folder within your source files. */,
-    "moduleResolution": "node16" /* Specify how TypeScript looks up a file from a given module specifier. */,
-    "baseUrl": "./" /* Specify the base directory to resolve non-relative module names. */,
-    "paths": {
-      "@/*": ["./src/*"],
-      "#/*": ["./tests/*"]
-    },
+    "module": "ES2022" /* Specify what module code is generated. */,
+    // "rootDir": "./",                                  /* Specify the root folder within your source files. */
+    "moduleResolution": "node",
+    // "moduleResolution": "node10",                     /* Specify how TypeScript looks up a file from a given module specifier. */
+    // "baseUrl": "./",                                  /* Specify the base directory to resolve non-relative module names. */
+    // "paths": {},                                      /* Specify a set of entries that re-map imports to additional lookup locations. */
     // "rootDirs": [],                                   /* Allow multiple folders to be treated as one when resolving modules. */
-    // "typeRoots": [],                                  /* Specify multiple folders that act like `./node_modules/@types`. */
+    // "typeRoots": [],                                  /* Specify multiple folders that act like './node_modules/@types'. */
     // "types": [],                                      /* Specify type package names to be included without being referenced in a source file. */
     // "allowUmdGlobalAccess": true,                     /* Allow accessing UMD globals from modules. */
-    // "resolveJsonModule": true,                        /* Enable importing .json files */
-    // "noResolve": true,                                /* Disallow `import`s, `require`s or `<reference>`s from expanding the number of files TypeScript should add to a project. */
+    // "moduleSuffixes": [],                             /* List of file name suffixes to search when resolving a module. */
+    // "allowImportingTsExtensions": true,               /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */
+    // "rewriteRelativeImportExtensions": true,          /* Rewrite '.ts', '.tsx', '.mts', and '.cts' file extensions in relative import paths to their JavaScript equivalent in output files. */
+    // "resolvePackageJsonExports": true,                /* Use the package.json 'exports' field when resolving package imports. */
+    // "resolvePackageJsonImports": true,                /* Use the package.json 'imports' field when resolving imports. */
+    // "customConditions": [],                           /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */
+    // "noUncheckedSideEffectImports": true,             /* Check side effect imports. */
+    // "resolveJsonModule": true,                        /* Enable importing .json files. */
+    // "allowArbitraryExtensions": true,                 /* Enable importing files with any extension, provided a declaration file is present. */
+    // "noResolve": true,                                /* Disallow 'import's, 'require's or '<reference>'s from expanding the number of files TypeScript should add to a project. */
 
     /* JavaScript Support */
-    "allowJs": true /* Allow JavaScript files to be a part of your program. Use the `checkJS` option to get errors from these files. */,
-    "checkJs": false /* Enable error reporting in type-checked JavaScript files. */,
-    // "maxNodeModuleJsDepth": 1,                        /* Specify the maximum folder depth used for checking JavaScript files from `node_modules`. Only applicable with `allowJs`. */
+    // "allowJs": true,                                  /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */
+    // "checkJs": true,                                  /* Enable error reporting in type-checked JavaScript files. */
+    // "maxNodeModuleJsDepth": 1,                        /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */
 
     /* Emit */
     // "declaration": true,                              /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
     // "declarationMap": true,                           /* Create sourcemaps for d.ts files. */
     // "emitDeclarationOnly": true,                      /* Only output d.ts files and not JavaScript files. */
-    "sourceMap": true /* Create source map files for emitted JavaScript files. */,
-    // "outFile": "./",                                  /* Specify a file that bundles all outputs into one JavaScript file. If `declaration` is true, also designates a file that bundles all .d.ts output. */
+    // "sourceMap": true,                                /* Create source map files for emitted JavaScript files. */
+    // "inlineSourceMap": true,                          /* Include sourcemap files inside the emitted JavaScript. */
+    // "noEmit": true,                                   /* Disable emitting files from a compilation. */
+    // "outFile": "./",                                  /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */
     "outDir": "./dist" /* Specify an output folder for all emitted files. */,
     // "removeComments": true,                           /* Disable emitting comments. */
-    // "noEmit": true,                                   /* Disable emitting files from a compilation. */
     // "importHelpers": true,                            /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
-    // "importsNotUsedAsValues": "remove",               /* Specify emit/checking behavior for imports that are only used for types */
     // "downlevelIteration": true,                       /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
     // "sourceRoot": "",                                 /* Specify the root path for debuggers to find the reference source code. */
     // "mapRoot": "",                                    /* Specify the location where debugger should locate map files instead of generated locations. */
-    // "inlineSourceMap": true,                          /* Include sourcemap files inside the emitted JavaScript. */
     // "inlineSources": true,                            /* Include source code in the sourcemaps inside the emitted JavaScript. */
     // "emitBOM": true,                                  /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
     // "newLine": "crlf",                                /* Set the newline character for emitting files. */
-    // "stripInternal": true,                            /* Disable emitting declarations that have `@internal` in their JSDoc comments. */
-    // "noEmitHelpers": true,                            /* Disable generating custom helper functions like `__extends` in compiled output. */
+    // "stripInternal": true,                            /* Disable emitting declarations that have '@internal' in their JSDoc comments. */
+    // "noEmitHelpers": true,                            /* Disable generating custom helper functions like '__extends' in compiled output. */
     // "noEmitOnError": true,                            /* Disable emitting files if any type checking errors are reported. */
-    // "preserveConstEnums": true,                       /* Disable erasing `const enum` declarations in generated code. */
+    // "preserveConstEnums": true,                       /* Disable erasing 'const enum' declarations in generated code. */
     // "declarationDir": "./",                           /* Specify the output directory for generated declaration files. */
-    // "preserveValueImports": true,                     /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */
 
     /* Interop Constraints */
     // "isolatedModules": true,                          /* Ensure that each file can be safely transpiled without relying on other imports. */
-    "allowSyntheticDefaultImports": true /* Allow 'import x from y' when a module doesn't have a default export. */,
-    "esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables `allowSyntheticDefaultImports` for type compatibility. */,
+    // "verbatimModuleSyntax": true,                     /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */
+    // "isolatedDeclarations": true,                     /* Require sufficient annotation on exports so other tools can trivially generate declaration files. */
+    // "allowSyntheticDefaultImports": true,             /* Allow 'import x from y' when a module doesn't have a default export. */
+    "esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */,
     // "preserveSymlinks": true,                         /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
     "forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */,
 
     /* Type Checking */
     "strict": true /* Enable all strict type-checking options. */,
-    // "noImplicitAny": true,                            /* Enable error reporting for expressions and declarations with an implied `any` type.. */
-    // "strictNullChecks": true,                         /* When type checking, take into account `null` and `undefined`. */
+    // "noImplicitAny": true,                            /* Enable error reporting for expressions and declarations with an implied 'any' type. */
+    // "strictNullChecks": true,                         /* When type checking, take into account 'null' and 'undefined'. */
     // "strictFunctionTypes": true,                      /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */
-    // "strictBindCallApply": true,                      /* Check that the arguments for `bind`, `call`, and `apply` methods match the original function. */
+    // "strictBindCallApply": true,                      /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */
     // "strictPropertyInitialization": true,             /* Check for class properties that are declared but not set in the constructor. */
-    // "noImplicitThis": true,                           /* Enable error reporting when `this` is given the type `any`. */
-    // "useUnknownInCatchVariables": true,               /* Type catch clause variables as 'unknown' instead of 'any'. */
+    // "strictBuiltinIteratorReturn": true,              /* Built-in iterators are instantiated with a 'TReturn' type of 'undefined' instead of 'any'. */
+    // "noImplicitThis": true,                           /* Enable error reporting when 'this' is given the type 'any'. */
+    // "useUnknownInCatchVariables": true,               /* Default catch clause variables as 'unknown' instead of 'any'. */
     // "alwaysStrict": true,                             /* Ensure 'use strict' is always emitted. */
-    // "noUnusedLocals": true,                           /* Enable error reporting when a local variables aren't read. */
-    // "noUnusedParameters": true,                       /* Raise an error when a function parameter isn't read */
+    // "noUnusedLocals": true,                           /* Enable error reporting when local variables aren't read. */
+    // "noUnusedParameters": true,                       /* Raise an error when a function parameter isn't read. */
     // "exactOptionalPropertyTypes": true,               /* Interpret optional property types as written, rather than adding 'undefined'. */
     // "noImplicitReturns": true,                        /* Enable error reporting for codepaths that do not explicitly return in a function. */
     // "noFallthroughCasesInSwitch": true,               /* Enable error reporting for fallthrough cases in switch statements. */
-    // "noUncheckedIndexedAccess": true,                 /* Include 'undefined' in index signature results */
+    // "noUncheckedIndexedAccess": true,                 /* Add 'undefined' to a type when accessed using an index. */
     // "noImplicitOverride": true,                       /* Ensure overriding members in derived classes are marked with an override modifier. */
-    // "noPropertyAccessFromIndexSignature": true,       /* Enforces using indexed accessors for keys declared using an indexed type */
+    // "noPropertyAccessFromIndexSignature": true,       /* Enforces using indexed accessors for keys declared using an indexed type. */
     // "allowUnusedLabels": true,                        /* Disable error reporting for unused labels. */
     // "allowUnreachableCode": true,                     /* Disable error reporting for unreachable code. */
 
     /* Completeness */
     // "skipDefaultLibCheck": true,                      /* Skip type checking .d.ts files that are included with TypeScript. */
     "skipLibCheck": true /* Skip type checking all .d.ts files. */
-  },
-  "ts-node": {
-    "swc": true
-  },
-  "include": ["src/**/*", "tests/**/*", "vitest.config.ts"],
-  "exclude": ["node_modules", "coverage", "reports"]
+  }
 }

packages/preview-service/vitest.config.ts

@@ -1,20 +0,0 @@
-import path from 'path'
-import { configDefaults, defineConfig } from 'vitest/config'
-
-export default defineConfig({
-  test: {
-    exclude: [...configDefaults.exclude],
-    globalSetup: ['./tests/hooks/globalSetup.ts'],
-    // reporters: ['verbose', 'hanging-process'] //uncomment to debug hanging processes etc.
-    sequence: {
-      shuffle: true,
-      concurrent: true
-    }
-  },
-  resolve: {
-    alias: {
-      '@': path.resolve(__dirname, './src'),
-      '#': path.resolve(__dirname, './tests')
-    }
-  }
-})

packages/preview-service/webpack.config.renderPage.cjs

@@ -1,72 +0,0 @@
-const HtmlWebpackPlugin = require('html-webpack-plugin')
-const { CleanWebpackPlugin } = require('clean-webpack-plugin')
-const path = require('path')
-const yargs = require('yargs')
-const env = yargs.argv.env
-
-const filename = 'viewer'
-
-let outputFile, mode
-
-if (env === 'build') {
-  mode = 'production'
-  outputFile = filename + '.min.js'
-} else {
-  mode = 'development'
-  outputFile = filename + '.js'
-}
-
-/**
- * @type {import('webpack').Configuration}
- */
-const config = {
-  mode,
-  entry: path.resolve(path.join(__dirname, 'renderPage', 'src', 'app.js')),
-  target: 'web',
-  devtool: 'source-map',
-  output: {
-    path: path.resolve(path.join(__dirname, 'dist', 'public', 'render')),
-    filename: outputFile
-  },
-  module: {
-    rules: [
-      {
-        test: /(\.jsx|\.js|\.ts|\.tsx)$/,
-        use: {
-          loader: 'babel-loader'
-        },
-        exclude: /(node_modules|bower_components)/
-      },
-      {
-        test: /\.(png|svg|jpg|jpeg|gif)$/i,
-        type: 'asset/resource'
-      }
-    ]
-  },
-  plugins: [
-    new CleanWebpackPlugin({ cleanStaleWebpackAssets: false }),
-    new HtmlWebpackPlugin({
-      title: 'Speckle Viewer Example',
-      template: 'renderPage/src/example.html',
-      filename: 'index.html',
-      favicon: 'renderPage/src/favicon.ico'
-    })
-  ],
-  resolve: {
-    modules: [
-      path.resolve('../../node_modules'),
-      path.resolve('./node_modules'),
-      path.resolve('.renderPage/src')
-    ],
-    extensions: ['.json', '.js']
-  },
-  devServer: {
-    contentBase: path.join(__dirname, 'example'),
-    compress: false,
-    port: 9000,
-    serveIndex: true,
-    writeToDisk: true
-  }
-}
-
-module.exports = config

packages/server/modules/core/domain/tokens/operations.ts

@@ -2,10 +2,10 @@ import {
   ApiToken,
   PersonalApiToken,
   TokenResourceAccessDefinition,
+  TokenResourceIdentifierType,
   TokenScope,
   UserServerAppToken
 } from '@/modules/core/domain/tokens/types'
-import { TokenResourceIdentifierInput } from '@/modules/core/graph/generated/graphql'
 import { TokenValidationResult } from '@/modules/core/helpers/types'
 import { NullableKeysToOptional, Optional, ServerScope } from '@speckle/shared'
 import { SetOptional } from 'type-fest'
@@ -60,6 +60,11 @@ export type UpdateApiToken = (
   token: Partial<ApiToken>
 ) => Promise<ApiToken>
 
+export type TokenResourceIdentifierInput = {
+  id: string
+  type: TokenResourceIdentifierType
+}
+
 export type CreateAndStoreUserToken = (params: {
   userId: string
   name: string

packages/server/modules/previews/domain/operations.ts

@@ -11,6 +11,30 @@ export type CreateObjectPreview = (
   params: Pick<ObjectPreview, 'streamId' | 'objectId' | 'priority'>
 ) => Promise<void>
 
+export type ObjectPreviewInput = Pick<
+  ObjectPreview,
+  'streamId' | 'objectId' | 'priority'
+>
+export type StoreObjectPreview = (params: ObjectPreviewInput) => Promise<void>
+export type UpsertObjectPreview = (params: {
+  objectPreview: ObjectPreview
+}) => Promise<void>
+
+export type ObjectPreviewRequest = {
+  url: string
+  token: string
+  jobId: string
+}
+
+export type Preview = {
+  id: string
+  data: Buffer
+}
+
+export type StorePreview = (params: { preview: Preview }) => Promise<void>
+
+export type RequestObjectPreview = (params: ObjectPreviewRequest) => Promise<void>
+
 export type GetPreviewImage = (params: {
   previewId: string
 }) => Promise<Nullable<Buffer>>

packages/server/modules/previews/index.ts

@@ -1,263 +1,162 @@
 /* istanbul ignore file */
-import { validateScopes, authorizeResolver } from '@/modules/shared'
+import { moduleLogger, previewLogger as logger } from '@/observability/logging'
+import { consumePreviewResultFactory } from '@/modules/previews/resultListener'
 
-import { makeOgImage } from '@/modules/previews/ogImage'
-import { moduleLogger } from '@/observability/logging'
-import { messageProcessor } from '@/modules/previews/resultListener'
 import { db } from '@/db/knex'
 import {
-  getObjectPreviewBufferOrFilepathFactory,
-  sendObjectPreviewFactory,
-  checkStreamPermissionsFactory
-} from '@/modules/previews/services/management'
-import {
-  getObjectPreviewInfoFactory,
-  createObjectPreviewFactory,
-  getPreviewImageFactory
-} from '@/modules/previews/repository/previews'
-import {
-  getCommitFactory,
-  getPaginatedBranchCommitsItemsFactory,
-  legacyGetPaginatedStreamCommitsPageFactory
-} from '@/modules/core/repositories/commits'
+  disablePreviews,
+  getPreviewServiceRedisUrl,
+  getRedisUrl,
+  getServerOrigin
+} from '@/modules/shared/helpers/envHelper'
+import Bull from 'bull'
+import Redis, { RedisOptions } from 'ioredis'
+import { createBullBoard } from 'bull-board'
+import { BullMQAdapter } from 'bull-board/bullMQAdapter'
+import { authMiddlewareCreator } from '@/modules/shared/middleware'
+import { Roles, TIME } from '@speckle/shared'
+import { validateServerRoleBuilderFactory } from '@/modules/shared/authz'
+import { getRolesFactory } from '@/modules/shared/repositories/roles'
+import { previewRouterFactory } from '@/modules/previews/rest/router'
 import { SpeckleModule } from '@/modules/shared/helpers/typeHelper'
-import { getStreamFactory } from '@/modules/core/repositories/streams'
-import { getPaginatedBranchCommitsItemsByNameFactory } from '@/modules/core/services/commit/retrieval'
-import { getStreamBranchByNameFactory } from '@/modules/core/repositories/branches'
-import { getFormattedObjectFactory } from '@/modules/core/repositories/objects'
+import { previewResultPayload } from '@speckle/shared/dist/commonjs/previews/job.js'
 import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector'
-import { listenFor } from '@/modules/core/utils/dbNotificationListener'
-import { disablePreviews } from '@/modules/shared/helpers/envHelper'
-import { corsMiddlewareFactory } from '@/modules/core/configs/cors'
+import {
+  storePreviewFactory,
+  upsertObjectPreviewFactory
+} from '@/modules/previews/repository/previews'
+import { getObjectCommitsWithStreamIdsFactory } from '@/modules/core/repositories/commits'
+import prometheusClient from 'prom-client'
+import { initializeMetrics } from '@/modules/previews/observability/metrics'
 
-const httpErrorImage = (httpErrorCode: number) =>
-  require.resolve(`#/assets/previews/images/preview_${httpErrorCode}.png`)
+const getPreviewQueues = (params: { responseQueueName: string }) => {
+  const { responseQueueName } = params
+  let client: Redis
+  let subscriber: Redis
+  const redisUrl = getPreviewServiceRedisUrl() ?? getRedisUrl()
 
-const noPreviewImage = require.resolve('#/assets/previews/images/no_preview.png')
+  const opts = {
+    // redisOpts here will contain at least a property of connectionName which will identify the queue based on its name
+    createClient(type: string, redisOpts: RedisOptions) {
+      switch (type) {
+        case 'client':
+          if (!client) {
+            client = new Redis(redisUrl, redisOpts)
+          }
+          return client
+        case 'subscriber':
+          if (!subscriber) {
+            subscriber = new Redis(redisUrl, {
+              ...redisOpts,
+              maxRetriesPerRequest: null,
+              enableReadyCheck: false
+            })
+          }
+          return subscriber
+        case 'bclient':
+          return new Redis(redisUrl, {
+            ...redisOpts,
+            maxRetriesPerRequest: null,
+            enableReadyCheck: false
+          })
+        default:
+          throw new Error('Unexpected connection type: ' + type)
+      }
+    }
+  }
+  const previewRequestQueue = new Bull('preview-service-jobs', opts)
+  // these events are published on the job queue, results come back on the response queue
+  previewRequestQueue.on('error', (err) => {
+    logger.error({ err }, 'Preview generation failed')
+  })
+  previewRequestQueue.on('failed', (job, err) => {
+    const jobId = 'jobId' in job.data ? job.data.jobId : undefined
+    logger.error({ err, jobId }, 'Preview job {jobId} failed.')
+  })
+  previewRequestQueue.on('active', (job) => {
+    const jobId = 'jobId' in job.data ? job.data.jobId : undefined
+    logger.info({ jobId }, 'Preview job {jobId} processing started.')
+  })
+  const previewResponseQueue = new Bull(responseQueueName, opts)
+  return { previewRequestQueue, previewResponseQueue }
+}
 
 export const init: SpeckleModule['init'] = ({ app, isInitial }) => {
-  if (disablePreviews()) {
-    moduleLogger.warn('📸 Object preview module is DISABLED')
-  } else {
-    moduleLogger.info('📸 Init object preview module')
-  }
-
-  app.options('/preview/:streamId/:angle?', corsMiddlewareFactory())
-  app.get('/preview/:streamId/:angle?', corsMiddlewareFactory(), async (req, res) => {
-    const projectDb = await getProjectDbClient({ projectId: req.params.streamId })
-    const checkStreamPermissions = checkStreamPermissionsFactory({
-      validateScopes,
-      authorizeResolver,
-      // getting the stream from the main DB, cause it needs to join on roles
-      getStream: getStreamFactory({ db })
-    })
-    const { hasPermissions, httpErrorCode } = await checkStreamPermissions(req)
-    if (!hasPermissions) {
-      // return res.status( httpErrorCode ).end()
-      return res.sendFile(httpErrorImage(httpErrorCode))
-    }
-
-    const getCommitsByStreamId = legacyGetPaginatedStreamCommitsPageFactory({
-      db: projectDb
-    })
-
-    const { commits } = await getCommitsByStreamId({
-      streamId: req.params.streamId,
-      limit: 1,
-      ignoreGlobalsBranch: true,
-      cursor: undefined
-    })
-    if (!commits || commits.length === 0) {
-      return res.sendFile(noPreviewImage)
-    }
-    const lastCommit = commits[0]
-    const getObjectPreviewBufferOrFilepath = getObjectPreviewBufferOrFilepathFactory({
-      getObject: getFormattedObjectFactory({ db: projectDb }),
-      getObjectPreviewInfo: getObjectPreviewInfoFactory({ db: projectDb }),
-      createObjectPreview: createObjectPreviewFactory({ db: projectDb }),
-      getPreviewImage: getPreviewImageFactory({ db: projectDb }),
-      logger: req.log
-    })
-
-    const sendObjectPreview = sendObjectPreviewFactory({
-      // getting the stream from the projectDb here, to handle preview data properly
-      getStream: getStreamFactory({ db: projectDb }),
-      getObjectPreviewBufferOrFilepath,
-      makeOgImage
-    })
-
-    return sendObjectPreview(
-      req,
-      res,
-      req.params.streamId,
-      lastCommit.referencedObject,
-      req.params.angle
-    )
-  })
-
-  app.options(
-    '/preview/:streamId/branches/:branchName/:angle?',
-    corsMiddlewareFactory()
-  )
-  app.get(
-    '/preview/:streamId/branches/:branchName/:angle?',
-    corsMiddlewareFactory(),
-    async (req, res) => {
-      const checkStreamPermissions = checkStreamPermissionsFactory({
-        validateScopes,
-        authorizeResolver,
-        // getting the stream from the main DB, cause it needs to join on roles
-        getStream: getStreamFactory({ db })
-      })
-      const { hasPermissions, httpErrorCode } = await checkStreamPermissions(req)
-      if (!hasPermissions) {
-        // return res.status( httpErrorCode ).end()
-        return res.sendFile(httpErrorImage(httpErrorCode))
-      }
-
-      const projectDb = await getProjectDbClient({ projectId: req.params.streamId })
-
-      let commitsObj
-      try {
-        const getCommitsByBranchName = getPaginatedBranchCommitsItemsByNameFactory({
-          getStreamBranchByName: getStreamBranchByNameFactory({ db: projectDb }),
-          getPaginatedBranchCommitsItems: getPaginatedBranchCommitsItemsFactory({
-            db: projectDb
-          })
-        })
-        commitsObj = await getCommitsByBranchName({
-          streamId: req.params.streamId,
-          branchName: req.params.branchName,
-          limit: 1,
-          cursor: undefined
-        })
-      } catch {
-        commitsObj = {}
-      }
-      const { commits } = commitsObj
-      if (!commits || commits.length === 0) {
-        return res.sendFile(noPreviewImage)
-      }
-      const lastCommit = commits[0]
-
-      const getObjectPreviewBufferOrFilepath = getObjectPreviewBufferOrFilepathFactory({
-        getObject: getFormattedObjectFactory({ db: projectDb }),
-        getObjectPreviewInfo: getObjectPreviewInfoFactory({ db: projectDb }),
-        createObjectPreview: createObjectPreviewFactory({ db: projectDb }),
-        getPreviewImage: getPreviewImageFactory({ db: projectDb }),
-        logger: req.log
-      })
-
-      const sendObjectPreview = sendObjectPreviewFactory({
-        // getting the stream from the projectDb here, to handle preview data properly
-        getStream: getStreamFactory({ db: projectDb }),
-        getObjectPreviewBufferOrFilepath,
-        makeOgImage
-      })
-
-      return sendObjectPreview(
-        req,
-        res,
-        req.params.streamId,
-        lastCommit.referencedObject,
-        req.params.angle
-      )
-    }
-  )
-
-  app.options('/preview/:streamId/commits/:commitId/:angle?', corsMiddlewareFactory())
-  app.get(
-    '/preview/:streamId/commits/:commitId/:angle?',
-    corsMiddlewareFactory(),
-    async (req, res) => {
-      const checkStreamPermissions = checkStreamPermissionsFactory({
-        validateScopes,
-        authorizeResolver,
-        // getting the stream from the main DB, cause it needs to join on roles
-        getStream: getStreamFactory({ db })
-      })
-      const { hasPermissions, httpErrorCode } = await checkStreamPermissions(req)
-      if (!hasPermissions) {
-        // return res.status( httpErrorCode ).end()
-        return res.sendFile(httpErrorImage(httpErrorCode))
-      }
-
-      const projectDb = await getProjectDbClient({ projectId: req.params.streamId })
-
-      const getCommit = getCommitFactory({ db: projectDb })
-      const commit = await getCommit(req.params.commitId, {
-        streamId: req.params.streamId
-      })
-      if (!commit) return res.sendFile(noPreviewImage)
-
-      const getObjectPreviewBufferOrFilepath = getObjectPreviewBufferOrFilepathFactory({
-        getObject: getFormattedObjectFactory({ db: projectDb }),
-        getObjectPreviewInfo: getObjectPreviewInfoFactory({ db: projectDb }),
-        createObjectPreview: createObjectPreviewFactory({ db: projectDb }),
-        getPreviewImage: getPreviewImageFactory({ db: projectDb }),
-        logger: req.log
-      })
-
-      const sendObjectPreview = sendObjectPreviewFactory({
-        // getting the stream from the projectDb here, to handle preview data properly
-        getStream: getStreamFactory({ db: projectDb }),
-        getObjectPreviewBufferOrFilepath,
-        makeOgImage
-      })
-      return sendObjectPreview(
-        req,
-        res,
-        req.params.streamId,
-        commit.referencedObject,
-        req.params.angle
-      )
-    }
-  )
-
-  app.options('/preview/:streamId/objects/:objectId/:angle?', corsMiddlewareFactory())
-  app.get(
-    '/preview/:streamId/objects/:objectId/:angle?',
-    corsMiddlewareFactory(),
-    async (req, res) => {
-      const checkStreamPermissions = checkStreamPermissionsFactory({
-        validateScopes,
-        authorizeResolver,
-        // getting the stream from the main DB, cause it needs to join on roles
-        getStream: getStreamFactory({ db })
-      })
-      const { hasPermissions } = await checkStreamPermissions(req)
-      if (!hasPermissions) {
-        return res.status(403).end()
-      }
-      const projectDb = await getProjectDbClient({ projectId: req.params.streamId })
-
-      const getObjectPreviewBufferOrFilepath = getObjectPreviewBufferOrFilepathFactory({
-        getObject: getFormattedObjectFactory({ db: projectDb }),
-        getObjectPreviewInfo: getObjectPreviewInfoFactory({ db: projectDb }),
-        createObjectPreview: createObjectPreviewFactory({ db: projectDb }),
-        getPreviewImage: getPreviewImageFactory({ db: projectDb }),
-        logger: req.log
-      })
-
-      const sendObjectPreview = sendObjectPreviewFactory({
-        // getting the stream from the projectDb here, to handle preview data properly
-        getStream: getStreamFactory({ db: projectDb }),
-        getObjectPreviewBufferOrFilepath,
-        makeOgImage
-      })
-
-      return sendObjectPreview(
-        req,
-        res,
-        req.params.streamId,
-        req.params.objectId,
-        req.params.angle
-      )
-    }
-  )
-
   if (isInitial) {
-    listenFor('preview_generation_update', messageProcessor)
+    if (disablePreviews()) {
+      moduleLogger.warn('📸 Object preview module is DISABLED')
+    } else {
+      moduleLogger.info('📸 Init object preview module')
+    }
+
+    const responseQueueName = `preview-service-results-${
+      new URL(getServerOrigin()).hostname
+    }`
+
+    const { previewRequestQueue, previewResponseQueue } = getPreviewQueues({
+      responseQueueName
+    })
+
+    const { previewJobsProcessedSummary } = initializeMetrics({
+      registers: [prometheusClient.register],
+      previewRequestQueue
+    })
+
+    const router = createBullBoard([
+      new BullMQAdapter(previewRequestQueue),
+      new BullMQAdapter(previewResponseQueue)
+    ]).router
+    app.use(
+      '/api/admin/preview-jobs',
+      async (req, res, next) => {
+        await authMiddlewareCreator([
+          validateServerRoleBuilderFactory({ getRoles: getRolesFactory({ db }) })({
+            requiredRole: Roles.Server.Admin
+          })
+        ])(req, res, next)
+      },
+      router
+    )
+
+    const previewRouter = previewRouterFactory({
+      previewRequestQueue,
+      responseQueueName
+    })
+    app.use(previewRouter)
+
+    previewResponseQueue.process(async (payload, done) => {
+      const parsedMessage = previewResultPayload.safeParse(payload.data)
+      if (!parsedMessage.success) {
+        logger.error(
+          { payload: payload.data, reason: parsedMessage.error },
+          'Failed to parse previewResult payload'
+        )
+        done(parsedMessage.error)
+        return
+      }
+      const [projectId, objectId] = parsedMessage.data.jobId.split('.')
+
+      const projectDb = await getProjectDbClient({ projectId })
+      await consumePreviewResultFactory({
+        logger,
+        storePreview: storePreviewFactory({ db: projectDb }),
+        upsertObjectPreview: upsertObjectPreviewFactory({ db: projectDb }),
+        getObjectCommitsWithStreamIds: getObjectCommitsWithStreamIdsFactory({
+          db: projectDb
+        })
+      })({
+        projectId,
+        objectId,
+        previewResult: parsedMessage.data
+      })
+
+      previewJobsProcessedSummary.observe(
+        { status: parsedMessage.data.status },
+        parsedMessage.data.result.durationSeconds * TIME.second
+      )
+
+      done()
+    })
   }
 }
 

packages/server/modules/previews/observability/metrics.ts

@@ -0,0 +1,52 @@
+import Bull from 'bull'
+import { type Registry, Counter, Summary, Gauge } from 'prom-client'
+
+export const initializeMetrics = (params: {
+  registers: Registry[]
+  previewRequestQueue: Bull.Queue
+}) => {
+  const { registers, previewRequestQueue } = params
+  // add a metric to gauge the length of the preview job queue
+  registers.forEach((r) =>
+    r.removeSingleMetric('speckle_server_preview_jobs_queue_pending')
+  )
+  new Gauge({
+    name: 'speckle_server_preview_jobs_queue_pending',
+    help: 'Number of preview jobs waiting in the job queue',
+    async collect() {
+      this.set(await previewRequestQueue.count())
+    }
+  })
+
+  registers.forEach((r) => r.removeSingleMetric('speckle_server_preview_jobs_count'))
+  const previewJobsCounter = new Counter({
+    name: 'speckle_server_preview_jobs_count',
+    help: 'Total number of preview jobs which have been requested to be processed.'
+  })
+
+  registers.forEach((r) =>
+    r.removeSingleMetric('speckle_server_preview_jobs_request_failed_count')
+  )
+  const previewJobsFailedCounter = new Counter({
+    name: 'speckle_server_preview_jobs_request_failed_count',
+    help: 'Total number of preview jobs which have been requested but failed to be processed.'
+  })
+
+  registers.forEach((r) =>
+    r.removeSingleMetric('speckle_server_preview_jobs_processed_duration_seconds')
+  )
+  const previewJobsProcessedSummary = new Summary<'status'>({
+    name: 'speckle_server_preview_jobs_processed_duration_seconds',
+    help: 'Duration of preview job processing, in seconds',
+    labelNames: ['status']
+  })
+
+  previewRequestQueue.on('added', () => {
+    previewJobsCounter.inc()
+  })
+  previewRequestQueue.on('failed', () => {
+    previewJobsFailedCounter.inc()
+  })
+
+  return { previewJobsProcessedSummary }
+}

packages/server/modules/previews/queues/previews.ts

@@ -0,0 +1,15 @@
+import { RequestObjectPreview } from '@/modules/previews/domain/operations'
+import type { Queue } from 'bull'
+
+export const requestObjectPreviewFactory =
+  ({
+    responseQueue,
+    queue
+  }: {
+    responseQueue: string
+    queue: Queue
+  }): RequestObjectPreview =>
+  async ({ jobId, token, url }) => {
+    const payload = { jobId, token, url, responseQueue }
+    await queue.add(payload, { removeOnComplete: true, attempts: 3 })
+  }

packages/server/modules/previews/repository/previews.ts

@@ -1,9 +1,10 @@
-/* istanbul ignore file */
 import { buildTableHelper } from '@/modules/core/dbSchema'
 import {
-  CreateObjectPreview,
   GetObjectPreviewInfo,
-  GetPreviewImage
+  GetPreviewImage,
+  StoreObjectPreview,
+  StorePreview,
+  UpsertObjectPreview
 } from '@/modules/previews/domain/operations'
 import {
   ObjectPreview as ObjectPreviewRecord,
@@ -37,8 +38,8 @@ export const getObjectPreviewInfoFactory =
       .first()
   }
 
-export const createObjectPreviewFactory =
-  ({ db }: { db: Knex }): CreateObjectPreview =>
+export const storeObjectPreviewFactory =
+  ({ db }: { db: Knex }): StoreObjectPreview =>
   async ({
     streamId,
     objectId,
@@ -51,11 +52,29 @@ export const createObjectPreviewFactory =
         priority,
         previewStatus: 0
       }
-    const sqlQuery =
-      tables.objectPreview(db).insert(insertionObject).toString() +
-      ' on conflict do nothing'
+    const sqlQuery = tables
+      .objectPreview(db)
+      .insert(insertionObject)
+      .onConflict()
+      .ignore()
 
-    await db.raw(sqlQuery)
+    await sqlQuery
+  }
+
+export const storePreviewFactory =
+  ({ db }: { db: Knex }): StorePreview =>
+  async ({ preview }) => {
+    await tables.previews(db).insert(preview).onConflict().ignore()
+  }
+
+export const upsertObjectPreviewFactory =
+  ({ db }: { db: Knex }): UpsertObjectPreview =>
+  async ({ objectPreview }) => {
+    await tables
+      .objectPreview(db)
+      .insert(objectPreview)
+      .onConflict(['streamId', 'objectId'])
+      .merge()
   }
 
 export const getPreviewImageFactory =

packages/server/modules/previews/rest/router.ts

@@ -0,0 +1,306 @@
+import { Router } from 'express'
+import cors from 'cors'
+
+import { validateScopes, authorizeResolver } from '@/modules/shared'
+
+import { makeOgImage } from '@/modules/previews/ogImage'
+
+import { db } from '@/db/knex'
+import {
+  getObjectPreviewBufferOrFilepathFactory,
+  sendObjectPreviewFactory,
+  checkStreamPermissionsFactory
+} from '@/modules/previews/services/management'
+import {
+  getObjectPreviewInfoFactory,
+  getPreviewImageFactory,
+  storeObjectPreviewFactory
+} from '@/modules/previews/repository/previews'
+import {
+  getCommitFactory,
+  getPaginatedBranchCommitsItemsFactory,
+  legacyGetPaginatedStreamCommitsPageFactory
+} from '@/modules/core/repositories/commits'
+import {
+  getStreamCollaboratorsFactory,
+  getStreamFactory
+} from '@/modules/core/repositories/streams'
+import { getPaginatedBranchCommitsItemsByNameFactory } from '@/modules/core/services/commit/retrieval'
+import { getStreamBranchByNameFactory } from '@/modules/core/repositories/branches'
+import { getFormattedObjectFactory } from '@/modules/core/repositories/objects'
+import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector'
+import { createObjectPreviewFactory } from '@/modules/previews/services/createObjectPreview'
+import { createAppTokenFactory } from '@/modules/core/services/tokens'
+import {
+  storeApiTokenFactory,
+  storeTokenResourceAccessDefinitionsFactory,
+  storeTokenScopesFactory,
+  storeUserServerAppTokenFactory
+} from '@/modules/core/repositories/tokens'
+import { getServerOrigin } from '@/modules/shared/helpers/envHelper'
+import { requestObjectPreviewFactory } from '@/modules/previews/queues/previews'
+import { Queue } from 'bull'
+import { Knex } from 'knex'
+
+const httpErrorImage = (httpErrorCode: number) =>
+  require.resolve(`#/assets/previews/images/preview_${httpErrorCode}.png`)
+
+const noPreviewImage = require.resolve('#/assets/previews/images/no_preview.png')
+
+const buildCreateObjectPreviewFunction = ({
+  projectDb,
+  previewRequestQueue,
+  responseQueueName
+}: {
+  projectDb: Knex
+  previewRequestQueue: Queue
+  responseQueueName: string
+}) => {
+  return createObjectPreviewFactory({
+    requestObjectPreview: requestObjectPreviewFactory({
+      queue: previewRequestQueue,
+      responseQueue: responseQueueName
+    }),
+    serverOrigin: getServerOrigin(),
+    storeObjectPreview: storeObjectPreviewFactory({ db: projectDb }),
+    getStreamCollaborators: getStreamCollaboratorsFactory({ db }),
+    createAppToken: createAppTokenFactory({
+      storeApiToken: storeApiTokenFactory({ db }),
+      storeTokenScopes: storeTokenScopesFactory({ db }),
+      storeTokenResourceAccessDefinitions: storeTokenResourceAccessDefinitionsFactory({
+        db
+      }),
+      storeUserServerAppToken: storeUserServerAppTokenFactory({ db })
+    })
+  })
+}
+
+export const previewRouterFactory = ({
+  previewRequestQueue,
+  responseQueueName
+}: {
+  previewRequestQueue: Queue
+  responseQueueName: string
+}): Router => {
+  const app = Router()
+
+  app.options('/preview/:streamId/:angle?', cors())
+  app.get('/preview/:streamId/:angle?', cors(), async (req, res) => {
+    const projectDb = await getProjectDbClient({ projectId: req.params.streamId })
+    const checkStreamPermissions = checkStreamPermissionsFactory({
+      validateScopes,
+      authorizeResolver,
+      // getting the stream from the main DB, cause it needs to join on roles
+      getStream: getStreamFactory({ db })
+    })
+    const { hasPermissions, httpErrorCode } = await checkStreamPermissions(req)
+    if (!hasPermissions) {
+      // return res.status( httpErrorCode ).end()
+      return res.sendFile(httpErrorImage(httpErrorCode))
+    }
+
+    const getCommitsByStreamId = legacyGetPaginatedStreamCommitsPageFactory({
+      db: projectDb
+    })
+
+    const { commits } = await getCommitsByStreamId({
+      streamId: req.params.streamId,
+      limit: 1,
+      ignoreGlobalsBranch: true,
+      cursor: undefined
+    })
+    if (!commits || commits.length === 0) {
+      return res.sendFile(noPreviewImage)
+    }
+    const lastCommit = commits[0]
+    const getObjectPreviewBufferOrFilepath = getObjectPreviewBufferOrFilepathFactory({
+      logger: req.log,
+      getObject: getFormattedObjectFactory({ db: projectDb }),
+      getObjectPreviewInfo: getObjectPreviewInfoFactory({ db: projectDb }),
+      createObjectPreview: buildCreateObjectPreviewFunction({
+        projectDb,
+        previewRequestQueue,
+        responseQueueName
+      }),
+      getPreviewImage: getPreviewImageFactory({ db: projectDb })
+    })
+
+    const sendObjectPreview = sendObjectPreviewFactory({
+      // getting the stream from the projectDb here, to handle preview data properly
+      getStream: getStreamFactory({ db: projectDb }),
+      getObjectPreviewBufferOrFilepath,
+      makeOgImage
+    })
+
+    return sendObjectPreview(
+      req,
+      res,
+      req.params.streamId,
+      lastCommit.referencedObject,
+      req.params.angle
+    )
+  })
+
+  app.options('/preview/:streamId/branches/:branchName/:angle?', cors())
+  app.get(
+    '/preview/:streamId/branches/:branchName/:angle?',
+    cors(),
+    async (req, res) => {
+      const checkStreamPermissions = checkStreamPermissionsFactory({
+        validateScopes,
+        authorizeResolver,
+        // getting the stream from the main DB, cause it needs to join on roles
+        getStream: getStreamFactory({ db })
+      })
+      const { hasPermissions, httpErrorCode } = await checkStreamPermissions(req)
+      if (!hasPermissions) {
+        // return res.status( httpErrorCode ).end()
+        return res.sendFile(httpErrorImage(httpErrorCode))
+      }
+
+      const projectDb = await getProjectDbClient({ projectId: req.params.streamId })
+
+      let commitsObj
+      try {
+        const getCommitsByBranchName = getPaginatedBranchCommitsItemsByNameFactory({
+          getStreamBranchByName: getStreamBranchByNameFactory({ db: projectDb }),
+          getPaginatedBranchCommitsItems: getPaginatedBranchCommitsItemsFactory({
+            db: projectDb
+          })
+        })
+        commitsObj = await getCommitsByBranchName({
+          streamId: req.params.streamId,
+          branchName: req.params.branchName,
+          limit: 1,
+          cursor: undefined
+        })
+      } catch {
+        commitsObj = {}
+      }
+      const { commits } = commitsObj
+      if (!commits || commits.length === 0) {
+        return res.sendFile(noPreviewImage)
+      }
+      const lastCommit = commits[0]
+
+      const getObjectPreviewBufferOrFilepath = getObjectPreviewBufferOrFilepathFactory({
+        logger: req.log,
+        getObject: getFormattedObjectFactory({ db: projectDb }),
+        getObjectPreviewInfo: getObjectPreviewInfoFactory({ db: projectDb }),
+        createObjectPreview: buildCreateObjectPreviewFunction({
+          projectDb,
+          previewRequestQueue,
+          responseQueueName
+        }),
+        getPreviewImage: getPreviewImageFactory({ db: projectDb })
+      })
+
+      const sendObjectPreview = sendObjectPreviewFactory({
+        // getting the stream from the projectDb here, to handle preview data properly
+        getStream: getStreamFactory({ db: projectDb }),
+        getObjectPreviewBufferOrFilepath,
+        makeOgImage
+      })
+
+      return sendObjectPreview(
+        req,
+        res,
+        req.params.streamId,
+        lastCommit.referencedObject,
+        req.params.angle
+      )
+    }
+  )
+
+  app.options('/preview/:streamId/commits/:commitId/:angle?', cors())
+  app.get('/preview/:streamId/commits/:commitId/:angle?', cors(), async (req, res) => {
+    const checkStreamPermissions = checkStreamPermissionsFactory({
+      validateScopes,
+      authorizeResolver,
+      // getting the stream from the main DB, cause it needs to join on roles
+      getStream: getStreamFactory({ db })
+    })
+    const { hasPermissions, httpErrorCode } = await checkStreamPermissions(req)
+    if (!hasPermissions) {
+      // return res.status( httpErrorCode ).end()
+      return res.sendFile(httpErrorImage(httpErrorCode))
+    }
+
+    const projectDb = await getProjectDbClient({ projectId: req.params.streamId })
+
+    const getCommit = getCommitFactory({ db: projectDb })
+    const commit = await getCommit(req.params.commitId, {
+      streamId: req.params.streamId
+    })
+    if (!commit) return res.sendFile(noPreviewImage)
+
+    const getObjectPreviewBufferOrFilepath = getObjectPreviewBufferOrFilepathFactory({
+      logger: req.log,
+      getObject: getFormattedObjectFactory({ db: projectDb }),
+      getObjectPreviewInfo: getObjectPreviewInfoFactory({ db: projectDb }),
+      createObjectPreview: buildCreateObjectPreviewFunction({
+        projectDb,
+        previewRequestQueue,
+        responseQueueName
+      }),
+      getPreviewImage: getPreviewImageFactory({ db: projectDb })
+    })
+
+    const sendObjectPreview = sendObjectPreviewFactory({
+      // getting the stream from the projectDb here, to handle preview data properly
+      getStream: getStreamFactory({ db: projectDb }),
+      getObjectPreviewBufferOrFilepath,
+      makeOgImage
+    })
+    return sendObjectPreview(
+      req,
+      res,
+      req.params.streamId,
+      commit.referencedObject,
+      req.params.angle
+    )
+  })
+
+  app.options('/preview/:streamId/objects/:objectId/:angle?', cors())
+  app.get('/preview/:streamId/objects/:objectId/:angle?', cors(), async (req, res) => {
+    const checkStreamPermissions = checkStreamPermissionsFactory({
+      validateScopes,
+      authorizeResolver,
+      // getting the stream from the main DB, cause it needs to join on roles
+      getStream: getStreamFactory({ db })
+    })
+    const { hasPermissions } = await checkStreamPermissions(req)
+    if (!hasPermissions) {
+      return res.status(403).end()
+    }
+    const projectDb = await getProjectDbClient({ projectId: req.params.streamId })
+
+    const getObjectPreviewBufferOrFilepath = getObjectPreviewBufferOrFilepathFactory({
+      logger: req.log,
+      getObject: getFormattedObjectFactory({ db: projectDb }),
+      getObjectPreviewInfo: getObjectPreviewInfoFactory({ db: projectDb }),
+      createObjectPreview: buildCreateObjectPreviewFunction({
+        projectDb,
+        previewRequestQueue,
+        responseQueueName
+      }),
+      getPreviewImage: getPreviewImageFactory({ db: projectDb })
+    })
+
+    const sendObjectPreview = sendObjectPreviewFactory({
+      // getting the stream from the projectDb here, to handle preview data properly
+      getStream: getStreamFactory({ db: projectDb }),
+      getObjectPreviewBufferOrFilepath,
+      makeOgImage
+    })
+
+    return sendObjectPreview(
+      req,
+      res,
+      req.params.streamId,
+      req.params.objectId,
+      req.params.angle
+    )
+  })
+  return app
+}

packages/server/modules/previews/resultListener.ts

@@ -3,6 +3,13 @@ import { MessageType } from '@/modules/core/utils/dbNotificationListener'
 import { getObjectCommitsWithStreamIdsFactory } from '@/modules/core/repositories/commits'
 import { publish } from '@/modules/shared/utils/subscriptions'
 import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector'
+import { PreviewResultPayload } from '@speckle/shared/dist/commonjs/previews/job.js'
+import { throwUncoveredError } from '@speckle/shared'
+import type { Logger } from '@/observability/logging'
+import crypto from 'crypto'
+import { StorePreview, UpsertObjectPreview } from '@/modules/previews/domain/operations'
+import { joinImages } from 'join-images'
+import { GetObjectCommitsWithStreamIds } from '@/modules/core/domain/commits/operations'
 
 const payloadRegexp = /^([\w\d]+):([\w\d]+):([\w\d]+)$/i
 
@@ -39,3 +46,120 @@ export const messageProcessor = async (msg: MessageType) => {
     )
   )
 }
+
+export const consumePreviewResultFactory =
+  ({
+    logger,
+    upsertObjectPreview,
+    storePreview,
+    getObjectCommitsWithStreamIds
+  }: {
+    logger: Logger
+    upsertObjectPreview: UpsertObjectPreview
+    storePreview: StorePreview
+    getObjectCommitsWithStreamIds: GetObjectCommitsWithStreamIds
+  }) =>
+  async ({
+    projectId,
+    objectId,
+    previewResult
+  }: {
+    projectId: string
+    objectId: string
+    previewResult: PreviewResultPayload
+  }) => {
+    const streamId = projectId
+    const lastUpdate = new Date()
+    const priority = 0
+    const previewStatus = 2
+    const log = logger.child({
+      ...previewResult,
+      projectId: streamId
+    })
+
+    switch (previewResult.status) {
+      case 'error':
+        await upsertObjectPreview({
+          objectPreview: {
+            objectId,
+            streamId,
+            lastUpdate,
+            preview: { err: previewResult.reason },
+            priority,
+            previewStatus
+          }
+        })
+
+        log.error('Preview generation failed for {jobId}.')
+        // store preview error in the db
+        break
+
+      case 'success':
+        log.info('Consumed preview generation {status} message payload for {jobId}.')
+        const preview: Record<string, string> = {}
+        const allImgsArr: Buffer[] = []
+        let i = 0
+        for (const [angle, value] of Object.entries(previewResult.result.screenshots)) {
+          const data = Buffer.from(
+            value.replace(/^data:image\/\w+;base64,/, ''),
+            'base64'
+          )
+
+          // @ts-expect-error this is a mismatch with node 18 and 22 types. upgrading to new node will fix it
+          const id = crypto.createHash('md5').update(data).digest('hex')
+
+          if (i++ === 0) {
+            await storePreview({ preview: { id, data } })
+            preview[angle] = id
+          }
+
+          allImgsArr.push(data)
+        }
+
+        const fullImg = await joinImages(allImgsArr, {
+          direction: 'horizontal',
+          offset: 700,
+          margin: '0 700 0 700',
+          color: { alpha: 0, r: 0, g: 0, b: 0 }
+        })
+        const png = fullImg.png({ quality: 95 })
+        const buff = await png.toBuffer()
+        // @ts-expect-error this is a mismatch with node 18 and 22 types. upgrading to new node will fix it
+        const fullImgId = crypto.createHash('md5').update(buff).digest('hex')
+
+        await storePreview({ preview: { id: fullImgId, data: buff } })
+
+        preview['all'] = fullImgId
+
+        await upsertObjectPreview({
+          objectPreview: {
+            objectId,
+            streamId,
+            lastUpdate,
+            preview,
+            priority,
+            previewStatus
+          }
+        })
+        const commits = await getObjectCommitsWithStreamIds([objectId], {
+          streamIds: [streamId]
+        })
+        if (!commits.length) break
+
+        await Promise.all(
+          commits.map((c) =>
+            publish(ProjectSubscriptions.ProjectVersionsPreviewGenerated, {
+              projectVersionsPreviewGenerated: {
+                versionId: c.id,
+                projectId: c.streamId,
+                objectId
+              }
+            })
+          )
+        )
+        break
+
+      default:
+        throwUncoveredError(previewResult)
+    }
+  }

packages/server/modules/previews/services/createObjectPreview.ts

@@ -0,0 +1,51 @@
+import { DefaultAppIds } from '@/modules/auth/defaultApps'
+import { GetStreamCollaborators } from '@/modules/core/domain/streams/operations'
+import { CreateAndStoreAppToken } from '@/modules/core/domain/tokens/operations'
+import {
+  CreateObjectPreview,
+  RequestObjectPreview,
+  StoreObjectPreview
+} from '@/modules/previews/domain/operations'
+import { Roles, Scopes } from '@speckle/shared'
+import { TokenResourceIdentifierType } from '@/modules/core/domain/tokens/types'
+
+export const createObjectPreviewFactory =
+  ({
+    getStreamCollaborators,
+    createAppToken,
+    requestObjectPreview,
+    storeObjectPreview,
+    serverOrigin
+  }: {
+    getStreamCollaborators: GetStreamCollaborators
+    serverOrigin: string
+    createAppToken: CreateAndStoreAppToken
+    requestObjectPreview: RequestObjectPreview
+    storeObjectPreview: StoreObjectPreview
+  }): CreateObjectPreview =>
+  async ({ streamId, objectId, priority }) => {
+    const owners = await getStreamCollaborators(streamId, Roles.Stream.Owner)
+    // there is always an owner, this is safe
+    const userId = owners[0].id
+
+    // we're running the preview generation in the name of a project owner
+    const token = await createAppToken({
+      appId: DefaultAppIds.Web,
+      name: `preview-${streamId}@${objectId}`,
+      userId,
+      scopes: [Scopes.Streams.Read],
+      lifespan: 120 * 60 * 1000, // for now, lets make this valid for 2 hours
+      limitResources: [
+        {
+          id: streamId,
+          type: TokenResourceIdentifierType.Project
+        }
+      ]
+    })
+    const url = new URL(
+      `/projects/${streamId}/models/${objectId}`,
+      serverOrigin
+    ).toString()
+    await requestObjectPreview({ jobId: `${streamId}.${objectId}`, token, url })
+    await storeObjectPreview({ streamId, objectId, priority })
+  }

packages/server/modules/previews/tests/unit/createObjectPreview.spec.ts

@@ -0,0 +1,58 @@
+import { LimitedUserWithStreamRole } from '@/modules/core/domain/streams/types'
+import {
+  ObjectPreviewInput,
+  ObjectPreviewRequest
+} from '@/modules/previews/domain/operations'
+import { createObjectPreviewFactory } from '@/modules/previews/services/createObjectPreview'
+import { Roles } from '@speckle/shared'
+import { expect } from 'chai'
+import cryptoRandomString from 'crypto-random-string'
+
+describe('object preview @previews', () => {
+  describe('createObjectPreviewFactory creates a function, that', () => {
+    it('requests and stores an object preview', async () => {
+      const appToken = cryptoRandomString({ length: 40 })
+      const streamOwner: LimitedUserWithStreamRole = {
+        id: cryptoRandomString({ length: 10 }),
+        avatar: null,
+        bio: null,
+        company: null,
+        createdAt: new Date(),
+        name: 'Preview User',
+        role: Roles.Server.User,
+        streamRole: Roles.Stream.Owner,
+        verified: true
+      }
+
+      let userId: string | undefined = undefined
+      let objectPreviewInput: ObjectPreviewInput | undefined = undefined
+      const serverOrigin = 'https://example.org'
+      let objectPreviewRequest: ObjectPreviewRequest | undefined = undefined
+      const createObjectPreview = createObjectPreviewFactory({
+        serverOrigin,
+        getStreamCollaborators: async () => [streamOwner],
+        createAppToken: async (tokenArgs) => {
+          userId = tokenArgs.userId
+          return appToken
+        },
+        storeObjectPreview: async (objectPreview) => {
+          objectPreviewInput = objectPreview
+        },
+        requestObjectPreview: async (previewRequest) => {
+          objectPreviewRequest = previewRequest
+        }
+      })
+      const objectId = cryptoRandomString({ length: 32 })
+      const streamId = cryptoRandomString({ length: 10 })
+      const priority = 0
+      await createObjectPreview({ objectId, streamId, priority })
+      expect(objectPreviewInput).to.deep.equal({ objectId, streamId, priority })
+      expect(userId).to.deep.equal(streamOwner.id)
+      expect(objectPreviewRequest).to.deep.equal({
+        url: `${serverOrigin}/projects/${streamId}/models/${objectId}`,
+        jobId: `${streamId}.${objectId}`,
+        token: appToken
+      })
+    })
+  })
+})

packages/server/modules/shared/authz.ts

@@ -306,6 +306,7 @@ export const streamWritePermissionsPipelineFactory = (
   validateServerRoleBuilderFactory(deps)({ requiredRole: Roles.Server.Guest }),
   validateScope({ requiredScope: Scopes.Streams.Write }),
   validateRequiredStreamFactory(deps),
+
   validateStreamRoleBuilderFactory(deps)({ requiredRole: Roles.Stream.Contributor }),
   validateResourceAccess
 ]

packages/server/modules/shared/helpers/envHelper.ts

@@ -86,6 +86,10 @@ export function getRedisUrl() {
   return getStringFromEnv('REDIS_URL')
 }
 
+export const getPreviewServiceRedisUrl = (): string | undefined => {
+  return process.env['PREVIEW_SERVICE_REDIS_URL']
+}
+
 export function getOidcDiscoveryUrl() {
   return getStringFromEnv('OIDC_DISCOVERY_URL')
 }

packages/server/observability/tests/metrics.spec.ts

@@ -18,15 +18,20 @@ describe('Observability', () => {
         metricsPageBody = await metricsResponse.text()
       })
       const testCases = [
+        // Apollo server
         'speckle_server_apollo_calls',
+        // express-prom-bundle
         'speckle_server_request_duration',
+        // Express (error handling middleware)
         'speckle_server_request_errors',
+        // http server
         'speckle_server_active_connections',
+        // apollo subscriptions
         'speckle_server_apollo_connect',
         'speckle_server_apollo_clients',
         'speckle_server_apollo_graphql_total_subscription_operations',
         'speckle_server_apollo_graphql_total_subscription_responses',
-        'speckle_server_active_connections',
+        // knex
         'speckle_server_knex_free',
         'speckle_server_knex_used',
         'speckle_server_knex_pending',
@@ -39,6 +44,7 @@ describe('Observability', () => {
         'speckle_server_knex_connection_acquisition_errors',
         'speckle_server_knex_connection_usage_duration',
         'speckle_server_knex_connection_pool_reaping_duration',
+        // high frequency metrics
         'nodejs_heap_size_total_bytes_high_frequency',
         'nodejs_heap_size_used_bytes_high_frequency',
         'nodejs_external_memory_bytes_high_frequency',
@@ -51,7 +57,12 @@ describe('Observability', () => {
         'knex_remaining_capacity_high_frequency',
         'process_cpu_user_seconds_total_high_frequency',
         'process_cpu_system_seconds_total_high_frequency',
-        'process_cpu_seconds_total_high_frequency'
+        'process_cpu_seconds_total_high_frequency',
+        // preview service
+        'speckle_server_preview_jobs_queue_pending',
+        'speckle_server_preview_jobs_count',
+        'speckle_server_preview_jobs_request_failed_count',
+        'speckle_server_preview_jobs_processed_duration_seconds'
       ]
 
       testCases.forEach((testCase) =>

packages/server/package.json

@@ -65,7 +65,8 @@
     "@speckle/shared": "workspace:^",
     "ajv": "^8.12.0",
     "bcrypt": "^5.0.0",
-    "bull": "^4.8.5",
+    "bull": "^4.16.4",
+    "bull-board": "^2.1.3",
     "busboy": "^1.4.0",
     "compression": "^1.7.4",
     "connect-redis": "^6.1.1",
@@ -88,6 +89,7 @@
     "graphql-subscriptions": "^2.0.0",
     "graphql-tag": "^2.12.6",
     "ioredis": "^5.2.2",
+    "join-images": "^1.1.5",
     "jose": "^5.6.3",
     "knex": "^2.5.1",
     "libsodium-wrappers": "^0.7.13",
@@ -164,6 +166,7 @@
     "@types/mock-require": "^2.0.1",
     "@types/module-alias": "^2.0.1",
     "@types/netmask": "^2.0.0",
+    "@types/node": "^18.19.38",
     "@types/node-cron": "^3.0.2",
     "@types/nodemailer": "^6.4.5",
     "@types/passport": "^1.0.16",
@@ -190,6 +193,7 @@
     "chai-http": "^4.3.0",
     "concurrently": "^7.0.0",
     "cross-env": "^7.0.3",
+    "csv-parse": "^5.6.0",
     "deep-equal-in-any-order": "^1.1.15",
     "enforce-unique": "^1.3.0",
     "eslint": "^8.11.0",

packages/server/type-augmentations/express.d.ts

@@ -8,17 +8,17 @@ declare module 'express' {
   }
 }
 
-declare module 'express-serve-static-core' {
-  interface Request {
-    context: AuthContext
-    mixpanel: ReturnType<typeof mixpanel>
-  }
-}
-
 declare module 'http' {
   interface IncomingMessage {
     context?: AuthContext
   }
 }
 
+declare module 'express-serve-static-core' {
+  interface Request {
+    context: AuthContext
+    mixpanel: ReturnType<typeof mixpanel>
+  }
+}
+
 export {}

packages/shared/src/previews/interface.ts

@@ -0,0 +1,13 @@
+import { PreviewResult } from './job.js'
+
+export interface PreviewGenerator {
+  takeScreenshot: TakeScreenshot
+  load: Load
+}
+
+export type TakeScreenshot = () => Promise<PreviewResult>
+
+export type LoadArgs = { url: string; token: string }
+export type Load = (args: LoadArgs) => Promise<void>
+
+export type { PreviewResult } from './job.js'

packages/shared/src/previews/job.ts

@@ -0,0 +1,49 @@
+import z from 'zod'
+
+const job = z.object({
+  jobId: z.string()
+})
+
+export const jobPayload = job.merge(
+  z.object({
+    url: z.string(),
+    token: z.string(),
+    responseQueue: z.string()
+  })
+)
+export type JobPayload = z.infer<typeof jobPayload>
+
+const previewResult = z.object({
+  durationSeconds: z.number().describe('Duration to generate the preview, in seconds'),
+  screenshots: z.record(z.string(), z.string())
+})
+
+export type PreviewResult = z.infer<typeof previewResult>
+
+const previewSuccessPayload = job.merge(
+  z.object({
+    status: z.literal('success'),
+    result: previewResult
+  })
+)
+
+export type PreviewSuccessPayload = z.infer<typeof previewSuccessPayload>
+
+const previewErrorPayload = job.merge(
+  z.object({
+    status: z.literal('error'),
+    reason: z.string(),
+    result: z.object({
+      durationSeconds: z
+        .number()
+        .describe('Duration spent processing the job before erroring, in seconds')
+    })
+  })
+)
+
+export const previewResultPayload = z.discriminatedUnion('status', [
+  previewSuccessPayload,
+  previewErrorPayload
+])
+
+export type PreviewResultPayload = z.infer<typeof previewResultPayload>

utils/helm/speckle-server/templates/_helpers.tpl

@@ -766,6 +766,15 @@ Generate the environment variables for Speckle server and Speckle objects deploy
       name: {{ default .Values.secretName .Values.redis.connectionString.secretName }}
       key: {{ default "redis_url" .Values.redis.connectionString.secretKey }}
 
+
+{{- if .Values.preview_service.dedicatedPreviewsQueue }}
+- name: PREVIEW_SERVICE_REDIS_URL
+  valueFrom:
+    secretKeyRef:
+      name: {{ default .Values.secretName .Values.redis.previewServiceConnectionString.secretName }}
+      key: {{ default "preview_service_redis_url" .Values.redis.previewServiceConnectionString.secretKey }}
+{{- end }}
+
 # *** PostgreSQL Database ***
 - name: POSTGRES_URL
   valueFrom:

utils/helm/speckle-server/templates/objects/serviceaccount.yml

@@ -44,4 +44,7 @@ secrets:
 {{- if .Values.featureFlags.gendoAIModuleEnabled }}
   - name: {{ default .Values.secretName .Values.server.gendoAI.key.secretName }}
 {{- end }}
+{{- if .Values.preview_service.dedicatedPreviewsQueue }}
+  - name: {{ default .Values.secretName .Values.redis.previewServiceConnectionString.secretName }}
+{{- end }}
 {{- end -}}

utils/helm/speckle-server/templates/preview_service/deployment.yml

@@ -35,7 +35,9 @@ spec:
             command:
               - node
               - -e
-              - {{ printf "process.exit(Date.now() - require('fs').readFileSync('/tmp/last_successful_query', 'utf8') > %s)" .Values.preview_service.puppeteer.timeoutMilliseconds }}
+              # just a dummy output for now, not yet sure how to do liveliness with the new setup.
+              # if there is no job in the queue, the service does nothing, so the prev solution would not work
+              - {{ printf "console.log(%s)" .Values.preview_service.puppeteer.timeoutMilliseconds }}
 
         resources:
           requests:
@@ -55,17 +57,6 @@ spec:
           runAsNonRoot: true
           runAsUser: 800
 
-        volumeMounts:
-        {{- if .Values.db.useCertificate }}
-          - name: postgres-certificate
-            mountPath: /postgres-certificate
-        {{- end }}
-        {{- if .Values.featureFlags.workspacesMultiRegionEnabled }}
-          - name: multi-region-config
-            mountPath: /multi-region-config
-            readOnly: true
-        {{- end }}
-
         env:
           - name: HOST
             value: '127.0.0.1' # Only accept connections from localhost, as preview service does not need to be exposed outside the container.
@@ -77,30 +68,29 @@ spec:
           - name: PROMETHEUS_METRICS_PORT
             value: {{ .Values.preview_service.monitoring.metricsPort | quote }}
 
-          - name: PG_CONNECTION_STRING
+
+
+
+          # *** Redis ***
+          {{- if .Values.preview_service.dedicatedPreviewsQueue }}
+          - name: REDIS_URL 
             valueFrom:
               secretKeyRef:
-                name: {{ default .Values.secretName .Values.db.connectionString.secretName }}
-                key: {{ default "postgres_url" .Values.db.connectionString.secretKey }}
-          {{- if .Values.preview_service.postgresMaxConnections }}
-          - name: POSTGRES_MAX_CONNECTIONS_PREVIEW_SERVICE
-            value: {{ .Values.preview_service.postgresMaxConnections | quote }}
+                name: {{ default .Values.secretName .Values.redis.previewServiceConnectionString.secretName }}
+                key: {{ default "preview_service_redis_url" .Values.redis.previewServiceConnectionString.secretKey }}
+          {{- else }}
+          - name: REDIS_URL
+            valueFrom:
+              secretKeyRef:
+                name: {{ default .Values.secretName .Values.redis.connectionString.secretName }}
+                key: {{ default "redis_url" .Values.redis.connectionString.secretKey }}
           {{- end }}
-          - name: POSTGRES_CONNECTION_CREATE_TIMEOUT_MILLIS
-            value: {{ .Values.db.connectionCreateTimeoutMillis | quote }}
-          - name: POSTGRES_CONNECTION_ACQUIRE_TIMEOUT_MILLIS
-            value: {{ .Values.db.connectionAcquireTimeoutMillis | quote }}
 
           - name: LOG_LEVEL
             value: {{ .Values.preview_service.logLevel | quote }}
           - name: LOG_PRETTY
             value: {{ .Values.preview_service.logPretty | quote }}
 
-          {{- if .Values.db.useCertificate }}
-          - name: NODE_EXTRA_CA_CERTS
-            value: "/postgres-certificate/ca-certificate.crt"
-          {{- end }}
-
           {{- if .Values.preview_service.puppeteer.userDataDirectory }}
           - name: USER_DATA_DIR
             value: {{ .Values.preview_service.puppeteer.userDataDirectory | quote }}
@@ -110,12 +100,6 @@ spec:
           - name: PREVIEW_TIMEOUT
             value: {{ .Values.preview_service.puppeteer.timeoutMilliseconds | quote }}
           {{- end }}
-          {{- if .Values.featureFlags.workspacesMultiRegionEnabled }}
-          - name: FF_WORKSPACES_MULTI_REGION_ENABLED
-            value: {{ .Values.featureFlags.workspacesMultiRegionEnabled | quote }}
-          - name: MULTI_REGION_CONFIG_PATH
-            value: "/multi-region-config/multi-region-config.json"
-          {{- end }}
 
       {{- if .Values.preview_service.affinity }}
       affinity: {{- include "speckle.renderTpl" (dict "value" .Values.preview_service.affinity "context" $) | nindent 8 }}
@@ -146,17 +130,3 @@ spec:
       # Should be > preview generation time ( 1 hour for good measure )
       terminationGracePeriodSeconds: 3600
 
-      volumes:
-      {{- if .Values.db.useCertificate }}
-        - name: postgres-certificate
-          configMap:
-            name: postgres-certificate
-      {{- end }}
-      {{- if .Values.featureFlags.workspacesMultiRegionEnabled }}
-        - name: multi-region-config
-          secret:
-            secretName: {{ .Values.multiRegion.config.secretName }}
-            items:
-            - key: {{ .Values.multiRegion.config.secretKey }}
-              path: "multi-region-config.json"
-      {{- end }}