chore: updating ws everywhere to resolve SNYK-JS-WS-7266574 (#2385)

* chore: updating ws in various ways

* puppeteer upgrade

* updating codegen deps

* workspaces cmd update

* various extra fixes

* minor preview-service troubleshooting improvements

* dockerfile fixes

* hopefully fixing docker build fe2

* try again

* try large again

.circleci/config.yml

@@ -875,6 +875,7 @@ jobs:
 
   docker-build-frontend-2:
     <<: *build-job
+    resource_class: large
     environment:
       SPECKLE_SERVER_PACKAGE: frontend-2
 
@@ -1010,15 +1011,15 @@ jobs:
 
       - run:
           name: build public packages
-          command: yarn workspaces foreach -ptv --no-private run build
+          command: yarn workspaces foreach -ptvW --no-private run build
       - run:
           name: bump all versions
           # bump all versions in dependency tree order but not in parallel
-          command: yarn workspaces foreach -tv version $IMAGE_VERSION_TAG
+          command: yarn workspaces foreach -tvW version $IMAGE_VERSION_TAG
 
       - run:
           name: publish to npm
-          command: 'yarn workspaces foreach -pv --no-private npm publish --access public'
+          command: 'yarn workspaces foreach -pvW --no-private npm publish --access public'
 
   publish-helm-chart:
     docker:

.yarnrc.yml

@@ -1,9 +1,7 @@
+compressionLevel: mixed
+
+enableGlobalCache: false
+
 nodeLinker: node-modules
 
-plugins:
-  - path: .yarn/plugins/@yarnpkg/plugin-workspace-tools.cjs
-    spec: '@yarnpkg/plugin-workspace-tools'
-  - path: .yarn/plugins/@yarnpkg/plugin-version.cjs
-    spec: '@yarnpkg/plugin-version'
-
-yarnPath: .yarn/releases/yarn-3.2.0.cjs
+yarnPath: .yarn/releases/yarn-4.3.0.cjs

package.json

@@ -1,5 +1,5 @@
 {
-  "packageManager": "yarn@3.2.0",
+  "packageManager": "yarn@4.3.0",
   "workspaces": [
     "packages/*"
   ],
@@ -9,9 +9,9 @@
     "node": "^18.19.0"
   },
   "scripts": {
-    "build": "yarn workspaces foreach -ptv run build",
-    "build:public": "yarn workspaces foreach -ptv --no-private run build",
-    "build:tailwind-deps": "yarn workspaces foreach -iv -j unlimited --include '{@speckle/shared,@speckle/tailwind-theme,@speckle/ui-components}' run build",
+    "build": "yarn workspaces foreach -ptvW run build",
+    "build:public": "yarn workspaces foreach -ptvW --no-private run build",
+    "build:tailwind-deps": "yarn workspaces foreach -ivW -j unlimited --include '{@speckle/shared,@speckle/tailwind-theme,@speckle/ui-components}' run build",
     "ensure:tailwind-deps": "node ./utils/ensure-tailwind-deps.mjs",
     "helm:readme:generate": "./utils/helm/update-schema-json.sh",
     "prettier:check": "prettier --check .",
@@ -26,10 +26,10 @@
     "dev:kind:helm:up": "yarn dev:kind:up && tilt up --file ./.circleci/deployment/Tiltfile.helm --context kind-speckle-server",
     "dev:kind:helm:down": "tilt down --file ./.circleci/deployment/Tiltfile.helm --context kind-speckle-server",
     "dev:kind:helm:ci": "tilt ci --file ./.circleci/deployment/Tiltfile.helm --context kind-speckle-server --timeout 10m",
-    "dev": "yarn workspaces foreach -piv -j unlimited run dev",
-    "dev:no-server": "yarn workspaces foreach --exclude @speckle/server -piv -j unlimited run dev",
-    "dev:minimal": "yarn workspaces foreach -piv -j unlimited --include '{@speckle/server,@speckle/frontend,@speckle/shared}' run dev",
-    "gqlgen": "yarn workspaces foreach -piv -j unlimited --include '{@speckle/server,@speckle/frontend,@speckle/frontend-2}' run gqlgen",
+    "dev": "yarn workspaces foreach -pivW -j unlimited run dev",
+    "dev:no-server": "yarn workspaces foreach --exclude @speckle/server -pivW -j unlimited run dev",
+    "dev:minimal": "yarn workspaces foreach -pivW -j unlimited --include '{@speckle/server,@speckle/frontend,@speckle/shared}' run dev",
+    "gqlgen": "yarn workspaces foreach -pivW -j unlimited --include '{@speckle/server,@speckle/frontend,@speckle/frontend-2}' run gqlgen",
     "dev:server": "yarn workspace @speckle/server dev",
     "dev:frontend": "yarn workspace @speckle/frontend dev",
     "dev:frontend-2": "yarn workspace @speckle/frontend-2 dev",
@@ -91,7 +91,9 @@
     "undici": "^5.28.4",
     "wait-on": ">=7.2.0",
     "word-wrap": "npm:@aashutoshrathi/word-wrap@^1.2.4",
-    "xml2js": ">=0.5.0"
+    "xml2js": ">=0.5.0",
+    "puppeteer-core/ws": "^8.17.1",
+    "@datadog/datadog-ci/ws": "^7.5.10"
   },
   "config": {
     "commitizen": {

packages/dui3/lib/common/generated/gql/gql.ts

@@ -10,7 +10,7 @@ import type { TypedDocumentNode as DocumentNode } from '@graphql-typed-document-
  * 2. It is not minifiable, so the string of a GraphQL query will be multiple times inside the bundle.
  * 3. It does not support dead code elimination, so it will add unused operations.
  *
- * Therefore it is highly recommended to use the babel-plugin for production.
+ * Therefore it is highly recommended to use the babel or swc plugin for production.
  */
 const documents = {
     "\n      query AcccountTestQuery {\n        serverInfo {\n          version\n          name\n          company\n        }\n      }\n    ": types.AcccountTestQueryDocument,
@@ -22,12 +22,12 @@ const documents = {
  *
  * @example
  * ```ts
- * const query = gql(`query GetUser($id: ID!) { user(id: $id) { name } }`);
+ * const query = graphql(`query GetUser($id: ID!) { user(id: $id) { name } }`);
  * ```
  *
  * The query argument is unknown!
  * Please regenerate the types.
-**/
+ */
 export function graphql(source: string): unknown;
 
 /**

packages/dui3/lib/common/generated/gql/index.ts

@@ -1 +1 @@
-export * from "./gql"
+export * from "./gql";

packages/dui3/package.json

@@ -43,10 +43,11 @@
     "vue-tippy": "^6.2.0"
   },
   "devDependencies": {
-    "@graphql-codegen/cli": "^2.13.6",
-    "@graphql-codegen/client-preset": "^1.2.5",
+    "@graphql-codegen/cli": "^5.0.2",
+    "@graphql-codegen/client-preset": "^4.3.0",
     "@nuxt/eslint": "^0.3.13",
     "@nuxtjs/tailwindcss": "^6.7.0",
+    "@parcel/watcher": "^2.4.1",
     "@types/apollo-upload-client": "^17.0.1",
     "@types/eslint": "^8.56.10",
     "@types/lodash-es": "^4.17.6",

packages/fileimport-service/Dockerfile

@@ -33,7 +33,7 @@ RUN yarn workspaces focus --all
 # build shared libraries
 COPY packages/shared ./packages/shared/
 COPY packages/fileimport-service ./packages/fileimport-service/
-RUN yarn workspaces foreach run build
+RUN yarn workspaces foreach -W run build
 
 # Install python virtual env and python dependencies
 RUN apt-get update && \

packages/frontend-2/Dockerfile

@@ -32,7 +32,7 @@ COPY packages/frontend-2 ./packages/frontend-2/
 # hadolint ignore=DL3059
 RUN yarn workspaces focus -A
 # hadolint ignore=DL3059
-RUN yarn workspaces foreach run build
+RUN yarn workspaces foreach -W run build
 # hadolint ignore=DL3059
 RUN find ./packages/frontend-2/.output/ -type f \( -name "*.js.map" -o -name "*.mjs.map" -o -name "*.cjs.map" \) -exec rm -f {} \;
 

packages/frontend-2/lib/common/generated/gql/gql.ts

@@ -10,7 +10,7 @@ import type { TypedDocumentNode as DocumentNode } from '@graphql-typed-document-
  * 2. It is not minifiable, so the string of a GraphQL query will be multiple times inside the bundle.
  * 3. It does not support dead code elimination, so it will add unused operations.
  *
- * Therefore it is highly recommended to use the babel-plugin for production.
+ * Therefore it is highly recommended to use the babel or swc plugin for production.
  */
 const documents = {
     "\n  fragment AuthRegisterPanelServerInfo on ServerInfo {\n    inviteOnly\n  }\n": types.AuthRegisterPanelServerInfoFragmentDoc,
@@ -246,12 +246,12 @@ const documents = {
  *
  * @example
  * ```ts
- * const query = gql(`query GetUser($id: ID!) { user(id: $id) { name } }`);
+ * const query = graphql(`query GetUser($id: ID!) { user(id: $id) { name } }`);
  * ```
  *
  * The query argument is unknown!
  * Please regenerate the types.
-**/
+ */
 export function graphql(source: string): unknown;
 
 /**

packages/frontend-2/lib/common/generated/gql/index.ts

@@ -1 +1 @@
-export * from "./gql"
+export * from "./gql";

packages/frontend-2/package.json

@@ -4,7 +4,7 @@
   "version": "0.0.1",
   "private": true,
   "scripts": {
-    "build": "nuxt build",
+    "build": "NODE_OPTIONS=--max-old-space-size=8192 nuxt build",
     "build:sourcemaps": "BUILD_SOURCEMAPS=true yarn build",
     "dev:nuxt": "nuxt dev",
     "dev:app": "concurrently \"nuxt dev\" \"yarn gqlgen:watch\"",
@@ -78,7 +78,7 @@
     "vee-validate": "^4.7.0",
     "vue-advanced-cropper": "^2.8.8",
     "vue-tippy": "^6.0.0",
-    "ws": "^8.9.0"
+    "ws": "^8.17.1"
   },
   "devDependencies": {
     "@babel/core": "^7.19.6",
@@ -87,11 +87,12 @@
     "@babel/preset-typescript": "^7.18.6",
     "@datadog/datadog-ci": "^2.37.0",
     "@eslint/config-inspector": "^0.4.10",
-    "@graphql-codegen/cli": "^2.13.6",
-    "@graphql-codegen/client-preset": "^1.2.5",
+    "@graphql-codegen/cli": "^5.0.2",
+    "@graphql-codegen/client-preset": "^4.3.0",
     "@nuxt/devtools": "^0.2.5",
     "@nuxt/eslint": "^0.3.13",
     "@nuxtjs/tailwindcss": "^6.3.0",
+    "@parcel/watcher": "^2.4.1",
     "@speckle/tailwind-theme": "workspace:^",
     "@tailwindcss/forms": "^0.5.3",
     "@tailwindcss/line-clamp": "^0.4.2",
@@ -117,7 +118,6 @@
     "eslint": "^9.4.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-plugin-vuejs-accessibility": "^2.3.0",
-    "jest": "27",
     "nuxt": "^3.12.2",
     "pino-pretty": "^10.0.1",
     "postcss": "^8.4.31",
@@ -127,7 +127,6 @@
     "prettier": "^2.7.1",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
-    "rollup-plugin-node-builtins": "^2.1.2",
     "stylelint": "^15.10.1",
     "stylelint-config-prettier": "^9.0.3",
     "stylelint-config-recommended-vue": "^1.4.0",

packages/frontend/Dockerfile

@@ -29,7 +29,7 @@ COPY packages/frontend ./packages/frontend/
 COPY packages/shared ./packages/shared/
 
 # This way the foreach only builds the frontend and its deps
-RUN yarn workspaces foreach run build
+RUN yarn workspaces foreach -W run build
 
 RUN DEBIAN_FRONTEND=noninteractive \
     apt-get -q update && \

packages/frontend/package.json

@@ -63,13 +63,14 @@
     "vuetify-image-input": "^19.1.0"
   },
   "devDependencies": {
-    "@graphql-codegen/cli": "^2.16.3",
-    "@graphql-codegen/introspection": "2.1.1",
-    "@graphql-codegen/typed-document-node": "^2.3.1",
-    "@graphql-codegen/typescript": "2.5.1",
-    "@graphql-codegen/typescript-document-nodes": "2.2.13",
-    "@graphql-codegen/typescript-operations": "2.4.2",
+    "@graphql-codegen/cli": "^5.0.2",
+    "@graphql-codegen/introspection": "^4.0.3",
+    "@graphql-codegen/typed-document-node": "^5.0.7",
+    "@graphql-codegen/typescript": "^4.0.7",
+    "@graphql-codegen/typescript-document-nodes": "^4.0.7",
+    "@graphql-codegen/typescript-operations": "^4.2.1",
     "@mdi/font": "^5.8.55",
+    "@parcel/watcher": "^2.4.1",
     "@swc/core": "^1.2.222",
     "@types/apollo-upload-client": "^17.0.1",
     "@types/dompurify": "^2.3.3",
@@ -90,7 +91,7 @@
     "type-fest": "^2.13.1",
     "typescript": "~4.5.5",
     "unplugin-vue-components": "^0.25.1",
-    "vite": "^4.5.2",
+    "vite": "^5.3.1",
     "vite-bundle-visualizer": "^0.7.0",
     "vite-plugin-simple-gql": "^0.5.0",
     "vue-tsc": "^1.8.8"

packages/preview-service/Dockerfile

@@ -34,7 +34,7 @@ COPY packages/viewer ./packages/viewer/
 COPY packages/preview-service ./packages/preview-service/
 
 # This way the foreach only builds the frontend and its deps
-RUN yarn workspaces foreach run build
+RUN yarn workspaces foreach -W run build
 
 FROM node:18-bookworm-slim@sha256:408f8cbbb7b33a5bb94bdb8862795a94d2b64c2d516856824fd86c4a5594a443 as node
 

packages/preview-service/app.js

@@ -27,7 +27,7 @@ app.use('/api', apiRouter)
 
 // catch 404 and forward to error handler
 app.use(function (req, res, next) {
-  next(createError(404))
+  next(createError(404, `Not Found: ${req.url}`))
 })
 
 // error handler

packages/preview-service/package.json

@@ -15,6 +15,7 @@
   },
   "scripts": {
     "dev": "LOG_PRETTY=true nodemon --trace-deprecation ./bin/www",
+    "dev:headed": "PREVIEWS_HEADED=true yarn dev",
     "build": "webpack --env dev --config webpack.config.render_page.js && webpack --env build --config webpack.config.render_page.js",
     "lint": "eslint ."
   },
@@ -29,6 +30,7 @@
     "http-errors": "~1.6.3",
     "join-images": "^1.1.3",
     "knex": "^2.4.1",
+    "lodash": "^4.17.21",
     "node-fetch": "^2.6.1",
     "pg": "^8.7.3",
     "pg-query-stream": "^4.2.3",
@@ -36,7 +38,7 @@
     "pino-http": "^8.2.1",
     "pino-pretty": "^9.1.1",
     "prom-client": "^14.0.1",
-    "puppeteer": "^13.3.2",
+    "puppeteer": "^22.11.1",
     "sharp": "^0.32.6",
     "yargs": "^17.3.0",
     "zlib": "^1.0.5"

packages/preview-service/render_page/src/example.html

@@ -9,6 +9,7 @@
       <link href="https://fonts.googleapis.com/css2?family=Space+Mono:ital,wght@0,400;0,700;1,700&display=swap" rel="stylesheet">
      <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/skeleton/2.0.4/skeleton.min.css" integrity="sha512-EZLkOqwILORob+p0BXZc+Vm3RgJBOe1Iq/0fiI7r/wJgzOFZMlsqTa29UEl6v6U6gsV4uIpsNZoV32YZqrCRCQ==" crossorigin="anonymous" />
 -->
+    <link href="{%=o.htmlWebpackPlugin.files.favicon%}" rel="shortcut icon" />
     <style type="text/css">
       body {
         font-family: 'Space Mono', monospace !important;

packages/preview-service/routes/preview.js

@@ -5,6 +5,9 @@ const express = require('express')
 const router = express.Router()
 const puppeteer = require('puppeteer')
 const { logger } = require('../observability/logging')
+const { reduce } = require('lodash')
+
+const shouldBeHeadless = process.env.PREVIEWS_HEADED !== 'true'
 
 async function pageFunction(objectUrl) {
   waitForAnimation = async (ms = 70) =>
@@ -56,7 +59,7 @@ async function pageFunction(objectUrl) {
 
 async function getScreenshot(objectUrl, boundLogger = logger) {
   const launchParams = {
-    headless: true,
+    headless: shouldBeHeadless,
     args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage']
   }
   // if ( process.env.PUPPETEER_SKIP_CHROMIUM_DOWNLOAD === 'true' ) {
@@ -97,7 +100,11 @@ async function getScreenshot(objectUrl, boundLogger = logger) {
   boundLogger.info(
     {
       durationSeconds: ret.duration,
-      totalMemoryMB: ret.mem.total / 1000000
+      totalMemoryMB: ret.mem.total / 1000000,
+      resultingImages: {
+        count: Object.keys(ret.scr || {}).length,
+        totalStringSize: reduce(ret.scr || {}, (acc, val) => acc + val.length, 0)
+      }
     },
     `Generated preview.`
   )

packages/preview-service/webpack.config.render_page.js

@@ -48,7 +48,8 @@ const config = {
     new HtmlWebpackPlugin({
       title: 'Speckle Viewer Example',
       template: 'render_page/src/example.html',
-      filename: 'index.html'
+      filename: 'index.html',
+      favicon: 'render_page/src/favicon.ico'
     })
   ],
   resolve: {

packages/server/Dockerfile

@@ -35,7 +35,7 @@ COPY packages/server ./packages/server/
 COPY packages/shared ./packages/shared/
 COPY packages/objectloader ./packages/objectloader/
 
-RUN yarn workspaces foreach run build
+RUN yarn workspaces foreach -W run build
 
 # install only production dependencies
 # we need a clean environment, free of build dependencies

packages/server/modules/previews/index.js

@@ -120,7 +120,8 @@ exports.init = (app, isInitial) => {
       res.set('X-Preview-Error-Code', previewBufferOrFile.errorCode)
     }
     if (previewBufferOrFile.type === 'file') {
-      res.set('Cache-Control', 'public, max-age=604800')
+      // we can't cache these cause they may switch to proper buffer previews in a sec
+      // res.set('Cache-Control', 'public, max-age=604800')
       res.sendFile(previewBufferOrFile.file)
     } else {
       res.contentType('image/png')

packages/server/package.json

@@ -40,7 +40,7 @@
     "@aws-sdk/lib-storage": "^3.100.0",
     "@faker-js/faker": "^7.1.0",
     "@godaddy/terminus": "^4.9.0",
-    "@graphql-tools/schema": "^9.0.4",
+    "@graphql-tools/schema": "^10.0.4",
     "@mailchimp/mailchimp_marketing": "^3.0.80",
     "@sentry/node": "^6.17.9",
     "@sentry/tracing": "^6.17.9",
@@ -115,11 +115,12 @@
   "devDependencies": {
     "@apollo/rover": "^0.23.0",
     "@bull-board/express": "^4.2.2",
-    "@graphql-codegen/cli": "^2.16.3",
-    "@graphql-codegen/typed-document-node": "^5.0.1",
-    "@graphql-codegen/typescript": "2.7.2",
-    "@graphql-codegen/typescript-operations": "^2.5.2",
-    "@graphql-codegen/typescript-resolvers": "2.7.2",
+    "@graphql-codegen/cli": "^5.0.2",
+    "@graphql-codegen/typed-document-node": "^5.0.7",
+    "@graphql-codegen/typescript": "^4.0.7",
+    "@graphql-codegen/typescript-operations": "^4.2.1",
+    "@graphql-codegen/typescript-resolvers": "^4.1.0",
+    "@parcel/watcher": "^2.4.1",
     "@swc/core": "^1.2.222",
     "@tiptap/core": "^2.0.0-beta.176",
     "@types/bcrypt": "^5.0.0",
@@ -173,7 +174,7 @@
     "type-fest": "^2.19.0",
     "typescript": "^4.6.4",
     "typescript-eslint": "^7.12.0",
-    "ws": "^7.5.7",
+    "ws": "^8.17.1",
     "yargs": "^17.3.1"
   },
   "config": {

packages/ui-components/.storybook/main.ts

@@ -17,9 +17,6 @@ const config: StorybookConfig = {
     name: getAbsolutePath('@storybook/vue3-vite'),
     options: {}
   },
-  docs: {
-    autodocs: true
-  },
   viteFinal(config) {
     // Remove dts plugin, we don't need it and it only causes issues
     // eslint-disable-next-line @typescript-eslint/no-floating-promises

packages/ui-components/.storybook/preview.ts

@@ -139,3 +139,5 @@ export const globalTypes = {
     }
   }
 }
+
+export const tags = ['autodocs']

packages/ui-components/package.json

@@ -48,7 +48,7 @@
     "@headlessui/vue": "^1.7.18",
     "@heroicons/vue": "^2.0.12",
     "@speckle/shared": "workspace:^",
-    "@storybook/test": "^8.0.5",
+    "@storybook/test": "^8.1.10",
     "@vueuse/core": "^9.13.0",
     "lodash": "^4.0.0",
     "lodash-es": "^4.0.0",
@@ -63,13 +63,13 @@
     "@babel/preset-react": "^7.18.6",
     "@rollup/plugin-typescript": "^11.1.0",
     "@speckle/tailwind-theme": "workspace:^",
-    "@storybook/addon-essentials": "^8.0.5",
-    "@storybook/addon-interactions": "^8.0.5",
-    "@storybook/addon-links": "^8.0.5",
-    "@storybook/blocks": "^8.0.5",
-    "@storybook/test-runner": "^0.17.0",
-    "@storybook/vue3": "^8.0.5",
-    "@storybook/vue3-vite": "^8.0.5",
+    "@storybook/addon-essentials": "^8.1.10",
+    "@storybook/addon-interactions": "^8.1.10",
+    "@storybook/addon-links": "^8.1.10",
+    "@storybook/blocks": "^8.1.10",
+    "@storybook/test-runner": "^0.18.2",
+    "@storybook/vue3": "^8.1.10",
+    "@storybook/vue3-vite": "^8.1.10",
     "@types/eslint": "^8.56.10",
     "@types/lodash": "^4.0.0",
     "@types/lodash-es": "^4.17.12",
@@ -89,7 +89,7 @@
     "postcss-nesting": "^10.2.0",
     "prettier": "^2.7.1",
     "rollup-plugin-node-builtins": "^2.1.2",
-    "storybook": "^8.0.5",
+    "storybook": "^8.1.10",
     "tailwindcss": "^3.3.2",
     "type-fest": "^2.13.1",
     "typescript": "^5.0.4",

packages/ui-components/src/stories/styling/semantic-colors/SemanticColors.stories.ts

@@ -13,5 +13,6 @@ export const SemanticColors: StoryObj = {
   }),
   parameters: {
     viewMode: 'docs'
-  }
+  },
+  tags: ['!autodocs']
 }

packages/webhook-service/Dockerfile

@@ -20,7 +20,7 @@ RUN yarn workspaces focus --all
 COPY packages/shared ./packages/shared/
 COPY packages/webhook-service/src ./packages/webhook-service/
 
-RUN yarn workspaces foreach run build
+RUN yarn workspaces foreach -W run build
 
 ARG WAIT_VERSION=2.8.0
 ENV WAIT_VERSION=${WAIT_VERSION}