Merge branch 'main' of github.com:specklesystems/speckle-server into gergo/adminFacelift

2023-08-01 15:27:51 +02:00
parent f6cc4b7835 b06959b092
commit 67260c2de4
15 changed files with 210 additions and 58 deletions
@@ -117,9 +117,9 @@ Essentially, ensure you have **Volar** enabled, and the built in **TypeScript an

 We have a [Maildev](https://github.com/maildev/maildev) container available that you can use to see all e-mails sent out from the app. Make sure your `server` .env file is configured properly to use it:

-```
+```bash
 EMAIL=true
-EMAIL_FROM="speckle@speckle.local"
+EMAIL_FROM="no-reply@example.org"
 EMAIL_HOST="localhost"
 EMAIL_PORT="1025"
 ```
@@ -51,6 +51,7 @@ services:
      S3_CREATE_BUCKET: 'true'
      S3_REGION: '' # optional, defaults to 'us-east-1'
      FILE_SIZE_LIMIT_MB: 100
+      EMAIL_FROM: 'no-reply@example.org'

  preview-service:
    build:
@@ -68,12 +68,10 @@ module.exports = class ServerAPI {
      totalChildrenCountByDepth
    )

-    const q1 = Objects().insert(insertionObject).toString() + ' on conflict do nothing'
-    await knex.raw(q1)
+    await Objects().insert(insertionObject).onConflict().ignore()

    if (closures.length > 0) {
-      const q2 = `${Closures().insert(closures).toString()} on conflict do nothing`
-      await knex.raw(q2)
+      await Closures().insert(closures).onConflict().ignore()
    }

    return insertionObject.id
@@ -125,10 +123,7 @@ module.exports = class ServerAPI {
      const batches = chunk(objsToInsert, objectsBatchSize)
      for (const [index, batch] of batches.entries()) {
        this.prepInsertionObjectBatch(batch)
-        await knex.transaction(async (trx) => {
-          const q = Objects().insert(batch).toString() + ' on conflict do nothing'
-          await trx.raw(q)
-        })
+        await Objects().insert(batch).onConflict().ignore()
        this.logger.info(
          `Inserted ${batch.length} objects from batch ${index + 1} of ${
            batches.length
@@ -143,10 +138,7 @@ module.exports = class ServerAPI {

      for (const [index, batch] of batches.entries()) {
        this.prepInsertionClosureBatch(batch)
-        await knex.transaction(async (trx) => {
-          const q = Closures().insert(batch).toString() + ' on conflict do nothing'
-          await trx.raw(q)
-        })
+        await Closures().insert(batch).onConflict().ignore()
        this.logger.info(
          `Inserted ${batch.length} closures from batch ${index + 1} of ${
            batches.length
@@ -64,8 +64,8 @@ S3_CREATE_BUCKET="true"
 # Emails
 ############################################################
 EMAIL=true
-EMAIL_FROM="speckle@speckle.local"
 EMAIL_HOST="127.0.0.1"
+EMAIL_FROM="no-reply@example.org"
 EMAIL_PORT="1025"

 # EMAIL_HOST="-> FILL IN <-"
@@ -33,7 +33,7 @@
      "console": "integratedTerminal"
    },
    {
-      "args": ["-g='@comments'", "--timeout=10000", "--exit"],
+      "args": ["-g='@ChunkInsertionObject'", "--timeout=10000", "--exit"],
      // "envFile": "${workspaceFolder}/.env",
      "env": {
        "PORT": "0",
@@ -48,7 +48,7 @@
      "program": "${workspaceFolder}/node_modules/mocha/bin/_mocha",
      "request": "launch",
      "skipFiles": ["<node_internals>/**"],
-      "type": "pwa-node"
+      "type": "node"
    },
    {
      "name": "NPM test",
@@ -65,10 +65,10 @@
      // "envFile": "${workspaceFolder}/.env",
      "runtimeExecutable": "npm",
      "skipFiles": ["<node_internals>/**"],
-      "type": "pwa-node"
+      "type": "node"
    },
    {
-      "type": "pwa-node",
+      "type": "node",
      "request": "launch",
      "name": "Launch Program",
      "skipFiles": ["<node_internals>/**"],
@@ -7,6 +7,7 @@ const { validatePermissionsWriteStream } = require('./authUtils')

 const { createObjectsBatched } = require('@/modules/core/services/objects')
 const { ObjectHandlingError } = require('@/modules/core/errors/object')
+const { estimateStringMegabyteSize } = require('@/modules/core/utils/chunking')

 const MAX_FILE_SIZE = 50 * 1024 * 1024

@@ -82,15 +83,17 @@ module.exports = (app) => {
          }

          const gunzippedBuffer = zlib.gunzipSync(gzippedBuffer).toString()
-          if (gunzippedBuffer.length > MAX_FILE_SIZE) {
+          const gunzippedBufferMegabyteSize =
+            estimateStringMegabyteSize(gunzippedBuffer)
+          if (gunzippedBufferMegabyteSize > MAX_FILE_SIZE) {
            req.log.error(
-              `Upload error: Batch size too large (${gunzippedBuffer.length} > ${MAX_FILE_SIZE})`
+              `upload error: batch size too large (${gunzippedBufferMegabyteSize} > ${MAX_FILE_SIZE})`
            )
            if (!requestDropped)
              res
                .status(400)
                .send(
-                  `File size too large (${gunzippedBuffer.length} > ${MAX_FILE_SIZE})`
+                  `File size too large (${gunzippedBufferMegabyteSize} > ${MAX_FILE_SIZE})`
                )
            requestDropped = true
          }
@@ -224,7 +227,7 @@ module.exports = (app) => {
          await promise
          req.log.info(
            {
-              uploadedSizeMB: buffer.length / 1000000,
+              uploadedSizeMB: estimateStringMegabyteSize(buffer),
              durationSeconds: (Date.now() - t0) / 1000,
              crtMemUsageMB: process.memoryUsage().heapUsed / 1024 / 1024,
              requestDropped
@@ -7,6 +7,10 @@ const knex = require(`@/db/knex`)
 const { servicesLogger } = require('@/logging/logging')
 const { getMaximumObjectSizeMB } = require('@/modules/shared/helpers/envHelper')
 const { ObjectHandlingError } = require('@/modules/core/errors/object')
+const {
+  chunkInsertionObjectArray,
+  estimateStringMegabyteSize
+} = require('@/modules/core/utils/chunking')

 const Objects = () => knex('objects')
 const Closures = () => knex('object_children_closure')
@@ -43,12 +47,10 @@ module.exports = {
      totalChildrenCountByDepth
    )

-    const q1 = Objects().insert(insertionObject).toString() + ' on conflict do nothing'
-    await knex.raw(q1)
+    await Objects().insert(insertionObject).onConflict().ignore()

    if (closures.length > 0) {
-      const q2 = `${Closures().insert(closures).toString()} on conflict do nothing`
-      await knex.raw(q2)
+      await Closures().insert(closures).onConflict().ignore()
    }

    return insertionObject.id
@@ -97,13 +99,15 @@ module.exports = {

    // step 1: insert objects
    if (objsToInsert.length > 0) {
-      const batches = chunk(objsToInsert, objectsBatchSize)
+      // const batches = chunk(objsToInsert, objectsBatchSize)
+      const batches = chunkInsertionObjectArray({
+        objects: objsToInsert,
+        chunkLengthLimit: objectsBatchSize,
+        chunkSizeLimitMb: 2
+      })
      for (const batch of batches) {
        prepInsertionObjectBatch(batch)
-        await knex.transaction(async (trx) => {
-          const q = Objects().insert(batch).toString() + ' on conflict do nothing'
-          await trx.raw(q)
-        })
+        await Objects().insert(batch).onConflict().ignore()
        servicesLogger.info(`Inserted ${batch.length} objects`)
      }
    }
@@ -114,10 +118,7 @@ module.exports = {

      for (const batch of batches) {
        prepInsertionClosureBatch(batch)
-        await knex.transaction(async (trx) => {
-          const q = Closures().insert(batch).toString() + ' on conflict do nothing'
-          await trx.raw(q)
-        })
+        await Closures().insert(batch).onConflict().ignore()
        servicesLogger.info(`Inserted ${batch.length} closures`)
      }
    }
@@ -180,14 +181,11 @@ module.exports = {
      })

      if (objsToInsert.length > 0) {
-        const queryObjs =
-          Objects().insert(objsToInsert).toString() + ' on conflict do nothing'
-        await knex.raw(queryObjs)
+        await Objects().insert(objsToInsert).onConflict().ignore()
      }

      if (closures.length > 0) {
-        const q2 = `${Closures().insert(closures).toString()} on conflict do nothing`
-        await knex.raw(q2)
+        await Closures().insert(closures).onConflict().ignore()
      }

      const t1 = performance.now()
@@ -596,8 +594,7 @@ module.exports = {
 // limitations when doing upserts - ignored fields are not always returned, hence
 // we cannot provide a full response back including all object hashes.
 function prepInsertionObject(streamId, obj) {
-  // let memNow = process.memoryUsage().heapUsed / 1024 / 1024
-  const MAX_OBJECT_SIZE = getMaximumObjectSizeMB() * 1024 * 1024
+  const MAX_OBJECT_SIZE_MB = getMaximumObjectSizeMB()

  if (obj.hash) obj.id = obj.hash
  else
@@ -605,13 +602,12 @@ function prepInsertionObject(streamId, obj) {
      obj.id || crypto.createHash('md5').update(JSON.stringify(obj)).digest('hex') // generate a hash if none is present

  const stringifiedObj = JSON.stringify(obj)
-  if (stringifiedObj.length > MAX_OBJECT_SIZE) {
+  const objectByteSize = estimateStringMegabyteSize(stringifiedObj)
+  if (objectByteSize > MAX_OBJECT_SIZE_MB) {
    throw new ObjectHandlingError(
-      `Object too large. (${stringifiedObj.length} > ${MAX_OBJECT_SIZE})`
+      `Object too large. (${objectByteSize} MB is > than limit, ${MAX_OBJECT_SIZE_MB} MB)`
    )
  }
-  // let memAfter = process.memoryUsage().heapUsed / 1024 / 1024
-
  return {
    data: stringifiedObj, // stored in jsonb column
    streamId,
@@ -0,0 +1,64 @@
+import { ArgumentError, chunkInsertionObjectArray } from '@/modules/core/utils/chunking'
+import { expect } from 'chai'
+
+describe('ChunkInsertionObjectArray', () => {
+  it('throws for invalid chunk length limits', () => {
+    expect(() =>
+      chunkInsertionObjectArray({
+        objects: [],
+        chunkLengthLimit: 0,
+        chunkSizeLimitMb: 10
+      })
+    ).to.throw(ArgumentError, 'Chunks must have a length limit > 1')
+  })
+
+  it('throws for invalid chunk size limits', () => {
+    expect(() =>
+      chunkInsertionObjectArray({
+        objects: [],
+        chunkLengthLimit: 1,
+        chunkSizeLimitMb: 0
+      })
+    ).to.throw(ArgumentError, 'Chunks must have a size in MB limit > 0')
+  })
+  it('creates an array-array of objects', () => {
+    const objects = [
+      {
+        data: 'fake'
+      }
+    ]
+    const chunkSizeLimitMb = 10
+    const chunkLengthLimit = 10
+    const insertionChunks = chunkInsertionObjectArray({
+      objects,
+      chunkSizeLimitMb,
+      chunkLengthLimit
+    })
+    expect(insertionChunks).deep.equals([objects])
+  })
+  it('breaks into chunks based on length limit', () => {
+    const object = { data: 'fake' }
+    const chunkSizeLimitMb = 10
+    const chunkLengthLimit = 2
+    const insertionChunks = chunkInsertionObjectArray({
+      objects: Array(10).fill(object),
+      chunkSizeLimitMb,
+      chunkLengthLimit
+    })
+    expect(insertionChunks).deep.equals(Array(5).fill([object, object]))
+  })
+  it('breaks into chunks based on size limit', () => {
+    // use 4 chars, thats 8 bytes
+    const object = { data: 'fake' }
+    // using 16 bytes as limit should result in chunks of 2 objects
+    const chunkSizeLimitMb = 8 / 1_000_000
+    const chunkLengthLimit = 10000
+    const insertionChunks = chunkInsertionObjectArray({
+      objects: Array(11).fill(object),
+      chunkSizeLimitMb,
+      chunkLengthLimit
+    })
+    const expected = Array(6).fill([object, object]).fill([object], -1)
+    expect(insertionChunks).deep.equals(expected)
+  })
+})
@@ -0,0 +1,72 @@
+import { BaseError } from '@/modules/shared/errors'
+import { Options } from 'verror'
+
+type InsertionObject = {
+  data: string
+}
+
+export class ArgumentError extends BaseError {
+  static defaultMessage = 'Invalid argument value provided'
+
+  constructor(message?: string | undefined, options?: Options | Error | undefined) {
+    super(message, options)
+  }
+}
+
+// since we're mostly using this for an artificial limit calculation
+// we can live with a somewhat imprecise but fast estimate
+// Js uses utf16 so the in memory string size in bytes is length * 2
+// this is just the in memory string size, not the utf-8 encoded byte size
+// since our data is mostly ascii characters, its prob safe to use
+// string.length is a slight underestimation of the actual size
+export const estimateStringByteSize = (str: string) => str.length
+export const estimateStringMegabyteSize = (str: string) =>
+  estimateStringByteSize(str) / 1_000_000
+
+export const chunkInsertionObjectArray = ({
+  objects,
+  chunkSizeLimitMb,
+  chunkLengthLimit
+}: {
+  chunkSizeLimitMb: number
+  chunkLengthLimit: number
+  objects: InsertionObject[]
+}): InsertionObject[][] => {
+  if (chunkLengthLimit < 1)
+    throw new ArgumentError('Chunks must have a length limit > 1')
+  if (chunkSizeLimitMb <= 0)
+    throw new ArgumentError('Chunks must have a size in MB limit > 0')
+
+  let currentChunkSize = 0
+  let currentChunkLength = 0
+  const chunkedObjects: InsertionObject[][] = []
+  let currentBatch: InsertionObject[] = []
+  for (const obj of objects) {
+    // if limits are exceeded start a new batch
+    if (
+      currentChunkSize >= chunkSizeLimitMb ||
+      currentChunkLength >= chunkLengthLimit
+    ) {
+      console.log(
+        `chunking on ${
+          currentChunkSize >= chunkSizeLimitMb ? 'object size' : 'chunk length'
+        }`
+      )
+      // push the current batch into the final chunks
+      chunkedObjects.push(currentBatch)
+      // reset the current batch
+      currentBatch = []
+      // reset limits
+      currentChunkSize = 0
+      currentChunkLength = 0
+    }
+    // do some proper chunking here
+    // insert the batch to returned chunks
+    currentChunkLength++
+    currentChunkSize += estimateStringMegabyteSize(obj.data)
+    currentBatch.push(obj)
+  }
+  // do not forget to push the final batch
+  chunkedObjects.push(currentBatch)
+  return chunkedObjects
+}
@@ -1,5 +1,6 @@
 import { logger } from '@/logging/logging'
 import { getTransporter } from '@/modules/emails/utils/transporter'
+import { getEmailFromAddress } from '@/modules/shared/helpers/envHelper'

 export type SendEmailParams = {
  from?: string
@@ -25,7 +26,7 @@ export async function sendEmail({
    return false
  }
  try {
-    const emailFrom = process.env.EMAIL_FROM || 'no-reply@speckle.systems'
+    const emailFrom = getEmailFromAddress()
    await transporter.sendMail({
      from: from || `"Speckle" <${emailFrom}>`,
      to,
@@ -144,7 +144,7 @@ export function getFrontendOrigin(forceFe2?: boolean) {
 export function getServerOrigin() {
  if (!process.env.CANONICAL_URL) {
    throw new MisconfiguredEnvironmentError(
-      'Server origin env var (CANONICAL_URL) not configured'
+      'Server origin environment variable (CANONICAL_URL) not configured'
    )
  }

@@ -204,3 +204,12 @@ export function getOnboardingStreamCacheBustNumber() {
  const val = process.env.ONBOARDING_STREAM_CACHE_BUST_NUMBER || '1'
  return parseInt(val) || 1
 }
+
+export function getEmailFromAddress() {
+  if (!process.env.EMAIL_FROM) {
+    throw new MisconfiguredEnvironmentError(
+      'Email From environment variable (EMAIL_FROM) is not configured'
+    )
+  }
+  return process.env.EMAIL_FROM
+}
@@ -110,7 +110,7 @@ const getStream = () => {
    // prettier-ignore
    // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8?c=%5B-7.66134,10.82932,6.41935,-0.07739,-13.88552,1.8697,0,1%5D'
    // Revit sample house (good for bim-like stuff with many display meshes)
-    // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8'
+    'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8'
    // 'https://latest.speckle.dev/streams/c1faab5c62/commits/6c6e43e5f3'
    // 'https://latest.speckle.dev/streams/58b5648c4d/commits/60371ecb2d'
    // 'Super' heavy revit shit
@@ -278,12 +278,13 @@ const getStream = () => {
    // 'https://latest.speckle.dev/streams/b68abcbf2e/commits/4e94ecad62'
    // Big ass mafa'
    // 'https://speckle.xyz/streams/88307505eb/objects/a232d760059046b81ff97e6c4530c985'
-    'https://latest.speckle.dev/streams/92b620fb17/commits/dfb9ca025d'
+    // 'https://latest.speckle.dev/streams/92b620fb17/commits/dfb9ca025d'
    // 'Blocks with elements
    // 'https://latest.speckle.dev/streams/e258b0e8db/commits/00e165cc1c'
    // 'https://latest.speckle.dev/streams/e258b0e8db/commits/e48cf53add'
    // 'https://latest.speckle.dev/streams/e258b0e8db/commits/c19577c7d6?c=%5B15.88776,-8.2182,12.17095,18.64059,1.48552,0.6025,0,1%5D'
    // 'https://speckle.xyz/streams/46caea9b53/commits/71938adcd1'
+    // 'https://speckle.xyz/streams/2f9f2f3021/commits/75bd13f513'
  )
 }

@@ -696,12 +696,25 @@ export class FilteringManager extends EventEmitter {
    const key = objectIds.join(',')

    if (this.idCache[key] && this.idCache[key].length) return this.idCache[key]
-
+    /** This doesn't return descendants correctly for some streams like:
+     * https://speckle.xyz/streams/2f9f2f3021/commits/75bd13f513
+     */
+    // this.WTI.walk((node: TreeNode) => {
+    //   if (objectIds.includes(node.model.raw.id) && node.model.raw.__closure) {
+    //     const ids = Object.keys(node.model.raw.__closure)
+    //     allIds.push(...ids)
+    //     this.idCache[node.model.raw.id] = ids
+    //   }
+    //   return true
+    // })
    this.WTI.walk((node: TreeNode) => {
-      if (objectIds.includes(node.model.raw.id) && node.model.raw.__closure) {
-        const ids = Object.keys(node.model.raw.__closure)
-        allIds.push(...ids)
-        this.idCache[node.model.raw.id] = ids
+      if (objectIds.includes(node.model.raw.id)) {
+        const subtree = node.all((node) => {
+          return node.model.raw !== undefined
+        })
+        const idList = subtree.map((node) => node.model.raw.id)
+        allIds.push(...idList)
+        this.idCache[node.model.raw.id] = idList
      }
      return true
    })
@@ -80,7 +80,7 @@
    "docker_image_tag": {
      "type": "string",
      "description": "Speckle is published as a Docker Image.  The version of the image which will be deployed is specified by this tag.",
-      "default": "2.13.3"
+      "default": "2"
    },
    "imagePullPolicy": {
      "type": "string",
@@ -65,7 +65,7 @@ ingress:
 ##
 ## @param docker_image_tag Speckle is published as a Docker Image.  The version of the image which will be deployed is specified by this tag.
 ##
-docker_image_tag: 2.13.3
+docker_image_tag: '2'

 ## @param imagePullPolicy Determines the conditions when the Docker Images for Speckle should be pulled from the Image registry.
 ## ref: https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy