diff --git a/README.md b/README.md index 4711d20ed..385b06885 100644 --- a/README.md +++ b/README.md @@ -117,9 +117,9 @@ Essentially, ensure you have **Volar** enabled, and the built in **TypeScript an We have a [Maildev](https://github.com/maildev/maildev) container available that you can use to see all e-mails sent out from the app. Make sure your `server` .env file is configured properly to use it: -``` +```bash EMAIL=true -EMAIL_FROM="speckle@speckle.local" +EMAIL_FROM="no-reply@example.org" EMAIL_HOST="localhost" EMAIL_PORT="1025" ``` diff --git a/docker-compose-speckle.yml b/docker-compose-speckle.yml index 291863c50..33519479d 100644 --- a/docker-compose-speckle.yml +++ b/docker-compose-speckle.yml @@ -51,6 +51,7 @@ services: S3_CREATE_BUCKET: 'true' S3_REGION: '' # optional, defaults to 'us-east-1' FILE_SIZE_LIMIT_MB: 100 + EMAIL_FROM: 'no-reply@example.org' preview-service: build: diff --git a/packages/fileimport-service/ifc/api.js b/packages/fileimport-service/ifc/api.js index 392590a92..ae80fe839 100644 --- a/packages/fileimport-service/ifc/api.js +++ b/packages/fileimport-service/ifc/api.js @@ -68,12 +68,10 @@ module.exports = class ServerAPI { totalChildrenCountByDepth ) - const q1 = Objects().insert(insertionObject).toString() + ' on conflict do nothing' - await knex.raw(q1) + await Objects().insert(insertionObject).onConflict().ignore() if (closures.length > 0) { - const q2 = `${Closures().insert(closures).toString()} on conflict do nothing` - await knex.raw(q2) + await Closures().insert(closures).onConflict().ignore() } return insertionObject.id @@ -125,10 +123,7 @@ module.exports = class ServerAPI { const batches = chunk(objsToInsert, objectsBatchSize) for (const [index, batch] of batches.entries()) { this.prepInsertionObjectBatch(batch) - await knex.transaction(async (trx) => { - const q = Objects().insert(batch).toString() + ' on conflict do nothing' - await trx.raw(q) - }) + await Objects().insert(batch).onConflict().ignore() this.logger.info( `Inserted ${batch.length} objects from batch ${index + 1} of ${ batches.length @@ -143,10 +138,7 @@ module.exports = class ServerAPI { for (const [index, batch] of batches.entries()) { this.prepInsertionClosureBatch(batch) - await knex.transaction(async (trx) => { - const q = Closures().insert(batch).toString() + ' on conflict do nothing' - await trx.raw(q) - }) + await Closures().insert(batch).onConflict().ignore() this.logger.info( `Inserted ${batch.length} closures from batch ${index + 1} of ${ batches.length diff --git a/packages/server/.env-example b/packages/server/.env-example index c8d1d75e8..8130a3a26 100644 --- a/packages/server/.env-example +++ b/packages/server/.env-example @@ -64,8 +64,8 @@ S3_CREATE_BUCKET="true" # Emails ############################################################ EMAIL=true -EMAIL_FROM="speckle@speckle.local" EMAIL_HOST="127.0.0.1" +EMAIL_FROM="no-reply@example.org" EMAIL_PORT="1025" # EMAIL_HOST="-> FILL IN <-" diff --git a/packages/server/.vscode/launch.json b/packages/server/.vscode/launch.json index 8f0b62712..f31af9b0f 100644 --- a/packages/server/.vscode/launch.json +++ b/packages/server/.vscode/launch.json @@ -33,7 +33,7 @@ "console": "integratedTerminal" }, { - "args": ["-g='@comments'", "--timeout=10000", "--exit"], + "args": ["-g='@ChunkInsertionObject'", "--timeout=10000", "--exit"], // "envFile": "${workspaceFolder}/.env", "env": { "PORT": "0", @@ -48,7 +48,7 @@ "program": "${workspaceFolder}/node_modules/mocha/bin/_mocha", "request": "launch", "skipFiles": ["/**"], - "type": "pwa-node" + "type": "node" }, { "name": "NPM test", @@ -65,10 +65,10 @@ // "envFile": "${workspaceFolder}/.env", "runtimeExecutable": "npm", "skipFiles": ["/**"], - "type": "pwa-node" + "type": "node" }, { - "type": "pwa-node", + "type": "node", "request": "launch", "name": "Launch Program", "skipFiles": ["/**"], diff --git a/packages/server/modules/core/rest/upload.js b/packages/server/modules/core/rest/upload.js index aae646ff2..d9a22753e 100644 --- a/packages/server/modules/core/rest/upload.js +++ b/packages/server/modules/core/rest/upload.js @@ -7,6 +7,7 @@ const { validatePermissionsWriteStream } = require('./authUtils') const { createObjectsBatched } = require('@/modules/core/services/objects') const { ObjectHandlingError } = require('@/modules/core/errors/object') +const { estimateStringMegabyteSize } = require('@/modules/core/utils/chunking') const MAX_FILE_SIZE = 50 * 1024 * 1024 @@ -82,15 +83,17 @@ module.exports = (app) => { } const gunzippedBuffer = zlib.gunzipSync(gzippedBuffer).toString() - if (gunzippedBuffer.length > MAX_FILE_SIZE) { + const gunzippedBufferMegabyteSize = + estimateStringMegabyteSize(gunzippedBuffer) + if (gunzippedBufferMegabyteSize > MAX_FILE_SIZE) { req.log.error( - `Upload error: Batch size too large (${gunzippedBuffer.length} > ${MAX_FILE_SIZE})` + `upload error: batch size too large (${gunzippedBufferMegabyteSize} > ${MAX_FILE_SIZE})` ) if (!requestDropped) res .status(400) .send( - `File size too large (${gunzippedBuffer.length} > ${MAX_FILE_SIZE})` + `File size too large (${gunzippedBufferMegabyteSize} > ${MAX_FILE_SIZE})` ) requestDropped = true } @@ -224,7 +227,7 @@ module.exports = (app) => { await promise req.log.info( { - uploadedSizeMB: buffer.length / 1000000, + uploadedSizeMB: estimateStringMegabyteSize(buffer), durationSeconds: (Date.now() - t0) / 1000, crtMemUsageMB: process.memoryUsage().heapUsed / 1024 / 1024, requestDropped diff --git a/packages/server/modules/core/services/objects.js b/packages/server/modules/core/services/objects.js index 194a1da55..f064e80a1 100644 --- a/packages/server/modules/core/services/objects.js +++ b/packages/server/modules/core/services/objects.js @@ -7,6 +7,10 @@ const knex = require(`@/db/knex`) const { servicesLogger } = require('@/logging/logging') const { getMaximumObjectSizeMB } = require('@/modules/shared/helpers/envHelper') const { ObjectHandlingError } = require('@/modules/core/errors/object') +const { + chunkInsertionObjectArray, + estimateStringMegabyteSize +} = require('@/modules/core/utils/chunking') const Objects = () => knex('objects') const Closures = () => knex('object_children_closure') @@ -43,12 +47,10 @@ module.exports = { totalChildrenCountByDepth ) - const q1 = Objects().insert(insertionObject).toString() + ' on conflict do nothing' - await knex.raw(q1) + await Objects().insert(insertionObject).onConflict().ignore() if (closures.length > 0) { - const q2 = `${Closures().insert(closures).toString()} on conflict do nothing` - await knex.raw(q2) + await Closures().insert(closures).onConflict().ignore() } return insertionObject.id @@ -97,13 +99,15 @@ module.exports = { // step 1: insert objects if (objsToInsert.length > 0) { - const batches = chunk(objsToInsert, objectsBatchSize) + // const batches = chunk(objsToInsert, objectsBatchSize) + const batches = chunkInsertionObjectArray({ + objects: objsToInsert, + chunkLengthLimit: objectsBatchSize, + chunkSizeLimitMb: 2 + }) for (const batch of batches) { prepInsertionObjectBatch(batch) - await knex.transaction(async (trx) => { - const q = Objects().insert(batch).toString() + ' on conflict do nothing' - await trx.raw(q) - }) + await Objects().insert(batch).onConflict().ignore() servicesLogger.info(`Inserted ${batch.length} objects`) } } @@ -114,10 +118,7 @@ module.exports = { for (const batch of batches) { prepInsertionClosureBatch(batch) - await knex.transaction(async (trx) => { - const q = Closures().insert(batch).toString() + ' on conflict do nothing' - await trx.raw(q) - }) + await Closures().insert(batch).onConflict().ignore() servicesLogger.info(`Inserted ${batch.length} closures`) } } @@ -180,14 +181,11 @@ module.exports = { }) if (objsToInsert.length > 0) { - const queryObjs = - Objects().insert(objsToInsert).toString() + ' on conflict do nothing' - await knex.raw(queryObjs) + await Objects().insert(objsToInsert).onConflict().ignore() } if (closures.length > 0) { - const q2 = `${Closures().insert(closures).toString()} on conflict do nothing` - await knex.raw(q2) + await Closures().insert(closures).onConflict().ignore() } const t1 = performance.now() @@ -596,8 +594,7 @@ module.exports = { // limitations when doing upserts - ignored fields are not always returned, hence // we cannot provide a full response back including all object hashes. function prepInsertionObject(streamId, obj) { - // let memNow = process.memoryUsage().heapUsed / 1024 / 1024 - const MAX_OBJECT_SIZE = getMaximumObjectSizeMB() * 1024 * 1024 + const MAX_OBJECT_SIZE_MB = getMaximumObjectSizeMB() if (obj.hash) obj.id = obj.hash else @@ -605,13 +602,12 @@ function prepInsertionObject(streamId, obj) { obj.id || crypto.createHash('md5').update(JSON.stringify(obj)).digest('hex') // generate a hash if none is present const stringifiedObj = JSON.stringify(obj) - if (stringifiedObj.length > MAX_OBJECT_SIZE) { + const objectByteSize = estimateStringMegabyteSize(stringifiedObj) + if (objectByteSize > MAX_OBJECT_SIZE_MB) { throw new ObjectHandlingError( - `Object too large. (${stringifiedObj.length} > ${MAX_OBJECT_SIZE})` + `Object too large. (${objectByteSize} MB is > than limit, ${MAX_OBJECT_SIZE_MB} MB)` ) } - // let memAfter = process.memoryUsage().heapUsed / 1024 / 1024 - return { data: stringifiedObj, // stored in jsonb column streamId, diff --git a/packages/server/modules/core/tests/chunking.spec.ts b/packages/server/modules/core/tests/chunking.spec.ts new file mode 100644 index 000000000..9c8f1c93a --- /dev/null +++ b/packages/server/modules/core/tests/chunking.spec.ts @@ -0,0 +1,64 @@ +import { ArgumentError, chunkInsertionObjectArray } from '@/modules/core/utils/chunking' +import { expect } from 'chai' + +describe('ChunkInsertionObjectArray', () => { + it('throws for invalid chunk length limits', () => { + expect(() => + chunkInsertionObjectArray({ + objects: [], + chunkLengthLimit: 0, + chunkSizeLimitMb: 10 + }) + ).to.throw(ArgumentError, 'Chunks must have a length limit > 1') + }) + + it('throws for invalid chunk size limits', () => { + expect(() => + chunkInsertionObjectArray({ + objects: [], + chunkLengthLimit: 1, + chunkSizeLimitMb: 0 + }) + ).to.throw(ArgumentError, 'Chunks must have a size in MB limit > 0') + }) + it('creates an array-array of objects', () => { + const objects = [ + { + data: 'fake' + } + ] + const chunkSizeLimitMb = 10 + const chunkLengthLimit = 10 + const insertionChunks = chunkInsertionObjectArray({ + objects, + chunkSizeLimitMb, + chunkLengthLimit + }) + expect(insertionChunks).deep.equals([objects]) + }) + it('breaks into chunks based on length limit', () => { + const object = { data: 'fake' } + const chunkSizeLimitMb = 10 + const chunkLengthLimit = 2 + const insertionChunks = chunkInsertionObjectArray({ + objects: Array(10).fill(object), + chunkSizeLimitMb, + chunkLengthLimit + }) + expect(insertionChunks).deep.equals(Array(5).fill([object, object])) + }) + it('breaks into chunks based on size limit', () => { + // use 4 chars, thats 8 bytes + const object = { data: 'fake' } + // using 16 bytes as limit should result in chunks of 2 objects + const chunkSizeLimitMb = 8 / 1_000_000 + const chunkLengthLimit = 10000 + const insertionChunks = chunkInsertionObjectArray({ + objects: Array(11).fill(object), + chunkSizeLimitMb, + chunkLengthLimit + }) + const expected = Array(6).fill([object, object]).fill([object], -1) + expect(insertionChunks).deep.equals(expected) + }) +}) diff --git a/packages/server/modules/core/utils/chunking.ts b/packages/server/modules/core/utils/chunking.ts new file mode 100644 index 000000000..303def15e --- /dev/null +++ b/packages/server/modules/core/utils/chunking.ts @@ -0,0 +1,72 @@ +import { BaseError } from '@/modules/shared/errors' +import { Options } from 'verror' + +type InsertionObject = { + data: string +} + +export class ArgumentError extends BaseError { + static defaultMessage = 'Invalid argument value provided' + + constructor(message?: string | undefined, options?: Options | Error | undefined) { + super(message, options) + } +} + +// since we're mostly using this for an artificial limit calculation +// we can live with a somewhat imprecise but fast estimate +// Js uses utf16 so the in memory string size in bytes is length * 2 +// this is just the in memory string size, not the utf-8 encoded byte size +// since our data is mostly ascii characters, its prob safe to use +// string.length is a slight underestimation of the actual size +export const estimateStringByteSize = (str: string) => str.length +export const estimateStringMegabyteSize = (str: string) => + estimateStringByteSize(str) / 1_000_000 + +export const chunkInsertionObjectArray = ({ + objects, + chunkSizeLimitMb, + chunkLengthLimit +}: { + chunkSizeLimitMb: number + chunkLengthLimit: number + objects: InsertionObject[] +}): InsertionObject[][] => { + if (chunkLengthLimit < 1) + throw new ArgumentError('Chunks must have a length limit > 1') + if (chunkSizeLimitMb <= 0) + throw new ArgumentError('Chunks must have a size in MB limit > 0') + + let currentChunkSize = 0 + let currentChunkLength = 0 + const chunkedObjects: InsertionObject[][] = [] + let currentBatch: InsertionObject[] = [] + for (const obj of objects) { + // if limits are exceeded start a new batch + if ( + currentChunkSize >= chunkSizeLimitMb || + currentChunkLength >= chunkLengthLimit + ) { + console.log( + `chunking on ${ + currentChunkSize >= chunkSizeLimitMb ? 'object size' : 'chunk length' + }` + ) + // push the current batch into the final chunks + chunkedObjects.push(currentBatch) + // reset the current batch + currentBatch = [] + // reset limits + currentChunkSize = 0 + currentChunkLength = 0 + } + // do some proper chunking here + // insert the batch to returned chunks + currentChunkLength++ + currentChunkSize += estimateStringMegabyteSize(obj.data) + currentBatch.push(obj) + } + // do not forget to push the final batch + chunkedObjects.push(currentBatch) + return chunkedObjects +} diff --git a/packages/server/modules/emails/services/sending.ts b/packages/server/modules/emails/services/sending.ts index 4ece6bdef..c84c60c4b 100644 --- a/packages/server/modules/emails/services/sending.ts +++ b/packages/server/modules/emails/services/sending.ts @@ -1,5 +1,6 @@ import { logger } from '@/logging/logging' import { getTransporter } from '@/modules/emails/utils/transporter' +import { getEmailFromAddress } from '@/modules/shared/helpers/envHelper' export type SendEmailParams = { from?: string @@ -25,7 +26,7 @@ export async function sendEmail({ return false } try { - const emailFrom = process.env.EMAIL_FROM || 'no-reply@speckle.systems' + const emailFrom = getEmailFromAddress() await transporter.sendMail({ from: from || `"Speckle" <${emailFrom}>`, to, diff --git a/packages/server/modules/shared/helpers/envHelper.ts b/packages/server/modules/shared/helpers/envHelper.ts index dfa449064..2a8621ac3 100644 --- a/packages/server/modules/shared/helpers/envHelper.ts +++ b/packages/server/modules/shared/helpers/envHelper.ts @@ -144,7 +144,7 @@ export function getFrontendOrigin(forceFe2?: boolean) { export function getServerOrigin() { if (!process.env.CANONICAL_URL) { throw new MisconfiguredEnvironmentError( - 'Server origin env var (CANONICAL_URL) not configured' + 'Server origin environment variable (CANONICAL_URL) not configured' ) } @@ -204,3 +204,12 @@ export function getOnboardingStreamCacheBustNumber() { const val = process.env.ONBOARDING_STREAM_CACHE_BUST_NUMBER || '1' return parseInt(val) || 1 } + +export function getEmailFromAddress() { + if (!process.env.EMAIL_FROM) { + throw new MisconfiguredEnvironmentError( + 'Email From environment variable (EMAIL_FROM) is not configured' + ) + } + return process.env.EMAIL_FROM +} diff --git a/packages/viewer-sandbox/src/main.ts b/packages/viewer-sandbox/src/main.ts index 778973f08..ee9428dfb 100644 --- a/packages/viewer-sandbox/src/main.ts +++ b/packages/viewer-sandbox/src/main.ts @@ -110,7 +110,7 @@ const getStream = () => { // prettier-ignore // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8?c=%5B-7.66134,10.82932,6.41935,-0.07739,-13.88552,1.8697,0,1%5D' // Revit sample house (good for bim-like stuff with many display meshes) - // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8' + 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8' // 'https://latest.speckle.dev/streams/c1faab5c62/commits/6c6e43e5f3' // 'https://latest.speckle.dev/streams/58b5648c4d/commits/60371ecb2d' // 'Super' heavy revit shit @@ -278,12 +278,13 @@ const getStream = () => { // 'https://latest.speckle.dev/streams/b68abcbf2e/commits/4e94ecad62' // Big ass mafa' // 'https://speckle.xyz/streams/88307505eb/objects/a232d760059046b81ff97e6c4530c985' - 'https://latest.speckle.dev/streams/92b620fb17/commits/dfb9ca025d' + // 'https://latest.speckle.dev/streams/92b620fb17/commits/dfb9ca025d' // 'Blocks with elements // 'https://latest.speckle.dev/streams/e258b0e8db/commits/00e165cc1c' // 'https://latest.speckle.dev/streams/e258b0e8db/commits/e48cf53add' // 'https://latest.speckle.dev/streams/e258b0e8db/commits/c19577c7d6?c=%5B15.88776,-8.2182,12.17095,18.64059,1.48552,0.6025,0,1%5D' // 'https://speckle.xyz/streams/46caea9b53/commits/71938adcd1' + // 'https://speckle.xyz/streams/2f9f2f3021/commits/75bd13f513' ) } diff --git a/packages/viewer/src/modules/filtering/FilteringManager.ts b/packages/viewer/src/modules/filtering/FilteringManager.ts index 78e327197..322ee3367 100644 --- a/packages/viewer/src/modules/filtering/FilteringManager.ts +++ b/packages/viewer/src/modules/filtering/FilteringManager.ts @@ -696,12 +696,25 @@ export class FilteringManager extends EventEmitter { const key = objectIds.join(',') if (this.idCache[key] && this.idCache[key].length) return this.idCache[key] - + /** This doesn't return descendants correctly for some streams like: + * https://speckle.xyz/streams/2f9f2f3021/commits/75bd13f513 + */ + // this.WTI.walk((node: TreeNode) => { + // if (objectIds.includes(node.model.raw.id) && node.model.raw.__closure) { + // const ids = Object.keys(node.model.raw.__closure) + // allIds.push(...ids) + // this.idCache[node.model.raw.id] = ids + // } + // return true + // }) this.WTI.walk((node: TreeNode) => { - if (objectIds.includes(node.model.raw.id) && node.model.raw.__closure) { - const ids = Object.keys(node.model.raw.__closure) - allIds.push(...ids) - this.idCache[node.model.raw.id] = ids + if (objectIds.includes(node.model.raw.id)) { + const subtree = node.all((node) => { + return node.model.raw !== undefined + }) + const idList = subtree.map((node) => node.model.raw.id) + allIds.push(...idList) + this.idCache[node.model.raw.id] = idList } return true }) diff --git a/utils/helm/speckle-server/values.schema.json b/utils/helm/speckle-server/values.schema.json index fd3555f81..6639153e4 100644 --- a/utils/helm/speckle-server/values.schema.json +++ b/utils/helm/speckle-server/values.schema.json @@ -80,7 +80,7 @@ "docker_image_tag": { "type": "string", "description": "Speckle is published as a Docker Image. The version of the image which will be deployed is specified by this tag.", - "default": "2.13.3" + "default": "2" }, "imagePullPolicy": { "type": "string", diff --git a/utils/helm/speckle-server/values.yaml b/utils/helm/speckle-server/values.yaml index c3e63b86d..c338603ed 100644 --- a/utils/helm/speckle-server/values.yaml +++ b/utils/helm/speckle-server/values.yaml @@ -65,7 +65,7 @@ ingress: ## ## @param docker_image_tag Speckle is published as a Docker Image. The version of the image which will be deployed is specified by this tag. ## -docker_image_tag: 2.13.3 +docker_image_tag: '2' ## @param imagePullPolicy Determines the conditions when the Docker Images for Speckle should be pulled from the Image registry. ## ref: https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy