From ef3703028700be2de67ba278e345eebdb231cb9d Mon Sep 17 00:00:00 2001 From: Kristaps Fabians Geikins Date: Wed, 13 Dec 2023 14:31:09 +0200 Subject: [PATCH] fix(server): optimized commit download (#1913) * fix(server): optimized commit download * added test --- packages/frontend-2/plugins/001-logger.ts | 4 ++-- .../server/modules/core/services/objects.js | 6 ++++- .../server/modules/core/tests/objects.spec.js | 16 ++++++++++++++ .../cross-server-sync/services/commit.ts | 22 ++++++++++++++++--- packages/server/package.json | 1 + workspace.code-workspace | 5 ++++- 6 files changed, 47 insertions(+), 7 deletions(-) diff --git a/packages/frontend-2/plugins/001-logger.ts b/packages/frontend-2/plugins/001-logger.ts index 68639ffde..bc5318a09 100644 --- a/packages/frontend-2/plugins/001-logger.ts +++ b/packages/frontend-2/plugins/001-logger.ts @@ -52,8 +52,8 @@ export default defineNuxtPlugin(async (nuxtApp) => { speckleServerVersion, serverName, frontendType: 'frontend-2', - route: route.path, - routeDefinition: route.matched[route.matched.length - 1].path, + route: route?.path, + routeDefinition: route.matched?.[route.matched.length - 1]?.path, ...collectBrowserInfo() } } diff --git a/packages/server/modules/core/services/objects.js b/packages/server/modules/core/services/objects.js index f064e80a1..699c4a612 100644 --- a/packages/server/modules/core/services/objects.js +++ b/packages/server/modules/core/services/objects.js @@ -50,7 +50,11 @@ module.exports = { await Objects().insert(insertionObject).onConflict().ignore() if (closures.length > 0) { - await Closures().insert(closures).onConflict().ignore() + const batchSize = 10000 + while (closures.length > 0) { + const closuresBatch = closures.splice(0, batchSize) // splice so that we don't take up more memory + await Closures().insert(closuresBatch).onConflict().ignore() + } } return insertionObject.id diff --git a/packages/server/modules/core/tests/objects.spec.js b/packages/server/modules/core/tests/objects.spec.js index 6b11cfdf1..16ec921f7 100644 --- a/packages/server/modules/core/tests/objects.spec.js +++ b/packages/server/modules/core/tests/objects.spec.js @@ -2,6 +2,7 @@ /* eslint-disable camelcase */ const expect = require('chai').expect const assert = require('assert') +const { cloneDeep, times, random, padStart } = require('lodash') const { beforeEachContext } = require('@/test/hooks') const { getAnIdForThisOnePlease } = require('@/test/helpers') @@ -115,6 +116,21 @@ describe('Objects @core-objects', () => { expect(myIds).to.have.lengthOf(objCount_2) }).timeout(30000) + it(`Should create a single object w/ a ton of closures`, async () => { + const obj = { + ...cloneDeep(sampleObject), + __closure: times(200000, (i) => [ + 'testa078f8b935d3e329e9080b' + padStart(i.toString(), 6, '0'), + random(1, 10) + ]).reduce((obj, [key, value]) => { + obj[key] = value + return obj + }, {}) + } + const id = await createObject(stream.id, obj) + expect(id).to.be.ok + }) + it('Should get a single object', async () => { const obj = await getObject({ streamId: stream.id, objectId: sampleCommit.id }) expect(obj).to.not.be.null diff --git a/packages/server/modules/cross-server-sync/services/commit.ts b/packages/server/modules/cross-server-sync/services/commit.ts index 69e65a432..e5c237dde 100644 --- a/packages/server/modules/cross-server-sync/services/commit.ts +++ b/packages/server/modules/cross-server-sync/services/commit.ts @@ -4,7 +4,7 @@ import { getFrontendOrigin } from '@/modules/shared/helpers/envHelper' import { CreateCommentInput } from '@/test/graphql/generated/graphql' import { getStreamBranchByName } from '@/modules/core/repositories/branches' import { getStream, getStreamCollaborators } from '@/modules/core/repositories/streams' -import { Roles } from '@speckle/shared' +import { Roles, timeoutAt } from '@speckle/shared' import { addCommitCreatedActivity } from '@/modules/activitystream/services/commitActivity' import { createObject } from '@/modules/core/services/objects' import { getObject } from '@/modules/core/repositories/objects' @@ -518,11 +518,27 @@ const loadAllObjectsFromParent = async ( // Iterate over all objects and download them into the DB const totalObjectCount = (sourceCommit.totalChildrenCount || 0) + 1 + const batchSize = 50 + let batchPromises: Promise[] = [] let processedObjectCount = 1 + for await (const obj of objectLoader.getObjectIterator()) { const typedObj = obj as ObjectLoaderObject - logger.debug(`Processing ${obj.id} - ${processedObjectCount++}/${totalObjectCount}`) - await createNewObject(typedObj, targetStreamId, { logger }) + const work = async () => { + const id = `${obj.id} - ${processedObjectCount++}/${totalObjectCount}` + logger.debug(`Processing ${id}...`) + await Promise.race([ + createNewObject(typedObj, targetStreamId, { logger }), + timeoutAt(60 * 1000, `Object create timed out! - ${id}`) + ]) + logger.debug(`Processed! ${id}`) + } + + batchPromises.push(work()) + if (batchPromises.length >= batchSize) { + await Promise.all(batchPromises) + batchPromises = [] + } } } diff --git a/packages/server/package.json b/packages/server/package.json index 3babad638..f613fe74e 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -27,6 +27,7 @@ "lint:tsc": "tsc --noEmit", "lint:eslint": "eslint . --ext .js,.ts", "cli": "cross-env LOG_PRETTY=true NODE_ENV=development ts-node ./modules/cli/index.js", + "cli:download:commit": "cross-env LOG_PRETTY=true LOG_LEVEL=debug yarn cli download commit", "migrate": "yarn cli db migrate", "migrate:test": "cross-env NODE_ENV=test ts-node ./modules/cli/index.js db migrate", "gqlgen": "graphql-codegen --config codegen.yml", diff --git a/workspace.code-workspace b/workspace.code-workspace index 2ab309000..309c1cb98 100644 --- a/workspace.code-workspace +++ b/workspace.code-workspace @@ -77,7 +77,7 @@ "editor.snippetSuggestions": "top", "editor.formatOnSave": true, "editor.codeActionsOnSave": { - "source.fixAll.eslint": true + "source.fixAll.eslint": "explicit" }, "editor.defaultFormatter": "esbenp.prettier-vscode", "search.useParentIgnoreFiles": true, @@ -107,6 +107,9 @@ }, "[dockercompose]": { "editor.defaultFormatter": "esbenp.prettier-vscode" + }, + "[json]": { + "editor.defaultFormatter": "esbenp.prettier-vscode" } }, "extensions": {