From 468eaccdfa19e20729793a716dd1d2828d7ca794 Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Thu, 27 Mar 2025 18:24:04 +0000 Subject: [PATCH 1/5] feat(fileimport-service): Make .NET parser default (#4272) * first pass removing the FF * legacyimporter * removed unused import --- .../src/controller/daemon.ts | 45 +++++++++---------- packages/shared/src/environment/index.ts | 6 --- .../fileimport_service/deployment.yml | 4 -- utils/helm/speckle-server/values.schema.json | 5 --- utils/helm/speckle-server/values.yaml | 2 - 5 files changed, 21 insertions(+), 41 deletions(-) diff --git a/packages/fileimport-service/src/controller/daemon.ts b/packages/fileimport-service/src/controller/daemon.ts index 53cd84fe8..c904a1b4c 100644 --- a/packages/fileimport-service/src/controller/daemon.ts +++ b/packages/fileimport-service/src/controller/daemon.ts @@ -1,4 +1,3 @@ -import Environment from '@speckle/shared/dist/commonjs/environment/index.js' import { initPrometheusMetrics, metricDuration, @@ -18,8 +17,6 @@ import { Nullable, Scopes, wait } from '@speckle/shared' import { Knex } from 'knex' import { Logger } from 'pino' -const { FF_FILEIMPORT_IFC_DOTNET_ENABLED } = Environment.getFeatureFlags() - const HEALTHCHECK_FILE_PATH = '/tmp/last_successful_query' const TMP_INPUT_DIR = '/tmp/file_to_import' @@ -156,27 +153,7 @@ async function doTask( taskLogger.info('Triggering importer for {fileType}') if (info.fileType.toLowerCase() === 'ifc') { - if (FF_FILEIMPORT_IFC_DOTNET_ENABLED) { - await runProcessWithTimeout( - taskLogger, - process.env['DOTNET_BINARY_PATH'] || 'dotnet', - [ - process.env['IFC_DOTNET_DLL_PATH'] || - '/speckle-server/packages/fileimport-service/src/ifc-dotnet/ifc-converter.dll', - TMP_FILE_PATH, - TMP_RESULTS_PATH, - info.streamId, - `File upload: ${info.fileName}`, - existingBranch?.id || '', - info.branchName, - regionName - ], - { - USER_TOKEN: tempUserToken - }, - TIME_LIMIT - ) - } else { + if (info.fileName.toLowerCase().endsWith('.legacyimporter.ifc')) { await runProcessWithTimeout( taskLogger, process.env['NODE_BINARY_PATH'] || 'node', @@ -199,6 +176,26 @@ async function doTask( }, TIME_LIMIT ) + } else { + await runProcessWithTimeout( + taskLogger, + process.env['DOTNET_BINARY_PATH'] || 'dotnet', + [ + process.env['IFC_DOTNET_DLL_PATH'] || + '/speckle-server/packages/fileimport-service/src/ifc-dotnet/ifc-converter.dll', + TMP_FILE_PATH, + TMP_RESULTS_PATH, + info.streamId, + `File upload: ${info.fileName}`, + existingBranch?.id || '', + info.branchName, + regionName + ], + { + USER_TOKEN: tempUserToken + }, + TIME_LIMIT + ) } } else if (info.fileType.toLowerCase() === 'stl') { await runProcessWithTimeout( diff --git a/packages/shared/src/environment/index.ts b/packages/shared/src/environment/index.ts index 548beaf38..b037457ae 100644 --- a/packages/shared/src/environment/index.ts +++ b/packages/shared/src/environment/index.ts @@ -64,11 +64,6 @@ export const parseFeatureFlags = ( schema: z.boolean(), defaults: { production: false, _: false } }, - // Toggles IFC parsing with experimental .Net parser - FF_FILEIMPORT_IFC_DOTNET_ENABLED: { - schema: z.boolean(), - defaults: { production: false, _: false } - }, // Forces onboarding for all users FF_FORCE_ONBOARDING: { schema: z.boolean(), @@ -114,7 +109,6 @@ export type FeatureFlags = { FF_GATEKEEPER_FORCE_FREE_PLAN: boolean FF_BILLING_INTEGRATION_ENABLED: boolean FF_WORKSPACES_MULTI_REGION_ENABLED: boolean - FF_FILEIMPORT_IFC_DOTNET_ENABLED: boolean FF_FORCE_ONBOARDING: boolean FF_OBJECTS_STREAMING_FIX: boolean FF_MOVE_PROJECT_REGION_ENABLED: boolean diff --git a/utils/helm/speckle-server/templates/fileimport_service/deployment.yml b/utils/helm/speckle-server/templates/fileimport_service/deployment.yml index 072936587..7ec17d592 100644 --- a/utils/helm/speckle-server/templates/fileimport_service/deployment.yml +++ b/utils/helm/speckle-server/templates/fileimport_service/deployment.yml @@ -115,10 +115,6 @@ spec: - name: MULTI_REGION_CONFIG_PATH value: "/multi-region-config/multi-region-config.json" {{- end }} - {{- if .Values.featureFlags.fileImportIFCDotNetEnabled }} - - name: FF_FILEIMPORT_IFC_DOTNET_ENABLED - value: {{ .Values.featureFlags.fileImportIFCDotNetEnabled | quote }} - {{- end }} {{- if .Values.fileimport_service.affinity }} affinity: {{- include "speckle.renderTpl" (dict "value" .Values.fileimport_service.affinity "context" $) | nindent 8 }} {{- end }} diff --git a/utils/helm/speckle-server/values.schema.json b/utils/helm/speckle-server/values.schema.json index 9034b3986..95d5c4a2c 100644 --- a/utils/helm/speckle-server/values.schema.json +++ b/utils/helm/speckle-server/values.schema.json @@ -75,11 +75,6 @@ "description": "Toggles whether multi-region is available within workspaces. workspacesModuleEnabled must also be enabled.", "default": false }, - "fileImportIFCDotNetEnabled": { - "type": "boolean", - "description": "Toggles whether the experimental .Net IFC importer is used for importing IFC files.", - "default": false - }, "forceEmailVerification": { "type": "boolean", "description": "Forces email verification for all users", diff --git a/utils/helm/speckle-server/values.yaml b/utils/helm/speckle-server/values.yaml index 68a8f7661..385a91d8b 100644 --- a/utils/helm/speckle-server/values.yaml +++ b/utils/helm/speckle-server/values.yaml @@ -51,8 +51,6 @@ featureFlags: billingIntegrationEnabled: false ## @param featureFlags.workspacesMultiRegionEnabled Toggles whether multi-region is available within workspaces. workspacesModuleEnabled must also be enabled. workspacesMultiRegionEnabled: false - ## @param featureFlags.fileImportIFCDotNetEnabled Toggles whether the experimental .Net IFC importer is used for importing IFC files. - fileImportIFCDotNetEnabled: false ## @param featureFlags.forceEmailVerification Forces email verification for all users forceEmailVerification: false ## @param featureFlags.forceOnboarding Forces onboarding for all users From 3e3de6df5b8a1f6b5e81963167c87a7c2c8e49a9 Mon Sep 17 00:00:00 2001 From: Alexandru Popovici Date: Thu, 27 Mar 2025 23:36:48 +0200 Subject: [PATCH 2/5] fix(viewer-lib): When resetting the selection, the shadowmap is re-rendered. This allows correct shadows after de-selecting outline only selections. Also re-enabled stream loading in th sandbox (#4280) --- packages/viewer-sandbox/src/main.ts | 20 +++++++++++--------- packages/viewer/src/modules/LegacyViewer.ts | 4 +++- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/packages/viewer-sandbox/src/main.ts b/packages/viewer-sandbox/src/main.ts index d74662891..e2f7b881c 100644 --- a/packages/viewer-sandbox/src/main.ts +++ b/packages/viewer-sandbox/src/main.ts @@ -94,22 +94,16 @@ const createViewer = async (containerName: string, _stream: string) => { sandbox.makeDiffUI() sandbox.makeMeasurementsUI() - await sandbox.objectLoaderOnly(_stream) - // await sandbox.loadUrl(_stream) + // await sandbox.objectLoaderOnly(_stream) + await sandbox.loadUrl(_stream) // await sandbox.loadJSON(JSONSpeckleStream) } const getStream = () => { return ( // prettier-ignore - `https://latest.speckle.systems/projects/97750296c2/models/767b70fc63@5386a0af02` - //crashing out of memory? - //`https://latest.speckle.systems/projects/97750296c2/models/767b70fc63@2a6fd781f2` - //too big? - // `https://latest.speckle.systems/projects/126cd4b7bb/models/032d09f716` - // 'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8?c=%5B-7.66134,10.82932,6.41935,-0.07739,-13.88552,1.8697,0,1%5D' // Revit sample house (good for bim-like stuff with many display meshes) - //'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8' + 'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8' // 'https://latest.speckle.systems/streams/c1faab5c62/commits/ab1a1ab2b6' // 'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8' // 'https://latest.speckle.systems/streams/58b5648c4d/commits/60371ecb2d' @@ -502,6 +496,14 @@ const getStream = () => { // SUPER slow tree build time (LARGE N-GONS TRIANGULATION) // 'https://app.speckle.systems/projects/0edb6ef628/models/ff3d8480bc@cd83d90a2c' + + /* ObjectLoader 2 tests */ + // `https://latest.speckle.systems/projects/97750296c2/models/767b70fc63@5386a0af02` + //crashing out of memory? + //`https://latest.speckle.systems/projects/97750296c2/models/767b70fc63@2a6fd781f2` + //too big? + // `https://latest.speckle.systems/projects/126cd4b7bb/models/032d09f716` + // 'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8?c=%5B-7.66134,10.82932,6.41935,-0.07739,-13.88552,1.8697,0,1%5D' ) } diff --git a/packages/viewer/src/modules/LegacyViewer.ts b/packages/viewer/src/modules/LegacyViewer.ts index 4af3f9a49..01c47d5ea 100644 --- a/packages/viewer/src/modules/LegacyViewer.ts +++ b/packages/viewer/src/modules/LegacyViewer.ts @@ -23,7 +23,8 @@ import { type SpeckleView, type SunLightConfiguration, type ViewerParams, - StencilOutlineType + StencilOutlineType, + UpdateFlags } from '../IViewer.js' import { Viewer } from './Viewer.js' import { SectionTool } from './extensions/SectionTool.js' @@ -207,6 +208,7 @@ export class LegacyViewer extends Viewer { this.selection.clearSelection() if (this.filtering.filteringState.selectedObjects) this.filtering.filteringState.selectedObjects.length = 0 + this.requestRender(UpdateFlags.RENDER | UpdateFlags.SHADOWS) return Promise.resolve(this.filtering.filteringState) } From 7be2e39913d11408dab99802cff10c90c135822a Mon Sep 17 00:00:00 2001 From: Chuck Driesler Date: Fri, 28 Mar 2025 00:28:38 +0000 Subject: [PATCH 3/5] chore(tests): dramatically reduce scope/time of multi-region tests (#4282) * chore(tests): limit multiregion ci to multiregion tests * fix(multiregion): unsure if this works * fix(multiregion): ope we got deleted --- .circleci/config.yml | 9 +- .circleci/multiregion.test-ci.json | 13 + packages/server/.env-example | 3 + .../core/tests/integration/subs.graph.spec.ts | 2 +- .../tests/e2e/projects.graph.spec.ts | 363 +++++++++++++++++ .../tests/e2e/serverAdmin.graph.spec.ts | 2 +- .../repositories/projectRegion.spec.ts | 0 .../tests/integration/projects.graph.spec.ts | 364 +----------------- .../tests/integration/subs.graph.spec.ts | 2 +- packages/server/multiregion.test.example.json | 14 + packages/server/package.json | 2 +- packages/server/test/hooks.ts | 4 +- 12 files changed, 410 insertions(+), 368 deletions(-) create mode 100644 packages/server/modules/multiregion/tests/e2e/projects.graph.spec.ts rename packages/server/modules/multiregion/tests/{intergration => integration}/repositories/projectRegion.spec.ts (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index caa7d9b4e..548b7b6c8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -522,7 +522,14 @@ jobs: - run: name: 'Run tests' # Extra formatting to get timestamps on each line in CI (for profiling purposes) - command: yarn test:report --color=always | while IFS= read -r line; do echo -e "$(date +%T.%3N) > $line"; done + command: | + GREP_FLAG="" + + if [ "$RUN_TESTS_IN_MULTIREGION_MODE" == "true" ]; then + GREP_FLAG="--grep @multiregion" + fi + + yarn test:report $GREP_FLAG --color=always | while IFS= read -r line; do echo -e "$(date +%T.%3N) > $line"; done working_directory: 'packages/server' no_output_timeout: 30m diff --git a/.circleci/multiregion.test-ci.json b/.circleci/multiregion.test-ci.json index 3d5a9ec1c..78619c2af 100644 --- a/.circleci/multiregion.test-ci.json +++ b/.circleci/multiregion.test-ci.json @@ -25,6 +25,19 @@ "endpoint": "http://127.0.0.1:9020", "s3Region": "us-east-1" } + }, + "region2": { + "postgres": { + "connectionUri": "postgresql://speckle:speckle@127.0.0.1:5434/speckle2_test" + }, + "blobStorage": { + "accessKey": "minioadmin", + "secretKey": "minioadmin", + "bucket": "speckle-server", + "createBucketIfNotExists": true, + "endpoint": "http://127.0.0.1:9040", + "s3Region": "us-east-1" + } } } } diff --git a/packages/server/.env-example b/packages/server/.env-example index 9ad363a0b..5891bccee 100644 --- a/packages/server/.env-example +++ b/packages/server/.env-example @@ -10,6 +10,9 @@ PORT=3000 CANONICAL_URL="http://127.0.0.1:3000" SESSION_SECRET="-> FILL IN <-" +# Optional license token for paid features like multiregion +# LICENSE_TOKEN= + # Redis connection: default for local development environment REDIS_URL="redis://127.0.0.1:6379" diff --git a/packages/server/modules/core/tests/integration/subs.graph.spec.ts b/packages/server/modules/core/tests/integration/subs.graph.spec.ts index e284457ab..cdd70da9b 100644 --- a/packages/server/modules/core/tests/integration/subs.graph.spec.ts +++ b/packages/server/modules/core/tests/integration/subs.graph.spec.ts @@ -222,7 +222,7 @@ describe('Core GraphQL Subscriptions (New)', () => { ] modes.forEach(({ isMultiRegion }) => { - describe(`W/${!isMultiRegion ? 'o' : ''} multiregion`, () => { + describe(`W/${!isMultiRegion ? 'o' : ''} @multiregion`, () => { const myMainWorkspace: BasicTestWorkspace = { id: '', ownerId: '', diff --git a/packages/server/modules/multiregion/tests/e2e/projects.graph.spec.ts b/packages/server/modules/multiregion/tests/e2e/projects.graph.spec.ts new file mode 100644 index 000000000..bc61839a6 --- /dev/null +++ b/packages/server/modules/multiregion/tests/e2e/projects.graph.spec.ts @@ -0,0 +1,363 @@ +import { db } from '@/db/knex' +import { AutomationRecord, AutomationRunRecord } from '@/modules/automate/helpers/types' +import { CommentRecord } from '@/modules/comments/helpers/types' +import { createRandomEmail } from '@/modules/core/helpers/testHelpers' +import { StreamRecord } from '@/modules/core/helpers/types' +import { getDb } from '@/modules/multiregion/utils/dbSelector' +import { + createWebhookConfigFactory, + createWebhookEventFactory +} from '@/modules/webhooks/repositories/webhooks' +import { + BasicTestWorkspace, + createTestWorkspace +} from '@/modules/workspaces/tests/helpers/creation' +import { BasicTestUser, createTestUser } from '@/test/authHelper' +import { + UpdateProjectRegionDocument, + GetProjectDocument, + GetRegionalProjectModelDocument, + GetRegionalProjectVersionDocument, + GetRegionalProjectObjectDocument, + GetRegionalProjectAutomationDocument, + GetRegionalProjectCommentDocument, + GetRegionalProjectWebhookDocument, + GetRegionalProjectBlobDocument +} from '@/test/graphql/generated/graphql' +import { TestApolloServer, testApolloServer } from '@/test/graphqlHelper' +import { + createTestAutomation, + createTestAutomationRun +} from '@/test/speckle-helpers/automationHelper' +import { createTestBlob } from '@/test/speckle-helpers/blobHelper' +import { BasicTestBranch, createTestBranch } from '@/test/speckle-helpers/branchHelper' +import { createTestComment } from '@/test/speckle-helpers/commentHelper' +import { + BasicTestCommit, + createTestObject, + createTestCommit +} from '@/test/speckle-helpers/commitHelper' +import { + isMultiRegionTestMode, + waitForRegionUser +} from '@/test/speckle-helpers/regions' +import { BasicTestStream, createTestStream } from '@/test/speckle-helpers/streamHelper' +import { retry, Roles } from '@speckle/shared' +import { expect } from 'chai' +import cryptoRandomString from 'crypto-random-string' +import { Knex } from 'knex' +import { SetOptional } from 'type-fest' + +const tables = { + projects: (db: Knex) => db.table('streams') +} + +const assertProjectRegion = async ( + projectId: string, + regionKey: string +): Promise => { + const project = await tables.projects(db).select('*').where('id', projectId).first() + + if (!project || project.regionKey !== regionKey) { + expect.fail('Project is not in expected region.') + } +} + +const ensureProjectRegion = async ( + projectId: string, + regionKey: string +): Promise => { + await retry(async () => assertProjectRegion(projectId, regionKey), 30, 500) +} + +isMultiRegionTestMode() + ? describe('Workspace project region changes @multiregion', () => { + const regionKey1 = 'region1' + const regionKey2 = 'region2' + + const adminUser: BasicTestUser = { + id: '', + name: 'John Speckle', + email: createRandomEmail(), + role: Roles.Server.Admin + } + + const testWorkspace: SetOptional = { + id: '', + ownerId: '', + name: 'Unlimited Workspace' + } + + const testProject: BasicTestStream = { + id: '', + ownerId: '', + name: 'Regional Project', + isPublic: true + } + + const testModel: BasicTestBranch = { + id: '', + name: cryptoRandomString({ length: 8 }), + streamId: '', + authorId: '' + } + + const testVersion: BasicTestCommit = { + id: '', + objectId: '', + streamId: '', + authorId: '' + } + + let testAutomation: AutomationRecord + let testAutomationRun: AutomationRunRecord + + let testComment: CommentRecord + let testWebhookId: string + let testBlobId: string + + let apollo: TestApolloServer + let sourceRegionDb: Knex + + before(async () => { + await createTestUser(adminUser) + await waitForRegionUser(adminUser) + + apollo = await testApolloServer({ authUserId: adminUser.id }) + sourceRegionDb = await getDb({ regionKey: regionKey1 }) + }) + + beforeEach(async () => { + delete testWorkspace.slug + + await createTestWorkspace(testWorkspace, adminUser, { + regionKey: regionKey1, + addPlan: { + name: 'unlimited', + status: 'valid' + } + }) + + testProject.workspaceId = testWorkspace.id + + await createTestStream(testProject, adminUser) + await createTestBranch({ + stream: testProject, + branch: testModel, + owner: adminUser + }) + + testVersion.branchName = testModel.name + testVersion.objectId = await createTestObject({ projectId: testProject.id }) + + await createTestCommit(testVersion, { + owner: adminUser, + stream: testProject + }) + + const { automation, revision } = await createTestAutomation({ + userId: adminUser.id, + projectId: testProject.id, + revision: { + functionId: cryptoRandomString({ length: 9 }), + functionReleaseId: cryptoRandomString({ length: 9 }) + } + }) + + if (!revision) { + throw new Error('Failed to create automation revision.') + } + + testAutomation = automation.automation + + const { automationRun } = await createTestAutomationRun({ + userId: adminUser.id, + projectId: testProject.id, + automationId: testAutomation.id + }) + + testAutomationRun = automationRun + + testComment = await createTestComment({ + userId: adminUser.id, + projectId: testProject.id, + objectId: testVersion.objectId + }) + + testWebhookId = await createWebhookConfigFactory({ db: sourceRegionDb })({ + id: cryptoRandomString({ length: 9 }), + streamId: testProject.id, + url: 'https://example.org', + description: cryptoRandomString({ length: 9 }), + secret: cryptoRandomString({ length: 9 }), + enabled: false, + triggers: ['branch_create'] + }) + await createWebhookEventFactory({ db: sourceRegionDb })({ + id: cryptoRandomString({ length: 9 }), + webhookId: testWebhookId, + payload: cryptoRandomString({ length: 9 }) + }) + + const testBlob = await createTestBlob({ + userId: adminUser.id, + projectId: testProject.id + }) + testBlobId = testBlob.blobId + + await assertProjectRegion(testProject.id, regionKey1) + }) + + it('moves project record to target regional db', async () => { + const resA = await apollo.execute(UpdateProjectRegionDocument, { + projectId: testProject.id, + regionKey: regionKey2 + }) + expect(resA).to.not.haveGraphQLErrors() + + await ensureProjectRegion(testProject.id, regionKey2) + + const resB = await apollo.execute(GetProjectDocument, { + id: testProject.id + }) + expect(resB).to.not.haveGraphQLErrors() + + expect(resB.data?.project.name).to.equal(testProject.name) + }) + + it('moves project models to target regional db', async () => { + const resA = await apollo.execute(UpdateProjectRegionDocument, { + projectId: testProject.id, + regionKey: regionKey2 + }) + expect(resA).to.not.haveGraphQLErrors() + + await ensureProjectRegion(testProject.id, regionKey2) + + const resB = await apollo.execute(GetRegionalProjectModelDocument, { + projectId: testProject.id, + modelId: testModel.id + }) + expect(resB).to.not.haveGraphQLErrors() + + expect(resB.data?.project.model.name).to.equal(testModel.name) + }) + + it('moves project model versions to target regional db', async () => { + const resA = await apollo.execute(UpdateProjectRegionDocument, { + projectId: testProject.id, + regionKey: regionKey2 + }) + expect(resA).to.not.haveGraphQLErrors() + + await ensureProjectRegion(testProject.id, regionKey2) + + const resB = await apollo.execute(GetRegionalProjectVersionDocument, { + projectId: testProject.id, + modelId: testModel.id, + versionId: testVersion.id + }) + expect(resB).to.not.haveGraphQLErrors() + + expect(resB.data?.project.model.version.referencedObject).to.equal( + testVersion.objectId + ) + }) + + it('moves project version objects to target regional db', async () => { + const resA = await apollo.execute(UpdateProjectRegionDocument, { + projectId: testProject.id, + regionKey: regionKey2 + }) + expect(resA).to.not.haveGraphQLErrors() + + await ensureProjectRegion(testProject.id, regionKey2) + + const resB = await apollo.execute(GetRegionalProjectObjectDocument, { + projectId: testProject.id, + objectId: testVersion.objectId + }) + expect(resB).to.not.haveGraphQLErrors() + + expect(resB.data?.project.object).to.not.be.undefined + }) + + it('moves project automations to target regional db', async () => { + const resA = await apollo.execute(UpdateProjectRegionDocument, { + projectId: testProject.id, + regionKey: regionKey2 + }) + expect(resA).to.not.haveGraphQLErrors() + + await ensureProjectRegion(testProject.id, regionKey2) + + const resB = await apollo.execute(GetRegionalProjectAutomationDocument, { + projectId: testProject.id, + automationId: testAutomation.id + }) + expect(resB).to.not.haveGraphQLErrors() + + expect(resB.data?.project.automation.id).to.equal(testAutomation.id) + expect(resB.data?.project.automation.runs.items.at(0)?.id).to.equal( + testAutomationRun.id + ) + expect( + resB.data?.project.automation.runs.items.at(0)?.functionRuns.length + ).to.not.equal(0) + }) + + it('moves project comments to target regional db', async () => { + const resA = await apollo.execute(UpdateProjectRegionDocument, { + projectId: testProject.id, + regionKey: regionKey2 + }) + expect(resA).to.not.haveGraphQLErrors() + + await ensureProjectRegion(testProject.id, regionKey2) + + const resB = await apollo.execute(GetRegionalProjectCommentDocument, { + projectId: testProject.id, + commentId: testComment.id + }) + expect(resB).to.not.haveGraphQLErrors() + + expect(resB.data?.project.comment).to.not.be.undefined + }) + + it('moves project webhooks to target regional db', async () => { + const resA = await apollo.execute(UpdateProjectRegionDocument, { + projectId: testProject.id, + regionKey: regionKey2 + }) + expect(resA).to.not.haveGraphQLErrors() + + await ensureProjectRegion(testProject.id, regionKey2) + + const resB = await apollo.execute(GetRegionalProjectWebhookDocument, { + projectId: testProject.id, + webhookId: testWebhookId + }) + expect(resB).to.not.haveGraphQLErrors() + + expect(resB.data?.project.webhooks.items.length).to.equal(1) + }) + + it('moves project files and associated blobs to target regional db and object storage', async () => { + const resA = await apollo.execute(UpdateProjectRegionDocument, { + projectId: testProject.id, + regionKey: regionKey2 + }) + expect(resA).to.not.haveGraphQLErrors() + + await ensureProjectRegion(testProject.id, regionKey2) + + const resB = await apollo.execute(GetRegionalProjectBlobDocument, { + projectId: testProject.id, + blobId: testBlobId + }) + expect(resB).to.not.haveGraphQLErrors() + + expect(resB.data?.project.blob).to.not.be.undefined + }) + }) + : void 0 diff --git a/packages/server/modules/multiregion/tests/e2e/serverAdmin.graph.spec.ts b/packages/server/modules/multiregion/tests/e2e/serverAdmin.graph.spec.ts index 54123761e..96f56c821 100644 --- a/packages/server/modules/multiregion/tests/e2e/serverAdmin.graph.spec.ts +++ b/packages/server/modules/multiregion/tests/e2e/serverAdmin.graph.spec.ts @@ -28,7 +28,7 @@ import { expect } from 'chai' const isEnabled = isMultiRegionEnabled() isEnabled - ? describe('Multi Region Server Settings', () => { + ? describe('Multi Region Server Settings @multiregion', () => { let testAdminUser: BasicTestUser let testBasicUser: BasicTestUser let apollo: TestApolloServer diff --git a/packages/server/modules/multiregion/tests/intergration/repositories/projectRegion.spec.ts b/packages/server/modules/multiregion/tests/integration/repositories/projectRegion.spec.ts similarity index 100% rename from packages/server/modules/multiregion/tests/intergration/repositories/projectRegion.spec.ts rename to packages/server/modules/multiregion/tests/integration/repositories/projectRegion.spec.ts diff --git a/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts b/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts index 0870f40cc..6aa314b26 100644 --- a/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts +++ b/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts @@ -1,17 +1,8 @@ import { db } from '@/db/knex' -import { AutomationRecord, AutomationRunRecord } from '@/modules/automate/helpers/types' -import { CommentRecord } from '@/modules/comments/helpers/types' import { AllScopes } from '@/modules/core/helpers/mainConstants' -import { createRandomEmail } from '@/modules/core/helpers/testHelpers' -import { StreamRecord } from '@/modules/core/helpers/types' import { grantStreamPermissionsFactory } from '@/modules/core/repositories/streams' import { WorkspaceSeatType } from '@/modules/gatekeeper/domain/billing' import { getWorkspaceUserSeatsFactory } from '@/modules/gatekeeper/repositories/workspaceSeat' -import { getDb } from '@/modules/multiregion/utils/dbSelector' -import { - createWebhookConfigFactory, - createWebhookEventFactory -} from '@/modules/webhooks/repositories/webhooks' import { WorkspaceInvalidRoleError } from '@/modules/workspaces/errors/workspace' import { assignToWorkspace, @@ -28,19 +19,10 @@ import { import { ActiveUserProjectsWorkspaceDocument, CreateWorkspaceProjectDocument, - GetProjectDocument, - GetRegionalProjectAutomationDocument, - GetRegionalProjectBlobDocument, - GetRegionalProjectCommentDocument, - GetRegionalProjectModelDocument, - GetRegionalProjectObjectDocument, - GetRegionalProjectVersionDocument, - GetRegionalProjectWebhookDocument, GetWorkspaceProjectsDocument, GetWorkspaceTeamDocument, MoveProjectToWorkspaceDocument, ProjectUpdateRoleInput, - UpdateProjectRegionDocument, UpdateProjectRoleDocument, UpdateWorkspaceProjectRoleDocument } from '@/test/graphql/generated/graphql' @@ -50,57 +32,15 @@ import { TestApolloServer } from '@/test/graphqlHelper' import { beforeEachContext } from '@/test/hooks' -import { - createTestAutomation, - createTestAutomationRun -} from '@/test/speckle-helpers/automationHelper' -import { createTestBlob } from '@/test/speckle-helpers/blobHelper' -import { BasicTestBranch, createTestBranch } from '@/test/speckle-helpers/branchHelper' -import { createTestComment } from '@/test/speckle-helpers/commentHelper' -import { - BasicTestCommit, - createTestCommit, - createTestObject -} from '@/test/speckle-helpers/commitHelper' -import { - getMainTestRegionKey, - isMultiRegionTestMode, - waitForRegionUser, - waitForRegionUsers -} from '@/test/speckle-helpers/regions' import { addToStream, BasicTestStream, createTestStream, getUserStreamRole } from '@/test/speckle-helpers/streamHelper' -import { Roles, retry } from '@speckle/shared' +import { Roles } from '@speckle/shared' import { expect } from 'chai' import cryptoRandomString from 'crypto-random-string' -import { Knex } from 'knex' -import { SetOptional } from 'type-fest' - -const tables = { - projects: (db: Knex) => db.table('streams') -} - -const assertProjectRegion = async ( - projectId: string, - regionKey: string -): Promise => { - const project = await tables.projects(db).select('*').where('id', projectId).first() - - if (!project || project.regionKey !== regionKey) { - expect.fail('Project is not in expected region.') - } -} - -const ensureProjectRegion = async ( - projectId: string, - regionKey: string -): Promise => { - await retry(async () => assertProjectRegion(projectId, regionKey), 20, 10) -} const grantStreamPermissions = grantStreamPermissionsFactory({ db }) @@ -182,12 +122,6 @@ describe('Workspace project GQL CRUD', () => { createTestUser(workspaceEditor), createTestUser(workspaceMemberViewer) ]) - await waitForRegionUsers([ - serverAdminUser, - workspaceGuest, - workspaceEditor, - workspaceMemberViewer - ]) }) describeEach( @@ -213,8 +147,7 @@ describe('Workspace project GQL CRUD', () => { await createTestWorkspace(roleWorkspace, serverAdminUser, { addPlan: oldPlan ? { name: 'business', status: 'valid' } - : { name: 'pro', status: 'valid' }, - regionKey: isMultiRegionTestMode() ? getMainTestRegionKey() : undefined + : { name: 'pro', status: 'valid' } }) roleProject.workspaceId = roleWorkspace.id @@ -512,296 +445,3 @@ describe('Workspace project GQL CRUD', () => { }) }) }) - -// TODO: These are very flaky for some reason -isMultiRegionTestMode() - ? describe.skip('Workspace project region changes', () => { - const regionKey1 = 'region1' - const regionKey2 = 'region2' - - const adminUser: BasicTestUser = { - id: '', - name: 'John Speckle', - email: createRandomEmail(), - role: Roles.Server.Admin - } - - const testWorkspace: SetOptional = { - id: '', - ownerId: '', - name: 'Unlimited Workspace' - } - - const testProject: BasicTestStream = { - id: '', - ownerId: '', - name: 'Regional Project', - isPublic: true - } - - const testModel: BasicTestBranch = { - id: '', - name: cryptoRandomString({ length: 8 }), - streamId: '', - authorId: '' - } - - const testVersion: BasicTestCommit = { - id: '', - objectId: '', - streamId: '', - authorId: '' - } - - let testAutomation: AutomationRecord - let testAutomationRun: AutomationRunRecord - - let testComment: CommentRecord - let testWebhookId: string - let testBlobId: string - - let apollo: TestApolloServer - let sourceRegionDb: Knex - - before(async () => { - await createTestUser(adminUser) - await waitForRegionUser(adminUser) - - apollo = await testApolloServer({ authUserId: adminUser.id }) - sourceRegionDb = await getDb({ regionKey: regionKey1 }) - }) - - beforeEach(async () => { - delete testWorkspace.slug - - await createTestWorkspace(testWorkspace, adminUser, { - regionKey: regionKey1, - addPlan: { - name: 'unlimited', - status: 'valid' - } - }) - - testProject.workspaceId = testWorkspace.id - - await createTestStream(testProject, adminUser) - await createTestBranch({ - stream: testProject, - branch: testModel, - owner: adminUser - }) - - testVersion.branchName = testModel.name - testVersion.objectId = await createTestObject({ projectId: testProject.id }) - - await createTestCommit(testVersion, { - owner: adminUser, - stream: testProject - }) - - const { automation, revision } = await createTestAutomation({ - userId: adminUser.id, - projectId: testProject.id, - revision: { - functionId: cryptoRandomString({ length: 9 }), - functionReleaseId: cryptoRandomString({ length: 9 }) - } - }) - - if (!revision) { - throw new Error('Failed to create automation revision.') - } - - testAutomation = automation.automation - - const { automationRun } = await createTestAutomationRun({ - userId: adminUser.id, - projectId: testProject.id, - automationId: testAutomation.id - }) - - testAutomationRun = automationRun - - testComment = await createTestComment({ - userId: adminUser.id, - projectId: testProject.id, - objectId: testVersion.objectId - }) - - testWebhookId = await createWebhookConfigFactory({ db: sourceRegionDb })({ - id: cryptoRandomString({ length: 9 }), - streamId: testProject.id, - url: 'https://example.org', - description: cryptoRandomString({ length: 9 }), - secret: cryptoRandomString({ length: 9 }), - enabled: false, - triggers: ['branch_create'] - }) - await createWebhookEventFactory({ db: sourceRegionDb })({ - id: cryptoRandomString({ length: 9 }), - webhookId: testWebhookId, - payload: cryptoRandomString({ length: 9 }) - }) - - const testBlob = await createTestBlob({ - userId: adminUser.id, - projectId: testProject.id - }) - testBlobId = testBlob.blobId - - await assertProjectRegion(testProject.id, regionKey1) - }) - - it('moves project record to target regional db', async () => { - const resA = await apollo.execute(UpdateProjectRegionDocument, { - projectId: testProject.id, - regionKey: regionKey2 - }) - expect(resA).to.not.haveGraphQLErrors() - - await ensureProjectRegion(testProject.id, regionKey2) - - const resB = await apollo.execute(GetProjectDocument, { - id: testProject.id - }) - expect(resB).to.not.haveGraphQLErrors() - - expect(resB.data?.project.name).to.equal(testProject.name) - }) - - it('moves project models to target regional db', async () => { - const resA = await apollo.execute(UpdateProjectRegionDocument, { - projectId: testProject.id, - regionKey: regionKey2 - }) - expect(resA).to.not.haveGraphQLErrors() - - await ensureProjectRegion(testProject.id, regionKey2) - - const resB = await apollo.execute(GetRegionalProjectModelDocument, { - projectId: testProject.id, - modelId: testModel.id - }) - expect(resB).to.not.haveGraphQLErrors() - - expect(resB.data?.project.model.name).to.equal(testModel.name) - }) - - it('moves project model versions to target regional db', async () => { - const resA = await apollo.execute(UpdateProjectRegionDocument, { - projectId: testProject.id, - regionKey: regionKey2 - }) - expect(resA).to.not.haveGraphQLErrors() - - await ensureProjectRegion(testProject.id, regionKey2) - - const resB = await apollo.execute(GetRegionalProjectVersionDocument, { - projectId: testProject.id, - modelId: testModel.id, - versionId: testVersion.id - }) - expect(resB).to.not.haveGraphQLErrors() - - expect(resB.data?.project.model.version.referencedObject).to.equal( - testVersion.objectId - ) - }) - - it('moves project version objects to target regional db', async () => { - const resA = await apollo.execute(UpdateProjectRegionDocument, { - projectId: testProject.id, - regionKey: regionKey2 - }) - expect(resA).to.not.haveGraphQLErrors() - - await ensureProjectRegion(testProject.id, regionKey2) - - const resB = await apollo.execute(GetRegionalProjectObjectDocument, { - projectId: testProject.id, - objectId: testVersion.objectId - }) - expect(resB).to.not.haveGraphQLErrors() - - expect(resB.data?.project.object).to.not.be.undefined - }) - - it('moves project automations to target regional db', async () => { - const resA = await apollo.execute(UpdateProjectRegionDocument, { - projectId: testProject.id, - regionKey: regionKey2 - }) - expect(resA).to.not.haveGraphQLErrors() - - await ensureProjectRegion(testProject.id, regionKey2) - - const resB = await apollo.execute(GetRegionalProjectAutomationDocument, { - projectId: testProject.id, - automationId: testAutomation.id - }) - expect(resB).to.not.haveGraphQLErrors() - - expect(resB.data?.project.automation.id).to.equal(testAutomation.id) - expect(resB.data?.project.automation.runs.items.at(0)?.id).to.equal( - testAutomationRun.id - ) - expect( - resB.data?.project.automation.runs.items.at(0)?.functionRuns.length - ).to.not.equal(0) - }) - - it('moves project comments to target regional db', async () => { - const resA = await apollo.execute(UpdateProjectRegionDocument, { - projectId: testProject.id, - regionKey: regionKey2 - }) - expect(resA).to.not.haveGraphQLErrors() - - await ensureProjectRegion(testProject.id, regionKey2) - - const resB = await apollo.execute(GetRegionalProjectCommentDocument, { - projectId: testProject.id, - commentId: testComment.id - }) - expect(resB).to.not.haveGraphQLErrors() - - expect(resB.data?.project.comment).to.not.be.undefined - }) - - it('moves project webhooks to target regional db', async () => { - const resA = await apollo.execute(UpdateProjectRegionDocument, { - projectId: testProject.id, - regionKey: regionKey2 - }) - expect(resA).to.not.haveGraphQLErrors() - - await ensureProjectRegion(testProject.id, regionKey2) - - const resB = await apollo.execute(GetRegionalProjectWebhookDocument, { - projectId: testProject.id, - webhookId: testWebhookId - }) - expect(resB).to.not.haveGraphQLErrors() - - expect(resB.data?.project.webhooks.items.length).to.equal(1) - }) - - it('moves project files and associated blobs to target regional db and object storage', async () => { - const resA = await apollo.execute(UpdateProjectRegionDocument, { - projectId: testProject.id, - regionKey: regionKey2 - }) - expect(resA).to.not.haveGraphQLErrors() - - await ensureProjectRegion(testProject.id, regionKey2) - - const resB = await apollo.execute(GetRegionalProjectBlobDocument, { - projectId: testProject.id, - blobId: testBlobId - }) - expect(resB).to.not.haveGraphQLErrors() - - expect(resB.data?.project.blob).to.not.be.undefined - }) - }) - : void 0 diff --git a/packages/server/modules/workspaces/tests/integration/subs.graph.spec.ts b/packages/server/modules/workspaces/tests/integration/subs.graph.spec.ts index f3beae25e..1fbb8fe15 100644 --- a/packages/server/modules/workspaces/tests/integration/subs.graph.spec.ts +++ b/packages/server/modules/workspaces/tests/integration/subs.graph.spec.ts @@ -93,7 +93,7 @@ describe('Workspace GQL Subscriptions', () => { ] modes.forEach(({ isMultiRegion }) => { - describe(`W/${!isMultiRegion ? 'o' : ''} multiregion`, () => { + describe(`W/${!isMultiRegion ? 'o' : ''} @multiregion`, () => { const myMainWorkspace: BasicTestWorkspace = { id: '', ownerId: '', diff --git a/packages/server/multiregion.test.example.json b/packages/server/multiregion.test.example.json index 0eff18956..6fc82a924 100644 --- a/packages/server/multiregion.test.example.json +++ b/packages/server/multiregion.test.example.json @@ -27,6 +27,20 @@ "endpoint": "http://127.0.0.1:9020", "s3Region": "us-east-1" } + }, + "region2": { + "postgres": { + "connectionUri": "postgresql://speckle:speckle@127.0.0.1:5402/speckle2_test", + "privateConnectionUri": "postgresql://speckle:speckle@postgres-region2:5432/speckle2_test" + }, + "blobStorage": { + "accessKey": "minioadmin", + "secretKey": "minioadmin", + "bucket": "test-speckle-server", + "createBucketIfNotExists": true, + "endpoint": "http://127.0.0.1:9040", + "s3Region": "us-east-1" + } } } } diff --git a/packages/server/package.json b/packages/server/package.json index 56a879fa0..1d38e83a9 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -25,7 +25,7 @@ "ts-mocha": "node --require ts-node/register ./bin/mocha", "test": "cross-env NODE_ENV=test LOG_LEVEL=silent LOG_PRETTY=true yarn ts-mocha", "test:all-ff": "cross-env ENABLE_ALL_FFS=true yarn test", - "test:multiregion": "cross-env RUN_TESTS_IN_MULTIREGION_MODE=true FF_WORKSPACES_MODULE_ENABLED=true FF_WORKSPACES_MULTI_REGION_ENABLED=true yarn test", + "test:multiregion": "cross-env RUN_TESTS_IN_MULTIREGION_MODE=true FF_WORKSPACES_MODULE_ENABLED=true FF_WORKSPACES_MULTI_REGION_ENABLED=true yarn test --grep @multiregion", "test:no-ff": "cross-env DISABLE_ALL_FFS=true yarn test", "test:coverage": "cross-env NODE_ENV=test LOG_LEVEL=silent LOG_PRETTY=true nyc --reporter lcov yarn ts-mocha", "test:report": "MOCHA_FILE=reports/test-results.xml yarn test:coverage -- --reporter mocha-multi --reporter-options spec=-,mocha-junit-reporter=reports/test-results.xml", diff --git a/packages/server/test/hooks.ts b/packages/server/test/hooks.ts index d5d494063..3bb73a083 100644 --- a/packages/server/test/hooks.ts +++ b/packages/server/test/hooks.ts @@ -196,15 +196,17 @@ export const resetPubSubFactory = (deps: { db: Knex }) => async () => { rows: Array<{ pubname: string }> } + // If we do not wait, the following call occasionally fails because a replication slot is still in use. const dropSubs = async (info: SubInfo) => { + await wait(1000) await deps.db.raw( `SELECT * FROM aiven_extras.pg_alter_subscription_disable('${info.subname}');` ) - // If we do not wait, the following call occasionally fails because a replication slot is still in use. await wait(1000) await deps.db.raw( `SELECT * FROM aiven_extras.pg_drop_subscription('${info.subname}');` ) + await wait(1000) await deps.db.raw( `SELECT * FROM aiven_extras.dblink_slot_create_or_drop('${info.subconninfo}', '${info.subslotname}', 'drop');` ) From 2f8d75bc67a009ac0cc347e6190641a4240898b8 Mon Sep 17 00:00:00 2001 From: Alexandru Popovici Date: Fri, 28 Mar 2025 11:33:00 +0200 Subject: [PATCH 4/5] fix(viewer-lib): Merging indices has been broken for over two years. Fixed now (#4098) --- packages/viewer/src/modules/converter/Geometry.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/viewer/src/modules/converter/Geometry.ts b/packages/viewer/src/modules/converter/Geometry.ts index ecb324cd0..85b4d81f9 100644 --- a/packages/viewer/src/modules/converter/Geometry.ts +++ b/packages/viewer/src/modules/converter/Geometry.ts @@ -101,15 +101,16 @@ export class Geometry { for (let i = 0; i < indexAttributes.length; ++i) { const index = indexAttributes[i] - if (!index || !positionAttributes) { + const positions = positionAttributes[i] + if (!index || !positions) { throw new Error('Cannot merge geometries. Indices or positions are undefined') } for (let j = 0; j < index.length; ++j) { - mergedIndex.push(index[j] + indexOffset) + mergedIndex.push(index[j] + indexOffset / 3) } - indexOffset += positionAttributes.length + indexOffset += positions.length } return mergedIndex } From 096e06abd884e7a73ba16ed77d6180165f222af5 Mon Sep 17 00:00:00 2001 From: Alexandru Popovici Date: Fri, 28 Mar 2025 11:35:15 +0200 Subject: [PATCH 5/5] Sandbox Loading Widget (#4283) * feat(sandbox): Vibez based developement of the loading widget thing * fix(sandbox): Fixed compler errors --- packages/viewer-sandbox/index.html | 17 ++++++++++++ packages/viewer-sandbox/src/Sandbox.ts | 8 ++++-- packages/viewer-sandbox/src/main.ts | 8 ++++++ packages/viewer-sandbox/src/style.css | 38 ++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 3 deletions(-) diff --git a/packages/viewer-sandbox/index.html b/packages/viewer-sandbox/index.html index 12bd5395e..ec371dd53 100644 --- a/packages/viewer-sandbox/index.html +++ b/packages/viewer-sandbox/index.html @@ -15,6 +15,23 @@ class="relative overflow-y-scroll h-full pointer-events-none w-auto" > +
+
+ + Grayscale Image + + Color Image +
+