From c382064585e67f962f240a4e6e76e24219d1be19 Mon Sep 17 00:00:00 2001 From: Chuck Driesler Date: Thu, 13 Feb 2025 14:39:23 +0000 Subject: [PATCH] feat(regions): move project branches and commits (#3843) * feat(regions): repo functions for copying project branches and commits * chore(regions): wire up move to resolver * chore(regions): successful basic test of project region change * fix(regions): sabrina carpenter please please please * fix(regions): repair multiregion test setup * chore(regions): appease ts * chore(multiregion): update test multiregion config * chore(multiregion): fix test docker config and test * chore(multiregion): use transaction * chore(multiregion): maybe this will work * fix(multiregion): drop subs synchronously * chore(multiregion): desperate test logs * chore(multiregion): somehow that worked? * chore(multiregion): add load-bearing log statement * chore(multiregion): move services * fix(multiregion): test drop waits * chore(regions): fix import * chore(regions): make test a bit more thorough for good measure * fix(regions): speed up inserts * fix(regions): ignore workspace conflict on move --- .circleci/config.yml | 10 +- .circleci/multiregion.test-ci.json | 13 ++ docker-compose-deps.yml | 28 +++ .../lib/common/generated/gql/graphql.ts | 12 + .../workspacesCore/typedefs/regions.graphql | 8 + .../modules/auth/tests/apps.graphql.spec.js | 6 - .../server/modules/auth/tests/auth.spec.js | 6 - .../modules/core/graph/generated/graphql.ts | 12 + .../graph/generated/graphql.ts | 11 + .../modules/multiregion/utils/dbSelector.ts | 13 +- .../server/modules/shared/helpers/dbHelper.ts | 4 +- .../server/modules/stats/tests/stats.spec.ts | 9 +- .../modules/webhooks/tests/webhooks.spec.js | 20 +- .../modules/workspaces/domain/operations.ts | 23 +- .../modules/workspaces/errors/regions.ts | 6 + .../workspaces/graph/resolvers/regions.ts | 56 ++++- .../workspaces/repositories/projectRegions.ts | 218 ++++++++++++++++++ .../workspaces/repositories/regions.ts | 4 +- .../workspaces/services/projectRegions.ts | 89 +++++++ .../modules/workspaces/services/regions.ts | 6 +- .../workspaces/tests/helpers/creation.ts | 4 +- .../tests/integration/projects.graph.spec.ts | 174 +++++++++++++- packages/server/multiregion.test.example.json | 14 ++ .../server/test/graphql/generated/graphql.ts | 20 ++ packages/server/test/graphql/multiRegion.ts | 12 + packages/server/test/hooks.ts | 10 +- packages/shared/src/environment/index.ts | 12 + 27 files changed, 739 insertions(+), 61 deletions(-) create mode 100644 packages/server/modules/workspaces/repositories/projectRegions.ts create mode 100644 packages/server/modules/workspaces/services/projectRegions.ts diff --git a/.circleci/config.yml b/.circleci/config.yml index 47030bcee..1ea6bcbd1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -598,16 +598,24 @@ jobs: POSTGRES_PASSWORD: speckle POSTGRES_USER: speckle command: -c 'max_connections=1000' -c 'port=5433' -c 'wal_level=logical' + - image: 'speckle/speckle-postgres' + environment: + POSTGRES_DB: speckle2_test + POSTGRES_PASSWORD: speckle + POSTGRES_USER: speckle + command: -c 'max_connections=1000' -c 'port=5434' -c 'wal_level=logical' - image: 'minio/minio' command: server /data --console-address ":9001" --address "0.0.0.0:9000" - image: 'minio/minio' command: server /data --console-address ":9021" --address "0.0.0.0:9020" + - image: 'minio/minio' + command: server /data --console-address ":9041" --address "0.0.0.0:9040" environment: # Same as test-server: NODE_ENV: test DATABASE_URL: 'postgres://speckle:speckle@127.0.0.1:5432/speckle2_test' PGDATABASE: speckle2_test - POSTGRES_MAX_CONNECTIONS_SERVER: 20 + POSTGRES_MAX_CONNECTIONS_SERVER: 50 PGUSER: speckle SESSION_SECRET: 'keyboard cat' STRATEGY_LOCAL: 'true' diff --git a/.circleci/multiregion.test-ci.json b/.circleci/multiregion.test-ci.json index 3d5a9ec1c..78619c2af 100644 --- a/.circleci/multiregion.test-ci.json +++ b/.circleci/multiregion.test-ci.json @@ -25,6 +25,19 @@ "endpoint": "http://127.0.0.1:9020", "s3Region": "us-east-1" } + }, + "region2": { + "postgres": { + "connectionUri": "postgresql://speckle:speckle@127.0.0.1:5434/speckle2_test" + }, + "blobStorage": { + "accessKey": "minioadmin", + "secretKey": "minioadmin", + "bucket": "speckle-server", + "createBucketIfNotExists": true, + "endpoint": "http://127.0.0.1:9040", + "s3Region": "us-east-1" + } } } } diff --git a/docker-compose-deps.yml b/docker-compose-deps.yml index d5068bf4e..e6e59ea02 100644 --- a/docker-compose-deps.yml +++ b/docker-compose-deps.yml @@ -34,6 +34,22 @@ services: ports: - '127.0.0.1:5401:5432' + postgres-region2: + build: + context: . + dockerfile: utils/postgres/Dockerfile + restart: always + environment: + POSTGRES_DB: speckle + POSTGRES_USER: speckle + POSTGRES_PASSWORD: speckle + volumes: + - postgres-region2-data:/var/lib/postgresql/data/ + - ./setup/db/10-docker_postgres_init.sql:/docker-entrypoint-initdb.d/10-docker_postgres_init.sql + - ./setup/db/11-docker_postgres_keycloack_init.sql:/docker-entrypoint-initdb.d/11-docker_postgres_keycloack_init.sql + ports: + - '127.0.0.1:5402:5432' + redis: image: 'redis:7-alpine' restart: always @@ -62,6 +78,16 @@ services: - '127.0.0.1:9020:9000' - '127.0.0.1:9021:9001' + minio-region2: + image: 'minio/minio' + command: server /data --console-address ":9001" + restart: always + volumes: + - minio-region2-data:/data + ports: + - '127.0.0.1:9040:9000' + - '127.0.0.1:9041:9001' + # Local OIDC provider for testing keycloak: image: quay.io/keycloak/keycloak:25.0 @@ -133,8 +159,10 @@ services: volumes: postgres-data: postgres-region1-data: + postgres-region2-data: redis-data: pgadmin-data: redis_insight-data: minio-data: minio-region1-data: + minio-region2-data: diff --git a/packages/frontend-2/lib/common/generated/gql/graphql.ts b/packages/frontend-2/lib/common/generated/gql/graphql.ts index 36ef136fa..5e0a4ab74 100644 --- a/packages/frontend-2/lib/common/generated/gql/graphql.ts +++ b/packages/frontend-2/lib/common/generated/gql/graphql.ts @@ -4587,6 +4587,11 @@ export type WorkspaceProjectInviteCreateInput = { export type WorkspaceProjectMutations = { __typename?: 'WorkspaceProjectMutations'; create: Project; + /** + * Update project region and move all regional data to new db. + * TODO: Currently performs all operations synchronously in request, should probably be scheduled. + */ + moveToRegion: Project; moveToWorkspace: Project; updateRole: Project; }; @@ -4597,6 +4602,12 @@ export type WorkspaceProjectMutationsCreateArgs = { }; +export type WorkspaceProjectMutationsMoveToRegionArgs = { + projectId: Scalars['String']['input']; + regionKey: Scalars['String']['input']; +}; + + export type WorkspaceProjectMutationsMoveToWorkspaceArgs = { projectId: Scalars['String']['input']; workspaceId: Scalars['String']['input']; @@ -8290,6 +8301,7 @@ export type WorkspacePlanFieldArgs = { } export type WorkspaceProjectMutationsFieldArgs = { create: WorkspaceProjectMutationsCreateArgs, + moveToRegion: WorkspaceProjectMutationsMoveToRegionArgs, moveToWorkspace: WorkspaceProjectMutationsMoveToWorkspaceArgs, updateRole: WorkspaceProjectMutationsUpdateRoleArgs, } diff --git a/packages/server/assets/workspacesCore/typedefs/regions.graphql b/packages/server/assets/workspacesCore/typedefs/regions.graphql index b394c8655..e61b93102 100644 --- a/packages/server/assets/workspacesCore/typedefs/regions.graphql +++ b/packages/server/assets/workspacesCore/typedefs/regions.graphql @@ -12,3 +12,11 @@ extend type WorkspaceMutations { """ setDefaultRegion(workspaceId: String!, regionKey: String!): Workspace! } + +extend type WorkspaceProjectMutations { + """ + Update project region and move all regional data to new db. + TODO: Currently performs all operations synchronously in request, should probably be scheduled. + """ + moveToRegion(projectId: String!, regionKey: String!): Project! +} diff --git a/packages/server/modules/auth/tests/apps.graphql.spec.js b/packages/server/modules/auth/tests/apps.graphql.spec.js index 6f651fde0..4e90f06f8 100644 --- a/packages/server/modules/auth/tests/apps.graphql.spec.js +++ b/packages/server/modules/auth/tests/apps.graphql.spec.js @@ -63,7 +63,6 @@ const { getServerInfoFactory } = require('@/modules/core/repositories/server') const { getEventBus } = require('@/modules/shared/services/eventBus') let sendRequest -let server const createAppToken = createAppTokenFactory({ storeApiToken: storeApiTokenFactory({ db }), @@ -128,7 +127,6 @@ describe('GraphQL @apps-api', () => { before(async () => { const ctx = await beforeEachContext() - server = ctx.server ;({ sendRequest } = await initializeTestServer(ctx)) testUser = { name: 'Dimitrie Stefanescu', @@ -157,10 +155,6 @@ describe('GraphQL @apps-api', () => { ])}` }) - after(async () => { - await server.close() - }) - let testAppId let testApp diff --git a/packages/server/modules/auth/tests/auth.spec.js b/packages/server/modules/auth/tests/auth.spec.js index 45f75cd95..67888198b 100644 --- a/packages/server/modules/auth/tests/auth.spec.js +++ b/packages/server/modules/auth/tests/auth.spec.js @@ -138,7 +138,6 @@ const expect = chai.expect let app let sendRequest -let server describe('Auth @auth', () => { describe('Local authN & authZ (token endpoints)', () => { @@ -160,7 +159,6 @@ describe('Auth @auth', () => { before(async () => { const ctx = await beforeEachContext() - server = ctx.server app = ctx.app ;({ sendRequest } = await initializeTestServer(ctx)) @@ -173,10 +171,6 @@ describe('Auth @auth', () => { ) }) - after(async () => { - await server.close() - }) - it('Should register a new user (speckle frontend)', async () => { await request(app) .post('/auth/local/register?challenge=test') diff --git a/packages/server/modules/core/graph/generated/graphql.ts b/packages/server/modules/core/graph/generated/graphql.ts index a98d8dd20..cfa220c24 100644 --- a/packages/server/modules/core/graph/generated/graphql.ts +++ b/packages/server/modules/core/graph/generated/graphql.ts @@ -4609,6 +4609,11 @@ export type WorkspaceProjectInviteCreateInput = { export type WorkspaceProjectMutations = { __typename?: 'WorkspaceProjectMutations'; create: Project; + /** + * Update project region and move all regional data to new db. + * TODO: Currently performs all operations synchronously in request, should probably be scheduled. + */ + moveToRegion: Project; moveToWorkspace: Project; updateRole: Project; }; @@ -4619,6 +4624,12 @@ export type WorkspaceProjectMutationsCreateArgs = { }; +export type WorkspaceProjectMutationsMoveToRegionArgs = { + projectId: Scalars['String']['input']; + regionKey: Scalars['String']['input']; +}; + + export type WorkspaceProjectMutationsMoveToWorkspaceArgs = { projectId: Scalars['String']['input']; workspaceId: Scalars['String']['input']; @@ -6943,6 +6954,7 @@ export type WorkspacePlanResolvers = { create?: Resolver>; + moveToRegion?: Resolver>; moveToWorkspace?: Resolver>; updateRole?: Resolver>; __isTypeOf?: IsTypeOfResolverFn; diff --git a/packages/server/modules/cross-server-sync/graph/generated/graphql.ts b/packages/server/modules/cross-server-sync/graph/generated/graphql.ts index 217626242..3888e479c 100644 --- a/packages/server/modules/cross-server-sync/graph/generated/graphql.ts +++ b/packages/server/modules/cross-server-sync/graph/generated/graphql.ts @@ -4590,6 +4590,11 @@ export type WorkspaceProjectInviteCreateInput = { export type WorkspaceProjectMutations = { __typename?: 'WorkspaceProjectMutations'; create: Project; + /** + * Update project region and move all regional data to new db. + * TODO: Currently performs all operations synchronously in request, should probably be scheduled. + */ + moveToRegion: Project; moveToWorkspace: Project; updateRole: Project; }; @@ -4600,6 +4605,12 @@ export type WorkspaceProjectMutationsCreateArgs = { }; +export type WorkspaceProjectMutationsMoveToRegionArgs = { + projectId: Scalars['String']['input']; + regionKey: Scalars['String']['input']; +}; + + export type WorkspaceProjectMutationsMoveToWorkspaceArgs = { projectId: Scalars['String']['input']; workspaceId: Scalars['String']['input']; diff --git a/packages/server/modules/multiregion/utils/dbSelector.ts b/packages/server/modules/multiregion/utils/dbSelector.ts index 69d1f4e1f..3887ba044 100644 --- a/packages/server/modules/multiregion/utils/dbSelector.ts +++ b/packages/server/modules/multiregion/utils/dbSelector.ts @@ -214,7 +214,7 @@ const setUpUserReplication = async ({ try { await from.public.raw(`CREATE PUBLICATION ${pubName} FOR TABLE users;`) } catch (err) { - if (!(err instanceof Error)) + if (!(err instanceof Error)) { throw new DatabaseError( 'Could not create publication {pubName} when setting up user replication for region {regionName}', from.public, @@ -223,7 +223,16 @@ const setUpUserReplication = async ({ info: { pubName, regionName } } ) - if (!err.message.includes('already exists')) throw err + } + + const errorMessage = err.message + + if ( + !['already exists', 'violates unique constraint'].some((message) => + errorMessage.includes(message) + ) + ) + throw err } const fromUrl = new URL( diff --git a/packages/server/modules/shared/helpers/dbHelper.ts b/packages/server/modules/shared/helpers/dbHelper.ts index f5c83f882..7252ee5bd 100644 --- a/packages/server/modules/shared/helpers/dbHelper.ts +++ b/packages/server/modules/shared/helpers/dbHelper.ts @@ -23,7 +23,7 @@ export async function* executeBatchedSelect< >( selectQuery: Knex.QueryBuilder, options?: Partial -): AsyncGenerator { +): AsyncGenerator, void, unknown> { const { batchSize = 100, trx } = options || {} if (trx) selectQuery.transacting(trx) @@ -34,7 +34,7 @@ export async function* executeBatchedSelect< let currentOffset = 0 while (hasMorePages) { const q = selectQuery.clone().offset(currentOffset) - const results = (await q) as TResult + const results = (await q) as Awaited if (!results.length) { hasMorePages = false diff --git a/packages/server/modules/stats/tests/stats.spec.ts b/packages/server/modules/stats/tests/stats.spec.ts index d06cf842a..42d2000f7 100644 --- a/packages/server/modules/stats/tests/stats.spec.ts +++ b/packages/server/modules/stats/tests/stats.spec.ts @@ -8,7 +8,6 @@ import { getTotalUserCountFactory } from '@/modules/stats/repositories/index' import { Scopes } from '@speckle/shared' -import { Server } from 'node:http' import { db } from '@/db/knex' import { createCommitByBranchIdFactory, @@ -194,8 +193,7 @@ describe('Server stats services @stats-services', function () { }) describe('Server stats api @stats-api', function () { - let server: Server, - sendRequest: Awaited>['sendRequest'] + let sendRequest: Awaited>['sendRequest'] const adminUser = { name: 'Dimitrie', @@ -233,7 +231,6 @@ describe('Server stats api @stats-api', function () { before(async function () { this.timeout(15000) const ctx = await beforeEachContext() - server = ctx.server ;({ sendRequest } = await initializeTestServer(ctx)) adminUser.id = await createUser(adminUser) @@ -263,10 +260,6 @@ describe('Server stats api @stats-api', function () { await seedDb(params) }) - after(async function () { - await server.close() - }) - it('Should not get stats if user is not admin', async () => { const res = await sendRequest(adminUser.badToken, { query: fullQuery }) expect(res.body.errors).to.exist diff --git a/packages/server/modules/webhooks/tests/webhooks.spec.js b/packages/server/modules/webhooks/tests/webhooks.spec.js index 0c8001a16..0befc5da0 100644 --- a/packages/server/modules/webhooks/tests/webhooks.spec.js +++ b/packages/server/modules/webhooks/tests/webhooks.spec.js @@ -2,11 +2,7 @@ const expect = require('chai').expect const assert = require('assert') -const { - beforeEachContext, - initializeTestServer, - truncateTables -} = require('@/test/hooks') +const { beforeEachContext, initializeTestServer } = require('@/test/hooks') const { noErrors } = require('@/test/helpers') const { Scopes, Roles } = require('@speckle/shared') const { @@ -26,7 +22,6 @@ const { deleteWebhookFactory, dispatchStreamEventFactory } = require('@/modules/webhooks/services/webhooks') -const { Users, Streams } = require('@/modules/core/dbSchema') const { getStreamFactory, createStreamFactory, @@ -166,7 +161,7 @@ const createPersonalAccessToken = createPersonalAccessTokenFactory({ describe('Webhooks @webhooks', () => { const getWebhook = getWebhookByIdFactory({ db }) - let server, sendRequest + let sendRequest const userOne = { name: 'User', @@ -191,7 +186,6 @@ describe('Webhooks @webhooks', () => { before(async () => { const ctx = await beforeEachContext() - server = ctx.server ;({ sendRequest } = await initializeTestServer(ctx)) userOne.id = await createUser(userOne) @@ -201,16 +195,6 @@ describe('Webhooks @webhooks', () => { webhookOne.streamId = streamOne.id }) - after(async () => { - await truncateTables([ - Users.name, - Streams.name, - 'webhooks_config', - 'webhooks_events' - ]) - await server.close() - }) - describe('Create, Read, Update, Delete Webhooks', () => { it('Should create a webhook', async () => { webhookOne.id = await createWebhookFactory({ diff --git a/packages/server/modules/workspaces/domain/operations.ts b/packages/server/modules/workspaces/domain/operations.ts index 614ffa36d..66f0983c3 100644 --- a/packages/server/modules/workspaces/domain/operations.ts +++ b/packages/server/modules/workspaces/domain/operations.ts @@ -283,7 +283,7 @@ export type GetAvailableRegions = (params: { workspaceId: string }) => Promise -export type AssignRegion = (params: { +export type AssignWorkspaceRegion = (params: { workspaceId: string regionKey: string }) => Promise @@ -342,3 +342,24 @@ export type ApproveWorkspaceJoinRequest = ( export type DenyWorkspaceJoinRequest = ( params: Pick ) => Promise + +/** + * Project regions + */ + +/** + * Updates project region and moves all regional data to target regional db + */ +export type UpdateProjectRegion = (params: { + projectId: string + regionKey: string +}) => Promise + +export type CopyWorkspace = (params: { workspaceId: string }) => Promise +export type CopyProjects = (params: { projectIds: string[] }) => Promise +export type CopyProjectModels = (params: { + projectIds: string[] +}) => Promise> +export type CopyProjectVersions = (params: { + projectIds: string[] +}) => Promise> diff --git a/packages/server/modules/workspaces/errors/regions.ts b/packages/server/modules/workspaces/errors/regions.ts index 60ce6774e..5a73b52bc 100644 --- a/packages/server/modules/workspaces/errors/regions.ts +++ b/packages/server/modules/workspaces/errors/regions.ts @@ -5,3 +5,9 @@ export class WorkspaceRegionAssignmentError extends BaseError { static code = 'WORKSPACE_REGION_ASSIGNMENT_ERROR' static statusCode = 400 } + +export class ProjectRegionAssignmentError extends BaseError { + static defaultMessage = 'Failed to assign region to project' + static code = 'PROJECT_REGION_ASSIGNMENT_ERROR' + static statusCode = 400 +} diff --git a/packages/server/modules/workspaces/graph/resolvers/regions.ts b/packages/server/modules/workspaces/graph/resolvers/regions.ts index a80df9e94..bb01c8195 100644 --- a/packages/server/modules/workspaces/graph/resolvers/regions.ts +++ b/packages/server/modules/workspaces/graph/resolvers/regions.ts @@ -2,22 +2,37 @@ import { db } from '@/db/knex' import { Resolvers } from '@/modules/core/graph/generated/graphql' import { getWorkspacePlanFactory } from '@/modules/gatekeeper/repositories/billing' import { canWorkspaceUseRegionsFactory } from '@/modules/gatekeeper/services/featureAuthorization' -import { getDb } from '@/modules/multiregion/utils/dbSelector' +import { getDb, getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' import { getRegionsFactory } from '@/modules/multiregion/repositories' import { authorizeResolver } from '@/modules/shared' import { getDefaultRegionFactory, upsertRegionAssignmentFactory } from '@/modules/workspaces/repositories/regions' +import { + copyProjectModelsFactory, + copyProjectsFactory, + copyProjectVersionsFactory, + copyWorkspaceFactory +} from '@/modules/workspaces/repositories/projectRegions' import { getWorkspaceFactory, upsertWorkspaceFactory } from '@/modules/workspaces/repositories/workspaces' import { - assignRegionFactory, + assignWorkspaceRegionFactory, getAvailableRegionsFactory } from '@/modules/workspaces/services/regions' +import { updateProjectRegionFactory } from '@/modules/workspaces/services/projectRegions' import { Roles } from '@speckle/shared' +import { getProjectFactory } from '@/modules/core/repositories/projects' +import { getStreamBranchCountFactory } from '@/modules/core/repositories/branches' +import { getStreamCommitCountFactory } from '@/modules/core/repositories/commits' +import { withTransaction } from '@/modules/shared/helpers/dbHelper' +import { getFeatureFlags, isTestEnv } from '@/modules/shared/helpers/envHelper' +import { WorkspacesNotYetImplementedError } from '@/modules/workspaces/errors/workspace' + +const { FF_MOVE_PROJECT_REGION_ENABLED } = getFeatureFlags() export default { Workspace: { @@ -37,7 +52,7 @@ export default { const regionDb = await getDb({ regionKey: args.regionKey }) - const assignRegion = assignRegionFactory({ + const assignRegion = assignWorkspaceRegionFactory({ getAvailableRegions: getAvailableRegionsFactory({ getRegions: getRegionsFactory({ db }), canWorkspaceUseRegions: canWorkspaceUseRegionsFactory({ @@ -53,5 +68,40 @@ export default { return await ctx.loaders.workspaces!.getWorkspace.load(args.workspaceId) } + }, + WorkspaceProjectMutations: { + moveToRegion: async (_parent, args, context) => { + if (!FF_MOVE_PROJECT_REGION_ENABLED && !isTestEnv()) { + throw new WorkspacesNotYetImplementedError() + } + + await authorizeResolver( + context.userId, + args.projectId, + Roles.Stream.Owner, + context.resourceAccessRules + ) + + const sourceDb = await getProjectDbClient({ projectId: args.projectId }) + const targetDb = await (await getDb({ regionKey: args.regionKey })).transaction() + + const updateProjectRegion = updateProjectRegionFactory({ + getProject: getProjectFactory({ db: sourceDb }), + countProjectModels: getStreamBranchCountFactory({ db: sourceDb }), + countProjectVersions: getStreamCommitCountFactory({ db: sourceDb }), + getAvailableRegions: getAvailableRegionsFactory({ + getRegions: getRegionsFactory({ db }), + canWorkspaceUseRegions: canWorkspaceUseRegionsFactory({ + getWorkspacePlan: getWorkspacePlanFactory({ db }) + }) + }), + copyWorkspace: copyWorkspaceFactory({ sourceDb, targetDb }), + copyProjects: copyProjectsFactory({ sourceDb, targetDb }), + copyProjectModels: copyProjectModelsFactory({ sourceDb, targetDb }), + copyProjectVersions: copyProjectVersionsFactory({ sourceDb, targetDb }) + }) + + return await withTransaction(updateProjectRegion(args), targetDb) + } } } as Resolvers diff --git a/packages/server/modules/workspaces/repositories/projectRegions.ts b/packages/server/modules/workspaces/repositories/projectRegions.ts new file mode 100644 index 000000000..6e1e021ef --- /dev/null +++ b/packages/server/modules/workspaces/repositories/projectRegions.ts @@ -0,0 +1,218 @@ +import { + BranchCommits, + Branches, + Commits, + StreamCommits, + StreamFavorites, + Streams, + StreamsMeta +} from '@/modules/core/dbSchema' +import { Branch } from '@/modules/core/domain/branches/types' +import { Commit } from '@/modules/core/domain/commits/types' +import { Stream } from '@/modules/core/domain/streams/types' +import { + BranchCommitRecord, + CommitRecord, + StreamCommitRecord, + StreamFavoriteRecord, + StreamRecord +} from '@/modules/core/helpers/types' +import { executeBatchedSelect } from '@/modules/shared/helpers/dbHelper' +import { + CopyProjectModels, + CopyProjects, + CopyProjectVersions, + CopyWorkspace +} from '@/modules/workspaces/domain/operations' +import { WorkspaceNotFoundError } from '@/modules/workspaces/errors/workspace' +import { Knex } from 'knex' +import { Workspace } from '@/modules/workspacesCore/domain/types' +import { Workspaces } from '@/modules/workspacesCore/helpers/db' + +const tables = { + workspaces: (db: Knex) => db(Workspaces.name), + projects: (db: Knex) => db(Streams.name), + models: (db: Knex) => db(Branches.name), + versions: (db: Knex) => db(Commits.name), + branchCommits: (db: Knex) => db(BranchCommits.name), + streamCommits: (db: Knex) => db(StreamCommits.name), + streamFavorites: (db: Knex) => db(StreamFavorites.name), + streamsMeta: (db: Knex) => db(StreamsMeta.name) +} + +export const copyWorkspaceFactory = + (deps: { sourceDb: Knex; targetDb: Knex }): CopyWorkspace => + async ({ workspaceId }) => { + const workspace = await tables + .workspaces(deps.sourceDb) + .select('*') + .where({ id: workspaceId }) + + if (!workspace) { + throw new WorkspaceNotFoundError() + } + + await tables + .workspaces(deps.targetDb) + .insert(workspace) + .onConflict(Workspaces.withoutTablePrefix.col.id) + .ignore() + + return workspaceId + } + +export const copyProjectsFactory = + (deps: { sourceDb: Knex; targetDb: Knex }): CopyProjects => + async ({ projectIds }) => { + const selectProjects = tables + .projects(deps.sourceDb) + .select('*') + .whereIn(Streams.col.id, projectIds) + const copiedProjectIds: string[] = [] + + // Copy project record + for await (const projects of executeBatchedSelect(selectProjects)) { + const projectIds = projects.map((project) => project.id) + copiedProjectIds.push(...projectIds) + + // Copy `streams` rows to target db + await tables + .projects(deps.targetDb) + .insert(projects) + .onConflict(Streams.withoutTablePrefix.col.id) + .merge(Streams.withoutTablePrefix.cols as (keyof StreamRecord)[]) + + // Fetch `stream_favorites` rows for projects in batch + const selectStreamFavorites = tables + .streamFavorites(deps.sourceDb) + .select('*') + .whereIn(StreamFavorites.col.streamId, projectIds) + + for await (const streamFavorites of executeBatchedSelect(selectStreamFavorites)) { + // Copy `stream_favorites` rows to target db + await tables + .streamFavorites(deps.targetDb) + .insert(streamFavorites) + .onConflict() + .ignore() + } + + // Fetch `streams_meta` rows for projects in batch + const selectStreamsMetadata = tables + .streamsMeta(deps.sourceDb) + .select('*') + .whereIn(StreamsMeta.col.streamId, projectIds) + + for await (const streamsMetadataBatch of executeBatchedSelect( + selectStreamsMetadata + )) { + // Copy `streams_meta` rows to target db + await tables + .streamsMeta(deps.targetDb) + .insert(streamsMetadataBatch) + .onConflict() + .ignore() + } + } + + return copiedProjectIds + } + +export const copyProjectModelsFactory = + (deps: { sourceDb: Knex; targetDb: Knex }): CopyProjectModels => + async ({ projectIds }) => { + const copiedModelCountByProjectId: Record = {} + + // Fetch `branches` rows for projects in batch + const selectModels = tables + .models(deps.sourceDb) + .select('*') + .whereIn(Branches.col.streamId, projectIds) + + for await (const models of executeBatchedSelect(selectModels)) { + // Copy `branches` rows to target db + await tables.models(deps.targetDb).insert(models).onConflict().ignore() + + for (const model of models) { + copiedModelCountByProjectId[model.streamId] ??= 0 + copiedModelCountByProjectId[model.streamId]++ + } + } + + return copiedModelCountByProjectId + } + +export const copyProjectVersionsFactory = + (deps: { sourceDb: Knex; targetDb: Knex }): CopyProjectVersions => + async ({ projectIds }) => { + const copiedVersionCountByProjectId: Record = {} + + const selectVersions = tables + .streamCommits(deps.sourceDb) + .select('*') + .join( + Commits.name, + Commits.col.id, + StreamCommits.col.commitId + ) + .whereIn(StreamCommits.col.streamId, projectIds) + + for await (const versions of executeBatchedSelect(selectVersions)) { + const { commitIds, commits } = versions.reduce( + (all, version) => { + const { commitId, streamId, ...commit } = version + + all.commitIds.push(commitId) + all.streamIds.push(streamId) + all.commits.push(commit) + + return all + }, + { commitIds: [], streamIds: [], commits: [] } as { + commitIds: string[] + streamIds: string[] + commits: CommitRecord[] + } + ) + + // Copy `commits` rows to target db + await tables.versions(deps.targetDb).insert(commits).onConflict().ignore() + + for (const version of versions) { + copiedVersionCountByProjectId[version.streamId] ??= 0 + copiedVersionCountByProjectId[version.streamId]++ + } + + // Fetch `branch_commits` rows for versions in batch + const selectBranchCommits = tables + .branchCommits(deps.sourceDb) + .select('*') + .whereIn(BranchCommits.col.commitId, commitIds) + + for await (const branchCommits of executeBatchedSelect(selectBranchCommits)) { + // Copy `branch_commits` row to target db + await tables + .branchCommits(deps.targetDb) + .insert(branchCommits) + .onConflict() + .ignore() + } + + // Fetch `stream_commits` rows for versions in batch + const selectStreamCommits = tables + .streamCommits(deps.sourceDb) + .select('*') + .whereIn(StreamCommits.col.commitId, commitIds) + + for await (const streamCommits of executeBatchedSelect(selectStreamCommits)) { + // Copy `stream_commits` row to target db + await tables + .streamCommits(deps.targetDb) + .insert(streamCommits) + .onConflict() + .ignore() + } + } + + return copiedVersionCountByProjectId + } diff --git a/packages/server/modules/workspaces/repositories/regions.ts b/packages/server/modules/workspaces/repositories/regions.ts index 20c267c62..dc9563829 100644 --- a/packages/server/modules/workspaces/repositories/regions.ts +++ b/packages/server/modules/workspaces/repositories/regions.ts @@ -14,8 +14,8 @@ export const WorkspaceRegions = buildTableHelper('workspace_regions', [ ]) const tables = { - workspaceRegions: (db: Knex) => db(WorkspaceRegions.name), - regions: (db: Knex) => db(Regions.name) + regions: (db: Knex) => db(Regions.name), + workspaceRegions: (db: Knex) => db(WorkspaceRegions.name) } export const upsertRegionAssignmentFactory = diff --git a/packages/server/modules/workspaces/services/projectRegions.ts b/packages/server/modules/workspaces/services/projectRegions.ts new file mode 100644 index 000000000..2bc60cd66 --- /dev/null +++ b/packages/server/modules/workspaces/services/projectRegions.ts @@ -0,0 +1,89 @@ +import { GetStreamBranchCount } from '@/modules/core/domain/branches/operations' +import { GetStreamCommitCount } from '@/modules/core/domain/commits/operations' +import { GetProject } from '@/modules/core/domain/projects/operations' +import { + CopyProjectModels, + CopyProjects, + CopyProjectVersions, + CopyWorkspace, + GetAvailableRegions, + UpdateProjectRegion +} from '@/modules/workspaces/domain/operations' +import { ProjectRegionAssignmentError } from '@/modules/workspaces/errors/regions' + +export const updateProjectRegionFactory = + (deps: { + getProject: GetProject + countProjectModels: GetStreamBranchCount + countProjectVersions: GetStreamCommitCount + getAvailableRegions: GetAvailableRegions + copyWorkspace: CopyWorkspace + copyProjects: CopyProjects + copyProjectModels: CopyProjectModels + copyProjectVersions: CopyProjectVersions + }): UpdateProjectRegion => + async (params) => { + const { projectId, regionKey } = params + + const project = await deps.getProject({ projectId }) + if (!project) { + throw new ProjectRegionAssignmentError('Project not found', { + info: { params } + }) + } + if (!project.workspaceId) { + throw new ProjectRegionAssignmentError('Project not a part of a workspace', { + info: { params } + }) + } + + const availableRegions = await deps.getAvailableRegions({ + workspaceId: project.workspaceId + }) + if (!availableRegions.find((region) => region.key === regionKey)) { + throw new ProjectRegionAssignmentError( + 'Specified region not available for workspace', + { + info: { + params, + workspaceId: project.workspaceId + } + } + ) + } + + // Move workspace + await deps.copyWorkspace({ workspaceId: project.workspaceId }) + + // Move commits + const projectIds = await deps.copyProjects({ projectIds: [projectId] }) + const modelIds = await deps.copyProjectModels({ projectIds }) + const versionIds = await deps.copyProjectVersions({ projectIds }) + + // TODO: Move objects + // TODO: Move automations + // TODO: Move comments + // TODO: Move file blobs + // TODO: Move webhooks + + // TODO: Validate state after move captures latest state of project + const sourceProjectModelCount = await deps.countProjectModels(projectId) + const sourceProjectVersionCount = await deps.countProjectVersions(projectId) + + const tests = [ + modelIds[projectId] === sourceProjectModelCount, + versionIds[projectId] === sourceProjectVersionCount + ] + + const isReconciled = tests.every((test) => !!test) + + if (!isReconciled) { + // TODO: Move failed or source project added data while changing regions. Retry move. + throw new ProjectRegionAssignmentError( + 'Missing data from source project in target region copy after move.' + ) + } + + // TODO: Update project region in db + return { ...project, regionKey } + } diff --git a/packages/server/modules/workspaces/services/regions.ts b/packages/server/modules/workspaces/services/regions.ts index 6d5c9cb22..88623c067 100644 --- a/packages/server/modules/workspaces/services/regions.ts +++ b/packages/server/modules/workspaces/services/regions.ts @@ -1,7 +1,7 @@ import { WorkspaceFeatureAccessFunction } from '@/modules/gatekeeper/domain/operations' import { GetRegions } from '@/modules/multiregion/domain/operations' import { - AssignRegion, + AssignWorkspaceRegion, GetAvailableRegions, GetDefaultRegion, GetWorkspace, @@ -25,14 +25,14 @@ export const getAvailableRegionsFactory = return await deps.getRegions() } -export const assignRegionFactory = +export const assignWorkspaceRegionFactory = (deps: { getAvailableRegions: GetAvailableRegions upsertRegionAssignment: UpsertRegionAssignment getDefaultRegion: GetDefaultRegion getWorkspace: GetWorkspace insertRegionWorkspace: UpsertWorkspace - }): AssignRegion => + }): AssignWorkspaceRegion => async (params) => { const { workspaceId, regionKey } = params diff --git a/packages/server/modules/workspaces/tests/helpers/creation.ts b/packages/server/modules/workspaces/tests/helpers/creation.ts index c93c750d9..90062488d 100644 --- a/packages/server/modules/workspaces/tests/helpers/creation.ts +++ b/packages/server/modules/workspaces/tests/helpers/creation.ts @@ -64,7 +64,7 @@ import { import { SetOptional } from 'type-fest' import { isMultiRegionTestMode } from '@/test/speckle-helpers/regions' import { - assignRegionFactory, + assignWorkspaceRegionFactory, getAvailableRegionsFactory } from '@/modules/workspaces/services/regions' import { getRegionsFactory } from '@/modules/multiregion/repositories' @@ -184,7 +184,7 @@ export const createTestWorkspace = async ( if (useRegion) { const regionDb = await getDb({ regionKey }) - const assignRegion = assignRegionFactory({ + const assignRegion = assignWorkspaceRegionFactory({ getAvailableRegions: getAvailableRegionsFactory({ getRegions: getRegionsFactory({ db }), canWorkspaceUseRegions: canWorkspaceUseRegionsFactory({ diff --git a/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts b/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts index d9c66f859..985e1c6ee 100644 --- a/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts +++ b/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts @@ -1,6 +1,15 @@ import { db } from '@/db/knex' import { AllScopes } from '@/modules/core/helpers/mainConstants' +import { createRandomEmail } from '@/modules/core/helpers/testHelpers' +import { + BranchCommitRecord, + BranchRecord, + CommitRecord, + StreamCommitRecord, + StreamRecord +} from '@/modules/core/helpers/types' import { grantStreamPermissionsFactory } from '@/modules/core/repositories/streams' +import { getDb } from '@/modules/multiregion/utils/dbSelector' import { BasicTestWorkspace, createTestWorkspace @@ -8,6 +17,7 @@ import { import { BasicTestUser, createAuthTokenForUser, + createTestUser, createTestUsers } from '@/test/authHelper' import { @@ -15,7 +25,8 @@ import { CreateWorkspaceProjectDocument, GetWorkspaceProjectsDocument, GetWorkspaceTeamDocument, - MoveProjectToWorkspaceDocument + MoveProjectToWorkspaceDocument, + UpdateProjectRegionDocument } from '@/test/graphql/generated/graphql' import { createTestContext, @@ -23,10 +34,22 @@ import { TestApolloServer } from '@/test/graphqlHelper' import { beforeEachContext } from '@/test/hooks' +import { BasicTestBranch, createTestBranch } from '@/test/speckle-helpers/branchHelper' +import { + BasicTestCommit, + createTestCommit, + createTestObject +} from '@/test/speckle-helpers/commitHelper' +import { + isMultiRegionTestMode, + waitForRegionUser +} from '@/test/speckle-helpers/regions' import { BasicTestStream, createTestStream } from '@/test/speckle-helpers/streamHelper' import { Roles } from '@speckle/shared' import { expect } from 'chai' import cryptoRandomString from 'crypto-random-string' +import { Knex } from 'knex' +import { SetOptional } from 'type-fest' const grantStreamPermissions = grantStreamPermissionsFactory({ db }) @@ -272,3 +295,152 @@ describe('Workspace project GQL CRUD', () => { }) }) }) + +isMultiRegionTestMode() + ? describe('Workspace project region changes', () => { + const regionKey1 = 'region1' + const regionKey2 = 'region2' + + const adminUser: BasicTestUser = { + id: '', + name: 'John Speckle', + email: createRandomEmail() + } + + const testWorkspace: SetOptional = { + id: '', + ownerId: '', + name: 'Unlimited Workspace' + } + + const testProject: BasicTestStream = { + id: '', + ownerId: '', + name: 'Regional Project', + isPublic: true + } + + const testModel: BasicTestBranch = { + id: '', + name: cryptoRandomString({ length: 8 }), + streamId: '', + authorId: '' + } + + const testVersion: BasicTestCommit = { + id: '', + objectId: '', + streamId: '', + authorId: '' + } + + let apollo: TestApolloServer + let targetRegionDb: Knex + + before(async () => { + await createTestUser(adminUser) + await waitForRegionUser(adminUser) + + apollo = await testApolloServer({ authUserId: adminUser.id }) + targetRegionDb = await getDb({ regionKey: regionKey2 }) + }) + + beforeEach(async () => { + delete testWorkspace.slug + + await createTestWorkspace(testWorkspace, adminUser, { + regionKey: regionKey1, + addPlan: { + name: 'unlimited', + status: 'valid' + } + }) + + testProject.workspaceId = testWorkspace.id + + await createTestStream(testProject, adminUser) + await createTestBranch({ + stream: testProject, + branch: testModel, + owner: adminUser + }) + + testVersion.branchName = testModel.name + testVersion.objectId = await createTestObject({ projectId: testProject.id }) + + await createTestCommit(testVersion, { + owner: adminUser, + stream: testProject + }) + }) + + it('moves project record to target regional db', async () => { + const res = await apollo.execute(UpdateProjectRegionDocument, { + projectId: testProject.id, + regionKey: regionKey2 + }) + + expect(res).to.not.haveGraphQLErrors() + + // TODO: Replace with gql query when possible + const project = await targetRegionDb + .table('streams') + .select('*') + .where({ id: testProject.id }) + .first() + + expect(project).to.not.be.undefined + }) + + it('moves project models to target regional db', async () => { + const res = await apollo.execute(UpdateProjectRegionDocument, { + projectId: testProject.id, + regionKey: regionKey2 + }) + + expect(res).to.not.haveGraphQLErrors() + + // TODO: Replace with gql query when possible + const branch = await targetRegionDb + .table('branches') + .select('*') + .where({ id: testModel.id }) + .first() + + expect(branch).to.not.be.undefined + }) + + it('moves project model versions to target regional db', async () => { + const res = await apollo.execute(UpdateProjectRegionDocument, { + projectId: testProject.id, + regionKey: regionKey2 + }) + + expect(res).to.not.haveGraphQLErrors() + + // TODO: Replace with gql query when possible + const version = await targetRegionDb + .table('commits') + .select('*') + .where({ id: testVersion.id }) + .first() + expect(version).to.not.be.undefined + + // TODO: Replace with gql query when possible + const streamCommitsRecord = await targetRegionDb + .table('stream_commits') + .select('*') + .where({ commitId: testVersion.id }) + .first() + expect(streamCommitsRecord).to.not.be.undefined + + // TODO: Replace with gql query when possible + const branchCommitsRecord = await targetRegionDb + .table('branch_commits') + .select('*') + .where({ commitId: testVersion.id }) + .first() + expect(branchCommitsRecord).to.not.be.undefined + }) + }) + : void 0 diff --git a/packages/server/multiregion.test.example.json b/packages/server/multiregion.test.example.json index 0eff18956..6fc82a924 100644 --- a/packages/server/multiregion.test.example.json +++ b/packages/server/multiregion.test.example.json @@ -27,6 +27,20 @@ "endpoint": "http://127.0.0.1:9020", "s3Region": "us-east-1" } + }, + "region2": { + "postgres": { + "connectionUri": "postgresql://speckle:speckle@127.0.0.1:5402/speckle2_test", + "privateConnectionUri": "postgresql://speckle:speckle@postgres-region2:5432/speckle2_test" + }, + "blobStorage": { + "accessKey": "minioadmin", + "secretKey": "minioadmin", + "bucket": "test-speckle-server", + "createBucketIfNotExists": true, + "endpoint": "http://127.0.0.1:9040", + "s3Region": "us-east-1" + } } } } diff --git a/packages/server/test/graphql/generated/graphql.ts b/packages/server/test/graphql/generated/graphql.ts index b2d31ffcb..be9dada4a 100644 --- a/packages/server/test/graphql/generated/graphql.ts +++ b/packages/server/test/graphql/generated/graphql.ts @@ -4591,6 +4591,11 @@ export type WorkspaceProjectInviteCreateInput = { export type WorkspaceProjectMutations = { __typename?: 'WorkspaceProjectMutations'; create: Project; + /** + * Update project region and move all regional data to new db. + * TODO: Currently performs all operations synchronously in request, should probably be scheduled. + */ + moveToRegion: Project; moveToWorkspace: Project; updateRole: Project; }; @@ -4601,6 +4606,12 @@ export type WorkspaceProjectMutationsCreateArgs = { }; +export type WorkspaceProjectMutationsMoveToRegionArgs = { + projectId: Scalars['String']['input']; + regionKey: Scalars['String']['input']; +}; + + export type WorkspaceProjectMutationsMoveToWorkspaceArgs = { projectId: Scalars['String']['input']; workspaceId: Scalars['String']['input']; @@ -5209,6 +5220,14 @@ export type UpdateRegionMutationVariables = Exact<{ export type UpdateRegionMutation = { __typename?: 'Mutation', serverInfoMutations: { __typename?: 'ServerInfoMutations', multiRegion: { __typename?: 'ServerRegionMutations', update: { __typename?: 'ServerRegionItem', id: string, key: string, name: string, description?: string | null } } } }; +export type UpdateProjectRegionMutationVariables = Exact<{ + projectId: Scalars['String']['input']; + regionKey: Scalars['String']['input']; +}>; + + +export type UpdateProjectRegionMutation = { __typename?: 'Mutation', workspaceMutations: { __typename?: 'WorkspaceMutations', projects: { __typename?: 'WorkspaceProjectMutations', moveToRegion: { __typename?: 'Project', id: string } } } }; + export type BasicProjectAccessRequestFieldsFragment = { __typename?: 'ProjectAccessRequest', id: string, requesterId: string, projectId: string, createdAt: string, requester: { __typename?: 'LimitedUser', id: string, name: string } }; export type CreateProjectAccessRequestMutationVariables = Exact<{ @@ -5751,6 +5770,7 @@ export const GetAvailableRegionKeysDocument = {"kind":"Document","definitions":[ export const CreateNewRegionDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"CreateNewRegion"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"CreateServerRegionInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"serverInfoMutations"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"multiRegion"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"create"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"MainRegionMetadata"}}]}}]}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"MainRegionMetadata"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"ServerRegionItem"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"key"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"description"}}]}}]} as unknown as DocumentNode; export const GetRegionsDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"GetRegions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"serverInfo"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"multiRegion"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"regions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"MainRegionMetadata"}}]}}]}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"MainRegionMetadata"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"ServerRegionItem"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"key"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"description"}}]}}]} as unknown as DocumentNode; export const UpdateRegionDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"UpdateRegion"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"UpdateServerRegionInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"serverInfoMutations"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"multiRegion"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"update"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"MainRegionMetadata"}}]}}]}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"MainRegionMetadata"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"ServerRegionItem"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"key"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"description"}}]}}]} as unknown as DocumentNode; +export const UpdateProjectRegionDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"UpdateProjectRegion"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"projectId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"regionKey"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"workspaceMutations"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"projects"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"moveToRegion"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"projectId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"projectId"}}},{"kind":"Argument","name":{"kind":"Name","value":"regionKey"},"value":{"kind":"Variable","name":{"kind":"Name","value":"regionKey"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}}]}}]}}]}}]}}]} as unknown as DocumentNode; export const CreateProjectAccessRequestDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"CreateProjectAccessRequest"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"projectId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"projectMutations"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"accessRequestMutations"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"create"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"projectId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"projectId"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"BasicProjectAccessRequestFields"}}]}}]}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"BasicProjectAccessRequestFields"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"ProjectAccessRequest"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"requester"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}}]}},{"kind":"Field","name":{"kind":"Name","value":"requesterId"}},{"kind":"Field","name":{"kind":"Name","value":"projectId"}},{"kind":"Field","name":{"kind":"Name","value":"createdAt"}}]}}]} as unknown as DocumentNode; export const GetActiveUserProjectAccessRequestDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"GetActiveUserProjectAccessRequest"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"projectId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"activeUser"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"projectAccessRequest"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"projectId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"projectId"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"BasicProjectAccessRequestFields"}}]}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"BasicProjectAccessRequestFields"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"ProjectAccessRequest"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"requester"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}}]}},{"kind":"Field","name":{"kind":"Name","value":"requesterId"}},{"kind":"Field","name":{"kind":"Name","value":"projectId"}},{"kind":"Field","name":{"kind":"Name","value":"createdAt"}}]}}]} as unknown as DocumentNode; export const GetActiveUserFullProjectAccessRequestDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"GetActiveUserFullProjectAccessRequest"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"projectId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"activeUser"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"projectAccessRequest"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"projectId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"projectId"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"BasicProjectAccessRequestFields"}},{"kind":"Field","name":{"kind":"Name","value":"project"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}}]}}]}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"BasicProjectAccessRequestFields"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"ProjectAccessRequest"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"requester"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}}]}},{"kind":"Field","name":{"kind":"Name","value":"requesterId"}},{"kind":"Field","name":{"kind":"Name","value":"projectId"}},{"kind":"Field","name":{"kind":"Name","value":"createdAt"}}]}}]} as unknown as DocumentNode; diff --git a/packages/server/test/graphql/multiRegion.ts b/packages/server/test/graphql/multiRegion.ts index ea44f957f..71a73e57b 100644 --- a/packages/server/test/graphql/multiRegion.ts +++ b/packages/server/test/graphql/multiRegion.ts @@ -60,3 +60,15 @@ export const updateRegionMutation = gql` ${mainRegionMetadataFragment} ` + +export const updateProjectRegionMutation = gql` + mutation UpdateProjectRegion($projectId: String!, $regionKey: String!) { + workspaceMutations { + projects { + moveToRegion(projectId: $projectId, regionKey: $regionKey) { + id + } + } + } + } +` diff --git a/packages/server/test/hooks.ts b/packages/server/test/hooks.ts index 8bc17fac2..99a3cf402 100644 --- a/packages/server/test/hooks.ts +++ b/packages/server/test/hooks.ts @@ -22,8 +22,7 @@ import { MaybeAsync, MaybeNullOrUndefined, Nullable, - Optional, - wait + Optional } from '@speckle/shared' import * as mocha from 'mocha' import { @@ -199,19 +198,18 @@ export const resetPubSubFactory = (deps: { db: Knex }) => async () => { await deps.db.raw( `SELECT * FROM aiven_extras.pg_alter_subscription_disable('${info.subname}');` ) - await wait(500) await deps.db.raw( `SELECT * FROM aiven_extras.pg_drop_subscription('${info.subname}');` ) - await wait(1000) await deps.db.raw( `SELECT * FROM aiven_extras.dblink_slot_create_or_drop('${info.subconninfo}', '${info.subslotname}', 'drop');` ) } // Drop all subs - // (concurrently, cause it seems possible and we have those delays there) - await Promise.all(subscriptions.rows.map(dropSubs)) + for (const sub of subscriptions.rows) { + await dropSubs(sub) + } // Drop all pubs for (const pub of publications.rows) { diff --git a/packages/shared/src/environment/index.ts b/packages/shared/src/environment/index.ts index dca6a7524..cbe7619f2 100644 --- a/packages/shared/src/environment/index.ts +++ b/packages/shared/src/environment/index.ts @@ -60,6 +60,16 @@ const parseFeatureFlags = () => { FF_FORCE_ONBOARDING: { schema: z.boolean(), defaults: { production: false, _: false } + }, + // Fixes the streaming of objects by ensuring that the database stream is closed properly + FF_OBJECTS_STREAMING_FIX: { + schema: z.boolean(), + defaults: { production: false, _: false } + }, + // Enables endpoint(s) for updating a project's region + FF_MOVE_PROJECT_REGION_ENABLED: { + schema: z.boolean(), + defaults: { production: false, _: true } } }) @@ -86,6 +96,8 @@ export function getFeatureFlags(): { FF_FILEIMPORT_IFC_DOTNET_ENABLED: boolean FF_FORCE_EMAIL_VERIFICATION: boolean FF_FORCE_ONBOARDING: boolean + FF_OBJECTS_STREAMING_FIX: boolean + FF_MOVE_PROJECT_REGION_ENABLED: boolean } { if (!parsedFlags) parsedFlags = parseFeatureFlags() return parsedFlags