From 9f83d1f74e7b2ae785e169a31fe223e75bcecc47 Mon Sep 17 00:00:00 2001 From: Chuck Driesler Date: Fri, 7 Mar 2025 09:18:12 +0000 Subject: [PATCH] feat(regions): trigger project move with job (#4010) --- .../lib/common/generated/gql/graphql.ts | 8 +- .../workspacesCore/typedefs/regions.graphql | 8 +- .../modules/core/graph/generated/graphql.ts | 10 +- .../graph/generated/graphql.ts | 8 +- .../modules/multiregion/errors/index.ts | 10 + packages/server/modules/multiregion/index.ts | 15 +- .../modules/multiregion/services/queue.ts | 209 ++++++++++++++++++ .../workspaces/graph/resolvers/regions.ts | 93 +------- .../tests/integration/projects.graph.spec.ts | 57 +++-- .../server/test/graphql/generated/graphql.ts | 12 +- packages/server/test/graphql/multiRegion.ts | 4 +- 11 files changed, 311 insertions(+), 123 deletions(-) create mode 100644 packages/server/modules/multiregion/services/queue.ts diff --git a/packages/frontend-2/lib/common/generated/gql/graphql.ts b/packages/frontend-2/lib/common/generated/gql/graphql.ts index 112e49e6c..6130056c0 100644 --- a/packages/frontend-2/lib/common/generated/gql/graphql.ts +++ b/packages/frontend-2/lib/common/generated/gql/graphql.ts @@ -4720,10 +4720,12 @@ export type WorkspaceProjectMutations = { __typename?: 'WorkspaceProjectMutations'; create: Project; /** - * Update project region and move all regional data to new db. - * TODO: Currently performs all operations synchronously in request, should probably be scheduled. + * Schedule a job that will: + * - Move all regional data to target region + * - Update project region key + * - TODO: Eventually delete data in previous region */ - moveToRegion: Project; + moveToRegion: Scalars['String']['output']; moveToWorkspace: Project; updateRole: Project; }; diff --git a/packages/server/assets/workspacesCore/typedefs/regions.graphql b/packages/server/assets/workspacesCore/typedefs/regions.graphql index e61b93102..2db19667b 100644 --- a/packages/server/assets/workspacesCore/typedefs/regions.graphql +++ b/packages/server/assets/workspacesCore/typedefs/regions.graphql @@ -15,8 +15,10 @@ extend type WorkspaceMutations { extend type WorkspaceProjectMutations { """ - Update project region and move all regional data to new db. - TODO: Currently performs all operations synchronously in request, should probably be scheduled. + Schedule a job that will: + - Move all regional data to target region + - Update project region key + - TODO: Eventually delete data in previous region """ - moveToRegion(projectId: String!, regionKey: String!): Project! + moveToRegion(projectId: String!, regionKey: String!): String! } diff --git a/packages/server/modules/core/graph/generated/graphql.ts b/packages/server/modules/core/graph/generated/graphql.ts index 896f81cb1..67482072d 100644 --- a/packages/server/modules/core/graph/generated/graphql.ts +++ b/packages/server/modules/core/graph/generated/graphql.ts @@ -4743,10 +4743,12 @@ export type WorkspaceProjectMutations = { __typename?: 'WorkspaceProjectMutations'; create: Project; /** - * Update project region and move all regional data to new db. - * TODO: Currently performs all operations synchronously in request, should probably be scheduled. + * Schedule a job that will: + * - Move all regional data to target region + * - Update project region key + * - TODO: Eventually delete data in previous region */ - moveToRegion: Project; + moveToRegion: Scalars['String']['output']; moveToWorkspace: Project; updateRole: Project; }; @@ -7155,7 +7157,7 @@ export type WorkspacePlanPriceResolvers = { create?: Resolver>; - moveToRegion?: Resolver>; + moveToRegion?: Resolver>; moveToWorkspace?: Resolver>; updateRole?: Resolver>; __isTypeOf?: IsTypeOfResolverFn; diff --git a/packages/server/modules/cross-server-sync/graph/generated/graphql.ts b/packages/server/modules/cross-server-sync/graph/generated/graphql.ts index e28b4800a..58c4d2422 100644 --- a/packages/server/modules/cross-server-sync/graph/generated/graphql.ts +++ b/packages/server/modules/cross-server-sync/graph/generated/graphql.ts @@ -4723,10 +4723,12 @@ export type WorkspaceProjectMutations = { __typename?: 'WorkspaceProjectMutations'; create: Project; /** - * Update project region and move all regional data to new db. - * TODO: Currently performs all operations synchronously in request, should probably be scheduled. + * Schedule a job that will: + * - Move all regional data to target region + * - Update project region key + * - TODO: Eventually delete data in previous region */ - moveToRegion: Project; + moveToRegion: Scalars['String']['output']; moveToWorkspace: Project; updateRole: Project; }; diff --git a/packages/server/modules/multiregion/errors/index.ts b/packages/server/modules/multiregion/errors/index.ts index 2f566a405..0cd3616a9 100644 --- a/packages/server/modules/multiregion/errors/index.ts +++ b/packages/server/modules/multiregion/errors/index.ts @@ -1,5 +1,10 @@ import { BaseError } from '@/modules/shared/errors' +export class MultiRegionNotYetImplementedError extends BaseError { + static code = 'MULTI_REGION_NOT_YET_IMPLEMENTED_ERROR' + static defaultMessage = 'Not yet implemented.' +} + export class MultiRegionSupportDisabledError extends BaseError { static code = 'MULTI_REGION_SUPPORT_DISABLED' static defaultMessage = 'Multi region support is disabled' @@ -22,3 +27,8 @@ export class RegionUpdateError extends BaseError { static defaultMessage = 'An error occurred while updating the region' static statusCode = 400 } + +export class MultiRegionInvalidJobError extends BaseError { + static code = 'MULTI_REGION_INVALID_JOB_ERROR' + static defaultMessage = 'Attempted to process malformed job in queue.' +} diff --git a/packages/server/modules/multiregion/index.ts b/packages/server/modules/multiregion/index.ts index de4dbd1f3..04553ff42 100644 --- a/packages/server/modules/multiregion/index.ts +++ b/packages/server/modules/multiregion/index.ts @@ -9,9 +9,14 @@ import { initializeRegisteredRegionClients as initBlobs, isMultiRegionBlobStorageEnabled } from '@/modules/multiregion/utils/blobStorageSelector' +import { + initializeQueue, + shutdownQueue, + startQueue +} from '@/modules/multiregion/services/queue' const multiRegion: SpeckleModule = { - async init() { + async init({ isInitial }) { const isEnabled = isMultiRegionEnabled() if (!isEnabled) { return @@ -29,6 +34,14 @@ const multiRegion: SpeckleModule = { moduleLogger.info('🌍 Init multiRegion blob storage') await initBlobs() } + + if (isInitial) { + await initializeQueue() + await startQueue() + } + }, + async shutdown() { + await shutdownQueue() } } diff --git a/packages/server/modules/multiregion/services/queue.ts b/packages/server/modules/multiregion/services/queue.ts new file mode 100644 index 000000000..6b85fd32a --- /dev/null +++ b/packages/server/modules/multiregion/services/queue.ts @@ -0,0 +1,209 @@ +import Bull from 'bull' +import { logger } from '@/observability/logging' +import { isProdEnv, isTestEnv } from '@/modules/shared/helpers/envHelper' +import cryptoRandomString from 'crypto-random-string' +import { Optional } from '@speckle/shared' +import { buildBaseQueueOptions } from '@/modules/shared/helpers/bullHelper' +import { UninitializedResourceAccessError } from '@/modules/shared/errors' +import { + MultiRegionInvalidJobError, + MultiRegionNotYetImplementedError +} from '@/modules/multiregion/errors' +import { getProjectDbClient, getRegionDb } from '@/modules/multiregion/utils/dbSelector' +import { + getProjectObjectStorage, + getRegionObjectStorage +} from '@/modules/multiregion/utils/blobStorageSelector' +import { + updateProjectRegionFactory, + validateProjectRegionCopyFactory +} from '@/modules/workspaces/services/projectRegions' +import { db } from '@/db/knex' +import { getProjectFactory } from '@/modules/core/repositories/projects' +import { getAvailableRegionsFactory } from '@/modules/workspaces/services/regions' +import { getRegionsFactory } from '@/modules/multiregion/repositories' +import { canWorkspaceUseRegionsFactory } from '@/modules/gatekeeper/services/featureAuthorization' +import { getProjectAutomationsTotalCountFactory } from '@/modules/automate/repositories/automations' +import { getStreamCommentCountFactory } from '@/modules/comments/repositories/comments' +import { getStreamBranchCountFactory } from '@/modules/core/repositories/branches' +import { getStreamCommitCountFactory } from '@/modules/core/repositories/commits' +import { getStreamObjectCountFactory } from '@/modules/core/repositories/objects' +import { getWorkspacePlanFactory } from '@/modules/gatekeeper/repositories/billing' +import { + upsertProjectRegionKeyFactory, + deleteRegionKeyFromCacheFactory +} from '@/modules/multiregion/repositories/projectRegion' +import { updateProjectRegionKeyFactory } from '@/modules/multiregion/services/projectRegion' +import { getGenericRedis } from '@/modules/shared/redis/redis' +import { getEventBus } from '@/modules/shared/services/eventBus' +import { getStreamWebhooksFactory } from '@/modules/webhooks/repositories/webhooks' +import { + copyWorkspaceFactory, + copyProjectsFactory, + copyProjectModelsFactory, + copyProjectVersionsFactory, + copyProjectObjectsFactory, + copyProjectAutomationsFactory, + copyProjectCommentsFactory, + copyProjectWebhooksFactory, + copyProjectBlobs +} from '@/modules/workspaces/repositories/projectRegions' +import { withTransaction } from '@/modules/shared/helpers/dbHelper' + +const MULTIREGION_QUEUE_NAME = isTestEnv() + ? `test:multiregion:${cryptoRandomString({ length: 5 })}` + : 'default:multiregion' + +if (isTestEnv()) { + logger.info(`Multiregion test queue ID: ${MULTIREGION_QUEUE_NAME}`) + logger.info(`Monitor using: 'yarn cli bull monitor ${MULTIREGION_QUEUE_NAME}'`) +} + +type MultiregionJob = + | { + type: 'move-project-region' + payload: { + projectId: string + regionKey: string + } + } + | { + type: 'delete-project-region-data' + payload: { + projectId: string + regionKey: string + } + } + +let queue: Optional> + +export const buildMultiregionQueue = (queueName: string) => + new Bull(queueName, { + ...buildBaseQueueOptions(), + ...(!isTestEnv() + ? { + limiter: { + max: 10, + duration: 1000 + } + } + : {}), + defaultJobOptions: { + attempts: 5, + timeout: 1000 * 60 * 15, // 15 minute timeout + backoff: { + type: 'fixed', + delay: 1000 * 60 * 5 + }, + removeOnComplete: isProdEnv(), + removeOnFail: false + } + }) + +export const getQueue = (): Bull.Queue => { + if (!queue) { + throw new UninitializedResourceAccessError( + 'Attempting to use uninitialized Bull queue' + ) + } + + return queue +} + +export const initializeQueue = () => { + queue = buildMultiregionQueue(MULTIREGION_QUEUE_NAME) +} + +/** + * Add a job to the multiregion job queue. + */ +export const scheduleJob = async (jobData: MultiregionJob): Promise => { + const queue = getQueue() + const job = await queue.add(jobData) + return job.id.toString() +} + +const isMultiregionJob = (job: Bull.Job): job is Bull.Job => { + const jobTypes: MultiregionJob['type'][] = [ + 'move-project-region', + 'delete-project-region-data' + ] + return !!job.data.type && jobTypes.includes(job.data.type) +} + +/** + * Start processing jobs in queue in current process. + */ +export const startQueue = async () => { + const queue = getQueue() + queue.process(async (job) => { + if (!isMultiregionJob(job)) { + throw new MultiRegionInvalidJobError() + } + + switch (job.data.type) { + case 'move-project-region': { + const { projectId, regionKey } = job.data.payload + + const sourceDb = await getProjectDbClient({ projectId }) + const sourceObjectStorage = await getProjectObjectStorage({ projectId }) + const targetDb = await (await getRegionDb({ regionKey })).transaction() + const targetObjectStorage = await getRegionObjectStorage({ regionKey }) + + const updateProjectRegion = updateProjectRegionFactory({ + getProject: getProjectFactory({ db: sourceDb }), + getAvailableRegions: getAvailableRegionsFactory({ + getRegions: getRegionsFactory({ db }), + canWorkspaceUseRegions: canWorkspaceUseRegionsFactory({ + getWorkspacePlan: getWorkspacePlanFactory({ db }) + }) + }), + copyWorkspace: copyWorkspaceFactory({ sourceDb, targetDb }), + copyProjects: copyProjectsFactory({ sourceDb, targetDb }), + copyProjectModels: copyProjectModelsFactory({ sourceDb, targetDb }), + copyProjectVersions: copyProjectVersionsFactory({ sourceDb, targetDb }), + copyProjectObjects: copyProjectObjectsFactory({ sourceDb, targetDb }), + copyProjectAutomations: copyProjectAutomationsFactory({ sourceDb, targetDb }), + copyProjectComments: copyProjectCommentsFactory({ sourceDb, targetDb }), + copyProjectWebhooks: copyProjectWebhooksFactory({ sourceDb, targetDb }), + copyProjectBlobs: copyProjectBlobs({ + sourceDb, + sourceObjectStorage, + targetDb, + targetObjectStorage + }), + validateProjectRegionCopy: validateProjectRegionCopyFactory({ + countProjectModels: getStreamBranchCountFactory({ db: sourceDb }), + countProjectVersions: getStreamCommitCountFactory({ db: sourceDb }), + countProjectObjects: getStreamObjectCountFactory({ db: sourceDb }), + countProjectAutomations: getProjectAutomationsTotalCountFactory({ + db: sourceDb + }), + countProjectComments: getStreamCommentCountFactory({ db: sourceDb }), + getProjectWebhooks: getStreamWebhooksFactory({ db: sourceDb }) + }), + updateProjectRegionKey: updateProjectRegionKeyFactory({ + upsertProjectRegionKey: upsertProjectRegionKeyFactory({ db }), + cacheDeleteRegionKey: deleteRegionKeyFromCacheFactory({ + redis: getGenericRedis() + }), + emitEvent: getEventBus().emit + }) + }) + + return await withTransaction( + updateProjectRegion({ projectId, regionKey }), + targetDb + ) + } + case 'delete-project-region-data': + default: + throw new MultiRegionNotYetImplementedError() + } + }) +} + +export const shutdownQueue = async () => { + if (!queue) return + await queue.close() +} diff --git a/packages/server/modules/workspaces/graph/resolvers/regions.ts b/packages/server/modules/workspaces/graph/resolvers/regions.ts index fec9eae92..506a4ef3a 100644 --- a/packages/server/modules/workspaces/graph/resolvers/regions.ts +++ b/packages/server/modules/workspaces/graph/resolvers/regions.ts @@ -2,24 +2,13 @@ import { db } from '@/db/knex' import { Resolvers } from '@/modules/core/graph/generated/graphql' import { getWorkspacePlanFactory } from '@/modules/gatekeeper/repositories/billing' import { canWorkspaceUseRegionsFactory } from '@/modules/gatekeeper/services/featureAuthorization' -import { getDb, getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' +import { getDb } from '@/modules/multiregion/utils/dbSelector' import { getRegionsFactory } from '@/modules/multiregion/repositories' import { authorizeResolver } from '@/modules/shared' import { getDefaultRegionFactory, upsertRegionAssignmentFactory } from '@/modules/workspaces/repositories/regions' -import { - copyProjectAutomationsFactory, - copyProjectCommentsFactory, - copyProjectBlobs, - copyProjectModelsFactory, - copyProjectObjectsFactory, - copyProjectsFactory, - copyProjectVersionsFactory, - copyProjectWebhooksFactory, - copyWorkspaceFactory -} from '@/modules/workspaces/repositories/projectRegions' import { getWorkspaceFactory, upsertWorkspaceFactory @@ -28,32 +17,10 @@ import { assignWorkspaceRegionFactory, getAvailableRegionsFactory } from '@/modules/workspaces/services/regions' -import { - updateProjectRegionFactory, - validateProjectRegionCopyFactory -} from '@/modules/workspaces/services/projectRegions' import { Roles } from '@speckle/shared' -import { getProjectFactory } from '@/modules/core/repositories/projects' -import { getStreamBranchCountFactory } from '@/modules/core/repositories/branches' -import { getStreamCommitCountFactory } from '@/modules/core/repositories/commits' -import { withTransaction } from '@/modules/shared/helpers/dbHelper' -import { getStreamObjectCountFactory } from '@/modules/core/repositories/objects' -import { getProjectAutomationsTotalCountFactory } from '@/modules/automate/repositories/automations' import { getFeatureFlags, isTestEnv } from '@/modules/shared/helpers/envHelper' import { WorkspacesNotYetImplementedError } from '@/modules/workspaces/errors/workspace' -import { getStreamCommentCountFactory } from '@/modules/comments/repositories/comments' -import { getStreamWebhooksFactory } from '@/modules/webhooks/repositories/webhooks' -import { - getProjectObjectStorage, - getRegionObjectStorage -} from '@/modules/multiregion/utils/blobStorageSelector' -import { updateProjectRegionKeyFactory } from '@/modules/multiregion/services/projectRegion' -import { - deleteRegionKeyFromCacheFactory, - upsertProjectRegionKeyFactory -} from '@/modules/multiregion/repositories/projectRegion' -import { getGenericRedis } from '@/modules/shared/redis/redis' -import { getEventBus } from '@/modules/shared/services/eventBus' +import { scheduleJob } from '@/modules/multiregion/services/queue' const { FF_MOVE_PROJECT_REGION_ENABLED } = getFeatureFlags() @@ -105,57 +72,13 @@ export default { context.resourceAccessRules ) - const sourceDb = await getProjectDbClient({ projectId: args.projectId }) - const sourceObjectStorage = await getProjectObjectStorage({ - projectId: args.projectId + return await scheduleJob({ + type: 'move-project-region', + payload: { + projectId: args.projectId, + regionKey: args.regionKey + } }) - const targetDb = await (await getDb({ regionKey: args.regionKey })).transaction() - const targetObjectStorage = await getRegionObjectStorage({ - regionKey: args.regionKey - }) - - const updateProjectRegion = updateProjectRegionFactory({ - getProject: getProjectFactory({ db: sourceDb }), - getAvailableRegions: getAvailableRegionsFactory({ - getRegions: getRegionsFactory({ db }), - canWorkspaceUseRegions: canWorkspaceUseRegionsFactory({ - getWorkspacePlan: getWorkspacePlanFactory({ db }) - }) - }), - copyWorkspace: copyWorkspaceFactory({ sourceDb, targetDb }), - copyProjects: copyProjectsFactory({ sourceDb, targetDb }), - copyProjectModels: copyProjectModelsFactory({ sourceDb, targetDb }), - copyProjectVersions: copyProjectVersionsFactory({ sourceDb, targetDb }), - copyProjectObjects: copyProjectObjectsFactory({ sourceDb, targetDb }), - copyProjectAutomations: copyProjectAutomationsFactory({ sourceDb, targetDb }), - copyProjectComments: copyProjectCommentsFactory({ sourceDb, targetDb }), - copyProjectWebhooks: copyProjectWebhooksFactory({ sourceDb, targetDb }), - copyProjectBlobs: copyProjectBlobs({ - sourceDb, - sourceObjectStorage, - targetDb, - targetObjectStorage - }), - validateProjectRegionCopy: validateProjectRegionCopyFactory({ - countProjectModels: getStreamBranchCountFactory({ db: sourceDb }), - countProjectVersions: getStreamCommitCountFactory({ db: sourceDb }), - countProjectObjects: getStreamObjectCountFactory({ db: sourceDb }), - countProjectAutomations: getProjectAutomationsTotalCountFactory({ - db: sourceDb - }), - countProjectComments: getStreamCommentCountFactory({ db: sourceDb }), - getProjectWebhooks: getStreamWebhooksFactory({ db: sourceDb }) - }), - updateProjectRegionKey: updateProjectRegionKeyFactory({ - upsertProjectRegionKey: upsertProjectRegionKeyFactory({ db }), - cacheDeleteRegionKey: deleteRegionKeyFromCacheFactory({ - redis: getGenericRedis() - }), - emitEvent: getEventBus().emit - }) - }) - - return await withTransaction(updateProjectRegion(args), targetDb) } } } as Resolvers diff --git a/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts b/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts index b238b9fc0..d9dd4dcf5 100644 --- a/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts +++ b/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts @@ -3,6 +3,7 @@ import { AutomationRecord, AutomationRunRecord } from '@/modules/automate/helper import { CommentRecord } from '@/modules/comments/helpers/types' import { AllScopes } from '@/modules/core/helpers/mainConstants' import { createRandomEmail } from '@/modules/core/helpers/testHelpers' +import { StreamRecord } from '@/modules/core/helpers/types' import { grantStreamPermissionsFactory } from '@/modules/core/repositories/streams' import { WorkspaceSeatType } from '@/modules/gatekeeper/domain/billing' import { getWorkspaceUserSeatsFactory } from '@/modules/gatekeeper/repositories/workspaceSeat' @@ -72,12 +73,34 @@ import { createTestStream, getUserStreamRole } from '@/test/speckle-helpers/streamHelper' -import { Roles } from '@speckle/shared' +import { Roles, retry } from '@speckle/shared' import { expect } from 'chai' import cryptoRandomString from 'crypto-random-string' import { Knex } from 'knex' import { SetOptional } from 'type-fest' +const tables = { + projects: (db: Knex) => db.table('streams') +} + +const assertProjectRegion = async ( + projectId: string, + regionKey: string +): Promise => { + const project = await tables.projects(db).select('*').where('id', projectId).first() + + if (!project || project.regionKey !== regionKey) { + expect.fail('Project is not in expected region.') + } +} + +const ensureProjectRegion = async ( + projectId: string, + regionKey: string +): Promise => { + await retry(async () => assertProjectRegion(projectId, regionKey), 20, 10) +} + const grantStreamPermissions = grantStreamPermissionsFactory({ db }) describe('Workspace project GQL CRUD', () => { @@ -624,6 +647,8 @@ isMultiRegionTestMode() projectId: testProject.id }) testBlobId = testBlob.blobId + + await assertProjectRegion(testProject.id, regionKey1) }) it('moves project record to target regional db', async () => { @@ -633,6 +658,8 @@ isMultiRegionTestMode() }) expect(resA).to.not.haveGraphQLErrors() + await ensureProjectRegion(testProject.id, regionKey2) + const resB = await apollo.execute(GetProjectDocument, { id: testProject.id }) @@ -648,6 +675,8 @@ isMultiRegionTestMode() }) expect(resA).to.not.haveGraphQLErrors() + await ensureProjectRegion(testProject.id, regionKey2) + const resB = await apollo.execute(GetRegionalProjectModelDocument, { projectId: testProject.id, modelId: testModel.id @@ -664,6 +693,8 @@ isMultiRegionTestMode() }) expect(resA).to.not.haveGraphQLErrors() + await ensureProjectRegion(testProject.id, regionKey2) + const resB = await apollo.execute(GetRegionalProjectVersionDocument, { projectId: testProject.id, modelId: testModel.id, @@ -683,6 +714,8 @@ isMultiRegionTestMode() }) expect(resA).to.not.haveGraphQLErrors() + await ensureProjectRegion(testProject.id, regionKey2) + const resB = await apollo.execute(GetRegionalProjectObjectDocument, { projectId: testProject.id, objectId: testVersion.objectId @@ -699,6 +732,8 @@ isMultiRegionTestMode() }) expect(resA).to.not.haveGraphQLErrors() + await ensureProjectRegion(testProject.id, regionKey2) + const resB = await apollo.execute(GetRegionalProjectAutomationDocument, { projectId: testProject.id, automationId: testAutomation.id @@ -721,6 +756,8 @@ isMultiRegionTestMode() }) expect(resA).to.not.haveGraphQLErrors() + await ensureProjectRegion(testProject.id, regionKey2) + const resB = await apollo.execute(GetRegionalProjectCommentDocument, { projectId: testProject.id, commentId: testComment.id @@ -737,6 +774,8 @@ isMultiRegionTestMode() }) expect(resA).to.not.haveGraphQLErrors() + await ensureProjectRegion(testProject.id, regionKey2) + const resB = await apollo.execute(GetRegionalProjectWebhookDocument, { projectId: testProject.id, webhookId: testWebhookId @@ -753,21 +792,7 @@ isMultiRegionTestMode() }) expect(resA).to.not.haveGraphQLErrors() - const resB = await apollo.execute(GetRegionalProjectBlobDocument, { - projectId: testProject.id, - blobId: testBlobId - }) - expect(resB).to.not.haveGraphQLErrors() - - expect(resB.data?.project.blob).to.not.be.undefined - }) - - it('moves project files and associated blobs to target regional db and object storage', async () => { - const resA = await apollo.execute(UpdateProjectRegionDocument, { - projectId: testProject.id, - regionKey: regionKey2 - }) - expect(resA).to.not.haveGraphQLErrors() + await ensureProjectRegion(testProject.id, regionKey2) const resB = await apollo.execute(GetRegionalProjectBlobDocument, { projectId: testProject.id, diff --git a/packages/server/test/graphql/generated/graphql.ts b/packages/server/test/graphql/generated/graphql.ts index cbaad7d4b..148c5f21b 100644 --- a/packages/server/test/graphql/generated/graphql.ts +++ b/packages/server/test/graphql/generated/graphql.ts @@ -4724,10 +4724,12 @@ export type WorkspaceProjectMutations = { __typename?: 'WorkspaceProjectMutations'; create: Project; /** - * Update project region and move all regional data to new db. - * TODO: Currently performs all operations synchronously in request, should probably be scheduled. + * Schedule a job that will: + * - Move all regional data to target region + * - Update project region key + * - TODO: Eventually delete data in previous region */ - moveToRegion: Project; + moveToRegion: Scalars['String']['output']; moveToWorkspace: Project; updateRole: Project; }; @@ -5372,7 +5374,7 @@ export type UpdateProjectRegionMutationVariables = Exact<{ }>; -export type UpdateProjectRegionMutation = { __typename?: 'Mutation', workspaceMutations: { __typename?: 'WorkspaceMutations', projects: { __typename?: 'WorkspaceProjectMutations', moveToRegion: { __typename?: 'Project', id: string } } } }; +export type UpdateProjectRegionMutation = { __typename?: 'Mutation', workspaceMutations: { __typename?: 'WorkspaceMutations', projects: { __typename?: 'WorkspaceProjectMutations', moveToRegion: string } } }; export type GetRegionalProjectModelQueryVariables = Exact<{ projectId: Scalars['String']['input']; @@ -6000,7 +6002,7 @@ export const GetAvailableRegionKeysDocument = {"kind":"Document","definitions":[ export const CreateNewRegionDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"CreateNewRegion"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"CreateServerRegionInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"serverInfoMutations"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"multiRegion"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"create"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"MainRegionMetadata"}}]}}]}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"MainRegionMetadata"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"ServerRegionItem"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"key"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"description"}}]}}]} as unknown as DocumentNode; export const GetRegionsDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"GetRegions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"serverInfo"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"multiRegion"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"regions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"MainRegionMetadata"}}]}}]}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"MainRegionMetadata"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"ServerRegionItem"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"key"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"description"}}]}}]} as unknown as DocumentNode; export const UpdateRegionDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"UpdateRegion"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"UpdateServerRegionInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"serverInfoMutations"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"multiRegion"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"update"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"MainRegionMetadata"}}]}}]}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"MainRegionMetadata"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"ServerRegionItem"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"key"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"description"}}]}}]} as unknown as DocumentNode; -export const UpdateProjectRegionDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"UpdateProjectRegion"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"projectId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"regionKey"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"workspaceMutations"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"projects"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"moveToRegion"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"projectId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"projectId"}}},{"kind":"Argument","name":{"kind":"Name","value":"regionKey"},"value":{"kind":"Variable","name":{"kind":"Name","value":"regionKey"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}}]}}]}}]}}]}}]} as unknown as DocumentNode; +export const UpdateProjectRegionDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"UpdateProjectRegion"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"projectId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"regionKey"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"workspaceMutations"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"projects"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"moveToRegion"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"projectId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"projectId"}}},{"kind":"Argument","name":{"kind":"Name","value":"regionKey"},"value":{"kind":"Variable","name":{"kind":"Name","value":"regionKey"}}}]}]}}]}}]}}]} as unknown as DocumentNode; export const GetRegionalProjectModelDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"GetRegionalProjectModel"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"projectId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"modelId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"project"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"projectId"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"model"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"modelId"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}}]}}]}}]}}]} as unknown as DocumentNode; export const GetRegionalProjectVersionDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"GetRegionalProjectVersion"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"projectId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"modelId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"versionId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"project"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"projectId"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"model"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"modelId"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"version"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"versionId"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"referencedObject"}}]}}]}}]}}]}}]} as unknown as DocumentNode; export const GetRegionalProjectObjectDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"GetRegionalProjectObject"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"projectId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"objectId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"project"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"projectId"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"object"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"objectId"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}}]}}]}}]}}]} as unknown as DocumentNode; diff --git a/packages/server/test/graphql/multiRegion.ts b/packages/server/test/graphql/multiRegion.ts index 6983dc929..e1670ce24 100644 --- a/packages/server/test/graphql/multiRegion.ts +++ b/packages/server/test/graphql/multiRegion.ts @@ -65,9 +65,7 @@ export const updateProjectRegionMutation = gql` mutation UpdateProjectRegion($projectId: String!, $regionKey: String!) { workspaceMutations { projects { - moveToRegion(projectId: $projectId, regionKey: $regionKey) { - id - } + moveToRegion(projectId: $projectId, regionKey: $regionKey) } } }