diff --git a/packages/server/modules/automate/domain/operations.ts b/packages/server/modules/automate/domain/operations.ts index dd2d78c27..ccf8cbcf4 100644 --- a/packages/server/modules/automate/domain/operations.ts +++ b/packages/server/modules/automate/domain/operations.ts @@ -1,6 +1,6 @@ import { InsertableAutomationFunctionRun } from '@/modules/automate/domain/types' import { - AutomateRevisionFunctionRecord, + AutomationRevisionFunctionRecord, AutomationFunctionRunRecord, AutomationRecord, AutomationRevisionRecord, @@ -175,7 +175,7 @@ export type GetRevisionsTriggerDefinitions = (params: { export type GetRevisionsFunctions = (params: { automationRevisionIds: string[] -}) => Promise<{ [automationRevisionId: string]: AutomateRevisionFunctionRecord[] }> +}) => Promise<{ [automationRevisionId: string]: AutomationRevisionFunctionRecord[] }> export type CreateStoredAuthCode = ( params: Omit @@ -204,3 +204,7 @@ export type TriggerAutomationRevisionRun = < manifest: M source: RunTriggerSource }) => Promise<{ automationRunId: string }> + +export type GetProjectAutomationCount = (params: { + projectId: string +}) => Promise diff --git a/packages/server/modules/automate/helpers/graphTypes.ts b/packages/server/modules/automate/helpers/graphTypes.ts index 40be15930..366a99f5c 100644 --- a/packages/server/modules/automate/helpers/graphTypes.ts +++ b/packages/server/modules/automate/helpers/graphTypes.ts @@ -1,5 +1,5 @@ import { - AutomateRevisionFunctionRecord, + AutomationRevisionFunctionRecord, AutomationFunctionRunRecord, AutomationRecord, AutomationRevisionRecord, @@ -56,7 +56,7 @@ export type AutomationRunTriggerGraphQLReturn = AutomationRunTriggerRecord & { } export type AutomationRevisionFunctionGraphQLReturn = Merge< - AutomateRevisionFunctionRecord, + AutomationRevisionFunctionRecord, { functionInputs: Nullable> release: AutomateFunctionReleaseGraphQLReturn diff --git a/packages/server/modules/automate/helpers/types.ts b/packages/server/modules/automate/helpers/types.ts index e360d87aa..eb95f5d22 100644 --- a/packages/server/modules/automate/helpers/types.ts +++ b/packages/server/modules/automate/helpers/types.ts @@ -66,7 +66,7 @@ export type AutomationRunRecord = { executionEngineRunId: string | null } -export type AutomateRevisionFunctionRecord = { +export type AutomationRevisionFunctionRecord = { functionReleaseId: string functionId: string functionInputs: string | null @@ -122,7 +122,7 @@ export type AutomationTokenRecord = { } export type AutomationRevisionWithTriggersFunctions = AutomationRevisionRecord & { - functions: AutomateRevisionFunctionRecord[] + functions: AutomationRevisionFunctionRecord[] triggers: AutomationTriggerDefinitionRecord[] } diff --git a/packages/server/modules/automate/repositories/automations.ts b/packages/server/modules/automate/repositories/automations.ts index c424b3a8e..c07a21b40 100644 --- a/packages/server/modules/automate/repositories/automations.ts +++ b/packages/server/modules/automate/repositories/automations.ts @@ -16,6 +16,7 @@ import { GetLatestAutomationRevision, GetLatestAutomationRevisions, GetLatestVersionAutomationRuns, + GetProjectAutomationCount, GetRevisionsFunctions, GetRevisionsTriggerDefinitions, StoreAutomation, @@ -37,7 +38,7 @@ import { AutomationRunRecord, AutomationTokenRecord, AutomationTriggerRecordBase, - AutomateRevisionFunctionRecord, + AutomationRevisionFunctionRecord, AutomationRunWithTriggersFunctionRuns, AutomationRunTriggerRecord, AutomationFunctionRunRecord, @@ -86,7 +87,7 @@ const tables = { automationRevisions: (db: Knex) => db(AutomationRevisions.name), automationRevisionFunctions: (db: Knex) => - db(AutomationRevisionFunctions.name), + db(AutomationRevisionFunctions.name), automationTriggers: (db: Knex) => db(AutomationTriggers.name), automationRuns: (db: Knex) => db(AutomationRuns.name), @@ -321,7 +322,7 @@ export const storeAutomationTokenFactory = } export type InsertableAutomationRevisionFunction = Omit< - AutomateRevisionFunctionRecord, + AutomationRevisionFunctionRecord, 'automationRevisionId' > @@ -369,7 +370,7 @@ export const storeAutomationRevisionFactory = .automationRevisionFunctions(deps.db) .insert( revision.functions.map( - (f): AutomateRevisionFunctionRecord => ({ + (f): AutomationRevisionFunctionRecord => ({ ...f, automationRevisionId: id }) @@ -742,10 +743,12 @@ const getProjectAutomationsBaseQueryFactory = } export const getProjectAutomationsTotalCountFactory = - (deps: { db: Knex }) => async (params: GetProjectAutomationsParams) => { - const q = getProjectAutomationsBaseQueryFactory(deps)(params).count< - [{ count: string }] - >(Automations.col.id) + (deps: { db: Knex }): GetProjectAutomationCount => + async ({ projectId }) => { + const q = getProjectAutomationsBaseQueryFactory(deps)({ + projectId, + args: {} + }).count<[{ count: string }]>(Automations.col.id) const [ret] = await q diff --git a/packages/server/modules/automate/services/encryption.ts b/packages/server/modules/automate/services/encryption.ts index 3b8228c8b..8cfc98d66 100644 --- a/packages/server/modules/automate/services/encryption.ts +++ b/packages/server/modules/automate/services/encryption.ts @@ -7,7 +7,7 @@ import { Nullable, Optional } from '@speckle/shared' import { MisconfiguredEnvironmentError } from '@/modules/shared/errors' import { AutomationFunctionInputEncryptionError } from '@/modules/automate/errors/management' import { KeyPair, buildDecryptor } from '@/modules/shared/utils/libsodium' -import { AutomateRevisionFunctionRecord } from '@/modules/automate/helpers/types' +import { AutomationRevisionFunctionRecord } from '@/modules/automate/helpers/types' import { AutomationRevisionFunctionGraphQLReturn } from '@/modules/automate/helpers/graphTypes' import { FunctionReleaseSchemaType } from '@/modules/automate/helpers/executionEngine' import { LibsodiumEncryptionError } from '@/modules/shared/errors/encryption' @@ -118,7 +118,7 @@ export type GetFunctionInputsForFrontendDeps = { } & GetFunctionInputDecryptorDeps export type AutomationRevisionFunctionForInputRedaction = Merge< - AutomateRevisionFunctionRecord, + AutomationRevisionFunctionRecord, { release: FunctionReleaseSchemaType } > diff --git a/packages/server/modules/automate/services/trigger.ts b/packages/server/modules/automate/services/trigger.ts index 0cdf02390..f40ff924c 100644 --- a/packages/server/modules/automate/services/trigger.ts +++ b/packages/server/modules/automate/services/trigger.ts @@ -46,6 +46,7 @@ import { ValidateStreamAccess } from '@/modules/core/domain/streams/operations' import { CreateAndStoreAppToken } from '@/modules/core/domain/tokens/operations' import { EventBusEmit } from '@/modules/shared/services/eventBus' import { AutomationRunEvents } from '@/modules/automate/domain/events' +import { isTestEnv } from '@/modules/shared/helpers/envHelper' export type OnModelVersionCreateDeps = { getAutomation: GetAutomation @@ -131,7 +132,7 @@ type CreateAutomationRunDataDeps = { getFunctionInputDecryptor: FunctionInputDecryptor } -const createAutomationRunDataFactory = +export const createAutomationRunDataFactory = (deps: CreateAutomationRunDataDeps) => async (params: { manifests: BaseTriggerManifest[] @@ -414,7 +415,7 @@ type ComposeTriggerDataDeps = { getBranchLatestCommits: GetBranchLatestCommits } -const composeTriggerDataFactory = +export const composeTriggerDataFactory = (deps: ComposeTriggerDataDeps) => async (params: { projectId: string @@ -575,7 +576,7 @@ export const createTestAutomationRunFactory = throw new TriggerAutomationError('Automation not found') } - if (!automationRecord.isTestAutomation) { + if (!isTestEnv() && !automationRecord.isTestAutomation) { throw new TriggerAutomationError( 'Automation is not a test automation and cannot create test function runs' ) diff --git a/packages/server/modules/automate/tests/automations.spec.ts b/packages/server/modules/automate/tests/automations.spec.ts index 6d02b202d..2ad7b4d30 100644 --- a/packages/server/modules/automate/tests/automations.spec.ts +++ b/packages/server/modules/automate/tests/automations.spec.ts @@ -510,9 +510,11 @@ const buildAutomationUpdate = () => { it('fails when refering to nonexistent function releases', async () => { const create = buildAutomationRevisionCreate({ - getFunctionRelease: async () => { - // TODO: Update once we know how exec engine should respond - throw new Error('Function release with ID XXX not found') + overrides: { + getFunctionRelease: async () => { + // TODO: Update once we know how exec engine should respond + throw new Error('Function release with ID XXX not found') + } } }) diff --git a/packages/server/modules/core/graph/dataloaders/index.ts b/packages/server/modules/core/graph/dataloaders/index.ts index 958019035..f6ed23fd1 100644 --- a/packages/server/modules/core/graph/dataloaders/index.ts +++ b/packages/server/modules/core/graph/dataloaders/index.ts @@ -56,7 +56,7 @@ import { Users } from '@/modules/core/dbSchema' import { getStreamPendingModelsFactory } from '@/modules/fileuploads/repositories/fileUploads' import { FileUploadRecord } from '@/modules/fileuploads/helpers/types' import { - AutomateRevisionFunctionRecord, + AutomationRevisionFunctionRecord, AutomationRecord, AutomationRevisionRecord, AutomationRunTriggerRecord, @@ -595,7 +595,7 @@ const dataLoadersDefinition = defineRequestDataloaders( }) return ids.map((i) => results[i] || []) }), - getRevisionFunctions: createLoader( + getRevisionFunctions: createLoader( async (ids) => { const results = await getRevisionsFunctions({ automationRevisionIds: ids.slice() diff --git a/packages/server/modules/workspaces/domain/operations.ts b/packages/server/modules/workspaces/domain/operations.ts index df4e770bb..0e330d04b 100644 --- a/packages/server/modules/workspaces/domain/operations.ts +++ b/packages/server/modules/workspaces/domain/operations.ts @@ -366,3 +366,6 @@ export type CopyProjectVersions = (params: { export type CopyProjectObjects = (params: { projectIds: string[] }) => Promise> +export type CopyProjectAutomations = (params: { + projectIds: string[] +}) => Promise> diff --git a/packages/server/modules/workspaces/graph/resolvers/regions.ts b/packages/server/modules/workspaces/graph/resolvers/regions.ts index 8ad3f15e9..1d7b890b8 100644 --- a/packages/server/modules/workspaces/graph/resolvers/regions.ts +++ b/packages/server/modules/workspaces/graph/resolvers/regions.ts @@ -10,6 +10,7 @@ import { upsertRegionAssignmentFactory } from '@/modules/workspaces/repositories/regions' import { + copyProjectAutomationsFactory, copyProjectModelsFactory, copyProjectObjectsFactory, copyProjectsFactory, @@ -31,6 +32,7 @@ import { getStreamBranchCountFactory } from '@/modules/core/repositories/branche import { getStreamCommitCountFactory } from '@/modules/core/repositories/commits' import { withTransaction } from '@/modules/shared/helpers/dbHelper' import { getStreamObjectCountFactory } from '@/modules/core/repositories/objects' +import { getProjectAutomationsTotalCountFactory } from '@/modules/automate/repositories/automations' import { getFeatureFlags, isTestEnv } from '@/modules/shared/helpers/envHelper' import { WorkspacesNotYetImplementedError } from '@/modules/workspaces/errors/workspace' @@ -92,6 +94,9 @@ export default { countProjectModels: getStreamBranchCountFactory({ db: sourceDb }), countProjectVersions: getStreamCommitCountFactory({ db: sourceDb }), countProjectObjects: getStreamObjectCountFactory({ db: sourceDb }), + countProjectAutomations: getProjectAutomationsTotalCountFactory({ + db: sourceDb + }), getAvailableRegions: getAvailableRegionsFactory({ getRegions: getRegionsFactory({ db }), canWorkspaceUseRegions: canWorkspaceUseRegionsFactory({ @@ -102,7 +107,8 @@ export default { copyProjects: copyProjectsFactory({ sourceDb, targetDb }), copyProjectModels: copyProjectModelsFactory({ sourceDb, targetDb }), copyProjectVersions: copyProjectVersionsFactory({ sourceDb, targetDb }), - copyProjectObjects: copyProjectObjectsFactory({ sourceDb, targetDb }) + copyProjectObjects: copyProjectObjectsFactory({ sourceDb, targetDb }), + copyProjectAutomations: copyProjectAutomationsFactory({ sourceDb, targetDb }) }) return await withTransaction(updateProjectRegion(args), targetDb) diff --git a/packages/server/modules/workspaces/repositories/projectRegions.ts b/packages/server/modules/workspaces/repositories/projectRegions.ts index e068162b6..0e05d0a9d 100644 --- a/packages/server/modules/workspaces/repositories/projectRegions.ts +++ b/packages/server/modules/workspaces/repositories/projectRegions.ts @@ -1,4 +1,12 @@ import { + AutomationFunctionRuns, + AutomationRevisionFunctions, + AutomationRevisions, + AutomationRuns, + AutomationRunTriggers, + Automations, + AutomationTokens, + AutomationTriggers, BranchCommits, Branches, Commits, @@ -21,6 +29,7 @@ import { } from '@/modules/core/helpers/types' import { executeBatchedSelect } from '@/modules/shared/helpers/dbHelper' import { + CopyProjectAutomations, CopyProjectModels, CopyProjectObjects, CopyProjects, @@ -32,6 +41,16 @@ import { Knex } from 'knex' import { Workspace } from '@/modules/workspacesCore/domain/types' import { Workspaces } from '@/modules/workspacesCore/helpers/db' import { ObjectPreview } from '@/modules/previews/domain/types' +import { + AutomationFunctionRunRecord, + AutomationRecord, + AutomationRevisionFunctionRecord, + AutomationRevisionRecord, + AutomationRunRecord, + AutomationRunTriggerRecord, + AutomationTokenRecord, + AutomationTriggerDefinitionRecord +} from '@/modules/automate/helpers/types' const tables = { workspaces: (db: Knex) => db(Workspaces.name), @@ -43,7 +62,20 @@ const tables = { streamFavorites: (db: Knex) => db(StreamFavorites.name), streamsMeta: (db: Knex) => db(StreamsMeta.name), objects: (db: Knex) => db(Objects.name), - objectPreviews: (db: Knex) => db('object_preview') + objectPreviews: (db: Knex) => db('object_preview'), + automations: (db: Knex) => db(Automations.name), + automationTokens: (db: Knex) => db(AutomationTokens.name), + automationRevisions: (db: Knex) => + db(AutomationRevisions.name), + automationTriggers: (db: Knex) => + db(AutomationTriggers.name), + automationRevisionFunctions: (db: Knex) => + db(AutomationRevisionFunctions.name), + automationRuns: (db: Knex) => db(AutomationRuns.name), + automationRunTriggers: (db: Knex) => + db(AutomationRunTriggers.name), + automationFunctionRuns: (db: Knex) => + db(AutomationFunctionRuns.name) } /** @@ -283,3 +315,147 @@ export const copyProjectObjectsFactory = return copiedObjectCountByProjectId } + +/** + * Copies rows from the following tables: + * - automations + * - automation_tokens + * - automation_revisions + * - automation_triggers + * - automation_revision_functions + * - automation_runs + * - automation_run_triggers + * - automation_function_runs + */ +export const copyProjectAutomationsFactory = + (deps: { sourceDb: Knex; targetDb: Knex }): CopyProjectAutomations => + async ({ projectIds }) => { + const copiedAutomationCountByProjectId: Record = {} + + // Copy `automations` table rows in batches + const selectAutomations = tables + .automations(deps.sourceDb) + .select('*') + .whereIn(Automations.col.projectId, projectIds) + + for await (const automations of executeBatchedSelect(selectAutomations)) { + const automationIds = automations.map((automation) => automation.id) + + // Write `automations` table rows to target db + await tables + .automations(deps.targetDb) + // Cast ignores unexpected behavior in how knex handles object union types + .insert(automations as unknown as AutomationRecord) + .onConflict() + .ignore() + + for (const automation of automations) { + copiedAutomationCountByProjectId[automation.projectId] ??= 0 + copiedAutomationCountByProjectId[automation.projectId]++ + } + + // Copy `automation_tokens` rows for automation + const selectAutomationTokens = tables + .automationTokens(deps.sourceDb) + .select('*') + .whereIn(AutomationTokens.col.automationId, automationIds) + + for await (const tokens of executeBatchedSelect(selectAutomationTokens)) { + // Write `automation_tokens` row to target db + await tables + .automationTokens(deps.targetDb) + .insert(tokens) + .onConflict() + .ignore() + } + + // Copy `automation_revisions` rows for automation + const selectAutomationRevisions = tables + .automationRevisions(deps.sourceDb) + .select('*') + .whereIn(AutomationRevisions.col.automationId, automationIds) + + for await (const automationRevisions of executeBatchedSelect( + selectAutomationRevisions + )) { + const automationRevisionIds = automationRevisions.map((revision) => revision.id) + + // Write `automation_revisions` rows to target db + await tables + .automationRevisions(deps.targetDb) + .insert(automationRevisions) + .onConflict() + .ignore() + + // Copy `automation_triggers` rows for automation revisions + const automationTriggers = await tables + .automationTriggers(deps.sourceDb) + .select('*') + .whereIn(AutomationTriggers.col.automationRevisionId, automationRevisionIds) + + await tables + .automationTriggers(deps.targetDb) + .insert(automationTriggers) + .onConflict() + .ignore() + + // Copy `automation_revision_functions` rows for automation revisions + const automationRevisionFunctions = await tables + .automationRevisionFunctions(deps.sourceDb) + .select('*') + .whereIn( + AutomationRevisionFunctions.col.automationRevisionId, + automationRevisionIds + ) + + await tables + .automationRevisionFunctions(deps.targetDb) + .insert(automationRevisionFunctions) + .onConflict() + .ignore() + + // Copy `automation_runs` rows for automation revision + const selectAutomationRuns = tables + .automationRuns(deps.sourceDb) + .select('*') + .whereIn(AutomationRuns.col.automationRevisionId, automationRevisionIds) + + for await (const automationRuns of executeBatchedSelect(selectAutomationRuns)) { + const automationRunIds = automationRuns.map((run) => run.id) + + // Write `automation_runs` row to target db + await tables + .automationRuns(deps.targetDb) + .insert(automationRuns) + .onConflict() + .ignore() + + // Copy `automation_run_triggers` rows for automation run + const automationRunTriggers = await tables + .automationRunTriggers(deps.sourceDb) + .select('*') + .whereIn(AutomationRunTriggers.col.automationRunId, automationRunIds) + + await tables + .automationRunTriggers(deps.targetDb) + .insert(automationRunTriggers) + .onConflict() + .ignore() + + // Copy `automation_function_runs` rows for automation run + const automationFunctionRuns = await tables + .automationFunctionRuns(deps.sourceDb) + .select('*') + .whereIn(AutomationFunctionRuns.col.runId, automationRunIds) + + await tables + .automationFunctionRuns(deps.targetDb) + .insert(automationFunctionRuns) + .onConflict() + .ignore() + } + } + } + + return copiedAutomationCountByProjectId + } diff --git a/packages/server/modules/workspaces/services/projectRegions.ts b/packages/server/modules/workspaces/services/projectRegions.ts index 84f5688ca..78cc9b7cf 100644 --- a/packages/server/modules/workspaces/services/projectRegions.ts +++ b/packages/server/modules/workspaces/services/projectRegions.ts @@ -1,8 +1,10 @@ +import { GetProjectAutomationCount } from '@/modules/automate/domain/operations' import { GetStreamBranchCount } from '@/modules/core/domain/branches/operations' import { GetStreamCommitCount } from '@/modules/core/domain/commits/operations' import { GetStreamObjectCount } from '@/modules/core/domain/objects/operations' import { GetProject } from '@/modules/core/domain/projects/operations' import { + CopyProjectAutomations, CopyProjectModels, CopyProjectObjects, CopyProjects, @@ -19,12 +21,14 @@ export const updateProjectRegionFactory = countProjectModels: GetStreamBranchCount countProjectVersions: GetStreamCommitCount countProjectObjects: GetStreamObjectCount + countProjectAutomations: GetProjectAutomationCount getAvailableRegions: GetAvailableRegions copyWorkspace: CopyWorkspace copyProjects: CopyProjects copyProjectModels: CopyProjectModels copyProjectVersions: CopyProjectVersions copyProjectObjects: CopyProjectObjects + copyProjectAutomations: CopyProjectAutomations }): UpdateProjectRegion => async (params) => { const { projectId, regionKey } = params @@ -67,7 +71,9 @@ export const updateProjectRegionFactory = // Move objects const copiedObjectCount = await deps.copyProjectObjects({ projectIds }) - // TODO: Move automations + // Move automations + const copiedAutomationCount = await deps.copyProjectAutomations({ projectIds }) + // TODO: Move comments // TODO: Move file blobs // TODO: Move webhooks @@ -78,11 +84,15 @@ export const updateProjectRegionFactory = const sourceProjectObjectCount = await deps.countProjectObjects({ streamId: projectId }) + const sourceProjectAutomationCount = await deps.countProjectAutomations({ + projectId + }) const tests = [ copiedModelCount[projectId] === sourceProjectModelCount, copiedVersionCount[projectId] === sourceProjectVersionCount, - copiedObjectCount[projectId] === sourceProjectObjectCount + copiedObjectCount[projectId] === sourceProjectObjectCount, + copiedAutomationCount[projectId] === sourceProjectAutomationCount ] if (!tests.every((test) => !!test)) { diff --git a/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts b/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts index f469a2669..291f8beb7 100644 --- a/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts +++ b/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts @@ -1,4 +1,23 @@ import { db } from '@/db/knex' +import { + AutomationFunctionRunRecord, + AutomationRecord, + AutomationRevisionFunctionRecord, + AutomationRevisionRecord, + AutomationRunRecord, + AutomationRunTriggerRecord, + AutomationTokenRecord, + AutomationTriggerDefinitionRecord +} from '@/modules/automate/helpers/types' +import { + AutomationFunctionRuns, + AutomationRevisionFunctions, + AutomationRevisions, + AutomationRuns, + AutomationRunTriggers, + AutomationTokens, + AutomationTriggers +} from '@/modules/core/dbSchema' import { AllScopes } from '@/modules/core/helpers/mainConstants' import { createRandomEmail } from '@/modules/core/helpers/testHelpers' import { @@ -35,6 +54,10 @@ import { TestApolloServer } from '@/test/graphqlHelper' import { beforeEachContext } from '@/test/hooks' +import { + createTestAutomation, + createTestAutomationRun +} from '@/test/speckle-helpers/automationHelper' import { BasicTestBranch, createTestBranch } from '@/test/speckle-helpers/branchHelper' import { BasicTestCommit, @@ -58,7 +81,20 @@ const tables = { versions: (db: Knex) => db.table('commits'), streamCommits: (db: Knex) => db.table('stream_commits'), branchCommits: (db: Knex) => db.table('branch_commits'), - objects: (db: Knex) => db.table('objects') + objects: (db: Knex) => db.table('objects'), + automations: (db: Knex) => db.table('automations'), + automationTokens: (db: Knex) => db(AutomationTokens.name), + automationRevisions: (db: Knex) => + db(AutomationRevisions.name), + automationTriggers: (db: Knex) => + db(AutomationTriggers.name), + automationRevisionFunctions: (db: Knex) => + db(AutomationRevisionFunctions.name), + automationRuns: (db: Knex) => db(AutomationRuns.name), + automationRunTriggers: (db: Knex) => + db(AutomationRunTriggers.name), + automationFunctionRuns: (db: Knex) => + db(AutomationFunctionRuns.name) } const grantStreamPermissions = grantStreamPermissionsFactory({ db }) @@ -344,6 +380,12 @@ isMultiRegionTestMode() authorId: '' } + let testAutomation: AutomationRecord + let testAutomationToken: AutomationTokenRecord + let testAutomationRevision: AutomationRevisionRecord + let testAutomationRun: AutomationRunRecord + let testAutomationFunctionRuns: AutomationFunctionRunRecord[] + let apollo: TestApolloServer let targetRegionDb: Knex @@ -382,6 +424,32 @@ isMultiRegionTestMode() owner: adminUser, stream: testProject }) + + const { automation, revision } = await createTestAutomation({ + userId: adminUser.id, + projectId: testProject.id, + revision: { + functionId: cryptoRandomString({ length: 9 }), + functionReleaseId: cryptoRandomString({ length: 9 }) + } + }) + + if (!revision) { + throw new Error('Failed to create automation revision.') + } + + testAutomation = automation.automation + testAutomationToken = automation.token + testAutomationRevision = revision + + const { automationRun, functionRuns } = await createTestAutomationRun({ + userId: adminUser.id, + projectId: testProject.id, + automationId: testAutomation.id + }) + + testAutomationRun = automationRun + testAutomationFunctionRuns = functionRuns }) it('moves project record to target regional db', async () => { @@ -468,5 +536,83 @@ isMultiRegionTestMode() expect(object).to.not.be.undefined }) + + it('moves project automation data to target regional db', async () => { + const res = await apollo.execute(UpdateProjectRegionDocument, { + projectId: testProject.id, + regionKey: regionKey2 + }) + + expect(res).to.not.haveGraphQLErrors() + + // TODO: Replace with gql query when possible + const automation = await tables + .automations(targetRegionDb) + .select('*') + .where({ id: testAutomation.id }) + .first() + expect(automation).to.not.be.undefined + + const automationToken = await tables + .automationTokens(targetRegionDb) + .select('*') + .where({ automationId: testAutomation.id }) + .first() + expect(automationToken).to.not.be.undefined + expect(automationToken?.automateToken).to.equal( + testAutomationToken.automateToken + ) + + const automationRevision = await tables + .automationRevisions(targetRegionDb) + .select('*') + .where({ automationId: testAutomation.id }) + .first() + expect(automationRevision).to.not.be.undefined + expect(automationRevision?.id).to.equal(testAutomationRevision.id) + + const automationTrigger = await tables + .automationTriggers(targetRegionDb) + .select('*') + .where({ automationRevisionId: testAutomationRevision.id }) + .first() + expect(automationTrigger).to.not.be.undefined + }) + + it('moves project automation runs to target regional db', async () => { + const res = await apollo.execute(UpdateProjectRegionDocument, { + projectId: testProject.id, + regionKey: regionKey2 + }) + + expect(res).to.not.haveGraphQLErrors() + + // TODO: Replace with gql query when possible + const automationRun = await tables + .automationRuns(targetRegionDb) + .select('*') + .where({ id: testAutomationRun.id }) + .first() + expect(automationRun).to.not.be.undefined + + const automationRunTriggers = await tables + .automationRunTriggers(targetRegionDb) + .select('*') + .where({ automationRunId: testAutomationRun.id }) + expect(automationRunTriggers.length).to.not.equal(0) + + const automationFunctionRuns = await tables + .automationFunctionRuns(targetRegionDb) + .select('*') + .where({ runId: testAutomationRun.id }) + expect(automationFunctionRuns.length).to.equal( + testAutomationFunctionRuns.length + ) + expect( + automationFunctionRuns.every((run) => + testAutomationFunctionRuns.some((testRun) => testRun.id === run.id) + ) + ) + }) }) : void 0 diff --git a/packages/server/test/speckle-helpers/automationHelper.ts b/packages/server/test/speckle-helpers/automationHelper.ts index 0e94fb315..a9b99b3d3 100644 --- a/packages/server/test/speckle-helpers/automationHelper.ts +++ b/packages/server/test/speckle-helpers/automationHelper.ts @@ -1,8 +1,12 @@ import { getAutomationFactory, + getFullAutomationRevisionMetadataFactory, + getFullAutomationRunByIdFactory, + getLatestAutomationRevisionFactory, storeAutomationFactory, storeAutomationRevisionFactory, - storeAutomationTokenFactory + storeAutomationTokenFactory, + upsertAutomationRunFactory } from '@/modules/automate/repositories/automations' import { CreateAutomationRevisionDeps, @@ -15,6 +19,7 @@ import cryptoRandomString from 'crypto-random-string' import { createAutomation as clientCreateAutomation } from '@/modules/automate/clients/executionEngine' import { getBranchesByIdsFactory, + getBranchLatestCommitsFactory, getLatestStreamBranchFactory } from '@/modules/core/repositories/branches' @@ -35,6 +40,7 @@ import { import { faker } from '@faker-js/faker' import { getEncryptionKeyPair, + getEncryptionKeyPairFor, getFunctionInputDecryptorFactory } from '@/modules/automate/services/encryption' import { buildDecryptor } from '@/modules/shared/utils/libsodium' @@ -42,22 +48,28 @@ import { db } from '@/db/knex' import { validateStreamAccessFactory } from '@/modules/core/services/streams/access' import { authorizeResolver } from '@/modules/shared' import { getEventBus } from '@/modules/shared/services/eventBus' +import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector' +import { Knex } from 'knex' +import { createTestAutomationRunFactory } from '@/modules/automate/services/trigger' -const storeAutomation = storeAutomationFactory({ db }) -const storeAutomationToken = storeAutomationTokenFactory({ db }) -const storeAutomationRevision = storeAutomationRevisionFactory({ db }) -const getAutomation = getAutomationFactory({ db }) -const getLatestStreamBranch = getLatestStreamBranchFactory({ db }) const validateStreamAccess = validateStreamAccessFactory({ authorizeResolver }) export const generateFunctionId = () => cryptoRandomString({ length: 10 }) export const generateFunctionReleaseId = () => cryptoRandomString({ length: 10 }) +/** + * @param overrides By default, we mock requests to the execution engine. You can replace those mocks here. + */ export const buildAutomationCreate = ( - overrides?: Partial<{ - createDbAutomation: typeof clientCreateAutomation - }> + params: { + dbClient?: Knex + overrides?: Partial<{ + createDbAutomation: typeof clientCreateAutomation + }> + } = {} ) => { + const { dbClient = db, overrides } = params + const create = createAutomationFactory({ createAuthCode: createStoredAuthCodeFactory({ redis: createInmemoryRedisClient() }), automateCreateAutomation: @@ -66,8 +78,8 @@ export const buildAutomationCreate = ( automationId: cryptoRandomString({ length: 10 }), token: cryptoRandomString({ length: 10 }) })), - storeAutomation, - storeAutomationToken, + storeAutomation: storeAutomationFactory({ db: dbClient }), + storeAutomationToken: storeAutomationTokenFactory({ db: dbClient }), validateStreamAccess, eventEmit: getEventBus().emit }) @@ -75,9 +87,17 @@ export const buildAutomationCreate = ( return create } +/** + * @param overrides By default, we mock requests to the execution engine. You can replace those mocks here. + */ export const buildAutomationRevisionCreate = ( - overrides?: Partial + params: { + dbClient?: Knex + overrides?: Partial + } = {} ) => { + const { dbClient = db, overrides } = params + const fakeGetRelease = (params: { functionReleaseId: string functionId: string @@ -91,9 +111,9 @@ export const buildAutomationRevisionCreate = ( }) const create = createAutomationRevisionFactory({ - getAutomation, - storeAutomationRevision, - getBranchesByIds: getBranchesByIdsFactory({ db }), + getAutomation: getAutomationFactory({ db: dbClient }), + storeAutomationRevision: storeAutomationRevisionFactory({ db: dbClient }), + getBranchesByIds: getBranchesByIdsFactory({ db: dbClient }), getFunctionRelease: async (params) => fakeGetRelease(params), getFunctionReleases: async (params) => params.ids.map(fakeGetRelease), getEncryptionKeyPair, @@ -128,8 +148,10 @@ export const createTestAutomation = async (params: { revision: { input: revisionInput, functionReleaseId, functionId } = {} } = params - const createAutomation = buildAutomationCreate() - const createRevision = buildAutomationRevisionCreate() + const projectDb = await getProjectDbClient({ projectId }) + + const createAutomation = buildAutomationCreate({ dbClient: projectDb }) + const createRevision = buildAutomationRevisionCreate({ dbClient: projectDb }) const automationRet = await createAutomation({ input: { @@ -143,7 +165,7 @@ export const createTestAutomation = async (params: { let revisionRet: Awaited> | null = null if (functionReleaseId?.length && functionId?.length) { - const firstModel = await getLatestStreamBranch(projectId) + const firstModel = await getLatestStreamBranchFactory({ db: projectDb })(projectId) if (!firstModel) throw new Error( @@ -186,6 +208,55 @@ export type TestAutomationWithRevision = Awaited< ReturnType > +export const createTestAutomationRun = async (params: { + userId: string + projectId: string + automationId: string +}) => { + const { userId, projectId, automationId } = params + + const projectDb = await getProjectDbClient({ projectId }) + + const { automationRunId } = await createTestAutomationRunFactory({ + getEncryptionKeyPairFor, + getFunctionInputDecryptor: getFunctionInputDecryptorFactory({ + buildDecryptor + }), + getAutomation: getAutomationFactory({ + db: projectDb + }), + getLatestAutomationRevision: getLatestAutomationRevisionFactory({ + db: projectDb + }), + getFullAutomationRevisionMetadata: getFullAutomationRevisionMetadataFactory({ + db: projectDb + }), + upsertAutomationRun: upsertAutomationRunFactory({ + db: projectDb + }), + getBranchLatestCommits: getBranchLatestCommitsFactory({ + db: projectDb + }), + validateStreamAccess + })({ projectId, automationId, userId }) + + const automationRunData = await getFullAutomationRunByIdFactory({ db: projectDb })( + automationRunId + ) + + if (!automationRunData) { + throw new Error('Failed to create test automation run!') + } + + const { triggers, functionRuns, ...automationRun } = automationRunData + + return { + automationRun, + functionRuns, + triggers + } +} + export const truncateAutomations = async () => { await truncateTables([ AutomationRunTriggers.name,