feat(regions): move project automations (#3925)
* feat(regions): repo functions for copying project branches and commits * chore(regions): wire up move to resolver * chore(regions): successful basic test of project region change * fix(regions): sabrina carpenter please please please * fix(regions): repair multiregion test setup * chore(regions): appease ts * chore(multiregion): update test multiregion config * chore(multiregion): fix test docker config and test * chore(multiregion): use transaction * chore(multiregion): maybe this will work * fix(multiregion): drop subs synchronously * chore(multiregion): desperate test logs * chore(multiregion): somehow that worked? * chore(multiregion): add load-bearing log statement * chore(multiregion): move services * fix(multiregion): test drop waits * chore(regions): fix import * chore(regions): make test a bit more thorough for good measure * fix(regions): move project objects * chore(regions): add tests for object move * feat(regions): move project automations * chore(regions): add tests for moving automations * chore(regions): more tests for moving automate data * fix(regions): speed up inserts * fix(regions): simplify postgres usage * chore(regions): repair build * fix(regions): improve queries * chore(regions): again
This commit is contained in:
Chuck Driesler
@@ -366,3 +366,6 @@ export type CopyProjectVersions = (params: {
|
||||
export type CopyProjectObjects = (params: {
|
||||
projectIds: string[]
|
||||
}) => Promise<Record<string, number>>
|
||||
export type CopyProjectAutomations = (params: {
|
||||
projectIds: string[]
|
||||
}) => Promise<Record<string, number>>
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
upsertRegionAssignmentFactory
|
||||
} from '@/modules/workspaces/repositories/regions'
|
||||
import {
|
||||
copyProjectAutomationsFactory,
|
||||
copyProjectModelsFactory,
|
||||
copyProjectObjectsFactory,
|
||||
copyProjectsFactory,
|
||||
@@ -31,6 +32,7 @@ import { getStreamBranchCountFactory } from '@/modules/core/repositories/branche
|
||||
import { getStreamCommitCountFactory } from '@/modules/core/repositories/commits'
|
||||
import { withTransaction } from '@/modules/shared/helpers/dbHelper'
|
||||
import { getStreamObjectCountFactory } from '@/modules/core/repositories/objects'
|
||||
import { getProjectAutomationsTotalCountFactory } from '@/modules/automate/repositories/automations'
|
||||
import { getFeatureFlags, isTestEnv } from '@/modules/shared/helpers/envHelper'
|
||||
import { WorkspacesNotYetImplementedError } from '@/modules/workspaces/errors/workspace'
|
||||
|
||||
@@ -92,6 +94,9 @@ export default {
|
||||
countProjectModels: getStreamBranchCountFactory({ db: sourceDb }),
|
||||
countProjectVersions: getStreamCommitCountFactory({ db: sourceDb }),
|
||||
countProjectObjects: getStreamObjectCountFactory({ db: sourceDb }),
|
||||
countProjectAutomations: getProjectAutomationsTotalCountFactory({
|
||||
db: sourceDb
|
||||
}),
|
||||
getAvailableRegions: getAvailableRegionsFactory({
|
||||
getRegions: getRegionsFactory({ db }),
|
||||
canWorkspaceUseRegions: canWorkspaceUseRegionsFactory({
|
||||
@@ -102,7 +107,8 @@ export default {
|
||||
copyProjects: copyProjectsFactory({ sourceDb, targetDb }),
|
||||
copyProjectModels: copyProjectModelsFactory({ sourceDb, targetDb }),
|
||||
copyProjectVersions: copyProjectVersionsFactory({ sourceDb, targetDb }),
|
||||
copyProjectObjects: copyProjectObjectsFactory({ sourceDb, targetDb })
|
||||
copyProjectObjects: copyProjectObjectsFactory({ sourceDb, targetDb }),
|
||||
copyProjectAutomations: copyProjectAutomationsFactory({ sourceDb, targetDb })
|
||||
})
|
||||
|
||||
return await withTransaction(updateProjectRegion(args), targetDb)
|
||||
|
||||
@@ -1,4 +1,12 @@
|
||||
import {
|
||||
AutomationFunctionRuns,
|
||||
AutomationRevisionFunctions,
|
||||
AutomationRevisions,
|
||||
AutomationRuns,
|
||||
AutomationRunTriggers,
|
||||
Automations,
|
||||
AutomationTokens,
|
||||
AutomationTriggers,
|
||||
BranchCommits,
|
||||
Branches,
|
||||
Commits,
|
||||
@@ -21,6 +29,7 @@ import {
|
||||
} from '@/modules/core/helpers/types'
|
||||
import { executeBatchedSelect } from '@/modules/shared/helpers/dbHelper'
|
||||
import {
|
||||
CopyProjectAutomations,
|
||||
CopyProjectModels,
|
||||
CopyProjectObjects,
|
||||
CopyProjects,
|
||||
@@ -32,6 +41,16 @@ import { Knex } from 'knex'
|
||||
import { Workspace } from '@/modules/workspacesCore/domain/types'
|
||||
import { Workspaces } from '@/modules/workspacesCore/helpers/db'
|
||||
import { ObjectPreview } from '@/modules/previews/domain/types'
|
||||
import {
|
||||
AutomationFunctionRunRecord,
|
||||
AutomationRecord,
|
||||
AutomationRevisionFunctionRecord,
|
||||
AutomationRevisionRecord,
|
||||
AutomationRunRecord,
|
||||
AutomationRunTriggerRecord,
|
||||
AutomationTokenRecord,
|
||||
AutomationTriggerDefinitionRecord
|
||||
} from '@/modules/automate/helpers/types'
|
||||
|
||||
const tables = {
|
||||
workspaces: (db: Knex) => db<Workspace>(Workspaces.name),
|
||||
@@ -43,7 +62,20 @@ const tables = {
|
||||
streamFavorites: (db: Knex) => db<StreamFavoriteRecord>(StreamFavorites.name),
|
||||
streamsMeta: (db: Knex) => db(StreamsMeta.name),
|
||||
objects: (db: Knex) => db<ObjectRecord>(Objects.name),
|
||||
objectPreviews: (db: Knex) => db<ObjectPreview>('object_preview')
|
||||
objectPreviews: (db: Knex) => db<ObjectPreview>('object_preview'),
|
||||
automations: (db: Knex) => db<AutomationRecord>(Automations.name),
|
||||
automationTokens: (db: Knex) => db<AutomationTokenRecord>(AutomationTokens.name),
|
||||
automationRevisions: (db: Knex) =>
|
||||
db<AutomationRevisionRecord>(AutomationRevisions.name),
|
||||
automationTriggers: (db: Knex) =>
|
||||
db<AutomationTriggerDefinitionRecord>(AutomationTriggers.name),
|
||||
automationRevisionFunctions: (db: Knex) =>
|
||||
db<AutomationRevisionFunctionRecord>(AutomationRevisionFunctions.name),
|
||||
automationRuns: (db: Knex) => db<AutomationRunRecord>(AutomationRuns.name),
|
||||
automationRunTriggers: (db: Knex) =>
|
||||
db<AutomationRunTriggerRecord>(AutomationRunTriggers.name),
|
||||
automationFunctionRuns: (db: Knex) =>
|
||||
db<AutomationFunctionRunRecord>(AutomationFunctionRuns.name)
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -283,3 +315,147 @@ export const copyProjectObjectsFactory =
|
||||
|
||||
return copiedObjectCountByProjectId
|
||||
}
|
||||
|
||||
/**
|
||||
* Copies rows from the following tables:
|
||||
* - automations
|
||||
* - automation_tokens
|
||||
* - automation_revisions
|
||||
* - automation_triggers
|
||||
* - automation_revision_functions
|
||||
* - automation_runs
|
||||
* - automation_run_triggers
|
||||
* - automation_function_runs
|
||||
*/
|
||||
export const copyProjectAutomationsFactory =
|
||||
(deps: { sourceDb: Knex; targetDb: Knex }): CopyProjectAutomations =>
|
||||
async ({ projectIds }) => {
|
||||
const copiedAutomationCountByProjectId: Record<string, number> = {}
|
||||
|
||||
// Copy `automations` table rows in batches
|
||||
const selectAutomations = tables
|
||||
.automations(deps.sourceDb)
|
||||
.select('*')
|
||||
.whereIn(Automations.col.projectId, projectIds)
|
||||
|
||||
for await (const automations of executeBatchedSelect(selectAutomations)) {
|
||||
const automationIds = automations.map((automation) => automation.id)
|
||||
|
||||
// Write `automations` table rows to target db
|
||||
await tables
|
||||
.automations(deps.targetDb)
|
||||
// Cast ignores unexpected behavior in how knex handles object union types
|
||||
.insert(automations as unknown as AutomationRecord)
|
||||
.onConflict()
|
||||
.ignore()
|
||||
|
||||
for (const automation of automations) {
|
||||
copiedAutomationCountByProjectId[automation.projectId] ??= 0
|
||||
copiedAutomationCountByProjectId[automation.projectId]++
|
||||
}
|
||||
|
||||
// Copy `automation_tokens` rows for automation
|
||||
const selectAutomationTokens = tables
|
||||
.automationTokens(deps.sourceDb)
|
||||
.select('*')
|
||||
.whereIn(AutomationTokens.col.automationId, automationIds)
|
||||
|
||||
for await (const tokens of executeBatchedSelect(selectAutomationTokens)) {
|
||||
// Write `automation_tokens` row to target db
|
||||
await tables
|
||||
.automationTokens(deps.targetDb)
|
||||
.insert(tokens)
|
||||
.onConflict()
|
||||
.ignore()
|
||||
}
|
||||
|
||||
// Copy `automation_revisions` rows for automation
|
||||
const selectAutomationRevisions = tables
|
||||
.automationRevisions(deps.sourceDb)
|
||||
.select('*')
|
||||
.whereIn(AutomationRevisions.col.automationId, automationIds)
|
||||
|
||||
for await (const automationRevisions of executeBatchedSelect(
|
||||
selectAutomationRevisions
|
||||
)) {
|
||||
const automationRevisionIds = automationRevisions.map((revision) => revision.id)
|
||||
|
||||
// Write `automation_revisions` rows to target db
|
||||
await tables
|
||||
.automationRevisions(deps.targetDb)
|
||||
.insert(automationRevisions)
|
||||
.onConflict()
|
||||
.ignore()
|
||||
|
||||
// Copy `automation_triggers` rows for automation revisions
|
||||
const automationTriggers = await tables
|
||||
.automationTriggers(deps.sourceDb)
|
||||
.select('*')
|
||||
.whereIn(AutomationTriggers.col.automationRevisionId, automationRevisionIds)
|
||||
|
||||
await tables
|
||||
.automationTriggers(deps.targetDb)
|
||||
.insert(automationTriggers)
|
||||
.onConflict()
|
||||
.ignore()
|
||||
|
||||
// Copy `automation_revision_functions` rows for automation revisions
|
||||
const automationRevisionFunctions = await tables
|
||||
.automationRevisionFunctions(deps.sourceDb)
|
||||
.select('*')
|
||||
.whereIn(
|
||||
AutomationRevisionFunctions.col.automationRevisionId,
|
||||
automationRevisionIds
|
||||
)
|
||||
|
||||
await tables
|
||||
.automationRevisionFunctions(deps.targetDb)
|
||||
.insert(automationRevisionFunctions)
|
||||
.onConflict()
|
||||
.ignore()
|
||||
|
||||
// Copy `automation_runs` rows for automation revision
|
||||
const selectAutomationRuns = tables
|
||||
.automationRuns(deps.sourceDb)
|
||||
.select('*')
|
||||
.whereIn(AutomationRuns.col.automationRevisionId, automationRevisionIds)
|
||||
|
||||
for await (const automationRuns of executeBatchedSelect(selectAutomationRuns)) {
|
||||
const automationRunIds = automationRuns.map((run) => run.id)
|
||||
|
||||
// Write `automation_runs` row to target db
|
||||
await tables
|
||||
.automationRuns(deps.targetDb)
|
||||
.insert(automationRuns)
|
||||
.onConflict()
|
||||
.ignore()
|
||||
|
||||
// Copy `automation_run_triggers` rows for automation run
|
||||
const automationRunTriggers = await tables
|
||||
.automationRunTriggers(deps.sourceDb)
|
||||
.select('*')
|
||||
.whereIn(AutomationRunTriggers.col.automationRunId, automationRunIds)
|
||||
|
||||
await tables
|
||||
.automationRunTriggers(deps.targetDb)
|
||||
.insert(automationRunTriggers)
|
||||
.onConflict()
|
||||
.ignore()
|
||||
|
||||
// Copy `automation_function_runs` rows for automation run
|
||||
const automationFunctionRuns = await tables
|
||||
.automationFunctionRuns(deps.sourceDb)
|
||||
.select('*')
|
||||
.whereIn(AutomationFunctionRuns.col.runId, automationRunIds)
|
||||
|
||||
await tables
|
||||
.automationFunctionRuns(deps.targetDb)
|
||||
.insert(automationFunctionRuns)
|
||||
.onConflict()
|
||||
.ignore()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return copiedAutomationCountByProjectId
|
||||
}
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
import { GetProjectAutomationCount } from '@/modules/automate/domain/operations'
|
||||
import { GetStreamBranchCount } from '@/modules/core/domain/branches/operations'
|
||||
import { GetStreamCommitCount } from '@/modules/core/domain/commits/operations'
|
||||
import { GetStreamObjectCount } from '@/modules/core/domain/objects/operations'
|
||||
import { GetProject } from '@/modules/core/domain/projects/operations'
|
||||
import {
|
||||
CopyProjectAutomations,
|
||||
CopyProjectModels,
|
||||
CopyProjectObjects,
|
||||
CopyProjects,
|
||||
@@ -19,12 +21,14 @@ export const updateProjectRegionFactory =
|
||||
countProjectModels: GetStreamBranchCount
|
||||
countProjectVersions: GetStreamCommitCount
|
||||
countProjectObjects: GetStreamObjectCount
|
||||
countProjectAutomations: GetProjectAutomationCount
|
||||
getAvailableRegions: GetAvailableRegions
|
||||
copyWorkspace: CopyWorkspace
|
||||
copyProjects: CopyProjects
|
||||
copyProjectModels: CopyProjectModels
|
||||
copyProjectVersions: CopyProjectVersions
|
||||
copyProjectObjects: CopyProjectObjects
|
||||
copyProjectAutomations: CopyProjectAutomations
|
||||
}): UpdateProjectRegion =>
|
||||
async (params) => {
|
||||
const { projectId, regionKey } = params
|
||||
@@ -67,7 +71,9 @@ export const updateProjectRegionFactory =
|
||||
// Move objects
|
||||
const copiedObjectCount = await deps.copyProjectObjects({ projectIds })
|
||||
|
||||
// TODO: Move automations
|
||||
// Move automations
|
||||
const copiedAutomationCount = await deps.copyProjectAutomations({ projectIds })
|
||||
|
||||
// TODO: Move comments
|
||||
// TODO: Move file blobs
|
||||
// TODO: Move webhooks
|
||||
@@ -78,11 +84,15 @@ export const updateProjectRegionFactory =
|
||||
const sourceProjectObjectCount = await deps.countProjectObjects({
|
||||
streamId: projectId
|
||||
})
|
||||
const sourceProjectAutomationCount = await deps.countProjectAutomations({
|
||||
projectId
|
||||
})
|
||||
|
||||
const tests = [
|
||||
copiedModelCount[projectId] === sourceProjectModelCount,
|
||||
copiedVersionCount[projectId] === sourceProjectVersionCount,
|
||||
copiedObjectCount[projectId] === sourceProjectObjectCount
|
||||
copiedObjectCount[projectId] === sourceProjectObjectCount,
|
||||
copiedAutomationCount[projectId] === sourceProjectAutomationCount
|
||||
]
|
||||
|
||||
if (!tests.every((test) => !!test)) {
|
||||
|
||||
@@ -1,4 +1,23 @@
|
||||
import { db } from '@/db/knex'
|
||||
import {
|
||||
AutomationFunctionRunRecord,
|
||||
AutomationRecord,
|
||||
AutomationRevisionFunctionRecord,
|
||||
AutomationRevisionRecord,
|
||||
AutomationRunRecord,
|
||||
AutomationRunTriggerRecord,
|
||||
AutomationTokenRecord,
|
||||
AutomationTriggerDefinitionRecord
|
||||
} from '@/modules/automate/helpers/types'
|
||||
import {
|
||||
AutomationFunctionRuns,
|
||||
AutomationRevisionFunctions,
|
||||
AutomationRevisions,
|
||||
AutomationRuns,
|
||||
AutomationRunTriggers,
|
||||
AutomationTokens,
|
||||
AutomationTriggers
|
||||
} from '@/modules/core/dbSchema'
|
||||
import { AllScopes } from '@/modules/core/helpers/mainConstants'
|
||||
import { createRandomEmail } from '@/modules/core/helpers/testHelpers'
|
||||
import {
|
||||
@@ -35,6 +54,10 @@ import {
|
||||
TestApolloServer
|
||||
} from '@/test/graphqlHelper'
|
||||
import { beforeEachContext } from '@/test/hooks'
|
||||
import {
|
||||
createTestAutomation,
|
||||
createTestAutomationRun
|
||||
} from '@/test/speckle-helpers/automationHelper'
|
||||
import { BasicTestBranch, createTestBranch } from '@/test/speckle-helpers/branchHelper'
|
||||
import {
|
||||
BasicTestCommit,
|
||||
@@ -58,7 +81,20 @@ const tables = {
|
||||
versions: (db: Knex) => db.table<CommitRecord>('commits'),
|
||||
streamCommits: (db: Knex) => db.table<StreamCommitRecord>('stream_commits'),
|
||||
branchCommits: (db: Knex) => db.table<BranchCommitRecord>('branch_commits'),
|
||||
objects: (db: Knex) => db.table<ObjectRecord>('objects')
|
||||
objects: (db: Knex) => db.table<ObjectRecord>('objects'),
|
||||
automations: (db: Knex) => db.table<AutomationRecord>('automations'),
|
||||
automationTokens: (db: Knex) => db<AutomationTokenRecord>(AutomationTokens.name),
|
||||
automationRevisions: (db: Knex) =>
|
||||
db<AutomationRevisionRecord>(AutomationRevisions.name),
|
||||
automationTriggers: (db: Knex) =>
|
||||
db<AutomationTriggerDefinitionRecord>(AutomationTriggers.name),
|
||||
automationRevisionFunctions: (db: Knex) =>
|
||||
db<AutomationRevisionFunctionRecord>(AutomationRevisionFunctions.name),
|
||||
automationRuns: (db: Knex) => db<AutomationRunRecord>(AutomationRuns.name),
|
||||
automationRunTriggers: (db: Knex) =>
|
||||
db<AutomationRunTriggerRecord>(AutomationRunTriggers.name),
|
||||
automationFunctionRuns: (db: Knex) =>
|
||||
db<AutomationFunctionRunRecord>(AutomationFunctionRuns.name)
|
||||
}
|
||||
|
||||
const grantStreamPermissions = grantStreamPermissionsFactory({ db })
|
||||
@@ -344,6 +380,12 @@ isMultiRegionTestMode()
|
||||
authorId: ''
|
||||
}
|
||||
|
||||
let testAutomation: AutomationRecord
|
||||
let testAutomationToken: AutomationTokenRecord
|
||||
let testAutomationRevision: AutomationRevisionRecord
|
||||
let testAutomationRun: AutomationRunRecord
|
||||
let testAutomationFunctionRuns: AutomationFunctionRunRecord[]
|
||||
|
||||
let apollo: TestApolloServer
|
||||
let targetRegionDb: Knex
|
||||
|
||||
@@ -382,6 +424,32 @@ isMultiRegionTestMode()
|
||||
owner: adminUser,
|
||||
stream: testProject
|
||||
})
|
||||
|
||||
const { automation, revision } = await createTestAutomation({
|
||||
userId: adminUser.id,
|
||||
projectId: testProject.id,
|
||||
revision: {
|
||||
functionId: cryptoRandomString({ length: 9 }),
|
||||
functionReleaseId: cryptoRandomString({ length: 9 })
|
||||
}
|
||||
})
|
||||
|
||||
if (!revision) {
|
||||
throw new Error('Failed to create automation revision.')
|
||||
}
|
||||
|
||||
testAutomation = automation.automation
|
||||
testAutomationToken = automation.token
|
||||
testAutomationRevision = revision
|
||||
|
||||
const { automationRun, functionRuns } = await createTestAutomationRun({
|
||||
userId: adminUser.id,
|
||||
projectId: testProject.id,
|
||||
automationId: testAutomation.id
|
||||
})
|
||||
|
||||
testAutomationRun = automationRun
|
||||
testAutomationFunctionRuns = functionRuns
|
||||
})
|
||||
|
||||
it('moves project record to target regional db', async () => {
|
||||
@@ -468,5 +536,83 @@ isMultiRegionTestMode()
|
||||
|
||||
expect(object).to.not.be.undefined
|
||||
})
|
||||
|
||||
it('moves project automation data to target regional db', async () => {
|
||||
const res = await apollo.execute(UpdateProjectRegionDocument, {
|
||||
projectId: testProject.id,
|
||||
regionKey: regionKey2
|
||||
})
|
||||
|
||||
expect(res).to.not.haveGraphQLErrors()
|
||||
|
||||
// TODO: Replace with gql query when possible
|
||||
const automation = await tables
|
||||
.automations(targetRegionDb)
|
||||
.select('*')
|
||||
.where({ id: testAutomation.id })
|
||||
.first()
|
||||
expect(automation).to.not.be.undefined
|
||||
|
||||
const automationToken = await tables
|
||||
.automationTokens(targetRegionDb)
|
||||
.select('*')
|
||||
.where({ automationId: testAutomation.id })
|
||||
.first()
|
||||
expect(automationToken).to.not.be.undefined
|
||||
expect(automationToken?.automateToken).to.equal(
|
||||
testAutomationToken.automateToken
|
||||
)
|
||||
|
||||
const automationRevision = await tables
|
||||
.automationRevisions(targetRegionDb)
|
||||
.select('*')
|
||||
.where({ automationId: testAutomation.id })
|
||||
.first()
|
||||
expect(automationRevision).to.not.be.undefined
|
||||
expect(automationRevision?.id).to.equal(testAutomationRevision.id)
|
||||
|
||||
const automationTrigger = await tables
|
||||
.automationTriggers(targetRegionDb)
|
||||
.select('*')
|
||||
.where({ automationRevisionId: testAutomationRevision.id })
|
||||
.first()
|
||||
expect(automationTrigger).to.not.be.undefined
|
||||
})
|
||||
|
||||
it('moves project automation runs to target regional db', async () => {
|
||||
const res = await apollo.execute(UpdateProjectRegionDocument, {
|
||||
projectId: testProject.id,
|
||||
regionKey: regionKey2
|
||||
})
|
||||
|
||||
expect(res).to.not.haveGraphQLErrors()
|
||||
|
||||
// TODO: Replace with gql query when possible
|
||||
const automationRun = await tables
|
||||
.automationRuns(targetRegionDb)
|
||||
.select('*')
|
||||
.where({ id: testAutomationRun.id })
|
||||
.first()
|
||||
expect(automationRun).to.not.be.undefined
|
||||
|
||||
const automationRunTriggers = await tables
|
||||
.automationRunTriggers(targetRegionDb)
|
||||
.select('*')
|
||||
.where({ automationRunId: testAutomationRun.id })
|
||||
expect(automationRunTriggers.length).to.not.equal(0)
|
||||
|
||||
const automationFunctionRuns = await tables
|
||||
.automationFunctionRuns(targetRegionDb)
|
||||
.select('*')
|
||||
.where({ runId: testAutomationRun.id })
|
||||
expect(automationFunctionRuns.length).to.equal(
|
||||
testAutomationFunctionRuns.length
|
||||
)
|
||||
expect(
|
||||
automationFunctionRuns.every((run) =>
|
||||
testAutomationFunctionRuns.some((testRun) => testRun.id === run.id)
|
||||
)
|
||||
)
|
||||
})
|
||||
})
|
||||
: void 0
|
||||
|
||||
Reference in New Issue
Block a user