feat(regions): move project automations (#3925)

* feat(regions): repo functions for copying project branches and commits

* chore(regions): wire up move to resolver

* chore(regions): successful basic test of project region change

* fix(regions): sabrina carpenter please please please

* fix(regions): repair multiregion test setup

* chore(regions): appease ts

* chore(multiregion): update test multiregion config

* chore(multiregion): fix test docker config and test

* chore(multiregion): use transaction

* chore(multiregion): maybe this will work

* fix(multiregion): drop subs synchronously

* chore(multiregion): desperate test logs

* chore(multiregion): somehow that worked?

* chore(multiregion): add load-bearing log statement

* chore(multiregion): move services

* fix(multiregion): test drop waits

* chore(regions): fix import

* chore(regions): make test a bit more thorough for good measure

* fix(regions): move project objects

* chore(regions): add tests for object move

* feat(regions): move project automations

* chore(regions): add tests for moving automations

* chore(regions): more tests for moving automate data

* fix(regions): speed up inserts

* fix(regions): simplify postgres usage

* chore(regions): repair build

* fix(regions): improve queries

* chore(regions): again
This commit is contained in:
Chuck Driesler
2025-02-18 15:48:00 +00:00
committed by GitHub
parent 974598b1fd
commit 86c113b29b
14 changed files with 469 additions and 47 deletions
@@ -1,6 +1,6 @@
import { InsertableAutomationFunctionRun } from '@/modules/automate/domain/types'
import {
AutomateRevisionFunctionRecord,
AutomationRevisionFunctionRecord,
AutomationFunctionRunRecord,
AutomationRecord,
AutomationRevisionRecord,
@@ -175,7 +175,7 @@ export type GetRevisionsTriggerDefinitions = (params: {
export type GetRevisionsFunctions = (params: {
automationRevisionIds: string[]
}) => Promise<{ [automationRevisionId: string]: AutomateRevisionFunctionRecord[] }>
}) => Promise<{ [automationRevisionId: string]: AutomationRevisionFunctionRecord[] }>
export type CreateStoredAuthCode = (
params: Omit<AuthCodePayload, 'code'>
@@ -204,3 +204,7 @@ export type TriggerAutomationRevisionRun = <
manifest: M
source: RunTriggerSource
}) => Promise<{ automationRunId: string }>
export type GetProjectAutomationCount = (params: {
projectId: string
}) => Promise<number>
@@ -1,5 +1,5 @@
import {
AutomateRevisionFunctionRecord,
AutomationRevisionFunctionRecord,
AutomationFunctionRunRecord,
AutomationRecord,
AutomationRevisionRecord,
@@ -56,7 +56,7 @@ export type AutomationRunTriggerGraphQLReturn = AutomationRunTriggerRecord & {
}
export type AutomationRevisionFunctionGraphQLReturn = Merge<
AutomateRevisionFunctionRecord,
AutomationRevisionFunctionRecord,
{
functionInputs: Nullable<Record<string, unknown>>
release: AutomateFunctionReleaseGraphQLReturn
@@ -66,7 +66,7 @@ export type AutomationRunRecord = {
executionEngineRunId: string | null
}
export type AutomateRevisionFunctionRecord = {
export type AutomationRevisionFunctionRecord = {
functionReleaseId: string
functionId: string
functionInputs: string | null
@@ -122,7 +122,7 @@ export type AutomationTokenRecord = {
}
export type AutomationRevisionWithTriggersFunctions = AutomationRevisionRecord & {
functions: AutomateRevisionFunctionRecord[]
functions: AutomationRevisionFunctionRecord[]
triggers: AutomationTriggerDefinitionRecord[]
}
@@ -16,6 +16,7 @@ import {
GetLatestAutomationRevision,
GetLatestAutomationRevisions,
GetLatestVersionAutomationRuns,
GetProjectAutomationCount,
GetRevisionsFunctions,
GetRevisionsTriggerDefinitions,
StoreAutomation,
@@ -37,7 +38,7 @@ import {
AutomationRunRecord,
AutomationTokenRecord,
AutomationTriggerRecordBase,
AutomateRevisionFunctionRecord,
AutomationRevisionFunctionRecord,
AutomationRunWithTriggersFunctionRuns,
AutomationRunTriggerRecord,
AutomationFunctionRunRecord,
@@ -86,7 +87,7 @@ const tables = {
automationRevisions: (db: Knex) =>
db<AutomationRevisionRecord>(AutomationRevisions.name),
automationRevisionFunctions: (db: Knex) =>
db<AutomateRevisionFunctionRecord>(AutomationRevisionFunctions.name),
db<AutomationRevisionFunctionRecord>(AutomationRevisionFunctions.name),
automationTriggers: (db: Knex) =>
db<AutomationTriggerDefinitionRecord>(AutomationTriggers.name),
automationRuns: (db: Knex) => db<AutomationRunRecord>(AutomationRuns.name),
@@ -321,7 +322,7 @@ export const storeAutomationTokenFactory =
}
export type InsertableAutomationRevisionFunction = Omit<
AutomateRevisionFunctionRecord,
AutomationRevisionFunctionRecord,
'automationRevisionId'
>
@@ -369,7 +370,7 @@ export const storeAutomationRevisionFactory =
.automationRevisionFunctions(deps.db)
.insert(
revision.functions.map(
(f): AutomateRevisionFunctionRecord => ({
(f): AutomationRevisionFunctionRecord => ({
...f,
automationRevisionId: id
})
@@ -742,10 +743,12 @@ const getProjectAutomationsBaseQueryFactory =
}
export const getProjectAutomationsTotalCountFactory =
(deps: { db: Knex }) => async (params: GetProjectAutomationsParams) => {
const q = getProjectAutomationsBaseQueryFactory(deps)(params).count<
[{ count: string }]
>(Automations.col.id)
(deps: { db: Knex }): GetProjectAutomationCount =>
async ({ projectId }) => {
const q = getProjectAutomationsBaseQueryFactory(deps)({
projectId,
args: {}
}).count<[{ count: string }]>(Automations.col.id)
const [ret] = await q
@@ -7,7 +7,7 @@ import { Nullable, Optional } from '@speckle/shared'
import { MisconfiguredEnvironmentError } from '@/modules/shared/errors'
import { AutomationFunctionInputEncryptionError } from '@/modules/automate/errors/management'
import { KeyPair, buildDecryptor } from '@/modules/shared/utils/libsodium'
import { AutomateRevisionFunctionRecord } from '@/modules/automate/helpers/types'
import { AutomationRevisionFunctionRecord } from '@/modules/automate/helpers/types'
import { AutomationRevisionFunctionGraphQLReturn } from '@/modules/automate/helpers/graphTypes'
import { FunctionReleaseSchemaType } from '@/modules/automate/helpers/executionEngine'
import { LibsodiumEncryptionError } from '@/modules/shared/errors/encryption'
@@ -118,7 +118,7 @@ export type GetFunctionInputsForFrontendDeps = {
} & GetFunctionInputDecryptorDeps
export type AutomationRevisionFunctionForInputRedaction = Merge<
AutomateRevisionFunctionRecord,
AutomationRevisionFunctionRecord,
{ release: FunctionReleaseSchemaType }
>
@@ -46,6 +46,7 @@ import { ValidateStreamAccess } from '@/modules/core/domain/streams/operations'
import { CreateAndStoreAppToken } from '@/modules/core/domain/tokens/operations'
import { EventBusEmit } from '@/modules/shared/services/eventBus'
import { AutomationRunEvents } from '@/modules/automate/domain/events'
import { isTestEnv } from '@/modules/shared/helpers/envHelper'
export type OnModelVersionCreateDeps = {
getAutomation: GetAutomation
@@ -131,7 +132,7 @@ type CreateAutomationRunDataDeps = {
getFunctionInputDecryptor: FunctionInputDecryptor
}
const createAutomationRunDataFactory =
export const createAutomationRunDataFactory =
(deps: CreateAutomationRunDataDeps) =>
async (params: {
manifests: BaseTriggerManifest[]
@@ -414,7 +415,7 @@ type ComposeTriggerDataDeps = {
getBranchLatestCommits: GetBranchLatestCommits
}
const composeTriggerDataFactory =
export const composeTriggerDataFactory =
(deps: ComposeTriggerDataDeps) =>
async (params: {
projectId: string
@@ -575,7 +576,7 @@ export const createTestAutomationRunFactory =
throw new TriggerAutomationError('Automation not found')
}
if (!automationRecord.isTestAutomation) {
if (!isTestEnv() && !automationRecord.isTestAutomation) {
throw new TriggerAutomationError(
'Automation is not a test automation and cannot create test function runs'
)
@@ -510,9 +510,11 @@ const buildAutomationUpdate = () => {
it('fails when refering to nonexistent function releases', async () => {
const create = buildAutomationRevisionCreate({
getFunctionRelease: async () => {
// TODO: Update once we know how exec engine should respond
throw new Error('Function release with ID XXX not found')
overrides: {
getFunctionRelease: async () => {
// TODO: Update once we know how exec engine should respond
throw new Error('Function release with ID XXX not found')
}
}
})
@@ -56,7 +56,7 @@ import { Users } from '@/modules/core/dbSchema'
import { getStreamPendingModelsFactory } from '@/modules/fileuploads/repositories/fileUploads'
import { FileUploadRecord } from '@/modules/fileuploads/helpers/types'
import {
AutomateRevisionFunctionRecord,
AutomationRevisionFunctionRecord,
AutomationRecord,
AutomationRevisionRecord,
AutomationRunTriggerRecord,
@@ -595,7 +595,7 @@ const dataLoadersDefinition = defineRequestDataloaders(
})
return ids.map((i) => results[i] || [])
}),
getRevisionFunctions: createLoader<string, AutomateRevisionFunctionRecord[]>(
getRevisionFunctions: createLoader<string, AutomationRevisionFunctionRecord[]>(
async (ids) => {
const results = await getRevisionsFunctions({
automationRevisionIds: ids.slice()
@@ -366,3 +366,6 @@ export type CopyProjectVersions = (params: {
export type CopyProjectObjects = (params: {
projectIds: string[]
}) => Promise<Record<string, number>>
export type CopyProjectAutomations = (params: {
projectIds: string[]
}) => Promise<Record<string, number>>
@@ -10,6 +10,7 @@ import {
upsertRegionAssignmentFactory
} from '@/modules/workspaces/repositories/regions'
import {
copyProjectAutomationsFactory,
copyProjectModelsFactory,
copyProjectObjectsFactory,
copyProjectsFactory,
@@ -31,6 +32,7 @@ import { getStreamBranchCountFactory } from '@/modules/core/repositories/branche
import { getStreamCommitCountFactory } from '@/modules/core/repositories/commits'
import { withTransaction } from '@/modules/shared/helpers/dbHelper'
import { getStreamObjectCountFactory } from '@/modules/core/repositories/objects'
import { getProjectAutomationsTotalCountFactory } from '@/modules/automate/repositories/automations'
import { getFeatureFlags, isTestEnv } from '@/modules/shared/helpers/envHelper'
import { WorkspacesNotYetImplementedError } from '@/modules/workspaces/errors/workspace'
@@ -92,6 +94,9 @@ export default {
countProjectModels: getStreamBranchCountFactory({ db: sourceDb }),
countProjectVersions: getStreamCommitCountFactory({ db: sourceDb }),
countProjectObjects: getStreamObjectCountFactory({ db: sourceDb }),
countProjectAutomations: getProjectAutomationsTotalCountFactory({
db: sourceDb
}),
getAvailableRegions: getAvailableRegionsFactory({
getRegions: getRegionsFactory({ db }),
canWorkspaceUseRegions: canWorkspaceUseRegionsFactory({
@@ -102,7 +107,8 @@ export default {
copyProjects: copyProjectsFactory({ sourceDb, targetDb }),
copyProjectModels: copyProjectModelsFactory({ sourceDb, targetDb }),
copyProjectVersions: copyProjectVersionsFactory({ sourceDb, targetDb }),
copyProjectObjects: copyProjectObjectsFactory({ sourceDb, targetDb })
copyProjectObjects: copyProjectObjectsFactory({ sourceDb, targetDb }),
copyProjectAutomations: copyProjectAutomationsFactory({ sourceDb, targetDb })
})
return await withTransaction(updateProjectRegion(args), targetDb)
@@ -1,4 +1,12 @@
import {
AutomationFunctionRuns,
AutomationRevisionFunctions,
AutomationRevisions,
AutomationRuns,
AutomationRunTriggers,
Automations,
AutomationTokens,
AutomationTriggers,
BranchCommits,
Branches,
Commits,
@@ -21,6 +29,7 @@ import {
} from '@/modules/core/helpers/types'
import { executeBatchedSelect } from '@/modules/shared/helpers/dbHelper'
import {
CopyProjectAutomations,
CopyProjectModels,
CopyProjectObjects,
CopyProjects,
@@ -32,6 +41,16 @@ import { Knex } from 'knex'
import { Workspace } from '@/modules/workspacesCore/domain/types'
import { Workspaces } from '@/modules/workspacesCore/helpers/db'
import { ObjectPreview } from '@/modules/previews/domain/types'
import {
AutomationFunctionRunRecord,
AutomationRecord,
AutomationRevisionFunctionRecord,
AutomationRevisionRecord,
AutomationRunRecord,
AutomationRunTriggerRecord,
AutomationTokenRecord,
AutomationTriggerDefinitionRecord
} from '@/modules/automate/helpers/types'
const tables = {
workspaces: (db: Knex) => db<Workspace>(Workspaces.name),
@@ -43,7 +62,20 @@ const tables = {
streamFavorites: (db: Knex) => db<StreamFavoriteRecord>(StreamFavorites.name),
streamsMeta: (db: Knex) => db(StreamsMeta.name),
objects: (db: Knex) => db<ObjectRecord>(Objects.name),
objectPreviews: (db: Knex) => db<ObjectPreview>('object_preview')
objectPreviews: (db: Knex) => db<ObjectPreview>('object_preview'),
automations: (db: Knex) => db<AutomationRecord>(Automations.name),
automationTokens: (db: Knex) => db<AutomationTokenRecord>(AutomationTokens.name),
automationRevisions: (db: Knex) =>
db<AutomationRevisionRecord>(AutomationRevisions.name),
automationTriggers: (db: Knex) =>
db<AutomationTriggerDefinitionRecord>(AutomationTriggers.name),
automationRevisionFunctions: (db: Knex) =>
db<AutomationRevisionFunctionRecord>(AutomationRevisionFunctions.name),
automationRuns: (db: Knex) => db<AutomationRunRecord>(AutomationRuns.name),
automationRunTriggers: (db: Knex) =>
db<AutomationRunTriggerRecord>(AutomationRunTriggers.name),
automationFunctionRuns: (db: Knex) =>
db<AutomationFunctionRunRecord>(AutomationFunctionRuns.name)
}
/**
@@ -283,3 +315,147 @@ export const copyProjectObjectsFactory =
return copiedObjectCountByProjectId
}
/**
* Copies rows from the following tables:
* - automations
* - automation_tokens
* - automation_revisions
* - automation_triggers
* - automation_revision_functions
* - automation_runs
* - automation_run_triggers
* - automation_function_runs
*/
export const copyProjectAutomationsFactory =
(deps: { sourceDb: Knex; targetDb: Knex }): CopyProjectAutomations =>
async ({ projectIds }) => {
const copiedAutomationCountByProjectId: Record<string, number> = {}
// Copy `automations` table rows in batches
const selectAutomations = tables
.automations(deps.sourceDb)
.select('*')
.whereIn(Automations.col.projectId, projectIds)
for await (const automations of executeBatchedSelect(selectAutomations)) {
const automationIds = automations.map((automation) => automation.id)
// Write `automations` table rows to target db
await tables
.automations(deps.targetDb)
// Cast ignores unexpected behavior in how knex handles object union types
.insert(automations as unknown as AutomationRecord)
.onConflict()
.ignore()
for (const automation of automations) {
copiedAutomationCountByProjectId[automation.projectId] ??= 0
copiedAutomationCountByProjectId[automation.projectId]++
}
// Copy `automation_tokens` rows for automation
const selectAutomationTokens = tables
.automationTokens(deps.sourceDb)
.select('*')
.whereIn(AutomationTokens.col.automationId, automationIds)
for await (const tokens of executeBatchedSelect(selectAutomationTokens)) {
// Write `automation_tokens` row to target db
await tables
.automationTokens(deps.targetDb)
.insert(tokens)
.onConflict()
.ignore()
}
// Copy `automation_revisions` rows for automation
const selectAutomationRevisions = tables
.automationRevisions(deps.sourceDb)
.select('*')
.whereIn(AutomationRevisions.col.automationId, automationIds)
for await (const automationRevisions of executeBatchedSelect(
selectAutomationRevisions
)) {
const automationRevisionIds = automationRevisions.map((revision) => revision.id)
// Write `automation_revisions` rows to target db
await tables
.automationRevisions(deps.targetDb)
.insert(automationRevisions)
.onConflict()
.ignore()
// Copy `automation_triggers` rows for automation revisions
const automationTriggers = await tables
.automationTriggers(deps.sourceDb)
.select('*')
.whereIn(AutomationTriggers.col.automationRevisionId, automationRevisionIds)
await tables
.automationTriggers(deps.targetDb)
.insert(automationTriggers)
.onConflict()
.ignore()
// Copy `automation_revision_functions` rows for automation revisions
const automationRevisionFunctions = await tables
.automationRevisionFunctions(deps.sourceDb)
.select('*')
.whereIn(
AutomationRevisionFunctions.col.automationRevisionId,
automationRevisionIds
)
await tables
.automationRevisionFunctions(deps.targetDb)
.insert(automationRevisionFunctions)
.onConflict()
.ignore()
// Copy `automation_runs` rows for automation revision
const selectAutomationRuns = tables
.automationRuns(deps.sourceDb)
.select('*')
.whereIn(AutomationRuns.col.automationRevisionId, automationRevisionIds)
for await (const automationRuns of executeBatchedSelect(selectAutomationRuns)) {
const automationRunIds = automationRuns.map((run) => run.id)
// Write `automation_runs` row to target db
await tables
.automationRuns(deps.targetDb)
.insert(automationRuns)
.onConflict()
.ignore()
// Copy `automation_run_triggers` rows for automation run
const automationRunTriggers = await tables
.automationRunTriggers(deps.sourceDb)
.select('*')
.whereIn(AutomationRunTriggers.col.automationRunId, automationRunIds)
await tables
.automationRunTriggers(deps.targetDb)
.insert(automationRunTriggers)
.onConflict()
.ignore()
// Copy `automation_function_runs` rows for automation run
const automationFunctionRuns = await tables
.automationFunctionRuns(deps.sourceDb)
.select('*')
.whereIn(AutomationFunctionRuns.col.runId, automationRunIds)
await tables
.automationFunctionRuns(deps.targetDb)
.insert(automationFunctionRuns)
.onConflict()
.ignore()
}
}
}
return copiedAutomationCountByProjectId
}
@@ -1,8 +1,10 @@
import { GetProjectAutomationCount } from '@/modules/automate/domain/operations'
import { GetStreamBranchCount } from '@/modules/core/domain/branches/operations'
import { GetStreamCommitCount } from '@/modules/core/domain/commits/operations'
import { GetStreamObjectCount } from '@/modules/core/domain/objects/operations'
import { GetProject } from '@/modules/core/domain/projects/operations'
import {
CopyProjectAutomations,
CopyProjectModels,
CopyProjectObjects,
CopyProjects,
@@ -19,12 +21,14 @@ export const updateProjectRegionFactory =
countProjectModels: GetStreamBranchCount
countProjectVersions: GetStreamCommitCount
countProjectObjects: GetStreamObjectCount
countProjectAutomations: GetProjectAutomationCount
getAvailableRegions: GetAvailableRegions
copyWorkspace: CopyWorkspace
copyProjects: CopyProjects
copyProjectModels: CopyProjectModels
copyProjectVersions: CopyProjectVersions
copyProjectObjects: CopyProjectObjects
copyProjectAutomations: CopyProjectAutomations
}): UpdateProjectRegion =>
async (params) => {
const { projectId, regionKey } = params
@@ -67,7 +71,9 @@ export const updateProjectRegionFactory =
// Move objects
const copiedObjectCount = await deps.copyProjectObjects({ projectIds })
// TODO: Move automations
// Move automations
const copiedAutomationCount = await deps.copyProjectAutomations({ projectIds })
// TODO: Move comments
// TODO: Move file blobs
// TODO: Move webhooks
@@ -78,11 +84,15 @@ export const updateProjectRegionFactory =
const sourceProjectObjectCount = await deps.countProjectObjects({
streamId: projectId
})
const sourceProjectAutomationCount = await deps.countProjectAutomations({
projectId
})
const tests = [
copiedModelCount[projectId] === sourceProjectModelCount,
copiedVersionCount[projectId] === sourceProjectVersionCount,
copiedObjectCount[projectId] === sourceProjectObjectCount
copiedObjectCount[projectId] === sourceProjectObjectCount,
copiedAutomationCount[projectId] === sourceProjectAutomationCount
]
if (!tests.every((test) => !!test)) {
@@ -1,4 +1,23 @@
import { db } from '@/db/knex'
import {
AutomationFunctionRunRecord,
AutomationRecord,
AutomationRevisionFunctionRecord,
AutomationRevisionRecord,
AutomationRunRecord,
AutomationRunTriggerRecord,
AutomationTokenRecord,
AutomationTriggerDefinitionRecord
} from '@/modules/automate/helpers/types'
import {
AutomationFunctionRuns,
AutomationRevisionFunctions,
AutomationRevisions,
AutomationRuns,
AutomationRunTriggers,
AutomationTokens,
AutomationTriggers
} from '@/modules/core/dbSchema'
import { AllScopes } from '@/modules/core/helpers/mainConstants'
import { createRandomEmail } from '@/modules/core/helpers/testHelpers'
import {
@@ -35,6 +54,10 @@ import {
TestApolloServer
} from '@/test/graphqlHelper'
import { beforeEachContext } from '@/test/hooks'
import {
createTestAutomation,
createTestAutomationRun
} from '@/test/speckle-helpers/automationHelper'
import { BasicTestBranch, createTestBranch } from '@/test/speckle-helpers/branchHelper'
import {
BasicTestCommit,
@@ -58,7 +81,20 @@ const tables = {
versions: (db: Knex) => db.table<CommitRecord>('commits'),
streamCommits: (db: Knex) => db.table<StreamCommitRecord>('stream_commits'),
branchCommits: (db: Knex) => db.table<BranchCommitRecord>('branch_commits'),
objects: (db: Knex) => db.table<ObjectRecord>('objects')
objects: (db: Knex) => db.table<ObjectRecord>('objects'),
automations: (db: Knex) => db.table<AutomationRecord>('automations'),
automationTokens: (db: Knex) => db<AutomationTokenRecord>(AutomationTokens.name),
automationRevisions: (db: Knex) =>
db<AutomationRevisionRecord>(AutomationRevisions.name),
automationTriggers: (db: Knex) =>
db<AutomationTriggerDefinitionRecord>(AutomationTriggers.name),
automationRevisionFunctions: (db: Knex) =>
db<AutomationRevisionFunctionRecord>(AutomationRevisionFunctions.name),
automationRuns: (db: Knex) => db<AutomationRunRecord>(AutomationRuns.name),
automationRunTriggers: (db: Knex) =>
db<AutomationRunTriggerRecord>(AutomationRunTriggers.name),
automationFunctionRuns: (db: Knex) =>
db<AutomationFunctionRunRecord>(AutomationFunctionRuns.name)
}
const grantStreamPermissions = grantStreamPermissionsFactory({ db })
@@ -344,6 +380,12 @@ isMultiRegionTestMode()
authorId: ''
}
let testAutomation: AutomationRecord
let testAutomationToken: AutomationTokenRecord
let testAutomationRevision: AutomationRevisionRecord
let testAutomationRun: AutomationRunRecord
let testAutomationFunctionRuns: AutomationFunctionRunRecord[]
let apollo: TestApolloServer
let targetRegionDb: Knex
@@ -382,6 +424,32 @@ isMultiRegionTestMode()
owner: adminUser,
stream: testProject
})
const { automation, revision } = await createTestAutomation({
userId: adminUser.id,
projectId: testProject.id,
revision: {
functionId: cryptoRandomString({ length: 9 }),
functionReleaseId: cryptoRandomString({ length: 9 })
}
})
if (!revision) {
throw new Error('Failed to create automation revision.')
}
testAutomation = automation.automation
testAutomationToken = automation.token
testAutomationRevision = revision
const { automationRun, functionRuns } = await createTestAutomationRun({
userId: adminUser.id,
projectId: testProject.id,
automationId: testAutomation.id
})
testAutomationRun = automationRun
testAutomationFunctionRuns = functionRuns
})
it('moves project record to target regional db', async () => {
@@ -468,5 +536,83 @@ isMultiRegionTestMode()
expect(object).to.not.be.undefined
})
it('moves project automation data to target regional db', async () => {
const res = await apollo.execute(UpdateProjectRegionDocument, {
projectId: testProject.id,
regionKey: regionKey2
})
expect(res).to.not.haveGraphQLErrors()
// TODO: Replace with gql query when possible
const automation = await tables
.automations(targetRegionDb)
.select('*')
.where({ id: testAutomation.id })
.first()
expect(automation).to.not.be.undefined
const automationToken = await tables
.automationTokens(targetRegionDb)
.select('*')
.where({ automationId: testAutomation.id })
.first()
expect(automationToken).to.not.be.undefined
expect(automationToken?.automateToken).to.equal(
testAutomationToken.automateToken
)
const automationRevision = await tables
.automationRevisions(targetRegionDb)
.select('*')
.where({ automationId: testAutomation.id })
.first()
expect(automationRevision).to.not.be.undefined
expect(automationRevision?.id).to.equal(testAutomationRevision.id)
const automationTrigger = await tables
.automationTriggers(targetRegionDb)
.select('*')
.where({ automationRevisionId: testAutomationRevision.id })
.first()
expect(automationTrigger).to.not.be.undefined
})
it('moves project automation runs to target regional db', async () => {
const res = await apollo.execute(UpdateProjectRegionDocument, {
projectId: testProject.id,
regionKey: regionKey2
})
expect(res).to.not.haveGraphQLErrors()
// TODO: Replace with gql query when possible
const automationRun = await tables
.automationRuns(targetRegionDb)
.select('*')
.where({ id: testAutomationRun.id })
.first()
expect(automationRun).to.not.be.undefined
const automationRunTriggers = await tables
.automationRunTriggers(targetRegionDb)
.select('*')
.where({ automationRunId: testAutomationRun.id })
expect(automationRunTriggers.length).to.not.equal(0)
const automationFunctionRuns = await tables
.automationFunctionRuns(targetRegionDb)
.select('*')
.where({ runId: testAutomationRun.id })
expect(automationFunctionRuns.length).to.equal(
testAutomationFunctionRuns.length
)
expect(
automationFunctionRuns.every((run) =>
testAutomationFunctionRuns.some((testRun) => testRun.id === run.id)
)
)
})
})
: void 0
@@ -1,8 +1,12 @@
import {
getAutomationFactory,
getFullAutomationRevisionMetadataFactory,
getFullAutomationRunByIdFactory,
getLatestAutomationRevisionFactory,
storeAutomationFactory,
storeAutomationRevisionFactory,
storeAutomationTokenFactory
storeAutomationTokenFactory,
upsertAutomationRunFactory
} from '@/modules/automate/repositories/automations'
import {
CreateAutomationRevisionDeps,
@@ -15,6 +19,7 @@ import cryptoRandomString from 'crypto-random-string'
import { createAutomation as clientCreateAutomation } from '@/modules/automate/clients/executionEngine'
import {
getBranchesByIdsFactory,
getBranchLatestCommitsFactory,
getLatestStreamBranchFactory
} from '@/modules/core/repositories/branches'
@@ -35,6 +40,7 @@ import {
import { faker } from '@faker-js/faker'
import {
getEncryptionKeyPair,
getEncryptionKeyPairFor,
getFunctionInputDecryptorFactory
} from '@/modules/automate/services/encryption'
import { buildDecryptor } from '@/modules/shared/utils/libsodium'
@@ -42,22 +48,28 @@ import { db } from '@/db/knex'
import { validateStreamAccessFactory } from '@/modules/core/services/streams/access'
import { authorizeResolver } from '@/modules/shared'
import { getEventBus } from '@/modules/shared/services/eventBus'
import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector'
import { Knex } from 'knex'
import { createTestAutomationRunFactory } from '@/modules/automate/services/trigger'
const storeAutomation = storeAutomationFactory({ db })
const storeAutomationToken = storeAutomationTokenFactory({ db })
const storeAutomationRevision = storeAutomationRevisionFactory({ db })
const getAutomation = getAutomationFactory({ db })
const getLatestStreamBranch = getLatestStreamBranchFactory({ db })
const validateStreamAccess = validateStreamAccessFactory({ authorizeResolver })
export const generateFunctionId = () => cryptoRandomString({ length: 10 })
export const generateFunctionReleaseId = () => cryptoRandomString({ length: 10 })
/**
* @param overrides By default, we mock requests to the execution engine. You can replace those mocks here.
*/
export const buildAutomationCreate = (
overrides?: Partial<{
createDbAutomation: typeof clientCreateAutomation
}>
params: {
dbClient?: Knex
overrides?: Partial<{
createDbAutomation: typeof clientCreateAutomation
}>
} = {}
) => {
const { dbClient = db, overrides } = params
const create = createAutomationFactory({
createAuthCode: createStoredAuthCodeFactory({ redis: createInmemoryRedisClient() }),
automateCreateAutomation:
@@ -66,8 +78,8 @@ export const buildAutomationCreate = (
automationId: cryptoRandomString({ length: 10 }),
token: cryptoRandomString({ length: 10 })
})),
storeAutomation,
storeAutomationToken,
storeAutomation: storeAutomationFactory({ db: dbClient }),
storeAutomationToken: storeAutomationTokenFactory({ db: dbClient }),
validateStreamAccess,
eventEmit: getEventBus().emit
})
@@ -75,9 +87,17 @@ export const buildAutomationCreate = (
return create
}
/**
* @param overrides By default, we mock requests to the execution engine. You can replace those mocks here.
*/
export const buildAutomationRevisionCreate = (
overrides?: Partial<CreateAutomationRevisionDeps>
params: {
dbClient?: Knex
overrides?: Partial<CreateAutomationRevisionDeps>
} = {}
) => {
const { dbClient = db, overrides } = params
const fakeGetRelease = (params: {
functionReleaseId: string
functionId: string
@@ -91,9 +111,9 @@ export const buildAutomationRevisionCreate = (
})
const create = createAutomationRevisionFactory({
getAutomation,
storeAutomationRevision,
getBranchesByIds: getBranchesByIdsFactory({ db }),
getAutomation: getAutomationFactory({ db: dbClient }),
storeAutomationRevision: storeAutomationRevisionFactory({ db: dbClient }),
getBranchesByIds: getBranchesByIdsFactory({ db: dbClient }),
getFunctionRelease: async (params) => fakeGetRelease(params),
getFunctionReleases: async (params) => params.ids.map(fakeGetRelease),
getEncryptionKeyPair,
@@ -128,8 +148,10 @@ export const createTestAutomation = async (params: {
revision: { input: revisionInput, functionReleaseId, functionId } = {}
} = params
const createAutomation = buildAutomationCreate()
const createRevision = buildAutomationRevisionCreate()
const projectDb = await getProjectDbClient({ projectId })
const createAutomation = buildAutomationCreate({ dbClient: projectDb })
const createRevision = buildAutomationRevisionCreate({ dbClient: projectDb })
const automationRet = await createAutomation({
input: {
@@ -143,7 +165,7 @@ export const createTestAutomation = async (params: {
let revisionRet: Awaited<ReturnType<typeof createRevision>> | null = null
if (functionReleaseId?.length && functionId?.length) {
const firstModel = await getLatestStreamBranch(projectId)
const firstModel = await getLatestStreamBranchFactory({ db: projectDb })(projectId)
if (!firstModel)
throw new Error(
@@ -186,6 +208,55 @@ export type TestAutomationWithRevision = Awaited<
ReturnType<typeof createTestAutomation>
>
export const createTestAutomationRun = async (params: {
userId: string
projectId: string
automationId: string
}) => {
const { userId, projectId, automationId } = params
const projectDb = await getProjectDbClient({ projectId })
const { automationRunId } = await createTestAutomationRunFactory({
getEncryptionKeyPairFor,
getFunctionInputDecryptor: getFunctionInputDecryptorFactory({
buildDecryptor
}),
getAutomation: getAutomationFactory({
db: projectDb
}),
getLatestAutomationRevision: getLatestAutomationRevisionFactory({
db: projectDb
}),
getFullAutomationRevisionMetadata: getFullAutomationRevisionMetadataFactory({
db: projectDb
}),
upsertAutomationRun: upsertAutomationRunFactory({
db: projectDb
}),
getBranchLatestCommits: getBranchLatestCommitsFactory({
db: projectDb
}),
validateStreamAccess
})({ projectId, automationId, userId })
const automationRunData = await getFullAutomationRunByIdFactory({ db: projectDb })(
automationRunId
)
if (!automationRunData) {
throw new Error('Failed to create test automation run!')
}
const { triggers, functionRuns, ...automationRun } = automationRunData
return {
automationRun,
functionRuns,
triggers
}
}
export const truncateAutomations = async () => {
await truncateTables([
AutomationRunTriggers.name,