feat(regions): move project automations (#3925)
* feat(regions): repo functions for copying project branches and commits * chore(regions): wire up move to resolver * chore(regions): successful basic test of project region change * fix(regions): sabrina carpenter please please please * fix(regions): repair multiregion test setup * chore(regions): appease ts * chore(multiregion): update test multiregion config * chore(multiregion): fix test docker config and test * chore(multiregion): use transaction * chore(multiregion): maybe this will work * fix(multiregion): drop subs synchronously * chore(multiregion): desperate test logs * chore(multiregion): somehow that worked? * chore(multiregion): add load-bearing log statement * chore(multiregion): move services * fix(multiregion): test drop waits * chore(regions): fix import * chore(regions): make test a bit more thorough for good measure * fix(regions): move project objects * chore(regions): add tests for object move * feat(regions): move project automations * chore(regions): add tests for moving automations * chore(regions): more tests for moving automate data * fix(regions): speed up inserts * fix(regions): simplify postgres usage * chore(regions): repair build * fix(regions): improve queries * chore(regions): again
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
import { InsertableAutomationFunctionRun } from '@/modules/automate/domain/types'
|
||||
import {
|
||||
AutomateRevisionFunctionRecord,
|
||||
AutomationRevisionFunctionRecord,
|
||||
AutomationFunctionRunRecord,
|
||||
AutomationRecord,
|
||||
AutomationRevisionRecord,
|
||||
@@ -175,7 +175,7 @@ export type GetRevisionsTriggerDefinitions = (params: {
|
||||
|
||||
export type GetRevisionsFunctions = (params: {
|
||||
automationRevisionIds: string[]
|
||||
}) => Promise<{ [automationRevisionId: string]: AutomateRevisionFunctionRecord[] }>
|
||||
}) => Promise<{ [automationRevisionId: string]: AutomationRevisionFunctionRecord[] }>
|
||||
|
||||
export type CreateStoredAuthCode = (
|
||||
params: Omit<AuthCodePayload, 'code'>
|
||||
@@ -204,3 +204,7 @@ export type TriggerAutomationRevisionRun = <
|
||||
manifest: M
|
||||
source: RunTriggerSource
|
||||
}) => Promise<{ automationRunId: string }>
|
||||
|
||||
export type GetProjectAutomationCount = (params: {
|
||||
projectId: string
|
||||
}) => Promise<number>
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {
|
||||
AutomateRevisionFunctionRecord,
|
||||
AutomationRevisionFunctionRecord,
|
||||
AutomationFunctionRunRecord,
|
||||
AutomationRecord,
|
||||
AutomationRevisionRecord,
|
||||
@@ -56,7 +56,7 @@ export type AutomationRunTriggerGraphQLReturn = AutomationRunTriggerRecord & {
|
||||
}
|
||||
|
||||
export type AutomationRevisionFunctionGraphQLReturn = Merge<
|
||||
AutomateRevisionFunctionRecord,
|
||||
AutomationRevisionFunctionRecord,
|
||||
{
|
||||
functionInputs: Nullable<Record<string, unknown>>
|
||||
release: AutomateFunctionReleaseGraphQLReturn
|
||||
|
||||
@@ -66,7 +66,7 @@ export type AutomationRunRecord = {
|
||||
executionEngineRunId: string | null
|
||||
}
|
||||
|
||||
export type AutomateRevisionFunctionRecord = {
|
||||
export type AutomationRevisionFunctionRecord = {
|
||||
functionReleaseId: string
|
||||
functionId: string
|
||||
functionInputs: string | null
|
||||
@@ -122,7 +122,7 @@ export type AutomationTokenRecord = {
|
||||
}
|
||||
|
||||
export type AutomationRevisionWithTriggersFunctions = AutomationRevisionRecord & {
|
||||
functions: AutomateRevisionFunctionRecord[]
|
||||
functions: AutomationRevisionFunctionRecord[]
|
||||
triggers: AutomationTriggerDefinitionRecord[]
|
||||
}
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
GetLatestAutomationRevision,
|
||||
GetLatestAutomationRevisions,
|
||||
GetLatestVersionAutomationRuns,
|
||||
GetProjectAutomationCount,
|
||||
GetRevisionsFunctions,
|
||||
GetRevisionsTriggerDefinitions,
|
||||
StoreAutomation,
|
||||
@@ -37,7 +38,7 @@ import {
|
||||
AutomationRunRecord,
|
||||
AutomationTokenRecord,
|
||||
AutomationTriggerRecordBase,
|
||||
AutomateRevisionFunctionRecord,
|
||||
AutomationRevisionFunctionRecord,
|
||||
AutomationRunWithTriggersFunctionRuns,
|
||||
AutomationRunTriggerRecord,
|
||||
AutomationFunctionRunRecord,
|
||||
@@ -86,7 +87,7 @@ const tables = {
|
||||
automationRevisions: (db: Knex) =>
|
||||
db<AutomationRevisionRecord>(AutomationRevisions.name),
|
||||
automationRevisionFunctions: (db: Knex) =>
|
||||
db<AutomateRevisionFunctionRecord>(AutomationRevisionFunctions.name),
|
||||
db<AutomationRevisionFunctionRecord>(AutomationRevisionFunctions.name),
|
||||
automationTriggers: (db: Knex) =>
|
||||
db<AutomationTriggerDefinitionRecord>(AutomationTriggers.name),
|
||||
automationRuns: (db: Knex) => db<AutomationRunRecord>(AutomationRuns.name),
|
||||
@@ -321,7 +322,7 @@ export const storeAutomationTokenFactory =
|
||||
}
|
||||
|
||||
export type InsertableAutomationRevisionFunction = Omit<
|
||||
AutomateRevisionFunctionRecord,
|
||||
AutomationRevisionFunctionRecord,
|
||||
'automationRevisionId'
|
||||
>
|
||||
|
||||
@@ -369,7 +370,7 @@ export const storeAutomationRevisionFactory =
|
||||
.automationRevisionFunctions(deps.db)
|
||||
.insert(
|
||||
revision.functions.map(
|
||||
(f): AutomateRevisionFunctionRecord => ({
|
||||
(f): AutomationRevisionFunctionRecord => ({
|
||||
...f,
|
||||
automationRevisionId: id
|
||||
})
|
||||
@@ -742,10 +743,12 @@ const getProjectAutomationsBaseQueryFactory =
|
||||
}
|
||||
|
||||
export const getProjectAutomationsTotalCountFactory =
|
||||
(deps: { db: Knex }) => async (params: GetProjectAutomationsParams) => {
|
||||
const q = getProjectAutomationsBaseQueryFactory(deps)(params).count<
|
||||
[{ count: string }]
|
||||
>(Automations.col.id)
|
||||
(deps: { db: Knex }): GetProjectAutomationCount =>
|
||||
async ({ projectId }) => {
|
||||
const q = getProjectAutomationsBaseQueryFactory(deps)({
|
||||
projectId,
|
||||
args: {}
|
||||
}).count<[{ count: string }]>(Automations.col.id)
|
||||
|
||||
const [ret] = await q
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ import { Nullable, Optional } from '@speckle/shared'
|
||||
import { MisconfiguredEnvironmentError } from '@/modules/shared/errors'
|
||||
import { AutomationFunctionInputEncryptionError } from '@/modules/automate/errors/management'
|
||||
import { KeyPair, buildDecryptor } from '@/modules/shared/utils/libsodium'
|
||||
import { AutomateRevisionFunctionRecord } from '@/modules/automate/helpers/types'
|
||||
import { AutomationRevisionFunctionRecord } from '@/modules/automate/helpers/types'
|
||||
import { AutomationRevisionFunctionGraphQLReturn } from '@/modules/automate/helpers/graphTypes'
|
||||
import { FunctionReleaseSchemaType } from '@/modules/automate/helpers/executionEngine'
|
||||
import { LibsodiumEncryptionError } from '@/modules/shared/errors/encryption'
|
||||
@@ -118,7 +118,7 @@ export type GetFunctionInputsForFrontendDeps = {
|
||||
} & GetFunctionInputDecryptorDeps
|
||||
|
||||
export type AutomationRevisionFunctionForInputRedaction = Merge<
|
||||
AutomateRevisionFunctionRecord,
|
||||
AutomationRevisionFunctionRecord,
|
||||
{ release: FunctionReleaseSchemaType }
|
||||
>
|
||||
|
||||
|
||||
@@ -46,6 +46,7 @@ import { ValidateStreamAccess } from '@/modules/core/domain/streams/operations'
|
||||
import { CreateAndStoreAppToken } from '@/modules/core/domain/tokens/operations'
|
||||
import { EventBusEmit } from '@/modules/shared/services/eventBus'
|
||||
import { AutomationRunEvents } from '@/modules/automate/domain/events'
|
||||
import { isTestEnv } from '@/modules/shared/helpers/envHelper'
|
||||
|
||||
export type OnModelVersionCreateDeps = {
|
||||
getAutomation: GetAutomation
|
||||
@@ -131,7 +132,7 @@ type CreateAutomationRunDataDeps = {
|
||||
getFunctionInputDecryptor: FunctionInputDecryptor
|
||||
}
|
||||
|
||||
const createAutomationRunDataFactory =
|
||||
export const createAutomationRunDataFactory =
|
||||
(deps: CreateAutomationRunDataDeps) =>
|
||||
async (params: {
|
||||
manifests: BaseTriggerManifest[]
|
||||
@@ -414,7 +415,7 @@ type ComposeTriggerDataDeps = {
|
||||
getBranchLatestCommits: GetBranchLatestCommits
|
||||
}
|
||||
|
||||
const composeTriggerDataFactory =
|
||||
export const composeTriggerDataFactory =
|
||||
(deps: ComposeTriggerDataDeps) =>
|
||||
async (params: {
|
||||
projectId: string
|
||||
@@ -575,7 +576,7 @@ export const createTestAutomationRunFactory =
|
||||
throw new TriggerAutomationError('Automation not found')
|
||||
}
|
||||
|
||||
if (!automationRecord.isTestAutomation) {
|
||||
if (!isTestEnv() && !automationRecord.isTestAutomation) {
|
||||
throw new TriggerAutomationError(
|
||||
'Automation is not a test automation and cannot create test function runs'
|
||||
)
|
||||
|
||||
@@ -510,9 +510,11 @@ const buildAutomationUpdate = () => {
|
||||
|
||||
it('fails when refering to nonexistent function releases', async () => {
|
||||
const create = buildAutomationRevisionCreate({
|
||||
getFunctionRelease: async () => {
|
||||
// TODO: Update once we know how exec engine should respond
|
||||
throw new Error('Function release with ID XXX not found')
|
||||
overrides: {
|
||||
getFunctionRelease: async () => {
|
||||
// TODO: Update once we know how exec engine should respond
|
||||
throw new Error('Function release with ID XXX not found')
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
@@ -56,7 +56,7 @@ import { Users } from '@/modules/core/dbSchema'
|
||||
import { getStreamPendingModelsFactory } from '@/modules/fileuploads/repositories/fileUploads'
|
||||
import { FileUploadRecord } from '@/modules/fileuploads/helpers/types'
|
||||
import {
|
||||
AutomateRevisionFunctionRecord,
|
||||
AutomationRevisionFunctionRecord,
|
||||
AutomationRecord,
|
||||
AutomationRevisionRecord,
|
||||
AutomationRunTriggerRecord,
|
||||
@@ -595,7 +595,7 @@ const dataLoadersDefinition = defineRequestDataloaders(
|
||||
})
|
||||
return ids.map((i) => results[i] || [])
|
||||
}),
|
||||
getRevisionFunctions: createLoader<string, AutomateRevisionFunctionRecord[]>(
|
||||
getRevisionFunctions: createLoader<string, AutomationRevisionFunctionRecord[]>(
|
||||
async (ids) => {
|
||||
const results = await getRevisionsFunctions({
|
||||
automationRevisionIds: ids.slice()
|
||||
|
||||
@@ -366,3 +366,6 @@ export type CopyProjectVersions = (params: {
|
||||
export type CopyProjectObjects = (params: {
|
||||
projectIds: string[]
|
||||
}) => Promise<Record<string, number>>
|
||||
export type CopyProjectAutomations = (params: {
|
||||
projectIds: string[]
|
||||
}) => Promise<Record<string, number>>
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
upsertRegionAssignmentFactory
|
||||
} from '@/modules/workspaces/repositories/regions'
|
||||
import {
|
||||
copyProjectAutomationsFactory,
|
||||
copyProjectModelsFactory,
|
||||
copyProjectObjectsFactory,
|
||||
copyProjectsFactory,
|
||||
@@ -31,6 +32,7 @@ import { getStreamBranchCountFactory } from '@/modules/core/repositories/branche
|
||||
import { getStreamCommitCountFactory } from '@/modules/core/repositories/commits'
|
||||
import { withTransaction } from '@/modules/shared/helpers/dbHelper'
|
||||
import { getStreamObjectCountFactory } from '@/modules/core/repositories/objects'
|
||||
import { getProjectAutomationsTotalCountFactory } from '@/modules/automate/repositories/automations'
|
||||
import { getFeatureFlags, isTestEnv } from '@/modules/shared/helpers/envHelper'
|
||||
import { WorkspacesNotYetImplementedError } from '@/modules/workspaces/errors/workspace'
|
||||
|
||||
@@ -92,6 +94,9 @@ export default {
|
||||
countProjectModels: getStreamBranchCountFactory({ db: sourceDb }),
|
||||
countProjectVersions: getStreamCommitCountFactory({ db: sourceDb }),
|
||||
countProjectObjects: getStreamObjectCountFactory({ db: sourceDb }),
|
||||
countProjectAutomations: getProjectAutomationsTotalCountFactory({
|
||||
db: sourceDb
|
||||
}),
|
||||
getAvailableRegions: getAvailableRegionsFactory({
|
||||
getRegions: getRegionsFactory({ db }),
|
||||
canWorkspaceUseRegions: canWorkspaceUseRegionsFactory({
|
||||
@@ -102,7 +107,8 @@ export default {
|
||||
copyProjects: copyProjectsFactory({ sourceDb, targetDb }),
|
||||
copyProjectModels: copyProjectModelsFactory({ sourceDb, targetDb }),
|
||||
copyProjectVersions: copyProjectVersionsFactory({ sourceDb, targetDb }),
|
||||
copyProjectObjects: copyProjectObjectsFactory({ sourceDb, targetDb })
|
||||
copyProjectObjects: copyProjectObjectsFactory({ sourceDb, targetDb }),
|
||||
copyProjectAutomations: copyProjectAutomationsFactory({ sourceDb, targetDb })
|
||||
})
|
||||
|
||||
return await withTransaction(updateProjectRegion(args), targetDb)
|
||||
|
||||
@@ -1,4 +1,12 @@
|
||||
import {
|
||||
AutomationFunctionRuns,
|
||||
AutomationRevisionFunctions,
|
||||
AutomationRevisions,
|
||||
AutomationRuns,
|
||||
AutomationRunTriggers,
|
||||
Automations,
|
||||
AutomationTokens,
|
||||
AutomationTriggers,
|
||||
BranchCommits,
|
||||
Branches,
|
||||
Commits,
|
||||
@@ -21,6 +29,7 @@ import {
|
||||
} from '@/modules/core/helpers/types'
|
||||
import { executeBatchedSelect } from '@/modules/shared/helpers/dbHelper'
|
||||
import {
|
||||
CopyProjectAutomations,
|
||||
CopyProjectModels,
|
||||
CopyProjectObjects,
|
||||
CopyProjects,
|
||||
@@ -32,6 +41,16 @@ import { Knex } from 'knex'
|
||||
import { Workspace } from '@/modules/workspacesCore/domain/types'
|
||||
import { Workspaces } from '@/modules/workspacesCore/helpers/db'
|
||||
import { ObjectPreview } from '@/modules/previews/domain/types'
|
||||
import {
|
||||
AutomationFunctionRunRecord,
|
||||
AutomationRecord,
|
||||
AutomationRevisionFunctionRecord,
|
||||
AutomationRevisionRecord,
|
||||
AutomationRunRecord,
|
||||
AutomationRunTriggerRecord,
|
||||
AutomationTokenRecord,
|
||||
AutomationTriggerDefinitionRecord
|
||||
} from '@/modules/automate/helpers/types'
|
||||
|
||||
const tables = {
|
||||
workspaces: (db: Knex) => db<Workspace>(Workspaces.name),
|
||||
@@ -43,7 +62,20 @@ const tables = {
|
||||
streamFavorites: (db: Knex) => db<StreamFavoriteRecord>(StreamFavorites.name),
|
||||
streamsMeta: (db: Knex) => db(StreamsMeta.name),
|
||||
objects: (db: Knex) => db<ObjectRecord>(Objects.name),
|
||||
objectPreviews: (db: Knex) => db<ObjectPreview>('object_preview')
|
||||
objectPreviews: (db: Knex) => db<ObjectPreview>('object_preview'),
|
||||
automations: (db: Knex) => db<AutomationRecord>(Automations.name),
|
||||
automationTokens: (db: Knex) => db<AutomationTokenRecord>(AutomationTokens.name),
|
||||
automationRevisions: (db: Knex) =>
|
||||
db<AutomationRevisionRecord>(AutomationRevisions.name),
|
||||
automationTriggers: (db: Knex) =>
|
||||
db<AutomationTriggerDefinitionRecord>(AutomationTriggers.name),
|
||||
automationRevisionFunctions: (db: Knex) =>
|
||||
db<AutomationRevisionFunctionRecord>(AutomationRevisionFunctions.name),
|
||||
automationRuns: (db: Knex) => db<AutomationRunRecord>(AutomationRuns.name),
|
||||
automationRunTriggers: (db: Knex) =>
|
||||
db<AutomationRunTriggerRecord>(AutomationRunTriggers.name),
|
||||
automationFunctionRuns: (db: Knex) =>
|
||||
db<AutomationFunctionRunRecord>(AutomationFunctionRuns.name)
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -283,3 +315,147 @@ export const copyProjectObjectsFactory =
|
||||
|
||||
return copiedObjectCountByProjectId
|
||||
}
|
||||
|
||||
/**
|
||||
* Copies rows from the following tables:
|
||||
* - automations
|
||||
* - automation_tokens
|
||||
* - automation_revisions
|
||||
* - automation_triggers
|
||||
* - automation_revision_functions
|
||||
* - automation_runs
|
||||
* - automation_run_triggers
|
||||
* - automation_function_runs
|
||||
*/
|
||||
export const copyProjectAutomationsFactory =
|
||||
(deps: { sourceDb: Knex; targetDb: Knex }): CopyProjectAutomations =>
|
||||
async ({ projectIds }) => {
|
||||
const copiedAutomationCountByProjectId: Record<string, number> = {}
|
||||
|
||||
// Copy `automations` table rows in batches
|
||||
const selectAutomations = tables
|
||||
.automations(deps.sourceDb)
|
||||
.select('*')
|
||||
.whereIn(Automations.col.projectId, projectIds)
|
||||
|
||||
for await (const automations of executeBatchedSelect(selectAutomations)) {
|
||||
const automationIds = automations.map((automation) => automation.id)
|
||||
|
||||
// Write `automations` table rows to target db
|
||||
await tables
|
||||
.automations(deps.targetDb)
|
||||
// Cast ignores unexpected behavior in how knex handles object union types
|
||||
.insert(automations as unknown as AutomationRecord)
|
||||
.onConflict()
|
||||
.ignore()
|
||||
|
||||
for (const automation of automations) {
|
||||
copiedAutomationCountByProjectId[automation.projectId] ??= 0
|
||||
copiedAutomationCountByProjectId[automation.projectId]++
|
||||
}
|
||||
|
||||
// Copy `automation_tokens` rows for automation
|
||||
const selectAutomationTokens = tables
|
||||
.automationTokens(deps.sourceDb)
|
||||
.select('*')
|
||||
.whereIn(AutomationTokens.col.automationId, automationIds)
|
||||
|
||||
for await (const tokens of executeBatchedSelect(selectAutomationTokens)) {
|
||||
// Write `automation_tokens` row to target db
|
||||
await tables
|
||||
.automationTokens(deps.targetDb)
|
||||
.insert(tokens)
|
||||
.onConflict()
|
||||
.ignore()
|
||||
}
|
||||
|
||||
// Copy `automation_revisions` rows for automation
|
||||
const selectAutomationRevisions = tables
|
||||
.automationRevisions(deps.sourceDb)
|
||||
.select('*')
|
||||
.whereIn(AutomationRevisions.col.automationId, automationIds)
|
||||
|
||||
for await (const automationRevisions of executeBatchedSelect(
|
||||
selectAutomationRevisions
|
||||
)) {
|
||||
const automationRevisionIds = automationRevisions.map((revision) => revision.id)
|
||||
|
||||
// Write `automation_revisions` rows to target db
|
||||
await tables
|
||||
.automationRevisions(deps.targetDb)
|
||||
.insert(automationRevisions)
|
||||
.onConflict()
|
||||
.ignore()
|
||||
|
||||
// Copy `automation_triggers` rows for automation revisions
|
||||
const automationTriggers = await tables
|
||||
.automationTriggers(deps.sourceDb)
|
||||
.select('*')
|
||||
.whereIn(AutomationTriggers.col.automationRevisionId, automationRevisionIds)
|
||||
|
||||
await tables
|
||||
.automationTriggers(deps.targetDb)
|
||||
.insert(automationTriggers)
|
||||
.onConflict()
|
||||
.ignore()
|
||||
|
||||
// Copy `automation_revision_functions` rows for automation revisions
|
||||
const automationRevisionFunctions = await tables
|
||||
.automationRevisionFunctions(deps.sourceDb)
|
||||
.select('*')
|
||||
.whereIn(
|
||||
AutomationRevisionFunctions.col.automationRevisionId,
|
||||
automationRevisionIds
|
||||
)
|
||||
|
||||
await tables
|
||||
.automationRevisionFunctions(deps.targetDb)
|
||||
.insert(automationRevisionFunctions)
|
||||
.onConflict()
|
||||
.ignore()
|
||||
|
||||
// Copy `automation_runs` rows for automation revision
|
||||
const selectAutomationRuns = tables
|
||||
.automationRuns(deps.sourceDb)
|
||||
.select('*')
|
||||
.whereIn(AutomationRuns.col.automationRevisionId, automationRevisionIds)
|
||||
|
||||
for await (const automationRuns of executeBatchedSelect(selectAutomationRuns)) {
|
||||
const automationRunIds = automationRuns.map((run) => run.id)
|
||||
|
||||
// Write `automation_runs` row to target db
|
||||
await tables
|
||||
.automationRuns(deps.targetDb)
|
||||
.insert(automationRuns)
|
||||
.onConflict()
|
||||
.ignore()
|
||||
|
||||
// Copy `automation_run_triggers` rows for automation run
|
||||
const automationRunTriggers = await tables
|
||||
.automationRunTriggers(deps.sourceDb)
|
||||
.select('*')
|
||||
.whereIn(AutomationRunTriggers.col.automationRunId, automationRunIds)
|
||||
|
||||
await tables
|
||||
.automationRunTriggers(deps.targetDb)
|
||||
.insert(automationRunTriggers)
|
||||
.onConflict()
|
||||
.ignore()
|
||||
|
||||
// Copy `automation_function_runs` rows for automation run
|
||||
const automationFunctionRuns = await tables
|
||||
.automationFunctionRuns(deps.sourceDb)
|
||||
.select('*')
|
||||
.whereIn(AutomationFunctionRuns.col.runId, automationRunIds)
|
||||
|
||||
await tables
|
||||
.automationFunctionRuns(deps.targetDb)
|
||||
.insert(automationFunctionRuns)
|
||||
.onConflict()
|
||||
.ignore()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return copiedAutomationCountByProjectId
|
||||
}
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
import { GetProjectAutomationCount } from '@/modules/automate/domain/operations'
|
||||
import { GetStreamBranchCount } from '@/modules/core/domain/branches/operations'
|
||||
import { GetStreamCommitCount } from '@/modules/core/domain/commits/operations'
|
||||
import { GetStreamObjectCount } from '@/modules/core/domain/objects/operations'
|
||||
import { GetProject } from '@/modules/core/domain/projects/operations'
|
||||
import {
|
||||
CopyProjectAutomations,
|
||||
CopyProjectModels,
|
||||
CopyProjectObjects,
|
||||
CopyProjects,
|
||||
@@ -19,12 +21,14 @@ export const updateProjectRegionFactory =
|
||||
countProjectModels: GetStreamBranchCount
|
||||
countProjectVersions: GetStreamCommitCount
|
||||
countProjectObjects: GetStreamObjectCount
|
||||
countProjectAutomations: GetProjectAutomationCount
|
||||
getAvailableRegions: GetAvailableRegions
|
||||
copyWorkspace: CopyWorkspace
|
||||
copyProjects: CopyProjects
|
||||
copyProjectModels: CopyProjectModels
|
||||
copyProjectVersions: CopyProjectVersions
|
||||
copyProjectObjects: CopyProjectObjects
|
||||
copyProjectAutomations: CopyProjectAutomations
|
||||
}): UpdateProjectRegion =>
|
||||
async (params) => {
|
||||
const { projectId, regionKey } = params
|
||||
@@ -67,7 +71,9 @@ export const updateProjectRegionFactory =
|
||||
// Move objects
|
||||
const copiedObjectCount = await deps.copyProjectObjects({ projectIds })
|
||||
|
||||
// TODO: Move automations
|
||||
// Move automations
|
||||
const copiedAutomationCount = await deps.copyProjectAutomations({ projectIds })
|
||||
|
||||
// TODO: Move comments
|
||||
// TODO: Move file blobs
|
||||
// TODO: Move webhooks
|
||||
@@ -78,11 +84,15 @@ export const updateProjectRegionFactory =
|
||||
const sourceProjectObjectCount = await deps.countProjectObjects({
|
||||
streamId: projectId
|
||||
})
|
||||
const sourceProjectAutomationCount = await deps.countProjectAutomations({
|
||||
projectId
|
||||
})
|
||||
|
||||
const tests = [
|
||||
copiedModelCount[projectId] === sourceProjectModelCount,
|
||||
copiedVersionCount[projectId] === sourceProjectVersionCount,
|
||||
copiedObjectCount[projectId] === sourceProjectObjectCount
|
||||
copiedObjectCount[projectId] === sourceProjectObjectCount,
|
||||
copiedAutomationCount[projectId] === sourceProjectAutomationCount
|
||||
]
|
||||
|
||||
if (!tests.every((test) => !!test)) {
|
||||
|
||||
@@ -1,4 +1,23 @@
|
||||
import { db } from '@/db/knex'
|
||||
import {
|
||||
AutomationFunctionRunRecord,
|
||||
AutomationRecord,
|
||||
AutomationRevisionFunctionRecord,
|
||||
AutomationRevisionRecord,
|
||||
AutomationRunRecord,
|
||||
AutomationRunTriggerRecord,
|
||||
AutomationTokenRecord,
|
||||
AutomationTriggerDefinitionRecord
|
||||
} from '@/modules/automate/helpers/types'
|
||||
import {
|
||||
AutomationFunctionRuns,
|
||||
AutomationRevisionFunctions,
|
||||
AutomationRevisions,
|
||||
AutomationRuns,
|
||||
AutomationRunTriggers,
|
||||
AutomationTokens,
|
||||
AutomationTriggers
|
||||
} from '@/modules/core/dbSchema'
|
||||
import { AllScopes } from '@/modules/core/helpers/mainConstants'
|
||||
import { createRandomEmail } from '@/modules/core/helpers/testHelpers'
|
||||
import {
|
||||
@@ -35,6 +54,10 @@ import {
|
||||
TestApolloServer
|
||||
} from '@/test/graphqlHelper'
|
||||
import { beforeEachContext } from '@/test/hooks'
|
||||
import {
|
||||
createTestAutomation,
|
||||
createTestAutomationRun
|
||||
} from '@/test/speckle-helpers/automationHelper'
|
||||
import { BasicTestBranch, createTestBranch } from '@/test/speckle-helpers/branchHelper'
|
||||
import {
|
||||
BasicTestCommit,
|
||||
@@ -58,7 +81,20 @@ const tables = {
|
||||
versions: (db: Knex) => db.table<CommitRecord>('commits'),
|
||||
streamCommits: (db: Knex) => db.table<StreamCommitRecord>('stream_commits'),
|
||||
branchCommits: (db: Knex) => db.table<BranchCommitRecord>('branch_commits'),
|
||||
objects: (db: Knex) => db.table<ObjectRecord>('objects')
|
||||
objects: (db: Knex) => db.table<ObjectRecord>('objects'),
|
||||
automations: (db: Knex) => db.table<AutomationRecord>('automations'),
|
||||
automationTokens: (db: Knex) => db<AutomationTokenRecord>(AutomationTokens.name),
|
||||
automationRevisions: (db: Knex) =>
|
||||
db<AutomationRevisionRecord>(AutomationRevisions.name),
|
||||
automationTriggers: (db: Knex) =>
|
||||
db<AutomationTriggerDefinitionRecord>(AutomationTriggers.name),
|
||||
automationRevisionFunctions: (db: Knex) =>
|
||||
db<AutomationRevisionFunctionRecord>(AutomationRevisionFunctions.name),
|
||||
automationRuns: (db: Knex) => db<AutomationRunRecord>(AutomationRuns.name),
|
||||
automationRunTriggers: (db: Knex) =>
|
||||
db<AutomationRunTriggerRecord>(AutomationRunTriggers.name),
|
||||
automationFunctionRuns: (db: Knex) =>
|
||||
db<AutomationFunctionRunRecord>(AutomationFunctionRuns.name)
|
||||
}
|
||||
|
||||
const grantStreamPermissions = grantStreamPermissionsFactory({ db })
|
||||
@@ -344,6 +380,12 @@ isMultiRegionTestMode()
|
||||
authorId: ''
|
||||
}
|
||||
|
||||
let testAutomation: AutomationRecord
|
||||
let testAutomationToken: AutomationTokenRecord
|
||||
let testAutomationRevision: AutomationRevisionRecord
|
||||
let testAutomationRun: AutomationRunRecord
|
||||
let testAutomationFunctionRuns: AutomationFunctionRunRecord[]
|
||||
|
||||
let apollo: TestApolloServer
|
||||
let targetRegionDb: Knex
|
||||
|
||||
@@ -382,6 +424,32 @@ isMultiRegionTestMode()
|
||||
owner: adminUser,
|
||||
stream: testProject
|
||||
})
|
||||
|
||||
const { automation, revision } = await createTestAutomation({
|
||||
userId: adminUser.id,
|
||||
projectId: testProject.id,
|
||||
revision: {
|
||||
functionId: cryptoRandomString({ length: 9 }),
|
||||
functionReleaseId: cryptoRandomString({ length: 9 })
|
||||
}
|
||||
})
|
||||
|
||||
if (!revision) {
|
||||
throw new Error('Failed to create automation revision.')
|
||||
}
|
||||
|
||||
testAutomation = automation.automation
|
||||
testAutomationToken = automation.token
|
||||
testAutomationRevision = revision
|
||||
|
||||
const { automationRun, functionRuns } = await createTestAutomationRun({
|
||||
userId: adminUser.id,
|
||||
projectId: testProject.id,
|
||||
automationId: testAutomation.id
|
||||
})
|
||||
|
||||
testAutomationRun = automationRun
|
||||
testAutomationFunctionRuns = functionRuns
|
||||
})
|
||||
|
||||
it('moves project record to target regional db', async () => {
|
||||
@@ -468,5 +536,83 @@ isMultiRegionTestMode()
|
||||
|
||||
expect(object).to.not.be.undefined
|
||||
})
|
||||
|
||||
it('moves project automation data to target regional db', async () => {
|
||||
const res = await apollo.execute(UpdateProjectRegionDocument, {
|
||||
projectId: testProject.id,
|
||||
regionKey: regionKey2
|
||||
})
|
||||
|
||||
expect(res).to.not.haveGraphQLErrors()
|
||||
|
||||
// TODO: Replace with gql query when possible
|
||||
const automation = await tables
|
||||
.automations(targetRegionDb)
|
||||
.select('*')
|
||||
.where({ id: testAutomation.id })
|
||||
.first()
|
||||
expect(automation).to.not.be.undefined
|
||||
|
||||
const automationToken = await tables
|
||||
.automationTokens(targetRegionDb)
|
||||
.select('*')
|
||||
.where({ automationId: testAutomation.id })
|
||||
.first()
|
||||
expect(automationToken).to.not.be.undefined
|
||||
expect(automationToken?.automateToken).to.equal(
|
||||
testAutomationToken.automateToken
|
||||
)
|
||||
|
||||
const automationRevision = await tables
|
||||
.automationRevisions(targetRegionDb)
|
||||
.select('*')
|
||||
.where({ automationId: testAutomation.id })
|
||||
.first()
|
||||
expect(automationRevision).to.not.be.undefined
|
||||
expect(automationRevision?.id).to.equal(testAutomationRevision.id)
|
||||
|
||||
const automationTrigger = await tables
|
||||
.automationTriggers(targetRegionDb)
|
||||
.select('*')
|
||||
.where({ automationRevisionId: testAutomationRevision.id })
|
||||
.first()
|
||||
expect(automationTrigger).to.not.be.undefined
|
||||
})
|
||||
|
||||
it('moves project automation runs to target regional db', async () => {
|
||||
const res = await apollo.execute(UpdateProjectRegionDocument, {
|
||||
projectId: testProject.id,
|
||||
regionKey: regionKey2
|
||||
})
|
||||
|
||||
expect(res).to.not.haveGraphQLErrors()
|
||||
|
||||
// TODO: Replace with gql query when possible
|
||||
const automationRun = await tables
|
||||
.automationRuns(targetRegionDb)
|
||||
.select('*')
|
||||
.where({ id: testAutomationRun.id })
|
||||
.first()
|
||||
expect(automationRun).to.not.be.undefined
|
||||
|
||||
const automationRunTriggers = await tables
|
||||
.automationRunTriggers(targetRegionDb)
|
||||
.select('*')
|
||||
.where({ automationRunId: testAutomationRun.id })
|
||||
expect(automationRunTriggers.length).to.not.equal(0)
|
||||
|
||||
const automationFunctionRuns = await tables
|
||||
.automationFunctionRuns(targetRegionDb)
|
||||
.select('*')
|
||||
.where({ runId: testAutomationRun.id })
|
||||
expect(automationFunctionRuns.length).to.equal(
|
||||
testAutomationFunctionRuns.length
|
||||
)
|
||||
expect(
|
||||
automationFunctionRuns.every((run) =>
|
||||
testAutomationFunctionRuns.some((testRun) => testRun.id === run.id)
|
||||
)
|
||||
)
|
||||
})
|
||||
})
|
||||
: void 0
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
import {
|
||||
getAutomationFactory,
|
||||
getFullAutomationRevisionMetadataFactory,
|
||||
getFullAutomationRunByIdFactory,
|
||||
getLatestAutomationRevisionFactory,
|
||||
storeAutomationFactory,
|
||||
storeAutomationRevisionFactory,
|
||||
storeAutomationTokenFactory
|
||||
storeAutomationTokenFactory,
|
||||
upsertAutomationRunFactory
|
||||
} from '@/modules/automate/repositories/automations'
|
||||
import {
|
||||
CreateAutomationRevisionDeps,
|
||||
@@ -15,6 +19,7 @@ import cryptoRandomString from 'crypto-random-string'
|
||||
import { createAutomation as clientCreateAutomation } from '@/modules/automate/clients/executionEngine'
|
||||
import {
|
||||
getBranchesByIdsFactory,
|
||||
getBranchLatestCommitsFactory,
|
||||
getLatestStreamBranchFactory
|
||||
} from '@/modules/core/repositories/branches'
|
||||
|
||||
@@ -35,6 +40,7 @@ import {
|
||||
import { faker } from '@faker-js/faker'
|
||||
import {
|
||||
getEncryptionKeyPair,
|
||||
getEncryptionKeyPairFor,
|
||||
getFunctionInputDecryptorFactory
|
||||
} from '@/modules/automate/services/encryption'
|
||||
import { buildDecryptor } from '@/modules/shared/utils/libsodium'
|
||||
@@ -42,22 +48,28 @@ import { db } from '@/db/knex'
|
||||
import { validateStreamAccessFactory } from '@/modules/core/services/streams/access'
|
||||
import { authorizeResolver } from '@/modules/shared'
|
||||
import { getEventBus } from '@/modules/shared/services/eventBus'
|
||||
import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector'
|
||||
import { Knex } from 'knex'
|
||||
import { createTestAutomationRunFactory } from '@/modules/automate/services/trigger'
|
||||
|
||||
const storeAutomation = storeAutomationFactory({ db })
|
||||
const storeAutomationToken = storeAutomationTokenFactory({ db })
|
||||
const storeAutomationRevision = storeAutomationRevisionFactory({ db })
|
||||
const getAutomation = getAutomationFactory({ db })
|
||||
const getLatestStreamBranch = getLatestStreamBranchFactory({ db })
|
||||
const validateStreamAccess = validateStreamAccessFactory({ authorizeResolver })
|
||||
|
||||
export const generateFunctionId = () => cryptoRandomString({ length: 10 })
|
||||
export const generateFunctionReleaseId = () => cryptoRandomString({ length: 10 })
|
||||
|
||||
/**
|
||||
* @param overrides By default, we mock requests to the execution engine. You can replace those mocks here.
|
||||
*/
|
||||
export const buildAutomationCreate = (
|
||||
overrides?: Partial<{
|
||||
createDbAutomation: typeof clientCreateAutomation
|
||||
}>
|
||||
params: {
|
||||
dbClient?: Knex
|
||||
overrides?: Partial<{
|
||||
createDbAutomation: typeof clientCreateAutomation
|
||||
}>
|
||||
} = {}
|
||||
) => {
|
||||
const { dbClient = db, overrides } = params
|
||||
|
||||
const create = createAutomationFactory({
|
||||
createAuthCode: createStoredAuthCodeFactory({ redis: createInmemoryRedisClient() }),
|
||||
automateCreateAutomation:
|
||||
@@ -66,8 +78,8 @@ export const buildAutomationCreate = (
|
||||
automationId: cryptoRandomString({ length: 10 }),
|
||||
token: cryptoRandomString({ length: 10 })
|
||||
})),
|
||||
storeAutomation,
|
||||
storeAutomationToken,
|
||||
storeAutomation: storeAutomationFactory({ db: dbClient }),
|
||||
storeAutomationToken: storeAutomationTokenFactory({ db: dbClient }),
|
||||
validateStreamAccess,
|
||||
eventEmit: getEventBus().emit
|
||||
})
|
||||
@@ -75,9 +87,17 @@ export const buildAutomationCreate = (
|
||||
return create
|
||||
}
|
||||
|
||||
/**
|
||||
* @param overrides By default, we mock requests to the execution engine. You can replace those mocks here.
|
||||
*/
|
||||
export const buildAutomationRevisionCreate = (
|
||||
overrides?: Partial<CreateAutomationRevisionDeps>
|
||||
params: {
|
||||
dbClient?: Knex
|
||||
overrides?: Partial<CreateAutomationRevisionDeps>
|
||||
} = {}
|
||||
) => {
|
||||
const { dbClient = db, overrides } = params
|
||||
|
||||
const fakeGetRelease = (params: {
|
||||
functionReleaseId: string
|
||||
functionId: string
|
||||
@@ -91,9 +111,9 @@ export const buildAutomationRevisionCreate = (
|
||||
})
|
||||
|
||||
const create = createAutomationRevisionFactory({
|
||||
getAutomation,
|
||||
storeAutomationRevision,
|
||||
getBranchesByIds: getBranchesByIdsFactory({ db }),
|
||||
getAutomation: getAutomationFactory({ db: dbClient }),
|
||||
storeAutomationRevision: storeAutomationRevisionFactory({ db: dbClient }),
|
||||
getBranchesByIds: getBranchesByIdsFactory({ db: dbClient }),
|
||||
getFunctionRelease: async (params) => fakeGetRelease(params),
|
||||
getFunctionReleases: async (params) => params.ids.map(fakeGetRelease),
|
||||
getEncryptionKeyPair,
|
||||
@@ -128,8 +148,10 @@ export const createTestAutomation = async (params: {
|
||||
revision: { input: revisionInput, functionReleaseId, functionId } = {}
|
||||
} = params
|
||||
|
||||
const createAutomation = buildAutomationCreate()
|
||||
const createRevision = buildAutomationRevisionCreate()
|
||||
const projectDb = await getProjectDbClient({ projectId })
|
||||
|
||||
const createAutomation = buildAutomationCreate({ dbClient: projectDb })
|
||||
const createRevision = buildAutomationRevisionCreate({ dbClient: projectDb })
|
||||
|
||||
const automationRet = await createAutomation({
|
||||
input: {
|
||||
@@ -143,7 +165,7 @@ export const createTestAutomation = async (params: {
|
||||
|
||||
let revisionRet: Awaited<ReturnType<typeof createRevision>> | null = null
|
||||
if (functionReleaseId?.length && functionId?.length) {
|
||||
const firstModel = await getLatestStreamBranch(projectId)
|
||||
const firstModel = await getLatestStreamBranchFactory({ db: projectDb })(projectId)
|
||||
|
||||
if (!firstModel)
|
||||
throw new Error(
|
||||
@@ -186,6 +208,55 @@ export type TestAutomationWithRevision = Awaited<
|
||||
ReturnType<typeof createTestAutomation>
|
||||
>
|
||||
|
||||
export const createTestAutomationRun = async (params: {
|
||||
userId: string
|
||||
projectId: string
|
||||
automationId: string
|
||||
}) => {
|
||||
const { userId, projectId, automationId } = params
|
||||
|
||||
const projectDb = await getProjectDbClient({ projectId })
|
||||
|
||||
const { automationRunId } = await createTestAutomationRunFactory({
|
||||
getEncryptionKeyPairFor,
|
||||
getFunctionInputDecryptor: getFunctionInputDecryptorFactory({
|
||||
buildDecryptor
|
||||
}),
|
||||
getAutomation: getAutomationFactory({
|
||||
db: projectDb
|
||||
}),
|
||||
getLatestAutomationRevision: getLatestAutomationRevisionFactory({
|
||||
db: projectDb
|
||||
}),
|
||||
getFullAutomationRevisionMetadata: getFullAutomationRevisionMetadataFactory({
|
||||
db: projectDb
|
||||
}),
|
||||
upsertAutomationRun: upsertAutomationRunFactory({
|
||||
db: projectDb
|
||||
}),
|
||||
getBranchLatestCommits: getBranchLatestCommitsFactory({
|
||||
db: projectDb
|
||||
}),
|
||||
validateStreamAccess
|
||||
})({ projectId, automationId, userId })
|
||||
|
||||
const automationRunData = await getFullAutomationRunByIdFactory({ db: projectDb })(
|
||||
automationRunId
|
||||
)
|
||||
|
||||
if (!automationRunData) {
|
||||
throw new Error('Failed to create test automation run!')
|
||||
}
|
||||
|
||||
const { triggers, functionRuns, ...automationRun } = automationRunData
|
||||
|
||||
return {
|
||||
automationRun,
|
||||
functionRuns,
|
||||
triggers
|
||||
}
|
||||
}
|
||||
|
||||
export const truncateAutomations = async () => {
|
||||
await truncateTables([
|
||||
AutomationRunTriggers.name,
|
||||
|
||||
Reference in New Issue
Block a user