diff --git a/packages/server/.env.example b/packages/server/.env.example index b429acee9..c292fcbad 100644 --- a/packages/server/.env.example +++ b/packages/server/.env.example @@ -185,9 +185,6 @@ OIDC_CLIENT_SECRET="gLb9IEutYQ0npyvA8iHxPsObY3duGB0w" # AUTODESK_INTEGRATION_CLIENT_ID= # AUTODESK_INTEGRATION_CLIENT_SECRET= -# AUTOMATE_FUNCTION_ACC_SVF2_IMPORTER_FUNCTION_ID= -# AUTOMATE_FUNCTION_ACC_SVF2_IMPORTER_FUNCTION_RELEASE_ID= - ########################################################## # Local dev settings ########################################################## diff --git a/packages/server/modules/acc/domain/constants.ts b/packages/server/modules/acc/domain/constants.ts new file mode 100644 index 000000000..2c51520fe --- /dev/null +++ b/packages/server/modules/acc/domain/constants.ts @@ -0,0 +1,6 @@ +export const ImporterAutomateFunctions = { + svf2: { + functionId: '2909d29a9d', + functionReleaseId: 'eeff138439' + } +} diff --git a/packages/server/modules/acc/graph/resolvers/accSyncItems.ts b/packages/server/modules/acc/graph/resolvers/accSyncItems.ts index 595ab1152..661468916 100644 --- a/packages/server/modules/acc/graph/resolvers/accSyncItems.ts +++ b/packages/server/modules/acc/graph/resolvers/accSyncItems.ts @@ -1,5 +1,5 @@ import type { AccSyncItem } from '@/modules/acc/domain/types' -import { createAccSyncItemAndNotifyFactory } from '@/modules/acc/repositories/accSyncItems' +import { createAccSyncItem } from '@/modules/acc/services/management' import { createAutomation, getFunctionReleaseFactory, @@ -123,7 +123,7 @@ const resolvers: Resolvers = { ) } - const createSyncItem = createAccSyncItemAndNotifyFactory({ + const createSyncItem = createAccSyncItem({ db: await getProjectDbClient({ projectId: input.projectId }), eventEmit: getEventBus().emit }) diff --git a/packages/server/modules/acc/oidcHelper.ts b/packages/server/modules/acc/helpers/oidcHelper.ts similarity index 100% rename from packages/server/modules/acc/oidcHelper.ts rename to packages/server/modules/acc/helpers/oidcHelper.ts diff --git a/packages/server/modules/acc/index.ts b/packages/server/modules/acc/index.ts index d9731f3c7..d1bbd4518 100644 --- a/packages/server/modules/acc/index.ts +++ b/packages/server/modules/acc/index.ts @@ -1,12 +1,15 @@ /* eslint-disable camelcase */ -import { createAccOidcFlow } from '@/modules/acc/oidcHelper' +import { createAccOidcFlow } from '@/modules/acc/helpers/oidcHelper' import { sessionMiddlewareFactory } from '@/modules/auth/middleware' import type { SpeckleModule } from '@/modules/shared/helpers/typeHelper' import { moduleLogger } from '@/observability/logging' import type { Express } from 'express' import { db } from '@/db/knex' -import { queryAllPendingAccSyncItemsFactory } from '@/modules/acc/repositories/accSyncItems' +import { + queryAllPendingAccSyncItemsFactory, + upsertAccSyncItemFactory +} from '@/modules/acc/repositories/accSyncItems' import { scheduleExecutionFactory } from '@/modules/core/services/taskScheduler' import { acquireTaskLockFactory, @@ -15,10 +18,7 @@ import { import { Scopes, TIME_MS } from '@speckle/shared' import type { ScheduleExecution } from '@/modules/core/domain/scheduledTasks/operations' import { AccSyncItems } from '@/modules/acc/dbSchema' -import type { - AccSyncItem, - ModelDerivativeServiceDesignManifest -} from '@/modules/acc/domain/types' +import type { AccSyncItem } from '@/modules/acc/domain/types' import type { InsertableAutomationRun } from '@/modules/automate/repositories/automations' import { getAutomationFactory, @@ -38,12 +38,18 @@ import { storeUserServerAppTokenFactory } from '@/modules/core/repositories/tokens' import { TokenResourceIdentifierType } from '@/modules/core/graph/generated/graphql' -import { getServerOrigin } from '@/modules/shared/helpers/envHelper' +import { + getAutodeskIntegrationClientId, + getAutodeskIntegrationClientSecret, + getFeatureFlags, + getServerOrigin +} from '@/modules/shared/helpers/envHelper' import type { VersionCreatedTriggerManifest } from '@/modules/automate/helpers/types' import { getManifestByUrn } from '@/modules/acc/clients/autodesk' import { isReadyForImport } from '@/modules/acc/domain/logic' +import { ImporterAutomateFunctions } from '@/modules/acc/domain/constants' -export default function accRestApi(app: Express) { +export function accRestApi(app: Express) { const sessionMiddleware = sessionMiddlewareFactory() app.post('/auth/acc/login', sessionMiddleware, async (req, res) => { const { projectId } = req.body @@ -141,57 +147,6 @@ export default function accRestApi(app: Express) { } }) - app.get('/acc/download', sessionMiddleware, async (req, res) => { - const clientId = '5Y2LzxsL3usaD1xAMyElBY8mcN6XKyfHfulZDV3up0jfhN5Y' - const clientSecret = - 'qHyGqaP4zCWLyS2lp04qBDOC1giIupPzJPmLFKGFHKZrPYYpan27zF8vlhQr1RYL' - - console.log('AAAAAA') - const token = Buffer.from(`${clientId}:${clientSecret}`, 'utf8').toString('base64') - const tokens = await fetch( - 'https://developer.api.autodesk.com/authentication/v2/token', - { - method: 'POST', - body: new URLSearchParams({ - grant_type: 'client_credentials', - scope: 'data:read account:read viewables:read' - }), - headers: { - Authorization: `Basic ${token}`, - Accept: 'application/json', - 'Content-Type': 'application/x-www-form-urlencoded' - } - } - ) - - const data = await tokens.json() - - console.log(data) - - const { access_token } = data - - // https://developer.api.autodesk.com/modelderivative/v2/designdata/{urn}/manifest - // (EMEA) https://developer.api.autodesk.com/modelderivative/v2/regions/eu/designdata/{urn}/manifest - - const urn = - 'dXJuOmFkc2sud2lwZW1lYTpmcy5maWxlOnZmLjVORWw3ajZJVF9PSmRDdjVDWFNHTlE_dmVyc2lvbj0xNA' - const response = await fetch( - `https://developer.api.autodesk.com/modelderivative/v2/regions/eu/designdata/${urn}/manifest`, - { - method: 'GET', - headers: { - Authorization: `Bearer ${access_token}` - } - } - ) - - const manifest: ModelDerivativeServiceDesignManifest = await response.json() - - console.log(manifest) - - res.status(200).send('OK') - }) - // Registered ACC webhooks are handled here // https://aps.autodesk.com/en/docs/webhooks/v1/reference/events/data_management_events/dm.version.added/ app.post('/api/v1/acc/webhook/callback', sessionMiddleware, async (req, res) => { @@ -246,31 +201,30 @@ const schedulePendingAccSyncItemsPoll = () => { }) return scheduleExecution( - '*/1 * * * *', // Every minute + '*/5 * * * *', 'pendingAccSyncItemPolling', async (now: Date, { logger }) => { - logger.info('Checking for pending ACC Sync items') + // logger.info('Checking for pending ACC Sync items') for await (const items of queryAllPendingAccSyncItemsFactory({ db })()) { for (const syncItem of items) { - console.log(`${syncItem.accFileVersionUrn} : ${syncItem.accFileName}`) - const projectDb = await getProjectDbClient({ projectId: syncItem.projectId }) - const urn = btoa(syncItem.accFileVersionUrn) - .replaceAll('+', '-') - .replaceAll('/', '_') - .replaceAll('=', '') + const manifest = await getManifestByUrn(syncItem.accFileVersionUrn) - const manifest = await getManifestByUrn(urn) + const isReady = isReadyForImport(manifest) - console.log(manifest) + logger.info( + { + syncItem, + manifest + }, + `ACC sync item {syncItem.id} is ${isReady ? '' : 'not'} ready` + ) - if (!isReadyForImport(manifest)) { - console.log('NOT READY') - continue - } + if (!isReady) continue - await projectDb.table(AccSyncItems.name).update({ + await upsertAccSyncItemFactory({ db: projectDb })({ + ...syncItem, status: 'SYNCING' }) @@ -304,14 +258,14 @@ const schedulePendingAccSyncItemsPoll = () => { ], functionRuns: [ { - functionId: '2909d29a9d', id: cryptoRandomString({ length: 15 }), + functionId: ImporterAutomateFunctions.svf2.functionId, + functionReleaseId: ImporterAutomateFunctions.svf2.functionId, status: 'pending' as const, elapsed: 0, results: null, contextView: null, statusMessage: null, - functionReleaseId: 'eeff138439', createdAt: new Date(), updatedAt: new Date() } @@ -332,8 +286,6 @@ const schedulePendingAccSyncItemsPoll = () => { appId: DefaultAppIds.Automate, name: `acct-${syncItem.id}`, userId: syncItem.authorId, - // for now this is a baked in constant - // should rely on the function definitions requesting the needed scopes scopes: [ Scopes.Profile.Read, Scopes.Streams.Read, @@ -352,8 +304,6 @@ const schedulePendingAccSyncItemsPoll = () => { syncItem.automationId ) - console.log({ automationToken }) - if (!automationToken) continue await triggerAutomationRun({ @@ -366,13 +316,10 @@ const schedulePendingAccSyncItemsPoll = () => { functionInputs: { projectId: syncItem.projectId, modelId: syncItem.modelId, - autodeskUrn: btoa(syncItem.accFileVersionUrn) - .replaceAll('/', '_') - .replaceAll('==', ''), - autodeskRegion: 1, - autodeskClientId: '5Y2LzxsL3usaD1xAMyElBY8mcN6XKyfHfulZDV3up0jfhN5Y', - autodeskClientSecret: - 'qHyGqaP4zCWLyS2lp04qBDOC1giIupPzJPmLFKGFHKZrPYYpan27zF8vlhQr1RYL' + autodeskUrn: syncItem.accFileVersionUrn, + autodeskRegion: syncItem.accRegion === 'EMEA' ? 1 : 0, + autodeskClientId: getAutodeskIntegrationClientId(), + autodeskClientSecret: getAutodeskIntegrationClientSecret() } })), manifests: [ @@ -388,21 +335,28 @@ const schedulePendingAccSyncItemsPoll = () => { } } }, - 30 * TIME_MS.second + 5 * TIME_MS.minute ) } -export const init: SpeckleModule['init'] = async ({ app }) => { - moduleLogger.info('🔑 Init acc module') +const { FF_ACC_INTEGRATION_ENABLED } = getFeatureFlags() - // Hoist rest - accRestApi(app) +const accModule: SpeckleModule = { + init: async ({ app, isInitial }) => { + if (!FF_ACC_INTEGRATION_ENABLED) return - scheduledTask = schedulePendingAccSyncItemsPoll() + moduleLogger.info('🖕 Init acc module') + + if (isInitial) { + accRestApi(app) + scheduledTask = schedulePendingAccSyncItemsPoll() + } + }, + shutdown: () => { + if (!FF_ACC_INTEGRATION_ENABLED) return + scheduledTask?.stop() + }, + finalize: () => {} } -export const shutdown: SpeckleModule['shutdown'] = async () => { - scheduledTask?.stop() -} - -export const finalize: SpeckleModule['finalize'] = async () => {} +export default accModule diff --git a/packages/server/modules/acc/repositories/accSyncItems.ts b/packages/server/modules/acc/repositories/accSyncItems.ts index e2f7d5f25..fd9dd3380 100644 --- a/packages/server/modules/acc/repositories/accSyncItems.ts +++ b/packages/server/modules/acc/repositories/accSyncItems.ts @@ -1,119 +1,37 @@ import { AccSyncItems } from '@/modules/acc/dbSchema' -import { AccSyncItemEvents } from '@/modules/acc/domain/events' import type { + DeleteAccSyncItem, QueryAllAccSyncItems, UpsertAccSyncItem } from '@/modules/acc/domain/operations' import { executeBatchedSelect } from '@/modules/shared/helpers/dbHelper' -import type { AccRegion, AccSyncItem } from '@/modules/acc/domain/types' -import type { EventBusEmit } from '@/modules/shared/services/eventBus' +import type { AccSyncItem } from '@/modules/acc/domain/types' import type { Knex } from 'knex' -import { tryRegisterAccWebhook } from '@/modules/acc/clients/autodesk' -import { getServerOrigin } from '@/modules/shared/helpers/envHelper' +import { without } from 'lodash-es' const tables = { accSyncItems: (db: Knex) => db(AccSyncItems.name) } -export type CreateAccSyncItemAndNotify = ( - input: Omit -) => Promise - -export const getAutodeskAccessToken = async (): Promise => { - try { - const clientId = '5Y2LzxsL3usaD1xAMyElBY8mcN6XKyfHfulZDV3up0jfhN5Y' - const clientSecret = - 'qHyGqaP4zCWLyS2lp04qBDOC1giIupPzJPmLFKGFHKZrPYYpan27zF8vlhQr1RYL' - - const basicAuth = Buffer.from(`${clientId}:${clientSecret}`).toString('base64') - - const response = await fetch( - 'https://developer.api.autodesk.com/authentication/v2/token', - { - method: 'POST', - headers: { - Authorization: `Basic ${basicAuth}`, - 'Content-Type': 'application/x-www-form-urlencoded', - Accept: 'application/json' - }, - body: new URLSearchParams({ - /* eslint-disable-next-line */ - grant_type: 'client_credentials', - scope: 'data:read account:read viewables:read' - }).toString() - } - ) - - if (!response.ok) { - const errText = await response.text() - throw new Error(`Failed to get access token: ${response.status} ${errText}`) - } - - const json = await response.json() - if (!json.access_token) { - throw new Error('access token is not found') - } - return json.access_token as string - } catch (error) { - console.log(error) - throw error - } -} - -export const createAccSyncItemAndNotifyFactory = (deps: { - db: Knex - eventEmit: EventBusEmit -}): CreateAccSyncItemAndNotify => { - return async (input) => { - const webhookId = await tryRegisterAccWebhook({ - rootProjectFolderUrn: input.accRootProjectFolderId, - // For local development, you may set your public tailscale url as your local server's canonical origin - callbackUrl: `${getServerOrigin()}/api/v1/acc/webhook/callback`, - region: input.accRegion as AccRegion, - event: 'dm.version.added' // NOTE ACC: you can register an event only once - }) - - if (webhookId) { - // TODO ACC: Update webhook id on sync record - } - - const now = new Date() - - const [item] = await tables - .accSyncItems(deps.db) - .insert({ - ...input, - createdAt: now, - updatedAt: now - }) - .returning('*') - - // TODO ACC: somehow i could not managed to get subsriptions work, doing stupid timeout refetch in FE after create/delete/update - // Once we have it properly TODO ogu: fix it on FE - await deps.eventEmit({ - eventName: AccSyncItemEvents.Created, - payload: { - syncItem: item, - projectId: item.projectId - } - }) - - return item - } -} - export const upsertAccSyncItemFactory = (deps: { db: Knex }): UpsertAccSyncItem => async (item) => { await tables .accSyncItems(deps.db) .insert(item) - .onConflict(AccSyncItems.col.id) - .merge([ - AccSyncItems.col.status, - AccSyncItems.col.accFileVersionIndex, - AccSyncItems.col.accFileVersionUrn - ] as (keyof AccSyncItem)[]) + .onConflict(AccSyncItems.withoutTablePrefix.col.id) + .merge( + without( + AccSyncItems.withoutTablePrefix.cols, + AccSyncItems.withoutTablePrefix.col.id + ) as (keyof AccSyncItem)[] + ) + } + +export const deleteAccSyncItemFactory = + (deps: { db: Knex }): DeleteAccSyncItem => + async ({ id }) => { + await tables.accSyncItems(deps.db).where({ id }).delete() } export const queryAllPendingAccSyncItemsFactory = diff --git a/packages/server/modules/acc/rest/oidc.ts b/packages/server/modules/acc/rest/oidc.ts new file mode 100644 index 000000000..e69de29bb diff --git a/packages/server/modules/acc/rest/webhooks.ts b/packages/server/modules/acc/rest/webhooks.ts new file mode 100644 index 000000000..e69de29bb diff --git a/packages/server/modules/acc/services/management.ts b/packages/server/modules/acc/services/management.ts new file mode 100644 index 000000000..6ea3bf55c --- /dev/null +++ b/packages/server/modules/acc/services/management.ts @@ -0,0 +1,53 @@ +import { tryRegisterAccWebhook } from '@/modules/acc/clients/autodesk' +import { AccSyncItems } from '@/modules/acc/dbSchema' +import { AccSyncItemEvents } from '@/modules/acc/domain/events' +import type { AccRegion, AccSyncItem } from '@/modules/acc/domain/types' +import { getServerOrigin } from '@/modules/shared/helpers/envHelper' +import type { EventBusEmit } from '@/modules/shared/services/eventBus' +import type { Knex } from 'knex' + +const tables = { + accSyncItems: (db: Knex) => db(AccSyncItems.name) +} + +export type CreateAccSyncItem = ( + input: Omit +) => Promise + +export const createAccSyncItem = (deps: { + db: Knex + eventEmit: EventBusEmit +}): CreateAccSyncItem => { + return async (input) => { + const webhookId = await tryRegisterAccWebhook({ + rootProjectFolderUrn: input.accRootProjectFolderId, + // For local development, you may set your public tailscale url as your local server's canonical origin + callbackUrl: `${getServerOrigin()}/api/v1/acc/webhook/callback`, + region: input.accRegion as AccRegion, + event: 'dm.version.added' // NOTE ACC: you can register an event only once + }) + + // TODO ACC: Upsert not db access + const [item] = await tables + .accSyncItems(deps.db) + .insert({ + ...input, + accWebhookId: webhookId ?? undefined, + createdAt: new Date(), + updatedAt: new Date() + }) + .returning('*') + + // TODO ACC: somehow i could not managed to get subsriptions work, doing stupid timeout refetch in FE after create/delete/update + // Once we have it properly TODO ogu: fix it on FE + await deps.eventEmit({ + eventName: AccSyncItemEvents.Created, + payload: { + syncItem: item, + projectId: item.projectId + } + }) + + return item + } +} diff --git a/packages/server/modules/shared/helpers/envHelper.ts b/packages/server/modules/shared/helpers/envHelper.ts index 7ee615f36..f94831651 100644 --- a/packages/server/modules/shared/helpers/envHelper.ts +++ b/packages/server/modules/shared/helpers/envHelper.ts @@ -535,3 +535,11 @@ export const getPreviewServiceMaxQueueBackpressure = (): number => { export const emailVerificationTimeoutMinutes = (): number => { return getIntFromEnv('EMAIL_VERIFICATION_TIMEOUT_MINUTES', '5') } + +export function getAutodeskIntegrationClientId() { + return getStringFromEnv('AUTODESK_INTEGRATION_CLIENT_ID') +} + +export function getAutodeskIntegrationClientSecret() { + return getStringFromEnv('AUTODESK_INTEGRATION_CLIENT_SECRET') +} diff --git a/packages/shared/src/environment/index.ts b/packages/shared/src/environment/index.ts index 616cd6146..26d121d56 100644 --- a/packages/shared/src/environment/index.ts +++ b/packages/shared/src/environment/index.ts @@ -134,6 +134,12 @@ export const parseFeatureFlags = ( description: 'Enables the IFC file importer based on IFCOpenShell (as of July 2025). Even if enabled, the previous webIFC & .Net importer can be accessed by appending `.dotnetimporter.ifc` to the uploaded file name.', defaults: { _: false } + }, + FF_ACC_INTEGRATION_ENABLED: { + schema: z.boolean(), + description: + 'Enables the integration with ACC. This synchronizes models with specified ACC assets.', + defaults: { _: false } } }) @@ -172,6 +178,7 @@ export type FeatureFlags = { FF_LEGACY_FILE_IMPORTS_ENABLED: boolean FF_LEGACY_IFC_IMPORTER_ENABLED: boolean FF_EXPERIMENTAL_IFC_IMPORTER_ENABLED: boolean + FF_ACC_INTEGRATION_ENABLED: boolean } export function getFeatureFlags(): FeatureFlags { diff --git a/utils/helm/speckle-server/templates/_helpers.tpl b/utils/helm/speckle-server/templates/_helpers.tpl index 13a142a38..f67894885 100644 --- a/utils/helm/speckle-server/templates/_helpers.tpl +++ b/utils/helm/speckle-server/templates/_helpers.tpl @@ -607,6 +607,21 @@ Generate the environment variables for Speckle server and Speckle objects deploy - name: FF_RETRY_ERRORED_PREVIEWS_ENABLED value: {{ .Values.featureFlags.retryErroredPreviewsEnabled | quote }} +- name: FF_ACC_INTEGRATION_ENABLED + value: {{ .Values.featureFlags.accIntegrationEnabled | quote }} + +{{- if .Values.featureFlags.accIntegrationEnabled }} +- name: AUTODESK_INTEGRATION_CLIENT_ID + value: {{ .Values.accIntegration.client_id }} + +- name: AUTODESK_INTEGRATION_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ default .Values.secretName .Values.accIntegration.clientSecret.secretName }} + key: {{ default "acc_integration_client_secret" .Values.accIntegration.clientSecret.secretKey }} + +{{- end }} + {{- if .Values.featureFlags.billingIntegrationEnabled }} - name: STRIPE_API_KEY valueFrom: diff --git a/utils/helm/speckle-server/templates/frontend_2/deployment.yml b/utils/helm/speckle-server/templates/frontend_2/deployment.yml index e179cfec2..b8100ad70 100644 --- a/utils/helm/speckle-server/templates/frontend_2/deployment.yml +++ b/utils/helm/speckle-server/templates/frontend_2/deployment.yml @@ -145,6 +145,8 @@ spec: value: {{ .Values.featureFlags.nextGenFileImporterEnabled | quote }} - name: NUXT_PUBLIC_FF_LEGACY_FILE_IMPORTS_ENABLED value: {{ .Values.featureFlags.legacyFileImportsEnabled | quote }} + - name: NUXT_PUBLIC_FF_ACC_INTEGRATION_ENABLED + value: {{ .Values.featureFlags.accIntegrationEnabled | quote }} {{- if .Values.analytics.intercom_app_id }} - name: NUXT_PUBLIC_INTERCOM_APP_ID value: {{ .Values.analytics.intercom_app_id | quote }}