huanld/speckle-server
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix(acc): env variables and repo functions

Browse Source
This commit is contained in:
Chuck Driesler
2025-07-25 21:18:43 +01:00
parent da338e1cd1
commit e894477cd3
13 changed files with 161 additions and 201 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/server/.env.example
-3
View File
@@ -185,9 +185,6 @@ OIDC_CLIENT_SECRET="gLb9IEutYQ0npyvA8iHxPsObY3duGB0w"
# AUTODESK_INTEGRATION_CLIENT_ID=
# AUTODESK_INTEGRATION_CLIENT_SECRET=
# AUTOMATE_FUNCTION_ACC_SVF2_IMPORTER_FUNCTION_ID=
# AUTOMATE_FUNCTION_ACC_SVF2_IMPORTER_FUNCTION_RELEASE_ID=
##########################################################
# Local dev settings
##########################################################
packages/server/modules/acc/domain/constants.ts
+6
View File
@@ -0,0 +1,6 @@
export const ImporterAutomateFunctions = {
svf2: {
functionId: '2909d29a9d',
functionReleaseId: 'eeff138439'
}
}
packages/server/modules/acc/graph/resolvers/accSyncItems.ts
+2 -2
View File
@@ -1,5 +1,5 @@
import type { AccSyncItem } from '@/modules/acc/domain/types'
import { createAccSyncItemAndNotifyFactory } from '@/modules/acc/repositories/accSyncItems'
import { createAccSyncItem } from '@/modules/acc/services/management'
import {
createAutomation,
getFunctionReleaseFactory,
@@ -123,7 +123,7 @@ const resolvers: Resolvers = {
)
}
const createSyncItem = createAccSyncItemAndNotifyFactory({
const createSyncItem = createAccSyncItem({
db: await getProjectDbClient({ projectId: input.projectId }),
eventEmit: getEventBus().emit
})
packages/server/modules/acc/oidcHelper.ts → packages/server/modules/acc/helpers/oidcHelper.ts
View File
packages/server/modules/acc/index.ts
+52 -98
View File
@@ -1,12 +1,15 @@
/* eslint-disable camelcase */
import { createAccOidcFlow } from '@/modules/acc/oidcHelper'
import { createAccOidcFlow } from '@/modules/acc/helpers/oidcHelper'
import { sessionMiddlewareFactory } from '@/modules/auth/middleware'
import type { SpeckleModule } from '@/modules/shared/helpers/typeHelper'
import { moduleLogger } from '@/observability/logging'
import type { Express } from 'express'
import { db } from '@/db/knex'
import { queryAllPendingAccSyncItemsFactory } from '@/modules/acc/repositories/accSyncItems'
import {
queryAllPendingAccSyncItemsFactory,
upsertAccSyncItemFactory
} from '@/modules/acc/repositories/accSyncItems'
import { scheduleExecutionFactory } from '@/modules/core/services/taskScheduler'
import {
acquireTaskLockFactory,
@@ -15,10 +18,7 @@ import {
import { Scopes, TIME_MS } from '@speckle/shared'
import type { ScheduleExecution } from '@/modules/core/domain/scheduledTasks/operations'
import { AccSyncItems } from '@/modules/acc/dbSchema'
import type {
AccSyncItem,
ModelDerivativeServiceDesignManifest
} from '@/modules/acc/domain/types'
import type { AccSyncItem } from '@/modules/acc/domain/types'
import type { InsertableAutomationRun } from '@/modules/automate/repositories/automations'
import {
getAutomationFactory,
@@ -38,12 +38,18 @@ import {
storeUserServerAppTokenFactory
} from '@/modules/core/repositories/tokens'
import { TokenResourceIdentifierType } from '@/modules/core/graph/generated/graphql'
import { getServerOrigin } from '@/modules/shared/helpers/envHelper'
import {
getAutodeskIntegrationClientId,
getAutodeskIntegrationClientSecret,
getFeatureFlags,
getServerOrigin
} from '@/modules/shared/helpers/envHelper'
import type { VersionCreatedTriggerManifest } from '@/modules/automate/helpers/types'
import { getManifestByUrn } from '@/modules/acc/clients/autodesk'
import { isReadyForImport } from '@/modules/acc/domain/logic'
import { ImporterAutomateFunctions } from '@/modules/acc/domain/constants'
export default function accRestApi(app: Express) {
export function accRestApi(app: Express) {
const sessionMiddleware = sessionMiddlewareFactory()
app.post('/auth/acc/login', sessionMiddleware, async (req, res) => {
const { projectId } = req.body
@@ -141,57 +147,6 @@ export default function accRestApi(app: Express) {
}
})
app.get('/acc/download', sessionMiddleware, async (req, res) => {
const clientId = '5Y2LzxsL3usaD1xAMyElBY8mcN6XKyfHfulZDV3up0jfhN5Y'
const clientSecret =
'qHyGqaP4zCWLyS2lp04qBDOC1giIupPzJPmLFKGFHKZrPYYpan27zF8vlhQr1RYL'
console.log('AAAAAA')
const token = Buffer.from(`${clientId}:${clientSecret}`, 'utf8').toString('base64')
const tokens = await fetch(
'https://developer.api.autodesk.com/authentication/v2/token',
{
method: 'POST',
body: new URLSearchParams({
grant_type: 'client_credentials',
scope: 'data:read account:read viewables:read'
}),
headers: {
Authorization: `Basic ${token}`,
Accept: 'application/json',
'Content-Type': 'application/x-www-form-urlencoded'
}
}
)
const data = await tokens.json()
console.log(data)
const { access_token } = data
// https://developer.api.autodesk.com/modelderivative/v2/designdata/{urn}/manifest
// (EMEA) https://developer.api.autodesk.com/modelderivative/v2/regions/eu/designdata/{urn}/manifest
const urn =
'dXJuOmFkc2sud2lwZW1lYTpmcy5maWxlOnZmLjVORWw3ajZJVF9PSmRDdjVDWFNHTlE_dmVyc2lvbj0xNA'
const response = await fetch(
`https://developer.api.autodesk.com/modelderivative/v2/regions/eu/designdata/${urn}/manifest`,
{
method: 'GET',
headers: {
Authorization: `Bearer ${access_token}`
}
}
)
const manifest: ModelDerivativeServiceDesignManifest = await response.json()
console.log(manifest)
res.status(200).send('OK')
})
// Registered ACC webhooks are handled here
// https://aps.autodesk.com/en/docs/webhooks/v1/reference/events/data_management_events/dm.version.added/
app.post('/api/v1/acc/webhook/callback', sessionMiddleware, async (req, res) => {
@@ -246,31 +201,30 @@ const schedulePendingAccSyncItemsPoll = () => {
})
return scheduleExecution(
'*/1 * * * *', // Every minute
'*/5 * * * *',
'pendingAccSyncItemPolling',
async (now: Date, { logger }) => {
logger.info('Checking for pending ACC Sync items')
// logger.info('Checking for pending ACC Sync items')
for await (const items of queryAllPendingAccSyncItemsFactory({ db })()) {
for (const syncItem of items) {
console.log(`${syncItem.accFileVersionUrn} : ${syncItem.accFileName}`)
const projectDb = await getProjectDbClient({ projectId: syncItem.projectId })
const urn = btoa(syncItem.accFileVersionUrn)
.replaceAll('+', '-')
.replaceAll('/', '_')
.replaceAll('=', '')
const manifest = await getManifestByUrn(syncItem.accFileVersionUrn)
const manifest = await getManifestByUrn(urn)
const isReady = isReadyForImport(manifest)
console.log(manifest)
logger.info(
{
syncItem,
manifest
},
`ACC sync item {syncItem.id} is ${isReady ? '' : 'not'} ready`
)
if (!isReadyForImport(manifest)) {
console.log('NOT READY')
continue
}
if (!isReady) continue
await projectDb.table<AccSyncItem>(AccSyncItems.name).update({
await upsertAccSyncItemFactory({ db: projectDb })({
...syncItem,
status: 'SYNCING'
})
@@ -304,14 +258,14 @@ const schedulePendingAccSyncItemsPoll = () => {
],
functionRuns: [
{
functionId: '2909d29a9d',
id: cryptoRandomString({ length: 15 }),
functionId: ImporterAutomateFunctions.svf2.functionId,
functionReleaseId: ImporterAutomateFunctions.svf2.functionId,
status: 'pending' as const,
elapsed: 0,
results: null,
contextView: null,
statusMessage: null,
functionReleaseId: 'eeff138439',
createdAt: new Date(),
updatedAt: new Date()
}
@@ -332,8 +286,6 @@ const schedulePendingAccSyncItemsPoll = () => {
appId: DefaultAppIds.Automate,
name: `acct-${syncItem.id}`,
userId: syncItem.authorId,
// for now this is a baked in constant
// should rely on the function definitions requesting the needed scopes
scopes: [
Scopes.Profile.Read,
Scopes.Streams.Read,
@@ -352,8 +304,6 @@ const schedulePendingAccSyncItemsPoll = () => {
syncItem.automationId
)
console.log({ automationToken })
if (!automationToken) continue
await triggerAutomationRun({
@@ -366,13 +316,10 @@ const schedulePendingAccSyncItemsPoll = () => {
functionInputs: {
projectId: syncItem.projectId,
modelId: syncItem.modelId,
autodeskUrn: btoa(syncItem.accFileVersionUrn)
.replaceAll('/', '_')
.replaceAll('==', ''),
autodeskRegion: 1,
autodeskClientId: '5Y2LzxsL3usaD1xAMyElBY8mcN6XKyfHfulZDV3up0jfhN5Y',
autodeskClientSecret:
'qHyGqaP4zCWLyS2lp04qBDOC1giIupPzJPmLFKGFHKZrPYYpan27zF8vlhQr1RYL'
autodeskUrn: syncItem.accFileVersionUrn,
autodeskRegion: syncItem.accRegion === 'EMEA' ? 1 : 0,
autodeskClientId: getAutodeskIntegrationClientId(),
autodeskClientSecret: getAutodeskIntegrationClientSecret()
}
})),
manifests: [
@@ -388,21 +335,28 @@ const schedulePendingAccSyncItemsPoll = () => {
}
}
},
30 * TIME_MS.second
5 * TIME_MS.minute
)
}
export const init: SpeckleModule['init'] = async ({ app }) => {
moduleLogger.info('🔑 Init acc module')
const { FF_ACC_INTEGRATION_ENABLED } = getFeatureFlags()
// Hoist rest
accRestApi(app)
const accModule: SpeckleModule = {
init: async ({ app, isInitial }) => {
if (!FF_ACC_INTEGRATION_ENABLED) return
scheduledTask = schedulePendingAccSyncItemsPoll()
moduleLogger.info('🖕 Init acc module')
if (isInitial) {
accRestApi(app)
scheduledTask = schedulePendingAccSyncItemsPoll()
}
},
shutdown: () => {
if (!FF_ACC_INTEGRATION_ENABLED) return
scheduledTask?.stop()
},
finalize: () => {}
}
export const shutdown: SpeckleModule['shutdown'] = async () => {
scheduledTask?.stop()
}
export const finalize: SpeckleModule['finalize'] = async () => {}
export default accModule
packages/server/modules/acc/repositories/accSyncItems.ts
+16 -98
View File
@@ -1,119 +1,37 @@
import { AccSyncItems } from '@/modules/acc/dbSchema'
import { AccSyncItemEvents } from '@/modules/acc/domain/events'
import type {
DeleteAccSyncItem,
QueryAllAccSyncItems,
UpsertAccSyncItem
} from '@/modules/acc/domain/operations'
import { executeBatchedSelect } from '@/modules/shared/helpers/dbHelper'
import type { AccRegion, AccSyncItem } from '@/modules/acc/domain/types'
import type { EventBusEmit } from '@/modules/shared/services/eventBus'
import type { AccSyncItem } from '@/modules/acc/domain/types'
import type { Knex } from 'knex'
import { tryRegisterAccWebhook } from '@/modules/acc/clients/autodesk'
import { getServerOrigin } from '@/modules/shared/helpers/envHelper'
import { without } from 'lodash-es'
const tables = {
accSyncItems: (db: Knex) => db<AccSyncItem>(AccSyncItems.name)
}
export type CreateAccSyncItemAndNotify = (
input: Omit<AccSyncItem, 'createdAt' | 'updatedAt'>
) => Promise<AccSyncItem>
export const getAutodeskAccessToken = async (): Promise<string> => {
try {
const clientId = '5Y2LzxsL3usaD1xAMyElBY8mcN6XKyfHfulZDV3up0jfhN5Y'
const clientSecret =
'qHyGqaP4zCWLyS2lp04qBDOC1giIupPzJPmLFKGFHKZrPYYpan27zF8vlhQr1RYL'
const basicAuth = Buffer.from(`${clientId}:${clientSecret}`).toString('base64')
const response = await fetch(
'https://developer.api.autodesk.com/authentication/v2/token',
{
method: 'POST',
headers: {
Authorization: `Basic ${basicAuth}`,
'Content-Type': 'application/x-www-form-urlencoded',
Accept: 'application/json'
},
body: new URLSearchParams({
/* eslint-disable-next-line */
grant_type: 'client_credentials',
scope: 'data:read account:read viewables:read'
}).toString()
}
)
if (!response.ok) {
const errText = await response.text()
throw new Error(`Failed to get access token: ${response.status} ${errText}`)
}
const json = await response.json()
if (!json.access_token) {
throw new Error('access token is not found')
}
return json.access_token as string
} catch (error) {
console.log(error)
throw error
}
}
export const createAccSyncItemAndNotifyFactory = (deps: {
db: Knex
eventEmit: EventBusEmit
}): CreateAccSyncItemAndNotify => {
return async (input) => {
const webhookId = await tryRegisterAccWebhook({
rootProjectFolderUrn: input.accRootProjectFolderId,
// For local development, you may set your public tailscale url as your local server's canonical origin
callbackUrl: `${getServerOrigin()}/api/v1/acc/webhook/callback`,
region: input.accRegion as AccRegion,
event: 'dm.version.added' // NOTE ACC: you can register an event only once
})
if (webhookId) {
// TODO ACC: Update webhook id on sync record
}
const now = new Date()
const [item] = await tables
.accSyncItems(deps.db)
.insert({
...input,
createdAt: now,
updatedAt: now
})
.returning('*')
// TODO ACC: somehow i could not managed to get subsriptions work, doing stupid timeout refetch in FE after create/delete/update
// Once we have it properly TODO ogu: fix it on FE
await deps.eventEmit({
eventName: AccSyncItemEvents.Created,
payload: {
syncItem: item,
projectId: item.projectId
}
})
return item
}
}
export const upsertAccSyncItemFactory =
(deps: { db: Knex }): UpsertAccSyncItem =>
async (item) => {
await tables
.accSyncItems(deps.db)
.insert(item)
.onConflict(AccSyncItems.col.id)
.merge([
AccSyncItems.col.status,
AccSyncItems.col.accFileVersionIndex,
AccSyncItems.col.accFileVersionUrn
] as (keyof AccSyncItem)[])
.onConflict(AccSyncItems.withoutTablePrefix.col.id)
.merge(
without(
AccSyncItems.withoutTablePrefix.cols,
AccSyncItems.withoutTablePrefix.col.id
) as (keyof AccSyncItem)[]
)
}
export const deleteAccSyncItemFactory =
(deps: { db: Knex }): DeleteAccSyncItem =>
async ({ id }) => {
await tables.accSyncItems(deps.db).where({ id }).delete()
}
export const queryAllPendingAccSyncItemsFactory =
packages/server/modules/acc/rest/oidc.ts
View File
packages/server/modules/acc/rest/webhooks.ts
View File
packages/server/modules/acc/services/management.ts
+53
View File
@@ -0,0 +1,53 @@
import { tryRegisterAccWebhook } from '@/modules/acc/clients/autodesk'
import { AccSyncItems } from '@/modules/acc/dbSchema'
import { AccSyncItemEvents } from '@/modules/acc/domain/events'
import type { AccRegion, AccSyncItem } from '@/modules/acc/domain/types'
import { getServerOrigin } from '@/modules/shared/helpers/envHelper'
import type { EventBusEmit } from '@/modules/shared/services/eventBus'
import type { Knex } from 'knex'
const tables = {
accSyncItems: (db: Knex) => db<AccSyncItem>(AccSyncItems.name)
}
export type CreateAccSyncItem = (
input: Omit<AccSyncItem, 'createdAt' | 'updatedAt'>
) => Promise<AccSyncItem>
export const createAccSyncItem = (deps: {
db: Knex
eventEmit: EventBusEmit
}): CreateAccSyncItem => {
return async (input) => {
const webhookId = await tryRegisterAccWebhook({
rootProjectFolderUrn: input.accRootProjectFolderId,
// For local development, you may set your public tailscale url as your local server's canonical origin
callbackUrl: `${getServerOrigin()}/api/v1/acc/webhook/callback`,
region: input.accRegion as AccRegion,
event: 'dm.version.added' // NOTE ACC: you can register an event only once
})
// TODO ACC: Upsert not db access
const [item] = await tables
.accSyncItems(deps.db)
.insert({
...input,
accWebhookId: webhookId ?? undefined,
createdAt: new Date(),
updatedAt: new Date()
})
.returning('*')
// TODO ACC: somehow i could not managed to get subsriptions work, doing stupid timeout refetch in FE after create/delete/update
// Once we have it properly TODO ogu: fix it on FE
await deps.eventEmit({
eventName: AccSyncItemEvents.Created,
payload: {
syncItem: item,
projectId: item.projectId
}
})
return item
}
}
packages/server/modules/shared/helpers/envHelper.ts
+8
View File
@@ -535,3 +535,11 @@ export const getPreviewServiceMaxQueueBackpressure = (): number => {
export const emailVerificationTimeoutMinutes = (): number => {
return getIntFromEnv('EMAIL_VERIFICATION_TIMEOUT_MINUTES', '5')
}
export function getAutodeskIntegrationClientId() {
return getStringFromEnv('AUTODESK_INTEGRATION_CLIENT_ID')
}
export function getAutodeskIntegrationClientSecret() {
return getStringFromEnv('AUTODESK_INTEGRATION_CLIENT_SECRET')
}
packages/shared/src/environment/index.ts
+7
View File
@@ -134,6 +134,12 @@ export const parseFeatureFlags = (
description:
'Enables the IFC file importer based on IFCOpenShell (as of July 2025). Even if enabled, the previous webIFC & .Net importer can be accessed by appending `.dotnetimporter.ifc` to the uploaded file name.',
defaults: { _: false }
},
FF_ACC_INTEGRATION_ENABLED: {
schema: z.boolean(),
description:
'Enables the integration with ACC. This synchronizes models with specified ACC assets.',
defaults: { _: false }
}
})
@@ -172,6 +178,7 @@ export type FeatureFlags = {
FF_LEGACY_FILE_IMPORTS_ENABLED: boolean
FF_LEGACY_IFC_IMPORTER_ENABLED: boolean
FF_EXPERIMENTAL_IFC_IMPORTER_ENABLED: boolean
FF_ACC_INTEGRATION_ENABLED: boolean
}
export function getFeatureFlags(): FeatureFlags {
utils/helm/speckle-server/templates/_helpers.tpl
+15
View File
@@ -607,6 +607,21 @@ Generate the environment variables for Speckle server and Speckle objects deploy
- name: FF_RETRY_ERRORED_PREVIEWS_ENABLED
value: {{ .Values.featureFlags.retryErroredPreviewsEnabled | quote }}
- name: FF_ACC_INTEGRATION_ENABLED
value: {{ .Values.featureFlags.accIntegrationEnabled | quote }}
{{- if .Values.featureFlags.accIntegrationEnabled }}
- name: AUTODESK_INTEGRATION_CLIENT_ID
value: {{ .Values.accIntegration.client_id }}
- name: AUTODESK_INTEGRATION_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: {{ default .Values.secretName .Values.accIntegration.clientSecret.secretName }}
key: {{ default "acc_integration_client_secret" .Values.accIntegration.clientSecret.secretKey }}
{{- end }}
{{- if .Values.featureFlags.billingIntegrationEnabled }}
- name: STRIPE_API_KEY
valueFrom:
utils/helm/speckle-server/templates/frontend_2/deployment.yml
+2
View File
@@ -145,6 +145,8 @@ spec:
value: {{ .Values.featureFlags.nextGenFileImporterEnabled | quote }}
- name: NUXT_PUBLIC_FF_LEGACY_FILE_IMPORTS_ENABLED
value: {{ .Values.featureFlags.legacyFileImportsEnabled | quote }}
- name: NUXT_PUBLIC_FF_ACC_INTEGRATION_ENABLED
value: {{ .Values.featureFlags.accIntegrationEnabled | quote }}
{{- if .Values.analytics.intercom_app_id }}
- name: NUXT_PUBLIC_INTERCOM_APP_ID
value: {{ .Values.analytics.intercom_app_id | quote }}