diff --git a/packages/fileimport-service/.env.example b/packages/fileimport-service/.env.example index 3e68740f0..7aecd2138 100644 --- a/packages/fileimport-service/.env.example +++ b/packages/fileimport-service/.env.example @@ -4,8 +4,6 @@ POSTGRES_MAX_CONNECTIONS_FILE_IMPORT_SERVICE='1' POSTGRES_CONNECTION_ACQUIRE_TIMEOUT_MILLIS='16000' POSTGRES_CONNECTION_CREATE_TIMEOUT_MILLIS='5000' FF_WORKSPACES_MULTI_REGION_ENABLED=false -FF_NEXT_GEN_FILE_IMPORTER_ENABLED=false -REDIS_URL="redis://127.0.0.1:6379" # IFC_DOTNET_DLL_PATH='packages/fileimport-service/src/ifc-dotnet/ifc-converter.dll' diff --git a/packages/fileimport-service/scripts/publishTask.ts b/packages/fileimport-service/scripts/publishTask.ts deleted file mode 100644 index c8f558e7a..000000000 --- a/packages/fileimport-service/scripts/publishTask.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { REDIS_URL, QUEUE_NAME } from '../src/nextGen/config.js' -import { initializeQueue } from '@speckle/shared/queue' -import { JobPayload } from '@speckle/shared/workers/fileimport' - -const jobQueue = await initializeQueue({ - queueName: QUEUE_NAME, - redisUrl: REDIS_URL -}) - -await jobQueue.add({ - serverUrl: '', - token: '', - jobId: '1', - projectId: '', - modelId: '', - blobId: '', - fileName: 'railing.ifc', - fileType: 'ifc', - timeOutSeconds: 100000000000 -}) - -console.log('published') - -process.exit() diff --git a/packages/fileimport-service/src/bin.ts b/packages/fileimport-service/src/bin.ts index c0b223597..6c5168a0a 100644 --- a/packages/fileimport-service/src/bin.ts +++ b/packages/fileimport-service/src/bin.ts @@ -1,16 +1,8 @@ import '@/bootstrap.js' // This has side-effects and has to be imported first -import * as Environment from '@speckle/shared/environment' - -const { FF_NEXT_GEN_FILE_IMPORTER_ENABLED } = Environment.getFeatureFlags() -import { main as oldMain } from '@/controller/daemon.js' -import { main } from '@/nextGen/main.js' +import { main } from '@/controller/daemon.js' const start = () => { - if (FF_NEXT_GEN_FILE_IMPORTER_ENABLED) { - void main() - } else { - void oldMain() - } + void main() } start() diff --git a/packages/fileimport-service/src/nextGen/config.ts b/packages/fileimport-service/src/nextGen/config.ts deleted file mode 100644 index cd0e80b12..000000000 --- a/packages/fileimport-service/src/nextGen/config.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { z } from 'zod' -import { parseEnv } from 'znv' - -export const { - REDIS_URL, - QUEUE_NAME, - FILE_IMPORT_TIME_LIMIT_MIN, - LOG_LEVEL, - LOG_PRETTY, - DOTNET_BINARY_PATH, - PYTHON_BINARY_PATH, - RHINO_IMPORTER_PATH -} = parseEnv(process.env, { - REDIS_URL: z.string().url(), - QUEUE_NAME: z.string().default('fileimport-service-jobs'), - FILE_IMPORT_TIME_LIMIT_MIN: z.number().default(30), // minutes - LOG_LEVEL: z.string().default('info'), - LOG_PRETTY: z.boolean().default(false), - DOTNET_BINARY_PATH: z.string().default('dotnet'), - PYTHON_BINARY_PATH: z.string().default('python3'), - RHINO_IMPORTER_PATH: z.string().default('rhino-importer.exe') -}) diff --git a/packages/fileimport-service/src/nextGen/healthcheck.ts b/packages/fileimport-service/src/nextGen/healthcheck.ts deleted file mode 100644 index a15f59c43..000000000 --- a/packages/fileimport-service/src/nextGen/healthcheck.ts +++ /dev/null @@ -1,21 +0,0 @@ -import http from 'node:http' -import { Logger } from 'pino' - -export const startHealthCheckServer = (params: { logger: Logger }) => { - const { logger } = params - const server = http.createServer((req, res) => { - if (req.url === '/healthz' && req.method === 'GET') { - res.writeHead(200, { 'Content-Type': 'text/plain' }) - res.end('OK') - } else { - res.writeHead(404, { 'Content-Type': 'text/plain' }) - res.end('Not Found') - } - }) - - server.listen(9080, '0.0.0.0', () => { - logger.info('Healthcheck server is running. Serving at 0.0.0.0:9080/healthz') - }) - - return server -} diff --git a/packages/fileimport-service/src/nextGen/jobProcessor.ts b/packages/fileimport-service/src/nextGen/jobProcessor.ts deleted file mode 100644 index f5ad26363..000000000 --- a/packages/fileimport-service/src/nextGen/jobProcessor.ts +++ /dev/null @@ -1,258 +0,0 @@ -import { downloadFile } from '@/controller/filesApi.js' -import { AppState } from '@speckle/shared/workers' -import { JobPayload, FileImportResultPayload } from '@speckle/shared/workers/fileimport' -import { Logger } from 'pino' -import { tmpdir } from 'node:os' -import { readFileSync } from 'node:fs' -import path from 'node:path' -import fs from 'fs' -import { runProcessWithTimeout } from '@/common/processHandling.js' -import { DOTNET_BINARY_PATH, RHINO_IMPORTER_PATH } from './config.js' -import { getIfcDllPath } from '@/controller/helpers/env.js' -import { z } from 'zod' -import { TIME_MS } from '@speckle/shared' - -const jobSuccess = z.object({ - success: z.literal(true), - commitId: z.string() -}) - -const jobError = z.object({ - success: z.literal(false), - error: z.string() -}) - -const jobResult = z.discriminatedUnion('success', [jobSuccess, jobError]) - -type JobArgs = { - job: JobPayload - timeout: number - logger: Logger - getAppState: () => AppState - getElapsed: () => number -} - -export const jobProcessor = async ({ - job, - timeout, - logger, - getAppState, - getElapsed -}: JobArgs): Promise => { - const taskLogger = logger - const jobMessage = - 'Processed job {jobId} with result {status}. It took {elapsed} seconds.' - - let parserUsed = 'none' - - const tmp = tmpdir() - const jobDir = path.join(tmp, job.jobId) - let downloadDurationSeconds = 0 - let parseDurationSeconds = 0 - - fs.rmSync(jobDir, { force: true, recursive: true }) - fs.mkdirSync(jobDir) - const fileType = job.fileType.toLowerCase() - const sourceFilePath = path.join(jobDir, job.fileName) - const resultsPath = path.join(jobDir, 'import_results.json') - - const elapsedDownloadDuration = (() => { - const start = new Date().getTime() - return () => (new Date().getTime() - start) / TIME_MS.second - })() - - try { - await downloadFile({ - speckleServerUrl: job.serverUrl, - fileId: job.blobId, - streamId: job.projectId, - token: job.token, - destination: sourceFilePath, - logger - }) - - downloadDurationSeconds = elapsedDownloadDuration() - } catch (err) { - if (!downloadDurationSeconds) downloadDurationSeconds = elapsedDownloadDuration() - - if (getAppState() === AppState.SHUTTINGDOWN) { - // likely that the job was cancelled due to the service shutting down - logger.warn( - { err, jobId: job.jobId, elapsed: getElapsed(), status: 'error' }, - jobMessage - ) - } else { - logger.error( - { err, jobId: job.jobId, elapsed: getElapsed(), status: 'error' }, - jobMessage - ) - } - - const reason = - err instanceof Error - ? err.stack ?? err.toString() - : 'unknown error while downloading file' - - try { - // try and clean up the job directory - fs.rmSync(jobDir, { recursive: true }) - } finally { - // return the error result whether or not the cleanup succeeded - return { - status: 'error', - result: { - parser: parserUsed, - durationSeconds: getElapsed(), - downloadDurationSeconds, - parseDurationSeconds - }, - reason - } - } - } - - const elapsedParseDuration = (() => { - const start = new Date().getTime() - return () => (new Date().getTime() - start) / TIME_MS.second - })() - - try { - switch (fileType) { - case 'ifc': - parserUsed = 'ifc' - const useDotnetIfcImporter = job.fileName - .toLowerCase() - .endsWith('.dotnetimporter.ifc') - - if (useDotnetIfcImporter) { - await runProcessWithTimeout( - taskLogger, - DOTNET_BINARY_PATH, - [ - getIfcDllPath(), - sourceFilePath, - resultsPath, - job.projectId, - `File upload: ${job.fileName}`, - job.modelId, - 'bogus', - 'regionName' - ], - { - SPECKLE_SERVER_URL: job.serverUrl, - USER_TOKEN: job.token - }, - Math.min(timeout, job.timeOutSeconds * TIME_MS.second), - resultsPath - ) - } else { - await runProcessWithTimeout( - taskLogger, - process.env['PYTHON_BINARY_PATH'] || 'python3', - [ - '-m', - 'speckleifc', - sourceFilePath, - resultsPath, - job.projectId, - `File upload: ${job.fileName}`, - job.modelId - ], - { - USER_TOKEN: job.token, - SPECKLE_SERVER_URL: job.serverUrl, - //speckleifc is not installed to sys (e.g. via pip), so we need to point it to the directory explicitly - PYTHONPATH: '/speckle-server/speckleifc/src/' - }, - Math.min(timeout, job.timeOutSeconds * TIME_MS.second), - resultsPath - ) - } - - break - case 'stl': - case 'obj': - case 'skp': - parserUsed = 'rhino' - await runProcessWithTimeout( - taskLogger, - RHINO_IMPORTER_PATH, - [ - sourceFilePath, - resultsPath, - job.projectId, - job.modelId, - job.serverUrl, - job.token - ], - { - SPECKLE_SERVER_URL: job.serverUrl, - USER_TOKEN: job.token - }, - Math.min(timeout, job.timeOutSeconds * TIME_MS.second), - resultsPath - ) - break - default: - throw new Error(`File type ${fileType} is not supported`) - } - - parseDurationSeconds = elapsedParseDuration() - - const output = jobResult.safeParse(JSON.parse(readFileSync(resultsPath, 'utf8'))) - - if (!output.success) { - throw new Error('could not parse the result file') - } - - if (!output.data.success) { - throw new Error(output.data.error) - } - - const versionId = output.data.commitId - return { - status: 'success', - result: { - versionId, - durationSeconds: getElapsed(), - downloadDurationSeconds, - parseDurationSeconds, - parser: parserUsed - }, - warnings: [] - } - } catch (err) { - if (!parseDurationSeconds) parseDurationSeconds = elapsedParseDuration() - - if (getAppState() === AppState.SHUTTINGDOWN) { - // likely that the job was cancelled due to the service shutting down - logger.warn( - { err, jobId: job.jobId, elapsed: getElapsed(), status: 'error' }, - jobMessage - ) - } else { - logger.error( - { err, jobId: job.jobId, elapsed: getElapsed(), status: 'error' }, - jobMessage - ) - } - - const reason = - err instanceof Error - ? err.stack ?? err.toString() - : 'unknown error while parsing file' - - return { - status: 'error', - result: { - parser: parserUsed, - durationSeconds: getElapsed(), - downloadDurationSeconds, - parseDurationSeconds - }, - reason - } - } finally { - fs.rmSync(jobDir, { recursive: true }) - } -} diff --git a/packages/fileimport-service/src/nextGen/main.ts b/packages/fileimport-service/src/nextGen/main.ts deleted file mode 100644 index 80fbeee34..000000000 --- a/packages/fileimport-service/src/nextGen/main.ts +++ /dev/null @@ -1,195 +0,0 @@ -import { AppState } from '@speckle/shared/workers' -import { initializeQueue } from '@speckle/shared/queue' -import { FILE_IMPORT_TIME_LIMIT_MIN, REDIS_URL, QUEUE_NAME } from '@/nextGen/config.js' -import type { - JobPayload, - FileImportResultPayload -} from '@speckle/shared/workers/fileimport' -import type Bull from 'bull' -import { logger } from '@/observability/logging.js' -import { Logger } from 'pino' -import { ensureError, TIME_MS } from '@speckle/shared' -import { jobProcessor } from './jobProcessor.js' -import { startHealthCheckServer } from './healthcheck.js' - -let jobQueue: Bull.Queue | undefined = undefined -let appState: AppState = AppState.STARTING -let currentJob: { logger: Logger; done: Bull.DoneCallback } | undefined = undefined -let healthCheckServer: ReturnType | undefined - -export const main = async () => { - logger.info('Starting FileUploads Service (nextGen 🚀)...') - // we discussed doing push based metrics from here - // await initMetrics({ app, registry: initPrometheusRegistry() }) - - // store this callback, so on shutdown we can error the job - - try { - jobQueue = await initializeQueue({ - queueName: QUEUE_NAME, - redisUrl: REDIS_URL - }) - } catch (e) { - const err = ensureError(e, 'Unknown error creating job queue') - logger.error({ err }, 'Error creating job queue') - - // the callback to server.listen has failed, so we need to exit the process and not just return - await beforeShutdown() // handle the shutdown gracefully - onShutdown() - process.exit(1) - } - appState = AppState.RUNNING - - healthCheckServer = startHealthCheckServer({ logger }) - - logger.debug(`Starting processing of "${QUEUE_NAME}" message queue`) - - await jobQueue.process(async (payload, done) => { - const elapsed = (() => { - const start = new Date().getTime() - return () => (new Date().getTime() - start) / TIME_MS.second - })() - let encounteredError = false - let jobLogger = logger.child({ - payloadId: payload.id, - jobPriorAttemptsMade: payload.attemptsMade - }) - - const job = payload.data - try { - currentJob = { done, logger: jobLogger } - jobLogger = jobLogger.child({ - jobId: job.jobId, - serverUrl: job.serverUrl - }) - const result = await jobProcessor({ - job, - logger, - timeout: FILE_IMPORT_TIME_LIMIT_MIN * TIME_MS.minute, - getAppState: () => appState, - getElapsed: elapsed - }) - await sendResult({ - ...job, - result - }) - } catch (err) { - if (appState === AppState.SHUTTINGDOWN) { - // likely that the job was cancelled due to the service shutting down - jobLogger.warn({ err }, 'Processing job {jobId} failed') - } else { - jobLogger.error({ err }, 'Processing job {jobId} failed') - } - if (err instanceof Error) { - encounteredError = true - try { - await sendResult({ - ...job, - result: { - status: 'error', - reason: err.message, - result: { - durationSeconds: 0, - downloadDurationSeconds: 0, - parseDurationSeconds: 0, - parser: 'none' - } - } - }) - } catch (sendErr) { - jobLogger.fatal({ err: sendErr }, 'Failed to send result for job {jobId}') - } finally { - done(err) - } - } else { - throw err - } - } finally { - if (!encounteredError) done() - currentJob = undefined - } - }) -} - -type GQLResponse = { - data?: { fileUploadMutations?: { finishFileImport?: null } } - errors?: object[] -} - -const sendResult = async ({ - serverUrl, - projectId, - jobId, - token, - result -}: { - serverUrl: string - projectId: string - jobId: string - token: string - result: FileImportResultPayload -}) => { - const gqlEndpoint = new URL(`/graphql`, serverUrl).toString() - const mutation = ` - mutation ($input: FinishFileImportInput!) { - fileUploadMutations { - finishFileImport(input: $input) - } - } - ` - const input = { - jobId, - projectId, - ...result - } - - const response = await fetch(gqlEndpoint, { - method: 'POST', - headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' }, - body: JSON.stringify({ - query: mutation, - variables: { - input - } - }) - }) - - const body = (await response.json()) as GQLResponse - - if (response.status !== 200 || body?.errors) { - currentJob?.logger.error( - { cause: JSON.stringify(body), sendResultUrl: gqlEndpoint, jobId, projectId }, - 'Failed to report result for job {jobId} to {sendResultUrl}' - ) - throw new Error(`Failed to report result for job ${jobId}`) - } -} - -const beforeShutdown = async () => { - logger.info('🛑 Beginning shut down, pausing all jobs') - appState = AppState.SHUTTINGDOWN - // stop accepting new jobs and kill any running jobs - if (jobQueue) { - await jobQueue.pause( - true, // just pausing this local worker of the queue - true // do not wait for active jobs to finish - ) - } - - if (currentJob) { - currentJob.logger.warn('Cancelling job due to fileimport-service shutdown') - currentJob.done(new Error('Job cancelled due to fileimport-service shutdown')) - } - // no need to close the job queue and redis client, when the process exits they will be closed automatically - - if (healthCheckServer) { - logger.info('Stopping health check server') - healthCheckServer.close(() => { - logger.info('Health check server stopped') - }) - } -} - -const onShutdown = () => { - logger.info('👋 Completed shut down, now exiting') -} diff --git a/packages/server/modules/fileuploads/domain/operations.ts b/packages/server/modules/fileuploads/domain/operations.ts index a4c41b04a..7cba65e0a 100644 --- a/packages/server/modules/fileuploads/domain/operations.ts +++ b/packages/server/modules/fileuploads/domain/operations.ts @@ -75,10 +75,6 @@ export type UpdateFileStatus = (params: { convertedCommitId: string | null }) => Promise -export type UpdateFileStatusForProjectFactory = (params: { - projectId: string -}) => Promise - export type UploadedFile = UploadResult & { userId: string } export type FileImportMessage = Pick< diff --git a/packages/server/modules/fileuploads/index.ts b/packages/server/modules/fileuploads/index.ts index 8aecc105f..d68caccc3 100644 --- a/packages/server/modules/fileuploads/index.ts +++ b/packages/server/modules/fileuploads/index.ts @@ -23,8 +23,7 @@ import { getEventBus } from '@/modules/shared/services/eventBus' import { expireOldPendingUploadsFactory, getFileInfoFactory, - updateFileUploadFactory, - updateFileStatusFactory + updateFileUploadFactory } from '@/modules/fileuploads/repositories/fileUploads' import { db } from '@/db/knex' import { getFileImportTimeLimitMinutes } from '@/modules/shared/helpers/envHelper' @@ -44,31 +43,19 @@ import { } from '@/modules/fileuploads/domain/consts' import { fileuploadRouterFactory } from '@/modules/fileuploads/rest/router' import { - initializeRhinoQueueFactory, - initializeIfcQueueFactory, shutdownQueues, - initializePostgresQueue, - initializeQueueFactory + initializePostgresQueue } from '@/modules/fileuploads/queues/fileimports' import { initializeEventListenersFactory } from '@/modules/fileuploads/events/eventListener' import type { ObserveResult } from '@/modules/fileuploads/observability/metrics' import { initializeMetrics } from '@/modules/fileuploads/observability/metrics' import { reportSubscriptionEventsFactory } from '@/modules/fileuploads/events/subscriptionListeners' import { configureClient } from '@/knexfile' -import { - requestActiveHandlerFactory, - requestErrorHandlerFactory, - requestFailedHandlerFactory -} from '@/modules/fileuploads/services/requestHandler' -import type { UpdateFileStatusForProjectFactory } from '@/modules/fileuploads/domain/operations' import { MisconfiguredEnvironmentError } from '@/modules/shared/errors' import { rhinoImporterSupportedFileExtensions } from '@speckle/shared/blobs' -const { - FF_NEXT_GEN_FILE_IMPORTER_ENABLED, - FF_BACKGROUND_JOBS_ENABLED, - FF_RHINO_FILE_IMPORTER_ENABLED -} = getFeatureFlags() +const { FF_NEXT_GEN_FILE_IMPORTER_ENABLED, FF_RHINO_FILE_IMPORTER_ENABLED } = + getFeatureFlags() let scheduledTasks: cron.ScheduledTask[] = [] @@ -115,12 +102,6 @@ const scheduleFileImportExpiry = async ({ ) } -const updateFileStatusBuilder: UpdateFileStatusForProjectFactory = async (params) => { - const { projectId } = params - const projectDb = await getProjectDbClient({ projectId }) - return updateFileStatusFactory({ db: projectDb }) -} - export const init: SpeckleModule['init'] = async ({ app, isInitial, @@ -140,80 +121,37 @@ export const init: SpeckleModule['init'] = async ({ if (isInitial) { // this feature flag is going away soon if (FF_NEXT_GEN_FILE_IMPORTER_ENABLED) { - // this freature flag is going away soon, it will be on by default - // once we switch stabilize the background jobs mechanism - if (FF_BACKGROUND_JOBS_ENABLED) { - moduleLogger.info('🗳️ Background Jobs are ENABLED') - const connectionUri = getFileImporterQueuePostgresUrl() - const queueDb = connectionUri - ? configureClient({ postgres: { connectionUri } }).public - : db - const requestQueues = [ + moduleLogger.info('🗳️ Next Gen File importer is ENABLED') + const connectionUri = getFileImporterQueuePostgresUrl() + const queueDb = connectionUri + ? configureClient({ postgres: { connectionUri } }).public + : db + const requestQueues = [ + await initializePostgresQueue({ + label: 'ifc', + supportedFileTypes: ['ifc'], + db: queueDb + }) + ] + if (FF_RHINO_FILE_IMPORTER_ENABLED) { + moduleLogger.info('🦏 Rhino File Importer is ENABLED') + if (!connectionUri) + throw new MisconfiguredEnvironmentError( + 'Need a dedicated queue for Rhino based fileimports' + ) + requestQueues.push( await initializePostgresQueue({ - label: 'ifc', - supportedFileTypes: ['ifc'], + label: 'rhino', + supportedFileTypes: [...rhinoImporterSupportedFileExtensions], + // using public here, as the private uri is not applicable here db: queueDb }) - ] - if (FF_RHINO_FILE_IMPORTER_ENABLED) { - moduleLogger.info('🦏 Rhino File Importer is ENABLED') - if (!connectionUri) - throw new MisconfiguredEnvironmentError( - 'Need a dedicated queue for Rhino based fileimports' - ) - requestQueues.push( - await initializePostgresQueue({ - label: 'rhino', - supportedFileTypes: [...rhinoImporterSupportedFileExtensions], - // using public here, as the private uri is not applicable here - db: queueDb - }) - ) - } - ;({ observeResult } = initializeMetrics({ - registers: [metricsRegister], - requestQueues - })) - } else { - const queueInits = [ - initializeIfcQueueFactory({ - initializeQueue: initializeQueueFactory({ - jobActiveHandler: requestActiveHandlerFactory({ - logger: moduleLogger, - updateFileStatusBuilder - }), - jobErrorHandler: requestErrorHandlerFactory({ logger: moduleLogger }), - jobFailedHandler: requestFailedHandlerFactory({ - logger: moduleLogger, - updateFileStatusForProjectFactory: updateFileStatusBuilder - }) - }) - })() - ] - if (FF_RHINO_FILE_IMPORTER_ENABLED) { - queueInits.push( - initializeRhinoQueueFactory({ - initializeQueue: initializeQueueFactory({ - jobActiveHandler: requestActiveHandlerFactory({ - logger: moduleLogger, - updateFileStatusBuilder - }), - jobErrorHandler: requestErrorHandlerFactory({ logger: moduleLogger }), - jobFailedHandler: requestFailedHandlerFactory({ - logger: moduleLogger, - updateFileStatusForProjectFactory: updateFileStatusBuilder - }) - }) - })() - ) - } - const requestQueues = await Promise.all(queueInits) - - ;({ observeResult } = initializeMetrics({ - registers: [metricsRegister], - requestQueues - })) + ) } + ;({ observeResult } = initializeMetrics({ + registers: [metricsRegister], + requestQueues + })) } const scheduleExecution = scheduleExecutionFactory({ diff --git a/packages/server/modules/fileuploads/observability/metrics.ts b/packages/server/modules/fileuploads/observability/metrics.ts index 187e0db46..7d1495378 100644 --- a/packages/server/modules/fileuploads/observability/metrics.ts +++ b/packages/server/modules/fileuploads/observability/metrics.ts @@ -3,7 +3,6 @@ import { type Registry, Counter, Summary, Gauge } from 'prom-client' import type { FileImportQueue } from '@/modules/fileuploads/domain/types' import type { FileImportResultPayload } from '@speckle/shared/workers/fileimport' import { JobResultStatus } from '@speckle/shared/workers/fileimport' -import type Bull from 'bull' export const FileImportJobDurationStep = { TOTAL: 'total', @@ -21,7 +20,7 @@ export type ObserveResult = (params: { jobResult: FileImportResultPayload }) => export const initializeMetrics = (params: { registers: Registry[] - requestQueues: (FileImportQueue & { queue?: Bull.Queue })[] + requestQueues: FileImportQueue[] }) => { const { registers, requestQueues } = params @@ -100,26 +99,6 @@ export const initializeMetrics = (params: { registers }) - const completedHandlerFactory = (queueLabel: string) => () => { - fileImportJobsRequestCompletedCounter.inc({ parser: queueLabel }) - } - - const failedHandlerFactory = (queueLabel: string) => () => { - fileImportJobsRequestFailedCounter.inc({ parser: queueLabel }) - } - - requestQueues.forEach((requestQueue) => { - if (!requestQueue.queue) return - - const completedHandler = completedHandlerFactory(requestQueue.label) - const failedHandler = failedHandlerFactory(requestQueue.label) - - requestQueue.queue.removeListener('completed', completedHandler) - requestQueue.queue.on('completed', completedHandler) - requestQueue.queue.removeListener('failed', failedHandler) - requestQueue.queue.on('failed', failedHandler) - }) - // ======= Responses ======= registers.forEach((r) => diff --git a/packages/server/modules/fileuploads/queues/fileimports.ts b/packages/server/modules/fileuploads/queues/fileimports.ts index dddee3eef..ef2293426 100644 --- a/packages/server/modules/fileuploads/queues/fileimports.ts +++ b/packages/server/modules/fileuploads/queues/fileimports.ts @@ -1,21 +1,11 @@ import { - getFileImportServiceIFCParserRedisUrl, - getFileImportServiceIFCQueueName, - getFileImportServiceRhinoParserRedisUrl, - getFileImportServiceRhinoQueueName, getFileImportTimeLimitMinutes, - getRedisUrl, - getServerOrigin, - isTestEnv + getServerOrigin } from '@/modules/shared/helpers/envHelper' import type { Logger } from '@/observability/logging' -import { fileUploadsLogger as logger } from '@/observability/logging' -import { TIME, TIME_MS } from '@speckle/shared' -import { initializeQueue as setupQueue } from '@speckle/shared/queue' +import { TIME_MS } from '@speckle/shared' import type { JobPayload } from '@speckle/shared/workers/fileimport' import type { FileImportQueue } from '@/modules/fileuploads/domain/types' -import type { ActiveEventCallback, ErrorEventCallback, FailedEventCallback } from 'bull' -import type Bull from 'bull' import { NumberOfFileImportRetries, DelayBetweenFileImportRetriesMinutes @@ -28,103 +18,14 @@ import { storeBackgroundJobFactory } from '@/modules/backgroundjobs/repositories' import { BackgroundJobStatus, BackgroundJobType } from '@/modules/backgroundjobs/domain' -import { rhinoImporterSupportedFileExtensions } from '@speckle/shared/blobs' - -const FILEIMPORT_SERVICE_RHINO_QUEUE_NAME = getFileImportServiceRhinoQueueName() -const FILEIMPORT_SERVICE_IFC_QUEUE_NAME = getFileImportServiceIFCQueueName() export const fileImportQueues: FileImportQueue[] = [] -if (isTestEnv()) { - logger.info(`Fileimport service test queue ID: ${FILEIMPORT_SERVICE_IFC_QUEUE_NAME}`) - logger.info( - `Monitor using: 'yarn cli bull monitor ${FILEIMPORT_SERVICE_IFC_QUEUE_NAME}'` - ) -} - -const limiter = { - max: 10, - duration: TIME_MS.second -} - const timeout = NumberOfFileImportRetries * (getFileImportTimeLimitMinutes() + DelayBetweenFileImportRetriesMinutes) * TIME_MS.minute -const defaultJobOptions = { - attempts: NumberOfFileImportRetries, - timeout, - backoff: { - type: 'fixed', - delay: DelayBetweenFileImportRetriesMinutes * TIME_MS.minute - }, - removeOnComplete: { - // retain completed jobs for 1 day or until it is the 100th completed job being retained, whichever comes first - age: 1 * TIME.day, - count: 100 - }, - removeOnFail: { - // retain completed jobs for 1 week or until it is the 1_000th failed job being retained, whichever comes first - age: 1 * TIME.week, - count: 1_000 - } -} - -export const initializeQueueFactory = - (deps: { - jobActiveHandler: ActiveEventCallback - jobErrorHandler: ErrorEventCallback - jobFailedHandler: FailedEventCallback - }) => - async (params: { - label: string - queueName: string - redisUrl: string - supportedFileTypes: string[] - }): Promise => { - const { label, queueName, redisUrl, supportedFileTypes } = params - const queue = await setupQueue({ - queueName, - redisUrl, - options: { - ...(!isTestEnv() ? { limiter } : {}), - defaultJobOptions - } - }) - - queue.removeListener('active', deps.jobActiveHandler) - queue.on('active', deps.jobActiveHandler) - - // The error event is triggered when an error in the Redis backend is thrown. - queue.removeListener('error', deps.jobErrorHandler) - queue.on('error', deps.jobErrorHandler) - - // The failed event is triggered when a job fails by throwing an exception during execution. - // https://api.docs.bullmq.io/interfaces/v5.QueueEventsListener.html#failed - queue.removeListener('failed', deps.jobFailedHandler) - queue.on('failed', deps.jobFailedHandler) - - const fileImportQueue = { - label, - queue, - supportedFileTypes: supportedFileTypes.map( - (type) => type.toLocaleLowerCase() // Normalize file types to lowercase (this is a safeguard to prevent stupid typos in the future) - ), - shutdown: async () => await queue.close(), - scheduleJob: async (jobData: JobPayload): Promise => { - await queue.add(jobData, defaultJobOptions) - }, - metrics: { - getPendingJobCount: () => queue.count(), - getWaitingJobCount: () => queue.getWaitingCount(), - getActiveJobCount: () => queue.getActiveCount() - } - } - fileImportQueues.push(fileImportQueue) - return fileImportQueue - } - export const initializePostgresQueue = async ({ label, supportedFileTypes, @@ -180,30 +81,6 @@ export const initializePostgresQueue = async ({ return fileImportQueue } -export const initializeRhinoQueueFactory = - (deps: { initializeQueue: ReturnType }) => - async () => { - const rhinoImportServiceRedisUrl = getFileImportServiceRhinoParserRedisUrl() - return deps.initializeQueue({ - label: 'rhino', - queueName: FILEIMPORT_SERVICE_RHINO_QUEUE_NAME, - redisUrl: rhinoImportServiceRedisUrl ? rhinoImportServiceRedisUrl : getRedisUrl(), - supportedFileTypes: [...rhinoImporterSupportedFileExtensions] - }) - } - -export const initializeIfcQueueFactory = - (deps: { initializeQueue: ReturnType }) => - async () => { - const ifcImportServiceRedisUrl = getFileImportServiceIFCParserRedisUrl() - return deps.initializeQueue({ - label: 'ifc', - queueName: FILEIMPORT_SERVICE_IFC_QUEUE_NAME, - redisUrl: ifcImportServiceRedisUrl ? ifcImportServiceRedisUrl : getRedisUrl(), - supportedFileTypes: ['ifc'] - }) - } - export const shutdownQueues = async (params: { logger: Logger }) => { for (const queue of fileImportQueues) { await queue.shutdown() diff --git a/packages/server/modules/fileuploads/services/requestHandler.ts b/packages/server/modules/fileuploads/services/requestHandler.ts deleted file mode 100644 index 449c8d833..000000000 --- a/packages/server/modules/fileuploads/services/requestHandler.ts +++ /dev/null @@ -1,83 +0,0 @@ -import type { Logger } from '@/observability/logging' -import type { ActiveEventCallback, ErrorEventCallback, FailedEventCallback } from 'bull' -import type { UpdateFileStatusForProjectFactory } from '@/modules/fileuploads/domain/operations' -import type { JobPayload } from '@speckle/shared/workers/fileimport' -import { FileUploadConvertedStatus } from '@speckle/shared/blobs' -import { ensureError } from '@speckle/shared' - -export const requestActiveHandlerFactory = - (deps: { - logger: Logger - updateFileStatusBuilder: UpdateFileStatusForProjectFactory - }): ActiveEventCallback => - async (job) => { - deps.logger.info( - { - jobId: job.id, - projectId: job.data.projectId, - streamId: job.data.projectId, //legacy - fileName: job.data.fileName, - fileType: job.data.fileType, - blobId: job.data.blobId, - modelId: job.data.modelId - }, - "File import job for file '${fileName}' accepted for project '${projectId} with file type '${fileType}'; updating status to 'Converting'." - ) - const updateFileStatus = await deps.updateFileStatusBuilder({ - projectId: job.data.projectId - }) - await updateFileStatus({ - fileId: job.data.blobId, - projectId: job.data.projectId, - status: FileUploadConvertedStatus.Converting, - convertedMessage: 'File import job accepted and converting started', - convertedCommitId: null - }) - } - -export const requestErrorHandlerFactory = - (deps: { logger: Logger }): ErrorEventCallback => - (e) => { - const err = ensureError( - e, - 'File import job errored for unknown reason, likely a Redis, networking, or application configuration problem' - ) - deps.logger.error({ err }, 'File import job errored (likely a Redis problem)') - // we do not have details about the job here, so we cannot update the file upload status - } - -export const requestFailedHandlerFactory = - (deps: { - logger: Logger - updateFileStatusForProjectFactory: UpdateFileStatusForProjectFactory - }): FailedEventCallback => - async (job, e) => { - const err = ensureError( - e, - 'File import job failed for an unknown reason. This may occur if the worker was killed, memory was exhausted, or a bug in the parsing of the file occurred which caused the worker to crash.' - ) - deps.logger.warn( - { - error: err, - jobId: job.id, - projectId: job.data.projectId, - streamId: job.data.projectId, //legacy - fileType: job.data.fileType, - fileName: job.data.fileName, - blobId: job.data.blobId, - modelId: job.data.modelId - }, - "File import job for file '${fileName}' failed for ${projectId} with file type ${fileType}. Updating status to 'Error'." - ) - const updateFileStatus = await deps.updateFileStatusForProjectFactory({ - projectId: job.data.projectId - }) - - await updateFileStatus({ - fileId: job.data.blobId, - projectId: job.data.projectId, - status: FileUploadConvertedStatus.Error, - convertedMessage: err.message, - convertedCommitId: null - }) - } diff --git a/packages/server/modules/shared/helpers/envHelper.ts b/packages/server/modules/shared/helpers/envHelper.ts index 8c1dead8f..22c6fbedc 100644 --- a/packages/server/modules/shared/helpers/envHelper.ts +++ b/packages/server/modules/shared/helpers/envHelper.ts @@ -130,28 +130,6 @@ export const fileImportServiceShouldUsePrivateObjectsServerUrl = (): boolean => return getBooleanFromEnv('FILEIMPORT_SERVICE_USE_PRIVATE_OBJECTS_SERVER_URL') } -export const getFileImportServiceRhinoParserRedisUrl = (): string | undefined => { - return getStringFromEnv('FILEIMPORT_SERVICE_RHINO_REDIS_URL', { unsafe: true }) -} - -export const getFileImportServiceRhinoQueueName = (): string => { - return ( - getStringFromEnv('FILEIMPORT_SERVICE_RHINO_QUEUE_NAME', { unsafe: true }) || - 'fileimport-service-jobs' - ) -} - -export const getFileImportServiceIFCParserRedisUrl = (): string | undefined => { - return getStringFromEnv('FILEIMPORT_SERVICE_IFC_REDIS_URL', { unsafe: true }) -} - -export const getFileImportServiceIFCQueueName = (): string => { - return ( - getStringFromEnv('FILEIMPORT_SERVICE_IFC_QUEUE_NAME', { unsafe: true }) || - 'fileimport-service-jobs' - ) -} - export const getPreviewServiceRedisUrl = (): string | undefined => { return process.env['PREVIEW_SERVICE_REDIS_URL'] } diff --git a/packages/shared/src/environment/featureFlags.ts b/packages/shared/src/environment/featureFlags.ts index bb4ee6cad..948abb604 100644 --- a/packages/shared/src/environment/featureFlags.ts +++ b/packages/shared/src/environment/featureFlags.ts @@ -17,7 +17,6 @@ export type FeatureFlags = { FF_PERSONAL_PROJECTS_LIMITS_ENABLED: boolean FF_NEXT_GEN_FILE_IMPORTER_ENABLED: boolean FF_RHINO_FILE_IMPORTER_ENABLED: boolean - FF_BACKGROUND_JOBS_ENABLED: boolean FF_LEGACY_FILE_IMPORTS_ENABLED: boolean FF_ACC_INTEGRATION_ENABLED: boolean FF_SAVED_VIEWS_ENABLED: boolean diff --git a/packages/shared/src/environment/index.ts b/packages/shared/src/environment/index.ts index 56ba1ac6c..1efb4936a 100644 --- a/packages/shared/src/environment/index.ts +++ b/packages/shared/src/environment/index.ts @@ -123,11 +123,6 @@ export const parseFeatureFlags = ( description: 'Enables the Rhino based file importer.', defaults: { _: false } }, - FF_BACKGROUND_JOBS_ENABLED: { - schema: z.boolean(), - description: 'Enables the postgres based background job mechanism', - defaults: { _: false } - }, FF_LEGACY_FILE_IMPORTS_ENABLED: { schema: z.boolean(), description: diff --git a/utils/helm/speckle-server/templates/_helpers.tpl b/utils/helm/speckle-server/templates/_helpers.tpl index 642b1032d..ad3a62270 100644 --- a/utils/helm/speckle-server/templates/_helpers.tpl +++ b/utils/helm/speckle-server/templates/_helpers.tpl @@ -578,9 +578,6 @@ Generate the environment variables for Speckle server and Speckle objects deploy - name: FF_MOVE_PROJECT_REGION_ENABLED value: {{ .Values.featureFlags.moveProjectRegionEnabled | quote }} -- name: FF_BACKGROUND_JOBS_ENABLED - value: {{ .Values.featureFlags.backgroundJobsEnabled | quote }} - {{- if .Values.featureFlags.gatekeeperModuleEnabled }} - name: LICENSE_TOKEN valueFrom: @@ -812,26 +809,6 @@ Generate the environment variables for Speckle server and Speckle objects deploy key: {{ default "preview_service_redis_url" .Values.redis.previewServiceConnectionString.secretKey }} {{- end }} -{{- if (and .Values.featureFlags.nextGenFileImporterEnabled (not .Values.featureFlags.backgroundJobsEnabled)) }} -- name: FILEIMPORT_SERVICE_RHINO_REDIS_URL - valueFrom: - secretKeyRef: - name: {{ default .Values.secretName .Values.redis.fileImportService.rhino.connectionString.secretName }} - key: {{ default "fileimport_service_rhino_redis_url" .Values.redis.fileImportService.rhino.connectionString.secretKey }} - -- name: FILEIMPORT_SERVICE_RHINO_QUEUE_NAME - value: {{ .Values.redis.fileImportService.rhino.queueName | quote }} - -- name: FILEIMPORT_SERVICE_IFC_REDIS_URL - valueFrom: - secretKeyRef: - name: {{ default .Values.secretName .Values.redis.fileImportService.ifc.connectionString.secretName }} - key: {{ default "fileimport_service_ifc_redis_url" .Values.redis.fileImportService.ifc.connectionString.secretKey }} - -- name: FILEIMPORT_SERVICE_IFC_QUEUE_NAME - value: {{ .Values.redis.fileImportService.ifc.queueName | quote }} -{{- end }} - # *** PostgreSQL Database *** - name: POSTGRES_URL valueFrom: @@ -1175,7 +1152,7 @@ Generate the environment variables for Speckle server and Speckle objects deploy value: {{ .Values.featureFlags.rhinoFileImporterEnabled | quote }} {{- end }} -{{- if .Values.featureFlags.backgroundJobsEnabled }} +{{- if .Values.featureFlags.nextGenFileImporterEnabled }} - name: FILEIMPORT_QUEUE_POSTGRES_URL valueFrom: secretKeyRef: diff --git a/utils/helm/speckle-server/templates/fileimport_service/deployment.yml b/utils/helm/speckle-server/templates/fileimport_service/deployment.yml index 1c979cc66..8415763a0 100644 --- a/utils/helm/speckle-server/templates/fileimport_service/deployment.yml +++ b/utils/helm/speckle-server/templates/fileimport_service/deployment.yml @@ -41,49 +41,31 @@ spec: periodSeconds: 10 failureThreshold: 18 # 10 * 18 = 180s to startup timeoutSeconds: 3 - {{- if .Values.featureFlags.nextGenFileImporterEnabled }} - httpGet: - path: /healthz - port: 9080 - {{- else }} exec: command: - /usr/bin/node - -e - "process.exit((Date.now() - require('fs').readFileSync('/tmp/last_successful_query', 'utf8') > 25 * 60 * 1000) ? 1 : 0)" - {{- end }} livenessProbe: initialDelaySeconds: 60 periodSeconds: 60 timeoutSeconds: 3 - {{- if .Values.featureFlags.nextGenFileImporterEnabled }} - httpGet: - path: /healthz - port: 9080 - {{- else }} exec: command: - /usr/bin/node - -e - "process.exit((Date.now() - require('fs').readFileSync('/tmp/last_successful_query', 'utf8') > 25 * 60 * 1000) ? 1 : 0)" - {{- end }} readinessProbe: initialDelaySeconds: 60 periodSeconds: 10 timeoutSeconds: 3 - {{- if .Values.featureFlags.nextGenFileImporterEnabled }} - httpGet: - path: /healthz - port: 9080 - {{- else }} exec: command: - /usr/bin/node - -e - "process.exit((Date.now() - require('fs').readFileSync('/tmp/last_successful_query', 'utf8') > 25 * 60 * 1000) ? 1 : 0)" - {{- end }} resources: {{- with .Values.fileimport_service.requests }} @@ -121,26 +103,9 @@ spec: {{- end }} env: - {{- if (not .Values.featureFlags.nextGenFileImporterEnabled) }} - name: SPECKLE_SERVER_URL value: {{ printf "http://%s:%s" ( include "objects.service.fqdn" $ ) ( include "objects.port" $ ) }} - {{- end }} - {{- if .Values.featureFlags.nextGenFileImporterEnabled }} - - name: REDIS_URL - valueFrom: - secretKeyRef: - name: {{ default .Values.secretName .Values.redis.fileImportService.ifc.connectionString.secretName }} - key: {{ default "fileimport_service_ifc_redis_url" .Values.redis.fileImportService.ifc.connectionString.secretKey }} - - name: QUEUE_NAME - value: {{ .Values.redis.fileImportService.ifc.queueName | quote }} - {{- else }} - # Dummy value for backwards compatibility - - name: REDIS_URL - value: "redis://redis" - {{- end }} - - {{- if (not .Values.featureFlags.nextGenFileImporterEnabled) }} - name: PG_CONNECTION_STRING valueFrom: secretKeyRef: @@ -157,7 +122,6 @@ spec: - name: LOG_LEVEL value: {{ .Values.fileimport_service.logLevel | quote }} - - name: LOG_PRETTY value: {{ .Values.fileimport_service.logPretty | quote }} @@ -165,25 +129,18 @@ spec: - name: NODE_EXTRA_CA_CERTS value: "/postgres-certificate/ca-certificate.crt" {{- end }} - {{- end }} - name: FILE_IMPORT_TIME_LIMIT_MIN value: {{ (or .Values.file_import_time_limit_min .Values.fileimport_service.time_limit_min) | quote }} - {{- if (not .Values.featureFlags.nextGenFileImporterEnabled) }} # DEPRECATED - name: MAX_OBJECT_SIZE_MB value: {{ .Values.server.max_object_size_mb | quote }} - {{- end }} - {{- if (and .Values.featureFlags.workspacesMultiRegionEnabled (not .Values.featureFlags.nextGenFileImporterEnabled)) }} + {{- if .Values.featureFlags.workspacesMultiRegionEnabled }} - name: FF_WORKSPACES_MULTI_REGION_ENABLED value: {{ .Values.featureFlags.workspacesMultiRegionEnabled | quote }} - name: MULTI_REGION_CONFIG_PATH value: "/multi-region-config/multi-region-config.json" {{- end }} - {{- if .Values.featureFlags.nextGenFileImporterEnabled }} - - name: FF_NEXT_GEN_FILE_IMPORTER_ENABLED - value: {{ .Values.featureFlags.nextGenFileImporterEnabled | quote }} - {{- end }} {{- with .Values.fileimport_service.additionalEnvVars }} {{- toYaml . | nindent 10}} {{- end }} diff --git a/utils/helm/speckle-server/templates/fileimport_service/serviceaccount.yml b/utils/helm/speckle-server/templates/fileimport_service/serviceaccount.yml index 4f5e58622..e2178c989 100644 --- a/utils/helm/speckle-server/templates/fileimport_service/serviceaccount.yml +++ b/utils/helm/speckle-server/templates/fileimport_service/serviceaccount.yml @@ -20,9 +20,5 @@ secrets: {{- if .Values.featureFlags.workspacesMultiRegionEnabled }} - name: {{ default .Values.secretName .Values.multiRegion.config.secretName }} {{- end }} -{{- if .Values.featureFlags.nextGenFileImporterEnabled }} - - name: {{ default .Values.secretName .Values.redis.fileImportService.rhino.connectionString.secretName }} - - name: {{ default .Values.secretName .Values.redis.fileImportService.ifc.connectionString.secretName }} -{{- end }} {{- end -}} {{- end }} diff --git a/utils/helm/speckle-server/templates/ifc_import_service/configmap-db-certificate.yml b/utils/helm/speckle-server/templates/ifc_import_service/configmap-db-certificate.yml index 3b8cc38b6..aab17b7c5 100644 --- a/utils/helm/speckle-server/templates/ifc_import_service/configmap-db-certificate.yml +++ b/utils/helm/speckle-server/templates/ifc_import_service/configmap-db-certificate.yml @@ -1,4 +1,4 @@ -{{ if ( and .Values.featureFlags.backgroundJobsEnabled .Values.ifc_import_service.db.certificate.enabled .Values.ifc_import_service.db.certificate.data ) }} +{{ if ( and .Values.ifc_import_service.enabled .Values.ifc_import_service.db.certificate.enabled .Values.ifc_import_service.db.certificate.data ) }} apiVersion: v1 kind: ConfigMap diff --git a/utils/helm/speckle-server/templates/ifc_import_service/deployment.yml b/utils/helm/speckle-server/templates/ifc_import_service/deployment.yml index 409f0c3f5..7f5704ea3 100644 --- a/utils/helm/speckle-server/templates/ifc_import_service/deployment.yml +++ b/utils/helm/speckle-server/templates/ifc_import_service/deployment.yml @@ -1,4 +1,4 @@ -{{- if (and .Values.ifc_import_service.enabled .Values.featureFlags.backgroundJobsEnabled) }} +{{- if .Values.ifc_import_service.enabled }} apiVersion: apps/v1 kind: Deployment metadata: @@ -30,6 +30,9 @@ spec: - name: metrics containerPort: 9093 protocol: TCP + - name: healthz + containerPort: 9080 + protocol: TCP startupProbe: periodSeconds: 10 diff --git a/utils/helm/speckle-server/templates/ifc_import_service/service.yml b/utils/helm/speckle-server/templates/ifc_import_service/service.yml index 744575317..4d5409457 100644 --- a/utils/helm/speckle-server/templates/ifc_import_service/service.yml +++ b/utils/helm/speckle-server/templates/ifc_import_service/service.yml @@ -1,4 +1,4 @@ -{{- if (and .Values.ifc_import_service.enabled .Values.featureFlags.backgroundJobsEnabled) }} +{{- if .Values.ifc_import_service.enabled }} apiVersion: v1 kind: Service metadata: diff --git a/utils/helm/speckle-server/templates/ifc_import_service/serviceaccount.yml b/utils/helm/speckle-server/templates/ifc_import_service/serviceaccount.yml index f5b7040fe..4d92ecdc7 100644 --- a/utils/helm/speckle-server/templates/ifc_import_service/serviceaccount.yml +++ b/utils/helm/speckle-server/templates/ifc_import_service/serviceaccount.yml @@ -1,4 +1,4 @@ -{{- if (and .Values.ifc_import_service.enabled .Values.featureFlags.backgroundJobsEnabled .Values.ifc_import_service.serviceAccount.create) }} +{{- if (and .Values.ifc_import_service.enabled .Values.ifc_import_service.serviceAccount.create) }} apiVersion: v1 kind: ServiceAccount metadata: diff --git a/utils/helm/speckle-server/templates/objects/serviceaccount.yml b/utils/helm/speckle-server/templates/objects/serviceaccount.yml index 3690cea02..246a3d750 100644 --- a/utils/helm/speckle-server/templates/objects/serviceaccount.yml +++ b/utils/helm/speckle-server/templates/objects/serviceaccount.yml @@ -52,10 +52,6 @@ secrets: {{- if .Values.preview_service.dedicatedPreviewsQueue }} - name: {{ default .Values.secretName .Values.redis.previewServiceConnectionString.secretName }} {{- end }} -{{- if .Values.featureFlags.nextGenFileImporterEnabled }} - - name: {{ default .Values.secretName .Values.redis.fileImportService.rhino.connectionString.secretName }} - - name: {{ default .Values.secretName .Values.redis.fileImportService.ifc.connectionString.secretName }} -{{- end }} {{- if .Values.featureFlags.accIntegrationEnabled }} - name: {{ default .Values.secretName .Values.server.accIntegration.clientSecret.secretName }} {{- end }} diff --git a/utils/helm/speckle-server/templates/server/serviceaccount.yml b/utils/helm/speckle-server/templates/server/serviceaccount.yml index 598f4eade..311e79c8d 100644 --- a/utils/helm/speckle-server/templates/server/serviceaccount.yml +++ b/utils/helm/speckle-server/templates/server/serviceaccount.yml @@ -52,10 +52,6 @@ secrets: {{- if .Values.preview_service.dedicatedPreviewsQueue }} - name: {{ default .Values.secretName .Values.redis.previewServiceConnectionString.secretName }} {{- end }} -{{- if .Values.featureFlags.nextGenFileImporterEnabled }} - - name: {{ default .Values.secretName .Values.redis.fileImportService.rhino.connectionString.secretName }} - - name: {{ default .Values.secretName .Values.redis.fileImportService.ifc.connectionString.secretName }} -{{- end }} {{- if .Values.featureFlags.accIntegrationEnabled }} - name: {{ default .Values.secretName .Values.server.accIntegration.clientSecret.secretName }} {{- end }} diff --git a/utils/helm/speckle-server/values.schema.json b/utils/helm/speckle-server/values.schema.json index fc16ca99d..f89666392 100644 --- a/utils/helm/speckle-server/values.schema.json +++ b/utils/helm/speckle-server/values.schema.json @@ -107,7 +107,7 @@ }, "nextGenFileImporterEnabled": { "type": "boolean", - "description": "Enables the next generation file importer", + "description": "Enables the next generation file importer. Requires ifc_import_service to be enabled and configured.", "default": false }, "legacyFileImportsEnabled": { @@ -115,11 +115,6 @@ "description": "Enables the legacy file upload mechanism, using REST API to proxy file uploads via the server", "default": false }, - "backgroundJobsEnabled": { - "type": "boolean", - "description": "Enables the ability to run background jobs (such as the IFC importer) in Speckle", - "default": false - }, "accIntegrationEnabled": { "type": "boolean", "description": "Enables the ability to import data from ACC", @@ -571,61 +566,6 @@ } } }, - "fileImportService": { - "type": "object", - "properties": { - "rhino": { - "type": "object", - "properties": { - "connectionString": { - "type": "object", - "properties": { - "secretName": { - "type": "string", - "description": "Required. A secret containing the full connection string to the Redis store (e.g. in format of `protocol://username:password@host:port/database`) stored within the Kubernetes cluster as an opaque Kubernetes Secret. Ref: https://kubernetes.io/docs/concepts/configuration/secret/#opaque-secrets", - "default": "" - }, - "secretKey": { - "type": "string", - "description": "Required. The key within the Kubernetes Secret holding the Redis connection string.", - "default": "" - } - } - }, - "queueName": { - "type": "string", - "description": "Optional. The name of the queue created within the Redis cluster.", - "default": "fileimport-service-jobs" - } - } - }, - "ifc": { - "type": "object", - "properties": { - "connectionString": { - "type": "object", - "properties": { - "secretName": { - "type": "string", - "description": "Required. A secret containing the full connection string to the Redis store (e.g. in format of `protocol://username:password@host:port/database`) stored within the Kubernetes cluster as an opaque Kubernetes Secret. Ref: https://kubernetes.io/docs/concepts/configuration/secret/#opaque-secrets", - "default": "" - }, - "secretKey": { - "type": "string", - "description": "Required. The key within the Kubernetes Secret holding the Redis connection string.", - "default": "" - } - } - }, - "queueName": { - "type": "string", - "description": "Optional. The name of the queue created within the Redis cluster.", - "default": "fileimport-service-jobs" - } - } - } - } - }, "networkPolicy": { "type": "object", "properties": { diff --git a/utils/helm/speckle-server/values.yaml b/utils/helm/speckle-server/values.yaml index 547c69c62..c333292fb 100644 --- a/utils/helm/speckle-server/values.yaml +++ b/utils/helm/speckle-server/values.yaml @@ -63,12 +63,10 @@ featureFlags: moveProjectRegionEnabled: false ## @param featureFlags.retryErroredPreviewsEnabled Toggles whether the system periodically retries preview requests which have previously errored. retryErroredPreviewsEnabled: false - ## @param featureFlags.nextGenFileImporterEnabled Enables the next generation file importer + ## @param featureFlags.nextGenFileImporterEnabled Enables the next generation file importer. Requires ifc_import_service to be enabled and configured. nextGenFileImporterEnabled: false ## @param featureFlags.legacyFileImportsEnabled Enables the legacy file upload mechanism, using REST API to proxy file uploads via the server legacyFileImportsEnabled: false - ## @param featureFlags.backgroundJobsEnabled Enables the ability to run background jobs (such as the IFC importer) in Speckle - backgroundJobsEnabled: false ## @param featureFlags.accIntegrationEnabled Enables the ability to import data from ACC accIntegrationEnabled: false ## @param featureFlags.rhinoFileImporterEnabled Enables the dedicated Rhino based file importer. This is not part of the deployment. @@ -421,30 +419,6 @@ redis: ## secretKey: '' - fileImportService: - rhino: - connectionString: - ## @param redis.fileImportService.rhino.connectionString.secretName Required. A secret containing the full connection string to the Redis store (e.g. in format of `protocol://username:password@host:port/database`) stored within the Kubernetes cluster as an opaque Kubernetes Secret. Ref: https://kubernetes.io/docs/concepts/configuration/secret/#opaque-secrets - ## - secretName: '' - ## @param redis.fileImportService.rhino.connectionString.secretKey Required. The key within the Kubernetes Secret holding the Redis connection string. - ## - secretKey: '' - ## @param redis.fileImportService.rhino.queueName Optional. The name of the queue created within the Redis cluster. - ## - queueName: 'fileimport-service-jobs' - ifc: - connectionString: - ## @param redis.fileImportService.ifc.connectionString.secretName Required. A secret containing the full connection string to the Redis store (e.g. in format of `protocol://username:password@host:port/database`) stored within the Kubernetes cluster as an opaque Kubernetes Secret. Ref: https://kubernetes.io/docs/concepts/configuration/secret/#opaque-secrets - ## - secretName: '' - ## @param redis.fileImportService.ifc.connectionString.secretKey Required. The key within the Kubernetes Secret holding the Redis connection string. - ## - secretKey: '' - ## @param redis.fileImportService.ifc.queueName Optional. The name of the queue created within the Redis cluster. - ## - queueName: 'fileimport-service-jobs' - ## @extra redis.networkPolicy If networkPolicy is enabled for Speckle server, this provides the NetworkPolicy with the necessary details to allow egress connections to the Redis store ## networkPolicy: