From 2e86a723c642c0e749f8ea700561c83bcc77cb1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Jedlicska?= <57442769+gjedlicska@users.noreply.github.com> Date: Fri, 23 May 2025 10:27:00 +0200 Subject: [PATCH] feat(fileimport-service): add next gen file importer (#4697) * feat(fileimport-service): add next gen file importer * feat(fileimports): integrate server and fileimporter * chore(dui3): remove leftover artifacts * fix(server): test typing fixes * fix(fileimports): test and pr comment fixes * feat(fileimports: moare test fixes * fix(fileimports): tests and yarn dedupe --- packages/fileimport-service/package.json | 4 +- .../fileimport-service/scripts/publishTask.ts | 24 ++ packages/fileimport-service/src/bin.ts | 3 +- .../fileimport-service/src/common/output.ts | 25 ++ .../src/common/processHandling.ts | 85 ++++++ .../fileimport-service/src/controller/api.ts | 2 +- .../src/controller/daemon.ts | 123 +------- .../src/controller/helpers/env.ts | 2 +- .../fileimport-service/src/nextGen/config.ts | 20 ++ .../src/nextGen/jobProcessor.ts | 144 +++++++++ .../fileimport-service/src/nextGen/main.ts | 150 ++++++++++ packages/fileimport-service/tsconfig.json | 2 +- packages/preview-service/eslint.config.mjs | 3 + packages/server/modules/core/errors/model.ts | 7 + .../modules/fileuploads/domain/operations.ts | 10 +- .../modules/fileuploads/helpers/convert.ts | 2 +- .../20250519074720_fileimport_message_text.ts | 16 + .../modules/fileuploads/queues/fileimports.ts | 24 +- .../fileuploads/repositories/fileUploads.ts | 5 +- .../modules/fileuploads/rest/nextGenRouter.ts | 29 +- .../fileuploads/services/createFileImport.ts | 47 +-- .../fileuploads/services/management.ts | 1 + .../fileuploads/services/resultHandler.ts | 11 +- .../fileuploads/tests/helpers/creation.ts | 1 + .../tests/integration/fileuploadsV2.spec.ts | 5 +- .../tests/integration/results.spec.ts | 59 ++-- .../tests/unit/fileuploads.spec.ts | 26 +- packages/shared/src/workers/fileimport/job.ts | 10 +- yarn.lock | 278 ++++++++++++++++++ 29 files changed, 889 insertions(+), 229 deletions(-) create mode 100644 packages/fileimport-service/scripts/publishTask.ts create mode 100644 packages/fileimport-service/src/common/output.ts create mode 100644 packages/fileimport-service/src/common/processHandling.ts create mode 100644 packages/fileimport-service/src/nextGen/config.ts create mode 100644 packages/fileimport-service/src/nextGen/jobProcessor.ts create mode 100644 packages/fileimport-service/src/nextGen/main.ts create mode 100644 packages/server/modules/core/errors/model.ts create mode 100644 packages/server/modules/fileuploads/migrations/20250519074720_fileimport_message_text.ts diff --git a/packages/fileimport-service/package.json b/packages/fileimport-service/package.json index fb266c9e8..60469ad4e 100644 --- a/packages/fileimport-service/package.json +++ b/packages/fileimport-service/package.json @@ -28,7 +28,8 @@ "lint:eslint": "eslint .", "start": "node --loader=./dist/src/aliasLoader.js ./bin/www.js", "test": "NODE_ENV=test LOG_LEVEL=silent LOG_PRETTY=true vitest run --sequence.shuffle", - "downloadBlob": "node scripts/downloadBlob.js" + "downloadBlob": "node scripts/downloadBlob.js", + "publishTask": "tsx --env-file=.env scripts/publishTask.ts" }, "dependencies": { "@speckle/shared": "workspace:^", @@ -63,6 +64,7 @@ "nodemon": "^2.0.20", "prettier": "^2.5.1", "rimraf": "^5.0.7", + "tsx": "^4.19.4", "typescript": "^4.6.4", "typescript-eslint": "^7.12.0", "vitest": "^1.6.0" diff --git a/packages/fileimport-service/scripts/publishTask.ts b/packages/fileimport-service/scripts/publishTask.ts new file mode 100644 index 000000000..c0393f057 --- /dev/null +++ b/packages/fileimport-service/scripts/publishTask.ts @@ -0,0 +1,24 @@ +import { REDIS_URL } from '../src/nextGen/config.js' +import { initializeQueue } from '@speckle/shared/queue' +import { JobPayload } from '@speckle/shared/workers/fileimport' + +const jobQueue = await initializeQueue({ + queueName: 'fileimport-service-jobs', + redisUrl: REDIS_URL +}) + +await jobQueue.add({ + serverUrl: '', + token: '', + jobId: '1', + projectId: '', + modelId: '', + blobId: '', + fileName: 'railing.ifc', + fileType: 'ifc', + timeOutSeconds: 100000000000 +}) + +console.log('published') + +process.exit() diff --git a/packages/fileimport-service/src/bin.ts b/packages/fileimport-service/src/bin.ts index 9af469c65..c0b223597 100644 --- a/packages/fileimport-service/src/bin.ts +++ b/packages/fileimport-service/src/bin.ts @@ -3,10 +3,11 @@ import * as Environment from '@speckle/shared/environment' const { FF_NEXT_GEN_FILE_IMPORTER_ENABLED } = Environment.getFeatureFlags() import { main as oldMain } from '@/controller/daemon.js' +import { main } from '@/nextGen/main.js' const start = () => { if (FF_NEXT_GEN_FILE_IMPORTER_ENABLED) { - throw new Error('Not yet implemented') + void main() } else { void oldMain() } diff --git a/packages/fileimport-service/src/common/output.ts b/packages/fileimport-service/src/common/output.ts new file mode 100644 index 000000000..c10bb5d0b --- /dev/null +++ b/packages/fileimport-service/src/common/output.ts @@ -0,0 +1,25 @@ +export function isSuccessOutput( + maybeSuccessOutput: unknown +): maybeSuccessOutput is { success: true; commitId: string } { + return ( + !!maybeSuccessOutput && + typeof maybeSuccessOutput === 'object' && + 'success' in maybeSuccessOutput && + typeof maybeSuccessOutput.success === 'boolean' && + maybeSuccessOutput.success && + 'commitId' in maybeSuccessOutput && + typeof maybeSuccessOutput.commitId === 'string' + ) +} + +export function isErrorOutput( + maybeErrorOutput: unknown +): maybeErrorOutput is { success: false; error: string } { + return ( + !!maybeErrorOutput && + typeof maybeErrorOutput === 'object' && + 'error' in maybeErrorOutput && + typeof maybeErrorOutput.error === 'string' && + !!maybeErrorOutput.error + ) +} diff --git a/packages/fileimport-service/src/common/processHandling.ts b/packages/fileimport-service/src/common/processHandling.ts new file mode 100644 index 000000000..472099425 --- /dev/null +++ b/packages/fileimport-service/src/common/processHandling.ts @@ -0,0 +1,85 @@ +import { Logger } from 'pino' +import { spawn } from 'child_process' +import fs from 'fs' + +export function runProcessWithTimeout( + processLogger: Logger, + cmd: string, + cmdArgs: string[], + extraEnv: Record, + timeoutMs: number, + resultsPath: string +): Promise { + return new Promise((resolve, reject) => { + let boundLogger = processLogger.child({ cmd, args: cmdArgs }) + boundLogger.info('Starting process.') + const childProc = spawn(cmd, cmdArgs, { env: { ...process.env, ...extraEnv } }) + + boundLogger = boundLogger.child({ pid: childProc.pid }) + childProc.stdout.on('data', (data) => { + handleData(data, false, boundLogger) + }) + + childProc.stderr.on('data', (data) => { + handleData(data, true, boundLogger) + }) + + let timedOut = false + + const timeout = setTimeout(() => { + boundLogger.warn('Process timed out. Killing process...') + + timedOut = true + childProc.kill(9) + const rejectionReason = `Timeout: Process took longer than ${timeoutMs} milliseconds to execute.` + const output = { + success: false, + error: rejectionReason + } + fs.writeFileSync(resultsPath, JSON.stringify(output)) + reject(new Error(rejectionReason)) + }, timeoutMs) + + childProc.on('close', (code) => { + boundLogger.info({ exitCode: code }, "Process exited with code '{exitCode}'") + + if (timedOut) { + return // ignore `close` calls after killing (the promise was already rejected) + } + + clearTimeout(timeout) + + if (code === 0) { + resolve() + } else { + reject(new Error(`Parser exited with code ${code}`)) + } + }) + }) +} + +export function handleData(data: unknown, isErr: boolean, logger: Logger) { + try { + if (!Buffer.isBuffer(data)) return + const dataAsString = data.toString() + dataAsString.split('\n').forEach((line) => { + if (!line) return + try { + JSON.parse(line) // verify if the data is already in JSON format + process.stdout.write('\n') + } catch { + wrapLogLine(line, isErr, logger) + } + }) + } catch { + wrapLogLine(JSON.stringify(data), isErr, logger) + } +} + +function wrapLogLine(line: string, isErr: boolean, logger: Logger) { + if (isErr) { + logger.error({ parserLogLine: line }, 'ParserLog: {parserLogLine}') + return + } + logger.info({ parserLogLine: line }, 'ParserLog: {parserLogLine}') +} diff --git a/packages/fileimport-service/src/controller/api.ts b/packages/fileimport-service/src/controller/api.ts index 7e8309320..485016b13 100644 --- a/packages/fileimport-service/src/controller/api.ts +++ b/packages/fileimport-service/src/controller/api.ts @@ -313,7 +313,7 @@ export class ServerAPI { .where({ id: tokenId.slice(0, 10) }) .del() - if (delCount === 0) throw new Error('Token revokation failed') + if (delCount === 0) throw new Error('Token revocation failed') return true } } diff --git a/packages/fileimport-service/src/controller/daemon.ts b/packages/fileimport-service/src/controller/daemon.ts index a1002ff7c..af03bb99e 100644 --- a/packages/fileimport-service/src/controller/daemon.ts +++ b/packages/fileimport-service/src/controller/daemon.ts @@ -8,15 +8,15 @@ import { getDbClients } from '@/clients/knex.js' import { downloadFile } from '@/controller/filesApi.js' import fs from 'fs' -import { spawn } from 'child_process' import { ServerAPI } from '@/controller/api.js' import { downloadDependencies } from '@/controller/objDependencies.js' import { logger } from '@/observability/logging.js' import { Nullable, Scopes, wait, TIME_MS } from '@speckle/shared' import { Knex } from 'knex' -import { Logger } from 'pino' import { getIfcDllPath, useLegacyIfcImporter } from '@/controller/helpers/env.js' +import { isErrorOutput, isSuccessOutput } from '@/common/output.js' +import { runProcessWithTimeout } from '@/common/processHandling.js' const HEALTHCHECK_FILE_PATH = '/tmp/last_successful_query' @@ -178,7 +178,8 @@ async function doTask( { USER_TOKEN: tempUserToken }, - TIME_LIMIT + TIME_LIMIT, + TMP_RESULTS_PATH ) } else { await runProcessWithTimeout( @@ -197,7 +198,8 @@ async function doTask( { USER_TOKEN: tempUserToken }, - TIME_LIMIT + TIME_LIMIT, + TMP_RESULTS_PATH ) } } else if (info.fileType.toLowerCase() === 'stl') { @@ -219,7 +221,8 @@ async function doTask( { USER_TOKEN: tempUserToken }, - TIME_LIMIT + TIME_LIMIT, + TMP_RESULTS_PATH ) } else if (info.fileType.toLowerCase() === 'obj') { await downloadDependencies({ @@ -249,7 +252,8 @@ async function doTask( { USER_TOKEN: tempUserToken }, - TIME_LIMIT + TIME_LIMIT, + TMP_RESULTS_PATH ) } else { throw new Error(`File type ${info.fileType} is not supported`) @@ -322,113 +326,6 @@ function maybeErrorToString(error: unknown): string { } } -function isSuccessOutput( - maybeSuccessOutput: unknown -): maybeSuccessOutput is { success: true; commitId: string } { - return ( - !!maybeSuccessOutput && - typeof maybeSuccessOutput === 'object' && - 'success' in maybeSuccessOutput && - typeof maybeSuccessOutput.success === 'boolean' && - maybeSuccessOutput.success && - 'commitId' in maybeSuccessOutput && - typeof maybeSuccessOutput.commitId === 'string' - ) -} - -function isErrorOutput( - maybeErrorOutput: unknown -): maybeErrorOutput is { success: false; error: string } { - return ( - !!maybeErrorOutput && - typeof maybeErrorOutput === 'object' && - 'error' in maybeErrorOutput && - typeof maybeErrorOutput.error === 'string' && - !!maybeErrorOutput.error - ) -} - -function runProcessWithTimeout( - processLogger: Logger, - cmd: string, - cmdArgs: string[], - extraEnv: Record, - timeoutMs: number -): Promise { - return new Promise((resolve, reject) => { - let boundLogger = processLogger.child({ cmd, args: cmdArgs }) - boundLogger.info('Starting process.') - const childProc = spawn(cmd, cmdArgs, { env: { ...process.env, ...extraEnv } }) - - boundLogger = boundLogger.child({ pid: childProc.pid }) - childProc.stdout.on('data', (data) => { - handleData(data, false, boundLogger) - }) - - childProc.stderr.on('data', (data) => { - handleData(data, true, boundLogger) - }) - - let timedOut = false - - const timeout = setTimeout(() => { - boundLogger.warn('Process timed out. Killing process...') - - timedOut = true - childProc.kill(9) - const rejectionReason = `Timeout: Process took longer than ${timeoutMs} milliseconds to execute.` - const output = { - success: false, - error: rejectionReason - } - fs.writeFileSync(TMP_RESULTS_PATH, JSON.stringify(output)) - reject(new Error(rejectionReason)) - }, timeoutMs) - - childProc.on('close', (code) => { - boundLogger.info({ exitCode: code }, "Process exited with code '{exitCode}'") - - if (timedOut) { - return // ignore `close` calls after killing (the promise was already rejected) - } - - clearTimeout(timeout) - - if (code === 0) { - resolve() - } else { - reject(new Error(`Parser exited with code ${code}`)) - } - }) - }) -} - -function handleData(data: unknown, isErr: boolean, logger: Logger) { - try { - if (!Buffer.isBuffer(data)) return - const dataAsString = data.toString() - dataAsString.split('\n').forEach((line) => { - if (!line) return - try { - JSON.parse(line) // verify if the data is already in JSON format - process.stdout.write('\n') - } catch { - wrapLogLine(line, isErr, logger) - } - }) - } catch { - wrapLogLine(JSON.stringify(data), isErr, logger) - } -} - -function wrapLogLine(line: string, isErr: boolean, logger: Logger) { - if (isErr) { - logger.error({ parserLogLine: line }, 'ParserLog: {parserLogLine}') - return - } - logger.info({ parserLogLine: line }, 'ParserLog: {parserLogLine}') -} - const doStuff = async () => { const dbClients = await getDbClients() const mainDb = dbClients.main.public diff --git a/packages/fileimport-service/src/controller/helpers/env.ts b/packages/fileimport-service/src/controller/helpers/env.ts index ba010df8d..f91048ed7 100644 --- a/packages/fileimport-service/src/controller/helpers/env.ts +++ b/packages/fileimport-service/src/controller/helpers/env.ts @@ -42,7 +42,7 @@ export const getIfcDllPath = () => { if (isDevOrTestEnv()) { const possiblePath = path.resolve( getPackageRootDirPath(), - './src/ifc-dotnet/output/ifc-converter.dll' + './src/ifc-dotnet/bin/Release/net8.0/ifc-converter.dll' ) if (file.existsSync(possiblePath)) { cachedIfcDllPath = absolutePath diff --git a/packages/fileimport-service/src/nextGen/config.ts b/packages/fileimport-service/src/nextGen/config.ts new file mode 100644 index 000000000..75d452bc9 --- /dev/null +++ b/packages/fileimport-service/src/nextGen/config.ts @@ -0,0 +1,20 @@ +import { z } from 'zod' +import { parseEnv } from 'znv' + +export const { + REDIS_URL, + FILEIMPORT_TIMEOUT, + LOG_LEVEL, + LOG_PRETTY, + DOTNET_BINARY_PATH, + PYTHON_BINARY_PATH, + RHINO_IMPORTER_PATH +} = parseEnv(process.env, { + REDIS_URL: z.string().url(), + FILEIMPORT_TIMEOUT: z.number().default(3600000), + LOG_LEVEL: z.string().default('info'), + LOG_PRETTY: z.boolean().default(false), + DOTNET_BINARY_PATH: z.string().default('dotnet'), + PYTHON_BINARY_PATH: z.string().default('python3'), + RHINO_IMPORTER_PATH: z.string().default('rhino-importer.exe') +}) diff --git a/packages/fileimport-service/src/nextGen/jobProcessor.ts b/packages/fileimport-service/src/nextGen/jobProcessor.ts new file mode 100644 index 000000000..2f9f331fb --- /dev/null +++ b/packages/fileimport-service/src/nextGen/jobProcessor.ts @@ -0,0 +1,144 @@ +import { downloadFile } from '@/controller/filesApi.js' +import { AppState } from '@speckle/shared/workers' +import { JobPayload, FileImportResultPayload } from '@speckle/shared/workers/fileimport' +import { Logger } from 'pino' +import { tmpdir } from 'node:os' +import { readFileSync } from 'node:fs' +import path from 'node:path' +import fs from 'fs' +import { runProcessWithTimeout } from '@/common/processHandling.js' +import { DOTNET_BINARY_PATH, RHINO_IMPORTER_PATH } from './config.js' +import { getIfcDllPath } from '@/controller/helpers/env.js' +import { z } from 'zod' + +const jobSuccess = z.object({ + success: z.literal(true), + commitId: z.string() +}) + +const jobError = z.object({ + success: z.literal(false), + error: z.string() +}) + +const jobResult = z.discriminatedUnion('success', [jobSuccess, jobError]) + +type JobArgs = { + job: JobPayload + timeout: number + logger: Logger + getAppState: () => AppState + getElapsed: () => number +} + +export const jobProcessor = async ({ + job, + timeout, + logger, + getAppState, + getElapsed +}: JobArgs): Promise => { + const taskLogger = logger + const jobMessage = + 'Processed job {jobId} with result {status}. It took {elapsed} seconds.' + + const tmp = tmpdir() + const jobDir = path.join(tmp, job.jobId) + fs.rmSync(jobDir, { force: true, recursive: true }) + fs.mkdirSync(jobDir) + try { + const fileType = job.fileType.toLowerCase() + const sourceFilePath = path.join(jobDir, job.fileName) + const resultsPath = path.join(jobDir, 'import_results.json') + + await downloadFile({ + speckleServerUrl: job.serverUrl, + fileId: job.blobId, + streamId: job.projectId, + token: job.token, + destination: sourceFilePath, + logger + }) + + switch (fileType) { + case 'ifc': + await runProcessWithTimeout( + taskLogger, + DOTNET_BINARY_PATH, + [ + getIfcDllPath(), + sourceFilePath, + resultsPath, + job.projectId, + `File upload: ${job.fileName}`, + job.modelId, + 'bogus', + 'regionName' + ], + { + USER_TOKEN: job.token + }, + timeout, + resultsPath + ) + break + case 'stl': + case 'obj': + await runProcessWithTimeout( + taskLogger, + RHINO_IMPORTER_PATH, + [ + sourceFilePath, + resultsPath, + job.projectId, + job.modelId, + job.serverUrl, + job.token + ], + { + USER_TOKEN: job.token + }, + timeout, + resultsPath + ) + break + default: + throw new Error(`File type ${fileType} is not supported`) + } + const output = jobResult.safeParse(JSON.parse(readFileSync(resultsPath, 'utf8'))) + + if (!output.success) { + throw new Error('could not parse the result file') + } + + if (!output.data.success) { + throw new Error(output.data.error) + } + + const versionId = output.data.commitId + return { + status: 'success', + result: { versionId, durationSeconds: getElapsed() }, + warnings: [] + } + } catch (err) { + if (getAppState() === AppState.SHUTTINGDOWN) { + // likely that the job was cancelled due to the service shutting down + logger.warn({ err, elapsed: getElapsed(), status: 'error' }, jobMessage) + } else { + logger.error({ err, elapsed: getElapsed(), status: 'error' }, jobMessage) + } + + const reason = err instanceof Error ? err.stack ?? err.toString() : 'unknown error' + + return { + status: 'error', + result: { + durationSeconds: getElapsed() + }, + reason + } + } finally { + fs.rmdirSync(jobDir, { recursive: true }) + } +} diff --git a/packages/fileimport-service/src/nextGen/main.ts b/packages/fileimport-service/src/nextGen/main.ts new file mode 100644 index 000000000..453ccbd32 --- /dev/null +++ b/packages/fileimport-service/src/nextGen/main.ts @@ -0,0 +1,150 @@ +import { AppState } from '@speckle/shared/workers' +import { initializeQueue } from '@speckle/shared/queue' +import { FILEIMPORT_TIMEOUT, REDIS_URL } from '@/nextGen/config.js' +import type { + JobPayload, + FileImportResultPayload +} from '@speckle/shared/workers/fileimport' +import type Bull from 'bull' +import { logger } from '@/observability/logging.js' +import { Logger } from 'pino' +import { ensureError, TIME_MS } from '@speckle/shared' +import { jobProcessor } from './jobProcessor.js' + +const JobQueueName = 'fileimport-service-jobs' +let jobQueue: Bull.Queue | undefined = undefined +let appState: AppState = AppState.STARTING +let currentJob: { logger: Logger; done: Bull.DoneCallback } | undefined = undefined + +export const main = async () => { + // we discussed doing push based metrics from here + // await initMetrics({ app, registry: initPrometheusRegistry() }) + + // store this callback, so on shutdown we can error the job + + try { + jobQueue = await initializeQueue({ + queueName: JobQueueName, + redisUrl: REDIS_URL + }) + } catch (e) { + const err = ensureError(e, 'Unknown error creating job queue') + logger.error({ err }, 'Error creating job queue') + + // the callback to server.listen has failed, so we need to exit the process and not just return + await beforeShutdown() // handle the shutdown gracefully + onShutdown() + process.exit(1) + } + appState = AppState.RUNNING + logger.debug(`Starting processing of "${JobQueueName}" message queue`) + + await jobQueue.process(async (payload, done) => { + const elapsed = (() => { + const start = new Date().getTime() + return () => (new Date().getTime() - start) / TIME_MS.second + })() + let encounteredError = false + let jobLogger = logger.child({ + payloadId: payload.id, + jobPriorAttemptsMade: payload.attemptsMade + }) + + const job = payload.data + try { + currentJob = { done, logger: jobLogger } + jobLogger = jobLogger.child({ + jobId: job.jobId, + serverUrl: job.serverUrl + }) + const result = await jobProcessor({ + job, + logger, + timeout: FILEIMPORT_TIMEOUT, + getAppState: () => appState, + getElapsed: elapsed + }) + await sendResult({ + ...job, + result + }) + } catch (err) { + if (appState === AppState.SHUTTINGDOWN) { + // likely that the job was cancelled due to the service shutting down + jobLogger.warn({ err }, 'Processing {jobId} failed') + } else { + jobLogger.error({ err }, 'Processing {jobId} failed') + } + if (err instanceof Error) { + encounteredError = true + done(err) + await sendResult({ + ...job, + result: { + status: 'error', + reason: err.message, + result: { + durationSeconds: 0 + } + } + }) + } else { + throw err + } + } finally { + if (!encounteredError) done() + currentJob = undefined + } + }) +} + +const sendResult = async ({ + serverUrl, + projectId, + jobId, + token, + result +}: { + serverUrl: string + projectId: string + jobId: string + token: string + result: FileImportResultPayload +}) => { + const response = await fetch( + `${serverUrl}/api/projects/${projectId}/fileimporter/jobs/${jobId}/results`, + { + method: 'POST', + headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' }, + + body: JSON.stringify(result) + } + ) + if (!response.ok) { + const text = await response.text() + currentJob?.logger.error({ cause: text }, 'Failed to report job result') + throw new Error(`Failed to report job result: ${text}`) + } +} + +const beforeShutdown = async () => { + logger.info('🛑 Beginning shut down, pausing all jobs') + appState = AppState.SHUTTINGDOWN + // stop accepting new jobs and kill any running jobs + if (jobQueue) { + await jobQueue.pause( + true, // just pausing this local worker of the queue + true // do not wait for active jobs to finish + ) + } + + if (currentJob) { + currentJob.logger.warn('Cancelling job due to fileimport-service shutdown') + currentJob.done(new Error('Job cancelled due to fileimport-service shutdown')) + } + // no need to close the job queue and redis client, when the process exits they will be closed automatically +} + +const onShutdown = () => { + logger.info('👋 Completed shut down, now exiting') +} diff --git a/packages/fileimport-service/tsconfig.json b/packages/fileimport-service/tsconfig.json index 4351d814b..a6f1c87bd 100644 --- a/packages/fileimport-service/tsconfig.json +++ b/packages/fileimport-service/tsconfig.json @@ -103,6 +103,6 @@ "ts-node": { "swc": true }, - "include": ["src/**/*", "vitest.config.ts"], + "include": ["src/**/*", "scripts/**/*", "vitest.config.ts"], "exclude": ["node_modules", "coverage", "reports"] } diff --git a/packages/preview-service/eslint.config.mjs b/packages/preview-service/eslint.config.mjs index cd8720229..5c67d7922 100644 --- a/packages/preview-service/eslint.config.mjs +++ b/packages/preview-service/eslint.config.mjs @@ -6,6 +6,9 @@ import tseslint from 'typescript-eslint' */ const configs = [ ...baseConfigs, + { + ignores: ['dist', 'public', 'docs'] + }, { files: ['**/*.js'], languageOptions: { diff --git a/packages/server/modules/core/errors/model.ts b/packages/server/modules/core/errors/model.ts new file mode 100644 index 000000000..23cfe9f6f --- /dev/null +++ b/packages/server/modules/core/errors/model.ts @@ -0,0 +1,7 @@ +import { BaseError } from '@/modules/shared/errors' + +export class ModelNotFoundError extends BaseError { + static defaultMessage = 'Attempting to work with a non-existant model' + static code = 'MODEL_NOT_FOUND' + static statusCode = 404 +} diff --git a/packages/server/modules/fileuploads/domain/operations.ts b/packages/server/modules/fileuploads/domain/operations.ts index 7c07577d6..8d4894a14 100644 --- a/packages/server/modules/fileuploads/domain/operations.ts +++ b/packages/server/modules/fileuploads/domain/operations.ts @@ -4,9 +4,8 @@ import { FileUploadRecordV2 } from '@/modules/fileuploads/helpers/types' import { Optional } from '@speckle/shared' -import { FileImportResultPayload } from '@speckle/shared/dist/commonjs/workers/fileimport/job.js' -import { JobFileImportPayload } from '@/modules/fileuploads/queues/fileimports' import { UploadResult } from '@/modules/blobstorage/domain/types' +import { FileImportResultPayload, JobPayload } from '@speckle/shared/workers/fileimport' export type GetFileInfo = (args: { fileId: string @@ -49,13 +48,16 @@ export type UpdateFileStatus = (params: { fileId: string status: FileUploadConvertedStatus convertedMessage: string + convertedCommitId: string | null }) => Promise export type UploadedFile = UploadResult & { userId: string } export type FileImportMessage = Pick< - JobFileImportPayload, - 'modelId' | 'projectId' | 'fileType' | 'blobId' + JobPayload, + 'modelId' | 'projectId' | 'fileType' | 'fileName' | 'blobId' > & { jobId: string; userId: string } +export type ScheduleFileimportJob = (args: JobPayload) => Promise + export type PushJobToFileImporter = (args: FileImportMessage) => Promise diff --git a/packages/server/modules/fileuploads/helpers/convert.ts b/packages/server/modules/fileuploads/helpers/convert.ts index 25642b08f..2618b2ed9 100644 --- a/packages/server/modules/fileuploads/helpers/convert.ts +++ b/packages/server/modules/fileuploads/helpers/convert.ts @@ -22,7 +22,7 @@ export const jobResultToConvertedMessage = (jobResult: FileImportResultPayload) case 'success': return jobResult.warnings.join('; ') case 'error': - return jobResult.reasons.join('; ') + return jobResult.reason default: throw new FileImportInvalidJobResultPayload('Unknown job result status') } diff --git a/packages/server/modules/fileuploads/migrations/20250519074720_fileimport_message_text.ts b/packages/server/modules/fileuploads/migrations/20250519074720_fileimport_message_text.ts new file mode 100644 index 000000000..32183f5c6 --- /dev/null +++ b/packages/server/modules/fileuploads/migrations/20250519074720_fileimport_message_text.ts @@ -0,0 +1,16 @@ +import { Knex } from 'knex' + +const FILEUPLOADS_TABLE = 'file_uploads' +const MESSAGE_FIELD = 'convertedMessage' + +export async function up(knex: Knex): Promise { + await knex.raw( + `ALTER TABLE ${FILEUPLOADS_TABLE} ALTER COLUMN "${MESSAGE_FIELD}" TYPE text;` + ) +} + +export async function down(knex: Knex): Promise { + await knex.raw( + `ALTER TABLE ${FILEUPLOADS_TABLE} ALTER COLUMN "${MESSAGE_FIELD}" TYPE varchar(255);` + ) +} diff --git a/packages/server/modules/fileuploads/queues/fileimports.ts b/packages/server/modules/fileuploads/queues/fileimports.ts index 85b5e143e..dda5d4707 100644 --- a/packages/server/modules/fileuploads/queues/fileimports.ts +++ b/packages/server/modules/fileuploads/queues/fileimports.ts @@ -10,27 +10,14 @@ import { Optional, TIME_MS } from '@speckle/shared' import Bull from 'bull' import cryptoRandomString from 'crypto-random-string' import { initializeQueue as setupQueue } from '@speckle/shared/dist/commonjs/queue/index.js' +import { JobPayload } from '@speckle/shared/workers/fileimport' +import { ScheduleFileimportJob } from '@/modules/fileuploads/domain/operations' const FILE_IMPORT_SERVICE_QUEUE_NAME = isTestEnv() ? `test:fileimport-service-jobs:${cryptoRandomString({ length: 5 })}` : 'fileimport-service-jobs' -export type JobFileImportPayload = { - blobId: string - modelId: string - projectId: string - url: string - token: string - fileType: string - timeOutSeconds: number -} - -export type FileImportJob = { - type: 'file-import' - payload: JobFileImportPayload -} - -let queue: Optional> +let queue: Optional> if (isTestEnv()) { logger.info(`Fileimport service test queue ID: ${FILE_IMPORT_SERVICE_QUEUE_NAME}`) @@ -71,13 +58,12 @@ export const shutdownQueue = async () => { await queue.close() } -export const scheduleJob = async (jobData: FileImportJob): Promise => { +export const scheduleJob: ScheduleFileimportJob = async (jobData) => { if (!queue) { throw new UninitializedResourceAccessError( 'Attempting to use uninitialized Bull queue' ) } - const job = await queue.add(jobData, { removeOnComplete: true, attempts: 3 }) - return job.id.toString() + await queue.add(jobData, { removeOnComplete: true, attempts: 3 }) } diff --git a/packages/server/modules/fileuploads/repositories/fileUploads.ts b/packages/server/modules/fileuploads/repositories/fileUploads.ts index 94b4f04e7..e21a2f27a 100644 --- a/packages/server/modules/fileuploads/repositories/fileUploads.ts +++ b/packages/server/modules/fileuploads/repositories/fileUploads.ts @@ -218,13 +218,14 @@ export const getBranchPendingVersionsFactory = export const updateFileStatusFactory = (deps: { db: Knex }): UpdateFileStatus => async (params) => { - const { fileId, status, convertedMessage } = params + const { fileId, status, convertedMessage, convertedCommitId } = params const fileInfos = await tables .fileUploads(deps.db) .update({ [FileUploads.withoutTablePrefix.col.convertedStatus]: status, [FileUploads.withoutTablePrefix.col.convertedLastUpdate]: knex.fn.now(), - [FileUploads.withoutTablePrefix.col.convertedMessage]: convertedMessage + [FileUploads.withoutTablePrefix.col.convertedMessage]: convertedMessage, + [FileUploads.withoutTablePrefix.col.convertedCommitId]: convertedCommitId }) .where({ [FileUploads.withoutTablePrefix.col.id]: fileId }) .returning('*') diff --git a/packages/server/modules/fileuploads/rest/nextGenRouter.ts b/packages/server/modules/fileuploads/rest/nextGenRouter.ts index f42874718..dbbb49a97 100644 --- a/packages/server/modules/fileuploads/rest/nextGenRouter.ts +++ b/packages/server/modules/fileuploads/rest/nextGenRouter.ts @@ -11,7 +11,6 @@ import { saveUploadFileFactoryV2, updateFileStatusFactory } from '@/modules/fileuploads/repositories/fileUploads' -import { FileImportInvalidJobResultPayload } from '@/modules/fileuploads/helpers/errors' import { validateRequest } from 'zod-express' import { z } from 'zod' import { processNewFileStreamFactory } from '@/modules/blobstorage/services/streams' @@ -32,26 +31,29 @@ import { import { pushJobToFileImporterFactory } from '@/modules/fileuploads/services/createFileImport' import { getServerOrigin } from '@/modules/shared/helpers/envHelper' import { scheduleJob } from '@/modules/fileuploads/queues/fileimports' +import { ModelNotFoundError } from '@/modules/core/errors/model' export const nextGenFileImporterRouterFactory = (): Router => { const processNewFileStream = processNewFileStreamFactory() const app = Router() app.post( - '/api/projects/:streamId/fileimporter/jobs', + '/api/projects/:streamId/models/:modelId/fileimporter/jobs', authMiddlewareCreator( streamWritePermissionsPipelineFactory({ getStream: getStreamFactory({ db }) }) ), validateRequest({ - query: z.object({ + params: z.object({ + // needs to be streamId, due to the auth context building + streamId: z.string(), modelId: z.string() }) }), async (req, res) => { const projectId = req.params.streamId - const modelId = req.query.modelId + const modelId = req.params.modelId const userId = req.context.userId if (!userId) throw new UnauthorizedError('User not authorized') @@ -65,7 +67,7 @@ export const nextGenFileImporterRouterFactory = (): Router => { const projectDb = await getProjectDbClient({ projectId }) const getModelsByIds = getBranchesByIdsFactory({ db: projectDb }) const [model] = await getModelsByIds([modelId], { streamId: projectId }) - if (!model) throw new UnauthorizedError() + if (!model) throw new ModelNotFoundError(undefined, { statusCode: 401 }) const pushJobToFileImporter = pushJobToFileImporterFactory({ getServerOrigin, @@ -142,6 +144,13 @@ export const nextGenFileImporterRouterFactory = (): Router => { getStream: getStreamFactory({ db }) }) ), + validateRequest({ + params: z.object({ + streamId: z.string(), + jobId: z.string() + }), + body: fileImportResultPayload + }), async (req, res) => { const userId = req.context.userId const projectId = req.params.streamId @@ -153,15 +162,7 @@ export const nextGenFileImporterRouterFactory = (): Router => { jobId }) - const parseJobOutput = fileImportResultPayload.safeParse(req.body) - if (!parseJobOutput.success) { - logger.error( - { err: parseJobOutput.error.format() }, - 'Error parsing file import job result' - ) - throw new FileImportInvalidJobResultPayload(parseJobOutput.error.message) - } - const jobResult = parseJobOutput.data + const jobResult = req.body const projectDb = await getProjectDbClient({ projectId }) diff --git a/packages/server/modules/fileuploads/services/createFileImport.ts b/packages/server/modules/fileuploads/services/createFileImport.ts index ebdf1abc9..1d86a41f1 100644 --- a/packages/server/modules/fileuploads/services/createFileImport.ts +++ b/packages/server/modules/fileuploads/services/createFileImport.ts @@ -2,24 +2,33 @@ import { CreateAndStoreAppToken } from '@/modules/core/domain/tokens/operations' import { DefaultAppIds } from '@/modules/auth/defaultApps' import { Scopes, TIME, TIME_MS } from '@speckle/shared' import { TokenResourceIdentifierType } from '@/test/graphql/generated/graphql' -import { getServerOrigin } from '@/modules/shared/helpers/envHelper' -import { scheduleJob } from '@/modules/fileuploads/queues/fileimports' -import { PushJobToFileImporter } from '@/modules/fileuploads/domain/operations' +import { + PushJobToFileImporter, + ScheduleFileimportJob +} from '@/modules/fileuploads/domain/operations' const twentyMinutes = 20 * TIME.minute export const pushJobToFileImporterFactory = (deps: { createAppToken: CreateAndStoreAppToken - getServerOrigin: typeof getServerOrigin - scheduleJob: typeof scheduleJob + getServerOrigin: () => string + scheduleJob: ScheduleFileimportJob }): PushJobToFileImporter => - async ({ modelId, projectId, userId, fileType, blobId, jobId }): Promise => { + async ({ + modelId, + projectId, + userId, + fileName, + fileType, + blobId, + jobId + }): Promise => { const token = await deps.createAppToken({ appId: DefaultAppIds.Web, name: `fileimport-${projectId}@${modelId}`, userId, - scopes: [Scopes.Streams.Write], + scopes: [Scopes.Streams.Write, Scopes.Streams.Read], lifespan: 2 * TIME_MS.hour, limitResources: [ { @@ -29,21 +38,15 @@ export const pushJobToFileImporterFactory = ] }) - const url = new URL( - `/projects/${projectId}/fileimporter/jobs/${jobId}/results`, - deps.getServerOrigin() - ).toString() - await deps.scheduleJob({ - type: 'file-import', - payload: { - token, - url, - modelId, - fileType, - projectId, - timeOutSeconds: twentyMinutes, - blobId - } + jobId, + fileName, + token, + serverUrl: deps.getServerOrigin(), + modelId, + fileType, + projectId, + timeOutSeconds: twentyMinutes, + blobId }) } diff --git a/packages/server/modules/fileuploads/services/management.ts b/packages/server/modules/fileuploads/services/management.ts index ff97f3d3b..a782534b4 100644 --- a/packages/server/modules/fileuploads/services/management.ts +++ b/packages/server/modules/fileuploads/services/management.ts @@ -81,6 +81,7 @@ export const insertNewUploadAndNotifyFactoryV2 = }) await deps.pushJobToFileImporter({ + fileName: file.fileName, fileType: file.fileType, projectId: file.projectId, modelId: upload.modelId, diff --git a/packages/server/modules/fileuploads/services/resultHandler.ts b/packages/server/modules/fileuploads/services/resultHandler.ts index a7dd62cd1..fee80bb12 100644 --- a/packages/server/modules/fileuploads/services/resultHandler.ts +++ b/packages/server/modules/fileuploads/services/resultHandler.ts @@ -34,12 +34,21 @@ export const onFileImportResultFactory = const status = jobResultStatusToFileUploadStatus(jobResult.status) const convertedMessage = jobResultToConvertedMessage(jobResult) + let convertedCommitId = null + switch (jobResult.status) { + case 'error': + break + case 'success': + convertedCommitId = jobResult.result.versionId + } + let updatedFile try { updatedFile = await deps.updateFileStatus({ fileId: jobId, status, - convertedMessage + convertedMessage, + convertedCommitId }) } catch (e) { const err = ensureError(e) diff --git a/packages/server/modules/fileuploads/tests/helpers/creation.ts b/packages/server/modules/fileuploads/tests/helpers/creation.ts index ac77744d3..bc004a704 100644 --- a/packages/server/modules/fileuploads/tests/helpers/creation.ts +++ b/packages/server/modules/fileuploads/tests/helpers/creation.ts @@ -29,6 +29,7 @@ export const buildFileUploadMessage = ( modelId: cryptoRandomString({ length: 10 }), projectId: cryptoRandomString({ length: 10 }), fileType: cryptoRandomString({ length: 10 }), + fileName: cryptoRandomString({ length: 10 }), blobId: cryptoRandomString({ length: 10 }), userId: cryptoRandomString({ length: 10 }), jobId: cryptoRandomString({ length: 10 }) diff --git a/packages/server/modules/fileuploads/tests/integration/fileuploadsV2.spec.ts b/packages/server/modules/fileuploads/tests/integration/fileuploadsV2.spec.ts index 9e8c9b527..99d11978b 100644 --- a/packages/server/modules/fileuploads/tests/integration/fileuploadsV2.spec.ts +++ b/packages/server/modules/fileuploads/tests/integration/fileuploadsV2.spec.ts @@ -20,9 +20,8 @@ import { BranchRecord } from '@/modules/core/helpers/types' const { createUser, createStream, createToken, createBranch } = initUploadTestEnvironment() -const fileImporterUrl = (projectOneId: string, modelId?: string) => - `/api/projects/${projectOneId}/fileimporter/jobs` + - (modelId ? `?modelId=${modelId}` : ``) +const fileImporterUrl = (projectOneId: string, modelId: string) => + `/api/projects/${projectOneId}/models/${modelId}/fileimporter/jobs` const { FF_NEXT_GEN_FILE_IMPORTER_ENABLED } = getFeatureFlags() diff --git a/packages/server/modules/fileuploads/tests/integration/results.spec.ts b/packages/server/modules/fileuploads/tests/integration/results.spec.ts index 2a31d8abe..05277400c 100644 --- a/packages/server/modules/fileuploads/tests/integration/results.spec.ts +++ b/packages/server/modules/fileuploads/tests/integration/results.spec.ts @@ -17,6 +17,10 @@ import type { Server } from 'http' import request from 'supertest' import { initUploadTestEnvironment } from '@/modules/fileuploads/tests/helpers/init' import { createFileUploadJob } from '@/modules/fileuploads/tests/helpers/creation' +import { + FileImportErrorPayload, + FileImportSuccessPayload +} from '@speckle/shared/workers/fileimport' const { createUser, createStream, createToken } = initUploadTestEnvironment() @@ -159,6 +163,14 @@ const { FF_NEXT_GEN_FILE_IMPORTER_ENABLED } = getFeatureFlags() expect(response.status).to.equal(400) }) it('should 400 if the job id cannot be found', async () => { + const payload: FileImportSuccessPayload = { + status: 'success', + warnings: [], + result: { + versionId: cryptoRandomString({ length: 10 }), + durationSeconds: randomInt(1, 3600) + } + } const response = await request(app) .post( `/api/projects/${projectOneId}/fileimporter/jobs/${cryptoRandomString({ @@ -167,34 +179,24 @@ const { FF_NEXT_GEN_FILE_IMPORTER_ENABLED } = getFeatureFlags() ) .set('Content-Type', 'application/json') .set('Authorization', `Bearer ${userOneToken}`) - .send( - JSON.stringify({ - status: 'success', - warnings: [], - result: { - versionId: cryptoRandomString({ length: 10 }), - durationSeconds: randomInt(1, 3600) - } - }) - ) + .send(JSON.stringify(payload)) expect(response.status).to.equal(404) }) it('should 200 if the payload reports a success result', async () => { + const payload: FileImportSuccessPayload = { + status: 'success', + warnings: [], + result: { + versionId: cryptoRandomString({ length: 10 }), + durationSeconds: randomInt(1, 3600) + } + } const response = await request(app) .post(`/api/projects/${projectOneId}/fileimporter/jobs/${jobOneId}/results`) .set('Authorization', `Bearer ${userOneToken}`) .set('Content-Type', 'application/json') - .send( - JSON.stringify({ - status: 'success', - warnings: [], - result: { - versionId: cryptoRandomString({ length: 10 }), - durationSeconds: randomInt(1, 3600) - } - }) - ) + .send(JSON.stringify(payload)) expect(response.status).to.equal(200) const gqlResponse = await getFileUploads(projectOneId, userOneToken) @@ -205,19 +207,18 @@ const { FF_NEXT_GEN_FILE_IMPORTER_ENABLED } = getFeatureFlags() ) }) it('should 200 if the payload reports an error result', async () => { + const payload: FileImportErrorPayload = { + status: 'error', + reason: cryptoRandomString({ length: 10 }), + result: { + durationSeconds: randomInt(1, 3600) + } + } const response = await request(app) .post(`/api/projects/${projectOneId}/fileimporter/jobs/${jobOneId}/results`) .set('Authorization', `Bearer ${userOneToken}`) .set('Content-Type', 'application/json') - .send( - JSON.stringify({ - status: 'error', - reasons: [cryptoRandomString({ length: 10 })], - result: { - durationSeconds: randomInt(1, 3600) - } - }) - ) + .send(JSON.stringify(payload)) expect(response.status).to.equal(200) const gqlResponse = await getFileUploads(projectOneId, userOneToken) diff --git a/packages/server/modules/fileuploads/tests/unit/fileuploads.spec.ts b/packages/server/modules/fileuploads/tests/unit/fileuploads.spec.ts index e89429bd0..fa6c4db48 100644 --- a/packages/server/modules/fileuploads/tests/unit/fileuploads.spec.ts +++ b/packages/server/modules/fileuploads/tests/unit/fileuploads.spec.ts @@ -17,6 +17,7 @@ import { pushJobToFileImporterFactory } from '@/modules/fileuploads/services/cre import { assign } from 'lodash' import { buildFileUploadMessage } from '@/modules/fileuploads/tests/helpers/creation' import { getFeatureFlags } from '@speckle/shared/environment' +import { JobPayload } from '@speckle/shared/workers/fileimport' const { createStream, createUser, garbageCollector } = initUploadTestEnvironment() @@ -110,7 +111,6 @@ describe('FileUploads @fileuploads', () => { getServerOrigin: () => serverOrigin, scheduleJob: async (jobData) => { assign(result, jobData) - return Promise.resolve(cryptoRandomString({ length: 10 })) }, createAppToken: (args) => { usedUserId = args.userId @@ -121,18 +121,18 @@ describe('FileUploads @fileuploads', () => { await pushJobToFileImporter(upload) expect(usedUserId).to.equal(upload.userId) - expect(result).to.deep.equal({ - type: 'file-import', - payload: { - token, - url: `${serverOrigin}/projects/${upload.projectId}/fileimporter/jobs/${upload.jobId}/results`, - modelId: upload.modelId, - fileType: upload.fileType, - projectId: upload.projectId, - timeOutSeconds: 1200, - blobId: upload.blobId - } - }) + const expected: JobPayload = { + jobId: upload.jobId, + fileName: upload.fileName, + token, + serverUrl: serverOrigin, + modelId: upload.modelId, + fileType: upload.fileType, + projectId: upload.projectId, + timeOutSeconds: 1200, + blobId: upload.blobId + } + expect(result).to.deep.equal(expected) }) } ) diff --git a/packages/shared/src/workers/fileimport/job.ts b/packages/shared/src/workers/fileimport/job.ts index f00008781..e8f8f3c4a 100644 --- a/packages/shared/src/workers/fileimport/job.ts +++ b/packages/shared/src/workers/fileimport/job.ts @@ -7,11 +7,13 @@ const job = z.object({ export const jobPayload = job.merge( z.object({ - url: z.string(), + serverUrl: z.string().url().describe('The url of the server'), + projectId: z.string(), + modelId: z.string(), token: z.string(), - responseUrl: z.string().url(), blobId: z.string(), fileType: z.string(), + fileName: z.string(), timeOutSeconds: z .number() .int() @@ -36,10 +38,12 @@ export type FileImportSuccessPayload = z.infer const fileImportErrorPayload = z.object({ status: z.literal('error'), - reasons: z.array(z.string()).min(1), + reason: z.string(), result: baseFileImportResult }) +export type FileImportErrorPayload = z.infer + export const fileImportResultPayload = z.discriminatedUnion('status', [ fileImportSuccessPayload, fileImportErrorPayload diff --git a/yarn.lock b/yarn.lock index ffd3a0221..ba74a43fe 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8879,6 +8879,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/aix-ppc64@npm:0.25.4": + version: 0.25.4 + resolution: "@esbuild/aix-ppc64@npm:0.25.4" + conditions: os=aix & cpu=ppc64 + languageName: node + linkType: hard + "@esbuild/android-arm64@npm:0.17.19": version: 0.17.19 resolution: "@esbuild/android-arm64@npm:0.17.19" @@ -8935,6 +8942,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/android-arm64@npm:0.25.4": + version: 0.25.4 + resolution: "@esbuild/android-arm64@npm:0.25.4" + conditions: os=android & cpu=arm64 + languageName: node + linkType: hard + "@esbuild/android-arm@npm:0.17.19": version: 0.17.19 resolution: "@esbuild/android-arm@npm:0.17.19" @@ -8991,6 +9005,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/android-arm@npm:0.25.4": + version: 0.25.4 + resolution: "@esbuild/android-arm@npm:0.25.4" + conditions: os=android & cpu=arm + languageName: node + linkType: hard + "@esbuild/android-x64@npm:0.17.19": version: 0.17.19 resolution: "@esbuild/android-x64@npm:0.17.19" @@ -9047,6 +9068,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/android-x64@npm:0.25.4": + version: 0.25.4 + resolution: "@esbuild/android-x64@npm:0.25.4" + conditions: os=android & cpu=x64 + languageName: node + linkType: hard + "@esbuild/darwin-arm64@npm:0.17.19": version: 0.17.19 resolution: "@esbuild/darwin-arm64@npm:0.17.19" @@ -9103,6 +9131,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/darwin-arm64@npm:0.25.4": + version: 0.25.4 + resolution: "@esbuild/darwin-arm64@npm:0.25.4" + conditions: os=darwin & cpu=arm64 + languageName: node + linkType: hard + "@esbuild/darwin-x64@npm:0.17.19": version: 0.17.19 resolution: "@esbuild/darwin-x64@npm:0.17.19" @@ -9159,6 +9194,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/darwin-x64@npm:0.25.4": + version: 0.25.4 + resolution: "@esbuild/darwin-x64@npm:0.25.4" + conditions: os=darwin & cpu=x64 + languageName: node + linkType: hard + "@esbuild/freebsd-arm64@npm:0.17.19": version: 0.17.19 resolution: "@esbuild/freebsd-arm64@npm:0.17.19" @@ -9215,6 +9257,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/freebsd-arm64@npm:0.25.4": + version: 0.25.4 + resolution: "@esbuild/freebsd-arm64@npm:0.25.4" + conditions: os=freebsd & cpu=arm64 + languageName: node + linkType: hard + "@esbuild/freebsd-x64@npm:0.17.19": version: 0.17.19 resolution: "@esbuild/freebsd-x64@npm:0.17.19" @@ -9271,6 +9320,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/freebsd-x64@npm:0.25.4": + version: 0.25.4 + resolution: "@esbuild/freebsd-x64@npm:0.25.4" + conditions: os=freebsd & cpu=x64 + languageName: node + linkType: hard + "@esbuild/linux-arm64@npm:0.17.19": version: 0.17.19 resolution: "@esbuild/linux-arm64@npm:0.17.19" @@ -9327,6 +9383,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/linux-arm64@npm:0.25.4": + version: 0.25.4 + resolution: "@esbuild/linux-arm64@npm:0.25.4" + conditions: os=linux & cpu=arm64 + languageName: node + linkType: hard + "@esbuild/linux-arm@npm:0.17.19": version: 0.17.19 resolution: "@esbuild/linux-arm@npm:0.17.19" @@ -9383,6 +9446,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/linux-arm@npm:0.25.4": + version: 0.25.4 + resolution: "@esbuild/linux-arm@npm:0.25.4" + conditions: os=linux & cpu=arm + languageName: node + linkType: hard + "@esbuild/linux-ia32@npm:0.17.19": version: 0.17.19 resolution: "@esbuild/linux-ia32@npm:0.17.19" @@ -9439,6 +9509,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/linux-ia32@npm:0.25.4": + version: 0.25.4 + resolution: "@esbuild/linux-ia32@npm:0.25.4" + conditions: os=linux & cpu=ia32 + languageName: node + linkType: hard + "@esbuild/linux-loong64@npm:0.17.19": version: 0.17.19 resolution: "@esbuild/linux-loong64@npm:0.17.19" @@ -9495,6 +9572,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/linux-loong64@npm:0.25.4": + version: 0.25.4 + resolution: "@esbuild/linux-loong64@npm:0.25.4" + conditions: os=linux & cpu=loong64 + languageName: node + linkType: hard + "@esbuild/linux-mips64el@npm:0.17.19": version: 0.17.19 resolution: "@esbuild/linux-mips64el@npm:0.17.19" @@ -9551,6 +9635,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/linux-mips64el@npm:0.25.4": + version: 0.25.4 + resolution: "@esbuild/linux-mips64el@npm:0.25.4" + conditions: os=linux & cpu=mips64el + languageName: node + linkType: hard + "@esbuild/linux-ppc64@npm:0.17.19": version: 0.17.19 resolution: "@esbuild/linux-ppc64@npm:0.17.19" @@ -9607,6 +9698,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/linux-ppc64@npm:0.25.4": + version: 0.25.4 + resolution: "@esbuild/linux-ppc64@npm:0.25.4" + conditions: os=linux & cpu=ppc64 + languageName: node + linkType: hard + "@esbuild/linux-riscv64@npm:0.17.19": version: 0.17.19 resolution: "@esbuild/linux-riscv64@npm:0.17.19" @@ -9663,6 +9761,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/linux-riscv64@npm:0.25.4": + version: 0.25.4 + resolution: "@esbuild/linux-riscv64@npm:0.25.4" + conditions: os=linux & cpu=riscv64 + languageName: node + linkType: hard + "@esbuild/linux-s390x@npm:0.17.19": version: 0.17.19 resolution: "@esbuild/linux-s390x@npm:0.17.19" @@ -9719,6 +9824,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/linux-s390x@npm:0.25.4": + version: 0.25.4 + resolution: "@esbuild/linux-s390x@npm:0.25.4" + conditions: os=linux & cpu=s390x + languageName: node + linkType: hard + "@esbuild/linux-x64@npm:0.17.19": version: 0.17.19 resolution: "@esbuild/linux-x64@npm:0.17.19" @@ -9775,6 +9887,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/linux-x64@npm:0.25.4": + version: 0.25.4 + resolution: "@esbuild/linux-x64@npm:0.25.4" + conditions: os=linux & cpu=x64 + languageName: node + linkType: hard + "@esbuild/netbsd-arm64@npm:0.24.2": version: 0.24.2 resolution: "@esbuild/netbsd-arm64@npm:0.24.2" @@ -9789,6 +9908,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/netbsd-arm64@npm:0.25.4": + version: 0.25.4 + resolution: "@esbuild/netbsd-arm64@npm:0.25.4" + conditions: os=netbsd & cpu=arm64 + languageName: node + linkType: hard + "@esbuild/netbsd-x64@npm:0.17.19": version: 0.17.19 resolution: "@esbuild/netbsd-x64@npm:0.17.19" @@ -9845,6 +9971,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/netbsd-x64@npm:0.25.4": + version: 0.25.4 + resolution: "@esbuild/netbsd-x64@npm:0.25.4" + conditions: os=netbsd & cpu=x64 + languageName: node + linkType: hard + "@esbuild/openbsd-arm64@npm:0.23.1": version: 0.23.1 resolution: "@esbuild/openbsd-arm64@npm:0.23.1" @@ -9866,6 +9999,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/openbsd-arm64@npm:0.25.4": + version: 0.25.4 + resolution: "@esbuild/openbsd-arm64@npm:0.25.4" + conditions: os=openbsd & cpu=arm64 + languageName: node + linkType: hard + "@esbuild/openbsd-x64@npm:0.17.19": version: 0.17.19 resolution: "@esbuild/openbsd-x64@npm:0.17.19" @@ -9922,6 +10062,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/openbsd-x64@npm:0.25.4": + version: 0.25.4 + resolution: "@esbuild/openbsd-x64@npm:0.25.4" + conditions: os=openbsd & cpu=x64 + languageName: node + linkType: hard + "@esbuild/sunos-x64@npm:0.17.19": version: 0.17.19 resolution: "@esbuild/sunos-x64@npm:0.17.19" @@ -9978,6 +10125,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/sunos-x64@npm:0.25.4": + version: 0.25.4 + resolution: "@esbuild/sunos-x64@npm:0.25.4" + conditions: os=sunos & cpu=x64 + languageName: node + linkType: hard + "@esbuild/win32-arm64@npm:0.17.19": version: 0.17.19 resolution: "@esbuild/win32-arm64@npm:0.17.19" @@ -10034,6 +10188,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/win32-arm64@npm:0.25.4": + version: 0.25.4 + resolution: "@esbuild/win32-arm64@npm:0.25.4" + conditions: os=win32 & cpu=arm64 + languageName: node + linkType: hard + "@esbuild/win32-ia32@npm:0.17.19": version: 0.17.19 resolution: "@esbuild/win32-ia32@npm:0.17.19" @@ -10090,6 +10251,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/win32-ia32@npm:0.25.4": + version: 0.25.4 + resolution: "@esbuild/win32-ia32@npm:0.25.4" + conditions: os=win32 & cpu=ia32 + languageName: node + linkType: hard + "@esbuild/win32-x64@npm:0.17.19": version: 0.17.19 resolution: "@esbuild/win32-x64@npm:0.17.19" @@ -10146,6 +10314,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/win32-x64@npm:0.25.4": + version: 0.25.4 + resolution: "@esbuild/win32-x64@npm:0.25.4" + conditions: os=win32 & cpu=x64 + languageName: node + linkType: hard + "@eslint-community/eslint-utils@npm:^4.2.0, @eslint-community/eslint-utils@npm:^4.4.0": version: 4.4.0 resolution: "@eslint-community/eslint-utils@npm:4.4.0" @@ -15650,6 +15825,7 @@ __metadata: prom-client: "npm:^14.0.1" rimraf: "npm:^5.0.7" tarn: "npm:^3.0.2" + tsx: "npm:^4.19.4" typescript: "npm:^4.6.4" typescript-eslint: "npm:^7.12.0" undici: "npm:^5.28.4" @@ -28181,6 +28357,92 @@ __metadata: languageName: node linkType: hard +"esbuild@npm:~0.25.0": + version: 0.25.4 + resolution: "esbuild@npm:0.25.4" + dependencies: + "@esbuild/aix-ppc64": "npm:0.25.4" + "@esbuild/android-arm": "npm:0.25.4" + "@esbuild/android-arm64": "npm:0.25.4" + "@esbuild/android-x64": "npm:0.25.4" + "@esbuild/darwin-arm64": "npm:0.25.4" + "@esbuild/darwin-x64": "npm:0.25.4" + "@esbuild/freebsd-arm64": "npm:0.25.4" + "@esbuild/freebsd-x64": "npm:0.25.4" + "@esbuild/linux-arm": "npm:0.25.4" + "@esbuild/linux-arm64": "npm:0.25.4" + "@esbuild/linux-ia32": "npm:0.25.4" + "@esbuild/linux-loong64": "npm:0.25.4" + "@esbuild/linux-mips64el": "npm:0.25.4" + "@esbuild/linux-ppc64": "npm:0.25.4" + "@esbuild/linux-riscv64": "npm:0.25.4" + "@esbuild/linux-s390x": "npm:0.25.4" + "@esbuild/linux-x64": "npm:0.25.4" + "@esbuild/netbsd-arm64": "npm:0.25.4" + "@esbuild/netbsd-x64": "npm:0.25.4" + "@esbuild/openbsd-arm64": "npm:0.25.4" + "@esbuild/openbsd-x64": "npm:0.25.4" + "@esbuild/sunos-x64": "npm:0.25.4" + "@esbuild/win32-arm64": "npm:0.25.4" + "@esbuild/win32-ia32": "npm:0.25.4" + "@esbuild/win32-x64": "npm:0.25.4" + dependenciesMeta: + "@esbuild/aix-ppc64": + optional: true + "@esbuild/android-arm": + optional: true + "@esbuild/android-arm64": + optional: true + "@esbuild/android-x64": + optional: true + "@esbuild/darwin-arm64": + optional: true + "@esbuild/darwin-x64": + optional: true + "@esbuild/freebsd-arm64": + optional: true + "@esbuild/freebsd-x64": + optional: true + "@esbuild/linux-arm": + optional: true + "@esbuild/linux-arm64": + optional: true + "@esbuild/linux-ia32": + optional: true + "@esbuild/linux-loong64": + optional: true + "@esbuild/linux-mips64el": + optional: true + "@esbuild/linux-ppc64": + optional: true + "@esbuild/linux-riscv64": + optional: true + "@esbuild/linux-s390x": + optional: true + "@esbuild/linux-x64": + optional: true + "@esbuild/netbsd-arm64": + optional: true + "@esbuild/netbsd-x64": + optional: true + "@esbuild/openbsd-arm64": + optional: true + "@esbuild/openbsd-x64": + optional: true + "@esbuild/sunos-x64": + optional: true + "@esbuild/win32-arm64": + optional: true + "@esbuild/win32-ia32": + optional: true + "@esbuild/win32-x64": + optional: true + bin: + esbuild: bin/esbuild + checksum: 10/227ffe9b31f0b184a0b0a0210bb9d32b2b115b8c5c9b09f08db2c3928cb470fc55a22dbba3c2894365d3abcc62c2089b85638be96a20691d1234d31990ea01b2 + languageName: node + linkType: hard + "escalade@npm:^3.1.1": version: 3.1.1 resolution: "escalade@npm:3.1.1" @@ -47416,6 +47678,22 @@ __metadata: languageName: node linkType: hard +"tsx@npm:^4.19.4": + version: 4.19.4 + resolution: "tsx@npm:4.19.4" + dependencies: + esbuild: "npm:~0.25.0" + fsevents: "npm:~2.3.3" + get-tsconfig: "npm:^4.7.5" + dependenciesMeta: + fsevents: + optional: true + bin: + tsx: dist/cli.mjs + checksum: 10/4dde315aeda70b9cadfecbc8d05b1625f5831018b9cb2db25cbbd03c5f5ee9c59cdc6652a0fd8492176b50944a5af1d5af352b944d024f4a719f58d6f2ac3a7f + languageName: node + linkType: hard + "tunnel-agent@npm:^0.6.0": version: 0.6.0 resolution: "tunnel-agent@npm:0.6.0"