diff --git a/.gitignore b/.gitignore index 72bf3f9d3..97264aa4d 100644 --- a/.gitignore +++ b/.gitignore @@ -71,6 +71,8 @@ redis-data/ .tshy-build obj/ bin/ +!packages/fileimport-service/src/obj +!packages/fileimport-service/bin !packages/monitor-deployment/bin !packages/preview-service/bin !packages/server/bin diff --git a/packages/fileimport-service/.env.example b/packages/fileimport-service/.env.example new file mode 100644 index 000000000..b6a66fe86 --- /dev/null +++ b/packages/fileimport-service/.env.example @@ -0,0 +1,7 @@ +FILE_IMPORT_TIME_LIMIT_MIN='10' +MAX_OBJECT_SIZE_MB='10' +POSTGRES_MAX_CONNECTIONS_FILE_IMPORT_SERVICE='1' +POSTGRES_CONNECTION_ACQUIRE_TIMEOUT_MILLIS='16000' +POSTGRES_CONNECTION_CREATE_TIMEOUT_MILLIS='5000' +FF_WORKSPACES_MULTI_REGION_ENABLED=false +# IFC_DOTNET_DLL_PATH='packages/fileimport-service/src/ifc-dotnet/ifc-converter.dll' diff --git a/packages/fileimport-service/.vscode/launch.json b/packages/fileimport-service/.vscode/launch.json deleted file mode 100644 index 02bb86b2b..000000000 --- a/packages/fileimport-service/.vscode/launch.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "name": "Launch via Yarn", - "request": "launch", - "console": "integratedTerminal", - "runtimeArgs": ["dev"], - "runtimeExecutable": "yarn", - "skipFiles": ["/**"], - "envFile": "${workspaceFolder}/.env", - "type": "node" - } - ] -} diff --git a/packages/fileimport-service/Dockerfile b/packages/fileimport-service/Dockerfile index 2ee9fc27f..c3191926f 100644 --- a/packages/fileimport-service/Dockerfile +++ b/packages/fileimport-service/Dockerfile @@ -2,7 +2,7 @@ ARG NODE_ENV=production FROM mcr.microsoft.com/dotnet/sdk:8.0-noble AS dotnet-build-stage WORKDIR /app -COPY packages/fileimport-service/ifc-dotnet . +COPY packages/fileimport-service/src/ifc-dotnet . RUN dotnet publish ifc-converter.csproj -c Release -o output/ @@ -69,8 +69,9 @@ ENV PYTHON_BINARY_PATH=${PYTHON_BINARY_PATH} ARG DOTNET_BINARY_PATH=/usr/bin/dotnet ENV DOTNET_BINARY_PATH=${DOTNET_BINARY_PATH} -COPY --link --from=dotnet-build-stage /app/output packages/fileimport-service/ifc-dotnet +COPY --link --from=dotnet-build-stage /app/output packages/fileimport-service/src/ifc-dotnet +ENV IFC_DOTNET_DLL_PATH='/speckle-server/packages/fileimport-service/src/ifc-dotnet/ifc-converter.dll' WORKDIR /speckle-server/packages/fileimport-service -ENTRYPOINT [ "tini", "--", "node", "--no-experimental-fetch", "src/daemon.js"] +ENTRYPOINT [ "tini", "--", "node", "--loader=./dist/src/aliasLoader.js", "bin/www.js" ] diff --git a/packages/fileimport-service/README.md b/packages/fileimport-service/README.md index 03cafa602..2efde3e9a 100644 --- a/packages/fileimport-service/README.md +++ b/packages/fileimport-service/README.md @@ -1,11 +1,13 @@ -# `ifc-parser` +# File Import Service -> TODO: description +## Description of how this works -## Usage +A micro-service which polls a Postgres database table `file_uploads` for new records and processes them. -``` -const ifcParser = require('ifc-parser'); +It retrieves a referenced file from an S3 bucket and stores it in a local directory for parsing. -// TODO: DEMONSTRATE API -``` +The File Import service can parse either STL, OBJ, or IFC files using external packages, written in either .Net or Python (_note_, there is a legacy IFC parser written in Node.js). These external packages are controlled via shell commands. + +The parsers are responsible for extracting the necessary data from the files and storing it in the database. They are also responsible for creating a new Speckle model if necessary. + +The service is then responsible for updating the status of the `file_uploads` table, and for posting a Postgres notification. diff --git a/packages/fileimport-service/bin/www.js b/packages/fileimport-service/bin/www.js new file mode 100755 index 000000000..5bbe59627 --- /dev/null +++ b/packages/fileimport-service/bin/www.js @@ -0,0 +1,2 @@ +#!/usr/bin/env node +import '../dist/src/bin.js' diff --git a/packages/fileimport-service/eslint.config.mjs b/packages/fileimport-service/eslint.config.mjs index d0bb88fbb..884f851ba 100644 --- a/packages/fileimport-service/eslint.config.mjs +++ b/packages/fileimport-service/eslint.config.mjs @@ -1,20 +1,61 @@ -import { baseConfigs, globals } from '../../eslint.config.mjs' +import tseslint from 'typescript-eslint' +import { + baseConfigs, + getESMDirname, + globals, + prettierConfig +} from '../../eslint.config.mjs' -/** - * @type {Array} - */ const configs = [ ...baseConfigs, { - ignores: ['**/ifc/**', '**/obj/**', '**/stl/**'] + ignores: ['dist', 'public', 'docs'] }, { + files: ['**/*.js'], + ignores: ['**/*.mjs'], + languageOptions: { + sourceType: 'module', + globals: { + ...globals.node + } + } + }, + { + files: ['bin/www'], + languageOptions: { + sourceType: 'module', + globals: { + ...globals.node + } + } + }, + ...tseslint.configs.recommendedTypeChecked.map((c) => ({ + ...c, + files: [...(c.files || []), '**/*.ts', '**/*.d.ts'] + })), + { + files: ['**/*.ts', '**/*.d.ts'], + languageOptions: { + parserOptions: { + tsconfigRootDir: getESMDirname(import.meta.url), + project: './tsconfig.json' + } + }, + rules: { + '@typescript-eslint/no-explicit-any': 'error', + '@typescript-eslint/no-unsafe-return': 'error' + } + }, + { + files: ['**/*.spec.{js,ts}'], languageOptions: { globals: { ...globals.node } } - } + }, + prettierConfig ] export default configs diff --git a/packages/fileimport-service/jsconfig.json b/packages/fileimport-service/jsconfig.json deleted file mode 100644 index 9027098ca..000000000 --- a/packages/fileimport-service/jsconfig.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "extends": "../../jsconfig.base.json", - "include": ["src", "ifc"] -} diff --git a/packages/fileimport-service/multiregion.example.json b/packages/fileimport-service/multiregion.example.json new file mode 100644 index 000000000..61bd76bf9 --- /dev/null +++ b/packages/fileimport-service/multiregion.example.json @@ -0,0 +1,34 @@ +{ + "main": { + "postgres": { + "connectionUri": "postgresql://speckle:speckle@127.0.0.1:5432/speckle", + "privateConnectionUri": "postgresql://speckle:speckle@postgres:5432/speckle", + "databaseName": "speckle" + }, + "blobStorage": { + "accessKey": "minioadmin", + "secretKey": "minioadmin", + "bucket": "speckle-server", + "createBucketIfNotExists": true, + "endpoint": "http://127.0.0.1:9000", + "s3Region": "us-east-1" + } + }, + "regions": { + "region1": { + "postgres": { + "connectionUri": "postgresql://speckle:speckle@127.0.0.1:5401/speckle", + "privateConnectionUri": "postgresql://speckle:speckle@postgres-region1:5432/speckle", + "databaseName": "speckle" + }, + "blobStorage": { + "accessKey": "minioadmin", + "secretKey": "minioadmin", + "bucket": "speckle-server", + "createBucketIfNotExists": true, + "endpoint": "http://127.0.0.1:9020", + "s3Region": "us-east-1" + } + } + } +} diff --git a/packages/fileimport-service/package.json b/packages/fileimport-service/package.json index 7e2dea43c..87a9e67c2 100644 --- a/packages/fileimport-service/package.json +++ b/packages/fileimport-service/package.json @@ -6,7 +6,8 @@ "author": "Speckle Systems ", "homepage": "https://github.com/specklesystems/speckle-server#readme", "license": "SEE LICENSE IN readme.md", - "main": "daemon.js", + "main": "./bin/www.js", + "type": "module", "repository": { "type": "git", "url": "git+https://github.com/specklesystems/speckle-server.git" @@ -15,34 +16,52 @@ "node": "^18.19.0" }, "scripts": { - "dev": "cross-env POSTGRES_URL=postgres://speckle:speckle@127.0.0.1/speckle NODE_ENV=development LOG_PRETTY=true SPECKLE_SERVER_URL=http://127.0.0.1:3000 nodemon --no-experimental-fetch ./src/daemon.js", - "parse:ifc": "node --no-experimental-fetch ./ifc/import_file.js ./ifc/ifcs/steelplates.ifc 33763848d6 2e4bfb467a main File upload: steelplates.ifc", - "lint": "eslint ." - }, - "bugs": { - "url": "https://github.com/specklesystems/speckle-server/issues" + "build:tsc:watch": "tsc -p ./tsconfig.build.json --watch", + "run:watch": "NODE_ENV=development LOG_PRETTY=true LOG_LEVEL=debug nodemon --exec \"yarn start\" --trace-deprecation --watch ./bin/www.js --watch ./dist", + "dev": "concurrently \"npm:build:tsc:watch\" \"npm:run:watch\"", + "dev:headed": "yarn dev", + "build:tsc": "rimraf ./dist/src && tsc -p ./tsconfig.build.json", + "build": "yarn build:tsc", + "lint": "yarn lint:tsc && yarn lint:eslint", + "lint:ci": "yarn lint:tsc", + "lint:tsc": "tsc --noEmit", + "lint:eslint": "eslint .", + "start": "node --loader=./dist/src/aliasLoader.js ./bin/www.js", + "test": "NODE_ENV=test LOG_LEVEL=silent LOG_PRETTY=true vitest run --sequence.shuffle", + "downloadBlob": "node scripts/downloadBlob.js" }, "dependencies": { "@speckle/shared": "workspace:^", - "bcrypt": "^5.0.1", - "crypto-random-string": "^3.3.1", + "bcrypt": "^5.0.0", + "crypto": "^1.0.1", + "crypto-random-string": "^3.2.0", + "dotenv": "^16.4.5", + "esm-module-alias": "^2.2.0", "knex": "^2.5.1", + "lodash": "^4.17.21", + "lodash-es": "^4.17.21", "pg": "^8.7.3", "pino": "^8.7.0", - "pino-http": "^8.0.0", "pino-pretty": "^9.1.1", "prom-client": "^14.0.1", - "undici": "^5.28.4", + "tarn": "^3.0.2", "valid-filename": "^3.1.0", - "web-ifc": "^0.0.36", - "znv": "^0.4.0", - "zod": "^3.22.4" + "web-ifc": "^0.0.36" }, "devDependencies": { - "cross-env": "^7.0.3", + "@types/bcrypt": "^5.0.0", + "@types/lodash-es": "^4.17.6", + "@types/node": "^18.19.38", + "@vitest/coverage-istanbul": "^1.6.0", + "concurrently": "^8.2.2", "eslint": "^9.4.0", "eslint-config-prettier": "^9.1.0", + "eslint-plugin-vitest": "^0.5.4", "nodemon": "^2.0.20", - "prettier": "^2.5.1" + "prettier": "^2.5.1", + "rimraf": "^5.0.7", + "typescript": "^4.6.4", + "typescript-eslint": "^7.12.0", + "vitest": "^1.6.0" } } diff --git a/packages/fileimport-service/scripts/downloadBlob.js b/packages/fileimport-service/scripts/downloadBlob.js new file mode 100644 index 000000000..72bac9c2a --- /dev/null +++ b/packages/fileimport-service/scripts/downloadBlob.js @@ -0,0 +1,16 @@ +const { downloadFile } = require('../src/controller/filesApi.js') +const { logger } = require('../src/observability/logging') + +//https://latest.speckle.systems/api/stream/c83a5b2d1f/blob/29fe85cffb +const speckleServerUrl = 'https://latest.speckle.systems' +const fileId = '29fe85cffb' +const streamId = 'c83a5b2d1f' + +downloadFile({ + speckleServerUrl, + fileId, + streamId, + destination: '/var/folders/p2/fczcvzfd62x5jcfdlw6ghf640000gn/T/tmp.U8MlF9KIxH', + token: process.env.SPECKLE_TOKEN || '', + logger: logger.child({ streamId, fileId }) +}) diff --git a/packages/fileimport-service/src/aliasLoader.ts b/packages/fileimport-service/src/aliasLoader.ts new file mode 100644 index 000000000..d77980f8d --- /dev/null +++ b/packages/fileimport-service/src/aliasLoader.ts @@ -0,0 +1,6 @@ +import generateAliasesResolver from 'esm-module-alias' +import { srcRoot } from './root.js' + +export const resolve = generateAliasesResolver({ + '@': srcRoot +}) diff --git a/packages/fileimport-service/src/bin.ts b/packages/fileimport-service/src/bin.ts new file mode 100644 index 000000000..5a087f0b8 --- /dev/null +++ b/packages/fileimport-service/src/bin.ts @@ -0,0 +1,9 @@ +import '@/bootstrap.js' // This has side-effects and has to be imported first + +import { main } from '@/controller/daemon.js' + +const start = () => { + void main() +} + +start() diff --git a/packages/fileimport-service/src/bootstrap.ts b/packages/fileimport-service/src/bootstrap.ts new file mode 100644 index 000000000..50c8721e6 --- /dev/null +++ b/packages/fileimport-service/src/bootstrap.ts @@ -0,0 +1,2 @@ +import dotenv from 'dotenv' +dotenv.config() diff --git a/packages/fileimport-service/src/api.js b/packages/fileimport-service/src/controller/api.ts similarity index 62% rename from packages/fileimport-service/src/api.js rename to packages/fileimport-service/src/controller/api.ts index 08e5b566c..b7523df0c 100644 --- a/packages/fileimport-service/src/api.js +++ b/packages/fileimport-service/src/controller/api.ts @@ -1,22 +1,59 @@ -'use strict' -const crypto = require('crypto') -const crs = require('crypto-random-string') -const bcrypt = require('bcrypt') -const { chunk } = require('lodash') -const { logger: parentLogger } = require('../observability/logging') +import crypto from 'crypto' +import crs from 'crypto-random-string' +import bcrypt from 'bcrypt' +import { chunk } from 'lodash-es' +import { logger as parentLogger } from '@/observability/logging.js' +import Observability from '@speckle/shared/dist/commonjs/observability/index.js' +import type { Knex } from 'knex' +import type { Logger } from 'pino' -const Observability = require('@speckle/shared/dist/commonjs/observability/index.js') - -const tables = (db) => ({ +const tables = (db: Knex) => ({ objects: db('objects'), - branches: db('branches'), + branches: db<{ + id: string + streamId: string + authorId: string + name: string + description: string + }>('branches'), streams: db('streams'), apiTokens: db('api_tokens'), tokenScopes: db('token_scopes') }) -module.exports = class ServerAPI { - constructor({ db, streamId, logger }) { +type SpeckleObject = { + id?: string + hash?: string + streamId: string + __closure?: Record + __tree?: unknown + speckleType: string + totalChildrenCount?: number + totalChildrenCountByDepth?: string + data: unknown +} + +type SpeckleObjectWithId = SpeckleObject & { + id: string +} + +export class ServerAPI { + tables: ReturnType + db: Knex + streamId: string + isSending: boolean + buffer: unknown[] + logger: Logger + + constructor({ + db, + streamId, + logger + }: { + db: Knex + streamId: string + logger: Logger + }) { this.tables = tables(db) this.db = db this.streamId = streamId @@ -27,7 +64,7 @@ module.exports = class ServerAPI { Observability.extendLoggerComponent(parentLogger.child({ streamId }), 'ifc') } - async saveObject(obj) { + async saveObject(obj: SpeckleObject) { if (!obj) throw new Error('Null object') if (!obj.id) { @@ -39,14 +76,20 @@ module.exports = class ServerAPI { return obj.id } - async saveObjectBatch(objs) { + async saveObjectBatch(objs: SpeckleObject[]) { return await this.createObjectsBatched(this.streamId, objs) } - async createObject({ streamId, object }) { + async createObject({ + streamId, + object + }: { + streamId: string + object: SpeckleObject + }) { const insertionObject = this.prepInsertionObject(streamId, object) - const totalChildrenCountByDepth = {} + const totalChildrenCountByDepth: Record = {} if (object.__closure !== null) { for (const prop in object.__closure) { if (totalChildrenCountByDepth[object.__closure[prop].toString()]) @@ -58,7 +101,7 @@ module.exports = class ServerAPI { delete insertionObject.__tree delete insertionObject.__closure - insertionObject.totalChildrenCount = object.__closures.length + insertionObject.totalChildrenCount = object.__closure?.length insertionObject.totalChildrenCountByDepth = JSON.stringify( totalChildrenCountByDepth ) @@ -68,15 +111,15 @@ module.exports = class ServerAPI { return insertionObject.id } - async createObjectsBatched(streamId, objects) { - const objsToInsert = [] - const ids = [] + async createObjectsBatched(streamId: string, objects: SpeckleObject[]) { + const objsToInsert: SpeckleObjectWithId[] = [] + const ids: string[] = [] // Prep objects up objects.forEach((obj) => { const insertionObject = this.prepInsertionObject(streamId, obj) let totalChildrenCountGlobal = 0 - const totalChildrenCountByDepth = {} + const totalChildrenCountByDepth: Record = {} if (obj.__closure !== null) { for (const prop in obj.__closure) { @@ -121,7 +164,7 @@ module.exports = class ServerAPI { return ids } - prepInsertionObject(streamId, obj) { + prepInsertionObject(streamId: string, obj: SpeckleObject): SpeckleObjectWithId { const maximumObjectSizeMB = parseInt(process.env['MAX_OBJECT_SIZE_MB'] || '10') const MAX_OBJECT_SIZE = maximumObjectSizeMB * 1024 * 1024 @@ -145,23 +188,49 @@ module.exports = class ServerAPI { } } - prepInsertionObjectBatch(batch) { + prepInsertionObjectBatch(batch: Array<{ id: string }>) { batch.sort((a, b) => (a.id > b.id ? 1 : -1)) } - prepInsertionClosureBatch(batch) { - batch.sort((a, b) => - a.parent > b.parent + prepInsertionClosureBatch( + batch: Array<{ parent: string | undefined; child: string | undefined }> + ) { + batch.sort((a, b) => { + return this.hasParent(a) && this.hasParent(b) && a.parent > b.parent ? 1 : a.parent === b.parent - ? a.child > b.child + ? this.hasChild(a) && this.hasChild(b) && a.child > b.child ? 1 : -1 : -1 + }) + } + + hasParent(maybeHasParent: unknown): maybeHasParent is { parent: string } { + return ( + !!maybeHasParent && + typeof maybeHasParent === 'object' && + 'parent' in maybeHasParent && + typeof maybeHasParent.parent === 'string' ) } - async getBranchByNameAndStreamId({ streamId, name }) { + hasChild(maybeHasChild: unknown): maybeHasChild is { child: string } { + return ( + !!maybeHasChild && + typeof maybeHasChild === 'object' && + 'child' in maybeHasChild && + typeof maybeHasChild.child === 'string' + ) + } + + async getBranchByNameAndStreamId({ + streamId, + name + }: { + streamId: string + name: string + }) { const query = this.tables.branches .select('*') .where({ streamId }) @@ -170,13 +239,24 @@ module.exports = class ServerAPI { return await query } - async createBranch({ name, description, streamId, authorId }) { - const branch = {} - branch.id = crs({ length: 10 }) - branch.streamId = streamId - branch.authorId = authorId - branch.name = name.toLowerCase() - branch.description = description + async createBranch({ + name, + description, + streamId, + authorId + }: { + name: string + description: string + streamId: string + authorId: string + }) { + const branch = { + id: crs({ length: 10 }), + streamId, + authorId, + name: name.toLowerCase(), + description + } await this.tables.branches.returning('id').insert(branch) @@ -197,7 +277,17 @@ module.exports = class ServerAPI { return { tokenId, tokenString, tokenHash, lastChars } } - async createToken({ userId, name, scopes, lifespan }) { + async createToken({ + userId, + name, + scopes, + lifespan + }: { + userId: string + name: string + scopes: string[] + lifespan: number + }) { const { tokenId, tokenString, tokenHash, lastChars } = await this.createBareToken() if (scopes.length === 0) throw new Error('No scopes provided') @@ -218,7 +308,7 @@ module.exports = class ServerAPI { return { id: tokenId, token: tokenId + tokenString } } - async revokeTokenById(tokenId) { + async revokeTokenById(tokenId: string) { const delCount = await this.tables.apiTokens .where({ id: tokenId.slice(0, 10) }) .del() diff --git a/packages/fileimport-service/src/daemon.js b/packages/fileimport-service/src/controller/daemon.ts similarity index 67% rename from packages/fileimport-service/src/daemon.js rename to packages/fileimport-service/src/controller/daemon.ts index 47970adef..53cd84fe8 100644 --- a/packages/fileimport-service/src/daemon.js +++ b/packages/fileimport-service/src/controller/daemon.ts @@ -1,22 +1,23 @@ -'use strict' - -const Environment = require('@speckle/shared/dist/commonjs/environment/index.js') -const { +import Environment from '@speckle/shared/dist/commonjs/environment/index.js' +import { initPrometheusMetrics, metricDuration, metricInputFileSize, metricOperationErrors -} = require('./prometheusMetrics') -const getDbClients = require('../knex') +} from '@/controller/prometheusMetrics.js' +import { getDbClients } from '@/knex.js' -const { downloadFile } = require('./filesApi') -const fs = require('fs') -const { spawn } = require('child_process') +import { downloadFile } from '@/controller/filesApi.js' +import fs from 'fs' +import { spawn } from 'child_process' + +import { ServerAPI } from '@/controller/api.js' +import { downloadDependencies } from '@/controller/objDependencies.js' +import { logger } from '@/observability/logging.js' +import { Nullable, Scopes, wait } from '@speckle/shared' +import { Knex } from 'knex' +import { Logger } from 'pino' -const ServerAPI = require('./api') -const objDependencies = require('./objDependencies') -const { logger } = require('../observability/logging') -const { Scopes, wait } = require('@speckle/shared') const { FF_FILEIMPORT_IFC_DOTNET_ENABLED } = Environment.getFeatureFlags() const HEALTHCHECK_FILE_PATH = '/tmp/last_successful_query' @@ -29,11 +30,11 @@ let shouldExit = false let TIME_LIMIT = 10 * 60 * 1000 -const providedTimeLimit = parseInt(process.env.FILE_IMPORT_TIME_LIMIT_MIN) +const providedTimeLimit = parseInt(process.env['FILE_IMPORT_TIME_LIMIT_MIN'] || '10') if (providedTimeLimit) TIME_LIMIT = providedTimeLimit * 60 * 1000 -async function startTask(knex) { - const { rows } = await knex.raw(` +async function startTask(knex: Knex) { + const { rows } = (await knex.raw(` UPDATE file_uploads SET "convertedStatus" = 1, @@ -46,18 +47,23 @@ async function startTask(knex) { ) as task WHERE file_uploads."id" = task."id" RETURNING file_uploads."id" - `) + `)) satisfies { rows: { id: string }[] } return rows[0] } -async function doTask(mainDb, regionName, taskDb, task) { +async function doTask( + mainDb: Knex, + regionName: string, + taskDb: Knex, + task: { id: string } +) { const taskId = task.id // Mark task as started await mainDb.raw(`NOTIFY file_import_started, '${task.id}'`) let taskLogger = logger.child({ taskId }) - let tempUserToken = null + let tempUserToken: Nullable = null let mainServerApi = null let taskServerApi = null let fileTypeForMetric = 'unknown' @@ -65,11 +71,24 @@ async function doTask(mainDb, regionName, taskDb, task) { const metricDurationEnd = metricDuration.startTimer() let newBranchCreated = false - let branchMetadata = { streamId: null, branchName: null } + let branchMetadata: { streamId: Nullable; branchName: Nullable } = { + streamId: null, + branchName: null + } try { taskLogger.info("Doing task '{taskId}'.") - const info = await taskDb('file_uploads').where({ id: taskId }).first() + const info = await taskDb<{ + id: string + fileType: string + fileSize: string + fileName: string + userId: string + streamId: string + branchName: string + }>('file_uploads') + .where({ id: taskId }) + .first() if (!info) { throw new Error('Internal error: DB inconsistent') } @@ -116,30 +135,40 @@ async function doTask(mainDb, regionName, taskDb, task) { const { token } = await mainServerApi.createToken({ userId: info.userId, name: 'temp upload token', - scopes: [Scopes.Streams.Write, Scopes.Streams.Read], + scopes: [Scopes.Streams.Write, Scopes.Streams.Read, Scopes.Profile.Read], lifespan: 1000000 }) tempUserToken = token + taskLogger.info('Downloading file {fileId}') + + const speckleServerUrl = process.env.SPECKLE_SERVER_URL || 'http://127.0.0.1:3000' + await downloadFile({ + speckleServerUrl, fileId: info.id, streamId: info.streamId, token, - destination: TMP_FILE_PATH + destination: TMP_FILE_PATH, + logger: taskLogger }) + taskLogger.info('Triggering importer for {fileType}') + if (info.fileType.toLowerCase() === 'ifc') { if (FF_FILEIMPORT_IFC_DOTNET_ENABLED) { await runProcessWithTimeout( taskLogger, process.env['DOTNET_BINARY_PATH'] || 'dotnet', [ - '/speckle-server/packages/fileimport-service/ifc-dotnet/ifc-converter.dll', + process.env['IFC_DOTNET_DLL_PATH'] || + '/speckle-server/packages/fileimport-service/src/ifc-dotnet/ifc-converter.dll', TMP_FILE_PATH, TMP_RESULTS_PATH, info.streamId, `File upload: ${info.fileName}`, existingBranch?.id || '', + info.branchName, regionName ], { @@ -153,7 +182,8 @@ async function doTask(mainDb, regionName, taskDb, task) { process.env['NODE_BINARY_PATH'] || 'node', [ '--no-experimental-fetch', - './ifc/import_file.js', + '--loader=./dist/src/aliasLoader.js', + './src/ifc/import_file.js', TMP_FILE_PATH, TMP_RESULTS_PATH, info.userId, @@ -175,7 +205,7 @@ async function doTask(mainDb, regionName, taskDb, task) { taskLogger, process.env['PYTHON_BINARY_PATH'] || 'python3', [ - './stl/import_file.py', + './src/stl/import_file.py', TMP_FILE_PATH, TMP_RESULTS_PATH, info.userId, @@ -192,7 +222,8 @@ async function doTask(mainDb, regionName, taskDb, task) { TIME_LIMIT ) } else if (info.fileType.toLowerCase() === 'obj') { - await objDependencies.downloadDependencies({ + await downloadDependencies({ + speckleServerUrl, objFilePath: TMP_FILE_PATH, streamId: info.streamId, destinationDir: TMP_INPUT_DIR, @@ -204,7 +235,7 @@ async function doTask(mainDb, regionName, taskDb, task) { process.env['PYTHON_BINARY_PATH'] || 'python3', [ '-u', - './obj/import_file.py', + './src/obj/import_file.py', TMP_FILE_PATH, TMP_RESULTS_PATH, info.userId, @@ -224,9 +255,11 @@ async function doTask(mainDb, regionName, taskDb, task) { throw new Error(`File type ${info.fileType} is not supported`) } - const output = JSON.parse(fs.readFileSync(TMP_RESULTS_PATH)) + const output: unknown = JSON.parse(fs.readFileSync(TMP_RESULTS_PATH, 'utf8')) - if (!output.success) throw new Error(output.error) + if (!isSuccessOutput(output)) { + throw new Error(isErrorOutput(output) ? output.error : 'Unknown error') + } const commitId = output.commitId @@ -243,7 +276,8 @@ async function doTask(mainDb, regionName, taskDb, task) { [commitId, task.id] ) } catch (err) { - taskLogger.error(err) + taskLogger.error(err, 'Error processing task') + const errorForDatabase = maybeErrorToString(err) await taskDb.raw( ` UPDATE file_uploads @@ -254,7 +288,7 @@ async function doTask(mainDb, regionName, taskDb, task) { WHERE "id" = ? `, // DB only accepts a varchar 255 - [err.toString().substring(0, 254), task.id] + [errorForDatabase.substring(0, 254), task.id] ) metricOperationErrors.labels(fileTypeForMetric).inc() } finally { @@ -271,12 +305,56 @@ async function doTask(mainDb, regionName, taskDb, task) { fs.rmSync(TMP_INPUT_DIR, { force: true, recursive: true }) if (fs.existsSync(TMP_RESULTS_PATH)) fs.unlinkSync(TMP_RESULTS_PATH) - if (tempUserToken) { + if (mainServerApi && tempUserToken) { await mainServerApi.revokeTokenById(tempUserToken) } } -function runProcessWithTimeout(processLogger, cmd, cmdArgs, extraEnv, timeoutMs) { +function maybeErrorToString(error: unknown): string { + const unknownError = 'Unknown error' + if (!error) return unknownError + if (typeof error === 'string') return error + if (error instanceof Error) return error.message + try { + return JSON.stringify(error) + } catch { + return unknownError + } +} + +function isSuccessOutput( + maybeSuccessOutput: unknown +): maybeSuccessOutput is { success: true; commitId: string } { + return ( + !!maybeSuccessOutput && + typeof maybeSuccessOutput === 'object' && + 'success' in maybeSuccessOutput && + typeof maybeSuccessOutput.success === 'boolean' && + maybeSuccessOutput.success && + 'commitId' in maybeSuccessOutput && + typeof maybeSuccessOutput.commitId === 'string' + ) +} + +function isErrorOutput( + maybeErrorOutput: unknown +): maybeErrorOutput is { success: false; error: string } { + return ( + !!maybeErrorOutput && + typeof maybeErrorOutput === 'object' && + 'error' in maybeErrorOutput && + typeof maybeErrorOutput.error === 'string' && + !!maybeErrorOutput.error + ) +} + +function runProcessWithTimeout( + processLogger: Logger, + cmd: string, + cmdArgs: string[], + extraEnv: Record, + timeoutMs: number +): Promise { return new Promise((resolve, reject) => { let boundLogger = processLogger.child({ cmd, args: cmdArgs }) boundLogger.info('Starting process.') @@ -304,7 +382,7 @@ function runProcessWithTimeout(processLogger, cmd, cmdArgs, extraEnv, timeoutMs) error: rejectionReason } fs.writeFileSync(TMP_RESULTS_PATH, JSON.stringify(output)) - reject(rejectionReason) + reject(new Error(rejectionReason)) }, timeoutMs) childProc.on('close', (code) => { @@ -319,20 +397,20 @@ function runProcessWithTimeout(processLogger, cmd, cmdArgs, extraEnv, timeoutMs) if (code === 0) { resolve() } else { - reject(`Parser exited with code ${code}`) + reject(new Error(`Parser exited with code ${code}`)) } }) }) } -function handleData(data, isErr, logger) { +function handleData(data: unknown, isErr: boolean, logger: Logger) { try { - Buffer.isBuffer(data) && (data = data.toString()) - data.split('\n').forEach((line) => { + if (!Buffer.isBuffer(data)) return + const dataAsString = data.toString() + dataAsString.split('\n').forEach((line) => { if (!line) return try { JSON.parse(line) // verify if the data is already in JSON format - process.stdout.write(line) process.stdout.write('\n') } catch { wrapLogLine(line, isErr, logger) @@ -343,7 +421,7 @@ function handleData(data, isErr, logger) { } } -function wrapLogLine(line, isErr, logger) { +function wrapLogLine(line: string, isErr: boolean, logger: Logger) { if (isErr) { logger.error({ parserLogLine: line }, 'ParserLog: {parserLogLine}') return @@ -356,7 +434,7 @@ const doStuff = async () => { const mainDb = dbClients.main.public const dbClientsIterator = infiniteDbClientsIterator(dbClients) while (!shouldExit) { - const [regionName, taskDb] = dbClientsIterator.next().value + const [regionName, taskDb]: [string, Knex] = dbClientsIterator.next().value try { const task = await startTask(taskDb) fs.writeFile(HEALTHCHECK_FILE_PATH, '' + Date.now(), () => {}) @@ -374,7 +452,7 @@ const doStuff = async () => { } } -async function main() { +export async function main() { logger.info('Starting FileUploads Service...') await initPrometheusMetrics() @@ -387,7 +465,9 @@ async function main() { process.exit(0) } -function* infiniteDbClientsIterator(dbClients) { +function* infiniteDbClientsIterator(dbClients: { + [key: string]: { public: Knex } +}): Generator<[string, Knex], [string, Knex], [string, Knex]> { let index = 0 const dbClientEntries = [...Object.entries(dbClients)] const clientCount = dbClientEntries.length @@ -399,5 +479,3 @@ function* infiniteDbClientsIterator(dbClients) { yield [regionName, dbConnection.public] } } - -main() diff --git a/packages/fileimport-service/src/controller/filesApi.ts b/packages/fileimport-service/src/controller/filesApi.ts new file mode 100644 index 000000000..02afaa469 --- /dev/null +++ b/packages/fileimport-service/src/controller/filesApi.ts @@ -0,0 +1,95 @@ +import { ensureError } from '@speckle/shared/dist/esm/index.js' +import fs from 'fs' +import path from 'node:path' +import { pipeline } from 'node:stream/promises' +import { Logger } from 'pino' + +export async function downloadFile({ + speckleServerUrl, + fileId, + streamId, + token, + destination, + logger +}: { + speckleServerUrl: string + fileId: string + streamId: string + token: string + destination: string + logger: Logger +}) { + try { + fs.mkdirSync(path.dirname(destination), { recursive: true }) + } catch (e) { + throw ensureError(e, 'Unknown error while creating directory') + } + + logger.info( + { destinationFile: destination }, + 'Downloading file {fileId} from {streamId} to {destinationFile}' + ) + + let response + try { + response = await fetch( + `${speckleServerUrl}/api/stream/${streamId}/blob/${fileId}`, + { + headers: { + Authorization: `Bearer ${token}` + } + } + ) + } catch (e) { + throw ensureError(e, 'Unknown error while fetching file') + } + + if (response === undefined || !response.ok) { + logger.error( + { status: response?.status, statusText: response?.statusText }, + 'Failed to download file {fileId}. HTTP {status}: {statusText}' + ) + throw new Error( + `Failed to download file ${fileId}. HTTP ${response?.status}: ${response?.statusText}` + ) + } + if (!response.body) { + throw new Error('Response body is undefined') + } + + const writer = fs.createWriteStream(destination) + + //handle errors + writer.on('error', (err) => { + logger.error(ensureError(err), `Error writing file ${destination}`) + throw err + }) + + //handle completion + writer.on('finish', () => { + logger.info(`File written to ${destination}`) + }) + + await pipeline(response.body, writer, { end: true }) +} +export async function getFileInfoByName({ + speckleServerUrl, + fileName, + streamId, + token +}: { + speckleServerUrl: string + fileName: string + streamId: string + token: string +}) { + const response = await fetch( + `${speckleServerUrl}/api/stream/${streamId}/blobs?fileName=${fileName}`, + { + headers: { + Authorization: `Bearer ${token}` + } + } + ) + return response.json() as Promise<{ blobs: { id: string }[] }> +} diff --git a/packages/fileimport-service/src/controller/objDependencies.ts b/packages/fileimport-service/src/controller/objDependencies.ts new file mode 100644 index 000000000..6e655f2c5 --- /dev/null +++ b/packages/fileimport-service/src/controller/objDependencies.ts @@ -0,0 +1,71 @@ +import events from 'events' +import fs from 'fs' +import readline from 'readline' +import path from 'path' +import isValidFilename from 'valid-filename' + +import { downloadFile, getFileInfoByName } from '@/controller/filesApi.js' +import { logger } from '@/observability/logging.js' + +const getReferencedMtlFiles = async ({ objFilePath }: { objFilePath: string }) => { + const mtlFiles: string[] = [] + + try { + const rl = readline.createInterface({ + input: fs.createReadStream(objFilePath), + crlfDelay: Infinity + }) + + rl.on('line', (line) => { + if (line.startsWith('mtllib ')) { + const mtlFile = line.slice('mtllib '.length).trim() + mtlFiles.push(mtlFile) + } + }) + + await events.once(rl, 'close') + } catch (err) { + logger.error(err, `Error getting dependencies for file ${objFilePath}`) + } + return mtlFiles +} + +export async function downloadDependencies({ + speckleServerUrl, + objFilePath, + streamId, + destinationDir, + token +}: { + speckleServerUrl: string + objFilePath: string + streamId: string + destinationDir: string + token: string +}) { + const dependencies = await getReferencedMtlFiles({ objFilePath }) + + logger.info(`Obj file depends on ${dependencies.join(', ')}`) + for (const mtlFile of dependencies) { + // there might be multiple files named with the same name, take the first... + const [file] = ( + await getFileInfoByName({ speckleServerUrl, fileName: mtlFile, streamId, token }) + ).blobs + if (!file) { + logger.info(`OBJ dependency file not found in stream: ${mtlFile}`) + continue + } + if (!isValidFilename(mtlFile)) { + logger.warn(`Invalid filename reference in OBJ dependencies: ${mtlFile}`) + continue + } + await downloadFile({ + speckleServerUrl, + fileId: file.id, + streamId, + token, + destination: path.join(destinationDir, mtlFile), + logger + }) + } +} diff --git a/packages/fileimport-service/src/controller/prometheusMetrics.ts b/packages/fileimport-service/src/controller/prometheusMetrics.ts new file mode 100644 index 000000000..172e59c52 --- /dev/null +++ b/packages/fileimport-service/src/controller/prometheusMetrics.ts @@ -0,0 +1,171 @@ +import http from 'http' +import prometheusClient, { Counter, Summary } from 'prom-client' +import { getDbClients } from '@/knex.js' +import { Knex } from 'knex' +import { Pool } from 'tarn' +import { isObject } from 'lodash-es' +import { IncomingMessage } from 'http' + +let metricQueryDuration: Summary | null = null +let metricQueryErrors: Counter | null = null + +const queryStartTime: Record = {} +prometheusClient.register.clear() +prometheusClient.register.setDefaultLabels({ + project: 'speckle-server', + app: 'fileimport-service' +}) +prometheusClient.collectDefaultMetrics() + +let prometheusInitialized = false + +const initDBPrometheusMetricsFactory = + ({ db }: { db: Knex }) => + () => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + const dbConnectionPool = db.client.pool as Pool + new prometheusClient.Gauge({ + name: 'speckle_server_knex_free', + help: 'Number of free DB connections', + collect() { + this.set(dbConnectionPool.numFree()) + } + }) + + new prometheusClient.Gauge({ + name: 'speckle_server_knex_used', + help: 'Number of used DB connections', + collect() { + this.set(dbConnectionPool.numUsed()) + } + }) + + new prometheusClient.Gauge({ + name: 'speckle_server_knex_pending', + help: 'Number of pending DB connection aquires', + collect() { + this.set(dbConnectionPool.numPendingAcquires()) + } + }) + + new prometheusClient.Gauge({ + name: 'speckle_server_knex_pending_creates', + help: 'Number of pending DB connection creates', + collect() { + this.set(dbConnectionPool.numPendingCreates()) + } + }) + + new prometheusClient.Gauge({ + name: 'speckle_server_knex_pending_validations', + help: 'Number of pending DB connection validations. This is a state between pending acquisition and acquiring a connection.', + collect() { + this.set(dbConnectionPool.numPendingValidations()) + } + }) + + new prometheusClient.Gauge({ + name: 'speckle_server_knex_remaining_capacity', + help: 'Remaining capacity of the DB connection pool', + collect() { + const postgresMaxConnections = parseInt( + process.env['POSTGRES_MAX_CONNECTIONS_FILE_IMPORT_SERVICE'] || '1' + ) + const demand = + dbConnectionPool.numUsed() + + dbConnectionPool.numPendingCreates() + + dbConnectionPool.numPendingValidations() + + dbConnectionPool.numPendingAcquires() + + this.set(Math.max(postgresMaxConnections - demand, 0)) + } + }) + + metricQueryDuration = new prometheusClient.Summary({ + name: 'speckle_server_knex_query_duration', + help: 'Summary of the DB query durations in seconds' + }) + + metricQueryErrors = new prometheusClient.Counter({ + name: 'speckle_server_knex_query_errors', + help: 'Number of DB queries with errors' + }) + + db.on('query', (data) => { + if (!isObject(data) || !('__knexQueryUid' in data)) return + const queryId = String(data.__knexQueryUid) + queryStartTime[queryId] = Date.now() + }) + + db.on('query-response', (_data, obj) => { + if (!isObject(obj) || !('__knexQueryUid' in obj)) return + const queryId = String(obj.__knexQueryUid) + const durationSec = (Date.now() - queryStartTime[queryId]) / 1000 + delete queryStartTime[queryId] + if (metricQueryDuration && !isNaN(durationSec)) + metricQueryDuration.observe(durationSec) + }) + + db.on('query-error', (_err, querySpec) => { + if (!isObject(querySpec) || !('__knexQueryUid' in querySpec)) return + const queryId = String(querySpec.__knexQueryUid) + const durationSec = (Date.now() - queryStartTime[queryId]) / 1000 + delete queryStartTime[queryId] + + if (metricQueryDuration && !isNaN(durationSec)) + metricQueryDuration.observe(durationSec) + if (metricQueryErrors) metricQueryErrors.inc() + }) + } + +export async function initPrometheusMetrics() { + if (prometheusInitialized) return + prometheusInitialized = true + + const db = (await getDbClients()).main.public + + initDBPrometheusMetricsFactory({ db })() + + const requestHandler = async (req: IncomingMessage, res: http.OutgoingMessage) => { + if (req.url === '/metrics') { + res.setHeader('Content-Type', prometheusClient.register.contentType) + const metrics = await prometheusClient.register.metrics() + res.end(metrics) + } else { + res.end('Speckle FileImport Service - prometheus metrics') + } + } + + // Define the HTTP server + // eslint-disable-next-line @typescript-eslint/no-misused-promises + const server = http.createServer(requestHandler) + server.listen(Number(process.env.PROMETHEUS_METRICS_PORT) || 9093) +} + +export const metricDuration = new prometheusClient.Histogram({ + name: 'speckle_server_operation_duration', + help: 'Summary of the operation durations in seconds', + buckets: [0.5, 1, 5, 10, 30, 60, 300, 600, 900, 1200], + labelNames: ['op'] +}) + +export const metricOperationErrors = new prometheusClient.Counter({ + name: 'speckle_server_operation_errors', + help: 'Number of operations with errors', + labelNames: ['op'] +}) + +export const metricInputFileSize = new prometheusClient.Histogram({ + name: 'speckle_server_operation_file_size', + help: 'Size of the operation input file size', + buckets: [ + 1000, + 100 * 1000, + 500 * 1000, + 1000 * 1000, + 5 * 1000 * 1000, + 10 * 1000 * 1000, + 100 * 1000 * 1000 + ], + labelNames: ['op'] +}) diff --git a/packages/fileimport-service/src/filesApi.js b/packages/fileimport-service/src/filesApi.js deleted file mode 100644 index 639e67c36..000000000 --- a/packages/fileimport-service/src/filesApi.js +++ /dev/null @@ -1,32 +0,0 @@ -/* istanbul ignore file */ -'use strict' -const fs = require('fs') -const path = require('node:path') -const { stream, fetch } = require('undici') - -module.exports = { - async downloadFile({ fileId, streamId, token, destination }) { - fs.mkdirSync(path.dirname(destination), { recursive: true }) - await stream( - `${process.env.SPECKLE_SERVER_URL}/api/stream/${streamId}/blob/${fileId}`, - { - opaque: fs.createWriteStream(destination), - headers: { - Authorization: `Bearer ${token}` - } - }, - ({ opaque }) => opaque - ) - }, - async getFileInfoByName({ fileName, streamId, token }) { - const response = await fetch( - `${process.env.SPECKLE_SERVER_URL}/api/stream/${streamId}/blobs?fileName=${fileName}`, - { - headers: { - Authorization: `Bearer ${token}` - } - } - ) - return response.json() - } -} diff --git a/packages/fileimport-service/ifc-dotnet/.config/dotnet-tools.json b/packages/fileimport-service/src/ifc-dotnet/.config/dotnet-tools.json similarity index 65% rename from packages/fileimport-service/ifc-dotnet/.config/dotnet-tools.json rename to packages/fileimport-service/src/ifc-dotnet/.config/dotnet-tools.json index db21a0aee..6e997c414 100644 --- a/packages/fileimport-service/ifc-dotnet/.config/dotnet-tools.json +++ b/packages/fileimport-service/src/ifc-dotnet/.config/dotnet-tools.json @@ -4,9 +4,7 @@ "tools": { "csharpier": { "version": "0.30.1", - "commands": [ - "dotnet-csharpier" - ] + "commands": ["dotnet-csharpier"] } } -} \ No newline at end of file +} diff --git a/packages/fileimport-service/ifc-dotnet/Program.cs b/packages/fileimport-service/src/ifc-dotnet/Program.cs similarity index 50% rename from packages/fileimport-service/ifc-dotnet/Program.cs rename to packages/fileimport-service/src/ifc-dotnet/Program.cs index 7a4b56ed0..55fc7ff50 100644 --- a/packages/fileimport-service/ifc-dotnet/Program.cs +++ b/packages/fileimport-service/src/ifc-dotnet/Program.cs @@ -2,46 +2,64 @@ using System.Text.Json; using Speckle.Importers.Ifc; using Speckle.Sdk.Common; +using Speckle.Sdk.Models.Extensions; -var filePathArgument = new Argument(name: "filePath"); +var filePathArgument = new Argument("filePath"); var outputPathArgument = new Argument("outputPath"); -var streamIdArgument = new Argument("streamId"); -var commitMessageArgument = new Argument("commitMessage"); +var projectIdArgument = new Argument("projectId"); +var versionMessageArgument = new Argument("versionMessage"); var modelIdArgument = new Argument("modelId"); +var modelNameArgument = new Argument("modelName"); var regionNameArgument = new Argument("regionName"); var rootCommand = new RootCommand { filePathArgument, outputPathArgument, - streamIdArgument, - commitMessageArgument, + projectIdArgument, + versionMessageArgument, modelIdArgument, + modelNameArgument, regionNameArgument, }; + rootCommand.SetHandler( - async (filePath, outputPath, streamId, commitMessage, modelId, _) => + async (filePath, outputPath, projectId, versionMessage, modelId, modelName, _) => { try { var token = Environment.GetEnvironmentVariable("USER_TOKEN").NotNull("USER_TOKEN is missing"); var url = Environment.GetEnvironmentVariable("SPECKLE_SERVER_URL") ?? "http://127.0.0.1:3000"; - var commitId = await Import.Ifc(url, filePath, streamId, modelId, commitMessage, token); - File.WriteAllText(outputPath, JsonSerializer.Serialize(new { success = true, commitId })); + ImporterArgs args = new() + { + ServerUrl = new(url), + FilePath = filePath, + ProjectId = projectId, + ModelId = modelId, + ModelName = modelName, + VersionMessage = versionMessage, + Token = token + }; + + var version = await Import.Ifc(args); + File.WriteAllText(outputPath, JsonSerializer.Serialize(new { success = true, commitId = version.id })); } catch (Exception e) { + Console.WriteLine($"IFC Importer failed with exception {e.ToFormattedString()}"); + File.WriteAllText( outputPath, - JsonSerializer.Serialize(new { success = false, error = e.ToString() }) + JsonSerializer.Serialize(new { success = false, error = e.ToFormattedString() }) ); } }, filePathArgument, outputPathArgument, - streamIdArgument, - commitMessageArgument, + projectIdArgument, + versionMessageArgument, modelIdArgument, + modelNameArgument, regionNameArgument ); await rootCommand.InvokeAsync(args); diff --git a/packages/fileimport-service/ifc-dotnet/ifc-converter.csproj b/packages/fileimport-service/src/ifc-dotnet/ifc-converter.csproj similarity index 96% rename from packages/fileimport-service/ifc-dotnet/ifc-converter.csproj rename to packages/fileimport-service/src/ifc-dotnet/ifc-converter.csproj index a600eedb5..42a5aad86 100644 --- a/packages/fileimport-service/ifc-dotnet/ifc-converter.csproj +++ b/packages/fileimport-service/src/ifc-dotnet/ifc-converter.csproj @@ -10,7 +10,7 @@ - + diff --git a/packages/fileimport-service/ifc-dotnet/ifc-converter.sln b/packages/fileimport-service/src/ifc-dotnet/ifc-converter.sln similarity index 100% rename from packages/fileimport-service/ifc-dotnet/ifc-converter.sln rename to packages/fileimport-service/src/ifc-dotnet/ifc-converter.sln diff --git a/packages/fileimport-service/ifc-dotnet/ifcs/171210AISC_Sculpture_brep.ifc b/packages/fileimport-service/src/ifc-dotnet/ifcs/171210AISC_Sculpture_brep.ifc similarity index 100% rename from packages/fileimport-service/ifc-dotnet/ifcs/171210AISC_Sculpture_brep.ifc rename to packages/fileimport-service/src/ifc-dotnet/ifcs/171210AISC_Sculpture_brep.ifc diff --git a/packages/fileimport-service/ifc-dotnet/ifcs/20160414office_model_CV2_fordesign.ifc b/packages/fileimport-service/src/ifc-dotnet/ifcs/20160414office_model_CV2_fordesign.ifc similarity index 100% rename from packages/fileimport-service/ifc-dotnet/ifcs/20160414office_model_CV2_fordesign.ifc rename to packages/fileimport-service/src/ifc-dotnet/ifcs/20160414office_model_CV2_fordesign.ifc diff --git a/packages/fileimport-service/ifc-dotnet/ifcs/20210219Architecture.ifc.zip b/packages/fileimport-service/src/ifc-dotnet/ifcs/20210219Architecture.ifc.zip similarity index 100% rename from packages/fileimport-service/ifc-dotnet/ifcs/20210219Architecture.ifc.zip rename to packages/fileimport-service/src/ifc-dotnet/ifcs/20210219Architecture.ifc.zip diff --git a/packages/fileimport-service/ifc-dotnet/ifcs/20210221PRIMARK.ifc b/packages/fileimport-service/src/ifc-dotnet/ifcs/20210221PRIMARK.ifc similarity index 100% rename from packages/fileimport-service/ifc-dotnet/ifcs/20210221PRIMARK.ifc rename to packages/fileimport-service/src/ifc-dotnet/ifcs/20210221PRIMARK.ifc diff --git a/packages/fileimport-service/ifc-dotnet/ifcs/23111023_IFCR2_Buildings_2.ifc b/packages/fileimport-service/src/ifc-dotnet/ifcs/23111023_IFCR2_Buildings_2.ifc similarity index 100% rename from packages/fileimport-service/ifc-dotnet/ifcs/23111023_IFCR2_Buildings_2.ifc rename to packages/fileimport-service/src/ifc-dotnet/ifcs/23111023_IFCR2_Buildings_2.ifc diff --git a/packages/fileimport-service/ifc-dotnet/ifcs/231110AC11-Institute-Var-2-IFC.ifc b/packages/fileimport-service/src/ifc-dotnet/ifcs/231110AC11-Institute-Var-2-IFC.ifc similarity index 100% rename from packages/fileimport-service/ifc-dotnet/ifcs/231110AC11-Institute-Var-2-IFC.ifc rename to packages/fileimport-service/src/ifc-dotnet/ifcs/231110AC11-Institute-Var-2-IFC.ifc diff --git a/packages/fileimport-service/ifc-dotnet/ifcs/231110ADT-FZK-Haus-2005-2006 (1).ifc b/packages/fileimport-service/src/ifc-dotnet/ifcs/231110ADT-FZK-Haus-2005-2006 (1).ifc similarity index 100% rename from packages/fileimport-service/ifc-dotnet/ifcs/231110ADT-FZK-Haus-2005-2006 (1).ifc rename to packages/fileimport-service/src/ifc-dotnet/ifcs/231110ADT-FZK-Haus-2005-2006 (1).ifc diff --git a/packages/fileimport-service/ifc-dotnet/ifcs/231110ADT-FZK-Haus-2005-2006.ifc b/packages/fileimport-service/src/ifc-dotnet/ifcs/231110ADT-FZK-Haus-2005-2006.ifc similarity index 100% rename from packages/fileimport-service/ifc-dotnet/ifcs/231110ADT-FZK-Haus-2005-2006.ifc rename to packages/fileimport-service/src/ifc-dotnet/ifcs/231110ADT-FZK-Haus-2005-2006.ifc diff --git a/packages/fileimport-service/ifc-dotnet/ifcs/example.ifc b/packages/fileimport-service/src/ifc-dotnet/ifcs/example.ifc similarity index 100% rename from packages/fileimport-service/ifc-dotnet/ifcs/example.ifc rename to packages/fileimport-service/src/ifc-dotnet/ifcs/example.ifc diff --git a/packages/fileimport-service/ifc-dotnet/ifcs/railing.ifc b/packages/fileimport-service/src/ifc-dotnet/ifcs/railing.ifc similarity index 100% rename from packages/fileimport-service/ifc-dotnet/ifcs/railing.ifc rename to packages/fileimport-service/src/ifc-dotnet/ifcs/railing.ifc diff --git a/packages/fileimport-service/ifc-dotnet/ifcs/small.ifc b/packages/fileimport-service/src/ifc-dotnet/ifcs/small.ifc similarity index 100% rename from packages/fileimport-service/ifc-dotnet/ifcs/small.ifc rename to packages/fileimport-service/src/ifc-dotnet/ifcs/small.ifc diff --git a/packages/fileimport-service/ifc-dotnet/ifcs/steelplates.ifc b/packages/fileimport-service/src/ifc-dotnet/ifcs/steelplates.ifc similarity index 100% rename from packages/fileimport-service/ifc-dotnet/ifcs/steelplates.ifc rename to packages/fileimport-service/src/ifc-dotnet/ifcs/steelplates.ifc diff --git a/packages/fileimport-service/ifc/import_file.js b/packages/fileimport-service/src/ifc/import_file.js similarity index 81% rename from packages/fileimport-service/ifc/import_file.js rename to packages/fileimport-service/src/ifc/import_file.js index 5eab2cf1c..a57845eb4 100644 --- a/packages/fileimport-service/ifc/import_file.js +++ b/packages/fileimport-service/src/ifc/import_file.js @@ -1,9 +1,8 @@ -const fs = require('fs') -const { logger: parentLogger } = require('../observability/logging') - -const { parseAndCreateCommitFactory } = require('./index') -const Observability = require('@speckle/shared/dist/commonjs/observability/index.js') -const getDbClients = require('../knex') +import fs from 'fs' +import Observability from '@speckle/shared/dist/commonjs/observability/index.js' +import { logger as parentLogger } from '@/observability/logging.js' +import { getDbClients } from '@/knex.js' +import { parseAndCreateCommitFactory } from '@/ifc/index.js' async function main() { const cmdArgs = process.argv.slice(2) diff --git a/packages/fileimport-service/ifc/index.js b/packages/fileimport-service/src/ifc/index.js similarity index 80% rename from packages/fileimport-service/ifc/index.js rename to packages/fileimport-service/src/ifc/index.js index ad3cd4a54..eea1e94c4 100644 --- a/packages/fileimport-service/ifc/index.js +++ b/packages/fileimport-service/src/ifc/index.js @@ -1,11 +1,11 @@ -const { performance } = require('perf_hooks') -const { fetch } = require('undici') -const Parser = require('./parser') -const ServerAPI = require('../src/api.js') -const Observability = require('@speckle/shared/dist/commonjs/observability/index.js') -const { logger: parentLogger } = require('../observability/logging') +import { performance } from 'perf_hooks' +import { fetch } from 'undici' +import Observability from '@speckle/shared/dist/commonjs/observability/index.js' +import { ServerAPI } from '@/controller/api.js' +import { logger as parentLogger } from '@/observability/logging.js' +import { IFCParser } from '@/ifc/parser.js' -const parseAndCreateCommitFactory = +export const parseAndCreateCommitFactory = ({ db }) => async ({ data, @@ -24,7 +24,7 @@ const parseAndCreateCommitFactory = ) } const serverApi = new ServerAPI({ db, streamId, logger }) - const myParser = new Parser({ serverApi, fileId, logger }) + const myParser = new IFCParser({ serverApi, fileId, logger }) const start = performance.now() const { id, tCount } = await myParser.parse(data) @@ -80,5 +80,3 @@ const parseAndCreateCommitFactory = return json.data.commitCreate } - -module.exports = { parseAndCreateCommitFactory } diff --git a/packages/fileimport-service/ifc/parser.js b/packages/fileimport-service/src/ifc/parser.js similarity index 97% rename from packages/fileimport-service/ifc/parser.js rename to packages/fileimport-service/src/ifc/parser.js index 251df7a11..e960fde30 100644 --- a/packages/fileimport-service/ifc/parser.js +++ b/packages/fileimport-service/src/ifc/parser.js @@ -1,16 +1,16 @@ -const { performance } = require('perf_hooks') -const WebIFC = require('web-ifc/web-ifc-api-node') -const { +import { performance } from 'perf_hooks' +import WebIFC from 'web-ifc/web-ifc-api-node.js' +import Observability from '@speckle/shared/dist/commonjs/observability/index.js' +import { logger as parentLogger } from '@/observability/logging.js' +import { getHash, IfcElements, PropNames, GeometryTypes, IfcTypesMap -} = require('./utils') -const Observability = require('@speckle/shared/dist/commonjs/observability/index.js') -const { logger: parentLogger } = require('../observability/logging') +} from '@/ifc/utils.js' -module.exports = class IFCParser { +export class IFCParser { constructor({ serverApi, fileId, logger }) { this.ifcapi = new WebIFC.IfcAPI() this.ifcapi.SetWasmPath('./', false) diff --git a/packages/fileimport-service/ifc/utils.js b/packages/fileimport-service/src/ifc/utils.js similarity index 99% rename from packages/fileimport-service/ifc/utils.js rename to packages/fileimport-service/src/ifc/utils.js index 3ce78d5d5..242443303 100644 --- a/packages/fileimport-service/ifc/utils.js +++ b/packages/fileimport-service/src/ifc/utils.js @@ -1,7 +1,7 @@ -const crypto = require('crypto') -const WebIFC = require('web-ifc/web-ifc-api-node') +import crypto from 'crypto' +import WebIFC from 'web-ifc/web-ifc-api-node.js' -const IfcElements = { +export const IfcElements = { 103090709: 'IFCPROJECT', 4097777520: 'IFCSITE', 4031249490: 'IFCBUILDING', @@ -141,7 +141,7 @@ const IfcElements = { 3009204131: 'IFCGRID' } -const GeometryTypes = new Set([ +export const GeometryTypes = new Set([ 1123145078, 574549367, 1675464909, 2059837836, 3798115385, 32440307, 3125803723, 3207858831, 2740243338, 2624227202, 4240577450, 3615266464, 3724593414, 220341763, 477187591, 1878645084, 1300840506, 3303107099, 1607154358, 1878645084, 846575682, @@ -163,7 +163,7 @@ const GeometryTypes = new Set([ 1682466193, 2519244187, 2839578677, 3958567839, 2513912981, 2830218821, 427810014 ]) -const IfcTypesMap = { +export const IfcTypesMap = { 3821786052: 'IFCACTIONREQUEST', 2296667514: 'IFCACTOR', 3630933823: 'IFCACTORROLE', @@ -982,7 +982,7 @@ const IfcTypesMap = { 1033361043: 'IFCZONE' } -const PropNames = { +export const PropNames = { aggregates: { name: WebIFC.IFCRELAGGREGATES, relating: 'RelatingObject', @@ -1015,14 +1015,6 @@ const PropNames = { } } -const getHash = (obj) => { +export const getHash = (obj) => { return crypto.createHash('md5').update(JSON.stringify(obj)).digest('hex') } - -module.exports = { - IfcElements, - IfcTypesMap, - PropNames, - GeometryTypes, - getHash -} diff --git a/packages/fileimport-service/ifc/web-ifc.wasm b/packages/fileimport-service/src/ifc/web-ifc.wasm similarity index 100% rename from packages/fileimport-service/ifc/web-ifc.wasm rename to packages/fileimport-service/src/ifc/web-ifc.wasm diff --git a/packages/fileimport-service/knex.js b/packages/fileimport-service/src/knex.ts similarity index 76% rename from packages/fileimport-service/knex.js rename to packages/fileimport-service/src/knex.ts index ff8285fef..6062b4761 100644 --- a/packages/fileimport-service/knex.js +++ b/packages/fileimport-service/src/knex.ts @@ -1,18 +1,18 @@ -'use strict' - -const Environment = require('@speckle/shared/dist/commonjs/environment/index.js') -const { +import Environment from '@speckle/shared/dist/commonjs/environment/index.js' +import { loadMultiRegionsConfig, configureKnexClient -} = require('@speckle/shared/dist/commonjs/environment/multiRegionConfig.js') -const { logger } = require('./observability/logging') +} from '@speckle/shared/dist/commonjs/environment/multiRegionConfig.js' +import { logger } from '@/observability/logging.js' +import { Knex } from 'knex' const { FF_WORKSPACES_MULTI_REGION_ENABLED } = Environment.getFeatureFlags() const isDevEnv = process.env.NODE_ENV !== 'production' -let dbClients -const getDbClients = async () => { +type DbClient = { public: Knex; private?: Knex } +let dbClients: { [key: string]: DbClient } +export const getDbClients = async () => { if (dbClients) return dbClients const maxConnections = parseInt( process.env['POSTGRES_MAX_CONNECTIONS_FILE_IMPORT_SERVICE'] || '1' @@ -49,7 +49,10 @@ const getDbClients = async () => { } else { const configPath = process.env.MULTI_REGION_CONFIG_PATH || 'multiregion.json' const config = await loadMultiRegionsConfig({ path: configPath }) - const clients = [['main', configureKnexClient(config.main, configArgs)]] + + const clients: [string, DbClient][] = [ + ['main', configureKnexClient(config.main, configArgs)] + ] Object.entries(config.regions).map(([key, config]) => { clients.push([key, configureKnexClient(config, configArgs)]) }) @@ -57,5 +60,3 @@ const getDbClients = async () => { } return dbClients } - -module.exports = getDbClients diff --git a/packages/fileimport-service/obj/import_file.py b/packages/fileimport-service/src/obj/import_file.py similarity index 100% rename from packages/fileimport-service/obj/import_file.py rename to packages/fileimport-service/src/obj/import_file.py diff --git a/packages/fileimport-service/obj/mtl_file_collection.py b/packages/fileimport-service/src/obj/mtl_file_collection.py similarity index 100% rename from packages/fileimport-service/obj/mtl_file_collection.py rename to packages/fileimport-service/src/obj/mtl_file_collection.py diff --git a/packages/fileimport-service/obj/obj_file.py b/packages/fileimport-service/src/obj/obj_file.py similarity index 100% rename from packages/fileimport-service/obj/obj_file.py rename to packages/fileimport-service/src/obj/obj_file.py diff --git a/packages/fileimport-service/obj/samples/untitled.mtl b/packages/fileimport-service/src/obj/samples/untitled.mtl similarity index 100% rename from packages/fileimport-service/obj/samples/untitled.mtl rename to packages/fileimport-service/src/obj/samples/untitled.mtl diff --git a/packages/fileimport-service/obj/samples/untitled.obj b/packages/fileimport-service/src/obj/samples/untitled.obj similarity index 100% rename from packages/fileimport-service/obj/samples/untitled.obj rename to packages/fileimport-service/src/obj/samples/untitled.obj diff --git a/packages/fileimport-service/src/objDependencies.js b/packages/fileimport-service/src/objDependencies.js deleted file mode 100644 index f7fa03ac4..000000000 --- a/packages/fileimport-service/src/objDependencies.js +++ /dev/null @@ -1,59 +0,0 @@ -'use strict' -const events = require('events') -const fs = require('fs') -const readline = require('readline') -const path = require('path') - -const { downloadFile, getFileInfoByName } = require('./filesApi') -const isValidFilename = require('valid-filename') -const { logger } = require('../observability/logging') - -const getReferencedMtlFiles = async ({ objFilePath }) => { - const mtlFiles = [] - - try { - const rl = readline.createInterface({ - input: fs.createReadStream(objFilePath), - crlfDelay: Infinity - }) - - rl.on('line', (line) => { - if (line.startsWith('mtllib ')) { - const mtlFile = line.slice('mtllib '.length).trim() - mtlFiles.push(mtlFile) - } - }) - - await events.once(rl, 'close') - } catch (err) { - logger.error(err, `Error getting dependencies for file ${objFilePath}`) - } - return mtlFiles -} - -module.exports = { - async downloadDependencies({ objFilePath, streamId, destinationDir, token }) { - const dependencies = await getReferencedMtlFiles({ objFilePath }) - - logger.info(`Obj file depends on ${dependencies}`) - for (const mtlFile of dependencies) { - // there might be multiple files named with the same name, take the first... - const [file] = (await getFileInfoByName({ fileName: mtlFile, streamId, token })) - .blobs - if (!file) { - logger.info(`OBJ dependency file not found in stream: ${mtlFile}`) - continue - } - if (!isValidFilename(mtlFile)) { - logger.warn(`Invalid filename reference in OBJ dependencies: ${mtlFile}`) - continue - } - await downloadFile({ - fileId: file.id, - streamId, - token, - destination: path.join(destinationDir, mtlFile) - }) - } - } -} diff --git a/packages/fileimport-service/observability/logging.js b/packages/fileimport-service/src/observability/logging.ts similarity index 53% rename from packages/fileimport-service/observability/logging.js rename to packages/fileimport-service/src/observability/logging.ts index 31f688f13..6ef8bca14 100644 --- a/packages/fileimport-service/observability/logging.js +++ b/packages/fileimport-service/src/observability/logging.ts @@ -1,14 +1,10 @@ -const Observability = require('@speckle/shared/dist/commonjs/observability/index.js') +import Observability from '@speckle/shared/dist/commonjs/observability/index.js' // loggers for specific components within normal operation -const logger = Observability.extendLoggerComponent( +export const logger = Observability.extendLoggerComponent( Observability.getLogger( process.env.LOG_LEVEL || 'info', process.env.LOG_PRETTY === 'true' ), 'fileimport-service' ) - -module.exports = { - logger -} diff --git a/packages/fileimport-service/src/prometheusMetrics.js b/packages/fileimport-service/src/prometheusMetrics.js deleted file mode 100644 index 6248636cf..000000000 --- a/packages/fileimport-service/src/prometheusMetrics.js +++ /dev/null @@ -1,166 +0,0 @@ -/* eslint-disable no-unused-vars */ -'use strict' - -const http = require('http') -const prometheusClient = require('prom-client') -const getDbClients = require('../knex') - -let metricFree = null -let metricUsed = null -let metricPendingAquires = null -let metricPendingCreates = null -let metricPendingValidations = null -let metricRemainingCapacity = null -let metricQueryDuration = null -let metricQueryErrors = null - -const queryStartTime = {} -prometheusClient.register.clear() -prometheusClient.register.setDefaultLabels({ - project: 'speckle-server', - app: 'fileimport-service' -}) -prometheusClient.collectDefaultMetrics() - -let prometheusInitialized = false - -const initDBPrometheusMetricsFactory = - ({ db }) => - () => { - metricFree = new prometheusClient.Gauge({ - name: 'speckle_server_knex_free', - help: 'Number of free DB connections', - collect() { - this.set(db.client.pool.numFree()) - } - }) - - metricUsed = new prometheusClient.Gauge({ - name: 'speckle_server_knex_used', - help: 'Number of used DB connections', - collect() { - this.set(db.client.pool.numUsed()) - } - }) - - metricPendingAquires = new prometheusClient.Gauge({ - name: 'speckle_server_knex_pending', - help: 'Number of pending DB connection aquires', - collect() { - this.set(db.client.pool.numPendingAcquires()) - } - }) - - metricPendingCreates = new prometheusClient.Gauge({ - name: 'speckle_server_knex_pending_creates', - help: 'Number of pending DB connection creates', - collect() { - this.set(db.client.pool.numPendingCreates()) - } - }) - - metricPendingValidations = new prometheusClient.Gauge({ - name: 'speckle_server_knex_pending_validations', - help: 'Number of pending DB connection validations. This is a state between pending acquisition and acquiring a connection.', - collect() { - this.set(db.client.pool.numPendingValidations()) - } - }) - - metricRemainingCapacity = new prometheusClient.Gauge({ - name: 'speckle_server_knex_remaining_capacity', - help: 'Remaining capacity of the DB connection pool', - collect() { - const postgresMaxConnections = - parseInt(process.env.POSTGRES_MAX_CONNECTIONS_FILE_IMPORT_SERVICE) || 1 - const demand = - db.client.pool.numUsed() + - db.client.pool.numPendingCreates() + - db.client.pool.numPendingValidations() + - db.client.pool.numPendingAcquires() - - this.set(Math.max(postgresMaxConnections - demand, 0)) - } - }) - - metricQueryDuration = new prometheusClient.Summary({ - name: 'speckle_server_knex_query_duration', - help: 'Summary of the DB query durations in seconds' - }) - - metricQueryErrors = new prometheusClient.Counter({ - name: 'speckle_server_knex_query_errors', - help: 'Number of DB queries with errors' - }) - - db.on('query', (data) => { - const queryId = data.__knexQueryUid + '' - queryStartTime[queryId] = Date.now() - }) - - db.on('query-response', (data, obj, builder) => { - const queryId = obj.__knexQueryUid + '' - const durationSec = (Date.now() - queryStartTime[queryId]) / 1000 - delete queryStartTime[queryId] - if (!isNaN(durationSec)) metricQueryDuration.observe(durationSec) - }) - - db.on('query-error', (err, querySpec) => { - const queryId = querySpec.__knexQueryUid + '' - const durationSec = (Date.now() - queryStartTime[queryId]) / 1000 - delete queryStartTime[queryId] - - if (!isNaN(durationSec)) metricQueryDuration.observe(durationSec) - metricQueryErrors.inc() - }) - } - -module.exports = { - async initPrometheusMetrics() { - if (prometheusInitialized) return - prometheusInitialized = true - - const db = (await getDbClients()).main.public - - initDBPrometheusMetricsFactory({ db })() - - // Define the HTTP server - const server = http.createServer(async (req, res) => { - if (req.url === '/metrics') { - res.setHeader('Content-Type', prometheusClient.register.contentType) - res.end(await prometheusClient.register.metrics()) - } else { - res.end('Speckle FileImport Service - prometheus metrics') - } - }) - server.listen(Number(process.env.PROMETHEUS_METRICS_PORT) || 9093) - }, - - metricDuration: new prometheusClient.Histogram({ - name: 'speckle_server_operation_duration', - help: 'Summary of the operation durations in seconds', - buckets: [0.5, 1, 5, 10, 30, 60, 300, 600, 900, 1200], - labelNames: ['op'] - }), - - metricOperationErrors: new prometheusClient.Counter({ - name: 'speckle_server_operation_errors', - help: 'Number of operations with errors', - labelNames: ['op'] - }), - - metricInputFileSize: new prometheusClient.Histogram({ - name: 'speckle_server_operation_file_size', - help: 'Size of the operation input file size', - buckets: [ - 1000, - 100 * 1000, - 500 * 1000, - 1000 * 1000, - 5 * 1000 * 1000, - 10 * 1000 * 1000, - 100 * 1000 * 1000 - ], - labelNames: ['op'] - }) -} diff --git a/packages/fileimport-service/src/root.ts b/packages/fileimport-service/src/root.ts new file mode 100644 index 000000000..13b51d800 --- /dev/null +++ b/packages/fileimport-service/src/root.ts @@ -0,0 +1,21 @@ +import path from 'node:path' +import fs from 'node:fs' +import { fileURLToPath } from 'url' + +/** + * Singleton module for src root and package root directory resolution + */ + +const __filename = fileURLToPath(import.meta.url) +const srcRoot = path.dirname(__filename) + +// Recursively walk back from __dirname till we find our package.json +let packageRoot = srcRoot +while (packageRoot !== '/') { + if (fs.readdirSync(packageRoot).includes('package.json')) { + break + } + packageRoot = path.resolve(packageRoot, '..') +} + +export { srcRoot, packageRoot } diff --git a/packages/fileimport-service/stl/import_file.py b/packages/fileimport-service/src/stl/import_file.py similarity index 100% rename from packages/fileimport-service/stl/import_file.py rename to packages/fileimport-service/src/stl/import_file.py diff --git a/packages/fileimport-service/stl/samples/Gizmo_Spoon_Rider_bin.stl b/packages/fileimport-service/src/stl/samples/Gizmo_Spoon_Rider_bin.stl similarity index 100% rename from packages/fileimport-service/stl/samples/Gizmo_Spoon_Rider_bin.stl rename to packages/fileimport-service/src/stl/samples/Gizmo_Spoon_Rider_bin.stl diff --git a/packages/fileimport-service/tsconfig.build.json b/packages/fileimport-service/tsconfig.build.json new file mode 100644 index 000000000..8e1bc28f9 --- /dev/null +++ b/packages/fileimport-service/tsconfig.build.json @@ -0,0 +1,5 @@ +{ + "extends": "./tsconfig.json", + "include": ["src/**/*"], + "exclude": ["**/*.spec.js", "**/*.spec.ts"] +} diff --git a/packages/fileimport-service/tsconfig.json b/packages/fileimport-service/tsconfig.json new file mode 100644 index 000000000..4351d814b --- /dev/null +++ b/packages/fileimport-service/tsconfig.json @@ -0,0 +1,108 @@ +{ + "compilerOptions": { + /* Visit https://aka.ms/tsconfig.json to read more about this file */ + + /* Projects */ + // "incremental": true, /* Enable incremental compilation */ + // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */ + // "tsBuildInfoFile": "./", /* Specify the folder for .tsbuildinfo incremental compilation files. */ + // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects */ + // "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */ + // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ + + /* Language and Environment */ + "target": "ES2022" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */, + // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ + // "jsx": "preserve", /* Specify what JSX code is generated. */ + // "experimentalDecorators": true, /* Enable experimental support for TC39 stage 2 draft decorators. */ + // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ + // "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h' */ + // "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */ + // "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using `jsx: react-jsx*`.` */ + // "reactNamespace": "", /* Specify the object invoked for `createElement`. This only applies when targeting `react` JSX emit. */ + // "noLib": true, /* Disable including any library files, including the default lib.d.ts. */ + // "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */ + + /* Modules */ + "module": "node16" /* Specify what module code is generated. */, + "rootDir": "./" /* Specify the root folder within your source files. */, + "moduleResolution": "node16" /* Specify how TypeScript looks up a file from a given module specifier. */, + "baseUrl": "./" /* Specify the base directory to resolve non-relative module names. */, + "paths": { + "@/*": ["./src/*"] + }, + // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ + // "typeRoots": [], /* Specify multiple folders that act like `./node_modules/@types`. */ + // "types": [], /* Specify type package names to be included without being referenced in a source file. */ + // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ + // "resolveJsonModule": true, /* Enable importing .json files */ + // "noResolve": true, /* Disallow `import`s, `require`s or ``s from expanding the number of files TypeScript should add to a project. */ + + /* JavaScript Support */ + "allowJs": true /* Allow JavaScript files to be a part of your program. Use the `checkJS` option to get errors from these files. */, + "checkJs": false /* Enable error reporting in type-checked JavaScript files. */, + // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from `node_modules`. Only applicable with `allowJs`. */ + + /* Emit */ + // "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ + // "declarationMap": true, /* Create sourcemaps for d.ts files. */ + // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ + "sourceMap": true /* Create source map files for emitted JavaScript files. */, + // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If `declaration` is true, also designates a file that bundles all .d.ts output. */ + "outDir": "./dist" /* Specify an output folder for all emitted files. */, + // "removeComments": true, /* Disable emitting comments. */ + // "noEmit": true, /* Disable emitting files from a compilation. */ + // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ + // "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types */ + // "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */ + // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */ + // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ + // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ + // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */ + // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */ + // "newLine": "crlf", /* Set the newline character for emitting files. */ + // "stripInternal": true, /* Disable emitting declarations that have `@internal` in their JSDoc comments. */ + // "noEmitHelpers": true, /* Disable generating custom helper functions like `__extends` in compiled output. */ + // "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */ + // "preserveConstEnums": true, /* Disable erasing `const enum` declarations in generated code. */ + // "declarationDir": "./", /* Specify the output directory for generated declaration files. */ + // "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */ + + /* Interop Constraints */ + // "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */ + "allowSyntheticDefaultImports": true /* Allow 'import x from y' when a module doesn't have a default export. */, + "esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables `allowSyntheticDefaultImports` for type compatibility. */, + // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ + "forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */, + + /* Type Checking */ + "strict": true /* Enable all strict type-checking options. */, + // "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied `any` type.. */ + // "strictNullChecks": true, /* When type checking, take into account `null` and `undefined`. */ + // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ + // "strictBindCallApply": true, /* Check that the arguments for `bind`, `call`, and `apply` methods match the original function. */ + // "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ + // "noImplicitThis": true, /* Enable error reporting when `this` is given the type `any`. */ + // "useUnknownInCatchVariables": true, /* Type catch clause variables as 'unknown' instead of 'any'. */ + // "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ + // "noUnusedLocals": true, /* Enable error reporting when a local variables aren't read. */ + // "noUnusedParameters": true, /* Raise an error when a function parameter isn't read */ + // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ + // "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ + // "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ + // "noUncheckedIndexedAccess": true, /* Include 'undefined' in index signature results */ + // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ + // "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type */ + // "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ + // "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ + + /* Completeness */ + // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ + "skipLibCheck": true /* Skip type checking all .d.ts files. */ + }, + "ts-node": { + "swc": true + }, + "include": ["src/**/*", "vitest.config.ts"], + "exclude": ["node_modules", "coverage", "reports"] +} diff --git a/packages/fileimport-service/vitest.config.ts b/packages/fileimport-service/vitest.config.ts new file mode 100644 index 000000000..dbf4dedd5 --- /dev/null +++ b/packages/fileimport-service/vitest.config.ts @@ -0,0 +1,18 @@ +import path from 'path' +import { configDefaults, defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + exclude: [...configDefaults.exclude], + // reporters: ['verbose', 'hanging-process'] //uncomment to debug hanging processes etc. + sequence: { + shuffle: true, + concurrent: true + } + }, + resolve: { + alias: { + '@': path.resolve(__dirname, './src') + } + } +}) diff --git a/packages/frontend-2/components/common/tiptap/MentionListItem.vue b/packages/frontend-2/components/common/tiptap/MentionListItem.vue index 0ff523955..8d9ca76a2 100644 --- a/packages/frontend-2/components/common/tiptap/MentionListItem.vue +++ b/packages/frontend-2/components/common/tiptap/MentionListItem.vue @@ -8,7 +8,7 @@ ]" @click="($event) => $emit('click', $event)" > - {{ item.company ? item.company : item.name }} + {{ item.name }} diff --git a/packages/frontend-2/components/connectors/Page.vue b/packages/frontend-2/components/connectors/Page.vue index 5a3329513..7343148fd 100644 --- a/packages/frontend-2/components/connectors/Page.vue +++ b/packages/frontend-2/components/connectors/Page.vue @@ -10,15 +10,15 @@

Extract and exchange data between the most popular AEC applications using our tailored connectors. - - Looking for V2 connectors? Get them - - here. - - +

+

+ Looking for V2 connectors? Get them + + here. +

diff --git a/packages/frontend-2/components/dashboard/sidebar/New.vue b/packages/frontend-2/components/dashboard/sidebar/New.vue new file mode 100644 index 000000000..24a12777c --- /dev/null +++ b/packages/frontend-2/components/dashboard/sidebar/New.vue @@ -0,0 +1,188 @@ + + + + diff --git a/packages/frontend-2/components/dashboard/Sidebar.vue b/packages/frontend-2/components/dashboard/sidebar/Sidebar.vue similarity index 100% rename from packages/frontend-2/components/dashboard/Sidebar.vue rename to packages/frontend-2/components/dashboard/sidebar/Sidebar.vue diff --git a/packages/frontend-2/components/dashboard/sidebar/WorkspaceItem.vue b/packages/frontend-2/components/dashboard/sidebar/WorkspaceItem.vue new file mode 100644 index 000000000..e4702e306 --- /dev/null +++ b/packages/frontend-2/components/dashboard/sidebar/WorkspaceItem.vue @@ -0,0 +1,39 @@ + + + diff --git a/packages/frontend-2/components/dashboard/sidebar/Wrapper.vue b/packages/frontend-2/components/dashboard/sidebar/Wrapper.vue new file mode 100644 index 000000000..40ca9b613 --- /dev/null +++ b/packages/frontend-2/components/dashboard/sidebar/Wrapper.vue @@ -0,0 +1,11 @@ + + + diff --git a/packages/frontend-2/components/header/NavBar.vue b/packages/frontend-2/components/header/NavBar.vue index 2c500fbc1..610941bb0 100644 --- a/packages/frontend-2/components/header/NavBar.vue +++ b/packages/frontend-2/components/header/NavBar.vue @@ -4,7 +4,15 @@
-
@@ -39,6 +49,8 @@ import { useActiveUser } from '~~/lib/auth/composables/activeUser' import { loginRoute } from '~~/lib/common/helpers/route' import type { Optional } from '@speckle/shared' +const isWorkspacesEnabled = useIsWorkspacesEnabled() +const isWorkspaceNewPlansEnabled = useWorkspaceNewPlansEnabled() const { activeUser } = useActiveUser() const route = useRoute() const router = useRouter() @@ -53,4 +65,9 @@ const loginUrl = computed(() => } }) ) + +const showWorkspaceSwitcher = computed( + () => + isWorkspacesEnabled.value && isWorkspaceNewPlansEnabled.value && activeUser.value +) diff --git a/packages/frontend-2/components/header/NavUserMenu.vue b/packages/frontend-2/components/header/NavUserMenu.vue index 85fdd8057..b3eb26d4b 100644 --- a/packages/frontend-2/components/header/NavUserMenu.vue +++ b/packages/frontend-2/components/header/NavUserMenu.vue @@ -19,77 +19,64 @@ -
+
+ + + Settings + + + + + Server settings + + - Connector downloads + {{ isDarkTheme ? 'Light mode' : 'Dark mode' }} + + + + + Invite to Speckle + + + + + Feedback
- - - Settings - - - - - Server settings - - - - - {{ isDarkTheme ? 'Light mode' : 'Dark mode' }} - - - - - Invite to Speckle - - - - - Feedback - -
+
+ + + Open workspace menu +
+ + + +
+ + + +
+
+ + + + + +
+

+ {{ displayName }} +

+

+ {{ activeWorkspace?.plan?.name }} · + {{ activeWorkspace?.team?.totalCount }} member{{ + activeWorkspace?.team?.totalCount > 1 ? 's' : '' + }} +

+

+ 2 projects to move +

+
+
+
+ + + Settings + + + + + Invite members + + +
+
+
+ +
+ + +
+ +
+ +
+ +

Join existing workspaces

+ + {{ discoverableWorkspacesCount }} + + +
+ + + + + + + + +
+ + diff --git a/packages/frontend-2/components/invite/dialog/Workspace.vue b/packages/frontend-2/components/invite/dialog/Workspace.vue index 41a88abd7..446d962f8 100644 --- a/packages/frontend-2/components/invite/dialog/Workspace.vue +++ b/packages/frontend-2/components/invite/dialog/Workspace.vue @@ -29,7 +29,7 @@ import { } from '~/lib/common/generated/gql/graphql' import type { InviteGenericItem } from '~~/lib/invites/helpers/types' import { emptyInviteGenericItem } from '~~/lib/invites/helpers/constants' -import { Roles } from '@speckle/shared' +import { Roles, type MaybeNullOrUndefined } from '@speckle/shared' import { useMixpanel } from '~/lib/core/composables/mp' import { mapMainRoleToGqlWorkspaceRole } from '~/lib/workspaces/helpers/roles' import { mapServerRoleToGqlServerRole } from '~/lib/common/helpers/roles' @@ -58,7 +58,7 @@ graphql(` `) const props = defineProps<{ - workspace: InviteDialogWorkspace_WorkspaceFragment + workspace?: MaybeNullOrUndefined }>() const isOpen = defineModel('open', { required: true }) @@ -75,7 +75,7 @@ const invites = ref([ ]) const allowedDomains = computed(() => - props.workspace.domainBasedMembershipProtectionEnabled + props.workspace?.domainBasedMembershipProtectionEnabled ? props.workspace.domains?.map((d) => d.domain) : null ) @@ -117,7 +117,7 @@ const onSelectUsersSubmit = async (updatedInvites: InviteGenericItem[]) => { : undefined })) - if (!inputs.length) return + if (!inputs.length || !props.workspace?.id) return await inviteToWorkspace({ workspaceId: props.workspace.id, inputs }) isOpen.value = false diff --git a/packages/frontend-2/components/onboarding/questions/Form.vue b/packages/frontend-2/components/onboarding/questions/Form.vue index 0d5e939bb..f1080b1f8 100644 --- a/packages/frontend-2/components/onboarding/questions/Form.vue +++ b/packages/frontend-2/components/onboarding/questions/Form.vue @@ -29,10 +29,11 @@ import { useForm } from 'vee-validate' import type { OnboardingRole, OnboardingPlan, OnboardingSource } from '@speckle/shared' import { useProcessOnboarding } from '~~/lib/auth/composables/onboarding' import { homeRoute, workspaceJoinRoute } from '~/lib/common/helpers/route' +import { useDiscoverableWorkspaces } from '~/lib/workspaces/composables/discoverableWorkspaces' const isOnboardingForced = useIsOnboardingForced() const isWorkspacesEnabled = useIsWorkspacesEnabled() -const isWorkspaceNewPlansEnabled = useWorkspaceNewPlansEnabled() +const { hasDiscoverableWorkspaces } = useDiscoverableWorkspaces() const { setUserOnboardingComplete, setMixpanelSegments } = useProcessOnboarding() @@ -53,7 +54,7 @@ const onSubmit = handleSubmit(async () => { plans: values.plan, source: values.source }) - if (!isWorkspaceNewPlansEnabled.value && isWorkspacesEnabled.value) { + if (isWorkspacesEnabled.value && hasDiscoverableWorkspaces.value) { navigateTo(workspaceJoinRoute) } else { navigateTo(homeRoute) diff --git a/packages/frontend-2/components/project/CardImportFileArea.vue b/packages/frontend-2/components/project/CardImportFileArea.vue index 6701ec588..63479d61b 100644 --- a/packages/frontend-2/components/project/CardImportFileArea.vue +++ b/packages/frontend-2/components/project/CardImportFileArea.vue @@ -37,12 +37,7 @@
Use our - + connectors to publish a {{ modelName ? '' : 'new model' }} version to @@ -56,7 +51,7 @@ import { useFileImport } from '~~/lib/core/composables/fileImport' import { useFileUploadProgressCore } from '~~/lib/form/composables/fileUpload' import { ExclamationTriangleIcon } from '@heroicons/vue/24/solid' -import { downloadManagerUrl } from '~/lib/common/helpers/route' +import { connectorsRoute } from '~/lib/common/helpers/route' import type { Nullable } from '@speckle/shared' const props = defineProps<{ diff --git a/packages/frontend-2/components/projects/Dashboard.vue b/packages/frontend-2/components/projects/Dashboard.vue index 5a3f76460..915e6036e 100644 --- a/packages/frontend-2/components/projects/Dashboard.vue +++ b/packages/frontend-2/components/projects/Dashboard.vue @@ -2,6 +2,7 @@
@@ -92,6 +93,7 @@ const showLoadingBar = ref(false) const areQueriesLoading = useQueryLoading() const isWorkspacesEnabled = useIsWorkspacesEnabled() const { isGuest } = useActiveUser() +const isWorkspaceNewPlansEnabled = useWorkspaceNewPlansEnabled() useUserProjectsUpdatedTracking() const { @@ -110,7 +112,8 @@ const { } = useQuery(projectsDashboardQuery, () => ({ filter: { search: (search.value || '').trim() || null, - onlyWithRoles: selectedRoles.value?.length ? selectedRoles.value : null + onlyWithRoles: selectedRoles.value?.length ? selectedRoles.value : null, + workspaceId: isWorkspaceNewPlansEnabled ? (null as Nullable) : undefined }, cursor: null as Nullable })) diff --git a/packages/frontend-2/components/settings/Sidebar.vue b/packages/frontend-2/components/settings/Sidebar.vue index 92b429af1..8454947e5 100644 --- a/packages/frontend-2/components/settings/Sidebar.vue +++ b/packages/frontend-2/components/settings/Sidebar.vue @@ -68,61 +68,103 @@ /> - - - + + + + @@ -144,16 +186,22 @@ import { } from '@speckle/ui-components' import { graphql } from '~~/lib/common/generated/gql' import type { WorkspaceRoles } from '@speckle/shared' -import { homeRoute, settingsWorkspaceRoutes } from '~/lib/common/helpers/route' +import { + homeRoute, + projectsRoute, + settingsWorkspaceRoutes, + workspaceRoute +} from '~/lib/common/helpers/route' import { WorkspacePlanStatuses, type SettingsMenu_WorkspaceFragment } from '~/lib/common/generated/gql/graphql' import { TailwindBreakpoints } from '~~/lib/common/helpers/tailwind' import { useBreakpoints } from '@vueuse/core' +import { useNavigation } from '~~/lib/navigation/composables/navigation' graphql(` - fragment SettingsDialog_Workspace on Workspace { + fragment SettingsSidebar_Workspace on Workspace { ...SettingsMenu_Workspace id slug @@ -162,6 +210,7 @@ graphql(` logo plan { status + name } creationState { completed @@ -170,20 +219,22 @@ graphql(` `) graphql(` - fragment SettingsDialog_User on User { + fragment SettingsSidebar_User on User { id workspaces { items { - ...SettingsDialog_Workspace + ...SettingsSidebar_Workspace } } } `) +const isWorkspaceNewPlansEnabled = useWorkspaceNewPlansEnabled() +const isWorkspacesEnabled = useIsWorkspacesEnabled() +const { activeWorkspaceSlug } = useNavigation() const settingsMenuState = useSettingsMenuState() const { isAdmin } = useActiveUser() const route = useRoute() -const isWorkspacesEnabled = useIsWorkspacesEnabled() const { result: workspaceResult } = useQuery(settingsSidebarQuery, null, { enabled: computed(() => isWorkspacesEnabled.value) }) @@ -200,6 +251,9 @@ const workspaceItems = computed( (item) => item.creationState?.completed !== false // Removed workspaces that are not completely created ) || [] ) +const activeWorkspaceItem = computed(() => + workspaceItems.value.find((item) => item.slug === activeWorkspaceSlug.value) +) const needsSsoSession = ( workspace: SettingsMenu_WorkspaceFragment, @@ -212,9 +266,19 @@ const needsSsoSession = ( } const exitSettingsRoute = computed(() => { - if (import.meta.server || !settingsMenuState.value.previousRoute) { - return homeRoute + if (import.meta.server) return homeRoute + if (!settingsMenuState.value.previousRoute) { + return activeWorkspaceSlug.value + ? workspaceRoute(activeWorkspaceSlug.value) + : projectsRoute } + return settingsMenuState.value.previousRoute }) + +const showWorkspaceSettings = computed(() => { + if (!isWorkspacesEnabled.value) return false + if (isWorkspaceNewPlansEnabled.value) return !!activeWorkspaceSlug.value + return true +}) diff --git a/packages/frontend-2/components/settings/workspaces/billing/PricingTable/Plan.vue b/packages/frontend-2/components/settings/workspaces/billing/PricingTable/Plan.vue index ed41b0135..72e6528a2 100644 --- a/packages/frontend-2/components/settings/workspaces/billing/PricingTable/Plan.vue +++ b/packages/frontend-2/components/settings/workspaces/billing/PricingTable/Plan.vue @@ -13,7 +13,7 @@

- {{ formatPrice(planPrice?.[Roles.Workspace.Member]) }} + {{ formatPrice(finalPlanPrice) }} per seat/month

@@ -141,6 +141,15 @@ const planPrice = computed( prices.value?.[props.plan]?.[props.yearlyIntervalSelected ? 'yearly' : 'monthly'] ) +const finalPlanPrice = computed(() => { + const basePrice = prices.value?.[props.plan].monthly?.['workspace:member'] + if (!basePrice) return undefined + return { + ...basePrice, + amount: props.yearlyIntervalSelected ? basePrice.amount * 0.8 : basePrice.amount + } +}) + const hasCta = computed(() => !!slots.cta) const canUpgradeToPlan = computed(() => { if (!props.currentPlan) return false diff --git a/packages/frontend-2/components/viewer/Base.vue b/packages/frontend-2/components/viewer/Base.vue index ce73ce0dd..f9a2b7ccf 100644 --- a/packages/frontend-2/components/viewer/Base.vue +++ b/packages/frontend-2/components/viewer/Base.vue @@ -1,8 +1,13 @@ diff --git a/packages/frontend-2/components/viewer/PreSetupWrapper.vue b/packages/frontend-2/components/viewer/PreSetupWrapper.vue index eaf6df817..3909cc249 100644 --- a/packages/frontend-2/components/viewer/PreSetupWrapper.vue +++ b/packages/frontend-2/components/viewer/PreSetupWrapper.vue @@ -97,7 +97,11 @@ :url="route.path" /> - +
@@ -113,17 +117,28 @@ import { useFilterUtilities } from '~/lib/viewer/composables/ui' import { projectsRoute } from '~~/lib/common/helpers/route' import { workspaceRoute } from '~/lib/common/helpers/route' import { useMixpanel } from '~/lib/core/composables/mp' +import { writableAsyncComputed } from '~/lib/common/composables/async' const emit = defineEmits<{ setup: [InjectableViewerState] }>() +const router = useRouter() const route = useRoute() const isWorkspacesEnabled = useIsWorkspacesEnabled() -const modelId = computed(() => route.params.modelId as string) - -const projectId = computed(() => route.params.id as string) +const resourceIdString = computed(() => route.params.modelId as string) +const projectId = writableAsyncComputed({ + get: () => route.params.id as string, + set: async (value: string) => { + // Just rewrite route id param + await router.push({ + params: { id: value } + }) + }, + initialState: route.params.id as string, + asyncRead: false +}) const state = useSetupViewer({ projectId diff --git a/packages/frontend-2/components/viewer/anchored-point/Thread.vue b/packages/frontend-2/components/viewer/anchored-point/Thread.vue index 1c70a1ef6..d4dacadc5 100644 --- a/packages/frontend-2/components/viewer/anchored-point/Thread.vue +++ b/packages/frontend-2/components/viewer/anchored-point/Thread.vue @@ -110,6 +110,24 @@ /> +
+
+ {{ bannerText }} +
+
+ + {{ bannerButton.text }} + +
+
@@ -117,22 +135,6 @@ ref="commentsContainer" class="max-h-[200px] sm:max-h-[300px] 2xl:max-h-[500px] overflow-y-auto simple-scrollbar flex flex-col space-y-1 py-2 sm:pr-3" > -
- Conversation started in a different version - - - -
) @@ -410,27 +406,6 @@ const canArchiveOrUnarchive = computed( project.value?.role === Roles.Stream.Owner) ) -const { resourceItems } = useInjectedViewerLoadedResources() - -const isThreadResourceLoaded = computed(() => { - const thread = props.modelValue - const loadedResources = resourceItems.value - const resourceLinks = thread.resources - - const objectLinks = resourceLinks - .filter((l) => l.resourceType === ResourceType.Object) - .map((l) => l.resourceId) - const commitLinks = resourceLinks - .filter((l) => l.resourceType === ResourceType.Commit) - .map((l) => l.resourceId) - - if (loadedResources.some((lr) => objectLinks.includes(lr.objectId))) return true - if (loadedResources.some((lr) => lr.versionId && commitLinks.includes(lr.versionId))) - return true - - return false -}) - const toggleCommentResolvedStatus = async () => { await archiveComment({ commentId: props.modelValue.id, @@ -467,13 +442,6 @@ const onThreadClick = () => { changeExpanded(!isExpanded.value) } -const onLoadThreadContext = async () => { - const state = props.modelValue.viewerState - if (!state) return - - await applyState(state, StateApplyMode.TheadFullContextOpen) -} - const onCopyLink = async () => { if (import.meta.server) return const url = getLinkToThread(projectId.value, props.modelValue) @@ -545,6 +513,41 @@ onMounted(() => { emit('update:expanded', true) } }) + +const showBanner = computed( + () => + threadResourceStatus.value.isDifferentVersion || + threadResourceStatus.value.isFederatedModel || + hasClickedFullContext.value +) + +const bannerText = computed(() => { + if (hasClickedFullContext.value) return 'Viewing full context' + if ( + threadResourceStatus.value.isDifferentVersion && + threadResourceStatus.value.isFederatedModel + ) + return 'References multiple models with different versions' + if (threadResourceStatus.value.isDifferentVersion) + return 'Conversation started in a different version' + if (threadResourceStatus.value.isFederatedModel) return 'References multiple models' + return '' +}) + +const bannerButton = computed(() => { + if (hasClickedFullContext.value) { + return { + text: 'Back', + icon: ArrowLeftIcon, + action: goBack + } + } + return { + text: 'Full context', + icon: ArrowUpRightIcon, + action: handleContextClick + } +})