diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 86f2ff040..f2bd0158f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -84,9 +84,6 @@ jobs: run: YARN_ENABLE_HARDENED_MODE=0 PUPPETEER_SKIP_DOWNLOAD=true PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 yarn --immutable - name: Build public packages run: yarn build:public - - name: Lint viewer-sandbox - run: yarn lint:ci - working-directory: 'packages/viewer-sandbox' - name: Build viewer-sandbox run: yarn build working-directory: 'packages/viewer-sandbox' diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a355782fb..d1e50b61a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -69,6 +69,7 @@ jobs: test-frontend-2: name: Frontend runs-on: blacksmith + if: false # disabled as there is nothing to run steps: - uses: actions/checkout@v4.2.2 - uses: useblacksmith/setup-node@v5 @@ -79,9 +80,6 @@ jobs: run: YARN_ENABLE_HARDENED_MODE=0 PUPPETEER_SKIP_DOWNLOAD=true PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 yarn --immutable - name: Build public packages run: yarn build:public - - name: Lint everything - run: yarn lint:ci - working-directory: 'packages/frontend-2' test-viewer: name: Viewer @@ -96,15 +94,9 @@ jobs: run: YARN_ENABLE_HARDENED_MODE=0 PUPPETEER_SKIP_DOWNLOAD=true PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 yarn --immutable - name: Build public packages run: yarn build:public - - name: Lint viewer - run: yarn lint:ci - working-directory: 'packages/viewer' - name: Run tests run: yarn test working-directory: 'packages/viewer' - - name: Lint viewer-sandbox - run: yarn lint:ci - working-directory: 'packages/viewer-sandbox' - name: Build viewer-sandbox run: yarn build working-directory: 'packages/viewer-sandbox' @@ -141,9 +133,6 @@ jobs: cache: yarn - name: Install dependencies run: YARN_ENABLE_HARDENED_MODE=0 PUPPETEER_SKIP_DOWNLOAD=true PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 yarn --immutable - - name: Lint - run: yarn lint:ci - working-directory: 'packages/shared' - name: Run tests (all FFs) run: ENABLE_ALL_FFS=1 yarn test:ci working-directory: 'packages/shared' @@ -179,7 +168,7 @@ jobs: run: YARN_ENABLE_HARDENED_MODE=0 PUPPETEER_SKIP_DOWNLOAD=true PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 yarn --immutable - name: Build public packages run: yarn build:public - - name: Lint everything + - name: Test object sender run: yarn test:ci working-directory: 'packages/objectsender' - uses: codecov/codecov-action@v5 @@ -202,15 +191,6 @@ jobs: run: YARN_ENABLE_HARDENED_MODE=0 PUPPETEER_SKIP_DOWNLOAD=true yarn --immutable # we need PLAYWRIGHT - name: Build public packages run: yarn build:public - - name: Lint tailwind theme - run: yarn lint:ci - working-directory: 'packages/tailwind-theme' - - name: Lint ui components - run: yarn lint:ci - working-directory: 'packages/ui-components' - - name: Lint component nuxt package - run: yarn lint:ci - working-directory: 'packages/ui-components-nuxt' - name: Test via Storybook run: yarn storybook:test:ci working-directory: 'packages/ui-components' @@ -218,6 +198,7 @@ jobs: test-preview-service: name: Preview service runs-on: blacksmith + if: false # disabled as there is nothing to run steps: - uses: actions/checkout@v4.2.2 - uses: useblacksmith/setup-node@v5 @@ -228,9 +209,6 @@ jobs: run: YARN_ENABLE_HARDENED_MODE=0 PUPPETEER_SKIP_DOWNLOAD=true PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 yarn --immutable - name: Build public packages run: yarn build:public - - name: Lint everything - run: yarn lint:ci - working-directory: 'packages/preview-service' docker-build-postgres-container: runs-on: blacksmith @@ -319,9 +297,6 @@ jobs: run: yarn build:public - run: cp .env.test-example .env.test working-directory: 'packages/server' - - name: 'Lint' - run: yarn lint:ci - working-directory: 'packages/server' - name: 'Run test' run: yarn test:report working-directory: 'packages/server' diff --git a/docker-compose-speckle.yml b/docker-compose-speckle.yml index 18d16e931..e8fb2b8c5 100644 --- a/docker-compose-speckle.yml +++ b/docker-compose-speckle.yml @@ -32,6 +32,7 @@ services: NUXT_PUBLIC_BACKEND_API_ORIGIN: 'http://speckle-server:3000' NUXT_PUBLIC_LOG_LEVEL: 'warn' NUXT_REDIS_URL: 'redis://redis' + NUXT_PUBLIC_FF_LARGE_FILE_IMPORTS_ENABLED: 'true' LOG_LEVEL: 'info' LOG_PRETTY: 'true' depends_on: @@ -79,9 +80,11 @@ services: REDIS_URL: 'redis://redis' PREVIEW_SERVICE_USE_PRIVATE_OBJECTS_SERVER_URL: 'true' PREVIEW_SERVICE_REDIS_URL: 'redis://redis' + FILEIMPORT_SERVICE_USE_PRIVATE_OBJECTS_SERVER_URL: 'true' FILEIMPORT_SERVICE_REDIS_URL: 'redis://redis' S3_ENDPOINT: 'http://minio:9000' + S3_PUBLIC_ENDPOINT: 'http://127.0.0.1:9000' S3_ACCESS_KEY: 'minioadmin' S3_SECRET_KEY: 'minioadmin' S3_BUCKET: 'speckle-server' @@ -92,6 +95,8 @@ services: FRONTEND_ORIGIN: 'http://127.0.0.1' ONBOARDING_STREAM_URL: 'https://latest.speckle.systems/projects/843d07eb10' + + FF_LARGE_FILE_IMPORTS_ENABLED: 'true' depends_on: [] # - minio diff --git a/packages/fileimport-service/.env.example b/packages/fileimport-service/.env.example index d162e5f97..0f9bb339f 100644 --- a/packages/fileimport-service/.env.example +++ b/packages/fileimport-service/.env.example @@ -4,7 +4,8 @@ POSTGRES_MAX_CONNECTIONS_FILE_IMPORT_SERVICE='1' POSTGRES_CONNECTION_ACQUIRE_TIMEOUT_MILLIS='16000' POSTGRES_CONNECTION_CREATE_TIMEOUT_MILLIS='5000' FF_WORKSPACES_MULTI_REGION_ENABLED=false -USE_LEGACY_IFC_IMPORTER=true +FF_LEGACY_IFC_IMPORTER_ENABLED=true +FF_EXPERIMENTAL_IFC_IMPORTER_ENABLED=false # IFC_DOTNET_DLL_PATH='packages/fileimport-service/src/ifc-dotnet/ifc-converter.dll' ########################################################## diff --git a/packages/fileimport-service/Dockerfile b/packages/fileimport-service/Dockerfile index 47bce1ba4..a62a15c72 100644 --- a/packages/fileimport-service/Dockerfile +++ b/packages/fileimport-service/Dockerfile @@ -12,6 +12,7 @@ WORKDIR /speckle-server # configure tini ARG TINI_VERSION=v0.19.0 +ARG SPECKLE_IFC_VERSION=0.2.0 # hadolint ignore=DL3008 RUN apt-get update -y \ @@ -20,6 +21,10 @@ RUN apt-get update -y \ ca-certificates=20240203 \ curl=8.5.0-2ubuntu10.6 \ gosu=1.17-1ubuntu0.24.04.3 \ + && curl -L -o speckleifc.tar.gz https://github.com/specklesystems/speckleifc/archive/refs/tags/v${SPECKLE_IFC_VERSION}.tar.gz \ + && mkdir speckleifc \ + && tar --strip-components=1 -C speckleifc -xzf speckleifc.tar.gz speckleifc-${SPECKLE_IFC_VERSION} \ + && rm speckleifc.tar.gz \ && curl -fsSL https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini -o /usr/bin/tini \ && chmod +x /usr/bin/tini \ && curl -fsSL https://deb.nodesource.com/setup_22.x -o nodesource_setup.sh \ diff --git a/packages/fileimport-service/requirements.txt b/packages/fileimport-service/requirements.txt index d5abd71fb..e3d3478b6 100644 --- a/packages/fileimport-service/requirements.txt +++ b/packages/fileimport-service/requirements.txt @@ -3,3 +3,4 @@ specklepy==3.0.1 structlog==23.3.0 numpy==1.26.4 # not directly required, pinned to avoid a vulnerability in <1.22.2 python-util==1.2.1 # not directly required, peer dependency of numpy-stl +ifcopenshell==0.8.2 # required for speckleifc diff --git a/packages/fileimport-service/src/controller/daemon.ts b/packages/fileimport-service/src/controller/daemon.ts index bc357cbb6..f14a5f612 100644 --- a/packages/fileimport-service/src/controller/daemon.ts +++ b/packages/fileimport-service/src/controller/daemon.ts @@ -5,26 +5,24 @@ import { metricOperationErrors } from '@/controller/prometheusMetrics.js' import { DbClient, getDbClients } from '@/clients/knex.js' - import { downloadFile } from '@/controller/filesApi.js' import fs from 'fs' - import { ServerAPI } from '@/controller/api.js' import { downloadDependencies } from '@/controller/objDependencies.js' import { logger } from '@/observability/logging.js' import { Nullable, Scopes, wait, TIME_MS } from '@speckle/shared' import { Knex } from 'knex' -import { - getIfcDllPath, - isProdEnv, - useLegacyIfcImporter -} from '@/controller/helpers/env.js' +import { getIfcDllPath, isProdEnv } from '@/controller/helpers/env.js' import { isErrorOutput, isSuccessOutput } from '@/common/output.js' import { runProcessWithTimeout } from '@/common/processHandling.js' import { getConnectionSettings, obfuscateConnectionString } from '@speckle/shared/environment/db' +import { getFeatureFlags } from '@speckle/shared/environment' + +const { FF_LEGACY_IFC_IMPORTER_ENABLED, FF_EXPERIMENTAL_IFC_IMPORTER_ENABLED } = + getFeatureFlags() const HEALTHCHECK_FILE_PATH = '/tmp/last_successful_query' @@ -183,7 +181,7 @@ async function doTask( if (info.fileType.toLowerCase() === 'ifc') { if ( info.fileName.toLowerCase().endsWith('.legacyimporter.ifc') || - useLegacyIfcImporter() + FF_LEGACY_IFC_IMPORTER_ENABLED ) { await runProcessWithTimeout( taskLogger, @@ -208,7 +206,10 @@ async function doTask( TIME_LIMIT, TMP_RESULTS_PATH ) - } else { + } else if ( + info.fileName.toLowerCase().endsWith('.dotnetimporter.ifc') || + !FF_EXPERIMENTAL_IFC_IMPORTER_ENABLED + ) { await runProcessWithTimeout( taskLogger, process.env['DOTNET_BINARY_PATH'] || 'dotnet', @@ -228,6 +229,27 @@ async function doTask( TIME_LIMIT, TMP_RESULTS_PATH ) + } else { + await runProcessWithTimeout( + taskLogger, + process.env['PYTHON_BINARY_PATH'] || 'python3', + [ + '-m', + 'speckleifc', + TMP_FILE_PATH, + TMP_RESULTS_PATH, + info.streamId, + `File upload: ${info.fileName}`, + existingBranch?.id || '' + ], + { + USER_TOKEN: tempUserToken, + //speckleifc is not installed to sys (e.g. via pip), so we need to point it to the directory explicitly + PYTHONPATH: '/speckle-server/speckleifc/src/' + }, + TIME_LIMIT, + TMP_RESULTS_PATH + ) } } else if (info.fileType.toLowerCase() === 'stl') { await runProcessWithTimeout( diff --git a/packages/fileimport-service/src/controller/helpers/env.ts b/packages/fileimport-service/src/controller/helpers/env.ts index 84b224267..07819fa96 100644 --- a/packages/fileimport-service/src/controller/helpers/env.ts +++ b/packages/fileimport-service/src/controller/helpers/env.ts @@ -16,10 +16,6 @@ export function isProdEnv() { export const isDevOrTestEnv = () => isDevEnv() || isTestEnv() -export const useLegacyIfcImporter = () => { - return ['true', '1'].includes(process.env.USE_LEGACY_IFC_IMPORTER || 'false') -} - export const getPackageRootDirPath = () => { const __filename = url.fileURLToPath(import.meta.url) const __dirname = path.dirname(__filename) diff --git a/packages/fileimport-service/src/nextGen/jobProcessor.ts b/packages/fileimport-service/src/nextGen/jobProcessor.ts index 289e23f70..b0262d853 100644 --- a/packages/fileimport-service/src/nextGen/jobProcessor.ts +++ b/packages/fileimport-service/src/nextGen/jobProcessor.ts @@ -11,6 +11,9 @@ import { DOTNET_BINARY_PATH, RHINO_IMPORTER_PATH } from './config.js' import { getIfcDllPath } from '@/controller/helpers/env.js' import { z } from 'zod' import { TIME_MS } from '@speckle/shared' +import { getFeatureFlags } from '@speckle/shared/environment' + +const { FF_EXPERIMENTAL_IFC_IMPORTER_ENABLED } = getFeatureFlags() const jobSuccess = z.object({ success: z.literal(true), @@ -120,26 +123,55 @@ export const jobProcessor = async ({ switch (fileType) { case 'ifc': parserUsed = 'ifc' - await runProcessWithTimeout( - taskLogger, - DOTNET_BINARY_PATH, - [ - getIfcDllPath(), - sourceFilePath, - resultsPath, - job.projectId, - `File upload: ${job.fileName}`, - job.modelId, - 'bogus', - 'regionName' - ], - { - SPECKLE_SERVER_URL: job.serverUrl, - USER_TOKEN: job.token - }, - Math.min(timeout, job.timeOutSeconds * TIME_MS.second), - resultsPath - ) + const useDotnetIfcImporter = + job.fileName.toLowerCase().endsWith('.dotnetimporter.ifc') || + !FF_EXPERIMENTAL_IFC_IMPORTER_ENABLED + + if (useDotnetIfcImporter) { + await runProcessWithTimeout( + taskLogger, + DOTNET_BINARY_PATH, + [ + getIfcDllPath(), + sourceFilePath, + resultsPath, + job.projectId, + `File upload: ${job.fileName}`, + job.modelId, + 'bogus', + 'regionName' + ], + { + SPECKLE_SERVER_URL: job.serverUrl, + USER_TOKEN: job.token + }, + Math.min(timeout, job.timeOutSeconds * TIME_MS.second), + resultsPath + ) + } else { + await runProcessWithTimeout( + taskLogger, + process.env['PYTHON_BINARY_PATH'] || 'python3', + [ + '-m', + 'speckleifc', + sourceFilePath, + resultsPath, + job.projectId, + `File upload: ${job.fileName}`, + job.modelId + ], + { + USER_TOKEN: job.token, + SPECKLE_SERVER_URL: job.serverUrl, + //speckleifc is not installed to sys (e.g. via pip), so we need to point it to the directory explicitly + PYTHONPATH: '/speckle-server/speckleifc/src/' + }, + Math.min(timeout, job.timeOutSeconds * TIME_MS.second), + resultsPath + ) + } + break case 'stl': case 'obj': diff --git a/packages/frontend-2/components/connectors/Card.vue b/packages/frontend-2/components/connectors/Card.vue index bb8dd0d55..eba8313b3 100644 --- a/packages/frontend-2/components/connectors/Card.vue +++ b/packages/frontend-2/components/connectors/Card.vue @@ -1,8 +1,6 @@