From c4778bfa4251003b624b9f82ddaa90304b9b1e5b Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Fri, 4 Jul 2025 17:01:25 +0100 Subject: [PATCH 01/18] feat(file-import): experimental ifc openshell importer (#5028) * first pass * second pass * undo local changes * Refactors env vars to feature flags, adds helm chart values * fix documentation of feature flag * fix typo * adjusted speckleifc pinned version * renamed enableLegacyIfcImporter helm ff to legacyIfcImporterEnabled * Added experimental ifcimporter to next-gen file importer * Fix next-gen import service redis url fallbakcs * add missing server url envvar --------- Co-authored-by: Iain Sproat <68657+iainsproat@users.noreply.github.com> --- packages/fileimport-service/.env.example | 3 +- packages/fileimport-service/Dockerfile | 5 ++ packages/fileimport-service/requirements.txt | 1 + .../src/controller/daemon.ts | 40 ++++++++--- .../src/controller/helpers/env.ts | 4 -- .../src/nextGen/jobProcessor.ts | 72 +++++++++++++------ .../modules/fileuploads/queues/fileimports.ts | 18 +++-- packages/shared/src/environment/index.ts | 14 ++++ .../fileimport_service/deployment.yml | 8 +++ utils/helm/speckle-server/values.schema.json | 10 +++ utils/helm/speckle-server/values.yaml | 4 ++ 11 files changed, 139 insertions(+), 40 deletions(-) diff --git a/packages/fileimport-service/.env.example b/packages/fileimport-service/.env.example index d162e5f97..0f9bb339f 100644 --- a/packages/fileimport-service/.env.example +++ b/packages/fileimport-service/.env.example @@ -4,7 +4,8 @@ POSTGRES_MAX_CONNECTIONS_FILE_IMPORT_SERVICE='1' POSTGRES_CONNECTION_ACQUIRE_TIMEOUT_MILLIS='16000' POSTGRES_CONNECTION_CREATE_TIMEOUT_MILLIS='5000' FF_WORKSPACES_MULTI_REGION_ENABLED=false -USE_LEGACY_IFC_IMPORTER=true +FF_LEGACY_IFC_IMPORTER_ENABLED=true +FF_EXPERIMENTAL_IFC_IMPORTER_ENABLED=false # IFC_DOTNET_DLL_PATH='packages/fileimport-service/src/ifc-dotnet/ifc-converter.dll' ########################################################## diff --git a/packages/fileimport-service/Dockerfile b/packages/fileimport-service/Dockerfile index 47bce1ba4..fdad3f992 100644 --- a/packages/fileimport-service/Dockerfile +++ b/packages/fileimport-service/Dockerfile @@ -12,6 +12,7 @@ WORKDIR /speckle-server # configure tini ARG TINI_VERSION=v0.19.0 +ARG SPECKLE_IFC_VERSION=0.1.2 # hadolint ignore=DL3008 RUN apt-get update -y \ @@ -20,6 +21,10 @@ RUN apt-get update -y \ ca-certificates=20240203 \ curl=8.5.0-2ubuntu10.6 \ gosu=1.17-1ubuntu0.24.04.3 \ + && curl -L -o speckleifc.tar.gz https://github.com/specklesystems/speckleifc/archive/refs/tags/v${SPECKLE_IFC_VERSION}.tar.gz \ + && mkdir speckleifc \ + && tar --strip-components=1 -C speckleifc -xzf speckleifc.tar.gz speckleifc-${SPECKLE_IFC_VERSION} \ + && rm speckleifc.tar.gz \ && curl -fsSL https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini -o /usr/bin/tini \ && chmod +x /usr/bin/tini \ && curl -fsSL https://deb.nodesource.com/setup_22.x -o nodesource_setup.sh \ diff --git a/packages/fileimport-service/requirements.txt b/packages/fileimport-service/requirements.txt index d5abd71fb..e3d3478b6 100644 --- a/packages/fileimport-service/requirements.txt +++ b/packages/fileimport-service/requirements.txt @@ -3,3 +3,4 @@ specklepy==3.0.1 structlog==23.3.0 numpy==1.26.4 # not directly required, pinned to avoid a vulnerability in <1.22.2 python-util==1.2.1 # not directly required, peer dependency of numpy-stl +ifcopenshell==0.8.2 # required for speckleifc diff --git a/packages/fileimport-service/src/controller/daemon.ts b/packages/fileimport-service/src/controller/daemon.ts index bc357cbb6..f14a5f612 100644 --- a/packages/fileimport-service/src/controller/daemon.ts +++ b/packages/fileimport-service/src/controller/daemon.ts @@ -5,26 +5,24 @@ import { metricOperationErrors } from '@/controller/prometheusMetrics.js' import { DbClient, getDbClients } from '@/clients/knex.js' - import { downloadFile } from '@/controller/filesApi.js' import fs from 'fs' - import { ServerAPI } from '@/controller/api.js' import { downloadDependencies } from '@/controller/objDependencies.js' import { logger } from '@/observability/logging.js' import { Nullable, Scopes, wait, TIME_MS } from '@speckle/shared' import { Knex } from 'knex' -import { - getIfcDllPath, - isProdEnv, - useLegacyIfcImporter -} from '@/controller/helpers/env.js' +import { getIfcDllPath, isProdEnv } from '@/controller/helpers/env.js' import { isErrorOutput, isSuccessOutput } from '@/common/output.js' import { runProcessWithTimeout } from '@/common/processHandling.js' import { getConnectionSettings, obfuscateConnectionString } from '@speckle/shared/environment/db' +import { getFeatureFlags } from '@speckle/shared/environment' + +const { FF_LEGACY_IFC_IMPORTER_ENABLED, FF_EXPERIMENTAL_IFC_IMPORTER_ENABLED } = + getFeatureFlags() const HEALTHCHECK_FILE_PATH = '/tmp/last_successful_query' @@ -183,7 +181,7 @@ async function doTask( if (info.fileType.toLowerCase() === 'ifc') { if ( info.fileName.toLowerCase().endsWith('.legacyimporter.ifc') || - useLegacyIfcImporter() + FF_LEGACY_IFC_IMPORTER_ENABLED ) { await runProcessWithTimeout( taskLogger, @@ -208,7 +206,10 @@ async function doTask( TIME_LIMIT, TMP_RESULTS_PATH ) - } else { + } else if ( + info.fileName.toLowerCase().endsWith('.dotnetimporter.ifc') || + !FF_EXPERIMENTAL_IFC_IMPORTER_ENABLED + ) { await runProcessWithTimeout( taskLogger, process.env['DOTNET_BINARY_PATH'] || 'dotnet', @@ -228,6 +229,27 @@ async function doTask( TIME_LIMIT, TMP_RESULTS_PATH ) + } else { + await runProcessWithTimeout( + taskLogger, + process.env['PYTHON_BINARY_PATH'] || 'python3', + [ + '-m', + 'speckleifc', + TMP_FILE_PATH, + TMP_RESULTS_PATH, + info.streamId, + `File upload: ${info.fileName}`, + existingBranch?.id || '' + ], + { + USER_TOKEN: tempUserToken, + //speckleifc is not installed to sys (e.g. via pip), so we need to point it to the directory explicitly + PYTHONPATH: '/speckle-server/speckleifc/src/' + }, + TIME_LIMIT, + TMP_RESULTS_PATH + ) } } else if (info.fileType.toLowerCase() === 'stl') { await runProcessWithTimeout( diff --git a/packages/fileimport-service/src/controller/helpers/env.ts b/packages/fileimport-service/src/controller/helpers/env.ts index 84b224267..07819fa96 100644 --- a/packages/fileimport-service/src/controller/helpers/env.ts +++ b/packages/fileimport-service/src/controller/helpers/env.ts @@ -16,10 +16,6 @@ export function isProdEnv() { export const isDevOrTestEnv = () => isDevEnv() || isTestEnv() -export const useLegacyIfcImporter = () => { - return ['true', '1'].includes(process.env.USE_LEGACY_IFC_IMPORTER || 'false') -} - export const getPackageRootDirPath = () => { const __filename = url.fileURLToPath(import.meta.url) const __dirname = path.dirname(__filename) diff --git a/packages/fileimport-service/src/nextGen/jobProcessor.ts b/packages/fileimport-service/src/nextGen/jobProcessor.ts index 289e23f70..b0262d853 100644 --- a/packages/fileimport-service/src/nextGen/jobProcessor.ts +++ b/packages/fileimport-service/src/nextGen/jobProcessor.ts @@ -11,6 +11,9 @@ import { DOTNET_BINARY_PATH, RHINO_IMPORTER_PATH } from './config.js' import { getIfcDllPath } from '@/controller/helpers/env.js' import { z } from 'zod' import { TIME_MS } from '@speckle/shared' +import { getFeatureFlags } from '@speckle/shared/environment' + +const { FF_EXPERIMENTAL_IFC_IMPORTER_ENABLED } = getFeatureFlags() const jobSuccess = z.object({ success: z.literal(true), @@ -120,26 +123,55 @@ export const jobProcessor = async ({ switch (fileType) { case 'ifc': parserUsed = 'ifc' - await runProcessWithTimeout( - taskLogger, - DOTNET_BINARY_PATH, - [ - getIfcDllPath(), - sourceFilePath, - resultsPath, - job.projectId, - `File upload: ${job.fileName}`, - job.modelId, - 'bogus', - 'regionName' - ], - { - SPECKLE_SERVER_URL: job.serverUrl, - USER_TOKEN: job.token - }, - Math.min(timeout, job.timeOutSeconds * TIME_MS.second), - resultsPath - ) + const useDotnetIfcImporter = + job.fileName.toLowerCase().endsWith('.dotnetimporter.ifc') || + !FF_EXPERIMENTAL_IFC_IMPORTER_ENABLED + + if (useDotnetIfcImporter) { + await runProcessWithTimeout( + taskLogger, + DOTNET_BINARY_PATH, + [ + getIfcDllPath(), + sourceFilePath, + resultsPath, + job.projectId, + `File upload: ${job.fileName}`, + job.modelId, + 'bogus', + 'regionName' + ], + { + SPECKLE_SERVER_URL: job.serverUrl, + USER_TOKEN: job.token + }, + Math.min(timeout, job.timeOutSeconds * TIME_MS.second), + resultsPath + ) + } else { + await runProcessWithTimeout( + taskLogger, + process.env['PYTHON_BINARY_PATH'] || 'python3', + [ + '-m', + 'speckleifc', + sourceFilePath, + resultsPath, + job.projectId, + `File upload: ${job.fileName}`, + job.modelId + ], + { + USER_TOKEN: job.token, + SPECKLE_SERVER_URL: job.serverUrl, + //speckleifc is not installed to sys (e.g. via pip), so we need to point it to the directory explicitly + PYTHONPATH: '/speckle-server/speckleifc/src/' + }, + Math.min(timeout, job.timeOutSeconds * TIME_MS.second), + resultsPath + ) + } + break case 'stl': case 'obj': diff --git a/packages/server/modules/fileuploads/queues/fileimports.ts b/packages/server/modules/fileuploads/queues/fileimports.ts index b7ca320a3..ea6a35916 100644 --- a/packages/server/modules/fileuploads/queues/fileimports.ts +++ b/packages/server/modules/fileuploads/queues/fileimports.ts @@ -87,21 +87,27 @@ const initializeQueue = async (params: { return fileImportQueue } -export const initializeRhinoQueue = async () => - initializeQueue({ +export const initializeRhinoQueue = async () => { + const rhinoImportServiceRedisUrl = getFileImportServiceRhinoParserRedisUrl() + + return initializeQueue({ label: 'rhino', queueName: FILEIMPORT_SERVICE_RHINO_QUEUE_NAME, - redisUrl: getFileImportServiceRhinoParserRedisUrl() ?? getRedisUrl(), + redisUrl: rhinoImportServiceRedisUrl ? rhinoImportServiceRedisUrl : getRedisUrl(), supportedFileTypes: ['obj', 'stl', 'skp'] }) +} -export const initializeIfcQueue = async () => - initializeQueue({ +export const initializeIfcQueue = async () => { + const ifcImportServiceRedisUrl = getFileImportServiceIFCParserRedisUrl() + + return initializeQueue({ label: 'ifc', queueName: FILEIMPORT_SERVICE_IFC_QUEUE_NAME, - redisUrl: getFileImportServiceIFCParserRedisUrl() ?? getRedisUrl(), + redisUrl: ifcImportServiceRedisUrl ? ifcImportServiceRedisUrl : getRedisUrl(), supportedFileTypes: ['ifc'] }) +} export const shutdownQueues = async (params: { logger: Logger }) => { for (const queue of fileImportQueues) { diff --git a/packages/shared/src/environment/index.ts b/packages/shared/src/environment/index.ts index 900a38f85..34a766d67 100644 --- a/packages/shared/src/environment/index.ts +++ b/packages/shared/src/environment/index.ts @@ -109,6 +109,18 @@ export const parseFeatureFlags = ( description: 'Enables the new file importer to handle large files via pre-signed URLs.', defaults: { _: false } + }, + FF_LEGACY_IFC_IMPORTER_ENABLED: { + schema: z.boolean(), + description: + 'Enables the legacy javascript based webIFC file importer (pre-2025). Even if disabled this importer can be accessed by appending `.legacyimporter.ifc` to the uploaded file name. This is deprecated and will be removed in the future.', + defaults: { _: false } + }, + FF_EXPERIMENTAL_IFC_IMPORTER_ENABLED: { + schema: z.boolean(), + description: + 'Enables the IFC file importer based on IFCOpenShell (as of July 2025). Even if enabled, the previous webIFC & .Net importer can be accessed by appending `.dotnetimporter.ifc` to the uploaded file name.', + defaults: { _: false } } }) @@ -142,6 +154,8 @@ export type FeatureFlags = { FF_PERSONAL_PROJECTS_LIMITS_ENABLED: boolean FF_NEXT_GEN_FILE_IMPORTER_ENABLED: boolean FF_LARGE_FILE_IMPORTS_ENABLED: boolean + FF_LEGACY_IFC_IMPORTER_ENABLED: boolean + FF_EXPERIMENTAL_IFC_IMPORTER_ENABLED: boolean } export function getFeatureFlags(): FeatureFlags { diff --git a/utils/helm/speckle-server/templates/fileimport_service/deployment.yml b/utils/helm/speckle-server/templates/fileimport_service/deployment.yml index b280d7553..5aeb5ce48 100644 --- a/utils/helm/speckle-server/templates/fileimport_service/deployment.yml +++ b/utils/helm/speckle-server/templates/fileimport_service/deployment.yml @@ -151,6 +151,14 @@ spec: - name: FF_NEXT_GEN_FILE_IMPORTER_ENABLED value: {{ .Values.featureFlags.nextGenFileImporterEnabled | quote }} {{- end }} + {{- if .Values.featureFlags.experimentalIfcImporterEnabled }} + - name: FF_EXPERIMENTAL_IFC_IMPORTER_ENABLED + value: {{ .Values.featureFlags.experimentalIfcImporterEnabled | quote }} + {{- end }} + {{- if .Values.featureFlags.legacyIfcImporterEnabled }} + - name: FF_LEGACY_IFC_IMPORTER_ENABLED + value: {{ .Values.featureFlags.legacyIfcImporterEnabled | quote }} + {{- end }} {{- with .Values.fileimport_service.additionalEnvVars }} {{- toYaml . | nindent 10}} {{- end }} diff --git a/utils/helm/speckle-server/values.schema.json b/utils/helm/speckle-server/values.schema.json index 16ae24923..218fb6cd0 100644 --- a/utils/helm/speckle-server/values.schema.json +++ b/utils/helm/speckle-server/values.schema.json @@ -109,6 +109,16 @@ "type": "boolean", "description": "Enables the ability to upload large files to Speckle", "default": false + }, + "experimentalIfcImporterEnabled": { + "type": "boolean", + "description": "Enables the ability to parse IFC files using the experimental IFC importer", + "default": false + }, + "legacyIfcImporterEnabled": { + "type": "boolean", + "description": "Enables the ability to parse IFC files using the legacy IFC importer.", + "default": false } } }, diff --git a/utils/helm/speckle-server/values.yaml b/utils/helm/speckle-server/values.yaml index f07725c66..9d532d233 100644 --- a/utils/helm/speckle-server/values.yaml +++ b/utils/helm/speckle-server/values.yaml @@ -65,6 +65,10 @@ featureFlags: nextGenFileImporterEnabled: false ## @param featureFlags.largeFileUploadsEnabled Enables the ability to upload large files to Speckle largeFileUploadsEnabled: false + ## @param featureFlags.experimentalIfcImporterEnabled Enables the ability to parse IFC files using the experimental IFC importer + experimentalIfcImporterEnabled: false + ## @param featureFlags.legacyIfcImporterEnabled Enables the ability to parse IFC files using the legacy IFC importer. + legacyIfcImporterEnabled: false analytics: ## @param analytics.enabled Enable or disable analytics From 556c2791b3eb27930f246e09cd4ab800eab0cdcc Mon Sep 17 00:00:00 2001 From: Iain Sproat <68657+iainsproat@users.noreply.github.com> Date: Mon, 7 Jul 2025 11:00:55 +0100 Subject: [PATCH 02/18] feat(file imports): large file uploads now work on docker compose (#5037) --- docker-compose-speckle.yml | 5 +++ .../blobstorage/clients/objectStorage.ts | 40 +++++++++++++++++++ .../server/modules/blobstorage/rest/router.ts | 6 ++- .../modules/blobstorage/services/streams.ts | 6 +-- .../blobstorage.integration.spec.ts | 9 +++-- .../integration/presigned.integration.spec.ts | 8 ++-- .../graph/resolvers/fileUploads.ts | 10 +++-- .../integration/presigned.integration.spec.ts | 6 +-- .../modules/gendo/graph/resolvers/index.ts | 4 +- packages/server/modules/gendo/rest/index.ts | 4 +- .../modules/multiregion/domain/operations.ts | 4 +- .../modules/multiregion/services/queue.ts | 6 ++- .../tests/e2e/serverAdmin.graph.spec.ts | 4 +- .../multiregion/utils/blobStorageSelector.ts | 40 ++++++++++++++++--- .../modules/shared/helpers/envHelper.ts | 8 ++++ .../server/test/speckle-helpers/blobHelper.ts | 2 +- packages/shared/src/environment/db.ts | 14 ++++++- 17 files changed, 143 insertions(+), 33 deletions(-) diff --git a/docker-compose-speckle.yml b/docker-compose-speckle.yml index 18d16e931..e8fb2b8c5 100644 --- a/docker-compose-speckle.yml +++ b/docker-compose-speckle.yml @@ -32,6 +32,7 @@ services: NUXT_PUBLIC_BACKEND_API_ORIGIN: 'http://speckle-server:3000' NUXT_PUBLIC_LOG_LEVEL: 'warn' NUXT_REDIS_URL: 'redis://redis' + NUXT_PUBLIC_FF_LARGE_FILE_IMPORTS_ENABLED: 'true' LOG_LEVEL: 'info' LOG_PRETTY: 'true' depends_on: @@ -79,9 +80,11 @@ services: REDIS_URL: 'redis://redis' PREVIEW_SERVICE_USE_PRIVATE_OBJECTS_SERVER_URL: 'true' PREVIEW_SERVICE_REDIS_URL: 'redis://redis' + FILEIMPORT_SERVICE_USE_PRIVATE_OBJECTS_SERVER_URL: 'true' FILEIMPORT_SERVICE_REDIS_URL: 'redis://redis' S3_ENDPOINT: 'http://minio:9000' + S3_PUBLIC_ENDPOINT: 'http://127.0.0.1:9000' S3_ACCESS_KEY: 'minioadmin' S3_SECRET_KEY: 'minioadmin' S3_BUCKET: 'speckle-server' @@ -92,6 +95,8 @@ services: FRONTEND_ORIGIN: 'http://127.0.0.1' ONBOARDING_STREAM_URL: 'https://latest.speckle.systems/projects/843d07eb10' + + FF_LARGE_FILE_IMPORTS_ENABLED: 'true' depends_on: [] # - minio diff --git a/packages/server/modules/blobstorage/clients/objectStorage.ts b/packages/server/modules/blobstorage/clients/objectStorage.ts index 615b0d01f..119034cd0 100644 --- a/packages/server/modules/blobstorage/clients/objectStorage.ts +++ b/packages/server/modules/blobstorage/clients/objectStorage.ts @@ -2,6 +2,7 @@ import { getS3AccessKey, getS3BucketName, getS3Endpoint, + getS3PublicEndpoint, getS3Region, getS3SecretKey } from '@/modules/shared/helpers/envHelper' @@ -51,9 +52,17 @@ export const getObjectStorage = (params: GetObjectStorageParams): ObjectStorage } let mainObjectStorage: Optional = undefined +let publicMainObjectStorage: Optional = undefined /** * Get main object storage client + * + * This is used for connecting the server to the S3 host. Where the S3 host is + * on the same private network as the server (e.g. in a Docker network), + * the S3_ENDPOINT can use the private IP or DNS name of the S3 host. + * + * S3_PUBLIC_ENDPOINT can be used to connect to the S3 host via the + * public internet (or localhost network if running locally or testing). */ export const getMainObjectStorage = (): ObjectStorage => { if (mainObjectStorage) return mainObjectStorage @@ -72,6 +81,37 @@ export const getMainObjectStorage = (): ObjectStorage => { return mainObjectStorage } +/** + * (Optional) Used to connect to the S3 host via the public endpoint. + * This is useful for clients that need to access the S3 bucket directly, e.g + * during testing or when the S3 host is not on the same private network as the server. + * + * If `S3_PUBLIC_ENDPOINT` is not set, it will return the same object storage + * as `getMainObjectStorage`. + */ +export const getPublicMainObjectStorage = (): ObjectStorage => { + if (publicMainObjectStorage) return publicMainObjectStorage + + const endpoint = getS3PublicEndpoint() + if (!endpoint) { + // If no public endpoint is set, return the main object storage + return getMainObjectStorage() + } + + const mainParams: GetObjectStorageParams = { + credentials: { + accessKeyId: getS3AccessKey(), + secretAccessKey: getS3SecretKey() + }, + endpoint, + region: getS3Region(), + bucket: getS3BucketName() + } + + publicMainObjectStorage = getObjectStorage(mainParams) + return publicMainObjectStorage +} + export const getSignedUrlFactory = (deps: { objectStorage: ObjectStorage }): GetSignedUrl => { diff --git a/packages/server/modules/blobstorage/rest/router.ts b/packages/server/modules/blobstorage/rest/router.ts index 594ffe6c8..de1214d99 100644 --- a/packages/server/modules/blobstorage/rest/router.ts +++ b/packages/server/modules/blobstorage/rest/router.ts @@ -124,7 +124,9 @@ export const blobStorageRouterFactory = (): Router => { const getBlobMetadata = getBlobMetadataFactory({ db: projectDb }) const getFileStream = getFileStreamFactory({ getBlobMetadata }) - const getObjectStream = getObjectStreamFactory({ storage: projectStorage }) + const getObjectStream = getObjectStreamFactory({ + storage: projectStorage.private + }) const { fileName } = await getBlobMetadata({ streamId: req.params.streamId, @@ -160,7 +162,7 @@ export const blobStorageRouterFactory = (): Router => { ]) const getBlobMetadata = getBlobMetadataFactory({ db: projectDb }) - const deleteObject = deleteObjectFactory({ storage: projectStorage }) + const deleteObject = deleteObjectFactory({ storage: projectStorage.private }) const deleteBlob = fullyDeleteBlobFactory({ getBlobMetadata, deleteBlob: deleteBlobFactory({ db: projectDb }), diff --git a/packages/server/modules/blobstorage/services/streams.ts b/packages/server/modules/blobstorage/services/streams.ts index ed87e0acb..6c886f215 100644 --- a/packages/server/modules/blobstorage/services/streams.ts +++ b/packages/server/modules/blobstorage/services/streams.ts @@ -45,7 +45,7 @@ export const processNewFileStreamFactory = (): NewFileStreamProcessor => { getProjectObjectStorage({ projectId: streamId }) ]) - const storeFileStream = storeFileStreamFactory({ storage: projectStorage }) + const storeFileStream = storeFileStreamFactory({ storage: projectStorage.private }) const updateBlob = updateBlobFactory({ db: projectDb }) const getBlobMetadata = getBlobMetadataFactory({ db: projectDb }) @@ -66,9 +66,9 @@ export const processNewFileStreamFactory = (): NewFileStreamProcessor => { }) const getObjectAttributes = getObjectAttributesFactory({ - storage: projectStorage + storage: projectStorage.private }) - const deleteObject = deleteObjectFactory({ storage: projectStorage }) + const deleteObject = deleteObjectFactory({ storage: projectStorage.private }) busboy.on( 'file', diff --git a/packages/server/modules/blobstorage/tests/integration/blobstorage.integration.spec.ts b/packages/server/modules/blobstorage/tests/integration/blobstorage.integration.spec.ts index dc7e04469..380e54238 100644 --- a/packages/server/modules/blobstorage/tests/integration/blobstorage.integration.spec.ts +++ b/packages/server/modules/blobstorage/tests/integration/blobstorage.integration.spec.ts @@ -23,7 +23,10 @@ import { BasicTestStream, createTestStream } from '@/test/speckle-helpers/stream import cryptoRandomString from 'crypto-random-string' import { BasicTestUser, createTestUser } from '@/test/authHelper' import { storeFileStreamFactory } from '@/modules/blobstorage/repositories/blobs' -import { getMainObjectStorage } from '@/modules/blobstorage/clients/objectStorage' +import { + getMainObjectStorage, + getPublicMainObjectStorage +} from '@/modules/blobstorage/clients/objectStorage' import { expect } from 'chai' import { UploadFileStream } from '@/modules/blobstorage/domain/operations' import { BlobStorageItem } from '@/modules/blobstorage/domain/types' @@ -43,8 +46,8 @@ const buildUploadFileStream = async (params: { streamId: string | null }) => { const storage = streamId ? await getProjectObjectStorage({ projectId: streamId }) - : getMainObjectStorage() - const storeFileStream = storeFileStreamFactory({ storage }) + : { private: getMainObjectStorage(), public: getPublicMainObjectStorage() } + const storeFileStream = storeFileStreamFactory({ storage: storage.public }) const uploadFileStream = uploadFileStreamFactory({ upsertBlob, updateBlob, diff --git a/packages/server/modules/blobstorage/tests/integration/presigned.integration.spec.ts b/packages/server/modules/blobstorage/tests/integration/presigned.integration.spec.ts index f80d22032..ad19720cd 100644 --- a/packages/server/modules/blobstorage/tests/integration/presigned.integration.spec.ts +++ b/packages/server/modules/blobstorage/tests/integration/presigned.integration.spec.ts @@ -52,7 +52,7 @@ const { FF_LARGE_FILE_IMPORTS_ENABLED } = getFeatureFlags() } let projectDb: Knex - let projectStorage: ObjectStorage + let projectStorage: { private: ObjectStorage; public: ObjectStorage } let getBlobMetadata: GetBlobMetadata before(async () => { @@ -77,7 +77,7 @@ const { FF_LARGE_FILE_IMPORTS_ENABLED } = getFeatureFlags() before(() => { SUT = generatePresignedUrlFactory({ getSignedUrl: getSignedUrlFactory({ - objectStorage: projectStorage + objectStorage: projectStorage.public }), upsertBlob: upsertBlobFactory({ db: projectDb @@ -117,7 +117,7 @@ const { FF_LARGE_FILE_IMPORTS_ENABLED } = getFeatureFlags() before(() => { generatePresignedUrl = generatePresignedUrlFactory({ getSignedUrl: getSignedUrlFactory({ - objectStorage: projectStorage + objectStorage: projectStorage.public }), upsertBlob: upsertBlobFactory({ db: projectDb @@ -126,7 +126,7 @@ const { FF_LARGE_FILE_IMPORTS_ENABLED } = getFeatureFlags() SUT = registerCompletedUploadFactory({ getBlob: getBlobFactory({ db: projectDb }), getBlobMetadata: getBlobMetadataFromStorage({ - objectStorage: projectStorage + objectStorage: projectStorage.public }), updateBlob: updateBlobFactory({ db: projectDb diff --git a/packages/server/modules/fileuploads/graph/resolvers/fileUploads.ts b/packages/server/modules/fileuploads/graph/resolvers/fileUploads.ts index 62bd4ba61..484a4b718 100644 --- a/packages/server/modules/fileuploads/graph/resolvers/fileUploads.ts +++ b/packages/server/modules/fileuploads/graph/resolvers/fileUploads.ts @@ -25,7 +25,9 @@ import { } from '@/modules/shared/errors' import { throwIfAuthNotOk } from '@/modules/shared/helpers/errorHelper' import { + fileImportServiceShouldUsePrivateObjectsServerUrl, getFileUploadUrlExpiryMinutes, + getPrivateObjectsServerOrigin, getServerOrigin, isFileUploadsEnabled } from '@/modules/shared/helpers/envHelper' @@ -134,7 +136,7 @@ const fileUploadMutations: Resolvers['FileUploadMutations'] = { const generatePresignedUrl = generatePresignedUrlFactory({ getSignedUrl: getSignedUrlFactory({ - objectStorage: projectStorage + objectStorage: projectStorage.public }), upsertBlob: upsertBlobFactory({ db: projectDb @@ -189,7 +191,9 @@ const fileUploadMutations: Resolvers['FileUploadMutations'] = { ]) const pushJobToFileImporter = pushJobToFileImporterFactory({ - getServerOrigin, + getServerOrigin: fileImportServiceShouldUsePrivateObjectsServerUrl() + ? getPrivateObjectsServerOrigin + : getServerOrigin, createAppToken: createAppTokenFactory({ storeApiToken: storeApiTokenFactory({ db }), storeTokenScopes: storeTokenScopesFactory({ db }), @@ -221,7 +225,7 @@ const fileUploadMutations: Resolvers['FileUploadMutations'] = { db: projectDb }), getBlobMetadata: getBlobMetadataFromStorage({ - objectStorage: projectStorage + objectStorage: projectStorage.private }) }), insertNewUploadAndNotify: FF_NEXT_GEN_FILE_IMPORTER_ENABLED diff --git a/packages/server/modules/fileuploads/tests/integration/presigned.integration.spec.ts b/packages/server/modules/fileuploads/tests/integration/presigned.integration.spec.ts index 5feb9a024..1a0977cfe 100644 --- a/packages/server/modules/fileuploads/tests/integration/presigned.integration.spec.ts +++ b/packages/server/modules/fileuploads/tests/integration/presigned.integration.spec.ts @@ -75,7 +75,7 @@ const { FF_LARGE_FILE_IMPORTS_ENABLED, FF_NEXT_GEN_FILE_IMPORTER_ENABLED } = } let projectDb: Knex - let projectStorage: ObjectStorage + let projectStorage: { private: ObjectStorage; public: ObjectStorage } before(async () => { await beforeEachContext() @@ -112,7 +112,7 @@ const { FF_LARGE_FILE_IMPORTS_ENABLED, FF_NEXT_GEN_FILE_IMPORTER_ENABLED } = before(() => { generatePresignedUrl = generatePresignedUrlFactory({ getSignedUrl: getSignedUrlFactory({ - objectStorage: projectStorage + objectStorage: projectStorage.public }), upsertBlob: upsertBlobFactory({ db: projectDb @@ -152,7 +152,7 @@ const { FF_LARGE_FILE_IMPORTS_ENABLED, FF_NEXT_GEN_FILE_IMPORTER_ENABLED } = registerCompletedUpload: registerCompletedUploadFactory({ getBlob: getBlobFactory({ db: projectDb }), getBlobMetadata: getBlobMetadataFromStorage({ - objectStorage: projectStorage + objectStorage: projectStorage.public }), updateBlob: updateBlobFactory({ db: projectDb diff --git a/packages/server/modules/gendo/graph/resolvers/index.ts b/packages/server/modules/gendo/graph/resolvers/index.ts index 3924b4912..9515a53f8 100644 --- a/packages/server/modules/gendo/graph/resolvers/index.ts +++ b/packages/server/modules/gendo/graph/resolvers/index.ts @@ -119,7 +119,9 @@ export = FF_GENDOAI_MODULE_ENABLED token: getGendoAIKey() }) - const storeFileStream = storeFileStreamFactory({ storage: projectStorage }) + const storeFileStream = storeFileStreamFactory({ + storage: projectStorage.private + }) const createRenderRequest = createRenderRequestFactory({ uploadFileStream: uploadFileStreamFactory({ storeFileStream, diff --git a/packages/server/modules/gendo/rest/index.ts b/packages/server/modules/gendo/rest/index.ts index 20251890a..587d8b272 100644 --- a/packages/server/modules/gendo/rest/index.ts +++ b/packages/server/modules/gendo/rest/index.ts @@ -59,7 +59,9 @@ export default function (app: express.Express) { getProjectObjectStorage({ projectId }) ]) - const storeFileStream = storeFileStreamFactory({ storage: projectStorage }) + const storeFileStream = storeFileStreamFactory({ + storage: projectStorage.private + }) const updateRenderRequest = updateRenderRequestFactory({ getRenderByGenerationId: getRenderByGenerationIdFactory({ db: projectDb }), uploadFileStream: uploadFileStreamFactory({ diff --git a/packages/server/modules/multiregion/domain/operations.ts b/packages/server/modules/multiregion/domain/operations.ts index 7dffb8114..ac79ee302 100644 --- a/packages/server/modules/multiregion/domain/operations.ts +++ b/packages/server/modules/multiregion/domain/operations.ts @@ -64,8 +64,8 @@ export type UpdateAndValidateRegion = (params: { export type GetProjectObjectStorage = (args: { projectId: string -}) => Promise +}) => Promise<{ private: ObjectStorage; public: ObjectStorage }> export type GetRegionObjectStorage = (args: { regionKey: string -}) => Promise +}) => Promise<{ private: ObjectStorage; public: ObjectStorage }> diff --git a/packages/server/modules/multiregion/services/queue.ts b/packages/server/modules/multiregion/services/queue.ts index ca57e3a80..277de80e5 100644 --- a/packages/server/modules/multiregion/services/queue.ts +++ b/packages/server/modules/multiregion/services/queue.ts @@ -165,9 +165,11 @@ export const startQueue = async () => { const { projectId, regionKey } = job.data.payload const sourceDb = await getProjectDbClient({ projectId }) - const sourceObjectStorage = await getProjectObjectStorage({ projectId }) + const sourceObjectStorage = (await getProjectObjectStorage({ projectId })) + .private const targetDb = await getRegionDb({ regionKey }) - const targetObjectStorage = await getRegionObjectStorage({ regionKey }) + const targetObjectStorage = (await getRegionObjectStorage({ regionKey })) + .private // Move project to target region const project = await withTransaction( diff --git a/packages/server/modules/multiregion/tests/e2e/serverAdmin.graph.spec.ts b/packages/server/modules/multiregion/tests/e2e/serverAdmin.graph.spec.ts index 96f56c821..bdce57304 100644 --- a/packages/server/modules/multiregion/tests/e2e/serverAdmin.graph.spec.ts +++ b/packages/server/modules/multiregion/tests/e2e/serverAdmin.graph.spec.ts @@ -74,7 +74,9 @@ isEnabled Promise.resolve() ) MultiRegionBlobStorageSelectorMock.mockFunction('initializeRegion', async () => - Promise.resolve(undefined as unknown as ObjectStorage) + Promise.resolve( + undefined as unknown as { private: ObjectStorage; public: ObjectStorage } + ) ) await beforeEachContext() diff --git a/packages/server/modules/multiregion/utils/blobStorageSelector.ts b/packages/server/modules/multiregion/utils/blobStorageSelector.ts index ddc0d0c9a..29d6c852b 100644 --- a/packages/server/modules/multiregion/utils/blobStorageSelector.ts +++ b/packages/server/modules/multiregion/utils/blobStorageSelector.ts @@ -1,6 +1,7 @@ import { getMainObjectStorage, getObjectStorage, + getPublicMainObjectStorage, ObjectStorage } from '@/modules/blobstorage/clients/objectStorage' import { ensureStorageAccessFactory } from '@/modules/blobstorage/repositories/blobs' @@ -20,7 +21,7 @@ import { Optional } from '@speckle/shared' import { BlobStorageConfig } from '@speckle/shared/environment/db' type RegionStorageClients = { - [regionKey: string]: ObjectStorage + [regionKey: string]: { private: ObjectStorage; public: ObjectStorage } } let initializedClients: Optional = undefined @@ -36,7 +37,8 @@ export const initializeRegion = async (params: { */ config?: BlobStorageConfig }) => { - if (!isMultiRegionBlobStorageEnabled()) return getMainObjectStorage() + if (!isMultiRegionBlobStorageEnabled()) + return { private: getMainObjectStorage(), public: getPublicMainObjectStorage() } const { regionKey } = params let config = params.config @@ -66,10 +68,33 @@ export const initializeRegion = async (params: { // Only add, if clients already initialized if (initializedClients) { - initializedClients[regionKey] = storage + initializedClients[regionKey] = { private: storage, public: storage } } - return storage + if (config.publicEndpoint) { + const publicStorage = getObjectStorage({ + credentials: { + accessKeyId: config.accessKey, + secretAccessKey: config.secretKey + }, + endpoint: config.publicEndpoint, + region: config.s3Region, + bucket: config.bucket + }) + + // ensure it works + const ensure = ensureStorageAccessFactory({ storage: publicStorage }) + await ensure({ createBucketIfNotExists: config.createBucketIfNotExists }) + + // Only add, if clients already initialized + if (initializedClients) { + initializedClients[regionKey] = { private: storage, public: publicStorage } + } + + return { private: storage, public: publicStorage } + } + + return { private: storage, public: storage } } /** @@ -99,7 +124,8 @@ export const getRegisteredRegionClients = async (): Promise { - if (!isMultiRegionBlobStorageEnabled()) return getMainObjectStorage() + if (!isMultiRegionBlobStorageEnabled()) + return { private: getMainObjectStorage(), public: getPublicMainObjectStorage() } const clients = await getRegisteredRegionClients() let storage = clients[regionKey] @@ -123,5 +149,7 @@ export const getProjectObjectStorage: GetProjectObjectStorage = async ({ projectId }) => { const regionKey = await getProjectRegionKey({ projectId }) - return regionKey ? getRegionObjectStorage({ regionKey }) : getMainObjectStorage() + return regionKey + ? getRegionObjectStorage({ regionKey }) + : { private: getMainObjectStorage(), public: getPublicMainObjectStorage() } } diff --git a/packages/server/modules/shared/helpers/envHelper.ts b/packages/server/modules/shared/helpers/envHelper.ts index 4da4b9798..7d4a9d501 100644 --- a/packages/server/modules/shared/helpers/envHelper.ts +++ b/packages/server/modules/shared/helpers/envHelper.ts @@ -125,6 +125,10 @@ export const previewServiceShouldUsePrivateObjectsServerUrl = (): boolean => { return getBooleanFromEnv('PREVIEW_SERVICE_USE_PRIVATE_OBJECTS_SERVER_URL') } +export const fileImportServiceShouldUsePrivateObjectsServerUrl = (): boolean => { + return getBooleanFromEnv('FILEIMPORT_SERVICE_USE_PRIVATE_OBJECTS_SERVER_URL') +} + export const getFileImportServiceRhinoParserRedisUrl = (): string | undefined => { return getStringFromEnv('FILEIMPORT_SERVICE_RHINO_REDIS_URL', { unsafe: true }) } @@ -429,6 +433,10 @@ export function getS3Endpoint() { return getStringFromEnv('S3_ENDPOINT') } +export function getS3PublicEndpoint() { + return getStringFromEnv('S3_PUBLIC_ENDPOINT', { unsafe: true }) +} + export function getS3Region(aDefault: string = 'us-east-1') { return process.env.S3_REGION || aDefault } diff --git a/packages/server/test/speckle-helpers/blobHelper.ts b/packages/server/test/speckle-helpers/blobHelper.ts index 21282952c..17fea6bd2 100644 --- a/packages/server/test/speckle-helpers/blobHelper.ts +++ b/packages/server/test/speckle-helpers/blobHelper.ts @@ -25,7 +25,7 @@ export const createTestBlob = async (params: { userId: string; projectId: string return await uploadFileStreamFactory({ upsertBlob: upsertBlobFactory({ db: projectDb }), updateBlob: updateBlobFactory({ db: projectDb }), - storeFileStream: storeFileStreamFactory({ storage: projectStorage }) + storeFileStream: storeFileStreamFactory({ storage: projectStorage.public }) })( { userId, diff --git a/packages/shared/src/environment/db.ts b/packages/shared/src/environment/db.ts index 7ed22f8dc..5a453b35f 100644 --- a/packages/shared/src/environment/db.ts +++ b/packages/shared/src/environment/db.ts @@ -32,7 +32,19 @@ const regionConfigSchema = z.object({ .optional() }), blobStorage: z.object({ - endpoint: z.string().url().describe('URL of the S3-compatible storage endpoint'), + endpoint: z + .string() + .url() + .describe( + 'URL of the S3-compatible storage endpoint, accessible from the server' + ), + publicEndpoint: z + .string() + .url() + .optional() + .describe( + 'Public URL of the S3-compatible storage endpoint, accessible from clients via the public internet' + ), accessKey: z.string().describe('Access key for the S3-compatible storage endpoint'), secretKey: z.string().describe('Secret key for the S3-compatible storage endpoint'), bucket: z.string().describe('Name of the S3-compatible storage bucket'), From 3e7e11b8a11ebff55c64bfbf3196ea5fb2b2405e Mon Sep 17 00:00:00 2001 From: Daniel Gak Anagrov Date: Mon, 7 Jul 2025 12:28:59 +0200 Subject: [PATCH 03/18] feat(gatekeeper): add error log on seat mismatch (#5004) * feat: added a specific errors on downscale issues --- .../modules/gatekeeper/clients/stripe.ts | 12 +- .../modules/gatekeeper/errors/billing.ts | 6 + .../gatekeeper/events/eventListener.ts | 8 +- .../gatekeeper/graph/resolvers/index.ts | 4 +- packages/server/modules/gatekeeper/index.ts | 10 +- .../server/modules/gatekeeper/rest/billing.ts | 6 +- .../manageSubscriptionDownscale.ts | 15 +- ...SubscriptionDataWithNewValidSeatNumbers.ts | 25 +++- .../gatekeeper/tests/helpers/stripe.ts | 11 ++ .../gatekeeper/tests/unit/stripe.spec.ts | 141 ++++++++++++++++++ .../tests/unit/subscriptions.spec.ts | 88 +++++++++++ 11 files changed, 303 insertions(+), 23 deletions(-) create mode 100644 packages/server/modules/gatekeeper/tests/helpers/stripe.ts create mode 100644 packages/server/modules/gatekeeper/tests/unit/stripe.spec.ts diff --git a/packages/server/modules/gatekeeper/clients/stripe.ts b/packages/server/modules/gatekeeper/clients/stripe.ts index 5438a1325..a799f06c6 100644 --- a/packages/server/modules/gatekeeper/clients/stripe.ts +++ b/packages/server/modules/gatekeeper/clients/stripe.ts @@ -41,7 +41,7 @@ export const createCustomerPortalUrlFactory = return session.url } -export const getSubscriptionDataFactory = +export const getStripeSubscriptionDataFactory = ({ stripe }: // getWorkspacePlanPrice @@ -92,9 +92,15 @@ export const parseSubscriptionData = ( // this should be a reconcile subscriptions, we keep an accurate state in the DB // on each change, we're reconciling that state to stripe export const reconcileWorkspaceSubscriptionFactory = - ({ stripe }: { stripe: Stripe }): ReconcileSubscriptionData => + ({ + stripe, + getStripeSubscriptionData + }: { + stripe: Stripe + getStripeSubscriptionData: GetSubscriptionData + }): ReconcileSubscriptionData => async ({ subscriptionData, prorationBehavior }) => { - const existingSubscriptionState = await getSubscriptionDataFactory({ stripe })({ + const existingSubscriptionState = await getStripeSubscriptionData({ subscriptionId: subscriptionData.subscriptionId }) const items: Stripe.SubscriptionUpdateParams.Item[] = [] diff --git a/packages/server/modules/gatekeeper/errors/billing.ts b/packages/server/modules/gatekeeper/errors/billing.ts index 4be2b0367..1027cf192 100644 --- a/packages/server/modules/gatekeeper/errors/billing.ts +++ b/packages/server/modules/gatekeeper/errors/billing.ts @@ -77,3 +77,9 @@ export class UnsupportedWorkspacePlanError extends BaseError { static code = 'UNSUPPORTED_WORKSPACE_PLAN_ERROR' static statusCode = 400 } + +export class SubscriptionStateError extends BaseError { + static defaultMessage = 'Subscription has an unexpected state' + static code = 'SUBSCRIPTION_STATE_ERROR' + static statusCode = 500 +} diff --git a/packages/server/modules/gatekeeper/events/eventListener.ts b/packages/server/modules/gatekeeper/events/eventListener.ts index 172134118..87e92e0a9 100644 --- a/packages/server/modules/gatekeeper/events/eventListener.ts +++ b/packages/server/modules/gatekeeper/events/eventListener.ts @@ -1,4 +1,7 @@ -import { reconcileWorkspaceSubscriptionFactory } from '@/modules/gatekeeper/clients/stripe' +import { + getStripeSubscriptionDataFactory, + reconcileWorkspaceSubscriptionFactory +} from '@/modules/gatekeeper/clients/stripe' import { getWorkspacePlanFactory, getWorkspaceSubscriptionFactory, @@ -28,7 +31,8 @@ export const initializeEventListenersFactory = getWorkspacePlanPriceId, getWorkspacePlanProductId, reconcileSubscriptionData: reconcileWorkspaceSubscriptionFactory({ - stripe + stripe, + getStripeSubscriptionData: getStripeSubscriptionDataFactory({ stripe }) }), upsertWorkspaceSubscription: upsertWorkspaceSubscriptionFactory({ db }), countSeatsByTypeInWorkspace: countSeatsByTypeInWorkspaceFactory({ db }) diff --git a/packages/server/modules/gatekeeper/graph/resolvers/index.ts b/packages/server/modules/gatekeeper/graph/resolvers/index.ts index 92e9349e9..efc9dd70b 100644 --- a/packages/server/modules/gatekeeper/graph/resolvers/index.ts +++ b/packages/server/modules/gatekeeper/graph/resolvers/index.ts @@ -17,6 +17,7 @@ import { db } from '@/db/knex' import { createCustomerPortalUrlFactory, getRecurringPricesFactory, + getStripeSubscriptionDataFactory, reconcileWorkspaceSubscriptionFactory } from '@/modules/gatekeeper/clients/stripe' import { @@ -458,7 +459,8 @@ export = FF_GATEKEEPER_MODULE_ENABLED const upgradeWorkspaceSubscription = upgradeWorkspaceSubscriptionFactory({ getWorkspacePlan: getWorkspacePlanFactory({ db }), reconcileSubscriptionData: reconcileWorkspaceSubscriptionFactory({ - stripe + stripe, + getStripeSubscriptionData: getStripeSubscriptionDataFactory({ stripe }) }), countSeatsByTypeInWorkspace: countSeatsByTypeInWorkspaceFactory({ db diff --git a/packages/server/modules/gatekeeper/index.ts b/packages/server/modules/gatekeeper/index.ts index 4ae390414..9b7cd22e5 100644 --- a/packages/server/modules/gatekeeper/index.ts +++ b/packages/server/modules/gatekeeper/index.ts @@ -25,7 +25,7 @@ import { upsertWorkspaceSubscriptionFactory } from '@/modules/gatekeeper/repositories/billing' import { - getSubscriptionDataFactory, + getStripeSubscriptionDataFactory, reconcileWorkspaceSubscriptionFactory } from '@/modules/gatekeeper/clients/stripe' import { ScheduleExecution } from '@/modules/core/domain/scheduledTasks/operations' @@ -53,17 +53,21 @@ const scheduleWorkspaceSubscriptionDownscale = ({ scheduleExecution: ScheduleExecution }) => { const stripe = getStripeClient() + const getStripeSubscriptionData = getStripeSubscriptionDataFactory({ stripe }) const manageSubscriptionDownscale = manageSubscriptionDownscaleFactory({ downscaleWorkspaceSubscription: downscaleWorkspaceSubscriptionFactory({ countSeatsByTypeInWorkspace: countSeatsByTypeInWorkspaceFactory({ db }), getWorkspacePlan: getWorkspacePlanFactory({ db }), - reconcileSubscriptionData: reconcileWorkspaceSubscriptionFactory({ stripe }), + reconcileSubscriptionData: reconcileWorkspaceSubscriptionFactory({ + stripe, + getStripeSubscriptionData + }), getWorkspacePlanProductId }), getWorkspaceSubscriptions: getWorkspaceSubscriptionsPastBillingCycleEndFactory({ db }), - getSubscriptionData: getSubscriptionDataFactory({ stripe }), + getStripeSubscriptionData, updateWorkspaceSubscription: upsertWorkspaceSubscriptionFactory({ db }) }) diff --git a/packages/server/modules/gatekeeper/rest/billing.ts b/packages/server/modules/gatekeeper/rest/billing.ts index cf4105697..5cc57e3cd 100644 --- a/packages/server/modules/gatekeeper/rest/billing.ts +++ b/packages/server/modules/gatekeeper/rest/billing.ts @@ -5,7 +5,7 @@ import { getStripeEndpointSigningKey } from '@/modules/shared/helpers/envHelper' import { db } from '@/db/knex' import { completeCheckoutSessionFactory } from '@/modules/gatekeeper/services/checkout' import { - getSubscriptionDataFactory, + getStripeSubscriptionDataFactory, parseSubscriptionData } from '@/modules/gatekeeper/clients/stripe' import { @@ -132,7 +132,7 @@ export const getBillingRouter = (): Router => { }), getWorkspacePlan: getWorkspacePlanFactory({ db }), getWorkspaceSubscription: getWorkspaceSubscriptionFactory({ db }), - getSubscriptionData: getSubscriptionDataFactory({ + getSubscriptionData: getStripeSubscriptionDataFactory({ stripe }), emitEvent: emit @@ -256,7 +256,7 @@ const getSubscriptionFromEventFactory = return null } if (typeof subscription === 'string') { - return await getSubscriptionDataFactory({ stripe })({ + return await getStripeSubscriptionDataFactory({ stripe })({ subscriptionId: subscription }) } diff --git a/packages/server/modules/gatekeeper/services/subscriptions/manageSubscriptionDownscale.ts b/packages/server/modules/gatekeeper/services/subscriptions/manageSubscriptionDownscale.ts index e427c44c7..7c1dd5f2a 100644 --- a/packages/server/modules/gatekeeper/services/subscriptions/manageSubscriptionDownscale.ts +++ b/packages/server/modules/gatekeeper/services/subscriptions/manageSubscriptionDownscale.ts @@ -65,6 +65,7 @@ export const downscaleWorkspaceSubscriptionFactory = }) const subscriptionData = cloneDeep(workspaceSubscription.subscriptionData) + mutateSubscriptionDataWithNewValidSeatNumbers({ seatCount: editorsCount, workspacePlan: workspacePlan.name, @@ -86,12 +87,12 @@ export const manageSubscriptionDownscaleFactory = getWorkspaceSubscriptions, downscaleWorkspaceSubscription, updateWorkspaceSubscription, - getSubscriptionData + getStripeSubscriptionData }: { getWorkspaceSubscriptions: GetWorkspaceSubscriptions downscaleWorkspaceSubscription: DownscaleWorkspaceSubscription updateWorkspaceSubscription: UpsertWorkspaceSubscription - getSubscriptionData: GetSubscriptionData + getStripeSubscriptionData: GetSubscriptionData }) => async (context: { logger: Logger }) => { const { logger } = context @@ -110,9 +111,15 @@ export const manageSubscriptionDownscaleFactory = log.info('Did not need to downscale the workspace subscription') } } catch (err) { - log.error({ err }, 'Failed to downscale workspace subscription') + log.error( + { + err, + workspaceId: workspaceSubscription.workspaceId + }, + 'Failed to downscale workspace subscription' + ) } - const subscriptionData = await getSubscriptionData( + const subscriptionData = await getStripeSubscriptionData( workspaceSubscription.subscriptionData ) const updatedWorkspaceSubscription = { diff --git a/packages/server/modules/gatekeeper/services/subscriptions/mutateSubscriptionDataWithNewValidSeatNumbers.ts b/packages/server/modules/gatekeeper/services/subscriptions/mutateSubscriptionDataWithNewValidSeatNumbers.ts index b8c13fddb..a92039a88 100644 --- a/packages/server/modules/gatekeeper/services/subscriptions/mutateSubscriptionDataWithNewValidSeatNumbers.ts +++ b/packages/server/modules/gatekeeper/services/subscriptions/mutateSubscriptionDataWithNewValidSeatNumbers.ts @@ -4,6 +4,7 @@ import { } from '@/modules/gatekeeper/domain/billing' import { WorkspacePricingProducts } from '@/modules/gatekeeperCore/domain/billing' import { LogicError } from '@/modules/shared/errors' +import { SubscriptionStateError } from '@/modules/gatekeeper/errors/billing' export const mutateSubscriptionDataWithNewValidSeatNumbers = ({ seatCount, @@ -20,15 +21,25 @@ export const mutateSubscriptionDataWithNewValidSeatNumbers = ({ const product = subscriptionData.products.find( (product) => product.productId === productId ) - if (seatCount < 0) throw new LogicError('Invalid seat count, cannot be negative') - if (seatCount === 0 && product === undefined) return - if (seatCount === 0 && product !== undefined) { + if (product === undefined && seatCount === 0) return + if (product === undefined) { + throw new LogicError('Product not found at mutation') + } + + if (seatCount < 0) { + throw new LogicError('Invalid seat count, cannot be negative') + } + + if (product.quantity < seatCount) { + throw new SubscriptionStateError('Subscription missing an upscale') + } + + if (seatCount === 0) { const prodIndex = subscriptionData.products.indexOf(product) subscriptionData.products.splice(prodIndex, 1) - } else if (product !== undefined && product.quantity >= seatCount) { - product.quantity = seatCount - } else { - throw new LogicError('Invalid subscription state') + return } + + product.quantity = seatCount } diff --git a/packages/server/modules/gatekeeper/tests/helpers/stripe.ts b/packages/server/modules/gatekeeper/tests/helpers/stripe.ts new file mode 100644 index 000000000..5b214a94b --- /dev/null +++ b/packages/server/modules/gatekeeper/tests/helpers/stripe.ts @@ -0,0 +1,11 @@ +import Stripe from 'stripe' + +export const buildFakeStripe = (updates: Record = {}): Stripe => { + return { + subscriptions: { + update: async (subscriptionId: string, params?: unknown) => { + updates[subscriptionId] = params + } + } + } as unknown as Stripe +} diff --git a/packages/server/modules/gatekeeper/tests/unit/stripe.spec.ts b/packages/server/modules/gatekeeper/tests/unit/stripe.spec.ts new file mode 100644 index 000000000..3fb59a018 --- /dev/null +++ b/packages/server/modules/gatekeeper/tests/unit/stripe.spec.ts @@ -0,0 +1,141 @@ +import { buildFakeStripe } from '@/modules/gatekeeper/tests/helpers/stripe' +import cryptoRandomString from 'crypto-random-string' +import { + buildTestSubscriptionData, + buildTestSubscriptionProduct +} from '@/modules/gatekeeper/tests/helpers/workspacePlan' +import { reconcileWorkspaceSubscriptionFactory } from '@/modules/gatekeeper/clients/stripe' +import { expect } from 'chai' + +describe('Stripe integration', () => { + describe('Reconciliation', () => { + it('does not send any delete or create anything in Stripe when existing subscription equals to the new one', async () => { + const updates = {} + const subscriptionId = cryptoRandomString({ length: 10 }) + const subscriptionItemId = cryptoRandomString({ length: 10 }) + const fakeStripe = buildFakeStripe(updates) + const subscriptionData = buildTestSubscriptionData({ + subscriptionId, + products: [ + buildTestSubscriptionProduct({ + subscriptionItemId, + quantity: 2 + }) + ] + }) + const reconcileWorkspaceSubscription = reconcileWorkspaceSubscriptionFactory({ + stripe: fakeStripe, + getStripeSubscriptionData: async () => subscriptionData + }) + + await reconcileWorkspaceSubscription({ + subscriptionData, + prorationBehavior: 'none' + }) + + expect(updates).to.be.deep.equal({ + [subscriptionId]: { + items: [{ quantity: 2, id: subscriptionItemId }], + // eslint-disable-next-line camelcase + proration_behavior: 'none' + } + }) + }) + + it('deletes the current products and adds only the needed when stripe has more products than we provided', async () => { + const updates = {} + const subscriptionId = cryptoRandomString({ length: 10 }) + const priceId = cryptoRandomString({ length: 10 }) + const subscriptionItemId = cryptoRandomString({ length: 10 }) + const fakeStripe = buildFakeStripe(updates) + const subscriptionData = buildTestSubscriptionData({ + subscriptionId, + products: [ + buildTestSubscriptionProduct({ + priceId, + subscriptionItemId, + quantity: 1 + }) + ] + }) + const reconcileWorkspaceSubscription = reconcileWorkspaceSubscriptionFactory({ + stripe: fakeStripe, + getStripeSubscriptionData: async () => + buildTestSubscriptionData({ + subscriptionId, + products: [ + buildTestSubscriptionProduct({ + priceId, + subscriptionItemId, + quantity: 2 + }) + ] + }) + }) + + await reconcileWorkspaceSubscription({ + subscriptionData, + prorationBehavior: 'none' + }) + + expect(updates).to.be.deep.equal({ + [subscriptionId]: { + items: [ + { quantity: 1, price: priceId }, + { deleted: true, id: subscriptionItemId } + ], + // eslint-disable-next-line camelcase + proration_behavior: 'none' + } + }) + }) + + it('deletes the current products and ads new ones when stripe has less products than we provided', async () => { + const updates = {} + const subscriptionId = cryptoRandomString({ length: 10 }) + const priceId = cryptoRandomString({ length: 10 }) + const subscriptionItemId = cryptoRandomString({ length: 10 }) + const fakeStripe = buildFakeStripe(updates) + const subscriptionData = buildTestSubscriptionData({ + subscriptionId, + products: [ + buildTestSubscriptionProduct({ + priceId, + subscriptionItemId, + quantity: 3 + }) + ] + }) + const reconcileWorkspaceSubscription = reconcileWorkspaceSubscriptionFactory({ + stripe: fakeStripe, + getStripeSubscriptionData: async () => + buildTestSubscriptionData({ + subscriptionId, + products: [ + buildTestSubscriptionProduct({ + priceId, + subscriptionItemId, + quantity: 2 + }) + ] + }) + }) + + await reconcileWorkspaceSubscription({ + subscriptionData, + prorationBehavior: 'none' + }) + + expect(updates).to.be.deep.equal({ + [subscriptionId]: { + items: [ + { quantity: 3, price: priceId }, + { deleted: true, id: subscriptionItemId } + ], + // eslint-disable-next-line camelcase + proration_behavior: 'none' + } + }) + }) + }) +}) diff --git a/packages/server/modules/gatekeeper/tests/unit/subscriptions.spec.ts b/packages/server/modules/gatekeeper/tests/unit/subscriptions.spec.ts index 226dad693..3534f42a6 100644 --- a/packages/server/modules/gatekeeper/tests/unit/subscriptions.spec.ts +++ b/packages/server/modules/gatekeeper/tests/unit/subscriptions.spec.ts @@ -30,6 +30,7 @@ import { omit } from 'lodash' import { upgradeWorkspaceSubscriptionFactory } from '@/modules/gatekeeper/services/subscriptions/upgradeWorkspaceSubscription' import { EventBusEmit } from '@/modules/shared/services/eventBus' import { testLogger } from '@/observability/logging' +import { mutateSubscriptionDataWithNewValidSeatNumbers } from '@/modules/gatekeeper/services/subscriptions/mutateSubscriptionDataWithNewValidSeatNumbers' describe('subscriptions @gatekeeper', () => { describe('handleSubscriptionUpdateFactory creates a function, that', () => { @@ -811,6 +812,93 @@ describe('subscriptions @gatekeeper', () => { }) }) + describe('mutateSubscriptionDataWithNewValidSeats', () => { + it('can totally downscale the subscription', () => { + const desiredSeatCount = 0 + const productId = cryptoRandomString({ length: 10 }) + const priceId = cryptoRandomString({ length: 10 }) + const subscriptionItemId = cryptoRandomString({ length: 10 }) + const currentQuantity = 2 + const subscriptionData = createTestSubscriptionData({ + products: [ + { priceId, productId, quantity: currentQuantity, subscriptionItemId } + ] + }) + + mutateSubscriptionDataWithNewValidSeatNumbers({ + seatCount: desiredSeatCount, + workspacePlan: PaidWorkspacePlans.Team, + getWorkspacePlanProductId: () => productId, + subscriptionData + }) + + expect(subscriptionData.products).to.has.lengthOf(0) + }) + + it('mutates the quantity when the count is less than the one given', () => { + const desiredSeatCount = 5 + const productId = cryptoRandomString({ length: 10 }) + const priceId = cryptoRandomString({ length: 10 }) + const subscriptionItemId = cryptoRandomString({ length: 10 }) + const currentQuantity = 10 + const subscriptionData = createTestSubscriptionData({ + products: [ + { priceId, productId, quantity: currentQuantity, subscriptionItemId } + ] + }) + + mutateSubscriptionDataWithNewValidSeatNumbers({ + seatCount: desiredSeatCount, + workspacePlan: PaidWorkspacePlans.Team, + getWorkspacePlanProductId: () => productId, + subscriptionData + }) + + expect(subscriptionData.products[0].quantity).to.be.equal(desiredSeatCount) + }) + + it('throws an exception to notify that instead of downscale, a subscription is broken and an upscale is required', () => { + const desiredSeatCount = 10 + const currentQuantity = 5 + const productId = cryptoRandomString({ length: 10 }) + const priceId = cryptoRandomString({ length: 10 }) + const subscriptionItemId = cryptoRandomString({ length: 10 }) + const subscriptionData = createTestSubscriptionData({ + products: [ + { priceId, productId, quantity: currentQuantity, subscriptionItemId } + ] + }) + + const mutation = () => + mutateSubscriptionDataWithNewValidSeatNumbers({ + seatCount: desiredSeatCount, + workspacePlan: PaidWorkspacePlans.Team, + getWorkspacePlanProductId: () => productId, + subscriptionData + }) + + expect(mutation).to.throw('Subscription missing an upscale') + }) + + it('throws an exception when the subscription is malformed', () => { + const desiredSeatCount = 10 + const productId = cryptoRandomString({ length: 10 }) + const subscriptionData = createTestSubscriptionData({ + products: [] + }) + + const mutation = () => + mutateSubscriptionDataWithNewValidSeatNumbers({ + seatCount: desiredSeatCount, + workspacePlan: PaidWorkspacePlans.Team, + getWorkspacePlanProductId: () => productId, + subscriptionData + }) + + expect(mutation).to.throw('Product not found at mutation') + }) + }) + describe('downscaleWorkspaceSubscriptionFactory creates a function, that', () => { it('throws an error if the workspace has no plan attached to it', async () => { const subscriptionData = createTestSubscriptionData() From aa29a09ebc27071b9a047b35ebacec95a2a30574 Mon Sep 17 00:00:00 2001 From: Chuck Driesler Date: Mon, 7 Jul 2025 12:02:38 +0100 Subject: [PATCH 04/18] feat(tokens): create embed-specific tokens (#5013) * feat(tokens): create embed-specific tokens * fix(tokens): repo functions and policy sketch * chore(authz): embed token policies and tests * chore(authz): fine * chore(gql): lint descriptions * fix(embedTokens): better api surface, repo structure * chore(embedTokens): test fixes * fix(embeds): check resource access * fix(embeds): use resource access util --- .../lib/common/generated/gql/graphql.ts | 42 ++++++ .../assets/core/typedefs/apitoken.graphql | 39 +++++ packages/server/codegen.yml | 1 + packages/server/modules/auth/helpers/types.ts | 7 + packages/server/modules/core/dbSchema.ts | 7 + .../modules/core/domain/tokens/operations.ts | 32 +++++ .../modules/core/domain/tokens/types.ts | 3 + .../modules/core/graph/generated/graphql.ts | 76 +++++++++- .../core/graph/resolvers/embedTokens.ts | 107 ++++++++++++++ .../server/modules/core/helpers/graphTypes.ts | 3 + .../20250630073647_add_embed_tokens.ts | 30 ++++ .../modules/core/repositories/embedTokens.ts | 67 +++++++++ .../modules/core/repositories/tokens.ts | 5 +- .../server/modules/core/services/tokens.ts | 48 ++++++- .../modules/core/tests/embedTokens.spec.ts | 120 ++++++++++++++++ .../graph/generated/graphql.ts | 45 ++++++ .../modules/shared/helpers/errorHelper.ts | 2 +- packages/server/test/graphql/embedTokens.ts | 11 ++ .../server/test/graphql/generated/graphql.ts | 53 +++++++ .../shared/src/authz/domain/authErrors.ts | 2 +- packages/shared/src/authz/policies/index.ts | 5 +- .../project/canUpdateEmbedTokens.spec.ts | 134 ++++++++++++++++++ .../policies/project/canUpdateEmbedTokens.ts | 101 +++++++++++++ .../canUseWorkspacePlanFeature.spec.ts | 4 +- .../workspace/canUseWorkspacePlanFeature.ts | 8 +- 25 files changed, 940 insertions(+), 12 deletions(-) create mode 100644 packages/server/modules/core/graph/resolvers/embedTokens.ts create mode 100644 packages/server/modules/core/migrations/20250630073647_add_embed_tokens.ts create mode 100644 packages/server/modules/core/repositories/embedTokens.ts create mode 100644 packages/server/modules/core/tests/embedTokens.spec.ts create mode 100644 packages/server/test/graphql/embedTokens.ts create mode 100644 packages/shared/src/authz/policies/project/canUpdateEmbedTokens.spec.ts create mode 100644 packages/shared/src/authz/policies/project/canUpdateEmbedTokens.ts diff --git a/packages/frontend-2/lib/common/generated/gql/graphql.ts b/packages/frontend-2/lib/common/generated/gql/graphql.ts index 85ba7e6ab..123826ecc 100644 --- a/packages/frontend-2/lib/common/generated/gql/graphql.ts +++ b/packages/frontend-2/lib/common/generated/gql/graphql.ts @@ -1000,6 +1000,22 @@ export type EmailVerificationRequestInput = { id: Scalars['ID']['input']; }; +export type EmbedToken = { + __typename?: 'EmbedToken'; + createdAt: Scalars['DateTime']['output']; + id: Scalars['String']['output']; + lastUsed: Scalars['DateTime']['output']; + lifespan: Scalars['BigInt']['output']; + modelIds: Scalars['String']['output']; + name: Scalars['String']['output']; +}; + +export type EmbedTokenCreateInput = { + lifespan?: InputMaybe; + modelIds: Scalars['String']['input']; + projectId: Scalars['String']['input']; +}; + export type FileUpload = { __typename?: 'FileUpload'; branchName: Scalars['String']['output']; @@ -2095,6 +2111,7 @@ export type Project = { description?: Maybe; /** Public project-level configuration for embedded viewer */ embedOptions: ProjectEmbedOptions; + embedTokens: Array; hasAccessToFeature: Scalars['Boolean']['output']; id: Scalars['ID']['output']; invitableCollaborators: WorkspaceCollaboratorCollection; @@ -2576,6 +2593,7 @@ export type ProjectMutations = { batchDelete: Scalars['Boolean']['output']; /** Create new project */ create: Project; + createEmbedToken: Scalars['String']['output']; /** * Create onboarding/tutorial project. If one is already created for the active user, that * one will be returned instead. @@ -2587,6 +2605,7 @@ export type ProjectMutations = { invites: ProjectInviteMutations; /** Leave a project. Only possible if you're not the last remaining owner. */ leave: Scalars['Boolean']['output']; + revokeEmbedToken: Scalars['Boolean']['output']; /** Updates an existing project */ update: Project; /** Update role for a collaborator */ @@ -2609,6 +2628,11 @@ export type ProjectMutationsCreateArgs = { }; +export type ProjectMutationsCreateEmbedTokenArgs = { + token: EmbedTokenCreateInput; +}; + + export type ProjectMutationsDeleteArgs = { id: Scalars['String']['input']; }; @@ -2619,6 +2643,11 @@ export type ProjectMutationsLeaveArgs = { }; +export type ProjectMutationsRevokeEmbedTokenArgs = { + token: Scalars['String']['input']; +}; + + export type ProjectMutationsUpdateArgs = { update: ProjectUpdateInput; }; @@ -7901,6 +7930,7 @@ export type AllObjectTypes = { CommitCollection: CommitCollection, CountOnlyCollection: CountOnlyCollection, CurrencyBasedPrices: CurrencyBasedPrices, + EmbedToken: EmbedToken, FileUpload: FileUpload, FileUploadCollection: FileUploadCollection, FileUploadMutations: FileUploadMutations, @@ -8363,6 +8393,14 @@ export type CurrencyBasedPricesFieldArgs = { gbp: {}, usd: {}, } +export type EmbedTokenFieldArgs = { + createdAt: {}, + id: {}, + lastUsed: {}, + lifespan: {}, + modelIds: {}, + name: {}, +} export type FileUploadFieldArgs = { branchName: {}, convertedCommitId: {}, @@ -8656,6 +8694,7 @@ export type ProjectFieldArgs = { createdAt: {}, description: {}, embedOptions: {}, + embedTokens: {}, hasAccessToFeature: ProjectHasAccessToFeatureArgs, id: {}, invitableCollaborators: ProjectInvitableCollaboratorsArgs, @@ -8762,10 +8801,12 @@ export type ProjectMutationsFieldArgs = { automationMutations: ProjectMutationsAutomationMutationsArgs, batchDelete: ProjectMutationsBatchDeleteArgs, create: ProjectMutationsCreateArgs, + createEmbedToken: ProjectMutationsCreateEmbedTokenArgs, createForOnboarding: {}, delete: ProjectMutationsDeleteArgs, invites: {}, leave: ProjectMutationsLeaveArgs, + revokeEmbedToken: ProjectMutationsRevokeEmbedTokenArgs, update: ProjectMutationsUpdateArgs, updateRole: ProjectMutationsUpdateRoleArgs, } @@ -9529,6 +9570,7 @@ export type AllObjectFieldArgTypes = { CommitCollection: CommitCollectionFieldArgs, CountOnlyCollection: CountOnlyCollectionFieldArgs, CurrencyBasedPrices: CurrencyBasedPricesFieldArgs, + EmbedToken: EmbedTokenFieldArgs, FileUpload: FileUploadFieldArgs, FileUploadCollection: FileUploadCollectionFieldArgs, FileUploadMutations: FileUploadMutationsFieldArgs, diff --git a/packages/server/assets/core/typedefs/apitoken.graphql b/packages/server/assets/core/typedefs/apitoken.graphql index 90f73cc50..b7f673cbd 100644 --- a/packages/server/assets/core/typedefs/apitoken.graphql +++ b/packages/server/assets/core/typedefs/apitoken.graphql @@ -55,6 +55,33 @@ input AppTokenCreateInput { limitResources: [TokenResourceIdentifierInput!] } +""" +A token used to enable an embedded viewer for a private project +""" +type EmbedToken { + tokenId: String! + projectId: String! + user: LimitedUser + resourceIdString: String! + createdAt: DateTime! + lifespan: BigInt! + lastUsed: DateTime! +} + +input EmbedTokenCreateInput { + projectId: String! + """ + The model(s) and version(s) string used in the embed url + """ + resourceIdString: String! + lifespan: BigInt +} + +type CreateEmbedTokenReturn { + token: String! + tokenMetadata: EmbedToken! +} + extend type Mutation { """ Creates an personal api token. @@ -77,3 +104,15 @@ extend type Mutation { @hasServerRole(role: SERVER_USER) @hasScope(scope: "tokens:write") } + +extend type ProjectMutations { + createEmbedToken(token: EmbedTokenCreateInput!): CreateEmbedTokenReturn! + @hasScope(scope: "tokens:write") + revokeEmbedToken(token: String!, projectId: String!): Boolean! + @hasScope(scope: "tokens:write") + revokeEmbedTokens(projectId: String!): Boolean! @hasScope(scope: "tokens:write") +} + +extend type Project { + embedTokens: [EmbedToken!]! +} diff --git a/packages/server/codegen.yml b/packages/server/codegen.yml index ba69456fe..5bd42aa19 100644 --- a/packages/server/codegen.yml +++ b/packages/server/codegen.yml @@ -24,6 +24,7 @@ generates: ProjectAccessRequestMutations: '@/modules/core/helpers/graphTypes#MutationsObjectGraphQLReturn' LimitedUser: '@/modules/core/helpers/graphTypes#LimitedUserGraphQLReturn' User: '@/modules/core/helpers/graphTypes#UserGraphQLReturn' + EmbedToken: '@/modules/core/helpers/graphTypes#EmbedTokenGraphQLReturn' ActiveUserMutations: '@/modules/core/helpers/graphTypes#MutationsObjectGraphQLReturn' UserMetaMutations: '@/modules/core/helpers/graphTypes#MutationsObjectGraphQLReturn' UserEmailMutations: '@/modules/core/helpers/graphTypes#MutationsObjectGraphQLReturn' diff --git a/packages/server/modules/auth/helpers/types.ts b/packages/server/modules/auth/helpers/types.ts index 024e552eb..10f084848 100644 --- a/packages/server/modules/auth/helpers/types.ts +++ b/packages/server/modules/auth/helpers/types.ts @@ -74,6 +74,13 @@ export type PersonalApiTokenRecord = { tokenId: string } +export type EmbedApiTokenRecord = { + projectId: string + tokenId: string + userId: string + resourceIdString: string +} + export type TokenScopeRecord = { tokenId: string scopeName: ServerScope diff --git a/packages/server/modules/core/dbSchema.ts b/packages/server/modules/core/dbSchema.ts index 201eca27b..d0f6528a8 100644 --- a/packages/server/modules/core/dbSchema.ts +++ b/packages/server/modules/core/dbSchema.ts @@ -406,6 +406,13 @@ export const PersonalApiTokens = buildTableHelper('personal_api_tokens', [ 'userId' ]) +export const EmbedApiTokens = buildTableHelper('embed_api_tokens', [ + 'tokenId', + 'projectId', + 'userId', + 'resourceIdString' +]) + export const UserServerAppTokens = buildTableHelper('user_server_app_tokens', [ 'appId', 'userId', diff --git a/packages/server/modules/core/domain/tokens/operations.ts b/packages/server/modules/core/domain/tokens/operations.ts index c26528852..89f1c1d41 100644 --- a/packages/server/modules/core/domain/tokens/operations.ts +++ b/packages/server/modules/core/domain/tokens/operations.ts @@ -1,5 +1,6 @@ import { ApiToken, + EmbedApiToken, PersonalApiToken, TokenResourceAccessDefinition, TokenResourceIdentifierType, @@ -31,6 +32,8 @@ export type StorePersonalApiToken = ( token: PersonalApiToken ) => Promise +export type StoreEmbedApiToken = (token: EmbedApiToken) => Promise + export type GetUserPersonalAccessTokens = (userId: string) => Promise< { id: string @@ -43,10 +46,25 @@ export type GetUserPersonalAccessTokens = (userId: string) => Promise< }[] > +export type ListProjectEmbedTokens = (args: { projectId: string }) => Promise< + (EmbedApiToken & { + createdAt: Date + lastUsed: Date + lifespan: number | bigint + })[] +> + export type RevokeTokenById = (tokenId: string) => Promise export type RevokeUserTokenById = (tokenId: string, userId: string) => Promise +export type RevokeEmbedTokenById = (args: { + tokenId: string + projectId: string +}) => Promise + +export type RevokeProjectEmbedTokens = (args: { projectId: string }) => Promise + export type GetApiTokenById = (tokenId: string) => Promise> export type GetTokenScopesById = (tokenId: string) => Promise @@ -86,4 +104,18 @@ export type CreateAndStorePersonalAccessToken = ( lifespan?: number | bigint ) => Promise +export type CreateAndStoreEmbedToken = (args: { + projectId: string + userId: string + /** + * The models (and optional versions) included in the embed. + * @example 'foo123,bar456@baz789' + */ + resourceIdString: string + lifespan?: number | bigint +}) => Promise<{ + token: string + tokenMetadata: EmbedApiToken +}> + export type ValidateToken = (tokenString: string) => Promise diff --git a/packages/server/modules/core/domain/tokens/types.ts b/packages/server/modules/core/domain/tokens/types.ts index 5b6f14888..7d0a00e96 100644 --- a/packages/server/modules/core/domain/tokens/types.ts +++ b/packages/server/modules/core/domain/tokens/types.ts @@ -1,4 +1,5 @@ import { + EmbedApiTokenRecord, PersonalApiTokenRecord, TokenScopeRecord, UserServerAppTokenRecord @@ -26,3 +27,5 @@ export type TokenResourceAccessDefinition = TokenResourceAccessRecord export type UserServerAppToken = UserServerAppTokenRecord export type PersonalApiToken = PersonalApiTokenRecord + +export type EmbedApiToken = EmbedApiTokenRecord diff --git a/packages/server/modules/core/graph/generated/graphql.ts b/packages/server/modules/core/graph/generated/graphql.ts index 7b58fb92a..795f3f156 100644 --- a/packages/server/modules/core/graph/generated/graphql.ts +++ b/packages/server/modules/core/graph/generated/graphql.ts @@ -1,5 +1,5 @@ import { GraphQLResolveInfo, GraphQLScalarType, GraphQLScalarTypeConfig } from 'graphql'; -import { StreamGraphQLReturn, CommitGraphQLReturn, ProjectGraphQLReturn, ObjectGraphQLReturn, VersionGraphQLReturn, ServerInviteGraphQLReturnType, ModelGraphQLReturn, ModelsTreeItemGraphQLReturn, MutationsObjectGraphQLReturn, LimitedUserGraphQLReturn, UserGraphQLReturn, GraphQLEmptyReturn, StreamCollaboratorGraphQLReturn, ProjectCollaboratorGraphQLReturn, ServerInfoGraphQLReturn, BranchGraphQLReturn, UserMetaGraphQLReturn, ProjectPermissionChecksGraphQLReturn, ModelPermissionChecksGraphQLReturn, VersionPermissionChecksGraphQLReturn, RootPermissionChecksGraphQLReturn } from '@/modules/core/helpers/graphTypes'; +import { StreamGraphQLReturn, CommitGraphQLReturn, ProjectGraphQLReturn, ObjectGraphQLReturn, VersionGraphQLReturn, ServerInviteGraphQLReturnType, ModelGraphQLReturn, ModelsTreeItemGraphQLReturn, MutationsObjectGraphQLReturn, LimitedUserGraphQLReturn, UserGraphQLReturn, EmbedTokenGraphQLReturn, GraphQLEmptyReturn, StreamCollaboratorGraphQLReturn, ProjectCollaboratorGraphQLReturn, ServerInfoGraphQLReturn, BranchGraphQLReturn, UserMetaGraphQLReturn, ProjectPermissionChecksGraphQLReturn, ModelPermissionChecksGraphQLReturn, VersionPermissionChecksGraphQLReturn, RootPermissionChecksGraphQLReturn } from '@/modules/core/helpers/graphTypes'; import { StreamAccessRequestGraphQLReturn, ProjectAccessRequestGraphQLReturn } from '@/modules/accessrequests/helpers/graphTypes'; import { CommentReplyAuthorCollectionGraphQLReturn, CommentGraphQLReturn, CommentPermissionChecksGraphQLReturn } from '@/modules/comments/helpers/graphTypes'; import { PendingStreamCollaboratorGraphQLReturn } from '@/modules/serverinvites/helpers/graphTypes'; @@ -945,6 +945,12 @@ export type CreateCommentReplyInput = { threadId: Scalars['String']['input']; }; +export type CreateEmbedTokenReturn = { + __typename?: 'CreateEmbedTokenReturn'; + token: Scalars['String']['output']; + tokenMetadata: EmbedToken; +}; + export type CreateModelInput = { description?: InputMaybe; name: Scalars['String']['input']; @@ -1023,6 +1029,25 @@ export type EmailVerificationRequestInput = { id: Scalars['ID']['input']; }; +/** A token used to enable an embedded viewer for a private project */ +export type EmbedToken = { + __typename?: 'EmbedToken'; + createdAt: Scalars['DateTime']['output']; + lastUsed: Scalars['DateTime']['output']; + lifespan: Scalars['BigInt']['output']; + projectId: Scalars['String']['output']; + resourceIdString: Scalars['String']['output']; + tokenId: Scalars['String']['output']; + user?: Maybe; +}; + +export type EmbedTokenCreateInput = { + lifespan?: InputMaybe; + projectId: Scalars['String']['input']; + /** The model(s) and version(s) string used in the embed url */ + resourceIdString: Scalars['String']['input']; +}; + export type FileUpload = { __typename?: 'FileUpload'; branchName: Scalars['String']['output']; @@ -2118,6 +2143,7 @@ export type Project = { description?: Maybe; /** Public project-level configuration for embedded viewer */ embedOptions: ProjectEmbedOptions; + embedTokens: Array; hasAccessToFeature: Scalars['Boolean']['output']; id: Scalars['ID']['output']; invitableCollaborators: WorkspaceCollaboratorCollection; @@ -2599,6 +2625,7 @@ export type ProjectMutations = { batchDelete: Scalars['Boolean']['output']; /** Create new project */ create: Project; + createEmbedToken: CreateEmbedTokenReturn; /** * Create onboarding/tutorial project. If one is already created for the active user, that * one will be returned instead. @@ -2610,6 +2637,8 @@ export type ProjectMutations = { invites: ProjectInviteMutations; /** Leave a project. Only possible if you're not the last remaining owner. */ leave: Scalars['Boolean']['output']; + revokeEmbedToken: Scalars['Boolean']['output']; + revokeEmbedTokens: Scalars['Boolean']['output']; /** Updates an existing project */ update: Project; /** Update role for a collaborator */ @@ -2632,6 +2661,11 @@ export type ProjectMutationsCreateArgs = { }; +export type ProjectMutationsCreateEmbedTokenArgs = { + token: EmbedTokenCreateInput; +}; + + export type ProjectMutationsDeleteArgs = { id: Scalars['String']['input']; }; @@ -2642,6 +2676,17 @@ export type ProjectMutationsLeaveArgs = { }; +export type ProjectMutationsRevokeEmbedTokenArgs = { + projectId: Scalars['String']['input']; + token: Scalars['String']['input']; +}; + + +export type ProjectMutationsRevokeEmbedTokensArgs = { + projectId: Scalars['String']['input']; +}; + + export type ProjectMutationsUpdateArgs = { update: ProjectUpdateInput; }; @@ -5429,6 +5474,7 @@ export type ResolversTypes = { CreateAutomateFunctionWithoutVersionInput: CreateAutomateFunctionWithoutVersionInput; CreateCommentInput: CreateCommentInput; CreateCommentReplyInput: CreateCommentReplyInput; + CreateEmbedTokenReturn: ResolverTypeWrapper & { tokenMetadata: ResolversTypes['EmbedToken'] }>; CreateModelInput: CreateModelInput; CreateServerRegionInput: CreateServerRegionInput; CreateUserEmailInput: CreateUserEmailInput; @@ -5444,6 +5490,8 @@ export type ResolversTypes = { DiscoverableStreamsSortingInput: DiscoverableStreamsSortingInput; EditCommentInput: EditCommentInput; EmailVerificationRequestInput: EmailVerificationRequestInput; + EmbedToken: ResolverTypeWrapper; + EmbedTokenCreateInput: EmbedTokenCreateInput; FileUpload: ResolverTypeWrapper; FileUploadCollection: ResolverTypeWrapper & { items: Array }>; FileUploadMutations: ResolverTypeWrapper; @@ -5773,6 +5821,7 @@ export type ResolversParentTypes = { CreateAutomateFunctionWithoutVersionInput: CreateAutomateFunctionWithoutVersionInput; CreateCommentInput: CreateCommentInput; CreateCommentReplyInput: CreateCommentReplyInput; + CreateEmbedTokenReturn: Omit & { tokenMetadata: ResolversParentTypes['EmbedToken'] }; CreateModelInput: CreateModelInput; CreateServerRegionInput: CreateServerRegionInput; CreateUserEmailInput: CreateUserEmailInput; @@ -5786,6 +5835,8 @@ export type ResolversParentTypes = { DiscoverableStreamsSortingInput: DiscoverableStreamsSortingInput; EditCommentInput: EditCommentInput; EmailVerificationRequestInput: EmailVerificationRequestInput; + EmbedToken: EmbedTokenGraphQLReturn; + EmbedTokenCreateInput: EmbedTokenCreateInput; FileUpload: FileUploadGraphQLReturn; FileUploadCollection: Omit & { items: Array }; FileUploadMutations: MutationsObjectGraphQLReturn; @@ -6458,6 +6509,12 @@ export type CountOnlyCollectionResolvers; }; +export type CreateEmbedTokenReturnResolvers = { + token?: Resolver; + tokenMetadata?: Resolver; + __isTypeOf?: IsTypeOfResolverFn; +}; + export type CurrencyBasedPricesResolvers = { gbp?: Resolver; usd?: Resolver; @@ -6468,6 +6525,17 @@ export interface DateTimeScalarConfig extends GraphQLScalarTypeConfig = { + createdAt?: Resolver; + lastUsed?: Resolver; + lifespan?: Resolver; + projectId?: Resolver; + resourceIdString?: Resolver; + tokenId?: Resolver; + user?: Resolver, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}; + export type FileUploadResolvers = { branchName?: Resolver; convertedCommitId?: Resolver, ParentType, ContextType>; @@ -6820,6 +6888,7 @@ export type ProjectResolvers; description?: Resolver, ParentType, ContextType>; embedOptions?: Resolver; + embedTokens?: Resolver, ParentType, ContextType>; hasAccessToFeature?: Resolver>; id?: Resolver; invitableCollaborators?: Resolver>; @@ -6954,10 +7023,13 @@ export type ProjectMutationsResolvers>; batchDelete?: Resolver>; create?: Resolver>; + createEmbedToken?: Resolver>; createForOnboarding?: Resolver; delete?: Resolver>; invites?: Resolver; leave?: Resolver>; + revokeEmbedToken?: Resolver>; + revokeEmbedTokens?: Resolver>; update?: Resolver>; updateRole?: Resolver>; __isTypeOf?: IsTypeOfResolverFn; @@ -7911,8 +7983,10 @@ export type Resolvers = { Commit?: CommitResolvers; CommitCollection?: CommitCollectionResolvers; CountOnlyCollection?: CountOnlyCollectionResolvers; + CreateEmbedTokenReturn?: CreateEmbedTokenReturnResolvers; CurrencyBasedPrices?: CurrencyBasedPricesResolvers; DateTime?: GraphQLScalarType; + EmbedToken?: EmbedTokenResolvers; FileUpload?: FileUploadResolvers; FileUploadCollection?: FileUploadCollectionResolvers; FileUploadMutations?: FileUploadMutationsResolvers; diff --git a/packages/server/modules/core/graph/resolvers/embedTokens.ts b/packages/server/modules/core/graph/resolvers/embedTokens.ts new file mode 100644 index 000000000..8b5aa0406 --- /dev/null +++ b/packages/server/modules/core/graph/resolvers/embedTokens.ts @@ -0,0 +1,107 @@ +import { db } from '@/db/knex' +import { Resolvers } from '@/modules/core/graph/generated/graphql' +import { + storeApiTokenFactory, + storeTokenResourceAccessDefinitionsFactory, + storeTokenScopesFactory +} from '@/modules/core/repositories/tokens' +import { + listProjectEmbedTokensFactory, + revokeEmbedTokenByIdFactory, + revokeProjectEmbedTokensFactory, + storeEmbedApiTokenFactory +} from '@/modules/core/repositories/embedTokens' +import { + createEmbedTokenFactory, + createTokenFactory +} from '@/modules/core/services/tokens' +import { throwIfAuthNotOk } from '@/modules/shared/helpers/errorHelper' +import { removeNullOrUndefinedKeys } from '@speckle/shared' +import { getUserFactory } from '@/modules/core/repositories/users' +import { throwIfResourceAccessNotAllowed } from '@/modules/core/helpers/token' + +const resolvers: Resolvers = { + EmbedToken: { + user: async (parent) => { + return await getUserFactory({ db })(parent.userId) + } + }, + Project: { + embedTokens: async (parent, _args, context) => { + const canReadEmbedTokens = await context.authPolicies.project.canReadEmbedTokens({ + userId: context.userId, + projectId: parent.id + }) + throwIfAuthNotOk(canReadEmbedTokens) + + return await listProjectEmbedTokensFactory({ db })({ + projectId: parent.id + }) + } + }, + ProjectMutations: { + createEmbedToken: async (_parent, args, context) => { + const canCreateEmbedToken = + await context.authPolicies.project.canUpdateEmbedTokens({ + userId: context.userId, + projectId: args.token.projectId + }) + throwIfAuthNotOk(canCreateEmbedToken) + throwIfResourceAccessNotAllowed({ + resourceId: args.token.projectId, + resourceType: 'project', + resourceAccessRules: context.resourceAccessRules + }) + + return await createEmbedTokenFactory({ + createToken: createTokenFactory({ + storeApiToken: storeApiTokenFactory({ db }), + storeTokenScopes: storeTokenScopesFactory({ db }), + storeTokenResourceAccessDefinitions: + storeTokenResourceAccessDefinitionsFactory({ db }) + }), + storeEmbedToken: storeEmbedApiTokenFactory({ db }) + })({ + ...removeNullOrUndefinedKeys(args.token), + userId: context.userId! + }) + }, + revokeEmbedToken: async (_parent, args, context) => { + const canRevokeEmbedToken = + await context.authPolicies.project.canUpdateEmbedTokens({ + userId: context.userId, + projectId: args.projectId + }) + throwIfAuthNotOk(canRevokeEmbedToken) + throwIfResourceAccessNotAllowed({ + resourceId: args.projectId, + resourceType: 'project', + resourceAccessRules: context.resourceAccessRules + }) + + return await revokeEmbedTokenByIdFactory({ db })({ + tokenId: args.token, + projectId: args.projectId + }) + }, + revokeEmbedTokens: async (_parent, args, context) => { + const canRevokeEmbedTokens = + await context.authPolicies.project.canUpdateEmbedTokens({ + userId: context.userId, + projectId: args.projectId + }) + throwIfAuthNotOk(canRevokeEmbedTokens) + throwIfResourceAccessNotAllowed({ + resourceId: args.projectId, + resourceType: 'project', + resourceAccessRules: context.resourceAccessRules + }) + + await revokeProjectEmbedTokensFactory({ db })({ projectId: args.projectId }) + + return true + } + } +} + +export default resolvers diff --git a/packages/server/modules/core/helpers/graphTypes.ts b/packages/server/modules/core/helpers/graphTypes.ts index 75500c608..43e0f734b 100644 --- a/packages/server/modules/core/helpers/graphTypes.ts +++ b/packages/server/modules/core/helpers/graphTypes.ts @@ -3,6 +3,7 @@ import { LegacyStreamCommit, LegacyUserCommit } from '@/modules/core/domain/commits/types' +import { EmbedApiToken } from '@/modules/core/domain/tokens/types' import { LimitedUser, StreamRole, @@ -154,3 +155,5 @@ export type VersionPermissionChecksGraphQLReturn = { versionId: string projectId: string } + +export type EmbedTokenGraphQLReturn = EmbedApiToken diff --git a/packages/server/modules/core/migrations/20250630073647_add_embed_tokens.ts b/packages/server/modules/core/migrations/20250630073647_add_embed_tokens.ts new file mode 100644 index 000000000..07db597c3 --- /dev/null +++ b/packages/server/modules/core/migrations/20250630073647_add_embed_tokens.ts @@ -0,0 +1,30 @@ +import { Knex } from 'knex' + +export async function up(knex: Knex): Promise { + await knex.schema.createTable('embed_api_tokens', (table) => { + table + .string('tokenId') + .notNullable() + .references('id') + .inTable('api_tokens') + .onDelete('cascade') + table + .string('projectId') + .notNullable() + .references('id') + .inTable('streams') + .onDelete('cascade') + table + .string('userId') + .notNullable() + .references('id') + .inTable('users') + .onDelete('cascade') + table.string('resourceIdString').notNullable() + table.primary(['projectId', 'tokenId']) + }) +} + +export async function down(knex: Knex): Promise { + await knex.schema.dropTableIfExists('embed_api_tokens') +} diff --git a/packages/server/modules/core/repositories/embedTokens.ts b/packages/server/modules/core/repositories/embedTokens.ts new file mode 100644 index 000000000..a3d84dd82 --- /dev/null +++ b/packages/server/modules/core/repositories/embedTokens.ts @@ -0,0 +1,67 @@ +import { EmbedApiTokenRecord } from '@/modules/auth/helpers/types' +import { ApiTokenRecord } from '@/modules/auth/repositories' +import { ApiTokens, EmbedApiTokens } from '@/modules/core/dbSchema' +import { + ListProjectEmbedTokens, + RevokeEmbedTokenById, + RevokeProjectEmbedTokens, + StoreEmbedApiToken +} from '@/modules/core/domain/tokens/operations' +import { UserInputError } from '@/modules/core/errors/userinput' +import { Knex } from 'knex' + +const tables = { + apiTokens: (db: Knex) => db(ApiTokens.name), + embedApiTokens: (db: Knex) => db(EmbedApiTokens.name) +} + +export const storeEmbedApiTokenFactory = + (deps: { db: Knex }): StoreEmbedApiToken => + async (token) => { + const [newToken] = await tables.embedApiTokens(deps.db).insert(token).returning('*') + return newToken + } + +export const listProjectEmbedTokensFactory = + (deps: { db: Knex }): ListProjectEmbedTokens => + async ({ projectId }) => { + return (await tables + .embedApiTokens(deps.db) + .select( + ...EmbedApiTokens.cols, + ApiTokens.col.createdAt, + ApiTokens.col.lastUsed, + ApiTokens.col.lifespan + ) + .orderBy(ApiTokens.col.createdAt, 'desc') + .leftJoin(ApiTokens.name, ApiTokens.col.id, EmbedApiTokens.col.tokenId) + .where(EmbedApiTokens.col.projectId, projectId)) as (EmbedApiTokenRecord & + Pick)[] + } + +export const revokeEmbedTokenByIdFactory = + (deps: { db: Knex }): RevokeEmbedTokenById => + async ({ tokenId: token, projectId }) => { + const tokenId = token.slice(0, 10) + const delCount = await tables + .embedApiTokens(deps.db) + .where({ tokenId, projectId }) + .delete() + if (delCount === 0) throw new UserInputError('Embed token not found') + await tables.apiTokens(deps.db).where(ApiTokens.col.id, tokenId).delete() + return true + } + +export const revokeProjectEmbedTokensFactory = + (deps: { db: Knex }): RevokeProjectEmbedTokens => + async ({ projectId }) => { + await tables + .apiTokens(deps.db) + .whereIn(ApiTokens.col.id, (builder) => { + return builder + .select('tokenId') + .from(EmbedApiTokens.name) + .where('projectId', projectId) + }) + .delete() + } diff --git a/packages/server/modules/core/repositories/tokens.ts b/packages/server/modules/core/repositories/tokens.ts index a1f53581c..a643d9d9c 100644 --- a/packages/server/modules/core/repositories/tokens.ts +++ b/packages/server/modules/core/repositories/tokens.ts @@ -1,4 +1,5 @@ import { + EmbedApiTokenRecord, PersonalApiTokenRecord, TokenScopeRecord, UserServerAppTokenRecord @@ -6,6 +7,7 @@ import { import { ApiTokenRecord } from '@/modules/auth/repositories' import { ApiTokens, + EmbedApiTokens, PersonalApiTokens, TokenResourceAccess, TokenScopes, @@ -38,7 +40,8 @@ const tables = { db(TokenResourceAccess.name), userServerAppTokens: (db: Knex) => db(UserServerAppTokens.name), - personalApiTokens: (db: Knex) => db(PersonalApiTokens.name) + personalApiTokens: (db: Knex) => db(PersonalApiTokens.name), + embedApiTokens: (db: Knex) => db(EmbedApiTokens.name) } export const storeApiTokenFactory = diff --git a/packages/server/modules/core/services/tokens.ts b/packages/server/modules/core/services/tokens.ts index a883c66ff..3e5d674f9 100644 --- a/packages/server/modules/core/services/tokens.ts +++ b/packages/server/modules/core/services/tokens.ts @@ -4,9 +4,10 @@ import { TokenResourceAccessRecord, TokenValidationResult } from '@/modules/core/helpers/types' -import { Optional, ServerScope } from '@speckle/shared' +import { Optional, Scopes, ServerScope } from '@speckle/shared' import { CreateAndStoreAppToken, + CreateAndStoreEmbedToken, CreateAndStorePersonalAccessToken, CreateAndStoreUserToken, GetApiTokenById, @@ -14,6 +15,7 @@ import { GetTokenScopesById, RevokeUserTokenById, StoreApiToken, + StoreEmbedApiToken, StorePersonalApiToken, StoreTokenResourceAccessDefinitions, StoreTokenScopes, @@ -24,6 +26,15 @@ import { import { GetTokenAppInfo } from '@/modules/auth/domain/operations' import { GetUserRole } from '@/modules/core/domain/users/operations' import { TokenCreateError } from '@/modules/core/errors/user' +import cryptoRandomString from 'crypto-random-string' +import { + EmbedApiToken, + TokenResourceIdentifierType +} from '@/modules/core/domain/tokens/types' +import { + createGetParamFromResources, + parseUrlParameters +} from '@speckle/shared/viewer/route' /* Tokens @@ -124,6 +135,41 @@ export const createPersonalAccessTokenFactory = return token } +export const createEmbedTokenFactory = + (deps: { + createToken: CreateAndStoreUserToken + storeEmbedToken: StoreEmbedApiToken + }): CreateAndStoreEmbedToken => + async ({ projectId, userId, resourceIdString, lifespan }) => { + const validatedResourceIdString = createGetParamFromResources( + parseUrlParameters(resourceIdString) + ) + + const { id, token } = await deps.createToken({ + userId, + name: cryptoRandomString({ length: 10 }), + scopes: [Scopes.Streams.Read], + limitResources: [ + { + id: projectId, + type: TokenResourceIdentifierType.Project + } + ], + lifespan + }) + + const tokenMetadata: EmbedApiToken = { + projectId, + tokenId: id, + userId, + resourceIdString: validatedResourceIdString + } + + await deps.storeEmbedToken(tokenMetadata) + + return { token, tokenMetadata } + } + export const validateTokenFactory = (deps: { revokeUserTokenById: RevokeUserTokenById diff --git a/packages/server/modules/core/tests/embedTokens.spec.ts b/packages/server/modules/core/tests/embedTokens.spec.ts new file mode 100644 index 000000000..b805729da --- /dev/null +++ b/packages/server/modules/core/tests/embedTokens.spec.ts @@ -0,0 +1,120 @@ +import { TokenResourceIdentifierType } from '@/modules/core/domain/tokens/types' +import { AllScopes } from '@/modules/core/helpers/mainConstants' +import { + createRandomEmail, + createRandomPassword +} from '@/modules/core/helpers/testHelpers' +import { + BasicTestWorkspace, + createTestWorkspace +} from '@/modules/workspaces/tests/helpers/creation' +import { BasicTestUser, createTestUser } from '@/test/authHelper' +import { + CreateEmbedTokenDocument, + GetActiveUserDocument, + GetProjectDocument, + GetWorkspaceDocument +} from '@/test/graphql/generated/graphql' +import { + createTestContext, + testApolloServer, + TestApolloServer +} from '@/test/graphqlHelper' +import { BasicTestStream, createTestStream } from '@/test/speckle-helpers/streamHelper' +import { Roles, Scopes } from '@speckle/shared' +import { expect } from 'chai' + +describe('Embed tokens', () => { + const adminUser: BasicTestUser = { + id: '', + name: 'John Speckle', + email: createRandomEmail(), + password: createRandomPassword() + } + + const workspace: BasicTestWorkspace = { + id: '', + ownerId: '', + name: 'My Workspace', + slug: '' + } + + const projectA: BasicTestStream = { + id: '', + ownerId: '', + name: 'My Project' + } + const projectB: BasicTestStream = { + id: '', + ownerId: '', + name: 'My Project 2' + } + + let apollo: TestApolloServer + + before(async () => { + await createTestUser(adminUser) + + await createTestWorkspace(workspace, adminUser) + + projectA.workspaceId = workspace.id + projectB.workspaceId = workspace.id + + await createTestStream(projectA, adminUser) + await createTestStream(projectB, adminUser) + + const adminApollo = await testApolloServer({ + context: await createTestContext({ + auth: true, + userId: adminUser.id, + role: Roles.Server.Admin, + scopes: AllScopes, + token: 'abc' + }) + }) + + const res = await adminApollo.execute(CreateEmbedTokenDocument, { + token: { + projectId: projectA.id, + resourceIdString: 'foo123' + } + }) + const token = res.data!.projectMutations.createEmbedToken.token + + apollo = await testApolloServer({ + context: await createTestContext({ + auth: true, + userId: adminUser.id, + role: Roles.Server.Admin, + scopes: [Scopes.Streams.Read], + resourceAccessRules: [ + { id: projectA.id, type: TokenResourceIdentifierType.Project } + ], + token + }) + }) + }) + + it('can read associated project data', async () => { + const res = await apollo.execute(GetProjectDocument, { id: projectA.id }) + expect(res).to.not.haveGraphQLErrors() + expect(res.data?.project.name).to.equal(projectA.name) + }) + + it('cannot read other project data, even if the source user has access', async () => { + const res = await apollo.execute(GetProjectDocument, { id: projectB.id }) + expect(res).to.haveGraphQLErrors() + }) + + it('cannot access source user profile', async () => { + const res = await apollo.execute(GetActiveUserDocument, {}) + expect(res).to.haveGraphQLErrors() + }) + + it('cannot access workspace data', async () => { + const res = await apollo.execute(GetWorkspaceDocument, { + workspaceId: workspace.id + }) + expect(res).to.haveGraphQLErrors() + }) +}) diff --git a/packages/server/modules/cross-server-sync/graph/generated/graphql.ts b/packages/server/modules/cross-server-sync/graph/generated/graphql.ts index 06542fdec..77654b2e9 100644 --- a/packages/server/modules/cross-server-sync/graph/generated/graphql.ts +++ b/packages/server/modules/cross-server-sync/graph/generated/graphql.ts @@ -925,6 +925,12 @@ export type CreateCommentReplyInput = { threadId: Scalars['String']['input']; }; +export type CreateEmbedTokenReturn = { + __typename?: 'CreateEmbedTokenReturn'; + token: Scalars['String']['output']; + tokenMetadata: EmbedToken; +}; + export type CreateModelInput = { description?: InputMaybe; name: Scalars['String']['input']; @@ -1003,6 +1009,25 @@ export type EmailVerificationRequestInput = { id: Scalars['ID']['input']; }; +/** A token used to enable an embedded viewer for a private project */ +export type EmbedToken = { + __typename?: 'EmbedToken'; + createdAt: Scalars['DateTime']['output']; + lastUsed: Scalars['DateTime']['output']; + lifespan: Scalars['BigInt']['output']; + projectId: Scalars['String']['output']; + resourceIdString: Scalars['String']['output']; + tokenId: Scalars['String']['output']; + user?: Maybe; +}; + +export type EmbedTokenCreateInput = { + lifespan?: InputMaybe; + projectId: Scalars['String']['input']; + /** The model(s) and version(s) string used in the embed url */ + resourceIdString: Scalars['String']['input']; +}; + export type FileUpload = { __typename?: 'FileUpload'; branchName: Scalars['String']['output']; @@ -2098,6 +2123,7 @@ export type Project = { description?: Maybe; /** Public project-level configuration for embedded viewer */ embedOptions: ProjectEmbedOptions; + embedTokens: Array; hasAccessToFeature: Scalars['Boolean']['output']; id: Scalars['ID']['output']; invitableCollaborators: WorkspaceCollaboratorCollection; @@ -2579,6 +2605,7 @@ export type ProjectMutations = { batchDelete: Scalars['Boolean']['output']; /** Create new project */ create: Project; + createEmbedToken: CreateEmbedTokenReturn; /** * Create onboarding/tutorial project. If one is already created for the active user, that * one will be returned instead. @@ -2590,6 +2617,8 @@ export type ProjectMutations = { invites: ProjectInviteMutations; /** Leave a project. Only possible if you're not the last remaining owner. */ leave: Scalars['Boolean']['output']; + revokeEmbedToken: Scalars['Boolean']['output']; + revokeEmbedTokens: Scalars['Boolean']['output']; /** Updates an existing project */ update: Project; /** Update role for a collaborator */ @@ -2612,6 +2641,11 @@ export type ProjectMutationsCreateArgs = { }; +export type ProjectMutationsCreateEmbedTokenArgs = { + token: EmbedTokenCreateInput; +}; + + export type ProjectMutationsDeleteArgs = { id: Scalars['String']['input']; }; @@ -2622,6 +2656,17 @@ export type ProjectMutationsLeaveArgs = { }; +export type ProjectMutationsRevokeEmbedTokenArgs = { + projectId: Scalars['String']['input']; + token: Scalars['String']['input']; +}; + + +export type ProjectMutationsRevokeEmbedTokensArgs = { + projectId: Scalars['String']['input']; +}; + + export type ProjectMutationsUpdateArgs = { update: ProjectUpdateInput; }; diff --git a/packages/server/modules/shared/helpers/errorHelper.ts b/packages/server/modules/shared/helpers/errorHelper.ts index 5fe0836d3..93f006046 100644 --- a/packages/server/modules/shared/helpers/errorHelper.ts +++ b/packages/server/modules/shared/helpers/errorHelper.ts @@ -41,7 +41,7 @@ export const mapAuthToServerError = (e: Authz.AllAuthErrors): BaseError => { case Authz.WorkspaceProjectMoveInvalidError.code: case Authz.CommentNoAccessError.code: case Authz.ProjectNotEnoughPermissionsError.code: - case Authz.WorkspaceNoFeatureAccessError.code: + case Authz.WorkspacePlanNoFeatureAccessError.code: case Authz.EligibleForExclusiveWorkspaceError.code: return new ForbiddenError(e.message) case Authz.WorkspaceSsoSessionNoAccessError.code: diff --git a/packages/server/test/graphql/embedTokens.ts b/packages/server/test/graphql/embedTokens.ts new file mode 100644 index 000000000..34d6c51ae --- /dev/null +++ b/packages/server/test/graphql/embedTokens.ts @@ -0,0 +1,11 @@ +import gql from 'graphql-tag' + +export const createEmbedTokenMutation = gql` + mutation CreateEmbedToken($token: EmbedTokenCreateInput!) { + projectMutations { + createEmbedToken(token: $token) { + token + } + } + } +` diff --git a/packages/server/test/graphql/generated/graphql.ts b/packages/server/test/graphql/generated/graphql.ts index 08956ba9e..bc706d47f 100644 --- a/packages/server/test/graphql/generated/graphql.ts +++ b/packages/server/test/graphql/generated/graphql.ts @@ -926,6 +926,12 @@ export type CreateCommentReplyInput = { threadId: Scalars['String']['input']; }; +export type CreateEmbedTokenReturn = { + __typename?: 'CreateEmbedTokenReturn'; + token: Scalars['String']['output']; + tokenMetadata: EmbedToken; +}; + export type CreateModelInput = { description?: InputMaybe; name: Scalars['String']['input']; @@ -1004,6 +1010,25 @@ export type EmailVerificationRequestInput = { id: Scalars['ID']['input']; }; +/** A token used to enable an embedded viewer for a private project */ +export type EmbedToken = { + __typename?: 'EmbedToken'; + createdAt: Scalars['DateTime']['output']; + lastUsed: Scalars['DateTime']['output']; + lifespan: Scalars['BigInt']['output']; + projectId: Scalars['String']['output']; + resourceIdString: Scalars['String']['output']; + tokenId: Scalars['String']['output']; + user?: Maybe; +}; + +export type EmbedTokenCreateInput = { + lifespan?: InputMaybe; + projectId: Scalars['String']['input']; + /** The model(s) and version(s) string used in the embed url */ + resourceIdString: Scalars['String']['input']; +}; + export type FileUpload = { __typename?: 'FileUpload'; branchName: Scalars['String']['output']; @@ -2099,6 +2124,7 @@ export type Project = { description?: Maybe; /** Public project-level configuration for embedded viewer */ embedOptions: ProjectEmbedOptions; + embedTokens: Array; hasAccessToFeature: Scalars['Boolean']['output']; id: Scalars['ID']['output']; invitableCollaborators: WorkspaceCollaboratorCollection; @@ -2580,6 +2606,7 @@ export type ProjectMutations = { batchDelete: Scalars['Boolean']['output']; /** Create new project */ create: Project; + createEmbedToken: CreateEmbedTokenReturn; /** * Create onboarding/tutorial project. If one is already created for the active user, that * one will be returned instead. @@ -2591,6 +2618,8 @@ export type ProjectMutations = { invites: ProjectInviteMutations; /** Leave a project. Only possible if you're not the last remaining owner. */ leave: Scalars['Boolean']['output']; + revokeEmbedToken: Scalars['Boolean']['output']; + revokeEmbedTokens: Scalars['Boolean']['output']; /** Updates an existing project */ update: Project; /** Update role for a collaborator */ @@ -2613,6 +2642,11 @@ export type ProjectMutationsCreateArgs = { }; +export type ProjectMutationsCreateEmbedTokenArgs = { + token: EmbedTokenCreateInput; +}; + + export type ProjectMutationsDeleteArgs = { id: Scalars['String']['input']; }; @@ -2623,6 +2657,17 @@ export type ProjectMutationsLeaveArgs = { }; +export type ProjectMutationsRevokeEmbedTokenArgs = { + projectId: Scalars['String']['input']; + token: Scalars['String']['input']; +}; + + +export type ProjectMutationsRevokeEmbedTokensArgs = { + projectId: Scalars['String']['input']; +}; + + export type ProjectMutationsUpdateArgs = { update: ProjectUpdateInput; }; @@ -5868,6 +5913,13 @@ export type DeleteCommitsMutationVariables = Exact<{ export type DeleteCommitsMutation = { __typename?: 'Mutation', commitsDelete: boolean }; +export type CreateEmbedTokenMutationVariables = Exact<{ + token: EmbedTokenCreateInput; +}>; + + +export type CreateEmbedTokenMutation = { __typename?: 'Mutation', projectMutations: { __typename?: 'ProjectMutations', createEmbedToken: { __typename?: 'CreateEmbedTokenReturn', token: string } } }; + export type GetWorkspacePlanPricesQueryVariables = Exact<{ [key: string]: never; }>; @@ -6615,6 +6667,7 @@ export const ReadOtherUsersCommitsDocument = {"kind":"Document","definitions":[{ export const ReadStreamBranchCommitsDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"ReadStreamBranchCommits"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"streamId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"branchName"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"cursor"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"limit"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"Int"}}},"defaultValue":{"kind":"IntValue","value":"10"}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"stream"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"streamId"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"role"}},{"kind":"Field","name":{"kind":"Name","value":"branch"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"name"},"value":{"kind":"Variable","name":{"kind":"Name","value":"branchName"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"description"}},{"kind":"Field","name":{"kind":"Name","value":"commits"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"cursor"},"value":{"kind":"Variable","name":{"kind":"Name","value":"cursor"}}},{"kind":"Argument","name":{"kind":"Name","value":"limit"},"value":{"kind":"Variable","name":{"kind":"Name","value":"limit"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"totalCount"}},{"kind":"Field","name":{"kind":"Name","value":"cursor"}},{"kind":"Field","name":{"kind":"Name","value":"items"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"BaseCommitFields"}}]}}]}}]}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"BaseCommitFields"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"Commit"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"authorName"}},{"kind":"Field","name":{"kind":"Name","value":"authorId"}},{"kind":"Field","name":{"kind":"Name","value":"authorAvatar"}},{"kind":"Field","name":{"kind":"Name","value":"streamId"}},{"kind":"Field","name":{"kind":"Name","value":"streamName"}},{"kind":"Field","name":{"kind":"Name","value":"sourceApplication"}},{"kind":"Field","name":{"kind":"Name","value":"message"}},{"kind":"Field","name":{"kind":"Name","value":"referencedObject"}},{"kind":"Field","name":{"kind":"Name","value":"createdAt"}},{"kind":"Field","name":{"kind":"Name","value":"commentCount"}}]}}]} as unknown as DocumentNode; export const MoveCommitsDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"MoveCommits"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"CommitsMoveInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"commitsMove"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}]}]}}]} as unknown as DocumentNode; export const DeleteCommitsDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"DeleteCommits"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"CommitsDeleteInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"commitsDelete"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}]}]}}]} as unknown as DocumentNode; +export const CreateEmbedTokenDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"CreateEmbedToken"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"token"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"EmbedTokenCreateInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"projectMutations"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"createEmbedToken"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"token"},"value":{"kind":"Variable","name":{"kind":"Name","value":"token"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"token"}}]}}]}}]}}]} as unknown as DocumentNode; export const GetWorkspacePlanPricesDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"GetWorkspacePlanPrices"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"serverInfo"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"workspaces"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"planPrices"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"usd"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"team"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"monthly"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"amount"}},{"kind":"Field","name":{"kind":"Name","value":"currency"}}]}},{"kind":"Field","name":{"kind":"Name","value":"yearly"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"amount"}},{"kind":"Field","name":{"kind":"Name","value":"currency"}}]}}]}},{"kind":"Field","name":{"kind":"Name","value":"teamUnlimited"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"monthly"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"amount"}},{"kind":"Field","name":{"kind":"Name","value":"currency"}}]}},{"kind":"Field","name":{"kind":"Name","value":"yearly"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"amount"}},{"kind":"Field","name":{"kind":"Name","value":"currency"}}]}}]}},{"kind":"Field","name":{"kind":"Name","value":"pro"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"monthly"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"amount"}},{"kind":"Field","name":{"kind":"Name","value":"currency"}}]}},{"kind":"Field","name":{"kind":"Name","value":"yearly"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"amount"}},{"kind":"Field","name":{"kind":"Name","value":"currency"}}]}}]}},{"kind":"Field","name":{"kind":"Name","value":"proUnlimited"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"monthly"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"amount"}},{"kind":"Field","name":{"kind":"Name","value":"currency"}}]}},{"kind":"Field","name":{"kind":"Name","value":"yearly"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"amount"}},{"kind":"Field","name":{"kind":"Name","value":"currency"}}]}}]}}]}},{"kind":"Field","name":{"kind":"Name","value":"gbp"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"team"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"monthly"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"amount"}},{"kind":"Field","name":{"kind":"Name","value":"currency"}}]}},{"kind":"Field","name":{"kind":"Name","value":"yearly"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"amount"}},{"kind":"Field","name":{"kind":"Name","value":"currency"}}]}}]}},{"kind":"Field","name":{"kind":"Name","value":"teamUnlimited"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"monthly"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"amount"}},{"kind":"Field","name":{"kind":"Name","value":"currency"}}]}},{"kind":"Field","name":{"kind":"Name","value":"yearly"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"amount"}},{"kind":"Field","name":{"kind":"Name","value":"currency"}}]}}]}},{"kind":"Field","name":{"kind":"Name","value":"pro"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"monthly"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"amount"}},{"kind":"Field","name":{"kind":"Name","value":"currency"}}]}},{"kind":"Field","name":{"kind":"Name","value":"yearly"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"amount"}},{"kind":"Field","name":{"kind":"Name","value":"currency"}}]}}]}},{"kind":"Field","name":{"kind":"Name","value":"proUnlimited"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"monthly"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"amount"}},{"kind":"Field","name":{"kind":"Name","value":"currency"}}]}},{"kind":"Field","name":{"kind":"Name","value":"yearly"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"amount"}},{"kind":"Field","name":{"kind":"Name","value":"currency"}}]}}]}}]}}]}}]}}]}}]}}]} as unknown as DocumentNode; export const CreateProjectModelDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"CreateProjectModel"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"CreateModelInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"modelMutations"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"create"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"description"}}]}}]}}]}}]} as unknown as DocumentNode; export const FindProjectModelByNameDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"FindProjectModelByName"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"projectId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"name"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"project"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"projectId"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"modelByName"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"name"},"value":{"kind":"Variable","name":{"kind":"Name","value":"name"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"description"}}]}}]}}]}}]} as unknown as DocumentNode; diff --git a/packages/shared/src/authz/domain/authErrors.ts b/packages/shared/src/authz/domain/authErrors.ts index ff6eecc9b..15b03c157 100644 --- a/packages/shared/src/authz/domain/authErrors.ts +++ b/packages/shared/src/authz/domain/authErrors.ts @@ -116,7 +116,7 @@ export const WorkspaceLimitsReachedError = defineAuthError< message: 'Workspace limits have been reached' }) -export const WorkspaceNoFeatureAccessError = defineAuthError({ +export const WorkspacePlanNoFeatureAccessError = defineAuthError({ code: 'WorkspaceNoFeatureAccess', message: 'Your workspace plan does not have access to this feature.' }) diff --git a/packages/shared/src/authz/policies/index.ts b/packages/shared/src/authz/policies/index.ts index 71ade434c..def12f934 100644 --- a/packages/shared/src/authz/policies/index.ts +++ b/packages/shared/src/authz/policies/index.ts @@ -31,6 +31,7 @@ import { canLoadPolicy } from './project/canLoad.js' import { canReadMemberEmailPolicy } from './workspace/canReadMemberEmail.js' import { canCreateWorkspacePolicy } from './workspace/canCreateWorkspace.js' import { canUseWorkspacePlanFeature } from './workspace/canUseWorkspacePlanFeature.js' +import { canUpdateEmbedTokensPolicy } from './project/canUpdateEmbedTokens.js' export const authPoliciesFactory = (loaders: AllAuthCheckContextLoaders) => ({ project: { @@ -68,7 +69,9 @@ export const authPoliciesFactory = (loaders: AllAuthCheckContextLoaders) => ({ canLeave: canLeaveProjectPolicy(loaders), canInvite: canInviteToProjectPolicy(loaders), canPublish: canPublishPolicy(loaders), - canLoad: canLoadPolicy(loaders) + canLoad: canLoadPolicy(loaders), + canReadEmbedTokens: canUpdateEmbedTokensPolicy(loaders), + canUpdateEmbedTokens: canUpdateEmbedTokensPolicy(loaders) }, workspace: { canCreateProject: canCreateWorkspaceProjectPolicy(loaders), diff --git a/packages/shared/src/authz/policies/project/canUpdateEmbedTokens.spec.ts b/packages/shared/src/authz/policies/project/canUpdateEmbedTokens.spec.ts new file mode 100644 index 000000000..1ffeb6e04 --- /dev/null +++ b/packages/shared/src/authz/policies/project/canUpdateEmbedTokens.spec.ts @@ -0,0 +1,134 @@ +import cryptoRandomString from 'crypto-random-string' +import { Roles } from '../../../core/constants.js' +import { parseFeatureFlags } from '../../../environment/index.js' +import { getProjectFake, getWorkspaceFake } from '../../../tests/fakes.js' +import { canUpdateEmbedTokensPolicy } from './canUpdateEmbedTokens.js' +import { assert, describe, expect, it } from 'vitest' +import { + ProjectNotEnoughPermissionsError, + ServerNoAccessError, + WorkspacePlanNoFeatureAccessError +} from '../../domain/authErrors.js' +import { OverridesOf } from '../../../tests/helpers/types.js' + +const buildCanUpdateEmbedTokens = ( + overrides?: OverridesOf +) => { + const workspaceId = cryptoRandomString({ length: 9 }) + + return canUpdateEmbedTokensPolicy({ + getEnv: async () => parseFeatureFlags({ FF_WORKSPACES_MODULE_ENABLED: 'true' }), + getServerRole: async () => { + return Roles.Server.User + }, + getProject: getProjectFake({ + id: 'project-id', + workspaceId + }), + getProjectRole: async () => { + return Roles.Stream.Owner + }, + getWorkspace: getWorkspaceFake({ + id: workspaceId + }), + getWorkspaceRole: async () => { + return Roles.Workspace.Admin + }, + getWorkspaceSsoProvider: async () => { + return null + }, + getWorkspaceSsoSession: async () => { + assert.fail() + }, + getWorkspacePlan: async () => { + return { + status: 'valid', + workspaceId, + name: 'unlimited', + createdAt: new Date(), + updatedAt: new Date() + } + }, + ...overrides + }) +} + +const canUpdateEmbedTokensArgs = () => ({ + userId: cryptoRandomString({ length: 9 }), + projectId: cryptoRandomString({ length: 9 }) +}) + +describe('canUpdateEmbedTokensArgs returns a function, that', () => { + it('requires a user session', async () => { + const result = await buildCanUpdateEmbedTokens({ + getServerRole: async () => { + return null + } + })(canUpdateEmbedTokensArgs()) + + expect(result).toBeAuthErrorResult({ + code: ServerNoAccessError.code + }) + }) + it('requires user to be project owner', async () => { + const result = await buildCanUpdateEmbedTokens({ + getWorkspaceRole: async () => { + return Roles.Workspace.Member + }, + getProjectRole: async () => { + return Roles.Stream.Contributor + } + })(canUpdateEmbedTokensArgs()) + + expect(result).toBeAuthErrorResult({ + code: ProjectNotEnoughPermissionsError.code + }) + }) + it('does not check workspace plan if workspaces not enabled', async () => { + const result = await buildCanUpdateEmbedTokens({ + getEnv: async () => + parseFeatureFlags({ + FF_WORKSPACES_MODULE_ENABLED: 'false' + }), + getWorkspacePlan: async () => { + assert.fail() + } + })(canUpdateEmbedTokensArgs()) + + expect(result).toBeAuthOKResult() + }) + it('does not check workspace plan if project is not in a workspace', async () => { + const result = await buildCanUpdateEmbedTokens({ + getProject: getProjectFake({ + id: 'project-id', + workspaceId: null + }), + getWorkspacePlan: async () => { + assert.fail() + } + })(canUpdateEmbedTokensArgs()) + + expect(result).toBeAuthOKResult() + }) + it('requires a paid workspace plan, if project is in a workspace', async () => { + const result = await buildCanUpdateEmbedTokens({ + getWorkspacePlan: async () => { + return { + status: 'valid', + workspaceId: 'foo', + name: 'free', + createdAt: new Date(), + updatedAt: new Date() + } + } + })(canUpdateEmbedTokensArgs()) + + expect(result).toBeAuthErrorResult({ + code: WorkspacePlanNoFeatureAccessError.code + }) + }) + it('allows action on paid workspace plans', async () => { + const result = await buildCanUpdateEmbedTokens()(canUpdateEmbedTokensArgs()) + expect(result).toBeAuthOKResult() + }) +}) diff --git a/packages/shared/src/authz/policies/project/canUpdateEmbedTokens.ts b/packages/shared/src/authz/policies/project/canUpdateEmbedTokens.ts new file mode 100644 index 000000000..27521c429 --- /dev/null +++ b/packages/shared/src/authz/policies/project/canUpdateEmbedTokens.ts @@ -0,0 +1,101 @@ +import { err, ok } from 'true-myth/result' +import { + ProjectNoAccessError, + ProjectNotEnoughPermissionsError, + ProjectNotFoundError, + ServerNoAccessError, + ServerNoSessionError, + ServerNotEnoughPermissionsError, + WorkspaceNoAccessError, + WorkspacePlanNoFeatureAccessError, + WorkspaceNotEnoughPermissionsError, + WorkspaceSsoSessionNoAccessError +} from '../../domain/authErrors.js' +import { MaybeUserContext, ProjectContext } from '../../domain/context.js' +import { Loaders } from '../../domain/loaders.js' +import { AuthPolicy } from '../../domain/policies.js' +import { + ensureImplicitProjectMemberWithWriteAccessFragment, + ensureMinimumProjectRoleFragment +} from '../../fragments/projects.js' +import { Roles } from '../../../core/constants.js' + +type PolicyLoaderKeys = + | typeof Loaders.getEnv + | typeof Loaders.getServerRole + | typeof Loaders.getProject + | typeof Loaders.getWorkspaceRole + | typeof Loaders.getWorkspace + | typeof Loaders.getWorkspaceSsoProvider + | typeof Loaders.getWorkspaceSsoSession + | typeof Loaders.getProjectRole + | typeof Loaders.getWorkspacePlan + +type PolicyArgs = MaybeUserContext & ProjectContext + +type PolicyErrors = InstanceType< + | typeof ServerNoAccessError + | typeof ServerNoSessionError + | typeof ServerNotEnoughPermissionsError + | typeof ProjectNotFoundError + | typeof ProjectNoAccessError + | typeof WorkspaceNoAccessError + | typeof WorkspaceSsoSessionNoAccessError + | typeof ProjectNotEnoughPermissionsError + | typeof WorkspaceNotEnoughPermissionsError + | typeof WorkspacePlanNoFeatureAccessError +> + +export const canUpdateEmbedTokensPolicy: AuthPolicy< + PolicyLoaderKeys, + PolicyArgs, + PolicyErrors +> = + (loaders) => + async ({ userId, projectId }) => { + const env = await loaders.getEnv() + const project = await loaders.getProject({ projectId }) + + if (!!project?.workspaceId && env.FF_WORKSPACES_MODULE_ENABLED) { + // Ensure owner-level access and valid plan + const ensuredProjectRole = + await ensureImplicitProjectMemberWithWriteAccessFragment(loaders)({ + userId, + projectId, + role: Roles.Stream.Owner + }) + if (ensuredProjectRole.isErr) { + return err(ensuredProjectRole.error) + } + + const plan = await loaders.getWorkspacePlan({ workspaceId: project.workspaceId }) + + switch (plan?.name) { + case 'academia': + case 'enterprise': + case 'pro': + case 'proUnlimited': + case 'proUnlimitedInvoiced': + case 'team': + case 'teamUnlimited': + case 'teamUnlimitedInvoiced': + case 'unlimited': + return ok() + case 'free': + default: + return err(new WorkspacePlanNoFeatureAccessError()) + } + } else { + // Ensure project owner + const isProjectOwner = await ensureMinimumProjectRoleFragment(loaders)({ + userId: userId!, + projectId, + role: Roles.Stream.Owner + }) + if (isProjectOwner.isErr) { + return err(isProjectOwner.error) + } + + return ok() + } + } diff --git a/packages/shared/src/authz/policies/workspace/canUseWorkspacePlanFeature.spec.ts b/packages/shared/src/authz/policies/workspace/canUseWorkspacePlanFeature.spec.ts index 26d82e8a1..bb8b7b47c 100644 --- a/packages/shared/src/authz/policies/workspace/canUseWorkspacePlanFeature.spec.ts +++ b/packages/shared/src/authz/policies/workspace/canUseWorkspacePlanFeature.spec.ts @@ -10,7 +10,7 @@ import { ServerNoSessionError, ServerNotEnoughPermissionsError, WorkspaceNoAccessError, - WorkspaceNoFeatureAccessError, + WorkspacePlanNoFeatureAccessError, WorkspaceNotEnoughPermissionsError, WorkspaceReadOnlyError } from '../../domain/authErrors.js' @@ -151,7 +151,7 @@ describe('canUseFeature', () => { }) expect(result).toBeAuthErrorResult({ - code: WorkspaceNoFeatureAccessError.code + code: WorkspacePlanNoFeatureAccessError.code }) }) diff --git a/packages/shared/src/authz/policies/workspace/canUseWorkspacePlanFeature.ts b/packages/shared/src/authz/policies/workspace/canUseWorkspacePlanFeature.ts index 230a164f5..5a093d014 100644 --- a/packages/shared/src/authz/policies/workspace/canUseWorkspacePlanFeature.ts +++ b/packages/shared/src/authz/policies/workspace/canUseWorkspacePlanFeature.ts @@ -4,7 +4,7 @@ import { ServerNoSessionError, ServerNotEnoughPermissionsError, WorkspaceNoAccessError, - WorkspaceNoFeatureAccessError, + WorkspacePlanNoFeatureAccessError, WorkspaceNotEnoughPermissionsError, WorkspaceReadOnlyError, WorkspacesNotEnabledError, @@ -43,7 +43,7 @@ type PolicyErrors = | InstanceType | InstanceType | InstanceType - | InstanceType + | InstanceType export const canUseWorkspacePlanFeature: AuthPolicy< PolicyLoaderKeys, @@ -63,10 +63,10 @@ export const canUseWorkspacePlanFeature: AuthPolicy< if (ensuredNotReadOnly.isErr) return err(ensuredNotReadOnly.error) const workspacePlan = await loaders.getWorkspacePlan({ workspaceId }) - if (!workspacePlan) return err(new WorkspaceNoFeatureAccessError()) + if (!workspacePlan) return err(new WorkspacePlanNoFeatureAccessError()) const canUseFeature = workspacePlanHasAccessToFeature({ plan: workspacePlan.name, feature }) - return canUseFeature ? ok() : err(new WorkspaceNoFeatureAccessError()) + return canUseFeature ? ok() : err(new WorkspacePlanNoFeatureAccessError()) } From f55083a933e4fa8b96aa2254cde33b481c1fdde1 Mon Sep 17 00:00:00 2001 From: Iain Sproat <68657+iainsproat@users.noreply.github.com> Date: Mon, 7 Jul 2025 12:08:24 +0100 Subject: [PATCH 05/18] feat(server): add deprecation notice to `/api/file/:fileType/:streamId/:branchName?` (#5032) --- .../fileuploads/graph/resolvers/fileUploads.ts | 10 +++++----- packages/server/modules/fileuploads/rest/router.ts | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/packages/server/modules/fileuploads/graph/resolvers/fileUploads.ts b/packages/server/modules/fileuploads/graph/resolvers/fileUploads.ts index 484a4b718..f33240092 100644 --- a/packages/server/modules/fileuploads/graph/resolvers/fileUploads.ts +++ b/packages/server/modules/fileuploads/graph/resolvers/fileUploads.ts @@ -70,6 +70,11 @@ import cryptoRandomString from 'crypto-random-string' import { getFeatureFlags } from '@speckle/shared/environment' import { throwIfResourceAccessNotAllowed } from '@/modules/core/helpers/token' import { TokenResourceIdentifierType } from '@/modules/core/domain/tokens/types' +import { getModelUploadsFactory } from '@/modules/fileuploads/services/management' +import { + FileUploadRecord, + FileUploadRecordV2 +} from '@/modules/fileuploads/helpers/types' const { FF_LARGE_FILE_IMPORTS_ENABLED, FF_NEXT_GEN_FILE_IMPORTER_ENABLED } = getFeatureFlags() @@ -253,11 +258,6 @@ const fileUploadMutations: Resolvers['FileUploadMutations'] = { } } } -import { getModelUploadsFactory } from '@/modules/fileuploads/services/management' -import { - FileUploadRecord, - FileUploadRecordV2 -} from '@/modules/fileuploads/helpers/types' export = { Stream: { diff --git a/packages/server/modules/fileuploads/rest/router.ts b/packages/server/modules/fileuploads/rest/router.ts index 0ce8637fe..af6baa9e4 100644 --- a/packages/server/modules/fileuploads/rest/router.ts +++ b/packages/server/modules/fileuploads/rest/router.ts @@ -13,12 +13,18 @@ import { UnauthorizedError } from '@/modules/shared/errors' import { ensureError, Nullable } from '@speckle/shared' import { UploadRequestErrorMessage } from '@/modules/fileuploads/helpers/rest' import { getEventBus } from '@/modules/shared/services/eventBus' +import { getFeatureFlags } from '@/modules/shared/helpers/envHelper' + +const { FF_LARGE_FILE_IMPORTS_ENABLED } = getFeatureFlags() export const fileuploadRouterFactory = (): Router => { const processNewFileStream = processNewFileStreamFactory() const app = Router() + /** + * @deprecated use POST /graphql (mutation.fileUploadMutations.generateUploadUrl), then PUT (to the provided url), then POST /graphql (mutation.fileUploadMutations.startFileImport) + */ app.post( '/api/file/:fileType/:streamId/:branchName?', authMiddlewareCreator( @@ -89,6 +95,14 @@ export const fileuploadRouterFactory = (): Router => { logger.error(ensureError(err), 'File importer handling error @deprecated') res.status(500) } + + if (FF_LARGE_FILE_IMPORTS_ENABLED) { + res.setHeader( + 'Warning', + 'Deprecated API; use POST /graphql (mutation.fileUploadMutations.generateUploadUrl), then PUT (to the provided url), then POST /graphql (mutation.fileUploadMutations.startFileImport)' + ) + } + res.status(201).send({ uploadResults }) }, onError: () => { From f769ff45729266f2c774d9d572d8097d4f99580d Mon Sep 17 00:00:00 2001 From: Iain Sproat <68657+iainsproat@users.noreply.github.com> Date: Tue, 8 Jul 2025 09:42:36 +0100 Subject: [PATCH 06/18] fix(server/fileuploads): longer time out before expiry, from last converted time (#5039) --- .../modules/fileuploads/domain/operations.ts | 5 + packages/server/modules/fileuploads/index.ts | 63 +++++++-- .../modules/fileuploads/queues/fileimports.ts | 123 +++++++++++------- .../fileuploads/repositories/fileUploads.ts | 37 +++++- .../fileuploads/services/requestHandler.ts | 83 ++++++++++++ .../tests/unit/fileuploads.spec.ts | 11 +- .../server/modules/previews/clients/bull.ts | 19 ++- 7 files changed, 271 insertions(+), 70 deletions(-) create mode 100644 packages/server/modules/fileuploads/services/requestHandler.ts diff --git a/packages/server/modules/fileuploads/domain/operations.ts b/packages/server/modules/fileuploads/domain/operations.ts index 71413e0bd..a4c41b04a 100644 --- a/packages/server/modules/fileuploads/domain/operations.ts +++ b/packages/server/modules/fileuploads/domain/operations.ts @@ -69,11 +69,16 @@ export type ProcessFileImportResult = (params: { export type UpdateFileStatus = (params: { fileId: string + projectId: string status: FileUploadConvertedStatus convertedMessage: string convertedCommitId: string | null }) => Promise +export type UpdateFileStatusForProjectFactory = (params: { + projectId: string +}) => Promise + export type UploadedFile = UploadResult & { userId: string } export type FileImportMessage = Pick< diff --git a/packages/server/modules/fileuploads/index.ts b/packages/server/modules/fileuploads/index.ts index f8764868e..d6c6ee956 100644 --- a/packages/server/modules/fileuploads/index.ts +++ b/packages/server/modules/fileuploads/index.ts @@ -22,7 +22,8 @@ import { getEventBus } from '@/modules/shared/services/eventBus' import { expireOldPendingUploadsFactory, getFileInfoFactory, - updateFileUploadFactory + updateFileUploadFactory, + updateFileStatusFactory } from '@/modules/fileuploads/repositories/fileUploads' import { db } from '@/db/knex' import { getFileImportTimeLimitMinutes } from '@/modules/shared/helpers/envHelper' @@ -35,14 +36,19 @@ import { import type { ScheduleExecution } from '@/modules/core/domain/scheduledTasks/operations' import { manageFileImportExpiryFactory } from '@/modules/fileuploads/services/tasks' import { TIME } from '@speckle/shared' -import { FileUploadDatabaseEvents } from '@/modules/fileuploads/domain/consts' +import { + DelayBetweenFileImportRetriesMinutes, + FileUploadDatabaseEvents, + NumberOfFileImportRetries +} from '@/modules/fileuploads/domain/consts' import { fileuploadRouterFactory } from '@/modules/fileuploads/rest/router' import { nextGenFileImporterRouterFactory } from '@/modules/fileuploads/rest/nextGenRouter' import { - initializeRhinoQueue, - initializeIfcQueue, + initializeRhinoQueueFactory, + initializeIfcQueueFactory, shutdownQueues, - fileImportQueues + fileImportQueues, + initializeQueueFactory } from '@/modules/fileuploads/queues/fileimports' import { initializeEventListenersFactory } from '@/modules/fileuploads/events/eventListener' import { @@ -50,6 +56,12 @@ import { ObserveResult } from '@/modules/fileuploads/observability/metrics' import { reportSubscriptionEventsFactory } from '@/modules/fileuploads/events/subscriptionListeners' +import { + requestActiveHandlerFactory, + requestErrorHandlerFactory, + requestFailedHandlerFactory +} from '@/modules/fileuploads/services/requestHandler' +import { UpdateFileStatusForProjectFactory } from '@/modules/fileuploads/domain/operations' const { FF_NEXT_GEN_FILE_IMPORTER_ENABLED } = getFeatureFlags() @@ -85,7 +97,12 @@ const scheduleFileImportExpiry = async ({ fileImportExpiryHandlers.map((handler) => handler({ logger, - timeoutThresholdSeconds: (getFileImportTimeLimitMinutes() + 1) * TIME.minute // additional buffer of 1 minute + timeoutThresholdSeconds: + (NumberOfFileImportRetries * + (getFileImportTimeLimitMinutes() + + DelayBetweenFileImportRetriesMinutes) + + 1) * // additional buffer of 1 minute + TIME.minute }) ) ) @@ -93,6 +110,12 @@ const scheduleFileImportExpiry = async ({ ) } +const updateFileStatusBuilder: UpdateFileStatusForProjectFactory = async (params) => { + const { projectId } = params + const projectDb = await getProjectDbClient({ projectId }) + return updateFileStatusFactory({ db: projectDb }) +} + export const init: SpeckleModule['init'] = async ({ app, isInitial, @@ -108,8 +131,32 @@ export const init: SpeckleModule['init'] = async ({ if (isInitial) { if (FF_NEXT_GEN_FILE_IMPORTER_ENABLED) { - const rhinoQueue = await initializeRhinoQueue() - const ifcQueue = await initializeIfcQueue() + const rhinoQueue = await initializeRhinoQueueFactory({ + initializeQueue: initializeQueueFactory({ + jobActiveHandler: requestActiveHandlerFactory({ + logger: moduleLogger, + updateFileStatusBuilder + }), + jobErrorHandler: requestErrorHandlerFactory({ logger: moduleLogger }), + jobFailedHandler: requestFailedHandlerFactory({ + logger: moduleLogger, + updateFileStatusForProjectFactory: updateFileStatusBuilder + }) + }) + })() + const ifcQueue = await initializeIfcQueueFactory({ + initializeQueue: initializeQueueFactory({ + jobActiveHandler: requestActiveHandlerFactory({ + logger: moduleLogger, + updateFileStatusBuilder + }), + jobErrorHandler: requestErrorHandlerFactory({ logger: moduleLogger }), + jobFailedHandler: requestFailedHandlerFactory({ + logger: moduleLogger, + updateFileStatusForProjectFactory: updateFileStatusBuilder + }) + }) + })() ;({ observeResult } = initializeMetrics({ registers: [metricsRegister], diff --git a/packages/server/modules/fileuploads/queues/fileimports.ts b/packages/server/modules/fileuploads/queues/fileimports.ts index ea6a35916..0059628b9 100644 --- a/packages/server/modules/fileuploads/queues/fileimports.ts +++ b/packages/server/modules/fileuploads/queues/fileimports.ts @@ -7,12 +7,16 @@ import { getRedisUrl, isTestEnv } from '@/modules/shared/helpers/envHelper' -import { Logger, logger } from '@/observability/logging' +import { Logger, fileUploadsLogger as logger } from '@/observability/logging' import { TIME, TIME_MS } from '@speckle/shared' import { initializeQueue as setupQueue } from '@speckle/shared/dist/commonjs/queue/index.js' import { JobPayload } from '@speckle/shared/workers/fileimport' import { FileImportQueue } from '@/modules/fileuploads/domain/types' -import Bull from 'bull' +import Bull, { + ActiveEventCallback, + ErrorEventCallback, + FailedEventCallback +} from 'bull' import { NumberOfFileImportRetries, DelayBetweenFileImportRetriesMinutes @@ -57,57 +61,78 @@ const defaultJobOptions = { } } -const initializeQueue = async (params: { - label: string - queueName: string - redisUrl: string - supportedFileTypes: string[] -}): Promise => { - const { label, queueName, redisUrl, supportedFileTypes } = params - const queue = await setupQueue({ - queueName, - redisUrl, - options: { - ...(!isTestEnv() ? { limiter } : {}), - defaultJobOptions - } - }) - const fileImportQueue = { - label, - queue, - supportedFileTypes: supportedFileTypes.map( - (type) => type.toLocaleLowerCase() // Normalize file types to lowercase (this is a safeguard to prevent stupid typos in the future) - ), - shutdown: async () => await queue.close(), - scheduleJob: async (jobData: JobPayload): Promise => { - await queue.add(jobData, defaultJobOptions) +export const initializeQueueFactory = + (deps: { + jobActiveHandler: ActiveEventCallback + jobErrorHandler: ErrorEventCallback + jobFailedHandler: FailedEventCallback + }) => + async (params: { + label: string + queueName: string + redisUrl: string + supportedFileTypes: string[] + }): Promise => { + const { label, queueName, redisUrl, supportedFileTypes } = params + const queue = await setupQueue({ + queueName, + redisUrl, + options: { + ...(!isTestEnv() ? { limiter } : {}), + defaultJobOptions + } + }) + + queue.removeListener('active', deps.jobActiveHandler) + queue.on('active', deps.jobActiveHandler) + + // The error event is triggered when an error in the Redis backend is thrown. + queue.removeListener('error', deps.jobErrorHandler) + queue.on('error', deps.jobErrorHandler) + + // The failed event is triggered when a job fails by throwing an exception during execution. + // https://api.docs.bullmq.io/interfaces/v5.QueueEventsListener.html#failed + queue.removeListener('failed', deps.jobFailedHandler) + queue.on('failed', deps.jobFailedHandler) + + const fileImportQueue = { + label, + queue, + supportedFileTypes: supportedFileTypes.map( + (type) => type.toLocaleLowerCase() // Normalize file types to lowercase (this is a safeguard to prevent stupid typos in the future) + ), + shutdown: async () => await queue.close(), + scheduleJob: async (jobData: JobPayload): Promise => { + await queue.add(jobData, defaultJobOptions) + } } + fileImportQueues.push(fileImportQueue) + return fileImportQueue } - fileImportQueues.push(fileImportQueue) - return fileImportQueue -} -export const initializeRhinoQueue = async () => { - const rhinoImportServiceRedisUrl = getFileImportServiceRhinoParserRedisUrl() +export const initializeRhinoQueueFactory = + (deps: { initializeQueue: ReturnType }) => + async () => { + const rhinoImportServiceRedisUrl = getFileImportServiceRhinoParserRedisUrl() + return deps.initializeQueue({ + label: 'rhino', + queueName: FILEIMPORT_SERVICE_RHINO_QUEUE_NAME, + redisUrl: rhinoImportServiceRedisUrl ? rhinoImportServiceRedisUrl : getRedisUrl(), + supportedFileTypes: ['obj', 'stl', 'skp'] + }) + } - return initializeQueue({ - label: 'rhino', - queueName: FILEIMPORT_SERVICE_RHINO_QUEUE_NAME, - redisUrl: rhinoImportServiceRedisUrl ? rhinoImportServiceRedisUrl : getRedisUrl(), - supportedFileTypes: ['obj', 'stl', 'skp'] - }) -} - -export const initializeIfcQueue = async () => { - const ifcImportServiceRedisUrl = getFileImportServiceIFCParserRedisUrl() - - return initializeQueue({ - label: 'ifc', - queueName: FILEIMPORT_SERVICE_IFC_QUEUE_NAME, - redisUrl: ifcImportServiceRedisUrl ? ifcImportServiceRedisUrl : getRedisUrl(), - supportedFileTypes: ['ifc'] - }) -} +export const initializeIfcQueueFactory = + (deps: { initializeQueue: ReturnType }) => + async () => { + const ifcImportServiceRedisUrl = getFileImportServiceIFCParserRedisUrl() + return deps.initializeQueue({ + label: 'ifc', + queueName: FILEIMPORT_SERVICE_IFC_QUEUE_NAME, + redisUrl: ifcImportServiceRedisUrl ? ifcImportServiceRedisUrl : getRedisUrl(), + supportedFileTypes: ['ifc'] + }) + } export const shutdownQueues = async (params: { logger: Logger }) => { for (const queue of fileImportQueues) { diff --git a/packages/server/modules/fileuploads/repositories/fileUploads.ts b/packages/server/modules/fileuploads/repositories/fileUploads.ts index 032d9d56d..231001cfe 100644 --- a/packages/server/modules/fileuploads/repositories/fileUploads.ts +++ b/packages/server/modules/fileuploads/repositories/fileUploads.ts @@ -10,7 +10,8 @@ import { UpdateFileUpload, GetModelUploadsItems, GetModelUploadsBaseArgs, - GetModelUploadsTotalCount + GetModelUploadsTotalCount, + UpdateFileStatus } from '@/modules/fileuploads/domain/operations' import { FileUploadConvertedStatus, @@ -162,17 +163,19 @@ export const expireOldPendingUploadsFactory = const updatedRows = await deps .db(FileUploads.name) .whereIn(FileUploads.withoutTablePrefix.col.convertedStatus, [ - FileUploadConvertedStatus.Converting, - FileUploadConvertedStatus.Queued + FileUploadConvertedStatus.Converting ]) .andWhere( - FileUploads.withoutTablePrefix.col.uploadDate, + FileUploads.withoutTablePrefix.col.convertedLastUpdate, '<', deps.db.raw(`now() - interval '${params.timeoutThresholdSeconds} seconds'`) ) .update({ [FileUploads.withoutTablePrefix.col.convertedStatus]: - FileUploadConvertedStatus.Error + FileUploadConvertedStatus.Error, + [FileUploads.withoutTablePrefix.col.convertedMessage]: + 'File import job timed out', + [FileUploads.withoutTablePrefix.col.convertedLastUpdate]: knex.fn.now() }) .returning('*') @@ -261,6 +264,30 @@ export const updateFileUploadFactory = return updatedFile[0] } +export const updateFileStatusFactory = + (deps: { db: Knex }): UpdateFileStatus => + async (params) => { + const updatedFile = await tables + .fileUploads(deps.db) + .update({ + [FileUploads.withoutTablePrefix.col.convertedStatus]: params.status, + [FileUploads.withoutTablePrefix.col.convertedMessage]: params.convertedMessage, + [FileUploads.withoutTablePrefix.col.convertedCommitId]: + params.convertedCommitId, + [FileUploads.withoutTablePrefix.col.convertedLastUpdate]: knex.fn.now() + }) + .where({ + [FileUploads.withoutTablePrefix.col.id]: params.fileId, + [FileUploads.withoutTablePrefix.col.streamId]: params.projectId + }) + .returning('*') + + if (updatedFile.length === 0) { + throw new FileImportJobNotFoundError(`File with id ${params.fileId} not found`) + } + return updatedFile[0] + } + const getModelUploadsBaseQueryFactory = (deps: { db: Knex }) => (params: GetModelUploadsBaseArgs) => { const { projectId, modelId } = params diff --git a/packages/server/modules/fileuploads/services/requestHandler.ts b/packages/server/modules/fileuploads/services/requestHandler.ts new file mode 100644 index 000000000..774bd57ac --- /dev/null +++ b/packages/server/modules/fileuploads/services/requestHandler.ts @@ -0,0 +1,83 @@ +import { Logger } from '@/observability/logging' +import { ActiveEventCallback, ErrorEventCallback, FailedEventCallback } from 'bull' +import { UpdateFileStatusForProjectFactory } from '@/modules/fileuploads/domain/operations' +import { JobPayload } from '@speckle/shared/workers/fileimport' +import { FileUploadConvertedStatus } from '@speckle/shared/blobs' +import { ensureError } from '@speckle/shared' + +export const requestActiveHandlerFactory = + (deps: { + logger: Logger + updateFileStatusBuilder: UpdateFileStatusForProjectFactory + }): ActiveEventCallback => + async (job) => { + deps.logger.info( + { + jobId: job.id, + projectId: job.data.projectId, + streamId: job.data.projectId, //legacy + fileName: job.data.fileName, + fileType: job.data.fileType, + blobId: job.data.blobId, + modelId: job.data.modelId + }, + "File import job for file '${fileName}' accepted for project '${projectId} with file type '${fileType}'; updating status to 'Converting'." + ) + const updateFileStatus = await deps.updateFileStatusBuilder({ + projectId: job.data.projectId + }) + await updateFileStatus({ + fileId: job.data.blobId, + projectId: job.data.projectId, + status: FileUploadConvertedStatus.Converting, + convertedMessage: 'File import job accepted and converting started', + convertedCommitId: null + }) + } + +export const requestErrorHandlerFactory = + (deps: { logger: Logger }): ErrorEventCallback => + (e) => { + const err = ensureError( + e, + 'File import job errored for unknown reason, likely a Redis, networking, or application configuration problem' + ) + deps.logger.error({ err }, 'File import job errored (likely a Redis problem)') + // we do not have details about the job here, so we cannot update the file upload status + } + +export const requestFailedHandlerFactory = + (deps: { + logger: Logger + updateFileStatusForProjectFactory: UpdateFileStatusForProjectFactory + }): FailedEventCallback => + async (job, e) => { + const err = ensureError( + e, + 'File import job failed for an unknown reason. This may occur if the worker was killed, memory was exhausted, or a bug in the parsing of the file occurred which caused the worker to crash.' + ) + deps.logger.warn( + { + error: err, + jobId: job.id, + projectId: job.data.projectId, + streamId: job.data.projectId, //legacy + fileType: job.data.fileType, + fileName: job.data.fileName, + blobId: job.data.blobId, + modelId: job.data.modelId + }, + "File import job for file '${fileName}' failed for ${projectId} with file type ${fileType}. Updating status to 'Error'." + ) + const updateFileStatus = await deps.updateFileStatusForProjectFactory({ + projectId: job.data.projectId + }) + + await updateFileStatus({ + fileId: job.data.blobId, + projectId: job.data.projectId, + status: FileUploadConvertedStatus.Error, + convertedMessage: err.message, + convertedCommitId: null + }) + } diff --git a/packages/server/modules/fileuploads/tests/unit/fileuploads.spec.ts b/packages/server/modules/fileuploads/tests/unit/fileuploads.spec.ts index 257796c71..fffa802d2 100644 --- a/packages/server/modules/fileuploads/tests/unit/fileuploads.spec.ts +++ b/packages/server/modules/fileuploads/tests/unit/fileuploads.spec.ts @@ -3,7 +3,8 @@ import { db } from '@/db/knex' import { getFileInfoFactory, saveUploadFileFactory, - saveUploadFileFactoryV2 + saveUploadFileFactoryV2, + updateFileStatusFactory } from '@/modules/fileuploads/repositories/fileUploads' import { insertNewUploadAndNotifyFactory, @@ -57,6 +58,7 @@ describe('FileUploads @fileuploads', () => { saveUploadFile: saveUploadFileFactory({ db }), emit: async () => {} }) + const updateFileStatus = updateFileStatusFactory({ db }) const fileId = cryptoRandomString({ length: 10 }) await insertNewUploadAndNotify({ streamId: createdStreamId, @@ -68,6 +70,13 @@ describe('FileUploads @fileuploads', () => { fileType: 'text/plain', modelId: null }) + await updateFileStatus({ + fileId, + projectId: createdStreamId, + status: FileUploadConvertedStatus.Converting, + convertedMessage: 'Converting started', + convertedCommitId: null + }) await sleep(2000) await garbageCollector({ logger, timeoutThresholdSeconds: 1 }) const results = await getFileInfoFactory({ db })({ diff --git a/packages/server/modules/previews/clients/bull.ts b/packages/server/modules/previews/clients/bull.ts index e75e9fc6d..917324cfa 100644 --- a/packages/server/modules/previews/clients/bull.ts +++ b/packages/server/modules/previews/clients/bull.ts @@ -1,4 +1,9 @@ -import { Queue, type Job } from 'bull' +import type { + ActiveEventCallback, + ErrorEventCallback, + FailedEventCallback, + Queue +} from 'bull' import type { EventEmitter } from 'stream' import { initializeQueue } from '@speckle/shared/queue' import { JobPayload, PreviewResultPayload } from '@speckle/shared/workers/previews' @@ -35,9 +40,9 @@ const defaultJobOptions = { export const addRequestQueueListeners = (params: { requestQueue: QueueEventEmitter - requestErrorHandler: (err: Error) => void - requestFailedHandler: (job: Job, err: Error) => void - requestActiveHandler: (job: Job) => void + requestErrorHandler: ErrorEventCallback + requestFailedHandler: FailedEventCallback + requestActiveHandler: ActiveEventCallback }) => { const { requestQueue, @@ -63,9 +68,9 @@ export const createRequestAndResponseQueues = async (params: { redisUrl: string requestQueueName: string responseQueueName: string - requestErrorHandler: (err: Error) => void - requestFailedHandler: (job: Job, err: Error) => void - requestActiveHandler: (job: Job) => void + requestErrorHandler: ErrorEventCallback + requestFailedHandler: FailedEventCallback + requestActiveHandler: ActiveEventCallback }): Promise<{ requestQueue: Queue responseQueue: Queue From c35b2ce8c2cbd08e59f09a5c5a71b8340986920a Mon Sep 17 00:00:00 2001 From: Chuck Driesler Date: Tue, 8 Jul 2025 13:17:26 +0100 Subject: [PATCH 07/18] fix(embeds): private embed pagination (#5040) * feat(tokens): create embed-specific tokens * fix(tokens): repo functions and policy sketch * chore(authz): embed token policies and tests * chore(authz): fine * chore(gql): lint descriptions * fix(embedTokens): better api surface, repo structure * chore(embedTokens): test fixes * fix(embeds): check resource access * fix(embeds): use resource access util * fix(embedTokens): paginate tokens, emit policy checks * fix(embeds): better query typing * chore(embeds): rename property --- .../assets/core/typedefs/apitoken.graphql | 10 ++++-- .../assets/core/typedefs/permissions.graphql | 3 ++ .../modules/core/domain/tokens/operations.ts | 22 +++++++++++- .../modules/core/graph/generated/graphql.ts | 33 +++++++++++++++-- .../core/graph/resolvers/embedTokens.ts | 14 +++++--- .../core/graph/resolvers/permissions.ts | 21 +++++++++++ .../modules/core/repositories/embedTokens.ts | 35 ++++++++++++++++--- .../server/modules/core/services/tokens.ts | 35 +++++++++++++++++++ .../graph/generated/graphql.ts | 18 +++++++++- .../server/test/graphql/generated/graphql.ts | 18 +++++++++- 10 files changed, 193 insertions(+), 16 deletions(-) diff --git a/packages/server/assets/core/typedefs/apitoken.graphql b/packages/server/assets/core/typedefs/apitoken.graphql index b7f673cbd..5357cbd4f 100644 --- a/packages/server/assets/core/typedefs/apitoken.graphql +++ b/packages/server/assets/core/typedefs/apitoken.graphql @@ -113,6 +113,12 @@ extend type ProjectMutations { revokeEmbedTokens(projectId: String!): Boolean! @hasScope(scope: "tokens:write") } -extend type Project { - embedTokens: [EmbedToken!]! +type EmbedTokenCollection { + items: [EmbedToken!]! + totalCount: Int! + cursor: String +} + +extend type Project { + embedTokens(cursor: String, limit: Int): EmbedTokenCollection! } diff --git a/packages/server/assets/core/typedefs/permissions.graphql b/packages/server/assets/core/typedefs/permissions.graphql index 9861eea99..72b5a17be 100644 --- a/packages/server/assets/core/typedefs/permissions.graphql +++ b/packages/server/assets/core/typedefs/permissions.graphql @@ -16,6 +16,9 @@ type ProjectPermissionChecks { canPublish: PermissionCheckResult! canLoad: PermissionCheckResult! canInvite: PermissionCheckResult! + canCreateEmbedTokens: PermissionCheckResult! + canRevokeEmbedTokens: PermissionCheckResult! + canReadEmbedTokens: PermissionCheckResult! } type RootPermissionChecks { diff --git a/packages/server/modules/core/domain/tokens/operations.ts b/packages/server/modules/core/domain/tokens/operations.ts index 89f1c1d41..85e2cf935 100644 --- a/packages/server/modules/core/domain/tokens/operations.ts +++ b/packages/server/modules/core/domain/tokens/operations.ts @@ -46,7 +46,13 @@ export type GetUserPersonalAccessTokens = (userId: string) => Promise< }[] > -export type ListProjectEmbedTokens = (args: { projectId: string }) => Promise< +export type ListProjectEmbedTokens = (args: { + projectId: string + filter?: { + limit?: number + createdBefore?: string | null + } +}) => Promise< (EmbedApiToken & { createdAt: Date lastUsed: Date @@ -54,6 +60,8 @@ export type ListProjectEmbedTokens = (args: { projectId: string }) => Promise< })[] > +export type CountProjectEmbedTokens = (args: { projectId: string }) => Promise + export type RevokeTokenById = (tokenId: string) => Promise export type RevokeUserTokenById = (tokenId: string, userId: string) => Promise @@ -118,4 +126,16 @@ export type CreateAndStoreEmbedToken = (args: { tokenMetadata: EmbedApiToken }> +export type GetPaginatedProjectEmbedTokens = (args: { + projectId: string + filter?: { + limit?: number + cursor?: string + } +}) => Promise<{ + items: EmbedApiToken[] + totalCount: number + cursor: string | null +}> + export type ValidateToken = (tokenString: string) => Promise diff --git a/packages/server/modules/core/graph/generated/graphql.ts b/packages/server/modules/core/graph/generated/graphql.ts index 795f3f156..7c9644a4d 100644 --- a/packages/server/modules/core/graph/generated/graphql.ts +++ b/packages/server/modules/core/graph/generated/graphql.ts @@ -1041,6 +1041,13 @@ export type EmbedToken = { user?: Maybe; }; +export type EmbedTokenCollection = { + __typename?: 'EmbedTokenCollection'; + cursor?: Maybe; + items: Array; + totalCount: Scalars['Int']['output']; +}; + export type EmbedTokenCreateInput = { lifespan?: InputMaybe; projectId: Scalars['String']['input']; @@ -2143,7 +2150,7 @@ export type Project = { description?: Maybe; /** Public project-level configuration for embedded viewer */ embedOptions: ProjectEmbedOptions; - embedTokens: Array; + embedTokens: EmbedTokenCollection; hasAccessToFeature: Scalars['Boolean']['output']; id: Scalars['ID']['output']; invitableCollaborators: WorkspaceCollaboratorCollection; @@ -2226,6 +2233,12 @@ export type ProjectCommentThreadsArgs = { }; +export type ProjectEmbedTokensArgs = { + cursor?: InputMaybe; + limit?: InputMaybe; +}; + + export type ProjectHasAccessToFeatureArgs = { featureName: WorkspaceFeatureName; }; @@ -2729,6 +2742,7 @@ export type ProjectPermissionChecks = { canBroadcastActivity: PermissionCheckResult; canCreateAutomation: PermissionCheckResult; canCreateComment: PermissionCheckResult; + canCreateEmbedTokens: PermissionCheckResult; canCreateModel: PermissionCheckResult; canDelete: PermissionCheckResult; canInvite: PermissionCheckResult; @@ -2737,9 +2751,11 @@ export type ProjectPermissionChecks = { canMoveToWorkspace: PermissionCheckResult; canPublish: PermissionCheckResult; canRead: PermissionCheckResult; + canReadEmbedTokens: PermissionCheckResult; canReadSettings: PermissionCheckResult; canReadWebhooks: PermissionCheckResult; canRequestRender: PermissionCheckResult; + canRevokeEmbedTokens: PermissionCheckResult; canUpdate: PermissionCheckResult; canUpdateAllowPublicComments: PermissionCheckResult; }; @@ -5491,6 +5507,7 @@ export type ResolversTypes = { EditCommentInput: EditCommentInput; EmailVerificationRequestInput: EmailVerificationRequestInput; EmbedToken: ResolverTypeWrapper; + EmbedTokenCollection: ResolverTypeWrapper & { items: Array }>; EmbedTokenCreateInput: EmbedTokenCreateInput; FileUpload: ResolverTypeWrapper; FileUploadCollection: ResolverTypeWrapper & { items: Array }>; @@ -5836,6 +5853,7 @@ export type ResolversParentTypes = { EditCommentInput: EditCommentInput; EmailVerificationRequestInput: EmailVerificationRequestInput; EmbedToken: EmbedTokenGraphQLReturn; + EmbedTokenCollection: Omit & { items: Array }; EmbedTokenCreateInput: EmbedTokenCreateInput; FileUpload: FileUploadGraphQLReturn; FileUploadCollection: Omit & { items: Array }; @@ -6536,6 +6554,13 @@ export type EmbedTokenResolvers; }; +export type EmbedTokenCollectionResolvers = { + cursor?: Resolver, ParentType, ContextType>; + items?: Resolver, ParentType, ContextType>; + totalCount?: Resolver; + __isTypeOf?: IsTypeOfResolverFn; +}; + export type FileUploadResolvers = { branchName?: Resolver; convertedCommitId?: Resolver, ParentType, ContextType>; @@ -6888,7 +6913,7 @@ export type ProjectResolvers; description?: Resolver, ParentType, ContextType>; embedOptions?: Resolver; - embedTokens?: Resolver, ParentType, ContextType>; + embedTokens?: Resolver>; hasAccessToFeature?: Resolver>; id?: Resolver; invitableCollaborators?: Resolver>; @@ -7053,6 +7078,7 @@ export type ProjectPermissionChecksResolvers; canCreateAutomation?: Resolver; canCreateComment?: Resolver; + canCreateEmbedTokens?: Resolver; canCreateModel?: Resolver; canDelete?: Resolver; canInvite?: Resolver; @@ -7061,9 +7087,11 @@ export type ProjectPermissionChecksResolvers>; canPublish?: Resolver; canRead?: Resolver; + canReadEmbedTokens?: Resolver; canReadSettings?: Resolver; canReadWebhooks?: Resolver; canRequestRender?: Resolver; + canRevokeEmbedTokens?: Resolver; canUpdate?: Resolver; canUpdateAllowPublicComments?: Resolver; __isTypeOf?: IsTypeOfResolverFn; @@ -7987,6 +8015,7 @@ export type Resolvers = { CurrencyBasedPrices?: CurrencyBasedPricesResolvers; DateTime?: GraphQLScalarType; EmbedToken?: EmbedTokenResolvers; + EmbedTokenCollection?: EmbedTokenCollectionResolvers; FileUpload?: FileUploadResolvers; FileUploadCollection?: FileUploadCollectionResolvers; FileUploadMutations?: FileUploadMutationsResolvers; diff --git a/packages/server/modules/core/graph/resolvers/embedTokens.ts b/packages/server/modules/core/graph/resolvers/embedTokens.ts index 8b5aa0406..fab53dbaa 100644 --- a/packages/server/modules/core/graph/resolvers/embedTokens.ts +++ b/packages/server/modules/core/graph/resolvers/embedTokens.ts @@ -6,6 +6,7 @@ import { storeTokenScopesFactory } from '@/modules/core/repositories/tokens' import { + countProjectEmbedTokensFactory, listProjectEmbedTokensFactory, revokeEmbedTokenByIdFactory, revokeProjectEmbedTokensFactory, @@ -13,7 +14,8 @@ import { } from '@/modules/core/repositories/embedTokens' import { createEmbedTokenFactory, - createTokenFactory + createTokenFactory, + getPaginatedProjectEmbedTokensFactory } from '@/modules/core/services/tokens' import { throwIfAuthNotOk } from '@/modules/shared/helpers/errorHelper' import { removeNullOrUndefinedKeys } from '@speckle/shared' @@ -27,15 +29,19 @@ const resolvers: Resolvers = { } }, Project: { - embedTokens: async (parent, _args, context) => { + embedTokens: async (parent, args, context) => { const canReadEmbedTokens = await context.authPolicies.project.canReadEmbedTokens({ userId: context.userId, projectId: parent.id }) throwIfAuthNotOk(canReadEmbedTokens) - return await listProjectEmbedTokensFactory({ db })({ - projectId: parent.id + return await getPaginatedProjectEmbedTokensFactory({ + listEmbedTokens: listProjectEmbedTokensFactory({ db }), + countEmbedTokens: countProjectEmbedTokensFactory({ db }) + })({ + projectId: parent.id, + filter: removeNullOrUndefinedKeys(args) }) } }, diff --git a/packages/server/modules/core/graph/resolvers/permissions.ts b/packages/server/modules/core/graph/resolvers/permissions.ts index 0691408fc..2b3c1a52c 100644 --- a/packages/server/modules/core/graph/resolvers/permissions.ts +++ b/packages/server/modules/core/graph/resolvers/permissions.ts @@ -113,6 +113,27 @@ export default { userId: ctx.userId }) return Authz.toGraphqlResult(canInvite) + }, + canReadEmbedTokens: async (parent, _args, ctx) => { + const canReadEmbedTokens = await ctx.authPolicies.project.canReadEmbedTokens({ + projectId: parent.projectId, + userId: ctx.userId + }) + return Authz.toGraphqlResult(canReadEmbedTokens) + }, + canCreateEmbedTokens: async (parent, _args, ctx) => { + const canCreateEmbedTokens = await ctx.authPolicies.project.canUpdateEmbedTokens({ + projectId: parent.projectId, + userId: ctx.userId + }) + return Authz.toGraphqlResult(canCreateEmbedTokens) + }, + canRevokeEmbedTokens: async (parent, _args, ctx) => { + const canUpdateEmbedTokens = await ctx.authPolicies.project.canUpdateEmbedTokens({ + projectId: parent.projectId, + userId: ctx.userId + }) + return Authz.toGraphqlResult(canUpdateEmbedTokens) } }, ModelPermissionChecks: { diff --git a/packages/server/modules/core/repositories/embedTokens.ts b/packages/server/modules/core/repositories/embedTokens.ts index a3d84dd82..e8459cca4 100644 --- a/packages/server/modules/core/repositories/embedTokens.ts +++ b/packages/server/modules/core/repositories/embedTokens.ts @@ -2,6 +2,7 @@ import { EmbedApiTokenRecord } from '@/modules/auth/helpers/types' import { ApiTokenRecord } from '@/modules/auth/repositories' import { ApiTokens, EmbedApiTokens } from '@/modules/core/dbSchema' import { + CountProjectEmbedTokens, ListProjectEmbedTokens, RevokeEmbedTokenById, RevokeProjectEmbedTokens, @@ -9,6 +10,7 @@ import { } from '@/modules/core/domain/tokens/operations' import { UserInputError } from '@/modules/core/errors/userinput' import { Knex } from 'knex' +import { clamp } from 'lodash' const tables = { apiTokens: (db: Knex) => db(ApiTokens.name), @@ -22,12 +24,29 @@ export const storeEmbedApiTokenFactory = return newToken } +export const countProjectEmbedTokensFactory = + (deps: { db: Knex }): CountProjectEmbedTokens => + async ({ projectId }) => { + const [{ count }] = await tables + .embedApiTokens(deps.db) + .where(EmbedApiTokens.col.projectId, projectId) + .count() + return Number.parseInt(count as string) + } + export const listProjectEmbedTokensFactory = (deps: { db: Knex }): ListProjectEmbedTokens => - async ({ projectId }) => { - return (await tables + async ({ projectId, filter = {} }) => { + const { limit = 10, createdBefore } = filter + + if (limit === 0) return [] + + const q = tables .embedApiTokens(deps.db) - .select( + .select< + (EmbedApiTokenRecord & + Pick)[] + >( ...EmbedApiTokens.cols, ApiTokens.col.createdAt, ApiTokens.col.lastUsed, @@ -35,8 +54,14 @@ export const listProjectEmbedTokensFactory = ) .orderBy(ApiTokens.col.createdAt, 'desc') .leftJoin(ApiTokens.name, ApiTokens.col.id, EmbedApiTokens.col.tokenId) - .where(EmbedApiTokens.col.projectId, projectId)) as (EmbedApiTokenRecord & - Pick)[] + .where(EmbedApiTokens.col.projectId, projectId) + .limit(clamp(limit, 0, 50)) + + if (createdBefore) { + q.andWhere(ApiTokens.col.createdAt, '<', createdBefore) + } + + return await q } export const revokeEmbedTokenByIdFactory = diff --git a/packages/server/modules/core/services/tokens.ts b/packages/server/modules/core/services/tokens.ts index 3e5d674f9..f952e0dea 100644 --- a/packages/server/modules/core/services/tokens.ts +++ b/packages/server/modules/core/services/tokens.ts @@ -6,13 +6,16 @@ import { } from '@/modules/core/helpers/types' import { Optional, Scopes, ServerScope } from '@speckle/shared' import { + CountProjectEmbedTokens, CreateAndStoreAppToken, CreateAndStoreEmbedToken, CreateAndStorePersonalAccessToken, CreateAndStoreUserToken, GetApiTokenById, + GetPaginatedProjectEmbedTokens, GetTokenResourceAccessDefinitionsById, GetTokenScopesById, + ListProjectEmbedTokens, RevokeUserTokenById, StoreApiToken, StoreEmbedApiToken, @@ -35,6 +38,10 @@ import { createGetParamFromResources, parseUrlParameters } from '@speckle/shared/viewer/route' +import { + decodeIsoDateCursor, + encodeIsoDateCursor +} from '@/modules/shared/helpers/dbHelper' /* Tokens @@ -170,6 +177,34 @@ export const createEmbedTokenFactory = return { token, tokenMetadata } } +export const getPaginatedProjectEmbedTokensFactory = + (deps: { + listEmbedTokens: ListProjectEmbedTokens + countEmbedTokens: CountProjectEmbedTokens + }): GetPaginatedProjectEmbedTokens => + async ({ projectId, filter = {} }) => { + const cursor = filter.cursor ? decodeIsoDateCursor(filter.cursor) : null + + const [items, totalCount] = await Promise.all([ + deps.listEmbedTokens({ + projectId, + filter: { + createdBefore: cursor, + limit: 10 + } + }), + deps.countEmbedTokens({ projectId }) + ]) + + const lastItem = items.at(-1) + + return { + items, + totalCount, + cursor: lastItem ? encodeIsoDateCursor(lastItem.createdAt) : null + } + } + export const validateTokenFactory = (deps: { revokeUserTokenById: RevokeUserTokenById diff --git a/packages/server/modules/cross-server-sync/graph/generated/graphql.ts b/packages/server/modules/cross-server-sync/graph/generated/graphql.ts index 77654b2e9..152973098 100644 --- a/packages/server/modules/cross-server-sync/graph/generated/graphql.ts +++ b/packages/server/modules/cross-server-sync/graph/generated/graphql.ts @@ -1021,6 +1021,13 @@ export type EmbedToken = { user?: Maybe; }; +export type EmbedTokenCollection = { + __typename?: 'EmbedTokenCollection'; + cursor?: Maybe; + items: Array; + totalCount: Scalars['Int']['output']; +}; + export type EmbedTokenCreateInput = { lifespan?: InputMaybe; projectId: Scalars['String']['input']; @@ -2123,7 +2130,7 @@ export type Project = { description?: Maybe; /** Public project-level configuration for embedded viewer */ embedOptions: ProjectEmbedOptions; - embedTokens: Array; + embedTokens: EmbedTokenCollection; hasAccessToFeature: Scalars['Boolean']['output']; id: Scalars['ID']['output']; invitableCollaborators: WorkspaceCollaboratorCollection; @@ -2206,6 +2213,12 @@ export type ProjectCommentThreadsArgs = { }; +export type ProjectEmbedTokensArgs = { + cursor?: InputMaybe; + limit?: InputMaybe; +}; + + export type ProjectHasAccessToFeatureArgs = { featureName: WorkspaceFeatureName; }; @@ -2709,6 +2722,7 @@ export type ProjectPermissionChecks = { canBroadcastActivity: PermissionCheckResult; canCreateAutomation: PermissionCheckResult; canCreateComment: PermissionCheckResult; + canCreateEmbedTokens: PermissionCheckResult; canCreateModel: PermissionCheckResult; canDelete: PermissionCheckResult; canInvite: PermissionCheckResult; @@ -2717,9 +2731,11 @@ export type ProjectPermissionChecks = { canMoveToWorkspace: PermissionCheckResult; canPublish: PermissionCheckResult; canRead: PermissionCheckResult; + canReadEmbedTokens: PermissionCheckResult; canReadSettings: PermissionCheckResult; canReadWebhooks: PermissionCheckResult; canRequestRender: PermissionCheckResult; + canRevokeEmbedTokens: PermissionCheckResult; canUpdate: PermissionCheckResult; canUpdateAllowPublicComments: PermissionCheckResult; }; diff --git a/packages/server/test/graphql/generated/graphql.ts b/packages/server/test/graphql/generated/graphql.ts index bc706d47f..e541cd176 100644 --- a/packages/server/test/graphql/generated/graphql.ts +++ b/packages/server/test/graphql/generated/graphql.ts @@ -1022,6 +1022,13 @@ export type EmbedToken = { user?: Maybe; }; +export type EmbedTokenCollection = { + __typename?: 'EmbedTokenCollection'; + cursor?: Maybe; + items: Array; + totalCount: Scalars['Int']['output']; +}; + export type EmbedTokenCreateInput = { lifespan?: InputMaybe; projectId: Scalars['String']['input']; @@ -2124,7 +2131,7 @@ export type Project = { description?: Maybe; /** Public project-level configuration for embedded viewer */ embedOptions: ProjectEmbedOptions; - embedTokens: Array; + embedTokens: EmbedTokenCollection; hasAccessToFeature: Scalars['Boolean']['output']; id: Scalars['ID']['output']; invitableCollaborators: WorkspaceCollaboratorCollection; @@ -2207,6 +2214,12 @@ export type ProjectCommentThreadsArgs = { }; +export type ProjectEmbedTokensArgs = { + cursor?: InputMaybe; + limit?: InputMaybe; +}; + + export type ProjectHasAccessToFeatureArgs = { featureName: WorkspaceFeatureName; }; @@ -2710,6 +2723,7 @@ export type ProjectPermissionChecks = { canBroadcastActivity: PermissionCheckResult; canCreateAutomation: PermissionCheckResult; canCreateComment: PermissionCheckResult; + canCreateEmbedTokens: PermissionCheckResult; canCreateModel: PermissionCheckResult; canDelete: PermissionCheckResult; canInvite: PermissionCheckResult; @@ -2718,9 +2732,11 @@ export type ProjectPermissionChecks = { canMoveToWorkspace: PermissionCheckResult; canPublish: PermissionCheckResult; canRead: PermissionCheckResult; + canReadEmbedTokens: PermissionCheckResult; canReadSettings: PermissionCheckResult; canReadWebhooks: PermissionCheckResult; canRequestRender: PermissionCheckResult; + canRevokeEmbedTokens: PermissionCheckResult; canUpdate: PermissionCheckResult; canUpdateAllowPublicComments: PermissionCheckResult; }; From d1e242bd2ecd3e3b8ce99d294e3b39d0e82ab9a4 Mon Sep 17 00:00:00 2001 From: Mike Date: Tue, 8 Jul 2025 14:54:05 +0200 Subject: [PATCH 08/18] Feat: Make connectors page public (#5043) --- .../frontend-2/components/connectors/Card.vue | 46 ++++++++++++------- .../frontend-2/components/connectors/Page.vue | 2 + .../frontend-2/pages/connectors/index.vue | 4 -- 3 files changed, 31 insertions(+), 21 deletions(-) diff --git a/packages/frontend-2/components/connectors/Card.vue b/packages/frontend-2/components/connectors/Card.vue index bb8dd0d55..eba8313b3 100644 --- a/packages/frontend-2/components/connectors/Card.vue +++ b/packages/frontend-2/components/connectors/Card.vue @@ -1,8 +1,6 @@