diff --git a/docker-compose-speckle.yml b/docker-compose-speckle.yml index 18d16e931..e8fb2b8c5 100644 --- a/docker-compose-speckle.yml +++ b/docker-compose-speckle.yml @@ -32,6 +32,7 @@ services: NUXT_PUBLIC_BACKEND_API_ORIGIN: 'http://speckle-server:3000' NUXT_PUBLIC_LOG_LEVEL: 'warn' NUXT_REDIS_URL: 'redis://redis' + NUXT_PUBLIC_FF_LARGE_FILE_IMPORTS_ENABLED: 'true' LOG_LEVEL: 'info' LOG_PRETTY: 'true' depends_on: @@ -79,9 +80,11 @@ services: REDIS_URL: 'redis://redis' PREVIEW_SERVICE_USE_PRIVATE_OBJECTS_SERVER_URL: 'true' PREVIEW_SERVICE_REDIS_URL: 'redis://redis' + FILEIMPORT_SERVICE_USE_PRIVATE_OBJECTS_SERVER_URL: 'true' FILEIMPORT_SERVICE_REDIS_URL: 'redis://redis' S3_ENDPOINT: 'http://minio:9000' + S3_PUBLIC_ENDPOINT: 'http://127.0.0.1:9000' S3_ACCESS_KEY: 'minioadmin' S3_SECRET_KEY: 'minioadmin' S3_BUCKET: 'speckle-server' @@ -92,6 +95,8 @@ services: FRONTEND_ORIGIN: 'http://127.0.0.1' ONBOARDING_STREAM_URL: 'https://latest.speckle.systems/projects/843d07eb10' + + FF_LARGE_FILE_IMPORTS_ENABLED: 'true' depends_on: [] # - minio diff --git a/packages/server/modules/blobstorage/clients/objectStorage.ts b/packages/server/modules/blobstorage/clients/objectStorage.ts index 615b0d01f..119034cd0 100644 --- a/packages/server/modules/blobstorage/clients/objectStorage.ts +++ b/packages/server/modules/blobstorage/clients/objectStorage.ts @@ -2,6 +2,7 @@ import { getS3AccessKey, getS3BucketName, getS3Endpoint, + getS3PublicEndpoint, getS3Region, getS3SecretKey } from '@/modules/shared/helpers/envHelper' @@ -51,9 +52,17 @@ export const getObjectStorage = (params: GetObjectStorageParams): ObjectStorage } let mainObjectStorage: Optional = undefined +let publicMainObjectStorage: Optional = undefined /** * Get main object storage client + * + * This is used for connecting the server to the S3 host. Where the S3 host is + * on the same private network as the server (e.g. in a Docker network), + * the S3_ENDPOINT can use the private IP or DNS name of the S3 host. + * + * S3_PUBLIC_ENDPOINT can be used to connect to the S3 host via the + * public internet (or localhost network if running locally or testing). */ export const getMainObjectStorage = (): ObjectStorage => { if (mainObjectStorage) return mainObjectStorage @@ -72,6 +81,37 @@ export const getMainObjectStorage = (): ObjectStorage => { return mainObjectStorage } +/** + * (Optional) Used to connect to the S3 host via the public endpoint. + * This is useful for clients that need to access the S3 bucket directly, e.g + * during testing or when the S3 host is not on the same private network as the server. + * + * If `S3_PUBLIC_ENDPOINT` is not set, it will return the same object storage + * as `getMainObjectStorage`. + */ +export const getPublicMainObjectStorage = (): ObjectStorage => { + if (publicMainObjectStorage) return publicMainObjectStorage + + const endpoint = getS3PublicEndpoint() + if (!endpoint) { + // If no public endpoint is set, return the main object storage + return getMainObjectStorage() + } + + const mainParams: GetObjectStorageParams = { + credentials: { + accessKeyId: getS3AccessKey(), + secretAccessKey: getS3SecretKey() + }, + endpoint, + region: getS3Region(), + bucket: getS3BucketName() + } + + publicMainObjectStorage = getObjectStorage(mainParams) + return publicMainObjectStorage +} + export const getSignedUrlFactory = (deps: { objectStorage: ObjectStorage }): GetSignedUrl => { diff --git a/packages/server/modules/blobstorage/rest/router.ts b/packages/server/modules/blobstorage/rest/router.ts index 594ffe6c8..de1214d99 100644 --- a/packages/server/modules/blobstorage/rest/router.ts +++ b/packages/server/modules/blobstorage/rest/router.ts @@ -124,7 +124,9 @@ export const blobStorageRouterFactory = (): Router => { const getBlobMetadata = getBlobMetadataFactory({ db: projectDb }) const getFileStream = getFileStreamFactory({ getBlobMetadata }) - const getObjectStream = getObjectStreamFactory({ storage: projectStorage }) + const getObjectStream = getObjectStreamFactory({ + storage: projectStorage.private + }) const { fileName } = await getBlobMetadata({ streamId: req.params.streamId, @@ -160,7 +162,7 @@ export const blobStorageRouterFactory = (): Router => { ]) const getBlobMetadata = getBlobMetadataFactory({ db: projectDb }) - const deleteObject = deleteObjectFactory({ storage: projectStorage }) + const deleteObject = deleteObjectFactory({ storage: projectStorage.private }) const deleteBlob = fullyDeleteBlobFactory({ getBlobMetadata, deleteBlob: deleteBlobFactory({ db: projectDb }), diff --git a/packages/server/modules/blobstorage/services/streams.ts b/packages/server/modules/blobstorage/services/streams.ts index ed87e0acb..6c886f215 100644 --- a/packages/server/modules/blobstorage/services/streams.ts +++ b/packages/server/modules/blobstorage/services/streams.ts @@ -45,7 +45,7 @@ export const processNewFileStreamFactory = (): NewFileStreamProcessor => { getProjectObjectStorage({ projectId: streamId }) ]) - const storeFileStream = storeFileStreamFactory({ storage: projectStorage }) + const storeFileStream = storeFileStreamFactory({ storage: projectStorage.private }) const updateBlob = updateBlobFactory({ db: projectDb }) const getBlobMetadata = getBlobMetadataFactory({ db: projectDb }) @@ -66,9 +66,9 @@ export const processNewFileStreamFactory = (): NewFileStreamProcessor => { }) const getObjectAttributes = getObjectAttributesFactory({ - storage: projectStorage + storage: projectStorage.private }) - const deleteObject = deleteObjectFactory({ storage: projectStorage }) + const deleteObject = deleteObjectFactory({ storage: projectStorage.private }) busboy.on( 'file', diff --git a/packages/server/modules/blobstorage/tests/integration/blobstorage.integration.spec.ts b/packages/server/modules/blobstorage/tests/integration/blobstorage.integration.spec.ts index dc7e04469..380e54238 100644 --- a/packages/server/modules/blobstorage/tests/integration/blobstorage.integration.spec.ts +++ b/packages/server/modules/blobstorage/tests/integration/blobstorage.integration.spec.ts @@ -23,7 +23,10 @@ import { BasicTestStream, createTestStream } from '@/test/speckle-helpers/stream import cryptoRandomString from 'crypto-random-string' import { BasicTestUser, createTestUser } from '@/test/authHelper' import { storeFileStreamFactory } from '@/modules/blobstorage/repositories/blobs' -import { getMainObjectStorage } from '@/modules/blobstorage/clients/objectStorage' +import { + getMainObjectStorage, + getPublicMainObjectStorage +} from '@/modules/blobstorage/clients/objectStorage' import { expect } from 'chai' import { UploadFileStream } from '@/modules/blobstorage/domain/operations' import { BlobStorageItem } from '@/modules/blobstorage/domain/types' @@ -43,8 +46,8 @@ const buildUploadFileStream = async (params: { streamId: string | null }) => { const storage = streamId ? await getProjectObjectStorage({ projectId: streamId }) - : getMainObjectStorage() - const storeFileStream = storeFileStreamFactory({ storage }) + : { private: getMainObjectStorage(), public: getPublicMainObjectStorage() } + const storeFileStream = storeFileStreamFactory({ storage: storage.public }) const uploadFileStream = uploadFileStreamFactory({ upsertBlob, updateBlob, diff --git a/packages/server/modules/blobstorage/tests/integration/presigned.integration.spec.ts b/packages/server/modules/blobstorage/tests/integration/presigned.integration.spec.ts index f80d22032..ad19720cd 100644 --- a/packages/server/modules/blobstorage/tests/integration/presigned.integration.spec.ts +++ b/packages/server/modules/blobstorage/tests/integration/presigned.integration.spec.ts @@ -52,7 +52,7 @@ const { FF_LARGE_FILE_IMPORTS_ENABLED } = getFeatureFlags() } let projectDb: Knex - let projectStorage: ObjectStorage + let projectStorage: { private: ObjectStorage; public: ObjectStorage } let getBlobMetadata: GetBlobMetadata before(async () => { @@ -77,7 +77,7 @@ const { FF_LARGE_FILE_IMPORTS_ENABLED } = getFeatureFlags() before(() => { SUT = generatePresignedUrlFactory({ getSignedUrl: getSignedUrlFactory({ - objectStorage: projectStorage + objectStorage: projectStorage.public }), upsertBlob: upsertBlobFactory({ db: projectDb @@ -117,7 +117,7 @@ const { FF_LARGE_FILE_IMPORTS_ENABLED } = getFeatureFlags() before(() => { generatePresignedUrl = generatePresignedUrlFactory({ getSignedUrl: getSignedUrlFactory({ - objectStorage: projectStorage + objectStorage: projectStorage.public }), upsertBlob: upsertBlobFactory({ db: projectDb @@ -126,7 +126,7 @@ const { FF_LARGE_FILE_IMPORTS_ENABLED } = getFeatureFlags() SUT = registerCompletedUploadFactory({ getBlob: getBlobFactory({ db: projectDb }), getBlobMetadata: getBlobMetadataFromStorage({ - objectStorage: projectStorage + objectStorage: projectStorage.public }), updateBlob: updateBlobFactory({ db: projectDb diff --git a/packages/server/modules/fileuploads/graph/resolvers/fileUploads.ts b/packages/server/modules/fileuploads/graph/resolvers/fileUploads.ts index 62bd4ba61..484a4b718 100644 --- a/packages/server/modules/fileuploads/graph/resolvers/fileUploads.ts +++ b/packages/server/modules/fileuploads/graph/resolvers/fileUploads.ts @@ -25,7 +25,9 @@ import { } from '@/modules/shared/errors' import { throwIfAuthNotOk } from '@/modules/shared/helpers/errorHelper' import { + fileImportServiceShouldUsePrivateObjectsServerUrl, getFileUploadUrlExpiryMinutes, + getPrivateObjectsServerOrigin, getServerOrigin, isFileUploadsEnabled } from '@/modules/shared/helpers/envHelper' @@ -134,7 +136,7 @@ const fileUploadMutations: Resolvers['FileUploadMutations'] = { const generatePresignedUrl = generatePresignedUrlFactory({ getSignedUrl: getSignedUrlFactory({ - objectStorage: projectStorage + objectStorage: projectStorage.public }), upsertBlob: upsertBlobFactory({ db: projectDb @@ -189,7 +191,9 @@ const fileUploadMutations: Resolvers['FileUploadMutations'] = { ]) const pushJobToFileImporter = pushJobToFileImporterFactory({ - getServerOrigin, + getServerOrigin: fileImportServiceShouldUsePrivateObjectsServerUrl() + ? getPrivateObjectsServerOrigin + : getServerOrigin, createAppToken: createAppTokenFactory({ storeApiToken: storeApiTokenFactory({ db }), storeTokenScopes: storeTokenScopesFactory({ db }), @@ -221,7 +225,7 @@ const fileUploadMutations: Resolvers['FileUploadMutations'] = { db: projectDb }), getBlobMetadata: getBlobMetadataFromStorage({ - objectStorage: projectStorage + objectStorage: projectStorage.private }) }), insertNewUploadAndNotify: FF_NEXT_GEN_FILE_IMPORTER_ENABLED diff --git a/packages/server/modules/fileuploads/tests/integration/presigned.integration.spec.ts b/packages/server/modules/fileuploads/tests/integration/presigned.integration.spec.ts index 5feb9a024..1a0977cfe 100644 --- a/packages/server/modules/fileuploads/tests/integration/presigned.integration.spec.ts +++ b/packages/server/modules/fileuploads/tests/integration/presigned.integration.spec.ts @@ -75,7 +75,7 @@ const { FF_LARGE_FILE_IMPORTS_ENABLED, FF_NEXT_GEN_FILE_IMPORTER_ENABLED } = } let projectDb: Knex - let projectStorage: ObjectStorage + let projectStorage: { private: ObjectStorage; public: ObjectStorage } before(async () => { await beforeEachContext() @@ -112,7 +112,7 @@ const { FF_LARGE_FILE_IMPORTS_ENABLED, FF_NEXT_GEN_FILE_IMPORTER_ENABLED } = before(() => { generatePresignedUrl = generatePresignedUrlFactory({ getSignedUrl: getSignedUrlFactory({ - objectStorage: projectStorage + objectStorage: projectStorage.public }), upsertBlob: upsertBlobFactory({ db: projectDb @@ -152,7 +152,7 @@ const { FF_LARGE_FILE_IMPORTS_ENABLED, FF_NEXT_GEN_FILE_IMPORTER_ENABLED } = registerCompletedUpload: registerCompletedUploadFactory({ getBlob: getBlobFactory({ db: projectDb }), getBlobMetadata: getBlobMetadataFromStorage({ - objectStorage: projectStorage + objectStorage: projectStorage.public }), updateBlob: updateBlobFactory({ db: projectDb diff --git a/packages/server/modules/gendo/graph/resolvers/index.ts b/packages/server/modules/gendo/graph/resolvers/index.ts index 3924b4912..9515a53f8 100644 --- a/packages/server/modules/gendo/graph/resolvers/index.ts +++ b/packages/server/modules/gendo/graph/resolvers/index.ts @@ -119,7 +119,9 @@ export = FF_GENDOAI_MODULE_ENABLED token: getGendoAIKey() }) - const storeFileStream = storeFileStreamFactory({ storage: projectStorage }) + const storeFileStream = storeFileStreamFactory({ + storage: projectStorage.private + }) const createRenderRequest = createRenderRequestFactory({ uploadFileStream: uploadFileStreamFactory({ storeFileStream, diff --git a/packages/server/modules/gendo/rest/index.ts b/packages/server/modules/gendo/rest/index.ts index 20251890a..587d8b272 100644 --- a/packages/server/modules/gendo/rest/index.ts +++ b/packages/server/modules/gendo/rest/index.ts @@ -59,7 +59,9 @@ export default function (app: express.Express) { getProjectObjectStorage({ projectId }) ]) - const storeFileStream = storeFileStreamFactory({ storage: projectStorage }) + const storeFileStream = storeFileStreamFactory({ + storage: projectStorage.private + }) const updateRenderRequest = updateRenderRequestFactory({ getRenderByGenerationId: getRenderByGenerationIdFactory({ db: projectDb }), uploadFileStream: uploadFileStreamFactory({ diff --git a/packages/server/modules/multiregion/domain/operations.ts b/packages/server/modules/multiregion/domain/operations.ts index 7dffb8114..ac79ee302 100644 --- a/packages/server/modules/multiregion/domain/operations.ts +++ b/packages/server/modules/multiregion/domain/operations.ts @@ -64,8 +64,8 @@ export type UpdateAndValidateRegion = (params: { export type GetProjectObjectStorage = (args: { projectId: string -}) => Promise +}) => Promise<{ private: ObjectStorage; public: ObjectStorage }> export type GetRegionObjectStorage = (args: { regionKey: string -}) => Promise +}) => Promise<{ private: ObjectStorage; public: ObjectStorage }> diff --git a/packages/server/modules/multiregion/services/queue.ts b/packages/server/modules/multiregion/services/queue.ts index ca57e3a80..277de80e5 100644 --- a/packages/server/modules/multiregion/services/queue.ts +++ b/packages/server/modules/multiregion/services/queue.ts @@ -165,9 +165,11 @@ export const startQueue = async () => { const { projectId, regionKey } = job.data.payload const sourceDb = await getProjectDbClient({ projectId }) - const sourceObjectStorage = await getProjectObjectStorage({ projectId }) + const sourceObjectStorage = (await getProjectObjectStorage({ projectId })) + .private const targetDb = await getRegionDb({ regionKey }) - const targetObjectStorage = await getRegionObjectStorage({ regionKey }) + const targetObjectStorage = (await getRegionObjectStorage({ regionKey })) + .private // Move project to target region const project = await withTransaction( diff --git a/packages/server/modules/multiregion/tests/e2e/serverAdmin.graph.spec.ts b/packages/server/modules/multiregion/tests/e2e/serverAdmin.graph.spec.ts index 96f56c821..bdce57304 100644 --- a/packages/server/modules/multiregion/tests/e2e/serverAdmin.graph.spec.ts +++ b/packages/server/modules/multiregion/tests/e2e/serverAdmin.graph.spec.ts @@ -74,7 +74,9 @@ isEnabled Promise.resolve() ) MultiRegionBlobStorageSelectorMock.mockFunction('initializeRegion', async () => - Promise.resolve(undefined as unknown as ObjectStorage) + Promise.resolve( + undefined as unknown as { private: ObjectStorage; public: ObjectStorage } + ) ) await beforeEachContext() diff --git a/packages/server/modules/multiregion/utils/blobStorageSelector.ts b/packages/server/modules/multiregion/utils/blobStorageSelector.ts index ddc0d0c9a..29d6c852b 100644 --- a/packages/server/modules/multiregion/utils/blobStorageSelector.ts +++ b/packages/server/modules/multiregion/utils/blobStorageSelector.ts @@ -1,6 +1,7 @@ import { getMainObjectStorage, getObjectStorage, + getPublicMainObjectStorage, ObjectStorage } from '@/modules/blobstorage/clients/objectStorage' import { ensureStorageAccessFactory } from '@/modules/blobstorage/repositories/blobs' @@ -20,7 +21,7 @@ import { Optional } from '@speckle/shared' import { BlobStorageConfig } from '@speckle/shared/environment/db' type RegionStorageClients = { - [regionKey: string]: ObjectStorage + [regionKey: string]: { private: ObjectStorage; public: ObjectStorage } } let initializedClients: Optional = undefined @@ -36,7 +37,8 @@ export const initializeRegion = async (params: { */ config?: BlobStorageConfig }) => { - if (!isMultiRegionBlobStorageEnabled()) return getMainObjectStorage() + if (!isMultiRegionBlobStorageEnabled()) + return { private: getMainObjectStorage(), public: getPublicMainObjectStorage() } const { regionKey } = params let config = params.config @@ -66,10 +68,33 @@ export const initializeRegion = async (params: { // Only add, if clients already initialized if (initializedClients) { - initializedClients[regionKey] = storage + initializedClients[regionKey] = { private: storage, public: storage } } - return storage + if (config.publicEndpoint) { + const publicStorage = getObjectStorage({ + credentials: { + accessKeyId: config.accessKey, + secretAccessKey: config.secretKey + }, + endpoint: config.publicEndpoint, + region: config.s3Region, + bucket: config.bucket + }) + + // ensure it works + const ensure = ensureStorageAccessFactory({ storage: publicStorage }) + await ensure({ createBucketIfNotExists: config.createBucketIfNotExists }) + + // Only add, if clients already initialized + if (initializedClients) { + initializedClients[regionKey] = { private: storage, public: publicStorage } + } + + return { private: storage, public: publicStorage } + } + + return { private: storage, public: storage } } /** @@ -99,7 +124,8 @@ export const getRegisteredRegionClients = async (): Promise { - if (!isMultiRegionBlobStorageEnabled()) return getMainObjectStorage() + if (!isMultiRegionBlobStorageEnabled()) + return { private: getMainObjectStorage(), public: getPublicMainObjectStorage() } const clients = await getRegisteredRegionClients() let storage = clients[regionKey] @@ -123,5 +149,7 @@ export const getProjectObjectStorage: GetProjectObjectStorage = async ({ projectId }) => { const regionKey = await getProjectRegionKey({ projectId }) - return regionKey ? getRegionObjectStorage({ regionKey }) : getMainObjectStorage() + return regionKey + ? getRegionObjectStorage({ regionKey }) + : { private: getMainObjectStorage(), public: getPublicMainObjectStorage() } } diff --git a/packages/server/modules/shared/helpers/envHelper.ts b/packages/server/modules/shared/helpers/envHelper.ts index 4da4b9798..7d4a9d501 100644 --- a/packages/server/modules/shared/helpers/envHelper.ts +++ b/packages/server/modules/shared/helpers/envHelper.ts @@ -125,6 +125,10 @@ export const previewServiceShouldUsePrivateObjectsServerUrl = (): boolean => { return getBooleanFromEnv('PREVIEW_SERVICE_USE_PRIVATE_OBJECTS_SERVER_URL') } +export const fileImportServiceShouldUsePrivateObjectsServerUrl = (): boolean => { + return getBooleanFromEnv('FILEIMPORT_SERVICE_USE_PRIVATE_OBJECTS_SERVER_URL') +} + export const getFileImportServiceRhinoParserRedisUrl = (): string | undefined => { return getStringFromEnv('FILEIMPORT_SERVICE_RHINO_REDIS_URL', { unsafe: true }) } @@ -429,6 +433,10 @@ export function getS3Endpoint() { return getStringFromEnv('S3_ENDPOINT') } +export function getS3PublicEndpoint() { + return getStringFromEnv('S3_PUBLIC_ENDPOINT', { unsafe: true }) +} + export function getS3Region(aDefault: string = 'us-east-1') { return process.env.S3_REGION || aDefault } diff --git a/packages/server/test/speckle-helpers/blobHelper.ts b/packages/server/test/speckle-helpers/blobHelper.ts index 21282952c..17fea6bd2 100644 --- a/packages/server/test/speckle-helpers/blobHelper.ts +++ b/packages/server/test/speckle-helpers/blobHelper.ts @@ -25,7 +25,7 @@ export const createTestBlob = async (params: { userId: string; projectId: string return await uploadFileStreamFactory({ upsertBlob: upsertBlobFactory({ db: projectDb }), updateBlob: updateBlobFactory({ db: projectDb }), - storeFileStream: storeFileStreamFactory({ storage: projectStorage }) + storeFileStream: storeFileStreamFactory({ storage: projectStorage.public }) })( { userId, diff --git a/packages/shared/src/environment/db.ts b/packages/shared/src/environment/db.ts index 7ed22f8dc..5a453b35f 100644 --- a/packages/shared/src/environment/db.ts +++ b/packages/shared/src/environment/db.ts @@ -32,7 +32,19 @@ const regionConfigSchema = z.object({ .optional() }), blobStorage: z.object({ - endpoint: z.string().url().describe('URL of the S3-compatible storage endpoint'), + endpoint: z + .string() + .url() + .describe( + 'URL of the S3-compatible storage endpoint, accessible from the server' + ), + publicEndpoint: z + .string() + .url() + .optional() + .describe( + 'Public URL of the S3-compatible storage endpoint, accessible from clients via the public internet' + ), accessKey: z.string().describe('Access key for the S3-compatible storage endpoint'), secretKey: z.string().describe('Secret key for the S3-compatible storage endpoint'), bucket: z.string().describe('Name of the S3-compatible storage bucket'),