feat(file imports): large file uploads now work on docker compose (#5037)

docker-compose-speckle.yml

@@ -32,6 +32,7 @@ services:
       NUXT_PUBLIC_BACKEND_API_ORIGIN: 'http://speckle-server:3000'
       NUXT_PUBLIC_LOG_LEVEL: 'warn'
       NUXT_REDIS_URL: 'redis://redis'
+      NUXT_PUBLIC_FF_LARGE_FILE_IMPORTS_ENABLED: 'true'
       LOG_LEVEL: 'info'
       LOG_PRETTY: 'true'
     depends_on:
@@ -79,9 +80,11 @@ services:
       REDIS_URL: 'redis://redis'
       PREVIEW_SERVICE_USE_PRIVATE_OBJECTS_SERVER_URL: 'true'
       PREVIEW_SERVICE_REDIS_URL: 'redis://redis'
+      FILEIMPORT_SERVICE_USE_PRIVATE_OBJECTS_SERVER_URL: 'true'
       FILEIMPORT_SERVICE_REDIS_URL: 'redis://redis'
 
       S3_ENDPOINT: 'http://minio:9000'
+      S3_PUBLIC_ENDPOINT: 'http://127.0.0.1:9000'
       S3_ACCESS_KEY: 'minioadmin'
       S3_SECRET_KEY: 'minioadmin'
       S3_BUCKET: 'speckle-server'
@@ -92,6 +95,8 @@ services:
 
       FRONTEND_ORIGIN: 'http://127.0.0.1'
       ONBOARDING_STREAM_URL: 'https://latest.speckle.systems/projects/843d07eb10'
+
+      FF_LARGE_FILE_IMPORTS_ENABLED: 'true'
     depends_on:
       []
       # - minio

packages/server/modules/blobstorage/clients/objectStorage.ts

@@ -2,6 +2,7 @@ import {
   getS3AccessKey,
   getS3BucketName,
   getS3Endpoint,
+  getS3PublicEndpoint,
   getS3Region,
   getS3SecretKey
 } from '@/modules/shared/helpers/envHelper'
@@ -51,9 +52,17 @@ export const getObjectStorage = (params: GetObjectStorageParams): ObjectStorage
 }
 
 let mainObjectStorage: Optional<ObjectStorage> = undefined
+let publicMainObjectStorage: Optional<ObjectStorage> = undefined
 
 /**
  * Get main object storage client
+ *
+ * This is used for connecting the server to the S3 host. Where the S3 host is
+ * on the same private network as the server (e.g. in a Docker network),
+ * the S3_ENDPOINT can use the private IP or DNS name of the S3 host.
+ *
+ * S3_PUBLIC_ENDPOINT can be used to connect to the S3 host via the
+ * public internet (or localhost network if running locally or testing).
  */
 export const getMainObjectStorage = (): ObjectStorage => {
   if (mainObjectStorage) return mainObjectStorage
@@ -72,6 +81,37 @@ export const getMainObjectStorage = (): ObjectStorage => {
   return mainObjectStorage
 }
 
+/**
+ * (Optional) Used to connect to the S3 host via the public endpoint.
+ * This is useful for clients that need to access the S3 bucket directly, e.g
+ * during testing or when the S3 host is not on the same private network as the server.
+ *
+ * If `S3_PUBLIC_ENDPOINT` is not set, it will return the same object storage
+ * as `getMainObjectStorage`.
+ */
+export const getPublicMainObjectStorage = (): ObjectStorage => {
+  if (publicMainObjectStorage) return publicMainObjectStorage
+
+  const endpoint = getS3PublicEndpoint()
+  if (!endpoint) {
+    // If no public endpoint is set, return the main object storage
+    return getMainObjectStorage()
+  }
+
+  const mainParams: GetObjectStorageParams = {
+    credentials: {
+      accessKeyId: getS3AccessKey(),
+      secretAccessKey: getS3SecretKey()
+    },
+    endpoint,
+    region: getS3Region(),
+    bucket: getS3BucketName()
+  }
+
+  publicMainObjectStorage = getObjectStorage(mainParams)
+  return publicMainObjectStorage
+}
+
 export const getSignedUrlFactory = (deps: {
   objectStorage: ObjectStorage
 }): GetSignedUrl => {

packages/server/modules/blobstorage/rest/router.ts

@@ -124,7 +124,9 @@ export const blobStorageRouterFactory = (): Router => {
 
       const getBlobMetadata = getBlobMetadataFactory({ db: projectDb })
       const getFileStream = getFileStreamFactory({ getBlobMetadata })
-      const getObjectStream = getObjectStreamFactory({ storage: projectStorage })
+      const getObjectStream = getObjectStreamFactory({
+        storage: projectStorage.private
+      })
 
       const { fileName } = await getBlobMetadata({
         streamId: req.params.streamId,
@@ -160,7 +162,7 @@ export const blobStorageRouterFactory = (): Router => {
       ])
 
       const getBlobMetadata = getBlobMetadataFactory({ db: projectDb })
-      const deleteObject = deleteObjectFactory({ storage: projectStorage })
+      const deleteObject = deleteObjectFactory({ storage: projectStorage.private })
       const deleteBlob = fullyDeleteBlobFactory({
         getBlobMetadata,
         deleteBlob: deleteBlobFactory({ db: projectDb }),

packages/server/modules/blobstorage/services/streams.ts

@@ -45,7 +45,7 @@ export const processNewFileStreamFactory = (): NewFileStreamProcessor => {
       getProjectObjectStorage({ projectId: streamId })
     ])
 
-    const storeFileStream = storeFileStreamFactory({ storage: projectStorage })
+    const storeFileStream = storeFileStreamFactory({ storage: projectStorage.private })
     const updateBlob = updateBlobFactory({ db: projectDb })
     const getBlobMetadata = getBlobMetadataFactory({ db: projectDb })
 
@@ -66,9 +66,9 @@ export const processNewFileStreamFactory = (): NewFileStreamProcessor => {
     })
 
     const getObjectAttributes = getObjectAttributesFactory({
-      storage: projectStorage
+      storage: projectStorage.private
     })
-    const deleteObject = deleteObjectFactory({ storage: projectStorage })
+    const deleteObject = deleteObjectFactory({ storage: projectStorage.private })
 
     busboy.on(
       'file',

packages/server/modules/blobstorage/tests/integration/blobstorage.integration.spec.ts

@@ -23,7 +23,10 @@ import { BasicTestStream, createTestStream } from '@/test/speckle-helpers/stream
 import cryptoRandomString from 'crypto-random-string'
 import { BasicTestUser, createTestUser } from '@/test/authHelper'
 import { storeFileStreamFactory } from '@/modules/blobstorage/repositories/blobs'
-import { getMainObjectStorage } from '@/modules/blobstorage/clients/objectStorage'
+import {
+  getMainObjectStorage,
+  getPublicMainObjectStorage
+} from '@/modules/blobstorage/clients/objectStorage'
 import { expect } from 'chai'
 import { UploadFileStream } from '@/modules/blobstorage/domain/operations'
 import { BlobStorageItem } from '@/modules/blobstorage/domain/types'
@@ -43,8 +46,8 @@ const buildUploadFileStream = async (params: { streamId: string | null }) => {
 
   const storage = streamId
     ? await getProjectObjectStorage({ projectId: streamId })
-    : getMainObjectStorage()
-  const storeFileStream = storeFileStreamFactory({ storage })
+    : { private: getMainObjectStorage(), public: getPublicMainObjectStorage() }
+  const storeFileStream = storeFileStreamFactory({ storage: storage.public })
   const uploadFileStream = uploadFileStreamFactory({
     upsertBlob,
     updateBlob,

packages/server/modules/blobstorage/tests/integration/presigned.integration.spec.ts

@@ -52,7 +52,7 @@ const { FF_LARGE_FILE_IMPORTS_ENABLED } = getFeatureFlags()
     }
 
     let projectDb: Knex
-    let projectStorage: ObjectStorage
+    let projectStorage: { private: ObjectStorage; public: ObjectStorage }
     let getBlobMetadata: GetBlobMetadata
 
     before(async () => {
@@ -77,7 +77,7 @@ const { FF_LARGE_FILE_IMPORTS_ENABLED } = getFeatureFlags()
       before(() => {
         SUT = generatePresignedUrlFactory({
           getSignedUrl: getSignedUrlFactory({
-            objectStorage: projectStorage
+            objectStorage: projectStorage.public
           }),
           upsertBlob: upsertBlobFactory({
             db: projectDb
@@ -117,7 +117,7 @@ const { FF_LARGE_FILE_IMPORTS_ENABLED } = getFeatureFlags()
       before(() => {
         generatePresignedUrl = generatePresignedUrlFactory({
           getSignedUrl: getSignedUrlFactory({
-            objectStorage: projectStorage
+            objectStorage: projectStorage.public
           }),
           upsertBlob: upsertBlobFactory({
             db: projectDb
@@ -126,7 +126,7 @@ const { FF_LARGE_FILE_IMPORTS_ENABLED } = getFeatureFlags()
         SUT = registerCompletedUploadFactory({
           getBlob: getBlobFactory({ db: projectDb }),
           getBlobMetadata: getBlobMetadataFromStorage({
-            objectStorage: projectStorage
+            objectStorage: projectStorage.public
           }),
           updateBlob: updateBlobFactory({
             db: projectDb

packages/server/modules/fileuploads/graph/resolvers/fileUploads.ts

@@ -25,7 +25,9 @@ import {
 } from '@/modules/shared/errors'
 import { throwIfAuthNotOk } from '@/modules/shared/helpers/errorHelper'
 import {
+  fileImportServiceShouldUsePrivateObjectsServerUrl,
   getFileUploadUrlExpiryMinutes,
+  getPrivateObjectsServerOrigin,
   getServerOrigin,
   isFileUploadsEnabled
 } from '@/modules/shared/helpers/envHelper'
@@ -134,7 +136,7 @@ const fileUploadMutations: Resolvers['FileUploadMutations'] = {
 
     const generatePresignedUrl = generatePresignedUrlFactory({
       getSignedUrl: getSignedUrlFactory({
-        objectStorage: projectStorage
+        objectStorage: projectStorage.public
       }),
       upsertBlob: upsertBlobFactory({
         db: projectDb
@@ -189,7 +191,9 @@ const fileUploadMutations: Resolvers['FileUploadMutations'] = {
     ])
 
     const pushJobToFileImporter = pushJobToFileImporterFactory({
-      getServerOrigin,
+      getServerOrigin: fileImportServiceShouldUsePrivateObjectsServerUrl()
+        ? getPrivateObjectsServerOrigin
+        : getServerOrigin,
       createAppToken: createAppTokenFactory({
         storeApiToken: storeApiTokenFactory({ db }),
         storeTokenScopes: storeTokenScopesFactory({ db }),
@@ -221,7 +225,7 @@ const fileUploadMutations: Resolvers['FileUploadMutations'] = {
             db: projectDb
           }),
           getBlobMetadata: getBlobMetadataFromStorage({
-            objectStorage: projectStorage
+            objectStorage: projectStorage.private
           })
         }),
         insertNewUploadAndNotify: FF_NEXT_GEN_FILE_IMPORTER_ENABLED

packages/server/modules/fileuploads/tests/integration/presigned.integration.spec.ts

@@ -75,7 +75,7 @@ const { FF_LARGE_FILE_IMPORTS_ENABLED, FF_NEXT_GEN_FILE_IMPORTER_ENABLED } =
     }
 
     let projectDb: Knex
-    let projectStorage: ObjectStorage
+    let projectStorage: { private: ObjectStorage; public: ObjectStorage }
 
     before(async () => {
       await beforeEachContext()
@@ -112,7 +112,7 @@ const { FF_LARGE_FILE_IMPORTS_ENABLED, FF_NEXT_GEN_FILE_IMPORTER_ENABLED } =
       before(() => {
         generatePresignedUrl = generatePresignedUrlFactory({
           getSignedUrl: getSignedUrlFactory({
-            objectStorage: projectStorage
+            objectStorage: projectStorage.public
           }),
           upsertBlob: upsertBlobFactory({
             db: projectDb
@@ -152,7 +152,7 @@ const { FF_LARGE_FILE_IMPORTS_ENABLED, FF_NEXT_GEN_FILE_IMPORTER_ENABLED } =
           registerCompletedUpload: registerCompletedUploadFactory({
             getBlob: getBlobFactory({ db: projectDb }),
             getBlobMetadata: getBlobMetadataFromStorage({
-              objectStorage: projectStorage
+              objectStorage: projectStorage.public
             }),
             updateBlob: updateBlobFactory({
               db: projectDb

packages/server/modules/gendo/graph/resolvers/index.ts

@@ -119,7 +119,9 @@ export = FF_GENDOAI_MODULE_ENABLED
             token: getGendoAIKey()
           })
 
-          const storeFileStream = storeFileStreamFactory({ storage: projectStorage })
+          const storeFileStream = storeFileStreamFactory({
+            storage: projectStorage.private
+          })
           const createRenderRequest = createRenderRequestFactory({
             uploadFileStream: uploadFileStreamFactory({
               storeFileStream,

packages/server/modules/gendo/rest/index.ts

@@ -59,7 +59,9 @@ export default function (app: express.Express) {
         getProjectObjectStorage({ projectId })
       ])
 
-      const storeFileStream = storeFileStreamFactory({ storage: projectStorage })
+      const storeFileStream = storeFileStreamFactory({
+        storage: projectStorage.private
+      })
       const updateRenderRequest = updateRenderRequestFactory({
         getRenderByGenerationId: getRenderByGenerationIdFactory({ db: projectDb }),
         uploadFileStream: uploadFileStreamFactory({

packages/server/modules/multiregion/domain/operations.ts

@@ -64,8 +64,8 @@ export type UpdateAndValidateRegion = (params: {
 
 export type GetProjectObjectStorage = (args: {
   projectId: string
-}) => Promise<ObjectStorage>
+}) => Promise<{ private: ObjectStorage; public: ObjectStorage }>
 
 export type GetRegionObjectStorage = (args: {
   regionKey: string
-}) => Promise<ObjectStorage>
+}) => Promise<{ private: ObjectStorage; public: ObjectStorage }>

packages/server/modules/multiregion/services/queue.ts

@@ -165,9 +165,11 @@ export const startQueue = async () => {
         const { projectId, regionKey } = job.data.payload
 
         const sourceDb = await getProjectDbClient({ projectId })
-        const sourceObjectStorage = await getProjectObjectStorage({ projectId })
+        const sourceObjectStorage = (await getProjectObjectStorage({ projectId }))
+          .private
         const targetDb = await getRegionDb({ regionKey })
-        const targetObjectStorage = await getRegionObjectStorage({ regionKey })
+        const targetObjectStorage = (await getRegionObjectStorage({ regionKey }))
+          .private
 
         // Move project to target region
         const project = await withTransaction(

packages/server/modules/multiregion/tests/e2e/serverAdmin.graph.spec.ts

@@ -74,7 +74,9 @@ isEnabled
           Promise.resolve()
         )
         MultiRegionBlobStorageSelectorMock.mockFunction('initializeRegion', async () =>
-          Promise.resolve(undefined as unknown as ObjectStorage)
+          Promise.resolve(
+            undefined as unknown as { private: ObjectStorage; public: ObjectStorage }
+          )
         )
 
         await beforeEachContext()

packages/server/modules/multiregion/utils/blobStorageSelector.ts

@@ -1,6 +1,7 @@
 import {
   getMainObjectStorage,
   getObjectStorage,
+  getPublicMainObjectStorage,
   ObjectStorage
 } from '@/modules/blobstorage/clients/objectStorage'
 import { ensureStorageAccessFactory } from '@/modules/blobstorage/repositories/blobs'
@@ -20,7 +21,7 @@ import { Optional } from '@speckle/shared'
 import { BlobStorageConfig } from '@speckle/shared/environment/db'
 
 type RegionStorageClients = {
-  [regionKey: string]: ObjectStorage
+  [regionKey: string]: { private: ObjectStorage; public: ObjectStorage }
 }
 
 let initializedClients: Optional<RegionStorageClients> = undefined
@@ -36,7 +37,8 @@ export const initializeRegion = async (params: {
    */
   config?: BlobStorageConfig
 }) => {
-  if (!isMultiRegionBlobStorageEnabled()) return getMainObjectStorage()
+  if (!isMultiRegionBlobStorageEnabled())
+    return { private: getMainObjectStorage(), public: getPublicMainObjectStorage() }
 
   const { regionKey } = params
   let config = params.config
@@ -66,10 +68,33 @@ export const initializeRegion = async (params: {
 
   // Only add, if clients already initialized
   if (initializedClients) {
-    initializedClients[regionKey] = storage
+    initializedClients[regionKey] = { private: storage, public: storage }
   }
 
-  return storage
+  if (config.publicEndpoint) {
+    const publicStorage = getObjectStorage({
+      credentials: {
+        accessKeyId: config.accessKey,
+        secretAccessKey: config.secretKey
+      },
+      endpoint: config.publicEndpoint,
+      region: config.s3Region,
+      bucket: config.bucket
+    })
+
+    // ensure it works
+    const ensure = ensureStorageAccessFactory({ storage: publicStorage })
+    await ensure({ createBucketIfNotExists: config.createBucketIfNotExists })
+
+    // Only add, if clients already initialized
+    if (initializedClients) {
+      initializedClients[regionKey] = { private: storage, public: publicStorage }
+    }
+
+    return { private: storage, public: publicStorage }
+  }
+
+  return { private: storage, public: storage }
 }
 
 /**
@@ -99,7 +124,8 @@ export const getRegisteredRegionClients = async (): Promise<RegionStorageClients
 }
 
 export const getRegionObjectStorage: GetRegionObjectStorage = async ({ regionKey }) => {
-  if (!isMultiRegionBlobStorageEnabled()) return getMainObjectStorage()
+  if (!isMultiRegionBlobStorageEnabled())
+    return { private: getMainObjectStorage(), public: getPublicMainObjectStorage() }
 
   const clients = await getRegisteredRegionClients()
   let storage = clients[regionKey]
@@ -123,5 +149,7 @@ export const getProjectObjectStorage: GetProjectObjectStorage = async ({
   projectId
 }) => {
   const regionKey = await getProjectRegionKey({ projectId })
-  return regionKey ? getRegionObjectStorage({ regionKey }) : getMainObjectStorage()
+  return regionKey
+    ? getRegionObjectStorage({ regionKey })
+    : { private: getMainObjectStorage(), public: getPublicMainObjectStorage() }
 }

packages/server/modules/shared/helpers/envHelper.ts

@@ -125,6 +125,10 @@ export const previewServiceShouldUsePrivateObjectsServerUrl = (): boolean => {
   return getBooleanFromEnv('PREVIEW_SERVICE_USE_PRIVATE_OBJECTS_SERVER_URL')
 }
 
+export const fileImportServiceShouldUsePrivateObjectsServerUrl = (): boolean => {
+  return getBooleanFromEnv('FILEIMPORT_SERVICE_USE_PRIVATE_OBJECTS_SERVER_URL')
+}
+
 export const getFileImportServiceRhinoParserRedisUrl = (): string | undefined => {
   return getStringFromEnv('FILEIMPORT_SERVICE_RHINO_REDIS_URL', { unsafe: true })
 }
@@ -429,6 +433,10 @@ export function getS3Endpoint() {
   return getStringFromEnv('S3_ENDPOINT')
 }
 
+export function getS3PublicEndpoint() {
+  return getStringFromEnv('S3_PUBLIC_ENDPOINT', { unsafe: true })
+}
+
 export function getS3Region(aDefault: string = 'us-east-1') {
   return process.env.S3_REGION || aDefault
 }

packages/server/test/speckle-helpers/blobHelper.ts

@@ -25,7 +25,7 @@ export const createTestBlob = async (params: { userId: string; projectId: string
   return await uploadFileStreamFactory({
     upsertBlob: upsertBlobFactory({ db: projectDb }),
     updateBlob: updateBlobFactory({ db: projectDb }),
-    storeFileStream: storeFileStreamFactory({ storage: projectStorage })
+    storeFileStream: storeFileStreamFactory({ storage: projectStorage.public })
   })(
     {
       userId,

packages/shared/src/environment/db.ts

@@ -32,7 +32,19 @@ const regionConfigSchema = z.object({
       .optional()
   }),
   blobStorage: z.object({
-    endpoint: z.string().url().describe('URL of the S3-compatible storage endpoint'),
+    endpoint: z
+      .string()
+      .url()
+      .describe(
+        'URL of the S3-compatible storage endpoint, accessible from the server'
+      ),
+    publicEndpoint: z
+      .string()
+      .url()
+      .optional()
+      .describe(
+        'Public URL of the S3-compatible storage endpoint, accessible from clients via the public internet'
+      ),
     accessKey: z.string().describe('Access key for the S3-compatible storage endpoint'),
     secretKey: z.string().describe('Secret key for the S3-compatible storage endpoint'),
     bucket: z.string().describe('Name of the S3-compatible storage bucket'),