huanld/speckle-server
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat(file uploads): large file uploads API is always available (#5103)

Browse Source
This commit is contained in:
Iain Sproat
2025-07-23 13:36:27 +01:00
committed by GitHub
parent 3a34c14461
commit d0e3377978
12 changed files with 1002 additions and 1051 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker-compose-speckle.yml
-3
View File
@@ -32,7 +32,6 @@ services:
NUXT_PUBLIC_BACKEND_API_ORIGIN: 'http://speckle-server:3000'
NUXT_PUBLIC_LOG_LEVEL: 'warn'
NUXT_REDIS_URL: 'redis://redis'
NUXT_PUBLIC_FF_LARGE_FILE_IMPORTS_ENABLED: 'true'
LOG_LEVEL: 'info'
LOG_PRETTY: 'true'
depends_on:
@@ -95,8 +94,6 @@ services:
FRONTEND_ORIGIN: 'http://127.0.0.1'
ONBOARDING_STREAM_URL: 'https://latest.speckle.systems/projects/843d07eb10'
FF_LARGE_FILE_IMPORTS_ENABLED: 'true'
depends_on:
[]
# - minio
packages/frontend-2/lib/core/composables/fileImport.ts
+2 -2
View File
@@ -204,7 +204,7 @@ const startFileImportMutation = graphql(`
export const useFileImportApi = () => {
const {
public: { FF_LARGE_FILE_IMPORTS_ENABLED }
public: { FF_LEGACY_FILE_IMPORTS_ENABLED }
} = useRuntimeConfig()
const apollo = useApolloClient().client
const { registerActiveUpload, unregisterActiveUpload } = useGlobalFileImportManager()
@@ -321,7 +321,7 @@ export const useFileImportApi = () => {
const uploadId = resolveUploadId()
try {
registerActiveUpload(uploadId)
return await (FF_LARGE_FILE_IMPORTS_ENABLED ? importFileV2 : importFileLegacy)(
return await (FF_LEGACY_FILE_IMPORTS_ENABLED ? importFileLegacy : importFileV2)(
...args
)
} finally {
packages/server/modules/blobstorage/tests/integration/presigned.integration.spec.ts
+326 -336
View File
@@ -37,341 +37,331 @@ import type {
GetBlobMetadata,
RegisterCompletedUpload
} from '@/modules/blobstorage/domain/operations'
import { getFeatureFlags } from '@speckle/shared/environment'
const { FF_LARGE_FILE_IMPORTS_ENABLED } = getFeatureFlags()
;(FF_LARGE_FILE_IMPORTS_ENABLED ? describe : describe.skip)(
'Presigned integration @blobstorage',
async () => {
const serverAdmin = { id: '', name: 'server admin', role: Roles.Server.Admin }
const ownedProject = {
id: '',
name: 'owned stream',
isPublic: false
}
let projectDb: Knex
let projectStorage: { private: ObjectStorage; public: ObjectStorage }
let getBlobMetadata: GetBlobMetadata
before(async () => {
await beforeEachContext()
serverAdmin.id = (await createTestUser(serverAdmin)).id
ownedProject.id = (
await createProject({
...ownedProject,
ownerId: serverAdmin.id
})
).id
;[projectDb, projectStorage] = await Promise.all([
getProjectDbClient({ projectId: ownedProject.id }),
getProjectObjectStorage({ projectId: ownedProject.id })
])
getBlobMetadata = getBlobMetadataFactory({ db: projectDb })
})
describe('generate a presigned URL', () => {
let SUT: ReturnType<typeof generatePresignedUrlFactory>
before(() => {
SUT = generatePresignedUrlFactory({
getSignedUrl: getSignedUrlFactory({
objectStorage: projectStorage.public
}),
upsertBlob: upsertBlobFactory({
db: projectDb
})
})
})
it('should provision a blob with uploadStatus 0 and return a presigned URL', async () => {
const blobId = cryptoRandomString({ length: 10 })
const fileName = `test-file-${cryptoRandomString({ length: 10 })}.stl`
const expiryDuration = 20
const url = await SUT({
blobId,
fileName,
projectId: ownedProject.id,
userId: serverAdmin.id,
urlExpiryDurationSeconds: expiryDuration
})
expect(url).to.contain(`/assets/${ownedProject.id}/${blobId}?`)
expect(url).to.contain(`X-Amz-Expires=${expiryDuration}`)
expect(url).to.contain('X-Amz-Credential') // we don't need to check the whole url, we can trust S3; only that it appears to be signed
const storedBlob = await getBlobMetadata({ blobId, streamId: ownedProject.id })
expect(storedBlob).to.exist
expect(storedBlob.id).to.equal(blobId)
expect(storedBlob.uploadStatus).to.equal(0)
expect(storedBlob.fileName).to.equal(fileName)
expect(storedBlob.streamId).to.equal(ownedProject.id)
expect(storedBlob.fileType).to.equal('stl')
})
})
describe('register completed upload', () => {
let generatePresignedUrl: GeneratePresignedUrl
let SUT: RegisterCompletedUpload
before(() => {
generatePresignedUrl = generatePresignedUrlFactory({
getSignedUrl: getSignedUrlFactory({
objectStorage: projectStorage.public
}),
upsertBlob: upsertBlobFactory({
db: projectDb
})
})
SUT = registerCompletedUploadFactory({
getBlob: getBlobFactory({ db: projectDb }),
getBlobMetadata: getBlobMetadataFromStorage({
objectStorage: projectStorage.public
}),
updateBlob: updateBlobFactory({
db: projectDb
}),
logger: testLogger
})
})
it('should update the blob with uploadStatus 1 given the correct ETag', async () => {
const blobId = cryptoRandomString({ length: 10 })
const fileName = `test-file-${cryptoRandomString({ length: 10 })}.stl`
const expiryDuration = 1 * TIME.minute
const url = await generatePresignedUrl({
blobId,
fileName,
projectId: ownedProject.id,
userId: serverAdmin.id,
urlExpiryDurationSeconds: expiryDuration
})
const fileSize = 100
const response = await axios.put(url, cryptoRandomString({ length: fileSize }))
expect(
response.status,
JSON.stringify({ statusText: response.statusText, body: response.data })
).to.equal(200)
expect(response.headers['etag'], JSON.stringify(response.headers)).to.exist
const expectedETag = response.headers['etag']
const storedBlob = await SUT({
blobId,
projectId: ownedProject.id,
expectedETag,
maximumFileSize: 1 * 1024 * 1024 // 1 MB
})
expect(storedBlob).to.exist
expect(storedBlob.uploadStatus).to.equal(BlobUploadStatus.Completed)
expect(storedBlob.fileHash).to.equal(expectedETag)
expect(storedBlob.fileSize).to.equal(fileSize)
})
it('should throw an StoredBlobAccessError if the blob cannot be found', async () => {
const blobId = cryptoRandomString({ length: 10 })
const fileName = `test-file-${cryptoRandomString({ length: 10 })}.stl`
const expiryDuration = 1 * TIME.minute
await generatePresignedUrl({
blobId,
fileName,
projectId: ownedProject.id,
userId: serverAdmin.id,
urlExpiryDurationSeconds: expiryDuration
})
// do not upload anything, skip straight to registering the 'completed' upload
const thrownError = await expectToThrow(
async () =>
await SUT({
blobId,
projectId: ownedProject.id,
expectedETag: cryptoRandomString({ length: 32 }),
maximumFileSize: 1 * 1024 * 1024 // 1 MB
})
)
expect(thrownError).to.be.instanceOf(StoredBlobAccessError)
})
it('should throw an UserInputError if the blob exceeds the maximum allowed size', async () => {
const blobId = cryptoRandomString({ length: 10 })
const fileName = `test-file-${cryptoRandomString({ length: 10 })}.stl`
const expiryDuration = 1 * TIME.minute
const url = await generatePresignedUrl({
blobId,
fileName,
projectId: ownedProject.id,
userId: serverAdmin.id,
urlExpiryDurationSeconds: expiryDuration
})
const response = await axios.put(url, 'test content') // more than 1 byte long
expect(
response.status,
JSON.stringify({ statusText: response.statusText, body: response.data })
).to.equal(200)
expect(response.headers['etag'], JSON.stringify(response.headers)).to.exist
const expectedETag = response.headers['etag']
const thrownError = await expectToThrow(
async () =>
await SUT({
blobId,
projectId: ownedProject.id,
expectedETag,
maximumFileSize: 1 // 1 byte max
})
)
expect(thrownError).to.be.instanceOf(UserInputError)
// Verify that the blob is now marked as in error state
const blobs = await getBlobsFactory({ db: projectDb })({
streamId: ownedProject.id,
blobIds: [blobId]
})
expect(blobs).to.have.lengthOf(1)
expect(blobs[0].uploadStatus).to.equal(BlobUploadStatus.Error)
expect(blobs[0].uploadError).to.include('[FILE_SIZE_EXCEEDED]')
})
it('re-registering should be idempotent', async () => {
const blobId = cryptoRandomString({ length: 10 })
const fileName = `test-file-${cryptoRandomString({ length: 10 })}.stl`
const expiryDuration = 1 * TIME.minute
const url = await generatePresignedUrl({
blobId,
fileName,
projectId: ownedProject.id,
userId: serverAdmin.id,
urlExpiryDurationSeconds: expiryDuration
})
const response = await axios.put(url, 'test content')
expect(
response.status,
JSON.stringify({ statusText: response.statusText, body: response.data })
).to.equal(200)
expect(response.headers['etag'], JSON.stringify(response.headers)).to.exist
const expectedETag = response.headers['etag']
const storedBlob = await SUT({
blobId,
projectId: ownedProject.id,
expectedETag,
maximumFileSize: 1 * 1024 * 1024 // 1 MB
})
expect(storedBlob).to.exist
expect(storedBlob.uploadStatus).to.equal(BlobUploadStatus.Completed)
expect(storedBlob.fileHash).to.equal(expectedETag)
const secondAttempt = await expectToThrow(
async () =>
await SUT({
blobId,
projectId: ownedProject.id,
expectedETag,
maximumFileSize: 1 * 1024 * 1024 // 1 MB
})
)
expect(secondAttempt).to.be.instanceOf(AlreadyRegisteredBlobError)
expect(secondAttempt.message).to.include(
'Blob already registered and completed'
)
})
it('re-registering with increased maximum file size after failure does not change anything', async () => {
const blobId = cryptoRandomString({ length: 10 })
const fileName = `test-file-${cryptoRandomString({ length: 10 })}.stl`
const expiryDuration = 1 * TIME.minute
const url = await generatePresignedUrl({
blobId,
fileName,
projectId: ownedProject.id,
userId: serverAdmin.id,
urlExpiryDurationSeconds: expiryDuration
})
const response = await axios.put(url, cryptoRandomString({ length: 100 })) // more than 1 byte long
expect(
response.status,
JSON.stringify({ statusText: response.statusText, body: response.data })
).to.equal(200)
expect(response.headers['etag'], JSON.stringify(response.headers)).to.exist
const expectedETag = response.headers['etag']
const thrownError = await expectToThrow(
async () =>
await SUT({
blobId,
projectId: ownedProject.id,
expectedETag,
maximumFileSize: 1 // our content exceeds this maximum file size
})
)
expect(thrownError).to.be.instanceOf(UserInputError)
const secondAttempt = await expectToThrow(
async () =>
await SUT({
blobId,
projectId: ownedProject.id,
expectedETag,
maximumFileSize: 1 * 1024 * 1024 // bigger allowed size, because maybe some environment variables changed
})
)
expect(secondAttempt).to.be.instanceOf(AlreadyRegisteredBlobError)
expect(secondAttempt.message).to.contain('[FILE_SIZE_EXCEEDED]')
})
it('re-registering with decreased maximum file size does not change anything', async () => {
const blobId = cryptoRandomString({ length: 10 })
const fileName = `test-file-${cryptoRandomString({ length: 10 })}.stl`
const expiryDuration = 1 * TIME.minute
const url = await generatePresignedUrl({
blobId,
fileName,
projectId: ownedProject.id,
userId: serverAdmin.id,
urlExpiryDurationSeconds: expiryDuration
})
const fileSize = 100
const response = await axios.put(url, cryptoRandomString({ length: fileSize }))
expect(
response.status,
JSON.stringify({ statusText: response.statusText, body: response.data })
).to.equal(200)
expect(response.headers['etag'], JSON.stringify(response.headers)).to.exist
const expectedETag = response.headers['etag']
const storedBlob = await SUT({
blobId,
projectId: ownedProject.id,
expectedETag,
maximumFileSize: 1 * 1024 * 1024 // larger than our content
})
expect(storedBlob).to.exist
expect(storedBlob.uploadStatus).to.equal(BlobUploadStatus.Completed)
expect(storedBlob.fileHash).to.equal(expectedETag)
expect(storedBlob.fileSize).to.equal(fileSize)
const secondAttempt = await expectToThrow(
async () =>
await SUT({
blobId,
projectId: ownedProject.id,
expectedETag,
maximumFileSize: 1 // smaller than our content. But as we're already registered, we should throw an error regardless of this
})
)
expect(secondAttempt).to.be.instanceOf(AlreadyRegisteredBlobError)
expect(secondAttempt.message).to.contain(
'Blob already registered and completed'
)
})
})
describe('Presigned integration @blobstorage', async () => {
const serverAdmin = { id: '', name: 'server admin', role: Roles.Server.Admin }
const ownedProject = {
id: '',
name: 'owned stream',
isPublic: false
}
)
let projectDb: Knex
let projectStorage: { private: ObjectStorage; public: ObjectStorage }
let getBlobMetadata: GetBlobMetadata
before(async () => {
await beforeEachContext()
serverAdmin.id = (await createTestUser(serverAdmin)).id
ownedProject.id = (
await createProject({
...ownedProject,
ownerId: serverAdmin.id
})
).id
;[projectDb, projectStorage] = await Promise.all([
getProjectDbClient({ projectId: ownedProject.id }),
getProjectObjectStorage({ projectId: ownedProject.id })
])
getBlobMetadata = getBlobMetadataFactory({ db: projectDb })
})
describe('generate a presigned URL', () => {
let SUT: ReturnType<typeof generatePresignedUrlFactory>
before(() => {
SUT = generatePresignedUrlFactory({
getSignedUrl: getSignedUrlFactory({
objectStorage: projectStorage.public
}),
upsertBlob: upsertBlobFactory({
db: projectDb
})
})
})
it('should provision a blob with uploadStatus 0 and return a presigned URL', async () => {
const blobId = cryptoRandomString({ length: 10 })
const fileName = `test-file-${cryptoRandomString({ length: 10 })}.stl`
const expiryDuration = 20
const url = await SUT({
blobId,
fileName,
projectId: ownedProject.id,
userId: serverAdmin.id,
urlExpiryDurationSeconds: expiryDuration
})
expect(url).to.contain(`/assets/${ownedProject.id}/${blobId}?`)
expect(url).to.contain(`X-Amz-Expires=${expiryDuration}`)
expect(url).to.contain('X-Amz-Credential') // we don't need to check the whole url, we can trust S3; only that it appears to be signed
const storedBlob = await getBlobMetadata({ blobId, streamId: ownedProject.id })
expect(storedBlob).to.exist
expect(storedBlob.id).to.equal(blobId)
expect(storedBlob.uploadStatus).to.equal(0)
expect(storedBlob.fileName).to.equal(fileName)
expect(storedBlob.streamId).to.equal(ownedProject.id)
expect(storedBlob.fileType).to.equal('stl')
})
})
describe('register completed upload', () => {
let generatePresignedUrl: GeneratePresignedUrl
let SUT: RegisterCompletedUpload
before(() => {
generatePresignedUrl = generatePresignedUrlFactory({
getSignedUrl: getSignedUrlFactory({
objectStorage: projectStorage.public
}),
upsertBlob: upsertBlobFactory({
db: projectDb
})
})
SUT = registerCompletedUploadFactory({
getBlob: getBlobFactory({ db: projectDb }),
getBlobMetadata: getBlobMetadataFromStorage({
objectStorage: projectStorage.public
}),
updateBlob: updateBlobFactory({
db: projectDb
}),
logger: testLogger
})
})
it('should update the blob with uploadStatus 1 given the correct ETag', async () => {
const blobId = cryptoRandomString({ length: 10 })
const fileName = `test-file-${cryptoRandomString({ length: 10 })}.stl`
const expiryDuration = 1 * TIME.minute
const url = await generatePresignedUrl({
blobId,
fileName,
projectId: ownedProject.id,
userId: serverAdmin.id,
urlExpiryDurationSeconds: expiryDuration
})
const fileSize = 100
const response = await axios.put(url, cryptoRandomString({ length: fileSize }))
expect(
response.status,
JSON.stringify({ statusText: response.statusText, body: response.data })
).to.equal(200)
expect(response.headers['etag'], JSON.stringify(response.headers)).to.exist
const expectedETag = response.headers['etag']
const storedBlob = await SUT({
blobId,
projectId: ownedProject.id,
expectedETag,
maximumFileSize: 1 * 1024 * 1024 // 1 MB
})
expect(storedBlob).to.exist
expect(storedBlob.uploadStatus).to.equal(BlobUploadStatus.Completed)
expect(storedBlob.fileHash).to.equal(expectedETag)
expect(storedBlob.fileSize).to.equal(fileSize)
})
it('should throw an StoredBlobAccessError if the blob cannot be found', async () => {
const blobId = cryptoRandomString({ length: 10 })
const fileName = `test-file-${cryptoRandomString({ length: 10 })}.stl`
const expiryDuration = 1 * TIME.minute
await generatePresignedUrl({
blobId,
fileName,
projectId: ownedProject.id,
userId: serverAdmin.id,
urlExpiryDurationSeconds: expiryDuration
})
// do not upload anything, skip straight to registering the 'completed' upload
const thrownError = await expectToThrow(
async () =>
await SUT({
blobId,
projectId: ownedProject.id,
expectedETag: cryptoRandomString({ length: 32 }),
maximumFileSize: 1 * 1024 * 1024 // 1 MB
})
)
expect(thrownError).to.be.instanceOf(StoredBlobAccessError)
})
it('should throw an UserInputError if the blob exceeds the maximum allowed size', async () => {
const blobId = cryptoRandomString({ length: 10 })
const fileName = `test-file-${cryptoRandomString({ length: 10 })}.stl`
const expiryDuration = 1 * TIME.minute
const url = await generatePresignedUrl({
blobId,
fileName,
projectId: ownedProject.id,
userId: serverAdmin.id,
urlExpiryDurationSeconds: expiryDuration
})
const response = await axios.put(url, 'test content') // more than 1 byte long
expect(
response.status,
JSON.stringify({ statusText: response.statusText, body: response.data })
).to.equal(200)
expect(response.headers['etag'], JSON.stringify(response.headers)).to.exist
const expectedETag = response.headers['etag']
const thrownError = await expectToThrow(
async () =>
await SUT({
blobId,
projectId: ownedProject.id,
expectedETag,
maximumFileSize: 1 // 1 byte max
})
)
expect(thrownError).to.be.instanceOf(UserInputError)
// Verify that the blob is now marked as in error state
const blobs = await getBlobsFactory({ db: projectDb })({
streamId: ownedProject.id,
blobIds: [blobId]
})
expect(blobs).to.have.lengthOf(1)
expect(blobs[0].uploadStatus).to.equal(BlobUploadStatus.Error)
expect(blobs[0].uploadError).to.include('[FILE_SIZE_EXCEEDED]')
})
it('re-registering should be idempotent', async () => {
const blobId = cryptoRandomString({ length: 10 })
const fileName = `test-file-${cryptoRandomString({ length: 10 })}.stl`
const expiryDuration = 1 * TIME.minute
const url = await generatePresignedUrl({
blobId,
fileName,
projectId: ownedProject.id,
userId: serverAdmin.id,
urlExpiryDurationSeconds: expiryDuration
})
const response = await axios.put(url, 'test content')
expect(
response.status,
JSON.stringify({ statusText: response.statusText, body: response.data })
).to.equal(200)
expect(response.headers['etag'], JSON.stringify(response.headers)).to.exist
const expectedETag = response.headers['etag']
const storedBlob = await SUT({
blobId,
projectId: ownedProject.id,
expectedETag,
maximumFileSize: 1 * 1024 * 1024 // 1 MB
})
expect(storedBlob).to.exist
expect(storedBlob.uploadStatus).to.equal(BlobUploadStatus.Completed)
expect(storedBlob.fileHash).to.equal(expectedETag)
const secondAttempt = await expectToThrow(
async () =>
await SUT({
blobId,
projectId: ownedProject.id,
expectedETag,
maximumFileSize: 1 * 1024 * 1024 // 1 MB
})
)
expect(secondAttempt).to.be.instanceOf(AlreadyRegisteredBlobError)
expect(secondAttempt.message).to.include('Blob already registered and completed')
})
it('re-registering with increased maximum file size after failure does not change anything', async () => {
const blobId = cryptoRandomString({ length: 10 })
const fileName = `test-file-${cryptoRandomString({ length: 10 })}.stl`
const expiryDuration = 1 * TIME.minute
const url = await generatePresignedUrl({
blobId,
fileName,
projectId: ownedProject.id,
userId: serverAdmin.id,
urlExpiryDurationSeconds: expiryDuration
})
const response = await axios.put(url, cryptoRandomString({ length: 100 })) // more than 1 byte long
expect(
response.status,
JSON.stringify({ statusText: response.statusText, body: response.data })
).to.equal(200)
expect(response.headers['etag'], JSON.stringify(response.headers)).to.exist
const expectedETag = response.headers['etag']
const thrownError = await expectToThrow(
async () =>
await SUT({
blobId,
projectId: ownedProject.id,
expectedETag,
maximumFileSize: 1 // our content exceeds this maximum file size
})
)
expect(thrownError).to.be.instanceOf(UserInputError)
const secondAttempt = await expectToThrow(
async () =>
await SUT({
blobId,
projectId: ownedProject.id,
expectedETag,
maximumFileSize: 1 * 1024 * 1024 // bigger allowed size, because maybe some environment variables changed
})
)
expect(secondAttempt).to.be.instanceOf(AlreadyRegisteredBlobError)
expect(secondAttempt.message).to.contain('[FILE_SIZE_EXCEEDED]')
})
it('re-registering with decreased maximum file size does not change anything', async () => {
const blobId = cryptoRandomString({ length: 10 })
const fileName = `test-file-${cryptoRandomString({ length: 10 })}.stl`
const expiryDuration = 1 * TIME.minute
const url = await generatePresignedUrl({
blobId,
fileName,
projectId: ownedProject.id,
userId: serverAdmin.id,
urlExpiryDurationSeconds: expiryDuration
})
const fileSize = 100
const response = await axios.put(url, cryptoRandomString({ length: fileSize }))
expect(
response.status,
JSON.stringify({ statusText: response.statusText, body: response.data })
).to.equal(200)
expect(response.headers['etag'], JSON.stringify(response.headers)).to.exist
const expectedETag = response.headers['etag']
const storedBlob = await SUT({
blobId,
projectId: ownedProject.id,
expectedETag,
maximumFileSize: 1 * 1024 * 1024 // larger than our content
})
expect(storedBlob).to.exist
expect(storedBlob.uploadStatus).to.equal(BlobUploadStatus.Completed)
expect(storedBlob.fileHash).to.equal(expectedETag)
expect(storedBlob.fileSize).to.equal(fileSize)
const secondAttempt = await expectToThrow(
async () =>
await SUT({
blobId,
projectId: ownedProject.id,
expectedETag,
maximumFileSize: 1 // smaller than our content. But as we're already registered, we should throw an error regardless of this
})
)
expect(secondAttempt).to.be.instanceOf(AlreadyRegisteredBlobError)
expect(secondAttempt.message).to.contain('Blob already registered and completed')
})
})
})
packages/server/modules/fileuploads/graph/resolvers/fileUploads.ts
+2 -18
View File
@@ -17,11 +17,7 @@ import {
filteredSubscribe
} from '@/modules/shared/utils/subscriptions'
import { getProjectDbClient } from '@/modules/multiregion/utils/dbSelector'
import {
BadRequestError,
ForbiddenError,
MisconfiguredEnvironmentError
} from '@/modules/shared/errors'
import { BadRequestError, ForbiddenError } from '@/modules/shared/errors'
import { throwIfAuthNotOk } from '@/modules/shared/helpers/errorHelper'
import {
fileImportServiceShouldUsePrivateObjectsServerUrl,
@@ -72,8 +68,7 @@ import type {
} from '@/modules/fileuploads/helpers/types'
import type { GraphQLContext } from '@/modules/shared/helpers/typeHelper'
const { FF_LARGE_FILE_IMPORTS_ENABLED, FF_NEXT_GEN_FILE_IMPORTER_ENABLED } =
getFeatureFlags()
const { FF_NEXT_GEN_FILE_IMPORTER_ENABLED } = getFeatureFlags()
const getFileUploadModel = async (params: {
upload: FileUploadRecord | FileUploadRecordV2
@@ -101,11 +96,6 @@ const getFileUploadModel = async (params: {
const fileUploadMutations: Resolvers['FileUploadMutations'] = {
async generateUploadUrl(_parent, args, ctx) {
if (!FF_LARGE_FILE_IMPORTS_ENABLED) {
throw new MisconfiguredEnvironmentError(
'The large file import feature is not enabled on this server. Please contact your Speckle administrator.'
)
}
const { projectId } = args.input
if (!ctx.userId) {
throw new ForbiddenError('No userId provided')
@@ -172,12 +162,6 @@ const fileUploadMutations: Resolvers['FileUploadMutations'] = {
if (!isFileUploadsEnabled())
throw new BadRequestError('File uploads are not enabled for this server')
if (!FF_LARGE_FILE_IMPORTS_ENABLED) {
throw new MisconfiguredEnvironmentError(
'The large file import feature is not enabled on this server. Please contact your Speckle administrator.'
)
}
const [projectDb, projectStorage] = await Promise.all([
getProjectDbClient({ projectId }),
getProjectObjectStorage({ projectId })
packages/server/modules/fileuploads/rest/router.ts
+4 -9
View File
@@ -14,9 +14,6 @@ import type { Nullable } from '@speckle/shared'
import { ensureError } from '@speckle/shared'
import { UploadRequestErrorMessage } from '@/modules/fileuploads/helpers/rest'
import { getEventBus } from '@/modules/shared/services/eventBus'
import { getFeatureFlags } from '@/modules/shared/helpers/envHelper'
const { FF_LARGE_FILE_IMPORTS_ENABLED } = getFeatureFlags()
export const fileuploadRouterFactory = (): Router => {
const processNewFileStream = processNewFileStreamFactory()
@@ -97,12 +94,10 @@ export const fileuploadRouterFactory = (): Router => {
res.status(500)
}
if (FF_LARGE_FILE_IMPORTS_ENABLED) {
res.setHeader(
'Warning',
'Deprecated API; use POST /graphql (mutation.fileUploadMutations.generateUploadUrl), then PUT (to the provided url), then POST /graphql (mutation.fileUploadMutations.startFileImport)'
)
}
res.setHeader(
'Warning',
'Deprecated API; use POST /graphql (mutation.fileUploadMutations.generateUploadUrl), then PUT (to the provided url), then POST /graphql (mutation.fileUploadMutations.startFileImport)'
)
res.status(201).send({ uploadResults })
},
packages/server/modules/fileuploads/tests/e2e/presigned.graph.spec.ts
+309 -315
View File
@@ -12,9 +12,6 @@ import { expect } from 'chai'
import cryptoRandomString from 'crypto-random-string'
import gql from 'graphql-tag'
import type { SetNonNullable } from 'type-fest'
import { getFeatureFlags } from '@speckle/shared/environment'
const { FF_LARGE_FILE_IMPORTS_ENABLED } = getFeatureFlags()
const testForbiddenResponse = (
// eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -193,336 +190,333 @@ const startFileImport = async (params: TestContext) => {
})
}
;(FF_LARGE_FILE_IMPORTS_ENABLED ? describe : describe.skip)(
'Presigned graph @fileuploads',
async () => {
const serverAdmin = { id: '', name: 'server admin', role: Roles.Server.Admin }
const regularServerUser = {
id: '',
name: 'regular server user',
role: Roles.Server.User
}
const archivedUser = {
id: '',
name: 'archived user',
role: Roles.Server.ArchivedUser
}
const unaffiliatedUser = {
id: '',
name: 'unaffiliated user',
role: Roles.Server.Guest
}
describe('Presigned graph @fileuploads', async () => {
const serverAdmin = { id: '', name: 'server admin', role: Roles.Server.Admin }
const regularServerUser = {
id: '',
name: 'regular server user',
role: Roles.Server.User
}
const archivedUser = {
id: '',
name: 'archived user',
role: Roles.Server.ArchivedUser
}
const unaffiliatedUser = {
id: '',
name: 'unaffiliated user',
role: Roles.Server.Guest
}
const ownedProject = {
id: '',
name: 'owned stream',
isPublic: false
}
const contributorProject = {
id: '',
name: 'contributions are welcome',
isPublic: false
}
const reviewerProject = {
id: '',
name: 'reviewer stream',
isPublic: false
}
const noAccessProject = {
id: '',
name: 'cannot touch this',
isPublic: false
}
const publicProject = {
id: '',
name: 'everyone can look',
isPublic: true
}
const ownedProject = {
id: '',
name: 'owned stream',
isPublic: false
}
const contributorProject = {
id: '',
name: 'contributions are welcome',
isPublic: false
}
const reviewerProject = {
id: '',
name: 'reviewer stream',
isPublic: false
}
const noAccessProject = {
id: '',
name: 'cannot touch this',
isPublic: false
}
const publicProject = {
id: '',
name: 'everyone can look',
isPublic: true
}
before(async () => {
await beforeEachContext()
serverAdmin.id = (await createTestUser(serverAdmin)).id
regularServerUser.id = (await createTestUser(regularServerUser)).id
archivedUser.id = (await createTestUser(archivedUser)).id
unaffiliatedUser.id = (await createTestUser(unaffiliatedUser)).id
before(async () => {
await beforeEachContext()
serverAdmin.id = (await createTestUser(serverAdmin)).id
regularServerUser.id = (await createTestUser(regularServerUser)).id
archivedUser.id = (await createTestUser(archivedUser)).id
unaffiliatedUser.id = (await createTestUser(unaffiliatedUser)).id
ownedProject.id = (
await createProject({
...ownedProject,
ownerId: serverAdmin.id
})
).id
ownedProject.id = (
await createProject({
...ownedProject,
ownerId: serverAdmin.id
})
).id
contributorProject.id = (
await createProject({
...contributorProject,
ownerId: serverAdmin.id
})
).id
contributorProject.id = (
await createProject({
...contributorProject,
ownerId: serverAdmin.id
})
).id
reviewerProject.id = (
await createProject({
...reviewerProject,
ownerId: serverAdmin.id
})
).id
reviewerProject.id = (
await createProject({
...reviewerProject,
ownerId: serverAdmin.id
})
).id
noAccessProject.id = (
await createProject({
...noAccessProject,
ownerId: serverAdmin.id
})
).id
noAccessProject.id = (
await createProject({
...noAccessProject,
ownerId: serverAdmin.id
})
).id
publicProject.id = (
await createProject({
...publicProject,
ownerId: serverAdmin.id
})
).id
})
publicProject.id = (
await createProject({
...publicProject,
ownerId: serverAdmin.id
})
).id
})
const testData: {
user: Nullable<{ id: string; name: string; role: ServerRoles }>
projectData: {
project: { id: string; name: string; isPublic: boolean }
projectRole: Nullable<StreamRoles>
cases: {
testCase: (params: TestContext) => Promise<void>
shouldSucceed: boolean
}[]
const testData: {
user: Nullable<{ id: string; name: string; role: ServerRoles }>
projectData: {
project: { id: string; name: string; isPublic: boolean }
projectRole: Nullable<StreamRoles>
cases: {
testCase: (params: TestContext) => Promise<void>
shouldSucceed: boolean
}[]
}[] = <const>[
{
user: regularServerUser,
projectData: [
{
project: ownedProject,
projectRole: Roles.Stream.Owner,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: true },
{ testCase: startFileImport, shouldSucceed: true }
]
},
{
project: contributorProject,
projectRole: Roles.Stream.Contributor,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: true },
{ testCase: startFileImport, shouldSucceed: true }
]
},
{
project: reviewerProject,
projectRole: Roles.Stream.Reviewer,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
},
{
project: noAccessProject,
projectRole: null,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
},
{
project: publicProject,
projectRole: null,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
}
]
},
{
user: archivedUser,
projectData: [
{
project: ownedProject,
projectRole: Roles.Stream.Owner,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
},
{
project: contributorProject,
projectRole: Roles.Stream.Contributor,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
},
{
project: reviewerProject,
projectRole: Roles.Stream.Reviewer,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
},
{
project: noAccessProject,
projectRole: null,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
},
{
project: publicProject,
projectRole: null,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
}
]
},
{
user: unaffiliatedUser,
projectData: [
{
project: ownedProject,
projectRole: null,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
},
{
project: contributorProject,
projectRole: null,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
},
{
project: reviewerProject,
projectRole: null,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
},
{
project: noAccessProject,
projectRole: null,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
},
{
project: publicProject,
projectRole: null,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
}
]
},
{
user: null,
projectData: [
{
project: ownedProject,
projectRole: null,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
},
{
project: contributorProject,
projectRole: null,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
},
{
project: reviewerProject,
projectRole: null,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
},
{
project: noAccessProject,
projectRole: null,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
},
{
project: publicProject,
projectRole: null,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
}
]
}
]
}[]
}[] = <const>[
{
user: regularServerUser,
projectData: [
{
project: ownedProject,
projectRole: Roles.Stream.Owner,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: true },
{ testCase: startFileImport, shouldSucceed: true }
]
},
{
project: contributorProject,
projectRole: Roles.Stream.Contributor,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: true },
{ testCase: startFileImport, shouldSucceed: true }
]
},
{
project: reviewerProject,
projectRole: Roles.Stream.Reviewer,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
},
{
project: noAccessProject,
projectRole: null,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
},
{
project: publicProject,
projectRole: null,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
}
]
},
{
user: archivedUser,
projectData: [
{
project: ownedProject,
projectRole: Roles.Stream.Owner,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
},
{
project: contributorProject,
projectRole: Roles.Stream.Contributor,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
},
{
project: reviewerProject,
projectRole: Roles.Stream.Reviewer,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
},
{
project: noAccessProject,
projectRole: null,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
},
{
project: publicProject,
projectRole: null,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
}
]
},
{
user: unaffiliatedUser,
projectData: [
{
project: ownedProject,
projectRole: null,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
},
{
project: contributorProject,
projectRole: null,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
},
{
project: reviewerProject,
projectRole: null,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
},
{
project: noAccessProject,
projectRole: null,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
},
{
project: publicProject,
projectRole: null,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
}
]
},
{
user: null,
projectData: [
{
project: ownedProject,
projectRole: null,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
},
{
project: contributorProject,
projectRole: null,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
},
{
project: reviewerProject,
projectRole: null,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
},
{
project: noAccessProject,
projectRole: null,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
},
{
project: publicProject,
projectRole: null,
cases: [
{ testCase: generateUploadUrl, shouldSucceed: false },
{ testCase: startFileImport, shouldSucceed: false }
]
}
]
}
]
testData.forEach(async (userContext) => {
const testUser = userContext.user
testData.forEach(async (userContext) => {
const testUser = userContext.user
describe(`User: ${testUser?.name ?? 'Anonymous'} as a ${
testUser?.role ?? 'anonymous user'
}`, async () => {
let apollo: Awaited<ReturnType<typeof testApolloServer>>
before(async () => {
apollo = await testApolloServer({
authUserId: testUser?.id
})
describe(`User: ${testUser?.name ?? 'Anonymous'} as a ${
testUser?.role ?? 'anonymous user'
}`, async () => {
let apollo: Awaited<ReturnType<typeof testApolloServer>>
before(async () => {
apollo = await testApolloServer({
authUserId: testUser?.id
})
})
userContext.projectData.forEach((projectContext) => {
const project = projectContext.project
const projectRole = projectContext.projectRole
userContext.projectData.forEach((projectContext) => {
const project = projectContext.project
const projectRole = projectContext.projectRole
describe(`testing ${projectContext.cases.length} cases for project "${
project.name
}" where I, "${testUser?.name ?? 'anonymous'}", ${
testUser && projectRole ? `have the role of ${projectRole}` : 'have no role'
}`, () => {
before(async () => {
if (testUser && projectRole) {
await grantProjectPermissions({
projectId: project.id,
userId: testUser.id,
role: projectRole
})
}
})
describe(`testing ${projectContext.cases.length} cases for project "${
project.name
}" where I, "${testUser?.name ?? 'anonymous'}", ${
testUser && projectRole ? `have the role of ${projectRole}` : 'have no role'
}`, () => {
before(async () => {
if (testUser && projectRole) {
await grantProjectPermissions({
projectId: project.id,
userId: testUser.id,
role: projectRole
})
}
})
projectContext.cases.forEach((value) => {
it(`${value.shouldSucceed ? 'should' : 'should not be allowed to'} ${
value.testCase.name
}`, async () => {
await value.testCase({
apollo,
projectId: project.id,
userId: testUser?.id,
fileName: `${cryptoRandomString({ length: 10 })}.${FILE_TYPE}`,
shouldSucceed: value.shouldSucceed
})
projectContext.cases.forEach((value) => {
it(`${value.shouldSucceed ? 'should' : 'should not be allowed to'} ${
value.testCase.name
}`, async () => {
await value.testCase({
apollo,
projectId: project.id,
userId: testUser?.id,
fileName: `${cryptoRandomString({ length: 10 })}.${FILE_TYPE}`,
shouldSucceed: value.shouldSucceed
})
})
})
})
})
})
}
)
})
})
packages/server/modules/fileuploads/tests/integration/presigned.integration.spec.ts
+349 -353
View File
@@ -56,367 +56,363 @@ import type { RegisterUploadCompleteAndStartFileImport } from '@/modules/fileupl
import type { BasicTestBranch } from '@/test/speckle-helpers/branchHelper'
import { createTestBranch } from '@/test/speckle-helpers/branchHelper'
const { FF_LARGE_FILE_IMPORTS_ENABLED, FF_NEXT_GEN_FILE_IMPORTER_ENABLED } =
getFeatureFlags()
const { FF_NEXT_GEN_FILE_IMPORTER_ENABLED } = getFeatureFlags()
;(FF_LARGE_FILE_IMPORTS_ENABLED ? describe : describe.skip)(
'Presigned integration @fileuploads',
async () => {
const serverAdmin = { id: '', name: 'server admin', role: Roles.Server.Admin }
const ownedProject = {
id: '',
name: 'owned stream',
isPublic: false
}
const model: BasicTestBranch = {
name: cryptoRandomString({ length: 10 }),
id: '',
streamId: '',
authorId: ''
}
describe('Presigned integration @fileuploads', async () => {
const serverAdmin = { id: '', name: 'server admin', role: Roles.Server.Admin }
const ownedProject = {
id: '',
name: 'owned stream',
isPublic: false
}
const model: BasicTestBranch = {
name: cryptoRandomString({ length: 10 }),
id: '',
streamId: '',
authorId: ''
}
let projectDb: Knex
let projectStorage: { private: ObjectStorage; public: ObjectStorage }
let projectDb: Knex
let projectStorage: { private: ObjectStorage; public: ObjectStorage }
before(async () => {
await beforeEachContext()
serverAdmin.id = (await createTestUser(serverAdmin)).id
ownedProject.id = (
await createProject({
...ownedProject,
ownerId: serverAdmin.id
})
).id
await createTestBranch({
branch: model,
stream: {
id: ownedProject.id,
name: '', //ignored
isPublic: false, //ignored
ownerId: '' //ignored
},
owner: {
name: '', //ignored
email: '', //ignored
id: serverAdmin.id
}
before(async () => {
await beforeEachContext()
serverAdmin.id = (await createTestUser(serverAdmin)).id
ownedProject.id = (
await createProject({
...ownedProject,
ownerId: serverAdmin.id
})
;[projectDb, projectStorage] = await Promise.all([
getProjectDbClient({ projectId: ownedProject.id }),
getProjectObjectStorage({ projectId: ownedProject.id })
])
).id
await createTestBranch({
branch: model,
stream: {
id: ownedProject.id,
name: '', //ignored
isPublic: false, //ignored
ownerId: '' //ignored
},
owner: {
name: '', //ignored
email: '', //ignored
id: serverAdmin.id
}
})
describe('register completed upload and start file import', () => {
let generatePresignedUrl: ReturnType<typeof generatePresignedUrlFactory>
let SUT: RegisterUploadCompleteAndStartFileImport
before(() => {
generatePresignedUrl = generatePresignedUrlFactory({
getSignedUrl: getSignedUrlFactory({
;[projectDb, projectStorage] = await Promise.all([
getProjectDbClient({ projectId: ownedProject.id }),
getProjectObjectStorage({ projectId: ownedProject.id })
])
})
describe('register completed upload and start file import', () => {
let generatePresignedUrl: ReturnType<typeof generatePresignedUrlFactory>
let SUT: RegisterUploadCompleteAndStartFileImport
before(() => {
generatePresignedUrl = generatePresignedUrlFactory({
getSignedUrl: getSignedUrlFactory({
objectStorage: projectStorage.public
}),
upsertBlob: upsertBlobFactory({
db: projectDb
})
})
const insertNewUploadAndNotify = FF_NEXT_GEN_FILE_IMPORTER_ENABLED
? insertNewUploadAndNotifyFactoryV2({
queues: [
{
supportedFileTypes: ['stl', 'obj', 'ifc'],
scheduleJob: () => Promise.resolve()
}
],
pushJobToFileImporter: pushJobToFileImporterFactory({
getServerOrigin,
createAppToken: createAppTokenFactory({
storeApiToken: storeApiTokenFactory({ db: projectDb }),
storeTokenScopes: storeTokenScopesFactory({ db: projectDb }),
storeTokenResourceAccessDefinitions:
storeTokenResourceAccessDefinitionsFactory({
db: projectDb
}),
storeUserServerAppToken: storeUserServerAppTokenFactory({
db: projectDb
})
})
}),
saveUploadFile: saveUploadFileFactoryV2({ db: projectDb }),
emit: getEventBus().emit
})
: insertNewUploadAndNotifyFactory({
saveUploadFile: saveUploadFileFactory({ db: projectDb }),
emit: getEventBus().emit
})
SUT = registerUploadCompleteAndStartFileImportFactory({
registerCompletedUpload: registerCompletedUploadFactory({
getBlob: getBlobFactory({ db: projectDb }),
getBlobMetadata: getBlobMetadataFromStorage({
objectStorage: projectStorage.public
}),
upsertBlob: upsertBlobFactory({
updateBlob: updateBlobFactory({
db: projectDb
})
})
const insertNewUploadAndNotify = FF_NEXT_GEN_FILE_IMPORTER_ENABLED
? insertNewUploadAndNotifyFactoryV2({
queues: [
{
supportedFileTypes: ['stl', 'obj', 'ifc'],
scheduleJob: () => Promise.resolve()
}
],
pushJobToFileImporter: pushJobToFileImporterFactory({
getServerOrigin,
createAppToken: createAppTokenFactory({
storeApiToken: storeApiTokenFactory({ db: projectDb }),
storeTokenScopes: storeTokenScopesFactory({ db: projectDb }),
storeTokenResourceAccessDefinitions:
storeTokenResourceAccessDefinitionsFactory({
db: projectDb
}),
storeUserServerAppToken: storeUserServerAppTokenFactory({
db: projectDb
})
})
}),
saveUploadFile: saveUploadFileFactoryV2({ db: projectDb }),
emit: getEventBus().emit
})
: insertNewUploadAndNotifyFactory({
saveUploadFile: saveUploadFileFactory({ db: projectDb }),
emit: getEventBus().emit
})
SUT = registerUploadCompleteAndStartFileImportFactory({
registerCompletedUpload: registerCompletedUploadFactory({
getBlob: getBlobFactory({ db: projectDb }),
getBlobMetadata: getBlobMetadataFromStorage({
objectStorage: projectStorage.public
}),
updateBlob: updateBlobFactory({
db: projectDb
}),
logger: testLogger
}),
insertNewUploadAndNotify,
getFileInfo: getFileInfoFactoryV2({ db: projectDb }),
getModelsByIds: getBranchesByIdsFactory({ db: projectDb })
})
})
it('should create a record for the uploaded file', async () => {
const fileId = cryptoRandomString({ length: 10 })
const fileSize = 10
const fileName = `test-file-${cryptoRandomString({ length: 10 })}.stl`
const expiryDuration = 1 * TIME.minute
const url = await generatePresignedUrl({
blobId: fileId,
fileName,
projectId: ownedProject.id,
userId: serverAdmin.id,
urlExpiryDurationSeconds: expiryDuration
})
const response = await axios.put(url, cryptoRandomString({ length: fileSize }))
expect(
response.status,
JSON.stringify({ statusText: response.statusText, body: response.data })
).to.equal(200)
expect(response.headers['etag'], JSON.stringify(response.headers)).to.exist
const expectedETag = response.headers['etag']
const storedFile = await SUT({
fileId,
modelId: model.id,
userId: serverAdmin.id,
projectId: ownedProject.id,
expectedETag,
maximumFileSize: 1 * 1024 * 1024 // 1 MB
})
expect(storedFile).to.exist
expect(storedFile.fileType).to.equal('stl')
expect(storedFile.fileSize).to.equal(fileSize)
expect(storedFile.uploadComplete).to.be.true
})
it('should throw a StoredBlobAccessError if the blob cannot be found', async () => {
const fileId = cryptoRandomString({ length: 10 })
const fileName = `test-file-${cryptoRandomString({ length: 10 })}.stl`
const expiryDuration = 1 * TIME.minute
await generatePresignedUrl({
blobId: fileId,
fileName,
projectId: ownedProject.id,
userId: serverAdmin.id,
urlExpiryDurationSeconds: expiryDuration
})
// Do not upload any file, and skip straight to requesting the file be imported
const thrownError = await expectToThrow(
async () =>
await SUT({
fileId,
projectId: ownedProject.id,
modelId: model.id,
userId: serverAdmin.id,
expectedETag: cryptoRandomString({ length: 32 }),
maximumFileSize: 1 * 1024 * 1024 // 1 MB
})
)
expect(thrownError).to.be.instanceOf(StoredBlobAccessError)
})
it('should throw an UserInputError if the file exceeds the maximum allowed size', async () => {
const fileId = cryptoRandomString({ length: 10 })
const fileName = `test-file-${cryptoRandomString({ length: 10 })}.stl`
const expiryDuration = 1 * TIME.minute
const url = await generatePresignedUrl({
blobId: fileId,
fileName,
projectId: ownedProject.id,
userId: serverAdmin.id,
urlExpiryDurationSeconds: expiryDuration
})
const response = await axios.put(url, 'test content') // more than 1 byte long
expect(
response.status,
JSON.stringify({ statusText: response.statusText, body: response.data })
).to.equal(200)
expect(response.headers['etag'], JSON.stringify(response.headers)).to.exist
const expectedETag = response.headers['etag']
const thrownError = await expectToThrow(
async () =>
await SUT({
fileId,
modelId: model.id,
userId: serverAdmin.id,
projectId: ownedProject.id,
expectedETag,
maximumFileSize: 1 // 1 byte max
})
)
expect(thrownError).to.be.instanceOf(UserInputError)
// Verify that the blob is now marked as in error state
const blobs = await getBlobsFactory({ db: projectDb })({
streamId: ownedProject.id,
blobIds: [fileId]
})
expect(blobs).to.have.lengthOf(1)
expect(blobs[0].uploadStatus).to.equal(BlobUploadStatus.Error)
expect(blobs[0].uploadError).to.include('[FILE_SIZE_EXCEEDED]')
})
it('re-registering should be idempotent', async () => {
const fileId = cryptoRandomString({ length: 10 })
const fileSize = 10
const fileName = `test-file-${cryptoRandomString({ length: 10 })}.stl`
const expiryDuration = 1 * TIME.minute
const url = await generatePresignedUrl({
blobId: fileId,
fileName,
projectId: ownedProject.id,
userId: serverAdmin.id,
urlExpiryDurationSeconds: expiryDuration
})
const response = await axios.put(url, cryptoRandomString({ length: fileSize }))
expect(
response.status,
JSON.stringify({ statusText: response.statusText, body: response.data })
).to.equal(200)
expect(response.headers['etag'], JSON.stringify(response.headers)).to.exist
const expectedETag = response.headers['etag']
const storedFile = await SUT({
fileId,
modelId: model.id,
userId: serverAdmin.id,
projectId: ownedProject.id,
expectedETag,
maximumFileSize: 1 * 1024 * 1024 // 1 MB
})
expect(storedFile).to.exist
expect(storedFile.fileType).to.equal('stl')
expect(storedFile.fileSize).to.equal(fileSize)
expect(storedFile.uploadComplete).to.be.true
const thrownError = await expectToThrow(
async () =>
await SUT({
fileId,
modelId: model.id,
userId: serverAdmin.id,
projectId: ownedProject.id,
expectedETag,
maximumFileSize: 1 * 1024 * 1024 // 1 MB
})
)
expect(thrownError).to.be.instanceOf(AlreadyRegisteredBlobError)
expect(thrownError.message).to.include('Blob already registered and completed')
})
it('re-registering with increased maximum file size after failure results in the file being processed', async () => {
const fileId = cryptoRandomString({ length: 10 })
const fileSize = 10
const fileName = `test-file-${cryptoRandomString({ length: 10 })}.stl`
const expiryDuration = 1 * TIME.minute
const url = await generatePresignedUrl({
blobId: fileId,
fileName,
projectId: ownedProject.id,
userId: serverAdmin.id,
urlExpiryDurationSeconds: expiryDuration
})
const response = await axios.put(url, cryptoRandomString({ length: fileSize }))
expect(
response.status,
JSON.stringify({ statusText: response.statusText, body: response.data })
).to.equal(200)
expect(response.headers['etag'], JSON.stringify(response.headers)).to.exist
const expectedETag = response.headers['etag']
const thrownError = await expectToThrow(
async () =>
await SUT({
fileId,
modelId: model.id,
userId: serverAdmin.id,
projectId: ownedProject.id,
expectedETag,
maximumFileSize: 1 // smaller than fileSize, so expected to throw
})
)
expect(thrownError).to.be.instanceOf(UserInputError)
const secondAttempt = await expectToThrow(
async () =>
await SUT({
fileId,
modelId: model.id,
userId: serverAdmin.id,
projectId: ownedProject.id,
expectedETag,
maximumFileSize: fileSize + 100 // an increased size, greater than the fileSize
})
)
expect(secondAttempt).to.be.instanceOf(AlreadyRegisteredBlobError)
expect(secondAttempt.message).to.contain('[FILE_SIZE_EXCEEDED]')
})
it('re-registering with decreased maximum file size does not change anything', async () => {
const fileId = cryptoRandomString({ length: 10 })
const fileSize = 10
const fileName = `test-file-${cryptoRandomString({ length: 10 })}.stl`
const expiryDuration = 1 * TIME.minute
const url = await generatePresignedUrl({
blobId: fileId,
fileName,
projectId: ownedProject.id,
userId: serverAdmin.id,
urlExpiryDurationSeconds: expiryDuration
})
const response = await axios.put(url, cryptoRandomString({ length: fileSize }))
expect(
response.status,
JSON.stringify({ statusText: response.statusText, body: response.data })
).to.equal(200)
expect(response.headers['etag'], JSON.stringify(response.headers)).to.exist
const expectedETag = response.headers['etag']
const storedFile = await SUT({
fileId,
modelId: model.id,
userId: serverAdmin.id,
projectId: ownedProject.id,
expectedETag,
maximumFileSize: fileSize + 100
})
expect(storedFile).to.exist
expect(storedFile.fileType).to.equal('stl')
expect(storedFile.fileSize).to.equal(fileSize)
expect(storedFile.uploadComplete).to.be.true
const thrownError = await expectToThrow(
async () =>
await SUT({
fileId,
modelId: model.id,
userId: serverAdmin.id,
projectId: ownedProject.id,
expectedETag,
maximumFileSize: 1 // smaller than our fileSize, but it is already registered so should throw
})
)
expect(thrownError).to.be.instanceOf(AlreadyRegisteredBlobError)
expect(thrownError.message).to.include('Blob already registered and completed')
logger: testLogger
}),
insertNewUploadAndNotify,
getFileInfo: getFileInfoFactoryV2({ db: projectDb }),
getModelsByIds: getBranchesByIdsFactory({ db: projectDb })
})
})
}
)
it('should create a record for the uploaded file', async () => {
const fileId = cryptoRandomString({ length: 10 })
const fileSize = 10
const fileName = `test-file-${cryptoRandomString({ length: 10 })}.stl`
const expiryDuration = 1 * TIME.minute
const url = await generatePresignedUrl({
blobId: fileId,
fileName,
projectId: ownedProject.id,
userId: serverAdmin.id,
urlExpiryDurationSeconds: expiryDuration
})
const response = await axios.put(url, cryptoRandomString({ length: fileSize }))
expect(
response.status,
JSON.stringify({ statusText: response.statusText, body: response.data })
).to.equal(200)
expect(response.headers['etag'], JSON.stringify(response.headers)).to.exist
const expectedETag = response.headers['etag']
const storedFile = await SUT({
fileId,
modelId: model.id,
userId: serverAdmin.id,
projectId: ownedProject.id,
expectedETag,
maximumFileSize: 1 * 1024 * 1024 // 1 MB
})
expect(storedFile).to.exist
expect(storedFile.fileType).to.equal('stl')
expect(storedFile.fileSize).to.equal(fileSize)
expect(storedFile.uploadComplete).to.be.true
})
it('should throw a StoredBlobAccessError if the blob cannot be found', async () => {
const fileId = cryptoRandomString({ length: 10 })
const fileName = `test-file-${cryptoRandomString({ length: 10 })}.stl`
const expiryDuration = 1 * TIME.minute
await generatePresignedUrl({
blobId: fileId,
fileName,
projectId: ownedProject.id,
userId: serverAdmin.id,
urlExpiryDurationSeconds: expiryDuration
})
// Do not upload any file, and skip straight to requesting the file be imported
const thrownError = await expectToThrow(
async () =>
await SUT({
fileId,
projectId: ownedProject.id,
modelId: model.id,
userId: serverAdmin.id,
expectedETag: cryptoRandomString({ length: 32 }),
maximumFileSize: 1 * 1024 * 1024 // 1 MB
})
)
expect(thrownError).to.be.instanceOf(StoredBlobAccessError)
})
it('should throw an UserInputError if the file exceeds the maximum allowed size', async () => {
const fileId = cryptoRandomString({ length: 10 })
const fileName = `test-file-${cryptoRandomString({ length: 10 })}.stl`
const expiryDuration = 1 * TIME.minute
const url = await generatePresignedUrl({
blobId: fileId,
fileName,
projectId: ownedProject.id,
userId: serverAdmin.id,
urlExpiryDurationSeconds: expiryDuration
})
const response = await axios.put(url, 'test content') // more than 1 byte long
expect(
response.status,
JSON.stringify({ statusText: response.statusText, body: response.data })
).to.equal(200)
expect(response.headers['etag'], JSON.stringify(response.headers)).to.exist
const expectedETag = response.headers['etag']
const thrownError = await expectToThrow(
async () =>
await SUT({
fileId,
modelId: model.id,
userId: serverAdmin.id,
projectId: ownedProject.id,
expectedETag,
maximumFileSize: 1 // 1 byte max
})
)
expect(thrownError).to.be.instanceOf(UserInputError)
// Verify that the blob is now marked as in error state
const blobs = await getBlobsFactory({ db: projectDb })({
streamId: ownedProject.id,
blobIds: [fileId]
})
expect(blobs).to.have.lengthOf(1)
expect(blobs[0].uploadStatus).to.equal(BlobUploadStatus.Error)
expect(blobs[0].uploadError).to.include('[FILE_SIZE_EXCEEDED]')
})
it('re-registering should be idempotent', async () => {
const fileId = cryptoRandomString({ length: 10 })
const fileSize = 10
const fileName = `test-file-${cryptoRandomString({ length: 10 })}.stl`
const expiryDuration = 1 * TIME.minute
const url = await generatePresignedUrl({
blobId: fileId,
fileName,
projectId: ownedProject.id,
userId: serverAdmin.id,
urlExpiryDurationSeconds: expiryDuration
})
const response = await axios.put(url, cryptoRandomString({ length: fileSize }))
expect(
response.status,
JSON.stringify({ statusText: response.statusText, body: response.data })
).to.equal(200)
expect(response.headers['etag'], JSON.stringify(response.headers)).to.exist
const expectedETag = response.headers['etag']
const storedFile = await SUT({
fileId,
modelId: model.id,
userId: serverAdmin.id,
projectId: ownedProject.id,
expectedETag,
maximumFileSize: 1 * 1024 * 1024 // 1 MB
})
expect(storedFile).to.exist
expect(storedFile.fileType).to.equal('stl')
expect(storedFile.fileSize).to.equal(fileSize)
expect(storedFile.uploadComplete).to.be.true
const thrownError = await expectToThrow(
async () =>
await SUT({
fileId,
modelId: model.id,
userId: serverAdmin.id,
projectId: ownedProject.id,
expectedETag,
maximumFileSize: 1 * 1024 * 1024 // 1 MB
})
)
expect(thrownError).to.be.instanceOf(AlreadyRegisteredBlobError)
expect(thrownError.message).to.include('Blob already registered and completed')
})
it('re-registering with increased maximum file size after failure results in the file being processed', async () => {
const fileId = cryptoRandomString({ length: 10 })
const fileSize = 10
const fileName = `test-file-${cryptoRandomString({ length: 10 })}.stl`
const expiryDuration = 1 * TIME.minute
const url = await generatePresignedUrl({
blobId: fileId,
fileName,
projectId: ownedProject.id,
userId: serverAdmin.id,
urlExpiryDurationSeconds: expiryDuration
})
const response = await axios.put(url, cryptoRandomString({ length: fileSize }))
expect(
response.status,
JSON.stringify({ statusText: response.statusText, body: response.data })
).to.equal(200)
expect(response.headers['etag'], JSON.stringify(response.headers)).to.exist
const expectedETag = response.headers['etag']
const thrownError = await expectToThrow(
async () =>
await SUT({
fileId,
modelId: model.id,
userId: serverAdmin.id,
projectId: ownedProject.id,
expectedETag,
maximumFileSize: 1 // smaller than fileSize, so expected to throw
})
)
expect(thrownError).to.be.instanceOf(UserInputError)
const secondAttempt = await expectToThrow(
async () =>
await SUT({
fileId,
modelId: model.id,
userId: serverAdmin.id,
projectId: ownedProject.id,
expectedETag,
maximumFileSize: fileSize + 100 // an increased size, greater than the fileSize
})
)
expect(secondAttempt).to.be.instanceOf(AlreadyRegisteredBlobError)
expect(secondAttempt.message).to.contain('[FILE_SIZE_EXCEEDED]')
})
it('re-registering with decreased maximum file size does not change anything', async () => {
const fileId = cryptoRandomString({ length: 10 })
const fileSize = 10
const fileName = `test-file-${cryptoRandomString({ length: 10 })}.stl`
const expiryDuration = 1 * TIME.minute
const url = await generatePresignedUrl({
blobId: fileId,
fileName,
projectId: ownedProject.id,
userId: serverAdmin.id,
urlExpiryDurationSeconds: expiryDuration
})
const response = await axios.put(url, cryptoRandomString({ length: fileSize }))
expect(
response.status,
JSON.stringify({ statusText: response.statusText, body: response.data })
).to.equal(200)
expect(response.headers['etag'], JSON.stringify(response.headers)).to.exist
const expectedETag = response.headers['etag']
const storedFile = await SUT({
fileId,
modelId: model.id,
userId: serverAdmin.id,
projectId: ownedProject.id,
expectedETag,
maximumFileSize: fileSize + 100
})
expect(storedFile).to.exist
expect(storedFile.fileType).to.equal('stl')
expect(storedFile.fileSize).to.equal(fileSize)
expect(storedFile.uploadComplete).to.be.true
const thrownError = await expectToThrow(
async () =>
await SUT({
fileId,
modelId: model.id,
userId: serverAdmin.id,
projectId: ownedProject.id,
expectedETag,
maximumFileSize: 1 // smaller than our fileSize, but it is already registered so should throw
})
)
expect(thrownError).to.be.instanceOf(AlreadyRegisteredBlobError)
expect(thrownError.message).to.include('Blob already registered and completed')
})
})
})
packages/shared/src/environment/index.ts
+4 -5
View File
@@ -104,8 +104,7 @@ export const parseFeatureFlags = (
},
FF_NEXT_GEN_FILE_IMPORTER_ENABLED: {
schema: z.boolean(),
description:
'Enables the new file importer. Requires FF_LARGE_FILE_IMPORTS_ENABLED to be true.',
description: 'Enables the new file importer.',
defaults: { _: false }
},
FF_RHINO_FILE_IMPORTER_ENABLED: {
@@ -118,10 +117,10 @@ export const parseFeatureFlags = (
description: 'Enables the postgres based background job mechanism',
defaults: { _: false }
},
FF_LARGE_FILE_IMPORTS_ENABLED: {
FF_LEGACY_FILE_IMPORTS_ENABLED: {
schema: z.boolean(),
description:
'Enables the new file importer to handle large files via pre-signed URLs.',
'Enables the legacy file importer. This proxies file uploads via REST API on the server instead of directly PUTing files to S3 via pre-signed urls.',
defaults: { _: false }
},
FF_LEGACY_IFC_IMPORTER_ENABLED: {
@@ -170,7 +169,7 @@ export type FeatureFlags = {
FF_NEXT_GEN_FILE_IMPORTER_ENABLED: boolean
FF_RHINO_FILE_IMPORTER_ENABLED: boolean
FF_BACKGROUND_JOBS_ENABLED: boolean
FF_LARGE_FILE_IMPORTS_ENABLED: boolean
FF_LEGACY_FILE_IMPORTS_ENABLED: boolean
FF_LEGACY_IFC_IMPORTER_ENABLED: boolean
FF_EXPERIMENTAL_IFC_IMPORTER_ENABLED: boolean
}
utils/helm/speckle-server/templates/_helpers.tpl
-4
View File
@@ -1155,10 +1155,6 @@ Generate the environment variables for Speckle server and Speckle objects deploy
name: {{ default .Values.secretName .Values.ifc_import_service.db.connectionString.secretName }}
key: {{ default "fileimport_queue_postgres_url" .Values.ifc_import_service.db.connectionString.secretKey }}
{{- end }}
{{- if .Values.featureFlags.largeFileUploadsEnabled }}
- name: FF_LARGE_FILE_IMPORTS_ENABLED
value: {{ .Values.featureFlags.largeFileUploadsEnabled | quote }}
- name: FILE_UPLOAD_URL_EXPIRY_MINUTES
value: {{ .Values.file_upload_url_expiry_minutes | quote }}
{{- end }}
{{- end }}
utils/helm/speckle-server/templates/frontend_2/deployment.yml
+2 -2
View File
@@ -143,8 +143,8 @@ spec:
value: {{ .Values.featureFlags.workspacesNewPlanEnabled | quote }}
- name: NUXT_PUBLIC_FF_NEXT_GEN_FILE_IMPORTER_ENABLED
value: {{ .Values.featureFlags.nextGenFileImporterEnabled | quote }}
- name: NUXT_PUBLIC_FF_LARGE_FILE_IMPORTS_ENABLED
value: {{ .Values.featureFlags.largeFileUploadsEnabled | quote }}
- name: NUXT_PUBLIC_FF_LEGACY_FILE_IMPORTS_ENABLED
value: {{ .Values.featureFlags.legacyFileImportsEnabled | quote }}
{{- if .Values.analytics.intercom_app_id }}
- name: NUXT_PUBLIC_INTERCOM_APP_ID
value: {{ .Values.analytics.intercom_app_id | quote }}
utils/helm/speckle-server/values.schema.json
+2 -2
View File
@@ -110,9 +110,9 @@
"description": "Enables the next generation file importer",
"default": false
},
"largeFileUploadsEnabled": {
"legacyFileImportsEnabled": {
"type": "boolean",
"description": "Enables the ability to upload large files to Speckle",
"description": "Enables the legacy file upload mechanism, using REST API to proxy file uploads via the server",
"default": false
},
"experimentalIfcImporterEnabled": {
utils/helm/speckle-server/values.yaml
+2 -2
View File
@@ -65,8 +65,8 @@ featureFlags:
retryErroredPreviewsEnabled: false
## @param featureFlags.nextGenFileImporterEnabled Enables the next generation file importer
nextGenFileImporterEnabled: false
## @param featureFlags.largeFileUploadsEnabled Enables the ability to upload large files to Speckle
largeFileUploadsEnabled: false
## @param featureFlags.legacyFileImportsEnabled Enables the legacy file upload mechanism, using REST API to proxy file uploads via the server
legacyFileImportsEnabled: false
## @param featureFlags.experimentalIfcImporterEnabled Enables the ability to parse IFC files using the experimental IFC importer
experimentalIfcImporterEnabled: false
## @param featureFlags.legacyIfcImporterEnabled Enables the ability to parse IFC files using the legacy IFC importer.