import { beforeEachContext } from '@/test/hooks' import { NotFoundError, BadRequestError } from '@/modules/shared/errors' import { range } from 'lodash-es' import { fakeIdGenerator, createBlobs } from '@/modules/blobstorage/tests/helpers' import { uploadFileStreamFactory, getFileStreamFactory, markUploadSuccessFactory, markUploadOverFileSizeLimitFactory, fullyDeleteBlobFactory } from '@/modules/blobstorage/services/management' import { upsertBlobFactory, updateBlobFactory, getBlobMetadataFactory, getBlobMetadataCollectionFactory, blobCollectionSummaryFactory, deleteBlobFactory } from '@/modules/blobstorage/repositories' import { db } from '@/db/knex' import { cursorFromRows, decodeCursor } from '@/modules/blobstorage/helpers/db' import { BasicTestStream, createTestStream } from '@/test/speckle-helpers/streamHelper' import cryptoRandomString from 'crypto-random-string' import { BasicTestUser, createTestUser } from '@/test/authHelper' import { storeFileStreamFactory } from '@/modules/blobstorage/repositories/blobs' import { getMainObjectStorage, getPublicMainObjectStorage } from '@/modules/blobstorage/clients/objectStorage' import { expect } from 'chai' import { UploadFileStream } from '@/modules/blobstorage/domain/operations' import { BlobStorageItem } from '@/modules/blobstorage/domain/types' import { BasicTestWorkspace, createTestWorkspace } from '@/modules/workspaces/tests/helpers/creation' import { waitForRegionUser } from '@/test/speckle-helpers/regions' import { getProjectObjectStorage } from '@/modules/multiregion/utils/blobStorageSelector' import { BlobUploadStatus } from '@speckle/shared/blobs' type UploadFileStreamStreamData = Parameters[0] type UploadFileStreamBlobData = Parameters[1] const buildUploadFileStream = async (params: { streamId: string | null }) => { const { streamId } = params const storage = streamId ? await getProjectObjectStorage({ projectId: streamId }) : { private: getMainObjectStorage(), public: getPublicMainObjectStorage() } const storeFileStream = storeFileStreamFactory({ storage: storage.public }) const uploadFileStream = uploadFileStreamFactory({ upsertBlob, updateBlob, storeFileStream }) return uploadFileStream } const fakeFileStreamStore = (fakeHash: string) => async () => ({ fileHash: fakeHash }) const upsertBlob = upsertBlobFactory({ db }) const updateBlob = updateBlobFactory({ db }) const getBlobMetadata = getBlobMetadataFactory({ db }) const getBlobMetadataCollection = getBlobMetadataCollectionFactory({ db }) const blobCollectionSummary = blobCollectionSummaryFactory({ db }) const getFileStream = getFileStreamFactory({ getBlobMetadata }) const markUploadSuccess = markUploadSuccessFactory({ getBlobMetadata, updateBlob }) const markUploadOverFileSizeLimit = markUploadOverFileSizeLimitFactory({ getBlobMetadata, updateBlob }) const deleteBlob = fullyDeleteBlobFactory({ getBlobMetadata, deleteBlob: deleteBlobFactory({ db }), deleteObject: async () => {} }) describe('Blob storage @blobstorage', () => { before(async () => { await beforeEachContext() }) describe('Upload file stream', () => { const invalidData: Array< [ caseName: string, streamData: UploadFileStreamStreamData, blobData: UploadFileStreamBlobData ] > = [ [ 'stream', { streamId: 'a'.padStart(1, 'a'), userId: 'a'.padStart(10, 'b') }, { blobId: 'a'.padStart(10, 'c') } as UploadFileStreamBlobData ], [ 'user', { streamId: 'a'.padStart(10, 'a'), userId: 'a'.padStart(1, 'b') }, { blobId: 'a'.padStart(10, 'c') } as UploadFileStreamBlobData ] ] invalidData.map(([caseName, streamData, blobData]) => it(`Should throw if ${caseName} id length is incorrect`, async () => { const uploadFileStream = await buildUploadFileStream({ streamId: null }) try { await uploadFileStream(streamData, blobData) } catch (err) { if (!(err instanceof BadRequestError)) throw err expect(err.message).to.equal(`The ${caseName} id has to be of length 10`) } }) ) it('Should store file stream', async () => { const fileName = `testFile_${fakeIdGenerator()}` const streamId = fakeIdGenerator() const blobId = fakeIdGenerator() const userId = fakeIdGenerator() const fileHash = fakeIdGenerator() const uploadFileStream = uploadFileStreamFactory({ upsertBlob, updateBlob, storeFileStream: fakeFileStreamStore(fileHash) }) const blobData = await uploadFileStream( { streamId, userId }, { blobId, fileName, fileType: '.something', fileStream: Buffer.from('') } ) expect(blobData).to.deep.equal({ blobId, fileName, fileHash }) }) }) describe('Get blob metadata', () => { const testUser1: BasicTestUser = { name: 'Blob Test User #1', email: 'testUser1@gmailll.com', id: '' } const testStream1: BasicTestStream = { name: 'Blob Test Stream #1', isPublic: false, ownerId: '', id: '' } const testWorkspace1: BasicTestWorkspace = { name: 'Blob Test Workspace #1', ownerId: '', id: '', slug: '' } let testStreamBlob1: BlobStorageItem before(async () => { // Insert blob await createTestUser(testUser1) await waitForRegionUser(testUser1) await createTestWorkspace(testWorkspace1, testUser1) testStream1.workspaceId = testWorkspace1.id await createTestStream(testStream1, testUser1) testStreamBlob1 = await upsertBlob({ id: cryptoRandomString({ length: 10 }), streamId: testStream1.id, userId: testUser1.id, objectKey: 'testObjectKey', fileName: 'testFileName', fileType: 'png' }) }) it('when no blob found throws NotFoundError', async () => { try { await getBlobMetadata({ streamId: 'foo', blobId: 'bar' }) throw new Error('This should have failed') } catch (err) { if (!(err instanceof NotFoundError)) throw err } }) it('when no streamId throws ResourceMismatch', async () => { try { await getBlobMetadata({ streamId: null as unknown as string, blobId: 'bar' }) throw new Error('This should have failed') } catch (err) { if (!(err instanceof BadRequestError)) throw err } }) it('for valid input return the data', async () => { const blobMetadata = await getBlobMetadata({ streamId: testStream1.id, blobId: testStreamBlob1.id }) expect(blobMetadata).to.be.ok expect(blobMetadata.streamId).to.eq(testStream1.id) expect(blobMetadata.id).to.eq(testStreamBlob1.id) }) }) describe('Query cursor handling', () => { describe('cursorFromRows', () => { it('returns base64 encoded date ISO string', () => { const cursorTarget = 'foo' const rowItem: Record = {} const cursorValue = new Date() rowItem[cursorTarget] = cursorValue const createdCursor = cursorFromRows([rowItem], cursorTarget)! expect(Buffer.from(createdCursor, 'base64').toString()).to.equal( cursorValue.toISOString() ) }) it('return null if rows is null or empty array', () => { expect(cursorFromRows([], 'cursorTarget')).to.be.null expect(cursorFromRows(null as unknown as [], 'cursorTarget')).to.be.null }) it("throws if the cursor target doesn't find a date object", () => { try { cursorFromRows([{}], 'cursorTarget' as never) throw new Error('This should have thrown') } catch (err) { if (!(err instanceof BadRequestError)) throw err expect(err.message).to.equal('The cursor target is not a date object') } }) }) describe('decodeCursor', () => { it('throws if cursor cannot be parsed into date', () => { try { decodeCursor('asdf') throw new Error('This should have thrown') } catch (err) { if (!(err instanceof BadRequestError)) throw err expect(err.message).to.equal('The cursor is not a base64 encoded date string') } }) it('should decode cursor', () => { const cursor = new Date().toISOString() const encodedCursor = Buffer.from(cursor).toString('base64') const decoded = decodeCursor(encodedCursor) expect(decoded).to.equal(cursor) }) }) }) describe('Get blob metadata collection', () => { it('When no blobs are found, no cursor returned', async () => { const noBlobs = await getBlobMetadataCollection({ streamId: 'foo' }) expect(noBlobs).to.deep.equal({ blobs: [], cursor: null }) }) it('Returns the correct data for good input', async () => { const streamId = fakeIdGenerator() const number = 4 const createdBlobs = await createBlobs({ streamId, number }) const blobData = await getBlobMetadataCollection({ streamId }) expect(blobData.blobs).to.have.lengthOf(number) expect(blobData.blobs.map((r) => r.id)).deep.equalInAnyOrder( createdBlobs.map((b) => b.id) ) }) it('Clamps limit to predefined maximum', async () => { const streamId = fakeIdGenerator() const number = 30 const limitMax = 25 await createBlobs({ streamId, number }) let blobData = await getBlobMetadataCollection({ streamId }) expect(blobData.blobs).to.have.lengthOf(limitMax) const localLimit = 3 blobData = await getBlobMetadataCollection({ streamId, limit: localLimit }) expect(blobData.blobs).to.have.lengthOf(localLimit) }) it('Cursor paginates query', async () => { const streamId = fakeIdGenerator() const number = 30 await createBlobs({ streamId, number }) let blobData = await getBlobMetadataCollection({ streamId }) expect(blobData.blobs).to.have.lengthOf(25) expect(blobData.blobs.map((blob) => blob.id)).to.deep.equalInAnyOrder( range(5, 30).map((index) => `${index}`.padStart(10, '0')) ) blobData = await getBlobMetadataCollection({ streamId, cursor: blobData.cursor }) expect(blobData.blobs).to.have.lengthOf(5) expect(blobData.blobs.map((blob) => blob.id)).to.deep.equalInAnyOrder( range(5).map((index) => `${index}`.padStart(10, '0')) ) }) it('Query matches for partial file name', async () => { const streamId = fakeIdGenerator() const number = 3 await createBlobs({ streamId, number }) let blobData = await getBlobMetadataCollection({ streamId, query: '00000' }) expect(blobData.blobs).to.have.lengthOf(number) blobData = await getBlobMetadataCollection({ streamId, query: '000002' }) expect(blobData.blobs).to.have.lengthOf(1) }) }) describe('Get blobCollectionSummary', () => { it('should set values to 0 if no blobs found', async () => { const summary = await blobCollectionSummary({ streamId: 'foo' }) expect(summary).to.deep.equal({ totalSize: 0, totalCount: 0 }) }) it('should report fileSize and count correctly', async () => { const streamId = fakeIdGenerator() const number = 30 const fileSize = 10 await createBlobs({ streamId, number, fileSize }) const summary = await blobCollectionSummary({ streamId }) expect(summary).to.deep.equal({ totalSize: number * fileSize, totalCount: number }) }) }) it('getFileStream should return content of getObjectStream', async () => { const fakeData = 'this is not a stream' const getObjectStream = async () => fakeData const streamId = fakeIdGenerator() const [blob] = await createBlobs({ streamId, number: 1 }) const fs = await getFileStream({ getObjectStream, streamId, blobId: blob.id }) expect(fs).to.equal(fakeData) }) it('deleteBlob should delete blob data', async () => { const streamId = fakeIdGenerator() const [blob] = await createBlobs({ streamId, number: 1 }) const blobId = blob.id const { objectKey } = await getBlobMetadata({ streamId, blobId }) expect(objectKey).to.equal(blob.objectKey) await deleteBlob({ streamId, blobId }) try { await getBlobMetadata({ streamId, blobId }) throw new Error('This should have thrown') } catch (err) { if (!(err instanceof NotFoundError)) throw err } }) it('markUploadOverFileSizeLimit calls delete object', async () => { let callCount = 0 async function deleteObjectSpy() { callCount++ } const streamId = fakeIdGenerator() const [blob] = await createBlobs({ streamId, number: 1 }) const blobId = blob.id const markResult = await markUploadOverFileSizeLimit( deleteObjectSpy, streamId, blobId ) expect(callCount).to.equal(1) expect(markResult).to.deep.equal({ blobId, fileName: blob.fileName, fileSize: null, uploadStatus: BlobUploadStatus.Error, uploadError: 'File size limit reached' }) }) it('markUploadSuccess returns with fileSize', async () => { const streamId = fakeIdGenerator() const [blob] = await createBlobs({ streamId, number: 1 }) const blobId = blob.id const fileSize = 12345 const getObjectAttributes = async () => ({ fileSize }) const markResult = await markUploadSuccess(getObjectAttributes, streamId, blobId) expect(markResult).to.deep.equal({ blobId, fileName: blob.fileName, uploadStatus: BlobUploadStatus.Completed, fileSize }) }) })