diff --git a/packages/objectloader2/src/helpers/batchedPool.ts b/packages/objectloader2/src/helpers/batchedPool.ts index 56c3390be..c51402adb 100644 --- a/packages/objectloader2/src/helpers/batchedPool.ts +++ b/packages/objectloader2/src/helpers/batchedPool.ts @@ -31,11 +31,7 @@ export default class BatchedPool { while (!this.#finished || this.#queue.length > 0) { if (this.#queue.length > 0) { const batch = this.getBatch(batchSize) - try { - await this.#processFunction(batch) - } catch (e) { - console.error(e) - } + await this.#processFunction(batch) } await this.#delay(this.#baseInterval) } diff --git a/packages/objectloader2/src/operations/downloaders/serverDownloader.spec.ts b/packages/objectloader2/src/operations/downloaders/serverDownloader.spec.ts index 7711264ef..a20404e93 100644 --- a/packages/objectloader2/src/operations/downloaders/serverDownloader.spec.ts +++ b/packages/objectloader2/src/operations/downloaders/serverDownloader.spec.ts @@ -55,6 +55,41 @@ describe('downloader', () => { r.push(x) } + expect(r).toMatchSnapshot() + }) + + test('download batch of three', async () => { + const fetchMocker = createFetchMock(vi) + const i1: Item = { baseId: 'id1', base: { id: 'id1', speckle_type: 'type' } } + const i2: Item = { baseId: 'id2', base: { id: 'id2', speckle_type: 'type' } } + const i3: Item = { baseId: 'id3', base: { id: 'id3', speckle_type: 'type' } } + fetchMocker.mockResponseOnce( + 'id1\t' + + JSON.stringify(i1.base) + + '\nid2\t' + + JSON.stringify(i2.base) + + '\nid3\t' + + JSON.stringify(i3.base) + + '\n' + ) + + const pump = new MemoryPump() + const downloader = new ServerDownloader({ + serverUrl: 'http://speckle.test', + streamId: 'streamId', + objectId: 'objectId', + token: 'token', + + fetch: fetchMocker + }) + downloader.initializePool({ results: pump, total: 2, maxDownloadBatchWait: 200 }) + downloader.add('id') + await downloader.disposeAsync() + const r = [] + for await (const x of pump.gather([i1.baseId, i2.baseId, i3.baseId])) { + r.push(x) + } + expect(r).toMatchSnapshot() await downloader.disposeAsync() }) diff --git a/packages/objectloader2/src/operations/downloaders/serverDownloader.ts b/packages/objectloader2/src/operations/downloaders/serverDownloader.ts index 084598d6f..49d3294b3 100644 --- a/packages/objectloader2/src/operations/downloaders/serverDownloader.ts +++ b/packages/objectloader2/src/operations/downloaders/serverDownloader.ts @@ -1,7 +1,7 @@ import BatchedPool from '../../helpers/batchedPool.js' import Queue from '../../helpers/queue.js' import { ObjectLoaderRuntimeError } from '../../types/errors.js' -import { Fetcher, isBase, Item } from '../../types/types.js' +import { Fetcher, isBase, Item, take } from '../../types/types.js' import { Downloader } from '../interfaces.js' export interface ServerDownloaderOptions { @@ -110,6 +110,7 @@ export default class ServerDownloader implements Downloader { headers: HeadersInit }): Promise { const { batch, url, headers } = params + const keys = new Set(batch) const response = await this.#fetch(url, { method: 'POST', headers: { ...headers, 'Content-Type': 'application/json' }, @@ -129,18 +130,24 @@ export default class ServerDownloader implements Downloader { const { done, value } = await reader.read() if (done) break - leftover = await this.processArray(leftover, value, async () => { + leftover = await this.processArray(leftover, value, keys, async () => { count++ if (count % 1000 === 0) { await new Promise((resolve) => setTimeout(resolve, 100)) //allow other stuff to happen } }) } + if (keys.size > 0) { + throw new Error( + 'Items requested were not downloaded: ' + take(keys.values(), 10).join(',') + ) + } } async processArray( leftover: Uint8Array, value: Uint8Array, + keys: Set, callback: () => Promise ): Promise { //this concat will allocate a new array @@ -156,6 +163,7 @@ export default class ServerDownloader implements Downloader { this.#results?.add(item) start = i + 1 await callback() + keys.delete(item.baseId) } } return combined.subarray(start) // carry over remainder diff --git a/packages/objectloader2/src/types/types.ts b/packages/objectloader2/src/types/types.ts index ad7d669b0..52a92f55e 100644 --- a/packages/objectloader2/src/types/types.ts +++ b/packages/objectloader2/src/types/types.ts @@ -59,3 +59,13 @@ export function isScalar( type === 'undefined' ) } + +export function take(it: Iterator, count: number): T[] { + const result: T[] = [] + for (let i = 0; i < count; i++) { + const itr = it.next() + if (itr.done) break + result.push(itr.value) + } + return result +} diff --git a/packages/viewer-sandbox/src/main.ts b/packages/viewer-sandbox/src/main.ts index 603d8a148..af059eed9 100644 --- a/packages/viewer-sandbox/src/main.ts +++ b/packages/viewer-sandbox/src/main.ts @@ -111,10 +111,14 @@ const getStream = () => { return ( // prettier-ignore // Revit sample house (good for bim-like stuff with many display meshes) - 'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8' + //'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8' // 'https://latest.speckle.systems/streams/c1faab5c62/commits/ab1a1ab2b6' // 'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8' // 'https://latest.speckle.systems/streams/58b5648c4d/commits/60371ecb2d' + + //bad commit! not all items uploaded to server + 'https://app.speckle.systems/projects/8e4347e65d/models/39bea37d69' + // 'Super' heavy revit shit // 'https://app.speckle.systems/streams/e6f9156405/commits/0694d53bb5' // IFC building (good for a tree based structure)