From 2cba323bfd57b16d6e0b9af2d230ad9aa188800a Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Thu, 22 May 2025 08:29:50 +0100 Subject: [PATCH] feat(main) objectloader2 should fail faster for missing json (#4578) * feat(viewer-sandbox): Made a sandbox function that will only invoke the object-loader loading objects * first pass of creating an objectloader2 * updated build + added vitest * try to get viewer sandbox to use new code * sandbox type fix * refactor a bit * can download root * intermediate commit for downloader/caching queue * can download stuff! * refactor files * intro isBase and fix isString * move single download to downloader * fix download * PR feedback * some intermediate commit * do clean up and download better * clean up promises and linting * can generate values while downloading and caching * add a finish method * remove unused functions * remove asBase * add temporary docs * add more docs with mermaid * add more test models * add response validation * add tests and redo options * add test for download batch * fix downloader tests and change Item to have clearer Base items * add tests and refactor a little * use fetch in downloader as an option * use optional in-memory indexdb instead of monkey patching the global one * more refactors for options for objectloader2 * add tests for objectloader2 * adjust single download * benchmark loading and adjust ol2 batches * download more! * adjust to use hash privates * refactored again with renaming * cleanup * make setupCacheDb throw instead * use BatchedPool for downloads! * fix tests * adjust timings and add adaptive waiting * Only wait if queue wasn't empty and queue size was full * fix tests * fix file names and some private usage * fix interval and private usage * rename vars * use params for methods * fix params for constructors and tests * fix params for constructors and tests again * using dexie * faster settings but doesn't end well * fixed end, optimized and removed logs * fix tests * fix types? * update lock with WSL * add e2e small model test * fix/update yarn.lock * Remove unused eslint ignore to fix pre-commit * prettier fixes * fix real DB usage * rename methods to better match OL1 * rename methods to better match OL1 again * add extra header collection * add headers correctly * test getTotalObjectCount * feat(viewer-lib): Replaced old object loader with Adam's objectloder2 * fix(viewer-lib): Removed the old object loader. Removed unneeded pause time in speckle loader * Testing * only deferred if not downloaded....don't save everything * Lockfile * pool isn't adjustable, adjust download buckets, dexie read is faster * chore * fix(viewer-lib): Fixed compiler errors * fix getObject access with real indexeddb...adjust buffer for deferred access * Fix disposal and pausing * don't index item! * fix dockerfiles to use OL2 * fix Dockerfile * Fix dockerfile * defer correctly and use record to add/lookup/remove to * delete stuff correctly * chore(sandbox): Enabled viewer loading * use objects instead of arrays to avoid findIndex * remove extra count * add a found cache to avoid some db hits * order matters for deferment * move found map to deferment * change option numbers * 2 level cache with expiry * defer everything, use loader to track what is requested....expire only found items * add deferment disposal * oops mismerge * chore(sandbox): Default stream * Beta version of CachePump and CacheReader * Clean up initialization * More clean up * chore(objectloader2): Fixed CI compiler error * chore(objectloader2): Fixed prettier * add cachePump tests * add cacheReader tests * fixed more tests * fixed final tests * moving stuff around and lock return value * try to move stuff out of objectloader2 * use a factory * rename factory * formatting * eslist fixes * try allocating no strings * add comments * small refactor and add another test * fix deferment expiration and have test * use byte size for max memory cache size * fix deferment manager tests * use a set of keys to determine if the whole batch was gathered * saved comment * fix(viewer-sandbox): Fixed compiler error * remove extra disposal * simplify the error handling --------- Co-authored-by: AlexandruPopovici Co-authored-by: Kristaps Fabians Geikins Co-authored-by: Iain Sproat <68657+iainsproat@users.noreply.github.com> --- .../objectloader2/src/helpers/batchedPool.ts | 6 +--- .../downloaders/serverDownloader.spec.ts | 35 +++++++++++++++++++ .../downloaders/serverDownloader.ts | 12 +++++-- packages/objectloader2/src/types/types.ts | 10 ++++++ packages/viewer-sandbox/src/main.ts | 6 +++- 5 files changed, 61 insertions(+), 8 deletions(-) diff --git a/packages/objectloader2/src/helpers/batchedPool.ts b/packages/objectloader2/src/helpers/batchedPool.ts index 56c3390be..c51402adb 100644 --- a/packages/objectloader2/src/helpers/batchedPool.ts +++ b/packages/objectloader2/src/helpers/batchedPool.ts @@ -31,11 +31,7 @@ export default class BatchedPool { while (!this.#finished || this.#queue.length > 0) { if (this.#queue.length > 0) { const batch = this.getBatch(batchSize) - try { - await this.#processFunction(batch) - } catch (e) { - console.error(e) - } + await this.#processFunction(batch) } await this.#delay(this.#baseInterval) } diff --git a/packages/objectloader2/src/operations/downloaders/serverDownloader.spec.ts b/packages/objectloader2/src/operations/downloaders/serverDownloader.spec.ts index 7711264ef..a20404e93 100644 --- a/packages/objectloader2/src/operations/downloaders/serverDownloader.spec.ts +++ b/packages/objectloader2/src/operations/downloaders/serverDownloader.spec.ts @@ -55,6 +55,41 @@ describe('downloader', () => { r.push(x) } + expect(r).toMatchSnapshot() + }) + + test('download batch of three', async () => { + const fetchMocker = createFetchMock(vi) + const i1: Item = { baseId: 'id1', base: { id: 'id1', speckle_type: 'type' } } + const i2: Item = { baseId: 'id2', base: { id: 'id2', speckle_type: 'type' } } + const i3: Item = { baseId: 'id3', base: { id: 'id3', speckle_type: 'type' } } + fetchMocker.mockResponseOnce( + 'id1\t' + + JSON.stringify(i1.base) + + '\nid2\t' + + JSON.stringify(i2.base) + + '\nid3\t' + + JSON.stringify(i3.base) + + '\n' + ) + + const pump = new MemoryPump() + const downloader = new ServerDownloader({ + serverUrl: 'http://speckle.test', + streamId: 'streamId', + objectId: 'objectId', + token: 'token', + + fetch: fetchMocker + }) + downloader.initializePool({ results: pump, total: 2, maxDownloadBatchWait: 200 }) + downloader.add('id') + await downloader.disposeAsync() + const r = [] + for await (const x of pump.gather([i1.baseId, i2.baseId, i3.baseId])) { + r.push(x) + } + expect(r).toMatchSnapshot() await downloader.disposeAsync() }) diff --git a/packages/objectloader2/src/operations/downloaders/serverDownloader.ts b/packages/objectloader2/src/operations/downloaders/serverDownloader.ts index 084598d6f..49d3294b3 100644 --- a/packages/objectloader2/src/operations/downloaders/serverDownloader.ts +++ b/packages/objectloader2/src/operations/downloaders/serverDownloader.ts @@ -1,7 +1,7 @@ import BatchedPool from '../../helpers/batchedPool.js' import Queue from '../../helpers/queue.js' import { ObjectLoaderRuntimeError } from '../../types/errors.js' -import { Fetcher, isBase, Item } from '../../types/types.js' +import { Fetcher, isBase, Item, take } from '../../types/types.js' import { Downloader } from '../interfaces.js' export interface ServerDownloaderOptions { @@ -110,6 +110,7 @@ export default class ServerDownloader implements Downloader { headers: HeadersInit }): Promise { const { batch, url, headers } = params + const keys = new Set(batch) const response = await this.#fetch(url, { method: 'POST', headers: { ...headers, 'Content-Type': 'application/json' }, @@ -129,18 +130,24 @@ export default class ServerDownloader implements Downloader { const { done, value } = await reader.read() if (done) break - leftover = await this.processArray(leftover, value, async () => { + leftover = await this.processArray(leftover, value, keys, async () => { count++ if (count % 1000 === 0) { await new Promise((resolve) => setTimeout(resolve, 100)) //allow other stuff to happen } }) } + if (keys.size > 0) { + throw new Error( + 'Items requested were not downloaded: ' + take(keys.values(), 10).join(',') + ) + } } async processArray( leftover: Uint8Array, value: Uint8Array, + keys: Set, callback: () => Promise ): Promise { //this concat will allocate a new array @@ -156,6 +163,7 @@ export default class ServerDownloader implements Downloader { this.#results?.add(item) start = i + 1 await callback() + keys.delete(item.baseId) } } return combined.subarray(start) // carry over remainder diff --git a/packages/objectloader2/src/types/types.ts b/packages/objectloader2/src/types/types.ts index ad7d669b0..52a92f55e 100644 --- a/packages/objectloader2/src/types/types.ts +++ b/packages/objectloader2/src/types/types.ts @@ -59,3 +59,13 @@ export function isScalar( type === 'undefined' ) } + +export function take(it: Iterator, count: number): T[] { + const result: T[] = [] + for (let i = 0; i < count; i++) { + const itr = it.next() + if (itr.done) break + result.push(itr.value) + } + return result +} diff --git a/packages/viewer-sandbox/src/main.ts b/packages/viewer-sandbox/src/main.ts index 603d8a148..af059eed9 100644 --- a/packages/viewer-sandbox/src/main.ts +++ b/packages/viewer-sandbox/src/main.ts @@ -111,10 +111,14 @@ const getStream = () => { return ( // prettier-ignore // Revit sample house (good for bim-like stuff with many display meshes) - 'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8' + //'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8' // 'https://latest.speckle.systems/streams/c1faab5c62/commits/ab1a1ab2b6' // 'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8' // 'https://latest.speckle.systems/streams/58b5648c4d/commits/60371ecb2d' + + //bad commit! not all items uploaded to server + 'https://app.speckle.systems/projects/8e4347e65d/models/39bea37d69' + // 'Super' heavy revit shit // 'https://app.speckle.systems/streams/e6f9156405/commits/0694d53bb5' // IFC building (good for a tree based structure)