From a385823b2d3aa60d3ec4b90d7a6dd12fde7faa32 Mon Sep 17 00:00:00 2001 From: Alexandru Popovici Date: Wed, 21 May 2025 10:05:50 +0300 Subject: [PATCH] feat(viewer): objectloader2 integration (#4267) * feat(viewer-sandbox): Made a sandbox function that will only invoke the object-loader loading objects * first pass of creating an objectloader2 * updated build + added vitest * try to get viewer sandbox to use new code * sandbox type fix * refactor a bit * can download root * intermediate commit for downloader/caching queue * can download stuff! * refactor files * intro isBase and fix isString * move single download to downloader * fix download * PR feedback * some intermediate commit * do clean up and download better * clean up promises and linting * can generate values while downloading and caching * add a finish method * remove unused functions * remove asBase * add temporary docs * add more docs with mermaid * add more test models * add response validation * add tests and redo options * add test for download batch * fix downloader tests and change Item to have clearer Base items * add tests and refactor a little * use fetch in downloader as an option * use optional in-memory indexdb instead of monkey patching the global one * more refactors for options for objectloader2 * add tests for objectloader2 * adjust single download * benchmark loading and adjust ol2 batches * download more! * adjust to use hash privates * refactored again with renaming * cleanup * make setupCacheDb throw instead * use BatchedPool for downloads! * fix tests * adjust timings and add adaptive waiting * Only wait if queue wasn't empty and queue size was full * fix tests * fix file names and some private usage * fix interval and private usage * rename vars * use params for methods * fix params for constructors and tests * fix params for constructors and tests again * using dexie * faster settings but doesn't end well * fixed end, optimized and removed logs * fix tests * fix types? * update lock with WSL * add e2e small model test * fix/update yarn.lock * Remove unused eslint ignore to fix pre-commit * prettier fixes * fix real DB usage * rename methods to better match OL1 * rename methods to better match OL1 again * add extra header collection * add headers correctly * test getTotalObjectCount * feat(viewer-lib): Replaced old object loader with Adam's objectloder2 * fix(viewer-lib): Removed the old object loader. Removed unneeded pause time in speckle loader * Testing * only deferred if not downloaded....don't save everything * Lockfile * pool isn't adjustable, adjust download buckets, dexie read is faster * chore * fix(viewer-lib): Fixed compiler errors * fix getObject access with real indexeddb...adjust buffer for deferred access * Fix disposal and pausing * don't index item! * fix dockerfiles to use OL2 * fix Dockerfile * Fix dockerfile * defer correctly and use record to add/lookup/remove to * delete stuff correctly * chore(sandbox): Enabled viewer loading * use objects instead of arrays to avoid findIndex * remove extra count * add a found cache to avoid some db hits * order matters for deferment * move found map to deferment * change option numbers * 2 level cache with expiry * defer everything, use loader to track what is requested....expire only found items * add deferment disposal * oops mismerge * chore(sandbox): Default stream * Beta version of CachePump and CacheReader * Clean up initialization * More clean up * chore(objectloader2): Fixed CI compiler error * chore(objectloader2): Fixed prettier * add cachePump tests * add cacheReader tests * fixed more tests * fixed final tests * moving stuff around and lock return value * try to move stuff out of objectloader2 * use a factory * rename factory * formatting * eslist fixes * try allocating no strings * add comments * small refactor and add another test * fix deferment expiration and have test * use byte size for max memory cache size * fix deferment manager tests * saved comment * fix(viewer-sandbox): Fixed compiler error * ignore tshy * chore(frontend): Attempt to make viewer loading sequential --------- Co-authored-by: Adam Hathcock Co-authored-by: Kristaps Fabians Geikins Co-authored-by: Iain Sproat <68657+iainsproat@users.noreply.github.com> --- .gitignore | 3 +- packages/frontend-2/Dockerfile | 4 +- .../lib/viewer/composables/setup/postSetup.ts | 18 +- packages/objectloader2/.tshy/build.json | 8 - packages/objectloader2/.tshy/commonjs.json | 17 -- packages/objectloader2/.tshy/esm.json | 16 -- packages/objectloader2/eslint.config.mjs | 4 +- .../__snapshots__/cachePump.spec.ts.snap | 31 +++ .../__snapshots__/cacheReader.spec.ts.snap | 8 + .../src/helpers/aggregateQueue.ts | 20 ++ .../objectloader2/src/helpers/batchedPool.ts | 2 +- .../src/helpers/batchingQueue.ts | 26 ++- .../src/helpers/cachePump.spec.ts | 74 ++++++ .../objectloader2/src/helpers/cachePump.ts | 98 ++++++++ .../src/helpers/cacheReader.spec.ts | 35 +++ .../objectloader2/src/helpers/cacheReader.ts | 64 +++++ .../src/helpers/defermentManager.spec.ts | 26 ++- .../src/helpers/defermentManager.ts | 134 ++++++++++- .../objectloader2/src/helpers/deferredBase.ts | 50 +++- .../objectloader2/src/helpers/keyedQueue.ts | 45 ++++ .../objectloader2/src/helpers/memoryPump.ts | 40 ++++ packages/objectloader2/src/helpers/pump.ts | 8 + packages/objectloader2/src/index.ts | 7 +- .../__snapshots__/objectLoader2.spec.ts.snap | 44 ++-- .../indexedDatabase.spec.ts.snap | 21 -- .../{ => databases}/indexedDatabase.spec.ts | 30 +-- .../operations/databases/indexedDatabase.ts | 150 ++++++++++++ .../operations/databases/memoryDatabase.ts | 43 ++++ .../serverDownloader.spec.ts.snap | 34 +++ .../{ => downloaders}/memoryDownloader.ts | 29 +-- .../serverDownloader.spec.ts | 86 +++---- .../{ => downloaders}/serverDownloader.ts | 119 +++++++--- .../src/operations/indexedDatabase.ts | 167 ------------- .../src/operations/interfaces.ts | 25 +- .../src/operations/memoryDatabase.ts | 42 ---- .../src/operations/objectLoader2.spec.ts | 220 ++++++------------ .../src/operations/objectLoader2.ts | 136 +++++------ .../src/operations/objectLoader2Factory.ts | 56 +++++ .../objectloader2/src/operations/options.ts | 57 ++--- .../src/operations/traverser.spec.ts | 2 +- .../objectloader2/src/operations/traverser.ts | 2 +- packages/objectloader2/src/test/e2e.spec.ts | 8 +- packages/objectloader2/src/types/types.ts | 1 + packages/preview-service/Dockerfile | 6 +- packages/viewer-sandbox/src/Sandbox.ts | 14 +- packages/viewer-sandbox/src/main.ts | 6 +- packages/viewer/package.json | 2 +- .../loaders/Speckle/SpeckleConverter.ts | 18 +- .../modules/loaders/Speckle/SpeckleLoader.ts | 27 +-- .../loaders/Speckle/SpeckleOfflineLoader.ts | 14 +- yarn.lock | 2 +- 51 files changed, 1317 insertions(+), 782 deletions(-) delete mode 100644 packages/objectloader2/.tshy/build.json delete mode 100644 packages/objectloader2/.tshy/commonjs.json delete mode 100644 packages/objectloader2/.tshy/esm.json create mode 100644 packages/objectloader2/src/helpers/__snapshots__/cachePump.spec.ts.snap create mode 100644 packages/objectloader2/src/helpers/__snapshots__/cacheReader.spec.ts.snap create mode 100644 packages/objectloader2/src/helpers/aggregateQueue.ts create mode 100644 packages/objectloader2/src/helpers/cachePump.spec.ts create mode 100644 packages/objectloader2/src/helpers/cachePump.ts create mode 100644 packages/objectloader2/src/helpers/cacheReader.spec.ts create mode 100644 packages/objectloader2/src/helpers/cacheReader.ts create mode 100644 packages/objectloader2/src/helpers/keyedQueue.ts create mode 100644 packages/objectloader2/src/helpers/memoryPump.ts create mode 100644 packages/objectloader2/src/helpers/pump.ts rename packages/objectloader2/src/operations/{ => databases}/__snapshots__/indexedDatabase.spec.ts.snap (59%) rename packages/objectloader2/src/operations/{ => databases}/indexedDatabase.spec.ts (57%) create mode 100644 packages/objectloader2/src/operations/databases/indexedDatabase.ts create mode 100644 packages/objectloader2/src/operations/databases/memoryDatabase.ts rename packages/objectloader2/src/operations/{ => downloaders}/__snapshots__/serverDownloader.spec.ts.snap (63%) rename packages/objectloader2/src/operations/{ => downloaders}/memoryDownloader.ts (52%) rename packages/objectloader2/src/operations/{ => downloaders}/serverDownloader.spec.ts (64%) rename packages/objectloader2/src/operations/{ => downloaders}/serverDownloader.ts (56%) delete mode 100644 packages/objectloader2/src/operations/indexedDatabase.ts delete mode 100644 packages/objectloader2/src/operations/memoryDatabase.ts create mode 100644 packages/objectloader2/src/operations/objectLoader2Factory.ts diff --git a/.gitignore b/.gitignore index 42c6d98b8..7ef035683 100644 --- a/.gitignore +++ b/.gitignore @@ -82,4 +82,5 @@ bin/ # Server multiregion.json -multiregion.test.json \ No newline at end of file +multiregion.test.json +packages/*/.tshy/ diff --git a/packages/frontend-2/Dockerfile b/packages/frontend-2/Dockerfile index cda52f5be..bbc070d45 100644 --- a/packages/frontend-2/Dockerfile +++ b/packages/frontend-2/Dockerfile @@ -13,7 +13,7 @@ COPY package.json yarn.lock ./ COPY utils/ensure-tailwind-deps.mjs ./utils/ COPY packages/viewer/package.json ./packages/viewer/ -COPY packages/objectloader/package.json ./packages/objectloader/ +COPY packages/objectloader2/package.json ./packages/objectloader2/ COPY packages/shared/package.json ./packages/shared/ COPY packages/ui-components/package.json ./packages/ui-components/ COPY packages/ui-components-nuxt/package.json ./packages/ui-components-nuxt/ @@ -21,7 +21,7 @@ COPY packages/tailwind-theme/package.json ./packages/tailwind-theme/ COPY packages/frontend-2/package.json ./packages/frontend-2/ COPY packages/frontend-2/type-augmentations ./packages/frontend-2/ -COPY packages/objectloader ./packages/objectloader/ +COPY packages/objectloader2 ./packages/objectloader2/ COPY packages/viewer ./packages/viewer/ COPY packages/shared ./packages/shared/ COPY packages/ui-components ./packages/ui-components/ diff --git a/packages/frontend-2/lib/viewer/composables/setup/postSetup.ts b/packages/frontend-2/lib/viewer/composables/setup/postSetup.ts index 8ef222c81..07d08ff40 100644 --- a/packages/frontend-2/lib/viewer/composables/setup/postSetup.ts +++ b/packages/frontend-2/lib/viewer/composables/setup/postSetup.ts @@ -111,7 +111,7 @@ function useViewerObjectAutoLoading() { const consolidateProgressThorttled = useThrottleFn(consolidateProgressInternal, 250) - const loadObject = ( + const loadObject = async ( objectId: string, unload?: boolean, options?: Partial<{ zoomToObject: boolean }> @@ -119,7 +119,7 @@ function useViewerObjectAutoLoading() { const objectUrl = getObjectUrl(projectId.value, objectId) if (unload) { - viewer.unloadObject(objectUrl) + return viewer.unloadObject(objectUrl) } else { const loader = new SpeckleLoader( viewer.getWorldTree(), @@ -135,7 +135,7 @@ function useViewerObjectAutoLoading() { consolidateProgressInternal({ id, progress: 1 }) }) - viewer.loadObject(loader, options?.zoomToObject) + return viewer.loadObject(loader, options?.zoomToObject) } } @@ -155,9 +155,15 @@ function useViewerObjectAutoLoading() { if (!newHasDoneInitialLoad) { const allObjectIds = getUniqueObjectIds(newResources) - const res = await Promise.all( - allObjectIds.map((i) => loadObject(i, false, { zoomToObject })) - ) + /** Load sequentially */ + const res = [] + for (const i of allObjectIds) { + res.push(await loadObject(i, false, { zoomToObject })) + } + /** Load in parallel */ + // const res = await Promise.all( + // allObjectIds.map((i) => loadObject(i, false, { zoomToObject })) + // ) if (res.length) { hasDoneInitialLoad.value = true } diff --git a/packages/objectloader2/.tshy/build.json b/packages/objectloader2/.tshy/build.json deleted file mode 100644 index aea1a9e93..000000000 --- a/packages/objectloader2/.tshy/build.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../tsconfig.json", - "compilerOptions": { - "rootDir": "../src", - "module": "nodenext", - "moduleResolution": "nodenext" - } -} diff --git a/packages/objectloader2/.tshy/commonjs.json b/packages/objectloader2/.tshy/commonjs.json deleted file mode 100644 index df704f961..000000000 --- a/packages/objectloader2/.tshy/commonjs.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "extends": "./build.json", - "include": [ - "../src/**/*.ts", - "../src/**/*.cts", - "../src/**/*.tsx", - "../src/**/*.json" - ], - "exclude": [ - "../**/*.spec.ts", - "../src/**/*.mts", - "../src/package.json" - ], - "compilerOptions": { - "outDir": "../.tshy-build/commonjs" - } -} diff --git a/packages/objectloader2/.tshy/esm.json b/packages/objectloader2/.tshy/esm.json deleted file mode 100644 index 0392150b7..000000000 --- a/packages/objectloader2/.tshy/esm.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "extends": "./build.json", - "include": [ - "../src/**/*.ts", - "../src/**/*.mts", - "../src/**/*.tsx", - "../src/**/*.json" - ], - "exclude": [ - "../**/*.spec.ts", - "../src/package.json" - ], - "compilerOptions": { - "outDir": "../.tshy-build/esm" - } -} diff --git a/packages/objectloader2/eslint.config.mjs b/packages/objectloader2/eslint.config.mjs index 105f47963..5e26bc71b 100644 --- a/packages/objectloader2/eslint.config.mjs +++ b/packages/objectloader2/eslint.config.mjs @@ -29,7 +29,9 @@ const configs = [ } }, rules: { - '@typescript-eslint/restrict-template-expressions': 'off' + '@typescript-eslint/restrict-template-expressions': 'off', + '@typescript-eslint/await-thenable': 'error', + '@typescript-eslint/explicit-function-return-type': 'error' } }, { diff --git a/packages/objectloader2/src/helpers/__snapshots__/cachePump.spec.ts.snap b/packages/objectloader2/src/helpers/__snapshots__/cachePump.spec.ts.snap new file mode 100644 index 000000000..500ad13fa --- /dev/null +++ b/packages/objectloader2/src/helpers/__snapshots__/cachePump.spec.ts.snap @@ -0,0 +1,31 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`CachePump testing > write two items to queue use pumpItems that are NOT found 1`] = `[]`; + +exports[`CachePump testing > write two items to queue use pumpItems that are NOT found 2`] = ` +[ + "id1", + "id2", +] +`; + +exports[`CachePump testing > write two items to queue use pumpItems that are found 1`] = ` +[ + { + "base": { + "id": "id", + "speckle_type": "type", + }, + "baseId": "id1", + }, + { + "base": { + "id": "id", + "speckle_type": "type", + }, + "baseId": "id2", + }, +] +`; + +exports[`CachePump testing > write two items to queue use pumpItems that are found 2`] = `[]`; diff --git a/packages/objectloader2/src/helpers/__snapshots__/cacheReader.spec.ts.snap b/packages/objectloader2/src/helpers/__snapshots__/cacheReader.spec.ts.snap new file mode 100644 index 000000000..1263f6254 --- /dev/null +++ b/packages/objectloader2/src/helpers/__snapshots__/cacheReader.spec.ts.snap @@ -0,0 +1,8 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`CacheReader testing > deferred getObject 1`] = ` +{ + "id": "id", + "speckle_type": "type", +} +`; diff --git a/packages/objectloader2/src/helpers/aggregateQueue.ts b/packages/objectloader2/src/helpers/aggregateQueue.ts new file mode 100644 index 000000000..db4e401f5 --- /dev/null +++ b/packages/objectloader2/src/helpers/aggregateQueue.ts @@ -0,0 +1,20 @@ +import Queue from './queue.js' + +export default class AggregateQueue implements Queue { + #queue1: Queue + #queue2: Queue + + constructor(queue1: Queue, queue2: Queue) { + this.#queue1 = queue1 + this.#queue2 = queue2 + } + + add(value: T): void { + this.#queue1.add(value) + this.#queue2.add(value) + } + + values(): T[] { + throw new Error('Not implemented') + } +} diff --git a/packages/objectloader2/src/helpers/batchedPool.ts b/packages/objectloader2/src/helpers/batchedPool.ts index acbb43abd..56c3390be 100644 --- a/packages/objectloader2/src/helpers/batchedPool.ts +++ b/packages/objectloader2/src/helpers/batchedPool.ts @@ -27,7 +27,7 @@ export default class BatchedPool { return this.#queue.splice(0, Math.min(batchSize, this.#queue.length)) } - async #runWorker(batchSize: number) { + async #runWorker(batchSize: number): Promise { while (!this.#finished || this.#queue.length > 0) { if (this.#queue.length > 0) { const batch = this.getBatch(batchSize) diff --git a/packages/objectloader2/src/helpers/batchingQueue.ts b/packages/objectloader2/src/helpers/batchingQueue.ts index edefa1f12..d9cd61f58 100644 --- a/packages/objectloader2/src/helpers/batchingQueue.ts +++ b/packages/objectloader2/src/helpers/batchingQueue.ts @@ -1,7 +1,7 @@ -import Queue from './queue.js' +import KeyedQueue from './keyedQueue.js' -export default class BatchingQueue implements Queue { - #queue: T[] = [] +export default class BatchingQueue { + #queue: KeyedQueue = new KeyedQueue() #batchSize: number #processFunction: (batch: T[]) => Promise @@ -30,28 +30,32 @@ export default class BatchingQueue implements Queue { await this.#processingLoop } - add(item: T): void { - this.#queue.push(item) + add(key: string, item: T): void { + this.#queue.enqueue(key, item) + } + + get(id: string): T | undefined { + return this.#queue.get(id) } count(): number { - return this.#queue.length + return this.#queue.size } #getBatch(batchSize: number): T[] { - return this.#queue.splice(0, Math.min(batchSize, this.#queue.length)) + return this.#queue.spliceValues(0, Math.min(batchSize, this.#queue.size)) } async #loop(): Promise { let interval = this.#baseInterval - while (!this.#finished || this.#queue.length > 0) { + while (!this.#finished || this.#queue.size > 0) { const startTime = performance.now() - if (this.#queue.length > 0) { + if (this.#queue.size > 0) { const batch = this.#getBatch(this.#batchSize) //console.log('running with queue size of ' + this.#queue.length) await this.#processFunction(batch) } - if (this.#queue.length < this.#batchSize / 2) { + if (this.#queue.size < this.#batchSize / 2) { //refigure interval const endTime = performance.now() const duration = endTime - startTime @@ -62,7 +66,7 @@ export default class BatchingQueue implements Queue { } /*console.log( 'queue is waiting ' + - interval / TIME_MS.second + + interval / 1000 + ' with queue size of ' + this.#queue.length )*/ diff --git a/packages/objectloader2/src/helpers/cachePump.spec.ts b/packages/objectloader2/src/helpers/cachePump.spec.ts new file mode 100644 index 000000000..bca6a356d --- /dev/null +++ b/packages/objectloader2/src/helpers/cachePump.spec.ts @@ -0,0 +1,74 @@ +import { describe, expect, test } from 'vitest' +import { CachePump } from './cachePump.js' +import { Base, Item } from '../types/types.js' +import BufferQueue from './bufferQueue.js' +import AsyncGeneratorQueue from './asyncGeneratorQueue.js' +import { DefermentManager } from './defermentManager.js' +import { MemoryDatabase } from '../operations/databases/memoryDatabase.js' + +describe('CachePump testing', () => { + test('write two items to queue use pumpItems that are NOT found', async () => { + const i1: Item = { baseId: 'id1', base: { id: 'id', speckle_type: 'type' } } + const i2: Item = { baseId: 'id2', base: { id: 'id', speckle_type: 'type' } } + + const gathered = new AsyncGeneratorQueue() + const deferments = new DefermentManager({ maxSize: 1, ttl: 1 }) + const cachePump = new CachePump(new MemoryDatabase({}), gathered, deferments, { + maxCacheReadSize: 1, + maxCacheWriteSize: 1, + maxCacheBatchWriteWait: 1, + maxCacheBatchReadWait: 1, + maxWriteQueueSize: 1 + }) + + const foundItems = new BufferQueue() + const notFoundItems = new BufferQueue() + + await cachePump.pumpItems({ + ids: [i1.baseId, i2.baseId], + foundItems, + notFoundItems + }) + + expect(foundItems.values()).toMatchSnapshot() + expect(notFoundItems.values()).toMatchSnapshot() + await cachePump.disposeAsync() + }) + + test('write two items to queue use pumpItems that are found', async () => { + const i1: Item = { baseId: 'id1', base: { id: 'id', speckle_type: 'type' } } + const i2: Item = { baseId: 'id2', base: { id: 'id', speckle_type: 'type' } } + + const db = new Map() + db.set(i1.baseId, i1.base) + db.set(i2.baseId, i2.base) + + const gathered = new AsyncGeneratorQueue() + const deferments = new DefermentManager({ maxSize: 1, ttl: 1 }) + const cachePump = new CachePump( + new MemoryDatabase({ items: db }), + gathered, + deferments, + { + maxCacheReadSize: 1, + maxCacheWriteSize: 1, + maxCacheBatchWriteWait: 1, + maxCacheBatchReadWait: 1, + maxWriteQueueSize: 1 + } + ) + + const foundItems = new BufferQueue() + const notFoundItems = new BufferQueue() + + await cachePump.pumpItems({ + ids: [i1.baseId, i2.baseId], + foundItems, + notFoundItems + }) + + expect(foundItems.values()).toMatchSnapshot() + expect(notFoundItems.values()).toMatchSnapshot() + await cachePump.disposeAsync() + }) +}) diff --git a/packages/objectloader2/src/helpers/cachePump.ts b/packages/objectloader2/src/helpers/cachePump.ts new file mode 100644 index 000000000..9cd04ab0d --- /dev/null +++ b/packages/objectloader2/src/helpers/cachePump.ts @@ -0,0 +1,98 @@ +import { TIME } from '@speckle/shared' +import { Database } from '../operations/interfaces.js' +import { CacheOptions } from '../operations/options.js' +import { CustomLogger, Item } from '../types/types.js' +import BatchingQueue from './batchingQueue.js' +import Queue from './queue.js' +import { Downloader } from '../operations/interfaces.js' +import { DefermentManager } from './defermentManager.js' +import AsyncGeneratorQueue from './asyncGeneratorQueue.js' +import { Pump } from './pump.js' + +export class CachePump implements Pump { + #writeQueue: BatchingQueue | undefined + #database: Database + #logger: CustomLogger + #deferments: DefermentManager + + #gathered: AsyncGeneratorQueue + + #options: CacheOptions + + constructor( + database: Database, + gathered: AsyncGeneratorQueue, + deferments: DefermentManager, + options: CacheOptions + ) { + this.#database = database + this.#gathered = gathered + this.#deferments = deferments + this.#options = options + this.#logger = options.logger || ((): void => {}) + } + + add(item: Item): void { + if (!this.#writeQueue) { + this.#writeQueue = new BatchingQueue({ + batchSize: this.#options.maxCacheWriteSize, + maxWaitTime: this.#options.maxCacheBatchWriteWait, + processFunction: (batch: Item[]): Promise => + this.#database.cacheSaveBatch({ batch }) + }) + } + this.#writeQueue.add(item.baseId, item) + } + + async disposeAsync(): Promise { + await this.#writeQueue?.disposeAsync() + } + + async pumpItems(params: { + ids: string[] + foundItems: Queue + notFoundItems: Queue + }): Promise { + const { ids, foundItems, notFoundItems } = params + const maxCacheReadSize = this.#options.maxCacheReadSize + + for (let i = 0; i < ids.length; ) { + if ((this.#writeQueue?.count() ?? 0) > this.#options.maxWriteQueueSize) { + this.#logger( + 'pausing reads (# in write queue: ' + this.#writeQueue?.count() + ')' + ) + await new Promise((resolve) => setTimeout(resolve, TIME.second)) // Pause for 1 second, protects against out of memory + continue + } + const batch = ids.slice(i, i + maxCacheReadSize) + const cachedData = await this.#database.getAll(batch) + for (let i = 0; i < cachedData.length; i++) { + if (cachedData[i]) { + foundItems.add(cachedData[i]!) + } else { + notFoundItems.add(batch[i]) + } + } + i += maxCacheReadSize + } + } + + async *gather(ids: string[], downloader: Downloader): AsyncGenerator { + const total = ids.length + const pumpPromise = this.pumpItems({ + ids, + foundItems: this.#gathered, + notFoundItems: downloader + }) + let count = 0 + for await (const item of this.#gathered.consume()) { + this.#deferments.undefer(item) + yield item + count++ + if (count >= total) { + this.#gathered.dispose() + } + } + await pumpPromise + } +} diff --git a/packages/objectloader2/src/helpers/cacheReader.spec.ts b/packages/objectloader2/src/helpers/cacheReader.spec.ts new file mode 100644 index 000000000..19ffc4c78 --- /dev/null +++ b/packages/objectloader2/src/helpers/cacheReader.spec.ts @@ -0,0 +1,35 @@ +import { describe, expect, test } from 'vitest' +import { Base, Item } from '../types/types.js' +import { DefermentManager } from './defermentManager.js' +import { CacheReader } from './cacheReader.js' +import { MemoryDatabase } from '../operations/databases/memoryDatabase.js' + +describe('CacheReader testing', () => { + test('deferred getObject', async () => { + const i1: Item = { baseId: 'id1', base: { id: 'id', speckle_type: 'type' } } + + const deferments = new DefermentManager({ maxSize: 1, ttl: 1 }) + const cacheReader = new CacheReader( + new MemoryDatabase({ + items: new Map([[i1.baseId, i1.base]]) + }), + deferments, + { + maxCacheReadSize: 1, + maxCacheWriteSize: 1, + maxCacheBatchWriteWait: 1, + maxCacheBatchReadWait: 1, + maxWriteQueueSize: 1 + } + ) + + const objPromise = cacheReader.getObject({ + id: i1.baseId + }) + deferments.undefer(i1) + const base = await objPromise + + expect(base).toMatchSnapshot() + await cacheReader.disposeAsync() + }) +}) diff --git a/packages/objectloader2/src/helpers/cacheReader.ts b/packages/objectloader2/src/helpers/cacheReader.ts new file mode 100644 index 000000000..55c980354 --- /dev/null +++ b/packages/objectloader2/src/helpers/cacheReader.ts @@ -0,0 +1,64 @@ +import { Database } from '../operations/interfaces.js' +import { CacheOptions } from '../operations/options.js' +import { Base, CustomLogger, Item } from '../types/types.js' +import BatchingQueue from './batchingQueue.js' +import { DefermentManager } from './defermentManager.js' + +export class CacheReader { + #database: Database + #defermentManager: DefermentManager + #logger: CustomLogger + #options: CacheOptions + #readQueue: BatchingQueue | undefined + + constructor( + database: Database, + defermentManager: DefermentManager, + options: CacheOptions + ) { + this.#database = database + this.#defermentManager = defermentManager + this.#options = options + this.#logger = options.logger || ((): void => {}) + } + + async getObject(params: { id: string }): Promise { + if (!this.#defermentManager.isDeferred(params.id)) { + this.#getItem(params.id) + } + return await this.#defermentManager.defer({ id: params.id }) + } + + #getItem(id: string): void { + if (!this.#readQueue) { + this.#readQueue = new BatchingQueue({ + batchSize: this.#options.maxCacheReadSize, + maxWaitTime: this.#options.maxCacheBatchReadWait, + processFunction: this.#processBatch + }) + } + if (!this.#readQueue.get(id)) { + this.#readQueue.add(id, id) + } + } + + async getAll(keys: string[]): Promise<(Item | undefined)[]> { + return this.#database.getAll(keys) + } + + #processBatch = async (batch: string[]): Promise => { + const items = await this.#database.getAll(batch) + for (let i = 0; i < items.length; i++) { + if (items[i]) { + this.#defermentManager.undefer(items[i]!) + } else { + //this is okay! + //this.#logger(`Item ${batch[i]} not found in cache`) + } + } + } + + async disposeAsync(): Promise { + await this.#readQueue?.disposeAsync() + } +} diff --git a/packages/objectloader2/src/helpers/defermentManager.spec.ts b/packages/objectloader2/src/helpers/defermentManager.spec.ts index 2707bcd42..99da71de2 100644 --- a/packages/objectloader2/src/helpers/defermentManager.spec.ts +++ b/packages/objectloader2/src/helpers/defermentManager.spec.ts @@ -1,13 +1,37 @@ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +/* eslint-disable @typescript-eslint/no-explicit-any */ import { describe, expect, test } from 'vitest' import { DefermentManager } from './defermentManager.js' describe('deferments', () => { test('defer one', async () => { - const deferments = new DefermentManager() + const deferments = new DefermentManager({ maxSizeInMb: 1, ttlms: 1 }) const x = deferments.defer({ id: 'id' }) expect(x).toBeInstanceOf(Promise) deferments.undefer({ baseId: 'id', base: { id: 'id', speckle_type: 'type' } }) const b = await x expect(b).toMatchSnapshot() }) + + test('expireAt timeout', async () => { + const now = 1 + const deferments = new DefermentManager({ maxSizeInMb: 1, ttlms: 1 }) + deferments['now'] = (): number => now + const x = deferments.defer({ id: 'id' }) + expect(x).toBeInstanceOf(Promise) + const d = deferments.get('id') + expect(d).toBeDefined() + expect(d?.getId()).toBe('id') + expect((d as any).expiresAt).toBe(2) + expect((d as any).ttl).toBe(1) + expect((d as any).item).toBeUndefined() + expect(d?.isExpired(1)).toBe(false) + deferments.undefer({ baseId: 'id', base: { id: 'id', speckle_type: 'type' } }) + await x + expect((d as any).expiresAt).toBe(2) + expect((d as any).ttl).toBe(1) + expect((d as any).item).toBeDefined() + expect(d?.isExpired(1)).toBe(false) + expect(d?.isExpired(3)).toBe(true) + }) }) diff --git a/packages/objectloader2/src/helpers/defermentManager.ts b/packages/objectloader2/src/helpers/defermentManager.ts index eb62fb9c5..3c23e9eb1 100644 --- a/packages/objectloader2/src/helpers/defermentManager.ts +++ b/packages/objectloader2/src/helpers/defermentManager.ts @@ -1,25 +1,135 @@ import { DeferredBase } from './deferredBase.js' -import { Base, Item } from '../types/types.js' +import { Base, CustomLogger, Item } from '../types/types.js' +import { DefermentManagerOptions } from '../operations/options.js' export class DefermentManager { - #deferments: DeferredBase[] = [] + private deferments: Map = new Map() + private timer?: ReturnType + private logger: CustomLogger + private currentSize = 0 + + constructor(private options: DefermentManagerOptions) { + this.resetGlobalTimer() + this.logger = options.logger || ((): void => {}) + } + + private now(): number { + return Date.now() + } + + isDeferred(id: string): boolean { + return this.deferments.has(id) + } + + get(id: string): DeferredBase | undefined { + return this.deferments.get(id) + } async defer(params: { id: string }): Promise { - const deferredBase = this.#deferments.find((x) => x.id === params.id) + const now = this.now() + const deferredBase = this.deferments.get(params.id) if (deferredBase) { - return await deferredBase.promise + deferredBase.setAccess(now) + return deferredBase.getPromise() } - const d = new DeferredBase(params.id) - this.#deferments.push(d) - return d.promise + const notYetFound = new DeferredBase( + this.options.ttlms, + params.id, + now + this.options.ttlms + ) + this.deferments.set(params.id, notYetFound) + return notYetFound.getPromise() } undefer(item: Item): void { - const deferredIndex = this.#deferments.findIndex((x) => x.id === item.baseId) - if (deferredIndex !== -1) { - const deferredBase = this.#deferments[deferredIndex] - deferredBase.resolve(item.base) - this.#deferments.splice(deferredIndex, 1) + const now = this.now() + this.currentSize += item.size || 0 + //order matters here with found before undefer + const deferredBase = this.deferments.get(item.baseId) + if (deferredBase) { + deferredBase.found(item) + deferredBase.setAccess(now) + } else { + const existing = new DeferredBase(this.options.ttlms, item.baseId, now) + existing.found(item) + this.deferments.set(item.baseId, existing) } } + + private resetGlobalTimer(): void { + const run = (): void => { + this.cleanDeferments() + this.timer = setTimeout(run, this.options.ttlms) + } + this.timer = setTimeout(run, this.options.ttlms) + } + + dispose(): void { + if (this.timer) { + clearTimeout(this.timer) + this.timer = undefined + } + this.clearDeferments() + } + + private clearDeferments(): void { + let waiting = 0 + for (const deferredBase of this.deferments.values()) { + deferredBase.done(0) + if (deferredBase.getItem() === undefined) { + waiting++ + } + } + this.currentSize = 0 + this.deferments.clear() + this.logger('cleared deferments, left', waiting) + } + + private cleanDeferments(): void { + const maxSizeBytes = this.options.maxSizeInMb * 1024 * 1024 + if (this.currentSize < maxSizeBytes) { + this.logger( + 'deferments size is ok, no need to clean', + this.currentSize, + maxSizeBytes + ) + return + } + const now = this.now() + let cleaned = 0 + const start = performance.now() + for (const deferredBase of Array.from(this.deferments.values()) + .filter((x) => x.isExpired(now)) + .sort((a, b) => this.compareMaybeBasesBySize(a.getItem(), b.getItem()))) { + if (deferredBase.done(now)) { + this.currentSize -= deferredBase.getItem()?.size || 0 + this.deferments.delete(deferredBase.getId()) + cleaned++ + if (this.currentSize < maxSizeBytes) { + break + } + } + } + this.logger( + 'cleaned deferments, cleaned, left', + cleaned, + this.deferments.size, + performance.now() - start + ) + return + } + + compareMaybeBasesBySize(a: Item | undefined, b: Item | undefined): number { + if (a === undefined && b === undefined) return 0 + if (a === undefined) return -1 + if (b === undefined) return 1 + return this.compareMaybe(a.size, b.size) + } + + compareMaybe(a: number | undefined, b: number | undefined): number { + if (a === undefined && b === undefined) return 0 + if (a === undefined) return -1 + if (b === undefined) return 1 + return a - b + } } diff --git a/packages/objectloader2/src/helpers/deferredBase.ts b/packages/objectloader2/src/helpers/deferredBase.ts index 7e74e18f5..c97b3ad43 100644 --- a/packages/objectloader2/src/helpers/deferredBase.ts +++ b/packages/objectloader2/src/helpers/deferredBase.ts @@ -1,17 +1,55 @@ -import { Base } from '../types/types.js' +import { Base, Item } from '../types/types.js' export class DeferredBase { - promise: Promise - resolve!: (value: Base) => void - reject!: (reason?: Error) => void + private promise: Promise + private resolve!: (value: Base) => void + private reject!: (reason?: Error) => void + private item?: Item - readonly id: string + private readonly id: string + private expiresAt: number // Timestamp in ms + private ttl: number // ttl in ms - constructor(id: string) { + constructor(ttl: number, id: string, expiresAt: number) { + this.expiresAt = expiresAt + this.ttl = ttl this.id = id this.promise = new Promise((resolve, reject) => { this.resolve = resolve this.reject = reject }) } + + getId(): string { + return this.id + } + + getItem(): Item | undefined { + return this.item + } + + getPromise(): Promise { + return this.promise + } + + isExpired(now: number): boolean { + return this.item !== undefined && now > this.expiresAt + } + setAccess(now: number): void { + this.expiresAt = now + this.ttl + } + + found(value: Item): void { + this.item = value + this.resolve(value.base) + } + done(now: number): boolean { + if (this.item) { + this.resolve(this.item.base) + } + if (this.isExpired(now)) { + return true + } + return false + } } diff --git a/packages/objectloader2/src/helpers/keyedQueue.ts b/packages/objectloader2/src/helpers/keyedQueue.ts new file mode 100644 index 000000000..949caaa7c --- /dev/null +++ b/packages/objectloader2/src/helpers/keyedQueue.ts @@ -0,0 +1,45 @@ +export default class KeyedQueue { + private _map: Map + private _order: K[] + + constructor() { + this._map = new Map() + this._order = [] + } + + enqueue(key: K, value: V): boolean { + if (this._map.has(key)) { + return false // Key already exists + } + this._map.set(key, value) + this._order.push(key) + return true + } + + get(key: K): V | undefined { + return this._map.get(key) + } + + has(key: K): boolean { + return this._map.has(key) + } + + get size(): number { + return this._order.length + } + + spliceValues(start: number, deleteCount: number): V[] { + const splicedKeys = this._order.splice(start, deleteCount) + const result: V[] = [] + + for (const key of splicedKeys) { + const value = this._map.get(key) + if (value !== undefined) { + result.push(value) + this._map.delete(key) + } + } + + return result + } +} diff --git a/packages/objectloader2/src/helpers/memoryPump.ts b/packages/objectloader2/src/helpers/memoryPump.ts new file mode 100644 index 000000000..eee22560a --- /dev/null +++ b/packages/objectloader2/src/helpers/memoryPump.ts @@ -0,0 +1,40 @@ +import { Item } from '../types/types.js' +import { Pump } from './pump.js' +import Queue from './queue.js' + +export class MemoryPump implements Pump { + #items: Map = new Map() + + add(item: Item): void { + this.#items.set(item.baseId, item) + } + + async pumpItems(params: { + ids: string[] + foundItems: Queue + notFoundItems: Queue + }): Promise { + const { ids, foundItems, notFoundItems } = params + for (const id of ids) { + const item = this.#items.get(id) + if (item) { + foundItems.add(item) + } else { + notFoundItems.add(id) + } + } + return Promise.resolve() + } + + async *gather(ids: string[]): AsyncGenerator { + for (const id of ids) { + const item = this.#items.get(id) + if (item) { + yield item + } + } + return Promise.resolve() + } + + async disposeAsync(): Promise {} +} diff --git a/packages/objectloader2/src/helpers/pump.ts b/packages/objectloader2/src/helpers/pump.ts new file mode 100644 index 000000000..431b71de1 --- /dev/null +++ b/packages/objectloader2/src/helpers/pump.ts @@ -0,0 +1,8 @@ +import { Downloader } from '../operations/interfaces.js' +import { Item } from '../types/types.js' +import Queue from './queue.js' + +export interface Pump extends Queue { + gather(ids: string[], downloader: Downloader): AsyncGenerator + disposeAsync(): Promise +} diff --git a/packages/objectloader2/src/index.ts b/packages/objectloader2/src/index.ts index 4dbbac01b..f699659bd 100644 --- a/packages/objectloader2/src/index.ts +++ b/packages/objectloader2/src/index.ts @@ -1,4 +1,3 @@ -import ObjectLoader2 from './operations/objectLoader2.js' - -export default ObjectLoader2 -export { MemoryDatabase } from './operations/memoryDatabase.js' +export { MemoryDatabase } from './operations/databases/memoryDatabase.js' +export { ObjectLoader2 } from './operations/objectLoader2.js' +export { ObjectLoader2Factory } from './operations/objectLoader2Factory.js' diff --git a/packages/objectloader2/src/operations/__snapshots__/objectLoader2.spec.ts.snap b/packages/objectloader2/src/operations/__snapshots__/objectLoader2.spec.ts.snap index bba7ad678..65653048b 100644 --- a/packages/objectloader2/src/operations/__snapshots__/objectLoader2.spec.ts.snap +++ b/packages/objectloader2/src/operations/__snapshots__/objectLoader2.spec.ts.snap @@ -2,35 +2,35 @@ exports[`objectloader2 > add extra header 1`] = ` { - "baseId": "baseId", -} -`; - -exports[`objectloader2 > can get a root object from cache 1`] = ` -{ - "baseId": "baseId", -} -`; - -exports[`objectloader2 > can get a root object from downloader 1`] = ` -{ - "baseId": "baseId", -} -`; - -exports[`objectloader2 > can get root/child object from cache using iterator 1`] = ` -[ - { + "base": { "__closure": { "child1Id": 100, }, "id": "rootId", "speckle_type": "type", }, - { - "id": "child1Id", + "baseId": "rootId", +} +`; + +exports[`objectloader2 > can get a root object from cache 1`] = ` +{ + "base": { + "id": "baseId", + "speckle_type": "type", }, -] + "baseId": "baseId", +} +`; + +exports[`objectloader2 > can get a root object from downloader 1`] = ` +{ + "base": { + "id": "baseId", + "speckle_type": "type", + }, + "baseId": "baseId", +} `; exports[`objectloader2 > can get root/child object from memory cache using iterator and getObject 1`] = ` diff --git a/packages/objectloader2/src/operations/__snapshots__/indexedDatabase.spec.ts.snap b/packages/objectloader2/src/operations/databases/__snapshots__/indexedDatabase.spec.ts.snap similarity index 59% rename from packages/objectloader2/src/operations/__snapshots__/indexedDatabase.spec.ts.snap rename to packages/objectloader2/src/operations/databases/__snapshots__/indexedDatabase.spec.ts.snap index 42387a565..1eb27e248 100644 --- a/packages/objectloader2/src/operations/__snapshots__/indexedDatabase.spec.ts.snap +++ b/packages/objectloader2/src/operations/databases/__snapshots__/indexedDatabase.spec.ts.snap @@ -29,24 +29,3 @@ exports[`database cache > write two items to queue use getItem 2`] = ` "baseId": "id2", } `; - -exports[`database cache > write two items to queue use processItems 1`] = ` -[ - { - "base": { - "id": "id", - "speckle_type": "type", - }, - "baseId": "id1", - }, - { - "base": { - "id": "id", - "speckle_type": "type", - }, - "baseId": "id2", - }, -] -`; - -exports[`database cache > write two items to queue use processItems 2`] = `[]`; diff --git a/packages/objectloader2/src/operations/indexedDatabase.spec.ts b/packages/objectloader2/src/operations/databases/indexedDatabase.spec.ts similarity index 57% rename from packages/objectloader2/src/operations/indexedDatabase.spec.ts rename to packages/objectloader2/src/operations/databases/indexedDatabase.spec.ts index 2b979933d..390862b5c 100644 --- a/packages/objectloader2/src/operations/indexedDatabase.spec.ts +++ b/packages/objectloader2/src/operations/databases/indexedDatabase.spec.ts @@ -1,16 +1,14 @@ import { describe, expect, test } from 'vitest' import IndexedDatabase from './indexedDatabase.js' import { IDBFactory, IDBKeyRange } from 'fake-indexeddb' -import { Item } from '../types/types.js' -import BufferQueue from '../helpers/bufferQueue.js' +import { Item } from '../../types/types.js' describe('database cache', () => { test('write single item to queue use getItem', async () => { const i: Item = { baseId: 'id', base: { id: 'id', speckle_type: 'type' } } const database = new IndexedDatabase({ indexedDB: new IDBFactory(), - keyRange: IDBKeyRange, - maxCacheBatchWriteWait: 200 + keyRange: IDBKeyRange }) await database.add(i) await database.disposeAsync() @@ -36,28 +34,4 @@ describe('database cache', () => { const x2 = await database.getItem({ id: i2.baseId }) expect(x2).toMatchSnapshot() }) - - test('write two items to queue use processItems', async () => { - const i1: Item = { baseId: 'id1', base: { id: 'id', speckle_type: 'type' } } - const i2: Item = { baseId: 'id2', base: { id: 'id', speckle_type: 'type' } } - const database = new IndexedDatabase({ - indexedDB: new IDBFactory(), - keyRange: IDBKeyRange - }) - await database.add(i1) - await database.add(i2) - await database.disposeAsync() - - const foundItems = new BufferQueue() - const notFoundItems = new BufferQueue() - - await database.processItems({ - ids: [i1.baseId, i2.baseId], - foundItems, - notFoundItems - }) - - expect(foundItems.values()).toMatchSnapshot() - expect(notFoundItems.values()).toMatchSnapshot() - }) }) diff --git a/packages/objectloader2/src/operations/databases/indexedDatabase.ts b/packages/objectloader2/src/operations/databases/indexedDatabase.ts new file mode 100644 index 000000000..03d0d9d4c --- /dev/null +++ b/packages/objectloader2/src/operations/databases/indexedDatabase.ts @@ -0,0 +1,150 @@ +/* eslint-disable @typescript-eslint/no-unsafe-function-type */ +import BatchingQueue from '../../helpers/batchingQueue.js' +import { CustomLogger, Item } from '../../types/types.js' +import { isSafari } from '@speckle/shared' +import { Dexie, DexieOptions, Table } from 'dexie' +import { Database } from '../interfaces.js' + +class ObjectStore extends Dexie { + static #databaseName: string = 'speckle-cache' + objects!: Table // Table type: + + constructor(options: DexieOptions) { + super(ObjectStore.#databaseName, options) + + this.version(1).stores({ + objects: 'baseId, item' // baseId is primary key + }) + } +} + +export interface IndexedDatabaseOptions { + logger?: CustomLogger + indexedDB?: IDBFactory + keyRange?: { + bound: Function + lowerBound: Function + upperBound: Function + } +} + +export default class IndexedDatabase implements Database { + #options: IndexedDatabaseOptions + #logger: CustomLogger + + #cacheDB?: ObjectStore + + #writeQueue: BatchingQueue | undefined + + // #count: number = 0 + + constructor(options: IndexedDatabaseOptions) { + this.#options = options + this.#logger = options.logger || ((): void => {}) + } + + async getAll(keys: string[]): Promise<(Item | undefined)[]> { + await this.#setupCacheDb() + let items: (Item | undefined)[] = [] + // this.#count++ + // const startTime = performance.now() + // this.#logger('Start read ' + x + ' ' + batch.length) + + //faster than BulkGet with dexie + await this.#cacheDB!.transaction('r', this.#cacheDB!.objects, async () => { + const gets = keys.map((key) => this.#cacheDB!.objects.get(key)) + const cachedData = await Promise.all(gets) + items = cachedData + }) + // const endTime = performance.now() + // const duration = endTime - startTime + //this.#logger('Saved batch ' + x + ' ' + batch.length + ' ' + duration / TIME_MS.second) + + return items + } + + async #openDatabase(): Promise { + const db = new ObjectStore({ + indexedDB: this.#options.indexedDB ?? globalThis.indexedDB, + IDBKeyRange: this.#options.keyRange ?? IDBKeyRange, + chromeTransactionDurability: 'relaxed' + }) + await db.open() + return db + } + + async #setupCacheDb(): Promise { + if (this.#cacheDB !== undefined) { + return + } + + // Initialize + await this.#safariFix() + this.#cacheDB = await this.#openDatabase() + } + + //this is for testing only - in the real world we will not use this + async add(item: Item): Promise { + await this.#setupCacheDb() + await this.#cacheDB!.transaction('rw', this.#cacheDB!.objects, async () => { + return await this.#cacheDB?.objects.add(item) + }) + } + + async getItem(params: { id: string }): Promise { + const { id } = params + await this.#setupCacheDb() + //might not be in the real DB yet, so check the write queue first + if (this.#writeQueue) { + const item = this.#writeQueue.get(id) + if (item) { + return item + } + } + + return this.#cacheDB!.transaction('r', this.#cacheDB!.objects, async () => { + return await this.#cacheDB?.objects.get(id) + }) + } + + async cacheSaveBatch(params: { batch: Item[] }): Promise { + await this.#setupCacheDb() + const { batch } = params + //const x = this.#count + //this.#count++ + + // const startTime = performance.now() + // this.#logger('Start save ' + x + ' ' + batch.length) + await this.#cacheDB!.objects.bulkPut(batch) + // const endTime = performance.now() + // const duration = endTime - startTime + //this.#logger('Saved batch ' + x + ' ' + batch.length + ' ' + duration / TIME_MS.second) + } + + /** + * Fixes a Safari bug where IndexedDB requests get lost and never resolve - invoke before you use IndexedDB + * @link Credits and more info: https://github.com/jakearchibald/safari-14-idb-fix + */ + async #safariFix(): Promise { + // No point putting other browsers or older versions of Safari through this mess. + if (!isSafari() || !this.#options.indexedDB?.databases) return Promise.resolve() + + let intervalId: ReturnType + + return new Promise((resolve: () => void) => { + const tryIdb = (): Promise | undefined => + this.#options.indexedDB?.databases().finally(resolve) + intervalId = setInterval(() => { + void tryIdb() + }, 100) + void tryIdb() + }).finally(() => clearInterval(intervalId)) + } + + async disposeAsync(): Promise { + this.#cacheDB?.close() + this.#cacheDB = undefined + await this.#writeQueue?.disposeAsync() + this.#writeQueue = undefined + } +} diff --git a/packages/objectloader2/src/operations/databases/memoryDatabase.ts b/packages/objectloader2/src/operations/databases/memoryDatabase.ts new file mode 100644 index 000000000..6e5d99e10 --- /dev/null +++ b/packages/objectloader2/src/operations/databases/memoryDatabase.ts @@ -0,0 +1,43 @@ +import { Base, Item } from '../../types/types.js' +import { Database } from '../interfaces.js' +import { MemoryDatabaseOptions } from '../options.js' + +export class MemoryDatabase implements Database { + private items: Map + + constructor(options?: MemoryDatabaseOptions) { + this.items = options?.items || new Map() + } + + getAll(keys: string[]): Promise<(Item | undefined)[]> { + const found: (Item | undefined)[] = [] + for (const key of keys) { + const item = this.items.get(key) + if (item) { + found.push({ baseId: key, base: item }) + } else { + found.push(undefined) + } + } + return Promise.resolve(found) + } + + cacheSaveBatch({ batch }: { batch: Item[] }): Promise { + for (const item of batch) { + this.items.set(item.baseId, item.base) + } + return Promise.resolve() + } + + getItem(params: { id: string }): Promise { + const item = this.items.get(params.id) + if (item) { + return Promise.resolve({ baseId: params.id, base: item }) + } + return Promise.resolve(undefined) + } + + disposeAsync(): Promise { + return Promise.resolve() + } +} diff --git a/packages/objectloader2/src/operations/__snapshots__/serverDownloader.spec.ts.snap b/packages/objectloader2/src/operations/downloaders/__snapshots__/serverDownloader.spec.ts.snap similarity index 63% rename from packages/objectloader2/src/operations/__snapshots__/serverDownloader.spec.ts.snap rename to packages/objectloader2/src/operations/downloaders/__snapshots__/serverDownloader.spec.ts.snap index 438aa6f2f..be81cb77d 100644 --- a/packages/objectloader2/src/operations/__snapshots__/serverDownloader.spec.ts.snap +++ b/packages/objectloader2/src/operations/downloaders/__snapshots__/serverDownloader.spec.ts.snap @@ -10,6 +10,7 @@ exports[`downloader > add extra header 1`] = ` "speckle_type": "type", }, "baseId": "id", + "size": 0, } `; @@ -21,6 +22,36 @@ exports[`downloader > download batch of one 1`] = ` "speckle_type": "type", }, "baseId": "id", + "size": 33, + }, +] +`; + +exports[`downloader > download batch of three 1`] = ` +[ + { + "base": { + "id": "id1", + "speckle_type": "type", + }, + "baseId": "id1", + "size": 34, + }, + { + "base": { + "id": "id2", + "speckle_type": "type", + }, + "baseId": "id2", + "size": 34, + }, + { + "base": { + "id": "id3", + "speckle_type": "type", + }, + "baseId": "id3", + "size": 34, }, ] `; @@ -33,6 +64,7 @@ exports[`downloader > download batch of two 1`] = ` "speckle_type": "type", }, "baseId": "id1", + "size": 34, }, { "base": { @@ -40,6 +72,7 @@ exports[`downloader > download batch of two 1`] = ` "speckle_type": "type", }, "baseId": "id2", + "size": 34, }, ] `; @@ -54,5 +87,6 @@ exports[`downloader > download single exists 1`] = ` "speckle_type": "type", }, "baseId": "id", + "size": 0, } `; diff --git a/packages/objectloader2/src/operations/memoryDownloader.ts b/packages/objectloader2/src/operations/downloaders/memoryDownloader.ts similarity index 52% rename from packages/objectloader2/src/operations/memoryDownloader.ts rename to packages/objectloader2/src/operations/downloaders/memoryDownloader.ts index a58d92a9f..383805279 100644 --- a/packages/objectloader2/src/operations/memoryDownloader.ts +++ b/packages/objectloader2/src/operations/downloaders/memoryDownloader.ts @@ -1,24 +1,25 @@ -import AsyncGeneratorQueue from '../helpers/asyncGeneratorQueue.js' -import { Base, Item } from '../types/types.js' -import { Downloader } from './interfaces.js' +import Queue from '../../helpers/queue.js' +import { Base, Item } from '../../types/types.js' +import { Downloader } from '../interfaces.js' export class MemoryDownloader implements Downloader { - #items: Record + #items: Map #rootId: string - #results?: AsyncGeneratorQueue + #results?: Queue - constructor( - rootId: string, - items: Record, - results?: AsyncGeneratorQueue - ) { + constructor(rootId: string, items: Map) { this.#rootId = rootId this.#items = items - this.#results = results } - initializePool(): void {} + initializePool(params: { + results: Queue + total: number + maxDownloadBatchWait?: number + }): void { + this.#results = params.results + } downloadSingle(): Promise { - const root = this.#items[this.#rootId] + const root = this.#items.get(this.#rootId) if (root) { return Promise.resolve({ baseId: this.#rootId, base: root }) } @@ -28,7 +29,7 @@ export class MemoryDownloader implements Downloader { return Promise.resolve() } add(id: string): void { - const base = this.#items[id] + const base = this.#items.get(id) if (base) { this.#results?.add({ baseId: id, base }) return diff --git a/packages/objectloader2/src/operations/serverDownloader.spec.ts b/packages/objectloader2/src/operations/downloaders/serverDownloader.spec.ts similarity index 64% rename from packages/objectloader2/src/operations/serverDownloader.spec.ts rename to packages/objectloader2/src/operations/downloaders/serverDownloader.spec.ts index c10afa901..e7d0e1bd1 100644 --- a/packages/objectloader2/src/operations/serverDownloader.spec.ts +++ b/packages/objectloader2/src/operations/downloaders/serverDownloader.spec.ts @@ -1,38 +1,28 @@ import { describe, expect, test } from 'vitest' import createFetchMock from 'vitest-fetch-mock' import { vi } from 'vitest' -import AsyncGeneratorQueue from '../helpers/asyncGeneratorQueue.js' -import { Item } from '../types/types.js' -import { Cache } from './interfaces.js' +import { Item } from '../../types/types.js' import ServerDownloader from './serverDownloader.js' +import { MemoryPump } from '../../helpers/memoryPump.js' describe('downloader', () => { test('download batch of one', async () => { const fetchMocker = createFetchMock(vi) const i: Item = { baseId: 'id', base: { id: 'id', speckle_type: 'type' } } fetchMocker.mockResponseOnce('id\t' + JSON.stringify(i.base) + '\n') - const results = new AsyncGeneratorQueue() - const db = { - async add(): Promise { - return Promise.resolve() - } - } as unknown as Cache + const pump = new MemoryPump() const downloader = new ServerDownloader({ - database: db, - results, serverUrl: 'http://speckle.test', streamId: 'streamId', objectId: 'objectId', token: 'token', - fetch: fetchMocker }) - downloader.initializePool({ total: 1, maxDownloadBatchWait: 200 }) + downloader.initializePool({ results: pump, total: 1, maxDownloadBatchWait: 200 }) downloader.add('id') await downloader.disposeAsync() - results.dispose() const r = [] - for await (const x of results.consume()) { + for await (const x of pump.gather([i.baseId])) { r.push(x) } @@ -46,15 +36,9 @@ describe('downloader', () => { fetchMocker.mockResponseOnce( 'id1\t' + JSON.stringify(i1.base) + '\nid2\t' + JSON.stringify(i2.base) + '\n' ) - const results = new AsyncGeneratorQueue() - const db = { - async add(): Promise { - return Promise.resolve() - } - } as unknown as Cache + + const pump = new MemoryPump() const downloader = new ServerDownloader({ - database: db, - results, serverUrl: 'http://speckle.test', streamId: 'streamId', objectId: 'objectId', @@ -62,12 +46,46 @@ describe('downloader', () => { fetch: fetchMocker }) - downloader.initializePool({ total: 2, maxDownloadBatchWait: 200 }) + downloader.initializePool({ results: pump, total: 2, maxDownloadBatchWait: 200 }) downloader.add('id') await downloader.disposeAsync() - results.dispose() const r = [] - for await (const x of results.consume()) { + for await (const x of pump.gather([i1.baseId, i2.baseId])) { + r.push(x) + } + + expect(r).toMatchSnapshot() + }) + + test('download batch of three', async () => { + const fetchMocker = createFetchMock(vi) + const i1: Item = { baseId: 'id1', base: { id: 'id1', speckle_type: 'type' } } + const i2: Item = { baseId: 'id2', base: { id: 'id2', speckle_type: 'type' } } + const i3: Item = { baseId: 'id3', base: { id: 'id3', speckle_type: 'type' } } + fetchMocker.mockResponseOnce( + 'id1\t' + + JSON.stringify(i1.base) + + '\nid2\t' + + JSON.stringify(i2.base) + + '\nid3\t' + + JSON.stringify(i3.base) + + '\n' + ) + + const pump = new MemoryPump() + const downloader = new ServerDownloader({ + serverUrl: 'http://speckle.test', + streamId: 'streamId', + objectId: 'objectId', + token: 'token', + + fetch: fetchMocker + }) + downloader.initializePool({ results: pump, total: 2, maxDownloadBatchWait: 200 }) + downloader.add('id') + await downloader.disposeAsync() + const r = [] + for await (const x of pump.gather([i1.baseId, i2.baseId, i3.baseId])) { r.push(x) } @@ -81,15 +99,7 @@ describe('downloader', () => { base: { id: 'id', speckle_type: 'type', __closure: { childIds: 1 } } } fetchMocker.mockResponseOnce(JSON.stringify(i.base)) - const results = new AsyncGeneratorQueue() - const db = { - async add(): Promise { - return Promise.resolve() - } - } as unknown as Cache const downloader = new ServerDownloader({ - database: db, - results, serverUrl: 'http://speckle.test', streamId: 'streamId', objectId: i.baseId, @@ -111,17 +121,9 @@ describe('downloader', () => { (req) => req.headers.get('x-test') === 'asdf', JSON.stringify(i.base) ) - const results = new AsyncGeneratorQueue() - const db = { - async add(): Promise { - return Promise.resolve() - } - } as unknown as Cache const headers = new Headers() headers.set('x-test', 'asdf') const downloader = new ServerDownloader({ - database: db, - results, serverUrl: 'http://speckle.test', headers, streamId: 'streamId', diff --git a/packages/objectloader2/src/operations/serverDownloader.ts b/packages/objectloader2/src/operations/downloaders/serverDownloader.ts similarity index 56% rename from packages/objectloader2/src/operations/serverDownloader.ts rename to packages/objectloader2/src/operations/downloaders/serverDownloader.ts index faf44a2de..8abe5e5d1 100644 --- a/packages/objectloader2/src/operations/serverDownloader.ts +++ b/packages/objectloader2/src/operations/downloaders/serverDownloader.ts @@ -1,22 +1,33 @@ -import BatchedPool from '../helpers/batchedPool.js' -import Queue from '../helpers/queue.js' -import { ObjectLoaderRuntimeError } from '../types/errors.js' -import { Fetcher, isBase, Item } from '../types/types.js' -import { Downloader } from './interfaces.js' -import { BaseDownloadOptions } from './options.js' +import BatchedPool from '../../helpers/batchedPool.js' +import Queue from '../../helpers/queue.js' +import { ObjectLoaderRuntimeError } from '../../types/errors.js' +import { Fetcher, isBase, Item } from '../../types/types.js' +import { Downloader } from '../interfaces.js' + +export interface ServerDownloaderOptions { + serverUrl: string + streamId: string + objectId: string + token?: string + headers?: Headers + fetch?: Fetcher +} export default class ServerDownloader implements Downloader { #requestUrlRootObj: string #requestUrlChildren: string #headers: HeadersInit - #options: BaseDownloadOptions + #options: ServerDownloaderOptions #fetch: Fetcher + #results?: Queue #downloadQueue?: BatchedPool + #decoder = new TextDecoder() - constructor(options: BaseDownloadOptions) { + constructor(options: ServerDownloaderOptions) { this.#options = options - this.#fetch = options.fetch ?? ((...args) => globalThis.fetch(...args)) + this.#fetch = + options.fetch ?? ((...args): Promise => globalThis.fetch(...args)) this.#headers = {} if (options.headers) { @@ -45,17 +56,21 @@ export default class ServerDownloader implements Downloader { return [10000, 30000, 10000, 1000] } - initializePool(params: { total: number; maxDownloadBatchWait?: number }) { - const { total } = params + initializePool(params: { + results: Queue + total: number + maxDownloadBatchWait?: number + }): void { + const { results, total } = params + this.#results = results this.#downloadQueue = new BatchedPool({ concurrencyAndSizes: this.#getDownloadCountAndSizes(total), maxWaitTime: params.maxDownloadBatchWait, - processFunction: (batch: string[]) => + processFunction: (batch: string[]): Promise => this.downloadBatch({ batch, url: this.#requestUrlChildren, - headers: this.#headers, - results: this.#options.results + headers: this.#headers }) }) } @@ -94,9 +109,8 @@ export default class ServerDownloader implements Downloader { batch: string[] url: string headers: HeadersInit - results: Queue }): Promise { - const { batch, url, headers, results } = params + const { batch, url, headers } = params const response = await this.#fetch(url, { method: 'POST', headers: { ...headers, 'Content-Type': 'application/json' }, @@ -109,35 +123,67 @@ export default class ServerDownloader implements Downloader { } const reader = response.body.getReader() - const decoder = new TextDecoder() - let buffer = '' // Temporary buffer to store incoming chunks + let leftover = new Uint8Array(0) let count = 0 while (true) { const { done, value } = await reader.read() if (done) break - // Decode the chunk and add to buffer - buffer += decoder.decode(value, { stream: true }) - // Try to process JSON objects from the buffer - let boundary = buffer.indexOf('\n') - while (boundary !== -1) { - const jsonString = buffer.slice(0, boundary) - buffer = buffer.slice(boundary + 1) - boundary = buffer.indexOf('\n') - if (jsonString) { - const pieces = jsonString.split('\t') - const [id, unparsedObj] = pieces - const item = this.#processJson(id, unparsedObj) - await this.#options.database.add(item) - results.add(item) - count++ - if (count % 1000 === 0) { - await new Promise((resolve) => setTimeout(resolve, 100)) //allow other stuff to happen - } + leftover = await this.processArray(leftover, value, async () => { + count++ + if (count % 1000 === 0) { + await new Promise((resolve) => setTimeout(resolve, 100)) //allow other stuff to happen } + }) + } + } + + async processArray( + leftover: Uint8Array, + value: Uint8Array, + callback: () => Promise + ): Promise { + //this concat will allocate a new array + const combined = this.concatUint8Arrays(leftover, value) + let start = 0 + + //subarray doesn't allocate + for (let i = 0; i < combined.length; i++) { + if (combined[i] === 0x0a) { + const line = combined.subarray(start, i) // line without \n + //strings are allocated here + const item = this.processLine(line) + this.#results?.add(item) + start = i + 1 + await callback() } } + return combined.subarray(start) // carry over remainder + } + + processLine(line: Uint8Array): Item { + for (let i = 0; i < line.length; i++) { + if (line[i] === 0x09) { + //this is a tab + const baseId = this.#decoder.decode(line.subarray(0, i)) + const json = line.subarray(i + 1) + const base = this.#decoder.decode(json) + const item = this.#processJson(baseId, base) + item.size = json.length + return item + } + } + throw new ObjectLoaderRuntimeError( + 'Invalid line format: ' + this.#decoder.decode(line) + ) + } + + concatUint8Arrays(a: Uint8Array, b: Uint8Array): Uint8Array { + const c = new Uint8Array(a.length + b.length) + c.set(a, 0) + c.set(b, a.length) + return c } async downloadSingle(): Promise { @@ -147,6 +193,7 @@ export default class ServerDownloader implements Downloader { this.#validateResponse(response) const responseText = await response.text() const item = this.#processJson(this.#options.objectId, responseText) + item.size = 0 return item } diff --git a/packages/objectloader2/src/operations/indexedDatabase.ts b/packages/objectloader2/src/operations/indexedDatabase.ts deleted file mode 100644 index f0900d053..000000000 --- a/packages/objectloader2/src/operations/indexedDatabase.ts +++ /dev/null @@ -1,167 +0,0 @@ -import BatchingQueue from '../helpers/batchingQueue.js' -import Queue from '../helpers/queue.js' -import { CustomLogger, Item } from '../types/types.js' -import { isSafari, TIME } from '@speckle/shared' -import { BaseDatabaseOptions } from './options.js' -import { Cache } from './interfaces.js' -import { Dexie, DexieOptions, Table } from 'dexie' - -class ObjectStore extends Dexie { - static #databaseName: string = 'speckle-cache' - objects!: Table // Table type: - - constructor(options: DexieOptions) { - super(ObjectStore.#databaseName, options) - - this.version(1).stores({ - objects: 'baseId, item' // baseId is primary key - }) - } -} - -export default class IndexedDatabase implements Cache { - #options: BaseDatabaseOptions - #logger: CustomLogger - - #cacheDB?: ObjectStore - - #writeQueue: BatchingQueue | undefined - - // #count: number = 0 - - constructor(options: BaseDatabaseOptions) { - this.#options = { - ...{ - maxCacheReadSize: 10000, - maxCacheBatchWriteWait: 1000 - }, - ...options - } - this.#logger = options.logger || (() => {}) - } - - async add(item: Item): Promise { - if (!this.#writeQueue) { - await this.#setupCacheDb() - this.#writeQueue = new BatchingQueue({ - batchSize: this.#options.maxCacheWriteSize ?? 10000, - maxWaitTime: this.#options.maxCacheBatchWriteWait, - processFunction: (batch: Item[]) => - this.#cacheSaveBatch({ batch, cacheDB: this.#cacheDB! }) - }) - } - this.#writeQueue.add(item) - } - - async disposeAsync(): Promise { - await this.#writeQueue?.disposeAsync() - } - - async #openDatabase(): Promise { - const db = new ObjectStore({ - indexedDB: this.#options.indexedDB ?? globalThis.indexedDB, - IDBKeyRange: this.#options.keyRange ?? IDBKeyRange, - chromeTransactionDurability: 'relaxed' - }) - await db.open() - return db - } - - async #setupCacheDb(): Promise { - if (this.#cacheDB !== undefined) { - return - } - - // Initialize - await this.#safariFix() - this.#cacheDB = await this.#openDatabase() - } - - async processItems(params: { - ids: string[] - foundItems: Queue - notFoundItems: Queue - }): Promise { - const { ids, foundItems, notFoundItems } = params - await this.#setupCacheDb() - const maxCacheReadSize = this.#options.maxCacheReadSize ?? 10000 - - for (let i = 0; i < ids.length; ) { - if ((this.#writeQueue?.count() ?? 0) > maxCacheReadSize * 2) { - this.#logger( - 'pausing reads (# in write queue: ' + this.#writeQueue?.count() + ')' - ) - await new Promise((resolve) => setTimeout(resolve, TIME.second)) // Pause for 1 second, protects against out of memory - continue - } - const batch = ids.slice(i, i + maxCacheReadSize) - // const x = this.#count - // this.#count++ - // const startTime = performance.now() - // this.#logger('Start read ' + x + ' ' + batch.length) - - //faster than BulkGet with dexie - await this.#cacheDB!.transaction('r', this.#cacheDB!.objects, async () => { - const gets = batch.map((key) => this.#cacheDB!.objects.get(key)) - const cachedData = await Promise.all(gets) - for (let i = 0; i < cachedData.length; i++) { - if (cachedData[i]) { - foundItems.add(cachedData[i]!) - } else { - notFoundItems.add(batch[i]) - } - } - }) - // const endTime = performance.now() - // const duration = endTime - startTime - // this.#logger('Read batch ' + x + ' ' + batch.length + ' ' + duration / TIME_MS.second) - - // interate down here to help with pausing - i += maxCacheReadSize - } - } - - async getItem(params: { id: string }): Promise { - const { id } = params - await this.#setupCacheDb() - - return this.#cacheDB!.transaction('r', this.#cacheDB!.objects, async () => { - return await this.#cacheDB?.objects.get(id) - }) - } - - async #cacheSaveBatch(params: { - batch: Item[] - cacheDB: ObjectStore - }): Promise { - const { batch, cacheDB } = params - //const x = this.#count - //this.#count++ - - // const startTime = performance.now() - // this.#logger('Start save ' + x + ' ' + batch.length) - await cacheDB.objects.bulkPut(batch) - // const endTime = performance.now() - // const duration = endTime - startTime - //this.#logger('Saved batch ' + x + ' ' + batch.length + ' ' + duration / TIME_MS.second) - } - - /** - * Fixes a Safari bug where IndexedDB requests get lost and never resolve - invoke before you use IndexedDB - * @link Credits and more info: https://github.com/jakearchibald/safari-14-idb-fix - */ - async #safariFix(): Promise { - // No point putting other browsers or older versions of Safari through this mess. - if (!isSafari() || !this.#options.indexedDB?.databases) return Promise.resolve() - - let intervalId: ReturnType - - return new Promise((resolve: () => void) => { - const tryIdb = () => this.#options.indexedDB?.databases().finally(resolve) - intervalId = setInterval(() => { - void tryIdb() - }, 100) - void tryIdb() - }).finally(() => clearInterval(intervalId)) - } -} diff --git a/packages/objectloader2/src/operations/interfaces.ts b/packages/objectloader2/src/operations/interfaces.ts index 08f12ace5..1be903feb 100644 --- a/packages/objectloader2/src/operations/interfaces.ts +++ b/packages/objectloader2/src/operations/interfaces.ts @@ -1,20 +1,19 @@ import Queue from '../helpers/queue.js' import { Item } from '../types/types.js' -export interface Cache { - getItem(params: { id: string }): Promise - processItems(params: { - ids: string[] - foundItems: Queue - notFoundItems: Queue - }): Promise - - add(item: Item): Promise - disposeAsync(): Promise -} - export interface Downloader extends Queue { - initializePool(params: { total: number }): void + initializePool(params: { + results: Queue + total: number + maxDownloadBatchWait?: number + }): void downloadSingle(): Promise disposeAsync(): Promise } + +export interface Database { + getAll(keys: string[]): Promise<(Item | undefined)[]> + getItem(params: { id: string }): Promise + cacheSaveBatch(params: { batch: Item[] }): Promise + disposeAsync(): Promise +} diff --git a/packages/objectloader2/src/operations/memoryDatabase.ts b/packages/objectloader2/src/operations/memoryDatabase.ts deleted file mode 100644 index 680f44576..000000000 --- a/packages/objectloader2/src/operations/memoryDatabase.ts +++ /dev/null @@ -1,42 +0,0 @@ -import Queue from '../helpers/queue.js' -import { Base, Item } from '../types/types.js' -import { Cache } from './interfaces.js' -import { MemoryDatabaseOptions } from './options.js' - -export class MemoryDatabase implements Cache { - #items: Record - constructor(options?: MemoryDatabaseOptions) { - this.#items = options?.items || {} - } - - getItem(params: { id: string }): Promise { - const item = this.#items[params.id] - if (item) { - return Promise.resolve({ baseId: params.id, base: item }) - } - return Promise.resolve(undefined) - } - processItems(params: { - ids: string[] - foundItems: Queue - notFoundItems: Queue - }): Promise { - const { ids, foundItems, notFoundItems } = params - for (const id of ids) { - const item = this.#items[id] - if (item) { - foundItems.add({ baseId: id, base: item }) - } else { - notFoundItems.add(id) - } - } - return Promise.resolve() - } - add(item: Item): Promise { - this.#items[item.baseId] = item.base - return Promise.resolve() - } - disposeAsync(): Promise { - return Promise.resolve() - } -} diff --git a/packages/objectloader2/src/operations/objectLoader2.spec.ts b/packages/objectloader2/src/operations/objectLoader2.spec.ts index ad592f076..6d2f2c698 100644 --- a/packages/objectloader2/src/operations/objectLoader2.spec.ts +++ b/packages/objectloader2/src/operations/objectLoader2.spec.ts @@ -1,56 +1,45 @@ import { describe, expect, test } from 'vitest' -import ObjectLoader2 from './objectLoader2.js' +import { ObjectLoader2 } from './objectLoader2.js' import { Base, Item } from '../types/types.js' -import { Cache, Downloader } from './interfaces.js' -import Queue from '../helpers/queue.js' -import { MemoryDatabase } from './memoryDatabase.js' -import { MemoryDownloader } from './memoryDownloader.js' -import AsyncGeneratorQueue from '../helpers/asyncGeneratorQueue.js' +import { MemoryDownloader } from './downloaders/memoryDownloader.js' +import { IDBFactory, IDBKeyRange } from 'fake-indexeddb' +import { MemoryDatabase } from './databases/memoryDatabase.js' +import IndexedDatabase from './databases/indexedDatabase.js' describe('objectloader2', () => { test('can get a root object from cache', async () => { - const root = { baseId: 'baseId' } as unknown as Item - const cache = { - getItem(params: { id: string }): Promise { - expect(params.id).toBe(root.baseId) - return Promise.resolve(root) - } - } as Cache - const downloader = {} as Downloader + const rootId = 'baseId' + const rootBase: Base = { id: 'baseId', speckle_type: 'type' } + const downloader = new MemoryDownloader( + rootId, + new Map([[rootId, rootBase]]) + ) const loader = new ObjectLoader2({ - serverUrl: 'a', - streamId: 'b', - objectId: root.baseId, - cache, - downloader + rootId, + downloader, + database: new IndexedDatabase({ + indexedDB: new IDBFactory(), + keyRange: IDBKeyRange + }) }) const x = await loader.getRootObject() expect(x).toMatchSnapshot() }) test('can get a root object from downloader', async () => { - const root = { baseId: 'baseId' } as unknown as Item - const cache = { - getItem(params: { id: string }): Promise { - expect(params.id).toBe(root.baseId) - return Promise.resolve(undefined) - }, - add(item: Item): Promise { - expect(item).toBe(root) - return Promise.resolve() - } - } as Cache - const downloader = { - downloadSingle(): Promise { - return Promise.resolve(root) - } - } as Downloader + const rootId = 'baseId' + const rootBase: Base = { id: 'baseId', speckle_type: 'type' } + const downloader = new MemoryDownloader( + rootId, + new Map([[rootId, rootBase]]) + ) const loader = new ObjectLoader2({ - serverUrl: 'a', - streamId: 'b', - objectId: root.baseId, - cache, - downloader + rootId, + downloader, + database: new IndexedDatabase({ + indexedDB: new IDBFactory(), + keyRange: IDBKeyRange + }) }) const x = await loader.getRootObject() expect(x).toMatchSnapshot() @@ -59,20 +48,18 @@ describe('objectloader2', () => { test('can get single object from cache using iterator', async () => { const rootId = 'baseId' const rootBase: Base = { id: 'baseId', speckle_type: 'type' } - const root = { baseId: rootId, base: rootBase } as unknown as Item - const cache = { - getItem(params: { id: string }): Promise { - expect(params.id).toBe(rootId) - return Promise.resolve(root) - } - } as Cache - const downloader = {} as Downloader + + const downloader = new MemoryDownloader( + rootId, + new Map([[rootId, rootBase]]) + ) const loader = new ObjectLoader2({ - serverUrl: 'a', - streamId: 'b', - objectId: rootId, - cache, - downloader + rootId, + downloader, + database: new IndexedDatabase({ + indexedDB: new IDBFactory(), + keyRange: IDBKeyRange + }) }) const r = [] for await (const x of loader.getObjectIterator()) { @@ -82,63 +69,6 @@ describe('objectloader2', () => { expect(r).toMatchSnapshot() }) - test('can get root/child object from cache using iterator', async () => { - const child1Base = { id: 'child1Id' } - const child1 = { baseId: 'child1Id', base: child1Base } as unknown as Item - - const rootId = 'rootId' - const rootBase: Base = { - id: 'rootId', - speckle_type: 'type', - __closure: { child1Id: 100 } - } - const root = { - baseId: rootId, - base: rootBase - } as unknown as Item - - const cache = { - getItem(params: { id: string }): Promise { - expect(params.id).toBe(root.baseId) - return Promise.resolve(root) - }, - processItems(params: { - ids: string[] - foundItems: Queue - - notFoundItems: Queue - }): Promise { - expect(params.ids.length).toBe(1) - expect(params.ids[0]).toBe(child1.baseId) - params.foundItems.add(child1) - return Promise.resolve() - }, - disposeAsync(): Promise { - return Promise.resolve() - } - } as Cache - const downloader = { - initializePool(params: { total: number }): void { - expect(params.total).toBe(1) - }, - disposeAsync(): Promise { - return Promise.resolve() - } - } as Downloader - const loader = new ObjectLoader2({ - serverUrl: 'a', - streamId: 'b', - objectId: root.baseId, - cache, - downloader - }) - const r = [] - for await (const x of loader.getObjectIterator()) { - r.push(x) - } - expect(r).toMatchSnapshot() - }) - test('can get root/child object from memory cache using iterator and getObject', async () => { const child1Base = { id: 'child1Id', speckle_type: 'type' } as Base const child1 = { baseId: 'child1Id', base: child1Base } as unknown as Item @@ -152,19 +82,18 @@ describe('objectloader2', () => { const root = { baseId: rootId, base: rootBase - } as unknown as Item + } as Item - const records: Record = {} - records[root.baseId] = rootBase - records[child1.baseId] = child1Base + const records: Map = new Map() + records.set(root.baseId, rootBase) + records.set(child1.baseId, child1Base) const loader = new ObjectLoader2({ - serverUrl: 'a', - streamId: 'b', - objectId: root.baseId, - cache: new MemoryDatabase({ items: records }), - downloader: new MemoryDownloader(rootId, records) + rootId: root.baseId, + downloader: new MemoryDownloader(rootId, records), + database: new MemoryDatabase({ items: records }) }) + const r = [] const obj = loader.getObject({ id: child1.baseId }) for await (const x of loader.getObjectIterator()) { @@ -193,18 +122,17 @@ describe('objectloader2', () => { base: rootBase } as unknown as Item - const records: Record = {} - records[root.baseId] = rootBase - records[child1.baseId] = child1Base + const records: Map = new Map() + records.set(root.baseId, rootBase) + records.set(child1.baseId, child1Base) - const results: AsyncGeneratorQueue = new AsyncGeneratorQueue() const loader = new ObjectLoader2({ - serverUrl: 'a', - streamId: 'b', - objectId: root.baseId, - results, - cache: new MemoryDatabase(), - downloader: new MemoryDownloader(rootId, records, results) + rootId: root.baseId, + downloader: new MemoryDownloader(rootId, records), + database: new IndexedDatabase({ + indexedDB: new IDBFactory(), + keyRange: IDBKeyRange + }) }) const r = [] const obj = loader.getObject({ id: child1.baseId }) @@ -220,26 +148,30 @@ describe('objectloader2', () => { }) test('add extra header', async () => { - const root = { baseId: 'baseId' } as unknown as Item - const cache = { - getItem(params: { id: string }): Promise { - expect(params.id).toBe(root.baseId) - return Promise.resolve(root) - } - } as Cache - const downloader = {} as Downloader + const rootId = 'rootId' + const rootBase: Base = { + id: 'rootId', + speckle_type: 'type', + __closure: { child1Id: 100 } + } + const root = { + baseId: rootId, + base: rootBase + } as Item + + const records: Map = new Map() + records.set(root.baseId, rootBase) const headers = new Headers() headers.set('x-test', 'asdf') const loader = new ObjectLoader2({ - serverUrl: 'a', - streamId: 'b', - objectId: root.baseId, - headers, - cache, - downloader + rootId: root.baseId, + downloader: new MemoryDownloader(rootId, records), + database: new IndexedDatabase({ + indexedDB: new IDBFactory(), + keyRange: IDBKeyRange + }) }) const x = await loader.getRootObject() - expect(x).toBe(root) expect(x).toMatchSnapshot() }) diff --git a/packages/objectloader2/src/operations/objectLoader2.ts b/packages/objectloader2/src/operations/objectLoader2.ts index 708f489af..81dd2ecf0 100644 --- a/packages/objectloader2/src/operations/objectLoader2.ts +++ b/packages/objectloader2/src/operations/objectLoader2.ts @@ -1,81 +1,79 @@ import AsyncGeneratorQueue from '../helpers/asyncGeneratorQueue.js' -import { Cache, Downloader } from './interfaces.js' -import IndexedDatabase from './indexedDatabase.js' -import ServerDownloader from './serverDownloader.js' +import { Downloader, Database } from './interfaces.js' import { CustomLogger, Base, Item } from '../types/types.js' -import { ObjectLoader2Options } from './options.js' -import { MemoryDownloader } from './memoryDownloader.js' -import { MemoryDatabase } from './memoryDatabase.js' +import { CacheOptions, ObjectLoader2Options } from './options.js' import { DefermentManager } from '../helpers/defermentManager.js' +import { CacheReader } from '../helpers/cacheReader.js' +import { CachePump } from '../helpers/cachePump.js' +import AggregateQueue from '../helpers/aggregateQueue.js' +import { ObjectLoader2Factory } from './objectLoader2Factory.js' -export default class ObjectLoader2 { - #objectId: string +export class ObjectLoader2 { + #rootId: string #logger: CustomLogger - #database: Cache + #database: Database #downloader: Downloader + #pump: CachePump + #cache: CacheReader #deferments: DefermentManager #gathered: AsyncGeneratorQueue - constructor(options: ObjectLoader2Options) { - this.#objectId = options.objectId + #root?: Item = undefined + constructor(options: ObjectLoader2Options) { + this.#rootId = options.rootId this.#logger = options.logger || console.log - this.#gathered = options.results || new AsyncGeneratorQueue() - this.#deferments = new DefermentManager() - this.#database = - options.cache || - new IndexedDatabase({ - logger: this.#logger, - maxCacheReadSize: 10_000, - maxCacheWriteSize: 5_000, - indexedDB: options.indexedDB, - keyRange: options.keyRange - }) - this.#downloader = - options.downloader || - new ServerDownloader({ - database: this.#database, - results: this.#gathered, - serverUrl: options.serverUrl, - streamId: options.streamId, - objectId: this.#objectId, - token: options.token, - headers: options.headers - }) + + const cacheOptions: CacheOptions = { + logger: this.#logger, + maxCacheReadSize: 10_000, + maxCacheWriteSize: 10_000, + maxWriteQueueSize: 40_000, + maxCacheBatchWriteWait: 3_000, + maxCacheBatchReadWait: 3_000 + } + + this.#gathered = new AsyncGeneratorQueue() + this.#database = options.database + this.#deferments = new DefermentManager({ + maxSizeInMb: 2_000, // 2 GBs + ttlms: 5_000, // 5 seconds + logger: this.#logger + }) + this.#cache = new CacheReader(this.#database, this.#deferments, cacheOptions) + this.#pump = new CachePump( + this.#database, + this.#gathered, + this.#deferments, + cacheOptions + ) + this.#downloader = options.downloader } async disposeAsync(): Promise { - await Promise.all([ - this.#database.disposeAsync(), - this.#downloader.disposeAsync(), - this.#gathered.dispose() - ]) + await Promise.all([this.#downloader.disposeAsync(), this.#cache.disposeAsync()]) + this.#deferments.dispose() } async getRootObject(): Promise { - const cachedRootObject = await this.#database.getItem({ id: this.#objectId }) - if (cachedRootObject) { - return cachedRootObject + if (!this.#root) { + this.#root = await this.#database.getItem({ id: this.#rootId }) + if (!this.#root) { + this.#root = await this.#downloader.downloadSingle() + } } - const rootItem = await this.#downloader.downloadSingle() - - await this.#database.add(rootItem) - return rootItem + return this.#root } async getObject(params: { id: string }): Promise { - const item = await this.#database.getItem({ id: params.id }) - if (item) { - return item.base - } - return await this.#deferments.defer({ id: params.id }) + return await this.#cache.getObject({ id: params.id }) } - async getTotalObjectCount() { + async getTotalObjectCount(): Promise { const rootObj = await this.getRootObject() const totalChildrenCount = Object.keys(rootObj?.base.__closure || {}).length return totalChildrenCount + 1 //count the root @@ -87,47 +85,27 @@ export default class ObjectLoader2 { this.#logger('No root object found!') return } + //only for root + this.#pump.add(rootItem) yield rootItem.base if (!rootItem.base.__closure) return const children = Object.keys(rootItem.base.__closure) const total = children.length - this.#downloader.initializePool({ total }) - const processPromise = this.#database.processItems({ - ids: children, - foundItems: this.#gathered, - notFoundItems: this.#downloader + this.#downloader.initializePool({ + results: new AggregateQueue(this.#gathered, this.#pump), + total }) - let count = 0 - for await (const item of this.#gathered.consume()) { - this.#deferments.undefer(item) + for await (const item of this.#pump.gather(children, this.#downloader)) { yield item.base - count++ - if (count >= total) { - await this.disposeAsync() - } } - await processPromise } static createFromObjects(objects: Base[]): ObjectLoader2 { - const root = objects[0] - const records: Record = {} - objects.forEach((element) => { - records[element.id] = element - }) - const loader = new ObjectLoader2({ - serverUrl: 'dummy', - streamId: 'dummy', - objectId: root.id, - cache: new MemoryDatabase({ items: records }), - downloader: new MemoryDownloader(root.id, records) - }) - return loader + return ObjectLoader2Factory.createFromObjects(objects) } static createFromJSON(json: string): ObjectLoader2 { - const jsonObj = JSON.parse(json) as Base[] - return this.createFromObjects(jsonObj) + return ObjectLoader2Factory.createFromJSON(json) } } diff --git a/packages/objectloader2/src/operations/objectLoader2Factory.ts b/packages/objectloader2/src/operations/objectLoader2Factory.ts new file mode 100644 index 000000000..8d182842a --- /dev/null +++ b/packages/objectloader2/src/operations/objectLoader2Factory.ts @@ -0,0 +1,56 @@ +import { Base, CustomLogger } from '../types/types.js' +import IndexedDatabase from './databases/indexedDatabase.js' +import { MemoryDatabase } from './databases/memoryDatabase.js' +import { MemoryDownloader } from './downloaders/memoryDownloader.js' +import ServerDownloader from './downloaders/serverDownloader.js' +import { ObjectLoader2 } from './objectLoader2.js' + +export class ObjectLoader2Factory { + static createFromObjects(objects: Base[]): ObjectLoader2 { + const root = objects[0] + const records: Map = new Map() + objects.forEach((element) => { + records.set(element.id, element) + }) + const loader = new ObjectLoader2({ + rootId: root.id, + database: new MemoryDatabase({ items: records }), + downloader: new MemoryDownloader(root.id, records) + }) + return loader + } + + static createFromJSON(json: string): ObjectLoader2 { + const jsonObj = JSON.parse(json) as Base[] + return this.createFromObjects(jsonObj) + } + + static createFromUrl(params: { + serverUrl: string + streamId: string + objectId: string + token?: string + headers?: Headers + // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type + keyRange?: { bound: Function; lowerBound: Function; upperBound: Function } + indexedDB?: IDBFactory + logger?: CustomLogger + }): ObjectLoader2 { + const loader = new ObjectLoader2({ + rootId: params.objectId, + downloader: new ServerDownloader({ + serverUrl: params.serverUrl, + streamId: params.streamId, + objectId: params.objectId, + token: params.token, + headers: params.headers + }), + database: new IndexedDatabase({ + logger: params.logger, + indexedDB: params.indexedDB, + keyRange: params.keyRange + }) + }) + return loader + } +} diff --git a/packages/objectloader2/src/operations/options.ts b/packages/objectloader2/src/operations/options.ts index 7bc7c6e81..39478f678 100644 --- a/packages/objectloader2/src/operations/options.ts +++ b/packages/objectloader2/src/operations/options.ts @@ -1,48 +1,29 @@ -/* eslint-disable @typescript-eslint/no-unsafe-function-type */ -import AsyncGeneratorQueue from '../helpers/asyncGeneratorQueue.js' -import Queue from '../helpers/queue.js' -import { Base, CustomLogger, Fetcher, Item } from '../types/types.js' -import { Cache, Downloader } from './interfaces.js' +import { Base, CustomLogger } from '../types/types.js' +import { Downloader, Database } from './interfaces.js' export interface ObjectLoader2Options { - keyRange?: { bound: Function; lowerBound: Function; upperBound: Function } - indexedDB?: IDBFactory - serverUrl: string - streamId: string - objectId: string - token?: string + rootId: string + downloader: Downloader + database: Database logger?: CustomLogger - headers?: Headers - results?: AsyncGeneratorQueue - cache?: Cache - downloader?: Downloader -} -export interface BaseDatabaseOptions { - logger?: CustomLogger - indexedDB?: IDBFactory - keyRange?: { - bound: Function - lowerBound: Function - upperBound: Function - } - maxCacheReadSize?: number - maxCacheWriteSize?: number - maxCacheBatchWriteWait?: number } -export interface BaseDownloadOptions { - serverUrl: string - streamId: string - objectId: string - token?: string - headers?: Headers - - fetch?: Fetcher - database: Cache - results: Queue +export interface CacheOptions { + logger?: CustomLogger + maxCacheReadSize: number + maxCacheWriteSize: number + maxCacheBatchWriteWait: number + maxCacheBatchReadWait: number + maxWriteQueueSize: number } export interface MemoryDatabaseOptions { logger?: CustomLogger - items?: Record + items?: Map +} + +export interface DefermentManagerOptions { + logger?: CustomLogger + maxSizeInMb: number + ttlms: number } diff --git a/packages/objectloader2/src/operations/traverser.spec.ts b/packages/objectloader2/src/operations/traverser.spec.ts index ebce891cd..620c0a983 100644 --- a/packages/objectloader2/src/operations/traverser.spec.ts +++ b/packages/objectloader2/src/operations/traverser.spec.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'vitest' import { Base } from '../types/types.js' -import ObjectLoader2 from './objectLoader2.js' +import { ObjectLoader2 } from './objectLoader2.js' import Traverser from './traverser.js' describe('Traverser', () => { diff --git a/packages/objectloader2/src/operations/traverser.ts b/packages/objectloader2/src/operations/traverser.ts index 2c8304908..72fbcba16 100644 --- a/packages/objectloader2/src/operations/traverser.ts +++ b/packages/objectloader2/src/operations/traverser.ts @@ -1,5 +1,5 @@ import { Base, DataChunk, isBase, isReference, isScalar } from '../types/types.js' -import ObjectLoader2 from './objectLoader2.js' +import { ObjectLoader2 } from './objectLoader2.js' export type ProgressStage = 'download' | 'construction' export type OnProgress = (e: { diff --git a/packages/objectloader2/src/test/e2e.spec.ts b/packages/objectloader2/src/test/e2e.spec.ts index cd735a95c..c3a3ffc75 100644 --- a/packages/objectloader2/src/test/e2e.spec.ts +++ b/packages/objectloader2/src/test/e2e.spec.ts @@ -1,8 +1,8 @@ import { describe, test, expect } from 'vitest' import { IDBFactory, IDBKeyRange } from 'fake-indexeddb' -import ObjectLoader2 from '../operations/objectLoader2.js' import { Base } from '../types/types.js' -import { TIME } from '@speckle/shared' +import { TIME_MS } from '@speckle/shared' +import { ObjectLoader2Factory } from '../operations/objectLoader2Factory.js' describe('e2e', () => { test( @@ -10,7 +10,7 @@ describe('e2e', () => { async () => { // Revit sample house (good for bim-like stuff with many display meshes) //const resource = 'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8' - const loader = new ObjectLoader2({ + const loader = ObjectLoader2Factory.createFromUrl({ serverUrl: 'https://app.speckle.systems', streamId: 'da9e320dad', objectId: '31d10c0cea569a1e26809658ed27e281', @@ -36,6 +36,6 @@ describe('e2e', () => { expect(base2).toBeDefined() expect(base2.id).toBe('3841e3cbc45d52c47bc2f1b7b0ad4eb9') }, - 10 * TIME.second + 10 * TIME_MS.second ) }) diff --git a/packages/objectloader2/src/types/types.ts b/packages/objectloader2/src/types/types.ts index af53829c1..ad7d669b0 100644 --- a/packages/objectloader2/src/types/types.ts +++ b/packages/objectloader2/src/types/types.ts @@ -8,6 +8,7 @@ export type Fetcher = ( export interface Item { baseId: string base: Base + size?: number } export interface Base { diff --git a/packages/preview-service/Dockerfile b/packages/preview-service/Dockerfile index 5e0628e27..8270f6479 100644 --- a/packages/preview-service/Dockerfile +++ b/packages/preview-service/Dockerfile @@ -30,14 +30,14 @@ COPY packages/frontend-2/type-augmentations/stubs ./packages/frontend-2/type-aug COPY packages/preview-frontend/package.json ./packages/preview-frontend/ COPY packages/preview-service/package.json ./packages/preview-service/ COPY packages/viewer/package.json ./packages/viewer/ -COPY packages/objectloader/package.json ./packages/objectloader/ +COPY packages/objectloader2/package.json ./packages/objectloader2/ COPY packages/shared/package.json ./packages/shared/ RUN PUPPETEER_SKIP_DOWNLOAD=true PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 yarn workspaces focus -A -# Onyl copy in the relevant source files for the dependencies +# Only copy in the relevant source files for the dependencies COPY packages/shared ./packages/shared/ -COPY packages/objectloader ./packages/objectloader/ +COPY packages/objectloader2 ./packages/objectloader2/ COPY packages/viewer ./packages/viewer/ COPY packages/preview-frontend ./packages/preview-frontend/ COPY packages/preview-service ./packages/preview-service/ diff --git a/packages/viewer-sandbox/src/Sandbox.ts b/packages/viewer-sandbox/src/Sandbox.ts index 264593398..f0d57d345 100644 --- a/packages/viewer-sandbox/src/Sandbox.ts +++ b/packages/viewer-sandbox/src/Sandbox.ts @@ -55,7 +55,7 @@ import Bright from '../assets/hdri/Bright.png' import { Euler, Vector3, Box3, Color, LinearFilter } from 'three' import { GeometryType } from '@speckle/viewer' import { MeshBatch } from '@speckle/viewer' -import ObjectLoader2 from '@speckle/objectloader2' +import { ObjectLoader2Factory } from '@speckle/objectloader2' export default class Sandbox { private viewer: Viewer @@ -1312,10 +1312,16 @@ export default class Sandbox { true, undefined ) + let progress = 0 /** Too spammy */ loader.on(LoaderEvent.LoadProgress, (arg: { progress: number; id: string }) => { - if (colorImage) - colorImage.style.clipPath = `inset(${(1 - arg.progress) * 100}% 0 0 0)` + const p = Math.floor(arg.progress * 100) + if (p > progress) { + if (colorImage) + colorImage.style.clipPath = `inset(${(1 - arg.progress) * 100}% 0 0 0)` + progress = p + console.log(`Loading ${p}%`) + } }) loader.on(LoaderEvent.LoadCancelled, (resource: string) => { console.warn(`Resource ${resource} loading was canceled`) @@ -1372,7 +1378,7 @@ export default class Sandbox { options: { enableCaching: true } })*/ - const loader = new ObjectLoader2({ + const loader = ObjectLoader2Factory.createFromUrl({ serverUrl, streamId, objectId, diff --git a/packages/viewer-sandbox/src/main.ts b/packages/viewer-sandbox/src/main.ts index b462f0d0f..1e21b2455 100644 --- a/packages/viewer-sandbox/src/main.ts +++ b/packages/viewer-sandbox/src/main.ts @@ -111,8 +111,10 @@ const getStream = () => { return ( // prettier-ignore // Revit sample house (good for bim-like stuff with many display meshes) - // 'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8' + 'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8' // 'https://latest.speckle.systems/streams/c1faab5c62/commits/ab1a1ab2b6' + // 'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8' + // 'https://latest.speckle.systems/streams/58b5648c4d/commits/60371ecb2d' // 'Super' heavy revit shit // 'https://app.speckle.systems/streams/e6f9156405/commits/0694d53bb5' // IFC building (good for a tree based structure) @@ -529,7 +531,7 @@ const getStream = () => { // 'https://app.speckle.systems/projects/f7bb16037a/models/5d090c6f07' // Large topological stuff - 'https://app.speckle.systems/projects/7a489ac0d4/models/146d5fbe27,3e481c9a58,65b4cf97d5,6d07577256,903850fa6f' + // 'https://app.speckle.systems/projects/7a489ac0d4/models/146d5fbe27,3e481c9a58,65b4cf97d5,6d07577256,903850fa6f' ) } diff --git a/packages/viewer/package.json b/packages/viewer/package.json index 8bbea4c11..b1cec59de 100644 --- a/packages/viewer/package.json +++ b/packages/viewer/package.json @@ -52,7 +52,7 @@ "threejs" ], "dependencies": { - "@speckle/objectloader": "workspace:^", + "@speckle/objectloader2": "workspace:^", "@speckle/shared": "workspace:^", "@types/flat": "^5.0.2", "flat": "^5.0.2", diff --git a/packages/viewer/src/modules/loaders/Speckle/SpeckleConverter.ts b/packages/viewer/src/modules/loaders/Speckle/SpeckleConverter.ts index 314280449..6e10b3e8d 100644 --- a/packages/viewer/src/modules/loaders/Speckle/SpeckleConverter.ts +++ b/packages/viewer/src/modules/loaders/Speckle/SpeckleConverter.ts @@ -3,8 +3,8 @@ import { MathUtils, Matrix4 } from 'three' import { type TreeNode, WorldTree } from '../../tree/WorldTree.js' import { NodeMap } from '../../tree/NodeMap.js' import { SpeckleType, type SpeckleObject } from '../../../index.js' -import type ObjectLoader from '@speckle/objectloader' import Logger from '../../utils/Logger.js' +import { ObjectLoader2 } from '@speckle/objectloader2' export type ConverterResultDelegate = () => Promise export type SpeckleConverterNodeDelegate = @@ -16,7 +16,7 @@ export type SpeckleConverterNodeDelegate = * Warning: HIC SVNT DRACONES. */ export default class SpeckleConverter { - protected objectLoader: ObjectLoader + protected objectLoader: ObjectLoader2 protected activePromises: number protected maxChildrenPromises: number protected spoofIDs = false @@ -63,7 +63,7 @@ export default class SpeckleConverter { protected readonly IgnoreNodes = ['Parameter'] - constructor(objectLoader: ObjectLoader, tree: WorldTree) { + constructor(objectLoader: ObjectLoader2, tree: WorldTree) { if (!objectLoader) { Logger.warn( 'Converter initialized without a corresponding object loader. Any objects that include references will throw errors.' @@ -276,9 +276,9 @@ export default class SpeckleConverter { const chunked: unknown[] = [] for (const ref of arr) { - const real: Record = await this.objectLoader.getObject( - ref.referencedId - ) + const real: Record = (await this.objectLoader.getObject({ + id: ref.referencedId + })) as unknown as Record chunked.push(real.data) // await this.asyncPause() } @@ -296,9 +296,9 @@ export default class SpeckleConverter { */ private async resolveReference(obj: SpeckleObject): Promise { if (obj.referencedId) { - const resolvedObj = (await this.objectLoader.getObject( - obj.referencedId - )) as SpeckleObject + const resolvedObj = (await this.objectLoader.getObject({ + id: obj.referencedId + })) as SpeckleObject // this.asyncPause() return resolvedObj } else return obj diff --git a/packages/viewer/src/modules/loaders/Speckle/SpeckleLoader.ts b/packages/viewer/src/modules/loaders/Speckle/SpeckleLoader.ts index 65286640c..e0449910b 100644 --- a/packages/viewer/src/modules/loaders/Speckle/SpeckleLoader.ts +++ b/packages/viewer/src/modules/loaders/Speckle/SpeckleLoader.ts @@ -1,14 +1,13 @@ import SpeckleConverter from './SpeckleConverter.js' import { Loader, LoaderEvent } from '../Loader.js' -import ObjectLoader from '@speckle/objectloader' import { SpeckleGeometryConverter } from './SpeckleGeometryConverter.js' import { WorldTree, type SpeckleObject } from '../../../index.js' -import { AsyncPause } from '../../World.js' import Logger from '../../utils/Logger.js' +import { ObjectLoader2, ObjectLoader2Factory } from '@speckle/objectloader2' import { TIME_MS } from '@speckle/shared' export class SpeckleLoader extends Loader { - protected loader: ObjectLoader + protected loader: ObjectLoader2 protected converter: SpeckleConverter protected tree: WorldTree protected isCancelled = false @@ -49,9 +48,9 @@ export class SpeckleLoader extends Loader { protected initObjectLoader( resource: string, authToken?: string, - enableCaching?: boolean, + _enableCaching?: boolean, resourceData?: unknown - ): ObjectLoader { + ): ObjectLoader2 { resourceData let token = undefined try { @@ -81,14 +80,7 @@ export class SpeckleLoader extends Loader { const streamId = segments[2] const objectId = segments[4] - return new ObjectLoader({ - serverUrl, - token, - streamId, - objectId, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - options: { enableCaching, customLogger: (Logger as any).log } - }) + return ObjectLoader2Factory.createFromUrl({ serverUrl, streamId, objectId, token }) } public async load(): Promise { @@ -101,8 +93,6 @@ export class SpeckleLoader extends Loader { Logger.warn('Downloading object ', this.resource) - const pause = new AsyncPause() - for await (const obj of this.loader.getObjectIterator()) { if (this.isCancelled) { this.emit(LoaderEvent.LoadCancelled, this.resource) @@ -114,10 +104,6 @@ export class SpeckleLoader extends Loader { obj as SpeckleObject, async () => { viewerLoads++ - pause.tick(100) - if (pause.needsWait) { - await pause.wait(16) - } } ) first = false @@ -138,6 +124,7 @@ export class SpeckleLoader extends Loader { (performance.now() - start) / TIME_MS.second } seconds. Node count: ${this.tree.nodeCount}` ) + await this.loader.disposeAsync() if (viewerLoads === 0) { Logger.warn(`Viewer: no 3d objects found in object ${this.resource}`) @@ -185,6 +172,6 @@ export class SpeckleLoader extends Loader { dispose() { super.dispose() - this.loader.dispose() + void this.loader.disposeAsync() } } diff --git a/packages/viewer/src/modules/loaders/Speckle/SpeckleOfflineLoader.ts b/packages/viewer/src/modules/loaders/Speckle/SpeckleOfflineLoader.ts index aa46a7bda..061b6fe07 100644 --- a/packages/viewer/src/modules/loaders/Speckle/SpeckleOfflineLoader.ts +++ b/packages/viewer/src/modules/loaders/Speckle/SpeckleOfflineLoader.ts @@ -1,7 +1,7 @@ -import ObjectLoader from '@speckle/objectloader' import { SpeckleLoader } from './SpeckleLoader.js' import { WorldTree } from '../../tree/WorldTree.js' import Logger from '../../utils/Logger.js' +import { ObjectLoader2, ObjectLoader2Factory } from '@speckle/objectloader2' export class SpeckleOfflineLoader extends SpeckleLoader { constructor(targetTree: WorldTree, resourceData: unknown, resourceId?: string) { @@ -13,8 +13,13 @@ export class SpeckleOfflineLoader extends SpeckleLoader { _authToken?: string, _enableCaching?: boolean, resourceData?: unknown - ): ObjectLoader { - return ObjectLoader.createFromJSON(resourceData as string) + ): ObjectLoader2 { + _resource + _authToken + _enableCaching + resourceData + /** TO DO: Implement either as part of ObjectLoader2 either separate */ + return ObjectLoader2Factory.createFromObjects([]) } public async load(): Promise { @@ -24,7 +29,8 @@ export class SpeckleOfflineLoader extends SpeckleLoader { return false } /** If not id is provided, we make one up based on the root object id */ - this._resource = this._resource || `/json/${rootObject.id as string}` + this._resource = + this._resource || `/json/${(rootObject?.baseId as string) ?? 'unnamed'}` return super.load() } } diff --git a/yarn.lock b/yarn.lock index 07e6c7f83..ffd3a0221 100644 --- a/yarn.lock +++ b/yarn.lock @@ -16359,7 +16359,7 @@ __metadata: "@rollup/plugin-babel": "npm:^5.3.1" "@rollup/plugin-image": "npm:^3.0.2" "@rollup/plugin-typescript": "npm:^11.1.6" - "@speckle/objectloader": "workspace:^" + "@speckle/objectloader2": "workspace:^" "@speckle/shared": "workspace:^" "@types/babel__core": "npm:^7.20.1" "@types/flat": "npm:^5.0.2"