From c5967a96166fa72786503b263139f050b86fa3cf Mon Sep 17 00:00:00 2001
From: Adam Hathcock <adamhathcock@users.noreply.github.com>
Date: Thu, 26 Jun 2025 13:28:50 +0100
Subject: [PATCH] (OL2) move files around to make more sense (#4950)

* Rename to saveBatch

* forgot a file

* first pass of cacheReader

* OL2 tests have infinite timeout

* OL2 refactor works

* fix for tests

* moved/removed types to make a more logical structure

* fixed imports

* rework loop to be in async generator for the expected count

* get rid of pumps and fix test

* lint fix

* redo mermaid diagrams

* add readme section on deferment

* always return root first

* fix linting

* revert the counting

* merge fixes

* remove unused var
---
 .../__snapshots__/objectLoader2.spec.ts.snap  |  0
 .../__snapshots__/traverser.spec.ts.snap      |  0
 .../src/{operations => core}/interfaces.ts    |  2 +-
 .../objectLoader2.spec.ts                     | 19 +++++--
 .../src/{operations => core}/objectLoader2.ts | 21 ++++----
 .../objectLoader2Factory.ts                   | 11 ++--
 .../src/{operations => core}/options.ts       |  3 +-
 .../__snapshots__/cacheReader.spec.ts.snap    |  0
 .../indexedDatabase.spec.ts.snap              |  0
 .../serverDownloader.spec.ts.snap             |  0
 .../stages}/cacheReader.spec.ts               |  8 +--
 .../{helpers => core/stages}/cacheReader.ts   | 13 ++---
 .../src/core/stages/cacheWriter.ts            | 49 ++++++++++++++++++
 .../stages}/indexedDatabase.spec.ts           |  4 +-
 .../stages}/indexedDatabase.ts                |  5 +-
 .../memoryDownloader.spec.ts.snap             |  0
 .../stages/memory}/memoryDatabase.spec.ts     |  2 +-
 .../stages/memory}/memoryDatabase.ts          |  6 +--
 .../stages/memory}/memoryDownloader.spec.ts   |  6 +--
 .../stages/memory}/memoryDownloader.ts        |  6 +--
 .../stages}/serverDownloader.spec.ts          | 15 ++----
 .../stages}/serverDownloader.ts               | 41 ++++++++-------
 .../{operations => core}/traverser.spec.ts    |  2 +-
 .../src/{operations => core}/traverser.ts     |  3 +-
 .../defermentManager.spec.ts.snap             |  0
 .../defermentManager.defermentTotals.spec.ts  |  2 +-
 .../defermentManager.disposal.spec.ts         |  2 +-
 .../defermentManager.spec.ts                  |  0
 .../defermentManager.ts                       |  5 +-
 .../{helpers => deferment}/deferredBase.ts    |  0
 packages/objectloader2/src/index.ts           |  4 +-
 .../src/{helpers => queues}/aggregateQueue.ts |  6 ++-
 .../asyncGeneratorQueue.ts                    |  3 +-
 .../src/{helpers => queues}/batchedPool.ts    |  0
 .../src/{helpers => queues}/batchingQueue.ts  |  0
 .../src/{helpers => queues}/bufferQueue.ts    |  3 ++
 .../src/{helpers => queues}/cacheWriter.ts    | 10 ++--
 .../src/{helpers => queues}/keyedQueue.ts     |  0
 .../src/{helpers => queues}/queue.ts          |  1 +
 packages/objectloader2/src/test/e2e.spec.ts   |  4 +-
 packages/objectloader2/src/types/functions.ts | 51 +++++++++++++++++++
 packages/objectloader2/src/types/types.ts     | 50 ------------------
 42 files changed, 215 insertions(+), 142 deletions(-)
 rename packages/objectloader2/src/{operations => core}/__snapshots__/objectLoader2.spec.ts.snap (100%)
 rename packages/objectloader2/src/{operations => core}/__snapshots__/traverser.spec.ts.snap (100%)
 rename packages/objectloader2/src/{operations => core}/interfaces.ts (91%)
 rename packages/objectloader2/src/{operations => core}/objectLoader2.spec.ts (94%)
 rename packages/objectloader2/src/{operations => core}/objectLoader2.ts (85%)
 rename packages/objectloader2/src/{operations => core}/objectLoader2Factory.ts (86%)
 rename packages/objectloader2/src/{operations => core}/options.ts (86%)
 rename packages/objectloader2/src/{helpers => core/stages}/__snapshots__/cacheReader.spec.ts.snap (100%)
 rename packages/objectloader2/src/{operations/databases => core/stages}/__snapshots__/indexedDatabase.spec.ts.snap (100%)
 rename packages/objectloader2/src/{operations/downloaders => core/stages}/__snapshots__/serverDownloader.spec.ts.snap (100%)
 rename packages/objectloader2/src/{helpers => core/stages}/cacheReader.spec.ts (78%)
 rename packages/objectloader2/src/{helpers => core/stages}/cacheReader.ts (83%)
 create mode 100644 packages/objectloader2/src/core/stages/cacheWriter.ts
 rename packages/objectloader2/src/{operations/databases => core/stages}/indexedDatabase.spec.ts (95%)
 rename packages/objectloader2/src/{operations/databases => core/stages}/indexedDatabase.ts (96%)
 rename packages/objectloader2/src/{operations/downloaders => core/stages/memory}/__snapshots__/memoryDownloader.spec.ts.snap (100%)
 rename packages/objectloader2/src/{operations/databases => core/stages/memory}/memoryDatabase.spec.ts (96%)
 rename packages/objectloader2/src/{operations/databases => core/stages/memory}/memoryDatabase.ts (84%)
 rename packages/objectloader2/src/{operations/downloaders => core/stages/memory}/memoryDownloader.spec.ts (90%)
 rename packages/objectloader2/src/{operations/downloaders => core/stages/memory}/memoryDownloader.ts (85%)
 rename packages/objectloader2/src/{operations/downloaders => core/stages}/serverDownloader.spec.ts (94%)
 rename packages/objectloader2/src/{operations/downloaders => core/stages}/serverDownloader.ts (93%)
 rename packages/objectloader2/src/{operations => core}/traverser.spec.ts (100%)
 rename packages/objectloader2/src/{operations => core}/traverser.ts (96%)
 rename packages/objectloader2/src/{helpers => deferment}/__snapshots__/defermentManager.spec.ts.snap (100%)
 rename packages/objectloader2/src/{helpers => deferment}/defermentManager.defermentTotals.spec.ts (96%)
 rename packages/objectloader2/src/{helpers => deferment}/defermentManager.disposal.spec.ts (93%)
 rename packages/objectloader2/src/{helpers => deferment}/defermentManager.spec.ts (100%)
 rename packages/objectloader2/src/{helpers => deferment}/defermentManager.ts (96%)
 rename packages/objectloader2/src/{helpers => deferment}/deferredBase.ts (100%)
 rename packages/objectloader2/src/{helpers => queues}/aggregateQueue.ts (72%)
 rename packages/objectloader2/src/{helpers => queues}/asyncGeneratorQueue.ts (93%)
 rename packages/objectloader2/src/{helpers => queues}/batchedPool.ts (100%)
 rename packages/objectloader2/src/{helpers => queues}/batchingQueue.ts (100%)
 rename packages/objectloader2/src/{helpers => queues}/bufferQueue.ts (76%)
 rename packages/objectloader2/src/{helpers => queues}/cacheWriter.ts (78%)
 rename packages/objectloader2/src/{helpers => queues}/keyedQueue.ts (100%)
 rename packages/objectloader2/src/{helpers => queues}/queue.ts (65%)
 create mode 100644 packages/objectloader2/src/types/functions.ts

diff --git a/packages/objectloader2/src/operations/__snapshots__/objectLoader2.spec.ts.snap b/packages/objectloader2/src/core/__snapshots__/objectLoader2.spec.ts.snap
similarity index 100%
rename from packages/objectloader2/src/operations/__snapshots__/objectLoader2.spec.ts.snap
rename to packages/objectloader2/src/core/__snapshots__/objectLoader2.spec.ts.snap
diff --git a/packages/objectloader2/src/operations/__snapshots__/traverser.spec.ts.snap b/packages/objectloader2/src/core/__snapshots__/traverser.spec.ts.snap
similarity index 100%
rename from packages/objectloader2/src/operations/__snapshots__/traverser.spec.ts.snap
rename to packages/objectloader2/src/core/__snapshots__/traverser.spec.ts.snap
diff --git a/packages/objectloader2/src/operations/interfaces.ts b/packages/objectloader2/src/core/interfaces.ts
similarity index 91%
rename from packages/objectloader2/src/operations/interfaces.ts
rename to packages/objectloader2/src/core/interfaces.ts
index 1ad33291e..add683213 100644
--- a/packages/objectloader2/src/operations/interfaces.ts
+++ b/packages/objectloader2/src/core/interfaces.ts
@@ -1,4 +1,4 @@
-import Queue from '../helpers/queue.js'
+import Queue from '../queues/queue.js'
 import { Item } from '../types/types.js'
 
 export interface Downloader extends Queue<string> {
diff --git a/packages/objectloader2/src/operations/objectLoader2.spec.ts b/packages/objectloader2/src/core/objectLoader2.spec.ts
similarity index 94%
rename from packages/objectloader2/src/operations/objectLoader2.spec.ts
rename to packages/objectloader2/src/core/objectLoader2.spec.ts
index 138205ddb..87b2649f9 100644
--- a/packages/objectloader2/src/operations/objectLoader2.spec.ts
+++ b/packages/objectloader2/src/core/objectLoader2.spec.ts
@@ -1,10 +1,10 @@
-import { describe, expect, test } from 'vitest'
-import { ObjectLoader2 } from './objectLoader2.js'
+import { describe, test, expect } from 'vitest'
 import { Base, Item } from '../types/types.js'
-import { MemoryDownloader } from './downloaders/memoryDownloader.js'
+import { ObjectLoader2 } from './objectLoader2.js'
+import IndexedDatabase from './stages/indexedDatabase.js'
 import { IDBFactory, IDBKeyRange } from 'fake-indexeddb'
-import { MemoryDatabase } from './databases/memoryDatabase.js'
-import IndexedDatabase from './databases/indexedDatabase.js'
+import { MemoryDatabase } from './stages/memory/memoryDatabase.js'
+import { MemoryDownloader } from './stages/memory/memoryDownloader.js'
 
 describe('objectloader2', () => {
   test('can get a root object from cache', async () => {
@@ -66,6 +66,9 @@ describe('objectloader2', () => {
     const r = []
     for await (const x of loader.getObjectIterator()) {
       r.push(x)
+      if (r.length >= 1) {
+        break
+      }
     }
     await loader.disposeAsync()
 
@@ -101,6 +104,9 @@ describe('objectloader2', () => {
     const obj = loader.getObject({ id: child1.baseId })
     for await (const x of loader.getObjectIterator()) {
       r.push(x)
+      if (r.length >= 2) {
+        break
+      }
     }
     await loader.disposeAsync()
 
@@ -139,6 +145,9 @@ describe('objectloader2', () => {
     const obj = loader.getObject({ id: child1.baseId })
     for await (const x of loader.getObjectIterator()) {
       r.push(x)
+      if (r.length >= 2) {
+        break
+      }
     }
     await loader.disposeAsync()
 
diff --git a/packages/objectloader2/src/operations/objectLoader2.ts b/packages/objectloader2/src/core/objectLoader2.ts
similarity index 85%
rename from packages/objectloader2/src/operations/objectLoader2.ts
rename to packages/objectloader2/src/core/objectLoader2.ts
index a68f916f1..934f9c9df 100644
--- a/packages/objectloader2/src/operations/objectLoader2.ts
+++ b/packages/objectloader2/src/core/objectLoader2.ts
@@ -1,12 +1,13 @@
-import AsyncGeneratorQueue from '../helpers/asyncGeneratorQueue.js'
-import { Downloader, Database } from './interfaces.js'
-import { CustomLogger, Base, Item } from '../types/types.js'
-import { CacheOptions, ObjectLoader2Options } from './options.js'
-import { DefermentManager } from '../helpers/defermentManager.js'
-import { CacheReader } from '../helpers/cacheReader.js'
-import AggregateQueue from '../helpers/aggregateQueue.js'
+import { DefermentManager } from '../deferment/defermentManager.js'
+import AggregateQueue from '../queues/aggregateQueue.js'
+import AsyncGeneratorQueue from '../queues/asyncGeneratorQueue.js'
+import { CustomLogger } from '../types/functions.js'
+import { Item, Base } from '../types/types.js'
+import { Database, Downloader } from './interfaces.js'
 import { ObjectLoader2Factory } from './objectLoader2Factory.js'
-import { CacheWriter } from '../helpers/cacheWriter.js'
+import { ObjectLoader2Options, CacheOptions } from './options.js'
+import { CacheReader } from './stages/cacheReader.js'
+import { CacheWriter } from './stages/cacheWriter.js'
 
 export class ObjectLoader2 {
   #rootId: string
@@ -52,8 +53,8 @@ export class ObjectLoader2 {
   }
 
   async disposeAsync(): Promise<void> {
-    this.#gathered.dispose()
     await Promise.all([
+      this.#gathered.disposeAsync(),
       this.#downloader.disposeAsync(),
       this.#cacheReader.disposeAsync(),
       this.#cacheWriter.disposeAsync()
@@ -93,7 +94,7 @@ export class ObjectLoader2 {
     }
 
     //sort the closures by their values descending
-    const sortedClosures = Object.entries(rootItem.base.__closure).sort(
+    const sortedClosures = Object.entries(rootItem.base.__closure ?? []).sort(
       (a, b) => b[1] - a[1]
     )
     const children = sortedClosures.map((x) => x[0])
diff --git a/packages/objectloader2/src/operations/objectLoader2Factory.ts b/packages/objectloader2/src/core/objectLoader2Factory.ts
similarity index 86%
rename from packages/objectloader2/src/operations/objectLoader2Factory.ts
rename to packages/objectloader2/src/core/objectLoader2Factory.ts
index 6f0f32506..6f3fde824 100644
--- a/packages/objectloader2/src/operations/objectLoader2Factory.ts
+++ b/packages/objectloader2/src/core/objectLoader2Factory.ts
@@ -1,9 +1,10 @@
-import { Base, CustomLogger } from '../types/types.js'
-import IndexedDatabase from './databases/indexedDatabase.js'
-import { MemoryDatabase } from './databases/memoryDatabase.js'
-import { MemoryDownloader } from './downloaders/memoryDownloader.js'
-import ServerDownloader from './downloaders/serverDownloader.js'
+import { CustomLogger } from '../types/functions.js'
+import { Base } from '../types/types.js'
 import { ObjectLoader2 } from './objectLoader2.js'
+import IndexedDatabase from './stages/indexedDatabase.js'
+import { MemoryDatabase } from './stages/memory/memoryDatabase.js'
+import { MemoryDownloader } from './stages/memory/memoryDownloader.js'
+import ServerDownloader from './stages/serverDownloader.js'
 
 export interface ObjectLoader2FactoryOptions {
   useMemoryCache?: boolean
diff --git a/packages/objectloader2/src/operations/options.ts b/packages/objectloader2/src/core/options.ts
similarity index 86%
rename from packages/objectloader2/src/operations/options.ts
rename to packages/objectloader2/src/core/options.ts
index 39478f678..1a47d7239 100644
--- a/packages/objectloader2/src/operations/options.ts
+++ b/packages/objectloader2/src/core/options.ts
@@ -1,4 +1,5 @@
-import { Base, CustomLogger } from '../types/types.js'
+import { CustomLogger } from '../types/functions.js'
+import { Base } from '../types/types.js'
 import { Downloader, Database } from './interfaces.js'
 
 export interface ObjectLoader2Options {
diff --git a/packages/objectloader2/src/helpers/__snapshots__/cacheReader.spec.ts.snap b/packages/objectloader2/src/core/stages/__snapshots__/cacheReader.spec.ts.snap
similarity index 100%
rename from packages/objectloader2/src/helpers/__snapshots__/cacheReader.spec.ts.snap
rename to packages/objectloader2/src/core/stages/__snapshots__/cacheReader.spec.ts.snap
diff --git a/packages/objectloader2/src/operations/databases/__snapshots__/indexedDatabase.spec.ts.snap b/packages/objectloader2/src/core/stages/__snapshots__/indexedDatabase.spec.ts.snap
similarity index 100%
rename from packages/objectloader2/src/operations/databases/__snapshots__/indexedDatabase.spec.ts.snap
rename to packages/objectloader2/src/core/stages/__snapshots__/indexedDatabase.spec.ts.snap
diff --git a/packages/objectloader2/src/operations/downloaders/__snapshots__/serverDownloader.spec.ts.snap b/packages/objectloader2/src/core/stages/__snapshots__/serverDownloader.spec.ts.snap
similarity index 100%
rename from packages/objectloader2/src/operations/downloaders/__snapshots__/serverDownloader.spec.ts.snap
rename to packages/objectloader2/src/core/stages/__snapshots__/serverDownloader.spec.ts.snap
diff --git a/packages/objectloader2/src/helpers/cacheReader.spec.ts b/packages/objectloader2/src/core/stages/cacheReader.spec.ts
similarity index 78%
rename from packages/objectloader2/src/helpers/cacheReader.spec.ts
rename to packages/objectloader2/src/core/stages/cacheReader.spec.ts
index 5604d9f84..2b1fbf9db 100644
--- a/packages/objectloader2/src/helpers/cacheReader.spec.ts
+++ b/packages/objectloader2/src/core/stages/cacheReader.spec.ts
@@ -1,8 +1,8 @@
-import { describe, expect, test } from 'vitest'
-import { Base, Item } from '../types/types.js'
-import { DefermentManager } from './defermentManager.js'
+import { describe, test, expect } from 'vitest'
+import { DefermentManager } from '../../deferment/defermentManager.js'
+import { Item, Base } from '../../types/types.js'
 import { CacheReader } from './cacheReader.js'
-import { MemoryDatabase } from '../operations/databases/memoryDatabase.js'
+import { MemoryDatabase } from './memory/memoryDatabase.js'
 
 describe('CacheReader testing', () => {
   test('deferred getObject', async () => {
diff --git a/packages/objectloader2/src/helpers/cacheReader.ts b/packages/objectloader2/src/core/stages/cacheReader.ts
similarity index 83%
rename from packages/objectloader2/src/helpers/cacheReader.ts
rename to packages/objectloader2/src/core/stages/cacheReader.ts
index ae853723c..7645af00b 100644
--- a/packages/objectloader2/src/helpers/cacheReader.ts
+++ b/packages/objectloader2/src/core/stages/cacheReader.ts
@@ -1,9 +1,10 @@
-import { Database } from '../operations/interfaces.js'
-import { CacheOptions } from '../operations/options.js'
-import { Base, CustomLogger, Item } from '../types/types.js'
-import BatchingQueue from './batchingQueue.js'
-import { DefermentManager } from './defermentManager.js'
-import Queue from './queue.js'
+import { DefermentManager } from '../../deferment/defermentManager.js'
+import BatchingQueue from '../../queues/batchingQueue.js'
+import Queue from '../../queues/queue.js'
+import { CustomLogger } from '../../types/functions.js'
+import { Item, Base } from '../../types/types.js'
+import { Database } from '../interfaces.js'
+import { CacheOptions } from '../options.js'
 
 export class CacheReader {
   #database: Database
diff --git a/packages/objectloader2/src/core/stages/cacheWriter.ts b/packages/objectloader2/src/core/stages/cacheWriter.ts
new file mode 100644
index 000000000..9387193cd
--- /dev/null
+++ b/packages/objectloader2/src/core/stages/cacheWriter.ts
@@ -0,0 +1,49 @@
+import { DefermentManager } from '../../deferment/defermentManager.js'
+import BatchingQueue from '../../queues/batchingQueue.js'
+import Queue from '../../queues/queue.js'
+import { CustomLogger } from '../../types/functions.js'
+import { Item } from '../../types/types.js'
+import { Database } from '../interfaces.js'
+import { CacheOptions } from '../options.js'
+
+export class CacheWriter implements Queue<Item> {
+  #writeQueue: BatchingQueue<Item> | undefined
+  #database: Database
+  #defermentManager: DefermentManager
+  #logger: CustomLogger
+  #options: CacheOptions
+  #disposed = false
+
+  constructor(
+    database: Database,
+    defermentManager: DefermentManager,
+    options: CacheOptions
+  ) {
+    this.#database = database
+    this.#defermentManager = defermentManager
+    this.#options = options
+    this.#logger = options.logger || ((): void => {})
+  }
+
+  add(item: Item): void {
+    if (!this.#writeQueue) {
+      this.#writeQueue = new BatchingQueue({
+        batchSize: this.#options.maxCacheWriteSize,
+        maxWaitTime: this.#options.maxCacheBatchWriteWait,
+        processFunction: (batch: Item[]): Promise<void> =>
+          this.#database.saveBatch({ batch })
+      })
+    }
+    this.#defermentManager.undefer(item)
+    this.#writeQueue.add(item.baseId, item)
+  }
+
+  async disposeAsync(): Promise<void> {
+    await this.#writeQueue?.disposeAsync()
+    this.#disposed = true
+  }
+
+  get isDisposed(): boolean {
+    return this.#disposed
+  }
+}
diff --git a/packages/objectloader2/src/operations/databases/indexedDatabase.spec.ts b/packages/objectloader2/src/core/stages/indexedDatabase.spec.ts
similarity index 95%
rename from packages/objectloader2/src/operations/databases/indexedDatabase.spec.ts
rename to packages/objectloader2/src/core/stages/indexedDatabase.spec.ts
index 11fdfe751..49acf7a4d 100644
--- a/packages/objectloader2/src/operations/databases/indexedDatabase.spec.ts
+++ b/packages/objectloader2/src/core/stages/indexedDatabase.spec.ts
@@ -1,8 +1,8 @@
 import { describe, it, expect, beforeEach, afterEach } from 'vitest'
 import { IDBFactory, IDBKeyRange } from 'fake-indexeddb'
-import IndexedDatabase, { IndexedDatabaseOptions } from './indexedDatabase.js'
 
-import { Item, Base } from '../../types/types.js'
+import { Base, Item } from '../../types/types.js'
+import IndexedDatabase, { IndexedDatabaseOptions } from './indexedDatabase.js'
 
 // Mock Item
 const defaultItem = (id: string): Item => ({
diff --git a/packages/objectloader2/src/operations/databases/indexedDatabase.ts b/packages/objectloader2/src/core/stages/indexedDatabase.ts
similarity index 96%
rename from packages/objectloader2/src/operations/databases/indexedDatabase.ts
rename to packages/objectloader2/src/core/stages/indexedDatabase.ts
index 4d110b89b..c10e60447 100644
--- a/packages/objectloader2/src/operations/databases/indexedDatabase.ts
+++ b/packages/objectloader2/src/core/stages/indexedDatabase.ts
@@ -1,9 +1,10 @@
 /* eslint-disable @typescript-eslint/no-unsafe-function-type */
-import BatchingQueue from '../../helpers/batchingQueue.js'
-import { CustomLogger, Item } from '../../types/types.js'
+import { CustomLogger } from '../../types/functions.js'
+import { Item } from '../../types/types.js'
 import { isSafari } from '@speckle/shared'
 import { Dexie, DexieOptions, Table } from 'dexie'
 import { Database } from '../interfaces.js'
+import BatchingQueue from '../../queues/batchingQueue.js'
 
 export class ObjectStore extends Dexie {
   static #databaseName: string = 'speckle-cache'
diff --git a/packages/objectloader2/src/operations/downloaders/__snapshots__/memoryDownloader.spec.ts.snap b/packages/objectloader2/src/core/stages/memory/__snapshots__/memoryDownloader.spec.ts.snap
similarity index 100%
rename from packages/objectloader2/src/operations/downloaders/__snapshots__/memoryDownloader.spec.ts.snap
rename to packages/objectloader2/src/core/stages/memory/__snapshots__/memoryDownloader.spec.ts.snap
diff --git a/packages/objectloader2/src/operations/databases/memoryDatabase.spec.ts b/packages/objectloader2/src/core/stages/memory/memoryDatabase.spec.ts
similarity index 96%
rename from packages/objectloader2/src/operations/databases/memoryDatabase.spec.ts
rename to packages/objectloader2/src/core/stages/memory/memoryDatabase.spec.ts
index 9bf03f631..a365a06ad 100644
--- a/packages/objectloader2/src/operations/databases/memoryDatabase.spec.ts
+++ b/packages/objectloader2/src/core/stages/memory/memoryDatabase.spec.ts
@@ -1,6 +1,6 @@
 import { describe, it, expect, beforeEach } from 'vitest'
+import { Item, Base } from '../../../types/types.js'
 import { MemoryDatabase } from './memoryDatabase.js'
-import { Base, Item } from '../../types/types.js'
 
 const makeItem = (id: string, foo = 'bar'): Item => ({
   baseId: id,
diff --git a/packages/objectloader2/src/operations/databases/memoryDatabase.ts b/packages/objectloader2/src/core/stages/memory/memoryDatabase.ts
similarity index 84%
rename from packages/objectloader2/src/operations/databases/memoryDatabase.ts
rename to packages/objectloader2/src/core/stages/memory/memoryDatabase.ts
index d97075592..cb2cbc19e 100644
--- a/packages/objectloader2/src/operations/databases/memoryDatabase.ts
+++ b/packages/objectloader2/src/core/stages/memory/memoryDatabase.ts
@@ -1,6 +1,6 @@
-import { Base, Item } from '../../types/types.js'
-import { Database } from '../interfaces.js'
-import { MemoryDatabaseOptions } from '../options.js'
+import { Base, Item } from '../../../types/types.js'
+import { Database } from '../../interfaces.js'
+import { MemoryDatabaseOptions } from '../../options.js'
 
 export class MemoryDatabase implements Database {
   private items: Map<string, Base>
diff --git a/packages/objectloader2/src/operations/downloaders/memoryDownloader.spec.ts b/packages/objectloader2/src/core/stages/memory/memoryDownloader.spec.ts
similarity index 90%
rename from packages/objectloader2/src/operations/downloaders/memoryDownloader.spec.ts
rename to packages/objectloader2/src/core/stages/memory/memoryDownloader.spec.ts
index c87e78e61..1c5e7b26d 100644
--- a/packages/objectloader2/src/operations/downloaders/memoryDownloader.spec.ts
+++ b/packages/objectloader2/src/core/stages/memory/memoryDownloader.spec.ts
@@ -1,8 +1,8 @@
 import { describe, it, expect, beforeEach } from 'vitest'
 import { MemoryDownloader } from './memoryDownloader.js'
-import { Base, Item } from '../../types/types.js'
-import Queue from '../../helpers/queue.js'
-import BufferQueue from '../../helpers/bufferQueue.js'
+import BufferQueue from '../../../queues/bufferQueue.js'
+import Queue from '../../../queues/queue.js'
+import { Base, Item } from '../../../types/types.js'
 
 const makeBase = (foo: string): Base => ({ foo } as unknown as Base)
 
diff --git a/packages/objectloader2/src/operations/downloaders/memoryDownloader.ts b/packages/objectloader2/src/core/stages/memory/memoryDownloader.ts
similarity index 85%
rename from packages/objectloader2/src/operations/downloaders/memoryDownloader.ts
rename to packages/objectloader2/src/core/stages/memory/memoryDownloader.ts
index 73a077a84..38f1fb8fa 100644
--- a/packages/objectloader2/src/operations/downloaders/memoryDownloader.ts
+++ b/packages/objectloader2/src/core/stages/memory/memoryDownloader.ts
@@ -1,6 +1,6 @@
-import Queue from '../../helpers/queue.js'
-import { Base, Item } from '../../types/types.js'
-import { Downloader } from '../interfaces.js'
+import Queue from '../../../queues/queue.js'
+import { Base, Item } from '../../../types/types.js'
+import { Downloader } from '../../interfaces.js'
 
 export class MemoryDownloader implements Downloader {
   #items: Map<string, Base>
diff --git a/packages/objectloader2/src/operations/downloaders/serverDownloader.spec.ts b/packages/objectloader2/src/core/stages/serverDownloader.spec.ts
similarity index 94%
rename from packages/objectloader2/src/operations/downloaders/serverDownloader.spec.ts
rename to packages/objectloader2/src/core/stages/serverDownloader.spec.ts
index 2bffce9f1..b1236b542 100644
--- a/packages/objectloader2/src/operations/downloaders/serverDownloader.spec.ts
+++ b/packages/objectloader2/src/core/stages/serverDownloader.spec.ts
@@ -3,7 +3,7 @@ import createFetchMock from 'vitest-fetch-mock'
 import { vi } from 'vitest'
 import { Item } from '../../types/types.js'
 import ServerDownloader from './serverDownloader.js'
-import AsyncGeneratorQueue from '../../helpers/asyncGeneratorQueue.js'
+import AsyncGeneratorQueue from '../../queues/asyncGeneratorQueue.js'
 
 describe('downloader', () => {
   test('download batch of one', async () => {
@@ -24,13 +24,10 @@ describe('downloader', () => {
       maxDownloadBatchWait: 200
     })
     downloader.add('id')
-    await downloader.disposeAsync()
     const r = []
-    let count = 0
     for await (const x of gathered.consume()) {
       r.push(x)
-      count++
-      if (count >= 1) {
+      if (r.length >= 1) {
         break
       }
     }
@@ -65,11 +62,9 @@ describe('downloader', () => {
     downloader.add('id2')
     await downloader.disposeAsync()
     const r = []
-    let count = 0
     for await (const x of gathered.consume()) {
       r.push(x)
-      count++
-      if (count >= 2) {
+      if (r.length >= 2) {
         break
       }
     }
@@ -112,11 +107,9 @@ describe('downloader', () => {
     downloader.add('id3')
     await downloader.disposeAsync()
     const r = []
-    let count = 0
     for await (const x of gathered.consume()) {
       r.push(x)
-      count++
-      if (count >= 3) {
+      if (r.length >= 3) {
         break
       }
     }
diff --git a/packages/objectloader2/src/operations/downloaders/serverDownloader.ts b/packages/objectloader2/src/core/stages/serverDownloader.ts
similarity index 93%
rename from packages/objectloader2/src/operations/downloaders/serverDownloader.ts
rename to packages/objectloader2/src/core/stages/serverDownloader.ts
index a0a24f730..e0c21e144 100644
--- a/packages/objectloader2/src/operations/downloaders/serverDownloader.ts
+++ b/packages/objectloader2/src/core/stages/serverDownloader.ts
@@ -1,7 +1,8 @@
-import BatchedPool from '../../helpers/batchedPool.js'
-import Queue from '../../helpers/queue.js'
+import BatchedPool from '../../queues/batchedPool.js'
+import Queue from '../../queues/queue.js'
 import { ObjectLoaderRuntimeError } from '../../types/errors.js'
-import { Fetcher, isBase, Item, take } from '../../types/types.js'
+import { Fetcher, isBase, take } from '../../types/functions.js'
+import { Item } from '../../types/types.js'
 import { Downloader } from '../interfaces.js'
 
 export interface ServerDownloaderOptions {
@@ -20,6 +21,7 @@ export default class ServerDownloader implements Downloader {
   #options: ServerDownloaderOptions
   #fetch: Fetcher
   #results?: Queue<Item>
+  #total?: number
 
   #downloadQueue?: BatchedPool<string>
   #decoder = new TextDecoder()
@@ -63,6 +65,7 @@ export default class ServerDownloader implements Downloader {
   }): void {
     const { results, total } = params
     this.#results = results
+    this.#total = total
     this.#downloadQueue = new BatchedPool<string>({
       concurrencyAndSizes: this.#getDownloadCountAndSizes(total),
       maxWaitTime: params.maxDownloadBatchWait,
@@ -90,20 +93,6 @@ export default class ServerDownloader implements Downloader {
     await this.#downloadQueue?.disposeAsync()
   }
 
-  #processJson(baseId: string, unparsedBase: string): Item {
-    let base: unknown
-    try {
-      base = JSON.parse(unparsedBase)
-    } catch (e: unknown) {
-      throw new Error(`Error parsing object ${baseId}: ${(e as Error).message}`)
-    }
-    if (isBase(base)) {
-      return { baseId, base }
-    } else {
-      throw new ObjectLoaderRuntimeError(`${baseId} is not a base`)
-    }
-  }
-
   async downloadBatch(params: {
     batch: string[]
     url: string
@@ -142,6 +131,10 @@ export default class ServerDownloader implements Downloader {
         'Items requested were not downloaded: ' + take(keys.values(), 10).join(',')
       )
     }
+    count += keys.size // count the leftovers
+    if (count >= this.#total!) {
+      await this.#results?.disposeAsync() // mark the queue as done
+    }
   }
 
   async processArray(
@@ -186,6 +179,20 @@ export default class ServerDownloader implements Downloader {
     )
   }
 
+  #processJson(baseId: string, unparsedBase: string): Item {
+    let base: unknown
+    try {
+      base = JSON.parse(unparsedBase)
+    } catch (e: unknown) {
+      throw new Error(`Error parsing object ${baseId}: ${(e as Error).message}`)
+    }
+    if (isBase(base)) {
+      return { baseId, base }
+    } else {
+      throw new ObjectLoaderRuntimeError(`${baseId} is not a base`)
+    }
+  }
+
   concatUint8Arrays(a: Uint8Array, b: Uint8Array): Uint8Array {
     const c = new Uint8Array(a.length + b.length)
     c.set(a, 0)
diff --git a/packages/objectloader2/src/operations/traverser.spec.ts b/packages/objectloader2/src/core/traverser.spec.ts
similarity index 100%
rename from packages/objectloader2/src/operations/traverser.spec.ts
rename to packages/objectloader2/src/core/traverser.spec.ts
index 620c0a983..aa01e46a7 100644
--- a/packages/objectloader2/src/operations/traverser.spec.ts
+++ b/packages/objectloader2/src/core/traverser.spec.ts
@@ -1,7 +1,7 @@
 import { describe, expect, test } from 'vitest'
-import { Base } from '../types/types.js'
 import { ObjectLoader2 } from './objectLoader2.js'
 import Traverser from './traverser.js'
+import { Base } from '../types/types.js'
 
 describe('Traverser', () => {
   test('root and two children with referenceId', async () => {
diff --git a/packages/objectloader2/src/operations/traverser.ts b/packages/objectloader2/src/core/traverser.ts
similarity index 96%
rename from packages/objectloader2/src/operations/traverser.ts
rename to packages/objectloader2/src/core/traverser.ts
index 72fbcba16..2f5bc036e 100644
--- a/packages/objectloader2/src/operations/traverser.ts
+++ b/packages/objectloader2/src/core/traverser.ts
@@ -1,4 +1,5 @@
-import { Base, DataChunk, isBase, isReference, isScalar } from '../types/types.js'
+import { isScalar, isBase, isReference } from '../types/functions.js'
+import { Base, DataChunk } from '../types/types.js'
 import { ObjectLoader2 } from './objectLoader2.js'
 
 export type ProgressStage = 'download' | 'construction'
diff --git a/packages/objectloader2/src/helpers/__snapshots__/defermentManager.spec.ts.snap b/packages/objectloader2/src/deferment/__snapshots__/defermentManager.spec.ts.snap
similarity index 100%
rename from packages/objectloader2/src/helpers/__snapshots__/defermentManager.spec.ts.snap
rename to packages/objectloader2/src/deferment/__snapshots__/defermentManager.spec.ts.snap
diff --git a/packages/objectloader2/src/helpers/defermentManager.defermentTotals.spec.ts b/packages/objectloader2/src/deferment/defermentManager.defermentTotals.spec.ts
similarity index 96%
rename from packages/objectloader2/src/helpers/defermentManager.defermentTotals.spec.ts
rename to packages/objectloader2/src/deferment/defermentManager.defermentTotals.spec.ts
index e61cbda0d..262d610a9 100644
--- a/packages/objectloader2/src/helpers/defermentManager.defermentTotals.spec.ts
+++ b/packages/objectloader2/src/deferment/defermentManager.defermentTotals.spec.ts
@@ -1,6 +1,6 @@
 import { describe, test, expect, beforeEach } from 'vitest'
 import { DefermentManager } from './defermentManager.js'
-import { DefermentManagerOptions } from '../operations/options.js'
+import { DefermentManagerOptions } from '../core/options.js'
 import { Base, Item } from '../types/types.js'
 
 const makeItem = (id: string, size = 1): Item => ({
diff --git a/packages/objectloader2/src/helpers/defermentManager.disposal.spec.ts b/packages/objectloader2/src/deferment/defermentManager.disposal.spec.ts
similarity index 93%
rename from packages/objectloader2/src/helpers/defermentManager.disposal.spec.ts
rename to packages/objectloader2/src/deferment/defermentManager.disposal.spec.ts
index 979b064d8..30d9f1e6b 100644
--- a/packages/objectloader2/src/helpers/defermentManager.disposal.spec.ts
+++ b/packages/objectloader2/src/deferment/defermentManager.disposal.spec.ts
@@ -1,6 +1,6 @@
 import { describe, it, expect } from 'vitest'
 import { DefermentManager } from './defermentManager.js'
-import { DefermentManagerOptions } from '../operations/options.js'
+import { DefermentManagerOptions } from '../core/options.js'
 import { Item } from '../types/types.js'
 
 describe('DefermentManager disposal', () => {
diff --git a/packages/objectloader2/src/helpers/defermentManager.spec.ts b/packages/objectloader2/src/deferment/defermentManager.spec.ts
similarity index 100%
rename from packages/objectloader2/src/helpers/defermentManager.spec.ts
rename to packages/objectloader2/src/deferment/defermentManager.spec.ts
diff --git a/packages/objectloader2/src/helpers/defermentManager.ts b/packages/objectloader2/src/deferment/defermentManager.ts
similarity index 96%
rename from packages/objectloader2/src/helpers/defermentManager.ts
rename to packages/objectloader2/src/deferment/defermentManager.ts
index 3434304f1..3a9404202 100644
--- a/packages/objectloader2/src/helpers/defermentManager.ts
+++ b/packages/objectloader2/src/deferment/defermentManager.ts
@@ -1,6 +1,7 @@
 import { DeferredBase } from './deferredBase.js'
-import { Base, CustomLogger, Item } from '../types/types.js'
-import { DefermentManagerOptions } from '../operations/options.js'
+import { CustomLogger } from '../types/functions.js'
+import { Item, Base } from '../types/types.js'
+import { DefermentManagerOptions } from '../core/options.js'
 
 export class DefermentManager {
   private deferments: Map<string, DeferredBase> = new Map()
diff --git a/packages/objectloader2/src/helpers/deferredBase.ts b/packages/objectloader2/src/deferment/deferredBase.ts
similarity index 100%
rename from packages/objectloader2/src/helpers/deferredBase.ts
rename to packages/objectloader2/src/deferment/deferredBase.ts
diff --git a/packages/objectloader2/src/index.ts b/packages/objectloader2/src/index.ts
index 2f26904b3..b6c0b2950 100644
--- a/packages/objectloader2/src/index.ts
+++ b/packages/objectloader2/src/index.ts
@@ -1,2 +1,2 @@
-export { ObjectLoader2 } from './operations/objectLoader2.js'
-export { ObjectLoader2Factory } from './operations/objectLoader2Factory.js'
+export { ObjectLoader2 } from './core/objectLoader2.js'
+export { ObjectLoader2Factory } from './core/objectLoader2Factory.js'
diff --git a/packages/objectloader2/src/helpers/aggregateQueue.ts b/packages/objectloader2/src/queues/aggregateQueue.ts
similarity index 72%
rename from packages/objectloader2/src/helpers/aggregateQueue.ts
rename to packages/objectloader2/src/queues/aggregateQueue.ts
index db4e401f5..50ca6097a 100644
--- a/packages/objectloader2/src/helpers/aggregateQueue.ts
+++ b/packages/objectloader2/src/queues/aggregateQueue.ts
@@ -8,13 +8,17 @@ export default class AggregateQueue<T> implements Queue<T> {
     this.#queue1 = queue1
     this.#queue2 = queue2
   }
+  async disposeAsync(): Promise<void> {
+    await this.#queue1.disposeAsync()
+    await this.#queue2.disposeAsync()
+  }
 
   add(value: T): void {
     this.#queue1.add(value)
     this.#queue2.add(value)
   }
 
-  values(): T[] {
+  values(): never {
     throw new Error('Not implemented')
   }
 }
diff --git a/packages/objectloader2/src/helpers/asyncGeneratorQueue.ts b/packages/objectloader2/src/queues/asyncGeneratorQueue.ts
similarity index 93%
rename from packages/objectloader2/src/helpers/asyncGeneratorQueue.ts
rename to packages/objectloader2/src/queues/asyncGeneratorQueue.ts
index 00451408e..e995c4c45 100644
--- a/packages/objectloader2/src/helpers/asyncGeneratorQueue.ts
+++ b/packages/objectloader2/src/queues/asyncGeneratorQueue.ts
@@ -29,7 +29,8 @@ export default class AsyncGeneratorQueue<T> implements Queue<T> {
       }
     }
   }
-  dispose(): void {
+  disposeAsync(): Promise<void> {
     this.#finished = true
+    return Promise.resolve()
   }
 }
diff --git a/packages/objectloader2/src/helpers/batchedPool.ts b/packages/objectloader2/src/queues/batchedPool.ts
similarity index 100%
rename from packages/objectloader2/src/helpers/batchedPool.ts
rename to packages/objectloader2/src/queues/batchedPool.ts
diff --git a/packages/objectloader2/src/helpers/batchingQueue.ts b/packages/objectloader2/src/queues/batchingQueue.ts
similarity index 100%
rename from packages/objectloader2/src/helpers/batchingQueue.ts
rename to packages/objectloader2/src/queues/batchingQueue.ts
diff --git a/packages/objectloader2/src/helpers/bufferQueue.ts b/packages/objectloader2/src/queues/bufferQueue.ts
similarity index 76%
rename from packages/objectloader2/src/helpers/bufferQueue.ts
rename to packages/objectloader2/src/queues/bufferQueue.ts
index d7b9da530..c1161283c 100644
--- a/packages/objectloader2/src/helpers/bufferQueue.ts
+++ b/packages/objectloader2/src/queues/bufferQueue.ts
@@ -9,4 +9,7 @@ export default class BufferQueue<T> implements Queue<T> {
   values(): T[] {
     return this.#buffer
   }
+  disposeAsync(): Promise<void> {
+    return Promise.resolve()
+  }
 }
diff --git a/packages/objectloader2/src/helpers/cacheWriter.ts b/packages/objectloader2/src/queues/cacheWriter.ts
similarity index 78%
rename from packages/objectloader2/src/helpers/cacheWriter.ts
rename to packages/objectloader2/src/queues/cacheWriter.ts
index d2e3c9c78..e104ed529 100644
--- a/packages/objectloader2/src/helpers/cacheWriter.ts
+++ b/packages/objectloader2/src/queues/cacheWriter.ts
@@ -1,15 +1,14 @@
-import { Database } from '../operations/interfaces.js'
-import { CacheOptions } from '../operations/options.js'
-import { CustomLogger, Item } from '../types/types.js'
+import { Database } from '../core/interfaces.js'
+import { CacheOptions } from '../core/options.js'
+import { DefermentManager } from '../deferment/defermentManager.js'
+import { Item } from '../types/types.js'
 import BatchingQueue from './batchingQueue.js'
-import { DefermentManager } from './defermentManager.js'
 import Queue from './queue.js'
 
 export class CacheWriter implements Queue<Item> {
   #writeQueue: BatchingQueue<Item> | undefined
   #database: Database
   #defermentManager: DefermentManager
-  #logger: CustomLogger
   #options: CacheOptions
   #disposed = false
 
@@ -21,7 +20,6 @@ export class CacheWriter implements Queue<Item> {
     this.#database = database
     this.#defermentManager = defermentManager
     this.#options = options
-    this.#logger = options.logger || ((): void => {})
   }
 
   add(item: Item): void {
diff --git a/packages/objectloader2/src/helpers/keyedQueue.ts b/packages/objectloader2/src/queues/keyedQueue.ts
similarity index 100%
rename from packages/objectloader2/src/helpers/keyedQueue.ts
rename to packages/objectloader2/src/queues/keyedQueue.ts
diff --git a/packages/objectloader2/src/helpers/queue.ts b/packages/objectloader2/src/queues/queue.ts
similarity index 65%
rename from packages/objectloader2/src/helpers/queue.ts
rename to packages/objectloader2/src/queues/queue.ts
index 505879553..4c295610f 100644
--- a/packages/objectloader2/src/helpers/queue.ts
+++ b/packages/objectloader2/src/queues/queue.ts
@@ -1,3 +1,4 @@
 export default interface Queue<T> {
   add(value: T): void
+  disposeAsync(): Promise<void>
 }
diff --git a/packages/objectloader2/src/test/e2e.spec.ts b/packages/objectloader2/src/test/e2e.spec.ts
index 07399958b..ac2487f74 100644
--- a/packages/objectloader2/src/test/e2e.spec.ts
+++ b/packages/objectloader2/src/test/e2e.spec.ts
@@ -1,8 +1,8 @@
 import { describe, test, expect } from 'vitest'
 import { IDBFactory, IDBKeyRange } from 'fake-indexeddb'
-import { Base } from '../types/types.js'
 import { TIME_MS } from '@speckle/shared'
-import { ObjectLoader2Factory } from '../operations/objectLoader2Factory.js'
+import { ObjectLoader2Factory } from '../core/objectLoader2Factory.js'
+import { Base } from '../types/types.js'
 
 describe('e2e', () => {
   test(
diff --git a/packages/objectloader2/src/types/functions.ts b/packages/objectloader2/src/types/functions.ts
new file mode 100644
index 000000000..4a44df982
--- /dev/null
+++ b/packages/objectloader2/src/types/functions.ts
@@ -0,0 +1,51 @@
+import { Base, Reference } from './types.js'
+
+export type CustomLogger = (message?: string, ...optionalParams: unknown[]) => void
+
+export type Fetcher = (
+  input: string | URL | Request,
+  init?: RequestInit
+) => Promise<Response>
+
+export function isBase(maybeBase?: unknown): maybeBase is Base {
+  return (
+    maybeBase !== null &&
+    typeof maybeBase === 'object' &&
+    'id' in maybeBase &&
+    typeof maybeBase.id === 'string'
+  )
+}
+
+export function isReference(maybeRef?: unknown): maybeRef is Reference {
+  return (
+    maybeRef !== null &&
+    typeof maybeRef === 'object' &&
+    'referencedId' in maybeRef &&
+    typeof maybeRef.referencedId === 'string'
+  )
+}
+
+export function isScalar(
+  value: unknown
+): value is string | number | boolean | bigint | symbol | undefined {
+  const type = typeof value
+  return (
+    value === null ||
+    type === 'string' ||
+    type === 'number' ||
+    type === 'boolean' ||
+    type === 'bigint' ||
+    type === 'symbol' ||
+    type === 'undefined'
+  )
+}
+
+export function take<T>(it: Iterator<T>, count: number): T[] {
+  const result: T[] = []
+  for (let i = 0; i < count; i++) {
+    const itr = it.next()
+    if (itr.done) break
+    result.push(itr.value)
+  }
+  return result
+}
diff --git a/packages/objectloader2/src/types/types.ts b/packages/objectloader2/src/types/types.ts
index f9c712634..a482b89f0 100644
--- a/packages/objectloader2/src/types/types.ts
+++ b/packages/objectloader2/src/types/types.ts
@@ -1,10 +1,3 @@
-export type CustomLogger = (message?: string, ...optionalParams: unknown[]) => void
-
-export type Fetcher = (
-  input: string | URL | Request,
-  init?: RequestInit
-) => Promise<Response>
-
 export interface Item {
   baseId: string
   base?: Base
@@ -26,46 +19,3 @@ export interface Reference {
 export interface DataChunk extends Base {
   data?: Base[]
 }
-
-export function isBase(maybeBase?: unknown): maybeBase is Base {
-  return (
-    maybeBase !== null &&
-    typeof maybeBase === 'object' &&
-    'id' in maybeBase &&
-    typeof maybeBase.id === 'string'
-  )
-}
-
-export function isReference(maybeRef?: unknown): maybeRef is Reference {
-  return (
-    maybeRef !== null &&
-    typeof maybeRef === 'object' &&
-    'referencedId' in maybeRef &&
-    typeof maybeRef.referencedId === 'string'
-  )
-}
-
-export function isScalar(
-  value: unknown
-): value is string | number | boolean | bigint | symbol | undefined {
-  const type = typeof value
-  return (
-    value === null ||
-    type === 'string' ||
-    type === 'number' ||
-    type === 'boolean' ||
-    type === 'bigint' ||
-    type === 'symbol' ||
-    type === 'undefined'
-  )
-}
-
-export function take<T>(it: Iterator<T>, count: number): T[] {
-  const result: T[] = []
-  for (let i = 0; i < count; i++) {
-    const itr = it.next()
-    if (itr.done) break
-    result.push(itr.value)
-  }
-  return result
-}