ol2(feat) remove dexie dependency (#5148)

* add new indexed db wrapper

* remove dexie and use new idb

* change storage durability

* allow undefined gets

* some clean up

* linting fixes

* add db close

* cleaner upgrade DB and more clean up

* fix database issues by deleting things instead of upgrading them

packages/objectloader2/package.json

@@ -33,8 +33,7 @@
   "author": "AEC Systems",
   "license": "Apache-2.0",
   "dependencies": {
-    "@speckle/shared": "workspace:^",
-    "dexie": "^4.0.11"
+    "@speckle/shared": "workspace:^"
   },
   "devDependencies": {
     "@types/lodash": "^4.17.5",

packages/objectloader2/src/core/objectLoader2Factory.ts

@@ -48,7 +48,6 @@ export class ObjectLoader2Factory {
     }
     if (getFeatureFlag(ObjectLoader2Flags.USE_CACHE) === 'true') {
       database = new IndexedDatabase({
-        logger: log,
         indexedDB: params.options?.indexedDB,
         keyRange: params.options?.keyRange
       })

packages/objectloader2/src/core/stages/ItemStore.ts

@@ -0,0 +1,196 @@
+/* eslint-disable @typescript-eslint/no-base-to-string */
+/* eslint-disable @typescript-eslint/prefer-promise-reject-errors */
+/* eslint-disable @typescript-eslint/no-explicit-any */
+/* eslint-disable @typescript-eslint/no-unsafe-function-type */
+import { isSafari } from '@speckle/shared'
+import { Item } from '../../types/types.js'
+
+/**
+ * A wrapper class for IndexedDB to simplify common database operations.
+ */
+export interface ItemStoreOptions {
+  indexedDB?: IDBFactory
+  keyRange?: {
+    bound: Function
+    lowerBound: Function
+    upperBound: Function
+  }
+}
+export class ItemStore {
+  #options: ItemStoreOptions
+
+  #db: IDBDatabase | undefined = undefined
+  readonly #dbName: string
+  readonly #storeName: string
+
+  constructor(options: ItemStoreOptions, dbName: string, storeName: string) {
+    this.#options = options
+    this.#dbName = dbName
+    this.#storeName = storeName
+  }
+
+  /**
+   * Initializes the database connection and creates the object store if needed.
+   * This must be called before any other database operations.
+   */
+  async init(): Promise<void> {
+    if (this.#db) return
+    await this.#safariFix()
+    return this.#openDatabase()
+  }
+
+  /**
+   * Opens the database, and if there's an error, deletes the database and tries again.
+   */
+  async #openDatabase(): Promise<void> {
+    const idb = this.#options.indexedDB ?? indexedDB
+
+    return new Promise((resolve, reject) => {
+      const request = idb.open(this.#dbName, 1)
+
+      request.onerror = (): any => {
+        console.warn(
+          `Failed to open database: ${this.#dbName}, deleting and trying again`
+        )
+        // Delete the database and try again
+        const deleteRequest = idb.deleteDatabase(this.#dbName)
+        deleteRequest.onsuccess = (): any => {
+          // Try opening again after deletion
+          void this.#openDatabase().then(resolve).catch(reject)
+        }
+        deleteRequest.onerror = (): any => {
+          reject(`Failed to delete and reopen database: ${this.#dbName}`)
+        }
+      }
+
+      request.onupgradeneeded = (event): any => {
+        const db = (event.target as IDBOpenDBRequest).result
+        if (db.objectStoreNames.contains(this.#storeName)) {
+          db.deleteObjectStore(this.#storeName)
+        }
+        db.createObjectStore(this.#storeName, { keyPath: 'baseId' })
+      }
+
+      request.onsuccess = (event): any => {
+        this.#db = (event.target as IDBOpenDBRequest).result
+        resolve()
+      }
+    })
+  }
+
+  #getDB(): IDBDatabase {
+    if (!this.#db) {
+      throw new Error('Database not initialized. Call init() first.')
+    }
+    return this.#db
+  }
+
+  /**
+   * Fixes a Safari bug where IndexedDB requests get lost and never resolve - invoke before you use IndexedDB
+   * @link Credits and more info: https://github.com/jakearchibald/safari-14-idb-fix
+   */
+  async #safariFix(): Promise<void> {
+    // No point putting other browsers or older versions of Safari through this mess.
+    if (!isSafari() || !this.#options.indexedDB?.databases) return Promise.resolve()
+
+    let intervalId: ReturnType<typeof setInterval>
+
+    return new Promise<void>((resolve: () => void) => {
+      const tryIdb = (): Promise<IDBDatabaseInfo[]> | undefined =>
+        this.#options.indexedDB?.databases().finally(resolve)
+      intervalId = setInterval(() => {
+        void tryIdb()
+      }, 100)
+      void tryIdb()
+    }).finally(() => clearInterval(intervalId))
+  }
+
+  /**
+   * Inserts or updates an array of items in a single transaction.
+   * @param data The array of items to insert.
+   */
+  bulkInsert(data: Item[]): Promise<void> {
+    return new Promise((resolve, reject) => {
+      try {
+        const transaction = this.#getDB().transaction(this.#storeName, 'readwrite', {
+          durability: 'relaxed'
+        })
+        const store = transaction.objectStore(this.#storeName)
+
+        transaction.onerror = (): any => {
+          reject(`Transaction error: ${transaction.error}`)
+        }
+        transaction.oncomplete = (): any => {
+          resolve()
+        }
+
+        data.forEach((item) => store.put(item))
+      } catch (error) {
+        reject(error)
+      }
+    })
+  }
+
+  /**
+   * Retrieves an array of items from the object store based on their IDs.
+   * @param ids The array of IDs to retrieve.
+   */
+  bulkGet(ids: string[]): Promise<(Item | undefined)[]> {
+    return new Promise((resolve, reject) => {
+      if (ids.length === 0) {
+        return resolve([])
+      }
+      try {
+        const transaction = this.#getDB().transaction(this.#storeName, 'readonly', {
+          durability: 'relaxed'
+        })
+        const store = transaction.objectStore(this.#storeName)
+        const promises: Promise<Item | undefined>[] = []
+
+        for (const id of ids) {
+          promises.push(
+            new Promise((resolveGet, rejectGet) => {
+              const request = store.get(id)
+              request.onerror = (): void =>
+                rejectGet(`Request error for id ${id}: ${request.error}`)
+              // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+              request.onsuccess = (): void => resolveGet(request.result)
+            })
+          )
+        }
+
+        Promise.all(promises)
+          .then((results) => {
+            resolve(results)
+          })
+          .catch(reject)
+      } catch (error) {
+        reject(error)
+      }
+    })
+  }
+
+  /**
+   * Retrieves all items from the object store.
+   */
+  getAll(): Promise<Item[]> {
+    return new Promise((resolve, reject) => {
+      try {
+        const transaction = this.#getDB().transaction(this.#storeName, 'readonly')
+        const store = transaction.objectStore(this.#storeName)
+        const request = store.getAll()
+
+        request.onerror = (): any => reject(`Request error: ${request.error}`)
+        request.onsuccess = (): any => resolve(request.result)
+      } catch (error) {
+        reject(error)
+      }
+    })
+  }
+
+  close(): void {
+    if (!this.#db) return
+    this.#db.close()
+    this.#db = undefined
+  }
+}

packages/objectloader2/src/core/stages/cacheReader.ts

@@ -75,7 +75,11 @@ export class CacheReader {
         this.#notFoundQueue?.add(batch[i])
       }
     }
-    this.#logger('readBatch: left, time', items.length, performance.now() - start)
+    this.#logger(
+      `readBatch: batch ${batch.length}, time ${
+        performance.now() - start
+      } ms left ${this.#readQueue?.count()}`
+    )
   }
 
   disposeAsync(): Promise<void> {

packages/objectloader2/src/core/stages/cacheWriter.ts

@@ -46,7 +46,11 @@ export class CacheWriter implements Queue<Item> {
   async writeAll(items: Item[]): Promise<void> {
     const start = performance.now()
     await this.#database.saveBatch({ batch: items })
-    this.#logger('writeBatch: left, time', items.length, performance.now() - start)
+    this.#logger(
+      `writeBatch: wrote ${items.length}, time ${
+        performance.now() - start
+      } ms left ${this.#writeQueue?.count()}`
+    )
   }
 
   async disposeAsync(): Promise<void> {

packages/objectloader2/src/core/stages/indexedDatabase.ts

@@ -1,26 +1,10 @@
 /* eslint-disable @typescript-eslint/no-unsafe-function-type */
-import { CustomLogger } from '../../types/functions.js'
 import { Item } from '../../types/types.js'
-import { isSafari } from '@speckle/shared'
-import { Dexie, DexieOptions, Table } from 'dexie'
 import { Database } from '../interfaces.js'
 import BatchingQueue from '../../queues/batchingQueue.js'
-
-export class ObjectStore extends Dexie {
-  static #databaseName: string = 'speckle-cache'
-  objects!: Table<Item, string> // Table type: <entity, primaryKey>
-
-  constructor(options: DexieOptions) {
-    super(ObjectStore.#databaseName, options)
-
-    this.version(1).stores({
-      objects: 'baseId, item' // baseId is primary key
-    })
-  }
-}
+import { ItemStore } from './ItemStore.js'
 
 export interface IndexedDatabaseOptions {
-  logger?: CustomLogger
   indexedDB?: IDBFactory
   keyRange?: {
     bound: Function
@@ -31,97 +15,35 @@ export interface IndexedDatabaseOptions {
 
 export default class IndexedDatabase implements Database {
   #options: IndexedDatabaseOptions
-  #logger: CustomLogger
-
-  #cacheDB?: ObjectStore
-
+  #cacheDB: ItemStore
   #writeQueue: BatchingQueue<Item> | undefined
 
-  // #count: number = 0
-
   constructor(options: IndexedDatabaseOptions) {
     this.#options = options
-    this.#logger = options.logger || ((): void => {})
+    this.#cacheDB = new ItemStore(
+      {
+        indexedDB: this.#options.indexedDB,
+        keyRange: this.#options.keyRange
+      },
+      'speckle-cache',
+      'cache'
+    )
   }
 
   async getAll(keys: string[]): Promise<(Item | undefined)[]> {
-    await this.#setupCacheDb()
-    let items: (Item | undefined)[] = []
-    //   this.#count++
-    //  const startTime = performance.now()
-    // this.#logger('Start read ' + x + ' ' + batch.length)
-
-    //faster than BulkGet with dexie
-    await this.#cacheDB!.transaction('r', this.#cacheDB!.objects, async () => {
-      const gets = keys.map((key) => this.#cacheDB!.objects.get(key))
-      const cachedData = await Promise.all(gets)
-      items = cachedData
-    })
-    // const endTime = performance.now()
-    // const duration = endTime - startTime
-    //this.#logger('Saved batch ' + x + ' ' + batch.length + ' ' + duration / TIME_MS.second)
-
-    return items
-  }
-
-  async #openDatabase(): Promise<ObjectStore> {
-    const db = new ObjectStore({
-      indexedDB: this.#options.indexedDB ?? globalThis.indexedDB,
-      IDBKeyRange: this.#options.keyRange ?? IDBKeyRange,
-      chromeTransactionDurability: 'relaxed'
-    })
-    await db.open()
-    return db
-  }
-
-  async #setupCacheDb(): Promise<void> {
-    if (this.#cacheDB !== undefined) {
-      return
-    }
-
-    // Initialize
-    await this.#safariFix()
-    this.#cacheDB = await this.#openDatabase()
+    await this.#cacheDB.init()
+    return await this.#cacheDB.bulkGet(keys)
   }
 
   async saveBatch(params: { batch: Item[] }): Promise<void> {
-    await this.#setupCacheDb()
+    await this.#cacheDB.init()
     const { batch } = params
-    //const x = this.#count
-    //this.#count++
-
-    // const startTime = performance.now()
-    //  this.#logger('Start save ' + x + ' ' + batch.length)
-    await this.#cacheDB!.objects.bulkPut(batch)
-    // const endTime = performance.now()
-    // const duration = endTime - startTime
-    //this.#logger('Saved batch ' + x + ' ' + batch.length + ' ' + duration / TIME_MS.second)
-  }
-
-  /**
-   * Fixes a Safari bug where IndexedDB requests get lost and never resolve - invoke before you use IndexedDB
-   * @link Credits and more info: https://github.com/jakearchibald/safari-14-idb-fix
-   */
-  async #safariFix(): Promise<void> {
-    // No point putting other browsers or older versions of Safari through this mess.
-    if (!isSafari() || !this.#options.indexedDB?.databases) return Promise.resolve()
-
-    let intervalId: ReturnType<typeof setInterval>
-
-    return new Promise<void>((resolve: () => void) => {
-      const tryIdb = (): Promise<IDBDatabaseInfo[]> | undefined =>
-        this.#options.indexedDB?.databases().finally(resolve)
-      intervalId = setInterval(() => {
-        void tryIdb()
-      }, 100)
-      void tryIdb()
-    }).finally(() => clearInterval(intervalId))
+    await this.#cacheDB.bulkInsert(batch)
   }
 
   async disposeAsync(): Promise<void> {
-    this.#cacheDB?.close()
-    this.#cacheDB = undefined
     await this.#writeQueue?.disposeAsync()
     this.#writeQueue = undefined
+    this.#cacheDB.close()
   }
 }

yarn.lock

@@ -16158,7 +16158,6 @@ __metadata:
     "@typescript-eslint/eslint-plugin": "npm:^7.12.0"
     "@typescript-eslint/parser": "npm:^7.12.0"
     "@vitest/ui": "npm:^3.0.9"
-    dexie: "npm:^4.0.11"
     eslint: "npm:^9.4.0"
     eslint-config-prettier: "npm:^9.1.0"
     fake-indexeddb: "npm:^6.0.0"