From 9220a3ee4b2e50f10a2e4a80f01bb0aa6493a766 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Mon, 28 Jul 2025 15:57:07 +0100 Subject: [PATCH] ol2(feat) remove dexie dependency (#5148) * add new indexed db wrapper * remove dexie and use new idb * change storage durability * allow undefined gets * some clean up * linting fixes * add db close * cleaner upgrade DB and more clean up * fix database issues by deleting things instead of upgrading them --- packages/objectloader2/package.json | 3 +- .../src/core/objectLoader2Factory.ts | 1 - .../src/core/stages/ItemStore.ts | 196 ++++++++++++++++++ .../src/core/stages/cacheReader.ts | 6 +- .../src/core/stages/cacheWriter.ts | 6 +- .../src/core/stages/indexedDatabase.ts | 108 ++-------- yarn.lock | 1 - 7 files changed, 222 insertions(+), 99 deletions(-) create mode 100644 packages/objectloader2/src/core/stages/ItemStore.ts diff --git a/packages/objectloader2/package.json b/packages/objectloader2/package.json index 2e976b7a1..226a835b0 100644 --- a/packages/objectloader2/package.json +++ b/packages/objectloader2/package.json @@ -33,8 +33,7 @@ "author": "AEC Systems", "license": "Apache-2.0", "dependencies": { - "@speckle/shared": "workspace:^", - "dexie": "^4.0.11" + "@speckle/shared": "workspace:^" }, "devDependencies": { "@types/lodash": "^4.17.5", diff --git a/packages/objectloader2/src/core/objectLoader2Factory.ts b/packages/objectloader2/src/core/objectLoader2Factory.ts index 641055d36..fc5f950d3 100644 --- a/packages/objectloader2/src/core/objectLoader2Factory.ts +++ b/packages/objectloader2/src/core/objectLoader2Factory.ts @@ -48,7 +48,6 @@ export class ObjectLoader2Factory { } if (getFeatureFlag(ObjectLoader2Flags.USE_CACHE) === 'true') { database = new IndexedDatabase({ - logger: log, indexedDB: params.options?.indexedDB, keyRange: params.options?.keyRange }) diff --git a/packages/objectloader2/src/core/stages/ItemStore.ts b/packages/objectloader2/src/core/stages/ItemStore.ts new file mode 100644 index 000000000..de57d193e --- /dev/null +++ b/packages/objectloader2/src/core/stages/ItemStore.ts @@ -0,0 +1,196 @@ +/* eslint-disable @typescript-eslint/no-base-to-string */ +/* eslint-disable @typescript-eslint/prefer-promise-reject-errors */ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable @typescript-eslint/no-unsafe-function-type */ +import { isSafari } from '@speckle/shared' +import { Item } from '../../types/types.js' + +/** + * A wrapper class for IndexedDB to simplify common database operations. + */ +export interface ItemStoreOptions { + indexedDB?: IDBFactory + keyRange?: { + bound: Function + lowerBound: Function + upperBound: Function + } +} +export class ItemStore { + #options: ItemStoreOptions + + #db: IDBDatabase | undefined = undefined + readonly #dbName: string + readonly #storeName: string + + constructor(options: ItemStoreOptions, dbName: string, storeName: string) { + this.#options = options + this.#dbName = dbName + this.#storeName = storeName + } + + /** + * Initializes the database connection and creates the object store if needed. + * This must be called before any other database operations. + */ + async init(): Promise { + if (this.#db) return + await this.#safariFix() + return this.#openDatabase() + } + + /** + * Opens the database, and if there's an error, deletes the database and tries again. + */ + async #openDatabase(): Promise { + const idb = this.#options.indexedDB ?? indexedDB + + return new Promise((resolve, reject) => { + const request = idb.open(this.#dbName, 1) + + request.onerror = (): any => { + console.warn( + `Failed to open database: ${this.#dbName}, deleting and trying again` + ) + // Delete the database and try again + const deleteRequest = idb.deleteDatabase(this.#dbName) + deleteRequest.onsuccess = (): any => { + // Try opening again after deletion + void this.#openDatabase().then(resolve).catch(reject) + } + deleteRequest.onerror = (): any => { + reject(`Failed to delete and reopen database: ${this.#dbName}`) + } + } + + request.onupgradeneeded = (event): any => { + const db = (event.target as IDBOpenDBRequest).result + if (db.objectStoreNames.contains(this.#storeName)) { + db.deleteObjectStore(this.#storeName) + } + db.createObjectStore(this.#storeName, { keyPath: 'baseId' }) + } + + request.onsuccess = (event): any => { + this.#db = (event.target as IDBOpenDBRequest).result + resolve() + } + }) + } + + #getDB(): IDBDatabase { + if (!this.#db) { + throw new Error('Database not initialized. Call init() first.') + } + return this.#db + } + + /** + * Fixes a Safari bug where IndexedDB requests get lost and never resolve - invoke before you use IndexedDB + * @link Credits and more info: https://github.com/jakearchibald/safari-14-idb-fix + */ + async #safariFix(): Promise { + // No point putting other browsers or older versions of Safari through this mess. + if (!isSafari() || !this.#options.indexedDB?.databases) return Promise.resolve() + + let intervalId: ReturnType + + return new Promise((resolve: () => void) => { + const tryIdb = (): Promise | undefined => + this.#options.indexedDB?.databases().finally(resolve) + intervalId = setInterval(() => { + void tryIdb() + }, 100) + void tryIdb() + }).finally(() => clearInterval(intervalId)) + } + + /** + * Inserts or updates an array of items in a single transaction. + * @param data The array of items to insert. + */ + bulkInsert(data: Item[]): Promise { + return new Promise((resolve, reject) => { + try { + const transaction = this.#getDB().transaction(this.#storeName, 'readwrite', { + durability: 'relaxed' + }) + const store = transaction.objectStore(this.#storeName) + + transaction.onerror = (): any => { + reject(`Transaction error: ${transaction.error}`) + } + transaction.oncomplete = (): any => { + resolve() + } + + data.forEach((item) => store.put(item)) + } catch (error) { + reject(error) + } + }) + } + + /** + * Retrieves an array of items from the object store based on their IDs. + * @param ids The array of IDs to retrieve. + */ + bulkGet(ids: string[]): Promise<(Item | undefined)[]> { + return new Promise((resolve, reject) => { + if (ids.length === 0) { + return resolve([]) + } + try { + const transaction = this.#getDB().transaction(this.#storeName, 'readonly', { + durability: 'relaxed' + }) + const store = transaction.objectStore(this.#storeName) + const promises: Promise[] = [] + + for (const id of ids) { + promises.push( + new Promise((resolveGet, rejectGet) => { + const request = store.get(id) + request.onerror = (): void => + rejectGet(`Request error for id ${id}: ${request.error}`) + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + request.onsuccess = (): void => resolveGet(request.result) + }) + ) + } + + Promise.all(promises) + .then((results) => { + resolve(results) + }) + .catch(reject) + } catch (error) { + reject(error) + } + }) + } + + /** + * Retrieves all items from the object store. + */ + getAll(): Promise { + return new Promise((resolve, reject) => { + try { + const transaction = this.#getDB().transaction(this.#storeName, 'readonly') + const store = transaction.objectStore(this.#storeName) + const request = store.getAll() + + request.onerror = (): any => reject(`Request error: ${request.error}`) + request.onsuccess = (): any => resolve(request.result) + } catch (error) { + reject(error) + } + }) + } + + close(): void { + if (!this.#db) return + this.#db.close() + this.#db = undefined + } +} diff --git a/packages/objectloader2/src/core/stages/cacheReader.ts b/packages/objectloader2/src/core/stages/cacheReader.ts index 448193ef9..3d526be2d 100644 --- a/packages/objectloader2/src/core/stages/cacheReader.ts +++ b/packages/objectloader2/src/core/stages/cacheReader.ts @@ -75,7 +75,11 @@ export class CacheReader { this.#notFoundQueue?.add(batch[i]) } } - this.#logger('readBatch: left, time', items.length, performance.now() - start) + this.#logger( + `readBatch: batch ${batch.length}, time ${ + performance.now() - start + } ms left ${this.#readQueue?.count()}` + ) } disposeAsync(): Promise { diff --git a/packages/objectloader2/src/core/stages/cacheWriter.ts b/packages/objectloader2/src/core/stages/cacheWriter.ts index d8683429b..8909d5697 100644 --- a/packages/objectloader2/src/core/stages/cacheWriter.ts +++ b/packages/objectloader2/src/core/stages/cacheWriter.ts @@ -46,7 +46,11 @@ export class CacheWriter implements Queue { async writeAll(items: Item[]): Promise { const start = performance.now() await this.#database.saveBatch({ batch: items }) - this.#logger('writeBatch: left, time', items.length, performance.now() - start) + this.#logger( + `writeBatch: wrote ${items.length}, time ${ + performance.now() - start + } ms left ${this.#writeQueue?.count()}` + ) } async disposeAsync(): Promise { diff --git a/packages/objectloader2/src/core/stages/indexedDatabase.ts b/packages/objectloader2/src/core/stages/indexedDatabase.ts index c10e60447..7dfb39d37 100644 --- a/packages/objectloader2/src/core/stages/indexedDatabase.ts +++ b/packages/objectloader2/src/core/stages/indexedDatabase.ts @@ -1,26 +1,10 @@ /* eslint-disable @typescript-eslint/no-unsafe-function-type */ -import { CustomLogger } from '../../types/functions.js' import { Item } from '../../types/types.js' -import { isSafari } from '@speckle/shared' -import { Dexie, DexieOptions, Table } from 'dexie' import { Database } from '../interfaces.js' import BatchingQueue from '../../queues/batchingQueue.js' - -export class ObjectStore extends Dexie { - static #databaseName: string = 'speckle-cache' - objects!: Table // Table type: - - constructor(options: DexieOptions) { - super(ObjectStore.#databaseName, options) - - this.version(1).stores({ - objects: 'baseId, item' // baseId is primary key - }) - } -} +import { ItemStore } from './ItemStore.js' export interface IndexedDatabaseOptions { - logger?: CustomLogger indexedDB?: IDBFactory keyRange?: { bound: Function @@ -31,97 +15,35 @@ export interface IndexedDatabaseOptions { export default class IndexedDatabase implements Database { #options: IndexedDatabaseOptions - #logger: CustomLogger - - #cacheDB?: ObjectStore - + #cacheDB: ItemStore #writeQueue: BatchingQueue | undefined - // #count: number = 0 - constructor(options: IndexedDatabaseOptions) { this.#options = options - this.#logger = options.logger || ((): void => {}) + this.#cacheDB = new ItemStore( + { + indexedDB: this.#options.indexedDB, + keyRange: this.#options.keyRange + }, + 'speckle-cache', + 'cache' + ) } async getAll(keys: string[]): Promise<(Item | undefined)[]> { - await this.#setupCacheDb() - let items: (Item | undefined)[] = [] - // this.#count++ - // const startTime = performance.now() - // this.#logger('Start read ' + x + ' ' + batch.length) - - //faster than BulkGet with dexie - await this.#cacheDB!.transaction('r', this.#cacheDB!.objects, async () => { - const gets = keys.map((key) => this.#cacheDB!.objects.get(key)) - const cachedData = await Promise.all(gets) - items = cachedData - }) - // const endTime = performance.now() - // const duration = endTime - startTime - //this.#logger('Saved batch ' + x + ' ' + batch.length + ' ' + duration / TIME_MS.second) - - return items - } - - async #openDatabase(): Promise { - const db = new ObjectStore({ - indexedDB: this.#options.indexedDB ?? globalThis.indexedDB, - IDBKeyRange: this.#options.keyRange ?? IDBKeyRange, - chromeTransactionDurability: 'relaxed' - }) - await db.open() - return db - } - - async #setupCacheDb(): Promise { - if (this.#cacheDB !== undefined) { - return - } - - // Initialize - await this.#safariFix() - this.#cacheDB = await this.#openDatabase() + await this.#cacheDB.init() + return await this.#cacheDB.bulkGet(keys) } async saveBatch(params: { batch: Item[] }): Promise { - await this.#setupCacheDb() + await this.#cacheDB.init() const { batch } = params - //const x = this.#count - //this.#count++ - - // const startTime = performance.now() - // this.#logger('Start save ' + x + ' ' + batch.length) - await this.#cacheDB!.objects.bulkPut(batch) - // const endTime = performance.now() - // const duration = endTime - startTime - //this.#logger('Saved batch ' + x + ' ' + batch.length + ' ' + duration / TIME_MS.second) - } - - /** - * Fixes a Safari bug where IndexedDB requests get lost and never resolve - invoke before you use IndexedDB - * @link Credits and more info: https://github.com/jakearchibald/safari-14-idb-fix - */ - async #safariFix(): Promise { - // No point putting other browsers or older versions of Safari through this mess. - if (!isSafari() || !this.#options.indexedDB?.databases) return Promise.resolve() - - let intervalId: ReturnType - - return new Promise((resolve: () => void) => { - const tryIdb = (): Promise | undefined => - this.#options.indexedDB?.databases().finally(resolve) - intervalId = setInterval(() => { - void tryIdb() - }, 100) - void tryIdb() - }).finally(() => clearInterval(intervalId)) + await this.#cacheDB.bulkInsert(batch) } async disposeAsync(): Promise { - this.#cacheDB?.close() - this.#cacheDB = undefined await this.#writeQueue?.disposeAsync() this.#writeQueue = undefined + this.#cacheDB.close() } } diff --git a/yarn.lock b/yarn.lock index 5453133d0..2885851bc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -16158,7 +16158,6 @@ __metadata: "@typescript-eslint/eslint-plugin": "npm:^7.12.0" "@typescript-eslint/parser": "npm:^7.12.0" "@vitest/ui": "npm:^3.0.9" - dexie: "npm:^4.0.11" eslint: "npm:^9.4.0" eslint-config-prettier: "npm:^9.1.0" fake-indexeddb: "npm:^6.0.0"