Files
speckle-server/packages/objectloader2/src/operations/objectLoader2.spec.ts
T
Alexandru Popovici a385823b2d feat(viewer): objectloader2 integration (#4267)
* feat(viewer-sandbox): Made a sandbox function that will only invoke the object-loader loading objects

* first pass of creating an objectloader2

* updated build + added vitest

* try to get viewer sandbox to use new code

* sandbox type fix

* refactor a bit

* can download root

* intermediate commit for downloader/caching queue

* can download stuff!

* refactor files

* intro isBase and fix isString

* move single download to downloader

* fix download

* PR feedback

* some intermediate commit

* do clean up and download better

* clean up promises and linting

* can generate values while downloading and caching

* add a finish method

* remove unused functions

* remove asBase

* add temporary docs

* add more docs with mermaid

* add more test models

* add response validation

* add tests and redo options

* add test for download batch

* fix downloader tests and change Item to have clearer Base items

* add tests and refactor a little

* use fetch in downloader as an option

* use optional in-memory indexdb instead of monkey patching the global one

* more refactors for options for objectloader2

* add tests for objectloader2

* adjust single download

* benchmark loading and adjust ol2 batches

* download more!

* adjust to use hash privates

* refactored again with renaming

* cleanup

* make setupCacheDb throw instead

* use BatchedPool for downloads!

* fix tests

* adjust timings and add adaptive waiting

* Only wait if queue wasn't empty and queue size was full

* fix tests

* fix file names and some private usage

* fix interval and private usage

* rename vars

* use params for methods

* fix params for constructors and tests

* fix params for constructors and tests again

* using dexie

* faster settings but doesn't end well

* fixed end, optimized and removed logs

* fix tests

* fix types?

* update lock with WSL

* add e2e small model test

* fix/update yarn.lock

* Remove unused eslint ignore to fix pre-commit

* prettier fixes

* fix real DB usage

* rename methods to better match OL1

* rename methods to better match OL1 again

* add extra header collection

* add headers correctly

* test getTotalObjectCount

* feat(viewer-lib): Replaced old object loader with Adam's  objectloder2

* fix(viewer-lib): Removed the old object loader. Removed unneeded pause time in speckle loader

* Testing

* only deferred if not downloaded....don't save everything

* Lockfile

* pool isn't adjustable, adjust download buckets, dexie read is faster

* chore

* fix(viewer-lib): Fixed compiler errors

* fix getObject access with real indexeddb...adjust buffer for deferred access

* Fix disposal and pausing

* don't index item!

* fix dockerfiles to use OL2

* fix Dockerfile

* Fix dockerfile

* defer correctly and use record to add/lookup/remove to

* delete stuff correctly

* chore(sandbox): Enabled viewer loading

* use objects instead of arrays to avoid findIndex

* remove extra count

* add a found cache to avoid some db hits

* order matters for deferment

* move found map to deferment

* change option numbers

* 2 level cache with expiry

* defer everything, use loader to track what is requested....expire only found items

* add deferment disposal

* oops mismerge

* chore(sandbox): Default stream

* Beta version of CachePump and CacheReader

* Clean up initialization

* More clean up

* chore(objectloader2): Fixed CI compiler error

* chore(objectloader2): Fixed prettier

* add cachePump tests

* add cacheReader tests

* fixed more tests

* fixed final tests

* moving stuff around and lock return value

* try to move stuff out of objectloader2

* use a factory

* rename factory

* formatting

* eslist fixes

* try allocating no strings

* add comments

* small refactor and add another test

* fix deferment expiration and have test

* use byte size for max memory cache size

* fix deferment manager tests

* saved comment

* fix(viewer-sandbox): Fixed compiler error

* ignore tshy

* chore(frontend): Attempt to make viewer loading sequential

---------

Co-authored-by: Adam Hathcock <adam@hathcock.uk>
Co-authored-by: Kristaps Fabians Geikins <fabis94@live.com>
Co-authored-by: Iain Sproat <68657+iainsproat@users.noreply.github.com>
2025-05-21 10:05:50 +03:00

230 lines
6.5 KiB
TypeScript

import { describe, expect, test } from 'vitest'
import { ObjectLoader2 } from './objectLoader2.js'
import { Base, Item } from '../types/types.js'
import { MemoryDownloader } from './downloaders/memoryDownloader.js'
import { IDBFactory, IDBKeyRange } from 'fake-indexeddb'
import { MemoryDatabase } from './databases/memoryDatabase.js'
import IndexedDatabase from './databases/indexedDatabase.js'
describe('objectloader2', () => {
test('can get a root object from cache', async () => {
const rootId = 'baseId'
const rootBase: Base = { id: 'baseId', speckle_type: 'type' }
const downloader = new MemoryDownloader(
rootId,
new Map<string, Base>([[rootId, rootBase]])
)
const loader = new ObjectLoader2({
rootId,
downloader,
database: new IndexedDatabase({
indexedDB: new IDBFactory(),
keyRange: IDBKeyRange
})
})
const x = await loader.getRootObject()
expect(x).toMatchSnapshot()
})
test('can get a root object from downloader', async () => {
const rootId = 'baseId'
const rootBase: Base = { id: 'baseId', speckle_type: 'type' }
const downloader = new MemoryDownloader(
rootId,
new Map<string, Base>([[rootId, rootBase]])
)
const loader = new ObjectLoader2({
rootId,
downloader,
database: new IndexedDatabase({
indexedDB: new IDBFactory(),
keyRange: IDBKeyRange
})
})
const x = await loader.getRootObject()
expect(x).toMatchSnapshot()
})
test('can get single object from cache using iterator', async () => {
const rootId = 'baseId'
const rootBase: Base = { id: 'baseId', speckle_type: 'type' }
const downloader = new MemoryDownloader(
rootId,
new Map<string, Base>([[rootId, rootBase]])
)
const loader = new ObjectLoader2({
rootId,
downloader,
database: new IndexedDatabase({
indexedDB: new IDBFactory(),
keyRange: IDBKeyRange
})
})
const r = []
for await (const x of loader.getObjectIterator()) {
r.push(x)
}
expect(r).toMatchSnapshot()
})
test('can get root/child object from memory cache using iterator and getObject', async () => {
const child1Base = { id: 'child1Id', speckle_type: 'type' } as Base
const child1 = { baseId: 'child1Id', base: child1Base } as unknown as Item
const rootId = 'rootId'
const rootBase: Base = {
id: 'rootId',
speckle_type: 'type',
__closure: { child1Id: 100 }
}
const root = {
baseId: rootId,
base: rootBase
} as Item
const records: Map<string, Base> = new Map<string, Base>()
records.set(root.baseId, rootBase)
records.set(child1.baseId, child1Base)
const loader = new ObjectLoader2({
rootId: root.baseId,
downloader: new MemoryDownloader(rootId, records),
database: new MemoryDatabase({ items: records })
})
const r = []
const obj = loader.getObject({ id: child1.baseId })
for await (const x of loader.getObjectIterator()) {
r.push(x)
}
expect(obj).toBeDefined()
expect(r).toMatchSnapshot()
const obj2 = await obj
expect(obj2).toBe(child1Base)
expect(obj2).toMatchSnapshot()
})
test('can get root/child object from memory downloader using iterator and getObject', async () => {
const child1Base = { id: 'child1Id', speckle_type: 'type' } as Base
const child1 = { baseId: 'child1Id', base: child1Base } as unknown as Item
const rootId = 'rootId'
const rootBase: Base = {
id: 'rootId',
speckle_type: 'type',
__closure: { child1Id: 100 }
}
const root = {
baseId: rootId,
base: rootBase
} as unknown as Item
const records: Map<string, Base> = new Map<string, Base>()
records.set(root.baseId, rootBase)
records.set(child1.baseId, child1Base)
const loader = new ObjectLoader2({
rootId: root.baseId,
downloader: new MemoryDownloader(rootId, records),
database: new IndexedDatabase({
indexedDB: new IDBFactory(),
keyRange: IDBKeyRange
})
})
const r = []
const obj = loader.getObject({ id: child1.baseId })
for await (const x of loader.getObjectIterator()) {
r.push(x)
}
expect(obj).toBeDefined()
expect(r).toMatchSnapshot()
const obj2 = await obj
expect(obj2).toBe(child1Base)
expect(obj2).toMatchSnapshot()
})
test('add extra header', async () => {
const rootId = 'rootId'
const rootBase: Base = {
id: 'rootId',
speckle_type: 'type',
__closure: { child1Id: 100 }
}
const root = {
baseId: rootId,
base: rootBase
} as Item
const records: Map<string, Base> = new Map<string, Base>()
records.set(root.baseId, rootBase)
const headers = new Headers()
headers.set('x-test', 'asdf')
const loader = new ObjectLoader2({
rootId: root.baseId,
downloader: new MemoryDownloader(rootId, records),
database: new IndexedDatabase({
indexedDB: new IDBFactory(),
keyRange: IDBKeyRange
})
})
const x = await loader.getRootObject()
expect(x).toMatchSnapshot()
})
test('createFromJSON test', async () => {
const root = `{
"list": [{
"speckle_type": "reference",
"referencedId": "0e61e61edee00404ec6e0f9f594bce24",
"__closure": null
}],
"list2": [{
"speckle_type": "reference",
"referencedId": "f70738e3e3e593ac11099a6ed6b71154",
"__closure": null
}],
"arr": null,
"detachedProp": null,
"detachedProp2": null,
"attachedProp": null,
"crazyProp": null,
"applicationId": "1",
"speckle_type": "Speckle.Core.Tests.Unit.Models.BaseTests+SampleObjectBase2",
"dynamicProp": 123,
"id": "efeadaca70a85ae6d3acfc93a8b380db",
"__closure": {
"0e61e61edee00404ec6e0f9f594bce24": 100,
"f70738e3e3e593ac11099a6ed6b71154": 100
}
}`
const list1 = `{
"data": [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0],
"applicationId": null,
"speckle_type": "Speckle.Core.Models.DataChunk",
"id": "0e61e61edee00404ec6e0f9f594bce24"
}`
const list2 = `{
"data": [1.0, 10.0],
"applicationId": null,
"speckle_type": "Speckle.Core.Models.DataChunk",
"id": "f70738e3e3e593ac11099a6ed6b71154"
}`
const rootObj = JSON.parse(root) as Base
const list1Obj = JSON.parse(list1) as Base
const list2Obj = JSON.parse(list2) as Base
const loader = ObjectLoader2.createFromObjects([rootObj, list1Obj, list2Obj])
const r = []
for await (const x of loader.getObjectIterator()) {
r.push(x)
}
expect(r).toMatchSnapshot()
})
})