a385823b2d
* feat(viewer-sandbox): Made a sandbox function that will only invoke the object-loader loading objects * first pass of creating an objectloader2 * updated build + added vitest * try to get viewer sandbox to use new code * sandbox type fix * refactor a bit * can download root * intermediate commit for downloader/caching queue * can download stuff! * refactor files * intro isBase and fix isString * move single download to downloader * fix download * PR feedback * some intermediate commit * do clean up and download better * clean up promises and linting * can generate values while downloading and caching * add a finish method * remove unused functions * remove asBase * add temporary docs * add more docs with mermaid * add more test models * add response validation * add tests and redo options * add test for download batch * fix downloader tests and change Item to have clearer Base items * add tests and refactor a little * use fetch in downloader as an option * use optional in-memory indexdb instead of monkey patching the global one * more refactors for options for objectloader2 * add tests for objectloader2 * adjust single download * benchmark loading and adjust ol2 batches * download more! * adjust to use hash privates * refactored again with renaming * cleanup * make setupCacheDb throw instead * use BatchedPool for downloads! * fix tests * adjust timings and add adaptive waiting * Only wait if queue wasn't empty and queue size was full * fix tests * fix file names and some private usage * fix interval and private usage * rename vars * use params for methods * fix params for constructors and tests * fix params for constructors and tests again * using dexie * faster settings but doesn't end well * fixed end, optimized and removed logs * fix tests * fix types? * update lock with WSL * add e2e small model test * fix/update yarn.lock * Remove unused eslint ignore to fix pre-commit * prettier fixes * fix real DB usage * rename methods to better match OL1 * rename methods to better match OL1 again * add extra header collection * add headers correctly * test getTotalObjectCount * feat(viewer-lib): Replaced old object loader with Adam's objectloder2 * fix(viewer-lib): Removed the old object loader. Removed unneeded pause time in speckle loader * Testing * only deferred if not downloaded....don't save everything * Lockfile * pool isn't adjustable, adjust download buckets, dexie read is faster * chore * fix(viewer-lib): Fixed compiler errors * fix getObject access with real indexeddb...adjust buffer for deferred access * Fix disposal and pausing * don't index item! * fix dockerfiles to use OL2 * fix Dockerfile * Fix dockerfile * defer correctly and use record to add/lookup/remove to * delete stuff correctly * chore(sandbox): Enabled viewer loading * use objects instead of arrays to avoid findIndex * remove extra count * add a found cache to avoid some db hits * order matters for deferment * move found map to deferment * change option numbers * 2 level cache with expiry * defer everything, use loader to track what is requested....expire only found items * add deferment disposal * oops mismerge * chore(sandbox): Default stream * Beta version of CachePump and CacheReader * Clean up initialization * More clean up * chore(objectloader2): Fixed CI compiler error * chore(objectloader2): Fixed prettier * add cachePump tests * add cacheReader tests * fixed more tests * fixed final tests * moving stuff around and lock return value * try to move stuff out of objectloader2 * use a factory * rename factory * formatting * eslist fixes * try allocating no strings * add comments * small refactor and add another test * fix deferment expiration and have test * use byte size for max memory cache size * fix deferment manager tests * saved comment * fix(viewer-sandbox): Fixed compiler error * ignore tshy * chore(frontend): Attempt to make viewer loading sequential --------- Co-authored-by: Adam Hathcock <adam@hathcock.uk> Co-authored-by: Kristaps Fabians Geikins <fabis94@live.com> Co-authored-by: Iain Sproat <68657+iainsproat@users.noreply.github.com>
230 lines
6.5 KiB
TypeScript
230 lines
6.5 KiB
TypeScript
import { describe, expect, test } from 'vitest'
|
|
import { ObjectLoader2 } from './objectLoader2.js'
|
|
import { Base, Item } from '../types/types.js'
|
|
import { MemoryDownloader } from './downloaders/memoryDownloader.js'
|
|
import { IDBFactory, IDBKeyRange } from 'fake-indexeddb'
|
|
import { MemoryDatabase } from './databases/memoryDatabase.js'
|
|
import IndexedDatabase from './databases/indexedDatabase.js'
|
|
|
|
describe('objectloader2', () => {
|
|
test('can get a root object from cache', async () => {
|
|
const rootId = 'baseId'
|
|
const rootBase: Base = { id: 'baseId', speckle_type: 'type' }
|
|
const downloader = new MemoryDownloader(
|
|
rootId,
|
|
new Map<string, Base>([[rootId, rootBase]])
|
|
)
|
|
const loader = new ObjectLoader2({
|
|
rootId,
|
|
downloader,
|
|
database: new IndexedDatabase({
|
|
indexedDB: new IDBFactory(),
|
|
keyRange: IDBKeyRange
|
|
})
|
|
})
|
|
const x = await loader.getRootObject()
|
|
expect(x).toMatchSnapshot()
|
|
})
|
|
|
|
test('can get a root object from downloader', async () => {
|
|
const rootId = 'baseId'
|
|
const rootBase: Base = { id: 'baseId', speckle_type: 'type' }
|
|
const downloader = new MemoryDownloader(
|
|
rootId,
|
|
new Map<string, Base>([[rootId, rootBase]])
|
|
)
|
|
const loader = new ObjectLoader2({
|
|
rootId,
|
|
downloader,
|
|
database: new IndexedDatabase({
|
|
indexedDB: new IDBFactory(),
|
|
keyRange: IDBKeyRange
|
|
})
|
|
})
|
|
const x = await loader.getRootObject()
|
|
expect(x).toMatchSnapshot()
|
|
})
|
|
|
|
test('can get single object from cache using iterator', async () => {
|
|
const rootId = 'baseId'
|
|
const rootBase: Base = { id: 'baseId', speckle_type: 'type' }
|
|
|
|
const downloader = new MemoryDownloader(
|
|
rootId,
|
|
new Map<string, Base>([[rootId, rootBase]])
|
|
)
|
|
const loader = new ObjectLoader2({
|
|
rootId,
|
|
downloader,
|
|
database: new IndexedDatabase({
|
|
indexedDB: new IDBFactory(),
|
|
keyRange: IDBKeyRange
|
|
})
|
|
})
|
|
const r = []
|
|
for await (const x of loader.getObjectIterator()) {
|
|
r.push(x)
|
|
}
|
|
|
|
expect(r).toMatchSnapshot()
|
|
})
|
|
|
|
test('can get root/child object from memory cache using iterator and getObject', async () => {
|
|
const child1Base = { id: 'child1Id', speckle_type: 'type' } as Base
|
|
const child1 = { baseId: 'child1Id', base: child1Base } as unknown as Item
|
|
|
|
const rootId = 'rootId'
|
|
const rootBase: Base = {
|
|
id: 'rootId',
|
|
speckle_type: 'type',
|
|
__closure: { child1Id: 100 }
|
|
}
|
|
const root = {
|
|
baseId: rootId,
|
|
base: rootBase
|
|
} as Item
|
|
|
|
const records: Map<string, Base> = new Map<string, Base>()
|
|
records.set(root.baseId, rootBase)
|
|
records.set(child1.baseId, child1Base)
|
|
|
|
const loader = new ObjectLoader2({
|
|
rootId: root.baseId,
|
|
downloader: new MemoryDownloader(rootId, records),
|
|
database: new MemoryDatabase({ items: records })
|
|
})
|
|
|
|
const r = []
|
|
const obj = loader.getObject({ id: child1.baseId })
|
|
for await (const x of loader.getObjectIterator()) {
|
|
r.push(x)
|
|
}
|
|
|
|
expect(obj).toBeDefined()
|
|
expect(r).toMatchSnapshot()
|
|
const obj2 = await obj
|
|
expect(obj2).toBe(child1Base)
|
|
expect(obj2).toMatchSnapshot()
|
|
})
|
|
|
|
test('can get root/child object from memory downloader using iterator and getObject', async () => {
|
|
const child1Base = { id: 'child1Id', speckle_type: 'type' } as Base
|
|
const child1 = { baseId: 'child1Id', base: child1Base } as unknown as Item
|
|
|
|
const rootId = 'rootId'
|
|
const rootBase: Base = {
|
|
id: 'rootId',
|
|
speckle_type: 'type',
|
|
__closure: { child1Id: 100 }
|
|
}
|
|
const root = {
|
|
baseId: rootId,
|
|
base: rootBase
|
|
} as unknown as Item
|
|
|
|
const records: Map<string, Base> = new Map<string, Base>()
|
|
records.set(root.baseId, rootBase)
|
|
records.set(child1.baseId, child1Base)
|
|
|
|
const loader = new ObjectLoader2({
|
|
rootId: root.baseId,
|
|
downloader: new MemoryDownloader(rootId, records),
|
|
database: new IndexedDatabase({
|
|
indexedDB: new IDBFactory(),
|
|
keyRange: IDBKeyRange
|
|
})
|
|
})
|
|
const r = []
|
|
const obj = loader.getObject({ id: child1.baseId })
|
|
for await (const x of loader.getObjectIterator()) {
|
|
r.push(x)
|
|
}
|
|
|
|
expect(obj).toBeDefined()
|
|
expect(r).toMatchSnapshot()
|
|
const obj2 = await obj
|
|
expect(obj2).toBe(child1Base)
|
|
expect(obj2).toMatchSnapshot()
|
|
})
|
|
|
|
test('add extra header', async () => {
|
|
const rootId = 'rootId'
|
|
const rootBase: Base = {
|
|
id: 'rootId',
|
|
speckle_type: 'type',
|
|
__closure: { child1Id: 100 }
|
|
}
|
|
const root = {
|
|
baseId: rootId,
|
|
base: rootBase
|
|
} as Item
|
|
|
|
const records: Map<string, Base> = new Map<string, Base>()
|
|
records.set(root.baseId, rootBase)
|
|
const headers = new Headers()
|
|
headers.set('x-test', 'asdf')
|
|
const loader = new ObjectLoader2({
|
|
rootId: root.baseId,
|
|
downloader: new MemoryDownloader(rootId, records),
|
|
database: new IndexedDatabase({
|
|
indexedDB: new IDBFactory(),
|
|
keyRange: IDBKeyRange
|
|
})
|
|
})
|
|
const x = await loader.getRootObject()
|
|
expect(x).toMatchSnapshot()
|
|
})
|
|
|
|
test('createFromJSON test', async () => {
|
|
const root = `{
|
|
"list": [{
|
|
"speckle_type": "reference",
|
|
"referencedId": "0e61e61edee00404ec6e0f9f594bce24",
|
|
"__closure": null
|
|
}],
|
|
"list2": [{
|
|
"speckle_type": "reference",
|
|
"referencedId": "f70738e3e3e593ac11099a6ed6b71154",
|
|
"__closure": null
|
|
}],
|
|
"arr": null,
|
|
"detachedProp": null,
|
|
"detachedProp2": null,
|
|
"attachedProp": null,
|
|
"crazyProp": null,
|
|
"applicationId": "1",
|
|
"speckle_type": "Speckle.Core.Tests.Unit.Models.BaseTests+SampleObjectBase2",
|
|
"dynamicProp": 123,
|
|
"id": "efeadaca70a85ae6d3acfc93a8b380db",
|
|
"__closure": {
|
|
"0e61e61edee00404ec6e0f9f594bce24": 100,
|
|
"f70738e3e3e593ac11099a6ed6b71154": 100
|
|
}
|
|
}`
|
|
|
|
const list1 = `{
|
|
"data": [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0],
|
|
"applicationId": null,
|
|
"speckle_type": "Speckle.Core.Models.DataChunk",
|
|
"id": "0e61e61edee00404ec6e0f9f594bce24"
|
|
}`
|
|
|
|
const list2 = `{
|
|
"data": [1.0, 10.0],
|
|
"applicationId": null,
|
|
"speckle_type": "Speckle.Core.Models.DataChunk",
|
|
"id": "f70738e3e3e593ac11099a6ed6b71154"
|
|
}`
|
|
const rootObj = JSON.parse(root) as Base
|
|
const list1Obj = JSON.parse(list1) as Base
|
|
const list2Obj = JSON.parse(list2) as Base
|
|
|
|
const loader = ObjectLoader2.createFromObjects([rootObj, list1Obj, list2Obj])
|
|
const r = []
|
|
for await (const x of loader.getObjectIterator()) {
|
|
r.push(x)
|
|
}
|
|
expect(r).toMatchSnapshot()
|
|
})
|
|
})
|