Offline Object Loader (#3717)

* Quick hack to demo how an offline loader would work with as little complication as possible

* Further simplified yielding objects in offline mode

* Commented out the URL thing

* Implemented SpeckleOfflineLoader. JSON parsing is implemented at object-loader level, completely isolated from the rest of the implementation in order to avoid regression

* Isolated ObjectLoader creation in base SpeckleLoader class so any extended classes can overwrite the way the object loader is created and used

* Removed the big json sample file

* Updated version

* Removed unused functions from objectloader

* Restored viewer package version

* Fixed typo

* Renamed and moved the sample offline Speckle JSON

* Replaced the default JSON object sample with a much smaller one since we don't want to increase the sandbox's build size by 10 megs

* Fixed a linting error

packages/objectloader/src/index.js

@@ -107,6 +107,69 @@ class ObjectLoader {
     }
   }
 
+  static createFromJSON(json) {
+    const start = performance.now()
+    const jsonObj = JSON.parse(json)
+    console.warn('JSON Parse Time -> ', performance.now() - start)
+
+    const rootObject = jsonObj[0]
+    const loader = new (class extends ObjectLoader {
+      constructor() {
+        super({
+          serverUrl: 'dummy',
+          streamId: 'dummy',
+          undefined,
+          objectId: rootObject.id
+        })
+
+        this.objectId = rootObject.id
+      }
+
+      async getRootObject() {
+        return rootObject
+      }
+
+      async getTotalObjectCount() {
+        return Object.keys(rootObject?.__closure || {}).length
+      }
+
+      async *getObjectIterator() {
+        const t0 = Date.now()
+        let count = 0
+        for await (const { id, obj } of this.getRawObjectIterator(jsonObj)) {
+          this.buffer[id] = obj
+          count += 1
+          yield obj
+        }
+        this.logger(`Loaded ${count} objects in: ${(Date.now() - t0) / 1000}`)
+      }
+
+      async *getRawObjectIterator(data) {
+        yield { id: data[0].id, obj: data[0] }
+
+        const rootObj = data[0]
+        if (!rootObj.__closure) return
+
+        // const childrenIds = Object.keys(rootObj.__closure)
+        //   .filter((id) => !id.includes('blob'))
+        //   .sort((a, b) => rootObj.__closure[a] - rootObj.__closure[b])
+
+        // for (const id of childrenIds) {
+        //   const obj = data.find((value) => value.id === id)
+        //   // Sleep 1 ms
+        //   await new Promise((resolve) => {
+        //     setTimeout(resolve, 1)
+        //   })
+        //   yield { id, obj }
+        // }
+        for (const item of data) {
+          yield { id: item.id, obj: item }
+        }
+      }
+    })()
+    return loader
+  }
+
   async asyncPause() {
     // Don't freeze the UI
     // while ( this.existingAsyncPause ) {
@@ -140,6 +203,11 @@ class ObjectLoader {
     return totalChildrenCount
   }
 
+  async getRootObject() {
+    const rootObjJson = await this.getRawRootObject()
+    return JSON.parse(rootObjJson)
+  }
+
   /**
    * Use this method to receive and construct the object. It will return the full, de-referenced and de-chunked original object.
    * @param {*} onProgress

packages/objectloader/types/index.d.ts

@@ -41,6 +41,8 @@ class ObjectLoader {
     }>
   })
 
+  static createFromJSON(input: string): ObjectLoader
+  async getRootObject(): Promise<SpeckleObject>
   async getTotalObjectCount(): Promise<number>
   async getAndConstructObject(
     onProgress: (e: { stage: ProgressStage; current: number; total: number }) => void

packages/viewer-sandbox/src/Sandbox.ts

@@ -16,6 +16,7 @@ import {
   OutputPass,
   Pipeline,
   SectionTool,
+  SpeckleOfflineLoader,
   SpeckleRenderer,
   SpeckleStandardMaterial,
   TAAPipeline,
@@ -1292,4 +1293,16 @@ export default class Sandbox {
     }
     localStorage.setItem('last-load-url', url)
   }
+
+  public async loadJSON(json: string) {
+    const loader = new SpeckleOfflineLoader(this.viewer.getWorldTree(), json)
+    loader.on(LoaderEvent.LoadCancelled, (resource: string) => {
+      console.warn(`Resource ${resource} loading was canceled`)
+    })
+    loader.on(LoaderEvent.LoadWarning, (arg: { message: string }) => {
+      console.error(`Loader warning: ${arg.message}`)
+    })
+
+    void this.viewer.loadObject(loader, true)
+  }
 }

packages/viewer-sandbox/src/main.ts

@@ -19,10 +19,11 @@ import {
 import { SectionTool } from '@speckle/viewer'
 import { SectionOutlines } from '@speckle/viewer'
 import { ViewModesKeys } from './Extensions/ViewModesKeys'
+import { JSONSpeckleStream } from './JSONSpeckleStream'
 import { BoxSelection } from './Extensions/BoxSelection'
 import { ExtendedSelection } from './Extensions/ExtendedSelection'
 
-const createViewer = async (containerName: string, stream: string) => {
+const createViewer = async (containerName: string, _stream: string) => {
   const container = document.querySelector<HTMLElement>(containerName)
 
   const controlsContainer = document.querySelector<HTMLElement>(
@@ -103,7 +104,8 @@ const createViewer = async (containerName: string, stream: string) => {
   sandbox.makeDiffUI()
   sandbox.makeMeasurementsUI()
 
-  await sandbox.loadUrl(stream)
+  // await sandbox.loadUrl(_stream)
+  await sandbox.loadJSON(JSONSpeckleStream)
 }
 
 const getStream = () => {
@@ -111,7 +113,7 @@ const getStream = () => {
     // prettier-ignore
     // 'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8?c=%5B-7.66134,10.82932,6.41935,-0.07739,-13.88552,1.8697,0,1%5D'
     // Revit sample house (good for bim-like stuff with many display meshes)
-    // 'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8'
+    'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8'
     // 'https://latest.speckle.systems/streams/c1faab5c62/commits/ab1a1ab2b6'
     // 'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8'
     // 'https://latest.speckle.systems/streams/58b5648c4d/commits/60371ecb2d'
@@ -451,13 +453,15 @@ const getStream = () => {
     // Far away house section tool
     // 'https://app.speckle.systems/projects/817c4e8daa/models/f0601ef5f9@80db5ff26a'
 
+    // 'https://app.speckle.systems/projects/00a5c443d6/models/de56edf901'
+    // 'https://latest.speckle.systems/projects/126cd4b7bb/models/49874f87a2ddd370bd2bf46b68c3660d'
     // Perfectly flat
     // 'https://app.speckle.systems/projects/344f803f81/models/5582ab673e'
 
     // 'https://speckle.xyz/streams/27e89d0ad6/commits/5ed4b74252'
 
     // DUI3 Mesh Colors
-    'https://app.speckle.systems/projects/93200a735d/models/cbacd3eaeb@344a397239'
+    // 'https://app.speckle.systems/projects/93200a735d/models/cbacd3eaeb@344a397239'
 
     // Instance toilets
     // 'https://app.speckle.systems/projects/e89b61b65c/models/2a0995f124'

packages/viewer/src/index.ts

@@ -129,6 +129,7 @@ import {
   FilterMaterialOptions,
   FilterMaterialType
 } from './modules/materials/Materials.js'
+import { SpeckleOfflineLoader } from './modules/loaders/Speckle/SpeckleOfflineLoader.js'
 import { AccelerationStructure } from './modules/objects/AccelerationStructure.js'
 import { TopLevelAccelerationStructure } from './modules/objects/TopLevelAccelerationStructure.js'
 import { ViewModeEvent, ViewModeEventPayload } from './modules/extensions/ViewModes.js'
@@ -222,6 +223,7 @@ export {
   FilterMaterial,
   FilterMaterialType,
   FilterMaterialOptions,
+  SpeckleOfflineLoader,
   NOT_INTERSECTED,
   INTERSECTED,
   CONTAINED,

packages/viewer/src/modules/loaders/Speckle/SpeckleLoader.ts

@@ -30,6 +30,29 @@ export class SpeckleLoader extends Loader {
   ) {
     super(resource, resourceData)
     this.tree = targetTree
+    try {
+      this.loader = this.initObjectLoader(
+        resource,
+        authToken,
+        enableCaching,
+        resourceData
+      )
+    } catch (e) {
+      Logger.error(e)
+      return
+    }
+
+    this.converter = new SpeckleConverter(this.loader, this.tree)
+  }
+
+  protected initObjectLoader(
+    resource: string,
+    authToken?: string,
+    enableCaching?: boolean,
+    resourceData?: string | ArrayBuffer
+  ): ObjectLoader {
+    resourceData
+
     let token = undefined
     try {
       token = authToken || (localStorage.getItem('AuthToken') as string | undefined)
@@ -58,7 +81,7 @@ export class SpeckleLoader extends Loader {
     const streamId = segments[2]
     const objectId = segments[4]
 
-    this.loader = new ObjectLoader({
+    return new ObjectLoader({
       serverUrl,
       token,
       streamId,
@@ -66,8 +89,6 @@ export class SpeckleLoader extends Loader {
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
       options: { enableCaching, customLogger: (Logger as any).log }
     })
-
-    this.converter = new SpeckleConverter(this.loader, this.tree)
   }
 
   public async load(): Promise<boolean> {
@@ -78,18 +99,18 @@ export class SpeckleLoader extends Loader {
     let viewerLoads = 0
     let firstObjectPromise = null
 
-    Logger.warn('Downloading object ', this._resource)
+    Logger.warn('Downloading object ', this.resource)
 
     const pause = new AsyncPause()
 
     for await (const obj of this.loader.getObjectIterator()) {
       if (this.isCancelled) {
-        this.emit(LoaderEvent.LoadCancelled, this._resource)
+        this.emit(LoaderEvent.LoadCancelled, this.resource)
         return Promise.resolve(false)
       }
       if (first) {
         firstObjectPromise = this.converter.traverse(
-          this._resource,
+          this.resource,
           obj as SpeckleObject,
           async () => {
             viewerLoads++
@@ -104,7 +125,7 @@ export class SpeckleLoader extends Loader {
       current++
       this.emit(LoaderEvent.LoadProgress, {
         progress: current / (total + 1),
-        id: this._resource
+        id: this.resource
       })
     }
 
@@ -113,15 +134,15 @@ export class SpeckleLoader extends Loader {
     }
 
     Logger.warn(
-      `Finished converting object ${this._resource} in ${
+      `Finished converting object ${this.resource} in ${
         (performance.now() - start) / 1000
       } seconds. Node count: ${this.tree.nodeCount}`
     )
 
     if (viewerLoads === 0) {
-      Logger.warn(`Viewer: no 3d objects found in object ${this._resource}`)
+      Logger.warn(`Viewer: no 3d objects found in object ${this.resource}`)
       this.emit(LoaderEvent.LoadWarning, {
-        message: `No displayable objects found in object ${this._resource}.`
+        message: `No displayable objects found in object ${this.resource}.`
       })
     }
     if (this.isCancelled) {
@@ -134,7 +155,7 @@ export class SpeckleLoader extends Loader {
     const t0 = performance.now()
     const geometryConverter = new SpeckleGeometryConverter()
 
-    const renderTree = this.tree.getRenderTree(this._resource)
+    const renderTree = this.tree.getRenderTree(this.resource)
     if (!renderTree) return Promise.resolve(false)
     const p = renderTree.buildRenderTree(geometryConverter)
 

packages/viewer/src/modules/loaders/Speckle/SpeckleOfflineLoader.ts

@@ -0,0 +1,30 @@
+import ObjectLoader from '@speckle/objectloader'
+import { SpeckleLoader } from './SpeckleLoader.js'
+import { WorldTree } from '../../tree/WorldTree.js'
+import Logger from '../../utils/Logger.js'
+
+export class SpeckleOfflineLoader extends SpeckleLoader {
+  constructor(targetTree: WorldTree, resourceData: string, resourceId?: string) {
+    super(targetTree, resourceId || '', undefined, undefined, resourceData)
+  }
+
+  protected initObjectLoader(
+    _resource: string,
+    _authToken?: string,
+    _enableCaching?: boolean,
+    resourceData?: string | ArrayBuffer
+  ): ObjectLoader {
+    return ObjectLoader.createFromJSON(resourceData as string)
+  }
+
+  public async load(): Promise<boolean> {
+    const rootObject = await this.loader.getRootObject()
+    if (!rootObject && this._resource) {
+      Logger.error('No root id set!')
+      return false
+    }
+    /** If not id is provided, we make one up based on the root object id */
+    this._resource = this._resource || `/json/${rootObject.id as string}`
+    return super.load()
+  }
+}