Alex/#1678 faster diff (#1688)

* Sped up diffing by several orders of magnitude. Also started on a different more fancy approach to diffing involving boolean operations on object maps * Finished with boolean version of diffing. Improved the speed of both by 50% on top of the previous speed improvements * #1690 Completely transparent objects are ignored during picking via a toggle-able flag in renderer
2023-07-26 10:40:47 +03:00
parent 37a0fa4094
commit 56058c04a3
11 changed files with 226 additions and 45 deletions
@@ -982,16 +982,16 @@ export default class Sandbox {
      diffResult = await this.viewer.diff(
        //building
        // 'https://latest.speckle.dev/streams/aea12cab71/objects/bcf37136dea9fe9397cdfd84012f616a',
-        // 'https://latest.speckle.dev/streams/aea12cab71/objects/94af0a6b4eaa318647180f8c230cb867'
+        // 'https://latest.speckle.dev/streams/aea12cab71/objects/94af0a6b4eaa318647180f8c230cb867',
        // cubes
        // 'https://latest.speckle.dev/streams/aea12cab71/objects/d2510c59c203b73473f8bbfe637e0552',
        // 'https://latest.speckle.dev/streams/aea12cab71/objects/1c327da824fdb04629eb48675101d7b7',
        // sketchup
        // 'https://latest.speckle.dev/streams/aea12cab71/objects/06bed1819e6c61d9df7196d424ab1eec',
-        // 'https://latest.speckle.dev/streams/aea12cab71/objects/9026f1d6495789b9eab31b5028c9a8ef'
+        // 'https://latest.speckle.dev/streams/aea12cab71/objects/9026f1d6495789b9eab31b5028c9a8ef',
        //latest
-        'https://latest.speckle.dev/streams/cdbe82b016/objects/c14d1a33fd68323193813ec215737472',
-        'https://latest.speckle.dev/streams/cdbe82b016/objects/16676fc95a9ead877f6a825d9e28cbe8',
+        // 'https://latest.speckle.dev/streams/cdbe82b016/objects/c14d1a33fd68323193813ec215737472',
+        // 'https://latest.speckle.dev/streams/cdbe82b016/objects/16676fc95a9ead877f6a825d9e28cbe8',
        //lines
        // 'https://latest.speckle.dev/streams/92b620fb17/objects/3b42d6ef51d3110b4e33b9f8cdc9f357',
        // 'https://latest.speckle.dev/streams/92b620fb17/objects/774384d431fb34d447d4696abbc4b816',
@@ -1016,6 +1016,12 @@ export default class Sandbox {
        // bug
        // 'https://latest.speckle.dev/streams/0c6ad366c4/objects/03f0a8bf0ed8064865eda87a865c7212',
        // 'https://latest.speckle.dev/streams/0c6ad366c4/objects/33ef6b9b547dc9688eb40157b967eab9',
+        // large
+        'https://speckle.xyz/streams/e6f9156405/objects/650f358d8aac50168d9e9226ef6f5cbc',
+        'https://latest.speckle.dev/streams/92b620fb17/objects/1154ca1d997ac631571db55f84cb703d',
+        // cubes
+        // 'https://latest.speckle.dev/streams/0c6ad366c4/objects/03f0a8bf0ed8064865eda87a865c7212',
+        // 'https://latest.speckle.dev/streams/0c6ad366c4/objects/33ef6b9b547dc9688eb40157b967eab9',

        VisualDiffMode.COLORED,
        localStorage.getItem('AuthTokenLatest') as string
@@ -110,7 +110,7 @@ const getStream = () => {
    // prettier-ignore
    // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8?c=%5B-7.66134,10.82932,6.41935,-0.07739,-13.88552,1.8697,0,1%5D'
    // Revit sample house (good for bim-like stuff with many display meshes)
-    'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8'
+    // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8'
    // 'https://latest.speckle.dev/streams/c1faab5c62/commits/6c6e43e5f3'
    // 'https://latest.speckle.dev/streams/58b5648c4d/commits/60371ecb2d'
    // 'Super' heavy revit shit
@@ -278,11 +278,12 @@ const getStream = () => {
    // 'https://latest.speckle.dev/streams/b68abcbf2e/commits/4e94ecad62'
    // Big ass mafa'
    // 'https://speckle.xyz/streams/88307505eb/objects/a232d760059046b81ff97e6c4530c985'
-    // 'https://latest.speckle.dev/streams/92b620fb17/commits/dfb9ca025d'
+    'https://latest.speckle.dev/streams/92b620fb17/commits/dfb9ca025d'
    // 'Blocks with elements
    // 'https://latest.speckle.dev/streams/e258b0e8db/commits/00e165cc1c'
    // 'https://latest.speckle.dev/streams/e258b0e8db/commits/e48cf53add'
    // 'https://latest.speckle.dev/streams/e258b0e8db/commits/c19577c7d6?c=%5B15.88776,-8.2182,12.17095,18.64059,1.48552,0.6025,0,1%5D'
+    // 'https://speckle.xyz/streams/46caea9b53/commits/71938adcd1'
  )
 }

@@ -61,7 +61,8 @@
    "three": "^0.140.0",
    "three-mesh-bvh": "0.5.17",
    "tree-model": "1.0.7",
-    "troika-three-text": "0.47.2"
+    "troika-three-text": "0.47.2",
+    "underscore": "1.13.6"
  },
  "devDependencies": {
    "@babel/core": "^7.18.2",
@@ -1,3 +1,4 @@
+/* eslint-disable @typescript-eslint/no-unused-vars */
 import { Color, FrontSide } from 'three'
 import { SpeckleTypeAllRenderables } from './converter/GeometryConverter'
 import SpeckleStandardMaterial from './materials/SpeckleStandardMaterial'
@@ -7,6 +8,7 @@ import { GeometryType } from './batching/Batch'
 import SpeckleLineMaterial from './materials/SpeckleLineMaterial'
 import Logger from 'js-logger'
 import { NodeRenderView } from './tree/NodeRenderView'
+import _, { omit } from 'underscore'

 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 type SpeckleObject = Record<string, any>
@@ -184,10 +186,41 @@ export class Differ {
    this.removedMaterialPoint.toneMapped = false
  }

-  public diff(urlA: string, urlB: string): Promise<DiffResult> {
-    const modifiedNew: Array<SpeckleObject> = []
-    const modifiedOld: Array<SpeckleObject> = []
+  private intersection(o1, o2) {
+    const [k1, k2] = [Object.keys(o1), Object.keys(o2)]
+    const [first, next] = k1.length > k2.length ? [k2, o1] : [k1, o2]
+    return first.filter((k) => k in next)
+  }

+  private buildIdMaps(
+    rvs: Array<TreeNode>,
+    idMap: { [id: string]: { node: TreeNode; applicationId: string } },
+    appIdMap: { [id: string]: number }
+  ) {
+    for (let k = 0; k < rvs.length; k++) {
+      const atomicRv = rvs[k]
+      const applicationId = atomicRv.model.raw.applicationId
+        ? atomicRv.model.raw.applicationId
+        : this.tree
+            .getAncestors(atomicRv)
+            .find((value) => value.model.raw.applicationId)?.model.raw.applicationId
+
+      idMap[atomicRv.model.raw.id] = {
+        node: atomicRv,
+        applicationId
+      }
+      if (applicationId) {
+        appIdMap[applicationId] = 1
+      }
+    }
+  }
+
+  public diff(urlA: string, urlB: string): Promise<DiffResult> {
+    return this.diffIterative(urlA, urlB)
+  }
+
+  private diffBoolean(urlA: string, urlB: string): Promise<DiffResult> {
+    const start = performance.now()
    const diffResult: DiffResult = {
      unchanged: [],
      added: [],
@@ -197,8 +230,6 @@ export class Differ {

    const renderTreeA = this.tree.getRenderTree(urlA)
    const renderTreeB = this.tree.getRenderTree(urlB)
-    const rootA = this.tree.findId(urlA)
-    const rootB = this.tree.findId(urlB)
    let rvsA = renderTreeA.getRenderableNodes(...SpeckleTypeAllRenderables)
    let rvsB = renderTreeB.getRenderableNodes(...SpeckleTypeAllRenderables)

@@ -213,18 +244,106 @@ export class Differ {
    rvsA = [...Array.from(new Set(rvsA))]
    rvsB = [...Array.from(new Set(rvsB))]

+    const idMapA = {}
+    const appIdMapA = {}
+    this.buildIdMaps(rvsA, idMapA, appIdMapA)
+
+    const idMapB = {}
+    const appIdMapB = {}
+    this.buildIdMaps(rvsB, idMapB, appIdMapB)
+
+    /** Get the ids which are common between the two maps. This will be objects
+     *  which have not changed
+     */
+    const unchanged: Array<string> = this.intersection(idMapA, idMapB)
+    /** We remove the unchanged objects from B and end up with changed + added */
+    const addedModified = _.omit(idMapB, unchanged)
+    /** We remove the unchanged objects from A and end up with changed + removed */
+    const removedModified = _.omit(idMapA, unchanged)
+    /** We remove the changed objects from B. An object from B is changed if
+     *  it's application ID exists in A
+     */
+    const added = _.omit(addedModified, function (value, key, object) {
+      return value.applicationId && appIdMapA[value.applicationId] !== undefined
+    })
+    /** We remove the changed objects from A. An object from A is changed if
+     *  it's application ID exists in B
+     */
+    const removed = _.omit(removedModified, function (value, key, object) {
+      return value.applicationId && appIdMapB[value.applicationId] !== undefined
+    })
+    /** We remove the removed objects from A, leaving us only changed objects */
+    const modifiedRemoved = _.omit(removedModified, Object.keys(removed))
+    /** We remove the removed objects from B, leaving us only changed objects */
+    const modifiedAdded = _.omit(addedModified, Object.keys(added))
+
+    /** We fill the arrays from here on out */
+    const modifiedOld = Object.values(modifiedRemoved).map(
+      (value: { node: TreeNode }) => value.node
+    )
+    const modifiedNew = Object.values(modifiedAdded).map(
+      (value: { node: TreeNode }) => value.node
+    )
+    diffResult.unchanged.push(...unchanged.map((value) => idMapA[value].node))
+    diffResult.unchanged.push(...unchanged.map((value) => idMapB[value].node))
+    diffResult.removed.push(
+      ...Object.values(removed).map((value: { node: TreeNode }) => value.node)
+    )
+    diffResult.added.push(
+      ...Object.values(added).map((value: { node: TreeNode }) => value.node)
+    )
+
+    modifiedOld.forEach((value, index) => {
+      value
+      diffResult.modified.push([modifiedOld[index], modifiedNew[index]])
+    })
+    console.warn('Boolean Time -> ', performance.now() - start)
+    return Promise.resolve(diffResult)
+  }
+
+  private diffIterative(urlA: string, urlB: string): Promise<DiffResult> {
+    const start = performance.now()
+    const modifiedNew: Array<SpeckleObject> = []
+    const modifiedOld: Array<SpeckleObject> = []
+
+    const diffResult: DiffResult = {
+      unchanged: [],
+      added: [],
+      removed: [],
+      modified: []
+    }
+
+    const renderTreeA = this.tree.getRenderTree(urlA)
+    const renderTreeB = this.tree.getRenderTree(urlB)
+    let rvsA = renderTreeA.getRenderableNodes(...SpeckleTypeAllRenderables)
+    let rvsB = renderTreeB.getRenderableNodes(...SpeckleTypeAllRenderables)
+
+    rvsA = rvsA.map((value) => {
+      return renderTreeA.getAtomicParent(value)
+    })
+
+    rvsB = rvsB.map((value) => {
+      return renderTreeB.getAtomicParent(value)
+    })
+
+    rvsA = [...Array.from(new Set(rvsA))]
+    rvsB = [...Array.from(new Set(rvsB))]
+
+    const idMapA = {}
+    const appIdMapA = {}
+    this.buildIdMaps(rvsA, idMapA, appIdMapA)
+
+    const idMapB = {}
+    const appIdMapB = {}
+    this.buildIdMaps(rvsB, idMapB, appIdMapB)
+
    for (let k = 0; k < rvsB.length; k++) {
-      const res = rootA.first((node: TreeNode) => {
-        return rvsB[k].model.raw.id === node.model.raw.id
-      })
+      const res = idMapA[rvsB[k].model.raw.id]?.node
+
      if (res) {
        diffResult.unchanged.push(res)
      } else {
-        const applicationId = rvsB[k].model.raw.applicationId
-          ? rvsB[k].model.raw.applicationId
-          : this.tree
-              .getAncestors(rvsB[k])
-              .find((value) => value.model.raw.applicationId)
+        const applicationId = idMapB[rvsB[k].model.raw.id].applicationId
        if (!applicationId) {
          Logger.error(
            `No application ID found. Object id:${rvsB[k].model.raw.id} is considered 'added'!`
@@ -232,9 +351,7 @@ export class Differ {
          diffResult.added.push(rvsB[k])
          continue
        }
-        const res2 = rootA.first((node: TreeNode) => {
-          return applicationId === node.model.raw.applicationId
-        })
+        const res2 = appIdMapA[applicationId]
        if (res2) {
          modifiedNew.push(rvsB[k])
        } else {
@@ -242,17 +359,10 @@ export class Differ {
        }
      }
    }
-
    for (let k = 0; k < rvsA.length; k++) {
-      const res = rootB.first((node: TreeNode) => {
-        return rvsA[k].model.raw.id === node.model.raw.id
-      })
+      const res = idMapB[rvsA[k].model.raw.id]?.node
      if (!res) {
-        const applicationId = rvsA[k].model.raw.applicationId
-          ? rvsA[k].model.raw.applicationId
-          : this.tree
-              .getAncestors(rvsA[k])
-              .find((value) => value.model.raw.applicationId)
+        const applicationId = idMapA[rvsA[k].model.raw.id].applicationId
        if (!applicationId) {
          Logger.error(
            `No application ID found. Object id:${rvsA[k].model.raw.id} is considered 'removed'!`
@@ -260,9 +370,7 @@ export class Differ {
          diffResult.removed.push(rvsA[k])
          continue
        }
-        const res2 = rootB.first((node: TreeNode) => {
-          return applicationId === node.model.raw.applicationId
-        })
+        const res2 = appIdMapB[applicationId]
        if (!res2) {
          diffResult.removed.push(rvsA[k])
        } else {
@@ -272,13 +380,11 @@ export class Differ {
        diffResult.unchanged.push(res)
      }
    }
-
    modifiedOld.forEach((value, index) => {
      value
      diffResult.modified.push([modifiedOld[index], modifiedNew[index]])
    })
-
-    console.warn(diffResult)
+    console.warn('Interative Time -> ', performance.now() - start)
    return Promise.resolve(diffResult)
  }

@@ -322,6 +428,7 @@ export class Differ {
      [id: string]: SpeckleStandardMaterial | SpecklePointMaterial | SpeckleLineMaterial
    }
  ) {
+    const start = performance.now()
    switch (mode) {
      case VisualDiffMode.COLORED:
        this._materialGroups = this.getColoredMaterialGroups(
@@ -337,6 +444,7 @@ export class Differ {
      default:
        Logger.error(`Unsupported visual diff mode ${mode}`)
    }
+    console.warn('Material groups -> ', performance.now() - start)
    return this._materialGroups
  }

@@ -77,6 +77,7 @@ export enum ObjectLayers {

 export default class SpeckleRenderer {
  private readonly SHOW_HELPERS = false
+  private readonly IGNORE_ZERO_OPACITY_OBJECTS = true
  public SHOW_BVH = false
  private container: HTMLElement
  private _renderer: WebGLRenderer
@@ -890,13 +891,28 @@ export default class SpeckleRenderer {
    const rvs = []
    const points = []
    for (let k = 0; k < results.length; k++) {
-      let rv = results[k].batchObject?.renderView
-      if (!rv) {
+      const batchObject = results[k].batchObject
+      let rv = null
+      if (batchObject) {
+        rv = batchObject.renderView
+        const material = (results[k].object as SpeckleMesh).getBatchObjectMaterial(
+          results[k].batchObject
+        )
+        if (material.opacity === 0 && this.IGNORE_ZERO_OPACITY_OBJECTS) continue
+      } else {
        rv = this.batcher.getRenderView(
          results[k].object.uuid,
          results[k].faceIndex !== undefined ? results[k].faceIndex : results[k].index
        )
+        if (rv) {
+          const material = this.batcher.getRenderViewMaterial(
+            results[k].object.uuid,
+            results[k].faceIndex !== undefined ? results[k].faceIndex : results[k].index
+          )
+          if (material.opacity === 0 && this.IGNORE_ZERO_OPACITY_OBJECTS) continue
+        }
      }
+
      if (rv) {
        rvs.push(rv)
        points.push(results[k].point)
@@ -31,6 +31,7 @@ export interface Batch {
  resetDrawRanges()
  buildBatch()
  getRenderView(index: number): NodeRenderView
+  getMaterialAtIndex(index: number): Material
  onUpdate(deltaTime: number)
  onRender(renderer: WebGLRenderer)
  purge()
@@ -460,6 +460,15 @@ export default class Batcher {
    return this.batches[batchId].getRenderView(index)
  }

+  public getRenderViewMaterial(batchId: string, index: number) {
+    if (!this.batches[batchId]) {
+      Logger.error('Invalid batch id!')
+      return null
+    }
+
+    return this.batches[batchId].getMaterialAtIndex(index)
+  }
+
  public resetBatchesDrawRanges() {
    for (const k in this.batches) {
      this.batches[k].resetDrawRanges()
@@ -576,7 +585,7 @@ export default class Batcher {
      if (k !== rv.batchId) {
        this.batches[k].setDrawRanges({
          offset: 0,
-          count: Infinity,
+          count: this.batches[k].getCount(),
          material: this.materials.getFilterMaterial(
            this.batches[k].renderViews[0],
            FilterMaterialType.GHOST
@@ -591,7 +600,7 @@ export default class Batcher {
      if (k !== batchId) {
        this.batches[k].setDrawRanges({
          offset: 0,
-          count: Infinity,
+          count: this.batches[k].getCount(),
          material: this.materials.getFilterMaterial(
            this.batches[k].renderViews[0],
            FilterMaterialType.GHOST
@@ -5,6 +5,7 @@ import {
  InstancedInterleavedBuffer,
  InterleavedBufferAttribute,
  Line,
+  Material,
  Object3D,
  Vector4,
  WebGLRenderer
@@ -253,6 +254,11 @@ export default class LineBatch implements Batch {
    }
  }

+  public getMaterialAtIndex(index: number): Material {
+    index
+    return this.batchMaterial
+  }
+
  private makeLineGeometry(position: Float64Array) {
    this.geometry = this.makeLineGeometryTriangle(new Float32Array(position))
    Geometry.updateRTEGeometry(this.geometry, position)
@@ -17,6 +17,7 @@ import {
 } from './Batch'
 import { GeometryConverter } from '../converter/GeometryConverter'
 import { ObjectLayers } from '../SpeckleRenderer'
+import Logger from 'js-logger'

 export default class PointBatch implements Batch {
  public id: string
@@ -353,6 +354,32 @@ export default class PointBatch implements Batch {
    }
  }

+  public getMaterialAtIndex(index: number): Material {
+    for (let k = 0; k < this.renderViews.length; k++) {
+      if (
+        index >= this.renderViews[k].batchStart &&
+        index < this.renderViews[k].batchEnd
+      ) {
+        const rv = this.renderViews[k]
+        const group = this.geometry.groups.find((value) => {
+          return (
+            rv.batchStart >= value.start &&
+            rv.batchStart + rv.batchCount <= value.count + value.start
+          )
+        })
+        if (!Array.isArray(this.mesh.material)) {
+          return this.mesh.material
+        } else {
+          if (!group) {
+            Logger.warn(`Malformed material index!`)
+            return null
+          }
+          return this.mesh.material[group.materialIndex]
+        }
+      }
+    }
+  }
+
  private makePointGeometry(
    position: Float64Array,
    color: Float32Array
@@ -131,9 +131,7 @@ export default class TextBatch implements Batch {
  }

  public getMaterialAtIndex(index: number): Material {
-    index
-    console.warn('Deprecated! Do not call this anymore')
-    return null
+    return this.batchMaterial
  }

  public purge() {
@@ -12881,6 +12881,7 @@ __metadata:
    tree-model: 1.0.7
    troika-three-text: 0.47.2
    typescript: ^4.5.4
+    underscore: 1.13.6
  languageName: unknown
  linkType: soft

@@ -43510,6 +43511,13 @@ __metadata:
  languageName: node
  linkType: hard

+"underscore@npm:1.13.6":
+  version: 1.13.6
+  resolution: "underscore@npm:1.13.6"
+  checksum: d5cedd14a9d0d91dd38c1ce6169e4455bb931f0aaf354108e47bd46d3f2da7464d49b2171a5cf786d61963204a42d01ea1332a903b7342ad428deaafaf70ec36
+  languageName: node
+  linkType: hard
+
 "undici@npm:^5.1.0, undici@npm:^5.12.0, undici@npm:^5.19.1, undici@npm:^5.22.0, undici@npm:^5.8.0":
  version: 5.22.1
  resolution: "undici@npm:5.22.1"