feat: Fix load/unload logic and cache to use objectIds

speckle-powerbi.code-workspace

@@ -16,7 +16,7 @@
   "settings": {
     "powerquery.general.mode": "SDK",
     "powerquery.sdk.defaultQueryFile": "${workspaceFolder}\\src\\powerbi-data-connector\\Speckle.query.pq",
-    "powerquery.sdk.defaultExtension": "${workspaceFolder}\\bin\\${workspaceFolderBasename}.mez",
+    "powerquery.sdk.defaultExtension": "${workspaceFolder}\\src\\powerbi-data-connector\\bin\\Speckle.mez",
     "files.eol": "\n",
     "files.watcherExclude": {
       "**/.git/objects/**": true,

src/powerbi-data-connector/Speckle.query.pq

@@ -1,7 +1,2 @@
 // Use this file to write queries to test your data connector
-let
+let result = Speckle.GetByUrl("https://app.speckle.systems/projects/e2988234fb/models/60b2300470@b1f31a351a") in result
-    result = Speckle.GetByUrl(
-        "https://app.speckle.systems/projects/e2988234fb/models/60b2300470@b1f31a351a,60b2300470"
-    )
-in
-    result

src/powerbi-data-connector/speckle/GetByUrl.pqm

@@ -44,15 +44,13 @@ in
             addSpeckleTypeCol = Table.AddColumn(
                 addObjectIdCol, "speckle_type", each try[data][speckle_type] otherwise null
             ),
+            // TODO: JSON Column must be added here so that any detached objects can be re-attached before doing the json thing. If we just pick the raw result from the API, it will only work in the simplest of objects.
+            addJsonCol = Table.AddColumn(
+                addSpeckleTypeCol, "json", each try Text.FromBinary(Json.FromValue([data])) otherwise null
+            ),
             final = Table.ReorderColumns(
-                addSpeckleTypeCol, {
+                addJsonCol,
-                    "Model URL",
+                {"Model URL", "URL Type", "Version Object ID", "Object ID", "speckle_type", "data", "json"}
-                    "URL Type",
-                    "Version Object ID",
-                    "Object ID",
-                    "speckle_type",
-                    "data"
-                }
             )
         in
             final

src/powerbi-visual/capabilities.json

@@ -20,6 +20,11 @@
       "kind": "Grouping",
       "name": "objectColorBy"
     },
+    {
+      "displayName": "JSON Data",
+      "kind": "Measure",
+      "name": "json"
+    },
     {
       "displayName": "Tooltip Data",
       "kind": "Measure",
@@ -36,16 +41,6 @@
             }
           },
           "select": [
-            {
-              "bind": {
-                "to": "stream"
-              }
-            },
-            {
-              "bind": {
-                "to": "parentObject"
-              }
-            },
             {
               "bind": {
                 "to": "objectColorBy"
@@ -60,6 +55,11 @@
         },
         "values": {
           "select": [
+            {
+              "bind": {
+                "to": "json"
+              }
+            },
             {
               "bind": {
                 "to": "objectData"
@@ -206,6 +206,7 @@
       "essential": true,
       "name": "WebAccess",
       "parameters": [
+        "https://*.speckle.systems",
         "https://speckle.xyz",
         "https://*.speckle.xyz",
         "https://latest.speckle.dev",

src/powerbi-visual/package-lock.json

@@ -6,6 +6,7 @@
   "packages": {
     "": {
       "name": "@specklesystems/powerbi-visual",
+      "version": "2.0.0",
       "license": "MIT",
       "dependencies": {
         "@babel/runtime": "^7.21.5",
@@ -70,8 +71,7 @@
         "webpack-bundle-analyzer": "^4.8.0",
         "webpack-cli": "^5.1.1",
         "webpack-dev-server": "^4.15.0"
-      },
+      }
-      "version": "2.0.0"
     },
     "node_modules/@alloc/quick-lru": {
       "version": "5.2.0",

src/powerbi-visual/src/handlers/viewerHandler.ts

@@ -90,62 +90,74 @@ export default class ViewerHandler {
   }
 
   public async unloadObjects(
-    objects: string[],
+    objectIds: string[],
     signal?: AbortSignal,
     onObjectUnloaded?: (url: string) => void
   ) {
-    for (const url of objects) {
+    console.log('Unloading objects', objectIds)
+    for (const speckleObjectId of objectIds) {
       if (signal?.aborted) return
-      await this.viewer
+      // TODO: Here's where the viewer unloads any objects that have been removed. It first ensures any loading is cancelled.
-        .cancelLoad(url, true)
+      // await this.viewer
-        .catch((e) => console.warn('Viewer Unload error', url, e))
+      //   .cancelLoad(url, true)
-        .finally(() => {
+      //   .catch((e) => console.warn('Viewer error while cancelling load', url, e))
-          if (this.loadedObjectsCache.has(url)) this.loadedObjectsCache.delete(url)
+
-          if (onObjectUnloaded) onObjectUnloaded(url)
+      if (this.loadedObjectsCache.has(speckleObjectId))
-        })
+        this.loadedObjectsCache.delete(speckleObjectId)
+      if (onObjectUnloaded) onObjectUnloaded(speckleObjectId)
     }
   }
 
   public async loadObjectsWithAutoUnload(
-    objectUrls: string[],
+    objects: any[],
     onLoad: (url: string, index: number) => void,
     onError: (url: string, error: Error) => void,
     signal: AbortSignal
   ) {
-    var objectsToUnload = _.difference([...this.loadedObjectsCache], objectUrls)
+    var objectsToUnload = _.difference(
+      [...this.loadedObjectsCache],
+      objects.map((o) => o.id as string)
+    )
     await this.unloadObjects(objectsToUnload, signal)
-    await this.loadObjects(objectUrls, onLoad, onError, signal)
+    await this.loadObjects(objects, onLoad, onError, signal)
   }
 
   public async loadObjects(
-    objectUrls: string[],
+    objectsToLoad: any[],
     onLoad: (url: string, index: number) => void,
     onError: (url: string, error: Error) => void,
     signal: AbortSignal
   ) {
+    console.log('Loading objects', objectsToLoad)
     try {
-      let index = 0
+      //let index = 0
       let promises = []
-      for (const url of objectUrls) {
+      for (const speckleObject of objectsToLoad) {
         signal.throwIfAborted()
-        console.log('Attempting to load', url)
+        console.log('Attempting to load', speckleObject.id, speckleObject)
-        if (!this.loadedObjectsCache.has(url)) {
+
+        if (!this.loadedObjectsCache.has(speckleObject.id)) {
           console.log('Object is not in cache')
-          const promise = this.viewer
+
-            .loadObjectAsync(url, this.config.authToken, false)
+          // TODO: Here's were the viewer loads each object, this used to be done via URL but is now changed to use JSON objects instead (already deserialized)
-            .then(() => onLoad(url, index++))
+
-            .catch((e: Error) => onError(url, e))
+          // const promise = this.viewer
-            .finally(() => {
+          //   .loadObjectAsync(speckleObject, this.config.authToken, false)
-              if (!this.loadedObjectsCache.has(url)) this.loadedObjectsCache.add(url)
+          //   .then(() => onLoad(speckleObject.id, index++))
-            })
+          //   .catch((e: Error) => onError(speckleObject, e))
-          promises.push(promise)
+
+          // promises.push(promise)
+
+          // If batchSize has been reached, wait till all promises resolve before continuing
           if (promises.length == this.config.batchSize) {
-            //this.promises.push(Promise.resolve(this.later(1000)))
             await Promise.all(promises)
             promises = []
           }
+
+          if (!this.loadedObjectsCache.has(speckleObject.id))
+            this.loadedObjectsCache.add(speckleObject.id)
         } else {
-          console.log('Object was already in cache')
+          console.log('Object was already in cache/already loaded')
         }
       }
       await Promise.all(promises)
@@ -174,6 +186,7 @@ export default class ViewerHandler {
       objects: res.objects
     }
   }
+
   public zoom(objectIds?: string[]) {
     this.viewer.zoom(objectIds)
   }
@@ -181,6 +194,7 @@ export default class ViewerHandler {
   public zoomExtents() {
     this.viewer.zoom()
   }
+
   public async unIsolateObjects() {
     if (this.state.isolatedObjects)
       this.state = await this.viewer.unIsolateObjects(this.state.isolatedObjects, 'powerbi', true)

src/powerbi-visual/src/utils/matrixViewUtils.ts

@@ -1,10 +1,6 @@
 import powerbi from 'powerbi-visuals-api'
 import { IViewerTooltip, IViewerTooltipData, SpeckleDataInput } from '../types'
 import { formattingSettings as fs } from 'powerbi-visuals-utils-formattingmodel'
-import {
-  createDataViewWildcardSelector,
-  DataViewWildcardMatchingOption
-} from 'powerbi-visuals-utils-dataviewutils/lib/dataViewWildcard'
 import VisualUpdateOptions = powerbi.extensibility.visual.VisualUpdateOptions
 import { SpeckleVisualSettingsModel } from 'src/settings/visualSettingsModel'
 
@@ -15,23 +11,18 @@ export function validateMatrixView(options: VisualUpdateOptions): {
   const matrixVew = options.dataViews[0].matrix
   if (!matrixVew) throw new Error('Data does not contain a matrix data view')
 
-  let hasStream = false,
+  let hasObject = false,
-    hasParentObject = false,
-    hasObject = false,
     hasColorFilter = false
 
   matrixVew.rows.levels.forEach((level) => {
     level.sources.forEach((source) => {
-      if (!hasStream) hasStream = source.roles['stream'] != undefined
-      if (!hasParentObject) hasParentObject = source.roles['parentObject'] != undefined
       if (!hasObject) hasObject = source.roles['object'] != undefined
       if (!hasColorFilter) hasColorFilter = source.roles['objectColorBy'] != undefined
     })
   })
 
-  if (!hasStream) throw new Error('Missing Stream ID input')
-  if (!hasParentObject) throw new Error('Missing Commit Object ID input')
   if (!hasObject) throw new Error('Missing Object Id input')
+
   return {
     hasColorFilter,
     view: matrixVew
@@ -65,7 +56,7 @@ function processObjectValues(
         displayName: colInfo.displayName,
         value: value.value.toString()
       }
-      objectData.push(propData)
+      if (value.valueSourceIndex) objectData.push(propData)
     })
   return { data: objectData, shouldColor, shouldSelect }
 }
@@ -126,94 +117,50 @@ export function processMatrixView(
   settings: SpeckleVisualSettingsModel,
   onSelectionPair: (objId: string, selectionId: powerbi.extensibility.ISelectionId) => void
 ): SpeckleDataInput {
-  const objectUrlsToLoad = [],
+  const objectJsonToLoad = [],
     objectIds = [],
     selectedIds = [],
     colorByIds = [],
     objectTooltipData = new Map<string, IViewerTooltip>()
 
-  matrixView.rows.root.children.forEach((streamUrlChild) => {
+  // Assume this has color filter
-    const url = streamUrlChild.value
+  matrixView.rows.root?.children?.forEach((colorByGroup) => {
+    const colorByValue = colorByGroup.value
+    console.log('Color by group', colorByValue, colorByGroup)
 
-    streamUrlChild.children?.forEach((parentObjectIdChild) => {
+    const colorGroup = createColorGroup(host, colorByGroup, matrixView)
-      const parentId = parentObjectIdChild.value
-      objectUrlsToLoad.push(`${url}/objects/${parentId}`)
 
-      if (!hasColorFilter) {
+    colorByGroup.children.forEach((objectIdGroup) => {
-        processObjectIdLevel(parentObjectIdChild, host, matrixView).forEach((objRes) => {
+      const uniqueId = objectIdGroup.value
-          objectIds.push(objRes.id)
+      const jsonValue = objectIdGroup.values[0] // TODO: Json value is set as first value in capabilities.json
-          onSelectionPair(objRes.id, objRes.selectionId)
-          if (objRes.shouldSelect) selectedIds.push(objRes.id)
-          if (objRes.color) {
-            let group = colorByIds.find((g) => g.color === objRes.color)
-            if (!group) {
-              group = {
-                color: objRes.color,
-                objectIds: []
-              }
-              colorByIds.push(group)
-            }
-            group.objectIds.push(objRes.id)
-          }
-          objectTooltipData.set(objRes.id, {
-            selectionId: objRes.selectionId,
-            data: objRes.data
-          })
-        })
-      } else {
-        if (previousPalette) host.colorPalette['colorPalette'] = previousPalette
-        parentObjectIdChild.children?.forEach((colorByChild) => {
-          const colorSelectionId = host
-            .createSelectionIdBuilder()
-            .withMatrixNode(colorByChild, matrixView.rows.levels)
-            .createSelectionId()
 
-          const color = host.colorPalette.getColor(colorByChild.value as string)
+      objectJsonToLoad.push(JSON.parse(jsonValue.value.toString()))
-          if (colorByChild.objects) {
+      colorGroup.objectIds.push(uniqueId)
-            console.log(
-              '⚠️COLOR NODE HAS objects',
-              colorByChild.objects,
-              colorByChild.objects.color?.fill
-            )
-          }
 
-          const colorSlice = new fs.ColorPicker({
+      if (jsonValue.highlight) console.log(uniqueId, jsonValue)
-            name: 'selectorFill',
+      var processedObject = processObjectNode(objectIdGroup, host, matrixView)
-            displayName: colorByChild.value.toString(),
+      console.log(processedObject)
-            value: {
-              value: color.value
-            },
-            selector: colorSelectionId.getSelector()
-          })
 
-          const colorGroup = {
+      onSelectionPair(uniqueId.toString(), processedObject.selectionId)
-            color: color.value,
-            slice: colorSlice,
-            objectIds: []
-          }
 
-          processObjectIdLevel(colorByChild, host, matrixView).forEach((objRes) => {
+      if (processedObject.shouldSelect) selectedIds.push(processedObject.id)
-            objectIds.push(objRes.id)
+      if (processedObject.shouldColor) colorGroup.objectIds.push(processedObject.id)
-            onSelectionPair(objRes.id, objRes.selectionId)
+
-            if (objRes.shouldSelect) selectedIds.push(objRes.id)
+      objectTooltipData.set(processedObject.id, {
-            if (objRes.shouldColor) {
+        selectionId: processedObject.selectionId,
-              colorGroup.objectIds.push(objRes.id)
+        data: processedObject.data
-            }
+      })
-            objectTooltipData.set(objRes.id, {
-              selectionId: objRes.selectionId,
-              data: objRes.data
-            })
-          })
-          if (colorGroup.objectIds.length > 0) colorByIds.push(colorGroup)
-        })
-      }
     })
+
+    if (colorGroup.objectIds.length > 0) colorByIds.push(colorGroup)
   })
 
+  // TODO: Code behavior without color filter
+
   previousPalette = host.colorPalette['colorPalette']
 
   return {
-    objectsToLoad: objectUrlsToLoad,
+    objectsToLoad: objectJsonToLoad,
     objectIds,
     selectedIds,
     colorByIds: colorByIds.length > 0 ? colorByIds : null,
@@ -221,3 +168,34 @@ export function processMatrixView(
     view: matrixView
   }
 }
+function createColorGroup(
+  host: powerbi.extensibility.visual.IVisualHost,
+  colorByGroup: powerbi.DataViewMatrixNode,
+  matrixView: powerbi.DataViewMatrix
+) {
+  const colorSelectionId = host
+    .createSelectionIdBuilder()
+    .withMatrixNode(colorByGroup, matrixView.rows.levels)
+    .createSelectionId()
+
+  const color = host.colorPalette.getColor(colorByGroup.value as string)
+  if (colorByGroup.objects) {
+    console.log('⚠️COLOR NODE HAS objects', colorByGroup.objects, colorByGroup.objects.color?.fill)
+  }
+
+  const colorSlice = new fs.ColorPicker({
+    name: 'selectorFill',
+    displayName: colorByGroup.value.toString(),
+    value: {
+      value: color.value
+    },
+    selector: colorSelectionId.getSelector()
+  })
+
+  const colorGroup = {
+    color: color.value,
+    slice: colorSlice,
+    objectIds: []
+  }
+  return colorGroup
+}

src/powerbi-visual/src/visual.ts

@@ -20,11 +20,6 @@ import VisualConstructorOptions = powerbi.extensibility.visual.VisualConstructor
 import VisualUpdateOptions = powerbi.extensibility.visual.VisualUpdateOptions
 import IVisual = powerbi.extensibility.visual.IVisual
 import ITooltipService = powerbi.extensibility.ITooltipService
-import {
-  createDataViewWildcardSelector,
-  DataViewWildcardMatchingOption
-} from 'powerbi-visuals-utils-dataviewutils/lib/dataViewWildcard'
-import { ColorSelectorSettings } from 'src/settings/colorSettings'
 
 // noinspection JSUnusedGlobalSymbols
 export class Visual implements IVisual {