Merge pull request #176 from specklesystems/dogukan/fix-function-registration

fix: helper function registration

.github/workflows/deploy.yml

@@ -1,4 +1,4 @@
-name: build_powerbi
+name: Build and deploy Connector and Visual
 on:
   push:
     branches: ["installer-test/**"]

.github/workflows/pr.yml

@@ -0,0 +1,30 @@
+name: Test Build Connector and Visual
+on: pull_request
+
+jobs:
+  build-connector:
+    runs-on: windows-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup MSBuild
+        uses: microsoft/setup-msbuild@v2
+
+      - name: Build Data Connector
+        working-directory: src/powerbi-data-connector
+        run: |
+          msbuild Speckle.proj /restore /consoleloggerparameters:NoSummary /property:GenerateFullPaths=true
+
+  build-visual:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - run: npm ci
+        working-directory: src/powerbi-visual
+      - run: npm run build
+        working-directory: src/powerbi-visual

src/powerbi-data-connector/Speckle.pq

@@ -75,6 +75,37 @@ shared Speckle.GetWorkspace = Value.ReplaceType(
     type function (url as Uri.Type) as record
 );
 
+shared Speckle.Objects.Properties = Value.ReplaceType(
+    Speckle.LoadFunction("Objects.Properties.pqm"),
+    type function (inputRecord as record, optional filterKeys as list) as record
+);
+
+
+shared Speckle.Utils.ExpandRecord = Value.ReplaceType(
+    Speckle.LoadFunction("Utils.ExpandRecord.pqm"),
+    type function (
+        table as table,
+        columnName as text,
+        optional FieldNames as list,
+        optional UseCombinedNames as logical
+    ) as table
+);
+
+shared Speckle.Objects.Collections = Value.ReplaceType(
+    Speckle.LoadFunction("Objects.Collections.pqm"),
+    type function (inputData as table) as table
+);
+
+shared Speckle.Objects.CompositeStructure = Value.ReplaceType(
+    Speckle.LoadFunction("Objects.CompositeStructure.pqm"),
+    type function (objectRecord as record, optional outputAsList as nullable logical) as any
+);
+
+shared Speckle.Objects.MaterialQuantities = Value.ReplaceType(
+    Speckle.LoadFunction("Objects.MaterialQuantities.pqm"),
+    type function (objectRecord as record, optional outputAsList as logical) as any
+);
+
 [DataSource.Kind = "Speckle", Publish="GetByUrl.Publish"]
 shared Speckle.GetByUrl = Value.ReplaceType(
     Speckle.LoadFunction("GetByUrl.pqm"),

src/powerbi-data-connector/speckle/api/GetStructuredData.pqm

@@ -59,7 +59,7 @@
             )
         ),
        
-        // Function to check if a row should be excluded based on speckle type
+        // function to check if a row should be excluded based on speckle type
         ShouldExcludeRow = (row as record) as logical =>
             let
                 speckleType = Record.FieldOrDefault(row[data], "speckle_type", "")

src/powerbi-data-connector/speckle/api/Objects.Collections.pqm

@@ -0,0 +1,163 @@
+// function for mapping collection names to referenced elements in Speckle data
+(inputData as table) as table =>
+let
+    // Helper function to safely get field value
+    SafeFieldValue = (record as record, fieldName as text) as any =>
+        if Record.HasFields(record, {fieldName}) then 
+            Record.Field(record, fieldName)
+        else 
+            null,
+    
+    // Helper function to safely get nested field value
+    SafeNestedValue = (record as record, path as list) as any =>
+        List.Accumulate(
+            path,
+            record,
+            (current, fieldName) =>
+                if current <> null and Value.Is(current, type record) and Record.HasFields(current, {fieldName}) then
+                    Record.Field(current, fieldName)
+                else
+                    null
+        ),
+    
+    // Step 1: Identify Collection Objects
+    CollectionObjects = Table.SelectRows(
+        inputData, 
+        each 
+            let
+                speckleType = SafeFieldValue(_, "Speckle Type")
+            in
+                speckleType <> null and Text.Contains(speckleType, "Collection")
+    ),
+    
+    // Step 2: Extract Collection Metadata
+    CollectionMetadata = Table.AddColumn(
+        CollectionObjects,
+        "CollectionInfo",
+        each
+            let
+                objectId = SafeFieldValue(_, "Object IDs"),
+                collectionName = SafeNestedValue(_, {"data", "name"}),
+                elements = SafeNestedValue(_, {"data", "elements"})
+            in
+                [
+                    ObjectId = objectId,
+                    CollectionName = if collectionName <> null then collectionName else "Unnamed Collection",
+                    Elements = if elements <> null and Value.Is(elements, type list) then elements else {}
+                ]
+    ),
+    
+    // Step 3: Build Collection Hierarchy Mapping
+    CollectionHierarchy = Table.AddColumn(
+        CollectionMetadata,
+        "CollectionReferences",
+        each
+            let
+                info = [CollectionInfo],
+                collectionName = info[CollectionName],
+                elements = info[Elements]
+            in
+                List.Transform(
+                    elements,
+                    (element) =>
+                        let
+                            referencedId = if Value.Is(element, type record) and Record.HasFields(element, {"referencedId"}) then
+                                element[referencedId]
+                            else
+                                null
+                        in
+                            if referencedId <> null then
+                                [
+                                    ReferencedId = referencedId,
+                                    CollectionName = collectionName,
+                                    ParentCollectionId = info[ObjectId]
+                                ]
+                            else
+                                null
+                )
+    ),
+    
+    // Step 4: Flatten Reference Mapping
+    FlattenedReferences = Table.SelectRows(
+        Table.ExpandListColumn(
+            Table.SelectColumns(CollectionHierarchy, {"CollectionReferences"}),
+            "CollectionReferences"
+        ),
+        each [CollectionReferences] <> null
+    ),
+    
+    ReferenceTable = Table.ExpandRecordColumn(
+        FlattenedReferences,
+        "CollectionReferences",
+        {"ReferencedId", "CollectionName", "ParentCollectionId"},
+        {"ReferencedId", "CollectionName", "ParentCollectionId"}
+    ),
+    
+    // Step 5: Build Hierarchical Collection Paths
+    BuildCollectionPath = (objectId as text, visited as list) as text =>
+        let
+            // Prevent infinite loops
+            _ = if List.Contains(visited, objectId) then
+                error "Circular reference detected in collection hierarchy"
+            else
+                null,
+                
+            newVisited = List.InsertRange(visited, 0, {objectId}),
+            
+            // Find if this object is referenced by any collection
+            parentReferences = Table.SelectRows(ReferenceTable, each [ReferencedId] = objectId),
+            
+            result = if Table.RowCount(parentReferences) = 0 then
+                // No parent collection found
+                ""
+            else
+                let
+                    parentRef = parentReferences{0},
+                    parentCollectionId = parentRef[ParentCollectionId],
+                    currentCollectionName = parentRef[CollectionName],
+                    
+                    // Recursively get parent path
+                    parentPath = @BuildCollectionPath(parentCollectionId, newVisited),
+                    
+                    // Build full path
+                    fullPath = if parentPath = "" then
+                        currentCollectionName
+                    else
+                        parentPath & "::" & currentCollectionName
+                in
+                    fullPath
+        in
+            result,
+    
+    // Step 6: Add Collection Paths to data field
+    FinalData = Table.TransformColumns(
+        inputData,
+        {
+            "data", each 
+                let
+                    currentData = _,
+                    currentRow = Table.SelectRows(inputData, each [data] = currentData){0},
+                    objectId = SafeFieldValue(currentRow, "Object IDs"),
+                    collectionPath = if objectId <> null then
+                        try
+                            BuildCollectionPath(objectId, {})
+                        otherwise
+                            ""
+                    else
+                        "",
+                    
+                    // Add CollectionPath field to the data record, set to null if empty
+                    enhancedData = if Value.Is(currentData, type record) then
+                        Record.AddField(
+                            currentData, 
+                            "collectionPath", 
+                            if collectionPath = "" then null else collectionPath
+                        )
+                    else
+                        currentData
+                in
+                    enhancedData
+        }
+    )
+in
+    FinalData 

src/powerbi-data-connector/speckle/api/Objects.CompositeStructure.pqm

@@ -0,0 +1,18 @@
+(objectRecord as record, optional outputAsList as nullable logical) as any =>
+    let
+        compositeStructure =
+            if Record.HasFields(objectRecord[properties], "Composite Structure") then
+                objectRecord[properties][Composite Structure]
+            else if Record.HasFields(objectRecord[properties], "Parameters") and 
+                    Record.HasFields(objectRecord[properties][Parameters], "Type Parameters") and
+                    Record.HasFields(objectRecord[properties][Parameters][Type Parameters], "Structure") then
+                objectRecord[properties][Parameters][Type Parameters][Structure]
+            else
+                null,
+        result =
+            if outputAsList = true then
+                if compositeStructure <> null then Record.ToList(compositeStructure) else null
+            else
+                compositeStructure
+    in
+        result

src/powerbi-data-connector/speckle/api/Objects.MaterialQuantities.pqm

@@ -0,0 +1,15 @@
+// Helper function to extract [properties][Material Quantities] and optionally output as list
+(objectRecord as record, optional outputAsList as logical) as any =>
+    let
+        // Ensure outputAsList is logical and defaults to false if not provided
+        OutputAsList = if outputAsList = null then false else outputAsList,
+        // Check if 'properties' and 'Material Quantities' exist
+        HasMaterialQuantities = Record.HasFields(objectRecord, {"properties"}) and Record.HasFields(Record.Field(objectRecord, "properties"), {"Material Quantities"}),
+        MaterialQuantities = if HasMaterialQuantities then Record.Field(Record.Field(objectRecord, "properties"), "Material Quantities") else null,
+        Result = if MaterialQuantities = null then null else
+            if OutputAsList then
+                Record.ToList(MaterialQuantities)
+            else
+                MaterialQuantities
+    in
+        Result

src/powerbi-data-connector/speckle/api/Objects.Properties.pqm

@@ -0,0 +1,196 @@
+// function for extracting and flattening properties from Speckle objects
+(inputRecord as record, optional filterKeys as list) as record =>
+let
+    // auto-extract properties if the input has a "properties" field
+    ActualInput = if Record.HasFields(inputRecord, {"properties"}) then 
+        inputRecord[properties] 
+    else 
+        inputRecord,
+    
+    // helper function to check if a key should be included
+    ShouldIncludeKey = (keyName as text) as logical =>
+        if filterKeys = null then 
+            true 
+        else 
+            List.Contains(filterKeys, keyName),
+    
+    // define excluded paths
+    ExcludedPaths = {
+        "Composite Structure",
+        "Material Quantities", 
+        "Parameters.Type Parameters.Structure"
+    },
+    
+    IsExcludedPath = (path as text) as logical =>
+        List.AnyTrue(
+            List.Transform(
+                ExcludedPaths,
+                (excludedPath) => Text.StartsWith(path, excludedPath)
+            )
+        ),
+    
+    // helper function to handle duplicate keys by adding suffixes
+    AddUniqueKey = (existingRecord as record, newKey as text, newValue as any) as record =>
+        let
+            originalKey = newKey,
+            counter = 1,
+            
+            // find a unique key by adding suffix if needed
+            FindUniqueKey = (testKey as text, testCounter as number) as text =>
+                if Record.HasFields(existingRecord, {testKey}) then
+                    @FindUniqueKey(originalKey & "_" & Text.From(testCounter), testCounter + 1)
+                else
+                    testKey,
+            
+            uniqueKey = FindUniqueKey(newKey, counter),
+            result = Record.AddField(existingRecord, uniqueKey, newValue)
+        in
+            result,
+    
+    // enhanced combine function that handles duplicates and filtering
+    SafeRecordCombine = (records as list) as record =>
+        List.Accumulate(
+            records,
+            [],
+            (state, current) => 
+                List.Accumulate(
+                    Record.FieldNames(current),
+                    state,
+                    (innerState, fieldName) => 
+                        if ShouldIncludeKey(fieldName) then
+                            AddUniqueKey(innerState, fieldName, Record.Field(current, fieldName))
+                        else
+                            innerState
+                )
+        ),
+    
+    // helper function to process name-value objects
+    ProcessNameValueObject = (obj as record) as record =>
+        let
+            // Assert that the object has required fields
+            _ = if not (Record.HasFields(obj, {"name"}) and Record.HasFields(obj, {"value"})) then
+                error [
+                    Reason = "Invalid Name-Value Object",
+                    Message = "Object must have both 'name' and 'value' fields"
+                ]
+            else
+                null,
+                
+            BaseName = obj[name],
+            BaseValue = obj[value],
+            
+            // only extract name and value if it should be included
+            Result = if ShouldIncludeKey(BaseName) then
+                Record.FromList({BaseValue}, {BaseName})
+            else
+                []
+        in
+            Result,
+    
+    // helper function to process direct key-value objects
+    ProcessDirectKeyValueObject = (obj as record) as record =>
+        let
+            // assert that input is a record
+            _ = if not Value.Is(obj, type record) then
+                error [
+                    Reason = "Invalid Input Type",
+                    Message = "Expected record for direct key-value processing"
+                ]
+            else
+                null,
+                
+            // extract all primitive key-value pairs
+            PrimitiveFields = List.Transform(
+                Record.FieldNames(obj),
+                (fieldName) => 
+                    let
+                        fieldValue = Record.Field(obj, fieldName)
+                    in
+                        if not Type.Is(Value.Type(fieldValue), type record) and ShouldIncludeKey(fieldName) then
+                            Record.FromList({fieldValue}, {fieldName})
+                        else
+                            []
+            ),
+            
+            // filter out empty records and combine using safe combine
+            FilteredFields = List.Select(PrimitiveFields, (rec) => Record.FieldCount(rec) > 0),
+            Result = SafeRecordCombine(FilteredFields)
+        in
+            Result,
+    
+    // helper functions for type checking
+    HasDirectKeyValuePattern = (obj as record) as logical =>
+        List.AllTrue(
+            List.Transform(
+                Record.FieldValues(obj),
+                (value) => not Type.Is(Value.Type(value), type record)
+            )
+        ),
+    
+    IsNestedContainer = (obj as record) as logical =>
+        List.AnyTrue(
+            List.Transform(
+                Record.FieldValues(obj),
+                (value) => Type.Is(Value.Type(value), type record)
+            )
+        ),
+    
+    // main processing function with path tracking
+    ProcessField = (fieldName as text, fieldValue as any, currentPath as text) as record =>
+        let
+            fieldPath = if currentPath = "" then fieldName else currentPath & "." & fieldName
+        in
+            if IsExcludedPath(fieldPath) then
+                []
+            else if Type.Is(Value.Type(fieldValue), type record) then
+                if Record.HasFields(fieldValue, {"name", "value"}) then
+                    ProcessNameValueObject(fieldValue)
+                else if HasDirectKeyValuePattern(fieldValue) then
+                    ProcessDirectKeyValueObject(fieldValue)
+                else if IsNestedContainer(fieldValue) then
+                    // recursive call for nested containers
+                    @ProcessRecord(fieldValue, fieldPath)
+                else
+                    // unknown record type, skip
+                    []
+            else
+                // primitive value - check if should be included
+                if ShouldIncludeKey(fieldName) then
+                    Record.FromList({fieldValue}, {fieldName})
+                else
+                    [],
+    
+    // process entire record recursively
+    ProcessRecord = (record as record, currentPath as text) as record =>
+        let
+            // assert that input is a record
+            _ = if not Value.Is(record, type record) then
+                error [
+                    Reason = "Invalid Record Type",
+                    Message = "Expected record for processing, but received: " & Text.From(Value.Type(record))
+                ]
+            else
+                null,
+                
+            ProcessedFields = List.Transform(
+                Record.FieldNames(record),
+                (fieldName) => ProcessField(fieldName, Record.Field(record, fieldName), currentPath)
+            ),
+            FilteredFields = List.Select(ProcessedFields, (rec) => Record.FieldCount(rec) > 0),
+            CombinedRecord = SafeRecordCombine(FilteredFields)
+        in
+            CombinedRecord,
+    
+    // assert that input is a record before processing
+    _ = if not Value.Is(ActualInput, type record) then
+        error [
+            Reason = "Invalid Input Type",
+            Message = "Input must be a record, but received: " & Text.From(Value.Type(ActualInput))
+        ]
+    else
+        null,
+    
+    // start processing from root
+    Result = ProcessRecord(ActualInput, "")
+in
+    Result 

src/powerbi-data-connector/speckle/api/SendToServer.pqm

@@ -1,5 +1,5 @@
 (url as text) as list =>
-    let
+    try let
         // Import required functions
         GetModel = Extension.LoadFunction("GetModel.pqm"),
         Parser = Extension.LoadFunction("Parser.pqm"),
@@ -70,4 +70,10 @@
         JsonResponse = Json.Document(Response)
 
     in
-        JsonResponse
+        JsonResponse
+    otherwise
+        error [
+            Reason = "Desktop Service Not Available", 
+            Message = "Cannot connect to Speckle Desktop Service. Please ensure the Desktop Service is running and try again.",
+            Detail = "The Speckle Desktop Service must be running to load data from Speckle. Please start the Desktop Service application and refresh your data connection."
+        ]

src/powerbi-data-connector/speckle/api/Utils.ExpandRecord.pqm

@@ -0,0 +1,31 @@
+// Expands a record column in a table, adding new columns for each field in the record.
+// If UseCombinedNames is true, columns are named as ColumnName.FieldName, otherwise just FieldName.
+// If FieldNames is provided (list), only those fields are expanded.
+(table as table, columnName as text, optional FieldNames as list, optional UseCombinedNames as logical) as table =>
+let
+    useCombined = if UseCombinedNames = null then false else UseCombinedNames,
+    // Determine which field names to expand
+    allFieldNames = if FieldNames <> null then FieldNames else List.Distinct(
+        List.Combine(
+            List.Transform(
+                Table.Column(table, columnName),
+                each if _ is record then Record.FieldNames(_) else {}
+            )
+        )
+    ),
+    // Add each field as a new column
+    addColumns = List.Accumulate(
+        allFieldNames,
+        table,
+        (state, field) =>
+            Table.AddColumn(
+                state,
+                if useCombined then columnName & "." & field else field,
+                (row) =>
+                    if Record.HasFields(row, columnName) and Record.Field(row, columnName) is record and Record.HasFields(Record.Field(row, columnName), field)
+                    then Record.Field(Record.Field(row, columnName), field)
+                    else null
+            )
+    )
+in
+    addColumns 

src/powerbi-visual/src/plugins/viewer.ts

@@ -241,7 +241,8 @@ export class ViewerHandler {
 
     Tracker.dataLoaded({
       sourceHostApp: store.receiveInfo.sourceApplication,
-      workspace_id: store.receiveInfo.workspaceId
+      workspace_id: store.receiveInfo.workspaceId,
+      core_version: store.receiveInfo.version
     })
     if (store.cameraPosition) {
       const position = new Vector3(

src/powerbi-visual/src/utils/matrixViewUtils.ts

@@ -296,9 +296,14 @@ async function fetchStreamedDataForModel(
     const endObjectCleanup = performance.now()
     console.log(`Objects cleaned up in: ${(endObjectCleanup - startObjectCleanup) / 1000} s`)
 
-    const sizeInBytes = new TextEncoder().encode(JSON.stringify(objects)).length
-    const sizeInMB = sizeInBytes / (1024 * 1024)
-    console.log(`Size of objects: ${sizeInMB} MB`)
+    try {
+      const sizeInBytes = new TextEncoder().encode(JSON.stringify(objects)).length
+      const sizeInMB = sizeInBytes / (1024 * 1024)
+      console.log(`Size of objects: ${sizeInMB} MB`)
+    } catch (error) {
+      console.log("Can't calculate the size of the model")
+      console.log(error)
+    }
 
     return objects
   } catch (error) {

src/powerbi-visual/src/utils/mixpanel.ts

@@ -31,7 +31,8 @@ export class Tracker {
         // eslint-disable-next-line camelcase
         server_id: hashedServer,
         email: receiveInfo.userEmail,
-        isAnonymous: receiveInfo.userEmail === ''
+        isAnonymous: receiveInfo.userEmail === '',
+        core_version: receiveInfo.version
       }
     }