Bump viewer for proxy fix (#210)

Co-authored-by: Oğuzhan Koral <45078678+oguzhankoral@users.noreply.github.com>

src/powerbi-data-connector/Speckle.pq

@@ -150,7 +150,7 @@ shared Speckle.GetWorkspace = Value.ReplaceType(
 
 shared Speckle.Objects.Properties = Value.ReplaceType(
     Speckle.LoadFunction("Objects.Properties.pqm"),
-    type function (inputRecord as record, optional filterKeys as list) as record
+    type function (inputRecord as any, optional filterKeys as list, optional parentPath as text, optional existingFields as list) as record
 );
 
 
@@ -179,19 +179,19 @@ shared Speckle.Objects.MaterialQuantities = Value.ReplaceType(
     type function (objectRecord as record, optional outputAsList as logical) as any
 );
 
-shared Speckle.MarkReceived = Value.ReplaceType(
-    Speckle.LoadFunction("MarkReceived.pqm"),
-    type function (server as text, projectId as text, versionId as text, sourceApplication as text) as logical
-);
-
 shared Speckle.Models.Federate = Value.ReplaceType(
     Speckle.LoadFunction("Models.Federate.pqm"),
     type function (tables as list, optional excludeData as logical) as table
 );
 
+shared Speckle.Models.MaterialQuantities = Value.ReplaceType(
+    Speckle.LoadFunction("Models.MaterialQuantities.pqm"),
+    type function (inputTable as table, optional addPrefix as logical) as table
+);
+
 [DataSource.Kind = "Speckle", Publish="GetByUrl.Publish"]
 shared Speckle.GetByUrl = Value.ReplaceType(
-    (url as text) => Speckle.LoadFunction("GetByUrl.pqm")(url, Speckle.MarkReceived),
+    Speckle.LoadFunction("GetByUrl.pqm"),
     type function (
         url as (
             Uri.Type meta [
@@ -199,6 +199,13 @@ shared Speckle.GetByUrl = Value.ReplaceType(
                 Documentation.FieldDescription = "The URL of a model in a Speckle server project. You can copy it directly from your browser.",
                 Documentation.SampleValues = {"https://app.speckle.systems/projects/7902de1f57/models/7f890a65df"}
             ]
+        ),
+        optional ExpandProperties as (
+            type logical meta [
+                Documentation.FieldCaption = "Expand Properties (may slow query)",
+                Documentation.FieldDescription = "Expand the properties column into individual columns for easier analysis. When checked, each property from the 'properties' record column will have its own column. This can slow down the query if you have a lot of properties.",
+                Documentation.AllowedValues = {true, false}
+            ]
         )
     ) as table meta [
         Documentation.Name = "Speckle - Get Data by URL",
@@ -247,7 +254,7 @@ GetByUrl.Icons = [
 Speckle = [
     // This is used when running the connector on an on-premises data gateway
     TestConnection = (path) => {"Speckle.GetUser", path},
-    // Authentication strategy
+    // Authentication strategy - OAuth only
     Authentication = [
         OAuth = [
             Label = "Speckle Account",
@@ -357,13 +364,6 @@ Speckle = [
                         ]
                 in
                     result
-        ],
-        Key = [
-            KeyLabel = "Personal Access Token",
-            Label = "Private Project"
-        ],
-        Implicit = [
-            Label = "Public Project"
         ]
     ],
     Label = "Speckle"

src/powerbi-data-connector/speckle/GetByUrl.pqm

@@ -1,5 +1,8 @@
-(url as text, MarkReceivedFunc as function) as table =>
+(url as text, optional ExpandProperties as logical) as table =>
     let
+        // set default value for ExpandProperties
+        shouldExpandProperties = if ExpandProperties = null then false else ExpandProperties,
+
         // import required functions
         GetStructuredData = Extension.LoadFunction("GetStructuredData.pqm"),
         SendToServer = Extension.LoadFunction("SendToServer.pqm"),
@@ -73,10 +76,19 @@
                     combinedData = Table.Combine(allTables),
                     
                     // replace the "Version Object ID" column with the combined root IDs
-                    finalData = Table.TransformColumns(
-                        combinedData, 
+                    transformedData = Table.TransformColumns(
+                        combinedData,
                         {"Version Object ID", each combinedRootIds}
-                    )
+                    ),
+
+                    // expand properties column if requested and if it exists
+                    finalData = if shouldExpandProperties and Table.HasColumns(transformedData, {"properties"}) then
+                        try
+                            Speckle.Utils.ExpandRecord(transformedData, "properties")
+                        otherwise
+                            transformedData  // fallback to original data if expansion fails
+                    else
+                        transformedData
                 in
                     finalData
             else
@@ -85,22 +97,22 @@
                     // get model name
                     modelInfo = GetModel(url),
                     modelName = modelInfo[modelName],
-                    
+
                     // get structured data
                     structuredData = GetStructuredData(url),
-    
-                    // Mark the version as received
-                    markReceived = MarkReceivedFunc(
-                        parsedUrl[baseUrl],
-                        parsedUrl[projectId],
-                        modelInfo[versionId],
-                        "PowerBI"
-                    ),
 
+                    // rename column based on send status
                     newColumnName = "Version Object ID",
-                    renamedTable = Table.RenameColumns(structuredData, {{"Version Object ID", newColumnName}}),
+                    renamedData = Table.RenameColumns(structuredData, {{"Version Object ID", newColumnName}}),
 
-                    result = if markReceived then renamedTable else renamedTable
+                    // expand properties column if requested and if it exists
+                    result = if shouldExpandProperties and Table.HasColumns(renamedData, {"properties"}) then
+                        try
+                            Speckle.Utils.ExpandRecord(renamedData, "properties")
+                        otherwise
+                            renamedData  // fallback to original data if expansion fails
+                    else
+                        renamedData
                 in
                     result
         else
@@ -130,11 +142,14 @@
                 // get structured data
                 structuredData = GetStructuredData(singleModelUrl),
                 
-                // add the model name as context
+                // add the model name as context - with version id if exists
                 result = Table.AddColumn(
-                    structuredData, 
-                    "Source Model", 
-                    each modelName,
+                    structuredData,
+                    "Source Model",
+                    each if versionId <> null then
+                        Text.Combine({modelName, "-", versionId})
+                    else
+                        modelName,
                     type text
                 )
             in

src/powerbi-data-connector/speckle/api/Api.Fetch.pqm

@@ -1,7 +1,7 @@
 (server as text, optional query as text, optional variables as record) as record =>
     let
         // Enhanced credential retrieval with OAuth2 support
-        apiKey = try Extension.CurrentCredential()[Key] otherwise try Extension.CurrentCredential()[access_token] otherwise null,
+        apiKey = try Extension.CurrentCredential()[access_token] otherwise null,
         
         defaultQuery = "query {
                 activeUser {

src/powerbi-data-connector/speckle/api/GetModel.pqm

@@ -29,7 +29,7 @@
         versionId = parsedUrl[versionId],
         
         // get API key if available
-        apiKey = try Extension.CurrentCredential()[Key] otherwise null,
+        apiKey = try Extension.CurrentCredential()[access_token] otherwise null,
         
         // graphql query to get model info including root object id
         // includes specific version if provided

src/powerbi-data-connector/speckle/api/GetUser.pqm

@@ -26,7 +26,7 @@ in
             parsedUrl = Parser(url),
             server = parsedUrl[baseUrl],
             
-            apiKey = try Extension.CurrentCredential()[Key] otherwise try Extension.CurrentCredential()[access_token] otherwise "",
+            apiKey = try Extension.CurrentCredential()[access_token] otherwise "",
             
             query = "query {
                     activeUser {

src/powerbi-data-connector/speckle/api/MarkReceived.pqm

@@ -1,39 +0,0 @@
-(server as text, projectId as text, versionId as text, sourceApplication as text) as logical =>
-    let
-        ApiFetch = Extension.LoadFunction("Api.Fetch.pqm"),
-        
-        Extension.LoadFunction = (fileName as text) =>
-            let
-                binary = Extension.Contents(fileName),
-                asText = Text.FromBinary(binary)
-            in
-                try
-                    Expression.Evaluate(asText, #shared)
-                catch (e) =>
-                    error
-                        [
-                            Reason = "Extension.LoadFunction Failure",
-                            Message.Format = "Loading '#{0}' failed - '#{1}': '#{2}'",
-                            Message.Parameters = {fileName, e[Reason], e[Message]},
-                            Detail = [File = fileName, Error = e]
-                        ],
-        
-        query = "
-            mutation MarkReceived($input: MarkReceivedVersionInput!) {
-              versionMutations {
-                markReceived(input: $input)
-              }
-            }
-        ",
-        
-        variables = [
-            input = [
-                versionId = versionId,
-                projectId = projectId,
-                sourceApplication = sourceApplication
-            ]
-        ],
-        
-        result = ApiFetch(server, query, variables)
-    in
-        result[versionMutations][markReceived] 

src/powerbi-data-connector/speckle/api/Models.MaterialQuantities.pqm

@@ -0,0 +1,22 @@
+// function for transforming a table to extract and expand Material Quantities data
+(inputTable as table, optional addPrefix as logical) as table =>
+let
+    // Default addPrefix to false if not provided
+    UsePrefix = if addPrefix = null then false else addPrefix,
+    
+    // Add mq column using existing MaterialQuantities function with list output
+    AddedMQ = Table.AddColumn(inputTable, "mq", each Speckle.Objects.MaterialQuantities([data], true)),
+    
+    // Expand the mq list column
+    ExpandMQ = Table.ExpandListColumn(AddedMQ, "mq"),
+    
+    // Add MQProperties column using Properties function with error handling
+    AddedMQProperties = Table.AddColumn(ExpandMQ, "MQ", each try Speckle.Objects.Properties([mq]) otherwise null),
+    
+    // Expand the MQProperties record using Utils.ExpandRecord
+    ExpandMQProperties = Speckle.Utils.ExpandRecord(AddedMQProperties, "MQ", null, UsePrefix),
+    
+    // Remove the temporary mq and MQProperties columns
+    FinalTable = Table.RemoveColumns(ExpandMQProperties, {"mq", "MQ"}, MissingField.Ignore)
+in
+    FinalTable

src/powerbi-data-connector/speckle/api/Objects.Properties.pqm

@@ -1,196 +1,257 @@
 // function for extracting and flattening properties from Speckle objects
-(inputRecord as record, optional filterKeys as list) as record =>
+(inputRecord as any, optional filterKeys as list, optional parentPath as text, optional existingFields as list) as record =>
 let
-    // auto-extract properties if the input has a "properties" field
-    ActualInput = if Record.HasFields(inputRecord, {"properties"}) then 
-        inputRecord[properties] 
-    else 
-        inputRecord,
-    
-    // helper function to check if a key should be included
-    ShouldIncludeKey = (keyName as text) as logical =>
-        if filterKeys = null then 
-            true 
-        else 
-            List.Contains(filterKeys, keyName),
-    
-    // define excluded paths
+    // Define excluded paths
     ExcludedPaths = {
         "Composite Structure",
         "Material Quantities", 
         "Parameters.Type Parameters.Structure"
     },
     
-    IsExcludedPath = (path as text) as logical =>
-        List.AnyTrue(
-            List.Transform(
-                ExcludedPaths,
-                (excludedPath) => Text.StartsWith(path, excludedPath)
-            )
-        ),
+    // Helper function to check if a path should be excluded
+    IsPathExcluded = (currentPath as text) as logical =>
+        List.AnyTrue(List.Transform(ExcludedPaths, each Text.Contains(currentPath, _))),
     
-    // helper function to handle duplicate keys by adding suffixes
-    AddUniqueKey = (existingRecord as record, newKey as text, newValue as any) as record =>
+    // Helper function to resolve naming conflicts
+    ResolveFieldName = (fieldName as text, parentPathParam as nullable text, existingFieldsParam as nullable list) as text =>
         let
-            originalKey = newKey,
-            counter = 1,
+            // Ensure we have valid inputs
+            parentPath = if parentPathParam = null then "" else parentPathParam,
+            existingFields = if existingFieldsParam = null then {} else existingFieldsParam,
             
-            // find a unique key by adding suffix if needed
-            FindUniqueKey = (testKey as text, testCounter as number) as text =>
-                if Record.HasFields(existingRecord, {testKey}) then
-                    @FindUniqueKey(originalKey & "_" & Text.From(testCounter), testCounter + 1)
-                else
-                    testKey,
+            // Try original field name first
+            candidateName = fieldName,
             
-            uniqueKey = FindUniqueKey(newKey, counter),
-            result = Record.AddField(existingRecord, uniqueKey, newValue)
+            // If no conflict, return original name
+            finalName = if not List.Contains(existingFields, candidateName) then
+                candidateName
+            else if parentPath = "" then 
+                fieldName  // No parent path available, keep original
+            else
+                let
+                    // Split parent path and try adding parents one by one
+                    pathParts = Text.Split(parentPath, "."),
+                    reversedParts = List.Reverse(pathParts),  // Start with immediate parent
+                    
+                    // Use iteration instead of recursion
+                    ResolveWithIteration = () =>
+                        let
+                            // Generate all possible candidates
+                            candidates = List.Generate(
+                                () => [depth = 1, candidate = fieldName & "." & List.First(reversedParts)],
+                                each [depth] <= List.Count(reversedParts),
+                                each [
+                                    depth = [depth] + 1,
+                                    candidate = fieldName & "." & Text.Combine(List.FirstN(reversedParts, [depth]), ".")
+                                ],
+                                each [candidate]
+                            ),
+                            
+                            // Find first non-conflicting candidate
+                            firstNonConflicting = List.First(
+                                List.Select(candidates, each not List.Contains(existingFields, _)),
+                                // If all conflict, use full path
+                                fieldName & "." & Text.Combine(reversedParts, ".")
+                            )
+                        in
+                            firstNonConflicting,
+                    
+                    resolvedName = ResolveWithIteration()
+                in
+                    resolvedName
+        in
+            finalName,
+    
+    // Create the main flattening function with self-reference capability
+    FlattenRecordImpl = (
+        flattenFn as function,
+        inputRecord as any, 
+        filterKeys as nullable list, 
+        parentPathParam as nullable text, 
+        existingFieldsParam as nullable list
+    ) as record =>
+        let
+            // Ensure non-null values for internal use
+            currentParentPath = if parentPathParam = null then "" else parentPathParam,
+            currentExistingFields = if existingFieldsParam = null then {} else existingFieldsParam,
+            currentfilterKeys = filterKeys,
+            
+            // Check if record has "properties" field and use it instead of the root record
+            recordToProcess = if inputRecord = null then 
+                null
+            else if Value.Is(inputRecord, type record) and Record.HasFields(inputRecord, {"properties"}) then
+                Record.Field(inputRecord, "properties")
+            else
+                inputRecord,
+            
+            // Helper function to check if a field should be included
+            ShouldIncludeField = (fieldName as text) as logical =>
+                if currentfilterKeys = null then true
+                else List.Contains(currentfilterKeys, fieldName),
+            
+            // Handle different input types
+            result = if recordToProcess = null then
+                []
+            else if Value.Is(recordToProcess, type record) then
+                let
+                    fieldNames = Record.FieldNames(recordToProcess),
+                    
+                    // Process each field
+                    processedFields = List.Accumulate(
+                        fieldNames,
+                        [FlattenedRecord = [], ExistingFieldsList = currentExistingFields],
+                        (state, fieldName) =>
+                            let
+                                fieldValue = Record.Field(recordToProcess, fieldName),
+                                newPath = if currentParentPath = "" then fieldName else currentParentPath & "." & fieldName,
+                                
+                                // Skip if path is excluded
+                                shouldProcess = not IsPathExcluded(newPath),
+                                
+                                processResult = if not shouldProcess then
+                                    state
+                                else
+                                    let
+                                        // Check if this is a name/value record
+                                        hasNameValue = Value.Is(fieldValue, type record) and 
+                                                     Record.HasFields(fieldValue, {"name", "value"}),
+                                        
+                                        finalResult = if hasNameValue then
+                                            let
+                                                nameField = Record.Field(fieldValue, "name"),
+                                                valueField = Record.Field(fieldValue, "value"),
+                                                // Check if this name field should be included
+                                                shouldInclude = if nameField = null then false else ShouldIncludeField(nameField),
+                                                result = if shouldInclude and nameField <> null then
+                                                    let
+                                                        resolvedName = ResolveFieldName(nameField, currentParentPath, state[ExistingFieldsList]),
+                                                        newRecord = Record.AddField(state[FlattenedRecord], resolvedName, valueField),
+                                                        newFieldsList = state[ExistingFieldsList] & {resolvedName}
+                                                    in
+                                                        [FlattenedRecord = newRecord, ExistingFieldsList = newFieldsList]
+                                                else
+                                                    state
+                                            in
+                                                result
+                                        else if fieldValue = null then
+                                            let
+                                                shouldInclude = ShouldIncludeField(fieldName),
+                                                result = if shouldInclude then
+                                                    let
+                                                        resolvedName = ResolveFieldName(fieldName, currentParentPath, state[ExistingFieldsList]),
+                                                        newRecord = Record.AddField(state[FlattenedRecord], resolvedName, null),
+                                                        newFieldsList = state[ExistingFieldsList] & {resolvedName}
+                                                    in
+                                                        [FlattenedRecord = newRecord, ExistingFieldsList = newFieldsList]
+                                                else
+                                                    state
+                                            in
+                                                result
+                                        else if Value.Is(fieldValue, type record) then
+                                            let
+                                                // Skip empty records
+                                                fieldCount = Record.FieldCount(fieldValue),
+                                                recursiveResult = if fieldCount = 0 then
+                                                    state
+                                                else
+                                                    let
+                                                        // Call the function through the passed reference
+                                                        // IMPORTANT: Pass the current state's existing fields list
+                                                        flattened = flattenFn(flattenFn, fieldValue, currentfilterKeys, newPath, state[ExistingFieldsList]),
+                                                        // Get all field names from the flattened result
+                                                        flattenedFieldNames = Record.FieldNames(flattened),
+                                                        // Merge the flattened record with the current state
+                                                        combinedRecord = flattened & state[FlattenedRecord],
+                                                        // Update the existing fields list with ALL fields from both records
+                                                        allFieldNames = List.Distinct(state[ExistingFieldsList] & flattenedFieldNames)
+                                                    in
+                                                        [FlattenedRecord = combinedRecord, ExistingFieldsList = allFieldNames]
+                                            in
+                                                recursiveResult
+                                        else if Value.Is(fieldValue, type list) then
+                                            let
+                                                listLength = List.Count(fieldValue),
+                                                // Skip empty lists
+                                                listResult = if listLength = 0 then
+                                                    state
+                                                else
+                                                    List.Accumulate(
+                                                        List.Positions(fieldValue),
+                                                        state,
+                                                        (listState, index) =>
+                                                            let
+                                                                listItem = fieldValue{index},
+                                                                indexSuffix = Text.From(index + 1), // 1-based indexing
+                                                                listFieldName = fieldName & "." & indexSuffix,
+                                                                listPath = if currentParentPath = "" then listFieldName else currentParentPath & "." & listFieldName,
+                                                                
+                                                                itemResult = if Value.Is(listItem, type record) then
+                                                                    let
+                                                                        itemFieldCount = Record.FieldCount(listItem),
+                                                                        itemFlattened = if itemFieldCount = 0 then
+                                                                            listState
+                                                                        else
+                                                                            let
+                                                                                // Call the function through the passed reference
+                                                                                flattened = flattenFn(flattenFn, listItem, currentfilterKeys, listPath, listState[ExistingFieldsList]),
+                                                                                // Get all field names from the flattened result
+                                                                                flattenedFieldNames = Record.FieldNames(flattened),
+                                                                                // Merge the flattened record with the current state
+                                                                                combinedRecord = flattened & listState[FlattenedRecord],
+                                                                                // Update the existing fields list with ALL fields
+                                                                                allFieldNames = List.Distinct(listState[ExistingFieldsList] & flattenedFieldNames)
+                                                                            in
+                                                                                [FlattenedRecord = combinedRecord, ExistingFieldsList = allFieldNames]
+                                                                    in
+                                                                        itemFlattened
+                                                                else
+                                                                    let
+                                                                        shouldInclude = ShouldIncludeField(listFieldName),
+                                                                        result = if shouldInclude then
+                                                                            let
+                                                                                resolvedName = ResolveFieldName(listFieldName, currentParentPath, listState[ExistingFieldsList]),
+                                                                                newRecord = Record.AddField(listState[FlattenedRecord], resolvedName, listItem),
+                                                                                newFieldsList = listState[ExistingFieldsList] & {resolvedName}
+                                                                            in
+                                                                                [FlattenedRecord = newRecord, ExistingFieldsList = newFieldsList]
+                                                                        else
+                                                                            listState
+                                                                    in
+                                                                        result
+                                                            in
+                                                                itemResult
+                                                    )
+                                            in
+                                                listResult
+                                        else
+                                            // Handle primitive values
+                                            let
+                                                shouldInclude = ShouldIncludeField(fieldName),
+                                                result = if shouldInclude then
+                                                    let
+                                                        resolvedName = ResolveFieldName(fieldName, currentParentPath, state[ExistingFieldsList]),
+                                                        newRecord = Record.AddField(state[FlattenedRecord], resolvedName, fieldValue),
+                                                        newFieldsList = state[ExistingFieldsList] & {resolvedName}
+                                                    in
+                                                        [FlattenedRecord = newRecord, ExistingFieldsList = newFieldsList]
+                                                else
+                                                    state
+                                            in
+                                                result
+                                    in
+                                        finalResult
+                            in
+                                processResult
+                    )
+                in
+                    processedFields[FlattenedRecord]
+            else
+                // If input is not a record, return it as is in a record wrapper
+                [Value = recordToProcess]
         in
             result,
     
-    // enhanced combine function that handles duplicates and filtering
-    SafeRecordCombine = (records as list) as record =>
-        List.Accumulate(
-            records,
-            [],
-            (state, current) => 
-                List.Accumulate(
-                    Record.FieldNames(current),
-                    state,
-                    (innerState, fieldName) => 
-                        if ShouldIncludeKey(fieldName) then
-                            AddUniqueKey(innerState, fieldName, Record.Field(current, fieldName))
-                        else
-                            innerState
-                )
-        ),
-    
-    // helper function to process name-value objects
-    ProcessNameValueObject = (obj as record) as record =>
-        let
-            // Assert that the object has required fields
-            _ = if not (Record.HasFields(obj, {"name"}) and Record.HasFields(obj, {"value"})) then
-                error [
-                    Reason = "Invalid Name-Value Object",
-                    Message = "Object must have both 'name' and 'value' fields"
-                ]
-            else
-                null,
-                
-            BaseName = obj[name],
-            BaseValue = obj[value],
-            
-            // only extract name and value if it should be included
-            Result = if ShouldIncludeKey(BaseName) then
-                Record.FromList({BaseValue}, {BaseName})
-            else
-                []
-        in
-            Result,
-    
-    // helper function to process direct key-value objects
-    ProcessDirectKeyValueObject = (obj as record) as record =>
-        let
-            // assert that input is a record
-            _ = if not Value.Is(obj, type record) then
-                error [
-                    Reason = "Invalid Input Type",
-                    Message = "Expected record for direct key-value processing"
-                ]
-            else
-                null,
-                
-            // extract all primitive key-value pairs
-            PrimitiveFields = List.Transform(
-                Record.FieldNames(obj),
-                (fieldName) => 
-                    let
-                        fieldValue = Record.Field(obj, fieldName)
-                    in
-                        if not Type.Is(Value.Type(fieldValue), type record) and ShouldIncludeKey(fieldName) then
-                            Record.FromList({fieldValue}, {fieldName})
-                        else
-                            []
-            ),
-            
-            // filter out empty records and combine using safe combine
-            FilteredFields = List.Select(PrimitiveFields, (rec) => Record.FieldCount(rec) > 0),
-            Result = SafeRecordCombine(FilteredFields)
-        in
-            Result,
-    
-    // helper functions for type checking
-    HasDirectKeyValuePattern = (obj as record) as logical =>
-        List.AllTrue(
-            List.Transform(
-                Record.FieldValues(obj),
-                (value) => not Type.Is(Value.Type(value), type record)
-            )
-        ),
-    
-    IsNestedContainer = (obj as record) as logical =>
-        List.AnyTrue(
-            List.Transform(
-                Record.FieldValues(obj),
-                (value) => Type.Is(Value.Type(value), type record)
-            )
-        ),
-    
-    // main processing function with path tracking
-    ProcessField = (fieldName as text, fieldValue as any, currentPath as text) as record =>
-        let
-            fieldPath = if currentPath = "" then fieldName else currentPath & "." & fieldName
-        in
-            if IsExcludedPath(fieldPath) then
-                []
-            else if Type.Is(Value.Type(fieldValue), type record) then
-                if Record.HasFields(fieldValue, {"name", "value"}) then
-                    ProcessNameValueObject(fieldValue)
-                else if HasDirectKeyValuePattern(fieldValue) then
-                    ProcessDirectKeyValueObject(fieldValue)
-                else if IsNestedContainer(fieldValue) then
-                    // recursive call for nested containers
-                    @ProcessRecord(fieldValue, fieldPath)
-                else
-                    // unknown record type, skip
-                    []
-            else
-                // primitive value - check if should be included
-                if ShouldIncludeKey(fieldName) then
-                    Record.FromList({fieldValue}, {fieldName})
-                else
-                    [],
-    
-    // process entire record recursively
-    ProcessRecord = (record as record, currentPath as text) as record =>
-        let
-            // assert that input is a record
-            _ = if not Value.Is(record, type record) then
-                error [
-                    Reason = "Invalid Record Type",
-                    Message = "Expected record for processing, but received: " & Text.From(Value.Type(record))
-                ]
-            else
-                null,
-                
-            ProcessedFields = List.Transform(
-                Record.FieldNames(record),
-                (fieldName) => ProcessField(fieldName, Record.Field(record, fieldName), currentPath)
-            ),
-            FilteredFields = List.Select(ProcessedFields, (rec) => Record.FieldCount(rec) > 0),
-            CombinedRecord = SafeRecordCombine(FilteredFields)
-        in
-            CombinedRecord,
-    
-    // assert that input is a record before processing
-    _ = if not Value.Is(ActualInput, type record) then
-        error [
-            Reason = "Invalid Input Type",
-            Message = "Input must be a record, but received: " & Text.From(Value.Type(ActualInput))
-        ]
-    else
-        null,
-    
-    // start processing from root
-    Result = ProcessRecord(ActualInput, "")
+    // Call the implementation with self-reference
+    result = FlattenRecordImpl(FlattenRecordImpl, inputRecord, filterKeys, parentPath, existingFields)
 in
-    Result 
+    result

src/powerbi-data-connector/speckle/api/SendToServer.pqm

@@ -1,5 +1,5 @@
 (url as text) as list =>
-    try let
+    let
         // Import required functions
         GetModel = Extension.LoadFunction("GetModel.pqm"),
         Parser = Extension.LoadFunction("Parser.pqm"),
@@ -24,56 +24,145 @@
                             Detail = [File = fileName, Error = e]
                         ],
 
+        // Get required information
         modelInfo = GetModel(url),
         parsedUrl = Parser(url),
         userInfo = GetUser(url),
-
         apiKey = userInfo[Token],
-        
         userEmail = userInfo[UserEmail],
-
-        // get version from Speckle.pq - look GetVersion.pqm
         connectorVersion = GetVersion(),
-
         workspaceInfo = GetWorkspace(url),
 
-        // Prepare request data
-        requestData = Json.FromValue([
-            Url = url,
-            Server = parsedUrl[baseUrl],
-            Email = userEmail,
-            ProjectId = parsedUrl[projectId],
-            ObjectId = modelInfo[rootObjectId],
-            SourceApplication = modelInfo[sourceApplication],
-            Token = apiKey,
-            Version = connectorVersion,
-            WorkspaceId = workspaceInfo[workspaceId],
-            WorkspaceName = workspaceInfo[workspaceName],
-            WorkspaceLogo = workspaceInfo[workspaceLogo],
-            CanHideBranding = workspaceInfo[canHideBranding]
-        ]),
+        // Function to check if Desktop Service is available
+        IsDesktopServiceAvailable = () =>
+            try
+                let
+                    PingResponse = Web.Contents(
+                        "http://127.0.0.1:29364/ping",
+                        [
+                            Headers = [#"Method" = "GET"],
+                            ManualStatusHandling = {400, 401, 403, 404, 500, 502, 503, 504},
+                            Timeout = #duration(0, 0, 0, 2) // 2 second timeout for ping
+                        ]
+                    ),
+                    StatusCode = Value.Metadata(PingResponse)[Response.Status]
+                in
+                    StatusCode = 200
+            otherwise
+                false,
+
+        // Function to use Desktop Service approach (only called if available)
+        UseDesktopService = () =>
+            let
+                // exchange powerful token for weak token via ds 
+                tokenExchangeData = Json.FromValue([
+                    PowerfulToken = apiKey,
+                    Scopes = {"profile:read", "streams:read", "users:read"},
+                    ProjectId = parsedUrl[projectId],
+                    ServerUrl = parsedUrl[baseUrl]
+                ]),
+                
+                tokenExchangeResponse = Web.Contents(
+                    "http://127.0.0.1:29364/auth/exchange-token",
+                    [
+                        Headers = [
+                            #"Content-Type" = "application/json",
+                            #"Method" = "POST"
+                        ],
+                        Content = tokenExchangeData,
+                        ManualStatusHandling = {400, 401, 403, 404, 500}
+                    ]
+                ),
+                
+                tokenExchangeJson = Json.Document(tokenExchangeResponse),
+                weakToken = tokenExchangeJson[token],
+
+                // prepare request data with weak token
+                requestData = Json.FromValue([
+                    Url = url,
+                    Server = parsedUrl[baseUrl],
+                    Email = userEmail,
+                    ProjectId = parsedUrl[projectId],
+                    RootObjectId = modelInfo[rootObjectId],
+                    SourceApplication = modelInfo[sourceApplication],
+                    Token = weakToken,
+                    Version = connectorVersion,
+                    VersionId = parsedUrl[versionId],
+                    WorkspaceId = workspaceInfo[workspaceId],
+                    WorkspaceName = workspaceInfo[workspaceName],
+                    WorkspaceLogo = workspaceInfo[workspaceLogo],
+                    CanHideBranding = workspaceInfo[canHideBranding]
+                ]),
+                
+                // Send request to local server
+                Response = Web.Contents(
+                    "http://127.0.0.1:29364/download",
+                    [
+                        Headers = [
+                            #"Content-Type" = "application/json",
+                            #"Method" = "POST"
+                        ],
+                        Content = requestData,
+                        ManualStatusHandling = {400, 401, 403, 404, 500}
+                    ]
+                ),
+                
+                // Parse response
+                JsonResponse = Json.Document(Response)
+            in
+                JsonResponse,
+
+        // Function to fallback to direct JSON download from Speckle server
+        FallbackToDirectDownload = () =>
+            let
+                // Construct the direct object URL: {baseUrl}/objects/{projectId}/{rootObjectId}
+                objectUrl = Text.Combine({
+                    parsedUrl[baseUrl], 
+                    "/objects/", 
+                    parsedUrl[projectId], 
+                    "/", 
+                    modelInfo[rootObjectId]
+                }),
+                
+                // Download JSON directly from Speckle server
+                Response = Web.Contents(
+                    objectUrl,
+                    [
+                        Headers = [
+                            #"Authorization" = "Bearer " & apiKey,
+                            #"Accept" = "application/json"
+                        ],
+                        ManualStatusHandling = {400, 401, 403, 404, 500, 502, 503, 504}
+                    ]
+                ),
+                
+                // Check response status
+                StatusCode = Value.Metadata(Response)[Response.Status],
+                
+                // Parse JSON response if successful
+                JsonResponse = if StatusCode >= 200 and StatusCode < 300 then
+                    Json.Document(Response)
+                else
+                    error [
+                        Reason = "DirectDownloadFailed",
+                        Message = "Failed to download model data directly from Speckle server",
+                        Detail = [
+                            StatusCode = StatusCode, 
+                            ObjectUrl = objectUrl,
+                            ProjectId = parsedUrl[projectId],
+                            RootObjectId = modelInfo[rootObjectId]
+                        ]
+                    ]
+            in
+                JsonResponse,
+
+        // Check Desktop Service availability and choose approach
+        DesktopServiceAvailable = IsDesktopServiceAvailable(),
         
-        // Send request to local server
-        Response = Web.Contents(
-            "http://127.0.0.1:29364/download",
-            [
-                Headers = [
-                    #"Content-Type" = "application/json",
-                    #"Method" = "POST"
-                ],
-                Content = requestData,
-                ManualStatusHandling = {400, 401, 403, 404, 500}
-            ]
-        ),
-        
-        // Parse response
-        JsonResponse = Json.Document(Response)
+        FinalResult = if DesktopServiceAvailable then
+            UseDesktopService()
+        else
+            FallbackToDirectDownload()
 
     in
-        JsonResponse
-    otherwise
-        error [
-            Reason = "Desktop Service Not Available", 
-            Message = "Cannot connect to Speckle Desktop Service. Please ensure the Desktop Service is running and try again.",
-            Detail = "The Speckle Desktop Service must be running to load data from Speckle. Please start the Desktop Service application and refresh your data connection."
-        ]
+        FinalResult

src/powerbi-visual/capabilities.json

@@ -133,6 +133,15 @@
         }
       }
     },
+    "dataLoading": {
+      "properties": {
+        "internalizeData": {
+          "type": {
+            "bool": true
+          }
+        }
+      }
+    },
     "color": {
       "properties": {
         "enabled": {

src/powerbi-visual/package.json

@@ -17,10 +17,10 @@
     "@babel/runtime-corejs3": "^7.21.5",
     "@headlessui/vue": "^1.7.13",
     "@heroicons/vue": "^2.0.12",
-    "@speckle/objectloader": "^2.23.8",
+    "@speckle/objectloader2": "2.26.5",
     "@speckle/tailwind-theme": "2.23.2",
     "@speckle/ui-components": "2.23.2",
-    "@speckle/viewer": "2.23.23",
+    "@speckle/viewer": "2.26.5",
     "color-interpolate": "^1.0.5",
     "core-js": "^3.30.2",
     "lodash": "^4.17.21",

src/powerbi-visual/src/components/ViewerWrapper.vue

@@ -1,5 +1,5 @@
 <template>
-  <div class="border">
+  <div>
     <transition name="slide-fade">
       <nav
         v-show="!visualStore.isNavbarHidden"
@@ -29,7 +29,7 @@
 
           <div class="flex items-center space-x-2">
             <FormButton
-              v-if="visualStore.latestAvailableVersion && !visualStore.isConnectorUpToDate"
+              v-if="visualStore.latestAvailableVersion && !visualStore.isConnectorUpToDate && visualStore.isRunningInDesktop"
               v-tippy="{
                 content: 'New connector version is available.<br>Click to download.',
                 allowHTML: true
@@ -165,7 +165,7 @@ onMounted(async () => {
   
   // Set up event listener for object clicks from the FilteredSelectionExtension
   viewerHandler.emitter.on('objectClicked', handleObjectClicked)
-  
+
   visualStore.setViewerEmitter(viewerHandler.emit)
 })
 

src/powerbi-visual/src/components/viewer/view-modes/ViewerViewModesMenu.vue

@@ -34,11 +34,9 @@ import ViewModes from '../../global/icon/ViewModes.vue'
 
 const viewModes = {
   [ViewMode.DEFAULT]: 'Default',
-  [ViewMode.DEFAULT_EDGES]: 'Edges',
   [ViewMode.SHADED]: 'Shaded',
   [ViewMode.PEN]: 'Pen',
-  [ViewMode.ARCTIC]: 'Arctic',
-  [ViewMode.COLORS]: 'Colors'
+  [ViewMode.ARCTIC]: 'Arctic'
 }
 
 const visualStore = useVisualStore()

src/powerbi-visual/src/laoder/SpeckleObjectsOfflineLoader.ts

@@ -1,31 +1,116 @@
-import ObjectLoader from '@speckle/objectloader'
+import { ObjectLoader2Factory } from '@speckle/objectloader2'
 import { SpeckleLoader, WorldTree } from '@speckle/viewer'
 
+// Base type from objectloader2 (has id, speckle_type properties)
+interface Base {
+  id: string
+  speckle_type: string
+  [key: string]: any
+}
+
 export class SpeckleObjectsOfflineLoader extends SpeckleLoader {
-  constructor(targetTree: WorldTree, resourceData: string, resourceId?: string) {
-    super(targetTree, resourceId || '', undefined, undefined, resourceData)
+  constructor(targetTree: WorldTree, resourceData: unknown, resourceId?: string) {
+    // Resource ID is not used for offline loading since we have objects in memory
+    // Pass empty string to avoid URL parsing issues
+    super(targetTree, '', undefined, undefined, resourceData)
   }
 
   protected initObjectLoader(
-    _resource: string,
-    _authToken?: string,
-    _enableCaching?: boolean,
-    resourceData?: string | ArrayBuffer
-  ): ObjectLoader {
-    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-    //@ts-ignore
-    // eslint-disable-next-line @typescript-eslint/no-unsafe-return
-    return ObjectLoader.createFromObjects(resourceData as unknown as [])
+    resource: string,
+    authToken?: string,
+    enableCaching?: boolean,
+    resourceData?: unknown
+  ): ReturnType<SpeckleLoader['initObjectLoader']> {
+    // Use ObjectLoader2Factory.createFromObjects for offline/memory-based loading
+    // The objects array must contain ALL objects (root + all children)
+    // The first object in the array must be the root object
+    const objects = (resourceData ?? this._resourceData) as Base[]
+
+    if (!objects || objects.length === 0) {
+      throw new Error('SpeckleObjectsOfflineLoader: No objects provided')
+    }
+
+    // Ensure all objects have an 'id' property
+    const missingIds = objects.filter((obj) => !obj.id)
+    if (missingIds.length > 0) {
+      console.error('Objects missing id property:', missingIds.slice(0, 5))
+      throw new Error(
+        `SpeckleObjectsOfflineLoader: ${missingIds.length} objects are missing 'id' property`
+      )
+    }
+
+    console.log(`Creating offline loader with ${objects.length} objects, root: ${objects[0].id}`)
+
+    // Create a Set of all object IDs for quick lookup
+    const objectIds = new Set(objects.map((obj) => obj.id))
+
+    // Check for references to objects that aren't in the array
+    const missingReferences = new Set<string>()
+    objects.forEach((obj) => {
+      // Check all properties for references (objects that look like { referencedId: "xxx" })
+      Object.values(obj).forEach((value) => {
+        if (value && typeof value === 'object') {
+          if ('referencedId' in value && typeof value.referencedId === 'string') {
+            if (!objectIds.has(value.referencedId)) {
+              missingReferences.add(value.referencedId)
+            }
+          }
+        }
+        // Check arrays for references
+        if (Array.isArray(value)) {
+          value.forEach((item) => {
+            if (item && typeof item === 'object' && 'referencedId' in item) {
+              if (!objectIds.has(item.referencedId)) {
+                missingReferences.add(item.referencedId)
+              }
+            }
+          })
+        }
+      })
+    })
+
+    if (missingReferences.size > 0) {
+      console.warn(
+        `⚠️ Found ${missingReferences.size} missing object references:`,
+        Array.from(missingReferences).slice(0, 10)
+      )
+    } else {
+      console.log('✅ All object references are present')
+    }
+
+    // @ts-ignore - Type compatibility issue between local objectloader2 and viewer's objectloader2
+    return ObjectLoader2Factory.createFromObjects(objects)
   }
 
   public async load(): Promise<boolean> {
     const rootObject = await this.loader.getRootObject()
-    if (!rootObject && this._resource) {
-      console.error('No root id set!')
+    if (!rootObject) {
+      console.error('No root object found!')
       return false
     }
-    /** If not id is provided, we make one up based on the root object id */
-    this._resource = this._resource || `/json/${rootObject.id as string}`
+
+    /** Set resource to a fake URL for logging purposes only */
+    this._resource = this._resource || `/json/${rootObject.baseId as string}`
+
+    console.log('Loading objects from memory (offline mode)')
+
+    // Call parent load() which will use our ObjectLoader2 to iterate through objects
+    // Since we're using MemoryDownloader, it won't actually download anything
     return super.load()
   }
+
+  /**
+   * Clean up the ObjectLoader2 resources
+   */
+  public async dispose(): Promise<void> {
+    try {
+      if (this.loader && 'disposeAsync' in this.loader) {
+        // @ts-ignore - disposeAsync exists on ObjectLoader2
+        await this.loader.disposeAsync()
+        console.log('SpeckleObjectsOfflineLoader: ObjectLoader2 disposed')
+      }
+    } catch (error) {
+      console.warn('Error disposing ObjectLoader2 in offline loader:', error)
+    }
+  }
 }

src/powerbi-visual/src/loader/SpeckleApiLoader.ts

@@ -0,0 +1,192 @@
+import { useVisualStore } from '@src/store/visualStore'
+import { ObjectLoader2Factory } from '@speckle/objectloader2'
+
+interface SpeckleObject {
+  id: string
+  speckle_type?: string
+  [key: string]: any
+}
+
+export class SpeckleApiLoader {
+  private serverUrl: string
+  private token: string
+  private projectId: string
+  private headers: Record<string, string>
+
+  constructor(serverUrl: string, projectId: string, token: string) {
+    this.serverUrl = serverUrl.replace(/\/$/, '')
+    this.projectId = projectId
+    this.token = token
+    this.headers = {
+      Authorization: `Bearer ${token}`,
+      'Content-Type': 'application/json'
+    }
+  }
+
+  async downloadObjectsWithChildren(
+    objectId: string,
+    onProgress?: (loaded: number, total: number) => void
+  ): Promise<SpeckleObject[]> {
+    const visualStore = useVisualStore()
+
+    visualStore.setLoadingProgress('Initializing object loader', 0)
+    console.log('Creating ObjectLoader v2 for Power BI environment')
+
+    const loader = ObjectLoader2Factory.createFromUrl({
+      serverUrl: this.serverUrl,
+      streamId: this.projectId,
+      objectId,
+      token: this.token,
+      attributeMask: { exclude: ['properties', 'encodedValue'] },
+      options: { useCache: false }
+    })
+
+    try {
+      // Get total count for progress tracking
+      const totalCount = await loader.getTotalObjectCount()
+      console.log(`Loading ${totalCount} objects using ObjectLoader v2`)
+
+      const objects: SpeckleObject[] = []
+      let loadedCount = 0
+
+      // Stream all objects using the async iterator
+      for await (const obj of loader.getObjectIterator()) {
+        objects.push(obj as SpeckleObject) // Type assertion for SpeckleObject interface
+        loadedCount++
+
+        // Update progress
+        if (onProgress) {
+          onProgress(loadedCount, totalCount)
+        }
+
+        const progress = totalCount > 0 ? loadedCount / totalCount : 0
+        visualStore.setLoadingProgress('🌍 Loading from Speckle', progress)
+
+        // Log progress every 100 objects
+        if (loadedCount % 100 === 0) {
+          console.log(`Loaded ${loadedCount}/${totalCount} objects`)
+        }
+      }
+
+      console.log(`Downloaded ${objects.length} objects using ObjectLoader v2`)
+
+      visualStore.setLoadingProgress('🔄 Finalizing object download...', 0.9)
+
+      // Recursively fetch all missing references until none remain
+      let iterationCount = 0
+      let totalFetched = 0
+
+      while (iterationCount < 10) {
+        // Safety limit: loop exits early when missingIds.size === 0 (line 108)
+        // This limit only prevents infinite loops if something goes wrong
+        iterationCount++
+
+        const objectIds = new Set(objects.map((obj) => obj.id))
+        const missingIds = new Set<string>()
+
+        // Check all objects for missing references
+        objects.forEach((obj) => {
+          Object.values(obj).forEach((value) => {
+            if (value && typeof value === 'object') {
+              if ('referencedId' in value && typeof value.referencedId === 'string') {
+                if (!objectIds.has(value.referencedId)) {
+                  missingIds.add(value.referencedId)
+                }
+              }
+            }
+            if (Array.isArray(value)) {
+              value.forEach((item) => {
+                if (item && typeof item === 'object' && 'referencedId' in item) {
+                  if (!objectIds.has(item.referencedId)) {
+                    missingIds.add(item.referencedId)
+                  }
+                }
+              })
+            }
+          })
+        })
+
+        if (missingIds.size === 0) {
+          console.log(
+            `✅ No more missing references. Complete after ${iterationCount} iteration(s)`
+          )
+          break
+        }
+
+        console.log(
+          `Iteration ${iterationCount}: Fetching ${missingIds.size} missing referenced objects...`
+        )
+
+        visualStore.setLoadingProgress(`🔄 Loading additional objects)`, 0.9)
+
+        // Fetch missing objects with progress tracking
+        const missingIdsArray = Array.from(missingIds)
+        let fetchedInIteration = 0
+
+        for (const missingId of missingIdsArray) {
+          try {
+            const missingObj = await loader.getObject({ id: missingId })
+            objects.push(missingObj as SpeckleObject)
+            totalFetched++
+            fetchedInIteration++
+
+            // Update progress within this iteration
+            const iterationProgress = fetchedInIteration / missingIdsArray.length
+            visualStore.setLoadingProgress(
+              `🔄 Loading objects (${objects.length} loaded)`,
+              0.9 + iterationProgress * 0.05 // Progress from 0.9 to 0.95
+            )
+          } catch (err) {
+            console.warn(`⚠️ Could not fetch missing object ${missingId}:`, err)
+          }
+        }
+
+        console.log(
+          `✅ Iteration ${iterationCount} complete. Fetched ${missingIdsArray.length} objects. Total: ${objects.length}`
+        )
+      }
+
+      if (iterationCount >= 10) {
+        console.warn(
+          '⚠️ Reached maximum iterations for fetching references. Some objects may still be missing.'
+        )
+      }
+
+      console.log(
+        `✅ Downloaded total of ${objects.length} objects (${totalFetched} additional references fetched)`
+      )
+
+      visualStore.setLoadingProgress('Download complete', 1)
+
+      return objects
+    } catch (error) {
+      console.error('Error loading objects:', error)
+      throw error
+    } finally {
+      // Clean up the loader resources
+      try {
+        await loader.disposeAsync()
+        console.log('ObjectLoader2 disposed successfully')
+      } catch (disposeError) {
+        console.warn('Error disposing ObjectLoader2:', disposeError)
+      }
+    }
+  }
+
+  async downloadFromVersionId(versionId: string): Promise<SpeckleObject[]> {
+    // For version IDs, we can't avoid GraphQL entirely as we need to resolve the referenced object
+    // However, this method might not be used if we're getting object IDs directly from the data connector
+    throw new Error('Version ID downloads not supported with weak tokens. Use object IDs directly.')
+  }
+
+  async downloadMultipleModels(objectIds: string[]): Promise<SpeckleObject[][]> {
+    const allObjects: SpeckleObject[][] = []
+
+    for (const objectId of objectIds) {
+      const objects = await this.downloadObjectsWithChildren(objectId)
+      allObjects.push(objects)
+    }
+
+    return allObjects
+  }
+}

src/powerbi-visual/src/plugins/viewer.ts

@@ -231,6 +231,11 @@ export class ViewerHandler {
       // Since you are setting another camera position, maybe you want the second argument to false
       await this.viewer.loadObject(loader, true)
       this.viewer.getRenderer().shadowcatcher.shadowcatcherMesh.visible = false // works fine only right after loadObjects
+
+      // Clean up loader resources after loading is complete
+      if (loader.dispose) {
+        await loader.dispose()
+      }
     }
 
     store.setSpeckleViews(speckleViews)

src/powerbi-visual/src/settings/dataLoadingSettings.ts

@@ -0,0 +1,15 @@
+import { formattingSettings as fs } from 'powerbi-visuals-utils-formattingmodel'
+
+export class DataLoadingSettings extends fs.SimpleCard {
+  name = 'dataLoading'
+  displayName = 'Data Management'
+
+  public internalizeData = new fs.ToggleSwitch({
+    name: 'internalizeData',
+    displayName: 'Internalize Data',
+    description: 'When enabled, objects are downloaded and stored in the Power BI file for offline access. When disabled, objects are loaded directly from Speckle servers (online mode).',
+    value: false
+  })
+
+  slices: fs.Slice[] = [this.internalizeData]
+}

src/powerbi-visual/src/settings/visualSettingsModel.ts

@@ -1,17 +1,18 @@
 import { formattingSettings as fs } from 'powerbi-visuals-utils-formattingmodel'
-import { ColorSelectorSettings, ColorSettings } from 'src/settings/colorSettings'
+import { ColorSettings } from 'src/settings/colorSettings'
 import { CameraSettings } from 'src/settings/cameraSettings'
 import { LightingSettings } from 'src/settings/lightingSettings'
+import { DataLoadingSettings } from 'src/settings/dataLoadingSettings'
 
 export class SpeckleVisualSettingsModel extends fs.Model {
   // Building my visual formatting settings card
   public color: ColorSettings = new ColorSettings()
 
-  public colorSelector: ColorSelectorSettings = new ColorSelectorSettings()
+  public dataLoading: DataLoadingSettings = new DataLoadingSettings()
 
   // public camera: CameraSettings = new CameraSettings()
 
   // public lighting: LightingSettings = new LightingSettings()
 
-  cards = [this.color]
+  cards = [this.color, this.dataLoading]
 }

src/powerbi-visual/src/store/visualStore.ts

@@ -3,8 +3,8 @@ import { Version } from '@src/composables/useUpdateConnector'
 import { ColorBy, IViewerEvents } from '@src/plugins/viewer'
 import { SpeckleVisualSettingsModel } from '@src/settings/visualSettingsModel'
 import { SpeckleDataInput } from '@src/types'
-import { zipModelObjects } from '@src/utils/compression'
 import { ReceiveInfo } from '@src/utils/matrixViewUtils'
+import { zipModelObjects } from '@src/utils/compression'
 import { defineStore } from 'pinia'
 import { Vector3 } from 'three'
 import { computed, ref, shallowRef } from 'vue'
@@ -26,7 +26,10 @@ export const useVisualStore = defineStore('visualStore', () => {
   const host = shallowRef<powerbi.extensibility.visual.IVisualHost>()
   const formattingSettings = ref<SpeckleVisualSettingsModel>()
   const loadingProgress = ref<LoadingProgress>(undefined)
-  const objectsFromStore = ref<object[]>(undefined)
+  const objectsFromStore = ref<object[][]>(undefined)
+
+  // State tracking for toggle reset prevention
+  const previousToggleState = ref<boolean | undefined>(undefined)
 
   const postFileSaveSkipNeeded = ref<boolean>(false)
   const postClickSkipNeeded = ref<boolean>(false)
@@ -83,7 +86,19 @@ export const useVisualStore = defineStore('visualStore', () => {
     host.value = hostToSet
   }
 
-  const setReceiveInfo = (newReceiveInfo: ReceiveInfo) => (receiveInfo.value = newReceiveInfo)
+  const setReceiveInfo = (newReceiveInfo: ReceiveInfo) => {
+    receiveInfo.value = newReceiveInfo
+    
+    // Always save receiveInfo to file for credentials persistence (contains token and metadata)
+    // This ensures weak tokens are available even when desktop service is unavailable
+    if (formattingSettings.value?.dataLoading.internalizeData.value && objectsFromStore.value) {
+      // If internalize is ON and we have objects, save both objects and receiveInfo together
+      writeObjectsToFile(objectsFromStore.value)
+    } else {
+      // Otherwise just save receiveInfo alone (credentials only)
+      writeReceiveInfoToFile()
+    }
+  }
 
   const setLatestAvailableVersion = (version: Version | null) => {
     latestAvailableVersion.value = version
@@ -96,6 +111,15 @@ export const useVisualStore = defineStore('visualStore', () => {
     return false
   })
 
+  // detecting the env to control the visibility of update button
+  // might use for different reasons in the future
+  const isRunningInDesktop = computed(() => {
+    // power bi hostEnv enum values:
+    // web = 1, desktop = 4
+    const hostEnv = host.value?.['hostEnv'] as number
+    return hostEnv === 4
+  })
+
   /**
    * Ideally one time set when onMounted of `ViewerWrapper.vue` component
    * @param emit picky emit function to trigger events under `IViewerEvents` interface
@@ -113,7 +137,7 @@ export const useVisualStore = defineStore('visualStore', () => {
     }
   }
 
-  const setObjectsFromStore = (newObjectsFromStore: object[]) => {
+  const setObjectsFromStore = (newObjectsFromStore: object[][]) => {
     objectsFromStore.value = newObjectsFromStore
   }
 
@@ -124,6 +148,23 @@ export const useVisualStore = defineStore('visualStore', () => {
     }
   }
 
+  const filterColorByIdsForSelection = (colorByIds: ColorBy[] | null | undefined, selectedIds: string[]): ColorBy[] => {
+    return colorByIds?.filter(colorGroup => {
+      const filteredObjectIds = colorGroup.objectIds.filter(objId =>
+        selectedIds.includes(objId)
+      )
+      if (filteredObjectIds.length > 0) {
+        return { ...colorGroup, objectIds: filteredObjectIds }
+      }
+      return false
+    }).map(colorGroup => ({
+      ...colorGroup,
+      objectIds: colorGroup.objectIds.filter(objId =>
+        selectedIds.includes(objId)
+      )
+    })) || []
+  }
+
   const clearLoadingProgress = () => {
     loadingProgress.value = undefined
   }
@@ -134,17 +175,24 @@ export const useVisualStore = defineStore('visualStore', () => {
   }
 
   const loadObjectsFromFile = async (objects: object[][]) => {
+    console.log('📁 loadObjectsFromFile called with:', objects.length, 'models')
     const savedVersionObjectId = objects.map((o) => (o[0] as SpeckleObject).id).join(',')
     lastLoadedRootObjectId.value = savedVersionObjectId
     viewerReloadNeeded.value = false
     console.log(`📦 Loading viewer from cached data with ${lastLoadedRootObjectId.value} id.`)
+    console.log('📁 About to call viewerEmit loadObjects...')
     await viewerEmit.value('loadObjects', objects)
+    console.log('📁 viewerEmit loadObjects completed')
     objectsFromStore.value = objects
     isViewerObjectsLoaded.value = true
     viewerReloadNeeded.value = false
     setIsLoadingFromFile(false)
+    console.log('📁 loadObjectsFromFile completed successfully')
   }
 
+  const setIsLoadingFromFile = (newValue: boolean) => (isLoadingFromFile.value = newValue)
+
+
   /**
    * Sets upcoming data input into store to be able to pass it through viewer by evaluating the data.
    * @param newValue new data input that user dragged and dropped to the fields in visual
@@ -159,14 +207,24 @@ export const useVisualStore = defineStore('visualStore', () => {
       await viewerEmit.value('loadObjects', dataInput.value.modelObjects)
       viewerReloadNeeded.value = false
       isViewerObjectsLoaded.value = true
-      setLoadingProgress('Storing objects into file', null)
-      writeObjectsToFile(dataInput.value.modelObjects)
+      
+      // Store the model objects for potential internalization
+      if (dataInput.value.modelObjects && dataInput.value.modelObjects.length > 0) {
+        console.log('📦 Storing modelObjects in visualStore for internalization:', dataInput.value.modelObjects.length, 'models')
+        objectsFromStore.value = dataInput.value.modelObjects
+      }
+      
+      // Note: Object internalization is now handled by toggle in visual.ts
       loadingProgress.value = undefined
     }
 
     if (dataInput.value.selectedIds.length > 0) {
       isFilterActive.value = true
       viewerEmit.value('filterSelection', dataInput.value.selectedIds, isGhostActive.value, isZoomOnFilterActive.value)
+
+      // When filtering, only apply colors to the selected/isolated objects
+      const filteredColorByIds = filterColorByIdsForSelection(dataInput.value.colorByIds, dataInput.value.selectedIds)
+      viewerEmit.value('colorObjectsByGroup', filteredColorByIds)
     } else {
       isFilterActive.value = false
       latestColorBy.value = dataInput.value.colorByIds
@@ -177,8 +235,9 @@ export const useVisualStore = defineStore('visualStore', () => {
         // No object IDs provided - show all objects without any filtering
         viewerEmit.value('unIsolateObjects')
       }
+      // When not filtering, apply all colors including conditional formatting
+      viewerEmit.value('colorObjectsByGroup', dataInput.value.colorByIds)
     }
-    viewerEmit.value('colorObjectsByGroup', dataInput.value.colorByIds)
   }
 
   const writeObjectsToFile = (modelObjects: object[][]) => {
@@ -192,6 +251,23 @@ export const useVisualStore = defineStore('visualStore', () => {
           objectName: 'storedData',
           properties: {
             speckleObjects: compressedChunks,
+            receiveInfo: JSON.stringify(receiveInfo.value) // Keep receiveInfo in sync when storing objects
+          },
+          selector: null
+        }
+      ]
+    })
+  }
+
+  const writeReceiveInfoToFile = () => {
+    // NOTE: need skipping the update function, it resets the viewer state unneccessarily.
+    postFileSaveSkipNeeded.value = true
+
+    host.value.persistProperties({
+      merge: [
+        {
+          objectName: 'storedData',
+          properties: {
             receiveInfo: JSON.stringify(receiveInfo.value)
           },
           selector: null
@@ -312,6 +388,22 @@ export const useVisualStore = defineStore('visualStore', () => {
     })
   }
 
+  const writeDataLoadingModeToFile = (internalizeData: boolean) => {
+    // NOTE: need skipping the update function, it resets the viewer state unneccessarily.
+    postFileSaveSkipNeeded.value = true
+    host.value.persistProperties({
+      merge: [
+        {
+          objectName: 'dataLoading',
+          properties: {
+            internalizeData: internalizeData
+          },
+          selector: null
+        }
+      ]
+    })
+  }
+
   const writeCameraPositionToFile = (position: Vector3, target: Vector3) => {
     // NOTE: need skipping the update function, it resets the viewer state unneccessarily.
     postFileSaveSkipNeeded.value = true
@@ -338,7 +430,6 @@ export const useVisualStore = defineStore('visualStore', () => {
 
   const clearDataInput = () => (dataInput.value = null)
 
-  const setIsLoadingFromFile = (newValue: boolean) => (isLoadingFromFile.value = newValue)
 
   const setViewerReadyToLoad = (newValue: boolean) => (isViewerReadyToLoad.value = newValue)
 
@@ -392,6 +483,7 @@ export const useVisualStore = defineStore('visualStore', () => {
       // No object IDs provided - show all objects without any filtering
       viewerEmit.value('unIsolateObjects')
     }
+    // When resetting filters, apply all colors including conditional formatting
     if (latestColorBy.value !== null) {
       viewerEmit.value('colorObjectsByGroup', latestColorBy.value)
     }
@@ -417,6 +509,10 @@ export const useVisualStore = defineStore('visualStore', () => {
       if (dataInput.value.selectedIds.length > 0) {
         isFilterActive.value = true
         viewerEmit.value('filterSelection', dataInput.value.selectedIds, isGhostActive.value, isZoomOnFilterActive.value)
+
+        // When filtering, only apply colors to the selected/isolated objects
+        const filteredColorByIds = filterColorByIdsForSelection(dataInput.value.colorByIds, dataInput.value.selectedIds)
+        viewerEmit.value('colorObjectsByGroup', filteredColorByIds)
       } else {
         isFilterActive.value = false
         latestColorBy.value = dataInput.value.colorByIds
@@ -427,16 +523,21 @@ export const useVisualStore = defineStore('visualStore', () => {
           // No object IDs provided - show all objects without any filtering
           viewerEmit.value('unIsolateObjects')
         }
+
+        // Restore color grouping for all objects when not filtering
+        viewerEmit.value('colorObjectsByGroup', dataInput.value.colorByIds)
       }
-      
-      // Restore color grouping
-      viewerEmit.value('colorObjectsByGroup', dataInput.value.colorByIds)
     }
     
     // Trigger host data refresh to synchronize with Power BI
     host.value.refreshHostData()
   }
 
+  // Toggle state tracking functions
+  const setPreviousToggleState = (state: boolean) => {
+    previousToggleState.value = state
+  }
+
   return {
     host,
     receiveInfo,
@@ -467,7 +568,9 @@ export const useVisualStore = defineStore('visualStore', () => {
     isZoomOnFilterActive,
     latestAvailableVersion,
     isConnectorUpToDate,
+    isRunningInDesktop,
     commonError,
+    previousToggleState,
     setCommonError,
     setLatestAvailableVersion,
     setIsOrthoProjection,
@@ -495,6 +598,7 @@ export const useVisualStore = defineStore('visualStore', () => {
     writeCameraPositionToFile,
     writeHideBrandingToFile,
     writeNavbarVisibilityToFile,
+    writeDataLoadingModeToFile,
     toggleBranding,
     toggleNavbar,
     setViewerEmitter,
@@ -507,6 +611,7 @@ export const useVisualStore = defineStore('visualStore', () => {
     setIsLoadingFromFile,
     resetFilters,
     downloadLatestVersion,
-    handleObjectsLoadedComplete
+    handleObjectsLoadedComplete,
+    setPreviousToggleState
   }
 })

src/powerbi-visual/src/utils/matrixViewUtils.ts

@@ -11,6 +11,8 @@ import { FieldInputState, useVisualStore } from '@src/store/visualStore'
 import { delay } from 'lodash'
 import { getSlugFromHostAppNameAndVersion } from './hostAppSlug'
 import { useUpdateConnector } from '@src/composables/useUpdateConnector'
+import { SpeckleApiLoader } from '@src/loader/SpeckleApiLoader'
+import { unzipModelObjects } from './compression'
 
 export class AsyncPause {
   private lastPauseTime = 0
@@ -129,7 +131,8 @@ function processObjectNode(
     console.log('⚠️ HAS objects', color)
     if (color) {
       res.color = color
-      res.shouldColor = true
+      // Don't override shouldColor for conditional formatting - keep the selection state
+      // res.shouldColor = true  // REMOVED: This was overriding cross-filter selection state
     }
   }
   return res
@@ -158,40 +161,8 @@ export type ReceiveInfo = {
   workspaceName?: string
   canHideBranding: boolean
   version?: string
-}
-
-export type PreGetObjects = {
-  modelExists: boolean
-  objectCount?: number
-}
-
-async function getPreGetObjects(commaSeparatedModelIds: string): Promise<PreGetObjects[]> {
-  const modelIds = (commaSeparatedModelIds as string).split(',')
-  const preGetObjects = []
-
-  for await (const id of modelIds) {
-    const res = await getPreGetObjectsForModel(id)
-    preGetObjects.push(res)
-  }
-  return preGetObjects
-}
-
-async function getPreGetObjectsForModel(id: string): Promise<PreGetObjects> {
-  try {
-    const preGetObjectsRes = await fetch(`http://localhost:29364/pre-get-objects/${id}`)
-
-    if (!preGetObjectsRes.body) {
-      console.log('No response body for pre get objects')
-      return {
-        modelExists: false,
-        objectCount: null
-      } as PreGetObjects
-    }
-
-    return (await preGetObjectsRes.json()) as PreGetObjects
-  } catch (error) {
-    console.log(error)
-  }
+  token: string
+  projectId?: string
 }
 
 async function getReceiveInfo(id) {
@@ -200,126 +171,40 @@ async function getReceiveInfo(id) {
     const response = await fetch(`http://localhost:29364/user-info/${ids[0]}`)
     if (!response.body) {
       console.error('No response body')
-      return
+      return { desktopServiceError: true }
     }
 
     return await response.json()
   } catch (error) {
     console.log(error)
-    console.log("User infp couldn't retrieved from local server.")
+    console.log("User info couldn't retrieved from local server.")
+    return { desktopServiceError: true }
   }
 }
 
-async function fetchStreamedData(commaSeparatedModelIds: string, totalObjectCount: number) {
-  const modelIds = (commaSeparatedModelIds as string).split(',')
+async function fetchFromSpeckleApi(
+  objectIds: string,
+  serverUrl: string,
+  projectId: string,
+  token: string
+): Promise<object[][]> {
+  const ids = objectIds.split(',')
   const modelObjects = []
 
-  let loadedObjectCount = 0
-
-  for await (const id of modelIds) {
-    const objects = await fetchStreamedDataForModel(id, totalObjectCount, loadedObjectCount)
-    modelObjects.push(objects)
-    loadedObjectCount += objects.length
-  }
-  return modelObjects
-}
-
-async function fetchStreamedDataForModel(
-  id: string,
-  totalObjectCount: number,
-  loadedObjectCount: number
-) {
-  console.log(loadedObjectCount, totalObjectCount)
-
-  try {
-    const visualStore = useVisualStore()
-    const response = await fetch(`http://localhost:29364/get-objects/${id}`)
-
-    if (!response.body) {
-      console.error('No response body')
-      return
-    }
-
-    const reader = response.body.getReader()
-    const decoder = new TextDecoder()
-    const objects = []
-    let buffer = ''
-
-    const start = performance.now()
-    console.log('Streaming started...')
-    for await (const chunk of readStream(reader)) {
-      // chucks.push(chuck)
-      buffer += decoder.decode(chunk, { stream: true })
-
-      let boundary
-      while ((boundary = buffer.indexOf('\n')) !== -1) {
-        const jsonString = buffer.slice(0, boundary)
-        buffer = buffer.slice(boundary + 1)
-
-        try {
-          const obj = JSON.parse(jsonString)
-          objects.push(obj)
-          visualStore.setLoadingProgress(
-            'Loading objects from storage',
-            (objects.length + loadedObjectCount) / totalObjectCount
-          )
-          // console.log('Loading', (objects.length + loadedObjectCount) / totalObjectCount)
-
-          // console.log('Received object:', jsonObject)
-        } catch (e) {
-          console.error('Invalid JSON chunk:', jsonString)
-        }
-      }
-    }
+  for (const objectId of ids) {
     try {
-      const obj = JSON.parse(buffer)
-      objects.push(obj)
-      // console.log('Received object:', jsonObject)
-    } catch (e) {
-      console.error('Invalid JSON chunk:', buffer)
-    }
-
-    const end = performance.now()
-    console.log(`Objects streamed in: ${(end - start) / 1000} s`)
-
-    const startObjectCleanup = performance.now()
-    // Skips first element
-    for (let i = 1; i < objects.length; i++) {
-      const obj = objects[i]
-      if (obj.speckle_type) {
-        if (obj.speckle_type.includes('Objects.Data.DataObject')) {
-          delete obj.properties
-        }
-      }
-      delete obj.__closure
-    }
-    const endObjectCleanup = performance.now()
-    console.log(`Objects cleaned up in: ${(endObjectCleanup - startObjectCleanup) / 1000} s`)
-
-    try {
-      const sizeInBytes = new TextEncoder().encode(JSON.stringify(objects)).length
-      const sizeInMB = sizeInBytes / (1024 * 1024)
-      console.log(`Size of objects: ${sizeInMB} MB`)
+      console.log(`Downloading from Speckle API: ${objectId}`)
+      const loader = new SpeckleApiLoader(serverUrl, projectId, token)
+      const objects = await loader.downloadObjectsWithChildren(objectId)
+      modelObjects.push(objects)
+      console.log(`Downloaded ${objects.length} objects from Speckle`)
     } catch (error) {
-      console.log("Can't calculate the size of the model")
-      console.log(error)
+      console.error(`Failed to download objects from Speckle:`, error)
+      throw error
     }
-
-    return objects
-  } catch (error) {
-    console.log(error)
-    console.log("Objects couldn't retrieved from local server.")
-  } finally {
-    console.log('Streaming finished!')
   }
-}
 
-async function* readStream(reader) {
-  while (true) {
-    const { done, value } = await reader.read()
-    if (done) break
-    yield value
-  }
+  return modelObjects
 }
 
 export async function processMatrixView(
@@ -327,7 +212,8 @@ export async function processMatrixView(
   host: powerbi.extensibility.visual.IVisualHost,
   hasColorFilter: boolean,
   settings: SpeckleVisualSettingsModel,
-  onSelectionPair: (objId: string, selectionId: powerbi.extensibility.ISelectionId) => void
+  onSelectionPair: (objId: string, selectionId: powerbi.extensibility.ISelectionId) => void,
+  internalizedData?: string
 ): Promise<SpeckleDataInput> {
   const visualStore = useVisualStore()
   const objectIds = [],
@@ -340,10 +226,92 @@ export async function processMatrixView(
   const localMatrixView = matrixView.rows.root.children
   let id = null
 
-  if (hasColorFilter) {
-    id = localMatrixView[0].children[0].values[0].value as unknown as string
-  } else {
-    id = localMatrixView[0].values[0].value as unknown as string
+  // Safety check for matrix data structure
+  if (!localMatrixView || localMatrixView.length === 0) {
+    throw new Error('Matrix view has no data rows')
+  }
+
+  try {
+    if (hasColorFilter) {
+      if (
+        !localMatrixView[0].children ||
+        localMatrixView[0].children.length === 0 ||
+        !localMatrixView[0].children[0].values
+      ) {
+        throw new Error('Matrix view structure is incomplete for color filter mode')
+      }
+      id = localMatrixView[0].children[0].values[0].value as unknown as string
+    } else {
+      if (!localMatrixView[0].values || !localMatrixView[0].values[0]) {
+        throw new Error('Matrix view structure is incomplete for normal mode')
+      }
+      id = localMatrixView[0].values[0].value as unknown as string
+    }
+  } catch (error) {
+    console.error('Error accessing matrix data:', error)
+    throw new Error(`Failed to extract root object ID from matrix: ${error.message}`)
+  }
+
+  // Check for internalized data but ONLY if it matches current matrix data
+  let internalizedModelObjects: object[][] | undefined = undefined
+  if (settings.dataLoading.internalizeData.value && internalizedData) {
+    console.log('📁 Checking internalized data in processMatrixView')
+
+    try {
+      internalizedModelObjects = unzipModelObjects(internalizedData)
+
+      if (internalizedModelObjects && internalizedModelObjects.length > 0) {
+        // CRITICAL: Validate that internalized data matches current matrix data
+        const internalizedRootId = (internalizedModelObjects[0][0] as any).id
+        if (internalizedRootId !== id) {
+          console.log(
+            `📁 Internalized data mismatch: stored=${internalizedRootId}, current=${id}. Using fresh data.`
+          )
+          internalizedModelObjects = undefined // Clear internalized data - use fresh data instead
+        } else {
+          console.log(
+            '📁 Successfully validated internalized data matches current matrix:',
+            internalizedModelObjects.length,
+            'models'
+          )
+        }
+      }
+
+      if (internalizedModelObjects && internalizedModelObjects.length > 0) {
+        // Set dummy receiveInfo to prevent UI errors
+        if (!visualStore.receiveInfo) {
+          visualStore.setReceiveInfo({
+            userEmail: 'offline@speckle.systems',
+            serverUrl: 'offline',
+            sourceApplication: 'PowerBI Offline',
+            workspaceId: 'offline',
+            workspaceName: 'Offline Workspace',
+            workspaceLogo: '',
+            version: '1.0.0',
+            canHideBranding: false,
+            token: 'offline',
+            projectId: 'offline'
+          })
+        }
+
+        // Only reload if switching models or not already loaded
+        const needsReload =
+          !visualStore.isViewerObjectsLoaded || visualStore.lastLoadedRootObjectId !== id
+        if (needsReload) {
+          console.log('🔄 Forcing viewer reload for internalized data (model switch or first load)')
+          visualStore.setViewerReloadNeeded()
+          visualStore.setViewerReadyToLoad(true)
+          visualStore.setLoadingProgress('📁 Loading from file', null)
+        } else {
+          console.log('📁 Internalized data already loaded, skipping reload')
+        }
+        visualStore.lastLoadedRootObjectId = id // Set to current ID to skip API calls
+      } else {
+        console.error('📁 Failed to unzip internalized data')
+      }
+    } catch (error) {
+      console.error('📁 Error processing internalized data:', error)
+    }
   }
 
   // const id = localMatrixView[0].values[0].value as unknown as string
@@ -352,48 +320,96 @@ export async function processMatrixView(
 
   let modelObjects: object[][] = undefined
 
-  if (visualStore.isLoadingFromFile) {
-    console.log('The data is loading from file, skipping the streaming it.')
-  }
-
-  if (visualStore.lastLoadedRootObjectId !== id && !visualStore.isLoadingFromFile) {
+  if (
+    visualStore.lastLoadedRootObjectId !== id &&
+    !visualStore.isLoadingFromFile &&
+    !internalizedModelObjects
+  ) {
     const start = performance.now()
 
-    const getPreGetObjectsRes: PreGetObjects[] = await getPreGetObjects(id)
-
-    if (getPreGetObjectsRes.some((preGetObjects) => preGetObjects.modelExists === false)) {
-      visualStore.setCommonError(
-        'Version Object ID is not found in storage. Please make sure you placed correct field or consider refreshing your data via data connector.'
-      )
-      visualStore.setViewerReadyToLoad(false)
-      return
-    }
-
+    // Get receive info from desktop service to populate visual store
     const receiveInfo = await getReceiveInfo(id)
-    if (receiveInfo) {
+    let desktopServiceUnavailable = false
+
+    if (receiveInfo && !receiveInfo.desktopServiceError) {
       visualStore.setReceiveInfo({
-        userEmail: receiveInfo.email,
-        serverUrl: receiveInfo.server,
-        sourceApplication: getSlugFromHostAppNameAndVersion(receiveInfo.sourceApplication),
-        workspaceId: receiveInfo.workspaceId,
-        workspaceName: receiveInfo.workspaceName,
-        workspaceLogo: receiveInfo.workspaceLogo,
-        version: receiveInfo.version,
-        canHideBranding: receiveInfo.canHideBranding
+        userEmail: receiveInfo.email || receiveInfo.Email,
+        serverUrl: receiveInfo.server || receiveInfo.Server,
+        sourceApplication: getSlugFromHostAppNameAndVersion(
+          receiveInfo.sourceApplication || receiveInfo.SourceApplication
+        ),
+        workspaceId: receiveInfo.workspaceId || receiveInfo.WorkspaceId,
+        workspaceName: receiveInfo.workspaceName || receiveInfo.WorkspaceName,
+        workspaceLogo: receiveInfo.workspaceLogo || receiveInfo.WorkspaceLogo,
+        version: receiveInfo.version || receiveInfo.Version,
+        canHideBranding: receiveInfo.canHideBranding ?? receiveInfo.CanHideBranding,
+        token: receiveInfo.weakToken || receiveInfo.WeakToken,
+        projectId: receiveInfo.projectId || receiveInfo.ProjectId
       })
-      console.log(`Receive info retrieved from desktop service`, receiveInfo)
+      console.log(`Receive info retrieved from desktop service - credentials loaded`)
+    } else {
+      desktopServiceUnavailable = true
+      console.log('Desktop service unavailable - cannot retrieve credentials')
     }
 
-    const totalObjectCount = getPreGetObjectsRes.reduce((sum, obj) => {
-      return sum + (obj.objectCount ?? 0)
-    }, 0)
+    // Now get the data from visual store for Speckle API download
+    const token = visualStore.receiveInfo?.token
+    const serverUrl = visualStore.receiveInfo?.serverUrl
+    const projectId = visualStore.receiveInfo?.projectId
+
+    if (!token || !serverUrl || !projectId) {
+      if (desktopServiceUnavailable) {
+        visualStore.setCommonError(
+          'Speckle Desktop Service is not running. Please start Speckle Desktop Services and refresh data.'
+        )
+      } else {
+        visualStore.setCommonError(
+          'Missing Speckle credentials. Please refresh the data from the data connector.'
+        )
+      }
+      visualStore.setViewerReadyToLoad(false)
+      return {
+        modelObjects: [],
+        objectIds: [],
+        selectedIds: [],
+        colorByIds: null,
+        objectTooltipData: new Map(),
+        isFromStore: false
+      }
+    }
 
     visualStore.setViewerReadyToLoad(true)
-    // stream data
-    modelObjects = await fetchStreamedData(id, totalObjectCount)
+
+    console.log('Downloading objects directly from Speckle API...')
+    console.log(`Server: ${serverUrl}, Project: ${projectId}, Object: ${id}`)
+    try {
+      modelObjects = await fetchFromSpeckleApi(id, serverUrl, projectId, token)
+      console.log('Successfully downloaded from Speckle API')
+
+      // Debug: Check what we're passing to the viewer
+      if (modelObjects && modelObjects.length > 0 && modelObjects[0].length > 0) {
+        console.log('ModelObjects structure:', {
+          totalModels: modelObjects.length,
+          firstModelObjectCount: modelObjects[0].length,
+          firstObject: modelObjects[0][0]
+        })
+      }
+    } catch (error) {
+      console.error('Failed to download from Speckle API:', error)
+      visualStore.setCommonError(`Failed to download objects from Speckle: ${error.message}`)
+      visualStore.setViewerReadyToLoad(false)
+      return {
+        modelObjects: [],
+        objectIds: [],
+        selectedIds: [],
+        colorByIds: null,
+        objectTooltipData: new Map(),
+        isFromStore: false
+      }
+    }
 
     visualStore.setViewerReloadNeeded() // they should be marked as deferred action bc of update function complexity.
-    visualStore.setLoadingProgress('Loading objects into viewer', null)
+    visualStore.setLoadingProgress('🌍 Loading objects into viewer', null)
     console.log(`🚀 Upload is completed in ${(performance.now() - start) / 1000} s!`)
   }
 
@@ -459,6 +475,7 @@ export async function processMatrixView(
     localMatrixView.forEach((obj) => {
       const processedObjectIdLevels = processObjectIdLevel(obj, host, matrixView)
 
+      // Apply conditional formatting color if present, regardless of selection state
       if (processedObjectIdLevels.color) {
         let group = colorByIds.find((g) => g.color === processedObjectIdLevels.color)
         if (!group) {
@@ -468,7 +485,11 @@ export async function processMatrixView(
           }
           colorByIds.push(group)
         }
+        // Always add to color group if color is specified (conditional formatting)
         group.objectIds.push(processedObjectIdLevels.id)
+      } else if (processedObjectIdLevels.shouldColor) {
+        // Only use shouldColor flag when there's no conditional formatting
+        // This preserves the original cross-filter coloring behavior
       }
 
       objectIds.push(processedObjectIdLevels.id)
@@ -538,11 +559,11 @@ export async function processMatrixView(
   previousPalette = host.colorPalette['colorPalette']
 
   return {
-    modelObjects,
+    modelObjects: internalizedModelObjects || modelObjects, // Use internalized data if available
     objectIds,
     selectedIds,
     colorByIds: colorByIds.length > 0 ? colorByIds : null,
     objectTooltipData,
-    isFromStore: false
+    isFromStore: !!internalizedModelObjects // true if loaded from internalized data
   }
 }

src/powerbi-visual/src/visual.ts

@@ -10,6 +10,7 @@ import { selectionHandlerKey, tooltipHandlerKey } from 'src/injectionKeys'
 import { SpeckleDataInput } from './types'
 import { processMatrixView, ReceiveInfo, validateMatrixView } from './utils/matrixViewUtils'
 import { SpeckleVisualSettingsModel } from './settings/visualSettingsModel'
+import { unzipModelObjects } from './utils/compression'
 
 import TooltipHandler from './handlers/tooltipHandler'
 import SelectionHandler from './handlers/selectionHandler'
@@ -21,7 +22,6 @@ import ITooltipService = powerbi.extensibility.ITooltipService
 
 import { pinia } from './plugins/pinia'
 import { useVisualStore } from './store/visualStore'
-import { unzipModelObjects } from './utils/compression'
 
 // noinspection JSUnusedGlobalSymbols
 export class Visual implements IVisual {
@@ -88,13 +88,43 @@ export class Visual implements IVisual {
 
     // @ts-ignore
     console.log('⤴️ Update type 👉', powerbi.VisualUpdateType[options.type])
+
     this.formattingSettings = this.formattingSettingsService.populateFormattingSettingsModel(
       SpeckleVisualSettingsModel,
       options.dataViews[0]
     )
 
     visualStore.setFormattingSettings(this.formattingSettings)
-    console.log('Selector colors', this.formattingSettings.colorSelector)
+    console.log(
+      'Data Loading - Internalize Data:',
+      this.formattingSettings.dataLoading.internalizeData.value
+    )
+
+    // Handle toggle state changes
+    const currentToggleState = this.formattingSettings.dataLoading.internalizeData.value
+    const previousToggleState = visualStore.previousToggleState
+
+    // Detect user toggle changes
+    if (previousToggleState !== undefined && currentToggleState !== previousToggleState) {
+      console.log('🔄 User changed toggle from', previousToggleState, 'to', currentToggleState)
+
+      if (currentToggleState) {
+        // Toggle switched ON - internalize via streaming
+        if (visualStore.isViewerObjectsLoaded && visualStore.lastLoadedRootObjectId) {
+          console.log('📁 Toggle ON - starting internalization')
+          await this.internalizeCurrentViewerData()
+        } else {
+          console.log('📁 Toggle ON - no active session to internalize')
+        }
+      } else {
+        // Toggle switched OFF - remove internalized data
+        console.log('🗑️ Toggle OFF - removing internalized data')
+        this.removeInternalizedData()
+      }
+    }
+
+    // CRITICAL: Always update the previous state for next comparison
+    visualStore.setPreviousToggleState(currentToggleState)
 
     try {
       const matrixView = options.dataViews[0].matrix
@@ -114,16 +144,10 @@ export class Visual implements IVisual {
           return
         case powerbi.VisualUpdateType.Data:
           try {
-            // read saved data from file if any
-            if (
-              !visualStore.isViewerObjectsLoaded &&
-              this.isFirstViewerLoad &&
-              options.dataViews[0].metadata.objects
-            ) {
-              const chunks = options.dataViews[0].metadata.objects.storedData
-                ?.speckleObjects as string
-              const objectsFromFile = unzipModelObjects(chunks)
+            // read saved settings from file if any
+            console.log('🔍 Checking for other saved settings:')
 
+            if (!visualStore.isViewerObjectsLoaded && options.dataViews[0].metadata.objects) {
               if (options.dataViews[0].metadata.objects.viewMode?.defaultViewMode as string) {
                 console.log(
                   `Default View Mode: ${
@@ -198,7 +222,9 @@ export class Visual implements IVisual {
 
               if (camera && 'zoomOnFilter' in camera) {
                 console.log(
-                  `Zoom on filter?: ${options.dataViews[0].metadata.objects.camera?.zoomOnFilter as boolean}`
+                  `Zoom on filter?: ${
+                    options.dataViews[0].metadata.objects.camera?.zoomOnFilter as boolean
+                  }`
                 )
 
                 visualStore.setIsZoomOnFilterActive(
@@ -206,31 +232,59 @@ export class Visual implements IVisual {
                 )
               }
 
-              // get receive info from file for mixpanel
+              // Log persisted data loading setting but don't force sync
+              if (
+                options.dataViews[0].metadata.objects.dataLoading?.internalizeData !== undefined
+              ) {
+                console.log(
+                  `Stored Data Loading - Internalize Data: ${
+                    options.dataViews[0].metadata.objects.dataLoading?.internalizeData as boolean
+                  }`
+                )
+              }
+
+              // get receive info from file for persistence
               try {
                 const receiveInfoFromFile = JSON.parse(
                   options.dataViews[0].metadata.objects.storedData?.receiveInfo as string
                 ) as ReceiveInfo
-                visualStore.setReceiveInfo(receiveInfoFromFile)
+                // Don't call setReceiveInfo here as it would trigger another save
+                visualStore.receiveInfo = receiveInfoFromFile
               } catch (error) {
                 console.warn(error)
-                console.log('missing mixpanel info')
-              }
-
-              const savedVersionObjectId = objectsFromFile.map((o) => o[0].id).join(',')
-              if (visualStore.lastLoadedRootObjectId !== savedVersionObjectId) {
-                this.tryReadFromFile(objectsFromFile, visualStore)
+                console.log('missing stored receive info')
               }
             }
 
+            // Check for internalized data
+            const internalizedData = options.dataViews[0].metadata.objects?.storedData
+              ?.speckleObjects as string
+
             const input = await processMatrixView(
               matrixView,
               this.host,
               validationResult.colorBy,
               this.formattingSettings,
-              (obj, id) => this.selectionHandler.set(obj, id)
+              (obj, id) => this.selectionHandler.set(obj, id),
+              internalizedData
             )
             this.updateViewer(input)
+
+            // Auto-internalize new API data if toggle is ON and this is fresh data (not from store)
+            // Imagine that user has a visual and select internalizing data and changes the data source
+            // This will automatically internalize the new data
+            if (
+              this.formattingSettings.dataLoading.internalizeData.value &&
+              input.modelObjects &&
+              input.modelObjects.length > 0 &&
+              !input.isFromStore
+            ) {
+              console.log('📦 Auto-internalizing new API data since toggle is ON')
+              // Trigger internalization after objects are loaded
+              setTimeout(() => {
+                this.internalizeCurrentViewerData()
+              }, 2000) // avoid a race condition (i know)
+            }
           } catch (error) {
             console.error('Data update error', error ?? 'Unknown')
           }
@@ -258,9 +312,8 @@ export class Visual implements IVisual {
   }
 
   public getFormattingModel(): powerbi.visuals.FormattingModel {
-    console.log('Showing Formatting settings', this.formattingSettings)
+    console.log('🎨 getFormattingModel called')
     const model = this.formattingSettingsService.buildFormattingModel(this.formattingSettings)
-    console.log('Formatting model was created', model)
     return model
   }
 
@@ -276,14 +329,13 @@ export class Visual implements IVisual {
       // we should give some time to Vue to render ViewerWrapper component to be able to have proper emitter setup. Happiness level 6/10
       setTimeout(() => {
         visualStore.setDataInput(input)
-        // visualStore.writeObjectsToFile(input.objects)
       }, 250)
     }
   }
 
   private tryReadFromFile(objectsFromFile: object[][], visualStore) {
     visualStore.setViewerReadyToLoad(true)
-    visualStore.setIsLoadingFromFile(true) // to block unnecessary streaming data if bg service is running
+    visualStore.setIsLoadingFromFile(true)
     setTimeout(() => {
       visualStore.loadObjectsFromFile(objectsFromFile)
       this.isFirstViewerLoad = false
@@ -291,6 +343,137 @@ export class Visual implements IVisual {
     console.log(`${objectsFromFile.length} objects retrieved from persistent properties!`)
   }
 
+  private async internalizeCurrentViewerData() {
+    const visualStore = useVisualStore()
+
+    // Get the current root object ID from the last loaded data
+    if (!visualStore.lastLoadedRootObjectId) {
+      console.log('📁 No root object ID to internalize')
+      return
+    }
+
+    try {
+      console.log('📁 Starting internalization via desktop service streaming...')
+
+      visualStore.setLoadingProgress('📦 Internalizing data...', null)
+
+      // Use desktop service for internalization
+      // TBD: getting objects from viewer caused two issue:
+      // - Data format -> we need to make an extra operation to match with the offline loader
+      // - Memory -> need to save data two times so sometimes causes memory issues
+      const rootObjectIds = visualStore.lastLoadedRootObjectId
+      const projectId = visualStore.receiveInfo?.projectId
+      
+      // Handle federated models by processing each object ID separately
+      const objectIds = rootObjectIds.split(',')
+      let allStreamedObjects = []
+      
+      for (const objectId of objectIds) {
+        console.log(`📁 Fetching objects for ID: ${objectId}`)
+        
+        // For federated models, pass project ID explicitly to avoid "project id is not set" error
+        const url = projectId 
+          ? `http://localhost:29364/get-objects/${objectId}?projectId=${projectId}`
+          : `http://localhost:29364/get-objects/${objectId}`
+        
+        const response = await fetch(url)
+        
+        if (!response.body) {
+          console.error(`📁 No response body from desktop service for ${objectId}`)
+          continue
+        }
+
+        const reader = response.body.getReader()
+        const decoder = new TextDecoder()
+        let allObjectsData = ''
+
+        console.log(`📁 Streaming objects from desktop service for ${objectId}...`)
+
+        while (true) {
+          const { done, value } = await reader.read()
+          if (done) break
+
+          allObjectsData += decoder.decode(value, { stream: true })
+        }
+
+        // Parse NDJSON (newline-delimited JSON) format
+        const lines = allObjectsData.trim().split('\n')
+        const objectsForThisId = lines.map((line) => JSON.parse(line))
+        
+        console.log(`📁 Streamed ${objectsForThisId.length} objects for ID ${objectId}`)
+        allStreamedObjects.push(...objectsForThisId)
+      }
+      
+      const streamedObjects = allStreamedObjects
+      
+      if (streamedObjects.length === 0) {
+        console.error('📁 No objects retrieved from desktop service')
+        visualStore.clearLoadingProgress()
+        return
+      }
+      
+      console.log(`📁 Retrieved ${streamedObjects.length} total objects from desktop service`)
+
+      // Clean up objects to reduce file size (same as desktop service does)
+      const cleanedObjects = streamedObjects.map((obj: any, index: number) => {
+        // Skip first object (root), clean others
+        if (index === 0) return obj
+
+        const cleanedObj = { ...obj }
+
+        // Remove unnecessary properties
+        if (cleanedObj.speckle_type?.includes('Objects.Data.DataObject')) {
+          delete cleanedObj.properties
+        }
+        delete cleanedObj.__closure
+
+        return cleanedObj
+      })
+
+      console.log(`📁 Cleaned objects: ${cleanedObjects.length} total`)
+
+      // Wrap in array format expected by viewer (object[][])
+      const modelObjectsArray = [cleanedObjects]
+
+      // Use existing writeObjectsToFile method from visualStore
+      visualStore.writeObjectsToFile(modelObjectsArray)
+
+      // Clear loading message immediately when done
+      visualStore.clearLoadingProgress()
+
+      console.log('📁 Successfully internalized data via desktop service!')
+    } catch (error) {
+      console.error('📁 Failed to internalize via desktop service:', error)
+
+      // Clear loading message immediately on error
+      visualStore.clearLoadingProgress()
+    }
+  }
+
+  private removeInternalizedData() {
+    const visualStore = useVisualStore()
+
+    try {
+      // Clear stored data from PowerBI file
+      this.host.persistProperties({
+        merge: [
+          {
+            objectName: 'storedData',
+            properties: {
+              speckleObjects: null,
+              receiveInfo: null
+            },
+            selector: null
+          }
+        ]
+      })
+
+      console.log('🗑️ Successfully removed internalized data from file!')
+    } catch (error) {
+      console.error('🗑️ Failed to remove internalized data:', error)
+    }
+  }
+
   public async destroy() {
     await this.clear()
   }