feat: Issues helper function (#194)

* helper func

* register

* query: remove code duplicate

* discussions gone, issues in.

* add issue urls

* add optional replies input

src/powerbi-data-connector/Speckle.pq

@@ -189,6 +189,12 @@ shared Speckle.Models.MaterialQuantities = Value.ReplaceType(
     type function (inputTable as table, optional addPrefix as logical) as table
 );
 
+[DataSource.Kind = "Speckle"]
+shared Speckle.Project.Issues = Value.ReplaceType(
+    Speckle.LoadFunction("Project.Issues.pqm"),
+    type function (url as Uri.Type, optional getReplies as logical) as table
+);
+
 [DataSource.Kind = "Speckle", Publish="GetByUrl.Publish"]
 shared Speckle.GetByUrl = Value.ReplaceType(
     Speckle.LoadFunction("GetByUrl.pqm"),

src/powerbi-data-connector/speckle/GetByUrl.pqm

@@ -101,9 +101,15 @@
                     // extract all data tables
                     allTables = List.Transform(modelsData, each [Data]),
 
+                    // extract all root object IDs
+                    allRootIds = List.Transform(modelsData, each [RootObjectId]),
+
                     // extract all encoded userInfo strings
                     allEncodedUserInfos = List.Transform(modelsData, each [EncodedUserInfo]),
 
+                    // combine all root object IDs into a comma-separated string
+                    combinedRootIds = Text.Combine(allRootIds, ","),
+
                     // combine all encoded userInfo strings with delimiter |||
                     // (delimiter chosen to avoid conflicts with base64 characters)
                     combinedEncodedUserInfos = Text.Combine(allEncodedUserInfos, "|||"),
@@ -111,10 +117,13 @@
                     // combine all data tables
                     combinedData = Table.Combine(allTables),
 
-                    // replace the "Model Info" column with the combined encoded strings
+                    // replace both columns with combined values
                     transformedData = Table.TransformColumns(
                         combinedData,
-                        {"Model Info", each combinedEncodedUserInfos}
+                        {
+                            {"Version Object ID", each combinedRootIds},
+                            {"Model Info", each combinedEncodedUserInfos}
+                        }
                     ),
 
                     // expand properties column if requested and if it exists
@@ -157,13 +166,20 @@
                         url = url
                     ],
 
+                    // try to send to desktop service for backward compatibility (non-blocking)
+                    // must be called BEFORE encoding to ensure it executes
+                    desktopServiceSent = TrySendToDesktopService(userInfoData),
+
                     // encode userInfoData as base64 JSON string
                     encodedUserInfo = EncodeUserInfo(userInfoData),
 
-                    // replace "Model Info" column with encoded string
+                    // replace both columns with appropriate values
                     transformedData = Table.TransformColumns(
                         structuredData,
-                        {"Model Info", each encodedUserInfo}
+                        {
+                            {"Version Object ID", each rootObjectId},
+                            {"Model Info", each if desktopServiceSent or not desktopServiceSent then encodedUserInfo else encodedUserInfo}
+                        }
                     ),
 
                     // expand properties column if requested and if it exists
@@ -182,6 +198,30 @@
                 {permissionCheck[message], permissionCheck[code]}
             ),
                 
+        // helper function to try sending user info to desktop service for backward compatibility
+        // returns true if successful, false otherwise (non-blocking)
+        TrySendToDesktopService = (userInfoData as record) =>
+            try
+                let
+                    userInfoJson = Json.FromValue(userInfoData),
+                    response = Web.Contents(
+                        "http://127.0.0.1:29364/store-user-info",
+                        [
+                            Headers = [
+                                #"Content-Type" = "application/json",
+                                #"Method" = "POST"
+                            ],
+                            Content = userInfoJson,
+                            ManualStatusHandling = {400, 401, 403, 404, 500, 502, 503, 504},
+                            Timeout = #duration(0, 0, 0, 2)
+                        ]
+                    ),
+                    statusCode = Value.Metadata(response)[Response.Status]
+                in
+                    statusCode >= 200 and statusCode < 300
+            otherwise
+                false,
+
         // function to process a single model and get its data
         ProcessSingleModel = (baseUrl, projectId, modelId, versionId) =>
             let
@@ -221,15 +261,22 @@
                     url = singleModelUrl
                 ],
 
+                // try to send to desktop service for backward compatibility (non-blocking)
+                // must be called BEFORE encoding to ensure it executes
+                desktopServiceSent = TrySendToDesktopService(userInfoData),
+
                 // encode userInfoData as base64 JSON string
                 encodedUserInfo = EncodeUserInfo(userInfoData),
 
                 // add the model name as context - with version id if exists
+                // reference desktopServiceSent to force evaluation
                 result = Table.AddColumn(
                     structuredData,
                     "Source Model",
                     each if versionId <> null then
                         Text.Combine({modelName, "-", versionId})
+                    else if desktopServiceSent or not desktopServiceSent then
+                        modelName
                     else
                         modelName,
                     type text

src/powerbi-data-connector/speckle/api/GetStructuredData.pqm

@@ -55,6 +55,7 @@
                     [
                         #"Object IDs" = record[id], // Object IDs
                         #"Speckle Type" = record[speckle_type], // Speckle Type
+                        #"Version Object ID" = rootId,
                         #"Model Info" = rootId,
                         data = cleanedRecord // Data
                     ]

src/powerbi-data-connector/speckle/api/Models.Federate.pqm

@@ -1,4 +1,4 @@
-// function for federating multiple tables by combining them and creating a concatenated Model Info field
+// function for federating multiple tables by combining them and creating concatenated Version Object ID and Model Info fields
 (tables as list, optional excludeData as logical) as table =>
 let
     ViewerOnly = if excludeData = null then false else excludeData,
@@ -8,19 +8,31 @@ let
         tables,
         each
             if ViewerOnly then
-                Table.SelectColumns(_, {"Model Info", "Object IDs"}, MissingField.Ignore)
+                Table.SelectColumns(_, {"Version Object ID", "Model Info", "Object IDs"}, MissingField.Ignore)
             else
                 _
     ),
 
     CombinedTable = Table.Combine(ProcessedTables),
 
-    DistinctModelInfo = List.Distinct(CombinedTable[Model Info]),
-    ConcatenatedModelInfo = Text.Combine(DistinctModelInfo, ","),
+    DistinctVersionObjectIDs = List.Distinct(CombinedTable[Version Object ID]),
+    ConcatenatedVersionObjectIDs = Text.Combine(DistinctVersionObjectIDs, ","),
 
-    // Replace all Metadata values with the concatenated string
-    FederatedTable = Table.ReplaceValue(
+    DistinctModelInfo = List.Distinct(CombinedTable[Model Info]),
+    ConcatenatedModelInfo = Text.Combine(DistinctModelInfo, "|||"),
+
+    // Replace all Version Object ID values with the concatenated string
+    TableWithVersionObjectID = Table.ReplaceValue(
         CombinedTable,
+        each [Version Object ID],
+        ConcatenatedVersionObjectIDs,
+        Replacer.ReplaceText,
+        {"Version Object ID"}
+    ),
+
+    // Replace all Model Info values with the concatenated string
+    FederatedTable = Table.ReplaceValue(
+        TableWithVersionObjectID,
         each [Model Info],
         ConcatenatedModelInfo,
         Replacer.ReplaceText,

src/powerbi-data-connector/speckle/api/Project.Issues.pqm

@@ -0,0 +1,164 @@
+// Function for getting issues from Speckle projects, models, or versions
+(url as text, optional getReplies as logical) as table =>
+    let
+        // Import required functions
+        Parser = Extension.LoadFunction("Parser.pqm"),
+        ApiFetch = Extension.LoadFunction("Api.Fetch.pqm"),
+        
+        // Set default value for getReplies parameter
+        getRepliesValue = if getReplies = null then false else getReplies,
+    
+        // Extension.LoadFunction logic for importing functions from other files
+        Extension.LoadFunction = (fileName as text) =>
+            let
+                binary = Extension.Contents(fileName),
+                asText = Text.FromBinary(binary)
+            in
+                try
+                    Expression.Evaluate(asText, #shared)
+                catch (e) =>
+                    error
+                        [
+                            Reason = "Extension.LoadFunction Failure",
+                            Message.Format = "Loading '#{0}' failed - '#{1}': '#{2}'",
+                            Message.Parameters = {fileName, e[Reason], e[Message]},
+                            Detail = [File = fileName, Error = e]
+                        ],
+        
+        // Parse the URL to get necessary components with fallback for project-only URLs
+        parsedUrl = try Parser(url) otherwise 
+            // Custom parsing for project-only URLs
+            let
+                urlParts = Uri.Parts(url),
+                baseUrl = Text.Combine({urlParts[Scheme], "://", urlParts[Host]}),
+                pathSegments = List.Select(Text.Split(urlParts[Path], "/"), each _ <> ""),
+                projectId = if List.Count(pathSegments) >= 2 and pathSegments{0} = "projects" 
+                    then pathSegments{1} else null
+            in
+                if projectId = null then
+                    error [
+                        Reason = "Invalid URL",
+                        Message = "The URL must be a valid Speckle project URL in the format 'https://server/projects/PROJECT_ID' or include models/versions"
+                    ]
+                else
+                    [
+                        baseUrl = baseUrl,
+                        projectId = projectId,
+                        modelId = null,
+                        versionId = null
+                    ],
+        
+        server = parsedUrl[baseUrl],
+        projectId = parsedUrl[projectId],
+        modelId = parsedUrl[modelId],
+        versionId = parsedUrl[versionId],
+        
+        // Define the GraphQL query (single query for all scopes)
+        issuesQuery = "query Project($projectId: String!, $input: ProjectIssuesInput" & 
+            (if getRepliesValue then ", $repliesInput2: IssueRepliesInput" else "") & ") {
+            project(id: $projectId) {
+                issues(input: $input) {
+                    items {
+                        identifier
+                        title
+                        rawDescription
+                        status
+                        priority
+                        assignee {
+                            user {
+                                name
+                            }
+                        }
+                        dueDate
+                        labels {
+                            name
+                        }
+                        createdAt
+                        updatedAt
+                        resourceIdString
+                        id" & 
+                        (if getRepliesValue then "
+                        replies(input: $repliesInput2) {
+                            items {
+                                issueId
+                                id
+                                rawDescription
+                                createdAt
+                                author {
+                                    user {
+                                        name
+                                    }
+                                }
+                            }
+                        }" else "") & "
+                    }
+                }
+            }
+        }",
+        
+        // Build input variable dynamically based on URL scope
+        inputVariable = 
+            if versionId <> null then
+                // Version URL: resourceIdString = "MODEL_ID@VERSION_ID"
+                [
+                    limit = 10000,
+                    resourceIdString = modelId & "@" & versionId
+                ]
+            else if modelId <> null then
+                // Model URL: resourceIdString = MODEL_ID
+                [
+                    limit = 10000,
+                    resourceIdString = modelId
+                ]
+            else
+                // Project URL: no resourceIdString
+                [
+                    limit = 10000
+                ],
+        
+        // Build query variables
+        queryVariables = if getRepliesValue then
+            [
+                projectId = projectId,
+                input = inputVariable,
+                repliesInput2 = [limit = 10000]
+            ]
+            else
+            [
+                projectId = projectId,
+                input = inputVariable
+            ],
+        
+        // Make the API request using ApiFetch
+        result = ApiFetch(server, issuesQuery, queryVariables),
+        
+        // Extract issues from the response
+        issues = result[project][issues][items],
+        
+        // Transform to table structure with specified columns
+        issuesTable = Table.FromRecords(
+            List.Transform(issues, (issue) =>
+                let
+                    baseRecord = [
+                        ID = issue[identifier],
+                        Title = issue[title],
+                        Description = try issue[rawDescription] otherwise null,
+                        Status = try issue[status] otherwise null,
+                        Priority = try issue[priority] otherwise null,
+                        Assignee = try issue[assignee][user][name] otherwise null,
+                        #"Due Date" = try DateTime.From(issue[dueDate]) otherwise null,
+                        Labels = try List.Transform(issue[labels], each _[name]) otherwise {},
+                        #"Created at" = try DateTime.From(issue[createdAt]) otherwise null,
+                        #"Updated at" = try DateTime.From(issue[updatedAt]) otherwise null,
+                        URL = server & "/projects/" & projectId & "/models/" & issue[resourceIdString] & "#threadId=" & issue[id]
+                    ],
+                    recordWithReplies = if getRepliesValue then
+                        baseRecord & [Replies = try issue[replies][items] otherwise null]
+                    else
+                        baseRecord
+                in
+                    recordWithReplies
+            )
+        )
+    in
+        issuesTable