Merge pull request #6 from specklesystems/alan/receive-receipts

feat: Added receive receipts to PowerBI and branch URL support

.github/workflows/close-issue.yml

@@ -0,0 +1,78 @@
+name: Update issue Status
+
+on:
+  issues:
+    types: [closed]
+
+jobs:
+  update_issue:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get project data
+        env:
+          GITHUB_TOKEN: ${{secrets.GHPROJECT_TOKEN}}
+          ORGANIZATION: specklesystems
+          PROJECT_NUMBER: 9
+        run: |
+          gh api graphql --header 'GraphQL-Features: projects_next_graphql' -f query='
+            query($org: String!, $number: Int!) {
+              organization(login: $org){
+                projectNext(number: $number) {
+                  id
+                  fields(first:20) {
+                    nodes {
+                      id
+                      name
+                      settings
+                    }
+                  }
+                }
+              }
+            }' -f org=$ORGANIZATION -F number=$PROJECT_NUMBER > project_data.json
+
+          echo 'PROJECT_ID='$(jq '.data.organization.projectNext.id' project_data.json) >> $GITHUB_ENV
+          echo 'STATUS_FIELD_ID='$(jq '.data.organization.projectNext.fields.nodes[] | select(.name== "Status") | .id' project_data.json) >> $GITHUB_ENV
+          
+          echo "$PROJECT_ID"
+          echo "$STATUS_FIELD_ID"
+          
+          echo 'DONE_ID='$(jq '.data.organization.projectNext.fields.nodes[] | select(.name== "Status") | .settings | fromjson | .options[] | select(.name== "Done") | .id' project_data.json) >> $GITHUB_ENV
+          echo "$DONE_ID"
+
+      - name: Add Issue to project #it's already in the project, but we do this to get its node id!
+        env:
+          GITHUB_TOKEN: ${{secrets.GHPROJECT_TOKEN}}
+          ISSUE_ID: ${{ github.event.issue.node_id }}
+        run: |
+          item_id="$( gh api graphql --header 'GraphQL-Features: projects_next_graphql' -f query='
+            mutation($project:ID!, $id:ID!) {
+              addProjectNextItem(input: {projectId: $project, contentId: $id}) {
+                projectNextItem {
+                  id
+                }
+              }
+            }' -f project=$PROJECT_ID -f id=$ISSUE_ID --jq '.data.addProjectNextItem.projectNextItem.id')"
+          
+          echo 'ITEM_ID='$item_id >> $GITHUB_ENV
+          
+      - name: Update Status
+        env:
+          GITHUB_TOKEN: ${{secrets.GHPROJECT_TOKEN}}
+          ISSUE_ID: ${{ github.event.issue.node_id }}
+        run: |
+          gh api graphql --header 'GraphQL-Features: projects_next_graphql' -f query='
+            mutation($project:ID!, $status:ID!, $id:ID!, $value:String!) {
+              set_status: updateProjectNextItemField(
+                input: {
+                  projectId: $project
+                  itemId: $id
+                  fieldId: $status
+                  value: $value
+                }
+              ) {
+                projectNextItem {
+                  id
+                }
+              }
+            }' -f project=$PROJECT_ID -f status=$STATUS_FIELD_ID -f id=$ITEM_ID -f value=${{ env.DONE_ID }}
+          

.github/workflows/open-issue.yml

@@ -0,0 +1,50 @@
+name: Move new issues into Project
+
+on:
+  issues:
+    types: [opened]
+
+jobs:
+  track_issue:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get project data
+        env:
+          GITHUB_TOKEN: ${{secrets.GHPROJECT_TOKEN}}
+          ORGANIZATION: specklesystems
+          PROJECT_NUMBER: 9
+        run: |
+          gh api graphql --header 'GraphQL-Features: projects_next_graphql' -f query='
+            query($org: String!, $number: Int!) {
+              organization(login: $org){
+                projectNext(number: $number) {
+                  id
+                  fields(first:20) {
+                    nodes {
+                      id
+                      name
+                      settings
+                    }
+                  }
+                }
+              }
+            }' -f org=$ORGANIZATION -F number=$PROJECT_NUMBER > project_data.json
+
+          echo 'PROJECT_ID='$(jq '.data.organization.projectNext.id' project_data.json) >> $GITHUB_ENV
+          echo 'STATUS_FIELD_ID='$(jq '.data.organization.projectNext.fields.nodes[] | select(.name== "Status") | .id' project_data.json) >> $GITHUB_ENV
+     
+      - name: Add Issue to project
+        env:
+          GITHUB_TOKEN: ${{secrets.GHPROJECT_TOKEN}}
+          ISSUE_ID: ${{ github.event.issue.node_id }}
+        run: |
+          item_id="$( gh api graphql --header 'GraphQL-Features: projects_next_graphql' -f query='
+            mutation($project:ID!, $id:ID!) {
+              addProjectNextItem(input: {projectId: $project, contentId: $id}) {
+                projectNextItem {
+                  id
+                }
+              }
+            }' -f project=$PROJECT_ID -f id=$ISSUE_ID --jq '.data.addProjectNextItem.projectNextItem.id')"
+          
+          echo 'ITEM_ID='$item_id >> $GITHUB_ENV

Speckle/Speckle.pq

@@ -48,14 +48,16 @@ Speckle.GetBranchAsList = (branchRecord) =>
     in
         list;
 */
-
 [DataSource.Kind="Speckle",  Publish="Speckle.Publish"]
 shared Speckle.Contents = Value.ReplaceType(CommitTable, type function (StreamUrl as Uri.Type) as any);
 
+/* INFO: Variables will not be instantiated (or any code run) until they are used */
 shared CommitTable = (url) as table => 
     let
         // Get server and streamId, and branchName / commitId / objectid from the input url
-        server = Text.Combine({"https://", Uri.Parts(url)[Host]}),
+        s = Text.Combine({"https://", Uri.Parts(url)[Host]}),
+        server = Speckle.LogToMatomo(s),
+
         segments = Text.Split(Text.AfterDelimiter(Uri.Parts(url)[Path], "/", 0), "/"),
         streamId = segments{1},
         branchName = if( List.Count(segments) = 4 and segments{2} = "branches" ) then segments{3} else null,
@@ -64,78 +66,164 @@ shared CommitTable = (url) as table =>
         
         commitTable = if (commitId <> null) then Speckle.GetObjectFromCommit(server, streamId, commitId)
         else if (objectId <> null) then Speckle.GetObjectFromObject(server, streamId, objectId, false)
-        else if (branchName <> null) then #table( { "Error" }, { { "Invalid URL, use a stream or commit or object url" } } ) // currently not implemented, see reason below
+        else if (branchName <> null) then Speckle.GetObjectFromBranch(server,streamId,branchName)
         else Speckle.GetObjectFromStream(server, streamId)
     in
         commitTable;
 
-Speckle.GetObjectFromStream = (server, streamId) =>
+Speckle.CommitReceived = (server, streamId, commitId) =>
     let
+        app= "PowerBI",
+        apiKey = try Extension.CurrentCredential()[Key] otherwise "",
+        Source = if apiKey = null then "wohoo" else Web.Contents(
+		Text.Combine({server, "graphql"}, "/"),
+		[
+			Headers=[
+				#"Method"="POST",
+                #"Content-Type"="application/json",
+                #"Authorization"= if apiKey = null then "" else Text.Format("Bearer #{0}",{apiKey})
+			],
+            ManualStatusHandling = {400},
+			Content=Text.ToBinary("{""query"": ""mutation { commitReceive(input: { streamId: \"""&streamId&"\"", commitId: \"""&commitId&"\"", sourceApplication: \"""&app&"\""}) }""}")		]
+		),
+		#"JSON" = Json.Document(Source),
+        Result = #"JSON"[data][commitReceive]
+    in
+        if apiKey = "" then [data="skipped"] else [data=Text.From(Result)];
+
+/* Since everything is lazily evaluated, we must join and split the result of the matomo call with the server, and spit back the server url for PowerBI to actually log the calls to Matomo */
+Speckle.LogToMatomo = (server) =>
+    let
+        matomoUrl = "https://speckle.matomo.cloud/matomo.php",
+        action = "receive/manual",
+        appName = "Power BI",
+        userId = "powerBIuser",
+
+        params = [
+            idsite = "2",
+            rec = "1",
+            apiv = "1",
+            uid = userId,
+            action_name = action,
+            url = Text.Combine({"http://connectors/PowerBI/", action}),
+            urlref = Text.Combine({"http://connectors/PowerBI/", action}),
+            _cvar = Text.FromBinary(Json.FromValue([hostApplication = appName]))
+        ],
+        visitQuery = Uri.BuildQueryString(params),
+        visitRes = Web.Contents(Text.Combine({matomoUrl, "?", visitQuery}),
+        [
+            Headers=[
+                #"Method"="POST"
+            ],
+            Content=Text.ToBinary(server)
+        ]),
+        
+        eventParams = [
+            idsite = "2",
+            rec = "1",
+            apiv = "1",
+            uid = userId,
+            _cvar = Text.FromBinary(Json.FromValue([hostApplication = appName])),
+            e_c = appName,
+            e_a = action
+        ],
+        eventQuery = Uri.BuildQueryString(eventParams),
+        eventRes = Web.Contents(Text.Combine({ matomoUrl, "?", eventQuery}),
+        [
+            Headers=[
+                #"Method"="POST"
+            ],
+            Content=Text.ToBinary(server)
+        ]),
+        Result = Text.FromBinary(visitRes) & Text.FromBinary(eventRes),
+        Combined = Text.Combine({server,Result},"___"),
+        Split = Text.Split(Combined,"___"){0}
+    in
+        Split;
+
+Speckle.GetObjectFromStream = (server, streamId) =>
+    
+    let
+        apiKey = try Extension.CurrentCredential()[Key] otherwise null,
         branchName = "main",
         Source = Web.Contents(
 		Text.Combine({server, "graphql"}, "/"),
 		[
 			Headers=[
 				#"Method"="POST",
-				#"Content-Type"="application/json"
+				#"Content-Type"="application/json",
+                #"Authorization"= if apiKey = null then "" else Text.Format("Bearer #{0}",{apiKey})
+
 			],
-			Content=Text.ToBinary("{""query"": ""query { stream( id: \"""&streamId&"\"" ) { branch (name: \"""&branchName&"\""){ commits (limit: 1) { items { referencedObject } } } } }""}")
+			Content=Text.ToBinary("{""query"": ""query { stream( id: \"""&streamId&"\"" ) { branch (name: \"""&branchName&"\""){ commits (limit: 1) { items { referencedObject, id } } } } }""}")
 		]
 		),
 		#"JSON" = Json.Document(Source),
         objectId = #"JSON"[data][stream][branch][commits][items]{0}[referencedObject],
-        objectsTable = Speckle.GetObjectFromObject(server, streamId, objectId, true)
+        commitId = #"JSON"[data][stream][branch][commits][items]{0}[id],
+        objectsTable = Speckle.GetObjectFromObject(server, streamId, objectId, true),
+        commitReceivedRes = Speckle.CommitReceived(server, streamId, commitId)
     in
-        objectsTable;
+        if commitReceivedRes[data] = "true" then objectsTable else objectsTable;
 
-/* Not implemented since power query M uri does not have a decode method...def not manually writing a method to handle special chars and emojis
 Speckle.GetObjectFromBranch = (server, streamId, branchName) => 
     let
+        decodedBranchName = Record.Field(Record.Field(Uri.Parts("http://www.dummy.com?A=" & branchName),"Query"),"A"),
+        apiKey = try Extension.CurrentCredential()[Key] otherwise null,
         Source = Web.Contents(
 		Text.Combine({server, "graphql"}, "/"),
 		[
 			Headers=[
 				#"Method"="POST",
-				#"Content-Type"="application/json"
+				#"Content-Type"="application/json",
+                #"Authorization"= if apiKey = null then "" else Text.Format("Bearer #{0}",{apiKey})
 			],
-			Content=Text.ToBinary("{""query"": ""query { stream( id: \"""&streamId&"\"" ) { branch (name: \"""&branchName&"\""){ commits (limit: 1) { items { referencedObject } } } } }""}")
+			Content=Text.ToBinary("{""query"": ""query { stream( id: \"""&streamId&"\"" ) { branch (name: \"""&decodedBranchName&"\""){ commits (limit: 1) { items { id referencedObject } } } } }""}")
 		]
 		),
 		#"JSON" = Json.Document(Source),
         objectId = #"JSON"[data][stream][branch][commits][items]{0}[referencedObject],
-        objectsTable = Speckle.GetObjectFromObject(server, streamId, objectId)
+        commitId = #"JSON"[data][stream][branch][commits][items]{0}[id],
+        objectsTable = Speckle.GetObjectFromObject(server, streamId, objectId, true),
+        commitReceivedRes = Speckle.CommitReceived(server, streamId, commitId)
     in
-        objectsTable;
-*/
+        if commitReceivedRes[data] = "true" then objectsTable else objectsTable;
+
 
 shared Speckle.GetObjectFromCommit = (server, streamId, commitId) =>
     let
+        apiKey = try Extension.CurrentCredential()[Key] otherwise null,
         Source = Web.Contents(
 		Text.Combine({server, "graphql"}, "/"),
 		[
 			Headers=[
 				#"Method"="POST",
-				#"Content-Type"="application/json"
+				#"Content-Type"="application/json",
+                #"Authorization"= if apiKey = null then "" else Text.Format("Bearer #{0}",{apiKey})
 			],
 			Content=Text.ToBinary("{""query"": ""query { stream( id: \"""&streamId&"\"" ) { commit (id: \"""&commitId&"\""){ referencedObject } } }""}")
 		]
 		),
 		#"JSON" = Json.Document(Source),
         objectId = #"JSON"[data][stream][commit][referencedObject],
-        objectsTable = Speckle.GetObjectFromObject(server, streamId, objectId, true)
+        objectsTable = Speckle.GetObjectFromObject(server, streamId, objectId, true),
+        commitReceivedRes = Speckle.CommitReceived(server, streamId, commitId),
+        forceCommitReceived = Table.RemoveFirstN(Table.InsertRows(objectsTable, 0, {commitReceivedRes}), 1)
     in
-        objectsTable;
+        if commitReceivedRes[data] = "true" then objectsTable else objectsTable;
 
 Speckle.GetObjectFromObject = (server, streamId, objectId, IsCommitObject) => 
     let
         query = if (IsCommitObject) then "{""query"": ""query { stream( id: \"""&streamId&"\"" ) { object (id: \"""&objectId&"\"") { children { objects { data } } } } }""}"
         else "{""query"": ""query { stream( id: \"""&streamId&"\"" ) { object (id: \"""&objectId&"\"") { data } } }""}",
+        apiKey = try Extension.CurrentCredential()[Key] otherwise null,        
         Source = Web.Contents(
 		Text.Combine({server, "graphql"}, "/"),
 		[
 			Headers=[
 				#"Method"="POST",
-				#"Content-Type"="application/json"
+				#"Content-Type"="application/json",
+                #"Authorization"= if apiKey = null then "" else Text.Format("Bearer #{0}",{apiKey})
 			],
 			Content=Text.ToBinary(query)
 		]
@@ -143,7 +231,11 @@ Speckle.GetObjectFromObject = (server, streamId, objectId, IsCommitObject) =>
 		#"JSON" = Json.Document(Source),
         objects = if (IsCommitObject) then #"JSON"[data][stream][object][children][objects]
         else {#"JSON"[data][stream][object][data]},
-        objectsTable = Table.FromRecords(objects)
+
+        // remove closures from records, and remove DataChunk records
+        removeClosureField = List.Transform(objects, each Record.RemoveFields(_, "__closure", MissingField.Ignore)),
+        removeDatachunkRecords = List.RemoveItems(removeClosureField, List.FindText(removeClosureField, "Speckle.Core.Models.DataChunk")),
+        objectsTable = Table.FromRecords(removeDatachunkRecords)
     in
         objectsTable;
 

Speckle/Speckle.query.pq

@@ -1,5 +1,5 @@
 // Use this file to write queries to test your data connector
 let
-    result = Speckle.Contents("https://speckle.xyz/streams/5dfbeb49c9/objects/ed4748572b27cfe008f2592a44ab85f1")
+    result = Speckle.Contents("https://speckle.xyz/streams/6ead215ec5/branches/main")
 in
     result