Merge pull request #4 from specklesystems/claire/remove-chunks-etc

PowerBI Improvements

.github/workflows/close-issue.yml

@@ -0,0 +1,78 @@
+name: Update issue Status
+
+on:
+  issues:
+    types: [closed]
+
+jobs:
+  update_issue:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get project data
+        env:
+          GITHUB_TOKEN: ${{secrets.GHPROJECT_TOKEN}}
+          ORGANIZATION: specklesystems
+          PROJECT_NUMBER: 9
+        run: |
+          gh api graphql --header 'GraphQL-Features: projects_next_graphql' -f query='
+            query($org: String!, $number: Int!) {
+              organization(login: $org){
+                projectNext(number: $number) {
+                  id
+                  fields(first:20) {
+                    nodes {
+                      id
+                      name
+                      settings
+                    }
+                  }
+                }
+              }
+            }' -f org=$ORGANIZATION -F number=$PROJECT_NUMBER > project_data.json
+
+          echo 'PROJECT_ID='$(jq '.data.organization.projectNext.id' project_data.json) >> $GITHUB_ENV
+          echo 'STATUS_FIELD_ID='$(jq '.data.organization.projectNext.fields.nodes[] | select(.name== "Status") | .id' project_data.json) >> $GITHUB_ENV
+          
+          echo "$PROJECT_ID"
+          echo "$STATUS_FIELD_ID"
+          
+          echo 'DONE_ID='$(jq '.data.organization.projectNext.fields.nodes[] | select(.name== "Status") | .settings | fromjson | .options[] | select(.name== "Done") | .id' project_data.json) >> $GITHUB_ENV
+          echo "$DONE_ID"
+
+      - name: Add Issue to project #it's already in the project, but we do this to get its node id!
+        env:
+          GITHUB_TOKEN: ${{secrets.GHPROJECT_TOKEN}}
+          ISSUE_ID: ${{ github.event.issue.node_id }}
+        run: |
+          item_id="$( gh api graphql --header 'GraphQL-Features: projects_next_graphql' -f query='
+            mutation($project:ID!, $id:ID!) {
+              addProjectNextItem(input: {projectId: $project, contentId: $id}) {
+                projectNextItem {
+                  id
+                }
+              }
+            }' -f project=$PROJECT_ID -f id=$ISSUE_ID --jq '.data.addProjectNextItem.projectNextItem.id')"
+          
+          echo 'ITEM_ID='$item_id >> $GITHUB_ENV
+          
+      - name: Update Status
+        env:
+          GITHUB_TOKEN: ${{secrets.GHPROJECT_TOKEN}}
+          ISSUE_ID: ${{ github.event.issue.node_id }}
+        run: |
+          gh api graphql --header 'GraphQL-Features: projects_next_graphql' -f query='
+            mutation($project:ID!, $status:ID!, $id:ID!, $value:String!) {
+              set_status: updateProjectNextItemField(
+                input: {
+                  projectId: $project
+                  itemId: $id
+                  fieldId: $status
+                  value: $value
+                }
+              ) {
+                projectNextItem {
+                  id
+                }
+              }
+            }' -f project=$PROJECT_ID -f status=$STATUS_FIELD_ID -f id=$ITEM_ID -f value=${{ env.DONE_ID }}
+          

.github/workflows/open-issue.yml

@@ -0,0 +1,50 @@
+name: Move new issues into Project
+
+on:
+  issues:
+    types: [opened]
+
+jobs:
+  track_issue:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get project data
+        env:
+          GITHUB_TOKEN: ${{secrets.GHPROJECT_TOKEN}}
+          ORGANIZATION: specklesystems
+          PROJECT_NUMBER: 9
+        run: |
+          gh api graphql --header 'GraphQL-Features: projects_next_graphql' -f query='
+            query($org: String!, $number: Int!) {
+              organization(login: $org){
+                projectNext(number: $number) {
+                  id
+                  fields(first:20) {
+                    nodes {
+                      id
+                      name
+                      settings
+                    }
+                  }
+                }
+              }
+            }' -f org=$ORGANIZATION -F number=$PROJECT_NUMBER > project_data.json
+
+          echo 'PROJECT_ID='$(jq '.data.organization.projectNext.id' project_data.json) >> $GITHUB_ENV
+          echo 'STATUS_FIELD_ID='$(jq '.data.organization.projectNext.fields.nodes[] | select(.name== "Status") | .id' project_data.json) >> $GITHUB_ENV
+     
+      - name: Add Issue to project
+        env:
+          GITHUB_TOKEN: ${{secrets.GHPROJECT_TOKEN}}
+          ISSUE_ID: ${{ github.event.issue.node_id }}
+        run: |
+          item_id="$( gh api graphql --header 'GraphQL-Features: projects_next_graphql' -f query='
+            mutation($project:ID!, $id:ID!) {
+              addProjectNextItem(input: {projectId: $project, contentId: $id}) {
+                projectNextItem {
+                  id
+                }
+              }
+            }' -f project=$PROJECT_ID -f id=$ISSUE_ID --jq '.data.addProjectNextItem.projectNextItem.id')"
+          
+          echo 'ITEM_ID='$item_id >> $GITHUB_ENV

Speckle/Speckle.pq

@@ -48,14 +48,16 @@ Speckle.GetBranchAsList = (branchRecord) =>
     in
         list;
 */
-
 [DataSource.Kind="Speckle",  Publish="Speckle.Publish"]
 shared Speckle.Contents = Value.ReplaceType(CommitTable, type function (StreamUrl as Uri.Type) as any);
 
+/* INFO: Variables will not be instantiated (or any code run) until they are used */
 shared CommitTable = (url) as table => 
     let
         // Get server and streamId, and branchName / commitId / objectid from the input url
-        server = Text.Combine({"https://", Uri.Parts(url)[Host]}),
+        s = Text.Combine({"https://", Uri.Parts(url)[Host]}),
+        server = Speckle.LogToMatomo(s),
+
         segments = Text.Split(Text.AfterDelimiter(Uri.Parts(url)[Path], "/", 0), "/"),
         streamId = segments{1},
         branchName = if( List.Count(segments) = 4 and segments{2} = "branches" ) then segments{3} else null,
@@ -69,7 +71,59 @@ shared CommitTable = (url) as table =>
     in
         commitTable;
 
+
+/* Since everything is lazily evaluated, we must join and split the result of the matomo call with the server, and spit back the server url for PowerBI to actually log the calls to Matomo */
+Speckle.LogToMatomo = (server) =>
+    let
+        matomoUrl = "https://speckle.matomo.cloud/matomo.php",
+        action = "receive/manual",
+        appName = "Power BI",
+        userId = "powerBIuser",
+
+        params = [
+            idsite = "2",
+            rec = "1",
+            apiv = "1",
+            uid = userId,
+            action_name = action,
+            url = Text.Combine({"http://connectors/PowerBI/", action}),
+            urlref = Text.Combine({"http://connectors/PowerBI/", action}),
+            _cvar = Text.FromBinary(Json.FromValue([hostApplication = appName]))
+        ],
+        visitQuery = Uri.BuildQueryString(params),
+        visitRes = Web.Contents(Text.Combine({matomoUrl, "?", visitQuery}),
+        [
+            Headers=[
+                #"Method"="POST"
+            ],
+            Content=Text.ToBinary(server)
+        ]),
+        
+        eventParams = [
+            idsite = "2",
+            rec = "1",
+            apiv = "1",
+            uid = userId,
+            _cvar = Text.FromBinary(Json.FromValue([hostApplication = appName])),
+            e_c = appName,
+            e_a = action
+        ],
+        eventQuery = Uri.BuildQueryString(eventParams),
+        eventRes = Web.Contents(Text.Combine({ matomoUrl, "?", eventQuery}),
+        [
+            Headers=[
+                #"Method"="POST"
+            ],
+            Content=Text.ToBinary(server)
+        ]),
+        Result = Text.FromBinary(visitRes) & Text.FromBinary(eventRes),
+        Combined = Text.Combine({server,Result},"___"),
+        Split = Text.Split(Combined,"___"){0}
+    in
+        Split;
+
 Speckle.GetObjectFromStream = (server, streamId) =>
+    
     let
         branchName = "main",
         Source = Web.Contents(
@@ -143,7 +197,12 @@ Speckle.GetObjectFromObject = (server, streamId, objectId, IsCommitObject) =>
 		#"JSON" = Json.Document(Source),
         objects = if (IsCommitObject) then #"JSON"[data][stream][object][children][objects]
         else {#"JSON"[data][stream][object][data]},
-        objectsTable = Table.FromRecords(objects)
+
+        // remove closures from records, and remove DataChunk records
+        removeClosureField = List.Transform(objects, each Record.RemoveFields(_, "__closure", MissingField.Ignore)),
+        removeDatachunkRecords = List.RemoveItems(removeClosureField, List.FindText(removeClosureField, "Speckle.Core.Models.DataChunk")),
+
+        objectsTable = Table.FromRecords(removeDatachunkRecords)
     in
         objectsTable;
 

Speckle/Speckle.query.pq

@@ -1,5 +1,5 @@
 // Use this file to write queries to test your data connector
 let
-    result = Speckle.Contents("https://speckle.xyz/streams/5dfbeb49c9/objects/ed4748572b27cfe008f2592a44ab85f1")
+    result = Speckle.Contents("https://speckle.xyz/streams/5b97b37b8b")
 in
     result