Merge pull request #7 from specklesystems/alan/table-pagination

Adds pagination to object queries

.github/workflows/close-issue.yml

@@ -0,0 +1,78 @@
+name: Update issue Status
+
+on:
+  issues:
+    types: [closed]
+
+jobs:
+  update_issue:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get project data
+        env:
+          GITHUB_TOKEN: ${{secrets.GHPROJECT_TOKEN}}
+          ORGANIZATION: specklesystems
+          PROJECT_NUMBER: 9
+        run: |
+          gh api graphql --header 'GraphQL-Features: projects_next_graphql' -f query='
+            query($org: String!, $number: Int!) {
+              organization(login: $org){
+                projectNext(number: $number) {
+                  id
+                  fields(first:20) {
+                    nodes {
+                      id
+                      name
+                      settings
+                    }
+                  }
+                }
+              }
+            }' -f org=$ORGANIZATION -F number=$PROJECT_NUMBER > project_data.json
+
+          echo 'PROJECT_ID='$(jq '.data.organization.projectNext.id' project_data.json) >> $GITHUB_ENV
+          echo 'STATUS_FIELD_ID='$(jq '.data.organization.projectNext.fields.nodes[] | select(.name== "Status") | .id' project_data.json) >> $GITHUB_ENV
+          
+          echo "$PROJECT_ID"
+          echo "$STATUS_FIELD_ID"
+          
+          echo 'DONE_ID='$(jq '.data.organization.projectNext.fields.nodes[] | select(.name== "Status") | .settings | fromjson | .options[] | select(.name== "Done") | .id' project_data.json) >> $GITHUB_ENV
+          echo "$DONE_ID"
+
+      - name: Add Issue to project #it's already in the project, but we do this to get its node id!
+        env:
+          GITHUB_TOKEN: ${{secrets.GHPROJECT_TOKEN}}
+          ISSUE_ID: ${{ github.event.issue.node_id }}
+        run: |
+          item_id="$( gh api graphql --header 'GraphQL-Features: projects_next_graphql' -f query='
+            mutation($project:ID!, $id:ID!) {
+              addProjectNextItem(input: {projectId: $project, contentId: $id}) {
+                projectNextItem {
+                  id
+                }
+              }
+            }' -f project=$PROJECT_ID -f id=$ISSUE_ID --jq '.data.addProjectNextItem.projectNextItem.id')"
+          
+          echo 'ITEM_ID='$item_id >> $GITHUB_ENV
+          
+      - name: Update Status
+        env:
+          GITHUB_TOKEN: ${{secrets.GHPROJECT_TOKEN}}
+          ISSUE_ID: ${{ github.event.issue.node_id }}
+        run: |
+          gh api graphql --header 'GraphQL-Features: projects_next_graphql' -f query='
+            mutation($project:ID!, $status:ID!, $id:ID!, $value:String!) {
+              set_status: updateProjectNextItemField(
+                input: {
+                  projectId: $project
+                  itemId: $id
+                  fieldId: $status
+                  value: $value
+                }
+              ) {
+                projectNextItem {
+                  id
+                }
+              }
+            }' -f project=$PROJECT_ID -f status=$STATUS_FIELD_ID -f id=$ITEM_ID -f value=${{ env.DONE_ID }}
+          

.github/workflows/open-issue.yml

@@ -0,0 +1,50 @@
+name: Move new issues into Project
+
+on:
+  issues:
+    types: [opened]
+
+jobs:
+  track_issue:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get project data
+        env:
+          GITHUB_TOKEN: ${{secrets.GHPROJECT_TOKEN}}
+          ORGANIZATION: specklesystems
+          PROJECT_NUMBER: 9
+        run: |
+          gh api graphql --header 'GraphQL-Features: projects_next_graphql' -f query='
+            query($org: String!, $number: Int!) {
+              organization(login: $org){
+                projectNext(number: $number) {
+                  id
+                  fields(first:20) {
+                    nodes {
+                      id
+                      name
+                      settings
+                    }
+                  }
+                }
+              }
+            }' -f org=$ORGANIZATION -F number=$PROJECT_NUMBER > project_data.json
+
+          echo 'PROJECT_ID='$(jq '.data.organization.projectNext.id' project_data.json) >> $GITHUB_ENV
+          echo 'STATUS_FIELD_ID='$(jq '.data.organization.projectNext.fields.nodes[] | select(.name== "Status") | .id' project_data.json) >> $GITHUB_ENV
+     
+      - name: Add Issue to project
+        env:
+          GITHUB_TOKEN: ${{secrets.GHPROJECT_TOKEN}}
+          ISSUE_ID: ${{ github.event.issue.node_id }}
+        run: |
+          item_id="$( gh api graphql --header 'GraphQL-Features: projects_next_graphql' -f query='
+            mutation($project:ID!, $id:ID!) {
+              addProjectNextItem(input: {projectId: $project, contentId: $id}) {
+                projectNextItem {
+                  id
+                }
+              }
+            }' -f project=$PROJECT_ID -f id=$ISSUE_ID --jq '.data.addProjectNextItem.projectNextItem.id')"
+          
+          echo 'ITEM_ID='$item_id >> $GITHUB_ENV

Speckle/Speckle.pq

@@ -1,166 +1,5 @@
 section Speckle;
 
-/* This is an additional nav bar that can display branches of a stream: not using this for now
-[DataSource.Kind="Speckle",  Publish="Speckle.Publish"]
-shared Speckle.Contents = Value.ReplaceType(NavigationTable.Simple, type function (url as Uri.Type) as any);
-
-// set up nav table
-shared NavigationTable.Simple = (url) as table => 
-    let
-        baseUrl = Uri.Parts(url)[Host] as text,
-        streamId = Text.Split(Uri.Parts(url)[Path], "/"){2},
-        objects = Speckle.GetBranches(baseUrl, streamId),
-        table = #table(
-        {"Name", "Key", "Data", "ItemKind", "ItemName", "IsLeaf"}, 
-        List.InsertRange(objects, List.Count(objects), {{"GetCommit", "GetCommit", Speckle.GetObjectFromObject(baseUrl, streamId), "Function", "Function", true}})
-        ),
-        NavTable = Table.ToNavigationTable(table, {"Key"}, "Name", "Data", "ItemKind", "ItemName", "IsLeaf")
-    in
-        NavTable;
-
-Speckle.GetBranches = (url, id) =>
-    let
-		Source = Web.Contents(
-		Text.Combine({"https:/", url, "graphql"}, "/"),
-		[
-			Headers=[
-				#"Method"="POST",
-				#"Content-Type"="application/json"
-			],
-			Content=Text.ToBinary("{""query"": ""query { stream( id: \"""&id&"\"" ) { branches { items { name commits { items { id message sourceApplication authorName createdAt } } } } } }""}")
-		]
-		),
-		#"JSON" = Json.Document(Source),
-        branches = #"JSON"[data][stream][branches][items],
-        branchList = List.Generate(
-            () => [x = 0, y = Speckle.GetBranchAsList(branches{x})],
-            each [x] < List.Count(branches),
-            each [x = [x] + 1, y = Speckle.GetBranchAsList(branches{x})],
-            each [y]
-        )
-	in
-		branchList;
-
-Speckle.GetBranchAsList = (branchRecord) =>
-    let
-        commits = Table.FromRecords(branchRecord[commits][items]),
-        list = {branchRecord[name], branchRecord[name], commits, "Table", "Table", true}
-    in
-        list;
-*/
-
-[DataSource.Kind="Speckle",  Publish="Speckle.Publish"]
-shared Speckle.Contents = Value.ReplaceType(CommitTable, type function (StreamUrl as Uri.Type) as any);
-
-shared CommitTable = (url) as table => 
-    let
-        // Get server and streamId, and branchName / commitId / objectid from the input url
-        server = Text.Combine({"https://", Uri.Parts(url)[Host]}),
-        segments = Text.Split(Text.AfterDelimiter(Uri.Parts(url)[Path], "/", 0), "/"),
-        streamId = segments{1},
-        branchName = if( List.Count(segments) = 4 and segments{2} = "branches" ) then segments{3} else null,
-        commitId = if (List.Count(segments) = 4 and segments{2} = "commits" ) then segments{3} else null,
-        objectId = if (List.Count(segments) = 4 and segments{2} = "objects" ) then segments{3} else null,
-        
-        commitTable = if (commitId <> null) then Speckle.GetObjectFromCommit(server, streamId, commitId)
-        else if (objectId <> null) then Speckle.GetObjectFromObject(server, streamId, objectId, false)
-        else if (branchName <> null) then #table( { "Error" }, { { "Invalid URL, use a stream or commit or object url" } } ) // currently not implemented, see reason below
-        else Speckle.GetObjectFromStream(server, streamId)
-    in
-        commitTable;
-
-Speckle.GetObjectFromStream = (server, streamId) =>
-    let
-        branchName = "main",
-        Source = Web.Contents(
-		Text.Combine({server, "graphql"}, "/"),
-		[
-			Headers=[
-				#"Method"="POST",
-				#"Content-Type"="application/json"
-			],
-			Content=Text.ToBinary("{""query"": ""query { stream( id: \"""&streamId&"\"" ) { branch (name: \"""&branchName&"\""){ commits (limit: 1) { items { referencedObject } } } } }""}")
-		]
-		),
-		#"JSON" = Json.Document(Source),
-        objectId = #"JSON"[data][stream][branch][commits][items]{0}[referencedObject],
-        objectsTable = Speckle.GetObjectFromObject(server, streamId, objectId, true)
-    in
-        objectsTable;
-
-/* Not implemented since power query M uri does not have a decode method...def not manually writing a method to handle special chars and emojis
-Speckle.GetObjectFromBranch = (server, streamId, branchName) => 
-    let
-        Source = Web.Contents(
-		Text.Combine({server, "graphql"}, "/"),
-		[
-			Headers=[
-				#"Method"="POST",
-				#"Content-Type"="application/json"
-			],
-			Content=Text.ToBinary("{""query"": ""query { stream( id: \"""&streamId&"\"" ) { branch (name: \"""&branchName&"\""){ commits (limit: 1) { items { referencedObject } } } } }""}")
-		]
-		),
-		#"JSON" = Json.Document(Source),
-        objectId = #"JSON"[data][stream][branch][commits][items]{0}[referencedObject],
-        objectsTable = Speckle.GetObjectFromObject(server, streamId, objectId)
-    in
-        objectsTable;
-*/
-
-shared Speckle.GetObjectFromCommit = (server, streamId, commitId) =>
-    let
-        Source = Web.Contents(
-		Text.Combine({server, "graphql"}, "/"),
-		[
-			Headers=[
-				#"Method"="POST",
-				#"Content-Type"="application/json"
-			],
-			Content=Text.ToBinary("{""query"": ""query { stream( id: \"""&streamId&"\"" ) { commit (id: \"""&commitId&"\""){ referencedObject } } }""}")
-		]
-		),
-		#"JSON" = Json.Document(Source),
-        objectId = #"JSON"[data][stream][commit][referencedObject],
-        objectsTable = Speckle.GetObjectFromObject(server, streamId, objectId, true)
-    in
-        objectsTable;
-
-Speckle.GetObjectFromObject = (server, streamId, objectId, IsCommitObject) => 
-    let
-        query = if (IsCommitObject) then "{""query"": ""query { stream( id: \"""&streamId&"\"" ) { object (id: \"""&objectId&"\"") { children { objects { data } } } } }""}"
-        else "{""query"": ""query { stream( id: \"""&streamId&"\"" ) { object (id: \"""&objectId&"\"") { data } } }""}",
-        Source = Web.Contents(
-		Text.Combine({server, "graphql"}, "/"),
-		[
-			Headers=[
-				#"Method"="POST",
-				#"Content-Type"="application/json"
-			],
-			Content=Text.ToBinary(query)
-		]
-		),
-		#"JSON" = Json.Document(Source),
-        objects = if (IsCommitObject) then #"JSON"[data][stream][object][children][objects]
-        else {#"JSON"[data][stream][object][data]},
-        objectsTable = Table.FromRecords(objects)
-    in
-        objectsTable;
-
-// Data Source Kind description
-Speckle = [
-    Authentication = [
-        Key = [
-            KeyLabel="Personal Access Token",
-            Label = "Private stream"
-        ],
-        Implicit = [
-            Label = "Public stream"
-        ]
-    ],
-    Label = Extension.LoadString("Speckle Connector")
-];
-
 // Data Source UI publishing description
 Speckle.Publish = [
     Beta = true,
@@ -176,6 +15,223 @@ Speckle.Icons = [
     Icon32 = { Extension.Contents("SpeckleLogo32.png"), Extension.Contents("SpeckleLogo40.png"), Extension.Contents("SpeckleLogo48.png"), Extension.Contents("SpeckleLogo64.png") }
 ];
 
+// Data Source Kind description
+Speckle = [
+    Authentication = [
+        Key = [
+            KeyLabel="Personal Access Token",
+            Label = "Private stream"
+        ],
+        Implicit = [
+            Label = "Public stream"
+        ]
+    ],
+    Label = Extension.LoadString("Speckle Connector")
+];
+
+[DataSource.Kind="Speckle",  Publish="Speckle.Publish"]
+shared Speckle.Contents = Value.ReplaceType(CommitTable, type function (StreamUrl as Uri.Type) as any);
+
+shared CommitTable = (url) as table => 
+    let
+        // Get server and streamId, and branchName / commitId / objectid from the input url
+        s = Text.Combine({"https://", Uri.Parts(url)[Host]}),
+        server = Speckle.LogToMatomo(s),
+
+        segments = Text.Split(Text.AfterDelimiter(Uri.Parts(url)[Path], "/", 0), "/"),
+        streamId = segments{1},
+        branchName = if( List.Count(segments) = 4 and segments{2} = "branches" ) then segments{3} else null,
+        commitId = if (List.Count(segments) = 4 and segments{2} = "commits" ) then segments{3} else null,
+        objectId = if (List.Count(segments) = 4 and segments{2} = "objects" ) then segments{3} else null,
+        
+        commitTable = if (commitId <> null) then Speckle.GetObjectFromCommit(server, streamId, commitId)
+        else if (objectId <> null) then Speckle.GetObjectFromObject(server, streamId, objectId)
+        else if (branchName <> null) then Speckle.GetObjectFromBranch(server,streamId,branchName)
+        else Speckle.GetObjectFromStream(server, streamId)
+    in
+        commitTable;
+
+shared Speckle.GetObjectFromStream = (server, streamId) => Speckle.GetObjectFromBranch(server, streamId, "main");
+
+shared Speckle.GetObjectFromBranch = (server, streamId, branchName) => 
+    let
+        decodedBranchName = Record.Field(Record.Field(Uri.Parts("http://www.dummy.com?A=" & branchName),"Query"),"A"), // Hacky way to decode base64 strings: Put them in a url query param and parse the URL
+        apiKey = try Extension.CurrentCredential()[Key] otherwise null,
+        query = "query($streamId: String!, $branchName: String!) { 
+            stream( id: $streamId ) { 
+                branch (name: $branchName ){
+                    commits (limit: 1) {
+                        items {
+                            id
+                            referencedObject
+                        }
+                    }
+                }
+            }
+        }",
+		#"JSON" = Speckle.Api.Fetch(server, query, [streamId=streamId, branchName=decodedBranchName]),
+        commit = #"JSON"[stream][branch][commits][items]{0},
+        objectsTable = Speckle.Api.GetAllObjectChildren(server, streamId, commit[referencedObject]),
+        commitReceivedRes = Speckle.CommitReceived(server, streamId, commit[id])
+    in
+        if commitReceivedRes[data] = "true" then objectsTable else objectsTable;
+
+shared Speckle.GetObjectFromCommit = (server, streamId, commitId) =>
+    let
+        apiKey = try Extension.CurrentCredential()[Key] otherwise null,
+        query = "query($streamId: String!, $commitId: String!) {
+            stream( id: $streamId ) {
+                commit (id: $commitId) {
+                    referencedObject
+                }
+            }
+        }",
+        variables = [streamId=streamId, commitId=commitId],
+        #"JSON" = Speckle.Api.Fetch(server, query, variables),
+        objectId = #"JSON"[stream][commit][referencedObject],
+        objectsTable = Speckle.Api.GetAllObjectChildren(server, streamId, objectId),
+        commitReceivedRes = Speckle.CommitReceived(server, streamId, commitId)
+    in
+        if commitReceivedRes[data] = "true" then objectsTable else objectsTable;
+
+shared Speckle.GetObjectFromObject = (server, streamId, objectId, optional limit, optional cursor) => 
+    Speckle.Api.GetAllObjectChildren(server, streamId, objectId, limit, cursor);
+
+Speckle.CleanUpObjects = (objects) =>
+    let
+        // remove closures from records, and remove DataChunk records
+        removeClosureField = List.Transform(objects, each Record.RemoveFields(_, "__closure", MissingField.Ignore)),
+        removeDatachunkRecords = List.RemoveItems(removeClosureField, List.FindText(removeClosureField, "Speckle.Core.Models.DataChunk"))
+    in
+        removeDatachunkRecords;
+
+shared Speckle.Api.Fetch = (server, query, optional variables) => 
+    let
+        apiKey = try Extension.CurrentCredential()[Key] otherwise null,        
+        Source = Web.Contents(
+		Text.Combine({server, "graphql"}, "/"),
+		[
+			Headers=[
+				#"Method"="POST",
+				#"Content-Type"="application/json",
+                #"Authorization"= if apiKey = null then "" else Text.Format("Bearer #{0}",{apiKey})
+			],
+            ManualStatusHandling = {400},
+			Content=Json.FromValue([query=Text.From(query),variables=variables])
+		]),
+		#"JSON" = Json.Document(Source)
+    in
+        // Check if response contains errors, if so, return first error. 
+        if Record.HasFields(#"JSON", {"errors"})
+        then error Text.FromBinary(Json.FromValue(#"JSON"[errors]{0}[message]) )
+        else #"JSON"[data];
+
+// Read all pages of data.
+// After every page, we check the "NextLink" record on the metadata of the previous request.
+// Table.GenerateByPage will keep asking for more pages until we return null.
+Speckle.Api.GetAllObjectChildren = (server as text, streamId as text, objectId as text, optional cursor as text) as table =>
+    Table.GenerateByPage((previous) => 
+        let
+            // if previous is null, then this is our first page of data
+            nextCursor = if (previous = null) then cursor else Value.Metadata(previous)[Cursor]?,
+            // if the cursor is null but the prevous page is not, we've reached the end
+            page = if (previous <> null and nextCursor = null) then  null else Speckle.Api.GetObjectChildren(server, streamId, objectId, 50, nextCursor)
+        in
+            page
+    );
+
+Speckle.Api.GetObjectChildren = (server as text, streamId as text, objectId as text, optional limit as number, optional cursor as text) => 
+    let
+        query = "query($streamId: String!, $objectId: String!, $limit: Int, $cursor: String) { 
+            stream( id: $streamId ) { 
+                object (id: $objectId) { 
+                    children(limit: $limit, cursor: $cursor) { 
+                        cursor
+                        objects { 
+                            data 
+                        } 
+                    } 
+                } 
+            } 
+        }",
+		#"JSON" = Speckle.Api.Fetch(server, query, [streamId=streamId, objectId=objectId, limit=limit, cursor=cursor]),
+        children = #"JSON"[stream][object][children],
+        objects = children[objects],
+        nextCursor=children[cursor],
+        clean=Speckle.CleanUpObjects(objects)
+    in
+       Table.FromRecords(clean) meta [Cursor=nextCursor];
+
+Speckle.CommitReceived = (server, streamId, commitId) =>
+    let
+        app= "PowerBI",
+        apiKey = try Extension.CurrentCredential()[Key] otherwise "",
+        Source = Web.Contents(
+		Text.Combine({server, "graphql"}, "/"),
+		[
+			Headers=[
+				#"Method"="POST",
+                #"Content-Type"="application/json",
+                #"Authorization"= if apiKey = null then "" else Text.Format("Bearer #{0}",{apiKey})
+			],
+            ManualStatusHandling = {400},
+			Content=Text.ToBinary("{""query"": ""mutation { commitReceive(input: { streamId: \"""&streamId&"\"", commitId: \"""&commitId&"\"", sourceApplication: \"""&app&"\""}) }""}")		]
+		),
+		#"JSON" = Json.Document(Source),
+        Result = #"JSON"[data][commitReceive]
+    in
+        if apiKey = "" then [data="skipped"] else [data=Text.From(Result)];
+
+/* Since everything is lazily evaluated, we must join and split the result of the matomo call with the server, and spit back the server url for PowerBI to actually log the calls to Matomo */
+Speckle.LogToMatomo = (server) =>
+    let
+        matomoUrl = "https://speckle.matomo.cloud/matomo.php",
+        action = "receive/manual",
+        appName = "Power BI",
+        userId = "powerBIuser",
+
+        params = [
+            idsite = "2",
+            rec = "1",
+            apiv = "1",
+            uid = userId,
+            action_name = action,
+            url = Text.Combine({"http://connectors/PowerBI/", action}),
+            urlref = Text.Combine({"http://connectors/PowerBI/", action}),
+            _cvar = Text.FromBinary(Json.FromValue([hostApplication = appName]))
+        ],
+        visitQuery = Uri.BuildQueryString(params),
+        visitRes = Web.Contents(Text.Combine({matomoUrl, "?", visitQuery}),
+        [
+            Headers=[
+                #"Method"="POST"
+            ],
+            Content=Text.ToBinary(server)
+        ]),
+        
+        eventParams = [
+            idsite = "2",
+            rec = "1",
+            apiv = "1",
+            uid = userId,
+            _cvar = Text.FromBinary(Json.FromValue([hostApplication = appName])),
+            e_c = appName,
+            e_a = action
+        ],
+        eventQuery = Uri.BuildQueryString(eventParams),
+        eventRes = Web.Contents(Text.Combine({ matomoUrl, "?", eventQuery}),
+        [
+            Headers=[
+                #"Method"="POST"
+            ],
+            Content=Text.ToBinary(server)
+        ]),
+        Result = Text.FromBinary(visitRes) & Text.FromBinary(eventRes),
+        Combined = Text.Combine({server,Result},"___"),
+        Split = Text.Split(Combined,"___"){0}
+    in
+        Split;
+
 // copy and pasted function from microsoft docs since it's not included yet in M standard lib
 Table.ToNavigationTable = (
     table as table,
@@ -199,3 +255,26 @@ Table.ToNavigationTable = (
         navigationTable = Value.ReplaceType(table, newTableType)
     in
         navigationTable;
+
+// The getNextPage function takes a single argument and is expected to return a nullable table
+Table.GenerateByPage = (getNextPage as function) as table =>
+    let        
+        listOfPages = List.Generate(
+            () => getNextPage(null),            // get the first page of data
+            (lastPage) => lastPage <> null,     // stop when the function returns null
+            (lastPage) => getNextPage(lastPage) // pass the previous page to the next function call
+        ),
+        // concatenate the pages together
+        tableOfPages = Table.FromList(listOfPages, Splitter.SplitByNothing(), {"Column1"}),
+        firstRow = tableOfPages{0}?
+    in
+        // if we didn't get back any pages of data, return an empty table
+        // otherwise set the table type based on the columns of the first page
+        if (firstRow = null) then
+            Table.FromRows({})
+        else        
+            Value.ReplaceType(
+                Table.ExpandTableColumn(tableOfPages, "Column1", Table.ColumnNames(firstRow[Column1])),
+                Value.Type(firstRow[Column1])
+            );
+

Speckle/Speckle.query.pq

@@ -1,5 +1,5 @@
 // Use this file to write queries to test your data connector
 let
-    result = Speckle.Contents("https://speckle.xyz/streams/5dfbeb49c9/objects/ed4748572b27cfe008f2592a44ab85f1")
+    result = Speckle.Contents("https://speckle.xyz/streams/6ead215ec5/branches/main")
 in
     result