From 4980796cd6d1b05266735af8ec0a91eba577aef7 Mon Sep 17 00:00:00 2001 From: Dimitrie Stefanescu Date: Tue, 10 Feb 2026 10:17:38 +0000 Subject: [PATCH 01/26] Dim/flopper experiments (#438) * Adds object flopper for experimental object uploads Implements a new mechanism for uploading objects to the server. * wip * pipe works * WIP * wip - refactors send pipeline --- Directory.Packages.props | 9 +- src/Speckle.Objects/packages.lock.json | 203 ++++++++-- .../packages.lock.json | 34 +- .../Models/DynamicBaseMemberType.cs | 6 +- src/Speckle.Sdk/Pipelines/ProgressStream.cs | 97 +++++ src/Speckle.Sdk/Pipelines/SendPipeline.cs | 59 +++ src/Speckle.Sdk/Pipelines/Serializer.cs | 372 ++++++++++++++++++ src/Speckle.Sdk/Pipelines/Uploader.cs | 175 ++++++++ src/Speckle.Sdk/Pipelines/Uploader_Old.cs | 83 ++++ .../Serialisation/V2/Send/BaseItem.cs | 8 +- .../Serialisation/V2/Send/ObjectFlopper.cs | 79 ++++ .../V2/Send/ObjectFlopperGandalf.cs | 90 +++++ .../Serialisation/V2/Send/SerializeProcess.cs | 11 +- .../V2/SerializeProcessFactory.cs | 3 +- src/Speckle.Sdk/Speckle.Sdk.csproj | 6 +- src/Speckle.Sdk/packages.lock.json | 257 +++++++++++- 16 files changed, 1413 insertions(+), 79 deletions(-) create mode 100644 src/Speckle.Sdk/Pipelines/ProgressStream.cs create mode 100644 src/Speckle.Sdk/Pipelines/SendPipeline.cs create mode 100644 src/Speckle.Sdk/Pipelines/Serializer.cs create mode 100644 src/Speckle.Sdk/Pipelines/Uploader.cs create mode 100644 src/Speckle.Sdk/Pipelines/Uploader_Old.cs create mode 100644 src/Speckle.Sdk/Serialisation/V2/Send/ObjectFlopper.cs create mode 100644 src/Speckle.Sdk/Serialisation/V2/Send/ObjectFlopperGandalf.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 86c51244..6bf24322 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -8,6 +8,8 @@ + + @@ -16,7 +18,6 @@ - @@ -28,7 +29,9 @@ - + + + @@ -39,4 +42,4 @@ - + \ No newline at end of file diff --git a/src/Speckle.Objects/packages.lock.json b/src/Speckle.Objects/packages.lock.json index 17abca99..2c2e04f1 100644 --- a/src/Speckle.Objects/packages.lock.json +++ b/src/Speckle.Objects/packages.lock.json @@ -130,6 +130,19 @@ "resolved": "8.0.0", "contentHash": "dk9JPxTCIevS75HyEQ0E4OVAFhB2N+V9ShCXf8Q6FkUQZDkgLI12y679Nym1YqsiSysuQskT7Z+6nUf3yab6Vw==" }, + "Newtonsoft.Json": { + "type": "Transitive", + "resolved": "13.0.1", + "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" + }, + "Newtonsoft.Json.Bson": { + "type": "Transitive", + "resolved": "1.0.2", + "contentHash": "QYFyxhaABwmq3p/21VrZNYvCg3DaEoN/wUuw5nmfAf0X3HLjgupwhkEWdgfb9nvGAUIv3osmZoD3kKl4jxEmYQ==", + "dependencies": { + "Newtonsoft.Json": "12.0.1" + } + }, "SQLitePCLRaw.bundle_e_sqlite3": { "type": "Transitive", "resolved": "2.1.4", @@ -162,8 +175,8 @@ }, "System.Buffers": { "type": "Transitive", - "resolved": "4.4.0", - "contentHash": "AwarXzzoDwX6BgrhjoJsk6tUezZEozOT5Y9QKF94Gl4JK91I4PIIBkBco9068Y9/Dra8Dkbie99kXB8+1BaYKw==" + "resolved": "4.6.1", + "contentHash": "N8GXpmiLMtljq7gwvyS+1QvKT/W2J8sNAvx+HVg4NGmsG/H+2k/y9QI23auLJRterrzCiDH+IWAw4V/GPwsMlw==" }, "System.ComponentModel.Annotations": { "type": "Transitive", @@ -172,18 +185,18 @@ }, "System.Memory": { "type": "Transitive", - "resolved": "4.5.3", - "contentHash": "3oDzvc/zzetpTKWMShs1AADwZjQ/36HnsufHRPcOjyRAAMLDlu2iD33MBI2opxnezcVUtXyqDXXjoFMOU9c7SA==", + "resolved": "4.6.3", + "contentHash": "qdcDOgnFZY40+Q9876JUHnlHu7bosOHX8XISRoH94fwk6hgaeQGSgfZd8srWRZNt5bV9ZW2TljcegDNxsf+96A==", "dependencies": { - "System.Buffers": "4.4.0", - "System.Numerics.Vectors": "4.4.0", - "System.Runtime.CompilerServices.Unsafe": "4.5.2" + "System.Buffers": "4.6.1", + "System.Numerics.Vectors": "4.6.1", + "System.Runtime.CompilerServices.Unsafe": "6.1.2" } }, "System.Numerics.Vectors": { "type": "Transitive", - "resolved": "4.4.0", - "contentHash": "UiLzLW+Lw6HLed1Hcg+8jSRttrbuXv7DANVj0DkL9g6EnnzbL75EB7EWsw5uRbhxd/4YdG8li5XizGWepmG3PQ==" + "resolved": "4.6.1", + "contentHash": "sQxefTnhagrhoq2ReR0D/6K0zJcr9Hrd6kikeXsA1I8kOCboTavcUC4r7TSfpKFeE163uMuxZcyfO1mGO3EN8Q==" }, "System.Reactive": { "type": "Transitive", @@ -205,8 +218,8 @@ }, "System.Runtime.CompilerServices.Unsafe": { "type": "Transitive", - "resolved": "4.5.3", - "contentHash": "3TIsJhD1EiiT0w2CcDMN/iSSwnNnsrnbzeVHSKkaEgV85txMprmuO+Yq2AdSbeVGcg28pdNDTPK87tJhX7VFHw==" + "resolved": "6.1.2", + "contentHash": "2hBr6zdbIBTDE3EhK7NSVNdX58uTK6iHW/P/Axmm9sl1xoGSLqDvMtpecn226TNwHByFokYwJmt/aQQNlO5CRw==" }, "System.Runtime.InteropServices.WindowsRuntime": { "type": "Transitive", @@ -216,26 +229,40 @@ "System.Runtime": "4.3.0" } }, + "System.Text.Encodings.Web": { + "type": "Transitive", + "resolved": "10.0.1", + "contentHash": "cVAka0o1rJJ5/De0pjNs7jcaZk5hUGf1HGzUyVmE2MEB1Vf0h/8qsWxImk1zjitCbeD2Avaq2P2+usdvqgbeVQ==", + "dependencies": { + "System.Buffers": "4.6.1", + "System.Memory": "4.6.3", + "System.Runtime.CompilerServices.Unsafe": "6.1.2" + } + }, "System.Threading.Tasks.Extensions": { "type": "Transitive", - "resolved": "4.5.4", - "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==", + "resolved": "4.6.3", + "contentHash": "7sCiwilJLYbTZELaKnc7RecBBXWXA+xMLQWZKWawBxYjp6DBlSE3v9/UcvKBvr1vv2tTOhipiogM8rRmxlhrVA==", "dependencies": { - "System.Runtime.CompilerServices.Unsafe": "4.5.3" + "System.Runtime.CompilerServices.Unsafe": "6.1.2" } }, "speckle.sdk": { "type": "Project", "dependencies": { "GraphQL.Client": "[6.0.0, )", - "Microsoft.Bcl.AsyncInterfaces": "[5.0.0, )", + "Microsoft.AspNet.WebApi.Client": "[6.0.0, )", + "Microsoft.Bcl.AsyncInterfaces": "[10.0.1, )", "Microsoft.CSharp": "[4.7.0, )", "Microsoft.Data.Sqlite": "[7.0.5, )", "Microsoft.Extensions.DependencyInjection.Abstractions": "[2.2.0, )", "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )", + "System.IO.Pipelines": "[10.0.1, )", + "System.Net.Http.Json": "[10.0.1, )", + "System.Threading.Channels": "[10.0.1, )" } }, "speckle.sdk.dependencies": { @@ -252,13 +279,25 @@ "System.Reactive": "5.0.0" } }, + "Microsoft.AspNet.WebApi.Client": { + "type": "CentralTransitive", + "requested": "[6.0.0, )", + "resolved": "6.0.0", + "contentHash": "zXeWP03dTo67AoDHUzR+/urck0KFssdCKOC+dq7Nv1V2YbFh/nIg09L0/3wSvyRONEdwxGB/ssEGmPNIIhAcAw==", + "dependencies": { + "Newtonsoft.Json": "13.0.1", + "Newtonsoft.Json.Bson": "1.0.2", + "System.Memory": "4.5.5", + "System.Threading.Tasks.Extensions": "4.5.4" + } + }, "Microsoft.Bcl.AsyncInterfaces": { "type": "CentralTransitive", - "requested": "[5.0.0, )", - "resolved": "5.0.0", - "contentHash": "W8DPQjkMScOMTtJbPwmPyj9c3zYSFGawDW3jwlBOOsnY+EzZFLgNQ/UMkK35JmkNOVPdCyPr2Tw7Vv9N+KA3ZQ==", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "E1HSLkPHXEO30JEij2pWbOuzz1Z5ND4a5l7IP1T2RgQuE0a0NzEIvtO64RNy3Otn6PFezbT80cfm3M/Cgt70PA==", "dependencies": { - "System.Threading.Tasks.Extensions": "4.5.4" + "System.Threading.Tasks.Extensions": "4.6.3" } }, "Microsoft.CSharp": { @@ -306,6 +345,54 @@ "requested": "[13.0.2, )", "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" + }, + "System.IO.Pipelines": { + "type": "CentralTransitive", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "26LbFXHKd7PmRnWlkjnYgmjd5B6HYVG+1MpTO25BdxTJnx6D0O16JPAC/S4YBqjtt4YpfGj1QO/Ss6SPMGEGQw==", + "dependencies": { + "System.Buffers": "4.6.1", + "System.Memory": "4.6.3", + "System.Threading.Tasks.Extensions": "4.6.3" + } + }, + "System.Net.Http.Json": { + "type": "CentralTransitive", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "XGOWt78ccgO9esyNlCemlS2b8JZPrH85pk/MvdHJxp6KwwUY/GnDaw2fPpJa7lgotiTkWnXnhip+ULNKaP7a8A==", + "dependencies": { + "System.Buffers": "4.6.1", + "System.Memory": "4.6.3", + "System.Text.Json": "10.0.1", + "System.Threading.Tasks.Extensions": "4.6.3" + } + }, + "System.Text.Json": { + "type": "CentralTransitive", + "requested": "[8.0.5, )", + "resolved": "10.0.1", + "contentHash": "EsgwDgU1PFqhrFA9l5n+RBu76wFhNGCEwu8ITrBNhjPP3MxLyklroU5GIF8o6JYpYg6T4KD/VICfMdgPAvNp5g==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "10.0.1", + "System.Buffers": "4.6.1", + "System.IO.Pipelines": "10.0.1", + "System.Memory": "4.6.3", + "System.Runtime.CompilerServices.Unsafe": "6.1.2", + "System.Text.Encodings.Web": "10.0.1", + "System.Threading.Tasks.Extensions": "4.6.3" + } + }, + "System.Threading.Channels": { + "type": "CentralTransitive", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "10.0.1", + "System.Threading.Tasks.Extensions": "4.6.3" + } } }, "net8.0": { @@ -418,6 +505,19 @@ "resolved": "8.0.0", "contentHash": "dk9JPxTCIevS75HyEQ0E4OVAFhB2N+V9ShCXf8Q6FkUQZDkgLI12y679Nym1YqsiSysuQskT7Z+6nUf3yab6Vw==" }, + "Newtonsoft.Json": { + "type": "Transitive", + "resolved": "13.0.1", + "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" + }, + "Newtonsoft.Json.Bson": { + "type": "Transitive", + "resolved": "1.0.2", + "contentHash": "QYFyxhaABwmq3p/21VrZNYvCg3DaEoN/wUuw5nmfAf0X3HLjgupwhkEWdgfb9nvGAUIv3osmZoD3kKl4jxEmYQ==", + "dependencies": { + "Newtonsoft.Json": "12.0.1" + } + }, "SQLitePCLRaw.bundle_e_sqlite3": { "type": "Transitive", "resolved": "2.1.4", @@ -455,8 +555,8 @@ }, "System.Memory": { "type": "Transitive", - "resolved": "4.5.3", - "contentHash": "3oDzvc/zzetpTKWMShs1AADwZjQ/36HnsufHRPcOjyRAAMLDlu2iD33MBI2opxnezcVUtXyqDXXjoFMOU9c7SA==" + "resolved": "4.5.5", + "contentHash": "XIWiDvKPXaTveaB7HVganDlOCRoj03l+jrwNvcge/t8vhGYKvqV+dMv6G4SAX2NoNmN0wZfVPTAlFwZcZvVOUw==" }, "System.Reactive": { "type": "Transitive", @@ -468,16 +568,30 @@ "resolved": "4.5.1", "contentHash": "Zh8t8oqolRaFa9vmOZfdQm/qKejdqz0J9kr7o2Fu0vPeoH3BL1EOXipKWwkWtLT1JPzjByrF19fGuFlNbmPpiw==" }, + "System.Text.Encodings.Web": { + "type": "Transitive", + "resolved": "10.0.1", + "contentHash": "cVAka0o1rJJ5/De0pjNs7jcaZk5hUGf1HGzUyVmE2MEB1Vf0h/8qsWxImk1zjitCbeD2Avaq2P2+usdvqgbeVQ==" + }, + "System.Threading.Tasks.Extensions": { + "type": "Transitive", + "resolved": "4.5.4", + "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==" + }, "speckle.sdk": { "type": "Project", "dependencies": { "GraphQL.Client": "[6.0.0, )", + "Microsoft.AspNet.WebApi.Client": "[6.0.0, )", "Microsoft.Data.Sqlite": "[7.0.5, )", "Microsoft.Extensions.DependencyInjection.Abstractions": "[2.2.0, )", "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )", + "System.IO.Pipelines": "[10.0.1, )", + "System.Net.Http.Json": "[10.0.1, )", + "System.Threading.Channels": "[10.0.1, )" } }, "speckle.sdk.dependencies": { @@ -494,6 +608,18 @@ "System.Reactive": "5.0.0" } }, + "Microsoft.AspNet.WebApi.Client": { + "type": "CentralTransitive", + "requested": "[6.0.0, )", + "resolved": "6.0.0", + "contentHash": "zXeWP03dTo67AoDHUzR+/urck0KFssdCKOC+dq7Nv1V2YbFh/nIg09L0/3wSvyRONEdwxGB/ssEGmPNIIhAcAw==", + "dependencies": { + "Newtonsoft.Json": "13.0.1", + "Newtonsoft.Json.Bson": "1.0.2", + "System.Memory": "4.5.5", + "System.Threading.Tasks.Extensions": "4.5.4" + } + }, "Microsoft.Data.Sqlite": { "type": "CentralTransitive", "requested": "[7.0.5, )", @@ -533,6 +659,37 @@ "requested": "[13.0.2, )", "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" + }, + "System.IO.Pipelines": { + "type": "CentralTransitive", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "26LbFXHKd7PmRnWlkjnYgmjd5B6HYVG+1MpTO25BdxTJnx6D0O16JPAC/S4YBqjtt4YpfGj1QO/Ss6SPMGEGQw==" + }, + "System.Net.Http.Json": { + "type": "CentralTransitive", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "XGOWt78ccgO9esyNlCemlS2b8JZPrH85pk/MvdHJxp6KwwUY/GnDaw2fPpJa7lgotiTkWnXnhip+ULNKaP7a8A==", + "dependencies": { + "System.Text.Json": "10.0.1" + } + }, + "System.Text.Json": { + "type": "CentralTransitive", + "requested": "[8.0.5, )", + "resolved": "10.0.1", + "contentHash": "EsgwDgU1PFqhrFA9l5n+RBu76wFhNGCEwu8ITrBNhjPP3MxLyklroU5GIF8o6JYpYg6T4KD/VICfMdgPAvNp5g==", + "dependencies": { + "System.IO.Pipelines": "10.0.1", + "System.Text.Encodings.Web": "10.0.1" + } + }, + "System.Threading.Channels": { + "type": "CentralTransitive", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" } } } diff --git a/src/Speckle.Sdk.Dependencies/packages.lock.json b/src/Speckle.Sdk.Dependencies/packages.lock.json index c664417c..1f256577 100644 --- a/src/Speckle.Sdk.Dependencies/packages.lock.json +++ b/src/Speckle.Sdk.Dependencies/packages.lock.json @@ -82,12 +82,12 @@ }, "System.Threading.Channels": { "type": "Direct", - "requested": "[9.0.4, )", - "resolved": "9.0.4", - "contentHash": "4qBn2H6/aXBpE/Pm3wY5yusY/pEvQz99NlWHrTUji0qCmOdbhhjaALcpmbfW2ksxlPM6i6S+QFLkpOQdyfeKYQ==", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==", "dependencies": { - "Microsoft.Bcl.AsyncInterfaces": "9.0.4", - "System.Threading.Tasks.Extensions": "4.5.4" + "Microsoft.Bcl.AsyncInterfaces": "10.0.1", + "System.Threading.Tasks.Extensions": "4.6.3" } }, "ILRepack": { @@ -141,24 +141,24 @@ }, "System.Runtime.CompilerServices.Unsafe": { "type": "Transitive", - "resolved": "6.0.0", - "contentHash": "/iUeP3tq1S0XdNNoMz5C9twLSrM/TH+qElHkXWaPvuNOt+99G75NrV0OS2EqHx5wMN7popYjpc8oTjC1y16DLg==" + "resolved": "6.1.2", + "contentHash": "2hBr6zdbIBTDE3EhK7NSVNdX58uTK6iHW/P/Axmm9sl1xoGSLqDvMtpecn226TNwHByFokYwJmt/aQQNlO5CRw==" }, "System.Threading.Tasks.Extensions": { "type": "Transitive", - "resolved": "4.5.4", - "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==", + "resolved": "4.6.3", + "contentHash": "7sCiwilJLYbTZELaKnc7RecBBXWXA+xMLQWZKWawBxYjp6DBlSE3v9/UcvKBvr1vv2tTOhipiogM8rRmxlhrVA==", "dependencies": { - "System.Runtime.CompilerServices.Unsafe": "4.5.3" + "System.Runtime.CompilerServices.Unsafe": "6.1.2" } }, "Microsoft.Bcl.AsyncInterfaces": { "type": "CentralTransitive", - "requested": "[5.0.0, )", - "resolved": "9.0.4", - "contentHash": "9VGI5kxIvrNG2mqLQZnUR6y/3fcnygD8eNpHR+CqfbnIXvea6nehnYknDKQTxZVPMpzpNca+7DxLBmpdB3q0Bw==", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "E1HSLkPHXEO30JEij2pWbOuzz1Z5ND4a5l7IP1T2RgQuE0a0NzEIvtO64RNy3Otn6PFezbT80cfm3M/Cgt70PA==", "dependencies": { - "System.Threading.Tasks.Extensions": "4.5.4" + "System.Threading.Tasks.Extensions": "4.6.3" } } }, @@ -229,9 +229,9 @@ }, "System.Threading.Channels": { "type": "Direct", - "requested": "[9.0.4, )", - "resolved": "9.0.4", - "contentHash": "4qBn2H6/aXBpE/Pm3wY5yusY/pEvQz99NlWHrTUji0qCmOdbhhjaALcpmbfW2ksxlPM6i6S+QFLkpOQdyfeKYQ==" + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" }, "ILRepack": { "type": "Transitive", diff --git a/src/Speckle.Sdk/Models/DynamicBaseMemberType.cs b/src/Speckle.Sdk/Models/DynamicBaseMemberType.cs index fc114380..b03c2579 100644 --- a/src/Speckle.Sdk/Models/DynamicBaseMemberType.cs +++ b/src/Speckle.Sdk/Models/DynamicBaseMemberType.cs @@ -17,7 +17,7 @@ public enum DynamicBaseMemberType Dynamic = 2, /// - /// The typed members flagged with attribute. + /// The typed members flagged with ObsoleteAttribute attribute. /// Obsolete = 4, @@ -27,12 +27,12 @@ public enum DynamicBaseMemberType SchemaComputed = 16, /// - /// All the typed members, including ones with attributes. + /// All the typed members, including ones with ObsoleteAttribute attributes. /// InstanceAll = Instance + Obsolete, /// - /// All the members, including dynamic and instance members flagged with attributes + /// All the members, including dynamic and instance members flagged with ObsoleteAttribute attributes /// All = InstanceAll + Dynamic, } diff --git a/src/Speckle.Sdk/Pipelines/ProgressStream.cs b/src/Speckle.Sdk/Pipelines/ProgressStream.cs new file mode 100644 index 00000000..ed62121c --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/ProgressStream.cs @@ -0,0 +1,97 @@ +namespace Speckle.Sdk.Pipelines; + +/// +/// Wraps a stream to report upload progress as bytes are read. +/// +public sealed class ProgressStream : Stream +{ + private readonly Stream _innerStream; + private readonly long _totalBytes; + private readonly IProgress<(long BytesSent, long TotalBytes)>? _progress; + private long _bytesSent; + + public ProgressStream( + Stream innerStream, + long totalBytes, + IProgress<(long BytesSent, long TotalBytes)>? progress = null + ) + { + _innerStream = innerStream; + _totalBytes = totalBytes; + _progress = progress; + _bytesSent = 0; + } + + public override bool CanRead => _innerStream.CanRead; + public override bool CanSeek => _innerStream.CanSeek; + public override bool CanWrite => false; + public override long Length => _innerStream.Length; + + public override long Position + { + get => _innerStream.Position; + set => _innerStream.Position = value; + } + + public override int Read(byte[] buffer, int offset, int count) + { + int bytesRead = _innerStream.Read(buffer, offset, count); + ReportProgress(bytesRead); + return bytesRead; + } + + public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + int bytesRead = await _innerStream +#if NET8_0_OR_GREATER + .ReadAsync(buffer.AsMemory(offset, count), cancellationToken) +#else + .ReadAsync(buffer, offset, count, cancellationToken) +#endif + .ConfigureAwait(false); + ReportProgress(bytesRead); + return bytesRead; + } + +#if NET8_0_OR_GREATER + public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + { + int bytesRead = await _innerStream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + ReportProgress(bytesRead); + return bytesRead; + } +#endif + + private void ReportProgress(int bytesRead) + { + _bytesSent += bytesRead; + _progress?.Report((_bytesSent, _totalBytes)); + } + + public override void Flush() => _innerStream.Flush(); + + public override Task FlushAsync(CancellationToken cancellationToken) => _innerStream.FlushAsync(cancellationToken); + + public override long Seek(long offset, SeekOrigin origin) => _innerStream.Seek(offset, origin); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + protected override void Dispose(bool disposing) + { + if (disposing) + { + _innerStream.Dispose(); + } + base.Dispose(disposing); + } + +#if NET8_0_OR_GREATER + public override async ValueTask DisposeAsync() + { + await _innerStream.DisposeAsync().ConfigureAwait(false); + await base.DisposeAsync().ConfigureAwait(false); + } +#endif +} diff --git a/src/Speckle.Sdk/Pipelines/SendPipeline.cs b/src/Speckle.Sdk/Pipelines/SendPipeline.cs new file mode 100644 index 00000000..4c580163 --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/SendPipeline.cs @@ -0,0 +1,59 @@ +using Speckle.Sdk.Credentials; +using Speckle.Sdk.Models; +using Speckle.Sdk.Serialisation; + +namespace Speckle.Sdk.Pipelines; + +public record UploadItem(string Id, Json Json, string SpeckleType, ObjectReference Reference); + +public sealed class SendPipeline : IDisposable +{ + private readonly CancellationToken _cancellationToken; + private readonly Serializer _serializer = new(); + private readonly Uploader _uploader; + + public SendPipeline( + Account account, + string projectId, + string modelId, + string ingestionId, + CancellationToken cancellationToken + ) + { + _cancellationToken = cancellationToken; + _uploader = new Uploader(projectId, modelId, ingestionId, account.serverInfo.url, account.token, cancellationToken); + } + + private UploadItem _lastItem; + + public async Task Process(Base @base) + { + var results = _serializer.Serialize(@base).ToArray(); + var first = results.First(); + foreach (var item in results) + { + // we're not doing fire and forget here so that we get the backpressure from the uploader + await _uploader.PushAsync(item).ConfigureAwait(false); + } + + // NOTE: this is important to keep track of. When we serialze an object, we get back a list of objects, with the first one being the original root. + // In the case of the commit root object, this means the last object is not necessarily the root; we therefore need to manually track its existance here + // and ensure it's the last one through in the uploader's stream. See WaitForUpload down below. + _lastItem = first; + return first.Reference; + } + + public async Task WaitForUpload() + { + await _uploader.PushAsync(_lastItem).ConfigureAwait(false); + await _uploader.CompleteAsync().ConfigureAwait(false); + } + + public async Task WaitForUploadAndServerProcessing() + { + // TODO: in some way, wait for the server to process the upload and return the actual new version id + return await Task.FromResult("todo").ConfigureAwait(false); + } + + public void Dispose() => _uploader.Dispose(); +} diff --git a/src/Speckle.Sdk/Pipelines/Serializer.cs b/src/Speckle.Sdk/Pipelines/Serializer.cs new file mode 100644 index 00000000..ed072a50 --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Serializer.cs @@ -0,0 +1,372 @@ +using System.Collections; +using System.Reflection; +using Speckle.DoubleNumerics; +using Speckle.Newtonsoft.Json; +using Speckle.Sdk.Dependencies; +using Speckle.Sdk.Helpers; +using Speckle.Sdk.Models; +using Speckle.Sdk.Serialisation; + +namespace Speckle.Sdk.Pipelines; + +/// +/// Another serializer, cleaner and meaner. Provides methods for serializing Speckle objects into a format suitable for upload or storage. +/// This class handles the conversion of and its derivatives +/// into serialized JSON structures along with associated metadata, closures, and references. +/// Any reference objects coming through are being "passed through" serialized - they do not get double encoded. +/// +public class Serializer +{ + private readonly record struct PropertyInfo(string Name, object? Value, bool IsDetachable); + + public IEnumerable Serialize(Base root) + { + // Special case: if root is already an ObjectReference, serialize it verbatim + if (root is ObjectReference existingRef) + { + var uploadItem = ReferenceToUploadItem(existingRef); + yield return uploadItem; + yield break; + } + + var detachedObjects = new List<(Id, Json, Dictionary, Base, string)>(); + var rootClosures = new Dictionary(); + + var (rootId, rootJson) = SerializeBase(root, false, rootClosures, detachedObjects); + + var rootReference = new ObjectReference + { + referencedId = rootId.Value, + applicationId = root.applicationId, + closure = rootClosures.Count > 0 ? rootClosures : null, + }; + + yield return new UploadItem(rootId.Value, rootJson, root.speckle_type, rootReference); + + foreach (var (id, json, closures, baseObj, speckleType) in detachedObjects) + { + var reference = new ObjectReference + { + referencedId = id.Value, + applicationId = baseObj.applicationId, + closure = closures.Count > 0 ? closures : null, + }; + + yield return new UploadItem(id.Value, json, speckleType, reference); + } + } + + private IEnumerable ExtractProperties(Base baseObj) + { + var typedProperties = baseObj.GetInstanceMembers(); + foreach (var prop in typedProperties) + { + if (prop.Name == "id" || prop.Name.StartsWith("__")) + { + continue; + } + + if (prop.IsDefined(typeof(JsonIgnoreAttribute), false)) + { + continue; + } + + var value = prop.GetValue(baseObj); + var isDetachable = prop.GetCustomAttribute(true)?.Detachable ?? false; + + yield return new PropertyInfo(prop.Name, value, isDetachable); + } + + foreach (var propName in baseObj.DynamicPropertyKeys) + { + if (propName.StartsWith("__")) + { + continue; + } + + var value = baseObj[propName]; + +#pragma warning disable CA1866 + var isDetachable = propName.StartsWith("@"); +#pragma warning restore CA1866 + + yield return new PropertyInfo(propName, value, isDetachable); + } + } + + private (Id, Json) SerializeBase( + Base baseObj, + bool forceDetach, + Dictionary closures, + List<(Id, Json, Dictionary, Base, string)> detachedObjects + ) + { + var childClosures = new Dictionary(); + + var sb = Pools.StringBuilders.Get(); + try + { + using var stringWriter = new StringWriter(sb); + using var jsonWriter = new JsonTextWriter(stringWriter); + using var idWriter = new SerializerIdWriter(jsonWriter); + + idWriter.WriteStartObject(); + + foreach (var prop in ExtractProperties(baseObj)) + { + idWriter.WritePropertyName(prop.Name); + SerializeValue(prop.Value, idWriter, prop.IsDetachable, childClosures, detachedObjects); + } + + var (jsonForId, finalWriter) = idWriter.FinishIdWriter(); + var id = IdGenerator.ComputeId(jsonForId); + + finalWriter.WritePropertyName("id"); + finalWriter.WriteValue(id.Value); + + baseObj.id = id.Value; + + if ((forceDetach || childClosures.Count > 0) && childClosures.Count > 0) + { + finalWriter.WritePropertyName("__closure"); + finalWriter.WriteStartObject(); + foreach (var kvp in childClosures) + { + finalWriter.WritePropertyName(kvp.Key); + finalWriter.WriteValue(kvp.Value); + } + finalWriter.WriteEndObject(); + + foreach (var kvp in childClosures) + { + closures[kvp.Key] = closures.TryGetValue(kvp.Key, out var existing) ? existing + kvp.Value : kvp.Value; + } + } + + finalWriter.WriteEndObject(); + finalWriter.Flush(); + + var json = new Json(stringWriter.ToString()); + return (id, json); + } + finally + { + Pools.StringBuilders.Return(sb); + } + } + + private void SerializeValue( + object? value, + JsonWriter writer, + bool isDetachable, + Dictionary closures, + List<(Id, Json, Dictionary, Base, string)> detachedObjects + ) + { + if (value == null) + { + writer.WriteNull(); + return; + } + + switch (value) + { + case string s: + writer.WriteValue(s); + return; + case int i: + writer.WriteValue(i); + return; + case long l: + writer.WriteValue(l); + return; + case double d: + writer.WriteValue(d); + return; + case float f: + writer.WriteValue(f); + return; + case bool b: + writer.WriteValue(b); + return; + case Enum e: + writer.WriteValue((int)(object)e); + return; + case Guid g: + writer.WriteValue(g.ToString()); + return; + case DateTime dt: + writer.WriteValue(dt.ToString("o")); + return; + + case Matrix4x4 md: + writer.WriteStartArray(); + writer.WriteValue(md.M11); + writer.WriteValue(md.M12); + writer.WriteValue(md.M13); + writer.WriteValue(md.M14); + writer.WriteValue(md.M21); + writer.WriteValue(md.M22); + writer.WriteValue(md.M23); + writer.WriteValue(md.M24); + writer.WriteValue(md.M31); + writer.WriteValue(md.M32); + writer.WriteValue(md.M33); + writer.WriteValue(md.M34); + writer.WriteValue(md.M41); + writer.WriteValue(md.M42); + writer.WriteValue(md.M43); + writer.WriteValue(md.M44); + writer.WriteEndArray(); + return; + } + + // Handle ObjectReference before Base (since ObjectReference extends Base) + // This prevents double-serialization and properly propagates closures + if (value is ObjectReference objRef) + { + writer.WriteStartObject(); + writer.WritePropertyName("speckle_type"); + writer.WriteValue("reference"); + writer.WritePropertyName("referencedId"); + writer.WriteValue(objRef.referencedId); + writer.WriteEndObject(); + + // Propagate closure: add the referenced ID + closures[objRef.referencedId] = closures.TryGetValue(objRef.referencedId, out var existing) ? existing + 1 : 1; + + // Propagate nested closures from the ObjectReference.closure dictionary + if (objRef.closure != null) + { + foreach (var kvp in objRef.closure) + { + closures[kvp.Key] = closures.TryGetValue(kvp.Key, out var existingDepth) + ? existingDepth + kvp.Value + : kvp.Value; + } + } + + return; + } + + if (value is Base baseObj) + { + if (isDetachable) + { + var childClosures = new Dictionary(); + var (childId, childJson) = SerializeBase(baseObj, true, childClosures, detachedObjects); + + detachedObjects.Add((childId, childJson, childClosures, baseObj, baseObj.speckle_type)); + + writer.WriteStartObject(); + writer.WritePropertyName("speckle_type"); + writer.WriteValue("reference"); + writer.WritePropertyName("referencedId"); + writer.WriteValue(childId.Value); + writer.WriteEndObject(); + + closures[childId.Value] = closures.TryGetValue(childId.Value, out var existing) ? existing + 1 : 1; + + foreach (var kvp in childClosures) + { + closures[kvp.Key] = closures.TryGetValue(kvp.Key, out var existingDepth) + ? existingDepth + kvp.Value + : kvp.Value; + } + } + else + { + var inlineClosures = new Dictionary(); + var (_, inlineJson) = SerializeBase(baseObj, false, inlineClosures, detachedObjects); + + writer.WriteRawValue(inlineJson.Value); + + foreach (var kvp in inlineClosures) + { + closures[kvp.Key] = closures.TryGetValue(kvp.Key, out var existingDepth) + ? existingDepth + kvp.Value + : kvp.Value; + } + } + return; + } + + if (value is IDictionary dict) + { + writer.WriteStartObject(); + foreach (DictionaryEntry kvp in dict) + { + if (kvp.Key is not string key) + { + throw new ArgumentException("Dictionary keys must be strings"); + } + + writer.WritePropertyName(key); + SerializeValue(kvp.Value, writer, false, closures, detachedObjects); + } + writer.WriteEndObject(); + return; + } + + if (value is IEnumerable enumerable and not string) + { + writer.WriteStartArray(); + foreach (var item in enumerable) + { + SerializeValue(item, writer, isDetachable, closures, detachedObjects); + } + writer.WriteEndArray(); + return; + } + + writer.WriteValue(value.ToString()); + } + + private UploadItem ReferenceToUploadItem(ObjectReference existingRef) + { + var sb = Pools.StringBuilders.Get(); + try + { + using var stringWriter = new StringWriter(sb); + using var jsonWriter = new JsonTextWriter(stringWriter); + + jsonWriter.WriteStartObject(); + jsonWriter.WritePropertyName("speckle_type"); + jsonWriter.WriteValue("reference"); + jsonWriter.WritePropertyName("referencedId"); + jsonWriter.WriteValue(existingRef.referencedId); + jsonWriter.WritePropertyName("__closure"); + + if (existingRef.closure != null && existingRef.closure.Count > 0) + { + jsonWriter.WriteStartObject(); + foreach (var kvp in existingRef.closure) + { + jsonWriter.WritePropertyName(kvp.Key); + jsonWriter.WriteValue(kvp.Value); + } + jsonWriter.WriteEndObject(); + } + else + { + jsonWriter.WriteNull(); + } + + jsonWriter.WriteEndObject(); + jsonWriter.Flush(); + + var refJson = new Json(stringWriter.ToString()); + + return new UploadItem( + existingRef.referencedId, + refJson, + existingRef.speckle_type, + existingRef // Pass through the original ObjectReference + ); + } + finally + { + Pools.StringBuilders.Return(sb); + } + } +} diff --git a/src/Speckle.Sdk/Pipelines/Uploader.cs b/src/Speckle.Sdk/Pipelines/Uploader.cs new file mode 100644 index 00000000..ec8133a4 --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Uploader.cs @@ -0,0 +1,175 @@ +using System.IO.Compression; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Threading.Channels; + +namespace Speckle.Sdk.Pipelines; + +public sealed class Uploader : IDisposable +{ + private readonly string _projectId; + private readonly string _modelId; + private readonly string _ingestionId; + private readonly CancellationToken _cancellationToken; + private readonly HttpClient _client; + private readonly Channel _channel; + private readonly Task _sendTask; + + public Uploader( + string projectId, + string modelId, + string ingestionId, + string apiEndpoint, + string authToken, + CancellationToken cancellationToken + ) + { + _projectId = projectId; + _modelId = modelId; + _ingestionId = ingestionId; + _cancellationToken = cancellationToken; + + Uri apiBaseUrl = new(new(apiEndpoint), "/api/v1/"); + _client = new HttpClient { BaseAddress = apiBaseUrl, Timeout = TimeSpan.FromMinutes(30) }; + + _client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", authToken); + + _channel = Channel.CreateBounded( + new BoundedChannelOptions(1000) { FullMode = BoundedChannelFullMode.Wait } + ); + + _sendTask = SendLoopAsync(); + } + + public ValueTask PushAsync(UploadItem item, CancellationToken ct = default) => _channel.Writer.WriteAsync(item, ct); + + public async Task CompleteAsync() + { + _channel.Writer.Complete(); + var result = await _sendTask.ConfigureAwait(false); + return result.IngestionId; + } + + private async Task SendLoopAsync() + { + // 1. Stream channel to temp file + string tempFilePath = Path.GetTempFileName(); + System.Diagnostics.Debug.WriteLine($"Temp file is at {tempFilePath}"); + try + { + long fileSizeBytes; + { + using var fileStream = new FileStream(tempFilePath, FileMode.Create, FileAccess.Write, FileShare.None); + using var gzip = new GZipStream(fileStream, CompressionLevel.Optimal); + using var writer = new StreamWriter(gzip); + await foreach (var item in _channel.Reader.ReadAllAsync(_cancellationToken).ConfigureAwait(false)) + { + await writer.WriteLineAsync($"{item.Id}\t{item.Json}\t{item.SpeckleType}").ConfigureAwait(false); + } + + await writer.FlushAsync().ConfigureAwait(false); + await gzip.FlushAsync(_cancellationToken).ConfigureAwait(false); + } + // fileStream.Flush(); + // fileStream.Close(); + fileSizeBytes = new FileInfo(tempFilePath).Length; + + // 2. Request presigned URL + var signUri = new Uri($"projects/{_projectId}/models/{_modelId}/uploads/sign", UriKind.Relative); + + var signResponse = await HttpClientExtensions + .PostAsJsonAsync(_client, signUri, _cancellationToken) + .ConfigureAwait(false); + signResponse.EnsureSuccessStatusCode(); + + var presignedUpload = + await signResponse.Content.ReadFromJsonAsync(_cancellationToken).ConfigureAwait(false) + ?? throw new InvalidOperationException("Failed to get presigned upload URL"); + + // 3. Upload to S3 + using var fileStreamUpload = new FileStream(tempFilePath, FileMode.Open, FileAccess.Read, FileShare.Read); + + Stream progressStream = fileStreamUpload; // TODO: wrap with progress stream + + using var streamContent = new StreamContent(progressStream); + streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); + streamContent.Headers.ContentLength = fileSizeBytes; + + using var uploadRequest = new HttpRequestMessage(HttpMethod.Put, new Uri(presignedUpload.Url, UriKind.Absolute)); + uploadRequest.Content = streamContent; + + using var s3Client = new HttpClient(); // NOTE: using a separate client for S3 as we DO NOT NEED THE AUTH HEADER, presigned url's don't work with it. + using var uploadResponse = await s3Client + .SendAsync(uploadRequest, HttpCompletionOption.ResponseHeadersRead, _cancellationToken) + .ConfigureAwait(false); + + uploadResponse.EnsureSuccessStatusCode(); + + // 4. Trigger processing + var processUri = new Uri($"projects/{_projectId}/models/{_modelId}/uploads/process", UriKind.Relative); + var processRequest = new ProcessUploadRequest { key = presignedUpload.Key, ingestionId = _ingestionId }; + + var processResponse = await HttpClientExtensions + .PostAsJsonAsync(_client, processUri, processRequest, _cancellationToken) + .ConfigureAwait(false); + processResponse.EnsureSuccessStatusCode(); + + var processResult = await processResponse + .Content.ReadFromJsonAsync(_cancellationToken) + .ConfigureAwait(false); + + if (processResult == null) + { + throw new InvalidOperationException("Failed to trigger upload processing"); + } + + return new UploadResult { IngestionId = processResult.ingestionId }; + } + catch (Exception ex) when (!ex.IsFatal()) + { + throw; + } + finally + { + // 5. Clean up temp file + if (File.Exists(tempFilePath)) + { + try + { + //File.Delete(tempFilePath); + } +#pragma warning disable CA1031 + catch +#pragma warning restore CA1031 + { + // Best effort + } + } + } + } + + public void Dispose() => _client.Dispose(); +} + +// DTOs +internal record PresignedUploadResponse +{ + public required string Url { get; init; } + public required string Key { get; init; } +} + +internal record ProcessUploadRequest +{ + public required string key { get; init; } + public required string ingestionId { get; init; } +} + +internal record ProcessUploadResponse +{ + public required string ingestionId { get; init; } +} + +internal record UploadResult +{ + public required string IngestionId { get; init; } +} diff --git a/src/Speckle.Sdk/Pipelines/Uploader_Old.cs b/src/Speckle.Sdk/Pipelines/Uploader_Old.cs new file mode 100644 index 00000000..fa8f73c6 --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Uploader_Old.cs @@ -0,0 +1,83 @@ +// using System.IO.Compression; +// using System.Net.Http.Headers; +// using System.Threading.Channels; +// +// namespace Speckle.Sdk.Pipeline; +// +// public sealed class UploaderOld : IDisposable +// { +// private readonly string _projectId; +// private readonly string _modelId; +// private readonly HttpClient _client; +// private readonly Channel _channel; +// private readonly Task _sendTask; +// +// public UploaderOld(string projectId, string modelId, string? authToken, string? apiEndpoint) +// { +// _projectId = projectId; +// _modelId = modelId; +// +// Uri apiBaseUrl = !string.IsNullOrEmpty(apiEndpoint) +// ? new Uri(apiEndpoint) +// // : new Uri("http://dimitries-macbook-pro.mermaid-emperor.ts.net/api/v1/"); +// : new Uri("http://zog.local:3000/api/v1/"); +// _client = new HttpClient { BaseAddress = apiBaseUrl, Timeout = TimeSpan.FromMinutes(10) }; +// +// if (authToken != null) +// { +// _client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", authToken); +// } +// +// _channel = Channel.CreateBounded( +// new BoundedChannelOptions(1000) { FullMode = BoundedChannelFullMode.Wait } // if we're not able to write fast enough, we'll block writes +// ); +// +// _sendTask = SendLoopAsync(projectId, "test"); +// } +// +// public ValueTask PushAsync(UploadItem item, CancellationToken ct = default) => _channel.Writer.WriteAsync(item, ct); +// +// public async Task CompleteAsync() +// { +// _channel.Writer.Complete(); +// await _sendTask.ConfigureAwait(false); +// } +// +// private async Task SendLoopAsync(string projectId, string modelId) +// { +// var content = new PushStreamContent( +// async (stream, _, _) => +// { +// var gzip = new GZipStream(stream, CompressionLevel.Optimal); +// var writer = new StreamWriter(gzip); // new StreamWriter(gzip, System.Text.Encoding.UTF8, 20 * 1024 * 1024); // potential lever for controlling memory pressure +// try +// { +// // extra levers for memory pressure in here: we can manually flush every x items or every x bytes +// await foreach (var item in _channel.Reader.ReadAllAsync().ConfigureAwait(false)) +// { +// await writer.WriteLineAsync($"{item.Id}\t{item.Json}\t{item.SpeckleType}").ConfigureAwait(false); +// } +// } +// finally +// { +// await writer.FlushAsync().ConfigureAwait(false); +// await gzip.FlushAsync().ConfigureAwait(false); +// writer.Dispose(); +// gzip.Dispose(); +// } +// }, +// new MediaTypeHeaderValue("application/x-ndjson") +// ); +// +// var uri = new Uri($"projects/{projectId}/models/{modelId}/versions", UriKind.Relative); +// var request = new HttpRequestMessage(HttpMethod.Post, uri) { Content = content }; +// request.Headers.TransferEncodingChunked = true; // NOTE: important for streaming to happen. +// var response = await _client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false); +// response.EnsureSuccessStatusCode(); +// +// // Consume the response body to fully complete the request +// await response.Content.ReadAsStringAsync().ConfigureAwait(false); +// } +// +// public void Dispose() => _client.Dispose(); +// } diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/BaseItem.cs b/src/Speckle.Sdk/Serialisation/V2/Send/BaseItem.cs index 2a88e3db..9c5b02bf 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Send/BaseItem.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Send/BaseItem.cs @@ -2,7 +2,13 @@ using System.Text; namespace Speckle.Sdk.Serialisation.V2.Send; -public sealed record BaseItem(Id Id, Json Json, bool NeedsStorage, Dictionary? Closures) : IHasByteSize +public sealed record BaseItem( + Id Id, + Json Json, + bool NeedsStorage, + Dictionary? Closures, + bool? IsReference = false +) : IHasByteSize { public int ByteSize { get; } = Encoding.UTF8.GetByteCount(Json.Value); diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/ObjectFlopper.cs b/src/Speckle.Sdk/Serialisation/V2/Send/ObjectFlopper.cs new file mode 100644 index 00000000..2027a6e3 --- /dev/null +++ b/src/Speckle.Sdk/Serialisation/V2/Send/ObjectFlopper.cs @@ -0,0 +1,79 @@ +using System.IO.Compression; +using System.Net.Http.Headers; +using System.Threading.Channels; + +namespace Speckle.Sdk.Serialisation.V2.Send; + +#pragma warning disable CA1001 +public sealed class ObjectFlopper +#pragma warning restore CA1001 +{ + private readonly Uri _url; + private readonly string _streamId; + private readonly HttpClient _client; + private readonly Channel _channel; + private readonly Task _sendTask; + + public ObjectFlopper(Uri _, string streamId, string? authToken) + { + _streamId = streamId; + _url = new Uri("http://zog.local:3000/api/v1/"); + _client = new HttpClient { BaseAddress = _url, Timeout = TimeSpan.FromMinutes(10) }; + + if (authToken != null) + { + _client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", authToken); + } + + _channel = Channel.CreateBounded( + new BoundedChannelOptions(1000) { FullMode = BoundedChannelFullMode.Wait } + ); + + _sendTask = SendLoopAsync(streamId, "test"); + } + + public ValueTask PushAsync(BaseItem item, CancellationToken ct = default) => _channel.Writer.WriteAsync(item, ct); + + public async Task CompleteAsync() + { + _channel.Writer.Complete(); + await _sendTask.ConfigureAwait(false); + } + + private async Task SendLoopAsync(string projectId, string modelId) + { + var content = new PushStreamContent( + async (stream, _, _) => + { + var gzip = new GZipStream(stream, CompressionLevel.Optimal); + var writer = new StreamWriter(gzip); //new StreamWriter(gzip, System.Text.Encoding.UTF8, 20 * 1024 * 1024); + try + { + await foreach (var item in _channel.Reader.ReadAllAsync().ConfigureAwait(false)) + { + await writer.WriteLineAsync($"{item.Id}\t{item.Json}").ConfigureAwait(false); + } + } + finally + { + await writer.FlushAsync().ConfigureAwait(false); + await gzip.FlushAsync().ConfigureAwait(false); + writer.Dispose(); + gzip.Dispose(); + } + }, + new MediaTypeHeaderValue("application/x-ndjson") + ); + + var uri = new Uri($"projects/{projectId}/models/{modelId}/versions", UriKind.Relative); + var request = new HttpRequestMessage(HttpMethod.Post, uri) { Content = content }; + request.Headers.TransferEncodingChunked = true; // NOTE: important for streaming to happen. + var response = await _client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + // Consume the response body to fully complete the request + await response.Content.ReadAsStringAsync().ConfigureAwait(false); + } + + public void Dispose() => _client.Dispose(); +} diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/ObjectFlopperGandalf.cs b/src/Speckle.Sdk/Serialisation/V2/Send/ObjectFlopperGandalf.cs new file mode 100644 index 00000000..abccd3a6 --- /dev/null +++ b/src/Speckle.Sdk/Serialisation/V2/Send/ObjectFlopperGandalf.cs @@ -0,0 +1,90 @@ +using System.IO.Compression; +using System.IO.Pipelines; +using System.Net.Http.Headers; +using System.Threading.Channels; + +namespace Speckle.Sdk.Serialisation.V2.Send; + +#pragma warning disable CA1001 +public sealed class ObjectFlopperGandalf +#pragma warning restore CA1001 +{ + private readonly Uri _url; + private readonly string _streamId; + private readonly HttpClient _client; + private readonly Channel _channel; + private readonly Task _sendTask; + + public ObjectFlopperGandalf(Uri _, string streamId, string? authToken) + { + _streamId = streamId; + _url = new Uri("http://bender-2.local:3000/api/v1/"); + _client = new HttpClient { BaseAddress = _url, Timeout = TimeSpan.FromMinutes(10) }; + + if (authToken != null) + { + _client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", authToken); + } + + _channel = Channel.CreateBounded( + new BoundedChannelOptions(1000) { FullMode = BoundedChannelFullMode.Wait } + ); + + _sendTask = SendLoopAsync(streamId, "test"); + } + + public ValueTask PushAsync(BaseItem item, CancellationToken ct = default) => _channel.Writer.WriteAsync(item, ct); + + public async Task CompleteAsync() + { + _channel.Writer.Complete(); + await _sendTask.ConfigureAwait(false); + } + + private async Task SendLoopAsync(string projectId, string modelId) + { + var pipe = new Pipe(); + + // Start writing to pipe immediately in background + var writeTask = Task.Run(async () => + { + var gzip = new GZipStream(pipe.Writer.AsStream(), CompressionLevel.Optimal); + var writer = new StreamWriter(gzip); + + try + { + await foreach (var item in _channel.Reader.ReadAllAsync().ConfigureAwait(false)) + { + await writer.WriteLineAsync($"{item.Id}\t{item.Json}").ConfigureAwait(false); + await writer.FlushAsync().ConfigureAwait(false); + } + } + finally + { + await writer.FlushAsync().ConfigureAwait(false); + await gzip.FlushAsync().ConfigureAwait(false); + writer.Dispose(); + gzip.Dispose(); + await pipe.Writer.CompleteAsync().ConfigureAwait(false); + } + }); + + // Start HTTP request immediately, reading from pipe + var content = new StreamContent(pipe.Reader.AsStream()); + content.Headers.ContentType = new MediaTypeHeaderValue("application/x-ndjson"); + content.Headers.ContentEncoding.Add("gzip"); + + var uri = new Uri($"projects/{projectId}/models/{modelId}/objects", UriKind.Relative); + var request = new HttpRequestMessage(HttpMethod.Post, uri) { Content = content }; + + var responseTask = _client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); + + // Wait for both + await Task.WhenAll(writeTask, responseTask).ConfigureAwait(false); + + var response = await responseTask.ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + } + + public void Dispose() => _client.Dispose(); +} diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs index 6dac6f6f..29965525 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs @@ -113,16 +113,6 @@ public sealed class SerializeProcess( _processSource.Token ); var findTotalObjectsTask = Task.CompletedTask; - if (!options.SkipFindTotalObjects) - { - ThrowIfFailed(); - findTotalObjectsTask = Task.Factory.StartNew( - () => TraverseTotal(root), - _processSource.Token, - TaskCreationOptions.AttachedToParent | TaskCreationOptions.PreferFairness, - _highest - ); - } await Traverse(root).ConfigureAwait(false); ThrowIfFailed(); @@ -133,6 +123,7 @@ public sealed class SerializeProcess( ThrowIfFailed(); await WaitForSchedulerCompletion().ConfigureAwait(false); ThrowIfFailed(); + return new(root.id.NotNull(), baseSerializer.ObjectReferences.Freeze()); } catch (OperationCanceledException) diff --git a/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs b/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs index b708f4d4..3c56b4af 100644 --- a/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs +++ b/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs @@ -35,7 +35,8 @@ public class SerializeProcessFactory( IServerObjectManager serverObjectManager, IProgress? progress, CancellationToken cancellationToken, - SerializeProcessOptions? options = null + SerializeProcessOptions? options = null, + ObjectFlopper? objectFlopper = null ) => new SerializeProcess( progress, diff --git a/src/Speckle.Sdk/Speckle.Sdk.csproj b/src/Speckle.Sdk/Speckle.Sdk.csproj index 5dbc5924..664d497a 100644 --- a/src/Speckle.Sdk/Speckle.Sdk.csproj +++ b/src/Speckle.Sdk/Speckle.Sdk.csproj @@ -23,19 +23,23 @@ + + + + + - diff --git a/src/Speckle.Sdk/packages.lock.json b/src/Speckle.Sdk/packages.lock.json index 5ea95453..b7195702 100644 --- a/src/Speckle.Sdk/packages.lock.json +++ b/src/Speckle.Sdk/packages.lock.json @@ -13,13 +13,25 @@ "System.Reactive": "5.0.0" } }, + "Microsoft.AspNet.WebApi.Client": { + "type": "Direct", + "requested": "[6.0.0, )", + "resolved": "6.0.0", + "contentHash": "zXeWP03dTo67AoDHUzR+/urck0KFssdCKOC+dq7Nv1V2YbFh/nIg09L0/3wSvyRONEdwxGB/ssEGmPNIIhAcAw==", + "dependencies": { + "Newtonsoft.Json": "13.0.1", + "Newtonsoft.Json.Bson": "1.0.2", + "System.Memory": "4.5.5", + "System.Threading.Tasks.Extensions": "4.5.4" + } + }, "Microsoft.Bcl.AsyncInterfaces": { "type": "Direct", - "requested": "[5.0.0, )", - "resolved": "5.0.0", - "contentHash": "W8DPQjkMScOMTtJbPwmPyj9c3zYSFGawDW3jwlBOOsnY+EzZFLgNQ/UMkK35JmkNOVPdCyPr2Tw7Vv9N+KA3ZQ==", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "E1HSLkPHXEO30JEij2pWbOuzz1Z5ND4a5l7IP1T2RgQuE0a0NzEIvtO64RNy3Otn6PFezbT80cfm3M/Cgt70PA==", "dependencies": { - "System.Threading.Tasks.Extensions": "4.5.4" + "System.Threading.Tasks.Extensions": "4.6.3" } }, "Microsoft.CSharp": { @@ -99,6 +111,39 @@ "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" }, + "System.IO.Pipelines": { + "type": "Direct", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "26LbFXHKd7PmRnWlkjnYgmjd5B6HYVG+1MpTO25BdxTJnx6D0O16JPAC/S4YBqjtt4YpfGj1QO/Ss6SPMGEGQw==", + "dependencies": { + "System.Buffers": "4.6.1", + "System.Memory": "4.6.3", + "System.Threading.Tasks.Extensions": "4.6.3" + } + }, + "System.Net.Http.Json": { + "type": "Direct", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "XGOWt78ccgO9esyNlCemlS2b8JZPrH85pk/MvdHJxp6KwwUY/GnDaw2fPpJa7lgotiTkWnXnhip+ULNKaP7a8A==", + "dependencies": { + "System.Buffers": "4.6.1", + "System.Memory": "4.6.3", + "System.Text.Json": "10.0.1", + "System.Threading.Tasks.Extensions": "4.6.3" + } + }, + "System.Threading.Channels": { + "type": "Direct", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "10.0.1", + "System.Threading.Tasks.Extensions": "4.6.3" + } + }, "GraphQL.Client.Abstractions": { "type": "Transitive", "resolved": "6.0.0", @@ -196,6 +241,19 @@ "resolved": "8.0.0", "contentHash": "dk9JPxTCIevS75HyEQ0E4OVAFhB2N+V9ShCXf8Q6FkUQZDkgLI12y679Nym1YqsiSysuQskT7Z+6nUf3yab6Vw==" }, + "Newtonsoft.Json": { + "type": "Transitive", + "resolved": "13.0.1", + "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" + }, + "Newtonsoft.Json.Bson": { + "type": "Transitive", + "resolved": "1.0.2", + "contentHash": "QYFyxhaABwmq3p/21VrZNYvCg3DaEoN/wUuw5nmfAf0X3HLjgupwhkEWdgfb9nvGAUIv3osmZoD3kKl4jxEmYQ==", + "dependencies": { + "Newtonsoft.Json": "12.0.1" + } + }, "SQLitePCLRaw.bundle_e_sqlite3": { "type": "Transitive", "resolved": "2.1.4", @@ -228,8 +286,8 @@ }, "System.Buffers": { "type": "Transitive", - "resolved": "4.4.0", - "contentHash": "AwarXzzoDwX6BgrhjoJsk6tUezZEozOT5Y9QKF94Gl4JK91I4PIIBkBco9068Y9/Dra8Dkbie99kXB8+1BaYKw==" + "resolved": "4.6.1", + "contentHash": "N8GXpmiLMtljq7gwvyS+1QvKT/W2J8sNAvx+HVg4NGmsG/H+2k/y9QI23auLJRterrzCiDH+IWAw4V/GPwsMlw==" }, "System.ComponentModel.Annotations": { "type": "Transitive", @@ -238,18 +296,18 @@ }, "System.Memory": { "type": "Transitive", - "resolved": "4.5.3", - "contentHash": "3oDzvc/zzetpTKWMShs1AADwZjQ/36HnsufHRPcOjyRAAMLDlu2iD33MBI2opxnezcVUtXyqDXXjoFMOU9c7SA==", + "resolved": "4.6.3", + "contentHash": "qdcDOgnFZY40+Q9876JUHnlHu7bosOHX8XISRoH94fwk6hgaeQGSgfZd8srWRZNt5bV9ZW2TljcegDNxsf+96A==", "dependencies": { - "System.Buffers": "4.4.0", - "System.Numerics.Vectors": "4.4.0", - "System.Runtime.CompilerServices.Unsafe": "4.5.2" + "System.Buffers": "4.6.1", + "System.Numerics.Vectors": "4.6.1", + "System.Runtime.CompilerServices.Unsafe": "6.1.2" } }, "System.Numerics.Vectors": { "type": "Transitive", - "resolved": "4.4.0", - "contentHash": "UiLzLW+Lw6HLed1Hcg+8jSRttrbuXv7DANVj0DkL9g6EnnzbL75EB7EWsw5uRbhxd/4YdG8li5XizGWepmG3PQ==" + "resolved": "4.6.1", + "contentHash": "sQxefTnhagrhoq2ReR0D/6K0zJcr9Hrd6kikeXsA1I8kOCboTavcUC4r7TSfpKFeE163uMuxZcyfO1mGO3EN8Q==" }, "System.Reactive": { "type": "Transitive", @@ -271,8 +329,8 @@ }, "System.Runtime.CompilerServices.Unsafe": { "type": "Transitive", - "resolved": "4.5.3", - "contentHash": "3TIsJhD1EiiT0w2CcDMN/iSSwnNnsrnbzeVHSKkaEgV85txMprmuO+Yq2AdSbeVGcg28pdNDTPK87tJhX7VFHw==" + "resolved": "6.1.2", + "contentHash": "2hBr6zdbIBTDE3EhK7NSVNdX58uTK6iHW/P/Axmm9sl1xoGSLqDvMtpecn226TNwHByFokYwJmt/aQQNlO5CRw==" }, "System.Runtime.InteropServices.WindowsRuntime": { "type": "Transitive", @@ -282,16 +340,75 @@ "System.Runtime": "4.3.0" } }, + "System.Text.Encodings.Web": { + "type": "Transitive", + "resolved": "10.0.1", + "contentHash": "cVAka0o1rJJ5/De0pjNs7jcaZk5hUGf1HGzUyVmE2MEB1Vf0h/8qsWxImk1zjitCbeD2Avaq2P2+usdvqgbeVQ==", + "dependencies": { + "System.Buffers": "4.6.1", + "System.Memory": "4.6.3", + "System.Runtime.CompilerServices.Unsafe": "6.1.2" + } + }, "System.Threading.Tasks.Extensions": { "type": "Transitive", - "resolved": "4.5.4", - "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==", + "resolved": "4.6.3", + "contentHash": "7sCiwilJLYbTZELaKnc7RecBBXWXA+xMLQWZKWawBxYjp6DBlSE3v9/UcvKBvr1vv2tTOhipiogM8rRmxlhrVA==", "dependencies": { - "System.Runtime.CompilerServices.Unsafe": "4.5.3" + "System.Runtime.CompilerServices.Unsafe": "6.1.2" + } + }, + "speckle.connectors.common": { + "type": "Project", + "dependencies": { + "Microsoft.Extensions.DependencyInjection": "[2.2.0, )", + "Speckle.Connectors.Logging": "[1.0.0, )", + "Speckle.Converters.Common": "[1.0.0, )", + "Speckle.Objects": "[1.0.0, )" + } + }, + "speckle.connectors.logging": { + "type": "Project" + }, + "speckle.converters.common": { + "type": "Project", + "dependencies": { + "Microsoft.Extensions.Logging.Abstractions": "[2.2.0, )", + "Speckle.Objects": "[1.0.0, )" + } + }, + "speckle.objects": { + "type": "Project", + "dependencies": { + "Speckle.Sdk": "[1.0.0, )" } }, "speckle.sdk.dependencies": { "type": "Project" + }, + "Microsoft.Extensions.DependencyInjection": { + "type": "CentralTransitive", + "requested": "[2.2.0, )", + "resolved": "2.2.0", + "contentHash": "MZtBIwfDFork5vfjpJdG5g8wuJFt7d/y3LOSVVtDK/76wlbtz6cjltfKHqLx2TKVqTj5/c41t77m1+h20zqtPA==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0" + } + }, + "System.Text.Json": { + "type": "CentralTransitive", + "requested": "[8.0.5, )", + "resolved": "10.0.1", + "contentHash": "EsgwDgU1PFqhrFA9l5n+RBu76wFhNGCEwu8ITrBNhjPP3MxLyklroU5GIF8o6JYpYg6T4KD/VICfMdgPAvNp5g==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "10.0.1", + "System.Buffers": "4.6.1", + "System.IO.Pipelines": "10.0.1", + "System.Memory": "4.6.3", + "System.Runtime.CompilerServices.Unsafe": "6.1.2", + "System.Text.Encodings.Web": "10.0.1", + "System.Threading.Tasks.Extensions": "4.6.3" + } } }, "net8.0": { @@ -306,6 +423,18 @@ "System.Reactive": "5.0.0" } }, + "Microsoft.AspNet.WebApi.Client": { + "type": "Direct", + "requested": "[6.0.0, )", + "resolved": "6.0.0", + "contentHash": "zXeWP03dTo67AoDHUzR+/urck0KFssdCKOC+dq7Nv1V2YbFh/nIg09L0/3wSvyRONEdwxGB/ssEGmPNIIhAcAw==", + "dependencies": { + "Newtonsoft.Json": "13.0.1", + "Newtonsoft.Json.Bson": "1.0.2", + "System.Memory": "4.5.5", + "System.Threading.Tasks.Extensions": "4.5.4" + } + }, "Microsoft.Data.Sqlite": { "type": "Direct", "requested": "[7.0.5, )", @@ -368,6 +497,27 @@ "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" }, + "System.IO.Pipelines": { + "type": "Direct", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "26LbFXHKd7PmRnWlkjnYgmjd5B6HYVG+1MpTO25BdxTJnx6D0O16JPAC/S4YBqjtt4YpfGj1QO/Ss6SPMGEGQw==" + }, + "System.Net.Http.Json": { + "type": "Direct", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "XGOWt78ccgO9esyNlCemlS2b8JZPrH85pk/MvdHJxp6KwwUY/GnDaw2fPpJa7lgotiTkWnXnhip+ULNKaP7a8A==", + "dependencies": { + "System.Text.Json": "10.0.1" + } + }, + "System.Threading.Channels": { + "type": "Direct", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" + }, "GraphQL.Client.Abstractions": { "type": "Transitive", "resolved": "6.0.0", @@ -455,6 +605,19 @@ "resolved": "8.0.0", "contentHash": "dk9JPxTCIevS75HyEQ0E4OVAFhB2N+V9ShCXf8Q6FkUQZDkgLI12y679Nym1YqsiSysuQskT7Z+6nUf3yab6Vw==" }, + "Newtonsoft.Json": { + "type": "Transitive", + "resolved": "13.0.1", + "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" + }, + "Newtonsoft.Json.Bson": { + "type": "Transitive", + "resolved": "1.0.2", + "contentHash": "QYFyxhaABwmq3p/21VrZNYvCg3DaEoN/wUuw5nmfAf0X3HLjgupwhkEWdgfb9nvGAUIv3osmZoD3kKl4jxEmYQ==", + "dependencies": { + "Newtonsoft.Json": "12.0.1" + } + }, "SQLitePCLRaw.bundle_e_sqlite3": { "type": "Transitive", "resolved": "2.1.4", @@ -492,8 +655,8 @@ }, "System.Memory": { "type": "Transitive", - "resolved": "4.5.3", - "contentHash": "3oDzvc/zzetpTKWMShs1AADwZjQ/36HnsufHRPcOjyRAAMLDlu2iD33MBI2opxnezcVUtXyqDXXjoFMOU9c7SA==" + "resolved": "4.5.5", + "contentHash": "XIWiDvKPXaTveaB7HVganDlOCRoj03l+jrwNvcge/t8vhGYKvqV+dMv6G4SAX2NoNmN0wZfVPTAlFwZcZvVOUw==" }, "System.Reactive": { "type": "Transitive", @@ -505,8 +668,62 @@ "resolved": "4.5.1", "contentHash": "Zh8t8oqolRaFa9vmOZfdQm/qKejdqz0J9kr7o2Fu0vPeoH3BL1EOXipKWwkWtLT1JPzjByrF19fGuFlNbmPpiw==" }, + "System.Text.Encodings.Web": { + "type": "Transitive", + "resolved": "10.0.1", + "contentHash": "cVAka0o1rJJ5/De0pjNs7jcaZk5hUGf1HGzUyVmE2MEB1Vf0h/8qsWxImk1zjitCbeD2Avaq2P2+usdvqgbeVQ==" + }, + "System.Threading.Tasks.Extensions": { + "type": "Transitive", + "resolved": "4.5.4", + "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==" + }, + "speckle.connectors.common": { + "type": "Project", + "dependencies": { + "Microsoft.Extensions.DependencyInjection": "[2.2.0, )", + "Speckle.Connectors.Logging": "[1.0.0, )", + "Speckle.Converters.Common": "[1.0.0, )", + "Speckle.Objects": "[1.0.0, )" + } + }, + "speckle.connectors.logging": { + "type": "Project" + }, + "speckle.converters.common": { + "type": "Project", + "dependencies": { + "Microsoft.Extensions.Logging.Abstractions": "[2.2.0, )", + "Speckle.Objects": "[1.0.0, )" + } + }, + "speckle.objects": { + "type": "Project", + "dependencies": { + "Speckle.Sdk": "[1.0.0, )" + } + }, "speckle.sdk.dependencies": { "type": "Project" + }, + "Microsoft.Extensions.DependencyInjection": { + "type": "CentralTransitive", + "requested": "[2.2.0, )", + "resolved": "2.2.0", + "contentHash": "MZtBIwfDFork5vfjpJdG5g8wuJFt7d/y3LOSVVtDK/76wlbtz6cjltfKHqLx2TKVqTj5/c41t77m1+h20zqtPA==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0" + } + }, + "System.Text.Json": { + "type": "CentralTransitive", + "requested": "[8.0.5, )", + "resolved": "10.0.1", + "contentHash": "EsgwDgU1PFqhrFA9l5n+RBu76wFhNGCEwu8ITrBNhjPP3MxLyklroU5GIF8o6JYpYg6T4KD/VICfMdgPAvNp5g==", + "dependencies": { + "System.IO.Pipelines": "10.0.1", + "System.Text.Encodings.Web": "10.0.1" + } } } } From 309cead189ec27ce7c7a1dc1f980e62f75aee64e Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Tue, 10 Feb 2026 12:25:05 +0000 Subject: [PATCH 02/26] changes to serializer value handilng (#440) --- src/Speckle.Sdk/Pipelines/Serializer.cs | 211 +++++++++++------------- 1 file changed, 95 insertions(+), 116 deletions(-) diff --git a/src/Speckle.Sdk/Pipelines/Serializer.cs b/src/Speckle.Sdk/Pipelines/Serializer.cs index ed072a50..3f551db3 100644 --- a/src/Speckle.Sdk/Pipelines/Serializer.cs +++ b/src/Speckle.Sdk/Pipelines/Serializer.cs @@ -1,4 +1,6 @@ using System.Collections; +using System.Drawing; +using System.Globalization; using System.Reflection; using Speckle.DoubleNumerics; using Speckle.Newtonsoft.Json; @@ -163,42 +165,20 @@ public class Serializer List<(Id, Json, Dictionary, Base, string)> detachedObjects ) { - if (value == null) - { - writer.WriteNull(); - return; - } - switch (value) { - case string s: - writer.WriteValue(s); - return; - case int i: - writer.WriteValue(i); - return; - case long l: - writer.WriteValue(l); - return; - case double d: - writer.WriteValue(d); - return; - case float f: - writer.WriteValue(f); - return; - case bool b: - writer.WriteValue(b); - return; - case Enum e: - writer.WriteValue((int)(object)e); + case Enum: + writer.WriteValue((int)value); return; case Guid g: writer.WriteValue(g.ToString()); return; - case DateTime dt: - writer.WriteValue(dt.ToString("o")); + case Color c: + writer.WriteValue(c.ToArgb()); + return; + case DateTime dt: + writer.WriteValue(dt.ToString("o", CultureInfo.InvariantCulture)); return; - case Matrix4x4 md: writer.WriteStartArray(); writer.WriteValue(md.M11); @@ -219,107 +199,106 @@ public class Serializer writer.WriteValue(md.M44); writer.WriteEndArray(); return; - } - - // Handle ObjectReference before Base (since ObjectReference extends Base) - // This prevents double-serialization and properly propagates closures - if (value is ObjectReference objRef) - { - writer.WriteStartObject(); - writer.WritePropertyName("speckle_type"); - writer.WriteValue("reference"); - writer.WritePropertyName("referencedId"); - writer.WriteValue(objRef.referencedId); - writer.WriteEndObject(); - - // Propagate closure: add the referenced ID - closures[objRef.referencedId] = closures.TryGetValue(objRef.referencedId, out var existing) ? existing + 1 : 1; - - // Propagate nested closures from the ObjectReference.closure dictionary - if (objRef.closure != null) + // Handle ObjectReference before Base (since ObjectReference extends Base) + // This prevents double-serialization and properly propagates closures + case ObjectReference objRef: { - foreach (var kvp in objRef.closure) - { - closures[kvp.Key] = closures.TryGetValue(kvp.Key, out var existingDepth) - ? existingDepth + kvp.Value - : kvp.Value; - } - } - - return; - } - - if (value is Base baseObj) - { - if (isDetachable) - { - var childClosures = new Dictionary(); - var (childId, childJson) = SerializeBase(baseObj, true, childClosures, detachedObjects); - - detachedObjects.Add((childId, childJson, childClosures, baseObj, baseObj.speckle_type)); - writer.WriteStartObject(); writer.WritePropertyName("speckle_type"); writer.WriteValue("reference"); writer.WritePropertyName("referencedId"); - writer.WriteValue(childId.Value); + writer.WriteValue(objRef.referencedId); writer.WriteEndObject(); - closures[childId.Value] = closures.TryGetValue(childId.Value, out var existing) ? existing + 1 : 1; + // Propagate closure: add the referenced ID + closures[objRef.referencedId] = closures.TryGetValue(objRef.referencedId, out var existing) ? existing + 1 : 1; - foreach (var kvp in childClosures) + // Propagate nested closures from the ObjectReference.closure dictionary + if (objRef.closure != null) { - closures[kvp.Key] = closures.TryGetValue(kvp.Key, out var existingDepth) - ? existingDepth + kvp.Value - : kvp.Value; - } - } - else - { - var inlineClosures = new Dictionary(); - var (_, inlineJson) = SerializeBase(baseObj, false, inlineClosures, detachedObjects); - - writer.WriteRawValue(inlineJson.Value); - - foreach (var kvp in inlineClosures) - { - closures[kvp.Key] = closures.TryGetValue(kvp.Key, out var existingDepth) - ? existingDepth + kvp.Value - : kvp.Value; - } - } - return; - } - - if (value is IDictionary dict) - { - writer.WriteStartObject(); - foreach (DictionaryEntry kvp in dict) - { - if (kvp.Key is not string key) - { - throw new ArgumentException("Dictionary keys must be strings"); + foreach (var kvp in objRef.closure) + { + closures[kvp.Key] = closures.TryGetValue(kvp.Key, out var existingDepth) + ? existingDepth + kvp.Value + : kvp.Value; + } } - writer.WritePropertyName(key); - SerializeValue(kvp.Value, writer, false, closures, detachedObjects); + return; } - writer.WriteEndObject(); - return; - } - - if (value is IEnumerable enumerable and not string) - { - writer.WriteStartArray(); - foreach (var item in enumerable) + case Base baseObj: { - SerializeValue(item, writer, isDetachable, closures, detachedObjects); - } - writer.WriteEndArray(); - return; - } + if (isDetachable) + { + var childClosures = new Dictionary(); + var (childId, childJson) = SerializeBase(baseObj, true, childClosures, detachedObjects); - writer.WriteValue(value.ToString()); + detachedObjects.Add((childId, childJson, childClosures, baseObj, baseObj.speckle_type)); + + writer.WriteStartObject(); + writer.WritePropertyName("speckle_type"); + writer.WriteValue("reference"); + writer.WritePropertyName("referencedId"); + writer.WriteValue(childId.Value); + writer.WriteEndObject(); + + closures[childId.Value] = closures.TryGetValue(childId.Value, out var existing) ? existing + 1 : 1; + + foreach (var kvp in childClosures) + { + closures[kvp.Key] = closures.TryGetValue(kvp.Key, out var existingDepth) + ? existingDepth + kvp.Value + : kvp.Value; + } + } + else + { + var inlineClosures = new Dictionary(); + var (_, inlineJson) = SerializeBase(baseObj, false, inlineClosures, detachedObjects); + + writer.WriteRawValue(inlineJson.Value); + + foreach (var kvp in inlineClosures) + { + closures[kvp.Key] = closures.TryGetValue(kvp.Key, out var existingDepth) + ? existingDepth + kvp.Value + : kvp.Value; + } + } + return; + } + case IDictionary dict: + { + writer.WriteStartObject(); + foreach (DictionaryEntry kvp in dict) + { + if (kvp.Key is not string key) + { + throw new ArgumentException("Dictionary keys must be strings", nameof(value)); + } + + writer.WritePropertyName(key); + SerializeValue(kvp.Value, writer, false, closures, detachedObjects); + } + writer.WriteEndObject(); + return; + } + case ICollection collection: + { + writer.WriteStartArray(); + foreach (var item in collection) + { + SerializeValue(item, writer, isDetachable, closures, detachedObjects); + } + writer.WriteEndArray(); + return; + } + default: + // This case will handle primitives and `null` + // Will throw JsonWriterException if not supported + writer.WriteValue(value); + return; + } } private UploadItem ReferenceToUploadItem(ObjectReference existingRef) From bf6ae0f6af2b626317830ff9f6f5227310c8897a Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Tue, 10 Feb 2026 16:29:27 +0000 Subject: [PATCH 03/26] Duck: Cleanup dependencies (#441) * Remove unused dependencies * remove unused package references * how's this * missed a bit --- Directory.Packages.props | 6 +- src/Speckle.Automate.Sdk/packages.lock.json | 50 ++-- src/Speckle.Objects/packages.lock.json | 179 ++----------- .../packages.lock.json | 17 +- src/Speckle.Sdk/Pipelines/SendPipeline.cs | 9 +- src/Speckle.Sdk/Pipelines/Uploader.cs | 53 ++-- .../Serialisation/V2/Send/ObjectFlopper.cs | 79 ------ .../V2/Send/ObjectFlopperGandalf.cs | 90 ------- .../V2/SerializeProcessFactory.cs | 3 +- src/Speckle.Sdk/Speckle.Sdk.csproj | 4 - src/Speckle.Sdk/packages.lock.json | 240 ++---------------- .../packages.lock.json | 9 +- .../packages.lock.json | 9 +- .../packages.lock.json | 20 +- .../packages.lock.json | 9 +- tests/Speckle.Sdk.Testing/packages.lock.json | 9 +- .../packages.lock.json | 9 +- .../packages.lock.json | 20 +- .../Speckle.Sdk.Tests.Unit/packages.lock.json | 9 +- 19 files changed, 193 insertions(+), 631 deletions(-) delete mode 100644 src/Speckle.Sdk/Serialisation/V2/Send/ObjectFlopper.cs delete mode 100644 src/Speckle.Sdk/Serialisation/V2/Send/ObjectFlopperGandalf.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 6bf24322..a2c96acf 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -8,8 +8,6 @@ - - @@ -29,8 +27,6 @@ - - @@ -42,4 +38,4 @@ - \ No newline at end of file + diff --git a/src/Speckle.Automate.Sdk/packages.lock.json b/src/Speckle.Automate.Sdk/packages.lock.json index 5e6185c6..1a222bbb 100644 --- a/src/Speckle.Automate.Sdk/packages.lock.json +++ b/src/Speckle.Automate.Sdk/packages.lock.json @@ -86,6 +86,14 @@ "resolved": "6.0.0", "contentHash": "yg72rrYDapfsIUrul7aF6wwNnTJBOFvuA9VdDTQpPa8AlAriHbufeXYLBcodKjfUdkCnaiggX1U/nEP08Zb5GA==" }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "Transitive", + "resolved": "10.0.1", + "contentHash": "E1HSLkPHXEO30JEij2pWbOuzz1Z5ND4a5l7IP1T2RgQuE0a0NzEIvtO64RNy3Otn6PFezbT80cfm3M/Cgt70PA==", + "dependencies": { + "System.Threading.Tasks.Extensions": "4.6.3" + } + }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", "resolved": "8.0.0", @@ -242,8 +250,8 @@ }, "System.Runtime.CompilerServices.Unsafe": { "type": "Transitive", - "resolved": "6.0.0", - "contentHash": "/iUeP3tq1S0XdNNoMz5C9twLSrM/TH+qElHkXWaPvuNOt+99G75NrV0OS2EqHx5wMN7popYjpc8oTjC1y16DLg==" + "resolved": "6.1.2", + "contentHash": "2hBr6zdbIBTDE3EhK7NSVNdX58uTK6iHW/P/Axmm9sl1xoGSLqDvMtpecn226TNwHByFokYwJmt/aQQNlO5CRw==" }, "System.Runtime.InteropServices.WindowsRuntime": { "type": "Transitive", @@ -265,10 +273,10 @@ }, "System.Threading.Tasks.Extensions": { "type": "Transitive", - "resolved": "4.5.4", - "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==", + "resolved": "4.6.3", + "contentHash": "7sCiwilJLYbTZELaKnc7RecBBXWXA+xMLQWZKWawBxYjp6DBlSE3v9/UcvKBvr1vv2tTOhipiogM8rRmxlhrVA==", "dependencies": { - "System.Runtime.CompilerServices.Unsafe": "4.5.3" + "System.Runtime.CompilerServices.Unsafe": "6.1.2" } }, "speckle.objects": { @@ -281,14 +289,14 @@ "type": "Project", "dependencies": { "GraphQL.Client": "[6.0.0, )", - "Microsoft.Bcl.AsyncInterfaces": "[5.0.0, )", "Microsoft.CSharp": "[4.7.0, )", "Microsoft.Data.Sqlite": "[7.0.5, )", "Microsoft.Extensions.DependencyInjection.Abstractions": "[2.2.0, )", "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )", + "System.Threading.Channels": "[10.0.1, )" } }, "speckle.sdk.dependencies": { @@ -305,15 +313,6 @@ "System.Reactive": "5.0.0" } }, - "Microsoft.Bcl.AsyncInterfaces": { - "type": "CentralTransitive", - "requested": "[5.0.0, )", - "resolved": "8.0.0", - "contentHash": "3WA9q9yVqJp222P3x1wYIGDAkpjAku0TMUaaQV22g6L67AI0LdOIrVS7Ht2vJfLHGSPVuqN94vIr15qn+HEkHw==", - "dependencies": { - "System.Threading.Tasks.Extensions": "4.5.4" - } - }, "Microsoft.CSharp": { "type": "CentralTransitive", "requested": "[4.7.0, )", @@ -359,6 +358,16 @@ "requested": "[13.0.2, )", "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" + }, + "System.Threading.Channels": { + "type": "CentralTransitive", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "10.0.1", + "System.Threading.Tasks.Extensions": "4.6.3" + } } }, "net8.0": { @@ -556,7 +565,8 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )", + "System.Threading.Channels": "[10.0.1, )" } }, "speckle.sdk.dependencies": { @@ -612,6 +622,12 @@ "requested": "[13.0.2, )", "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" + }, + "System.Threading.Channels": { + "type": "CentralTransitive", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" } } } diff --git a/src/Speckle.Objects/packages.lock.json b/src/Speckle.Objects/packages.lock.json index 2c2e04f1..5042eec7 100644 --- a/src/Speckle.Objects/packages.lock.json +++ b/src/Speckle.Objects/packages.lock.json @@ -54,6 +54,14 @@ "resolved": "6.0.0", "contentHash": "yg72rrYDapfsIUrul7aF6wwNnTJBOFvuA9VdDTQpPa8AlAriHbufeXYLBcodKjfUdkCnaiggX1U/nEP08Zb5GA==" }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "Transitive", + "resolved": "10.0.1", + "contentHash": "E1HSLkPHXEO30JEij2pWbOuzz1Z5ND4a5l7IP1T2RgQuE0a0NzEIvtO64RNy3Otn6PFezbT80cfm3M/Cgt70PA==", + "dependencies": { + "System.Threading.Tasks.Extensions": "4.6.3" + } + }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", "resolved": "8.0.0", @@ -130,19 +138,6 @@ "resolved": "8.0.0", "contentHash": "dk9JPxTCIevS75HyEQ0E4OVAFhB2N+V9ShCXf8Q6FkUQZDkgLI12y679Nym1YqsiSysuQskT7Z+6nUf3yab6Vw==" }, - "Newtonsoft.Json": { - "type": "Transitive", - "resolved": "13.0.1", - "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" - }, - "Newtonsoft.Json.Bson": { - "type": "Transitive", - "resolved": "1.0.2", - "contentHash": "QYFyxhaABwmq3p/21VrZNYvCg3DaEoN/wUuw5nmfAf0X3HLjgupwhkEWdgfb9nvGAUIv3osmZoD3kKl4jxEmYQ==", - "dependencies": { - "Newtonsoft.Json": "12.0.1" - } - }, "SQLitePCLRaw.bundle_e_sqlite3": { "type": "Transitive", "resolved": "2.1.4", @@ -175,8 +170,8 @@ }, "System.Buffers": { "type": "Transitive", - "resolved": "4.6.1", - "contentHash": "N8GXpmiLMtljq7gwvyS+1QvKT/W2J8sNAvx+HVg4NGmsG/H+2k/y9QI23auLJRterrzCiDH+IWAw4V/GPwsMlw==" + "resolved": "4.4.0", + "contentHash": "AwarXzzoDwX6BgrhjoJsk6tUezZEozOT5Y9QKF94Gl4JK91I4PIIBkBco9068Y9/Dra8Dkbie99kXB8+1BaYKw==" }, "System.ComponentModel.Annotations": { "type": "Transitive", @@ -185,18 +180,18 @@ }, "System.Memory": { "type": "Transitive", - "resolved": "4.6.3", - "contentHash": "qdcDOgnFZY40+Q9876JUHnlHu7bosOHX8XISRoH94fwk6hgaeQGSgfZd8srWRZNt5bV9ZW2TljcegDNxsf+96A==", + "resolved": "4.5.3", + "contentHash": "3oDzvc/zzetpTKWMShs1AADwZjQ/36HnsufHRPcOjyRAAMLDlu2iD33MBI2opxnezcVUtXyqDXXjoFMOU9c7SA==", "dependencies": { - "System.Buffers": "4.6.1", - "System.Numerics.Vectors": "4.6.1", - "System.Runtime.CompilerServices.Unsafe": "6.1.2" + "System.Buffers": "4.4.0", + "System.Numerics.Vectors": "4.4.0", + "System.Runtime.CompilerServices.Unsafe": "4.5.2" } }, "System.Numerics.Vectors": { "type": "Transitive", - "resolved": "4.6.1", - "contentHash": "sQxefTnhagrhoq2ReR0D/6K0zJcr9Hrd6kikeXsA1I8kOCboTavcUC4r7TSfpKFeE163uMuxZcyfO1mGO3EN8Q==" + "resolved": "4.4.0", + "contentHash": "UiLzLW+Lw6HLed1Hcg+8jSRttrbuXv7DANVj0DkL9g6EnnzbL75EB7EWsw5uRbhxd/4YdG8li5XizGWepmG3PQ==" }, "System.Reactive": { "type": "Transitive", @@ -229,16 +224,6 @@ "System.Runtime": "4.3.0" } }, - "System.Text.Encodings.Web": { - "type": "Transitive", - "resolved": "10.0.1", - "contentHash": "cVAka0o1rJJ5/De0pjNs7jcaZk5hUGf1HGzUyVmE2MEB1Vf0h/8qsWxImk1zjitCbeD2Avaq2P2+usdvqgbeVQ==", - "dependencies": { - "System.Buffers": "4.6.1", - "System.Memory": "4.6.3", - "System.Runtime.CompilerServices.Unsafe": "6.1.2" - } - }, "System.Threading.Tasks.Extensions": { "type": "Transitive", "resolved": "4.6.3", @@ -251,8 +236,6 @@ "type": "Project", "dependencies": { "GraphQL.Client": "[6.0.0, )", - "Microsoft.AspNet.WebApi.Client": "[6.0.0, )", - "Microsoft.Bcl.AsyncInterfaces": "[10.0.1, )", "Microsoft.CSharp": "[4.7.0, )", "Microsoft.Data.Sqlite": "[7.0.5, )", "Microsoft.Extensions.DependencyInjection.Abstractions": "[2.2.0, )", @@ -260,8 +243,6 @@ "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", "Speckle.Sdk.Dependencies": "[1.0.0, )", - "System.IO.Pipelines": "[10.0.1, )", - "System.Net.Http.Json": "[10.0.1, )", "System.Threading.Channels": "[10.0.1, )" } }, @@ -279,27 +260,6 @@ "System.Reactive": "5.0.0" } }, - "Microsoft.AspNet.WebApi.Client": { - "type": "CentralTransitive", - "requested": "[6.0.0, )", - "resolved": "6.0.0", - "contentHash": "zXeWP03dTo67AoDHUzR+/urck0KFssdCKOC+dq7Nv1V2YbFh/nIg09L0/3wSvyRONEdwxGB/ssEGmPNIIhAcAw==", - "dependencies": { - "Newtonsoft.Json": "13.0.1", - "Newtonsoft.Json.Bson": "1.0.2", - "System.Memory": "4.5.5", - "System.Threading.Tasks.Extensions": "4.5.4" - } - }, - "Microsoft.Bcl.AsyncInterfaces": { - "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "E1HSLkPHXEO30JEij2pWbOuzz1Z5ND4a5l7IP1T2RgQuE0a0NzEIvtO64RNy3Otn6PFezbT80cfm3M/Cgt70PA==", - "dependencies": { - "System.Threading.Tasks.Extensions": "4.6.3" - } - }, "Microsoft.CSharp": { "type": "CentralTransitive", "requested": "[4.7.0, )", @@ -346,44 +306,6 @@ "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" }, - "System.IO.Pipelines": { - "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "26LbFXHKd7PmRnWlkjnYgmjd5B6HYVG+1MpTO25BdxTJnx6D0O16JPAC/S4YBqjtt4YpfGj1QO/Ss6SPMGEGQw==", - "dependencies": { - "System.Buffers": "4.6.1", - "System.Memory": "4.6.3", - "System.Threading.Tasks.Extensions": "4.6.3" - } - }, - "System.Net.Http.Json": { - "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "XGOWt78ccgO9esyNlCemlS2b8JZPrH85pk/MvdHJxp6KwwUY/GnDaw2fPpJa7lgotiTkWnXnhip+ULNKaP7a8A==", - "dependencies": { - "System.Buffers": "4.6.1", - "System.Memory": "4.6.3", - "System.Text.Json": "10.0.1", - "System.Threading.Tasks.Extensions": "4.6.3" - } - }, - "System.Text.Json": { - "type": "CentralTransitive", - "requested": "[8.0.5, )", - "resolved": "10.0.1", - "contentHash": "EsgwDgU1PFqhrFA9l5n+RBu76wFhNGCEwu8ITrBNhjPP3MxLyklroU5GIF8o6JYpYg6T4KD/VICfMdgPAvNp5g==", - "dependencies": { - "Microsoft.Bcl.AsyncInterfaces": "10.0.1", - "System.Buffers": "4.6.1", - "System.IO.Pipelines": "10.0.1", - "System.Memory": "4.6.3", - "System.Runtime.CompilerServices.Unsafe": "6.1.2", - "System.Text.Encodings.Web": "10.0.1", - "System.Threading.Tasks.Extensions": "4.6.3" - } - }, "System.Threading.Channels": { "type": "CentralTransitive", "requested": "[10.0.1, )", @@ -505,19 +427,6 @@ "resolved": "8.0.0", "contentHash": "dk9JPxTCIevS75HyEQ0E4OVAFhB2N+V9ShCXf8Q6FkUQZDkgLI12y679Nym1YqsiSysuQskT7Z+6nUf3yab6Vw==" }, - "Newtonsoft.Json": { - "type": "Transitive", - "resolved": "13.0.1", - "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" - }, - "Newtonsoft.Json.Bson": { - "type": "Transitive", - "resolved": "1.0.2", - "contentHash": "QYFyxhaABwmq3p/21VrZNYvCg3DaEoN/wUuw5nmfAf0X3HLjgupwhkEWdgfb9nvGAUIv3osmZoD3kKl4jxEmYQ==", - "dependencies": { - "Newtonsoft.Json": "12.0.1" - } - }, "SQLitePCLRaw.bundle_e_sqlite3": { "type": "Transitive", "resolved": "2.1.4", @@ -555,8 +464,8 @@ }, "System.Memory": { "type": "Transitive", - "resolved": "4.5.5", - "contentHash": "XIWiDvKPXaTveaB7HVganDlOCRoj03l+jrwNvcge/t8vhGYKvqV+dMv6G4SAX2NoNmN0wZfVPTAlFwZcZvVOUw==" + "resolved": "4.5.3", + "contentHash": "3oDzvc/zzetpTKWMShs1AADwZjQ/36HnsufHRPcOjyRAAMLDlu2iD33MBI2opxnezcVUtXyqDXXjoFMOU9c7SA==" }, "System.Reactive": { "type": "Transitive", @@ -568,29 +477,16 @@ "resolved": "4.5.1", "contentHash": "Zh8t8oqolRaFa9vmOZfdQm/qKejdqz0J9kr7o2Fu0vPeoH3BL1EOXipKWwkWtLT1JPzjByrF19fGuFlNbmPpiw==" }, - "System.Text.Encodings.Web": { - "type": "Transitive", - "resolved": "10.0.1", - "contentHash": "cVAka0o1rJJ5/De0pjNs7jcaZk5hUGf1HGzUyVmE2MEB1Vf0h/8qsWxImk1zjitCbeD2Avaq2P2+usdvqgbeVQ==" - }, - "System.Threading.Tasks.Extensions": { - "type": "Transitive", - "resolved": "4.5.4", - "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==" - }, "speckle.sdk": { "type": "Project", "dependencies": { "GraphQL.Client": "[6.0.0, )", - "Microsoft.AspNet.WebApi.Client": "[6.0.0, )", "Microsoft.Data.Sqlite": "[7.0.5, )", "Microsoft.Extensions.DependencyInjection.Abstractions": "[2.2.0, )", "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", "Speckle.Sdk.Dependencies": "[1.0.0, )", - "System.IO.Pipelines": "[10.0.1, )", - "System.Net.Http.Json": "[10.0.1, )", "System.Threading.Channels": "[10.0.1, )" } }, @@ -608,18 +504,6 @@ "System.Reactive": "5.0.0" } }, - "Microsoft.AspNet.WebApi.Client": { - "type": "CentralTransitive", - "requested": "[6.0.0, )", - "resolved": "6.0.0", - "contentHash": "zXeWP03dTo67AoDHUzR+/urck0KFssdCKOC+dq7Nv1V2YbFh/nIg09L0/3wSvyRONEdwxGB/ssEGmPNIIhAcAw==", - "dependencies": { - "Newtonsoft.Json": "13.0.1", - "Newtonsoft.Json.Bson": "1.0.2", - "System.Memory": "4.5.5", - "System.Threading.Tasks.Extensions": "4.5.4" - } - }, "Microsoft.Data.Sqlite": { "type": "CentralTransitive", "requested": "[7.0.5, )", @@ -660,31 +544,6 @@ "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" }, - "System.IO.Pipelines": { - "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "26LbFXHKd7PmRnWlkjnYgmjd5B6HYVG+1MpTO25BdxTJnx6D0O16JPAC/S4YBqjtt4YpfGj1QO/Ss6SPMGEGQw==" - }, - "System.Net.Http.Json": { - "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "XGOWt78ccgO9esyNlCemlS2b8JZPrH85pk/MvdHJxp6KwwUY/GnDaw2fPpJa7lgotiTkWnXnhip+ULNKaP7a8A==", - "dependencies": { - "System.Text.Json": "10.0.1" - } - }, - "System.Text.Json": { - "type": "CentralTransitive", - "requested": "[8.0.5, )", - "resolved": "10.0.1", - "contentHash": "EsgwDgU1PFqhrFA9l5n+RBu76wFhNGCEwu8ITrBNhjPP3MxLyklroU5GIF8o6JYpYg6T4KD/VICfMdgPAvNp5g==", - "dependencies": { - "System.IO.Pipelines": "10.0.1", - "System.Text.Encodings.Web": "10.0.1" - } - }, "System.Threading.Channels": { "type": "CentralTransitive", "requested": "[10.0.1, )", diff --git a/src/Speckle.Sdk.Dependencies/packages.lock.json b/src/Speckle.Sdk.Dependencies/packages.lock.json index 1f256577..aa13bc43 100644 --- a/src/Speckle.Sdk.Dependencies/packages.lock.json +++ b/src/Speckle.Sdk.Dependencies/packages.lock.json @@ -95,6 +95,14 @@ "resolved": "2.0.33", "contentHash": "xb2h1CsOepoYwdXEPui9VcQglwABQwNf9cccZbf+acarEzF5PUp8Xx71nFXIhOgEdm6wrxAoF6xAxK4m/XFRUQ==" }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "Transitive", + "resolved": "10.0.1", + "contentHash": "E1HSLkPHXEO30JEij2pWbOuzz1Z5ND4a5l7IP1T2RgQuE0a0NzEIvtO64RNy3Otn6PFezbT80cfm3M/Cgt70PA==", + "dependencies": { + "System.Threading.Tasks.Extensions": "4.6.3" + } + }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", "resolved": "8.0.0", @@ -151,15 +159,6 @@ "dependencies": { "System.Runtime.CompilerServices.Unsafe": "6.1.2" } - }, - "Microsoft.Bcl.AsyncInterfaces": { - "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "E1HSLkPHXEO30JEij2pWbOuzz1Z5ND4a5l7IP1T2RgQuE0a0NzEIvtO64RNy3Otn6PFezbT80cfm3M/Cgt70PA==", - "dependencies": { - "System.Threading.Tasks.Extensions": "4.6.3" - } } }, "net8.0": { diff --git a/src/Speckle.Sdk/Pipelines/SendPipeline.cs b/src/Speckle.Sdk/Pipelines/SendPipeline.cs index 4c580163..1079f1f7 100644 --- a/src/Speckle.Sdk/Pipelines/SendPipeline.cs +++ b/src/Speckle.Sdk/Pipelines/SendPipeline.cs @@ -21,7 +21,14 @@ public sealed class SendPipeline : IDisposable ) { _cancellationToken = cancellationToken; - _uploader = new Uploader(projectId, modelId, ingestionId, account.serverInfo.url, account.token, cancellationToken); + _uploader = new Uploader( + projectId, + modelId, + ingestionId, + new(account.serverInfo.url), + account.token, + cancellationToken + ); } private UploadItem _lastItem; diff --git a/src/Speckle.Sdk/Pipelines/Uploader.cs b/src/Speckle.Sdk/Pipelines/Uploader.cs index ec8133a4..a7bf6e7e 100644 --- a/src/Speckle.Sdk/Pipelines/Uploader.cs +++ b/src/Speckle.Sdk/Pipelines/Uploader.cs @@ -1,7 +1,8 @@ using System.IO.Compression; using System.Net.Http.Headers; -using System.Net.Http.Json; +using System.Text; using System.Threading.Channels; +using Speckle.Newtonsoft.Json; namespace Speckle.Sdk.Pipelines; @@ -19,7 +20,7 @@ public sealed class Uploader : IDisposable string projectId, string modelId, string ingestionId, - string apiEndpoint, + Uri serverUrl, string authToken, CancellationToken cancellationToken ) @@ -28,14 +29,14 @@ public sealed class Uploader : IDisposable _modelId = modelId; _ingestionId = ingestionId; _cancellationToken = cancellationToken; + Uri apiBaseUrl = new(serverUrl, "/api/v1/"); - Uri apiBaseUrl = new(new(apiEndpoint), "/api/v1/"); _client = new HttpClient { BaseAddress = apiBaseUrl, Timeout = TimeSpan.FromMinutes(30) }; _client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", authToken); _channel = Channel.CreateBounded( - new BoundedChannelOptions(1000) { FullMode = BoundedChannelFullMode.Wait } + new BoundedChannelOptions(1000) { FullMode = BoundedChannelFullMode.Wait, SingleReader = true } ); _sendTask = SendLoopAsync(); @@ -66,9 +67,11 @@ public sealed class Uploader : IDisposable { await writer.WriteLineAsync($"{item.Id}\t{item.Json}\t{item.SpeckleType}").ConfigureAwait(false); } - +#if NET8_0_OR_GREATER + await writer.FlushAsync(_cancellationToken).ConfigureAwait(false); +#else await writer.FlushAsync().ConfigureAwait(false); - await gzip.FlushAsync(_cancellationToken).ConfigureAwait(false); +#endif } // fileStream.Flush(); // fileStream.Close(); @@ -77,13 +80,18 @@ public sealed class Uploader : IDisposable // 2. Request presigned URL var signUri = new Uri($"projects/{_projectId}/models/{_modelId}/uploads/sign", UriKind.Relative); - var signResponse = await HttpClientExtensions - .PostAsJsonAsync(_client, signUri, _cancellationToken) - .ConfigureAwait(false); + using var signResponse = await _client.PostAsync(signUri, null, _cancellationToken).ConfigureAwait(false); signResponse.EnsureSuccessStatusCode(); - var presignedUpload = - await signResponse.Content.ReadFromJsonAsync(_cancellationToken).ConfigureAwait(false) +#if NET5_0_OR_GREATER + string signResponseString = await signResponse + .Content.ReadAsStringAsync(_cancellationToken) + .ConfigureAwait(false); +#else + string signResponseString = await signResponse.Content.ReadAsStringAsync().ConfigureAwait(false); +#endif + PresignedUploadResponse presignedUpload = + JsonConvert.DeserializeObject(signResponseString) ?? throw new InvalidOperationException("Failed to get presigned upload URL"); // 3. Upload to S3 @@ -109,25 +117,24 @@ public sealed class Uploader : IDisposable var processUri = new Uri($"projects/{_projectId}/models/{_modelId}/uploads/process", UriKind.Relative); var processRequest = new ProcessUploadRequest { key = presignedUpload.Key, ingestionId = _ingestionId }; - var processResponse = await HttpClientExtensions - .PostAsJsonAsync(_client, processUri, processRequest, _cancellationToken) - .ConfigureAwait(false); + using StringContent content = new(JsonConvert.SerializeObject(processRequest), Encoding.UTF8, "application/json"); + var processResponse = await _client.PostAsync(processUri, content, _cancellationToken).ConfigureAwait(false); + processResponse.EnsureSuccessStatusCode(); - var processResult = await processResponse - .Content.ReadFromJsonAsync(_cancellationToken) - .ConfigureAwait(false); +#if NET5_0_OR_GREATER + string processResult = await processResponse.Content.ReadAsStringAsync(_cancellationToken).ConfigureAwait(false); +#else + string processResult = await processResponse.Content.ReadAsStringAsync().ConfigureAwait(false); +#endif + var json = JsonConvert.DeserializeObject(processResult); - if (processResult == null) + if (json is null) { throw new InvalidOperationException("Failed to trigger upload processing"); } - return new UploadResult { IngestionId = processResult.ingestionId }; - } - catch (Exception ex) when (!ex.IsFatal()) - { - throw; + return new UploadResult { IngestionId = json.ingestionId }; } finally { diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/ObjectFlopper.cs b/src/Speckle.Sdk/Serialisation/V2/Send/ObjectFlopper.cs deleted file mode 100644 index 2027a6e3..00000000 --- a/src/Speckle.Sdk/Serialisation/V2/Send/ObjectFlopper.cs +++ /dev/null @@ -1,79 +0,0 @@ -using System.IO.Compression; -using System.Net.Http.Headers; -using System.Threading.Channels; - -namespace Speckle.Sdk.Serialisation.V2.Send; - -#pragma warning disable CA1001 -public sealed class ObjectFlopper -#pragma warning restore CA1001 -{ - private readonly Uri _url; - private readonly string _streamId; - private readonly HttpClient _client; - private readonly Channel _channel; - private readonly Task _sendTask; - - public ObjectFlopper(Uri _, string streamId, string? authToken) - { - _streamId = streamId; - _url = new Uri("http://zog.local:3000/api/v1/"); - _client = new HttpClient { BaseAddress = _url, Timeout = TimeSpan.FromMinutes(10) }; - - if (authToken != null) - { - _client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", authToken); - } - - _channel = Channel.CreateBounded( - new BoundedChannelOptions(1000) { FullMode = BoundedChannelFullMode.Wait } - ); - - _sendTask = SendLoopAsync(streamId, "test"); - } - - public ValueTask PushAsync(BaseItem item, CancellationToken ct = default) => _channel.Writer.WriteAsync(item, ct); - - public async Task CompleteAsync() - { - _channel.Writer.Complete(); - await _sendTask.ConfigureAwait(false); - } - - private async Task SendLoopAsync(string projectId, string modelId) - { - var content = new PushStreamContent( - async (stream, _, _) => - { - var gzip = new GZipStream(stream, CompressionLevel.Optimal); - var writer = new StreamWriter(gzip); //new StreamWriter(gzip, System.Text.Encoding.UTF8, 20 * 1024 * 1024); - try - { - await foreach (var item in _channel.Reader.ReadAllAsync().ConfigureAwait(false)) - { - await writer.WriteLineAsync($"{item.Id}\t{item.Json}").ConfigureAwait(false); - } - } - finally - { - await writer.FlushAsync().ConfigureAwait(false); - await gzip.FlushAsync().ConfigureAwait(false); - writer.Dispose(); - gzip.Dispose(); - } - }, - new MediaTypeHeaderValue("application/x-ndjson") - ); - - var uri = new Uri($"projects/{projectId}/models/{modelId}/versions", UriKind.Relative); - var request = new HttpRequestMessage(HttpMethod.Post, uri) { Content = content }; - request.Headers.TransferEncodingChunked = true; // NOTE: important for streaming to happen. - var response = await _client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); - - // Consume the response body to fully complete the request - await response.Content.ReadAsStringAsync().ConfigureAwait(false); - } - - public void Dispose() => _client.Dispose(); -} diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/ObjectFlopperGandalf.cs b/src/Speckle.Sdk/Serialisation/V2/Send/ObjectFlopperGandalf.cs deleted file mode 100644 index abccd3a6..00000000 --- a/src/Speckle.Sdk/Serialisation/V2/Send/ObjectFlopperGandalf.cs +++ /dev/null @@ -1,90 +0,0 @@ -using System.IO.Compression; -using System.IO.Pipelines; -using System.Net.Http.Headers; -using System.Threading.Channels; - -namespace Speckle.Sdk.Serialisation.V2.Send; - -#pragma warning disable CA1001 -public sealed class ObjectFlopperGandalf -#pragma warning restore CA1001 -{ - private readonly Uri _url; - private readonly string _streamId; - private readonly HttpClient _client; - private readonly Channel _channel; - private readonly Task _sendTask; - - public ObjectFlopperGandalf(Uri _, string streamId, string? authToken) - { - _streamId = streamId; - _url = new Uri("http://bender-2.local:3000/api/v1/"); - _client = new HttpClient { BaseAddress = _url, Timeout = TimeSpan.FromMinutes(10) }; - - if (authToken != null) - { - _client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", authToken); - } - - _channel = Channel.CreateBounded( - new BoundedChannelOptions(1000) { FullMode = BoundedChannelFullMode.Wait } - ); - - _sendTask = SendLoopAsync(streamId, "test"); - } - - public ValueTask PushAsync(BaseItem item, CancellationToken ct = default) => _channel.Writer.WriteAsync(item, ct); - - public async Task CompleteAsync() - { - _channel.Writer.Complete(); - await _sendTask.ConfigureAwait(false); - } - - private async Task SendLoopAsync(string projectId, string modelId) - { - var pipe = new Pipe(); - - // Start writing to pipe immediately in background - var writeTask = Task.Run(async () => - { - var gzip = new GZipStream(pipe.Writer.AsStream(), CompressionLevel.Optimal); - var writer = new StreamWriter(gzip); - - try - { - await foreach (var item in _channel.Reader.ReadAllAsync().ConfigureAwait(false)) - { - await writer.WriteLineAsync($"{item.Id}\t{item.Json}").ConfigureAwait(false); - await writer.FlushAsync().ConfigureAwait(false); - } - } - finally - { - await writer.FlushAsync().ConfigureAwait(false); - await gzip.FlushAsync().ConfigureAwait(false); - writer.Dispose(); - gzip.Dispose(); - await pipe.Writer.CompleteAsync().ConfigureAwait(false); - } - }); - - // Start HTTP request immediately, reading from pipe - var content = new StreamContent(pipe.Reader.AsStream()); - content.Headers.ContentType = new MediaTypeHeaderValue("application/x-ndjson"); - content.Headers.ContentEncoding.Add("gzip"); - - var uri = new Uri($"projects/{projectId}/models/{modelId}/objects", UriKind.Relative); - var request = new HttpRequestMessage(HttpMethod.Post, uri) { Content = content }; - - var responseTask = _client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); - - // Wait for both - await Task.WhenAll(writeTask, responseTask).ConfigureAwait(false); - - var response = await responseTask.ConfigureAwait(false); - response.EnsureSuccessStatusCode(); - } - - public void Dispose() => _client.Dispose(); -} diff --git a/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs b/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs index 3c56b4af..b708f4d4 100644 --- a/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs +++ b/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs @@ -35,8 +35,7 @@ public class SerializeProcessFactory( IServerObjectManager serverObjectManager, IProgress? progress, CancellationToken cancellationToken, - SerializeProcessOptions? options = null, - ObjectFlopper? objectFlopper = null + SerializeProcessOptions? options = null ) => new SerializeProcess( progress, diff --git a/src/Speckle.Sdk/Speckle.Sdk.csproj b/src/Speckle.Sdk/Speckle.Sdk.csproj index 664d497a..1e9cf643 100644 --- a/src/Speckle.Sdk/Speckle.Sdk.csproj +++ b/src/Speckle.Sdk/Speckle.Sdk.csproj @@ -23,12 +23,9 @@ - - - @@ -36,7 +33,6 @@ - diff --git a/src/Speckle.Sdk/packages.lock.json b/src/Speckle.Sdk/packages.lock.json index b7195702..17478611 100644 --- a/src/Speckle.Sdk/packages.lock.json +++ b/src/Speckle.Sdk/packages.lock.json @@ -13,27 +13,6 @@ "System.Reactive": "5.0.0" } }, - "Microsoft.AspNet.WebApi.Client": { - "type": "Direct", - "requested": "[6.0.0, )", - "resolved": "6.0.0", - "contentHash": "zXeWP03dTo67AoDHUzR+/urck0KFssdCKOC+dq7Nv1V2YbFh/nIg09L0/3wSvyRONEdwxGB/ssEGmPNIIhAcAw==", - "dependencies": { - "Newtonsoft.Json": "13.0.1", - "Newtonsoft.Json.Bson": "1.0.2", - "System.Memory": "4.5.5", - "System.Threading.Tasks.Extensions": "4.5.4" - } - }, - "Microsoft.Bcl.AsyncInterfaces": { - "type": "Direct", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "E1HSLkPHXEO30JEij2pWbOuzz1Z5ND4a5l7IP1T2RgQuE0a0NzEIvtO64RNy3Otn6PFezbT80cfm3M/Cgt70PA==", - "dependencies": { - "System.Threading.Tasks.Extensions": "4.6.3" - } - }, "Microsoft.CSharp": { "type": "Direct", "requested": "[4.7.0, )", @@ -111,29 +90,6 @@ "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" }, - "System.IO.Pipelines": { - "type": "Direct", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "26LbFXHKd7PmRnWlkjnYgmjd5B6HYVG+1MpTO25BdxTJnx6D0O16JPAC/S4YBqjtt4YpfGj1QO/Ss6SPMGEGQw==", - "dependencies": { - "System.Buffers": "4.6.1", - "System.Memory": "4.6.3", - "System.Threading.Tasks.Extensions": "4.6.3" - } - }, - "System.Net.Http.Json": { - "type": "Direct", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "XGOWt78ccgO9esyNlCemlS2b8JZPrH85pk/MvdHJxp6KwwUY/GnDaw2fPpJa7lgotiTkWnXnhip+ULNKaP7a8A==", - "dependencies": { - "System.Buffers": "4.6.1", - "System.Memory": "4.6.3", - "System.Text.Json": "10.0.1", - "System.Threading.Tasks.Extensions": "4.6.3" - } - }, "System.Threading.Channels": { "type": "Direct", "requested": "[10.0.1, )", @@ -165,6 +121,14 @@ "resolved": "6.0.0", "contentHash": "yg72rrYDapfsIUrul7aF6wwNnTJBOFvuA9VdDTQpPa8AlAriHbufeXYLBcodKjfUdkCnaiggX1U/nEP08Zb5GA==" }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "Transitive", + "resolved": "10.0.1", + "contentHash": "E1HSLkPHXEO30JEij2pWbOuzz1Z5ND4a5l7IP1T2RgQuE0a0NzEIvtO64RNy3Otn6PFezbT80cfm3M/Cgt70PA==", + "dependencies": { + "System.Threading.Tasks.Extensions": "4.6.3" + } + }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", "resolved": "8.0.0", @@ -241,19 +205,6 @@ "resolved": "8.0.0", "contentHash": "dk9JPxTCIevS75HyEQ0E4OVAFhB2N+V9ShCXf8Q6FkUQZDkgLI12y679Nym1YqsiSysuQskT7Z+6nUf3yab6Vw==" }, - "Newtonsoft.Json": { - "type": "Transitive", - "resolved": "13.0.1", - "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" - }, - "Newtonsoft.Json.Bson": { - "type": "Transitive", - "resolved": "1.0.2", - "contentHash": "QYFyxhaABwmq3p/21VrZNYvCg3DaEoN/wUuw5nmfAf0X3HLjgupwhkEWdgfb9nvGAUIv3osmZoD3kKl4jxEmYQ==", - "dependencies": { - "Newtonsoft.Json": "12.0.1" - } - }, "SQLitePCLRaw.bundle_e_sqlite3": { "type": "Transitive", "resolved": "2.1.4", @@ -286,8 +237,8 @@ }, "System.Buffers": { "type": "Transitive", - "resolved": "4.6.1", - "contentHash": "N8GXpmiLMtljq7gwvyS+1QvKT/W2J8sNAvx+HVg4NGmsG/H+2k/y9QI23auLJRterrzCiDH+IWAw4V/GPwsMlw==" + "resolved": "4.4.0", + "contentHash": "AwarXzzoDwX6BgrhjoJsk6tUezZEozOT5Y9QKF94Gl4JK91I4PIIBkBco9068Y9/Dra8Dkbie99kXB8+1BaYKw==" }, "System.ComponentModel.Annotations": { "type": "Transitive", @@ -296,18 +247,18 @@ }, "System.Memory": { "type": "Transitive", - "resolved": "4.6.3", - "contentHash": "qdcDOgnFZY40+Q9876JUHnlHu7bosOHX8XISRoH94fwk6hgaeQGSgfZd8srWRZNt5bV9ZW2TljcegDNxsf+96A==", + "resolved": "4.5.3", + "contentHash": "3oDzvc/zzetpTKWMShs1AADwZjQ/36HnsufHRPcOjyRAAMLDlu2iD33MBI2opxnezcVUtXyqDXXjoFMOU9c7SA==", "dependencies": { - "System.Buffers": "4.6.1", - "System.Numerics.Vectors": "4.6.1", - "System.Runtime.CompilerServices.Unsafe": "6.1.2" + "System.Buffers": "4.4.0", + "System.Numerics.Vectors": "4.4.0", + "System.Runtime.CompilerServices.Unsafe": "4.5.2" } }, "System.Numerics.Vectors": { "type": "Transitive", - "resolved": "4.6.1", - "contentHash": "sQxefTnhagrhoq2ReR0D/6K0zJcr9Hrd6kikeXsA1I8kOCboTavcUC4r7TSfpKFeE163uMuxZcyfO1mGO3EN8Q==" + "resolved": "4.4.0", + "contentHash": "UiLzLW+Lw6HLed1Hcg+8jSRttrbuXv7DANVj0DkL9g6EnnzbL75EB7EWsw5uRbhxd/4YdG8li5XizGWepmG3PQ==" }, "System.Reactive": { "type": "Transitive", @@ -340,16 +291,6 @@ "System.Runtime": "4.3.0" } }, - "System.Text.Encodings.Web": { - "type": "Transitive", - "resolved": "10.0.1", - "contentHash": "cVAka0o1rJJ5/De0pjNs7jcaZk5hUGf1HGzUyVmE2MEB1Vf0h/8qsWxImk1zjitCbeD2Avaq2P2+usdvqgbeVQ==", - "dependencies": { - "System.Buffers": "4.6.1", - "System.Memory": "4.6.3", - "System.Runtime.CompilerServices.Unsafe": "6.1.2" - } - }, "System.Threading.Tasks.Extensions": { "type": "Transitive", "resolved": "4.6.3", @@ -358,57 +299,8 @@ "System.Runtime.CompilerServices.Unsafe": "6.1.2" } }, - "speckle.connectors.common": { - "type": "Project", - "dependencies": { - "Microsoft.Extensions.DependencyInjection": "[2.2.0, )", - "Speckle.Connectors.Logging": "[1.0.0, )", - "Speckle.Converters.Common": "[1.0.0, )", - "Speckle.Objects": "[1.0.0, )" - } - }, - "speckle.connectors.logging": { - "type": "Project" - }, - "speckle.converters.common": { - "type": "Project", - "dependencies": { - "Microsoft.Extensions.Logging.Abstractions": "[2.2.0, )", - "Speckle.Objects": "[1.0.0, )" - } - }, - "speckle.objects": { - "type": "Project", - "dependencies": { - "Speckle.Sdk": "[1.0.0, )" - } - }, "speckle.sdk.dependencies": { "type": "Project" - }, - "Microsoft.Extensions.DependencyInjection": { - "type": "CentralTransitive", - "requested": "[2.2.0, )", - "resolved": "2.2.0", - "contentHash": "MZtBIwfDFork5vfjpJdG5g8wuJFt7d/y3LOSVVtDK/76wlbtz6cjltfKHqLx2TKVqTj5/c41t77m1+h20zqtPA==", - "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0" - } - }, - "System.Text.Json": { - "type": "CentralTransitive", - "requested": "[8.0.5, )", - "resolved": "10.0.1", - "contentHash": "EsgwDgU1PFqhrFA9l5n+RBu76wFhNGCEwu8ITrBNhjPP3MxLyklroU5GIF8o6JYpYg6T4KD/VICfMdgPAvNp5g==", - "dependencies": { - "Microsoft.Bcl.AsyncInterfaces": "10.0.1", - "System.Buffers": "4.6.1", - "System.IO.Pipelines": "10.0.1", - "System.Memory": "4.6.3", - "System.Runtime.CompilerServices.Unsafe": "6.1.2", - "System.Text.Encodings.Web": "10.0.1", - "System.Threading.Tasks.Extensions": "4.6.3" - } } }, "net8.0": { @@ -423,18 +315,6 @@ "System.Reactive": "5.0.0" } }, - "Microsoft.AspNet.WebApi.Client": { - "type": "Direct", - "requested": "[6.0.0, )", - "resolved": "6.0.0", - "contentHash": "zXeWP03dTo67AoDHUzR+/urck0KFssdCKOC+dq7Nv1V2YbFh/nIg09L0/3wSvyRONEdwxGB/ssEGmPNIIhAcAw==", - "dependencies": { - "Newtonsoft.Json": "13.0.1", - "Newtonsoft.Json.Bson": "1.0.2", - "System.Memory": "4.5.5", - "System.Threading.Tasks.Extensions": "4.5.4" - } - }, "Microsoft.Data.Sqlite": { "type": "Direct", "requested": "[7.0.5, )", @@ -497,21 +377,6 @@ "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" }, - "System.IO.Pipelines": { - "type": "Direct", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "26LbFXHKd7PmRnWlkjnYgmjd5B6HYVG+1MpTO25BdxTJnx6D0O16JPAC/S4YBqjtt4YpfGj1QO/Ss6SPMGEGQw==" - }, - "System.Net.Http.Json": { - "type": "Direct", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "XGOWt78ccgO9esyNlCemlS2b8JZPrH85pk/MvdHJxp6KwwUY/GnDaw2fPpJa7lgotiTkWnXnhip+ULNKaP7a8A==", - "dependencies": { - "System.Text.Json": "10.0.1" - } - }, "System.Threading.Channels": { "type": "Direct", "requested": "[10.0.1, )", @@ -605,19 +470,6 @@ "resolved": "8.0.0", "contentHash": "dk9JPxTCIevS75HyEQ0E4OVAFhB2N+V9ShCXf8Q6FkUQZDkgLI12y679Nym1YqsiSysuQskT7Z+6nUf3yab6Vw==" }, - "Newtonsoft.Json": { - "type": "Transitive", - "resolved": "13.0.1", - "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" - }, - "Newtonsoft.Json.Bson": { - "type": "Transitive", - "resolved": "1.0.2", - "contentHash": "QYFyxhaABwmq3p/21VrZNYvCg3DaEoN/wUuw5nmfAf0X3HLjgupwhkEWdgfb9nvGAUIv3osmZoD3kKl4jxEmYQ==", - "dependencies": { - "Newtonsoft.Json": "12.0.1" - } - }, "SQLitePCLRaw.bundle_e_sqlite3": { "type": "Transitive", "resolved": "2.1.4", @@ -655,8 +507,8 @@ }, "System.Memory": { "type": "Transitive", - "resolved": "4.5.5", - "contentHash": "XIWiDvKPXaTveaB7HVganDlOCRoj03l+jrwNvcge/t8vhGYKvqV+dMv6G4SAX2NoNmN0wZfVPTAlFwZcZvVOUw==" + "resolved": "4.5.3", + "contentHash": "3oDzvc/zzetpTKWMShs1AADwZjQ/36HnsufHRPcOjyRAAMLDlu2iD33MBI2opxnezcVUtXyqDXXjoFMOU9c7SA==" }, "System.Reactive": { "type": "Transitive", @@ -668,62 +520,8 @@ "resolved": "4.5.1", "contentHash": "Zh8t8oqolRaFa9vmOZfdQm/qKejdqz0J9kr7o2Fu0vPeoH3BL1EOXipKWwkWtLT1JPzjByrF19fGuFlNbmPpiw==" }, - "System.Text.Encodings.Web": { - "type": "Transitive", - "resolved": "10.0.1", - "contentHash": "cVAka0o1rJJ5/De0pjNs7jcaZk5hUGf1HGzUyVmE2MEB1Vf0h/8qsWxImk1zjitCbeD2Avaq2P2+usdvqgbeVQ==" - }, - "System.Threading.Tasks.Extensions": { - "type": "Transitive", - "resolved": "4.5.4", - "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==" - }, - "speckle.connectors.common": { - "type": "Project", - "dependencies": { - "Microsoft.Extensions.DependencyInjection": "[2.2.0, )", - "Speckle.Connectors.Logging": "[1.0.0, )", - "Speckle.Converters.Common": "[1.0.0, )", - "Speckle.Objects": "[1.0.0, )" - } - }, - "speckle.connectors.logging": { - "type": "Project" - }, - "speckle.converters.common": { - "type": "Project", - "dependencies": { - "Microsoft.Extensions.Logging.Abstractions": "[2.2.0, )", - "Speckle.Objects": "[1.0.0, )" - } - }, - "speckle.objects": { - "type": "Project", - "dependencies": { - "Speckle.Sdk": "[1.0.0, )" - } - }, "speckle.sdk.dependencies": { "type": "Project" - }, - "Microsoft.Extensions.DependencyInjection": { - "type": "CentralTransitive", - "requested": "[2.2.0, )", - "resolved": "2.2.0", - "contentHash": "MZtBIwfDFork5vfjpJdG5g8wuJFt7d/y3LOSVVtDK/76wlbtz6cjltfKHqLx2TKVqTj5/c41t77m1+h20zqtPA==", - "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0" - } - }, - "System.Text.Json": { - "type": "CentralTransitive", - "requested": "[8.0.5, )", - "resolved": "10.0.1", - "contentHash": "EsgwDgU1PFqhrFA9l5n+RBu76wFhNGCEwu8ITrBNhjPP3MxLyklroU5GIF8o6JYpYg6T4KD/VICfMdgPAvNp5g==", - "dependencies": { - "System.IO.Pipelines": "10.0.1", - "System.Text.Encodings.Web": "10.0.1" - } } } } diff --git a/tests/Speckle.Automate.Sdk.Integration/packages.lock.json b/tests/Speckle.Automate.Sdk.Integration/packages.lock.json index 78c3841b..d0818532 100644 --- a/tests/Speckle.Automate.Sdk.Integration/packages.lock.json +++ b/tests/Speckle.Automate.Sdk.Integration/packages.lock.json @@ -370,7 +370,8 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )", + "System.Threading.Channels": "[10.0.1, )" } }, "speckle.sdk.dependencies": { @@ -491,6 +492,12 @@ "System.Text.Encodings.Web": "8.0.0" } }, + "System.Threading.Channels": { + "type": "CentralTransitive", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" + }, "Verify.Quibble": { "type": "CentralTransitive", "requested": "[2.1.1, )", diff --git a/tests/Speckle.Objects.Tests.Unit/packages.lock.json b/tests/Speckle.Objects.Tests.Unit/packages.lock.json index fe375e71..3ef2d70b 100644 --- a/tests/Speckle.Objects.Tests.Unit/packages.lock.json +++ b/tests/Speckle.Objects.Tests.Unit/packages.lock.json @@ -348,7 +348,8 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )", + "System.Threading.Channels": "[10.0.1, )" } }, "speckle.sdk.dependencies": { @@ -432,6 +433,12 @@ "System.Text.Encodings.Web": "8.0.0" } }, + "System.Threading.Channels": { + "type": "CentralTransitive", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" + }, "Verify.Quibble": { "type": "CentralTransitive", "requested": "[2.1.1, )", diff --git a/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json b/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json index 046c2c3c..022252ea 100644 --- a/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json +++ b/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json @@ -74,6 +74,11 @@ "resolved": "1.17.0", "contentHash": "8x+HCVTl/HHTGpscH3vMBhV8sknN/muZFw9s3TsI8SA6+c43cOTCi2+jE4KsU8pNLbJ++iF2ZFcpcXHXtDglnw==" }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "Transitive", + "resolved": "1.1.0", + "contentHash": "1Am6l4Vpn3/K32daEqZI+FFr96OlZkgwK2LcT3pZ2zWubR5zTPW3/FkO1Rat9kb7oQOa4rxgl9LJHc5tspCWfg==" + }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", "resolved": "8.0.0", @@ -337,7 +342,8 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )", + "System.Threading.Channels": "[10.0.1, )" } }, "speckle.sdk.dependencies": { @@ -380,12 +386,6 @@ "System.Reactive": "5.0.0" } }, - "Microsoft.Bcl.AsyncInterfaces": { - "type": "CentralTransitive", - "requested": "[5.0.0, )", - "resolved": "1.1.0", - "contentHash": "1Am6l4Vpn3/K32daEqZI+FFr96OlZkgwK2LcT3pZ2zWubR5zTPW3/FkO1Rat9kb7oQOa4rxgl9LJHc5tspCWfg==" - }, "Microsoft.Data.Sqlite": { "type": "CentralTransitive", "requested": "[7.0.5, )", @@ -425,6 +425,12 @@ "requested": "[13.0.2, )", "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" + }, + "System.Threading.Channels": { + "type": "CentralTransitive", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" } } } diff --git a/tests/Speckle.Sdk.Serialization.Tests/packages.lock.json b/tests/Speckle.Sdk.Serialization.Tests/packages.lock.json index 2ac574fb..11691412 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/packages.lock.json +++ b/tests/Speckle.Sdk.Serialization.Tests/packages.lock.json @@ -393,7 +393,8 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )", + "System.Threading.Channels": "[10.0.1, )" } }, "speckle.sdk.dependencies": { @@ -474,6 +475,12 @@ "System.Text.Encodings.Web": "8.0.0" } }, + "System.Threading.Channels": { + "type": "CentralTransitive", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" + }, "Verify.Quibble": { "type": "CentralTransitive", "requested": "[2.1.1, )", diff --git a/tests/Speckle.Sdk.Testing/packages.lock.json b/tests/Speckle.Sdk.Testing/packages.lock.json index 6e2cfd77..9cb33c89 100644 --- a/tests/Speckle.Sdk.Testing/packages.lock.json +++ b/tests/Speckle.Sdk.Testing/packages.lock.json @@ -311,7 +311,8 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )", + "System.Threading.Channels": "[10.0.1, )" } }, "speckle.sdk.dependencies": { @@ -376,6 +377,12 @@ "dependencies": { "System.Text.Encodings.Web": "8.0.0" } + }, + "System.Threading.Channels": { + "type": "CentralTransitive", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" } } } diff --git a/tests/Speckle.Sdk.Tests.Integration/packages.lock.json b/tests/Speckle.Sdk.Tests.Integration/packages.lock.json index 6138bda1..59e1a4d2 100644 --- a/tests/Speckle.Sdk.Tests.Integration/packages.lock.json +++ b/tests/Speckle.Sdk.Tests.Integration/packages.lock.json @@ -370,7 +370,8 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )", + "System.Threading.Channels": "[10.0.1, )" } }, "speckle.sdk.dependencies": { @@ -454,6 +455,12 @@ "System.Text.Encodings.Web": "8.0.0" } }, + "System.Threading.Channels": { + "type": "CentralTransitive", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" + }, "Verify.Quibble": { "type": "CentralTransitive", "requested": "[2.1.1, )", diff --git a/tests/Speckle.Sdk.Tests.Performance/packages.lock.json b/tests/Speckle.Sdk.Tests.Performance/packages.lock.json index 16d806d9..a96f59c2 100644 --- a/tests/Speckle.Sdk.Tests.Performance/packages.lock.json +++ b/tests/Speckle.Sdk.Tests.Performance/packages.lock.json @@ -92,6 +92,11 @@ "resolved": "1.17.0", "contentHash": "8x+HCVTl/HHTGpscH3vMBhV8sknN/muZFw9s3TsI8SA6+c43cOTCi2+jE4KsU8pNLbJ++iF2ZFcpcXHXtDglnw==" }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "Transitive", + "resolved": "1.1.0", + "contentHash": "1Am6l4Vpn3/K32daEqZI+FFr96OlZkgwK2LcT3pZ2zWubR5zTPW3/FkO1Rat9kb7oQOa4rxgl9LJHc5tspCWfg==" + }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", "resolved": "8.0.0", @@ -355,7 +360,8 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )", + "System.Threading.Channels": "[10.0.1, )" } }, "speckle.sdk.dependencies": { @@ -372,12 +378,6 @@ "System.Reactive": "5.0.0" } }, - "Microsoft.Bcl.AsyncInterfaces": { - "type": "CentralTransitive", - "requested": "[5.0.0, )", - "resolved": "1.1.0", - "contentHash": "1Am6l4Vpn3/K32daEqZI+FFr96OlZkgwK2LcT3pZ2zWubR5zTPW3/FkO1Rat9kb7oQOa4rxgl9LJHc5tspCWfg==" - }, "Microsoft.Data.Sqlite": { "type": "CentralTransitive", "requested": "[7.0.5, )", @@ -417,6 +417,12 @@ "requested": "[13.0.2, )", "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" + }, + "System.Threading.Channels": { + "type": "CentralTransitive", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" } } } diff --git a/tests/Speckle.Sdk.Tests.Unit/packages.lock.json b/tests/Speckle.Sdk.Tests.Unit/packages.lock.json index 8b50e6f9..4faff1c8 100644 --- a/tests/Speckle.Sdk.Tests.Unit/packages.lock.json +++ b/tests/Speckle.Sdk.Tests.Unit/packages.lock.json @@ -363,7 +363,8 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )", + "System.Threading.Channels": "[10.0.1, )" } }, "speckle.sdk.dependencies": { @@ -441,6 +442,12 @@ "System.Text.Encodings.Web": "8.0.0" } }, + "System.Threading.Channels": { + "type": "CentralTransitive", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" + }, "Verify.Quibble": { "type": "CentralTransitive", "requested": "[2.1.1, )", From 57843cc45454032df54618aaea3d9dafe772b895 Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Thu, 12 Feb 2026 18:20:55 +0000 Subject: [PATCH 04/26] Test(integration): Add reusable workflow for running integration tests from the server repo (#443) * experiment * rename * we'll give this a go * fix path * correct path * the correct path this time * build docker image first * correct docker build * try this * ensure editor config is observed * typo --- ...gration-test-callable-from-server-repo.yml | 61 +++++++++++++++++++ .github/workflows/integration-test.yml | 2 +- Speckle.Sdk.slnx | 1 + docker-compose-internal.yml | 2 +- 4 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/integration-test-callable-from-server-repo.yml diff --git a/.github/workflows/integration-test-callable-from-server-repo.yml b/.github/workflows/integration-test-callable-from-server-repo.yml new file mode 100644 index 00000000..c6179c1c --- /dev/null +++ b/.github/workflows/integration-test-callable-from-server-repo.yml @@ -0,0 +1,61 @@ +name: Integration Test + +on: + workflow_call: + inputs: + speckle-sharp-sdk-ref: + required: true + type: string + +jobs: + integration-test: + env: + CLIENT_DIR: "./client" + CLIENT_REPO: "specklesystems/speckle-sharp-sdk" + SERVER_DIR: "./server" + SERVER_REPO: "specklesystems/speckle-server-internal" + SOLUTION: "Speckle.Sdk.sln" + SPECKLE_SERVER_IMAGE: "speckle-server:local" + runs-on: ubuntu-latest + steps: + - name: Checkout ${{ env.CLIENT_REPO }} + uses: actions/checkout@v6 + with: + path: ${{ env.CLIENT_DIR }} + repository: ${{ env.CLIENT_REPO }} + ref: ${{ inputs.speckle-sharp-sdk-ref }} + + - name: Checkout ${{ env.SERVER_REPO }} + uses: actions/checkout@v6 + with: + repository: ${{ env.SERVER_REPO }} + path: ${{ env.SERVER_DIR }} + + - name: Setup .NET SDK + uses: actions/setup-dotnet@v5 + with: + dotnet-version: 8.x.x + # cache: true + # cache-dependency-path: "**/packages.lock.json" + + - name: 🏗️ Build Server + run: docker build --file "./packages/server/Dockerfile" --tag ${{ env.SPECKLE_SERVER_IMAGE }} . + working-directory: ${{ env.SERVER_DIR }} + + - name: ⚙️ Spin up Server + run: docker compose --file "../${{ env.CLIENT_DIR }}/docker-compose-internal.yml" up --wait + working-directory: ${{ env.SERVER_DIR }} + env: + SPECKLE_SERVER_IMAGE: ${{ env.SPECKLE_SERVER_IMAGE }} + + - name: 📦 Restore .NET Solution + run: dotnet restore ${{ env.SOLUTION }} --locked-mode + working-directory: ${{ env.CLIENT_DIR }} + + - name: 🏗️ Build .NET Solution + run: dotnet build ${{ env.SOLUTION }} --configuration Release --no-restore -warnaserror + working-directory: ${{ env.CLIENT_DIR }} + + - name: 🔨 Run .NET Integration Tests + run: dotnet test ${{ env.SOLUTION }} --filter "(Category=Integration)&(Server!=Public)" --configuration Release --no-build --no-restore --verbosity=normal + working-directory: ${{ env.CLIENT_DIR }} diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index ec74abd5..3ed1c7b8 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -37,7 +37,7 @@ jobs: password: ${{ github.token }} - name: ⚙️ Spin up Server - run: docker compose -f ${{ inputs.docker-compose-file }} up --wait + run: docker compose --file ${{ inputs.docker-compose-file }} up --wait - name: 📦 Restore run: dotnet restore ${{ env.Solution }} --locked-mode diff --git a/Speckle.Sdk.slnx b/Speckle.Sdk.slnx index 3f7ded3f..2461b2a0 100644 --- a/Speckle.Sdk.slnx +++ b/Speckle.Sdk.slnx @@ -18,6 +18,7 @@ + diff --git a/docker-compose-internal.yml b/docker-compose-internal.yml index a4eee0df..11d2d9b6 100644 --- a/docker-compose-internal.yml +++ b/docker-compose-internal.yml @@ -52,7 +52,7 @@ services: start_period: 10s speckle-server: - image: ghcr.io/specklesystems/speckle-server:latest + image: ${SPECKLE_SERVER_IMAGE:-ghcr.io/specklesystems/speckle-server:latest} restart: always healthcheck: test: From a81aaca8feb525b407e67ec6addd81204ef238a7 Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Fri, 20 Feb 2026 12:02:34 +0000 Subject: [PATCH 05/26] Updated tests for recent server changes (#445) --- .../Api/GraphQL/Resources/ProjectResourceExceptionalTests.cs | 2 +- .../Api/GraphQL/Resources/SubscriptionResourceTests.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/ProjectResourceExceptionalTests.cs b/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/ProjectResourceExceptionalTests.cs index 94034402..ab14c650 100644 --- a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/ProjectResourceExceptionalTests.cs +++ b/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/ProjectResourceExceptionalTests.cs @@ -106,7 +106,7 @@ public class ProjectResourceExceptionalTests : IAsyncLifetime ProjectUpdateRoleInput input = new(_secondUser.Account.id.NotNull(), "NonExistentProject", newRole); var ex = await Assert.ThrowsAsync(async () => _ = await Sut.UpdateRole(input)); - ex.InnerExceptions.Single().Should().BeOfType(); + ex.InnerExceptions.Single().Should().BeAssignableTo(); //v3 server responds with SpeckleGraphQLStreamNotFoundException exception, v2 reponds with SpeckleGraphQLForbiddenException } [Theory] diff --git a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/SubscriptionResourceTests.cs b/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/SubscriptionResourceTests.cs index 3ef89104..dff86650 100644 --- a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/SubscriptionResourceTests.cs +++ b/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/SubscriptionResourceTests.cs @@ -15,7 +15,7 @@ public class SubscriptionResourceTests : IAsyncLifetime #else private const int WAIT_PERIOD = 400; // For CI runs, a much smaller wait time is acceptable #endif - private const int TIMEOUT = WAIT_PERIOD + 1000; + private const int TIMEOUT = WAIT_PERIOD + WAIT_PERIOD + 600; private IClient _testUser; private Project _testProject; private Model _testModel; From 43ebc84881e167158e4d2095ef77493acbd8b918 Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Fri, 20 Feb 2026 14:20:29 +0000 Subject: [PATCH 06/26] feat(api): DI Refactor for Duck DB + Gergo's API endpoint changes (#444) * di * di2 * Gergo's api changes --- src/Speckle.Sdk/Helpers/DisposableFile.cs | 34 ++++ src/Speckle.Sdk/Pipelines/ProgressStream.cs | 102 +++++----- .../Pipelines/{ => Send}/SendPipeline.cs | 40 ++-- .../Pipelines/{ => Send}/Serializer.cs | 4 +- src/Speckle.Sdk/Pipelines/Send/Uploader.cs | 164 ++++++++++++++++ .../Pipelines/Send/UploaderDTOs.cs | 22 +++ src/Speckle.Sdk/Pipelines/Uploader.cs | 182 ------------------ src/Speckle.Sdk/Pipelines/Uploader_Old.cs | 83 -------- 8 files changed, 298 insertions(+), 333 deletions(-) create mode 100644 src/Speckle.Sdk/Helpers/DisposableFile.cs rename src/Speckle.Sdk/Pipelines/{ => Send}/SendPipeline.cs (77%) rename src/Speckle.Sdk/Pipelines/{ => Send}/Serializer.cs (99%) create mode 100644 src/Speckle.Sdk/Pipelines/Send/Uploader.cs create mode 100644 src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs delete mode 100644 src/Speckle.Sdk/Pipelines/Uploader.cs delete mode 100644 src/Speckle.Sdk/Pipelines/Uploader_Old.cs diff --git a/src/Speckle.Sdk/Helpers/DisposableFile.cs b/src/Speckle.Sdk/Helpers/DisposableFile.cs new file mode 100644 index 00000000..b4200678 --- /dev/null +++ b/src/Speckle.Sdk/Helpers/DisposableFile.cs @@ -0,0 +1,34 @@ +using Microsoft.Extensions.Logging; + +namespace Speckle.Sdk.Helpers; + +/// +/// wrapper around the downloaded file to try and delete the file on Dispose +/// +/// +/// We're using a similar pattern in the Rhino File Importer codebase (see ImportJobFile) +/// +/// +/// +public sealed class DisposableFile(FileInfo file, ILogger logger, bool deleteOnDispose = true) : IDisposable +{ + public FileInfo FileInfo => file; + + public void Dispose() + { + if (!deleteOnDispose) + { + return; + } + + try + { + file.Delete(); + logger.LogInformation("Cleaned up {File}", file); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + logger.LogWarning(ex, "Failed to clean up {File}", file); + } + } +} diff --git a/src/Speckle.Sdk/Pipelines/ProgressStream.cs b/src/Speckle.Sdk/Pipelines/ProgressStream.cs index ed62121c..b23998ae 100644 --- a/src/Speckle.Sdk/Pipelines/ProgressStream.cs +++ b/src/Speckle.Sdk/Pipelines/ProgressStream.cs @@ -1,54 +1,46 @@ +using System.Diagnostics.CodeAnalysis; + namespace Speckle.Sdk.Pipelines; +public readonly record struct StreamProgressArgs(long BytesStreamed, long ExpectedTotalBytes); + /// /// Wraps a stream to report upload progress as bytes are read. /// -public sealed class ProgressStream : Stream +public sealed class ProgressStream( + Stream innerStream, + long expectedTotalBytesStreamed, + IProgress? progress = null +) : Stream { - private readonly Stream _innerStream; - private readonly long _totalBytes; - private readonly IProgress<(long BytesSent, long TotalBytes)>? _progress; - private long _bytesSent; + private long _bytesStreamed; - public ProgressStream( - Stream innerStream, - long totalBytes, - IProgress<(long BytesSent, long TotalBytes)>? progress = null - ) - { - _innerStream = innerStream; - _totalBytes = totalBytes; - _progress = progress; - _bytesSent = 0; - } - - public override bool CanRead => _innerStream.CanRead; - public override bool CanSeek => _innerStream.CanSeek; + public override bool CanRead => innerStream.CanRead; + public override bool CanSeek => innerStream.CanSeek; public override bool CanWrite => false; - public override long Length => _innerStream.Length; + public override long Length => innerStream.Length; public override long Position { - get => _innerStream.Position; - set => _innerStream.Position = value; + get => innerStream.Position; + set => innerStream.Position = value; } public override int Read(byte[] buffer, int offset, int count) { - int bytesRead = _innerStream.Read(buffer, offset, count); + int bytesRead = innerStream.Read(buffer, offset, count); ReportProgress(bytesRead); return bytesRead; } + [SuppressMessage( + "Performance", + "CA1835:Prefer the \'Memory\'-based overloads for \'ReadAsync\' and \'WriteAsync\'", + Justification = "Analyser warning forwarded to caller" + )] public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) { - int bytesRead = await _innerStream -#if NET8_0_OR_GREATER - .ReadAsync(buffer.AsMemory(offset, count), cancellationToken) -#else - .ReadAsync(buffer, offset, count, cancellationToken) -#endif - .ConfigureAwait(false); + int bytesRead = await innerStream.ReadAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); ReportProgress(bytesRead); return bytesRead; } @@ -56,41 +48,61 @@ public sealed class ProgressStream : Stream #if NET8_0_OR_GREATER public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) { - int bytesRead = await _innerStream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + int bytesRead = await innerStream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); ReportProgress(bytesRead); return bytesRead; } #endif - private void ReportProgress(int bytesRead) + private void ReportProgress(int newBytesProcessed) { - _bytesSent += bytesRead; - _progress?.Report((_bytesSent, _totalBytes)); + _bytesStreamed += newBytesProcessed; + progress?.Report(new(_bytesStreamed, expectedTotalBytesStreamed)); } - public override void Flush() => _innerStream.Flush(); + public override void Flush() => innerStream.Flush(); - public override Task FlushAsync(CancellationToken cancellationToken) => _innerStream.FlushAsync(cancellationToken); + public override Task FlushAsync(CancellationToken cancellationToken) => innerStream.FlushAsync(cancellationToken); - public override long Seek(long offset, SeekOrigin origin) => _innerStream.Seek(offset, origin); + public override long Seek(long offset, SeekOrigin origin) => innerStream.Seek(offset, origin); - public override void SetLength(long value) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); //intentionally not supporting, as changing length of stream mid-flight will fuck up progress - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) + { + innerStream.Write(buffer, offset, count); + ReportProgress(count); + } + + [SuppressMessage( + "Performance", + "CA1835:Prefer the \'Memory\'-based overloads for \'ReadAsync\' and \'WriteAsync\'", + Justification = "Analyser warning forwarded to caller" + )] + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + await innerStream.WriteAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); + ReportProgress(count); + } + +#if NET6_0_OR_GREATER + public override async ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) + { + await innerStream.WriteAsync(buffer, cancellationToken).ConfigureAwait(false); + ReportProgress(buffer.Length); + } +#endif protected override void Dispose(bool disposing) { - if (disposing) - { - _innerStream.Dispose(); - } + innerStream.Dispose(); base.Dispose(disposing); } -#if NET8_0_OR_GREATER +#if NET6_0_OR_GREATER public override async ValueTask DisposeAsync() { - await _innerStream.DisposeAsync().ConfigureAwait(false); + await innerStream.DisposeAsync().ConfigureAwait(false); await base.DisposeAsync().ConfigureAwait(false); } #endif diff --git a/src/Speckle.Sdk/Pipelines/SendPipeline.cs b/src/Speckle.Sdk/Pipelines/Send/SendPipeline.cs similarity index 77% rename from src/Speckle.Sdk/Pipelines/SendPipeline.cs rename to src/Speckle.Sdk/Pipelines/Send/SendPipeline.cs index 1079f1f7..e1a97a40 100644 --- a/src/Speckle.Sdk/Pipelines/SendPipeline.cs +++ b/src/Speckle.Sdk/Pipelines/Send/SendPipeline.cs @@ -1,34 +1,32 @@ +using Speckle.InterfaceGenerator; using Speckle.Sdk.Credentials; using Speckle.Sdk.Models; -using Speckle.Sdk.Serialisation; -namespace Speckle.Sdk.Pipelines; +namespace Speckle.Sdk.Pipelines.Send; -public record UploadItem(string Id, Json Json, string SpeckleType, ObjectReference Reference); - -public sealed class SendPipeline : IDisposable +[GenerateAutoInterface] +public sealed class SendPipelineFactory(IUploaderFactory uploaderFactory) : ISendPipelineFactory { - private readonly CancellationToken _cancellationToken; - private readonly Serializer _serializer = new(); - private readonly Uploader _uploader; - - public SendPipeline( - Account account, + public SendPipeline CreateInstance( string projectId, - string modelId, string ingestionId, + Account account, CancellationToken cancellationToken ) { - _cancellationToken = cancellationToken; - _uploader = new Uploader( - projectId, - modelId, - ingestionId, - new(account.serverInfo.url), - account.token, - cancellationToken - ); + var uploader = uploaderFactory.CreateInstance(projectId, ingestionId, account, cancellationToken); + return new SendPipeline(uploader); + } +} + +public sealed class SendPipeline : IDisposable +{ + private readonly Serializer _serializer = new(); + private readonly Uploader _uploader; + + internal SendPipeline(Uploader uploader) + { + _uploader = uploader; } private UploadItem _lastItem; diff --git a/src/Speckle.Sdk/Pipelines/Serializer.cs b/src/Speckle.Sdk/Pipelines/Send/Serializer.cs similarity index 99% rename from src/Speckle.Sdk/Pipelines/Serializer.cs rename to src/Speckle.Sdk/Pipelines/Send/Serializer.cs index 3f551db3..ed78ce73 100644 --- a/src/Speckle.Sdk/Pipelines/Serializer.cs +++ b/src/Speckle.Sdk/Pipelines/Send/Serializer.cs @@ -9,7 +9,7 @@ using Speckle.Sdk.Helpers; using Speckle.Sdk.Models; using Speckle.Sdk.Serialisation; -namespace Speckle.Sdk.Pipelines; +namespace Speckle.Sdk.Pipelines.Send; /// /// Another serializer, cleaner and meaner. Provides methods for serializing Speckle objects into a format suitable for upload or storage. @@ -17,7 +17,7 @@ namespace Speckle.Sdk.Pipelines; /// into serialized JSON structures along with associated metadata, closures, and references. /// Any reference objects coming through are being "passed through" serialized - they do not get double encoded. /// -public class Serializer +internal sealed class Serializer { private readonly record struct PropertyInfo(string Name, object? Value, bool IsDetachable); diff --git a/src/Speckle.Sdk/Pipelines/Send/Uploader.cs b/src/Speckle.Sdk/Pipelines/Send/Uploader.cs new file mode 100644 index 00000000..9d7103d9 --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Send/Uploader.cs @@ -0,0 +1,164 @@ +using System.IO.Compression; +using System.Net.Http.Headers; +using System.Threading.Channels; +using Microsoft.Extensions.Logging; +using Speckle.InterfaceGenerator; +using Speckle.Newtonsoft.Json; +using Speckle.Sdk.Credentials; +using Speckle.Sdk.Helpers; + +namespace Speckle.Sdk.Pipelines.Send; + +[GenerateAutoInterface] +public sealed class UploaderFactory(ISpeckleHttp httpClientFactory, ILogger logger) : IUploaderFactory +{ + public Uploader CreateInstance( + string projectId, + string ingestionId, + Account account, + CancellationToken cancellationToken + ) => new(projectId, ingestionId, logger, httpClientFactory, account, cancellationToken); +} + +public sealed class Uploader : IDisposable +{ + private readonly string _projectId; + private readonly string _ingestionId; + private readonly CancellationToken _cancellationToken; + private readonly HttpClient _speckleClient; + private readonly HttpClient _s3Client; + private readonly Channel _channel; + private readonly Task _sendTask; + private readonly ILogger _logger; + + internal Uploader( + string projectId, + string ingestionId, + ILogger logger, + ISpeckleHttp httpClientFactory, + Account speckleAccount, + CancellationToken cancellationToken + ) + { + _projectId = projectId; + _ingestionId = ingestionId; + _logger = logger; + _cancellationToken = cancellationToken; + + _speckleClient = httpClientFactory.CreateHttpClient( + null, + (int)TimeSpan.FromMinutes(30).TotalSeconds, + speckleAccount.token + ); + _speckleClient.BaseAddress = new(new(speckleAccount.serverInfo.url), "/api/v1/"); + + _s3Client = httpClientFactory.CreateHttpClient(); + + _channel = Channel.CreateBounded( + new BoundedChannelOptions(1000) { FullMode = BoundedChannelFullMode.Wait, SingleReader = true } + ); + + _sendTask = Task.Run(SendLoopAsync, cancellationToken); + } + + public ValueTask PushAsync(UploadItem item) => _channel.Writer.WriteAsync(item, _cancellationToken); + + public async Task CompleteAsync() + { + _channel.Writer.Complete(); + var result = await _sendTask.ConfigureAwait(false); + return result.IngestionId; + } + + private async Task SendLoopAsync() + { + using DisposableFile tempFile = await WriteFile().ConfigureAwait(false); + + PresignedUploadResponse presignedUpload = await GetPresignedUrl().ConfigureAwait(false); + await UploadToS3(tempFile.FileInfo, presignedUpload.Url).ConfigureAwait(false); + + return await TriggerProcessing().ConfigureAwait(false); + } + + /// + /// Reads from the Channel and streams the s to a temporary file on disk. + /// Will keep reading until is called. + /// + /// the file that was written + private async Task WriteFile() + { + string tempFilePath = Path.GetTempFileName(); + _logger.LogInformation("Writing temp file to {TempFilePath}", tempFilePath); + + using var fileStream = new FileStream(tempFilePath, FileMode.Create, FileAccess.Write, FileShare.None); + using var gzip = new GZipStream(fileStream, CompressionLevel.Optimal); + using var writer = new StreamWriter(gzip); + await foreach (var item in _channel.Reader.ReadAllAsync(_cancellationToken).ConfigureAwait(false)) + { + await writer.WriteLineAsync($"{item.Id}\t{item.Json}\t{item.SpeckleType}").ConfigureAwait(false); + } +#if NET8_0_OR_GREATER + await writer.FlushAsync(_cancellationToken).ConfigureAwait(false); +#else + await writer.FlushAsync().ConfigureAwait(false); +#endif + return new DisposableFile(new FileInfo(tempFilePath), _logger); + } + + private async Task GetPresignedUrl() + { + var signUri = new Uri($"projects/{_projectId}/modelingestion/{_ingestionId}/uploads/sign", UriKind.Relative); + + using var signResponse = await _speckleClient.PostAsync(signUri, null, _cancellationToken).ConfigureAwait(false); + signResponse.EnsureSuccessStatusCode(); + +#if NET5_0_OR_GREATER + string signResponseString = await signResponse.Content.ReadAsStringAsync(_cancellationToken).ConfigureAwait(false); +#else + string signResponseString = await signResponse.Content.ReadAsStringAsync().ConfigureAwait(false); +#endif + PresignedUploadResponse presignedUpload = + JsonConvert.DeserializeObject(signResponseString) + ?? throw new InvalidOperationException("Failed to get presigned upload URL"); + return presignedUpload; + } + + private async Task UploadToS3(FileInfo file, Uri s3Url) + { + using var fileStreamUpload = new FileStream(file.FullName, FileMode.Open, FileAccess.Read, FileShare.Read); + + Stream progressStream = fileStreamUpload; // TODO: wrap with progress stream + + using var streamContent = new StreamContent(progressStream); + streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); + streamContent.Headers.ContentLength = file.Length; + + using var uploadRequest = new HttpRequestMessage(HttpMethod.Put, s3Url); + uploadRequest.Content = streamContent; + + using var uploadResponse = await _s3Client + .SendAsync(uploadRequest, HttpCompletionOption.ResponseHeadersRead, _cancellationToken) + .ConfigureAwait(false); + + uploadResponse.EnsureSuccessStatusCode(); + } + + private async Task TriggerProcessing() + { + Uri processUri = new($"projects/{_projectId}/modelingestion/{_ingestionId}/uploads/process", UriKind.Relative); + + using HttpResponseMessage processResponse = await _speckleClient + .PostAsync(processUri, null, _cancellationToken) + .ConfigureAwait(false); + + processResponse.EnsureSuccessStatusCode(); + + return new UploadResult { IngestionId = _ingestionId }; + } + + public void Dispose() + { + _speckleClient.Dispose(); + _s3Client.Dispose(); + } +} diff --git a/src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs b/src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs new file mode 100644 index 00000000..fb020de4 --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs @@ -0,0 +1,22 @@ +using Speckle.Sdk.Models; +using Speckle.Sdk.Serialisation; + +namespace Speckle.Sdk.Pipelines.Send; + +public record UploadItem(string Id, Json Json, string SpeckleType, ObjectReference Reference); + +internal record PresignedUploadResponse +{ + public required Uri Url { get; init; } + public required string Key { get; init; } +} + +internal record ProcessUploadResponse +{ + public required string ingestionId { get; init; } +} + +internal record UploadResult +{ + public required string IngestionId { get; init; } +} diff --git a/src/Speckle.Sdk/Pipelines/Uploader.cs b/src/Speckle.Sdk/Pipelines/Uploader.cs deleted file mode 100644 index a7bf6e7e..00000000 --- a/src/Speckle.Sdk/Pipelines/Uploader.cs +++ /dev/null @@ -1,182 +0,0 @@ -using System.IO.Compression; -using System.Net.Http.Headers; -using System.Text; -using System.Threading.Channels; -using Speckle.Newtonsoft.Json; - -namespace Speckle.Sdk.Pipelines; - -public sealed class Uploader : IDisposable -{ - private readonly string _projectId; - private readonly string _modelId; - private readonly string _ingestionId; - private readonly CancellationToken _cancellationToken; - private readonly HttpClient _client; - private readonly Channel _channel; - private readonly Task _sendTask; - - public Uploader( - string projectId, - string modelId, - string ingestionId, - Uri serverUrl, - string authToken, - CancellationToken cancellationToken - ) - { - _projectId = projectId; - _modelId = modelId; - _ingestionId = ingestionId; - _cancellationToken = cancellationToken; - Uri apiBaseUrl = new(serverUrl, "/api/v1/"); - - _client = new HttpClient { BaseAddress = apiBaseUrl, Timeout = TimeSpan.FromMinutes(30) }; - - _client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", authToken); - - _channel = Channel.CreateBounded( - new BoundedChannelOptions(1000) { FullMode = BoundedChannelFullMode.Wait, SingleReader = true } - ); - - _sendTask = SendLoopAsync(); - } - - public ValueTask PushAsync(UploadItem item, CancellationToken ct = default) => _channel.Writer.WriteAsync(item, ct); - - public async Task CompleteAsync() - { - _channel.Writer.Complete(); - var result = await _sendTask.ConfigureAwait(false); - return result.IngestionId; - } - - private async Task SendLoopAsync() - { - // 1. Stream channel to temp file - string tempFilePath = Path.GetTempFileName(); - System.Diagnostics.Debug.WriteLine($"Temp file is at {tempFilePath}"); - try - { - long fileSizeBytes; - { - using var fileStream = new FileStream(tempFilePath, FileMode.Create, FileAccess.Write, FileShare.None); - using var gzip = new GZipStream(fileStream, CompressionLevel.Optimal); - using var writer = new StreamWriter(gzip); - await foreach (var item in _channel.Reader.ReadAllAsync(_cancellationToken).ConfigureAwait(false)) - { - await writer.WriteLineAsync($"{item.Id}\t{item.Json}\t{item.SpeckleType}").ConfigureAwait(false); - } -#if NET8_0_OR_GREATER - await writer.FlushAsync(_cancellationToken).ConfigureAwait(false); -#else - await writer.FlushAsync().ConfigureAwait(false); -#endif - } - // fileStream.Flush(); - // fileStream.Close(); - fileSizeBytes = new FileInfo(tempFilePath).Length; - - // 2. Request presigned URL - var signUri = new Uri($"projects/{_projectId}/models/{_modelId}/uploads/sign", UriKind.Relative); - - using var signResponse = await _client.PostAsync(signUri, null, _cancellationToken).ConfigureAwait(false); - signResponse.EnsureSuccessStatusCode(); - -#if NET5_0_OR_GREATER - string signResponseString = await signResponse - .Content.ReadAsStringAsync(_cancellationToken) - .ConfigureAwait(false); -#else - string signResponseString = await signResponse.Content.ReadAsStringAsync().ConfigureAwait(false); -#endif - PresignedUploadResponse presignedUpload = - JsonConvert.DeserializeObject(signResponseString) - ?? throw new InvalidOperationException("Failed to get presigned upload URL"); - - // 3. Upload to S3 - using var fileStreamUpload = new FileStream(tempFilePath, FileMode.Open, FileAccess.Read, FileShare.Read); - - Stream progressStream = fileStreamUpload; // TODO: wrap with progress stream - - using var streamContent = new StreamContent(progressStream); - streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); - streamContent.Headers.ContentLength = fileSizeBytes; - - using var uploadRequest = new HttpRequestMessage(HttpMethod.Put, new Uri(presignedUpload.Url, UriKind.Absolute)); - uploadRequest.Content = streamContent; - - using var s3Client = new HttpClient(); // NOTE: using a separate client for S3 as we DO NOT NEED THE AUTH HEADER, presigned url's don't work with it. - using var uploadResponse = await s3Client - .SendAsync(uploadRequest, HttpCompletionOption.ResponseHeadersRead, _cancellationToken) - .ConfigureAwait(false); - - uploadResponse.EnsureSuccessStatusCode(); - - // 4. Trigger processing - var processUri = new Uri($"projects/{_projectId}/models/{_modelId}/uploads/process", UriKind.Relative); - var processRequest = new ProcessUploadRequest { key = presignedUpload.Key, ingestionId = _ingestionId }; - - using StringContent content = new(JsonConvert.SerializeObject(processRequest), Encoding.UTF8, "application/json"); - var processResponse = await _client.PostAsync(processUri, content, _cancellationToken).ConfigureAwait(false); - - processResponse.EnsureSuccessStatusCode(); - -#if NET5_0_OR_GREATER - string processResult = await processResponse.Content.ReadAsStringAsync(_cancellationToken).ConfigureAwait(false); -#else - string processResult = await processResponse.Content.ReadAsStringAsync().ConfigureAwait(false); -#endif - var json = JsonConvert.DeserializeObject(processResult); - - if (json is null) - { - throw new InvalidOperationException("Failed to trigger upload processing"); - } - - return new UploadResult { IngestionId = json.ingestionId }; - } - finally - { - // 5. Clean up temp file - if (File.Exists(tempFilePath)) - { - try - { - //File.Delete(tempFilePath); - } -#pragma warning disable CA1031 - catch -#pragma warning restore CA1031 - { - // Best effort - } - } - } - } - - public void Dispose() => _client.Dispose(); -} - -// DTOs -internal record PresignedUploadResponse -{ - public required string Url { get; init; } - public required string Key { get; init; } -} - -internal record ProcessUploadRequest -{ - public required string key { get; init; } - public required string ingestionId { get; init; } -} - -internal record ProcessUploadResponse -{ - public required string ingestionId { get; init; } -} - -internal record UploadResult -{ - public required string IngestionId { get; init; } -} diff --git a/src/Speckle.Sdk/Pipelines/Uploader_Old.cs b/src/Speckle.Sdk/Pipelines/Uploader_Old.cs deleted file mode 100644 index fa8f73c6..00000000 --- a/src/Speckle.Sdk/Pipelines/Uploader_Old.cs +++ /dev/null @@ -1,83 +0,0 @@ -// using System.IO.Compression; -// using System.Net.Http.Headers; -// using System.Threading.Channels; -// -// namespace Speckle.Sdk.Pipeline; -// -// public sealed class UploaderOld : IDisposable -// { -// private readonly string _projectId; -// private readonly string _modelId; -// private readonly HttpClient _client; -// private readonly Channel _channel; -// private readonly Task _sendTask; -// -// public UploaderOld(string projectId, string modelId, string? authToken, string? apiEndpoint) -// { -// _projectId = projectId; -// _modelId = modelId; -// -// Uri apiBaseUrl = !string.IsNullOrEmpty(apiEndpoint) -// ? new Uri(apiEndpoint) -// // : new Uri("http://dimitries-macbook-pro.mermaid-emperor.ts.net/api/v1/"); -// : new Uri("http://zog.local:3000/api/v1/"); -// _client = new HttpClient { BaseAddress = apiBaseUrl, Timeout = TimeSpan.FromMinutes(10) }; -// -// if (authToken != null) -// { -// _client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", authToken); -// } -// -// _channel = Channel.CreateBounded( -// new BoundedChannelOptions(1000) { FullMode = BoundedChannelFullMode.Wait } // if we're not able to write fast enough, we'll block writes -// ); -// -// _sendTask = SendLoopAsync(projectId, "test"); -// } -// -// public ValueTask PushAsync(UploadItem item, CancellationToken ct = default) => _channel.Writer.WriteAsync(item, ct); -// -// public async Task CompleteAsync() -// { -// _channel.Writer.Complete(); -// await _sendTask.ConfigureAwait(false); -// } -// -// private async Task SendLoopAsync(string projectId, string modelId) -// { -// var content = new PushStreamContent( -// async (stream, _, _) => -// { -// var gzip = new GZipStream(stream, CompressionLevel.Optimal); -// var writer = new StreamWriter(gzip); // new StreamWriter(gzip, System.Text.Encoding.UTF8, 20 * 1024 * 1024); // potential lever for controlling memory pressure -// try -// { -// // extra levers for memory pressure in here: we can manually flush every x items or every x bytes -// await foreach (var item in _channel.Reader.ReadAllAsync().ConfigureAwait(false)) -// { -// await writer.WriteLineAsync($"{item.Id}\t{item.Json}\t{item.SpeckleType}").ConfigureAwait(false); -// } -// } -// finally -// { -// await writer.FlushAsync().ConfigureAwait(false); -// await gzip.FlushAsync().ConfigureAwait(false); -// writer.Dispose(); -// gzip.Dispose(); -// } -// }, -// new MediaTypeHeaderValue("application/x-ndjson") -// ); -// -// var uri = new Uri($"projects/{projectId}/models/{modelId}/versions", UriKind.Relative); -// var request = new HttpRequestMessage(HttpMethod.Post, uri) { Content = content }; -// request.Headers.TransferEncodingChunked = true; // NOTE: important for streaming to happen. -// var response = await _client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false); -// response.EnsureSuccessStatusCode(); -// -// // Consume the response body to fully complete the request -// await response.Content.ReadAsStringAsync().ConfigureAwait(false); -// } -// -// public void Dispose() => _client.Dispose(); -// } From 9bf6995b157dd55ca58d0fe8ad0f0ee3f8849438 Mon Sep 17 00:00:00 2001 From: Dimitrie Stefanescu Date: Tue, 24 Feb 2026 11:57:16 +0000 Subject: [PATCH 07/26] Add headers for Azure S3 compat --- src/Speckle.Sdk/Pipelines/Send/Uploader.cs | 13 +++++++++---- src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs | 1 + 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/Speckle.Sdk/Pipelines/Send/Uploader.cs b/src/Speckle.Sdk/Pipelines/Send/Uploader.cs index 9d7103d9..1cd96f13 100644 --- a/src/Speckle.Sdk/Pipelines/Send/Uploader.cs +++ b/src/Speckle.Sdk/Pipelines/Send/Uploader.cs @@ -74,8 +74,8 @@ public sealed class Uploader : IDisposable { using DisposableFile tempFile = await WriteFile().ConfigureAwait(false); - PresignedUploadResponse presignedUpload = await GetPresignedUrl().ConfigureAwait(false); - await UploadToS3(tempFile.FileInfo, presignedUpload.Url).ConfigureAwait(false); + PresignedUploadResponse presignedUploadResponse = await GetPresignedUrl().ConfigureAwait(false); + await UploadToS3(tempFile.FileInfo, presignedUploadResponse).ConfigureAwait(false); return await TriggerProcessing().ConfigureAwait(false); } @@ -123,7 +123,7 @@ public sealed class Uploader : IDisposable return presignedUpload; } - private async Task UploadToS3(FileInfo file, Uri s3Url) + private async Task UploadToS3(FileInfo file, PresignedUploadResponse presignedUploadResponse) { using var fileStreamUpload = new FileStream(file.FullName, FileMode.Open, FileAccess.Read, FileShare.Read); @@ -133,7 +133,12 @@ public sealed class Uploader : IDisposable streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); streamContent.Headers.ContentLength = file.Length; - using var uploadRequest = new HttpRequestMessage(HttpMethod.Put, s3Url); + using var uploadRequest = new HttpRequestMessage(HttpMethod.Put, presignedUploadResponse.Url); + foreach (var kvp in presignedUploadResponse.AdditionalRequestHeaders) + { + uploadRequest.Headers.Add(kvp.Key, kvp.Value); + } + uploadRequest.Content = streamContent; using var uploadResponse = await _s3Client diff --git a/src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs b/src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs index fb020de4..cff5775b 100644 --- a/src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs +++ b/src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs @@ -9,6 +9,7 @@ internal record PresignedUploadResponse { public required Uri Url { get; init; } public required string Key { get; init; } + public Dictionary AdditionalRequestHeaders { get; init; } = new(); } internal record ProcessUploadResponse From 070f21b07500086d64c7db80176cfc7de04d9c79 Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Thu, 26 Feb 2026 10:45:48 +0000 Subject: [PATCH 08/26] feat(progress): Add progress reporting to uploader (#446) * refactor uploader for progress * progress * not so many decimal places * small tweak to RenderStreamProgress * fix unit tests * uploading data --- .../Helpers/StopwatchPollyfills.cs | 21 +++++ .../Pipelines/Progress/AggregateProgress.cs | 12 +++ .../Progress/IngestionProgressManager.cs | 89 +++++++++++++++++++ .../IngestionProgressManagerFactory.cs | 22 +++++ .../Pipelines/Progress/ProgressArgs.cs | 6 ++ .../{ => Progress}/ProgressStream.cs | 16 ++-- .../Progress/RenderedStreamProgress.cs | 40 +++++++++ src/Speckle.Sdk/Pipelines/Send/DiskStore.cs | 77 ++++++++++++++++ .../Pipelines/Send/SendPipeline.cs | 30 +++++-- src/Speckle.Sdk/Pipelines/Send/Uploader.cs | 78 ++++------------ .../Pipelines/Send/UploaderDTOs.cs | 5 -- src/Speckle.Sdk/ServiceRegistration.cs | 4 +- 12 files changed, 313 insertions(+), 87 deletions(-) create mode 100644 src/Speckle.Sdk/Helpers/StopwatchPollyfills.cs create mode 100644 src/Speckle.Sdk/Pipelines/Progress/AggregateProgress.cs create mode 100644 src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManager.cs create mode 100644 src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManagerFactory.cs create mode 100644 src/Speckle.Sdk/Pipelines/Progress/ProgressArgs.cs rename src/Speckle.Sdk/Pipelines/{ => Progress}/ProgressStream.cs (87%) create mode 100644 src/Speckle.Sdk/Pipelines/Progress/RenderedStreamProgress.cs create mode 100644 src/Speckle.Sdk/Pipelines/Send/DiskStore.cs diff --git a/src/Speckle.Sdk/Helpers/StopwatchPollyfills.cs b/src/Speckle.Sdk/Helpers/StopwatchPollyfills.cs new file mode 100644 index 00000000..83dd0e05 --- /dev/null +++ b/src/Speckle.Sdk/Helpers/StopwatchPollyfills.cs @@ -0,0 +1,21 @@ +using System.Diagnostics; + +namespace Speckle.Sdk.Helpers; + +public static class StopwatchPolyfills +{ +#if !NET7_0_OR_GREATER + private static readonly double s_tickFrequency = (double)TimeSpan.TicksPerSecond / Stopwatch.Frequency; +#endif + + public static TimeSpan GetElapsedTime(long startingTimestamp) + { +#if NET7_0_OR_GREATER + return Stopwatch.GetElapsedTime(startingTimestamp); +#else + + long elapsedTicks = Stopwatch.GetTimestamp() - startingTimestamp; + return new TimeSpan((long)(elapsedTicks * s_tickFrequency)); +#endif + } +} diff --git a/src/Speckle.Sdk/Pipelines/Progress/AggregateProgress.cs b/src/Speckle.Sdk/Pipelines/Progress/AggregateProgress.cs new file mode 100644 index 00000000..74dfa327 --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Progress/AggregateProgress.cs @@ -0,0 +1,12 @@ +namespace Speckle.Sdk.Pipelines.Progress; + +public sealed class AggregateProgress(params IProgress[] progresses) : IProgress +{ + public void Report(T value) + { + foreach (var progress in progresses) + { + progress.Report(value); + } + } +} diff --git a/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManager.cs b/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManager.cs new file mode 100644 index 00000000..be141c65 --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManager.cs @@ -0,0 +1,89 @@ +using System.Diagnostics; +using Microsoft.Extensions.Logging; +using Speckle.InterfaceGenerator; +using Speckle.Sdk.Api; +using Speckle.Sdk.Api.GraphQL.Inputs; +using Speckle.Sdk.Api.GraphQL.Models; +using Speckle.Sdk.Helpers; + +namespace Speckle.Sdk.Pipelines.Progress; + +public partial interface IIngestionProgressManager : IProgress; + +/// +/// An implementation for the entire client side Ingestion progress update reporting +/// Will throttles ingestion progress messages and reports their progress +/// +[GenerateAutoInterface] +public sealed class IngestionProgressManager( + ILogger logger, + IClient speckleClient, + ModelIngestion ingestion, + string projectId, + TimeSpan updateInterval, + CancellationToken cancellationToken +) : IIngestionProgressManager +{ + /// + /// Normally we would pick quite a coarse throttle window to try and avoid over pressure (1-5s) + /// + private Task? _lastUpdate; + private long _lastUpdatedAt; + private readonly object _lock = new(); + + [AutoInterfaceIgnore] + public void Report(CardProgress value) + { + cancellationToken.ThrowIfCancellationRequested(); + + string trimmedMessage; + lock (_lock) + { + if (ShouldIgnoreProgressUpdate()) + { + return; + } + + _lastUpdatedAt = Stopwatch.GetTimestamp(); + + trimmedMessage = value.Status.TrimEnd('.'); + + _lastUpdate = speckleClient + .Ingestion.UpdateProgress( + new ModelIngestionUpdateInput(ingestion.id, projectId, trimmedMessage, value.Progress), + cancellationToken + ) + .ContinueWith( + HandleFaultedContinuation, + CancellationToken.None, + TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default + ); + } + + logger.LogInformation("Progress update {Message} {Progress}", trimmedMessage, value.Progress); + } + + /// if the update should be ignored, otherwise + private bool ShouldIgnoreProgressUpdate() + { + if (_lastUpdate is not null && !_lastUpdate.IsCompleted) + { + return true; + } + + TimeSpan msSinceLastUpdate = StopwatchPolyfills.GetElapsedTime(_lastUpdatedAt); + return msSinceLastUpdate < updateInterval; + } + + private void HandleFaultedContinuation(Task updateTask) + { + // The progress report failed... could be many reasons. + // For now, we're not letting this fail the Ingestion in any way + // we'll log but otherwise let it slide while leaving no unobserved task exceptions + if (updateTask.IsFaulted) + { + logger.LogWarning(updateTask.Exception, "A progress update failed unexpectedly"); + } + } +} diff --git a/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManagerFactory.cs b/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManagerFactory.cs new file mode 100644 index 00000000..998431c2 --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManagerFactory.cs @@ -0,0 +1,22 @@ +using Microsoft.Extensions.Logging; +using Speckle.InterfaceGenerator; +using Speckle.Sdk.Api; +using Speckle.Sdk.Api.GraphQL.Models; + +namespace Speckle.Sdk.Pipelines.Progress; + +[GenerateAutoInterface] +public sealed class IngestionProgressManagerFactory(ILogger logger) + : IIngestionProgressManagerFactory +{ + public IIngestionProgressManager CreateInstance( + IClient speckleClient, + ModelIngestion ingestion, + string projectId, + TimeSpan updateInterval, + CancellationToken cancellationToken + ) + { + return new IngestionProgressManager(logger, speckleClient, ingestion, projectId, updateInterval, cancellationToken); + } +} diff --git a/src/Speckle.Sdk/Pipelines/Progress/ProgressArgs.cs b/src/Speckle.Sdk/Pipelines/Progress/ProgressArgs.cs new file mode 100644 index 00000000..bfc47dbc --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Progress/ProgressArgs.cs @@ -0,0 +1,6 @@ +namespace Speckle.Sdk.Pipelines.Progress; + +//TODO: rename PipelineProgressArgs +public readonly record struct CardProgress(string Status, double? Progress); + +public readonly record struct StreamProgressArgs(long BytesStreamed, long ExpectedTotalBytes); diff --git a/src/Speckle.Sdk/Pipelines/ProgressStream.cs b/src/Speckle.Sdk/Pipelines/Progress/ProgressStream.cs similarity index 87% rename from src/Speckle.Sdk/Pipelines/ProgressStream.cs rename to src/Speckle.Sdk/Pipelines/Progress/ProgressStream.cs index b23998ae..f75c7837 100644 --- a/src/Speckle.Sdk/Pipelines/ProgressStream.cs +++ b/src/Speckle.Sdk/Pipelines/Progress/ProgressStream.cs @@ -1,23 +1,17 @@ using System.Diagnostics.CodeAnalysis; -namespace Speckle.Sdk.Pipelines; - -public readonly record struct StreamProgressArgs(long BytesStreamed, long ExpectedTotalBytes); +namespace Speckle.Sdk.Pipelines.Progress; /// -/// Wraps a stream to report upload progress as bytes are read. +/// Wraps to report streaming progress as bytes are read/written. /// -public sealed class ProgressStream( - Stream innerStream, - long expectedTotalBytesStreamed, - IProgress? progress = null -) : Stream +public sealed class ProgressStream(Stream innerStream, IProgress? progress = null) : Stream { private long _bytesStreamed; public override bool CanRead => innerStream.CanRead; public override bool CanSeek => innerStream.CanSeek; - public override bool CanWrite => false; + public override bool CanWrite => innerStream.CanWrite; public override long Length => innerStream.Length; public override long Position @@ -57,7 +51,7 @@ public sealed class ProgressStream( private void ReportProgress(int newBytesProcessed) { _bytesStreamed += newBytesProcessed; - progress?.Report(new(_bytesStreamed, expectedTotalBytesStreamed)); + progress?.Report(new(_bytesStreamed, Length)); } public override void Flush() => innerStream.Flush(); diff --git a/src/Speckle.Sdk/Pipelines/Progress/RenderedStreamProgress.cs b/src/Speckle.Sdk/Pipelines/Progress/RenderedStreamProgress.cs new file mode 100644 index 00000000..81f41d6c --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Progress/RenderedStreamProgress.cs @@ -0,0 +1,40 @@ +namespace Speckle.Sdk.Pipelines.Progress; + +/// +/// Renders "low level" data stream updates +/// into "high level" that is expected by Ingestion progress and DUI3 +/// +/// +public sealed class RenderedStreamProgress(IProgress progress) : IProgress +{ + public void Report(StreamProgressArgs value) + { + var (suffix, scaleFactor) = GetFileSizeRendering(value.ExpectedTotalBytes); + progress.Report( + new( + $"Uploading data... ({value.BytesStreamed * scaleFactor:F1}/{value.ExpectedTotalBytes * scaleFactor:F1} {suffix})", + (double)value.BytesStreamed / value.ExpectedTotalBytes + ) + ); + } + + private static readonly string[] s_suffixes = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"]; + + private static (string suffix, double scaleFactor) GetFileSizeRendering(long value) + { + if (value <= 0) + { + return (s_suffixes[0], 1d); + } + + for (int i = 0; i < s_suffixes.Length; i++) + { + if (value <= Math.Pow(1024, i + 1)) + { + return (s_suffixes[i], 1 / Math.Pow(1024, i)); + } + } + + throw new ArgumentOutOfRangeException(nameof(value), "Value is too large to convert to a file size"); + } +} diff --git a/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs b/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs new file mode 100644 index 00000000..0ac685fb --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs @@ -0,0 +1,77 @@ +using System.IO.Compression; +using System.Threading.Channels; +using Microsoft.Extensions.Logging; +using Speckle.InterfaceGenerator; +using Speckle.Sdk.Helpers; + +namespace Speckle.Sdk.Pipelines.Send; + +[GenerateAutoInterface] +public sealed class DiskStoreFactory(ILogger logger) : IDiskStoreFactory +{ + public DiskStore CreateInstance(CancellationToken cancellationToken) => new(logger, cancellationToken); +} + +public sealed class DiskStore +{ + private readonly Channel _channel; + private readonly Task _writeToDiskTask; + private readonly ILogger _logger; + private readonly CancellationToken _cancellationToken; + + internal DiskStore(ILogger logger, CancellationToken cancellationToken) + { + _logger = logger; + _cancellationToken = cancellationToken; + + _channel = Channel.CreateBounded( + new BoundedChannelOptions(1000) { FullMode = BoundedChannelFullMode.Wait, SingleReader = true } + ); + _writeToDiskTask = Task.Run(WriteFile, cancellationToken); + } + + public ValueTask PushAsync(UploadItem item) => _channel.Writer.WriteAsync(item, _cancellationToken); + + public async Task CompleteAsync() + { + _channel.Writer.Complete(); + return await _writeToDiskTask.ConfigureAwait(false); + } + + /// + /// Reads from the Channel and streams the s to a temporary file on disk. + /// Will keep reading until is called. + /// + /// the file that was written + private async Task WriteFile() + { + string tempFilePath = Path.GetTempFileName(); + var tempFile = new DisposableFile(new FileInfo(tempFilePath), _logger); + _logger.LogInformation("Writing temp file to {TempFilePath}", tempFilePath); + + try + { + using var fileStream = new FileStream(tempFilePath, FileMode.Create, FileAccess.Write, FileShare.None); + using var gzip = new GZipStream(fileStream, CompressionLevel.Optimal); + using var writer = new StreamWriter(gzip); + + await foreach (var item in _channel.Reader.ReadAllAsync(_cancellationToken).ConfigureAwait(false)) + { + await writer.WriteLineAsync($"{item.Id}\t{item.Json}\t{item.SpeckleType}").ConfigureAwait(false); + } +#if NET8_0_OR_GREATER + await writer.FlushAsync(_cancellationToken).ConfigureAwait(false); +#else + await writer.FlushAsync().ConfigureAwait(false); +#endif + tempFile.FileInfo.Refresh(); + + return tempFile; + } + catch + { + tempFile.Dispose(); + throw; + } + } +} diff --git a/src/Speckle.Sdk/Pipelines/Send/SendPipeline.cs b/src/Speckle.Sdk/Pipelines/Send/SendPipeline.cs index e1a97a40..e00439ff 100644 --- a/src/Speckle.Sdk/Pipelines/Send/SendPipeline.cs +++ b/src/Speckle.Sdk/Pipelines/Send/SendPipeline.cs @@ -1,21 +1,26 @@ using Speckle.InterfaceGenerator; using Speckle.Sdk.Credentials; +using Speckle.Sdk.Helpers; using Speckle.Sdk.Models; +using Speckle.Sdk.Pipelines.Progress; namespace Speckle.Sdk.Pipelines.Send; [GenerateAutoInterface] -public sealed class SendPipelineFactory(IUploaderFactory uploaderFactory) : ISendPipelineFactory +public sealed class SendPipelineFactory(IUploaderFactory uploaderFactory, IDiskStoreFactory diskStoreFactory) + : ISendPipelineFactory { public SendPipeline CreateInstance( string projectId, string ingestionId, Account account, + IProgress uploadProgress, CancellationToken cancellationToken ) { - var uploader = uploaderFactory.CreateInstance(projectId, ingestionId, account, cancellationToken); - return new SendPipeline(uploader); + var uploader = uploaderFactory.CreateInstance(projectId, ingestionId, account, uploadProgress, cancellationToken); + var diskStore = diskStoreFactory.CreateInstance(cancellationToken); + return new SendPipeline(uploader, diskStore); } } @@ -23,10 +28,12 @@ public sealed class SendPipeline : IDisposable { private readonly Serializer _serializer = new(); private readonly Uploader _uploader; + private readonly DiskStore _diskStore; - internal SendPipeline(Uploader uploader) + internal SendPipeline(Uploader uploader, DiskStore diskStore) { _uploader = uploader; + _diskStore = diskStore; } private UploadItem _lastItem; @@ -38,7 +45,7 @@ public sealed class SendPipeline : IDisposable foreach (var item in results) { // we're not doing fire and forget here so that we get the backpressure from the uploader - await _uploader.PushAsync(item).ConfigureAwait(false); + await _diskStore.PushAsync(item).ConfigureAwait(false); } // NOTE: this is important to keep track of. When we serialze an object, we get back a list of objects, with the first one being the original root. @@ -50,8 +57,17 @@ public sealed class SendPipeline : IDisposable public async Task WaitForUpload() { - await _uploader.PushAsync(_lastItem).ConfigureAwait(false); - await _uploader.CompleteAsync().ConfigureAwait(false); + await _diskStore.PushAsync(_lastItem).ConfigureAwait(false); + using DisposableFile tempFile = await _diskStore.CompleteAsync().ConfigureAwait(false); + + using Stream fileStreamUpload = new FileStream( + tempFile.FileInfo.FullName, + FileMode.Open, + FileAccess.Read, + FileShare.Read + ); + + await _uploader.Send(fileStreamUpload).ConfigureAwait(false); } public async Task WaitForUploadAndServerProcessing() diff --git a/src/Speckle.Sdk/Pipelines/Send/Uploader.cs b/src/Speckle.Sdk/Pipelines/Send/Uploader.cs index 1cd96f13..63a17881 100644 --- a/src/Speckle.Sdk/Pipelines/Send/Uploader.cs +++ b/src/Speckle.Sdk/Pipelines/Send/Uploader.cs @@ -1,11 +1,10 @@ -using System.IO.Compression; using System.Net.Http.Headers; -using System.Threading.Channels; using Microsoft.Extensions.Logging; using Speckle.InterfaceGenerator; using Speckle.Newtonsoft.Json; using Speckle.Sdk.Credentials; using Speckle.Sdk.Helpers; +using Speckle.Sdk.Pipelines.Progress; namespace Speckle.Sdk.Pipelines.Send; @@ -16,8 +15,9 @@ public sealed class UploaderFactory(ISpeckleHttp httpClientFactory, ILogger progress, CancellationToken cancellationToken - ) => new(projectId, ingestionId, logger, httpClientFactory, account, cancellationToken); + ) => new(projectId, ingestionId, logger, httpClientFactory, account, progress, cancellationToken); } public sealed class Uploader : IDisposable @@ -27,9 +27,8 @@ public sealed class Uploader : IDisposable private readonly CancellationToken _cancellationToken; private readonly HttpClient _speckleClient; private readonly HttpClient _s3Client; - private readonly Channel _channel; - private readonly Task _sendTask; private readonly ILogger _logger; + private readonly IProgress _progress; internal Uploader( string projectId, @@ -37,6 +36,7 @@ public sealed class Uploader : IDisposable ILogger logger, ISpeckleHttp httpClientFactory, Account speckleAccount, + IProgress progress, CancellationToken cancellationToken ) { @@ -44,65 +44,19 @@ public sealed class Uploader : IDisposable _ingestionId = ingestionId; _logger = logger; _cancellationToken = cancellationToken; - - _speckleClient = httpClientFactory.CreateHttpClient( - null, - (int)TimeSpan.FromMinutes(30).TotalSeconds, - speckleAccount.token - ); + _progress = progress; + _speckleClient = httpClientFactory.CreateHttpClient(authorizationToken: speckleAccount.token); _speckleClient.BaseAddress = new(new(speckleAccount.serverInfo.url), "/api/v1/"); _s3Client = httpClientFactory.CreateHttpClient(); - - _channel = Channel.CreateBounded( - new BoundedChannelOptions(1000) { FullMode = BoundedChannelFullMode.Wait, SingleReader = true } - ); - - _sendTask = Task.Run(SendLoopAsync, cancellationToken); } - public ValueTask PushAsync(UploadItem item) => _channel.Writer.WriteAsync(item, _cancellationToken); - - public async Task CompleteAsync() + public async Task Send(Stream fileStream) { - _channel.Writer.Complete(); - var result = await _sendTask.ConfigureAwait(false); - return result.IngestionId; - } - - private async Task SendLoopAsync() - { - using DisposableFile tempFile = await WriteFile().ConfigureAwait(false); - PresignedUploadResponse presignedUploadResponse = await GetPresignedUrl().ConfigureAwait(false); - await UploadToS3(tempFile.FileInfo, presignedUploadResponse).ConfigureAwait(false); + await UploadToS3(fileStream, presignedUploadResponse).ConfigureAwait(false); - return await TriggerProcessing().ConfigureAwait(false); - } - - /// - /// Reads from the Channel and streams the s to a temporary file on disk. - /// Will keep reading until is called. - /// - /// the file that was written - private async Task WriteFile() - { - string tempFilePath = Path.GetTempFileName(); - _logger.LogInformation("Writing temp file to {TempFilePath}", tempFilePath); - - using var fileStream = new FileStream(tempFilePath, FileMode.Create, FileAccess.Write, FileShare.None); - using var gzip = new GZipStream(fileStream, CompressionLevel.Optimal); - using var writer = new StreamWriter(gzip); - await foreach (var item in _channel.Reader.ReadAllAsync(_cancellationToken).ConfigureAwait(false)) - { - await writer.WriteLineAsync($"{item.Id}\t{item.Json}\t{item.SpeckleType}").ConfigureAwait(false); - } -#if NET8_0_OR_GREATER - await writer.FlushAsync(_cancellationToken).ConfigureAwait(false); -#else - await writer.FlushAsync().ConfigureAwait(false); -#endif - return new DisposableFile(new FileInfo(tempFilePath), _logger); + await TriggerProcessing().ConfigureAwait(false); } private async Task GetPresignedUrl() @@ -123,15 +77,15 @@ public sealed class Uploader : IDisposable return presignedUpload; } - private async Task UploadToS3(FileInfo file, PresignedUploadResponse presignedUploadResponse) + private async Task UploadToS3(Stream fileStream, PresignedUploadResponse presignedUploadResponse) { - using var fileStreamUpload = new FileStream(file.FullName, FileMode.Open, FileAccess.Read, FileShare.Read); + _logger.LogInformation("Uploading file to pre-signed url"); - Stream progressStream = fileStreamUpload; // TODO: wrap with progress stream + Stream progressStream = new ProgressStream(fileStream, _progress); using var streamContent = new StreamContent(progressStream); streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); - streamContent.Headers.ContentLength = file.Length; + streamContent.Headers.ContentLength = fileStream.Length; using var uploadRequest = new HttpRequestMessage(HttpMethod.Put, presignedUploadResponse.Url); foreach (var kvp in presignedUploadResponse.AdditionalRequestHeaders) @@ -148,7 +102,7 @@ public sealed class Uploader : IDisposable uploadResponse.EnsureSuccessStatusCode(); } - private async Task TriggerProcessing() + private async Task TriggerProcessing() { Uri processUri = new($"projects/{_projectId}/modelingestion/{_ingestionId}/uploads/process", UriKind.Relative); @@ -157,8 +111,6 @@ public sealed class Uploader : IDisposable .ConfigureAwait(false); processResponse.EnsureSuccessStatusCode(); - - return new UploadResult { IngestionId = _ingestionId }; } public void Dispose() diff --git a/src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs b/src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs index cff5775b..8ff9a546 100644 --- a/src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs +++ b/src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs @@ -16,8 +16,3 @@ internal record ProcessUploadResponse { public required string ingestionId { get; init; } } - -internal record UploadResult -{ - public required string IngestionId { get; init; } -} diff --git a/src/Speckle.Sdk/ServiceRegistration.cs b/src/Speckle.Sdk/ServiceRegistration.cs index 94e147cc..9028ae7c 100644 --- a/src/Speckle.Sdk/ServiceRegistration.cs +++ b/src/Speckle.Sdk/ServiceRegistration.cs @@ -8,6 +8,7 @@ using Speckle.Sdk.Dependencies; using Speckle.Sdk.Host; using Speckle.Sdk.Logging; using Speckle.Sdk.Models.GraphTraversal; +using Speckle.Sdk.Pipelines.Progress; using Speckle.Sdk.Serialisation.V2; using Speckle.Sdk.Serialisation.V2.Receive; using Speckle.Sdk.Serialisation.V2.Send; @@ -96,7 +97,8 @@ public static class ServiceRegistration typeof(DeserializeProcess), typeof(ObjectLoader), typeof(TraversalRule), - typeof(Client) + typeof(Client), + typeof(IngestionProgressManager) ); serviceCollection.AddMatchingInterfacesAsTransient(typeof(GraphQLRetry).Assembly); return serviceCollection; From 0ca9162e7b3a4e062a8d6b4a7ff878f5e2d24f92 Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Thu, 26 Feb 2026 16:09:19 +0000 Subject: [PATCH 09/26] pass etag to trigger processing (#447) --- src/Speckle.Sdk/Api/Blob/BlobApi.cs | 25 +--------------- src/Speckle.Sdk/Helpers/BlobApiHelpers.cs | 29 +++++++++++++++++++ src/Speckle.Sdk/Pipelines/Send/Uploader.cs | 16 ++++++---- .../Pipelines/Send/UploaderDTOs.cs | 6 ++-- 4 files changed, 45 insertions(+), 31 deletions(-) create mode 100644 src/Speckle.Sdk/Helpers/BlobApiHelpers.cs diff --git a/src/Speckle.Sdk/Api/Blob/BlobApi.cs b/src/Speckle.Sdk/Api/Blob/BlobApi.cs index 1dd68eb5..10939f6d 100644 --- a/src/Speckle.Sdk/Api/Blob/BlobApi.cs +++ b/src/Speckle.Sdk/Api/Blob/BlobApi.cs @@ -193,30 +193,7 @@ public sealed class BlobApi : IBlobApi using var response = await _unauthedClient.SendAsync(requestMessage, cancellationToken).ConfigureAwait(false); response.EnsureSuccessStatusCode(); - return ParseEtagHeader(response.Headers); - } - - private static string ParseEtagHeader(HttpResponseHeaders headers) - { - if (!headers.TryGetValues("ETag", out var etagValues)) - { - throw new ArgumentException( - "Response does not have an ETag attached to it, cannot use this as an upload", - nameof(headers) - ); - } - - var etagValuesArray = etagValues.ToArray(); - - if (etagValuesArray.Length != 1) - { - throw new ArgumentException( - $"Expected Etag header to have a single value but got {etagValuesArray.Length}", - nameof(headers) - ); - } - - return etagValuesArray[0]; + return BlobApiHelpers.ParseEtagHeader(response.Headers); } /// diff --git a/src/Speckle.Sdk/Helpers/BlobApiHelpers.cs b/src/Speckle.Sdk/Helpers/BlobApiHelpers.cs new file mode 100644 index 00000000..c7979567 --- /dev/null +++ b/src/Speckle.Sdk/Helpers/BlobApiHelpers.cs @@ -0,0 +1,29 @@ +using System.Net.Http.Headers; + +namespace Speckle.Sdk.Helpers; + +public static class BlobApiHelpers +{ + public static string ParseEtagHeader(HttpResponseHeaders headers) + { + if (!headers.TryGetValues("ETag", out var etagValues)) + { + throw new ArgumentException( + "Response does not have an ETag attached to it, cannot use this as an upload", + nameof(headers) + ); + } + + var etagValuesArray = etagValues.ToArray(); + + if (etagValuesArray.Length != 1) + { + throw new ArgumentException( + $"Expected Etag header to have a single value but got {etagValuesArray.Length}", + nameof(headers) + ); + } + + return etagValuesArray[0]; + } +} diff --git a/src/Speckle.Sdk/Pipelines/Send/Uploader.cs b/src/Speckle.Sdk/Pipelines/Send/Uploader.cs index 63a17881..1cd4f8f4 100644 --- a/src/Speckle.Sdk/Pipelines/Send/Uploader.cs +++ b/src/Speckle.Sdk/Pipelines/Send/Uploader.cs @@ -1,4 +1,5 @@ using System.Net.Http.Headers; +using System.Text; using Microsoft.Extensions.Logging; using Speckle.InterfaceGenerator; using Speckle.Newtonsoft.Json; @@ -54,9 +55,9 @@ public sealed class Uploader : IDisposable public async Task Send(Stream fileStream) { PresignedUploadResponse presignedUploadResponse = await GetPresignedUrl().ConfigureAwait(false); - await UploadToS3(fileStream, presignedUploadResponse).ConfigureAwait(false); + var etag = await UploadToS3(fileStream, presignedUploadResponse).ConfigureAwait(false); - await TriggerProcessing().ConfigureAwait(false); + await TriggerProcessing(new() { Etag = etag }).ConfigureAwait(false); } private async Task GetPresignedUrl() @@ -77,7 +78,7 @@ public sealed class Uploader : IDisposable return presignedUpload; } - private async Task UploadToS3(Stream fileStream, PresignedUploadResponse presignedUploadResponse) + private async Task UploadToS3(Stream fileStream, PresignedUploadResponse presignedUploadResponse) { _logger.LogInformation("Uploading file to pre-signed url"); @@ -100,16 +101,21 @@ public sealed class Uploader : IDisposable .ConfigureAwait(false); uploadResponse.EnsureSuccessStatusCode(); + + return BlobApiHelpers.ParseEtagHeader(uploadResponse.Headers); } - private async Task TriggerProcessing() + private async Task TriggerProcessing(TriggerUploadRequest request) { Uri processUri = new($"projects/{_projectId}/modelingestion/{_ingestionId}/uploads/process", UriKind.Relative); + string requestBody = JsonConvert.SerializeObject(request); + using var content = new StringContent(requestBody, Encoding.UTF8, "application/json"); using HttpResponseMessage processResponse = await _speckleClient - .PostAsync(processUri, null, _cancellationToken) + .PostAsync(processUri, content, _cancellationToken) .ConfigureAwait(false); + string body = await processResponse.Content.ReadAsStringAsync().ConfigureAwait(false); processResponse.EnsureSuccessStatusCode(); } diff --git a/src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs b/src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs index 8ff9a546..11df1420 100644 --- a/src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs +++ b/src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs @@ -1,3 +1,4 @@ +using Speckle.Newtonsoft.Json; using Speckle.Sdk.Models; using Speckle.Sdk.Serialisation; @@ -12,7 +13,8 @@ internal record PresignedUploadResponse public Dictionary AdditionalRequestHeaders { get; init; } = new(); } -internal record ProcessUploadResponse +internal readonly struct TriggerUploadRequest { - public required string ingestionId { get; init; } + [JsonProperty("etag")] + public required string Etag { get; init; } } From 9a74195b24bee0c6b3eea247eb05472c26556955 Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Thu, 26 Feb 2026 17:10:54 +0000 Subject: [PATCH 10/26] feat(api): Add versionId to ModelIngestion (#448) * Add version id to ingestion query * and the sub * Add project & user ids * test --- .../Api/GraphQL/Models/ModelIngestion.cs | 3 ++- .../Models/ModelIngestionStatusData.cs | 1 + .../Resources/ModelIngestionResource.cs | 20 +++++++++++++++++++ .../GraphQL/Resources/SubscriptionResource.cs | 6 ++++++ .../Resources/ModelIngestionResourceTests.cs | 7 +++++++ 5 files changed, 36 insertions(+), 1 deletion(-) diff --git a/src/Speckle.Sdk/Api/GraphQL/Models/ModelIngestion.cs b/src/Speckle.Sdk/Api/GraphQL/Models/ModelIngestion.cs index 58d0dc67..e5c08dfc 100644 --- a/src/Speckle.Sdk/Api/GraphQL/Models/ModelIngestion.cs +++ b/src/Speckle.Sdk/Api/GraphQL/Models/ModelIngestion.cs @@ -6,7 +6,8 @@ public sealed class ModelIngestion public required DateTime createdAt { get; init; } public required DateTime updatedAt { get; init; } public required string modelId { get; init; } + public required string projectId { get; init; } + public required string userId { get; init; } public required bool cancellationRequested { get; init; } public required ModelIngestionStatusData statusData { get; init; } - // public required LimitedUser user { get; init; } } diff --git a/src/Speckle.Sdk/Api/GraphQL/Models/ModelIngestionStatusData.cs b/src/Speckle.Sdk/Api/GraphQL/Models/ModelIngestionStatusData.cs index a195bdfc..8e98d1e6 100644 --- a/src/Speckle.Sdk/Api/GraphQL/Models/ModelIngestionStatusData.cs +++ b/src/Speckle.Sdk/Api/GraphQL/Models/ModelIngestionStatusData.cs @@ -6,4 +6,5 @@ public sealed class ModelIngestionStatusData { public required ModelIngestionStatus status { get; init; } public required string? progressMessage { get; init; } + public required string? versionId { get; init; } } diff --git a/src/Speckle.Sdk/Api/GraphQL/Resources/ModelIngestionResource.cs b/src/Speckle.Sdk/Api/GraphQL/Resources/ModelIngestionResource.cs index d029a61d..d01a6a10 100644 --- a/src/Speckle.Sdk/Api/GraphQL/Resources/ModelIngestionResource.cs +++ b/src/Speckle.Sdk/Api/GraphQL/Resources/ModelIngestionResource.cs @@ -44,6 +44,8 @@ public sealed class ModelIngestionResource createdAt updatedAt modelId + projectId + userId cancellationRequested statusData { ... on HasModelIngestionStatus { @@ -94,6 +96,8 @@ public sealed class ModelIngestionResource createdAt updatedAt modelId + projectId + userId cancellationRequested statusData { ... on HasModelIngestionStatus { @@ -102,6 +106,10 @@ public sealed class ModelIngestionResource ... on HasProgressMessage { progressMessage } + ... on ModelIngestionSuccessStatus + { + versionId + } } } } @@ -142,6 +150,8 @@ public sealed class ModelIngestionResource createdAt updatedAt modelId + projectId + userId cancellationRequested statusData { ... on HasModelIngestionStatus { @@ -194,6 +204,8 @@ public sealed class ModelIngestionResource createdAt updatedAt modelId + projectId + userId cancellationRequested statusData { ... on HasModelIngestionStatus { @@ -245,6 +257,8 @@ public sealed class ModelIngestionResource createdAt updatedAt modelId + projectId + userId cancellationRequested statusData { ... on HasModelIngestionStatus { @@ -343,6 +357,8 @@ public sealed class ModelIngestionResource createdAt updatedAt modelId + projectId + userId cancellationRequested statusData { ... on HasModelIngestionStatus { @@ -398,6 +414,8 @@ public sealed class ModelIngestionResource createdAt updatedAt modelId + projectId + userId cancellationRequested statusData { ... on HasModelIngestionStatus { @@ -457,6 +475,8 @@ public sealed class ModelIngestionResource createdAt updatedAt modelId + projectId + userId cancellationRequested statusData { ... on HasModelIngestionStatus { diff --git a/src/Speckle.Sdk/Api/GraphQL/Resources/SubscriptionResource.cs b/src/Speckle.Sdk/Api/GraphQL/Resources/SubscriptionResource.cs index 9a014cff..8ba201d1 100644 --- a/src/Speckle.Sdk/Api/GraphQL/Resources/SubscriptionResource.cs +++ b/src/Speckle.Sdk/Api/GraphQL/Resources/SubscriptionResource.cs @@ -229,6 +229,8 @@ public sealed class SubscriptionResource : IDisposable createdAt updatedAt modelId + projectId + userId cancellationRequested statusData { ... on HasModelIngestionStatus { @@ -237,6 +239,10 @@ public sealed class SubscriptionResource : IDisposable ... on HasProgressMessage { progressMessage } + ... on ModelIngestionSuccessStatus + { + versionId + } } } type diff --git a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/ModelIngestionResourceTests.cs b/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/ModelIngestionResourceTests.cs index 88ff78d2..7b8e1552 100644 --- a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/ModelIngestionResourceTests.cs +++ b/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/ModelIngestionResourceTests.cs @@ -129,8 +129,10 @@ public sealed class ModelIngestionResourceTests : IAsyncLifetime ModelIngestionSuccessInput finish = new(ingest.id, _project.id, sendResult.RootId, "yay!"); string versionId = await Sut.Complete(finish); Version version = await _testUser.Version.Get(versionId, _project.id); + ModelIngestion finalIngestion = await _testUser.Ingestion.Get(ingest.id, _project.id); Assert.Equal(version.id, versionId); Assert.Equal(sendResult.RootId, version.referencedObject); + Assert.Equal(finalIngestion.statusData.versionId, versionId); } [Fact] @@ -147,6 +149,11 @@ public sealed class ModelIngestionResourceTests : IAsyncLifetime ModelIngestion res = await Sut.Get(ingest.id, _project.id); Assert.Equal(ingest.id, res.id); Assert.Equal(ingest.statusData.status, res.statusData.status); + Assert.Equal(ingest.statusData.versionId, res.statusData.versionId); + Assert.Null(res.statusData.versionId); + Assert.Equal(_model.id, res.modelId); + Assert.Equal(_project.id, res.projectId); + Assert.Equal(_testUser.Account.userInfo.id, res.userId); } [Fact] From a662fb54c2f928a533c0f4639062816dee279b98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?O=C4=9Fuzhan=20Koral?= <45078678+oguzhankoral@users.noreply.github.com> Date: Fri, 27 Feb 2026 13:32:16 +0300 Subject: [PATCH 11/26] fix: channels reference only for net standard (#450) --- Directory.Packages.props | 2 +- src/Speckle.Automate.Sdk/packages.lock.json | 23 +++++++------------ src/Speckle.Objects/packages.lock.json | 23 +++++++------------ .../packages.lock.json | 18 +++++++-------- src/Speckle.Sdk/Speckle.Sdk.csproj | 2 +- src/Speckle.Sdk/packages.lock.json | 18 +++++---------- .../packages.lock.json | 9 +------- .../packages.lock.json | 9 +------- .../packages.lock.json | 9 +------- .../packages.lock.json | 9 +------- tests/Speckle.Sdk.Testing/packages.lock.json | 9 +------- .../packages.lock.json | 9 +------- .../packages.lock.json | 9 +------- .../Speckle.Sdk.Tests.Unit/packages.lock.json | 9 +------- 14 files changed, 41 insertions(+), 117 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index a2c96acf..f2f4c427 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -27,7 +27,7 @@ - + diff --git a/src/Speckle.Automate.Sdk/packages.lock.json b/src/Speckle.Automate.Sdk/packages.lock.json index 1a222bbb..dd5d4c56 100644 --- a/src/Speckle.Automate.Sdk/packages.lock.json +++ b/src/Speckle.Automate.Sdk/packages.lock.json @@ -88,8 +88,8 @@ }, "Microsoft.Bcl.AsyncInterfaces": { "type": "Transitive", - "resolved": "10.0.1", - "contentHash": "E1HSLkPHXEO30JEij2pWbOuzz1Z5ND4a5l7IP1T2RgQuE0a0NzEIvtO64RNy3Otn6PFezbT80cfm3M/Cgt70PA==", + "resolved": "10.0.0", + "contentHash": "vFuwSLj9QJBbNR0NeNO4YVASUbokxs+i/xbuu8B+Fs4FAZg5QaFa6eGrMaRqTzzNI5tAb97T7BhSxtLckFyiRA==", "dependencies": { "System.Threading.Tasks.Extensions": "4.6.3" } @@ -296,7 +296,7 @@ "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", "Speckle.Sdk.Dependencies": "[1.0.0, )", - "System.Threading.Channels": "[10.0.1, )" + "System.Threading.Channels": "[10.0.0, )" } }, "speckle.sdk.dependencies": { @@ -361,11 +361,11 @@ }, "System.Threading.Channels": { "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==", + "requested": "[10.0.0, )", + "resolved": "10.0.0", + "contentHash": "fwRdkJpKisUEVNaEdsL5w5EwidzuVw0BOTfzDvYB1Yg8sq1pqNfUZxBOVFgSj6i6tNhpT3HP8BEDXf1+kFkTDA==", "dependencies": { - "Microsoft.Bcl.AsyncInterfaces": "10.0.1", + "Microsoft.Bcl.AsyncInterfaces": "10.0.0", "System.Threading.Tasks.Extensions": "4.6.3" } } @@ -565,8 +565,7 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )", - "System.Threading.Channels": "[10.0.1, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )" } }, "speckle.sdk.dependencies": { @@ -622,12 +621,6 @@ "requested": "[13.0.2, )", "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" - }, - "System.Threading.Channels": { - "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" } } } diff --git a/src/Speckle.Objects/packages.lock.json b/src/Speckle.Objects/packages.lock.json index 5042eec7..9ac413aa 100644 --- a/src/Speckle.Objects/packages.lock.json +++ b/src/Speckle.Objects/packages.lock.json @@ -56,8 +56,8 @@ }, "Microsoft.Bcl.AsyncInterfaces": { "type": "Transitive", - "resolved": "10.0.1", - "contentHash": "E1HSLkPHXEO30JEij2pWbOuzz1Z5ND4a5l7IP1T2RgQuE0a0NzEIvtO64RNy3Otn6PFezbT80cfm3M/Cgt70PA==", + "resolved": "10.0.0", + "contentHash": "vFuwSLj9QJBbNR0NeNO4YVASUbokxs+i/xbuu8B+Fs4FAZg5QaFa6eGrMaRqTzzNI5tAb97T7BhSxtLckFyiRA==", "dependencies": { "System.Threading.Tasks.Extensions": "4.6.3" } @@ -243,7 +243,7 @@ "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", "Speckle.Sdk.Dependencies": "[1.0.0, )", - "System.Threading.Channels": "[10.0.1, )" + "System.Threading.Channels": "[10.0.0, )" } }, "speckle.sdk.dependencies": { @@ -308,11 +308,11 @@ }, "System.Threading.Channels": { "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==", + "requested": "[10.0.0, )", + "resolved": "10.0.0", + "contentHash": "fwRdkJpKisUEVNaEdsL5w5EwidzuVw0BOTfzDvYB1Yg8sq1pqNfUZxBOVFgSj6i6tNhpT3HP8BEDXf1+kFkTDA==", "dependencies": { - "Microsoft.Bcl.AsyncInterfaces": "10.0.1", + "Microsoft.Bcl.AsyncInterfaces": "10.0.0", "System.Threading.Tasks.Extensions": "4.6.3" } } @@ -486,8 +486,7 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )", - "System.Threading.Channels": "[10.0.1, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )" } }, "speckle.sdk.dependencies": { @@ -543,12 +542,6 @@ "requested": "[13.0.2, )", "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" - }, - "System.Threading.Channels": { - "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" } } } diff --git a/src/Speckle.Sdk.Dependencies/packages.lock.json b/src/Speckle.Sdk.Dependencies/packages.lock.json index aa13bc43..760480ee 100644 --- a/src/Speckle.Sdk.Dependencies/packages.lock.json +++ b/src/Speckle.Sdk.Dependencies/packages.lock.json @@ -82,11 +82,11 @@ }, "System.Threading.Channels": { "type": "Direct", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==", + "requested": "[10.0.0, )", + "resolved": "10.0.0", + "contentHash": "fwRdkJpKisUEVNaEdsL5w5EwidzuVw0BOTfzDvYB1Yg8sq1pqNfUZxBOVFgSj6i6tNhpT3HP8BEDXf1+kFkTDA==", "dependencies": { - "Microsoft.Bcl.AsyncInterfaces": "10.0.1", + "Microsoft.Bcl.AsyncInterfaces": "10.0.0", "System.Threading.Tasks.Extensions": "4.6.3" } }, @@ -97,8 +97,8 @@ }, "Microsoft.Bcl.AsyncInterfaces": { "type": "Transitive", - "resolved": "10.0.1", - "contentHash": "E1HSLkPHXEO30JEij2pWbOuzz1Z5ND4a5l7IP1T2RgQuE0a0NzEIvtO64RNy3Otn6PFezbT80cfm3M/Cgt70PA==", + "resolved": "10.0.0", + "contentHash": "vFuwSLj9QJBbNR0NeNO4YVASUbokxs+i/xbuu8B+Fs4FAZg5QaFa6eGrMaRqTzzNI5tAb97T7BhSxtLckFyiRA==", "dependencies": { "System.Threading.Tasks.Extensions": "4.6.3" } @@ -228,9 +228,9 @@ }, "System.Threading.Channels": { "type": "Direct", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" + "requested": "[10.0.0, )", + "resolved": "10.0.0", + "contentHash": "fwRdkJpKisUEVNaEdsL5w5EwidzuVw0BOTfzDvYB1Yg8sq1pqNfUZxBOVFgSj6i6tNhpT3HP8BEDXf1+kFkTDA==" }, "ILRepack": { "type": "Transitive", diff --git a/src/Speckle.Sdk/Speckle.Sdk.csproj b/src/Speckle.Sdk/Speckle.Sdk.csproj index 1e9cf643..350b4330 100644 --- a/src/Speckle.Sdk/Speckle.Sdk.csproj +++ b/src/Speckle.Sdk/Speckle.Sdk.csproj @@ -26,7 +26,6 @@ - @@ -36,6 +35,7 @@ + diff --git a/src/Speckle.Sdk/packages.lock.json b/src/Speckle.Sdk/packages.lock.json index 17478611..5702b969 100644 --- a/src/Speckle.Sdk/packages.lock.json +++ b/src/Speckle.Sdk/packages.lock.json @@ -92,11 +92,11 @@ }, "System.Threading.Channels": { "type": "Direct", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==", + "requested": "[10.0.0, )", + "resolved": "10.0.0", + "contentHash": "fwRdkJpKisUEVNaEdsL5w5EwidzuVw0BOTfzDvYB1Yg8sq1pqNfUZxBOVFgSj6i6tNhpT3HP8BEDXf1+kFkTDA==", "dependencies": { - "Microsoft.Bcl.AsyncInterfaces": "10.0.1", + "Microsoft.Bcl.AsyncInterfaces": "10.0.0", "System.Threading.Tasks.Extensions": "4.6.3" } }, @@ -123,8 +123,8 @@ }, "Microsoft.Bcl.AsyncInterfaces": { "type": "Transitive", - "resolved": "10.0.1", - "contentHash": "E1HSLkPHXEO30JEij2pWbOuzz1Z5ND4a5l7IP1T2RgQuE0a0NzEIvtO64RNy3Otn6PFezbT80cfm3M/Cgt70PA==", + "resolved": "10.0.0", + "contentHash": "vFuwSLj9QJBbNR0NeNO4YVASUbokxs+i/xbuu8B+Fs4FAZg5QaFa6eGrMaRqTzzNI5tAb97T7BhSxtLckFyiRA==", "dependencies": { "System.Threading.Tasks.Extensions": "4.6.3" } @@ -377,12 +377,6 @@ "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" }, - "System.Threading.Channels": { - "type": "Direct", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" - }, "GraphQL.Client.Abstractions": { "type": "Transitive", "resolved": "6.0.0", diff --git a/tests/Speckle.Automate.Sdk.Integration/packages.lock.json b/tests/Speckle.Automate.Sdk.Integration/packages.lock.json index d0818532..78c3841b 100644 --- a/tests/Speckle.Automate.Sdk.Integration/packages.lock.json +++ b/tests/Speckle.Automate.Sdk.Integration/packages.lock.json @@ -370,8 +370,7 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )", - "System.Threading.Channels": "[10.0.1, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )" } }, "speckle.sdk.dependencies": { @@ -492,12 +491,6 @@ "System.Text.Encodings.Web": "8.0.0" } }, - "System.Threading.Channels": { - "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" - }, "Verify.Quibble": { "type": "CentralTransitive", "requested": "[2.1.1, )", diff --git a/tests/Speckle.Objects.Tests.Unit/packages.lock.json b/tests/Speckle.Objects.Tests.Unit/packages.lock.json index 3ef2d70b..fe375e71 100644 --- a/tests/Speckle.Objects.Tests.Unit/packages.lock.json +++ b/tests/Speckle.Objects.Tests.Unit/packages.lock.json @@ -348,8 +348,7 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )", - "System.Threading.Channels": "[10.0.1, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )" } }, "speckle.sdk.dependencies": { @@ -433,12 +432,6 @@ "System.Text.Encodings.Web": "8.0.0" } }, - "System.Threading.Channels": { - "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" - }, "Verify.Quibble": { "type": "CentralTransitive", "requested": "[2.1.1, )", diff --git a/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json b/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json index 022252ea..765c1137 100644 --- a/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json +++ b/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json @@ -342,8 +342,7 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )", - "System.Threading.Channels": "[10.0.1, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )" } }, "speckle.sdk.dependencies": { @@ -425,12 +424,6 @@ "requested": "[13.0.2, )", "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" - }, - "System.Threading.Channels": { - "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" } } } diff --git a/tests/Speckle.Sdk.Serialization.Tests/packages.lock.json b/tests/Speckle.Sdk.Serialization.Tests/packages.lock.json index 11691412..2ac574fb 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/packages.lock.json +++ b/tests/Speckle.Sdk.Serialization.Tests/packages.lock.json @@ -393,8 +393,7 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )", - "System.Threading.Channels": "[10.0.1, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )" } }, "speckle.sdk.dependencies": { @@ -475,12 +474,6 @@ "System.Text.Encodings.Web": "8.0.0" } }, - "System.Threading.Channels": { - "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" - }, "Verify.Quibble": { "type": "CentralTransitive", "requested": "[2.1.1, )", diff --git a/tests/Speckle.Sdk.Testing/packages.lock.json b/tests/Speckle.Sdk.Testing/packages.lock.json index 9cb33c89..6e2cfd77 100644 --- a/tests/Speckle.Sdk.Testing/packages.lock.json +++ b/tests/Speckle.Sdk.Testing/packages.lock.json @@ -311,8 +311,7 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )", - "System.Threading.Channels": "[10.0.1, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )" } }, "speckle.sdk.dependencies": { @@ -377,12 +376,6 @@ "dependencies": { "System.Text.Encodings.Web": "8.0.0" } - }, - "System.Threading.Channels": { - "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" } } } diff --git a/tests/Speckle.Sdk.Tests.Integration/packages.lock.json b/tests/Speckle.Sdk.Tests.Integration/packages.lock.json index 59e1a4d2..6138bda1 100644 --- a/tests/Speckle.Sdk.Tests.Integration/packages.lock.json +++ b/tests/Speckle.Sdk.Tests.Integration/packages.lock.json @@ -370,8 +370,7 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )", - "System.Threading.Channels": "[10.0.1, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )" } }, "speckle.sdk.dependencies": { @@ -455,12 +454,6 @@ "System.Text.Encodings.Web": "8.0.0" } }, - "System.Threading.Channels": { - "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" - }, "Verify.Quibble": { "type": "CentralTransitive", "requested": "[2.1.1, )", diff --git a/tests/Speckle.Sdk.Tests.Performance/packages.lock.json b/tests/Speckle.Sdk.Tests.Performance/packages.lock.json index a96f59c2..bff226e4 100644 --- a/tests/Speckle.Sdk.Tests.Performance/packages.lock.json +++ b/tests/Speckle.Sdk.Tests.Performance/packages.lock.json @@ -360,8 +360,7 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )", - "System.Threading.Channels": "[10.0.1, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )" } }, "speckle.sdk.dependencies": { @@ -417,12 +416,6 @@ "requested": "[13.0.2, )", "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" - }, - "System.Threading.Channels": { - "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" } } } diff --git a/tests/Speckle.Sdk.Tests.Unit/packages.lock.json b/tests/Speckle.Sdk.Tests.Unit/packages.lock.json index 4faff1c8..8b50e6f9 100644 --- a/tests/Speckle.Sdk.Tests.Unit/packages.lock.json +++ b/tests/Speckle.Sdk.Tests.Unit/packages.lock.json @@ -363,8 +363,7 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )", - "System.Threading.Channels": "[10.0.1, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )" } }, "speckle.sdk.dependencies": { @@ -442,12 +441,6 @@ "System.Text.Encodings.Web": "8.0.0" } }, - "System.Threading.Channels": { - "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" - }, "Verify.Quibble": { "type": "CentralTransitive", "requested": "[2.1.1, )", From c4e956cdb411c4362d0e8eccbe02e04dcf92b108 Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Thu, 5 Mar 2026 12:47:56 +0000 Subject: [PATCH 12/26] fix(build): Option 2 - "keep channels il-repacked" (#453) * Repacked channel * transitive for sdk * restore solution * comment tweaks * Fix print line --- Directory.Packages.props | 3 +- src/Speckle.Automate.Sdk/packages.lock.json | 45 +++++++---------- src/Speckle.Objects/packages.lock.json | 45 +++++++---------- .../RepackedChannel.cs | 49 +++++++++++++++++++ .../Speckle.Sdk.Dependencies.csproj | 32 ++++++++++++ .../packages.lock.json | 43 ++++++++-------- src/Speckle.Sdk/Pipelines/Send/DiskStore.cs | 15 +++--- src/Speckle.Sdk/Speckle.Sdk.csproj | 1 - src/Speckle.Sdk/packages.lock.json | 42 +++++++--------- .../packages.lock.json | 11 +++-- .../packages.lock.json | 11 +++-- 11 files changed, 180 insertions(+), 117 deletions(-) create mode 100644 src/Speckle.Sdk.Dependencies/RepackedChannel.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index f2f4c427..8c51176a 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -15,6 +15,7 @@ + @@ -27,7 +28,7 @@ - + diff --git a/src/Speckle.Automate.Sdk/packages.lock.json b/src/Speckle.Automate.Sdk/packages.lock.json index dd5d4c56..f7174f7c 100644 --- a/src/Speckle.Automate.Sdk/packages.lock.json +++ b/src/Speckle.Automate.Sdk/packages.lock.json @@ -86,14 +86,6 @@ "resolved": "6.0.0", "contentHash": "yg72rrYDapfsIUrul7aF6wwNnTJBOFvuA9VdDTQpPa8AlAriHbufeXYLBcodKjfUdkCnaiggX1U/nEP08Zb5GA==" }, - "Microsoft.Bcl.AsyncInterfaces": { - "type": "Transitive", - "resolved": "10.0.0", - "contentHash": "vFuwSLj9QJBbNR0NeNO4YVASUbokxs+i/xbuu8B+Fs4FAZg5QaFa6eGrMaRqTzzNI5tAb97T7BhSxtLckFyiRA==", - "dependencies": { - "System.Threading.Tasks.Extensions": "4.6.3" - } - }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", "resolved": "8.0.0", @@ -250,8 +242,8 @@ }, "System.Runtime.CompilerServices.Unsafe": { "type": "Transitive", - "resolved": "6.1.2", - "contentHash": "2hBr6zdbIBTDE3EhK7NSVNdX58uTK6iHW/P/Axmm9sl1xoGSLqDvMtpecn226TNwHByFokYwJmt/aQQNlO5CRw==" + "resolved": "6.0.0", + "contentHash": "/iUeP3tq1S0XdNNoMz5C9twLSrM/TH+qElHkXWaPvuNOt+99G75NrV0OS2EqHx5wMN7popYjpc8oTjC1y16DLg==" }, "System.Runtime.InteropServices.WindowsRuntime": { "type": "Transitive", @@ -273,10 +265,10 @@ }, "System.Threading.Tasks.Extensions": { "type": "Transitive", - "resolved": "4.6.3", - "contentHash": "7sCiwilJLYbTZELaKnc7RecBBXWXA+xMLQWZKWawBxYjp6DBlSE3v9/UcvKBvr1vv2tTOhipiogM8rRmxlhrVA==", + "resolved": "4.5.4", + "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==", "dependencies": { - "System.Runtime.CompilerServices.Unsafe": "6.1.2" + "System.Runtime.CompilerServices.Unsafe": "4.5.3" } }, "speckle.objects": { @@ -295,12 +287,14 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )", - "System.Threading.Channels": "[10.0.0, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )" } }, "speckle.sdk.dependencies": { - "type": "Project" + "type": "Project", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "[9.0.4, )" + } }, "GraphQL.Client": { "type": "CentralTransitive", @@ -313,6 +307,15 @@ "System.Reactive": "5.0.0" } }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "CentralTransitive", + "requested": "[9.0.4, )", + "resolved": "9.0.4", + "contentHash": "9VGI5kxIvrNG2mqLQZnUR6y/3fcnygD8eNpHR+CqfbnIXvea6nehnYknDKQTxZVPMpzpNca+7DxLBmpdB3q0Bw==", + "dependencies": { + "System.Threading.Tasks.Extensions": "4.5.4" + } + }, "Microsoft.CSharp": { "type": "CentralTransitive", "requested": "[4.7.0, )", @@ -358,16 +361,6 @@ "requested": "[13.0.2, )", "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" - }, - "System.Threading.Channels": { - "type": "CentralTransitive", - "requested": "[10.0.0, )", - "resolved": "10.0.0", - "contentHash": "fwRdkJpKisUEVNaEdsL5w5EwidzuVw0BOTfzDvYB1Yg8sq1pqNfUZxBOVFgSj6i6tNhpT3HP8BEDXf1+kFkTDA==", - "dependencies": { - "Microsoft.Bcl.AsyncInterfaces": "10.0.0", - "System.Threading.Tasks.Extensions": "4.6.3" - } } }, "net8.0": { diff --git a/src/Speckle.Objects/packages.lock.json b/src/Speckle.Objects/packages.lock.json index 9ac413aa..1fe1cf2d 100644 --- a/src/Speckle.Objects/packages.lock.json +++ b/src/Speckle.Objects/packages.lock.json @@ -54,14 +54,6 @@ "resolved": "6.0.0", "contentHash": "yg72rrYDapfsIUrul7aF6wwNnTJBOFvuA9VdDTQpPa8AlAriHbufeXYLBcodKjfUdkCnaiggX1U/nEP08Zb5GA==" }, - "Microsoft.Bcl.AsyncInterfaces": { - "type": "Transitive", - "resolved": "10.0.0", - "contentHash": "vFuwSLj9QJBbNR0NeNO4YVASUbokxs+i/xbuu8B+Fs4FAZg5QaFa6eGrMaRqTzzNI5tAb97T7BhSxtLckFyiRA==", - "dependencies": { - "System.Threading.Tasks.Extensions": "4.6.3" - } - }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", "resolved": "8.0.0", @@ -213,8 +205,8 @@ }, "System.Runtime.CompilerServices.Unsafe": { "type": "Transitive", - "resolved": "6.1.2", - "contentHash": "2hBr6zdbIBTDE3EhK7NSVNdX58uTK6iHW/P/Axmm9sl1xoGSLqDvMtpecn226TNwHByFokYwJmt/aQQNlO5CRw==" + "resolved": "4.5.3", + "contentHash": "3TIsJhD1EiiT0w2CcDMN/iSSwnNnsrnbzeVHSKkaEgV85txMprmuO+Yq2AdSbeVGcg28pdNDTPK87tJhX7VFHw==" }, "System.Runtime.InteropServices.WindowsRuntime": { "type": "Transitive", @@ -226,10 +218,10 @@ }, "System.Threading.Tasks.Extensions": { "type": "Transitive", - "resolved": "4.6.3", - "contentHash": "7sCiwilJLYbTZELaKnc7RecBBXWXA+xMLQWZKWawBxYjp6DBlSE3v9/UcvKBvr1vv2tTOhipiogM8rRmxlhrVA==", + "resolved": "4.5.4", + "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==", "dependencies": { - "System.Runtime.CompilerServices.Unsafe": "6.1.2" + "System.Runtime.CompilerServices.Unsafe": "4.5.3" } }, "speckle.sdk": { @@ -242,12 +234,14 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )", - "System.Threading.Channels": "[10.0.0, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )" } }, "speckle.sdk.dependencies": { - "type": "Project" + "type": "Project", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "[9.0.4, )" + } }, "GraphQL.Client": { "type": "CentralTransitive", @@ -260,6 +254,15 @@ "System.Reactive": "5.0.0" } }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "CentralTransitive", + "requested": "[9.0.4, )", + "resolved": "9.0.4", + "contentHash": "9VGI5kxIvrNG2mqLQZnUR6y/3fcnygD8eNpHR+CqfbnIXvea6nehnYknDKQTxZVPMpzpNca+7DxLBmpdB3q0Bw==", + "dependencies": { + "System.Threading.Tasks.Extensions": "4.5.4" + } + }, "Microsoft.CSharp": { "type": "CentralTransitive", "requested": "[4.7.0, )", @@ -305,16 +308,6 @@ "requested": "[13.0.2, )", "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" - }, - "System.Threading.Channels": { - "type": "CentralTransitive", - "requested": "[10.0.0, )", - "resolved": "10.0.0", - "contentHash": "fwRdkJpKisUEVNaEdsL5w5EwidzuVw0BOTfzDvYB1Yg8sq1pqNfUZxBOVFgSj6i6tNhpT3HP8BEDXf1+kFkTDA==", - "dependencies": { - "Microsoft.Bcl.AsyncInterfaces": "10.0.0", - "System.Threading.Tasks.Extensions": "4.6.3" - } } }, "net8.0": { diff --git a/src/Speckle.Sdk.Dependencies/RepackedChannel.cs b/src/Speckle.Sdk.Dependencies/RepackedChannel.cs new file mode 100644 index 00000000..a5f2e73b --- /dev/null +++ b/src/Speckle.Sdk.Dependencies/RepackedChannel.cs @@ -0,0 +1,49 @@ +using System.Threading.Channels; + +namespace Speckle.Sdk.Dependencies; + +/// +/// For various reasons related to our use of ILRepack.FullAuto, +/// we cannot use Channels from the SDK project. +/// We have to keep usage of it inside the Sdk.Dependencies project. +/// +/// For the sake of quick development, I've wrapped the class here in a type +/// that is safe to use from the SDK project. +/// +/// As and when we need more functions, we can add them here. +/// +/// And yes... I'm not very happy about the way we've set this up +/// +/// +public sealed class RepackedChannel +{ + private readonly Channel _channel; + + public RepackedChannel(int capacity, bool singleReader, bool singleWriter) + { + _channel = Channel.CreateBounded( + new BoundedChannelOptions(capacity) + { + FullMode = BoundedChannelFullMode.Wait, + SingleReader = singleReader, + SingleWriter = singleWriter, + } + ); + } + + public void CompleteWriter() => _channel.Writer.Complete(); + + public ValueTask WriteAsync(T item, CancellationToken cancellationToken) => + _channel.Writer.WriteAsync(item, cancellationToken); + + public IAsyncEnumerable ReadAllAsync(CancellationToken cancellationToken) => + _channel.Reader.ReadAllAsync(cancellationToken); + + // public async Task ReadAllAsync(Func callback, CancellationToken cancellationToken) + // { + // await foreach (T item in _channel.Reader.ReadAllAsync(cancellationToken)) + // { + // await callback.Invoke(item).ConfigureAwait(false); + // } + // } +} diff --git a/src/Speckle.Sdk.Dependencies/Speckle.Sdk.Dependencies.csproj b/src/Speckle.Sdk.Dependencies/Speckle.Sdk.Dependencies.csproj index 043b7876..fe91aa91 100644 --- a/src/Speckle.Sdk.Dependencies/Speckle.Sdk.Dependencies.csproj +++ b/src/Speckle.Sdk.Dependencies/Speckle.Sdk.Dependencies.csproj @@ -28,4 +28,36 @@ + + + + + + + <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)System.Numerics.Vectors.dll" /> + <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)System.Runtime.CompilerServices.Unsafe.dll" /> + <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)System.Memory.dll" /> + <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)Open.ChannelExtensions.dll" /> + <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)System.Threading.Channels.dll" /> + <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)System.Collections.Immutable.dll" /> + <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)Polly.dll" /> + <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)Polly.Contrib.WaitAndRetry.dll" /> + <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)Polly.Extensions.Http.dll" /> + <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)Microsoft.Extensions.ObjectPool.dll" /> + <_ILRepackExcludeAssemblies_Items Include="$(OutputPath)*.dll" Exclude="@(_ILRepackIncludeAssemblies_Items)" /> + + + + @(_ILRepackExcludeAssemblies_Items) + + diff --git a/src/Speckle.Sdk.Dependencies/packages.lock.json b/src/Speckle.Sdk.Dependencies/packages.lock.json index 760480ee..ad39f5df 100644 --- a/src/Speckle.Sdk.Dependencies/packages.lock.json +++ b/src/Speckle.Sdk.Dependencies/packages.lock.json @@ -11,6 +11,15 @@ "ILRepack": "2.0.33" } }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "Direct", + "requested": "[9.0.4, )", + "resolved": "9.0.4", + "contentHash": "9VGI5kxIvrNG2mqLQZnUR6y/3fcnygD8eNpHR+CqfbnIXvea6nehnYknDKQTxZVPMpzpNca+7DxLBmpdB3q0Bw==", + "dependencies": { + "System.Threading.Tasks.Extensions": "4.5.4" + } + }, "Microsoft.Extensions.ObjectPool": { "type": "Direct", "requested": "[9.0.4, )", @@ -82,12 +91,12 @@ }, "System.Threading.Channels": { "type": "Direct", - "requested": "[10.0.0, )", - "resolved": "10.0.0", - "contentHash": "fwRdkJpKisUEVNaEdsL5w5EwidzuVw0BOTfzDvYB1Yg8sq1pqNfUZxBOVFgSj6i6tNhpT3HP8BEDXf1+kFkTDA==", + "requested": "[9.0.4, )", + "resolved": "9.0.4", + "contentHash": "4qBn2H6/aXBpE/Pm3wY5yusY/pEvQz99NlWHrTUji0qCmOdbhhjaALcpmbfW2ksxlPM6i6S+QFLkpOQdyfeKYQ==", "dependencies": { - "Microsoft.Bcl.AsyncInterfaces": "10.0.0", - "System.Threading.Tasks.Extensions": "4.6.3" + "Microsoft.Bcl.AsyncInterfaces": "9.0.4", + "System.Threading.Tasks.Extensions": "4.5.4" } }, "ILRepack": { @@ -95,14 +104,6 @@ "resolved": "2.0.33", "contentHash": "xb2h1CsOepoYwdXEPui9VcQglwABQwNf9cccZbf+acarEzF5PUp8Xx71nFXIhOgEdm6wrxAoF6xAxK4m/XFRUQ==" }, - "Microsoft.Bcl.AsyncInterfaces": { - "type": "Transitive", - "resolved": "10.0.0", - "contentHash": "vFuwSLj9QJBbNR0NeNO4YVASUbokxs+i/xbuu8B+Fs4FAZg5QaFa6eGrMaRqTzzNI5tAb97T7BhSxtLckFyiRA==", - "dependencies": { - "System.Threading.Tasks.Extensions": "4.6.3" - } - }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", "resolved": "8.0.0", @@ -149,15 +150,15 @@ }, "System.Runtime.CompilerServices.Unsafe": { "type": "Transitive", - "resolved": "6.1.2", - "contentHash": "2hBr6zdbIBTDE3EhK7NSVNdX58uTK6iHW/P/Axmm9sl1xoGSLqDvMtpecn226TNwHByFokYwJmt/aQQNlO5CRw==" + "resolved": "6.0.0", + "contentHash": "/iUeP3tq1S0XdNNoMz5C9twLSrM/TH+qElHkXWaPvuNOt+99G75NrV0OS2EqHx5wMN7popYjpc8oTjC1y16DLg==" }, "System.Threading.Tasks.Extensions": { "type": "Transitive", - "resolved": "4.6.3", - "contentHash": "7sCiwilJLYbTZELaKnc7RecBBXWXA+xMLQWZKWawBxYjp6DBlSE3v9/UcvKBvr1vv2tTOhipiogM8rRmxlhrVA==", + "resolved": "4.5.4", + "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==", "dependencies": { - "System.Runtime.CompilerServices.Unsafe": "6.1.2" + "System.Runtime.CompilerServices.Unsafe": "4.5.3" } } }, @@ -228,9 +229,9 @@ }, "System.Threading.Channels": { "type": "Direct", - "requested": "[10.0.0, )", - "resolved": "10.0.0", - "contentHash": "fwRdkJpKisUEVNaEdsL5w5EwidzuVw0BOTfzDvYB1Yg8sq1pqNfUZxBOVFgSj6i6tNhpT3HP8BEDXf1+kFkTDA==" + "requested": "[9.0.4, )", + "resolved": "9.0.4", + "contentHash": "4qBn2H6/aXBpE/Pm3wY5yusY/pEvQz99NlWHrTUji0qCmOdbhhjaALcpmbfW2ksxlPM6i6S+QFLkpOQdyfeKYQ==" }, "ILRepack": { "type": "Transitive", diff --git a/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs b/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs index 0ac685fb..140f9ad2 100644 --- a/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs +++ b/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs @@ -1,7 +1,7 @@ using System.IO.Compression; -using System.Threading.Channels; using Microsoft.Extensions.Logging; using Speckle.InterfaceGenerator; +using Speckle.Sdk.Dependencies; using Speckle.Sdk.Helpers; namespace Speckle.Sdk.Pipelines.Send; @@ -14,7 +14,7 @@ public sealed class DiskStoreFactory(ILogger logger) : IDiskStoreFact public sealed class DiskStore { - private readonly Channel _channel; + private readonly RepackedChannel _channel; private readonly Task _writeToDiskTask; private readonly ILogger _logger; private readonly CancellationToken _cancellationToken; @@ -24,17 +24,16 @@ public sealed class DiskStore _logger = logger; _cancellationToken = cancellationToken; - _channel = Channel.CreateBounded( - new BoundedChannelOptions(1000) { FullMode = BoundedChannelFullMode.Wait, SingleReader = true } - ); + _channel = new RepackedChannel(1000, true, false); _writeToDiskTask = Task.Run(WriteFile, cancellationToken); } - public ValueTask PushAsync(UploadItem item) => _channel.Writer.WriteAsync(item, _cancellationToken); + public async Task PushAsync(UploadItem item) => + await _channel.WriteAsync(item, _cancellationToken).ConfigureAwait(false); public async Task CompleteAsync() { - _channel.Writer.Complete(); + _channel.CompleteWriter(); return await _writeToDiskTask.ConfigureAwait(false); } @@ -55,7 +54,7 @@ public sealed class DiskStore using var gzip = new GZipStream(fileStream, CompressionLevel.Optimal); using var writer = new StreamWriter(gzip); - await foreach (var item in _channel.Reader.ReadAllAsync(_cancellationToken).ConfigureAwait(false)) + await foreach (var item in _channel.ReadAllAsync(_cancellationToken).ConfigureAwait(false)) { await writer.WriteLineAsync($"{item.Id}\t{item.Json}\t{item.SpeckleType}").ConfigureAwait(false); } diff --git a/src/Speckle.Sdk/Speckle.Sdk.csproj b/src/Speckle.Sdk/Speckle.Sdk.csproj index 350b4330..000ee2bf 100644 --- a/src/Speckle.Sdk/Speckle.Sdk.csproj +++ b/src/Speckle.Sdk/Speckle.Sdk.csproj @@ -35,7 +35,6 @@ - diff --git a/src/Speckle.Sdk/packages.lock.json b/src/Speckle.Sdk/packages.lock.json index 5702b969..1426afb3 100644 --- a/src/Speckle.Sdk/packages.lock.json +++ b/src/Speckle.Sdk/packages.lock.json @@ -90,16 +90,6 @@ "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" }, - "System.Threading.Channels": { - "type": "Direct", - "requested": "[10.0.0, )", - "resolved": "10.0.0", - "contentHash": "fwRdkJpKisUEVNaEdsL5w5EwidzuVw0BOTfzDvYB1Yg8sq1pqNfUZxBOVFgSj6i6tNhpT3HP8BEDXf1+kFkTDA==", - "dependencies": { - "Microsoft.Bcl.AsyncInterfaces": "10.0.0", - "System.Threading.Tasks.Extensions": "4.6.3" - } - }, "GraphQL.Client.Abstractions": { "type": "Transitive", "resolved": "6.0.0", @@ -121,14 +111,6 @@ "resolved": "6.0.0", "contentHash": "yg72rrYDapfsIUrul7aF6wwNnTJBOFvuA9VdDTQpPa8AlAriHbufeXYLBcodKjfUdkCnaiggX1U/nEP08Zb5GA==" }, - "Microsoft.Bcl.AsyncInterfaces": { - "type": "Transitive", - "resolved": "10.0.0", - "contentHash": "vFuwSLj9QJBbNR0NeNO4YVASUbokxs+i/xbuu8B+Fs4FAZg5QaFa6eGrMaRqTzzNI5tAb97T7BhSxtLckFyiRA==", - "dependencies": { - "System.Threading.Tasks.Extensions": "4.6.3" - } - }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", "resolved": "8.0.0", @@ -280,8 +262,8 @@ }, "System.Runtime.CompilerServices.Unsafe": { "type": "Transitive", - "resolved": "6.1.2", - "contentHash": "2hBr6zdbIBTDE3EhK7NSVNdX58uTK6iHW/P/Axmm9sl1xoGSLqDvMtpecn226TNwHByFokYwJmt/aQQNlO5CRw==" + "resolved": "4.5.3", + "contentHash": "3TIsJhD1EiiT0w2CcDMN/iSSwnNnsrnbzeVHSKkaEgV85txMprmuO+Yq2AdSbeVGcg28pdNDTPK87tJhX7VFHw==" }, "System.Runtime.InteropServices.WindowsRuntime": { "type": "Transitive", @@ -293,14 +275,26 @@ }, "System.Threading.Tasks.Extensions": { "type": "Transitive", - "resolved": "4.6.3", - "contentHash": "7sCiwilJLYbTZELaKnc7RecBBXWXA+xMLQWZKWawBxYjp6DBlSE3v9/UcvKBvr1vv2tTOhipiogM8rRmxlhrVA==", + "resolved": "4.5.4", + "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==", "dependencies": { - "System.Runtime.CompilerServices.Unsafe": "6.1.2" + "System.Runtime.CompilerServices.Unsafe": "4.5.3" } }, "speckle.sdk.dependencies": { - "type": "Project" + "type": "Project", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "[9.0.4, )" + } + }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "CentralTransitive", + "requested": "[9.0.4, )", + "resolved": "9.0.4", + "contentHash": "9VGI5kxIvrNG2mqLQZnUR6y/3fcnygD8eNpHR+CqfbnIXvea6nehnYknDKQTxZVPMpzpNca+7DxLBmpdB3q0Bw==", + "dependencies": { + "System.Threading.Tasks.Extensions": "4.5.4" + } } }, "net8.0": { diff --git a/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json b/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json index 765c1137..efd29581 100644 --- a/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json +++ b/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json @@ -74,11 +74,6 @@ "resolved": "1.17.0", "contentHash": "8x+HCVTl/HHTGpscH3vMBhV8sknN/muZFw9s3TsI8SA6+c43cOTCi2+jE4KsU8pNLbJ++iF2ZFcpcXHXtDglnw==" }, - "Microsoft.Bcl.AsyncInterfaces": { - "type": "Transitive", - "resolved": "1.1.0", - "contentHash": "1Am6l4Vpn3/K32daEqZI+FFr96OlZkgwK2LcT3pZ2zWubR5zTPW3/FkO1Rat9kb7oQOa4rxgl9LJHc5tspCWfg==" - }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", "resolved": "8.0.0", @@ -385,6 +380,12 @@ "System.Reactive": "5.0.0" } }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "CentralTransitive", + "requested": "[9.0.4, )", + "resolved": "1.1.0", + "contentHash": "1Am6l4Vpn3/K32daEqZI+FFr96OlZkgwK2LcT3pZ2zWubR5zTPW3/FkO1Rat9kb7oQOa4rxgl9LJHc5tspCWfg==" + }, "Microsoft.Data.Sqlite": { "type": "CentralTransitive", "requested": "[7.0.5, )", diff --git a/tests/Speckle.Sdk.Tests.Performance/packages.lock.json b/tests/Speckle.Sdk.Tests.Performance/packages.lock.json index bff226e4..23a76189 100644 --- a/tests/Speckle.Sdk.Tests.Performance/packages.lock.json +++ b/tests/Speckle.Sdk.Tests.Performance/packages.lock.json @@ -92,11 +92,6 @@ "resolved": "1.17.0", "contentHash": "8x+HCVTl/HHTGpscH3vMBhV8sknN/muZFw9s3TsI8SA6+c43cOTCi2+jE4KsU8pNLbJ++iF2ZFcpcXHXtDglnw==" }, - "Microsoft.Bcl.AsyncInterfaces": { - "type": "Transitive", - "resolved": "1.1.0", - "contentHash": "1Am6l4Vpn3/K32daEqZI+FFr96OlZkgwK2LcT3pZ2zWubR5zTPW3/FkO1Rat9kb7oQOa4rxgl9LJHc5tspCWfg==" - }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", "resolved": "8.0.0", @@ -377,6 +372,12 @@ "System.Reactive": "5.0.0" } }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "CentralTransitive", + "requested": "[9.0.4, )", + "resolved": "1.1.0", + "contentHash": "1Am6l4Vpn3/K32daEqZI+FFr96OlZkgwK2LcT3pZ2zWubR5zTPW3/FkO1Rat9kb7oQOa4rxgl9LJHc5tspCWfg==" + }, "Microsoft.Data.Sqlite": { "type": "CentralTransitive", "requested": "[7.0.5, )", From f777050c102bc9f6e207142b3965bfc24a489146 Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Thu, 5 Mar 2026 13:40:18 +0000 Subject: [PATCH 13/26] change order of tab delimited json (#454) --- src/Speckle.Sdk/Pipelines/Send/DiskStore.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs b/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs index 140f9ad2..cc808450 100644 --- a/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs +++ b/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs @@ -56,7 +56,7 @@ public sealed class DiskStore await foreach (var item in _channel.ReadAllAsync(_cancellationToken).ConfigureAwait(false)) { - await writer.WriteLineAsync($"{item.Id}\t{item.Json}\t{item.SpeckleType}").ConfigureAwait(false); + await writer.WriteLineAsync($"{item.Id}\t{item.SpeckleType}\t{item.Json}").ConfigureAwait(false); } #if NET8_0_OR_GREATER await writer.FlushAsync(_cancellationToken).ConfigureAwait(false); From abf86eda03392bc08e655e65fbbdc47c0cad3aab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Mar 2026 09:50:34 +0000 Subject: [PATCH 14/26] chore(deps): bump docker/login-action from 3 to 4 (#456) Bumps [docker/login-action](https://github.com/docker/login-action) from 3 to 4. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/v3...v4) --- updated-dependencies: - dependency-name: docker/login-action dependency-version: '4' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/integration-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 3ed1c7b8..865b1143 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -30,7 +30,7 @@ jobs: - name: 🔐 Login to Github Container Registry if: ${{ inputs.use-internal-image }} - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: "ghcr.io" username: ${{ github.actor }} From 515d45528d1b95e45336c151cc9eff303551ccea Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Wed, 11 Mar 2026 14:51:58 +0000 Subject: [PATCH 15/26] feat(otel): traces for new send pipeline (#457) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 😃 extra traces * default --- .../GraphQL/Inputs/ModelIngestionInputs.cs | 3 ++- src/Speckle.Sdk/Pipelines/Send/DiskStore.cs | 15 ++++++++++++--- src/Speckle.Sdk/Pipelines/Send/Uploader.cs | 19 ++++++++++++------- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/src/Speckle.Sdk/Api/GraphQL/Inputs/ModelIngestionInputs.cs b/src/Speckle.Sdk/Api/GraphQL/Inputs/ModelIngestionInputs.cs index 1215d034..87cbde66 100644 --- a/src/Speckle.Sdk/Api/GraphQL/Inputs/ModelIngestionInputs.cs +++ b/src/Speckle.Sdk/Api/GraphQL/Inputs/ModelIngestionInputs.cs @@ -14,7 +14,8 @@ public record ModelIngestionCreateInput( string modelId, string projectId, string progressMessage, - SourceDataInput sourceData + SourceDataInput sourceData, + int? maxIdleTimeoutSeconds = null ); public record ModelIngestionUpdateInput(string ingestionId, string projectId, string progressMessage, double? progress); diff --git a/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs b/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs index cc808450..88443633 100644 --- a/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs +++ b/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs @@ -3,13 +3,15 @@ using Microsoft.Extensions.Logging; using Speckle.InterfaceGenerator; using Speckle.Sdk.Dependencies; using Speckle.Sdk.Helpers; +using Speckle.Sdk.Logging; namespace Speckle.Sdk.Pipelines.Send; [GenerateAutoInterface] -public sealed class DiskStoreFactory(ILogger logger) : IDiskStoreFactory +public sealed class DiskStoreFactory(ILogger logger, ISdkActivityFactory activityFactory) : IDiskStoreFactory { - public DiskStore CreateInstance(CancellationToken cancellationToken) => new(logger, cancellationToken); + public DiskStore CreateInstance(CancellationToken cancellationToken) => + new(logger, activityFactory, cancellationToken); } public sealed class DiskStore @@ -17,11 +19,17 @@ public sealed class DiskStore private readonly RepackedChannel _channel; private readonly Task _writeToDiskTask; private readonly ILogger _logger; + private readonly ISdkActivityFactory _activityFactory; private readonly CancellationToken _cancellationToken; - internal DiskStore(ILogger logger, CancellationToken cancellationToken) + internal DiskStore( + ILogger logger, + ISdkActivityFactory activityFactory, + CancellationToken cancellationToken + ) { _logger = logger; + _activityFactory = activityFactory; _cancellationToken = cancellationToken; _channel = new RepackedChannel(1000, true, false); @@ -33,6 +41,7 @@ public sealed class DiskStore public async Task CompleteAsync() { + using var a = _activityFactory.Start("Waiting for DiskStore to complete"); _channel.CompleteWriter(); return await _writeToDiskTask.ConfigureAwait(false); } diff --git a/src/Speckle.Sdk/Pipelines/Send/Uploader.cs b/src/Speckle.Sdk/Pipelines/Send/Uploader.cs index 1cd4f8f4..0b7f2126 100644 --- a/src/Speckle.Sdk/Pipelines/Send/Uploader.cs +++ b/src/Speckle.Sdk/Pipelines/Send/Uploader.cs @@ -1,16 +1,17 @@ using System.Net.Http.Headers; using System.Text; -using Microsoft.Extensions.Logging; using Speckle.InterfaceGenerator; using Speckle.Newtonsoft.Json; using Speckle.Sdk.Credentials; using Speckle.Sdk.Helpers; +using Speckle.Sdk.Logging; using Speckle.Sdk.Pipelines.Progress; namespace Speckle.Sdk.Pipelines.Send; [GenerateAutoInterface] -public sealed class UploaderFactory(ISpeckleHttp httpClientFactory, ILogger logger) : IUploaderFactory +public sealed class UploaderFactory(ISpeckleHttp httpClientFactory, ISdkActivityFactory activityFactory) + : IUploaderFactory { public Uploader CreateInstance( string projectId, @@ -18,7 +19,7 @@ public sealed class UploaderFactory(ISpeckleHttp httpClientFactory, ILogger progress, CancellationToken cancellationToken - ) => new(projectId, ingestionId, logger, httpClientFactory, account, progress, cancellationToken); + ) => new(projectId, ingestionId, activityFactory, httpClientFactory, account, progress, cancellationToken); } public sealed class Uploader : IDisposable @@ -28,13 +29,13 @@ public sealed class Uploader : IDisposable private readonly CancellationToken _cancellationToken; private readonly HttpClient _speckleClient; private readonly HttpClient _s3Client; - private readonly ILogger _logger; + private readonly ISdkActivityFactory _activity; private readonly IProgress _progress; internal Uploader( string projectId, string ingestionId, - ILogger logger, + ISdkActivityFactory activity, ISpeckleHttp httpClientFactory, Account speckleAccount, IProgress progress, @@ -43,7 +44,7 @@ public sealed class Uploader : IDisposable { _projectId = projectId; _ingestionId = ingestionId; - _logger = logger; + _activity = activity; _cancellationToken = cancellationToken; _progress = progress; _speckleClient = httpClientFactory.CreateHttpClient(authorizationToken: speckleAccount.token); @@ -62,6 +63,8 @@ public sealed class Uploader : IDisposable private async Task GetPresignedUrl() { + using var a = _activity.Start("Get Presigned Url"); + var signUri = new Uri($"projects/{_projectId}/modelingestion/{_ingestionId}/uploads/sign", UriKind.Relative); using var signResponse = await _speckleClient.PostAsync(signUri, null, _cancellationToken).ConfigureAwait(false); @@ -80,7 +83,7 @@ public sealed class Uploader : IDisposable private async Task UploadToS3(Stream fileStream, PresignedUploadResponse presignedUploadResponse) { - _logger.LogInformation("Uploading file to pre-signed url"); + using var a = _activity.Start("Uploading file to pre-signed url"); Stream progressStream = new ProgressStream(fileStream, _progress); @@ -107,6 +110,8 @@ public sealed class Uploader : IDisposable private async Task TriggerProcessing(TriggerUploadRequest request) { + using var a = _activity.Start("Triggering Processing"); + Uri processUri = new($"projects/{_projectId}/modelingestion/{_ingestionId}/uploads/process", UriKind.Relative); string requestBody = JsonConvert.SerializeObject(request); using var content = new StringContent(requestBody, Encoding.UTF8, "application/json"); From 906ff9c3ff094f49a937c89ca292f32648a866e9 Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Wed, 11 Mar 2026 16:30:24 +0000 Subject: [PATCH 16/26] test: duck tests (#455) * lets get some testing action! * revert url * we'll try these fixes * we'll try this, if it doesn't work I'm deleting the test * try this * Add optional parent id to traces * how about now * and as zero * Fix build * fix code cov * FML subscription tests are so flaky --- .github/workflows/pr.yml | 6 +- .github/workflows/release.yml | 7 -- .../ISdkActivityFactory.cs | 6 +- .../Logging/NullActivityFactory.cs | 2 +- .../Progress/IngestionProgressManager.cs | 22 +++--- .../IngestionProgressManagerFactory.cs | 3 +- .../Progress/RenderedStreamProgress.cs | 4 +- .../Pipelines/Send/SendPipeline.cs | 6 -- .../ServerObjectManagerTests.cs | 8 +- .../Resources/SubscriptionResourceTests.cs | 2 +- .../Progress/IngestionProgressManagerTests.cs | 76 +++++++++++++++++++ .../Speckle.Sdk.Tests.Integration.csproj | 3 + .../Progress/AggregateProgressTests.cs | 21 +++++ .../Pipelines/Progress/ProgressStreamTests.cs | 72 ++++++++++++++++++ .../Progress/RenderedStreamProgressTests.cs | 46 +++++++++++ 15 files changed, 248 insertions(+), 36 deletions(-) create mode 100644 tests/Speckle.Sdk.Tests.Integration/Pipelines/Progress/IngestionProgressManagerTests.cs create mode 100644 tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/AggregateProgressTests.cs create mode 100644 tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/ProgressStreamTests.cs create mode 100644 tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/RenderedStreamProgressTests.cs diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 8068e7c2..637c7da8 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1,7 +1,11 @@ name: PR Test on: - pull_request: + pull_request: {} + push: + branches: + - "main" # Need to run for codecov to compare against the BASE + jobs: build: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 669d0541..9f47df08 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -46,13 +46,6 @@ jobs: SEMVER: ${{ steps.set-version.outputs.SEMVER }} FILE_VERSION: ${{ steps.set-version.outputs.FILE_VERSION }} - - name: Upload coverage reports to Codecov with GitHub Action - uses: codecov/codecov-action@v5 - continue-on-error: true - with: - fail_ci_if_error: true - files: tests/**/coverage.xml - token: ${{ secrets.CODECOV_TOKEN }} - name: NuGet login (OIDC → temp API key) uses: NuGet/login@v1 diff --git a/src/Speckle.Sdk.Dependencies/ISdkActivityFactory.cs b/src/Speckle.Sdk.Dependencies/ISdkActivityFactory.cs index 6db37c9a..2ed61881 100644 --- a/src/Speckle.Sdk.Dependencies/ISdkActivityFactory.cs +++ b/src/Speckle.Sdk.Dependencies/ISdkActivityFactory.cs @@ -4,5 +4,9 @@ namespace Speckle.Sdk.Logging; public interface ISdkActivityFactory : IDisposable { - ISdkActivity? Start(string? name = default, [CallerMemberName] string source = ""); + /// + /// + /// Only need to set if the parent is coming from an external source (e.g.to trace between client and server) + /// + ISdkActivity? Start(string? name = default, [CallerMemberName] string source = "", string? parentId = null); } diff --git a/src/Speckle.Sdk/Logging/NullActivityFactory.cs b/src/Speckle.Sdk/Logging/NullActivityFactory.cs index 580cc5ef..a8aeb098 100644 --- a/src/Speckle.Sdk/Logging/NullActivityFactory.cs +++ b/src/Speckle.Sdk/Logging/NullActivityFactory.cs @@ -4,5 +4,5 @@ public sealed class NullActivityFactory : ISdkActivityFactory { public void Dispose() { } - public ISdkActivity? Start(string? name = default, string source = "") => null; + public ISdkActivity? Start(string? name = default, string source = "", string? parentId = null) => null; } diff --git a/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManager.cs b/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManager.cs index be141c65..8e146ca4 100644 --- a/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManager.cs +++ b/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManager.cs @@ -14,20 +14,20 @@ public partial interface IIngestionProgressManager : IProgress; /// An implementation for the entire client side Ingestion progress update reporting /// Will throttles ingestion progress messages and reports their progress /// +/// +/// Normally we would pick quite a coarse updateInterval to try and spamming the server (1-5s) +/// [GenerateAutoInterface] public sealed class IngestionProgressManager( ILogger logger, IClient speckleClient, ModelIngestion ingestion, - string projectId, TimeSpan updateInterval, CancellationToken cancellationToken ) : IIngestionProgressManager { - /// - /// Normally we would pick quite a coarse throttle window to try and avoid over pressure (1-5s) - /// - private Task? _lastUpdate; + public Task? LastUpdate { get; private set; } + private long _lastUpdatedAt; private readonly object _lock = new(); @@ -48,15 +48,15 @@ public sealed class IngestionProgressManager( trimmedMessage = value.Status.TrimEnd('.'); - _lastUpdate = speckleClient + LastUpdate = speckleClient .Ingestion.UpdateProgress( - new ModelIngestionUpdateInput(ingestion.id, projectId, trimmedMessage, value.Progress), + new ModelIngestionUpdateInput(ingestion.id, ingestion.projectId, trimmedMessage, value.Progress), cancellationToken ) .ContinueWith( - HandleFaultedContinuation, + Continuation, CancellationToken.None, - TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously, + TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default ); } @@ -67,7 +67,7 @@ public sealed class IngestionProgressManager( /// if the update should be ignored, otherwise private bool ShouldIgnoreProgressUpdate() { - if (_lastUpdate is not null && !_lastUpdate.IsCompleted) + if (LastUpdate is not null && !LastUpdate.IsCompleted) { return true; } @@ -76,7 +76,7 @@ public sealed class IngestionProgressManager( return msSinceLastUpdate < updateInterval; } - private void HandleFaultedContinuation(Task updateTask) + private void Continuation(Task updateTask) { // The progress report failed... could be many reasons. // For now, we're not letting this fail the Ingestion in any way diff --git a/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManagerFactory.cs b/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManagerFactory.cs index 998431c2..522aeff0 100644 --- a/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManagerFactory.cs +++ b/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManagerFactory.cs @@ -12,11 +12,10 @@ public sealed class IngestionProgressManagerFactory(ILogger progress) : I ); } - private static readonly string[] s_suffixes = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"]; + private static readonly string[] s_suffixes = ["B", "KB", "MB", "GB", "TB", "PB"]; - private static (string suffix, double scaleFactor) GetFileSizeRendering(long value) + internal static (string suffix, double scaleFactor) GetFileSizeRendering(long value) { if (value <= 0) { diff --git a/src/Speckle.Sdk/Pipelines/Send/SendPipeline.cs b/src/Speckle.Sdk/Pipelines/Send/SendPipeline.cs index e00439ff..3ddd4249 100644 --- a/src/Speckle.Sdk/Pipelines/Send/SendPipeline.cs +++ b/src/Speckle.Sdk/Pipelines/Send/SendPipeline.cs @@ -70,11 +70,5 @@ public sealed class SendPipeline : IDisposable await _uploader.Send(fileStreamUpload).ConfigureAwait(false); } - public async Task WaitForUploadAndServerProcessing() - { - // TODO: in some way, wait for the server to process the upload and return the actual new version id - return await Task.FromResult("todo").ConfigureAwait(false); - } - public void Dispose() => _uploader.Dispose(); } diff --git a/tests/Speckle.Sdk.Serialization.Tests/ServerObjectManagerTests.cs b/tests/Speckle.Sdk.Serialization.Tests/ServerObjectManagerTests.cs index c81e734e..1ba692c9 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/ServerObjectManagerTests.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/ServerObjectManagerTests.cs @@ -52,7 +52,7 @@ public class ServerObjectManagerTests : MoqTest http.Setup(x => x.CreateHttpClient(It.IsAny(), timeout, token)).Returns(httpClient); var activityFactory = Create(); - activityFactory.Setup(x => x.Start(null, "DownloadObjects")).Returns((ISdkActivity?)null); + activityFactory.Setup(x => x.Start(null, "DownloadObjects", null)).Returns((ISdkActivity?)null); var serverObjectManager = new ServerObjectManager( http.Object, @@ -91,7 +91,7 @@ public class ServerObjectManagerTests : MoqTest http.Setup(x => x.CreateHttpClient(It.IsAny(), timeout, token)).Returns(httpClient); var activityFactory = Create(); - activityFactory.Setup(x => x.Start(null, "DownloadSingleObject")).Returns((ISdkActivity?)null); + activityFactory.Setup(x => x.Start(null, "DownloadSingleObject", null)).Returns((ISdkActivity?)null); var serverObjectManager = new ServerObjectManager( http.Object, @@ -132,7 +132,7 @@ public class ServerObjectManagerTests : MoqTest http.Setup(x => x.CreateHttpClient(It.IsAny(), timeout, token)).Returns(httpClient); var activityFactory = Create(); - activityFactory.Setup(x => x.Start(null, "HasObjects")).Returns((ISdkActivity?)null); + activityFactory.Setup(x => x.Start(null, "HasObjects", null)).Returns((ISdkActivity?)null); var serverObjectManager = new ServerObjectManager( http.Object, @@ -171,7 +171,7 @@ public class ServerObjectManagerTests : MoqTest http.Setup(x => x.CreateHttpClient(It.IsAny(), timeout, token)).Returns(httpClient); var activityFactory = Create(); - activityFactory.Setup(x => x.Start(null, "UploadObjects")).Returns((ISdkActivity?)null); + activityFactory.Setup(x => x.Start(null, "UploadObjects", null)).Returns((ISdkActivity?)null); var serverObjectManager = new ServerObjectManager( http.Object, diff --git a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/SubscriptionResourceTests.cs b/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/SubscriptionResourceTests.cs index dff86650..5ce49cb8 100644 --- a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/SubscriptionResourceTests.cs +++ b/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/SubscriptionResourceTests.cs @@ -13,7 +13,7 @@ public class SubscriptionResourceTests : IAsyncLifetime #if DEBUG private const int WAIT_PERIOD = 3000; // WSL is slow AF, so for local runs, we're being extra generous #else - private const int WAIT_PERIOD = 400; // For CI runs, a much smaller wait time is acceptable + private const int WAIT_PERIOD = 600; // For CI runs, a much smaller wait time is acceptable #endif private const int TIMEOUT = WAIT_PERIOD + WAIT_PERIOD + 600; private IClient _testUser; diff --git a/tests/Speckle.Sdk.Tests.Integration/Pipelines/Progress/IngestionProgressManagerTests.cs b/tests/Speckle.Sdk.Tests.Integration/Pipelines/Progress/IngestionProgressManagerTests.cs new file mode 100644 index 00000000..5ca8c8b5 --- /dev/null +++ b/tests/Speckle.Sdk.Tests.Integration/Pipelines/Progress/IngestionProgressManagerTests.cs @@ -0,0 +1,76 @@ +using Microsoft.Extensions.DependencyInjection; +using Speckle.Sdk.Api; +using Speckle.Sdk.Api.GraphQL.Models; +using Speckle.Sdk.Common; +using Speckle.Sdk.Pipelines.Progress; + +namespace Speckle.Sdk.Tests.Integration.Pipelines.Progress; + +[Trait("Server", "Internal")] +public class IngestionProgressManagerTests : IAsyncLifetime +{ + private IIngestionProgressManagerFactory _factory; + private IClient _client; + private Project _project; + private Model _model; + private ModelIngestion _ingestion; + + public async Task InitializeAsync() + { + var serviceProvider = TestServiceSetup.GetServiceProvider(); + _factory = serviceProvider.GetRequiredService(); + + _client = await Fixtures.SeedUserWithClient(); + _project = await _client.Project.Create(new("test", null, default)); + _model = await _client.Model.Create(new("test", null, _project.id)); + _ingestion = await _client.Ingestion.Create( + new(_model.id, _project.id, "Testing ingestion", new("integrationTests", "0.0.0", null, null)) + ); + } + + [Fact] + public async Task TestProgress_NoThrottle() + { + var sut = _factory.CreateInstance(_client, _ingestion, TimeSpan.Zero, CancellationToken.None); + const string FIRST_MESSAGE = "This is a test 123"; + const string SECOND_MESSAGE = "This is another test 321"; + + // first message (should go through) + sut.Report(new CardProgress(FIRST_MESSAGE, 0.123123123d)); + await sut.LastUpdate.NotNull(); + var res = await _client.Ingestion.Get(_ingestion.id, _project.id, CancellationToken.None); + + Assert.Equal(FIRST_MESSAGE, res.statusData.progressMessage); + + // second message (should also go through) + sut.Report(new CardProgress(SECOND_MESSAGE, 0.321321321d)); + await sut.LastUpdate.NotNull(); + res = await _client.Ingestion.Get(_ingestion.id, _project.id, CancellationToken.None); + + Assert.Equal(SECOND_MESSAGE, res.statusData.progressMessage); + } + + [Fact] + public async Task TestProgress_WithThrottle() + { + var sut = _factory.CreateInstance(_client, _ingestion, TimeSpan.FromMilliseconds(500), CancellationToken.None); + const string EXPECTED_MESSAGE = "First message should go through 123"; + + await Task.Delay(TimeSpan.FromMilliseconds(600)); + + // first message (should go through) + sut.Report(new CardProgress(EXPECTED_MESSAGE, 0.123123123d)); + // second message (should be dropped) + sut.Report(new CardProgress("Second message, should be dropped", 0.321321321d)); + await sut.LastUpdate.NotNull(); + var res = await _client.Ingestion.Get(_ingestion.id, _project.id, CancellationToken.None); + + Assert.Equal(EXPECTED_MESSAGE, res.statusData.progressMessage); + } + + public Task DisposeAsync() + { + _client.Dispose(); + return Task.CompletedTask; + } +} diff --git a/tests/Speckle.Sdk.Tests.Integration/Speckle.Sdk.Tests.Integration.csproj b/tests/Speckle.Sdk.Tests.Integration/Speckle.Sdk.Tests.Integration.csproj index 778355be..dfd7fa55 100644 --- a/tests/Speckle.Sdk.Tests.Integration/Speckle.Sdk.Tests.Integration.csproj +++ b/tests/Speckle.Sdk.Tests.Integration/Speckle.Sdk.Tests.Integration.csproj @@ -16,4 +16,7 @@ + + + diff --git a/tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/AggregateProgressTests.cs b/tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/AggregateProgressTests.cs new file mode 100644 index 00000000..50b0b878 --- /dev/null +++ b/tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/AggregateProgressTests.cs @@ -0,0 +1,21 @@ +using Moq; +using Speckle.Sdk.Pipelines.Progress; + +namespace Speckle.Sdk.Tests.Unit.Pipelines.Progress; + +public class AggregateProgressTests +{ + [Fact] + public void Report_InvokesReportOnAllInnerProgresses() + { + var mock1 = new Mock>(); + var mock2 = new Mock>(); + const int TEST_VALUE = 42; + var target = new AggregateProgress(mock1.Object, mock2.Object); + + target.Report(TEST_VALUE); + + mock1.Verify(x => x.Report(TEST_VALUE), Times.Once); + mock2.Verify(x => x.Report(TEST_VALUE), Times.Once); + } +} diff --git a/tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/ProgressStreamTests.cs b/tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/ProgressStreamTests.cs new file mode 100644 index 00000000..8563341c --- /dev/null +++ b/tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/ProgressStreamTests.cs @@ -0,0 +1,72 @@ +using System.Diagnostics.CodeAnalysis; +using Moq; +using Speckle.Sdk.Pipelines.Progress; + +namespace Speckle.Sdk.Tests.Unit.Pipelines.Progress; + +[SuppressMessage( + "Performance", + "CA1835:Prefer the \'Memory\'-based overloads for \'ReadAsync\' and \'WriteAsync\'", + Justification = "Need to test it" +)] +public class ProgressStreamTests : IDisposable +{ + private readonly Mock _innerStreamMock; + private readonly Mock> _progressMock; + private readonly ProgressStream _sut; + + public ProgressStreamTests() + { + // Setup the mocks + _innerStreamMock = new Mock(); + _innerStreamMock.Setup(s => s.Length).Returns(1024L); + + _progressMock = new Mock>(); + + // Inject mocks into the System Under Test + _sut = new ProgressStream(_innerStreamMock.Object, _progressMock.Object); + } + + [Fact] + public async Task ReadAsync_Should_CallInnerStreamAndReportProgress() + { + // Arrange + var buffer = new byte[10]; + _innerStreamMock + .Setup(s => s.ReadAsync(buffer, 0, buffer.Length, CancellationToken.None)) + .Returns(Task.FromResult(5)); + + // Act + await _sut.ReadAsync(buffer, 0, buffer.Length); + + // Assert - Inner Stream Read was called + _innerStreamMock.Verify(s => s.ReadAsync(buffer, 0, buffer.Length, CancellationToken.None), Times.Once); + + // Assert - Progress Report was called with the correct byte count + _progressMock.Verify(p => p.Report(It.IsAny()), Times.Once); + } + + [Fact] + public async Task WriteAsync_Should_CallInnerStreamAndReportProgress() + { + // Arrange + var buffer = new byte[10]; + _innerStreamMock + .Setup(s => s.WriteAsync(buffer, 0, buffer.Length, CancellationToken.None)) + .Returns(Task.FromResult(5)); + + // Act + await _sut.WriteAsync(buffer, 0, buffer.Length); + + // Assert - Inner Stream Write was called + _innerStreamMock.Verify(s => s.WriteAsync(buffer, 0, buffer.Length, CancellationToken.None), Times.Once); + + // Assert - Progress Report was called with the correct byte count + _progressMock.Verify(p => p.Report(It.IsAny()), Times.Once); + } + + public void Dispose() + { + _sut.Dispose(); + } +} diff --git a/tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/RenderedStreamProgressTests.cs b/tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/RenderedStreamProgressTests.cs new file mode 100644 index 00000000..33007f6c --- /dev/null +++ b/tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/RenderedStreamProgressTests.cs @@ -0,0 +1,46 @@ +using Speckle.Sdk.Pipelines.Progress; + +namespace Speckle.Sdk.Tests.Unit.Pipelines.Progress; + +public class RenderedStreamProgressTests +{ + [Theory] + [InlineData(1, "B", 1.0)] + [InlineData(1024, "B", 1.0)] + [InlineData(1024 + 1, "KB", 1.0 / 1024)] + [InlineData(1024 * 1024, "KB", 1.0 / 1024)] + [InlineData(1024 * 1024 + 1, "MB", 1.0 / (1024 * 1024))] + [InlineData(1024 * 1024 * 1024, "MB", 1.0 / (1024 * 1024))] + [InlineData(1024 * 1024 * 1024 + 1, "GB", 1.0 / (1024 * 1024 * 1024))] + [InlineData(1024L * 1024L * 1024L * 1024L, "GB", 1.0 / (1024L * 1024L * 1024L))] + public void GetFileSizeRendering_WithPositiveValue_ReturnsCorrectSuffix( + long value, + string expectedSuffix, + double expectedScaleFactor + ) + { + var result = RenderedStreamProgress.GetFileSizeRendering(value); + + Assert.Equal(expectedSuffix, result.suffix); + Assert.Equal(expectedScaleFactor, result.scaleFactor); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(-1000)] + public void GetFileSizeRendering_WithNonPositiveValue_ReturnsBytesSuffix(long value) + { + var result = RenderedStreamProgress.GetFileSizeRendering(value); + + Assert.Equal("B", result.suffix); + Assert.Equal(1d, result.scaleFactor); + } + + [Theory] + [InlineData(long.MaxValue)] + public void GetFileSizeRendering_WithVeryLargeValue_ThrowsArgumentOutOfRangeException(long value) + { + Assert.Throws(() => RenderedStreamProgress.GetFileSizeRendering(value)); + } +} From a2c99a537a43184020fde52ccfd46da71399e8fc Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Thu, 12 Mar 2026 12:52:10 +0000 Subject: [PATCH 17/26] feat(otel): Add proper support for remote spans (#458) * Add proper support for remote spans * Fix tests --- src/Speckle.Sdk.Dependencies/ISdkActivity.cs | 1 + src/Speckle.Sdk.Dependencies/ISdkActivityFactory.cs | 9 +++++++-- src/Speckle.Sdk/Logging/NullActivityFactory.cs | 5 ++++- .../ServerObjectManagerTests.cs | 8 ++++---- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/Speckle.Sdk.Dependencies/ISdkActivity.cs b/src/Speckle.Sdk.Dependencies/ISdkActivity.cs index d832c769..0b3d1cea 100644 --- a/src/Speckle.Sdk.Dependencies/ISdkActivity.cs +++ b/src/Speckle.Sdk.Dependencies/ISdkActivity.cs @@ -5,6 +5,7 @@ public interface ISdkActivity : IDisposable void SetTag(string key, object? value); void RecordException(Exception e); string TraceId { get; } + string SpanId { get; } void SetStatus(SdkActivityStatusCode code); void InjectHeaders(Action header); diff --git a/src/Speckle.Sdk.Dependencies/ISdkActivityFactory.cs b/src/Speckle.Sdk.Dependencies/ISdkActivityFactory.cs index 2ed61881..ce69c387 100644 --- a/src/Speckle.Sdk.Dependencies/ISdkActivityFactory.cs +++ b/src/Speckle.Sdk.Dependencies/ISdkActivityFactory.cs @@ -6,7 +6,12 @@ public interface ISdkActivityFactory : IDisposable { /// /// - /// Only need to set if the parent is coming from an external source (e.g.to trace between client and server) /// - ISdkActivity? Start(string? name = default, [CallerMemberName] string source = "", string? parentId = null); + ISdkActivity? Start(string? name = default, [CallerMemberName] string source = ""); + ISdkActivity? StartRemote( + string traceId, + string parentSpanId, + string? name = default, + [CallerMemberName] string source = "" + ); } diff --git a/src/Speckle.Sdk/Logging/NullActivityFactory.cs b/src/Speckle.Sdk/Logging/NullActivityFactory.cs index a8aeb098..e761a9bd 100644 --- a/src/Speckle.Sdk/Logging/NullActivityFactory.cs +++ b/src/Speckle.Sdk/Logging/NullActivityFactory.cs @@ -4,5 +4,8 @@ public sealed class NullActivityFactory : ISdkActivityFactory { public void Dispose() { } - public ISdkActivity? Start(string? name = default, string source = "", string? parentId = null) => null; + public ISdkActivity? Start(string? name = default, string source = "") => null; + + public ISdkActivity? StartRemote(string traceId, string parentSpanId, string? name = default, string source = "") => + null; } diff --git a/tests/Speckle.Sdk.Serialization.Tests/ServerObjectManagerTests.cs b/tests/Speckle.Sdk.Serialization.Tests/ServerObjectManagerTests.cs index 1ba692c9..c81e734e 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/ServerObjectManagerTests.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/ServerObjectManagerTests.cs @@ -52,7 +52,7 @@ public class ServerObjectManagerTests : MoqTest http.Setup(x => x.CreateHttpClient(It.IsAny(), timeout, token)).Returns(httpClient); var activityFactory = Create(); - activityFactory.Setup(x => x.Start(null, "DownloadObjects", null)).Returns((ISdkActivity?)null); + activityFactory.Setup(x => x.Start(null, "DownloadObjects")).Returns((ISdkActivity?)null); var serverObjectManager = new ServerObjectManager( http.Object, @@ -91,7 +91,7 @@ public class ServerObjectManagerTests : MoqTest http.Setup(x => x.CreateHttpClient(It.IsAny(), timeout, token)).Returns(httpClient); var activityFactory = Create(); - activityFactory.Setup(x => x.Start(null, "DownloadSingleObject", null)).Returns((ISdkActivity?)null); + activityFactory.Setup(x => x.Start(null, "DownloadSingleObject")).Returns((ISdkActivity?)null); var serverObjectManager = new ServerObjectManager( http.Object, @@ -132,7 +132,7 @@ public class ServerObjectManagerTests : MoqTest http.Setup(x => x.CreateHttpClient(It.IsAny(), timeout, token)).Returns(httpClient); var activityFactory = Create(); - activityFactory.Setup(x => x.Start(null, "HasObjects", null)).Returns((ISdkActivity?)null); + activityFactory.Setup(x => x.Start(null, "HasObjects")).Returns((ISdkActivity?)null); var serverObjectManager = new ServerObjectManager( http.Object, @@ -171,7 +171,7 @@ public class ServerObjectManagerTests : MoqTest http.Setup(x => x.CreateHttpClient(It.IsAny(), timeout, token)).Returns(httpClient); var activityFactory = Create(); - activityFactory.Setup(x => x.Start(null, "UploadObjects", null)).Returns((ISdkActivity?)null); + activityFactory.Setup(x => x.Start(null, "UploadObjects")).Returns((ISdkActivity?)null); var serverObjectManager = new ServerObjectManager( http.Object, From 70acc06f37ab9fe6d7e51075521cfc85432fecf0 Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Mon, 16 Mar 2026 15:32:33 +0000 Subject: [PATCH 18/26] feat(otel): Change sig for remote (#460) * Remote changes * Fix tests * Fix tests * ditto --- .../ISdkActivityFactory.cs | 17 ++++++----- .../SpeckleActivityKind.cs | 30 +++++++++++++++++++ .../Logging/NullActivityFactory.cs | 9 +++--- .../ServerObjectManagerTests.cs | 8 ++--- ...esourceTests.TestGetProjects.verified.json | 8 ----- ...sourceTests.TestGetWorkspace.verified.json | 8 ----- .../Resources/WorkspaceResourceTests.cs | 6 ++-- 7 files changed, 53 insertions(+), 33 deletions(-) create mode 100644 src/Speckle.Sdk.Dependencies/SpeckleActivityKind.cs delete mode 100644 tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.TestGetProjects.verified.json delete mode 100644 tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.TestGetWorkspace.verified.json diff --git a/src/Speckle.Sdk.Dependencies/ISdkActivityFactory.cs b/src/Speckle.Sdk.Dependencies/ISdkActivityFactory.cs index ce69c387..927d0f70 100644 --- a/src/Speckle.Sdk.Dependencies/ISdkActivityFactory.cs +++ b/src/Speckle.Sdk.Dependencies/ISdkActivityFactory.cs @@ -1,17 +1,20 @@ using System.Runtime.CompilerServices; +using Speckle.Connectors.Logging; namespace Speckle.Sdk.Logging; public interface ISdkActivityFactory : IDisposable { - /// - /// - /// - ISdkActivity? Start(string? name = default, [CallerMemberName] string source = ""); + ISdkActivity? Start( + string? name = null, + SdkActivityKind kind = SdkActivityKind.Internal, + [CallerMemberName] string source = "" + ); + ISdkActivity? StartRemote( - string traceId, - string parentSpanId, - string? name = default, + string traceContext, + SdkActivityKind kind, + string? name = null, [CallerMemberName] string source = "" ); } diff --git a/src/Speckle.Sdk.Dependencies/SpeckleActivityKind.cs b/src/Speckle.Sdk.Dependencies/SpeckleActivityKind.cs new file mode 100644 index 00000000..8ca93543 --- /dev/null +++ b/src/Speckle.Sdk.Dependencies/SpeckleActivityKind.cs @@ -0,0 +1,30 @@ +namespace Speckle.Connectors.Logging; + +public enum SdkActivityKind +{ + /// + /// Default value. + /// Indicates that the Activity represents an internal operation within an application, as opposed to an operations with remote parents or children. + /// + Internal = 0, + + /// + /// Server activity represents request incoming from external component. + /// + Server = 1, + + /// + /// Client activity represents outgoing request to the external component. + /// + Client = 2, + + /// + /// Producer activity represents output provided to external components. + /// + Producer = 3, + + /// + /// Consumer activity represents output received from an external component. + /// + Consumer = 4, +} diff --git a/src/Speckle.Sdk/Logging/NullActivityFactory.cs b/src/Speckle.Sdk/Logging/NullActivityFactory.cs index e761a9bd..039f91b8 100644 --- a/src/Speckle.Sdk/Logging/NullActivityFactory.cs +++ b/src/Speckle.Sdk/Logging/NullActivityFactory.cs @@ -1,11 +1,12 @@ -namespace Speckle.Sdk.Logging; +using Speckle.Connectors.Logging; + +namespace Speckle.Sdk.Logging; public sealed class NullActivityFactory : ISdkActivityFactory { public void Dispose() { } - public ISdkActivity? Start(string? name = default, string source = "") => null; + public ISdkActivity? Start(string? name, SdkActivityKind kind, string source) => null; - public ISdkActivity? StartRemote(string traceId, string parentSpanId, string? name = default, string source = "") => - null; + public ISdkActivity? StartRemote(string traceContext, SdkActivityKind kind, string? name, string source) => null; } diff --git a/tests/Speckle.Sdk.Serialization.Tests/ServerObjectManagerTests.cs b/tests/Speckle.Sdk.Serialization.Tests/ServerObjectManagerTests.cs index c81e734e..90455343 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/ServerObjectManagerTests.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/ServerObjectManagerTests.cs @@ -52,7 +52,7 @@ public class ServerObjectManagerTests : MoqTest http.Setup(x => x.CreateHttpClient(It.IsAny(), timeout, token)).Returns(httpClient); var activityFactory = Create(); - activityFactory.Setup(x => x.Start(null, "DownloadObjects")).Returns((ISdkActivity?)null); + activityFactory.Setup(x => x.Start(null, default, "DownloadObjects")).Returns((ISdkActivity?)null); var serverObjectManager = new ServerObjectManager( http.Object, @@ -91,7 +91,7 @@ public class ServerObjectManagerTests : MoqTest http.Setup(x => x.CreateHttpClient(It.IsAny(), timeout, token)).Returns(httpClient); var activityFactory = Create(); - activityFactory.Setup(x => x.Start(null, "DownloadSingleObject")).Returns((ISdkActivity?)null); + activityFactory.Setup(x => x.Start(null, default, "DownloadSingleObject")).Returns((ISdkActivity?)null); var serverObjectManager = new ServerObjectManager( http.Object, @@ -132,7 +132,7 @@ public class ServerObjectManagerTests : MoqTest http.Setup(x => x.CreateHttpClient(It.IsAny(), timeout, token)).Returns(httpClient); var activityFactory = Create(); - activityFactory.Setup(x => x.Start(null, "HasObjects")).Returns((ISdkActivity?)null); + activityFactory.Setup(x => x.Start(null, default, "HasObjects")).Returns((ISdkActivity?)null); var serverObjectManager = new ServerObjectManager( http.Object, @@ -171,7 +171,7 @@ public class ServerObjectManagerTests : MoqTest http.Setup(x => x.CreateHttpClient(It.IsAny(), timeout, token)).Returns(httpClient); var activityFactory = Create(); - activityFactory.Setup(x => x.Start(null, "UploadObjects")).Returns((ISdkActivity?)null); + activityFactory.Setup(x => x.Start(null, default, "UploadObjects")).Returns((ISdkActivity?)null); var serverObjectManager = new ServerObjectManager( http.Object, diff --git a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.TestGetProjects.verified.json b/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.TestGetProjects.verified.json deleted file mode 100644 index 781efcaf..00000000 --- a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.TestGetProjects.verified.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "Type": "AggregateException", - "InnerException": { - "Data": {}, - "Message": "FORBIDDEN: Your auth token does not have the required scope: workspace:read.", - "Type": "SpeckleGraphQLForbiddenException" - } -} diff --git a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.TestGetWorkspace.verified.json b/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.TestGetWorkspace.verified.json deleted file mode 100644 index 781efcaf..00000000 --- a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.TestGetWorkspace.verified.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "Type": "AggregateException", - "InnerException": { - "Data": {}, - "Message": "FORBIDDEN: Your auth token does not have the required scope: workspace:read.", - "Type": "SpeckleGraphQLForbiddenException" - } -} diff --git a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.cs b/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.cs index 8b73b43d..d597472d 100644 --- a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.cs +++ b/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.cs @@ -25,13 +25,15 @@ public class WorkspaceResourceTests public async Task TestGetWorkspace() { var ex = await Assert.ThrowsAsync(async () => _ = await Sut.Get("non-existent-id")); - await Verify(ex); + Assert.Single(ex.InnerExceptions); + Assert.All(ex.InnerExceptions, item => Assert.IsType(item)); } [Fact] public async Task TestGetProjects() { var ex = await Assert.ThrowsAsync(async () => _ = await Sut.GetProjects("non-existent-id")); - await Verify(ex); + Assert.Single(ex.InnerExceptions); + Assert.All(ex.InnerExceptions, item => Assert.IsType(item)); } } From edbc884d74c0838de8982d014694393acfc96cdb Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Mon, 23 Mar 2026 10:45:57 +0000 Subject: [PATCH 19/26] Clean diff --- src/Speckle.Sdk/Models/DynamicBaseMemberType.cs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/Speckle.Sdk/Models/DynamicBaseMemberType.cs b/src/Speckle.Sdk/Models/DynamicBaseMemberType.cs index b03c2579..4522b61e 100644 --- a/src/Speckle.Sdk/Models/DynamicBaseMemberType.cs +++ b/src/Speckle.Sdk/Models/DynamicBaseMemberType.cs @@ -7,32 +7,33 @@ namespace Speckle.Sdk.Models; public enum DynamicBaseMemberType { /// - /// The typed members of the DynamicBase object + /// The typed members of the object /// Instance = 1, /// - /// The dynamically added members of the DynamicBase object + /// The dynamically added members of the object /// Dynamic = 2, /// - /// The typed members flagged with ObsoleteAttribute attribute. + /// The typed members flagged with attribute. /// Obsolete = 4, /// - /// The typed methods flagged with TODO: + /// Old feature supported in v2 for grasshopper /// + [Obsolete("Feature no longer supported")] SchemaComputed = 16, /// - /// All the typed members, including ones with ObsoleteAttribute attributes. + /// All the typed members, including ones with attributes. /// InstanceAll = Instance + Obsolete, /// - /// All the members, including dynamic and instance members flagged with ObsoleteAttribute attributes + /// All the members, including dynamic and instance members flagged with attributes /// All = InstanceAll + Dynamic, } From 7042cdb06a64ad2edd8ceee7b701fb5f7fec1d9c Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Tue, 24 Mar 2026 17:17:25 +0000 Subject: [PATCH 20/26] feat(auth): Add new authflow functions (#462) * Auth functions * Avoid Microsoft.BCL.Memory polyfil * refactor updating accounts * fix tests * Fix tests * better auth messages * Remote changes * Fix tests * Fix tests * ditto * typo * More tests * fix exception types * revert temp change to test url * use new endpoint * new endpoint * Revert temp change to server url * Fix mistakes in tests * using * extra tests * test server url * no need to repack bcl memory since it's not a dependencgy * Avoid parallelising account manager tests * potential fix * disable parallelisation for authflow tests --- src/Speckle.Sdk/Credentials/Account.cs | 34 +- src/Speckle.Sdk/Credentials/AccountManager.cs | 675 ++++-------------- src/Speckle.Sdk/Credentials/AuthApp.cs | 13 + src/Speckle.Sdk/Credentials/AuthFlow.cs | 327 +++++++++ .../Credentials/AuthFlowException.cs | 14 - src/Speckle.Sdk/Credentials/Exceptions.cs | 22 +- src/Speckle.Sdk/Credentials/Responses.cs | 13 +- .../Resources/WorkspaceResourceTests.cs | 11 +- .../Collections.cs | 7 + .../Credentials/AccountManagerTests.cs | 107 +++ .../Credentials/AuthFlowExceptionalTests.cs | 84 +++ .../Credentials/AuthFlowTests.cs | 94 +++ .../Speckle.Sdk.Tests.Integration/Fixtures.cs | 16 +- .../Credentials/AccountManagerTests.cs | 83 +-- .../Credentials/Accounts.cs | 25 +- .../Credentials/AuthFlowTests.cs | 36 + tests/Speckle.Sdk.Tests.Unit/Fixtures.cs | 14 - 17 files changed, 852 insertions(+), 723 deletions(-) create mode 100644 src/Speckle.Sdk/Credentials/AuthApp.cs create mode 100644 src/Speckle.Sdk/Credentials/AuthFlow.cs delete mode 100644 src/Speckle.Sdk/Credentials/AuthFlowException.cs create mode 100644 tests/Speckle.Sdk.Tests.Integration/Collections.cs create mode 100644 tests/Speckle.Sdk.Tests.Integration/Credentials/AccountManagerTests.cs create mode 100644 tests/Speckle.Sdk.Tests.Integration/Credentials/AuthFlowExceptionalTests.cs create mode 100644 tests/Speckle.Sdk.Tests.Integration/Credentials/AuthFlowTests.cs create mode 100644 tests/Speckle.Sdk.Tests.Unit/Credentials/AuthFlowTests.cs diff --git a/src/Speckle.Sdk/Credentials/Account.cs b/src/Speckle.Sdk/Credentials/Account.cs index a9dbbbe2..dc3a92a2 100644 --- a/src/Speckle.Sdk/Credentials/Account.cs +++ b/src/Speckle.Sdk/Credentials/Account.cs @@ -1,11 +1,8 @@ -using System.Runtime.InteropServices; using Speckle.Sdk.Api.GraphQL.Models; using Speckle.Sdk.Common; namespace Speckle.Sdk.Credentials; -[ClassInterface(ClassInterfaceType.AutoDual)] -[ComVisible(true)] public class Account : IEquatable { private string _id; @@ -37,6 +34,8 @@ public class Account : IEquatable public string? refreshToken { get; set; } public bool isDefault { get; set; } + + [Obsolete("Not used in v3")] public bool isOnline { get; set; } = true; public ServerInfo serverInfo { get; set; } @@ -101,33 +100,4 @@ public class Account : IEquatable } #endregion - - internal const string LOCAL_IDENTIFIER_DEPRECATION_MESSAGE = "Local identifiers no longer nesseary"; - - /// - /// Retrieves the local identifier for the current user. - /// - /// - /// Returns a object representing the local identifier for the current user. - /// The local identifier is created by appending the user ID as a query parameter to the server URL. - /// - /// - /// Notice that the generated Uri is not intended to be used as a functioning Uri, but rather as a - /// unique identifier for a specific account in a local environment. The format of the Uri, containing a query parameter with the user ID, - /// serves this specific purpose. Therefore, it should not be used for forming network requests or - /// expecting it to lead to an actual webpage. The primary intent of this Uri is for unique identification in a Uri format. - /// - /// - /// This sample shows how to call the GetLocalIdentifier method. - /// - /// Uri localIdentifier = GetLocalIdentifier(); - /// Console.WriteLine(localIdentifier); - /// - /// For a fictional `User ID: 123` and `Server: https://speckle.xyz`, the output might look like this: - /// - /// https://speckle.xyz?id=123 - /// - /// - [Obsolete(LOCAL_IDENTIFIER_DEPRECATION_MESSAGE)] - internal Uri GetLocalIdentifier() => new($"{serverInfo.url}?id={userInfo.id}"); } diff --git a/src/Speckle.Sdk/Credentials/AccountManager.cs b/src/Speckle.Sdk/Credentials/AccountManager.cs index 7fb97826..d20a3473 100644 --- a/src/Speckle.Sdk/Credentials/AccountManager.cs +++ b/src/Speckle.Sdk/Credentials/AccountManager.cs @@ -1,145 +1,35 @@ -using System.Diagnostics; -using System.Net; -using System.Net.Http.Headers; -using System.Security.Cryptography; -using System.Text; -using System.Text.RegularExpressions; -using GraphQL; using GraphQL.Client.Http; using Microsoft.Extensions.Logging; using Speckle.InterfaceGenerator; using Speckle.Newtonsoft.Json; -using Speckle.Sdk.Api.GraphQL; using Speckle.Sdk.Api.GraphQL.Models; -using Speckle.Sdk.Api.GraphQL.Models.Responses; using Speckle.Sdk.Common; -using Speckle.Sdk.Helpers; using Speckle.Sdk.Logging; using Speckle.Sdk.SQLite; -using Stream = System.IO.Stream; namespace Speckle.Sdk.Credentials; public partial interface IAccountManager : IDisposable; /// -/// Manage accounts locally for desktop applications. +/// Manages data in the local sqlite account store /// [GenerateAutoInterface] public sealed class AccountManager( - ISpeckleApplication application, ILogger logger, - IGraphQLClientFactory graphQLClientFactory, - ISpeckleHttp speckleHttp, IAccountFactory accountFactory, + IAuthFlow authFlow, ISqLiteJsonCacheManagerFactory sqLiteJsonCacheManagerFactory ) : IAccountManager { public const string DEFAULT_SERVER_URL = "https://app.speckle.systems"; private readonly ISqLiteJsonCacheManager _accountStorage = sqLiteJsonCacheManagerFactory.CreateForUser("Accounts"); - private static volatile bool s_isAddingAccount; - private readonly ISqLiteJsonCacheManager _accountAddLockStorage = sqLiteJsonCacheManagerFactory.CreateForUser( - "AccountAddFlow" - ); [AutoInterfaceIgnore] public void Dispose() { _accountStorage.Dispose(); - _accountAddLockStorage.Dispose(); - } - - /// - /// Gets the basic information about a server. - /// - /// Server Information - /// - /// Request failed on the HTTP layer (received a non-successful response code) - /// - public async Task GetServerInfo(Uri server, CancellationToken cancellationToken = default) - { - using var gqlClient = graphQLClientFactory.CreateGraphQLClient(server, null); - - //lang=graphql - const string QUERY_STRING = "query { serverInfo { name company migration { movedFrom movedTo } } }"; - - var request = new GraphQLRequest { Query = QUERY_STRING }; - - var response = await gqlClient.SendQueryAsync(request, cancellationToken).ConfigureAwait(false); - - response.EnsureGraphQLSuccess(); - - ServerInfo serverInfo = response.Data.serverInfo; - serverInfo.url = server.ToString().TrimEnd('/'); - - return response.Data.serverInfo; - } - - /// - /// Gets basic user information given a token and a server. - /// - /// - /// Server URL - /// - /// Request failed on the HTTP layer (received a non-successful response code) - /// - public async Task GetUserInfo(string token, Uri server, CancellationToken cancellationToken = default) - { - using var gqlClient = graphQLClientFactory.CreateGraphQLClient(server, token); - - //language=graphql - const string QUERY = """ - query { - data:activeUser { - name - email - id - company - } - } - """; - var request = new GraphQLRequest { Query = QUERY }; - - var response = await gqlClient - .SendQueryAsync>(request, cancellationToken) - .ConfigureAwait(false); - - response.EnsureGraphQLSuccess(); - - return response.Data.data; - } - - /// - /// The Default Server URL for authentication, can be overridden by placing a file with the alternatrive url in the Speckle folder or with an ENV_VAR - /// - public Uri GetDefaultServerUrl() - { - var customServerUrl = ""; - - // first mechanism, check for local file - var customServerFile = Path.Combine(SpecklePathProvider.UserSpeckleFolderPath, "server"); - if (File.Exists(customServerFile)) - { - customServerUrl = File.ReadAllText(customServerFile); - } - - // second mechanism, check ENV VAR - var customServerEnvVar = Environment.GetEnvironmentVariable("SPECKLE_SERVER"); - if (!string.IsNullOrEmpty(customServerEnvVar)) - { - customServerUrl = customServerEnvVar; - } - - if (!string.IsNullOrEmpty(customServerUrl)) - { - if (Uri.TryCreate(customServerUrl, UriKind.Absolute, out Uri? url)) - { - return url; - } - } - - return new Uri(DEFAULT_SERVER_URL); } /// The Id of the account to fetch @@ -151,37 +41,6 @@ public sealed class AccountManager( ?? throw new SpeckleAccountManagerException($"Account {id} not found"); } - /// - /// Upgrades an account from the account.serverInfo.movedFrom account to the account.serverInfo.movedTo account - /// - /// Id of the account to upgrade - public void UpgradeAccount(string id) - { - Account account = GetAccount(id); - - if (account.serverInfo.migration?.movedTo is not Uri upgradeUri) - { - throw new SpeckleAccountManagerException( - $"Server with url {account.serverInfo.url} does not have information about the upgraded server" - ); - } - - account.serverInfo.migration.movedTo = null; - account.serverInfo.migration.movedFrom = new Uri(account.serverInfo.url); - account.serverInfo.url = upgradeUri.ToString().TrimEnd('/'); - - // setting the id to null will force it to be recreated - account.id = null!; //TODO this is gross so remove when id is nullable - - RemoveAccount(id); - _accountStorage.UpdateObject(account.id.NotNull(), JsonConvert.SerializeObject(account)); - } - - public IEnumerable GetAccounts(string serverUrl) - { - return GetAccounts(new Uri(serverUrl)); - } - /// /// Returns all unique accounts matching the serverUrl provided. If an account exists on more than one server, /// typically because it has been migrated, then only the upgraded account (and therefore server) are returned. @@ -245,7 +104,6 @@ public sealed class AccountManager( static bool IsInvalid(Account ac) => ac.userInfo == null || ac.serverInfo == null; var sqlAccounts = _accountStorage.GetAllObjects().Select(x => JsonConvert.DeserializeObject(x.Json)); - var localAccounts = GetLocalAccounts(); foreach (var acc in sqlAccounts) { @@ -259,119 +117,88 @@ public sealed class AccountManager( yield return acc; } } - - foreach (var acc in localAccounts) - { - yield return acc; - } } /// - /// Gets the local accounts - /// These are accounts not handled by Manager and are stored in json format in a local directory + /// Refetches all local accounts (in local db), including and . + /// If the looks to be expired, this function will also attempt to use the to refresh it. + /// Will write the changes to the local accounts db /// - /// - private IList GetLocalAccounts() + /// + /// + /// + public async Task UpdateAccount(Account account, CancellationToken cancellationToken = default) { - var accountsDir = SpecklePathProvider.AccountsFolderPath; - if (!Directory.Exists(accountsDir)) + string oldAccountId = account.id; + await UpdateAccountInMemory(account, cancellationToken).ConfigureAwait(false); + + if (oldAccountId != account.id) { - return Array.Empty(); + // ID may have changed, e.g. users email changed, or server url migrated + _accountStorage.DeleteObject(oldAccountId); } - - var accounts = new List(); - string[] files = Directory.GetFiles(accountsDir, "*.json", SearchOption.AllDirectories); - foreach (var file in files) - { - try - { - var json = File.ReadAllText(file); - Account? account = JsonConvert.DeserializeObject(json); - - if ( - account is not null - && !string.IsNullOrEmpty(account.token) - && !string.IsNullOrEmpty(account.userInfo.id) - && !string.IsNullOrEmpty(account.userInfo.email) - && !string.IsNullOrEmpty(account.userInfo.name) - && !string.IsNullOrEmpty(account.serverInfo.url) - && !string.IsNullOrEmpty(account.serverInfo.name) - ) - { - accounts.Add(account); - } - } - catch (Exception ex) when (!ex.IsFatal()) - { - logger.LogWarning(ex, "Failed to load json account at {filePath}", file); - } - } - - return accounts; + _accountStorage.UpdateObject(account.id, JsonConvert.SerializeObject(account)); } /// - /// Refetches user and server info for each account + /// Refetches the information, including and + /// If the looks to be expired, this function will also attempt to use the to refresh it. + /// + /// Will only mutate in memory only, and only if successful. /// - /// It is defaultAppId in the server. By default it is "sca" to not break existing parts that this function involves. - /// - public async Task UpdateAccounts(CancellationToken ct = default, string app = "sca") + /// + /// + /// + /// Thrown if + public async Task UpdateAccountInMemory(Account account, CancellationToken cancellationToken = default) { - // need to ToList() the GetAccounts call or the UpdateObject call at the end of this method - // will not work because sqlite does not support concurrent db calls - foreach (var account in GetAccounts().ToList()) + Uri url = account.serverInfo.migration?.movedTo ?? new(account.serverInfo.url); + ActiveUserServerInfoResponse userServerInfo; + + try { + userServerInfo = await accountFactory + .GetUserServerInfo(url, account.token, cancellationToken) + .ConfigureAwait(false); + } + catch (GraphQLHttpRequestException ex) + { + // Failed to fetch info, perhaps the token is expired? + // Attempt to refresh it + TokenExchangeResponse refreshTokenResponse; try { - Uri url = new(account.serverInfo.url); - var userServerInfo = await accountFactory.GetUserServerInfo(url, account.token, ct).ConfigureAwait(false); + refreshTokenResponse = await authFlow + .GetRefreshedToken( + account.refreshToken.NotNull("No refresh token provided"), + url, + AuthApp.ConnectorsV3, + cancellationToken + ) + .ConfigureAwait(false); - //the token has expired - //TODO: once we get a token expired exception from the server use that instead - if (userServerInfo.activeUser == null || userServerInfo.serverInfo == null) - { - // We were initially was handling refresh token here bc quite a while ago server was returning null - // for activeUser and serverInfo instead of throwing exception. In short, our logic moved into catch block to cover both. - throw new SpeckleException("Token is expired"); - } - - account.isOnline = true; - account.userInfo = userServerInfo.activeUser; - account.serverInfo = userServerInfo.serverInfo; + userServerInfo = await accountFactory + .GetUserServerInfo(url, refreshTokenResponse.token, cancellationToken) + .ConfigureAwait(false); } - catch (OperationCanceledException) + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; } - catch (Exception ex) when (!ex.IsFatal()) + catch (Exception ex2) { - await RefreshAndSetAccountToken(account, app).ConfigureAwait(false); + throw new AggregateException("Failed to update account information", ex, ex2); } - ct.ThrowIfCancellationRequested(); - _accountStorage.UpdateObject(account.id, JsonConvert.SerializeObject(account)); - } - } - - /// - /// Mutates the account with new tokens. - /// - /// - /// - private async Task RefreshAndSetAccountToken(Account account, string app) - { - try - { - Uri url = new(account.serverInfo.url); - var tokenResponse = await GetRefreshedToken(account.refreshToken, url, app).ConfigureAwait(false); - account.token = tokenResponse.token; - account.refreshToken = tokenResponse.refreshToken; - account.isOnline = true; - } - catch (Exception ex) when (!ex.IsFatal()) - { - account.isOnline = false; + account.token = refreshTokenResponse.token; + account.refreshToken = refreshTokenResponse.refreshToken; + logger.LogInformation(ex, "Account token has been refreshed"); } + account.userInfo = userServerInfo.activeUser.NotNull(); + account.serverInfo = userServerInfo.serverInfo; + //This is a bit gross, since id is not marked nullable + //but this will force re-generate the id (e.g. if the user's email, or servers url has changed) + account.id = null!; } /// @@ -412,325 +239,103 @@ public sealed class AccountManager( } /// - /// Retrieves the local identifier for the specified account. + /// Adds an account to local storage by prompting the user to log in via their browser. /// - /// The account for which to retrieve the local identifier. - /// The local identifier for the specified account in the form of "SERVER_URL?u=USER_ID". - /// - /// - /// - [Obsolete(Account.LOCAL_IDENTIFIER_DEPRECATION_MESSAGE)] - public Uri? GetLocalIdentifierForAccount(Account account) - { - var identifier = account.GetLocalIdentifier(); - - // Validate account is stored locally - var searchResult = GetAccountForLocalIdentifier(identifier); - - return searchResult == null ? null : identifier; - } - - public async Task Validate(Account account) - { - Uri server = new(account.serverInfo.url); - return await GetUserInfo(account.token, server).ConfigureAwait(false); - } - - /// - /// Gets the account that corresponds to the given local identifier. - /// - /// The local identifier of the account. - /// The account that matches the local identifier, or null if no match is found. - [Obsolete(Account.LOCAL_IDENTIFIER_DEPRECATION_MESSAGE)] - public Account? GetAccountForLocalIdentifier(Uri localIdentifier) - { - var searchResult = GetAccounts() - .FirstOrDefault(acc => - { - var id = acc.GetLocalIdentifier(); - return id == localIdentifier; - }); - - return searchResult; - } - - private Uri EnsureCorrectServerUrl(Uri? server) - { - var localUrl = server; - if (localUrl == null) - { - localUrl = GetDefaultServerUrl(); - logger.LogDebug("The provided server url was null or empty. Changed to the default url {serverUrl}", localUrl); - } - return localUrl; - } - - private void EnsureGetAccessCodeFlowIsSupported() - { - if (!HttpListener.IsSupported) - { - logger.LogError("HttpListener not supported"); - throw new PlatformNotSupportedException("Your operating system is not supported"); - } - } - - private async Task GetAccessCode(Uri server, string challenge, TimeSpan timeout) - { - EnsureGetAccessCodeFlowIsSupported(); - - logger.LogDebug("Starting auth process for {server}/authn/verify/sca/{challenge}", server, challenge); - - var accessCode = ""; - - Process.Start(new ProcessStartInfo($"{server}/authn/verify/sca/{challenge}") { UseShellExecute = true }); - - var task = Task.Run(() => - { - using var listener = new HttpListener(); - var localUrl = "http://localhost:29363/"; - listener.Prefixes.Add(localUrl); - listener.Start(); - logger.LogDebug("Listening for auth redirects on {localUrl}", localUrl); - // Note: The GetContext method blocks while waiting for a request. - HttpListenerContext context = listener.GetContext(); - HttpListenerRequest request = context.Request; - HttpListenerResponse response = context.Response; - - accessCode = request.QueryString["access_code"]; - logger.LogDebug("Got access code {accessCode}", accessCode); - - string message = - accessCode != null - ? "Success!

You can close this window now." - : "Oups, something went wrong...!"; - - var responseString = - $"
{message}"; - byte[] buffer = Encoding.UTF8.GetBytes(responseString); - response.ContentLength64 = buffer.Length; - Stream output = response.OutputStream; - output.Write(buffer, 0, buffer.Length); - output.Close(); - logger.LogDebug("Processed finished processing the access code"); - listener.Stop(); - listener.Close(); - }); - - var completedTask = await Task.WhenAny(task, Task.Delay(timeout)).ConfigureAwait(false); - - // this is means the task timed out - if (completedTask != task) - { - logger.LogWarning( - "Local auth flow failed to complete within the timeout window. Access code is {accessCode}", - accessCode - ); - throw new AuthFlowException("Local auth flow failed to complete within the timeout window"); - } - - if (task.IsFaulted && task.Exception is not null) - { - logger.LogError( - task.Exception, - "Getting access code flow failed with {exceptionMessage}", - task.Exception.Message - ); - throw new AuthFlowException($"Auth flow failed: {task.Exception.Message}", task.Exception); - } - - // task completed within timeout - logger.LogInformation( - "Local auth flow completed successfully within the timeout window. Access code is {accessCode}", - accessCode - ); - return accessCode; - } - - private async Task CreateAccount(string accessCode, string challenge, Uri server) - { - try - { - var tokenResponse = await GetToken(accessCode, challenge, server).ConfigureAwait(false); - - var account = await accountFactory - .CreateAccount(server, tokenResponse.token, tokenResponse.refreshToken) - .ConfigureAwait(false); - - account.isDefault = !GetAccounts().Any(); - - logger.LogInformation("Successfully created account for {serverUrl}", server); - - return account; - } - catch (Exception ex) when (!ex.IsFatal()) - { - throw new SpeckleAccountManagerException("Failed to create account from access code and challenge", ex); - } - } - - private void TryLockAccountAddFlow(TimeSpan timespan) - { - // use a static variable to quickly - // prevent launching this flow multiple times - if (s_isAddingAccount) - { - // this should probably throw with an error message - throw new SpeckleAccountFlowLockedException("The account add flow is already launched."); - } - - // this uses the SQLite transport to store locks - var lockIds = _accountAddLockStorage.GetAllObjects().Select(x => x.Id).OrderByDescending(d => d).ToList(); - var now = DateTime.Now; - foreach (var l in lockIds) - { - var lockArray = l.Split('@'); - var lockName = lockArray.Length == 2 ? lockArray[0] : "the other app"; - var lockTime = - lockArray.Length == 2 - ? DateTime.ParseExact(lockArray[1], "o", null) - : DateTime.ParseExact(lockArray[0], "o", null); - - if (lockTime > now) - { - var lockString = string.Format("{0:mm} minutes {0:ss} seconds", lockTime - now); - throw new SpeckleAccountFlowLockedException( - $"The account add flow was already started in {lockName}, retry in {lockString}" - ); - } - } - - var lockId = application.ApplicationAndVersion + "@" + DateTime.Now.Add(timespan).ToString("o"); - - // using the lock release time as an id and value - // for ease of deletion and retrieval - _accountAddLockStorage.SaveObject(lockId, lockId); - s_isAddingAccount = true; - } - - private void UnlockAccountAddFlow() - { - s_isAddingAccount = false; - // make sure all old locks are removed - foreach (var (id, _) in _accountAddLockStorage.GetAllObjects()) - { - _accountAddLockStorage.DeleteObject(id); - } - } - - /// - /// Adds an account by propting the user to log in via a web flow - /// - /// Server to use to add the account, if not provied the default Server will be used + /// + /// + /// Account account = await AuthenticateAccount(new Uri("https://app.speckle.systems"), TimeSpan.FromMinutes(1)); + /// + /// + /// + /// Timeout for user to auth with browser, recommend 1 min timeout + /// /// - public async Task AddAccount(Uri? server = null) + public async Task AuthenticateAccount(Uri serverUrl, TimeSpan timeout, CancellationToken cancellationToken) { - logger.LogDebug("Starting to add account for {serverUrl}", server); + logger.LogDebug("Starting to add account for {ServerUrl}", serverUrl); - server = EnsureCorrectServerUrl(server); + TokenExchangeResponse tokenResponse = await authFlow + .TriggerAuthFlowWithTimeout(serverUrl, AuthApp.ConnectorsV3, timeout, cancellationToken) + .ConfigureAwait(false); - // locking for 1 minute - var timeout = TimeSpan.FromMinutes(1); - // this is not part of the try finally block - // we do not want to clean up the existing locks - TryLockAccountAddFlow(timeout); - var challenge = GenerateChallenge(); + return await CreateAndAddAccount(serverUrl, tokenResponse, cancellationToken).ConfigureAwait(false); + } - try + public async Task CreateAndAddAccount( + Uri serverUrl, + TokenExchangeResponse tokenResponse, + CancellationToken cancellationToken + ) + { + var account = await accountFactory + .CreateAccount(serverUrl, tokenResponse.token, tokenResponse.refreshToken, cancellationToken) + .ConfigureAwait(false); + account.isDefault = !GetAccounts().Any(); + + _accountStorage.SaveObject(account.id, JsonConvert.SerializeObject(account)); + logger.LogInformation("Successfully authenticated account {AccountId} for {ServerUrl}", account.id, serverUrl); + return account; + } + + /// + /// The Default Server URL for authentication, can be overridden by placing a file with the alternative url in the Speckle folder or with an ENV_VAR + /// + [Obsolete("Unused")] + public Uri GetDefaultServerUrl() + { + var customServerUrl = ""; + + // first mechanism, check for local file + var customServerFile = Path.Combine(SpecklePathProvider.UserSpeckleFolderPath, "server"); + if (File.Exists(customServerFile)) { - string accessCode = await GetAccessCode(server, challenge, timeout).ConfigureAwait(false); - if (string.IsNullOrEmpty(accessCode)) + customServerUrl = File.ReadAllText(customServerFile); + } + + // second mechanism, check ENV VAR + var customServerEnvVar = Environment.GetEnvironmentVariable("SPECKLE_SERVER"); + if (!string.IsNullOrEmpty(customServerEnvVar)) + { + customServerUrl = customServerEnvVar; + } + + if (!string.IsNullOrEmpty(customServerUrl)) + { + if (Uri.TryCreate(customServerUrl, UriKind.Absolute, out Uri? url)) { - throw new SpeckleAccountManagerException("Access code is invalid"); + return url; } + } - var account = await CreateAccount(accessCode, challenge, server).ConfigureAwait(false); - - //if the account already exists it will not be added again - _accountStorage.SaveObject(account.id, JsonConvert.SerializeObject(account)); - logger.LogDebug("Finished adding account {accountId} for {serverUrl}", account.id, server); - } - catch (SpeckleAccountManagerException ex) - { - logger.LogCritical(ex, "Failed to add account: {exceptionMessage}", ex.Message); - // rethrowing any known errors - throw; - } - catch (Exception ex) when (!ex.IsFatal()) - { - logger.LogCritical(ex, "Failed to add account: {exceptionMessage}", ex.Message); - throw new SpeckleAccountManagerException($"Failed to add account: {ex.Message}", ex); - } - finally - { - UnlockAccountAddFlow(); - } + return new Uri(DEFAULT_SERVER_URL); } - private async Task GetToken(string accessCode, string challenge, Uri server) + [Obsolete("Use Uri overload")] + public IEnumerable GetAccounts(string serverUrl) { - try - { - using var client = speckleHttp.CreateHttpClient(); - - var body = new - { - appId = "sca", - appSecret = "sca", - accessCode, - challenge, - }; - - using var content = new StringContent(JsonConvert.SerializeObject(body)); - content.Headers.ContentType = new MediaTypeHeaderValue("application/json"); - var response = await client.PostAsync(new Uri(server, "/auth/token"), content).ConfigureAwait(false); - - return JsonConvert - .DeserializeObject(await response.Content.ReadAsStringAsync().ConfigureAwait(false)) - .NotNull(); - } - catch (Exception ex) when (!ex.IsFatal()) - { - throw new SpeckleException($"Failed to get authentication token from {server}", ex); - } + return GetAccounts(new Uri(serverUrl)); } - private async Task GetRefreshedToken(string? refreshToken, Uri server, string app = "sca") - { - try - { - using var client = speckleHttp.CreateHttpClient(); + [Obsolete("Use UpdateAccount instead for more control over error handling", true)] + public Task UpdateAccounts(CancellationToken ct = default, string app = "sca") => throw new NotImplementedException(); - var body = new - { - appId = app, - appSecret = app, - refreshToken, - }; + [Obsolete("Use UpdateAccount instead", true)] + public void UpgradeAccount(string id) => throw new NotImplementedException(); - using var content = new StringContent(JsonConvert.SerializeObject(body)); - content.Headers.ContentType = new MediaTypeHeaderValue("application/json"); - var response = await client.PostAsync(new Uri(server, "/auth/token"), content).ConfigureAwait(false); + [Obsolete($"Use {nameof(AuthenticateAccount)} instead", true)] + public Task AddAccount(Uri? server = null) => throw new NotImplementedException(); - return JsonConvert - .DeserializeObject(await response.Content.ReadAsStringAsync().ConfigureAwait(false)) - .NotNull(); - } - catch (Exception ex) when (!ex.IsFatal()) - { - throw new SpeckleException($"Failed to get refreshed token from {server}", ex); - } - } + [Obsolete("Use serverInfo stored on a client instead", true)] + public Task GetServerInfo(Uri server, CancellationToken cancellationToken = default) => + throw new NotImplementedException(); - private static string GenerateChallenge() - { -#if NET8_0 - byte[] challengeData = RandomNumberGenerator.GetBytes(32); -#else - using RNGCryptoServiceProvider rng = new(); - byte[] challengeData = new byte[32]; - rng.GetBytes(challengeData); -#endif - //escaped chars like % do not play nice with the server - return Regex.Replace(Convert.ToBase64String(challengeData), @"[^\w\.@-]", ""); - } + [Obsolete("Use userInfo stored on a client instead", true)] + public Task GetUserInfo(string token, Uri server, CancellationToken cancellationToken = default) => + throw new NotImplementedException(); + + [Obsolete("Accounts must now be stored in sqlite db, no more json workaround", true)] + public IList GetLocalAccounts() => throw new NotImplementedException(); + + [Obsolete("Use UpdateAccount or UpdateAccountInMemory Instead", true)] + public IList Validate() => throw new NotImplementedException(); } diff --git a/src/Speckle.Sdk/Credentials/AuthApp.cs b/src/Speckle.Sdk/Credentials/AuthApp.cs new file mode 100644 index 00000000..00478414 --- /dev/null +++ b/src/Speckle.Sdk/Credentials/AuthApp.cs @@ -0,0 +1,13 @@ +namespace Speckle.Sdk.Credentials; + +public readonly record struct AuthApp(string AppId, string AppSecret, Uri CallbackUrl) +{ + //These values are defined on the server, and specify the scopes the app is requesting + public static AuthApp ConnectorsV3 { get; } = + new() + { + AppId = "connectrV3", + AppSecret = "connectrV3", + CallbackUrl = new Uri("http://localhost:29355"), + }; +} diff --git a/src/Speckle.Sdk/Credentials/AuthFlow.cs b/src/Speckle.Sdk/Credentials/AuthFlow.cs new file mode 100644 index 00000000..f849da97 --- /dev/null +++ b/src/Speckle.Sdk/Credentials/AuthFlow.cs @@ -0,0 +1,327 @@ +using System.Diagnostics; +using System.Net; +using System.Net.Http.Headers; +using System.Security.Cryptography; +using System.Text; +using Speckle.InterfaceGenerator; +using Speckle.Newtonsoft.Json; +using Speckle.Sdk.Common; +using Speckle.Sdk.Helpers; +using Speckle.Sdk.Logging; + +namespace Speckle.Sdk.Credentials; + +/// +/// Authentication flow with the Speckle Server to create a application token for the connectorsV3 application +/// Starts the browser based authentication flow where the user's browser will be opened, they'll be asked to +/// confirm permission, then an access code will be given via a which will be exchanged +/// for a +/// +/// +/// Note, this class is not coupled in any way to +/// lets keep it that way... +/// See instead +/// +[GenerateAutoInterface] +public sealed class AuthFlow(ISdkActivityFactory activityFactory, ISpeckleHttp speckleHttp) : IAuthFlow +{ + private readonly JsonSerializerSettings _serializerSettings = new() + { + MissingMemberHandling = MissingMemberHandling.Error, + NullValueHandling = NullValueHandling.Ignore, + }; + + public async Task TriggerAuthFlowWithTimeout( + Uri serverUrl, + AuthApp authApp, + TimeSpan timeout, + CancellationToken cancellationToken + ) + { + using HttpClient client = speckleHttp.CreateHttpClient(); + + Uri tokenEndpoint = new(serverUrl, "/oauth/token"); + string codeVerifier = GenerateCodeVerifier(); + string challenge; + string codeChallengeMethod; + var req = await client.GetAsync(tokenEndpoint, cancellationToken).ConfigureAwait(false); + bool useLegacyEndpoint = req.StatusCode != HttpStatusCode.OK; + + if (useLegacyEndpoint) + { + challenge = codeVerifier; + tokenEndpoint = new(serverUrl, "/auth/token"); + codeChallengeMethod = ""; + } + else + { + challenge = GenerateCodeChallenge(codeVerifier); + codeChallengeMethod = "?code_challenge_method=S256"; + } + + Uri endpoint = new(serverUrl, $"/authn/verify/{authApp.AppId}/{challenge}{codeChallengeMethod}"); + _ = Process.Start(new ProcessStartInfo(endpoint.ToString()) { UseShellExecute = true }); + string accessCode = await RunListenerWithTimeout(authApp.CallbackUrl, timeout, cancellationToken) + .ConfigureAwait(false); + + return await ExchangeAccessCodeForToken( + client, + accessCode, + authApp, + useLegacyEndpoint ? challenge : null, + !useLegacyEndpoint ? codeVerifier : null, + tokenEndpoint, + cancellationToken + ) + .ConfigureAwait(false); + } + + /// + /// + /// + /// + /// + /// + /// + /// requested cancel + /// timeout was reached + public async Task RunListenerWithTimeout( + Uri applicationCallbackUrl, + TimeSpan timeout, + CancellationToken userCancellation + ) + { + using CancellationTokenSource cancelOnTimeout = new(timeout); + using CancellationTokenSource linkedSource = CancellationTokenSource.CreateLinkedTokenSource( + cancelOnTimeout.Token, + userCancellation + ); + + try + { + using var activity = activityFactory.Start("Listening for authflow access code"); + + return await RunListener(applicationCallbackUrl, linkedSource.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) when (userCancellation.IsCancellationRequested) + { + throw; + } + catch (OperationCanceledException ex) when (cancelOnTimeout.IsCancellationRequested) + { + throw new TimeoutException($"Auth flow was cancelled after {timeout:g} timeout", ex); + } + } + + /// + /// + /// + /// + /// + /// Auth app, needs to match the app that generated the refresh token originally + /// + /// HTTP exceptions + /// Server response was invalid or partial + /// Invalid (must be absolute url) + /// requested cancel + /// + public async Task GetRefreshedToken( + string? refreshToken, + Uri serverUrl, + AuthApp authApp, + CancellationToken cancellationToken + ) + { + using var client = speckleHttp.CreateHttpClient(); + + var body = new + { + appId = authApp.AppId, + appSecret = authApp.AppSecret, + refreshToken = refreshToken, + }; + + using var content = new StringContent(JsonConvert.SerializeObject(body)); + content.Headers.ContentType = new MediaTypeHeaderValue("application/json"); + var response = await client + .PostAsync(new Uri(serverUrl, "/auth/token"), content, cancellationToken) + .ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + +#if NET8_0_OR_GREATER + string read = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); +#else + string read = await response.Content.ReadAsStringAsync().ConfigureAwait(false); +#endif + return JsonConvert.DeserializeObject(read, _serializerSettings).NotNull(); + } + + private static async Task GetContext(HttpListener listener, CancellationToken cancellationToken) + { + //GetContextAsync doesn't support cancellation, so we have to do this song and dance... + Task timeoutTask = Task.Delay(Timeout.Infinite, cancellationToken); + Task getContextTask = listener.GetContextAsync(); + + Task completed = await Task.WhenAny(getContextTask, timeoutTask).ConfigureAwait(false); + if (completed == getContextTask) + { + return getContextTask.Result; + } + + cancellationToken.ThrowIfCancellationRequested(); + + throw new InvalidOperationException("Cancellation should have thrown, this shouldn't be possible"); + } + + public static async Task RunListener(Uri localUrl, CancellationToken cancellationToken) + { + using HttpListener listener = new(); + listener.Prefixes.Add(localUrl.ToString()); + listener.Start(); + + HttpListenerContext context = await GetContext(listener, cancellationToken).ConfigureAwait(false); + HttpListenerRequest request = context.Request; + using HttpListenerResponse response = context.Response; + + string? accessCode = request.QueryString["access_code"]; + string? denied = request.QueryString["denied"]; + bool isDenied = denied == "true"; + + if (isDenied) + { + //lang=html + WriteResponse( + """ +

Denied!

+

+ Please close this window and return to your Speckle Connector. + """ + ); + throw new AuthFlowException("Authentication flow was denied"); //denied presumably by the user + } + else if (accessCode != null) + { + //lang=html + WriteResponse( + """ +

Success!

+

+ Your Speckle Connector is now authorized +

+ You may now close this window and return to your Speckle Connector + """ + ); + return accessCode; + } + else + { + //lang=html + WriteResponse( + """ +

Failed!

+

+ Something went wrong trying to authorize your Speckle Connector +

+ Please close this window and try again from your Speckle Connector. + """ + ); + throw new AuthFlowException("Failed to receive access code"); + } + + void WriteResponse(string message) + { + //lang=html + string responseString = $""" + + +
+ {message} + + + """; + + byte[] buffer = Encoding.UTF8.GetBytes(responseString); + response.ContentLength64 = buffer.Length; + response.OutputStream.Write(buffer, 0, buffer.Length); + } + } + + private async Task ExchangeAccessCodeForToken( + HttpClient client, + string accessCode, + AuthApp authApp, + string? challenge, + string? codeVerifier, + Uri tokenEndpoint, + CancellationToken cancellationToken + ) + { + var body = new + { + appId = authApp.AppId, + appSecret = authApp.AppSecret, + accessCode = accessCode, + challenge = challenge, + codeVerifier = codeVerifier, + }; + + using StringContent content = new(JsonConvert.SerializeObject(body, _serializerSettings)); + content.Headers.ContentType = new MediaTypeHeaderValue("application/json"); + + using HttpResponseMessage response = await client + .PostAsync(tokenEndpoint, content, cancellationToken) + .ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + +#if NET8_0_OR_GREATER + string read = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); +#else + string read = await response.Content.ReadAsStringAsync().ConfigureAwait(false); +#endif + + return JsonConvert.DeserializeObject(read, _serializerSettings).NotNull(); + } + + public static string GenerateCodeVerifier() + { +#if NET8_0_OR_GREATER + Span codeVerifierData = stackalloc byte[32]; + RandomNumberGenerator.Fill(codeVerifierData); +#else + using RNGCryptoServiceProvider rng = new(); + byte[] codeVerifierData = new byte[32]; + rng.GetBytes(codeVerifierData); +#endif + + return Base64UrlEncode(codeVerifierData); + } + + public static string GenerateCodeChallenge(string codeVerifier) + { +#if NET8_0_OR_GREATER + int byteCount = Encoding.UTF8.GetByteCount(codeVerifier.AsSpan()); + Span codeVerifierBytes = stackalloc byte[byteCount]; + Encoding.UTF8.GetBytes(codeVerifier, codeVerifierBytes); + Span challengeData = stackalloc byte[SHA256.HashSizeInBytes]; + SHA256.HashData(codeVerifierBytes, challengeData); +#else + byte[] codeVerifierBytes = Encoding.UTF8.GetBytes(codeVerifier); + using SHA256 hash = SHA256.Create(); + byte[] challengeData = hash.ComputeHash(codeVerifierBytes); +#endif + return Base64UrlEncode(challengeData); + } + + private static string Base64UrlEncode( +#if NET8_0_OR_GREATER + ReadOnlySpan bytes +#else + byte[] bytes +#endif + ) + { + // Base64Url is available in .NET 9, or via the Microsoft.Bcl.Memory polyfill + // But for simplicity r.e. dll dependencies, we're doing it the dumb way... + return Convert.ToBase64String(bytes).Replace('+', '-').Replace('/', '_').TrimEnd('='); + } +} diff --git a/src/Speckle.Sdk/Credentials/AuthFlowException.cs b/src/Speckle.Sdk/Credentials/AuthFlowException.cs deleted file mode 100644 index cb720d00..00000000 --- a/src/Speckle.Sdk/Credentials/AuthFlowException.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace Speckle.Sdk.Credentials; - -#pragma warning disable CA2237 -public sealed class AuthFlowException : Exception -#pragma warning restore CA2237 -{ - public AuthFlowException(string? message, Exception? innerException) - : base(message, innerException) { } - - public AuthFlowException(string? message) - : base(message) { } - - public AuthFlowException() { } -} diff --git a/src/Speckle.Sdk/Credentials/Exceptions.cs b/src/Speckle.Sdk/Credentials/Exceptions.cs index e33738a0..9c85c017 100644 --- a/src/Speckle.Sdk/Credentials/Exceptions.cs +++ b/src/Speckle.Sdk/Credentials/Exceptions.cs @@ -1,5 +1,16 @@ namespace Speckle.Sdk.Credentials; +public sealed class AuthFlowException : SpeckleException +{ + public AuthFlowException(string? message, Exception? innerException) + : base(message, innerException) { } + + public AuthFlowException(string? message) + : base(message) { } + + public AuthFlowException() { } +} + public class SpeckleAccountManagerException : SpeckleException { public SpeckleAccountManagerException(string message) @@ -10,14 +21,3 @@ public class SpeckleAccountManagerException : SpeckleException public SpeckleAccountManagerException() { } } - -public class SpeckleAccountFlowLockedException : SpeckleAccountManagerException -{ - public SpeckleAccountFlowLockedException(string message) - : base(message) { } - - public SpeckleAccountFlowLockedException() { } - - public SpeckleAccountFlowLockedException(string message, Exception? innerException) - : base(message, innerException) { } -} diff --git a/src/Speckle.Sdk/Credentials/Responses.cs b/src/Speckle.Sdk/Credentials/Responses.cs index 24f12117..996bb520 100644 --- a/src/Speckle.Sdk/Credentials/Responses.cs +++ b/src/Speckle.Sdk/Credentials/Responses.cs @@ -6,16 +6,19 @@ namespace Speckle.Sdk.Credentials; internal sealed class ActiveUserServerInfoResponse { [property: JsonProperty(Required = Required.AllowNull)] - public UserInfo? activeUser { get; init; } + public required UserInfo? activeUser { get; init; } [property: JsonProperty(Required = Required.Always)] - public ServerInfo serverInfo { get; init; } + public required ServerInfo serverInfo { get; init; } } -internal sealed class TokenExchangeResponse +public sealed class TokenExchangeResponse { - public string token { get; init; } - public string refreshToken { get; init; } + [JsonRequired] + public required string token { get; init; } + + [JsonRequired] + public required string refreshToken { get; init; } } public sealed class UserInfo diff --git a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.cs b/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.cs index d597472d..3f4ed00c 100644 --- a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.cs +++ b/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.cs @@ -1,4 +1,5 @@ -using Speckle.Sdk.Api; +using FluentAssertions; +using Speckle.Sdk.Api; using Speckle.Sdk.Api.GraphQL.Resources; namespace Speckle.Sdk.Tests.Integration.API.GraphQL.Resources; @@ -25,15 +26,15 @@ public class WorkspaceResourceTests public async Task TestGetWorkspace() { var ex = await Assert.ThrowsAsync(async () => _ = await Sut.Get("non-existent-id")); - Assert.Single(ex.InnerExceptions); - Assert.All(ex.InnerExceptions, item => Assert.IsType(item)); + ex.InnerExceptions.Should().HaveCount(1); + ex.InnerExceptions.Should().AllBeOfType(); } [Fact] public async Task TestGetProjects() { var ex = await Assert.ThrowsAsync(async () => _ = await Sut.GetProjects("non-existent-id")); - Assert.Single(ex.InnerExceptions); - Assert.All(ex.InnerExceptions, item => Assert.IsType(item)); + ex.InnerExceptions.Should().HaveCount(1); + ex.InnerExceptions.Should().AllBeOfType(); } } diff --git a/tests/Speckle.Sdk.Tests.Integration/Collections.cs b/tests/Speckle.Sdk.Tests.Integration/Collections.cs new file mode 100644 index 00000000..d5a850db --- /dev/null +++ b/tests/Speckle.Sdk.Tests.Integration/Collections.cs @@ -0,0 +1,7 @@ +namespace Speckle.Sdk.Tests.Integration; + +[CollectionDefinition(nameof(RequiresSqLiteAccountDb), DisableParallelization = true)] +public sealed class RequiresSqLiteAccountDb; + +[CollectionDefinition(nameof(RequiresAuthFlowPort), DisableParallelization = true)] +public sealed class RequiresAuthFlowPort; diff --git a/tests/Speckle.Sdk.Tests.Integration/Credentials/AccountManagerTests.cs b/tests/Speckle.Sdk.Tests.Integration/Credentials/AccountManagerTests.cs new file mode 100644 index 00000000..c044d452 --- /dev/null +++ b/tests/Speckle.Sdk.Tests.Integration/Credentials/AccountManagerTests.cs @@ -0,0 +1,107 @@ +using Microsoft.Extensions.DependencyInjection; +using Speckle.Sdk.Api; +using Speckle.Sdk.Api.GraphQL.Models; +using Speckle.Sdk.Credentials; + +namespace Speckle.Sdk.Tests.Integration.Credentials; + +[Collection(nameof(RequiresSqLiteAccountDb))] +public class AccountManagerTests +{ + private IAccountManager _sut; + + public AccountManagerTests() + { + _sut = Fixtures.ServiceProvider.GetRequiredService(); + } + + [Fact] + public async Task UpdateAccount_UpdatesUserInfo() + { + using IClient user = await Fixtures.SeedUserWithClient(); + string realAccountId = user.Account.id; + UserInfo realUserData = user.Account.userInfo; + UserInfo staleData = new() + { + avatar = "my old avatar", + company = "my old company", + email = "my.old.email@example.com", + id = realUserData.id, + name = "my old name", + }; + // Mutate with "fake" data to simulate a stale account data + user.Account.userInfo = staleData; + user.Account.id = null!; //force re-generate id + + Assert.NotEqual(realAccountId, user.Account.id); + + await _sut.UpdateAccountInMemory(user.Account); + + Assert.Equal(realUserData.avatar, user.Account.userInfo.avatar); + Assert.Equal(realUserData.company, user.Account.userInfo.company); + Assert.Equal(realUserData.email, user.Account.userInfo.email); + Assert.Equal(realUserData.id, user.Account.userInfo.id); + Assert.Equal(realUserData.name, user.Account.userInfo.name); + Assert.Equal(realAccountId, user.Account.id); + } + + [Fact] + public async Task UpdateAccount_UpdatesServerInfo() + { + using IClient user = await Fixtures.SeedUserWithClient(); + string realAccountId = user.Account.id; + ServerInfo realServerData = user.Account.serverInfo; + ServerInfo staleData = new() + { + company = "This old company", + description = "this old description", + name = "This old name", + url = realServerData.url, + version = "0.0.123", + }; + // Mutate with "fake" data to simulate a stale account data + user.Account.serverInfo = staleData; + user.Account.id = null!; //force re-generate id + + Assert.Equal(realAccountId, user.Account.id); //account id should not change since we didn't change server url + + await _sut.UpdateAccountInMemory(user.Account); + + Assert.Equal(realServerData.company, user.Account.serverInfo.company); + Assert.Equal(realServerData.description, user.Account.serverInfo.description); + Assert.Equal(realServerData.name, user.Account.serverInfo.name); + Assert.Equal(realServerData.url, user.Account.serverInfo.url); + Assert.Equal(realServerData.version, user.Account.serverInfo.version); + Assert.Equal(realAccountId, user.Account.id); + } + + [Fact] + public async Task UpdateAccount_ServerInfoMigration() + { + using IClient user = await Fixtures.SeedUserWithClient(); + string realAccountId = user.Account.id; + ServerInfo realServerData = user.Account.serverInfo; + ServerInfo staleData = new() + { + company = "This old company", + description = "this old description", + name = "This old name", + url = realServerData.url, + version = "0.0.123", + }; + // Mutate with "fake" data to simulate a stale account data + user.Account.serverInfo = staleData; + user.Account.id = null!; //force re-generate id + + Assert.Equal(realAccountId, user.Account.id); //account id should not change since we didn't change server url + + await _sut.UpdateAccountInMemory(user.Account); + + Assert.Equal(realServerData.company, user.Account.serverInfo.company); + Assert.Equal(realServerData.description, user.Account.serverInfo.description); + Assert.Equal(realServerData.name, user.Account.serverInfo.name); + Assert.Equal(realServerData.url, user.Account.serverInfo.url); + Assert.Equal(realServerData.version, user.Account.serverInfo.version); + Assert.Equal(realAccountId, user.Account.id); + } +} diff --git a/tests/Speckle.Sdk.Tests.Integration/Credentials/AuthFlowExceptionalTests.cs b/tests/Speckle.Sdk.Tests.Integration/Credentials/AuthFlowExceptionalTests.cs new file mode 100644 index 00000000..32a0afd3 --- /dev/null +++ b/tests/Speckle.Sdk.Tests.Integration/Credentials/AuthFlowExceptionalTests.cs @@ -0,0 +1,84 @@ +using System.Net; +using Microsoft.Extensions.DependencyInjection; +using Speckle.Sdk.Api; +using Speckle.Sdk.Credentials; + +namespace Speckle.Sdk.Tests.Integration.Credentials; + +[Collection(nameof(RequiresAuthFlowPort))] +public class AuthFlowExceptionalTests : IAsyncLifetime +{ + private IAuthFlow _authFlow; + private IClient _client; + private readonly Uri _url = AuthApp.ConnectorsV3.CallbackUrl; + + [Fact] + public async Task GetRefreshToken_Cancellation() + { + await Assert.ThrowsAnyAsync(async () => + _ = await _authFlow.GetRefreshedToken( + _client.Account.refreshToken, + _client.ServerUrl, + Fixtures.TestAuthApp, + new(true) + ) + ); + } + + [Fact] + public async Task GetRefreshToken_UnknownApp() + { + //interestingly, the server responds with a 401 Unauthorized despite internally being a bad request + await Assert.ThrowsAnyAsync(async () => + _ = await _authFlow.GetRefreshedToken( + _client.Account.refreshToken, + _client.ServerUrl, + new() + { + AppId = "doesn't exist", + AppSecret = "doesn't exist", + CallbackUrl = new("invalid://localhost"), + }, + CancellationToken.None + ) + ); + } + + [Fact] + public async Task GetRefreshToken_NullRefreshToken() + { + await Assert.ThrowsAnyAsync(async () => + _ = await _authFlow.GetRefreshedToken(null, _client.ServerUrl, AuthApp.ConnectorsV3, CancellationToken.None) + ); + } + + [Fact] + public async Task SimultaneousListeners_SamePort_OneFails() + { + using CancellationTokenSource ct = new(); + var task1 = AuthFlow.RunListener(_url, ct.Token); + await Task.Delay(50, CancellationToken.None); + + await Assert.ThrowsAsync(async () => await AuthFlow.RunListener(_url, ct.Token)); + + if (task1.IsCompleted) + { + throw new InvalidOperationException("Was expecting task to still be running", task1.Exception); + } + + await ct.CancelAsync(); + await Assert.ThrowsAnyAsync(async () => await task1); + } + + public async Task InitializeAsync() + { + _authFlow = Fixtures.ServiceProvider.GetRequiredService(); + _client = await Fixtures.SeedUserWithClient(); + } + + public Task DisposeAsync() + { + _client.Dispose(); + return Task.CompletedTask; + } +} diff --git a/tests/Speckle.Sdk.Tests.Integration/Credentials/AuthFlowTests.cs b/tests/Speckle.Sdk.Tests.Integration/Credentials/AuthFlowTests.cs new file mode 100644 index 00000000..b5719f7c --- /dev/null +++ b/tests/Speckle.Sdk.Tests.Integration/Credentials/AuthFlowTests.cs @@ -0,0 +1,94 @@ +using Microsoft.Extensions.DependencyInjection; +using Speckle.Sdk.Credentials; + +namespace Speckle.Sdk.Tests.Integration.Credentials; + +[Collection(nameof(RequiresAuthFlowPort))] +public sealed class AuthFlowTests +{ + private readonly IAuthFlow _authFlow; + private readonly Uri _url = AuthApp.ConnectorsV3.CallbackUrl; + + public AuthFlowTests() + { + _authFlow = Fixtures.ServiceProvider.GetRequiredService(); + } + + [Fact] + public async Task RunListener_ReturnsAccessCode_WhenQueryContainsAccessCode() + { + var listenerTask = AuthFlow.RunListener(_url, CancellationToken.None); + using var client = new HttpClient(); + const string EXPECTED_ACCESS_CODE = "abcdef123456"; + + var response = await client.GetAsync(new Uri(_url, $"?access_code={EXPECTED_ACCESS_CODE}")); + response.EnsureSuccessStatusCode(); + + string result = await listenerTask; + + Assert.Equal(EXPECTED_ACCESS_CODE, result); + } + + [Fact] + public async Task RunListener_Throws_InvalidAccessCode() + { + var listenerTask = AuthFlow.RunListener(_url, CancellationToken.None); + using var client = new HttpClient(); + + var response = await client.GetAsync(new Uri(_url, "")); + response.EnsureSuccessStatusCode(); + + await Assert.ThrowsAsync(async () => + { + _ = await listenerTask; + }); + } + + [Fact] + public async Task RunListener_Throws_Cancellation() + { + using CancellationTokenSource cancellationTokenSource = new(); + var listenerTask = AuthFlow.RunListener(_url, cancellationTokenSource.Token); + + await cancellationTokenSource.CancelAsync(); + + await Assert.ThrowsAsync(async () => + { + _ = await listenerTask; + }); + } + + [Theory] + [InlineData(0.1)] + [InlineData(1)] + [InlineData(5)] + public async Task RunListener_Timeout(double timeS) + { + await Assert.ThrowsAsync(async () => + { + _ = await _authFlow.RunListenerWithTimeout(_url, TimeSpan.FromSeconds(timeS), CancellationToken.None); + }); + } + + [Fact] + public async Task CanGetRefreshToken() + { + using var user = await Fixtures.SeedUserWithClient(); + var tokenExchange = await _authFlow.GetRefreshedToken( + user.Account.refreshToken, + user.ServerUrl, + Fixtures.TestAuthApp, + CancellationToken.None + ); + + Assert.NotNull(tokenExchange.token); + Assert.NotNull(tokenExchange.refreshToken); + + user.Account.token = tokenExchange.token; + user.Account.refreshToken = tokenExchange.refreshToken; + + var apiTest = await user.ActiveUser.Get(); + + Assert.NotNull(apiTest); + } +} diff --git a/tests/Speckle.Sdk.Tests.Integration/Fixtures.cs b/tests/Speckle.Sdk.Tests.Integration/Fixtures.cs index 95da56e5..8cab800b 100644 --- a/tests/Speckle.Sdk.Tests.Integration/Fixtures.cs +++ b/tests/Speckle.Sdk.Tests.Integration/Fixtures.cs @@ -27,7 +27,12 @@ namespace Speckle.Sdk.Tests.Integration; public static class Fixtures { public static readonly ServerInfo Server = new() { url = "http://localhost:3000", name = "Docker Server" }; - + public static readonly AuthApp TestAuthApp = new() + { + AppId = "spklwebapp", + AppSecret = "spklwebapp", + CallbackUrl = new Uri("invaid://localhost"), + }; public static IServiceProvider ServiceProvider { get; set; } static Fixtures() @@ -95,8 +100,8 @@ public static class Fixtures Dictionary tokenBody = new() { ["accessCode"] = accessCode, - ["appId"] = "spklwebapp", - ["appSecret"] = "spklwebapp", + ["appId"] = TestAuthApp.AppId, + ["appSecret"] = TestAuthApp.AppSecret, ["challenge"] = "challengingchallenge", }; @@ -109,8 +114,11 @@ public static class Fixtures ); var token = deserialised.NotNull()["token"].NotNull(); + var refreshToken = deserialised.NotNull()["refreshToken"].NotNull(); - return await ServiceProvider.GetRequiredService().CreateAccount(new(Server.url), token); + return await ServiceProvider + .GetRequiredService() + .CreateAccount(new(Server.url), token, refreshToken); } public static Base GenerateSimpleObject() diff --git a/tests/Speckle.Sdk.Tests.Unit/Credentials/AccountManagerTests.cs b/tests/Speckle.Sdk.Tests.Unit/Credentials/AccountManagerTests.cs index 91594975..3e2f5bff 100644 --- a/tests/Speckle.Sdk.Tests.Unit/Credentials/AccountManagerTests.cs +++ b/tests/Speckle.Sdk.Tests.Unit/Credentials/AccountManagerTests.cs @@ -3,7 +3,6 @@ using Moq; using Speckle.Newtonsoft.Json; using Speckle.Sdk.Api.GraphQL.Models; using Speckle.Sdk.Credentials; -using Speckle.Sdk.Helpers; using Speckle.Sdk.SQLite; using Speckle.Sdk.Testing; @@ -28,14 +27,11 @@ public sealed class AccountManagerTests : MoqTest ) => throw new NotImplementedException(); } - private readonly Mock _mockApplication; private readonly Mock> _mockLogger; - private readonly Mock _mockGraphQLClientFactory; - private readonly Mock _mockSpeckleHttp; private readonly IAccountFactory _mockAccountFactory; private readonly Mock _mockSqLiteJsonCacheManagerFactory; private readonly Mock _mockAccountStorage; - private readonly Mock _mockAccountAddLockStorage; + private readonly Mock _mockAuthFlow; #pragma warning disable CA2213 private readonly AccountManager _accountManager; @@ -43,27 +39,19 @@ public sealed class AccountManagerTests : MoqTest public AccountManagerTests() { - _mockApplication = Create(); _mockLogger = Create>(MockBehavior.Loose); - _mockGraphQLClientFactory = Create(); - _mockSpeckleHttp = Create(); _mockAccountFactory = new TestAccountFactory(); _mockSqLiteJsonCacheManagerFactory = Create(); + _mockAuthFlow = Create(); _mockAccountStorage = Create(); - _mockAccountAddLockStorage = Create(); _mockSqLiteJsonCacheManagerFactory.Setup(f => f.CreateForUser("Accounts")).Returns(_mockAccountStorage.Object); - _mockSqLiteJsonCacheManagerFactory - .Setup(f => f.CreateForUser("AccountAddFlow")) - .Returns(_mockAccountAddLockStorage.Object); _accountManager = new AccountManager( - _mockApplication.Object, _mockLogger.Object, - _mockGraphQLClientFactory.Object, - _mockSpeckleHttp.Object, _mockAccountFactory, + _mockAuthFlow.Object, _mockSqLiteJsonCacheManagerFactory.Object ); } @@ -330,71 +318,6 @@ public sealed class AccountManagerTests : MoqTest ); } - [Fact] - public void GetLocalIdentifierForAccount_ReturnsIdentifier_WhenAccountExists() - { - // Arrange - var account = CreateTestAccount("test-account"); - var expectedUri = new Uri($"{account.serverInfo.url}?id={account.userInfo.id}"); - - _mockAccountStorage.Setup(s => s.GetAllObjects()).Returns(new[] { ("bad", JsonConvert.SerializeObject(account)) }); - - // Act - var result = _accountManager.GetLocalIdentifierForAccount(account); - - // Assert - Assert.NotNull(result); - Assert.Equal(expectedUri, result); - } - - [Fact] - public void GetLocalIdentifierForAccount_ReturnsNull_WhenAccountDoesNotExist() - { - // Arrange - var account = CreateTestAccount("non-existent-account"); - - _mockAccountStorage.Setup(s => s.GetAllObjects()).Returns([]); - - // Act - var result = _accountManager.GetLocalIdentifierForAccount(account); - - // Assert - Assert.Null(result); - } - - [Fact] - public void GetAccountForLocalIdentifier_ReturnsAccount_WhenMatches() - { - // Arrange - var account = CreateTestAccount("test-account"); - var localIdentifier = new Uri($"{account.serverInfo.url}?id={account.userInfo.id}"); - - _mockAccountStorage.Setup(s => s.GetAllObjects()).Returns(new[] { ("bad", JsonConvert.SerializeObject(account)) }); - - // Act - var result = _accountManager.GetAccountForLocalIdentifier(localIdentifier); - - // Assert - Assert.NotNull(result); - Assert.Equal(account.id, result!.id); - } - - [Fact] - public void GetAccountForLocalIdentifier_ReturnsNull_WhenNoMatch() - { - // Arrange - var account = CreateTestAccount("test-account"); - var localIdentifier = new Uri("https://different.url?u=different-user"); - - _mockAccountStorage.Setup(s => s.GetAllObjects()).Returns(new[] { ("bad", JsonConvert.SerializeObject(account)) }); - - // Act - var result = _accountManager.GetAccountForLocalIdentifier(localIdentifier); - - // Assert - Assert.Null(result); - } - // Helper method to create a test account private static Account CreateTestAccount(string id) { diff --git a/tests/Speckle.Sdk.Tests.Unit/Credentials/Accounts.cs b/tests/Speckle.Sdk.Tests.Unit/Credentials/Accounts.cs index 44607509..d9046d61 100644 --- a/tests/Speckle.Sdk.Tests.Unit/Credentials/Accounts.cs +++ b/tests/Speckle.Sdk.Tests.Unit/Credentials/Accounts.cs @@ -48,7 +48,7 @@ public class CredentialInfrastructure : IDisposable { Fixtures.UpdateOrSaveAccount(s_testAccount1); Fixtures.UpdateOrSaveAccount(s_testAccount2); - Fixtures.SaveLocalAccount(s_testAccount3); + Fixtures.UpdateOrSaveAccount(s_testAccount3); var serviceProvider = TestServiceSetup.GetServiceProvider(); _accountManager = serviceProvider.GetRequiredService(); @@ -60,7 +60,6 @@ public class CredentialInfrastructure : IDisposable Fixtures.DeleteLocalAccount(s_testAccount1.id); Fixtures.DeleteLocalAccount(s_testAccount2.id); Fixtures.DeleteLocalAccount(s_testAccount3.id); - Fixtures.DeleteLocalAccountFile(); } [Fact] @@ -93,7 +92,7 @@ public class CredentialInfrastructure : IDisposable { var accs = _accountManager.GetAccounts(target.serverInfo.url).ToList(); - accs.Count.Should().Be(1); + accs.Should().HaveCount(1); var acc = accs[0]; @@ -103,24 +102,4 @@ public class CredentialInfrastructure : IDisposable acc.refreshToken.Should().Be(target.refreshToken); acc.token.Should().Be(target.token); } - - [Fact] - public void EnsureLocalIdentifiers_AreUniqueAcrossServers() - { - // Accounts with the same user ID in different servers should always result in different local identifiers. - string id = "12345"; - var acc1 = new Account - { - serverInfo = new ServerInfo { url = "https://speckle.xyz" }, - userInfo = new UserInfo { id = id }, - }.GetLocalIdentifier(); - - var acc2 = new Account - { - serverInfo = new ServerInfo { url = "https://app.speckle.systems" }, - userInfo = new UserInfo { id = id }, - }.GetLocalIdentifier(); - - acc1.Should().NotBe(acc2); - } } diff --git a/tests/Speckle.Sdk.Tests.Unit/Credentials/AuthFlowTests.cs b/tests/Speckle.Sdk.Tests.Unit/Credentials/AuthFlowTests.cs new file mode 100644 index 00000000..44397f3b --- /dev/null +++ b/tests/Speckle.Sdk.Tests.Unit/Credentials/AuthFlowTests.cs @@ -0,0 +1,36 @@ +using Speckle.Sdk.Credentials; + +namespace Speckle.Sdk.Tests.Unit.Credentials; + +public class AuthFlowTests +{ + private const int REPEAT = 20; + + [Fact] + public void GenerateChallenge_ReturnsValidUniqueChallenge() + { + var codeVerifiers = Enumerable.Range(0, REPEAT).Select(_ => AuthFlow.GenerateCodeVerifier()).ToArray(); + + Assert.All( + codeVerifiers, + item => + { + Assert.Equal(43, item.Length); + Assert.Matches(@"^[A-Za-z0-9\-_+/]*$", item); + } + ); + + Assert.Equivalent(codeVerifiers, codeVerifiers.Distinct()); + var challenges = codeVerifiers.Select(AuthFlow.GenerateCodeChallenge).ToArray(); + + Assert.All( + challenges, + item => + { + Assert.Equal(43, item.Length); + Assert.Matches(@"^[A-Za-z0-9\-_+/]*$", item); + } + ); + Assert.Equivalent(challenges, challenges.Distinct()); + } +} diff --git a/tests/Speckle.Sdk.Tests.Unit/Fixtures.cs b/tests/Speckle.Sdk.Tests.Unit/Fixtures.cs index 65f49f12..596a3c9e 100644 --- a/tests/Speckle.Sdk.Tests.Unit/Fixtures.cs +++ b/tests/Speckle.Sdk.Tests.Unit/Fixtures.cs @@ -1,7 +1,6 @@ using Newtonsoft.Json; using Speckle.Sdk.Common; using Speckle.Sdk.Credentials; -using Speckle.Sdk.Logging; using Speckle.Sdk.Transports; namespace Speckle.Sdk.Tests.Unit; @@ -10,11 +9,6 @@ public abstract class Fixtures { private static readonly SQLiteTransport s_accountStorage = new(scope: "Accounts"); - private static readonly string s_accountPath = Path.Combine( - SpecklePathProvider.AccountsFolderPath, - "TestAccount.json" - ); - public static void UpdateOrSaveAccount(Account account) { DeleteLocalAccount(account.id.NotNull()); @@ -22,13 +16,5 @@ public abstract class Fixtures s_accountStorage.SaveObjectSync(account.id, serializedObject); } - public static void SaveLocalAccount(Account account) - { - var json = JsonConvert.SerializeObject(account); - File.WriteAllText(s_accountPath, json); - } - public static void DeleteLocalAccount(string id) => s_accountStorage.DeleteObject(id); - - public static void DeleteLocalAccountFile() => File.Delete(s_accountPath); } From 7f50987201c5afa4540776400b53793a8524af03 Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Thu, 26 Mar 2026 13:42:21 +0000 Subject: [PATCH 21/26] Auth flow updates (#464) * Auth flow updates * remove obsolete flag on refresh --- src/Speckle.Sdk/Credentials/AccountManager.cs | 49 +++------------- src/Speckle.Sdk/Credentials/AuthFlow.cs | 57 ++++++++++--------- 2 files changed, 38 insertions(+), 68 deletions(-) diff --git a/src/Speckle.Sdk/Credentials/AccountManager.cs b/src/Speckle.Sdk/Credentials/AccountManager.cs index d20a3473..d464f4e9 100644 --- a/src/Speckle.Sdk/Credentials/AccountManager.cs +++ b/src/Speckle.Sdk/Credentials/AccountManager.cs @@ -142,59 +142,26 @@ public sealed class AccountManager( /// /// Refetches the information, including and - /// If the looks to be expired, this function will also attempt to use the to refresh it. /// /// Will only mutate in memory only, and only if successful. /// /// /// /// - /// Thrown if + /// public async Task UpdateAccountInMemory(Account account, CancellationToken cancellationToken = default) { Uri url = account.serverInfo.migration?.movedTo ?? new(account.serverInfo.url); - ActiveUserServerInfoResponse userServerInfo; - try + ActiveUserServerInfoResponse userServerInfo = await accountFactory + .GetUserServerInfo(url, account.token, cancellationToken) + .ConfigureAwait(false); + + if (userServerInfo.activeUser == null) { - userServerInfo = await accountFactory - .GetUserServerInfo(url, account.token, cancellationToken) - .ConfigureAwait(false); + throw new SpeckleException("GraphQL response indicated that the ActiveUser could not be found"); } - catch (GraphQLHttpRequestException ex) - { - // Failed to fetch info, perhaps the token is expired? - // Attempt to refresh it - TokenExchangeResponse refreshTokenResponse; - try - { - refreshTokenResponse = await authFlow - .GetRefreshedToken( - account.refreshToken.NotNull("No refresh token provided"), - url, - AuthApp.ConnectorsV3, - cancellationToken - ) - .ConfigureAwait(false); - - userServerInfo = await accountFactory - .GetUserServerInfo(url, refreshTokenResponse.token, cancellationToken) - .ConfigureAwait(false); - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - throw; - } - catch (Exception ex2) - { - throw new AggregateException("Failed to update account information", ex, ex2); - } - - account.token = refreshTokenResponse.token; - account.refreshToken = refreshTokenResponse.refreshToken; - logger.LogInformation(ex, "Account token has been refreshed"); - } - account.userInfo = userServerInfo.activeUser.NotNull(); + account.userInfo = userServerInfo.activeUser; account.serverInfo = userServerInfo.serverInfo; //This is a bit gross, since id is not marked nullable //but this will force re-generate the id (e.g. if the user's email, or servers url has changed) diff --git a/src/Speckle.Sdk/Credentials/AuthFlow.cs b/src/Speckle.Sdk/Credentials/AuthFlow.cs index f849da97..6a380dd3 100644 --- a/src/Speckle.Sdk/Credentials/AuthFlow.cs +++ b/src/Speckle.Sdk/Credentials/AuthFlow.cs @@ -1,4 +1,5 @@ using System.Diagnostics; +using System.Diagnostics.Contracts; using System.Net; using System.Net.Http.Headers; using System.Security.Cryptography; @@ -42,34 +43,45 @@ public sealed class AuthFlow(ISdkActivityFactory activityFactory, ISpeckleHttp s Uri tokenEndpoint = new(serverUrl, "/oauth/token"); string codeVerifier = GenerateCodeVerifier(); - string challenge; - string codeChallengeMethod; - var req = await client.GetAsync(tokenEndpoint, cancellationToken).ConfigureAwait(false); + Uri authnVerify; + using var req = await client.GetAsync(tokenEndpoint, cancellationToken).ConfigureAwait(false); bool useLegacyEndpoint = req.StatusCode != HttpStatusCode.OK; if (useLegacyEndpoint) { - challenge = codeVerifier; + string challenge = codeVerifier; // Old endpoint only supports PKCE "plain" method + authnVerify = new($"/authn/verify/{authApp.AppId}/{challenge}", UriKind.Relative); tokenEndpoint = new(serverUrl, "/auth/token"); - codeChallengeMethod = ""; } else { - challenge = GenerateCodeChallenge(codeVerifier); - codeChallengeMethod = "?code_challenge_method=S256"; + string challenge = GenerateCodeChallenge(codeVerifier); + authnVerify = new($"/authn/verify/{authApp.AppId}/{challenge}?code_challenge_method=S256", UriKind.Relative); } - Uri endpoint = new(serverUrl, $"/authn/verify/{authApp.AppId}/{challenge}{codeChallengeMethod}"); + Uri endpoint = new(serverUrl, authnVerify); _ = Process.Start(new ProcessStartInfo(endpoint.ToString()) { UseShellExecute = true }); string accessCode = await RunListenerWithTimeout(authApp.CallbackUrl, timeout, cancellationToken) .ConfigureAwait(false); + object body = useLegacyEndpoint + ? new + { + appId = authApp.AppId, + appSecret = authApp.AppSecret, + accessCode = accessCode, + challenge = codeVerifier, + } + : new + { + appId = authApp.AppId, + accessCode = accessCode, + codeVerifier = codeVerifier, + }; + return await ExchangeAccessCodeForToken( client, - accessCode, - authApp, - useLegacyEndpoint ? challenge : null, - !useLegacyEndpoint ? codeVerifier : null, + JsonConvert.SerializeObject(body, _serializerSettings), tokenEndpoint, cancellationToken ) @@ -141,7 +153,7 @@ public sealed class AuthFlow(ISdkActivityFactory activityFactory, ISpeckleHttp s refreshToken = refreshToken, }; - using var content = new StringContent(JsonConvert.SerializeObject(body)); + using var content = new StringContent(JsonConvert.SerializeObject(body, _serializerSettings)); content.Headers.ContentType = new MediaTypeHeaderValue("application/json"); var response = await client .PostAsync(new Uri(serverUrl, "/auth/token"), content, cancellationToken) @@ -248,24 +260,12 @@ public sealed class AuthFlow(ISdkActivityFactory activityFactory, ISpeckleHttp s private async Task ExchangeAccessCodeForToken( HttpClient client, - string accessCode, - AuthApp authApp, - string? challenge, - string? codeVerifier, + string jsonContent, Uri tokenEndpoint, CancellationToken cancellationToken ) { - var body = new - { - appId = authApp.AppId, - appSecret = authApp.AppSecret, - accessCode = accessCode, - challenge = challenge, - codeVerifier = codeVerifier, - }; - - using StringContent content = new(JsonConvert.SerializeObject(body, _serializerSettings)); + using StringContent content = new(jsonContent); content.Headers.ContentType = new MediaTypeHeaderValue("application/json"); using HttpResponseMessage response = await client @@ -282,6 +282,7 @@ public sealed class AuthFlow(ISdkActivityFactory activityFactory, ISpeckleHttp s return JsonConvert.DeserializeObject(read, _serializerSettings).NotNull(); } + [Pure] public static string GenerateCodeVerifier() { #if NET8_0_OR_GREATER @@ -296,6 +297,7 @@ public sealed class AuthFlow(ISdkActivityFactory activityFactory, ISpeckleHttp s return Base64UrlEncode(codeVerifierData); } + [Pure] public static string GenerateCodeChallenge(string codeVerifier) { #if NET8_0_OR_GREATER @@ -312,6 +314,7 @@ public sealed class AuthFlow(ISdkActivityFactory activityFactory, ISpeckleHttp s return Base64UrlEncode(challengeData); } + [Pure] private static string Base64UrlEncode( #if NET8_0_OR_GREATER ReadOnlySpan bytes From c81692ee5ab1d3372182b07167c8b249a1eb6001 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Mar 2026 21:42:44 +0100 Subject: [PATCH 22/26] chore(deps): bump codecov/codecov-action from 5 to 6 (#465) Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 5 to 6. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v5...v6) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/integration-test.yml | 2 +- .github/workflows/pr.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 865b1143..7bdc42b4 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -54,7 +54,7 @@ jobs: run: dotnet test ${{ env.Solution }} --filter "(Category=Integration)&(Server!=Public)" --configuration Release --no-build --no-restore --verbosity=normal /p:AltCover=true /p:AltCoverAttributeFilter=ExcludeFromCodeCoverage - name: Upload coverage reports to Codecov with GitHub Action - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@v6 continue-on-error: true with: fail_ci_if_error: true diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 637c7da8..bfb62cf5 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -42,7 +42,7 @@ jobs: run: dotnet pack ${{ env.Solution }} --configuration Release --no-build - name: Upload coverage reports to Codecov with GitHub Action - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@v6 continue-on-error: true with: fail_ci_if_error: true From 74d40e40a920217325b9fbd5b21bc1a8fb79aad2 Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Tue, 31 Mar 2026 11:07:08 +0100 Subject: [PATCH 23/26] Optimization for the disk store string writing to avoid memory allocations (#466) --- src/Speckle.Sdk/Api/GraphQL/Client.cs | 4 ++-- src/Speckle.Sdk/Pipelines/Send/DiskStore.cs | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/Speckle.Sdk/Api/GraphQL/Client.cs b/src/Speckle.Sdk/Api/GraphQL/Client.cs index e53064be..dc66df02 100644 --- a/src/Speckle.Sdk/Api/GraphQL/Client.cs +++ b/src/Speckle.Sdk/Api/GraphQL/Client.cs @@ -154,10 +154,10 @@ public sealed class Client : ISpeckleGraphQLClient, IClient activity?.SetStatus(SdkActivityStatusCode.Ok); return ret; } - catch (Exception) + catch (Exception ex) { activity?.SetStatus(SdkActivityStatusCode.Error); - // Don't record exception as it's rethrown. + activity?.RecordException(ex); throw; } } diff --git a/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs b/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs index 88443633..172de132 100644 --- a/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs +++ b/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs @@ -65,7 +65,12 @@ public sealed class DiskStore await foreach (var item in _channel.ReadAllAsync(_cancellationToken).ConfigureAwait(false)) { - await writer.WriteLineAsync($"{item.Id}\t{item.SpeckleType}\t{item.Json}").ConfigureAwait(false); + await writer.WriteAsync(item.Id).ConfigureAwait(false); + await writer.WriteAsync('\t').ConfigureAwait(false); + await writer.WriteAsync(item.SpeckleType).ConfigureAwait(false); + await writer.WriteAsync('\t').ConfigureAwait(false); + await writer.WriteAsync(item.Json.Value).ConfigureAwait(false); + await writer.WriteLineAsync().ConfigureAwait(false); } #if NET8_0_OR_GREATER await writer.FlushAsync(_cancellationToken).ConfigureAwait(false); From 719581bc12c083de08fa58c2f682c36a3f24b21a Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Thu, 2 Apr 2026 12:36:02 +0100 Subject: [PATCH 24/26] Record activity exceptions (#467) --- src/Speckle.Sdk/Pipelines/Send/Uploader.cs | 103 +++++++++++++-------- 1 file changed, 65 insertions(+), 38 deletions(-) diff --git a/src/Speckle.Sdk/Pipelines/Send/Uploader.cs b/src/Speckle.Sdk/Pipelines/Send/Uploader.cs index 0b7f2126..133b3d8e 100644 --- a/src/Speckle.Sdk/Pipelines/Send/Uploader.cs +++ b/src/Speckle.Sdk/Pipelines/Send/Uploader.cs @@ -65,63 +65,90 @@ public sealed class Uploader : IDisposable { using var a = _activity.Start("Get Presigned Url"); - var signUri = new Uri($"projects/{_projectId}/modelingestion/{_ingestionId}/uploads/sign", UriKind.Relative); + try + { + var signUri = new Uri($"projects/{_projectId}/modelingestion/{_ingestionId}/uploads/sign", UriKind.Relative); - using var signResponse = await _speckleClient.PostAsync(signUri, null, _cancellationToken).ConfigureAwait(false); - signResponse.EnsureSuccessStatusCode(); + using var signResponse = await _speckleClient.PostAsync(signUri, null, _cancellationToken).ConfigureAwait(false); + signResponse.EnsureSuccessStatusCode(); #if NET5_0_OR_GREATER - string signResponseString = await signResponse.Content.ReadAsStringAsync(_cancellationToken).ConfigureAwait(false); + string signResponseString = await signResponse + .Content.ReadAsStringAsync(_cancellationToken) + .ConfigureAwait(false); #else - string signResponseString = await signResponse.Content.ReadAsStringAsync().ConfigureAwait(false); + string signResponseString = await signResponse.Content.ReadAsStringAsync().ConfigureAwait(false); #endif - PresignedUploadResponse presignedUpload = - JsonConvert.DeserializeObject(signResponseString) - ?? throw new InvalidOperationException("Failed to get presigned upload URL"); - return presignedUpload; + PresignedUploadResponse presignedUpload = + JsonConvert.DeserializeObject(signResponseString) + ?? throw new InvalidOperationException("Failed to get presigned upload URL"); + return presignedUpload; + } + catch (Exception ex) + { + a?.SetStatus(SdkActivityStatusCode.Error); + a?.RecordException(ex); + throw; + } } private async Task UploadToS3(Stream fileStream, PresignedUploadResponse presignedUploadResponse) { using var a = _activity.Start("Uploading file to pre-signed url"); - - Stream progressStream = new ProgressStream(fileStream, _progress); - - using var streamContent = new StreamContent(progressStream); - streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); - streamContent.Headers.ContentLength = fileStream.Length; - - using var uploadRequest = new HttpRequestMessage(HttpMethod.Put, presignedUploadResponse.Url); - foreach (var kvp in presignedUploadResponse.AdditionalRequestHeaders) + try { - uploadRequest.Headers.Add(kvp.Key, kvp.Value); + Stream progressStream = new ProgressStream(fileStream, _progress); + + using var streamContent = new StreamContent(progressStream); + streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); + streamContent.Headers.ContentLength = fileStream.Length; + + using var uploadRequest = new HttpRequestMessage(HttpMethod.Put, presignedUploadResponse.Url); + foreach (var kvp in presignedUploadResponse.AdditionalRequestHeaders) + { + uploadRequest.Headers.Add(kvp.Key, kvp.Value); + } + + uploadRequest.Content = streamContent; + + using var uploadResponse = await _s3Client + .SendAsync(uploadRequest, HttpCompletionOption.ResponseHeadersRead, _cancellationToken) + .ConfigureAwait(false); + + uploadResponse.EnsureSuccessStatusCode(); + + return BlobApiHelpers.ParseEtagHeader(uploadResponse.Headers); + } + catch (Exception ex) + { + a?.SetStatus(SdkActivityStatusCode.Error); + a?.RecordException(ex); + throw; } - - uploadRequest.Content = streamContent; - - using var uploadResponse = await _s3Client - .SendAsync(uploadRequest, HttpCompletionOption.ResponseHeadersRead, _cancellationToken) - .ConfigureAwait(false); - - uploadResponse.EnsureSuccessStatusCode(); - - return BlobApiHelpers.ParseEtagHeader(uploadResponse.Headers); } private async Task TriggerProcessing(TriggerUploadRequest request) { using var a = _activity.Start("Triggering Processing"); + try + { + Uri processUri = new($"projects/{_projectId}/modelingestion/{_ingestionId}/uploads/process", UriKind.Relative); + string requestBody = JsonConvert.SerializeObject(request); + using var content = new StringContent(requestBody, Encoding.UTF8, "application/json"); - Uri processUri = new($"projects/{_projectId}/modelingestion/{_ingestionId}/uploads/process", UriKind.Relative); - string requestBody = JsonConvert.SerializeObject(request); - using var content = new StringContent(requestBody, Encoding.UTF8, "application/json"); + using HttpResponseMessage processResponse = await _speckleClient + .PostAsync(processUri, content, _cancellationToken) + .ConfigureAwait(false); - using HttpResponseMessage processResponse = await _speckleClient - .PostAsync(processUri, content, _cancellationToken) - .ConfigureAwait(false); - - string body = await processResponse.Content.ReadAsStringAsync().ConfigureAwait(false); - processResponse.EnsureSuccessStatusCode(); + string body = await processResponse.Content.ReadAsStringAsync().ConfigureAwait(false); + processResponse.EnsureSuccessStatusCode(); + } + catch (Exception ex) + { + a?.SetStatus(SdkActivityStatusCode.Error); + a?.RecordException(ex); + throw; + } } public void Dispose() From f06b361fc03c1fa527800cd21e9192da70900ebf Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Mon, 13 Apr 2026 11:51:31 +0100 Subject: [PATCH 25/26] Write objects in reverse order instead of root object twice (#470) --- src/Speckle.Sdk/Pipelines/Send/SendPipeline.cs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/Speckle.Sdk/Pipelines/Send/SendPipeline.cs b/src/Speckle.Sdk/Pipelines/Send/SendPipeline.cs index 3ddd4249..55613c6d 100644 --- a/src/Speckle.Sdk/Pipelines/Send/SendPipeline.cs +++ b/src/Speckle.Sdk/Pipelines/Send/SendPipeline.cs @@ -36,28 +36,22 @@ public sealed class SendPipeline : IDisposable _diskStore = diskStore; } - private UploadItem _lastItem; - public async Task Process(Base @base) { var results = _serializer.Serialize(@base).ToArray(); var first = results.First(); - foreach (var item in results) + // .Reverse ensures the root commit object is written last. + foreach (var item in results.Reverse()) { // we're not doing fire and forget here so that we get the backpressure from the uploader await _diskStore.PushAsync(item).ConfigureAwait(false); } - // NOTE: this is important to keep track of. When we serialze an object, we get back a list of objects, with the first one being the original root. - // In the case of the commit root object, this means the last object is not necessarily the root; we therefore need to manually track its existance here - // and ensure it's the last one through in the uploader's stream. See WaitForUpload down below. - _lastItem = first; return first.Reference; } public async Task WaitForUpload() { - await _diskStore.PushAsync(_lastItem).ConfigureAwait(false); using DisposableFile tempFile = await _diskStore.CompleteAsync().ConfigureAwait(false); using Stream fileStreamUpload = new FileStream( From 35ff9cb2bf6cae4ce7f8493fbd48cb26d390ed86 Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Mon, 13 Apr 2026 11:51:46 +0100 Subject: [PATCH 26/26] nullprogress (#469) --- src/Speckle.Sdk/Pipelines/Progress/NullProgress.cs | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 src/Speckle.Sdk/Pipelines/Progress/NullProgress.cs diff --git a/src/Speckle.Sdk/Pipelines/Progress/NullProgress.cs b/src/Speckle.Sdk/Pipelines/Progress/NullProgress.cs new file mode 100644 index 00000000..0cf95b0e --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Progress/NullProgress.cs @@ -0,0 +1,6 @@ +namespace Speckle.Sdk.Pipelines.Progress; + +public sealed class NullProgress : IProgress +{ + public void Report(T value) { } +}