From 4980796cd6d1b05266735af8ec0a91eba577aef7 Mon Sep 17 00:00:00 2001 From: Dimitrie Stefanescu Date: Tue, 10 Feb 2026 10:17:38 +0000 Subject: [PATCH 01/16] Dim/flopper experiments (#438) * Adds object flopper for experimental object uploads Implements a new mechanism for uploading objects to the server. * wip * pipe works * WIP * wip - refactors send pipeline --- Directory.Packages.props | 9 +- src/Speckle.Objects/packages.lock.json | 203 ++++++++-- .../packages.lock.json | 34 +- .../Models/DynamicBaseMemberType.cs | 6 +- src/Speckle.Sdk/Pipelines/ProgressStream.cs | 97 +++++ src/Speckle.Sdk/Pipelines/SendPipeline.cs | 59 +++ src/Speckle.Sdk/Pipelines/Serializer.cs | 372 ++++++++++++++++++ src/Speckle.Sdk/Pipelines/Uploader.cs | 175 ++++++++ src/Speckle.Sdk/Pipelines/Uploader_Old.cs | 83 ++++ .../Serialisation/V2/Send/BaseItem.cs | 8 +- .../Serialisation/V2/Send/ObjectFlopper.cs | 79 ++++ .../V2/Send/ObjectFlopperGandalf.cs | 90 +++++ .../Serialisation/V2/Send/SerializeProcess.cs | 11 +- .../V2/SerializeProcessFactory.cs | 3 +- src/Speckle.Sdk/Speckle.Sdk.csproj | 6 +- src/Speckle.Sdk/packages.lock.json | 257 +++++++++++- 16 files changed, 1413 insertions(+), 79 deletions(-) create mode 100644 src/Speckle.Sdk/Pipelines/ProgressStream.cs create mode 100644 src/Speckle.Sdk/Pipelines/SendPipeline.cs create mode 100644 src/Speckle.Sdk/Pipelines/Serializer.cs create mode 100644 src/Speckle.Sdk/Pipelines/Uploader.cs create mode 100644 src/Speckle.Sdk/Pipelines/Uploader_Old.cs create mode 100644 src/Speckle.Sdk/Serialisation/V2/Send/ObjectFlopper.cs create mode 100644 src/Speckle.Sdk/Serialisation/V2/Send/ObjectFlopperGandalf.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 86c51244..6bf24322 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -8,6 +8,8 @@ + + @@ -16,7 +18,6 @@ - @@ -28,7 +29,9 @@ - + + + @@ -39,4 +42,4 @@ - + \ No newline at end of file diff --git a/src/Speckle.Objects/packages.lock.json b/src/Speckle.Objects/packages.lock.json index 17abca99..2c2e04f1 100644 --- a/src/Speckle.Objects/packages.lock.json +++ b/src/Speckle.Objects/packages.lock.json @@ -130,6 +130,19 @@ "resolved": "8.0.0", "contentHash": "dk9JPxTCIevS75HyEQ0E4OVAFhB2N+V9ShCXf8Q6FkUQZDkgLI12y679Nym1YqsiSysuQskT7Z+6nUf3yab6Vw==" }, + "Newtonsoft.Json": { + "type": "Transitive", + "resolved": "13.0.1", + "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" + }, + "Newtonsoft.Json.Bson": { + "type": "Transitive", + "resolved": "1.0.2", + "contentHash": "QYFyxhaABwmq3p/21VrZNYvCg3DaEoN/wUuw5nmfAf0X3HLjgupwhkEWdgfb9nvGAUIv3osmZoD3kKl4jxEmYQ==", + "dependencies": { + "Newtonsoft.Json": "12.0.1" + } + }, "SQLitePCLRaw.bundle_e_sqlite3": { "type": "Transitive", "resolved": "2.1.4", @@ -162,8 +175,8 @@ }, "System.Buffers": { "type": "Transitive", - "resolved": "4.4.0", - "contentHash": "AwarXzzoDwX6BgrhjoJsk6tUezZEozOT5Y9QKF94Gl4JK91I4PIIBkBco9068Y9/Dra8Dkbie99kXB8+1BaYKw==" + "resolved": "4.6.1", + "contentHash": "N8GXpmiLMtljq7gwvyS+1QvKT/W2J8sNAvx+HVg4NGmsG/H+2k/y9QI23auLJRterrzCiDH+IWAw4V/GPwsMlw==" }, "System.ComponentModel.Annotations": { "type": "Transitive", @@ -172,18 +185,18 @@ }, "System.Memory": { "type": "Transitive", - "resolved": "4.5.3", - "contentHash": "3oDzvc/zzetpTKWMShs1AADwZjQ/36HnsufHRPcOjyRAAMLDlu2iD33MBI2opxnezcVUtXyqDXXjoFMOU9c7SA==", + "resolved": "4.6.3", + "contentHash": "qdcDOgnFZY40+Q9876JUHnlHu7bosOHX8XISRoH94fwk6hgaeQGSgfZd8srWRZNt5bV9ZW2TljcegDNxsf+96A==", "dependencies": { - "System.Buffers": "4.4.0", - "System.Numerics.Vectors": "4.4.0", - "System.Runtime.CompilerServices.Unsafe": "4.5.2" + "System.Buffers": "4.6.1", + "System.Numerics.Vectors": "4.6.1", + "System.Runtime.CompilerServices.Unsafe": "6.1.2" } }, "System.Numerics.Vectors": { "type": "Transitive", - "resolved": "4.4.0", - "contentHash": "UiLzLW+Lw6HLed1Hcg+8jSRttrbuXv7DANVj0DkL9g6EnnzbL75EB7EWsw5uRbhxd/4YdG8li5XizGWepmG3PQ==" + "resolved": "4.6.1", + "contentHash": "sQxefTnhagrhoq2ReR0D/6K0zJcr9Hrd6kikeXsA1I8kOCboTavcUC4r7TSfpKFeE163uMuxZcyfO1mGO3EN8Q==" }, "System.Reactive": { "type": "Transitive", @@ -205,8 +218,8 @@ }, "System.Runtime.CompilerServices.Unsafe": { "type": "Transitive", - "resolved": "4.5.3", - "contentHash": "3TIsJhD1EiiT0w2CcDMN/iSSwnNnsrnbzeVHSKkaEgV85txMprmuO+Yq2AdSbeVGcg28pdNDTPK87tJhX7VFHw==" + "resolved": "6.1.2", + "contentHash": "2hBr6zdbIBTDE3EhK7NSVNdX58uTK6iHW/P/Axmm9sl1xoGSLqDvMtpecn226TNwHByFokYwJmt/aQQNlO5CRw==" }, "System.Runtime.InteropServices.WindowsRuntime": { "type": "Transitive", @@ -216,26 +229,40 @@ "System.Runtime": "4.3.0" } }, + "System.Text.Encodings.Web": { + "type": "Transitive", + "resolved": "10.0.1", + "contentHash": "cVAka0o1rJJ5/De0pjNs7jcaZk5hUGf1HGzUyVmE2MEB1Vf0h/8qsWxImk1zjitCbeD2Avaq2P2+usdvqgbeVQ==", + "dependencies": { + "System.Buffers": "4.6.1", + "System.Memory": "4.6.3", + "System.Runtime.CompilerServices.Unsafe": "6.1.2" + } + }, "System.Threading.Tasks.Extensions": { "type": "Transitive", - "resolved": "4.5.4", - "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==", + "resolved": "4.6.3", + "contentHash": "7sCiwilJLYbTZELaKnc7RecBBXWXA+xMLQWZKWawBxYjp6DBlSE3v9/UcvKBvr1vv2tTOhipiogM8rRmxlhrVA==", "dependencies": { - "System.Runtime.CompilerServices.Unsafe": "4.5.3" + "System.Runtime.CompilerServices.Unsafe": "6.1.2" } }, "speckle.sdk": { "type": "Project", "dependencies": { "GraphQL.Client": "[6.0.0, )", - "Microsoft.Bcl.AsyncInterfaces": "[5.0.0, )", + "Microsoft.AspNet.WebApi.Client": "[6.0.0, )", + "Microsoft.Bcl.AsyncInterfaces": "[10.0.1, )", "Microsoft.CSharp": "[4.7.0, )", "Microsoft.Data.Sqlite": "[7.0.5, )", "Microsoft.Extensions.DependencyInjection.Abstractions": "[2.2.0, )", "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )", + "System.IO.Pipelines": "[10.0.1, )", + "System.Net.Http.Json": "[10.0.1, )", + "System.Threading.Channels": "[10.0.1, )" } }, "speckle.sdk.dependencies": { @@ -252,13 +279,25 @@ "System.Reactive": "5.0.0" } }, + "Microsoft.AspNet.WebApi.Client": { + "type": "CentralTransitive", + "requested": "[6.0.0, )", + "resolved": "6.0.0", + "contentHash": "zXeWP03dTo67AoDHUzR+/urck0KFssdCKOC+dq7Nv1V2YbFh/nIg09L0/3wSvyRONEdwxGB/ssEGmPNIIhAcAw==", + "dependencies": { + "Newtonsoft.Json": "13.0.1", + "Newtonsoft.Json.Bson": "1.0.2", + "System.Memory": "4.5.5", + "System.Threading.Tasks.Extensions": "4.5.4" + } + }, "Microsoft.Bcl.AsyncInterfaces": { "type": "CentralTransitive", - "requested": "[5.0.0, )", - "resolved": "5.0.0", - "contentHash": "W8DPQjkMScOMTtJbPwmPyj9c3zYSFGawDW3jwlBOOsnY+EzZFLgNQ/UMkK35JmkNOVPdCyPr2Tw7Vv9N+KA3ZQ==", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "E1HSLkPHXEO30JEij2pWbOuzz1Z5ND4a5l7IP1T2RgQuE0a0NzEIvtO64RNy3Otn6PFezbT80cfm3M/Cgt70PA==", "dependencies": { - "System.Threading.Tasks.Extensions": "4.5.4" + "System.Threading.Tasks.Extensions": "4.6.3" } }, "Microsoft.CSharp": { @@ -306,6 +345,54 @@ "requested": "[13.0.2, )", "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" + }, + "System.IO.Pipelines": { + "type": "CentralTransitive", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "26LbFXHKd7PmRnWlkjnYgmjd5B6HYVG+1MpTO25BdxTJnx6D0O16JPAC/S4YBqjtt4YpfGj1QO/Ss6SPMGEGQw==", + "dependencies": { + "System.Buffers": "4.6.1", + "System.Memory": "4.6.3", + "System.Threading.Tasks.Extensions": "4.6.3" + } + }, + "System.Net.Http.Json": { + "type": "CentralTransitive", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "XGOWt78ccgO9esyNlCemlS2b8JZPrH85pk/MvdHJxp6KwwUY/GnDaw2fPpJa7lgotiTkWnXnhip+ULNKaP7a8A==", + "dependencies": { + "System.Buffers": "4.6.1", + "System.Memory": "4.6.3", + "System.Text.Json": "10.0.1", + "System.Threading.Tasks.Extensions": "4.6.3" + } + }, + "System.Text.Json": { + "type": "CentralTransitive", + "requested": "[8.0.5, )", + "resolved": "10.0.1", + "contentHash": "EsgwDgU1PFqhrFA9l5n+RBu76wFhNGCEwu8ITrBNhjPP3MxLyklroU5GIF8o6JYpYg6T4KD/VICfMdgPAvNp5g==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "10.0.1", + "System.Buffers": "4.6.1", + "System.IO.Pipelines": "10.0.1", + "System.Memory": "4.6.3", + "System.Runtime.CompilerServices.Unsafe": "6.1.2", + "System.Text.Encodings.Web": "10.0.1", + "System.Threading.Tasks.Extensions": "4.6.3" + } + }, + "System.Threading.Channels": { + "type": "CentralTransitive", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "10.0.1", + "System.Threading.Tasks.Extensions": "4.6.3" + } } }, "net8.0": { @@ -418,6 +505,19 @@ "resolved": "8.0.0", "contentHash": "dk9JPxTCIevS75HyEQ0E4OVAFhB2N+V9ShCXf8Q6FkUQZDkgLI12y679Nym1YqsiSysuQskT7Z+6nUf3yab6Vw==" }, + "Newtonsoft.Json": { + "type": "Transitive", + "resolved": "13.0.1", + "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" + }, + "Newtonsoft.Json.Bson": { + "type": "Transitive", + "resolved": "1.0.2", + "contentHash": "QYFyxhaABwmq3p/21VrZNYvCg3DaEoN/wUuw5nmfAf0X3HLjgupwhkEWdgfb9nvGAUIv3osmZoD3kKl4jxEmYQ==", + "dependencies": { + "Newtonsoft.Json": "12.0.1" + } + }, "SQLitePCLRaw.bundle_e_sqlite3": { "type": "Transitive", "resolved": "2.1.4", @@ -455,8 +555,8 @@ }, "System.Memory": { "type": "Transitive", - "resolved": "4.5.3", - "contentHash": "3oDzvc/zzetpTKWMShs1AADwZjQ/36HnsufHRPcOjyRAAMLDlu2iD33MBI2opxnezcVUtXyqDXXjoFMOU9c7SA==" + "resolved": "4.5.5", + "contentHash": "XIWiDvKPXaTveaB7HVganDlOCRoj03l+jrwNvcge/t8vhGYKvqV+dMv6G4SAX2NoNmN0wZfVPTAlFwZcZvVOUw==" }, "System.Reactive": { "type": "Transitive", @@ -468,16 +568,30 @@ "resolved": "4.5.1", "contentHash": "Zh8t8oqolRaFa9vmOZfdQm/qKejdqz0J9kr7o2Fu0vPeoH3BL1EOXipKWwkWtLT1JPzjByrF19fGuFlNbmPpiw==" }, + "System.Text.Encodings.Web": { + "type": "Transitive", + "resolved": "10.0.1", + "contentHash": "cVAka0o1rJJ5/De0pjNs7jcaZk5hUGf1HGzUyVmE2MEB1Vf0h/8qsWxImk1zjitCbeD2Avaq2P2+usdvqgbeVQ==" + }, + "System.Threading.Tasks.Extensions": { + "type": "Transitive", + "resolved": "4.5.4", + "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==" + }, "speckle.sdk": { "type": "Project", "dependencies": { "GraphQL.Client": "[6.0.0, )", + "Microsoft.AspNet.WebApi.Client": "[6.0.0, )", "Microsoft.Data.Sqlite": "[7.0.5, )", "Microsoft.Extensions.DependencyInjection.Abstractions": "[2.2.0, )", "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )", + "System.IO.Pipelines": "[10.0.1, )", + "System.Net.Http.Json": "[10.0.1, )", + "System.Threading.Channels": "[10.0.1, )" } }, "speckle.sdk.dependencies": { @@ -494,6 +608,18 @@ "System.Reactive": "5.0.0" } }, + "Microsoft.AspNet.WebApi.Client": { + "type": "CentralTransitive", + "requested": "[6.0.0, )", + "resolved": "6.0.0", + "contentHash": "zXeWP03dTo67AoDHUzR+/urck0KFssdCKOC+dq7Nv1V2YbFh/nIg09L0/3wSvyRONEdwxGB/ssEGmPNIIhAcAw==", + "dependencies": { + "Newtonsoft.Json": "13.0.1", + "Newtonsoft.Json.Bson": "1.0.2", + "System.Memory": "4.5.5", + "System.Threading.Tasks.Extensions": "4.5.4" + } + }, "Microsoft.Data.Sqlite": { "type": "CentralTransitive", "requested": "[7.0.5, )", @@ -533,6 +659,37 @@ "requested": "[13.0.2, )", "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" + }, + "System.IO.Pipelines": { + "type": "CentralTransitive", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "26LbFXHKd7PmRnWlkjnYgmjd5B6HYVG+1MpTO25BdxTJnx6D0O16JPAC/S4YBqjtt4YpfGj1QO/Ss6SPMGEGQw==" + }, + "System.Net.Http.Json": { + "type": "CentralTransitive", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "XGOWt78ccgO9esyNlCemlS2b8JZPrH85pk/MvdHJxp6KwwUY/GnDaw2fPpJa7lgotiTkWnXnhip+ULNKaP7a8A==", + "dependencies": { + "System.Text.Json": "10.0.1" + } + }, + "System.Text.Json": { + "type": "CentralTransitive", + "requested": "[8.0.5, )", + "resolved": "10.0.1", + "contentHash": "EsgwDgU1PFqhrFA9l5n+RBu76wFhNGCEwu8ITrBNhjPP3MxLyklroU5GIF8o6JYpYg6T4KD/VICfMdgPAvNp5g==", + "dependencies": { + "System.IO.Pipelines": "10.0.1", + "System.Text.Encodings.Web": "10.0.1" + } + }, + "System.Threading.Channels": { + "type": "CentralTransitive", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" } } } diff --git a/src/Speckle.Sdk.Dependencies/packages.lock.json b/src/Speckle.Sdk.Dependencies/packages.lock.json index c664417c..1f256577 100644 --- a/src/Speckle.Sdk.Dependencies/packages.lock.json +++ b/src/Speckle.Sdk.Dependencies/packages.lock.json @@ -82,12 +82,12 @@ }, "System.Threading.Channels": { "type": "Direct", - "requested": "[9.0.4, )", - "resolved": "9.0.4", - "contentHash": "4qBn2H6/aXBpE/Pm3wY5yusY/pEvQz99NlWHrTUji0qCmOdbhhjaALcpmbfW2ksxlPM6i6S+QFLkpOQdyfeKYQ==", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==", "dependencies": { - "Microsoft.Bcl.AsyncInterfaces": "9.0.4", - "System.Threading.Tasks.Extensions": "4.5.4" + "Microsoft.Bcl.AsyncInterfaces": "10.0.1", + "System.Threading.Tasks.Extensions": "4.6.3" } }, "ILRepack": { @@ -141,24 +141,24 @@ }, "System.Runtime.CompilerServices.Unsafe": { "type": "Transitive", - "resolved": "6.0.0", - "contentHash": "/iUeP3tq1S0XdNNoMz5C9twLSrM/TH+qElHkXWaPvuNOt+99G75NrV0OS2EqHx5wMN7popYjpc8oTjC1y16DLg==" + "resolved": "6.1.2", + "contentHash": "2hBr6zdbIBTDE3EhK7NSVNdX58uTK6iHW/P/Axmm9sl1xoGSLqDvMtpecn226TNwHByFokYwJmt/aQQNlO5CRw==" }, "System.Threading.Tasks.Extensions": { "type": "Transitive", - "resolved": "4.5.4", - "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==", + "resolved": "4.6.3", + "contentHash": "7sCiwilJLYbTZELaKnc7RecBBXWXA+xMLQWZKWawBxYjp6DBlSE3v9/UcvKBvr1vv2tTOhipiogM8rRmxlhrVA==", "dependencies": { - "System.Runtime.CompilerServices.Unsafe": "4.5.3" + "System.Runtime.CompilerServices.Unsafe": "6.1.2" } }, "Microsoft.Bcl.AsyncInterfaces": { "type": "CentralTransitive", - "requested": "[5.0.0, )", - "resolved": "9.0.4", - "contentHash": "9VGI5kxIvrNG2mqLQZnUR6y/3fcnygD8eNpHR+CqfbnIXvea6nehnYknDKQTxZVPMpzpNca+7DxLBmpdB3q0Bw==", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "E1HSLkPHXEO30JEij2pWbOuzz1Z5ND4a5l7IP1T2RgQuE0a0NzEIvtO64RNy3Otn6PFezbT80cfm3M/Cgt70PA==", "dependencies": { - "System.Threading.Tasks.Extensions": "4.5.4" + "System.Threading.Tasks.Extensions": "4.6.3" } } }, @@ -229,9 +229,9 @@ }, "System.Threading.Channels": { "type": "Direct", - "requested": "[9.0.4, )", - "resolved": "9.0.4", - "contentHash": "4qBn2H6/aXBpE/Pm3wY5yusY/pEvQz99NlWHrTUji0qCmOdbhhjaALcpmbfW2ksxlPM6i6S+QFLkpOQdyfeKYQ==" + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" }, "ILRepack": { "type": "Transitive", diff --git a/src/Speckle.Sdk/Models/DynamicBaseMemberType.cs b/src/Speckle.Sdk/Models/DynamicBaseMemberType.cs index fc114380..b03c2579 100644 --- a/src/Speckle.Sdk/Models/DynamicBaseMemberType.cs +++ b/src/Speckle.Sdk/Models/DynamicBaseMemberType.cs @@ -17,7 +17,7 @@ public enum DynamicBaseMemberType Dynamic = 2, /// - /// The typed members flagged with attribute. + /// The typed members flagged with ObsoleteAttribute attribute. /// Obsolete = 4, @@ -27,12 +27,12 @@ public enum DynamicBaseMemberType SchemaComputed = 16, /// - /// All the typed members, including ones with attributes. + /// All the typed members, including ones with ObsoleteAttribute attributes. /// InstanceAll = Instance + Obsolete, /// - /// All the members, including dynamic and instance members flagged with attributes + /// All the members, including dynamic and instance members flagged with ObsoleteAttribute attributes /// All = InstanceAll + Dynamic, } diff --git a/src/Speckle.Sdk/Pipelines/ProgressStream.cs b/src/Speckle.Sdk/Pipelines/ProgressStream.cs new file mode 100644 index 00000000..ed62121c --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/ProgressStream.cs @@ -0,0 +1,97 @@ +namespace Speckle.Sdk.Pipelines; + +/// +/// Wraps a stream to report upload progress as bytes are read. +/// +public sealed class ProgressStream : Stream +{ + private readonly Stream _innerStream; + private readonly long _totalBytes; + private readonly IProgress<(long BytesSent, long TotalBytes)>? _progress; + private long _bytesSent; + + public ProgressStream( + Stream innerStream, + long totalBytes, + IProgress<(long BytesSent, long TotalBytes)>? progress = null + ) + { + _innerStream = innerStream; + _totalBytes = totalBytes; + _progress = progress; + _bytesSent = 0; + } + + public override bool CanRead => _innerStream.CanRead; + public override bool CanSeek => _innerStream.CanSeek; + public override bool CanWrite => false; + public override long Length => _innerStream.Length; + + public override long Position + { + get => _innerStream.Position; + set => _innerStream.Position = value; + } + + public override int Read(byte[] buffer, int offset, int count) + { + int bytesRead = _innerStream.Read(buffer, offset, count); + ReportProgress(bytesRead); + return bytesRead; + } + + public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + int bytesRead = await _innerStream +#if NET8_0_OR_GREATER + .ReadAsync(buffer.AsMemory(offset, count), cancellationToken) +#else + .ReadAsync(buffer, offset, count, cancellationToken) +#endif + .ConfigureAwait(false); + ReportProgress(bytesRead); + return bytesRead; + } + +#if NET8_0_OR_GREATER + public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + { + int bytesRead = await _innerStream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + ReportProgress(bytesRead); + return bytesRead; + } +#endif + + private void ReportProgress(int bytesRead) + { + _bytesSent += bytesRead; + _progress?.Report((_bytesSent, _totalBytes)); + } + + public override void Flush() => _innerStream.Flush(); + + public override Task FlushAsync(CancellationToken cancellationToken) => _innerStream.FlushAsync(cancellationToken); + + public override long Seek(long offset, SeekOrigin origin) => _innerStream.Seek(offset, origin); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + protected override void Dispose(bool disposing) + { + if (disposing) + { + _innerStream.Dispose(); + } + base.Dispose(disposing); + } + +#if NET8_0_OR_GREATER + public override async ValueTask DisposeAsync() + { + await _innerStream.DisposeAsync().ConfigureAwait(false); + await base.DisposeAsync().ConfigureAwait(false); + } +#endif +} diff --git a/src/Speckle.Sdk/Pipelines/SendPipeline.cs b/src/Speckle.Sdk/Pipelines/SendPipeline.cs new file mode 100644 index 00000000..4c580163 --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/SendPipeline.cs @@ -0,0 +1,59 @@ +using Speckle.Sdk.Credentials; +using Speckle.Sdk.Models; +using Speckle.Sdk.Serialisation; + +namespace Speckle.Sdk.Pipelines; + +public record UploadItem(string Id, Json Json, string SpeckleType, ObjectReference Reference); + +public sealed class SendPipeline : IDisposable +{ + private readonly CancellationToken _cancellationToken; + private readonly Serializer _serializer = new(); + private readonly Uploader _uploader; + + public SendPipeline( + Account account, + string projectId, + string modelId, + string ingestionId, + CancellationToken cancellationToken + ) + { + _cancellationToken = cancellationToken; + _uploader = new Uploader(projectId, modelId, ingestionId, account.serverInfo.url, account.token, cancellationToken); + } + + private UploadItem _lastItem; + + public async Task Process(Base @base) + { + var results = _serializer.Serialize(@base).ToArray(); + var first = results.First(); + foreach (var item in results) + { + // we're not doing fire and forget here so that we get the backpressure from the uploader + await _uploader.PushAsync(item).ConfigureAwait(false); + } + + // NOTE: this is important to keep track of. When we serialze an object, we get back a list of objects, with the first one being the original root. + // In the case of the commit root object, this means the last object is not necessarily the root; we therefore need to manually track its existance here + // and ensure it's the last one through in the uploader's stream. See WaitForUpload down below. + _lastItem = first; + return first.Reference; + } + + public async Task WaitForUpload() + { + await _uploader.PushAsync(_lastItem).ConfigureAwait(false); + await _uploader.CompleteAsync().ConfigureAwait(false); + } + + public async Task WaitForUploadAndServerProcessing() + { + // TODO: in some way, wait for the server to process the upload and return the actual new version id + return await Task.FromResult("todo").ConfigureAwait(false); + } + + public void Dispose() => _uploader.Dispose(); +} diff --git a/src/Speckle.Sdk/Pipelines/Serializer.cs b/src/Speckle.Sdk/Pipelines/Serializer.cs new file mode 100644 index 00000000..ed072a50 --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Serializer.cs @@ -0,0 +1,372 @@ +using System.Collections; +using System.Reflection; +using Speckle.DoubleNumerics; +using Speckle.Newtonsoft.Json; +using Speckle.Sdk.Dependencies; +using Speckle.Sdk.Helpers; +using Speckle.Sdk.Models; +using Speckle.Sdk.Serialisation; + +namespace Speckle.Sdk.Pipelines; + +/// +/// Another serializer, cleaner and meaner. Provides methods for serializing Speckle objects into a format suitable for upload or storage. +/// This class handles the conversion of and its derivatives +/// into serialized JSON structures along with associated metadata, closures, and references. +/// Any reference objects coming through are being "passed through" serialized - they do not get double encoded. +/// +public class Serializer +{ + private readonly record struct PropertyInfo(string Name, object? Value, bool IsDetachable); + + public IEnumerable Serialize(Base root) + { + // Special case: if root is already an ObjectReference, serialize it verbatim + if (root is ObjectReference existingRef) + { + var uploadItem = ReferenceToUploadItem(existingRef); + yield return uploadItem; + yield break; + } + + var detachedObjects = new List<(Id, Json, Dictionary, Base, string)>(); + var rootClosures = new Dictionary(); + + var (rootId, rootJson) = SerializeBase(root, false, rootClosures, detachedObjects); + + var rootReference = new ObjectReference + { + referencedId = rootId.Value, + applicationId = root.applicationId, + closure = rootClosures.Count > 0 ? rootClosures : null, + }; + + yield return new UploadItem(rootId.Value, rootJson, root.speckle_type, rootReference); + + foreach (var (id, json, closures, baseObj, speckleType) in detachedObjects) + { + var reference = new ObjectReference + { + referencedId = id.Value, + applicationId = baseObj.applicationId, + closure = closures.Count > 0 ? closures : null, + }; + + yield return new UploadItem(id.Value, json, speckleType, reference); + } + } + + private IEnumerable ExtractProperties(Base baseObj) + { + var typedProperties = baseObj.GetInstanceMembers(); + foreach (var prop in typedProperties) + { + if (prop.Name == "id" || prop.Name.StartsWith("__")) + { + continue; + } + + if (prop.IsDefined(typeof(JsonIgnoreAttribute), false)) + { + continue; + } + + var value = prop.GetValue(baseObj); + var isDetachable = prop.GetCustomAttribute(true)?.Detachable ?? false; + + yield return new PropertyInfo(prop.Name, value, isDetachable); + } + + foreach (var propName in baseObj.DynamicPropertyKeys) + { + if (propName.StartsWith("__")) + { + continue; + } + + var value = baseObj[propName]; + +#pragma warning disable CA1866 + var isDetachable = propName.StartsWith("@"); +#pragma warning restore CA1866 + + yield return new PropertyInfo(propName, value, isDetachable); + } + } + + private (Id, Json) SerializeBase( + Base baseObj, + bool forceDetach, + Dictionary closures, + List<(Id, Json, Dictionary, Base, string)> detachedObjects + ) + { + var childClosures = new Dictionary(); + + var sb = Pools.StringBuilders.Get(); + try + { + using var stringWriter = new StringWriter(sb); + using var jsonWriter = new JsonTextWriter(stringWriter); + using var idWriter = new SerializerIdWriter(jsonWriter); + + idWriter.WriteStartObject(); + + foreach (var prop in ExtractProperties(baseObj)) + { + idWriter.WritePropertyName(prop.Name); + SerializeValue(prop.Value, idWriter, prop.IsDetachable, childClosures, detachedObjects); + } + + var (jsonForId, finalWriter) = idWriter.FinishIdWriter(); + var id = IdGenerator.ComputeId(jsonForId); + + finalWriter.WritePropertyName("id"); + finalWriter.WriteValue(id.Value); + + baseObj.id = id.Value; + + if ((forceDetach || childClosures.Count > 0) && childClosures.Count > 0) + { + finalWriter.WritePropertyName("__closure"); + finalWriter.WriteStartObject(); + foreach (var kvp in childClosures) + { + finalWriter.WritePropertyName(kvp.Key); + finalWriter.WriteValue(kvp.Value); + } + finalWriter.WriteEndObject(); + + foreach (var kvp in childClosures) + { + closures[kvp.Key] = closures.TryGetValue(kvp.Key, out var existing) ? existing + kvp.Value : kvp.Value; + } + } + + finalWriter.WriteEndObject(); + finalWriter.Flush(); + + var json = new Json(stringWriter.ToString()); + return (id, json); + } + finally + { + Pools.StringBuilders.Return(sb); + } + } + + private void SerializeValue( + object? value, + JsonWriter writer, + bool isDetachable, + Dictionary closures, + List<(Id, Json, Dictionary, Base, string)> detachedObjects + ) + { + if (value == null) + { + writer.WriteNull(); + return; + } + + switch (value) + { + case string s: + writer.WriteValue(s); + return; + case int i: + writer.WriteValue(i); + return; + case long l: + writer.WriteValue(l); + return; + case double d: + writer.WriteValue(d); + return; + case float f: + writer.WriteValue(f); + return; + case bool b: + writer.WriteValue(b); + return; + case Enum e: + writer.WriteValue((int)(object)e); + return; + case Guid g: + writer.WriteValue(g.ToString()); + return; + case DateTime dt: + writer.WriteValue(dt.ToString("o")); + return; + + case Matrix4x4 md: + writer.WriteStartArray(); + writer.WriteValue(md.M11); + writer.WriteValue(md.M12); + writer.WriteValue(md.M13); + writer.WriteValue(md.M14); + writer.WriteValue(md.M21); + writer.WriteValue(md.M22); + writer.WriteValue(md.M23); + writer.WriteValue(md.M24); + writer.WriteValue(md.M31); + writer.WriteValue(md.M32); + writer.WriteValue(md.M33); + writer.WriteValue(md.M34); + writer.WriteValue(md.M41); + writer.WriteValue(md.M42); + writer.WriteValue(md.M43); + writer.WriteValue(md.M44); + writer.WriteEndArray(); + return; + } + + // Handle ObjectReference before Base (since ObjectReference extends Base) + // This prevents double-serialization and properly propagates closures + if (value is ObjectReference objRef) + { + writer.WriteStartObject(); + writer.WritePropertyName("speckle_type"); + writer.WriteValue("reference"); + writer.WritePropertyName("referencedId"); + writer.WriteValue(objRef.referencedId); + writer.WriteEndObject(); + + // Propagate closure: add the referenced ID + closures[objRef.referencedId] = closures.TryGetValue(objRef.referencedId, out var existing) ? existing + 1 : 1; + + // Propagate nested closures from the ObjectReference.closure dictionary + if (objRef.closure != null) + { + foreach (var kvp in objRef.closure) + { + closures[kvp.Key] = closures.TryGetValue(kvp.Key, out var existingDepth) + ? existingDepth + kvp.Value + : kvp.Value; + } + } + + return; + } + + if (value is Base baseObj) + { + if (isDetachable) + { + var childClosures = new Dictionary(); + var (childId, childJson) = SerializeBase(baseObj, true, childClosures, detachedObjects); + + detachedObjects.Add((childId, childJson, childClosures, baseObj, baseObj.speckle_type)); + + writer.WriteStartObject(); + writer.WritePropertyName("speckle_type"); + writer.WriteValue("reference"); + writer.WritePropertyName("referencedId"); + writer.WriteValue(childId.Value); + writer.WriteEndObject(); + + closures[childId.Value] = closures.TryGetValue(childId.Value, out var existing) ? existing + 1 : 1; + + foreach (var kvp in childClosures) + { + closures[kvp.Key] = closures.TryGetValue(kvp.Key, out var existingDepth) + ? existingDepth + kvp.Value + : kvp.Value; + } + } + else + { + var inlineClosures = new Dictionary(); + var (_, inlineJson) = SerializeBase(baseObj, false, inlineClosures, detachedObjects); + + writer.WriteRawValue(inlineJson.Value); + + foreach (var kvp in inlineClosures) + { + closures[kvp.Key] = closures.TryGetValue(kvp.Key, out var existingDepth) + ? existingDepth + kvp.Value + : kvp.Value; + } + } + return; + } + + if (value is IDictionary dict) + { + writer.WriteStartObject(); + foreach (DictionaryEntry kvp in dict) + { + if (kvp.Key is not string key) + { + throw new ArgumentException("Dictionary keys must be strings"); + } + + writer.WritePropertyName(key); + SerializeValue(kvp.Value, writer, false, closures, detachedObjects); + } + writer.WriteEndObject(); + return; + } + + if (value is IEnumerable enumerable and not string) + { + writer.WriteStartArray(); + foreach (var item in enumerable) + { + SerializeValue(item, writer, isDetachable, closures, detachedObjects); + } + writer.WriteEndArray(); + return; + } + + writer.WriteValue(value.ToString()); + } + + private UploadItem ReferenceToUploadItem(ObjectReference existingRef) + { + var sb = Pools.StringBuilders.Get(); + try + { + using var stringWriter = new StringWriter(sb); + using var jsonWriter = new JsonTextWriter(stringWriter); + + jsonWriter.WriteStartObject(); + jsonWriter.WritePropertyName("speckle_type"); + jsonWriter.WriteValue("reference"); + jsonWriter.WritePropertyName("referencedId"); + jsonWriter.WriteValue(existingRef.referencedId); + jsonWriter.WritePropertyName("__closure"); + + if (existingRef.closure != null && existingRef.closure.Count > 0) + { + jsonWriter.WriteStartObject(); + foreach (var kvp in existingRef.closure) + { + jsonWriter.WritePropertyName(kvp.Key); + jsonWriter.WriteValue(kvp.Value); + } + jsonWriter.WriteEndObject(); + } + else + { + jsonWriter.WriteNull(); + } + + jsonWriter.WriteEndObject(); + jsonWriter.Flush(); + + var refJson = new Json(stringWriter.ToString()); + + return new UploadItem( + existingRef.referencedId, + refJson, + existingRef.speckle_type, + existingRef // Pass through the original ObjectReference + ); + } + finally + { + Pools.StringBuilders.Return(sb); + } + } +} diff --git a/src/Speckle.Sdk/Pipelines/Uploader.cs b/src/Speckle.Sdk/Pipelines/Uploader.cs new file mode 100644 index 00000000..ec8133a4 --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Uploader.cs @@ -0,0 +1,175 @@ +using System.IO.Compression; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Threading.Channels; + +namespace Speckle.Sdk.Pipelines; + +public sealed class Uploader : IDisposable +{ + private readonly string _projectId; + private readonly string _modelId; + private readonly string _ingestionId; + private readonly CancellationToken _cancellationToken; + private readonly HttpClient _client; + private readonly Channel _channel; + private readonly Task _sendTask; + + public Uploader( + string projectId, + string modelId, + string ingestionId, + string apiEndpoint, + string authToken, + CancellationToken cancellationToken + ) + { + _projectId = projectId; + _modelId = modelId; + _ingestionId = ingestionId; + _cancellationToken = cancellationToken; + + Uri apiBaseUrl = new(new(apiEndpoint), "/api/v1/"); + _client = new HttpClient { BaseAddress = apiBaseUrl, Timeout = TimeSpan.FromMinutes(30) }; + + _client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", authToken); + + _channel = Channel.CreateBounded( + new BoundedChannelOptions(1000) { FullMode = BoundedChannelFullMode.Wait } + ); + + _sendTask = SendLoopAsync(); + } + + public ValueTask PushAsync(UploadItem item, CancellationToken ct = default) => _channel.Writer.WriteAsync(item, ct); + + public async Task CompleteAsync() + { + _channel.Writer.Complete(); + var result = await _sendTask.ConfigureAwait(false); + return result.IngestionId; + } + + private async Task SendLoopAsync() + { + // 1. Stream channel to temp file + string tempFilePath = Path.GetTempFileName(); + System.Diagnostics.Debug.WriteLine($"Temp file is at {tempFilePath}"); + try + { + long fileSizeBytes; + { + using var fileStream = new FileStream(tempFilePath, FileMode.Create, FileAccess.Write, FileShare.None); + using var gzip = new GZipStream(fileStream, CompressionLevel.Optimal); + using var writer = new StreamWriter(gzip); + await foreach (var item in _channel.Reader.ReadAllAsync(_cancellationToken).ConfigureAwait(false)) + { + await writer.WriteLineAsync($"{item.Id}\t{item.Json}\t{item.SpeckleType}").ConfigureAwait(false); + } + + await writer.FlushAsync().ConfigureAwait(false); + await gzip.FlushAsync(_cancellationToken).ConfigureAwait(false); + } + // fileStream.Flush(); + // fileStream.Close(); + fileSizeBytes = new FileInfo(tempFilePath).Length; + + // 2. Request presigned URL + var signUri = new Uri($"projects/{_projectId}/models/{_modelId}/uploads/sign", UriKind.Relative); + + var signResponse = await HttpClientExtensions + .PostAsJsonAsync(_client, signUri, _cancellationToken) + .ConfigureAwait(false); + signResponse.EnsureSuccessStatusCode(); + + var presignedUpload = + await signResponse.Content.ReadFromJsonAsync(_cancellationToken).ConfigureAwait(false) + ?? throw new InvalidOperationException("Failed to get presigned upload URL"); + + // 3. Upload to S3 + using var fileStreamUpload = new FileStream(tempFilePath, FileMode.Open, FileAccess.Read, FileShare.Read); + + Stream progressStream = fileStreamUpload; // TODO: wrap with progress stream + + using var streamContent = new StreamContent(progressStream); + streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); + streamContent.Headers.ContentLength = fileSizeBytes; + + using var uploadRequest = new HttpRequestMessage(HttpMethod.Put, new Uri(presignedUpload.Url, UriKind.Absolute)); + uploadRequest.Content = streamContent; + + using var s3Client = new HttpClient(); // NOTE: using a separate client for S3 as we DO NOT NEED THE AUTH HEADER, presigned url's don't work with it. + using var uploadResponse = await s3Client + .SendAsync(uploadRequest, HttpCompletionOption.ResponseHeadersRead, _cancellationToken) + .ConfigureAwait(false); + + uploadResponse.EnsureSuccessStatusCode(); + + // 4. Trigger processing + var processUri = new Uri($"projects/{_projectId}/models/{_modelId}/uploads/process", UriKind.Relative); + var processRequest = new ProcessUploadRequest { key = presignedUpload.Key, ingestionId = _ingestionId }; + + var processResponse = await HttpClientExtensions + .PostAsJsonAsync(_client, processUri, processRequest, _cancellationToken) + .ConfigureAwait(false); + processResponse.EnsureSuccessStatusCode(); + + var processResult = await processResponse + .Content.ReadFromJsonAsync(_cancellationToken) + .ConfigureAwait(false); + + if (processResult == null) + { + throw new InvalidOperationException("Failed to trigger upload processing"); + } + + return new UploadResult { IngestionId = processResult.ingestionId }; + } + catch (Exception ex) when (!ex.IsFatal()) + { + throw; + } + finally + { + // 5. Clean up temp file + if (File.Exists(tempFilePath)) + { + try + { + //File.Delete(tempFilePath); + } +#pragma warning disable CA1031 + catch +#pragma warning restore CA1031 + { + // Best effort + } + } + } + } + + public void Dispose() => _client.Dispose(); +} + +// DTOs +internal record PresignedUploadResponse +{ + public required string Url { get; init; } + public required string Key { get; init; } +} + +internal record ProcessUploadRequest +{ + public required string key { get; init; } + public required string ingestionId { get; init; } +} + +internal record ProcessUploadResponse +{ + public required string ingestionId { get; init; } +} + +internal record UploadResult +{ + public required string IngestionId { get; init; } +} diff --git a/src/Speckle.Sdk/Pipelines/Uploader_Old.cs b/src/Speckle.Sdk/Pipelines/Uploader_Old.cs new file mode 100644 index 00000000..fa8f73c6 --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Uploader_Old.cs @@ -0,0 +1,83 @@ +// using System.IO.Compression; +// using System.Net.Http.Headers; +// using System.Threading.Channels; +// +// namespace Speckle.Sdk.Pipeline; +// +// public sealed class UploaderOld : IDisposable +// { +// private readonly string _projectId; +// private readonly string _modelId; +// private readonly HttpClient _client; +// private readonly Channel _channel; +// private readonly Task _sendTask; +// +// public UploaderOld(string projectId, string modelId, string? authToken, string? apiEndpoint) +// { +// _projectId = projectId; +// _modelId = modelId; +// +// Uri apiBaseUrl = !string.IsNullOrEmpty(apiEndpoint) +// ? new Uri(apiEndpoint) +// // : new Uri("http://dimitries-macbook-pro.mermaid-emperor.ts.net/api/v1/"); +// : new Uri("http://zog.local:3000/api/v1/"); +// _client = new HttpClient { BaseAddress = apiBaseUrl, Timeout = TimeSpan.FromMinutes(10) }; +// +// if (authToken != null) +// { +// _client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", authToken); +// } +// +// _channel = Channel.CreateBounded( +// new BoundedChannelOptions(1000) { FullMode = BoundedChannelFullMode.Wait } // if we're not able to write fast enough, we'll block writes +// ); +// +// _sendTask = SendLoopAsync(projectId, "test"); +// } +// +// public ValueTask PushAsync(UploadItem item, CancellationToken ct = default) => _channel.Writer.WriteAsync(item, ct); +// +// public async Task CompleteAsync() +// { +// _channel.Writer.Complete(); +// await _sendTask.ConfigureAwait(false); +// } +// +// private async Task SendLoopAsync(string projectId, string modelId) +// { +// var content = new PushStreamContent( +// async (stream, _, _) => +// { +// var gzip = new GZipStream(stream, CompressionLevel.Optimal); +// var writer = new StreamWriter(gzip); // new StreamWriter(gzip, System.Text.Encoding.UTF8, 20 * 1024 * 1024); // potential lever for controlling memory pressure +// try +// { +// // extra levers for memory pressure in here: we can manually flush every x items or every x bytes +// await foreach (var item in _channel.Reader.ReadAllAsync().ConfigureAwait(false)) +// { +// await writer.WriteLineAsync($"{item.Id}\t{item.Json}\t{item.SpeckleType}").ConfigureAwait(false); +// } +// } +// finally +// { +// await writer.FlushAsync().ConfigureAwait(false); +// await gzip.FlushAsync().ConfigureAwait(false); +// writer.Dispose(); +// gzip.Dispose(); +// } +// }, +// new MediaTypeHeaderValue("application/x-ndjson") +// ); +// +// var uri = new Uri($"projects/{projectId}/models/{modelId}/versions", UriKind.Relative); +// var request = new HttpRequestMessage(HttpMethod.Post, uri) { Content = content }; +// request.Headers.TransferEncodingChunked = true; // NOTE: important for streaming to happen. +// var response = await _client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false); +// response.EnsureSuccessStatusCode(); +// +// // Consume the response body to fully complete the request +// await response.Content.ReadAsStringAsync().ConfigureAwait(false); +// } +// +// public void Dispose() => _client.Dispose(); +// } diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/BaseItem.cs b/src/Speckle.Sdk/Serialisation/V2/Send/BaseItem.cs index 2a88e3db..9c5b02bf 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Send/BaseItem.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Send/BaseItem.cs @@ -2,7 +2,13 @@ using System.Text; namespace Speckle.Sdk.Serialisation.V2.Send; -public sealed record BaseItem(Id Id, Json Json, bool NeedsStorage, Dictionary? Closures) : IHasByteSize +public sealed record BaseItem( + Id Id, + Json Json, + bool NeedsStorage, + Dictionary? Closures, + bool? IsReference = false +) : IHasByteSize { public int ByteSize { get; } = Encoding.UTF8.GetByteCount(Json.Value); diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/ObjectFlopper.cs b/src/Speckle.Sdk/Serialisation/V2/Send/ObjectFlopper.cs new file mode 100644 index 00000000..2027a6e3 --- /dev/null +++ b/src/Speckle.Sdk/Serialisation/V2/Send/ObjectFlopper.cs @@ -0,0 +1,79 @@ +using System.IO.Compression; +using System.Net.Http.Headers; +using System.Threading.Channels; + +namespace Speckle.Sdk.Serialisation.V2.Send; + +#pragma warning disable CA1001 +public sealed class ObjectFlopper +#pragma warning restore CA1001 +{ + private readonly Uri _url; + private readonly string _streamId; + private readonly HttpClient _client; + private readonly Channel _channel; + private readonly Task _sendTask; + + public ObjectFlopper(Uri _, string streamId, string? authToken) + { + _streamId = streamId; + _url = new Uri("http://zog.local:3000/api/v1/"); + _client = new HttpClient { BaseAddress = _url, Timeout = TimeSpan.FromMinutes(10) }; + + if (authToken != null) + { + _client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", authToken); + } + + _channel = Channel.CreateBounded( + new BoundedChannelOptions(1000) { FullMode = BoundedChannelFullMode.Wait } + ); + + _sendTask = SendLoopAsync(streamId, "test"); + } + + public ValueTask PushAsync(BaseItem item, CancellationToken ct = default) => _channel.Writer.WriteAsync(item, ct); + + public async Task CompleteAsync() + { + _channel.Writer.Complete(); + await _sendTask.ConfigureAwait(false); + } + + private async Task SendLoopAsync(string projectId, string modelId) + { + var content = new PushStreamContent( + async (stream, _, _) => + { + var gzip = new GZipStream(stream, CompressionLevel.Optimal); + var writer = new StreamWriter(gzip); //new StreamWriter(gzip, System.Text.Encoding.UTF8, 20 * 1024 * 1024); + try + { + await foreach (var item in _channel.Reader.ReadAllAsync().ConfigureAwait(false)) + { + await writer.WriteLineAsync($"{item.Id}\t{item.Json}").ConfigureAwait(false); + } + } + finally + { + await writer.FlushAsync().ConfigureAwait(false); + await gzip.FlushAsync().ConfigureAwait(false); + writer.Dispose(); + gzip.Dispose(); + } + }, + new MediaTypeHeaderValue("application/x-ndjson") + ); + + var uri = new Uri($"projects/{projectId}/models/{modelId}/versions", UriKind.Relative); + var request = new HttpRequestMessage(HttpMethod.Post, uri) { Content = content }; + request.Headers.TransferEncodingChunked = true; // NOTE: important for streaming to happen. + var response = await _client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + // Consume the response body to fully complete the request + await response.Content.ReadAsStringAsync().ConfigureAwait(false); + } + + public void Dispose() => _client.Dispose(); +} diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/ObjectFlopperGandalf.cs b/src/Speckle.Sdk/Serialisation/V2/Send/ObjectFlopperGandalf.cs new file mode 100644 index 00000000..abccd3a6 --- /dev/null +++ b/src/Speckle.Sdk/Serialisation/V2/Send/ObjectFlopperGandalf.cs @@ -0,0 +1,90 @@ +using System.IO.Compression; +using System.IO.Pipelines; +using System.Net.Http.Headers; +using System.Threading.Channels; + +namespace Speckle.Sdk.Serialisation.V2.Send; + +#pragma warning disable CA1001 +public sealed class ObjectFlopperGandalf +#pragma warning restore CA1001 +{ + private readonly Uri _url; + private readonly string _streamId; + private readonly HttpClient _client; + private readonly Channel _channel; + private readonly Task _sendTask; + + public ObjectFlopperGandalf(Uri _, string streamId, string? authToken) + { + _streamId = streamId; + _url = new Uri("http://bender-2.local:3000/api/v1/"); + _client = new HttpClient { BaseAddress = _url, Timeout = TimeSpan.FromMinutes(10) }; + + if (authToken != null) + { + _client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", authToken); + } + + _channel = Channel.CreateBounded( + new BoundedChannelOptions(1000) { FullMode = BoundedChannelFullMode.Wait } + ); + + _sendTask = SendLoopAsync(streamId, "test"); + } + + public ValueTask PushAsync(BaseItem item, CancellationToken ct = default) => _channel.Writer.WriteAsync(item, ct); + + public async Task CompleteAsync() + { + _channel.Writer.Complete(); + await _sendTask.ConfigureAwait(false); + } + + private async Task SendLoopAsync(string projectId, string modelId) + { + var pipe = new Pipe(); + + // Start writing to pipe immediately in background + var writeTask = Task.Run(async () => + { + var gzip = new GZipStream(pipe.Writer.AsStream(), CompressionLevel.Optimal); + var writer = new StreamWriter(gzip); + + try + { + await foreach (var item in _channel.Reader.ReadAllAsync().ConfigureAwait(false)) + { + await writer.WriteLineAsync($"{item.Id}\t{item.Json}").ConfigureAwait(false); + await writer.FlushAsync().ConfigureAwait(false); + } + } + finally + { + await writer.FlushAsync().ConfigureAwait(false); + await gzip.FlushAsync().ConfigureAwait(false); + writer.Dispose(); + gzip.Dispose(); + await pipe.Writer.CompleteAsync().ConfigureAwait(false); + } + }); + + // Start HTTP request immediately, reading from pipe + var content = new StreamContent(pipe.Reader.AsStream()); + content.Headers.ContentType = new MediaTypeHeaderValue("application/x-ndjson"); + content.Headers.ContentEncoding.Add("gzip"); + + var uri = new Uri($"projects/{projectId}/models/{modelId}/objects", UriKind.Relative); + var request = new HttpRequestMessage(HttpMethod.Post, uri) { Content = content }; + + var responseTask = _client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); + + // Wait for both + await Task.WhenAll(writeTask, responseTask).ConfigureAwait(false); + + var response = await responseTask.ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + } + + public void Dispose() => _client.Dispose(); +} diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs index 6dac6f6f..29965525 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs @@ -113,16 +113,6 @@ public sealed class SerializeProcess( _processSource.Token ); var findTotalObjectsTask = Task.CompletedTask; - if (!options.SkipFindTotalObjects) - { - ThrowIfFailed(); - findTotalObjectsTask = Task.Factory.StartNew( - () => TraverseTotal(root), - _processSource.Token, - TaskCreationOptions.AttachedToParent | TaskCreationOptions.PreferFairness, - _highest - ); - } await Traverse(root).ConfigureAwait(false); ThrowIfFailed(); @@ -133,6 +123,7 @@ public sealed class SerializeProcess( ThrowIfFailed(); await WaitForSchedulerCompletion().ConfigureAwait(false); ThrowIfFailed(); + return new(root.id.NotNull(), baseSerializer.ObjectReferences.Freeze()); } catch (OperationCanceledException) diff --git a/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs b/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs index b708f4d4..3c56b4af 100644 --- a/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs +++ b/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs @@ -35,7 +35,8 @@ public class SerializeProcessFactory( IServerObjectManager serverObjectManager, IProgress? progress, CancellationToken cancellationToken, - SerializeProcessOptions? options = null + SerializeProcessOptions? options = null, + ObjectFlopper? objectFlopper = null ) => new SerializeProcess( progress, diff --git a/src/Speckle.Sdk/Speckle.Sdk.csproj b/src/Speckle.Sdk/Speckle.Sdk.csproj index 5dbc5924..664d497a 100644 --- a/src/Speckle.Sdk/Speckle.Sdk.csproj +++ b/src/Speckle.Sdk/Speckle.Sdk.csproj @@ -23,19 +23,23 @@ + + + + + - diff --git a/src/Speckle.Sdk/packages.lock.json b/src/Speckle.Sdk/packages.lock.json index 5ea95453..b7195702 100644 --- a/src/Speckle.Sdk/packages.lock.json +++ b/src/Speckle.Sdk/packages.lock.json @@ -13,13 +13,25 @@ "System.Reactive": "5.0.0" } }, + "Microsoft.AspNet.WebApi.Client": { + "type": "Direct", + "requested": "[6.0.0, )", + "resolved": "6.0.0", + "contentHash": "zXeWP03dTo67AoDHUzR+/urck0KFssdCKOC+dq7Nv1V2YbFh/nIg09L0/3wSvyRONEdwxGB/ssEGmPNIIhAcAw==", + "dependencies": { + "Newtonsoft.Json": "13.0.1", + "Newtonsoft.Json.Bson": "1.0.2", + "System.Memory": "4.5.5", + "System.Threading.Tasks.Extensions": "4.5.4" + } + }, "Microsoft.Bcl.AsyncInterfaces": { "type": "Direct", - "requested": "[5.0.0, )", - "resolved": "5.0.0", - "contentHash": "W8DPQjkMScOMTtJbPwmPyj9c3zYSFGawDW3jwlBOOsnY+EzZFLgNQ/UMkK35JmkNOVPdCyPr2Tw7Vv9N+KA3ZQ==", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "E1HSLkPHXEO30JEij2pWbOuzz1Z5ND4a5l7IP1T2RgQuE0a0NzEIvtO64RNy3Otn6PFezbT80cfm3M/Cgt70PA==", "dependencies": { - "System.Threading.Tasks.Extensions": "4.5.4" + "System.Threading.Tasks.Extensions": "4.6.3" } }, "Microsoft.CSharp": { @@ -99,6 +111,39 @@ "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" }, + "System.IO.Pipelines": { + "type": "Direct", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "26LbFXHKd7PmRnWlkjnYgmjd5B6HYVG+1MpTO25BdxTJnx6D0O16JPAC/S4YBqjtt4YpfGj1QO/Ss6SPMGEGQw==", + "dependencies": { + "System.Buffers": "4.6.1", + "System.Memory": "4.6.3", + "System.Threading.Tasks.Extensions": "4.6.3" + } + }, + "System.Net.Http.Json": { + "type": "Direct", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "XGOWt78ccgO9esyNlCemlS2b8JZPrH85pk/MvdHJxp6KwwUY/GnDaw2fPpJa7lgotiTkWnXnhip+ULNKaP7a8A==", + "dependencies": { + "System.Buffers": "4.6.1", + "System.Memory": "4.6.3", + "System.Text.Json": "10.0.1", + "System.Threading.Tasks.Extensions": "4.6.3" + } + }, + "System.Threading.Channels": { + "type": "Direct", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "10.0.1", + "System.Threading.Tasks.Extensions": "4.6.3" + } + }, "GraphQL.Client.Abstractions": { "type": "Transitive", "resolved": "6.0.0", @@ -196,6 +241,19 @@ "resolved": "8.0.0", "contentHash": "dk9JPxTCIevS75HyEQ0E4OVAFhB2N+V9ShCXf8Q6FkUQZDkgLI12y679Nym1YqsiSysuQskT7Z+6nUf3yab6Vw==" }, + "Newtonsoft.Json": { + "type": "Transitive", + "resolved": "13.0.1", + "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" + }, + "Newtonsoft.Json.Bson": { + "type": "Transitive", + "resolved": "1.0.2", + "contentHash": "QYFyxhaABwmq3p/21VrZNYvCg3DaEoN/wUuw5nmfAf0X3HLjgupwhkEWdgfb9nvGAUIv3osmZoD3kKl4jxEmYQ==", + "dependencies": { + "Newtonsoft.Json": "12.0.1" + } + }, "SQLitePCLRaw.bundle_e_sqlite3": { "type": "Transitive", "resolved": "2.1.4", @@ -228,8 +286,8 @@ }, "System.Buffers": { "type": "Transitive", - "resolved": "4.4.0", - "contentHash": "AwarXzzoDwX6BgrhjoJsk6tUezZEozOT5Y9QKF94Gl4JK91I4PIIBkBco9068Y9/Dra8Dkbie99kXB8+1BaYKw==" + "resolved": "4.6.1", + "contentHash": "N8GXpmiLMtljq7gwvyS+1QvKT/W2J8sNAvx+HVg4NGmsG/H+2k/y9QI23auLJRterrzCiDH+IWAw4V/GPwsMlw==" }, "System.ComponentModel.Annotations": { "type": "Transitive", @@ -238,18 +296,18 @@ }, "System.Memory": { "type": "Transitive", - "resolved": "4.5.3", - "contentHash": "3oDzvc/zzetpTKWMShs1AADwZjQ/36HnsufHRPcOjyRAAMLDlu2iD33MBI2opxnezcVUtXyqDXXjoFMOU9c7SA==", + "resolved": "4.6.3", + "contentHash": "qdcDOgnFZY40+Q9876JUHnlHu7bosOHX8XISRoH94fwk6hgaeQGSgfZd8srWRZNt5bV9ZW2TljcegDNxsf+96A==", "dependencies": { - "System.Buffers": "4.4.0", - "System.Numerics.Vectors": "4.4.0", - "System.Runtime.CompilerServices.Unsafe": "4.5.2" + "System.Buffers": "4.6.1", + "System.Numerics.Vectors": "4.6.1", + "System.Runtime.CompilerServices.Unsafe": "6.1.2" } }, "System.Numerics.Vectors": { "type": "Transitive", - "resolved": "4.4.0", - "contentHash": "UiLzLW+Lw6HLed1Hcg+8jSRttrbuXv7DANVj0DkL9g6EnnzbL75EB7EWsw5uRbhxd/4YdG8li5XizGWepmG3PQ==" + "resolved": "4.6.1", + "contentHash": "sQxefTnhagrhoq2ReR0D/6K0zJcr9Hrd6kikeXsA1I8kOCboTavcUC4r7TSfpKFeE163uMuxZcyfO1mGO3EN8Q==" }, "System.Reactive": { "type": "Transitive", @@ -271,8 +329,8 @@ }, "System.Runtime.CompilerServices.Unsafe": { "type": "Transitive", - "resolved": "4.5.3", - "contentHash": "3TIsJhD1EiiT0w2CcDMN/iSSwnNnsrnbzeVHSKkaEgV85txMprmuO+Yq2AdSbeVGcg28pdNDTPK87tJhX7VFHw==" + "resolved": "6.1.2", + "contentHash": "2hBr6zdbIBTDE3EhK7NSVNdX58uTK6iHW/P/Axmm9sl1xoGSLqDvMtpecn226TNwHByFokYwJmt/aQQNlO5CRw==" }, "System.Runtime.InteropServices.WindowsRuntime": { "type": "Transitive", @@ -282,16 +340,75 @@ "System.Runtime": "4.3.0" } }, + "System.Text.Encodings.Web": { + "type": "Transitive", + "resolved": "10.0.1", + "contentHash": "cVAka0o1rJJ5/De0pjNs7jcaZk5hUGf1HGzUyVmE2MEB1Vf0h/8qsWxImk1zjitCbeD2Avaq2P2+usdvqgbeVQ==", + "dependencies": { + "System.Buffers": "4.6.1", + "System.Memory": "4.6.3", + "System.Runtime.CompilerServices.Unsafe": "6.1.2" + } + }, "System.Threading.Tasks.Extensions": { "type": "Transitive", - "resolved": "4.5.4", - "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==", + "resolved": "4.6.3", + "contentHash": "7sCiwilJLYbTZELaKnc7RecBBXWXA+xMLQWZKWawBxYjp6DBlSE3v9/UcvKBvr1vv2tTOhipiogM8rRmxlhrVA==", "dependencies": { - "System.Runtime.CompilerServices.Unsafe": "4.5.3" + "System.Runtime.CompilerServices.Unsafe": "6.1.2" + } + }, + "speckle.connectors.common": { + "type": "Project", + "dependencies": { + "Microsoft.Extensions.DependencyInjection": "[2.2.0, )", + "Speckle.Connectors.Logging": "[1.0.0, )", + "Speckle.Converters.Common": "[1.0.0, )", + "Speckle.Objects": "[1.0.0, )" + } + }, + "speckle.connectors.logging": { + "type": "Project" + }, + "speckle.converters.common": { + "type": "Project", + "dependencies": { + "Microsoft.Extensions.Logging.Abstractions": "[2.2.0, )", + "Speckle.Objects": "[1.0.0, )" + } + }, + "speckle.objects": { + "type": "Project", + "dependencies": { + "Speckle.Sdk": "[1.0.0, )" } }, "speckle.sdk.dependencies": { "type": "Project" + }, + "Microsoft.Extensions.DependencyInjection": { + "type": "CentralTransitive", + "requested": "[2.2.0, )", + "resolved": "2.2.0", + "contentHash": "MZtBIwfDFork5vfjpJdG5g8wuJFt7d/y3LOSVVtDK/76wlbtz6cjltfKHqLx2TKVqTj5/c41t77m1+h20zqtPA==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0" + } + }, + "System.Text.Json": { + "type": "CentralTransitive", + "requested": "[8.0.5, )", + "resolved": "10.0.1", + "contentHash": "EsgwDgU1PFqhrFA9l5n+RBu76wFhNGCEwu8ITrBNhjPP3MxLyklroU5GIF8o6JYpYg6T4KD/VICfMdgPAvNp5g==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "10.0.1", + "System.Buffers": "4.6.1", + "System.IO.Pipelines": "10.0.1", + "System.Memory": "4.6.3", + "System.Runtime.CompilerServices.Unsafe": "6.1.2", + "System.Text.Encodings.Web": "10.0.1", + "System.Threading.Tasks.Extensions": "4.6.3" + } } }, "net8.0": { @@ -306,6 +423,18 @@ "System.Reactive": "5.0.0" } }, + "Microsoft.AspNet.WebApi.Client": { + "type": "Direct", + "requested": "[6.0.0, )", + "resolved": "6.0.0", + "contentHash": "zXeWP03dTo67AoDHUzR+/urck0KFssdCKOC+dq7Nv1V2YbFh/nIg09L0/3wSvyRONEdwxGB/ssEGmPNIIhAcAw==", + "dependencies": { + "Newtonsoft.Json": "13.0.1", + "Newtonsoft.Json.Bson": "1.0.2", + "System.Memory": "4.5.5", + "System.Threading.Tasks.Extensions": "4.5.4" + } + }, "Microsoft.Data.Sqlite": { "type": "Direct", "requested": "[7.0.5, )", @@ -368,6 +497,27 @@ "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" }, + "System.IO.Pipelines": { + "type": "Direct", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "26LbFXHKd7PmRnWlkjnYgmjd5B6HYVG+1MpTO25BdxTJnx6D0O16JPAC/S4YBqjtt4YpfGj1QO/Ss6SPMGEGQw==" + }, + "System.Net.Http.Json": { + "type": "Direct", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "XGOWt78ccgO9esyNlCemlS2b8JZPrH85pk/MvdHJxp6KwwUY/GnDaw2fPpJa7lgotiTkWnXnhip+ULNKaP7a8A==", + "dependencies": { + "System.Text.Json": "10.0.1" + } + }, + "System.Threading.Channels": { + "type": "Direct", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" + }, "GraphQL.Client.Abstractions": { "type": "Transitive", "resolved": "6.0.0", @@ -455,6 +605,19 @@ "resolved": "8.0.0", "contentHash": "dk9JPxTCIevS75HyEQ0E4OVAFhB2N+V9ShCXf8Q6FkUQZDkgLI12y679Nym1YqsiSysuQskT7Z+6nUf3yab6Vw==" }, + "Newtonsoft.Json": { + "type": "Transitive", + "resolved": "13.0.1", + "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" + }, + "Newtonsoft.Json.Bson": { + "type": "Transitive", + "resolved": "1.0.2", + "contentHash": "QYFyxhaABwmq3p/21VrZNYvCg3DaEoN/wUuw5nmfAf0X3HLjgupwhkEWdgfb9nvGAUIv3osmZoD3kKl4jxEmYQ==", + "dependencies": { + "Newtonsoft.Json": "12.0.1" + } + }, "SQLitePCLRaw.bundle_e_sqlite3": { "type": "Transitive", "resolved": "2.1.4", @@ -492,8 +655,8 @@ }, "System.Memory": { "type": "Transitive", - "resolved": "4.5.3", - "contentHash": "3oDzvc/zzetpTKWMShs1AADwZjQ/36HnsufHRPcOjyRAAMLDlu2iD33MBI2opxnezcVUtXyqDXXjoFMOU9c7SA==" + "resolved": "4.5.5", + "contentHash": "XIWiDvKPXaTveaB7HVganDlOCRoj03l+jrwNvcge/t8vhGYKvqV+dMv6G4SAX2NoNmN0wZfVPTAlFwZcZvVOUw==" }, "System.Reactive": { "type": "Transitive", @@ -505,8 +668,62 @@ "resolved": "4.5.1", "contentHash": "Zh8t8oqolRaFa9vmOZfdQm/qKejdqz0J9kr7o2Fu0vPeoH3BL1EOXipKWwkWtLT1JPzjByrF19fGuFlNbmPpiw==" }, + "System.Text.Encodings.Web": { + "type": "Transitive", + "resolved": "10.0.1", + "contentHash": "cVAka0o1rJJ5/De0pjNs7jcaZk5hUGf1HGzUyVmE2MEB1Vf0h/8qsWxImk1zjitCbeD2Avaq2P2+usdvqgbeVQ==" + }, + "System.Threading.Tasks.Extensions": { + "type": "Transitive", + "resolved": "4.5.4", + "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==" + }, + "speckle.connectors.common": { + "type": "Project", + "dependencies": { + "Microsoft.Extensions.DependencyInjection": "[2.2.0, )", + "Speckle.Connectors.Logging": "[1.0.0, )", + "Speckle.Converters.Common": "[1.0.0, )", + "Speckle.Objects": "[1.0.0, )" + } + }, + "speckle.connectors.logging": { + "type": "Project" + }, + "speckle.converters.common": { + "type": "Project", + "dependencies": { + "Microsoft.Extensions.Logging.Abstractions": "[2.2.0, )", + "Speckle.Objects": "[1.0.0, )" + } + }, + "speckle.objects": { + "type": "Project", + "dependencies": { + "Speckle.Sdk": "[1.0.0, )" + } + }, "speckle.sdk.dependencies": { "type": "Project" + }, + "Microsoft.Extensions.DependencyInjection": { + "type": "CentralTransitive", + "requested": "[2.2.0, )", + "resolved": "2.2.0", + "contentHash": "MZtBIwfDFork5vfjpJdG5g8wuJFt7d/y3LOSVVtDK/76wlbtz6cjltfKHqLx2TKVqTj5/c41t77m1+h20zqtPA==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0" + } + }, + "System.Text.Json": { + "type": "CentralTransitive", + "requested": "[8.0.5, )", + "resolved": "10.0.1", + "contentHash": "EsgwDgU1PFqhrFA9l5n+RBu76wFhNGCEwu8ITrBNhjPP3MxLyklroU5GIF8o6JYpYg6T4KD/VICfMdgPAvNp5g==", + "dependencies": { + "System.IO.Pipelines": "10.0.1", + "System.Text.Encodings.Web": "10.0.1" + } } } } From 309cead189ec27ce7c7a1dc1f980e62f75aee64e Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Tue, 10 Feb 2026 12:25:05 +0000 Subject: [PATCH 02/16] changes to serializer value handilng (#440) --- src/Speckle.Sdk/Pipelines/Serializer.cs | 211 +++++++++++------------- 1 file changed, 95 insertions(+), 116 deletions(-) diff --git a/src/Speckle.Sdk/Pipelines/Serializer.cs b/src/Speckle.Sdk/Pipelines/Serializer.cs index ed072a50..3f551db3 100644 --- a/src/Speckle.Sdk/Pipelines/Serializer.cs +++ b/src/Speckle.Sdk/Pipelines/Serializer.cs @@ -1,4 +1,6 @@ using System.Collections; +using System.Drawing; +using System.Globalization; using System.Reflection; using Speckle.DoubleNumerics; using Speckle.Newtonsoft.Json; @@ -163,42 +165,20 @@ public class Serializer List<(Id, Json, Dictionary, Base, string)> detachedObjects ) { - if (value == null) - { - writer.WriteNull(); - return; - } - switch (value) { - case string s: - writer.WriteValue(s); - return; - case int i: - writer.WriteValue(i); - return; - case long l: - writer.WriteValue(l); - return; - case double d: - writer.WriteValue(d); - return; - case float f: - writer.WriteValue(f); - return; - case bool b: - writer.WriteValue(b); - return; - case Enum e: - writer.WriteValue((int)(object)e); + case Enum: + writer.WriteValue((int)value); return; case Guid g: writer.WriteValue(g.ToString()); return; - case DateTime dt: - writer.WriteValue(dt.ToString("o")); + case Color c: + writer.WriteValue(c.ToArgb()); + return; + case DateTime dt: + writer.WriteValue(dt.ToString("o", CultureInfo.InvariantCulture)); return; - case Matrix4x4 md: writer.WriteStartArray(); writer.WriteValue(md.M11); @@ -219,107 +199,106 @@ public class Serializer writer.WriteValue(md.M44); writer.WriteEndArray(); return; - } - - // Handle ObjectReference before Base (since ObjectReference extends Base) - // This prevents double-serialization and properly propagates closures - if (value is ObjectReference objRef) - { - writer.WriteStartObject(); - writer.WritePropertyName("speckle_type"); - writer.WriteValue("reference"); - writer.WritePropertyName("referencedId"); - writer.WriteValue(objRef.referencedId); - writer.WriteEndObject(); - - // Propagate closure: add the referenced ID - closures[objRef.referencedId] = closures.TryGetValue(objRef.referencedId, out var existing) ? existing + 1 : 1; - - // Propagate nested closures from the ObjectReference.closure dictionary - if (objRef.closure != null) + // Handle ObjectReference before Base (since ObjectReference extends Base) + // This prevents double-serialization and properly propagates closures + case ObjectReference objRef: { - foreach (var kvp in objRef.closure) - { - closures[kvp.Key] = closures.TryGetValue(kvp.Key, out var existingDepth) - ? existingDepth + kvp.Value - : kvp.Value; - } - } - - return; - } - - if (value is Base baseObj) - { - if (isDetachable) - { - var childClosures = new Dictionary(); - var (childId, childJson) = SerializeBase(baseObj, true, childClosures, detachedObjects); - - detachedObjects.Add((childId, childJson, childClosures, baseObj, baseObj.speckle_type)); - writer.WriteStartObject(); writer.WritePropertyName("speckle_type"); writer.WriteValue("reference"); writer.WritePropertyName("referencedId"); - writer.WriteValue(childId.Value); + writer.WriteValue(objRef.referencedId); writer.WriteEndObject(); - closures[childId.Value] = closures.TryGetValue(childId.Value, out var existing) ? existing + 1 : 1; + // Propagate closure: add the referenced ID + closures[objRef.referencedId] = closures.TryGetValue(objRef.referencedId, out var existing) ? existing + 1 : 1; - foreach (var kvp in childClosures) + // Propagate nested closures from the ObjectReference.closure dictionary + if (objRef.closure != null) { - closures[kvp.Key] = closures.TryGetValue(kvp.Key, out var existingDepth) - ? existingDepth + kvp.Value - : kvp.Value; - } - } - else - { - var inlineClosures = new Dictionary(); - var (_, inlineJson) = SerializeBase(baseObj, false, inlineClosures, detachedObjects); - - writer.WriteRawValue(inlineJson.Value); - - foreach (var kvp in inlineClosures) - { - closures[kvp.Key] = closures.TryGetValue(kvp.Key, out var existingDepth) - ? existingDepth + kvp.Value - : kvp.Value; - } - } - return; - } - - if (value is IDictionary dict) - { - writer.WriteStartObject(); - foreach (DictionaryEntry kvp in dict) - { - if (kvp.Key is not string key) - { - throw new ArgumentException("Dictionary keys must be strings"); + foreach (var kvp in objRef.closure) + { + closures[kvp.Key] = closures.TryGetValue(kvp.Key, out var existingDepth) + ? existingDepth + kvp.Value + : kvp.Value; + } } - writer.WritePropertyName(key); - SerializeValue(kvp.Value, writer, false, closures, detachedObjects); + return; } - writer.WriteEndObject(); - return; - } - - if (value is IEnumerable enumerable and not string) - { - writer.WriteStartArray(); - foreach (var item in enumerable) + case Base baseObj: { - SerializeValue(item, writer, isDetachable, closures, detachedObjects); - } - writer.WriteEndArray(); - return; - } + if (isDetachable) + { + var childClosures = new Dictionary(); + var (childId, childJson) = SerializeBase(baseObj, true, childClosures, detachedObjects); - writer.WriteValue(value.ToString()); + detachedObjects.Add((childId, childJson, childClosures, baseObj, baseObj.speckle_type)); + + writer.WriteStartObject(); + writer.WritePropertyName("speckle_type"); + writer.WriteValue("reference"); + writer.WritePropertyName("referencedId"); + writer.WriteValue(childId.Value); + writer.WriteEndObject(); + + closures[childId.Value] = closures.TryGetValue(childId.Value, out var existing) ? existing + 1 : 1; + + foreach (var kvp in childClosures) + { + closures[kvp.Key] = closures.TryGetValue(kvp.Key, out var existingDepth) + ? existingDepth + kvp.Value + : kvp.Value; + } + } + else + { + var inlineClosures = new Dictionary(); + var (_, inlineJson) = SerializeBase(baseObj, false, inlineClosures, detachedObjects); + + writer.WriteRawValue(inlineJson.Value); + + foreach (var kvp in inlineClosures) + { + closures[kvp.Key] = closures.TryGetValue(kvp.Key, out var existingDepth) + ? existingDepth + kvp.Value + : kvp.Value; + } + } + return; + } + case IDictionary dict: + { + writer.WriteStartObject(); + foreach (DictionaryEntry kvp in dict) + { + if (kvp.Key is not string key) + { + throw new ArgumentException("Dictionary keys must be strings", nameof(value)); + } + + writer.WritePropertyName(key); + SerializeValue(kvp.Value, writer, false, closures, detachedObjects); + } + writer.WriteEndObject(); + return; + } + case ICollection collection: + { + writer.WriteStartArray(); + foreach (var item in collection) + { + SerializeValue(item, writer, isDetachable, closures, detachedObjects); + } + writer.WriteEndArray(); + return; + } + default: + // This case will handle primitives and `null` + // Will throw JsonWriterException if not supported + writer.WriteValue(value); + return; + } } private UploadItem ReferenceToUploadItem(ObjectReference existingRef) From bf6ae0f6af2b626317830ff9f6f5227310c8897a Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Tue, 10 Feb 2026 16:29:27 +0000 Subject: [PATCH 03/16] Duck: Cleanup dependencies (#441) * Remove unused dependencies * remove unused package references * how's this * missed a bit --- Directory.Packages.props | 6 +- src/Speckle.Automate.Sdk/packages.lock.json | 50 ++-- src/Speckle.Objects/packages.lock.json | 179 ++----------- .../packages.lock.json | 17 +- src/Speckle.Sdk/Pipelines/SendPipeline.cs | 9 +- src/Speckle.Sdk/Pipelines/Uploader.cs | 53 ++-- .../Serialisation/V2/Send/ObjectFlopper.cs | 79 ------ .../V2/Send/ObjectFlopperGandalf.cs | 90 ------- .../V2/SerializeProcessFactory.cs | 3 +- src/Speckle.Sdk/Speckle.Sdk.csproj | 4 - src/Speckle.Sdk/packages.lock.json | 240 ++---------------- .../packages.lock.json | 9 +- .../packages.lock.json | 9 +- .../packages.lock.json | 20 +- .../packages.lock.json | 9 +- tests/Speckle.Sdk.Testing/packages.lock.json | 9 +- .../packages.lock.json | 9 +- .../packages.lock.json | 20 +- .../Speckle.Sdk.Tests.Unit/packages.lock.json | 9 +- 19 files changed, 193 insertions(+), 631 deletions(-) delete mode 100644 src/Speckle.Sdk/Serialisation/V2/Send/ObjectFlopper.cs delete mode 100644 src/Speckle.Sdk/Serialisation/V2/Send/ObjectFlopperGandalf.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 6bf24322..a2c96acf 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -8,8 +8,6 @@ - - @@ -29,8 +27,6 @@ - - @@ -42,4 +38,4 @@ - \ No newline at end of file + diff --git a/src/Speckle.Automate.Sdk/packages.lock.json b/src/Speckle.Automate.Sdk/packages.lock.json index 5e6185c6..1a222bbb 100644 --- a/src/Speckle.Automate.Sdk/packages.lock.json +++ b/src/Speckle.Automate.Sdk/packages.lock.json @@ -86,6 +86,14 @@ "resolved": "6.0.0", "contentHash": "yg72rrYDapfsIUrul7aF6wwNnTJBOFvuA9VdDTQpPa8AlAriHbufeXYLBcodKjfUdkCnaiggX1U/nEP08Zb5GA==" }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "Transitive", + "resolved": "10.0.1", + "contentHash": "E1HSLkPHXEO30JEij2pWbOuzz1Z5ND4a5l7IP1T2RgQuE0a0NzEIvtO64RNy3Otn6PFezbT80cfm3M/Cgt70PA==", + "dependencies": { + "System.Threading.Tasks.Extensions": "4.6.3" + } + }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", "resolved": "8.0.0", @@ -242,8 +250,8 @@ }, "System.Runtime.CompilerServices.Unsafe": { "type": "Transitive", - "resolved": "6.0.0", - "contentHash": "/iUeP3tq1S0XdNNoMz5C9twLSrM/TH+qElHkXWaPvuNOt+99G75NrV0OS2EqHx5wMN7popYjpc8oTjC1y16DLg==" + "resolved": "6.1.2", + "contentHash": "2hBr6zdbIBTDE3EhK7NSVNdX58uTK6iHW/P/Axmm9sl1xoGSLqDvMtpecn226TNwHByFokYwJmt/aQQNlO5CRw==" }, "System.Runtime.InteropServices.WindowsRuntime": { "type": "Transitive", @@ -265,10 +273,10 @@ }, "System.Threading.Tasks.Extensions": { "type": "Transitive", - "resolved": "4.5.4", - "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==", + "resolved": "4.6.3", + "contentHash": "7sCiwilJLYbTZELaKnc7RecBBXWXA+xMLQWZKWawBxYjp6DBlSE3v9/UcvKBvr1vv2tTOhipiogM8rRmxlhrVA==", "dependencies": { - "System.Runtime.CompilerServices.Unsafe": "4.5.3" + "System.Runtime.CompilerServices.Unsafe": "6.1.2" } }, "speckle.objects": { @@ -281,14 +289,14 @@ "type": "Project", "dependencies": { "GraphQL.Client": "[6.0.0, )", - "Microsoft.Bcl.AsyncInterfaces": "[5.0.0, )", "Microsoft.CSharp": "[4.7.0, )", "Microsoft.Data.Sqlite": "[7.0.5, )", "Microsoft.Extensions.DependencyInjection.Abstractions": "[2.2.0, )", "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )", + "System.Threading.Channels": "[10.0.1, )" } }, "speckle.sdk.dependencies": { @@ -305,15 +313,6 @@ "System.Reactive": "5.0.0" } }, - "Microsoft.Bcl.AsyncInterfaces": { - "type": "CentralTransitive", - "requested": "[5.0.0, )", - "resolved": "8.0.0", - "contentHash": "3WA9q9yVqJp222P3x1wYIGDAkpjAku0TMUaaQV22g6L67AI0LdOIrVS7Ht2vJfLHGSPVuqN94vIr15qn+HEkHw==", - "dependencies": { - "System.Threading.Tasks.Extensions": "4.5.4" - } - }, "Microsoft.CSharp": { "type": "CentralTransitive", "requested": "[4.7.0, )", @@ -359,6 +358,16 @@ "requested": "[13.0.2, )", "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" + }, + "System.Threading.Channels": { + "type": "CentralTransitive", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "10.0.1", + "System.Threading.Tasks.Extensions": "4.6.3" + } } }, "net8.0": { @@ -556,7 +565,8 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )", + "System.Threading.Channels": "[10.0.1, )" } }, "speckle.sdk.dependencies": { @@ -612,6 +622,12 @@ "requested": "[13.0.2, )", "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" + }, + "System.Threading.Channels": { + "type": "CentralTransitive", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" } } } diff --git a/src/Speckle.Objects/packages.lock.json b/src/Speckle.Objects/packages.lock.json index 2c2e04f1..5042eec7 100644 --- a/src/Speckle.Objects/packages.lock.json +++ b/src/Speckle.Objects/packages.lock.json @@ -54,6 +54,14 @@ "resolved": "6.0.0", "contentHash": "yg72rrYDapfsIUrul7aF6wwNnTJBOFvuA9VdDTQpPa8AlAriHbufeXYLBcodKjfUdkCnaiggX1U/nEP08Zb5GA==" }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "Transitive", + "resolved": "10.0.1", + "contentHash": "E1HSLkPHXEO30JEij2pWbOuzz1Z5ND4a5l7IP1T2RgQuE0a0NzEIvtO64RNy3Otn6PFezbT80cfm3M/Cgt70PA==", + "dependencies": { + "System.Threading.Tasks.Extensions": "4.6.3" + } + }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", "resolved": "8.0.0", @@ -130,19 +138,6 @@ "resolved": "8.0.0", "contentHash": "dk9JPxTCIevS75HyEQ0E4OVAFhB2N+V9ShCXf8Q6FkUQZDkgLI12y679Nym1YqsiSysuQskT7Z+6nUf3yab6Vw==" }, - "Newtonsoft.Json": { - "type": "Transitive", - "resolved": "13.0.1", - "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" - }, - "Newtonsoft.Json.Bson": { - "type": "Transitive", - "resolved": "1.0.2", - "contentHash": "QYFyxhaABwmq3p/21VrZNYvCg3DaEoN/wUuw5nmfAf0X3HLjgupwhkEWdgfb9nvGAUIv3osmZoD3kKl4jxEmYQ==", - "dependencies": { - "Newtonsoft.Json": "12.0.1" - } - }, "SQLitePCLRaw.bundle_e_sqlite3": { "type": "Transitive", "resolved": "2.1.4", @@ -175,8 +170,8 @@ }, "System.Buffers": { "type": "Transitive", - "resolved": "4.6.1", - "contentHash": "N8GXpmiLMtljq7gwvyS+1QvKT/W2J8sNAvx+HVg4NGmsG/H+2k/y9QI23auLJRterrzCiDH+IWAw4V/GPwsMlw==" + "resolved": "4.4.0", + "contentHash": "AwarXzzoDwX6BgrhjoJsk6tUezZEozOT5Y9QKF94Gl4JK91I4PIIBkBco9068Y9/Dra8Dkbie99kXB8+1BaYKw==" }, "System.ComponentModel.Annotations": { "type": "Transitive", @@ -185,18 +180,18 @@ }, "System.Memory": { "type": "Transitive", - "resolved": "4.6.3", - "contentHash": "qdcDOgnFZY40+Q9876JUHnlHu7bosOHX8XISRoH94fwk6hgaeQGSgfZd8srWRZNt5bV9ZW2TljcegDNxsf+96A==", + "resolved": "4.5.3", + "contentHash": "3oDzvc/zzetpTKWMShs1AADwZjQ/36HnsufHRPcOjyRAAMLDlu2iD33MBI2opxnezcVUtXyqDXXjoFMOU9c7SA==", "dependencies": { - "System.Buffers": "4.6.1", - "System.Numerics.Vectors": "4.6.1", - "System.Runtime.CompilerServices.Unsafe": "6.1.2" + "System.Buffers": "4.4.0", + "System.Numerics.Vectors": "4.4.0", + "System.Runtime.CompilerServices.Unsafe": "4.5.2" } }, "System.Numerics.Vectors": { "type": "Transitive", - "resolved": "4.6.1", - "contentHash": "sQxefTnhagrhoq2ReR0D/6K0zJcr9Hrd6kikeXsA1I8kOCboTavcUC4r7TSfpKFeE163uMuxZcyfO1mGO3EN8Q==" + "resolved": "4.4.0", + "contentHash": "UiLzLW+Lw6HLed1Hcg+8jSRttrbuXv7DANVj0DkL9g6EnnzbL75EB7EWsw5uRbhxd/4YdG8li5XizGWepmG3PQ==" }, "System.Reactive": { "type": "Transitive", @@ -229,16 +224,6 @@ "System.Runtime": "4.3.0" } }, - "System.Text.Encodings.Web": { - "type": "Transitive", - "resolved": "10.0.1", - "contentHash": "cVAka0o1rJJ5/De0pjNs7jcaZk5hUGf1HGzUyVmE2MEB1Vf0h/8qsWxImk1zjitCbeD2Avaq2P2+usdvqgbeVQ==", - "dependencies": { - "System.Buffers": "4.6.1", - "System.Memory": "4.6.3", - "System.Runtime.CompilerServices.Unsafe": "6.1.2" - } - }, "System.Threading.Tasks.Extensions": { "type": "Transitive", "resolved": "4.6.3", @@ -251,8 +236,6 @@ "type": "Project", "dependencies": { "GraphQL.Client": "[6.0.0, )", - "Microsoft.AspNet.WebApi.Client": "[6.0.0, )", - "Microsoft.Bcl.AsyncInterfaces": "[10.0.1, )", "Microsoft.CSharp": "[4.7.0, )", "Microsoft.Data.Sqlite": "[7.0.5, )", "Microsoft.Extensions.DependencyInjection.Abstractions": "[2.2.0, )", @@ -260,8 +243,6 @@ "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", "Speckle.Sdk.Dependencies": "[1.0.0, )", - "System.IO.Pipelines": "[10.0.1, )", - "System.Net.Http.Json": "[10.0.1, )", "System.Threading.Channels": "[10.0.1, )" } }, @@ -279,27 +260,6 @@ "System.Reactive": "5.0.0" } }, - "Microsoft.AspNet.WebApi.Client": { - "type": "CentralTransitive", - "requested": "[6.0.0, )", - "resolved": "6.0.0", - "contentHash": "zXeWP03dTo67AoDHUzR+/urck0KFssdCKOC+dq7Nv1V2YbFh/nIg09L0/3wSvyRONEdwxGB/ssEGmPNIIhAcAw==", - "dependencies": { - "Newtonsoft.Json": "13.0.1", - "Newtonsoft.Json.Bson": "1.0.2", - "System.Memory": "4.5.5", - "System.Threading.Tasks.Extensions": "4.5.4" - } - }, - "Microsoft.Bcl.AsyncInterfaces": { - "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "E1HSLkPHXEO30JEij2pWbOuzz1Z5ND4a5l7IP1T2RgQuE0a0NzEIvtO64RNy3Otn6PFezbT80cfm3M/Cgt70PA==", - "dependencies": { - "System.Threading.Tasks.Extensions": "4.6.3" - } - }, "Microsoft.CSharp": { "type": "CentralTransitive", "requested": "[4.7.0, )", @@ -346,44 +306,6 @@ "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" }, - "System.IO.Pipelines": { - "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "26LbFXHKd7PmRnWlkjnYgmjd5B6HYVG+1MpTO25BdxTJnx6D0O16JPAC/S4YBqjtt4YpfGj1QO/Ss6SPMGEGQw==", - "dependencies": { - "System.Buffers": "4.6.1", - "System.Memory": "4.6.3", - "System.Threading.Tasks.Extensions": "4.6.3" - } - }, - "System.Net.Http.Json": { - "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "XGOWt78ccgO9esyNlCemlS2b8JZPrH85pk/MvdHJxp6KwwUY/GnDaw2fPpJa7lgotiTkWnXnhip+ULNKaP7a8A==", - "dependencies": { - "System.Buffers": "4.6.1", - "System.Memory": "4.6.3", - "System.Text.Json": "10.0.1", - "System.Threading.Tasks.Extensions": "4.6.3" - } - }, - "System.Text.Json": { - "type": "CentralTransitive", - "requested": "[8.0.5, )", - "resolved": "10.0.1", - "contentHash": "EsgwDgU1PFqhrFA9l5n+RBu76wFhNGCEwu8ITrBNhjPP3MxLyklroU5GIF8o6JYpYg6T4KD/VICfMdgPAvNp5g==", - "dependencies": { - "Microsoft.Bcl.AsyncInterfaces": "10.0.1", - "System.Buffers": "4.6.1", - "System.IO.Pipelines": "10.0.1", - "System.Memory": "4.6.3", - "System.Runtime.CompilerServices.Unsafe": "6.1.2", - "System.Text.Encodings.Web": "10.0.1", - "System.Threading.Tasks.Extensions": "4.6.3" - } - }, "System.Threading.Channels": { "type": "CentralTransitive", "requested": "[10.0.1, )", @@ -505,19 +427,6 @@ "resolved": "8.0.0", "contentHash": "dk9JPxTCIevS75HyEQ0E4OVAFhB2N+V9ShCXf8Q6FkUQZDkgLI12y679Nym1YqsiSysuQskT7Z+6nUf3yab6Vw==" }, - "Newtonsoft.Json": { - "type": "Transitive", - "resolved": "13.0.1", - "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" - }, - "Newtonsoft.Json.Bson": { - "type": "Transitive", - "resolved": "1.0.2", - "contentHash": "QYFyxhaABwmq3p/21VrZNYvCg3DaEoN/wUuw5nmfAf0X3HLjgupwhkEWdgfb9nvGAUIv3osmZoD3kKl4jxEmYQ==", - "dependencies": { - "Newtonsoft.Json": "12.0.1" - } - }, "SQLitePCLRaw.bundle_e_sqlite3": { "type": "Transitive", "resolved": "2.1.4", @@ -555,8 +464,8 @@ }, "System.Memory": { "type": "Transitive", - "resolved": "4.5.5", - "contentHash": "XIWiDvKPXaTveaB7HVganDlOCRoj03l+jrwNvcge/t8vhGYKvqV+dMv6G4SAX2NoNmN0wZfVPTAlFwZcZvVOUw==" + "resolved": "4.5.3", + "contentHash": "3oDzvc/zzetpTKWMShs1AADwZjQ/36HnsufHRPcOjyRAAMLDlu2iD33MBI2opxnezcVUtXyqDXXjoFMOU9c7SA==" }, "System.Reactive": { "type": "Transitive", @@ -568,29 +477,16 @@ "resolved": "4.5.1", "contentHash": "Zh8t8oqolRaFa9vmOZfdQm/qKejdqz0J9kr7o2Fu0vPeoH3BL1EOXipKWwkWtLT1JPzjByrF19fGuFlNbmPpiw==" }, - "System.Text.Encodings.Web": { - "type": "Transitive", - "resolved": "10.0.1", - "contentHash": "cVAka0o1rJJ5/De0pjNs7jcaZk5hUGf1HGzUyVmE2MEB1Vf0h/8qsWxImk1zjitCbeD2Avaq2P2+usdvqgbeVQ==" - }, - "System.Threading.Tasks.Extensions": { - "type": "Transitive", - "resolved": "4.5.4", - "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==" - }, "speckle.sdk": { "type": "Project", "dependencies": { "GraphQL.Client": "[6.0.0, )", - "Microsoft.AspNet.WebApi.Client": "[6.0.0, )", "Microsoft.Data.Sqlite": "[7.0.5, )", "Microsoft.Extensions.DependencyInjection.Abstractions": "[2.2.0, )", "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", "Speckle.Sdk.Dependencies": "[1.0.0, )", - "System.IO.Pipelines": "[10.0.1, )", - "System.Net.Http.Json": "[10.0.1, )", "System.Threading.Channels": "[10.0.1, )" } }, @@ -608,18 +504,6 @@ "System.Reactive": "5.0.0" } }, - "Microsoft.AspNet.WebApi.Client": { - "type": "CentralTransitive", - "requested": "[6.0.0, )", - "resolved": "6.0.0", - "contentHash": "zXeWP03dTo67AoDHUzR+/urck0KFssdCKOC+dq7Nv1V2YbFh/nIg09L0/3wSvyRONEdwxGB/ssEGmPNIIhAcAw==", - "dependencies": { - "Newtonsoft.Json": "13.0.1", - "Newtonsoft.Json.Bson": "1.0.2", - "System.Memory": "4.5.5", - "System.Threading.Tasks.Extensions": "4.5.4" - } - }, "Microsoft.Data.Sqlite": { "type": "CentralTransitive", "requested": "[7.0.5, )", @@ -660,31 +544,6 @@ "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" }, - "System.IO.Pipelines": { - "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "26LbFXHKd7PmRnWlkjnYgmjd5B6HYVG+1MpTO25BdxTJnx6D0O16JPAC/S4YBqjtt4YpfGj1QO/Ss6SPMGEGQw==" - }, - "System.Net.Http.Json": { - "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "XGOWt78ccgO9esyNlCemlS2b8JZPrH85pk/MvdHJxp6KwwUY/GnDaw2fPpJa7lgotiTkWnXnhip+ULNKaP7a8A==", - "dependencies": { - "System.Text.Json": "10.0.1" - } - }, - "System.Text.Json": { - "type": "CentralTransitive", - "requested": "[8.0.5, )", - "resolved": "10.0.1", - "contentHash": "EsgwDgU1PFqhrFA9l5n+RBu76wFhNGCEwu8ITrBNhjPP3MxLyklroU5GIF8o6JYpYg6T4KD/VICfMdgPAvNp5g==", - "dependencies": { - "System.IO.Pipelines": "10.0.1", - "System.Text.Encodings.Web": "10.0.1" - } - }, "System.Threading.Channels": { "type": "CentralTransitive", "requested": "[10.0.1, )", diff --git a/src/Speckle.Sdk.Dependencies/packages.lock.json b/src/Speckle.Sdk.Dependencies/packages.lock.json index 1f256577..aa13bc43 100644 --- a/src/Speckle.Sdk.Dependencies/packages.lock.json +++ b/src/Speckle.Sdk.Dependencies/packages.lock.json @@ -95,6 +95,14 @@ "resolved": "2.0.33", "contentHash": "xb2h1CsOepoYwdXEPui9VcQglwABQwNf9cccZbf+acarEzF5PUp8Xx71nFXIhOgEdm6wrxAoF6xAxK4m/XFRUQ==" }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "Transitive", + "resolved": "10.0.1", + "contentHash": "E1HSLkPHXEO30JEij2pWbOuzz1Z5ND4a5l7IP1T2RgQuE0a0NzEIvtO64RNy3Otn6PFezbT80cfm3M/Cgt70PA==", + "dependencies": { + "System.Threading.Tasks.Extensions": "4.6.3" + } + }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", "resolved": "8.0.0", @@ -151,15 +159,6 @@ "dependencies": { "System.Runtime.CompilerServices.Unsafe": "6.1.2" } - }, - "Microsoft.Bcl.AsyncInterfaces": { - "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "E1HSLkPHXEO30JEij2pWbOuzz1Z5ND4a5l7IP1T2RgQuE0a0NzEIvtO64RNy3Otn6PFezbT80cfm3M/Cgt70PA==", - "dependencies": { - "System.Threading.Tasks.Extensions": "4.6.3" - } } }, "net8.0": { diff --git a/src/Speckle.Sdk/Pipelines/SendPipeline.cs b/src/Speckle.Sdk/Pipelines/SendPipeline.cs index 4c580163..1079f1f7 100644 --- a/src/Speckle.Sdk/Pipelines/SendPipeline.cs +++ b/src/Speckle.Sdk/Pipelines/SendPipeline.cs @@ -21,7 +21,14 @@ public sealed class SendPipeline : IDisposable ) { _cancellationToken = cancellationToken; - _uploader = new Uploader(projectId, modelId, ingestionId, account.serverInfo.url, account.token, cancellationToken); + _uploader = new Uploader( + projectId, + modelId, + ingestionId, + new(account.serverInfo.url), + account.token, + cancellationToken + ); } private UploadItem _lastItem; diff --git a/src/Speckle.Sdk/Pipelines/Uploader.cs b/src/Speckle.Sdk/Pipelines/Uploader.cs index ec8133a4..a7bf6e7e 100644 --- a/src/Speckle.Sdk/Pipelines/Uploader.cs +++ b/src/Speckle.Sdk/Pipelines/Uploader.cs @@ -1,7 +1,8 @@ using System.IO.Compression; using System.Net.Http.Headers; -using System.Net.Http.Json; +using System.Text; using System.Threading.Channels; +using Speckle.Newtonsoft.Json; namespace Speckle.Sdk.Pipelines; @@ -19,7 +20,7 @@ public sealed class Uploader : IDisposable string projectId, string modelId, string ingestionId, - string apiEndpoint, + Uri serverUrl, string authToken, CancellationToken cancellationToken ) @@ -28,14 +29,14 @@ public sealed class Uploader : IDisposable _modelId = modelId; _ingestionId = ingestionId; _cancellationToken = cancellationToken; + Uri apiBaseUrl = new(serverUrl, "/api/v1/"); - Uri apiBaseUrl = new(new(apiEndpoint), "/api/v1/"); _client = new HttpClient { BaseAddress = apiBaseUrl, Timeout = TimeSpan.FromMinutes(30) }; _client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", authToken); _channel = Channel.CreateBounded( - new BoundedChannelOptions(1000) { FullMode = BoundedChannelFullMode.Wait } + new BoundedChannelOptions(1000) { FullMode = BoundedChannelFullMode.Wait, SingleReader = true } ); _sendTask = SendLoopAsync(); @@ -66,9 +67,11 @@ public sealed class Uploader : IDisposable { await writer.WriteLineAsync($"{item.Id}\t{item.Json}\t{item.SpeckleType}").ConfigureAwait(false); } - +#if NET8_0_OR_GREATER + await writer.FlushAsync(_cancellationToken).ConfigureAwait(false); +#else await writer.FlushAsync().ConfigureAwait(false); - await gzip.FlushAsync(_cancellationToken).ConfigureAwait(false); +#endif } // fileStream.Flush(); // fileStream.Close(); @@ -77,13 +80,18 @@ public sealed class Uploader : IDisposable // 2. Request presigned URL var signUri = new Uri($"projects/{_projectId}/models/{_modelId}/uploads/sign", UriKind.Relative); - var signResponse = await HttpClientExtensions - .PostAsJsonAsync(_client, signUri, _cancellationToken) - .ConfigureAwait(false); + using var signResponse = await _client.PostAsync(signUri, null, _cancellationToken).ConfigureAwait(false); signResponse.EnsureSuccessStatusCode(); - var presignedUpload = - await signResponse.Content.ReadFromJsonAsync(_cancellationToken).ConfigureAwait(false) +#if NET5_0_OR_GREATER + string signResponseString = await signResponse + .Content.ReadAsStringAsync(_cancellationToken) + .ConfigureAwait(false); +#else + string signResponseString = await signResponse.Content.ReadAsStringAsync().ConfigureAwait(false); +#endif + PresignedUploadResponse presignedUpload = + JsonConvert.DeserializeObject(signResponseString) ?? throw new InvalidOperationException("Failed to get presigned upload URL"); // 3. Upload to S3 @@ -109,25 +117,24 @@ public sealed class Uploader : IDisposable var processUri = new Uri($"projects/{_projectId}/models/{_modelId}/uploads/process", UriKind.Relative); var processRequest = new ProcessUploadRequest { key = presignedUpload.Key, ingestionId = _ingestionId }; - var processResponse = await HttpClientExtensions - .PostAsJsonAsync(_client, processUri, processRequest, _cancellationToken) - .ConfigureAwait(false); + using StringContent content = new(JsonConvert.SerializeObject(processRequest), Encoding.UTF8, "application/json"); + var processResponse = await _client.PostAsync(processUri, content, _cancellationToken).ConfigureAwait(false); + processResponse.EnsureSuccessStatusCode(); - var processResult = await processResponse - .Content.ReadFromJsonAsync(_cancellationToken) - .ConfigureAwait(false); +#if NET5_0_OR_GREATER + string processResult = await processResponse.Content.ReadAsStringAsync(_cancellationToken).ConfigureAwait(false); +#else + string processResult = await processResponse.Content.ReadAsStringAsync().ConfigureAwait(false); +#endif + var json = JsonConvert.DeserializeObject(processResult); - if (processResult == null) + if (json is null) { throw new InvalidOperationException("Failed to trigger upload processing"); } - return new UploadResult { IngestionId = processResult.ingestionId }; - } - catch (Exception ex) when (!ex.IsFatal()) - { - throw; + return new UploadResult { IngestionId = json.ingestionId }; } finally { diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/ObjectFlopper.cs b/src/Speckle.Sdk/Serialisation/V2/Send/ObjectFlopper.cs deleted file mode 100644 index 2027a6e3..00000000 --- a/src/Speckle.Sdk/Serialisation/V2/Send/ObjectFlopper.cs +++ /dev/null @@ -1,79 +0,0 @@ -using System.IO.Compression; -using System.Net.Http.Headers; -using System.Threading.Channels; - -namespace Speckle.Sdk.Serialisation.V2.Send; - -#pragma warning disable CA1001 -public sealed class ObjectFlopper -#pragma warning restore CA1001 -{ - private readonly Uri _url; - private readonly string _streamId; - private readonly HttpClient _client; - private readonly Channel _channel; - private readonly Task _sendTask; - - public ObjectFlopper(Uri _, string streamId, string? authToken) - { - _streamId = streamId; - _url = new Uri("http://zog.local:3000/api/v1/"); - _client = new HttpClient { BaseAddress = _url, Timeout = TimeSpan.FromMinutes(10) }; - - if (authToken != null) - { - _client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", authToken); - } - - _channel = Channel.CreateBounded( - new BoundedChannelOptions(1000) { FullMode = BoundedChannelFullMode.Wait } - ); - - _sendTask = SendLoopAsync(streamId, "test"); - } - - public ValueTask PushAsync(BaseItem item, CancellationToken ct = default) => _channel.Writer.WriteAsync(item, ct); - - public async Task CompleteAsync() - { - _channel.Writer.Complete(); - await _sendTask.ConfigureAwait(false); - } - - private async Task SendLoopAsync(string projectId, string modelId) - { - var content = new PushStreamContent( - async (stream, _, _) => - { - var gzip = new GZipStream(stream, CompressionLevel.Optimal); - var writer = new StreamWriter(gzip); //new StreamWriter(gzip, System.Text.Encoding.UTF8, 20 * 1024 * 1024); - try - { - await foreach (var item in _channel.Reader.ReadAllAsync().ConfigureAwait(false)) - { - await writer.WriteLineAsync($"{item.Id}\t{item.Json}").ConfigureAwait(false); - } - } - finally - { - await writer.FlushAsync().ConfigureAwait(false); - await gzip.FlushAsync().ConfigureAwait(false); - writer.Dispose(); - gzip.Dispose(); - } - }, - new MediaTypeHeaderValue("application/x-ndjson") - ); - - var uri = new Uri($"projects/{projectId}/models/{modelId}/versions", UriKind.Relative); - var request = new HttpRequestMessage(HttpMethod.Post, uri) { Content = content }; - request.Headers.TransferEncodingChunked = true; // NOTE: important for streaming to happen. - var response = await _client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); - - // Consume the response body to fully complete the request - await response.Content.ReadAsStringAsync().ConfigureAwait(false); - } - - public void Dispose() => _client.Dispose(); -} diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/ObjectFlopperGandalf.cs b/src/Speckle.Sdk/Serialisation/V2/Send/ObjectFlopperGandalf.cs deleted file mode 100644 index abccd3a6..00000000 --- a/src/Speckle.Sdk/Serialisation/V2/Send/ObjectFlopperGandalf.cs +++ /dev/null @@ -1,90 +0,0 @@ -using System.IO.Compression; -using System.IO.Pipelines; -using System.Net.Http.Headers; -using System.Threading.Channels; - -namespace Speckle.Sdk.Serialisation.V2.Send; - -#pragma warning disable CA1001 -public sealed class ObjectFlopperGandalf -#pragma warning restore CA1001 -{ - private readonly Uri _url; - private readonly string _streamId; - private readonly HttpClient _client; - private readonly Channel _channel; - private readonly Task _sendTask; - - public ObjectFlopperGandalf(Uri _, string streamId, string? authToken) - { - _streamId = streamId; - _url = new Uri("http://bender-2.local:3000/api/v1/"); - _client = new HttpClient { BaseAddress = _url, Timeout = TimeSpan.FromMinutes(10) }; - - if (authToken != null) - { - _client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", authToken); - } - - _channel = Channel.CreateBounded( - new BoundedChannelOptions(1000) { FullMode = BoundedChannelFullMode.Wait } - ); - - _sendTask = SendLoopAsync(streamId, "test"); - } - - public ValueTask PushAsync(BaseItem item, CancellationToken ct = default) => _channel.Writer.WriteAsync(item, ct); - - public async Task CompleteAsync() - { - _channel.Writer.Complete(); - await _sendTask.ConfigureAwait(false); - } - - private async Task SendLoopAsync(string projectId, string modelId) - { - var pipe = new Pipe(); - - // Start writing to pipe immediately in background - var writeTask = Task.Run(async () => - { - var gzip = new GZipStream(pipe.Writer.AsStream(), CompressionLevel.Optimal); - var writer = new StreamWriter(gzip); - - try - { - await foreach (var item in _channel.Reader.ReadAllAsync().ConfigureAwait(false)) - { - await writer.WriteLineAsync($"{item.Id}\t{item.Json}").ConfigureAwait(false); - await writer.FlushAsync().ConfigureAwait(false); - } - } - finally - { - await writer.FlushAsync().ConfigureAwait(false); - await gzip.FlushAsync().ConfigureAwait(false); - writer.Dispose(); - gzip.Dispose(); - await pipe.Writer.CompleteAsync().ConfigureAwait(false); - } - }); - - // Start HTTP request immediately, reading from pipe - var content = new StreamContent(pipe.Reader.AsStream()); - content.Headers.ContentType = new MediaTypeHeaderValue("application/x-ndjson"); - content.Headers.ContentEncoding.Add("gzip"); - - var uri = new Uri($"projects/{projectId}/models/{modelId}/objects", UriKind.Relative); - var request = new HttpRequestMessage(HttpMethod.Post, uri) { Content = content }; - - var responseTask = _client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); - - // Wait for both - await Task.WhenAll(writeTask, responseTask).ConfigureAwait(false); - - var response = await responseTask.ConfigureAwait(false); - response.EnsureSuccessStatusCode(); - } - - public void Dispose() => _client.Dispose(); -} diff --git a/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs b/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs index 3c56b4af..b708f4d4 100644 --- a/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs +++ b/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs @@ -35,8 +35,7 @@ public class SerializeProcessFactory( IServerObjectManager serverObjectManager, IProgress? progress, CancellationToken cancellationToken, - SerializeProcessOptions? options = null, - ObjectFlopper? objectFlopper = null + SerializeProcessOptions? options = null ) => new SerializeProcess( progress, diff --git a/src/Speckle.Sdk/Speckle.Sdk.csproj b/src/Speckle.Sdk/Speckle.Sdk.csproj index 664d497a..1e9cf643 100644 --- a/src/Speckle.Sdk/Speckle.Sdk.csproj +++ b/src/Speckle.Sdk/Speckle.Sdk.csproj @@ -23,12 +23,9 @@ - - - @@ -36,7 +33,6 @@ - diff --git a/src/Speckle.Sdk/packages.lock.json b/src/Speckle.Sdk/packages.lock.json index b7195702..17478611 100644 --- a/src/Speckle.Sdk/packages.lock.json +++ b/src/Speckle.Sdk/packages.lock.json @@ -13,27 +13,6 @@ "System.Reactive": "5.0.0" } }, - "Microsoft.AspNet.WebApi.Client": { - "type": "Direct", - "requested": "[6.0.0, )", - "resolved": "6.0.0", - "contentHash": "zXeWP03dTo67AoDHUzR+/urck0KFssdCKOC+dq7Nv1V2YbFh/nIg09L0/3wSvyRONEdwxGB/ssEGmPNIIhAcAw==", - "dependencies": { - "Newtonsoft.Json": "13.0.1", - "Newtonsoft.Json.Bson": "1.0.2", - "System.Memory": "4.5.5", - "System.Threading.Tasks.Extensions": "4.5.4" - } - }, - "Microsoft.Bcl.AsyncInterfaces": { - "type": "Direct", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "E1HSLkPHXEO30JEij2pWbOuzz1Z5ND4a5l7IP1T2RgQuE0a0NzEIvtO64RNy3Otn6PFezbT80cfm3M/Cgt70PA==", - "dependencies": { - "System.Threading.Tasks.Extensions": "4.6.3" - } - }, "Microsoft.CSharp": { "type": "Direct", "requested": "[4.7.0, )", @@ -111,29 +90,6 @@ "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" }, - "System.IO.Pipelines": { - "type": "Direct", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "26LbFXHKd7PmRnWlkjnYgmjd5B6HYVG+1MpTO25BdxTJnx6D0O16JPAC/S4YBqjtt4YpfGj1QO/Ss6SPMGEGQw==", - "dependencies": { - "System.Buffers": "4.6.1", - "System.Memory": "4.6.3", - "System.Threading.Tasks.Extensions": "4.6.3" - } - }, - "System.Net.Http.Json": { - "type": "Direct", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "XGOWt78ccgO9esyNlCemlS2b8JZPrH85pk/MvdHJxp6KwwUY/GnDaw2fPpJa7lgotiTkWnXnhip+ULNKaP7a8A==", - "dependencies": { - "System.Buffers": "4.6.1", - "System.Memory": "4.6.3", - "System.Text.Json": "10.0.1", - "System.Threading.Tasks.Extensions": "4.6.3" - } - }, "System.Threading.Channels": { "type": "Direct", "requested": "[10.0.1, )", @@ -165,6 +121,14 @@ "resolved": "6.0.0", "contentHash": "yg72rrYDapfsIUrul7aF6wwNnTJBOFvuA9VdDTQpPa8AlAriHbufeXYLBcodKjfUdkCnaiggX1U/nEP08Zb5GA==" }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "Transitive", + "resolved": "10.0.1", + "contentHash": "E1HSLkPHXEO30JEij2pWbOuzz1Z5ND4a5l7IP1T2RgQuE0a0NzEIvtO64RNy3Otn6PFezbT80cfm3M/Cgt70PA==", + "dependencies": { + "System.Threading.Tasks.Extensions": "4.6.3" + } + }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", "resolved": "8.0.0", @@ -241,19 +205,6 @@ "resolved": "8.0.0", "contentHash": "dk9JPxTCIevS75HyEQ0E4OVAFhB2N+V9ShCXf8Q6FkUQZDkgLI12y679Nym1YqsiSysuQskT7Z+6nUf3yab6Vw==" }, - "Newtonsoft.Json": { - "type": "Transitive", - "resolved": "13.0.1", - "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" - }, - "Newtonsoft.Json.Bson": { - "type": "Transitive", - "resolved": "1.0.2", - "contentHash": "QYFyxhaABwmq3p/21VrZNYvCg3DaEoN/wUuw5nmfAf0X3HLjgupwhkEWdgfb9nvGAUIv3osmZoD3kKl4jxEmYQ==", - "dependencies": { - "Newtonsoft.Json": "12.0.1" - } - }, "SQLitePCLRaw.bundle_e_sqlite3": { "type": "Transitive", "resolved": "2.1.4", @@ -286,8 +237,8 @@ }, "System.Buffers": { "type": "Transitive", - "resolved": "4.6.1", - "contentHash": "N8GXpmiLMtljq7gwvyS+1QvKT/W2J8sNAvx+HVg4NGmsG/H+2k/y9QI23auLJRterrzCiDH+IWAw4V/GPwsMlw==" + "resolved": "4.4.0", + "contentHash": "AwarXzzoDwX6BgrhjoJsk6tUezZEozOT5Y9QKF94Gl4JK91I4PIIBkBco9068Y9/Dra8Dkbie99kXB8+1BaYKw==" }, "System.ComponentModel.Annotations": { "type": "Transitive", @@ -296,18 +247,18 @@ }, "System.Memory": { "type": "Transitive", - "resolved": "4.6.3", - "contentHash": "qdcDOgnFZY40+Q9876JUHnlHu7bosOHX8XISRoH94fwk6hgaeQGSgfZd8srWRZNt5bV9ZW2TljcegDNxsf+96A==", + "resolved": "4.5.3", + "contentHash": "3oDzvc/zzetpTKWMShs1AADwZjQ/36HnsufHRPcOjyRAAMLDlu2iD33MBI2opxnezcVUtXyqDXXjoFMOU9c7SA==", "dependencies": { - "System.Buffers": "4.6.1", - "System.Numerics.Vectors": "4.6.1", - "System.Runtime.CompilerServices.Unsafe": "6.1.2" + "System.Buffers": "4.4.0", + "System.Numerics.Vectors": "4.4.0", + "System.Runtime.CompilerServices.Unsafe": "4.5.2" } }, "System.Numerics.Vectors": { "type": "Transitive", - "resolved": "4.6.1", - "contentHash": "sQxefTnhagrhoq2ReR0D/6K0zJcr9Hrd6kikeXsA1I8kOCboTavcUC4r7TSfpKFeE163uMuxZcyfO1mGO3EN8Q==" + "resolved": "4.4.0", + "contentHash": "UiLzLW+Lw6HLed1Hcg+8jSRttrbuXv7DANVj0DkL9g6EnnzbL75EB7EWsw5uRbhxd/4YdG8li5XizGWepmG3PQ==" }, "System.Reactive": { "type": "Transitive", @@ -340,16 +291,6 @@ "System.Runtime": "4.3.0" } }, - "System.Text.Encodings.Web": { - "type": "Transitive", - "resolved": "10.0.1", - "contentHash": "cVAka0o1rJJ5/De0pjNs7jcaZk5hUGf1HGzUyVmE2MEB1Vf0h/8qsWxImk1zjitCbeD2Avaq2P2+usdvqgbeVQ==", - "dependencies": { - "System.Buffers": "4.6.1", - "System.Memory": "4.6.3", - "System.Runtime.CompilerServices.Unsafe": "6.1.2" - } - }, "System.Threading.Tasks.Extensions": { "type": "Transitive", "resolved": "4.6.3", @@ -358,57 +299,8 @@ "System.Runtime.CompilerServices.Unsafe": "6.1.2" } }, - "speckle.connectors.common": { - "type": "Project", - "dependencies": { - "Microsoft.Extensions.DependencyInjection": "[2.2.0, )", - "Speckle.Connectors.Logging": "[1.0.0, )", - "Speckle.Converters.Common": "[1.0.0, )", - "Speckle.Objects": "[1.0.0, )" - } - }, - "speckle.connectors.logging": { - "type": "Project" - }, - "speckle.converters.common": { - "type": "Project", - "dependencies": { - "Microsoft.Extensions.Logging.Abstractions": "[2.2.0, )", - "Speckle.Objects": "[1.0.0, )" - } - }, - "speckle.objects": { - "type": "Project", - "dependencies": { - "Speckle.Sdk": "[1.0.0, )" - } - }, "speckle.sdk.dependencies": { "type": "Project" - }, - "Microsoft.Extensions.DependencyInjection": { - "type": "CentralTransitive", - "requested": "[2.2.0, )", - "resolved": "2.2.0", - "contentHash": "MZtBIwfDFork5vfjpJdG5g8wuJFt7d/y3LOSVVtDK/76wlbtz6cjltfKHqLx2TKVqTj5/c41t77m1+h20zqtPA==", - "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0" - } - }, - "System.Text.Json": { - "type": "CentralTransitive", - "requested": "[8.0.5, )", - "resolved": "10.0.1", - "contentHash": "EsgwDgU1PFqhrFA9l5n+RBu76wFhNGCEwu8ITrBNhjPP3MxLyklroU5GIF8o6JYpYg6T4KD/VICfMdgPAvNp5g==", - "dependencies": { - "Microsoft.Bcl.AsyncInterfaces": "10.0.1", - "System.Buffers": "4.6.1", - "System.IO.Pipelines": "10.0.1", - "System.Memory": "4.6.3", - "System.Runtime.CompilerServices.Unsafe": "6.1.2", - "System.Text.Encodings.Web": "10.0.1", - "System.Threading.Tasks.Extensions": "4.6.3" - } } }, "net8.0": { @@ -423,18 +315,6 @@ "System.Reactive": "5.0.0" } }, - "Microsoft.AspNet.WebApi.Client": { - "type": "Direct", - "requested": "[6.0.0, )", - "resolved": "6.0.0", - "contentHash": "zXeWP03dTo67AoDHUzR+/urck0KFssdCKOC+dq7Nv1V2YbFh/nIg09L0/3wSvyRONEdwxGB/ssEGmPNIIhAcAw==", - "dependencies": { - "Newtonsoft.Json": "13.0.1", - "Newtonsoft.Json.Bson": "1.0.2", - "System.Memory": "4.5.5", - "System.Threading.Tasks.Extensions": "4.5.4" - } - }, "Microsoft.Data.Sqlite": { "type": "Direct", "requested": "[7.0.5, )", @@ -497,21 +377,6 @@ "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" }, - "System.IO.Pipelines": { - "type": "Direct", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "26LbFXHKd7PmRnWlkjnYgmjd5B6HYVG+1MpTO25BdxTJnx6D0O16JPAC/S4YBqjtt4YpfGj1QO/Ss6SPMGEGQw==" - }, - "System.Net.Http.Json": { - "type": "Direct", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "XGOWt78ccgO9esyNlCemlS2b8JZPrH85pk/MvdHJxp6KwwUY/GnDaw2fPpJa7lgotiTkWnXnhip+ULNKaP7a8A==", - "dependencies": { - "System.Text.Json": "10.0.1" - } - }, "System.Threading.Channels": { "type": "Direct", "requested": "[10.0.1, )", @@ -605,19 +470,6 @@ "resolved": "8.0.0", "contentHash": "dk9JPxTCIevS75HyEQ0E4OVAFhB2N+V9ShCXf8Q6FkUQZDkgLI12y679Nym1YqsiSysuQskT7Z+6nUf3yab6Vw==" }, - "Newtonsoft.Json": { - "type": "Transitive", - "resolved": "13.0.1", - "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" - }, - "Newtonsoft.Json.Bson": { - "type": "Transitive", - "resolved": "1.0.2", - "contentHash": "QYFyxhaABwmq3p/21VrZNYvCg3DaEoN/wUuw5nmfAf0X3HLjgupwhkEWdgfb9nvGAUIv3osmZoD3kKl4jxEmYQ==", - "dependencies": { - "Newtonsoft.Json": "12.0.1" - } - }, "SQLitePCLRaw.bundle_e_sqlite3": { "type": "Transitive", "resolved": "2.1.4", @@ -655,8 +507,8 @@ }, "System.Memory": { "type": "Transitive", - "resolved": "4.5.5", - "contentHash": "XIWiDvKPXaTveaB7HVganDlOCRoj03l+jrwNvcge/t8vhGYKvqV+dMv6G4SAX2NoNmN0wZfVPTAlFwZcZvVOUw==" + "resolved": "4.5.3", + "contentHash": "3oDzvc/zzetpTKWMShs1AADwZjQ/36HnsufHRPcOjyRAAMLDlu2iD33MBI2opxnezcVUtXyqDXXjoFMOU9c7SA==" }, "System.Reactive": { "type": "Transitive", @@ -668,62 +520,8 @@ "resolved": "4.5.1", "contentHash": "Zh8t8oqolRaFa9vmOZfdQm/qKejdqz0J9kr7o2Fu0vPeoH3BL1EOXipKWwkWtLT1JPzjByrF19fGuFlNbmPpiw==" }, - "System.Text.Encodings.Web": { - "type": "Transitive", - "resolved": "10.0.1", - "contentHash": "cVAka0o1rJJ5/De0pjNs7jcaZk5hUGf1HGzUyVmE2MEB1Vf0h/8qsWxImk1zjitCbeD2Avaq2P2+usdvqgbeVQ==" - }, - "System.Threading.Tasks.Extensions": { - "type": "Transitive", - "resolved": "4.5.4", - "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==" - }, - "speckle.connectors.common": { - "type": "Project", - "dependencies": { - "Microsoft.Extensions.DependencyInjection": "[2.2.0, )", - "Speckle.Connectors.Logging": "[1.0.0, )", - "Speckle.Converters.Common": "[1.0.0, )", - "Speckle.Objects": "[1.0.0, )" - } - }, - "speckle.connectors.logging": { - "type": "Project" - }, - "speckle.converters.common": { - "type": "Project", - "dependencies": { - "Microsoft.Extensions.Logging.Abstractions": "[2.2.0, )", - "Speckle.Objects": "[1.0.0, )" - } - }, - "speckle.objects": { - "type": "Project", - "dependencies": { - "Speckle.Sdk": "[1.0.0, )" - } - }, "speckle.sdk.dependencies": { "type": "Project" - }, - "Microsoft.Extensions.DependencyInjection": { - "type": "CentralTransitive", - "requested": "[2.2.0, )", - "resolved": "2.2.0", - "contentHash": "MZtBIwfDFork5vfjpJdG5g8wuJFt7d/y3LOSVVtDK/76wlbtz6cjltfKHqLx2TKVqTj5/c41t77m1+h20zqtPA==", - "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0" - } - }, - "System.Text.Json": { - "type": "CentralTransitive", - "requested": "[8.0.5, )", - "resolved": "10.0.1", - "contentHash": "EsgwDgU1PFqhrFA9l5n+RBu76wFhNGCEwu8ITrBNhjPP3MxLyklroU5GIF8o6JYpYg6T4KD/VICfMdgPAvNp5g==", - "dependencies": { - "System.IO.Pipelines": "10.0.1", - "System.Text.Encodings.Web": "10.0.1" - } } } } diff --git a/tests/Speckle.Automate.Sdk.Integration/packages.lock.json b/tests/Speckle.Automate.Sdk.Integration/packages.lock.json index 78c3841b..d0818532 100644 --- a/tests/Speckle.Automate.Sdk.Integration/packages.lock.json +++ b/tests/Speckle.Automate.Sdk.Integration/packages.lock.json @@ -370,7 +370,8 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )", + "System.Threading.Channels": "[10.0.1, )" } }, "speckle.sdk.dependencies": { @@ -491,6 +492,12 @@ "System.Text.Encodings.Web": "8.0.0" } }, + "System.Threading.Channels": { + "type": "CentralTransitive", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" + }, "Verify.Quibble": { "type": "CentralTransitive", "requested": "[2.1.1, )", diff --git a/tests/Speckle.Objects.Tests.Unit/packages.lock.json b/tests/Speckle.Objects.Tests.Unit/packages.lock.json index fe375e71..3ef2d70b 100644 --- a/tests/Speckle.Objects.Tests.Unit/packages.lock.json +++ b/tests/Speckle.Objects.Tests.Unit/packages.lock.json @@ -348,7 +348,8 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )", + "System.Threading.Channels": "[10.0.1, )" } }, "speckle.sdk.dependencies": { @@ -432,6 +433,12 @@ "System.Text.Encodings.Web": "8.0.0" } }, + "System.Threading.Channels": { + "type": "CentralTransitive", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" + }, "Verify.Quibble": { "type": "CentralTransitive", "requested": "[2.1.1, )", diff --git a/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json b/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json index 046c2c3c..022252ea 100644 --- a/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json +++ b/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json @@ -74,6 +74,11 @@ "resolved": "1.17.0", "contentHash": "8x+HCVTl/HHTGpscH3vMBhV8sknN/muZFw9s3TsI8SA6+c43cOTCi2+jE4KsU8pNLbJ++iF2ZFcpcXHXtDglnw==" }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "Transitive", + "resolved": "1.1.0", + "contentHash": "1Am6l4Vpn3/K32daEqZI+FFr96OlZkgwK2LcT3pZ2zWubR5zTPW3/FkO1Rat9kb7oQOa4rxgl9LJHc5tspCWfg==" + }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", "resolved": "8.0.0", @@ -337,7 +342,8 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )", + "System.Threading.Channels": "[10.0.1, )" } }, "speckle.sdk.dependencies": { @@ -380,12 +386,6 @@ "System.Reactive": "5.0.0" } }, - "Microsoft.Bcl.AsyncInterfaces": { - "type": "CentralTransitive", - "requested": "[5.0.0, )", - "resolved": "1.1.0", - "contentHash": "1Am6l4Vpn3/K32daEqZI+FFr96OlZkgwK2LcT3pZ2zWubR5zTPW3/FkO1Rat9kb7oQOa4rxgl9LJHc5tspCWfg==" - }, "Microsoft.Data.Sqlite": { "type": "CentralTransitive", "requested": "[7.0.5, )", @@ -425,6 +425,12 @@ "requested": "[13.0.2, )", "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" + }, + "System.Threading.Channels": { + "type": "CentralTransitive", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" } } } diff --git a/tests/Speckle.Sdk.Serialization.Tests/packages.lock.json b/tests/Speckle.Sdk.Serialization.Tests/packages.lock.json index 2ac574fb..11691412 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/packages.lock.json +++ b/tests/Speckle.Sdk.Serialization.Tests/packages.lock.json @@ -393,7 +393,8 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )", + "System.Threading.Channels": "[10.0.1, )" } }, "speckle.sdk.dependencies": { @@ -474,6 +475,12 @@ "System.Text.Encodings.Web": "8.0.0" } }, + "System.Threading.Channels": { + "type": "CentralTransitive", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" + }, "Verify.Quibble": { "type": "CentralTransitive", "requested": "[2.1.1, )", diff --git a/tests/Speckle.Sdk.Testing/packages.lock.json b/tests/Speckle.Sdk.Testing/packages.lock.json index 6e2cfd77..9cb33c89 100644 --- a/tests/Speckle.Sdk.Testing/packages.lock.json +++ b/tests/Speckle.Sdk.Testing/packages.lock.json @@ -311,7 +311,8 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )", + "System.Threading.Channels": "[10.0.1, )" } }, "speckle.sdk.dependencies": { @@ -376,6 +377,12 @@ "dependencies": { "System.Text.Encodings.Web": "8.0.0" } + }, + "System.Threading.Channels": { + "type": "CentralTransitive", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" } } } diff --git a/tests/Speckle.Sdk.Tests.Integration/packages.lock.json b/tests/Speckle.Sdk.Tests.Integration/packages.lock.json index 6138bda1..59e1a4d2 100644 --- a/tests/Speckle.Sdk.Tests.Integration/packages.lock.json +++ b/tests/Speckle.Sdk.Tests.Integration/packages.lock.json @@ -370,7 +370,8 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )", + "System.Threading.Channels": "[10.0.1, )" } }, "speckle.sdk.dependencies": { @@ -454,6 +455,12 @@ "System.Text.Encodings.Web": "8.0.0" } }, + "System.Threading.Channels": { + "type": "CentralTransitive", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" + }, "Verify.Quibble": { "type": "CentralTransitive", "requested": "[2.1.1, )", diff --git a/tests/Speckle.Sdk.Tests.Performance/packages.lock.json b/tests/Speckle.Sdk.Tests.Performance/packages.lock.json index 16d806d9..a96f59c2 100644 --- a/tests/Speckle.Sdk.Tests.Performance/packages.lock.json +++ b/tests/Speckle.Sdk.Tests.Performance/packages.lock.json @@ -92,6 +92,11 @@ "resolved": "1.17.0", "contentHash": "8x+HCVTl/HHTGpscH3vMBhV8sknN/muZFw9s3TsI8SA6+c43cOTCi2+jE4KsU8pNLbJ++iF2ZFcpcXHXtDglnw==" }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "Transitive", + "resolved": "1.1.0", + "contentHash": "1Am6l4Vpn3/K32daEqZI+FFr96OlZkgwK2LcT3pZ2zWubR5zTPW3/FkO1Rat9kb7oQOa4rxgl9LJHc5tspCWfg==" + }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", "resolved": "8.0.0", @@ -355,7 +360,8 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )", + "System.Threading.Channels": "[10.0.1, )" } }, "speckle.sdk.dependencies": { @@ -372,12 +378,6 @@ "System.Reactive": "5.0.0" } }, - "Microsoft.Bcl.AsyncInterfaces": { - "type": "CentralTransitive", - "requested": "[5.0.0, )", - "resolved": "1.1.0", - "contentHash": "1Am6l4Vpn3/K32daEqZI+FFr96OlZkgwK2LcT3pZ2zWubR5zTPW3/FkO1Rat9kb7oQOa4rxgl9LJHc5tspCWfg==" - }, "Microsoft.Data.Sqlite": { "type": "CentralTransitive", "requested": "[7.0.5, )", @@ -417,6 +417,12 @@ "requested": "[13.0.2, )", "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" + }, + "System.Threading.Channels": { + "type": "CentralTransitive", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" } } } diff --git a/tests/Speckle.Sdk.Tests.Unit/packages.lock.json b/tests/Speckle.Sdk.Tests.Unit/packages.lock.json index 8b50e6f9..4faff1c8 100644 --- a/tests/Speckle.Sdk.Tests.Unit/packages.lock.json +++ b/tests/Speckle.Sdk.Tests.Unit/packages.lock.json @@ -363,7 +363,8 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )", + "System.Threading.Channels": "[10.0.1, )" } }, "speckle.sdk.dependencies": { @@ -441,6 +442,12 @@ "System.Text.Encodings.Web": "8.0.0" } }, + "System.Threading.Channels": { + "type": "CentralTransitive", + "requested": "[10.0.1, )", + "resolved": "10.0.1", + "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" + }, "Verify.Quibble": { "type": "CentralTransitive", "requested": "[2.1.1, )", From 43ebc84881e167158e4d2095ef77493acbd8b918 Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Fri, 20 Feb 2026 14:20:29 +0000 Subject: [PATCH 04/16] feat(api): DI Refactor for Duck DB + Gergo's API endpoint changes (#444) * di * di2 * Gergo's api changes --- src/Speckle.Sdk/Helpers/DisposableFile.cs | 34 ++++ src/Speckle.Sdk/Pipelines/ProgressStream.cs | 102 +++++----- .../Pipelines/{ => Send}/SendPipeline.cs | 40 ++-- .../Pipelines/{ => Send}/Serializer.cs | 4 +- src/Speckle.Sdk/Pipelines/Send/Uploader.cs | 164 ++++++++++++++++ .../Pipelines/Send/UploaderDTOs.cs | 22 +++ src/Speckle.Sdk/Pipelines/Uploader.cs | 182 ------------------ src/Speckle.Sdk/Pipelines/Uploader_Old.cs | 83 -------- 8 files changed, 298 insertions(+), 333 deletions(-) create mode 100644 src/Speckle.Sdk/Helpers/DisposableFile.cs rename src/Speckle.Sdk/Pipelines/{ => Send}/SendPipeline.cs (77%) rename src/Speckle.Sdk/Pipelines/{ => Send}/Serializer.cs (99%) create mode 100644 src/Speckle.Sdk/Pipelines/Send/Uploader.cs create mode 100644 src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs delete mode 100644 src/Speckle.Sdk/Pipelines/Uploader.cs delete mode 100644 src/Speckle.Sdk/Pipelines/Uploader_Old.cs diff --git a/src/Speckle.Sdk/Helpers/DisposableFile.cs b/src/Speckle.Sdk/Helpers/DisposableFile.cs new file mode 100644 index 00000000..b4200678 --- /dev/null +++ b/src/Speckle.Sdk/Helpers/DisposableFile.cs @@ -0,0 +1,34 @@ +using Microsoft.Extensions.Logging; + +namespace Speckle.Sdk.Helpers; + +/// +/// wrapper around the downloaded file to try and delete the file on Dispose +/// +/// +/// We're using a similar pattern in the Rhino File Importer codebase (see ImportJobFile) +/// +/// +/// +public sealed class DisposableFile(FileInfo file, ILogger logger, bool deleteOnDispose = true) : IDisposable +{ + public FileInfo FileInfo => file; + + public void Dispose() + { + if (!deleteOnDispose) + { + return; + } + + try + { + file.Delete(); + logger.LogInformation("Cleaned up {File}", file); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + logger.LogWarning(ex, "Failed to clean up {File}", file); + } + } +} diff --git a/src/Speckle.Sdk/Pipelines/ProgressStream.cs b/src/Speckle.Sdk/Pipelines/ProgressStream.cs index ed62121c..b23998ae 100644 --- a/src/Speckle.Sdk/Pipelines/ProgressStream.cs +++ b/src/Speckle.Sdk/Pipelines/ProgressStream.cs @@ -1,54 +1,46 @@ +using System.Diagnostics.CodeAnalysis; + namespace Speckle.Sdk.Pipelines; +public readonly record struct StreamProgressArgs(long BytesStreamed, long ExpectedTotalBytes); + /// /// Wraps a stream to report upload progress as bytes are read. /// -public sealed class ProgressStream : Stream +public sealed class ProgressStream( + Stream innerStream, + long expectedTotalBytesStreamed, + IProgress? progress = null +) : Stream { - private readonly Stream _innerStream; - private readonly long _totalBytes; - private readonly IProgress<(long BytesSent, long TotalBytes)>? _progress; - private long _bytesSent; + private long _bytesStreamed; - public ProgressStream( - Stream innerStream, - long totalBytes, - IProgress<(long BytesSent, long TotalBytes)>? progress = null - ) - { - _innerStream = innerStream; - _totalBytes = totalBytes; - _progress = progress; - _bytesSent = 0; - } - - public override bool CanRead => _innerStream.CanRead; - public override bool CanSeek => _innerStream.CanSeek; + public override bool CanRead => innerStream.CanRead; + public override bool CanSeek => innerStream.CanSeek; public override bool CanWrite => false; - public override long Length => _innerStream.Length; + public override long Length => innerStream.Length; public override long Position { - get => _innerStream.Position; - set => _innerStream.Position = value; + get => innerStream.Position; + set => innerStream.Position = value; } public override int Read(byte[] buffer, int offset, int count) { - int bytesRead = _innerStream.Read(buffer, offset, count); + int bytesRead = innerStream.Read(buffer, offset, count); ReportProgress(bytesRead); return bytesRead; } + [SuppressMessage( + "Performance", + "CA1835:Prefer the \'Memory\'-based overloads for \'ReadAsync\' and \'WriteAsync\'", + Justification = "Analyser warning forwarded to caller" + )] public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) { - int bytesRead = await _innerStream -#if NET8_0_OR_GREATER - .ReadAsync(buffer.AsMemory(offset, count), cancellationToken) -#else - .ReadAsync(buffer, offset, count, cancellationToken) -#endif - .ConfigureAwait(false); + int bytesRead = await innerStream.ReadAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); ReportProgress(bytesRead); return bytesRead; } @@ -56,41 +48,61 @@ public sealed class ProgressStream : Stream #if NET8_0_OR_GREATER public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) { - int bytesRead = await _innerStream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + int bytesRead = await innerStream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); ReportProgress(bytesRead); return bytesRead; } #endif - private void ReportProgress(int bytesRead) + private void ReportProgress(int newBytesProcessed) { - _bytesSent += bytesRead; - _progress?.Report((_bytesSent, _totalBytes)); + _bytesStreamed += newBytesProcessed; + progress?.Report(new(_bytesStreamed, expectedTotalBytesStreamed)); } - public override void Flush() => _innerStream.Flush(); + public override void Flush() => innerStream.Flush(); - public override Task FlushAsync(CancellationToken cancellationToken) => _innerStream.FlushAsync(cancellationToken); + public override Task FlushAsync(CancellationToken cancellationToken) => innerStream.FlushAsync(cancellationToken); - public override long Seek(long offset, SeekOrigin origin) => _innerStream.Seek(offset, origin); + public override long Seek(long offset, SeekOrigin origin) => innerStream.Seek(offset, origin); - public override void SetLength(long value) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); //intentionally not supporting, as changing length of stream mid-flight will fuck up progress - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) + { + innerStream.Write(buffer, offset, count); + ReportProgress(count); + } + + [SuppressMessage( + "Performance", + "CA1835:Prefer the \'Memory\'-based overloads for \'ReadAsync\' and \'WriteAsync\'", + Justification = "Analyser warning forwarded to caller" + )] + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + await innerStream.WriteAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); + ReportProgress(count); + } + +#if NET6_0_OR_GREATER + public override async ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) + { + await innerStream.WriteAsync(buffer, cancellationToken).ConfigureAwait(false); + ReportProgress(buffer.Length); + } +#endif protected override void Dispose(bool disposing) { - if (disposing) - { - _innerStream.Dispose(); - } + innerStream.Dispose(); base.Dispose(disposing); } -#if NET8_0_OR_GREATER +#if NET6_0_OR_GREATER public override async ValueTask DisposeAsync() { - await _innerStream.DisposeAsync().ConfigureAwait(false); + await innerStream.DisposeAsync().ConfigureAwait(false); await base.DisposeAsync().ConfigureAwait(false); } #endif diff --git a/src/Speckle.Sdk/Pipelines/SendPipeline.cs b/src/Speckle.Sdk/Pipelines/Send/SendPipeline.cs similarity index 77% rename from src/Speckle.Sdk/Pipelines/SendPipeline.cs rename to src/Speckle.Sdk/Pipelines/Send/SendPipeline.cs index 1079f1f7..e1a97a40 100644 --- a/src/Speckle.Sdk/Pipelines/SendPipeline.cs +++ b/src/Speckle.Sdk/Pipelines/Send/SendPipeline.cs @@ -1,34 +1,32 @@ +using Speckle.InterfaceGenerator; using Speckle.Sdk.Credentials; using Speckle.Sdk.Models; -using Speckle.Sdk.Serialisation; -namespace Speckle.Sdk.Pipelines; +namespace Speckle.Sdk.Pipelines.Send; -public record UploadItem(string Id, Json Json, string SpeckleType, ObjectReference Reference); - -public sealed class SendPipeline : IDisposable +[GenerateAutoInterface] +public sealed class SendPipelineFactory(IUploaderFactory uploaderFactory) : ISendPipelineFactory { - private readonly CancellationToken _cancellationToken; - private readonly Serializer _serializer = new(); - private readonly Uploader _uploader; - - public SendPipeline( - Account account, + public SendPipeline CreateInstance( string projectId, - string modelId, string ingestionId, + Account account, CancellationToken cancellationToken ) { - _cancellationToken = cancellationToken; - _uploader = new Uploader( - projectId, - modelId, - ingestionId, - new(account.serverInfo.url), - account.token, - cancellationToken - ); + var uploader = uploaderFactory.CreateInstance(projectId, ingestionId, account, cancellationToken); + return new SendPipeline(uploader); + } +} + +public sealed class SendPipeline : IDisposable +{ + private readonly Serializer _serializer = new(); + private readonly Uploader _uploader; + + internal SendPipeline(Uploader uploader) + { + _uploader = uploader; } private UploadItem _lastItem; diff --git a/src/Speckle.Sdk/Pipelines/Serializer.cs b/src/Speckle.Sdk/Pipelines/Send/Serializer.cs similarity index 99% rename from src/Speckle.Sdk/Pipelines/Serializer.cs rename to src/Speckle.Sdk/Pipelines/Send/Serializer.cs index 3f551db3..ed78ce73 100644 --- a/src/Speckle.Sdk/Pipelines/Serializer.cs +++ b/src/Speckle.Sdk/Pipelines/Send/Serializer.cs @@ -9,7 +9,7 @@ using Speckle.Sdk.Helpers; using Speckle.Sdk.Models; using Speckle.Sdk.Serialisation; -namespace Speckle.Sdk.Pipelines; +namespace Speckle.Sdk.Pipelines.Send; /// /// Another serializer, cleaner and meaner. Provides methods for serializing Speckle objects into a format suitable for upload or storage. @@ -17,7 +17,7 @@ namespace Speckle.Sdk.Pipelines; /// into serialized JSON structures along with associated metadata, closures, and references. /// Any reference objects coming through are being "passed through" serialized - they do not get double encoded. /// -public class Serializer +internal sealed class Serializer { private readonly record struct PropertyInfo(string Name, object? Value, bool IsDetachable); diff --git a/src/Speckle.Sdk/Pipelines/Send/Uploader.cs b/src/Speckle.Sdk/Pipelines/Send/Uploader.cs new file mode 100644 index 00000000..9d7103d9 --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Send/Uploader.cs @@ -0,0 +1,164 @@ +using System.IO.Compression; +using System.Net.Http.Headers; +using System.Threading.Channels; +using Microsoft.Extensions.Logging; +using Speckle.InterfaceGenerator; +using Speckle.Newtonsoft.Json; +using Speckle.Sdk.Credentials; +using Speckle.Sdk.Helpers; + +namespace Speckle.Sdk.Pipelines.Send; + +[GenerateAutoInterface] +public sealed class UploaderFactory(ISpeckleHttp httpClientFactory, ILogger logger) : IUploaderFactory +{ + public Uploader CreateInstance( + string projectId, + string ingestionId, + Account account, + CancellationToken cancellationToken + ) => new(projectId, ingestionId, logger, httpClientFactory, account, cancellationToken); +} + +public sealed class Uploader : IDisposable +{ + private readonly string _projectId; + private readonly string _ingestionId; + private readonly CancellationToken _cancellationToken; + private readonly HttpClient _speckleClient; + private readonly HttpClient _s3Client; + private readonly Channel _channel; + private readonly Task _sendTask; + private readonly ILogger _logger; + + internal Uploader( + string projectId, + string ingestionId, + ILogger logger, + ISpeckleHttp httpClientFactory, + Account speckleAccount, + CancellationToken cancellationToken + ) + { + _projectId = projectId; + _ingestionId = ingestionId; + _logger = logger; + _cancellationToken = cancellationToken; + + _speckleClient = httpClientFactory.CreateHttpClient( + null, + (int)TimeSpan.FromMinutes(30).TotalSeconds, + speckleAccount.token + ); + _speckleClient.BaseAddress = new(new(speckleAccount.serverInfo.url), "/api/v1/"); + + _s3Client = httpClientFactory.CreateHttpClient(); + + _channel = Channel.CreateBounded( + new BoundedChannelOptions(1000) { FullMode = BoundedChannelFullMode.Wait, SingleReader = true } + ); + + _sendTask = Task.Run(SendLoopAsync, cancellationToken); + } + + public ValueTask PushAsync(UploadItem item) => _channel.Writer.WriteAsync(item, _cancellationToken); + + public async Task CompleteAsync() + { + _channel.Writer.Complete(); + var result = await _sendTask.ConfigureAwait(false); + return result.IngestionId; + } + + private async Task SendLoopAsync() + { + using DisposableFile tempFile = await WriteFile().ConfigureAwait(false); + + PresignedUploadResponse presignedUpload = await GetPresignedUrl().ConfigureAwait(false); + await UploadToS3(tempFile.FileInfo, presignedUpload.Url).ConfigureAwait(false); + + return await TriggerProcessing().ConfigureAwait(false); + } + + /// + /// Reads from the Channel and streams the s to a temporary file on disk. + /// Will keep reading until is called. + /// + /// the file that was written + private async Task WriteFile() + { + string tempFilePath = Path.GetTempFileName(); + _logger.LogInformation("Writing temp file to {TempFilePath}", tempFilePath); + + using var fileStream = new FileStream(tempFilePath, FileMode.Create, FileAccess.Write, FileShare.None); + using var gzip = new GZipStream(fileStream, CompressionLevel.Optimal); + using var writer = new StreamWriter(gzip); + await foreach (var item in _channel.Reader.ReadAllAsync(_cancellationToken).ConfigureAwait(false)) + { + await writer.WriteLineAsync($"{item.Id}\t{item.Json}\t{item.SpeckleType}").ConfigureAwait(false); + } +#if NET8_0_OR_GREATER + await writer.FlushAsync(_cancellationToken).ConfigureAwait(false); +#else + await writer.FlushAsync().ConfigureAwait(false); +#endif + return new DisposableFile(new FileInfo(tempFilePath), _logger); + } + + private async Task GetPresignedUrl() + { + var signUri = new Uri($"projects/{_projectId}/modelingestion/{_ingestionId}/uploads/sign", UriKind.Relative); + + using var signResponse = await _speckleClient.PostAsync(signUri, null, _cancellationToken).ConfigureAwait(false); + signResponse.EnsureSuccessStatusCode(); + +#if NET5_0_OR_GREATER + string signResponseString = await signResponse.Content.ReadAsStringAsync(_cancellationToken).ConfigureAwait(false); +#else + string signResponseString = await signResponse.Content.ReadAsStringAsync().ConfigureAwait(false); +#endif + PresignedUploadResponse presignedUpload = + JsonConvert.DeserializeObject(signResponseString) + ?? throw new InvalidOperationException("Failed to get presigned upload URL"); + return presignedUpload; + } + + private async Task UploadToS3(FileInfo file, Uri s3Url) + { + using var fileStreamUpload = new FileStream(file.FullName, FileMode.Open, FileAccess.Read, FileShare.Read); + + Stream progressStream = fileStreamUpload; // TODO: wrap with progress stream + + using var streamContent = new StreamContent(progressStream); + streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); + streamContent.Headers.ContentLength = file.Length; + + using var uploadRequest = new HttpRequestMessage(HttpMethod.Put, s3Url); + uploadRequest.Content = streamContent; + + using var uploadResponse = await _s3Client + .SendAsync(uploadRequest, HttpCompletionOption.ResponseHeadersRead, _cancellationToken) + .ConfigureAwait(false); + + uploadResponse.EnsureSuccessStatusCode(); + } + + private async Task TriggerProcessing() + { + Uri processUri = new($"projects/{_projectId}/modelingestion/{_ingestionId}/uploads/process", UriKind.Relative); + + using HttpResponseMessage processResponse = await _speckleClient + .PostAsync(processUri, null, _cancellationToken) + .ConfigureAwait(false); + + processResponse.EnsureSuccessStatusCode(); + + return new UploadResult { IngestionId = _ingestionId }; + } + + public void Dispose() + { + _speckleClient.Dispose(); + _s3Client.Dispose(); + } +} diff --git a/src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs b/src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs new file mode 100644 index 00000000..fb020de4 --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs @@ -0,0 +1,22 @@ +using Speckle.Sdk.Models; +using Speckle.Sdk.Serialisation; + +namespace Speckle.Sdk.Pipelines.Send; + +public record UploadItem(string Id, Json Json, string SpeckleType, ObjectReference Reference); + +internal record PresignedUploadResponse +{ + public required Uri Url { get; init; } + public required string Key { get; init; } +} + +internal record ProcessUploadResponse +{ + public required string ingestionId { get; init; } +} + +internal record UploadResult +{ + public required string IngestionId { get; init; } +} diff --git a/src/Speckle.Sdk/Pipelines/Uploader.cs b/src/Speckle.Sdk/Pipelines/Uploader.cs deleted file mode 100644 index a7bf6e7e..00000000 --- a/src/Speckle.Sdk/Pipelines/Uploader.cs +++ /dev/null @@ -1,182 +0,0 @@ -using System.IO.Compression; -using System.Net.Http.Headers; -using System.Text; -using System.Threading.Channels; -using Speckle.Newtonsoft.Json; - -namespace Speckle.Sdk.Pipelines; - -public sealed class Uploader : IDisposable -{ - private readonly string _projectId; - private readonly string _modelId; - private readonly string _ingestionId; - private readonly CancellationToken _cancellationToken; - private readonly HttpClient _client; - private readonly Channel _channel; - private readonly Task _sendTask; - - public Uploader( - string projectId, - string modelId, - string ingestionId, - Uri serverUrl, - string authToken, - CancellationToken cancellationToken - ) - { - _projectId = projectId; - _modelId = modelId; - _ingestionId = ingestionId; - _cancellationToken = cancellationToken; - Uri apiBaseUrl = new(serverUrl, "/api/v1/"); - - _client = new HttpClient { BaseAddress = apiBaseUrl, Timeout = TimeSpan.FromMinutes(30) }; - - _client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", authToken); - - _channel = Channel.CreateBounded( - new BoundedChannelOptions(1000) { FullMode = BoundedChannelFullMode.Wait, SingleReader = true } - ); - - _sendTask = SendLoopAsync(); - } - - public ValueTask PushAsync(UploadItem item, CancellationToken ct = default) => _channel.Writer.WriteAsync(item, ct); - - public async Task CompleteAsync() - { - _channel.Writer.Complete(); - var result = await _sendTask.ConfigureAwait(false); - return result.IngestionId; - } - - private async Task SendLoopAsync() - { - // 1. Stream channel to temp file - string tempFilePath = Path.GetTempFileName(); - System.Diagnostics.Debug.WriteLine($"Temp file is at {tempFilePath}"); - try - { - long fileSizeBytes; - { - using var fileStream = new FileStream(tempFilePath, FileMode.Create, FileAccess.Write, FileShare.None); - using var gzip = new GZipStream(fileStream, CompressionLevel.Optimal); - using var writer = new StreamWriter(gzip); - await foreach (var item in _channel.Reader.ReadAllAsync(_cancellationToken).ConfigureAwait(false)) - { - await writer.WriteLineAsync($"{item.Id}\t{item.Json}\t{item.SpeckleType}").ConfigureAwait(false); - } -#if NET8_0_OR_GREATER - await writer.FlushAsync(_cancellationToken).ConfigureAwait(false); -#else - await writer.FlushAsync().ConfigureAwait(false); -#endif - } - // fileStream.Flush(); - // fileStream.Close(); - fileSizeBytes = new FileInfo(tempFilePath).Length; - - // 2. Request presigned URL - var signUri = new Uri($"projects/{_projectId}/models/{_modelId}/uploads/sign", UriKind.Relative); - - using var signResponse = await _client.PostAsync(signUri, null, _cancellationToken).ConfigureAwait(false); - signResponse.EnsureSuccessStatusCode(); - -#if NET5_0_OR_GREATER - string signResponseString = await signResponse - .Content.ReadAsStringAsync(_cancellationToken) - .ConfigureAwait(false); -#else - string signResponseString = await signResponse.Content.ReadAsStringAsync().ConfigureAwait(false); -#endif - PresignedUploadResponse presignedUpload = - JsonConvert.DeserializeObject(signResponseString) - ?? throw new InvalidOperationException("Failed to get presigned upload URL"); - - // 3. Upload to S3 - using var fileStreamUpload = new FileStream(tempFilePath, FileMode.Open, FileAccess.Read, FileShare.Read); - - Stream progressStream = fileStreamUpload; // TODO: wrap with progress stream - - using var streamContent = new StreamContent(progressStream); - streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); - streamContent.Headers.ContentLength = fileSizeBytes; - - using var uploadRequest = new HttpRequestMessage(HttpMethod.Put, new Uri(presignedUpload.Url, UriKind.Absolute)); - uploadRequest.Content = streamContent; - - using var s3Client = new HttpClient(); // NOTE: using a separate client for S3 as we DO NOT NEED THE AUTH HEADER, presigned url's don't work with it. - using var uploadResponse = await s3Client - .SendAsync(uploadRequest, HttpCompletionOption.ResponseHeadersRead, _cancellationToken) - .ConfigureAwait(false); - - uploadResponse.EnsureSuccessStatusCode(); - - // 4. Trigger processing - var processUri = new Uri($"projects/{_projectId}/models/{_modelId}/uploads/process", UriKind.Relative); - var processRequest = new ProcessUploadRequest { key = presignedUpload.Key, ingestionId = _ingestionId }; - - using StringContent content = new(JsonConvert.SerializeObject(processRequest), Encoding.UTF8, "application/json"); - var processResponse = await _client.PostAsync(processUri, content, _cancellationToken).ConfigureAwait(false); - - processResponse.EnsureSuccessStatusCode(); - -#if NET5_0_OR_GREATER - string processResult = await processResponse.Content.ReadAsStringAsync(_cancellationToken).ConfigureAwait(false); -#else - string processResult = await processResponse.Content.ReadAsStringAsync().ConfigureAwait(false); -#endif - var json = JsonConvert.DeserializeObject(processResult); - - if (json is null) - { - throw new InvalidOperationException("Failed to trigger upload processing"); - } - - return new UploadResult { IngestionId = json.ingestionId }; - } - finally - { - // 5. Clean up temp file - if (File.Exists(tempFilePath)) - { - try - { - //File.Delete(tempFilePath); - } -#pragma warning disable CA1031 - catch -#pragma warning restore CA1031 - { - // Best effort - } - } - } - } - - public void Dispose() => _client.Dispose(); -} - -// DTOs -internal record PresignedUploadResponse -{ - public required string Url { get; init; } - public required string Key { get; init; } -} - -internal record ProcessUploadRequest -{ - public required string key { get; init; } - public required string ingestionId { get; init; } -} - -internal record ProcessUploadResponse -{ - public required string ingestionId { get; init; } -} - -internal record UploadResult -{ - public required string IngestionId { get; init; } -} diff --git a/src/Speckle.Sdk/Pipelines/Uploader_Old.cs b/src/Speckle.Sdk/Pipelines/Uploader_Old.cs deleted file mode 100644 index fa8f73c6..00000000 --- a/src/Speckle.Sdk/Pipelines/Uploader_Old.cs +++ /dev/null @@ -1,83 +0,0 @@ -// using System.IO.Compression; -// using System.Net.Http.Headers; -// using System.Threading.Channels; -// -// namespace Speckle.Sdk.Pipeline; -// -// public sealed class UploaderOld : IDisposable -// { -// private readonly string _projectId; -// private readonly string _modelId; -// private readonly HttpClient _client; -// private readonly Channel _channel; -// private readonly Task _sendTask; -// -// public UploaderOld(string projectId, string modelId, string? authToken, string? apiEndpoint) -// { -// _projectId = projectId; -// _modelId = modelId; -// -// Uri apiBaseUrl = !string.IsNullOrEmpty(apiEndpoint) -// ? new Uri(apiEndpoint) -// // : new Uri("http://dimitries-macbook-pro.mermaid-emperor.ts.net/api/v1/"); -// : new Uri("http://zog.local:3000/api/v1/"); -// _client = new HttpClient { BaseAddress = apiBaseUrl, Timeout = TimeSpan.FromMinutes(10) }; -// -// if (authToken != null) -// { -// _client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", authToken); -// } -// -// _channel = Channel.CreateBounded( -// new BoundedChannelOptions(1000) { FullMode = BoundedChannelFullMode.Wait } // if we're not able to write fast enough, we'll block writes -// ); -// -// _sendTask = SendLoopAsync(projectId, "test"); -// } -// -// public ValueTask PushAsync(UploadItem item, CancellationToken ct = default) => _channel.Writer.WriteAsync(item, ct); -// -// public async Task CompleteAsync() -// { -// _channel.Writer.Complete(); -// await _sendTask.ConfigureAwait(false); -// } -// -// private async Task SendLoopAsync(string projectId, string modelId) -// { -// var content = new PushStreamContent( -// async (stream, _, _) => -// { -// var gzip = new GZipStream(stream, CompressionLevel.Optimal); -// var writer = new StreamWriter(gzip); // new StreamWriter(gzip, System.Text.Encoding.UTF8, 20 * 1024 * 1024); // potential lever for controlling memory pressure -// try -// { -// // extra levers for memory pressure in here: we can manually flush every x items or every x bytes -// await foreach (var item in _channel.Reader.ReadAllAsync().ConfigureAwait(false)) -// { -// await writer.WriteLineAsync($"{item.Id}\t{item.Json}\t{item.SpeckleType}").ConfigureAwait(false); -// } -// } -// finally -// { -// await writer.FlushAsync().ConfigureAwait(false); -// await gzip.FlushAsync().ConfigureAwait(false); -// writer.Dispose(); -// gzip.Dispose(); -// } -// }, -// new MediaTypeHeaderValue("application/x-ndjson") -// ); -// -// var uri = new Uri($"projects/{projectId}/models/{modelId}/versions", UriKind.Relative); -// var request = new HttpRequestMessage(HttpMethod.Post, uri) { Content = content }; -// request.Headers.TransferEncodingChunked = true; // NOTE: important for streaming to happen. -// var response = await _client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false); -// response.EnsureSuccessStatusCode(); -// -// // Consume the response body to fully complete the request -// await response.Content.ReadAsStringAsync().ConfigureAwait(false); -// } -// -// public void Dispose() => _client.Dispose(); -// } From 9bf6995b157dd55ca58d0fe8ad0f0ee3f8849438 Mon Sep 17 00:00:00 2001 From: Dimitrie Stefanescu Date: Tue, 24 Feb 2026 11:57:16 +0000 Subject: [PATCH 05/16] Add headers for Azure S3 compat --- src/Speckle.Sdk/Pipelines/Send/Uploader.cs | 13 +++++++++---- src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs | 1 + 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/Speckle.Sdk/Pipelines/Send/Uploader.cs b/src/Speckle.Sdk/Pipelines/Send/Uploader.cs index 9d7103d9..1cd96f13 100644 --- a/src/Speckle.Sdk/Pipelines/Send/Uploader.cs +++ b/src/Speckle.Sdk/Pipelines/Send/Uploader.cs @@ -74,8 +74,8 @@ public sealed class Uploader : IDisposable { using DisposableFile tempFile = await WriteFile().ConfigureAwait(false); - PresignedUploadResponse presignedUpload = await GetPresignedUrl().ConfigureAwait(false); - await UploadToS3(tempFile.FileInfo, presignedUpload.Url).ConfigureAwait(false); + PresignedUploadResponse presignedUploadResponse = await GetPresignedUrl().ConfigureAwait(false); + await UploadToS3(tempFile.FileInfo, presignedUploadResponse).ConfigureAwait(false); return await TriggerProcessing().ConfigureAwait(false); } @@ -123,7 +123,7 @@ public sealed class Uploader : IDisposable return presignedUpload; } - private async Task UploadToS3(FileInfo file, Uri s3Url) + private async Task UploadToS3(FileInfo file, PresignedUploadResponse presignedUploadResponse) { using var fileStreamUpload = new FileStream(file.FullName, FileMode.Open, FileAccess.Read, FileShare.Read); @@ -133,7 +133,12 @@ public sealed class Uploader : IDisposable streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); streamContent.Headers.ContentLength = file.Length; - using var uploadRequest = new HttpRequestMessage(HttpMethod.Put, s3Url); + using var uploadRequest = new HttpRequestMessage(HttpMethod.Put, presignedUploadResponse.Url); + foreach (var kvp in presignedUploadResponse.AdditionalRequestHeaders) + { + uploadRequest.Headers.Add(kvp.Key, kvp.Value); + } + uploadRequest.Content = streamContent; using var uploadResponse = await _s3Client diff --git a/src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs b/src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs index fb020de4..cff5775b 100644 --- a/src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs +++ b/src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs @@ -9,6 +9,7 @@ internal record PresignedUploadResponse { public required Uri Url { get; init; } public required string Key { get; init; } + public Dictionary AdditionalRequestHeaders { get; init; } = new(); } internal record ProcessUploadResponse From 070f21b07500086d64c7db80176cfc7de04d9c79 Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Thu, 26 Feb 2026 10:45:48 +0000 Subject: [PATCH 06/16] feat(progress): Add progress reporting to uploader (#446) * refactor uploader for progress * progress * not so many decimal places * small tweak to RenderStreamProgress * fix unit tests * uploading data --- .../Helpers/StopwatchPollyfills.cs | 21 +++++ .../Pipelines/Progress/AggregateProgress.cs | 12 +++ .../Progress/IngestionProgressManager.cs | 89 +++++++++++++++++++ .../IngestionProgressManagerFactory.cs | 22 +++++ .../Pipelines/Progress/ProgressArgs.cs | 6 ++ .../{ => Progress}/ProgressStream.cs | 16 ++-- .../Progress/RenderedStreamProgress.cs | 40 +++++++++ src/Speckle.Sdk/Pipelines/Send/DiskStore.cs | 77 ++++++++++++++++ .../Pipelines/Send/SendPipeline.cs | 30 +++++-- src/Speckle.Sdk/Pipelines/Send/Uploader.cs | 78 ++++------------ .../Pipelines/Send/UploaderDTOs.cs | 5 -- src/Speckle.Sdk/ServiceRegistration.cs | 4 +- 12 files changed, 313 insertions(+), 87 deletions(-) create mode 100644 src/Speckle.Sdk/Helpers/StopwatchPollyfills.cs create mode 100644 src/Speckle.Sdk/Pipelines/Progress/AggregateProgress.cs create mode 100644 src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManager.cs create mode 100644 src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManagerFactory.cs create mode 100644 src/Speckle.Sdk/Pipelines/Progress/ProgressArgs.cs rename src/Speckle.Sdk/Pipelines/{ => Progress}/ProgressStream.cs (87%) create mode 100644 src/Speckle.Sdk/Pipelines/Progress/RenderedStreamProgress.cs create mode 100644 src/Speckle.Sdk/Pipelines/Send/DiskStore.cs diff --git a/src/Speckle.Sdk/Helpers/StopwatchPollyfills.cs b/src/Speckle.Sdk/Helpers/StopwatchPollyfills.cs new file mode 100644 index 00000000..83dd0e05 --- /dev/null +++ b/src/Speckle.Sdk/Helpers/StopwatchPollyfills.cs @@ -0,0 +1,21 @@ +using System.Diagnostics; + +namespace Speckle.Sdk.Helpers; + +public static class StopwatchPolyfills +{ +#if !NET7_0_OR_GREATER + private static readonly double s_tickFrequency = (double)TimeSpan.TicksPerSecond / Stopwatch.Frequency; +#endif + + public static TimeSpan GetElapsedTime(long startingTimestamp) + { +#if NET7_0_OR_GREATER + return Stopwatch.GetElapsedTime(startingTimestamp); +#else + + long elapsedTicks = Stopwatch.GetTimestamp() - startingTimestamp; + return new TimeSpan((long)(elapsedTicks * s_tickFrequency)); +#endif + } +} diff --git a/src/Speckle.Sdk/Pipelines/Progress/AggregateProgress.cs b/src/Speckle.Sdk/Pipelines/Progress/AggregateProgress.cs new file mode 100644 index 00000000..74dfa327 --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Progress/AggregateProgress.cs @@ -0,0 +1,12 @@ +namespace Speckle.Sdk.Pipelines.Progress; + +public sealed class AggregateProgress(params IProgress[] progresses) : IProgress +{ + public void Report(T value) + { + foreach (var progress in progresses) + { + progress.Report(value); + } + } +} diff --git a/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManager.cs b/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManager.cs new file mode 100644 index 00000000..be141c65 --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManager.cs @@ -0,0 +1,89 @@ +using System.Diagnostics; +using Microsoft.Extensions.Logging; +using Speckle.InterfaceGenerator; +using Speckle.Sdk.Api; +using Speckle.Sdk.Api.GraphQL.Inputs; +using Speckle.Sdk.Api.GraphQL.Models; +using Speckle.Sdk.Helpers; + +namespace Speckle.Sdk.Pipelines.Progress; + +public partial interface IIngestionProgressManager : IProgress; + +/// +/// An implementation for the entire client side Ingestion progress update reporting +/// Will throttles ingestion progress messages and reports their progress +/// +[GenerateAutoInterface] +public sealed class IngestionProgressManager( + ILogger logger, + IClient speckleClient, + ModelIngestion ingestion, + string projectId, + TimeSpan updateInterval, + CancellationToken cancellationToken +) : IIngestionProgressManager +{ + /// + /// Normally we would pick quite a coarse throttle window to try and avoid over pressure (1-5s) + /// + private Task? _lastUpdate; + private long _lastUpdatedAt; + private readonly object _lock = new(); + + [AutoInterfaceIgnore] + public void Report(CardProgress value) + { + cancellationToken.ThrowIfCancellationRequested(); + + string trimmedMessage; + lock (_lock) + { + if (ShouldIgnoreProgressUpdate()) + { + return; + } + + _lastUpdatedAt = Stopwatch.GetTimestamp(); + + trimmedMessage = value.Status.TrimEnd('.'); + + _lastUpdate = speckleClient + .Ingestion.UpdateProgress( + new ModelIngestionUpdateInput(ingestion.id, projectId, trimmedMessage, value.Progress), + cancellationToken + ) + .ContinueWith( + HandleFaultedContinuation, + CancellationToken.None, + TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default + ); + } + + logger.LogInformation("Progress update {Message} {Progress}", trimmedMessage, value.Progress); + } + + /// if the update should be ignored, otherwise + private bool ShouldIgnoreProgressUpdate() + { + if (_lastUpdate is not null && !_lastUpdate.IsCompleted) + { + return true; + } + + TimeSpan msSinceLastUpdate = StopwatchPolyfills.GetElapsedTime(_lastUpdatedAt); + return msSinceLastUpdate < updateInterval; + } + + private void HandleFaultedContinuation(Task updateTask) + { + // The progress report failed... could be many reasons. + // For now, we're not letting this fail the Ingestion in any way + // we'll log but otherwise let it slide while leaving no unobserved task exceptions + if (updateTask.IsFaulted) + { + logger.LogWarning(updateTask.Exception, "A progress update failed unexpectedly"); + } + } +} diff --git a/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManagerFactory.cs b/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManagerFactory.cs new file mode 100644 index 00000000..998431c2 --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManagerFactory.cs @@ -0,0 +1,22 @@ +using Microsoft.Extensions.Logging; +using Speckle.InterfaceGenerator; +using Speckle.Sdk.Api; +using Speckle.Sdk.Api.GraphQL.Models; + +namespace Speckle.Sdk.Pipelines.Progress; + +[GenerateAutoInterface] +public sealed class IngestionProgressManagerFactory(ILogger logger) + : IIngestionProgressManagerFactory +{ + public IIngestionProgressManager CreateInstance( + IClient speckleClient, + ModelIngestion ingestion, + string projectId, + TimeSpan updateInterval, + CancellationToken cancellationToken + ) + { + return new IngestionProgressManager(logger, speckleClient, ingestion, projectId, updateInterval, cancellationToken); + } +} diff --git a/src/Speckle.Sdk/Pipelines/Progress/ProgressArgs.cs b/src/Speckle.Sdk/Pipelines/Progress/ProgressArgs.cs new file mode 100644 index 00000000..bfc47dbc --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Progress/ProgressArgs.cs @@ -0,0 +1,6 @@ +namespace Speckle.Sdk.Pipelines.Progress; + +//TODO: rename PipelineProgressArgs +public readonly record struct CardProgress(string Status, double? Progress); + +public readonly record struct StreamProgressArgs(long BytesStreamed, long ExpectedTotalBytes); diff --git a/src/Speckle.Sdk/Pipelines/ProgressStream.cs b/src/Speckle.Sdk/Pipelines/Progress/ProgressStream.cs similarity index 87% rename from src/Speckle.Sdk/Pipelines/ProgressStream.cs rename to src/Speckle.Sdk/Pipelines/Progress/ProgressStream.cs index b23998ae..f75c7837 100644 --- a/src/Speckle.Sdk/Pipelines/ProgressStream.cs +++ b/src/Speckle.Sdk/Pipelines/Progress/ProgressStream.cs @@ -1,23 +1,17 @@ using System.Diagnostics.CodeAnalysis; -namespace Speckle.Sdk.Pipelines; - -public readonly record struct StreamProgressArgs(long BytesStreamed, long ExpectedTotalBytes); +namespace Speckle.Sdk.Pipelines.Progress; /// -/// Wraps a stream to report upload progress as bytes are read. +/// Wraps to report streaming progress as bytes are read/written. /// -public sealed class ProgressStream( - Stream innerStream, - long expectedTotalBytesStreamed, - IProgress? progress = null -) : Stream +public sealed class ProgressStream(Stream innerStream, IProgress? progress = null) : Stream { private long _bytesStreamed; public override bool CanRead => innerStream.CanRead; public override bool CanSeek => innerStream.CanSeek; - public override bool CanWrite => false; + public override bool CanWrite => innerStream.CanWrite; public override long Length => innerStream.Length; public override long Position @@ -57,7 +51,7 @@ public sealed class ProgressStream( private void ReportProgress(int newBytesProcessed) { _bytesStreamed += newBytesProcessed; - progress?.Report(new(_bytesStreamed, expectedTotalBytesStreamed)); + progress?.Report(new(_bytesStreamed, Length)); } public override void Flush() => innerStream.Flush(); diff --git a/src/Speckle.Sdk/Pipelines/Progress/RenderedStreamProgress.cs b/src/Speckle.Sdk/Pipelines/Progress/RenderedStreamProgress.cs new file mode 100644 index 00000000..81f41d6c --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Progress/RenderedStreamProgress.cs @@ -0,0 +1,40 @@ +namespace Speckle.Sdk.Pipelines.Progress; + +/// +/// Renders "low level" data stream updates +/// into "high level" that is expected by Ingestion progress and DUI3 +/// +/// +public sealed class RenderedStreamProgress(IProgress progress) : IProgress +{ + public void Report(StreamProgressArgs value) + { + var (suffix, scaleFactor) = GetFileSizeRendering(value.ExpectedTotalBytes); + progress.Report( + new( + $"Uploading data... ({value.BytesStreamed * scaleFactor:F1}/{value.ExpectedTotalBytes * scaleFactor:F1} {suffix})", + (double)value.BytesStreamed / value.ExpectedTotalBytes + ) + ); + } + + private static readonly string[] s_suffixes = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"]; + + private static (string suffix, double scaleFactor) GetFileSizeRendering(long value) + { + if (value <= 0) + { + return (s_suffixes[0], 1d); + } + + for (int i = 0; i < s_suffixes.Length; i++) + { + if (value <= Math.Pow(1024, i + 1)) + { + return (s_suffixes[i], 1 / Math.Pow(1024, i)); + } + } + + throw new ArgumentOutOfRangeException(nameof(value), "Value is too large to convert to a file size"); + } +} diff --git a/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs b/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs new file mode 100644 index 00000000..0ac685fb --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs @@ -0,0 +1,77 @@ +using System.IO.Compression; +using System.Threading.Channels; +using Microsoft.Extensions.Logging; +using Speckle.InterfaceGenerator; +using Speckle.Sdk.Helpers; + +namespace Speckle.Sdk.Pipelines.Send; + +[GenerateAutoInterface] +public sealed class DiskStoreFactory(ILogger logger) : IDiskStoreFactory +{ + public DiskStore CreateInstance(CancellationToken cancellationToken) => new(logger, cancellationToken); +} + +public sealed class DiskStore +{ + private readonly Channel _channel; + private readonly Task _writeToDiskTask; + private readonly ILogger _logger; + private readonly CancellationToken _cancellationToken; + + internal DiskStore(ILogger logger, CancellationToken cancellationToken) + { + _logger = logger; + _cancellationToken = cancellationToken; + + _channel = Channel.CreateBounded( + new BoundedChannelOptions(1000) { FullMode = BoundedChannelFullMode.Wait, SingleReader = true } + ); + _writeToDiskTask = Task.Run(WriteFile, cancellationToken); + } + + public ValueTask PushAsync(UploadItem item) => _channel.Writer.WriteAsync(item, _cancellationToken); + + public async Task CompleteAsync() + { + _channel.Writer.Complete(); + return await _writeToDiskTask.ConfigureAwait(false); + } + + /// + /// Reads from the Channel and streams the s to a temporary file on disk. + /// Will keep reading until is called. + /// + /// the file that was written + private async Task WriteFile() + { + string tempFilePath = Path.GetTempFileName(); + var tempFile = new DisposableFile(new FileInfo(tempFilePath), _logger); + _logger.LogInformation("Writing temp file to {TempFilePath}", tempFilePath); + + try + { + using var fileStream = new FileStream(tempFilePath, FileMode.Create, FileAccess.Write, FileShare.None); + using var gzip = new GZipStream(fileStream, CompressionLevel.Optimal); + using var writer = new StreamWriter(gzip); + + await foreach (var item in _channel.Reader.ReadAllAsync(_cancellationToken).ConfigureAwait(false)) + { + await writer.WriteLineAsync($"{item.Id}\t{item.Json}\t{item.SpeckleType}").ConfigureAwait(false); + } +#if NET8_0_OR_GREATER + await writer.FlushAsync(_cancellationToken).ConfigureAwait(false); +#else + await writer.FlushAsync().ConfigureAwait(false); +#endif + tempFile.FileInfo.Refresh(); + + return tempFile; + } + catch + { + tempFile.Dispose(); + throw; + } + } +} diff --git a/src/Speckle.Sdk/Pipelines/Send/SendPipeline.cs b/src/Speckle.Sdk/Pipelines/Send/SendPipeline.cs index e1a97a40..e00439ff 100644 --- a/src/Speckle.Sdk/Pipelines/Send/SendPipeline.cs +++ b/src/Speckle.Sdk/Pipelines/Send/SendPipeline.cs @@ -1,21 +1,26 @@ using Speckle.InterfaceGenerator; using Speckle.Sdk.Credentials; +using Speckle.Sdk.Helpers; using Speckle.Sdk.Models; +using Speckle.Sdk.Pipelines.Progress; namespace Speckle.Sdk.Pipelines.Send; [GenerateAutoInterface] -public sealed class SendPipelineFactory(IUploaderFactory uploaderFactory) : ISendPipelineFactory +public sealed class SendPipelineFactory(IUploaderFactory uploaderFactory, IDiskStoreFactory diskStoreFactory) + : ISendPipelineFactory { public SendPipeline CreateInstance( string projectId, string ingestionId, Account account, + IProgress uploadProgress, CancellationToken cancellationToken ) { - var uploader = uploaderFactory.CreateInstance(projectId, ingestionId, account, cancellationToken); - return new SendPipeline(uploader); + var uploader = uploaderFactory.CreateInstance(projectId, ingestionId, account, uploadProgress, cancellationToken); + var diskStore = diskStoreFactory.CreateInstance(cancellationToken); + return new SendPipeline(uploader, diskStore); } } @@ -23,10 +28,12 @@ public sealed class SendPipeline : IDisposable { private readonly Serializer _serializer = new(); private readonly Uploader _uploader; + private readonly DiskStore _diskStore; - internal SendPipeline(Uploader uploader) + internal SendPipeline(Uploader uploader, DiskStore diskStore) { _uploader = uploader; + _diskStore = diskStore; } private UploadItem _lastItem; @@ -38,7 +45,7 @@ public sealed class SendPipeline : IDisposable foreach (var item in results) { // we're not doing fire and forget here so that we get the backpressure from the uploader - await _uploader.PushAsync(item).ConfigureAwait(false); + await _diskStore.PushAsync(item).ConfigureAwait(false); } // NOTE: this is important to keep track of. When we serialze an object, we get back a list of objects, with the first one being the original root. @@ -50,8 +57,17 @@ public sealed class SendPipeline : IDisposable public async Task WaitForUpload() { - await _uploader.PushAsync(_lastItem).ConfigureAwait(false); - await _uploader.CompleteAsync().ConfigureAwait(false); + await _diskStore.PushAsync(_lastItem).ConfigureAwait(false); + using DisposableFile tempFile = await _diskStore.CompleteAsync().ConfigureAwait(false); + + using Stream fileStreamUpload = new FileStream( + tempFile.FileInfo.FullName, + FileMode.Open, + FileAccess.Read, + FileShare.Read + ); + + await _uploader.Send(fileStreamUpload).ConfigureAwait(false); } public async Task WaitForUploadAndServerProcessing() diff --git a/src/Speckle.Sdk/Pipelines/Send/Uploader.cs b/src/Speckle.Sdk/Pipelines/Send/Uploader.cs index 1cd96f13..63a17881 100644 --- a/src/Speckle.Sdk/Pipelines/Send/Uploader.cs +++ b/src/Speckle.Sdk/Pipelines/Send/Uploader.cs @@ -1,11 +1,10 @@ -using System.IO.Compression; using System.Net.Http.Headers; -using System.Threading.Channels; using Microsoft.Extensions.Logging; using Speckle.InterfaceGenerator; using Speckle.Newtonsoft.Json; using Speckle.Sdk.Credentials; using Speckle.Sdk.Helpers; +using Speckle.Sdk.Pipelines.Progress; namespace Speckle.Sdk.Pipelines.Send; @@ -16,8 +15,9 @@ public sealed class UploaderFactory(ISpeckleHttp httpClientFactory, ILogger progress, CancellationToken cancellationToken - ) => new(projectId, ingestionId, logger, httpClientFactory, account, cancellationToken); + ) => new(projectId, ingestionId, logger, httpClientFactory, account, progress, cancellationToken); } public sealed class Uploader : IDisposable @@ -27,9 +27,8 @@ public sealed class Uploader : IDisposable private readonly CancellationToken _cancellationToken; private readonly HttpClient _speckleClient; private readonly HttpClient _s3Client; - private readonly Channel _channel; - private readonly Task _sendTask; private readonly ILogger _logger; + private readonly IProgress _progress; internal Uploader( string projectId, @@ -37,6 +36,7 @@ public sealed class Uploader : IDisposable ILogger logger, ISpeckleHttp httpClientFactory, Account speckleAccount, + IProgress progress, CancellationToken cancellationToken ) { @@ -44,65 +44,19 @@ public sealed class Uploader : IDisposable _ingestionId = ingestionId; _logger = logger; _cancellationToken = cancellationToken; - - _speckleClient = httpClientFactory.CreateHttpClient( - null, - (int)TimeSpan.FromMinutes(30).TotalSeconds, - speckleAccount.token - ); + _progress = progress; + _speckleClient = httpClientFactory.CreateHttpClient(authorizationToken: speckleAccount.token); _speckleClient.BaseAddress = new(new(speckleAccount.serverInfo.url), "/api/v1/"); _s3Client = httpClientFactory.CreateHttpClient(); - - _channel = Channel.CreateBounded( - new BoundedChannelOptions(1000) { FullMode = BoundedChannelFullMode.Wait, SingleReader = true } - ); - - _sendTask = Task.Run(SendLoopAsync, cancellationToken); } - public ValueTask PushAsync(UploadItem item) => _channel.Writer.WriteAsync(item, _cancellationToken); - - public async Task CompleteAsync() + public async Task Send(Stream fileStream) { - _channel.Writer.Complete(); - var result = await _sendTask.ConfigureAwait(false); - return result.IngestionId; - } - - private async Task SendLoopAsync() - { - using DisposableFile tempFile = await WriteFile().ConfigureAwait(false); - PresignedUploadResponse presignedUploadResponse = await GetPresignedUrl().ConfigureAwait(false); - await UploadToS3(tempFile.FileInfo, presignedUploadResponse).ConfigureAwait(false); + await UploadToS3(fileStream, presignedUploadResponse).ConfigureAwait(false); - return await TriggerProcessing().ConfigureAwait(false); - } - - /// - /// Reads from the Channel and streams the s to a temporary file on disk. - /// Will keep reading until is called. - /// - /// the file that was written - private async Task WriteFile() - { - string tempFilePath = Path.GetTempFileName(); - _logger.LogInformation("Writing temp file to {TempFilePath}", tempFilePath); - - using var fileStream = new FileStream(tempFilePath, FileMode.Create, FileAccess.Write, FileShare.None); - using var gzip = new GZipStream(fileStream, CompressionLevel.Optimal); - using var writer = new StreamWriter(gzip); - await foreach (var item in _channel.Reader.ReadAllAsync(_cancellationToken).ConfigureAwait(false)) - { - await writer.WriteLineAsync($"{item.Id}\t{item.Json}\t{item.SpeckleType}").ConfigureAwait(false); - } -#if NET8_0_OR_GREATER - await writer.FlushAsync(_cancellationToken).ConfigureAwait(false); -#else - await writer.FlushAsync().ConfigureAwait(false); -#endif - return new DisposableFile(new FileInfo(tempFilePath), _logger); + await TriggerProcessing().ConfigureAwait(false); } private async Task GetPresignedUrl() @@ -123,15 +77,15 @@ public sealed class Uploader : IDisposable return presignedUpload; } - private async Task UploadToS3(FileInfo file, PresignedUploadResponse presignedUploadResponse) + private async Task UploadToS3(Stream fileStream, PresignedUploadResponse presignedUploadResponse) { - using var fileStreamUpload = new FileStream(file.FullName, FileMode.Open, FileAccess.Read, FileShare.Read); + _logger.LogInformation("Uploading file to pre-signed url"); - Stream progressStream = fileStreamUpload; // TODO: wrap with progress stream + Stream progressStream = new ProgressStream(fileStream, _progress); using var streamContent = new StreamContent(progressStream); streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); - streamContent.Headers.ContentLength = file.Length; + streamContent.Headers.ContentLength = fileStream.Length; using var uploadRequest = new HttpRequestMessage(HttpMethod.Put, presignedUploadResponse.Url); foreach (var kvp in presignedUploadResponse.AdditionalRequestHeaders) @@ -148,7 +102,7 @@ public sealed class Uploader : IDisposable uploadResponse.EnsureSuccessStatusCode(); } - private async Task TriggerProcessing() + private async Task TriggerProcessing() { Uri processUri = new($"projects/{_projectId}/modelingestion/{_ingestionId}/uploads/process", UriKind.Relative); @@ -157,8 +111,6 @@ public sealed class Uploader : IDisposable .ConfigureAwait(false); processResponse.EnsureSuccessStatusCode(); - - return new UploadResult { IngestionId = _ingestionId }; } public void Dispose() diff --git a/src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs b/src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs index cff5775b..8ff9a546 100644 --- a/src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs +++ b/src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs @@ -16,8 +16,3 @@ internal record ProcessUploadResponse { public required string ingestionId { get; init; } } - -internal record UploadResult -{ - public required string IngestionId { get; init; } -} diff --git a/src/Speckle.Sdk/ServiceRegistration.cs b/src/Speckle.Sdk/ServiceRegistration.cs index 94e147cc..9028ae7c 100644 --- a/src/Speckle.Sdk/ServiceRegistration.cs +++ b/src/Speckle.Sdk/ServiceRegistration.cs @@ -8,6 +8,7 @@ using Speckle.Sdk.Dependencies; using Speckle.Sdk.Host; using Speckle.Sdk.Logging; using Speckle.Sdk.Models.GraphTraversal; +using Speckle.Sdk.Pipelines.Progress; using Speckle.Sdk.Serialisation.V2; using Speckle.Sdk.Serialisation.V2.Receive; using Speckle.Sdk.Serialisation.V2.Send; @@ -96,7 +97,8 @@ public static class ServiceRegistration typeof(DeserializeProcess), typeof(ObjectLoader), typeof(TraversalRule), - typeof(Client) + typeof(Client), + typeof(IngestionProgressManager) ); serviceCollection.AddMatchingInterfacesAsTransient(typeof(GraphQLRetry).Assembly); return serviceCollection; From 0ca9162e7b3a4e062a8d6b4a7ff878f5e2d24f92 Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Thu, 26 Feb 2026 16:09:19 +0000 Subject: [PATCH 07/16] pass etag to trigger processing (#447) --- src/Speckle.Sdk/Api/Blob/BlobApi.cs | 25 +--------------- src/Speckle.Sdk/Helpers/BlobApiHelpers.cs | 29 +++++++++++++++++++ src/Speckle.Sdk/Pipelines/Send/Uploader.cs | 16 ++++++---- .../Pipelines/Send/UploaderDTOs.cs | 6 ++-- 4 files changed, 45 insertions(+), 31 deletions(-) create mode 100644 src/Speckle.Sdk/Helpers/BlobApiHelpers.cs diff --git a/src/Speckle.Sdk/Api/Blob/BlobApi.cs b/src/Speckle.Sdk/Api/Blob/BlobApi.cs index 1dd68eb5..10939f6d 100644 --- a/src/Speckle.Sdk/Api/Blob/BlobApi.cs +++ b/src/Speckle.Sdk/Api/Blob/BlobApi.cs @@ -193,30 +193,7 @@ public sealed class BlobApi : IBlobApi using var response = await _unauthedClient.SendAsync(requestMessage, cancellationToken).ConfigureAwait(false); response.EnsureSuccessStatusCode(); - return ParseEtagHeader(response.Headers); - } - - private static string ParseEtagHeader(HttpResponseHeaders headers) - { - if (!headers.TryGetValues("ETag", out var etagValues)) - { - throw new ArgumentException( - "Response does not have an ETag attached to it, cannot use this as an upload", - nameof(headers) - ); - } - - var etagValuesArray = etagValues.ToArray(); - - if (etagValuesArray.Length != 1) - { - throw new ArgumentException( - $"Expected Etag header to have a single value but got {etagValuesArray.Length}", - nameof(headers) - ); - } - - return etagValuesArray[0]; + return BlobApiHelpers.ParseEtagHeader(response.Headers); } /// diff --git a/src/Speckle.Sdk/Helpers/BlobApiHelpers.cs b/src/Speckle.Sdk/Helpers/BlobApiHelpers.cs new file mode 100644 index 00000000..c7979567 --- /dev/null +++ b/src/Speckle.Sdk/Helpers/BlobApiHelpers.cs @@ -0,0 +1,29 @@ +using System.Net.Http.Headers; + +namespace Speckle.Sdk.Helpers; + +public static class BlobApiHelpers +{ + public static string ParseEtagHeader(HttpResponseHeaders headers) + { + if (!headers.TryGetValues("ETag", out var etagValues)) + { + throw new ArgumentException( + "Response does not have an ETag attached to it, cannot use this as an upload", + nameof(headers) + ); + } + + var etagValuesArray = etagValues.ToArray(); + + if (etagValuesArray.Length != 1) + { + throw new ArgumentException( + $"Expected Etag header to have a single value but got {etagValuesArray.Length}", + nameof(headers) + ); + } + + return etagValuesArray[0]; + } +} diff --git a/src/Speckle.Sdk/Pipelines/Send/Uploader.cs b/src/Speckle.Sdk/Pipelines/Send/Uploader.cs index 63a17881..1cd4f8f4 100644 --- a/src/Speckle.Sdk/Pipelines/Send/Uploader.cs +++ b/src/Speckle.Sdk/Pipelines/Send/Uploader.cs @@ -1,4 +1,5 @@ using System.Net.Http.Headers; +using System.Text; using Microsoft.Extensions.Logging; using Speckle.InterfaceGenerator; using Speckle.Newtonsoft.Json; @@ -54,9 +55,9 @@ public sealed class Uploader : IDisposable public async Task Send(Stream fileStream) { PresignedUploadResponse presignedUploadResponse = await GetPresignedUrl().ConfigureAwait(false); - await UploadToS3(fileStream, presignedUploadResponse).ConfigureAwait(false); + var etag = await UploadToS3(fileStream, presignedUploadResponse).ConfigureAwait(false); - await TriggerProcessing().ConfigureAwait(false); + await TriggerProcessing(new() { Etag = etag }).ConfigureAwait(false); } private async Task GetPresignedUrl() @@ -77,7 +78,7 @@ public sealed class Uploader : IDisposable return presignedUpload; } - private async Task UploadToS3(Stream fileStream, PresignedUploadResponse presignedUploadResponse) + private async Task UploadToS3(Stream fileStream, PresignedUploadResponse presignedUploadResponse) { _logger.LogInformation("Uploading file to pre-signed url"); @@ -100,16 +101,21 @@ public sealed class Uploader : IDisposable .ConfigureAwait(false); uploadResponse.EnsureSuccessStatusCode(); + + return BlobApiHelpers.ParseEtagHeader(uploadResponse.Headers); } - private async Task TriggerProcessing() + private async Task TriggerProcessing(TriggerUploadRequest request) { Uri processUri = new($"projects/{_projectId}/modelingestion/{_ingestionId}/uploads/process", UriKind.Relative); + string requestBody = JsonConvert.SerializeObject(request); + using var content = new StringContent(requestBody, Encoding.UTF8, "application/json"); using HttpResponseMessage processResponse = await _speckleClient - .PostAsync(processUri, null, _cancellationToken) + .PostAsync(processUri, content, _cancellationToken) .ConfigureAwait(false); + string body = await processResponse.Content.ReadAsStringAsync().ConfigureAwait(false); processResponse.EnsureSuccessStatusCode(); } diff --git a/src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs b/src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs index 8ff9a546..11df1420 100644 --- a/src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs +++ b/src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs @@ -1,3 +1,4 @@ +using Speckle.Newtonsoft.Json; using Speckle.Sdk.Models; using Speckle.Sdk.Serialisation; @@ -12,7 +13,8 @@ internal record PresignedUploadResponse public Dictionary AdditionalRequestHeaders { get; init; } = new(); } -internal record ProcessUploadResponse +internal readonly struct TriggerUploadRequest { - public required string ingestionId { get; init; } + [JsonProperty("etag")] + public required string Etag { get; init; } } From 9a74195b24bee0c6b3eea247eb05472c26556955 Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Thu, 26 Feb 2026 17:10:54 +0000 Subject: [PATCH 08/16] feat(api): Add versionId to ModelIngestion (#448) * Add version id to ingestion query * and the sub * Add project & user ids * test --- .../Api/GraphQL/Models/ModelIngestion.cs | 3 ++- .../Models/ModelIngestionStatusData.cs | 1 + .../Resources/ModelIngestionResource.cs | 20 +++++++++++++++++++ .../GraphQL/Resources/SubscriptionResource.cs | 6 ++++++ .../Resources/ModelIngestionResourceTests.cs | 7 +++++++ 5 files changed, 36 insertions(+), 1 deletion(-) diff --git a/src/Speckle.Sdk/Api/GraphQL/Models/ModelIngestion.cs b/src/Speckle.Sdk/Api/GraphQL/Models/ModelIngestion.cs index 58d0dc67..e5c08dfc 100644 --- a/src/Speckle.Sdk/Api/GraphQL/Models/ModelIngestion.cs +++ b/src/Speckle.Sdk/Api/GraphQL/Models/ModelIngestion.cs @@ -6,7 +6,8 @@ public sealed class ModelIngestion public required DateTime createdAt { get; init; } public required DateTime updatedAt { get; init; } public required string modelId { get; init; } + public required string projectId { get; init; } + public required string userId { get; init; } public required bool cancellationRequested { get; init; } public required ModelIngestionStatusData statusData { get; init; } - // public required LimitedUser user { get; init; } } diff --git a/src/Speckle.Sdk/Api/GraphQL/Models/ModelIngestionStatusData.cs b/src/Speckle.Sdk/Api/GraphQL/Models/ModelIngestionStatusData.cs index a195bdfc..8e98d1e6 100644 --- a/src/Speckle.Sdk/Api/GraphQL/Models/ModelIngestionStatusData.cs +++ b/src/Speckle.Sdk/Api/GraphQL/Models/ModelIngestionStatusData.cs @@ -6,4 +6,5 @@ public sealed class ModelIngestionStatusData { public required ModelIngestionStatus status { get; init; } public required string? progressMessage { get; init; } + public required string? versionId { get; init; } } diff --git a/src/Speckle.Sdk/Api/GraphQL/Resources/ModelIngestionResource.cs b/src/Speckle.Sdk/Api/GraphQL/Resources/ModelIngestionResource.cs index d029a61d..d01a6a10 100644 --- a/src/Speckle.Sdk/Api/GraphQL/Resources/ModelIngestionResource.cs +++ b/src/Speckle.Sdk/Api/GraphQL/Resources/ModelIngestionResource.cs @@ -44,6 +44,8 @@ public sealed class ModelIngestionResource createdAt updatedAt modelId + projectId + userId cancellationRequested statusData { ... on HasModelIngestionStatus { @@ -94,6 +96,8 @@ public sealed class ModelIngestionResource createdAt updatedAt modelId + projectId + userId cancellationRequested statusData { ... on HasModelIngestionStatus { @@ -102,6 +106,10 @@ public sealed class ModelIngestionResource ... on HasProgressMessage { progressMessage } + ... on ModelIngestionSuccessStatus + { + versionId + } } } } @@ -142,6 +150,8 @@ public sealed class ModelIngestionResource createdAt updatedAt modelId + projectId + userId cancellationRequested statusData { ... on HasModelIngestionStatus { @@ -194,6 +204,8 @@ public sealed class ModelIngestionResource createdAt updatedAt modelId + projectId + userId cancellationRequested statusData { ... on HasModelIngestionStatus { @@ -245,6 +257,8 @@ public sealed class ModelIngestionResource createdAt updatedAt modelId + projectId + userId cancellationRequested statusData { ... on HasModelIngestionStatus { @@ -343,6 +357,8 @@ public sealed class ModelIngestionResource createdAt updatedAt modelId + projectId + userId cancellationRequested statusData { ... on HasModelIngestionStatus { @@ -398,6 +414,8 @@ public sealed class ModelIngestionResource createdAt updatedAt modelId + projectId + userId cancellationRequested statusData { ... on HasModelIngestionStatus { @@ -457,6 +475,8 @@ public sealed class ModelIngestionResource createdAt updatedAt modelId + projectId + userId cancellationRequested statusData { ... on HasModelIngestionStatus { diff --git a/src/Speckle.Sdk/Api/GraphQL/Resources/SubscriptionResource.cs b/src/Speckle.Sdk/Api/GraphQL/Resources/SubscriptionResource.cs index 9a014cff..8ba201d1 100644 --- a/src/Speckle.Sdk/Api/GraphQL/Resources/SubscriptionResource.cs +++ b/src/Speckle.Sdk/Api/GraphQL/Resources/SubscriptionResource.cs @@ -229,6 +229,8 @@ public sealed class SubscriptionResource : IDisposable createdAt updatedAt modelId + projectId + userId cancellationRequested statusData { ... on HasModelIngestionStatus { @@ -237,6 +239,10 @@ public sealed class SubscriptionResource : IDisposable ... on HasProgressMessage { progressMessage } + ... on ModelIngestionSuccessStatus + { + versionId + } } } type diff --git a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/ModelIngestionResourceTests.cs b/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/ModelIngestionResourceTests.cs index 88ff78d2..7b8e1552 100644 --- a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/ModelIngestionResourceTests.cs +++ b/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/ModelIngestionResourceTests.cs @@ -129,8 +129,10 @@ public sealed class ModelIngestionResourceTests : IAsyncLifetime ModelIngestionSuccessInput finish = new(ingest.id, _project.id, sendResult.RootId, "yay!"); string versionId = await Sut.Complete(finish); Version version = await _testUser.Version.Get(versionId, _project.id); + ModelIngestion finalIngestion = await _testUser.Ingestion.Get(ingest.id, _project.id); Assert.Equal(version.id, versionId); Assert.Equal(sendResult.RootId, version.referencedObject); + Assert.Equal(finalIngestion.statusData.versionId, versionId); } [Fact] @@ -147,6 +149,11 @@ public sealed class ModelIngestionResourceTests : IAsyncLifetime ModelIngestion res = await Sut.Get(ingest.id, _project.id); Assert.Equal(ingest.id, res.id); Assert.Equal(ingest.statusData.status, res.statusData.status); + Assert.Equal(ingest.statusData.versionId, res.statusData.versionId); + Assert.Null(res.statusData.versionId); + Assert.Equal(_model.id, res.modelId); + Assert.Equal(_project.id, res.projectId); + Assert.Equal(_testUser.Account.userInfo.id, res.userId); } [Fact] From a662fb54c2f928a533c0f4639062816dee279b98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?O=C4=9Fuzhan=20Koral?= <45078678+oguzhankoral@users.noreply.github.com> Date: Fri, 27 Feb 2026 13:32:16 +0300 Subject: [PATCH 09/16] fix: channels reference only for net standard (#450) --- Directory.Packages.props | 2 +- src/Speckle.Automate.Sdk/packages.lock.json | 23 +++++++------------ src/Speckle.Objects/packages.lock.json | 23 +++++++------------ .../packages.lock.json | 18 +++++++-------- src/Speckle.Sdk/Speckle.Sdk.csproj | 2 +- src/Speckle.Sdk/packages.lock.json | 18 +++++---------- .../packages.lock.json | 9 +------- .../packages.lock.json | 9 +------- .../packages.lock.json | 9 +------- .../packages.lock.json | 9 +------- tests/Speckle.Sdk.Testing/packages.lock.json | 9 +------- .../packages.lock.json | 9 +------- .../packages.lock.json | 9 +------- .../Speckle.Sdk.Tests.Unit/packages.lock.json | 9 +------- 14 files changed, 41 insertions(+), 117 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index a2c96acf..f2f4c427 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -27,7 +27,7 @@ - + diff --git a/src/Speckle.Automate.Sdk/packages.lock.json b/src/Speckle.Automate.Sdk/packages.lock.json index 1a222bbb..dd5d4c56 100644 --- a/src/Speckle.Automate.Sdk/packages.lock.json +++ b/src/Speckle.Automate.Sdk/packages.lock.json @@ -88,8 +88,8 @@ }, "Microsoft.Bcl.AsyncInterfaces": { "type": "Transitive", - "resolved": "10.0.1", - "contentHash": "E1HSLkPHXEO30JEij2pWbOuzz1Z5ND4a5l7IP1T2RgQuE0a0NzEIvtO64RNy3Otn6PFezbT80cfm3M/Cgt70PA==", + "resolved": "10.0.0", + "contentHash": "vFuwSLj9QJBbNR0NeNO4YVASUbokxs+i/xbuu8B+Fs4FAZg5QaFa6eGrMaRqTzzNI5tAb97T7BhSxtLckFyiRA==", "dependencies": { "System.Threading.Tasks.Extensions": "4.6.3" } @@ -296,7 +296,7 @@ "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", "Speckle.Sdk.Dependencies": "[1.0.0, )", - "System.Threading.Channels": "[10.0.1, )" + "System.Threading.Channels": "[10.0.0, )" } }, "speckle.sdk.dependencies": { @@ -361,11 +361,11 @@ }, "System.Threading.Channels": { "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==", + "requested": "[10.0.0, )", + "resolved": "10.0.0", + "contentHash": "fwRdkJpKisUEVNaEdsL5w5EwidzuVw0BOTfzDvYB1Yg8sq1pqNfUZxBOVFgSj6i6tNhpT3HP8BEDXf1+kFkTDA==", "dependencies": { - "Microsoft.Bcl.AsyncInterfaces": "10.0.1", + "Microsoft.Bcl.AsyncInterfaces": "10.0.0", "System.Threading.Tasks.Extensions": "4.6.3" } } @@ -565,8 +565,7 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )", - "System.Threading.Channels": "[10.0.1, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )" } }, "speckle.sdk.dependencies": { @@ -622,12 +621,6 @@ "requested": "[13.0.2, )", "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" - }, - "System.Threading.Channels": { - "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" } } } diff --git a/src/Speckle.Objects/packages.lock.json b/src/Speckle.Objects/packages.lock.json index 5042eec7..9ac413aa 100644 --- a/src/Speckle.Objects/packages.lock.json +++ b/src/Speckle.Objects/packages.lock.json @@ -56,8 +56,8 @@ }, "Microsoft.Bcl.AsyncInterfaces": { "type": "Transitive", - "resolved": "10.0.1", - "contentHash": "E1HSLkPHXEO30JEij2pWbOuzz1Z5ND4a5l7IP1T2RgQuE0a0NzEIvtO64RNy3Otn6PFezbT80cfm3M/Cgt70PA==", + "resolved": "10.0.0", + "contentHash": "vFuwSLj9QJBbNR0NeNO4YVASUbokxs+i/xbuu8B+Fs4FAZg5QaFa6eGrMaRqTzzNI5tAb97T7BhSxtLckFyiRA==", "dependencies": { "System.Threading.Tasks.Extensions": "4.6.3" } @@ -243,7 +243,7 @@ "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", "Speckle.Sdk.Dependencies": "[1.0.0, )", - "System.Threading.Channels": "[10.0.1, )" + "System.Threading.Channels": "[10.0.0, )" } }, "speckle.sdk.dependencies": { @@ -308,11 +308,11 @@ }, "System.Threading.Channels": { "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==", + "requested": "[10.0.0, )", + "resolved": "10.0.0", + "contentHash": "fwRdkJpKisUEVNaEdsL5w5EwidzuVw0BOTfzDvYB1Yg8sq1pqNfUZxBOVFgSj6i6tNhpT3HP8BEDXf1+kFkTDA==", "dependencies": { - "Microsoft.Bcl.AsyncInterfaces": "10.0.1", + "Microsoft.Bcl.AsyncInterfaces": "10.0.0", "System.Threading.Tasks.Extensions": "4.6.3" } } @@ -486,8 +486,7 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )", - "System.Threading.Channels": "[10.0.1, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )" } }, "speckle.sdk.dependencies": { @@ -543,12 +542,6 @@ "requested": "[13.0.2, )", "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" - }, - "System.Threading.Channels": { - "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" } } } diff --git a/src/Speckle.Sdk.Dependencies/packages.lock.json b/src/Speckle.Sdk.Dependencies/packages.lock.json index aa13bc43..760480ee 100644 --- a/src/Speckle.Sdk.Dependencies/packages.lock.json +++ b/src/Speckle.Sdk.Dependencies/packages.lock.json @@ -82,11 +82,11 @@ }, "System.Threading.Channels": { "type": "Direct", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==", + "requested": "[10.0.0, )", + "resolved": "10.0.0", + "contentHash": "fwRdkJpKisUEVNaEdsL5w5EwidzuVw0BOTfzDvYB1Yg8sq1pqNfUZxBOVFgSj6i6tNhpT3HP8BEDXf1+kFkTDA==", "dependencies": { - "Microsoft.Bcl.AsyncInterfaces": "10.0.1", + "Microsoft.Bcl.AsyncInterfaces": "10.0.0", "System.Threading.Tasks.Extensions": "4.6.3" } }, @@ -97,8 +97,8 @@ }, "Microsoft.Bcl.AsyncInterfaces": { "type": "Transitive", - "resolved": "10.0.1", - "contentHash": "E1HSLkPHXEO30JEij2pWbOuzz1Z5ND4a5l7IP1T2RgQuE0a0NzEIvtO64RNy3Otn6PFezbT80cfm3M/Cgt70PA==", + "resolved": "10.0.0", + "contentHash": "vFuwSLj9QJBbNR0NeNO4YVASUbokxs+i/xbuu8B+Fs4FAZg5QaFa6eGrMaRqTzzNI5tAb97T7BhSxtLckFyiRA==", "dependencies": { "System.Threading.Tasks.Extensions": "4.6.3" } @@ -228,9 +228,9 @@ }, "System.Threading.Channels": { "type": "Direct", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" + "requested": "[10.0.0, )", + "resolved": "10.0.0", + "contentHash": "fwRdkJpKisUEVNaEdsL5w5EwidzuVw0BOTfzDvYB1Yg8sq1pqNfUZxBOVFgSj6i6tNhpT3HP8BEDXf1+kFkTDA==" }, "ILRepack": { "type": "Transitive", diff --git a/src/Speckle.Sdk/Speckle.Sdk.csproj b/src/Speckle.Sdk/Speckle.Sdk.csproj index 1e9cf643..350b4330 100644 --- a/src/Speckle.Sdk/Speckle.Sdk.csproj +++ b/src/Speckle.Sdk/Speckle.Sdk.csproj @@ -26,7 +26,6 @@ - @@ -36,6 +35,7 @@ + diff --git a/src/Speckle.Sdk/packages.lock.json b/src/Speckle.Sdk/packages.lock.json index 17478611..5702b969 100644 --- a/src/Speckle.Sdk/packages.lock.json +++ b/src/Speckle.Sdk/packages.lock.json @@ -92,11 +92,11 @@ }, "System.Threading.Channels": { "type": "Direct", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==", + "requested": "[10.0.0, )", + "resolved": "10.0.0", + "contentHash": "fwRdkJpKisUEVNaEdsL5w5EwidzuVw0BOTfzDvYB1Yg8sq1pqNfUZxBOVFgSj6i6tNhpT3HP8BEDXf1+kFkTDA==", "dependencies": { - "Microsoft.Bcl.AsyncInterfaces": "10.0.1", + "Microsoft.Bcl.AsyncInterfaces": "10.0.0", "System.Threading.Tasks.Extensions": "4.6.3" } }, @@ -123,8 +123,8 @@ }, "Microsoft.Bcl.AsyncInterfaces": { "type": "Transitive", - "resolved": "10.0.1", - "contentHash": "E1HSLkPHXEO30JEij2pWbOuzz1Z5ND4a5l7IP1T2RgQuE0a0NzEIvtO64RNy3Otn6PFezbT80cfm3M/Cgt70PA==", + "resolved": "10.0.0", + "contentHash": "vFuwSLj9QJBbNR0NeNO4YVASUbokxs+i/xbuu8B+Fs4FAZg5QaFa6eGrMaRqTzzNI5tAb97T7BhSxtLckFyiRA==", "dependencies": { "System.Threading.Tasks.Extensions": "4.6.3" } @@ -377,12 +377,6 @@ "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" }, - "System.Threading.Channels": { - "type": "Direct", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" - }, "GraphQL.Client.Abstractions": { "type": "Transitive", "resolved": "6.0.0", diff --git a/tests/Speckle.Automate.Sdk.Integration/packages.lock.json b/tests/Speckle.Automate.Sdk.Integration/packages.lock.json index d0818532..78c3841b 100644 --- a/tests/Speckle.Automate.Sdk.Integration/packages.lock.json +++ b/tests/Speckle.Automate.Sdk.Integration/packages.lock.json @@ -370,8 +370,7 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )", - "System.Threading.Channels": "[10.0.1, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )" } }, "speckle.sdk.dependencies": { @@ -492,12 +491,6 @@ "System.Text.Encodings.Web": "8.0.0" } }, - "System.Threading.Channels": { - "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" - }, "Verify.Quibble": { "type": "CentralTransitive", "requested": "[2.1.1, )", diff --git a/tests/Speckle.Objects.Tests.Unit/packages.lock.json b/tests/Speckle.Objects.Tests.Unit/packages.lock.json index 3ef2d70b..fe375e71 100644 --- a/tests/Speckle.Objects.Tests.Unit/packages.lock.json +++ b/tests/Speckle.Objects.Tests.Unit/packages.lock.json @@ -348,8 +348,7 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )", - "System.Threading.Channels": "[10.0.1, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )" } }, "speckle.sdk.dependencies": { @@ -433,12 +432,6 @@ "System.Text.Encodings.Web": "8.0.0" } }, - "System.Threading.Channels": { - "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" - }, "Verify.Quibble": { "type": "CentralTransitive", "requested": "[2.1.1, )", diff --git a/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json b/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json index 022252ea..765c1137 100644 --- a/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json +++ b/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json @@ -342,8 +342,7 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )", - "System.Threading.Channels": "[10.0.1, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )" } }, "speckle.sdk.dependencies": { @@ -425,12 +424,6 @@ "requested": "[13.0.2, )", "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" - }, - "System.Threading.Channels": { - "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" } } } diff --git a/tests/Speckle.Sdk.Serialization.Tests/packages.lock.json b/tests/Speckle.Sdk.Serialization.Tests/packages.lock.json index 11691412..2ac574fb 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/packages.lock.json +++ b/tests/Speckle.Sdk.Serialization.Tests/packages.lock.json @@ -393,8 +393,7 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )", - "System.Threading.Channels": "[10.0.1, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )" } }, "speckle.sdk.dependencies": { @@ -475,12 +474,6 @@ "System.Text.Encodings.Web": "8.0.0" } }, - "System.Threading.Channels": { - "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" - }, "Verify.Quibble": { "type": "CentralTransitive", "requested": "[2.1.1, )", diff --git a/tests/Speckle.Sdk.Testing/packages.lock.json b/tests/Speckle.Sdk.Testing/packages.lock.json index 9cb33c89..6e2cfd77 100644 --- a/tests/Speckle.Sdk.Testing/packages.lock.json +++ b/tests/Speckle.Sdk.Testing/packages.lock.json @@ -311,8 +311,7 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )", - "System.Threading.Channels": "[10.0.1, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )" } }, "speckle.sdk.dependencies": { @@ -377,12 +376,6 @@ "dependencies": { "System.Text.Encodings.Web": "8.0.0" } - }, - "System.Threading.Channels": { - "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" } } } diff --git a/tests/Speckle.Sdk.Tests.Integration/packages.lock.json b/tests/Speckle.Sdk.Tests.Integration/packages.lock.json index 59e1a4d2..6138bda1 100644 --- a/tests/Speckle.Sdk.Tests.Integration/packages.lock.json +++ b/tests/Speckle.Sdk.Tests.Integration/packages.lock.json @@ -370,8 +370,7 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )", - "System.Threading.Channels": "[10.0.1, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )" } }, "speckle.sdk.dependencies": { @@ -455,12 +454,6 @@ "System.Text.Encodings.Web": "8.0.0" } }, - "System.Threading.Channels": { - "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" - }, "Verify.Quibble": { "type": "CentralTransitive", "requested": "[2.1.1, )", diff --git a/tests/Speckle.Sdk.Tests.Performance/packages.lock.json b/tests/Speckle.Sdk.Tests.Performance/packages.lock.json index a96f59c2..bff226e4 100644 --- a/tests/Speckle.Sdk.Tests.Performance/packages.lock.json +++ b/tests/Speckle.Sdk.Tests.Performance/packages.lock.json @@ -360,8 +360,7 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )", - "System.Threading.Channels": "[10.0.1, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )" } }, "speckle.sdk.dependencies": { @@ -417,12 +416,6 @@ "requested": "[13.0.2, )", "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" - }, - "System.Threading.Channels": { - "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" } } } diff --git a/tests/Speckle.Sdk.Tests.Unit/packages.lock.json b/tests/Speckle.Sdk.Tests.Unit/packages.lock.json index 4faff1c8..8b50e6f9 100644 --- a/tests/Speckle.Sdk.Tests.Unit/packages.lock.json +++ b/tests/Speckle.Sdk.Tests.Unit/packages.lock.json @@ -363,8 +363,7 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )", - "System.Threading.Channels": "[10.0.1, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )" } }, "speckle.sdk.dependencies": { @@ -442,12 +441,6 @@ "System.Text.Encodings.Web": "8.0.0" } }, - "System.Threading.Channels": { - "type": "CentralTransitive", - "requested": "[10.0.1, )", - "resolved": "10.0.1", - "contentHash": "YRqU6Y2Cl6C+HrG5h1ftgKZ5VDTSA7j1wMKs5RtlauPeQ2EZ639Jt5aOFHdX3naP01hDDWFOWPApmNDVKwOpmg==" - }, "Verify.Quibble": { "type": "CentralTransitive", "requested": "[2.1.1, )", From c4e956cdb411c4362d0e8eccbe02e04dcf92b108 Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Thu, 5 Mar 2026 12:47:56 +0000 Subject: [PATCH 10/16] fix(build): Option 2 - "keep channels il-repacked" (#453) * Repacked channel * transitive for sdk * restore solution * comment tweaks * Fix print line --- Directory.Packages.props | 3 +- src/Speckle.Automate.Sdk/packages.lock.json | 45 +++++++---------- src/Speckle.Objects/packages.lock.json | 45 +++++++---------- .../RepackedChannel.cs | 49 +++++++++++++++++++ .../Speckle.Sdk.Dependencies.csproj | 32 ++++++++++++ .../packages.lock.json | 43 ++++++++-------- src/Speckle.Sdk/Pipelines/Send/DiskStore.cs | 15 +++--- src/Speckle.Sdk/Speckle.Sdk.csproj | 1 - src/Speckle.Sdk/packages.lock.json | 42 +++++++--------- .../packages.lock.json | 11 +++-- .../packages.lock.json | 11 +++-- 11 files changed, 180 insertions(+), 117 deletions(-) create mode 100644 src/Speckle.Sdk.Dependencies/RepackedChannel.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index f2f4c427..8c51176a 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -15,6 +15,7 @@ + @@ -27,7 +28,7 @@ - + diff --git a/src/Speckle.Automate.Sdk/packages.lock.json b/src/Speckle.Automate.Sdk/packages.lock.json index dd5d4c56..f7174f7c 100644 --- a/src/Speckle.Automate.Sdk/packages.lock.json +++ b/src/Speckle.Automate.Sdk/packages.lock.json @@ -86,14 +86,6 @@ "resolved": "6.0.0", "contentHash": "yg72rrYDapfsIUrul7aF6wwNnTJBOFvuA9VdDTQpPa8AlAriHbufeXYLBcodKjfUdkCnaiggX1U/nEP08Zb5GA==" }, - "Microsoft.Bcl.AsyncInterfaces": { - "type": "Transitive", - "resolved": "10.0.0", - "contentHash": "vFuwSLj9QJBbNR0NeNO4YVASUbokxs+i/xbuu8B+Fs4FAZg5QaFa6eGrMaRqTzzNI5tAb97T7BhSxtLckFyiRA==", - "dependencies": { - "System.Threading.Tasks.Extensions": "4.6.3" - } - }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", "resolved": "8.0.0", @@ -250,8 +242,8 @@ }, "System.Runtime.CompilerServices.Unsafe": { "type": "Transitive", - "resolved": "6.1.2", - "contentHash": "2hBr6zdbIBTDE3EhK7NSVNdX58uTK6iHW/P/Axmm9sl1xoGSLqDvMtpecn226TNwHByFokYwJmt/aQQNlO5CRw==" + "resolved": "6.0.0", + "contentHash": "/iUeP3tq1S0XdNNoMz5C9twLSrM/TH+qElHkXWaPvuNOt+99G75NrV0OS2EqHx5wMN7popYjpc8oTjC1y16DLg==" }, "System.Runtime.InteropServices.WindowsRuntime": { "type": "Transitive", @@ -273,10 +265,10 @@ }, "System.Threading.Tasks.Extensions": { "type": "Transitive", - "resolved": "4.6.3", - "contentHash": "7sCiwilJLYbTZELaKnc7RecBBXWXA+xMLQWZKWawBxYjp6DBlSE3v9/UcvKBvr1vv2tTOhipiogM8rRmxlhrVA==", + "resolved": "4.5.4", + "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==", "dependencies": { - "System.Runtime.CompilerServices.Unsafe": "6.1.2" + "System.Runtime.CompilerServices.Unsafe": "4.5.3" } }, "speckle.objects": { @@ -295,12 +287,14 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )", - "System.Threading.Channels": "[10.0.0, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )" } }, "speckle.sdk.dependencies": { - "type": "Project" + "type": "Project", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "[9.0.4, )" + } }, "GraphQL.Client": { "type": "CentralTransitive", @@ -313,6 +307,15 @@ "System.Reactive": "5.0.0" } }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "CentralTransitive", + "requested": "[9.0.4, )", + "resolved": "9.0.4", + "contentHash": "9VGI5kxIvrNG2mqLQZnUR6y/3fcnygD8eNpHR+CqfbnIXvea6nehnYknDKQTxZVPMpzpNca+7DxLBmpdB3q0Bw==", + "dependencies": { + "System.Threading.Tasks.Extensions": "4.5.4" + } + }, "Microsoft.CSharp": { "type": "CentralTransitive", "requested": "[4.7.0, )", @@ -358,16 +361,6 @@ "requested": "[13.0.2, )", "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" - }, - "System.Threading.Channels": { - "type": "CentralTransitive", - "requested": "[10.0.0, )", - "resolved": "10.0.0", - "contentHash": "fwRdkJpKisUEVNaEdsL5w5EwidzuVw0BOTfzDvYB1Yg8sq1pqNfUZxBOVFgSj6i6tNhpT3HP8BEDXf1+kFkTDA==", - "dependencies": { - "Microsoft.Bcl.AsyncInterfaces": "10.0.0", - "System.Threading.Tasks.Extensions": "4.6.3" - } } }, "net8.0": { diff --git a/src/Speckle.Objects/packages.lock.json b/src/Speckle.Objects/packages.lock.json index 9ac413aa..1fe1cf2d 100644 --- a/src/Speckle.Objects/packages.lock.json +++ b/src/Speckle.Objects/packages.lock.json @@ -54,14 +54,6 @@ "resolved": "6.0.0", "contentHash": "yg72rrYDapfsIUrul7aF6wwNnTJBOFvuA9VdDTQpPa8AlAriHbufeXYLBcodKjfUdkCnaiggX1U/nEP08Zb5GA==" }, - "Microsoft.Bcl.AsyncInterfaces": { - "type": "Transitive", - "resolved": "10.0.0", - "contentHash": "vFuwSLj9QJBbNR0NeNO4YVASUbokxs+i/xbuu8B+Fs4FAZg5QaFa6eGrMaRqTzzNI5tAb97T7BhSxtLckFyiRA==", - "dependencies": { - "System.Threading.Tasks.Extensions": "4.6.3" - } - }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", "resolved": "8.0.0", @@ -213,8 +205,8 @@ }, "System.Runtime.CompilerServices.Unsafe": { "type": "Transitive", - "resolved": "6.1.2", - "contentHash": "2hBr6zdbIBTDE3EhK7NSVNdX58uTK6iHW/P/Axmm9sl1xoGSLqDvMtpecn226TNwHByFokYwJmt/aQQNlO5CRw==" + "resolved": "4.5.3", + "contentHash": "3TIsJhD1EiiT0w2CcDMN/iSSwnNnsrnbzeVHSKkaEgV85txMprmuO+Yq2AdSbeVGcg28pdNDTPK87tJhX7VFHw==" }, "System.Runtime.InteropServices.WindowsRuntime": { "type": "Transitive", @@ -226,10 +218,10 @@ }, "System.Threading.Tasks.Extensions": { "type": "Transitive", - "resolved": "4.6.3", - "contentHash": "7sCiwilJLYbTZELaKnc7RecBBXWXA+xMLQWZKWawBxYjp6DBlSE3v9/UcvKBvr1vv2tTOhipiogM8rRmxlhrVA==", + "resolved": "4.5.4", + "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==", "dependencies": { - "System.Runtime.CompilerServices.Unsafe": "6.1.2" + "System.Runtime.CompilerServices.Unsafe": "4.5.3" } }, "speckle.sdk": { @@ -242,12 +234,14 @@ "Microsoft.Extensions.Logging": "[2.2.0, )", "Speckle.DoubleNumerics": "[4.1.0, )", "Speckle.Newtonsoft.Json": "[13.0.2, )", - "Speckle.Sdk.Dependencies": "[1.0.0, )", - "System.Threading.Channels": "[10.0.0, )" + "Speckle.Sdk.Dependencies": "[1.0.0, )" } }, "speckle.sdk.dependencies": { - "type": "Project" + "type": "Project", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "[9.0.4, )" + } }, "GraphQL.Client": { "type": "CentralTransitive", @@ -260,6 +254,15 @@ "System.Reactive": "5.0.0" } }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "CentralTransitive", + "requested": "[9.0.4, )", + "resolved": "9.0.4", + "contentHash": "9VGI5kxIvrNG2mqLQZnUR6y/3fcnygD8eNpHR+CqfbnIXvea6nehnYknDKQTxZVPMpzpNca+7DxLBmpdB3q0Bw==", + "dependencies": { + "System.Threading.Tasks.Extensions": "4.5.4" + } + }, "Microsoft.CSharp": { "type": "CentralTransitive", "requested": "[4.7.0, )", @@ -305,16 +308,6 @@ "requested": "[13.0.2, )", "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" - }, - "System.Threading.Channels": { - "type": "CentralTransitive", - "requested": "[10.0.0, )", - "resolved": "10.0.0", - "contentHash": "fwRdkJpKisUEVNaEdsL5w5EwidzuVw0BOTfzDvYB1Yg8sq1pqNfUZxBOVFgSj6i6tNhpT3HP8BEDXf1+kFkTDA==", - "dependencies": { - "Microsoft.Bcl.AsyncInterfaces": "10.0.0", - "System.Threading.Tasks.Extensions": "4.6.3" - } } }, "net8.0": { diff --git a/src/Speckle.Sdk.Dependencies/RepackedChannel.cs b/src/Speckle.Sdk.Dependencies/RepackedChannel.cs new file mode 100644 index 00000000..a5f2e73b --- /dev/null +++ b/src/Speckle.Sdk.Dependencies/RepackedChannel.cs @@ -0,0 +1,49 @@ +using System.Threading.Channels; + +namespace Speckle.Sdk.Dependencies; + +/// +/// For various reasons related to our use of ILRepack.FullAuto, +/// we cannot use Channels from the SDK project. +/// We have to keep usage of it inside the Sdk.Dependencies project. +/// +/// For the sake of quick development, I've wrapped the class here in a type +/// that is safe to use from the SDK project. +/// +/// As and when we need more functions, we can add them here. +/// +/// And yes... I'm not very happy about the way we've set this up +/// +/// +public sealed class RepackedChannel +{ + private readonly Channel _channel; + + public RepackedChannel(int capacity, bool singleReader, bool singleWriter) + { + _channel = Channel.CreateBounded( + new BoundedChannelOptions(capacity) + { + FullMode = BoundedChannelFullMode.Wait, + SingleReader = singleReader, + SingleWriter = singleWriter, + } + ); + } + + public void CompleteWriter() => _channel.Writer.Complete(); + + public ValueTask WriteAsync(T item, CancellationToken cancellationToken) => + _channel.Writer.WriteAsync(item, cancellationToken); + + public IAsyncEnumerable ReadAllAsync(CancellationToken cancellationToken) => + _channel.Reader.ReadAllAsync(cancellationToken); + + // public async Task ReadAllAsync(Func callback, CancellationToken cancellationToken) + // { + // await foreach (T item in _channel.Reader.ReadAllAsync(cancellationToken)) + // { + // await callback.Invoke(item).ConfigureAwait(false); + // } + // } +} diff --git a/src/Speckle.Sdk.Dependencies/Speckle.Sdk.Dependencies.csproj b/src/Speckle.Sdk.Dependencies/Speckle.Sdk.Dependencies.csproj index 043b7876..fe91aa91 100644 --- a/src/Speckle.Sdk.Dependencies/Speckle.Sdk.Dependencies.csproj +++ b/src/Speckle.Sdk.Dependencies/Speckle.Sdk.Dependencies.csproj @@ -28,4 +28,36 @@ + + + + + + + <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)System.Numerics.Vectors.dll" /> + <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)System.Runtime.CompilerServices.Unsafe.dll" /> + <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)System.Memory.dll" /> + <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)Open.ChannelExtensions.dll" /> + <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)System.Threading.Channels.dll" /> + <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)System.Collections.Immutable.dll" /> + <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)Polly.dll" /> + <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)Polly.Contrib.WaitAndRetry.dll" /> + <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)Polly.Extensions.Http.dll" /> + <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)Microsoft.Extensions.ObjectPool.dll" /> + <_ILRepackExcludeAssemblies_Items Include="$(OutputPath)*.dll" Exclude="@(_ILRepackIncludeAssemblies_Items)" /> + + + + @(_ILRepackExcludeAssemblies_Items) + + diff --git a/src/Speckle.Sdk.Dependencies/packages.lock.json b/src/Speckle.Sdk.Dependencies/packages.lock.json index 760480ee..ad39f5df 100644 --- a/src/Speckle.Sdk.Dependencies/packages.lock.json +++ b/src/Speckle.Sdk.Dependencies/packages.lock.json @@ -11,6 +11,15 @@ "ILRepack": "2.0.33" } }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "Direct", + "requested": "[9.0.4, )", + "resolved": "9.0.4", + "contentHash": "9VGI5kxIvrNG2mqLQZnUR6y/3fcnygD8eNpHR+CqfbnIXvea6nehnYknDKQTxZVPMpzpNca+7DxLBmpdB3q0Bw==", + "dependencies": { + "System.Threading.Tasks.Extensions": "4.5.4" + } + }, "Microsoft.Extensions.ObjectPool": { "type": "Direct", "requested": "[9.0.4, )", @@ -82,12 +91,12 @@ }, "System.Threading.Channels": { "type": "Direct", - "requested": "[10.0.0, )", - "resolved": "10.0.0", - "contentHash": "fwRdkJpKisUEVNaEdsL5w5EwidzuVw0BOTfzDvYB1Yg8sq1pqNfUZxBOVFgSj6i6tNhpT3HP8BEDXf1+kFkTDA==", + "requested": "[9.0.4, )", + "resolved": "9.0.4", + "contentHash": "4qBn2H6/aXBpE/Pm3wY5yusY/pEvQz99NlWHrTUji0qCmOdbhhjaALcpmbfW2ksxlPM6i6S+QFLkpOQdyfeKYQ==", "dependencies": { - "Microsoft.Bcl.AsyncInterfaces": "10.0.0", - "System.Threading.Tasks.Extensions": "4.6.3" + "Microsoft.Bcl.AsyncInterfaces": "9.0.4", + "System.Threading.Tasks.Extensions": "4.5.4" } }, "ILRepack": { @@ -95,14 +104,6 @@ "resolved": "2.0.33", "contentHash": "xb2h1CsOepoYwdXEPui9VcQglwABQwNf9cccZbf+acarEzF5PUp8Xx71nFXIhOgEdm6wrxAoF6xAxK4m/XFRUQ==" }, - "Microsoft.Bcl.AsyncInterfaces": { - "type": "Transitive", - "resolved": "10.0.0", - "contentHash": "vFuwSLj9QJBbNR0NeNO4YVASUbokxs+i/xbuu8B+Fs4FAZg5QaFa6eGrMaRqTzzNI5tAb97T7BhSxtLckFyiRA==", - "dependencies": { - "System.Threading.Tasks.Extensions": "4.6.3" - } - }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", "resolved": "8.0.0", @@ -149,15 +150,15 @@ }, "System.Runtime.CompilerServices.Unsafe": { "type": "Transitive", - "resolved": "6.1.2", - "contentHash": "2hBr6zdbIBTDE3EhK7NSVNdX58uTK6iHW/P/Axmm9sl1xoGSLqDvMtpecn226TNwHByFokYwJmt/aQQNlO5CRw==" + "resolved": "6.0.0", + "contentHash": "/iUeP3tq1S0XdNNoMz5C9twLSrM/TH+qElHkXWaPvuNOt+99G75NrV0OS2EqHx5wMN7popYjpc8oTjC1y16DLg==" }, "System.Threading.Tasks.Extensions": { "type": "Transitive", - "resolved": "4.6.3", - "contentHash": "7sCiwilJLYbTZELaKnc7RecBBXWXA+xMLQWZKWawBxYjp6DBlSE3v9/UcvKBvr1vv2tTOhipiogM8rRmxlhrVA==", + "resolved": "4.5.4", + "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==", "dependencies": { - "System.Runtime.CompilerServices.Unsafe": "6.1.2" + "System.Runtime.CompilerServices.Unsafe": "4.5.3" } } }, @@ -228,9 +229,9 @@ }, "System.Threading.Channels": { "type": "Direct", - "requested": "[10.0.0, )", - "resolved": "10.0.0", - "contentHash": "fwRdkJpKisUEVNaEdsL5w5EwidzuVw0BOTfzDvYB1Yg8sq1pqNfUZxBOVFgSj6i6tNhpT3HP8BEDXf1+kFkTDA==" + "requested": "[9.0.4, )", + "resolved": "9.0.4", + "contentHash": "4qBn2H6/aXBpE/Pm3wY5yusY/pEvQz99NlWHrTUji0qCmOdbhhjaALcpmbfW2ksxlPM6i6S+QFLkpOQdyfeKYQ==" }, "ILRepack": { "type": "Transitive", diff --git a/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs b/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs index 0ac685fb..140f9ad2 100644 --- a/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs +++ b/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs @@ -1,7 +1,7 @@ using System.IO.Compression; -using System.Threading.Channels; using Microsoft.Extensions.Logging; using Speckle.InterfaceGenerator; +using Speckle.Sdk.Dependencies; using Speckle.Sdk.Helpers; namespace Speckle.Sdk.Pipelines.Send; @@ -14,7 +14,7 @@ public sealed class DiskStoreFactory(ILogger logger) : IDiskStoreFact public sealed class DiskStore { - private readonly Channel _channel; + private readonly RepackedChannel _channel; private readonly Task _writeToDiskTask; private readonly ILogger _logger; private readonly CancellationToken _cancellationToken; @@ -24,17 +24,16 @@ public sealed class DiskStore _logger = logger; _cancellationToken = cancellationToken; - _channel = Channel.CreateBounded( - new BoundedChannelOptions(1000) { FullMode = BoundedChannelFullMode.Wait, SingleReader = true } - ); + _channel = new RepackedChannel(1000, true, false); _writeToDiskTask = Task.Run(WriteFile, cancellationToken); } - public ValueTask PushAsync(UploadItem item) => _channel.Writer.WriteAsync(item, _cancellationToken); + public async Task PushAsync(UploadItem item) => + await _channel.WriteAsync(item, _cancellationToken).ConfigureAwait(false); public async Task CompleteAsync() { - _channel.Writer.Complete(); + _channel.CompleteWriter(); return await _writeToDiskTask.ConfigureAwait(false); } @@ -55,7 +54,7 @@ public sealed class DiskStore using var gzip = new GZipStream(fileStream, CompressionLevel.Optimal); using var writer = new StreamWriter(gzip); - await foreach (var item in _channel.Reader.ReadAllAsync(_cancellationToken).ConfigureAwait(false)) + await foreach (var item in _channel.ReadAllAsync(_cancellationToken).ConfigureAwait(false)) { await writer.WriteLineAsync($"{item.Id}\t{item.Json}\t{item.SpeckleType}").ConfigureAwait(false); } diff --git a/src/Speckle.Sdk/Speckle.Sdk.csproj b/src/Speckle.Sdk/Speckle.Sdk.csproj index 350b4330..000ee2bf 100644 --- a/src/Speckle.Sdk/Speckle.Sdk.csproj +++ b/src/Speckle.Sdk/Speckle.Sdk.csproj @@ -35,7 +35,6 @@ - diff --git a/src/Speckle.Sdk/packages.lock.json b/src/Speckle.Sdk/packages.lock.json index 5702b969..1426afb3 100644 --- a/src/Speckle.Sdk/packages.lock.json +++ b/src/Speckle.Sdk/packages.lock.json @@ -90,16 +90,6 @@ "resolved": "13.0.2", "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" }, - "System.Threading.Channels": { - "type": "Direct", - "requested": "[10.0.0, )", - "resolved": "10.0.0", - "contentHash": "fwRdkJpKisUEVNaEdsL5w5EwidzuVw0BOTfzDvYB1Yg8sq1pqNfUZxBOVFgSj6i6tNhpT3HP8BEDXf1+kFkTDA==", - "dependencies": { - "Microsoft.Bcl.AsyncInterfaces": "10.0.0", - "System.Threading.Tasks.Extensions": "4.6.3" - } - }, "GraphQL.Client.Abstractions": { "type": "Transitive", "resolved": "6.0.0", @@ -121,14 +111,6 @@ "resolved": "6.0.0", "contentHash": "yg72rrYDapfsIUrul7aF6wwNnTJBOFvuA9VdDTQpPa8AlAriHbufeXYLBcodKjfUdkCnaiggX1U/nEP08Zb5GA==" }, - "Microsoft.Bcl.AsyncInterfaces": { - "type": "Transitive", - "resolved": "10.0.0", - "contentHash": "vFuwSLj9QJBbNR0NeNO4YVASUbokxs+i/xbuu8B+Fs4FAZg5QaFa6eGrMaRqTzzNI5tAb97T7BhSxtLckFyiRA==", - "dependencies": { - "System.Threading.Tasks.Extensions": "4.6.3" - } - }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", "resolved": "8.0.0", @@ -280,8 +262,8 @@ }, "System.Runtime.CompilerServices.Unsafe": { "type": "Transitive", - "resolved": "6.1.2", - "contentHash": "2hBr6zdbIBTDE3EhK7NSVNdX58uTK6iHW/P/Axmm9sl1xoGSLqDvMtpecn226TNwHByFokYwJmt/aQQNlO5CRw==" + "resolved": "4.5.3", + "contentHash": "3TIsJhD1EiiT0w2CcDMN/iSSwnNnsrnbzeVHSKkaEgV85txMprmuO+Yq2AdSbeVGcg28pdNDTPK87tJhX7VFHw==" }, "System.Runtime.InteropServices.WindowsRuntime": { "type": "Transitive", @@ -293,14 +275,26 @@ }, "System.Threading.Tasks.Extensions": { "type": "Transitive", - "resolved": "4.6.3", - "contentHash": "7sCiwilJLYbTZELaKnc7RecBBXWXA+xMLQWZKWawBxYjp6DBlSE3v9/UcvKBvr1vv2tTOhipiogM8rRmxlhrVA==", + "resolved": "4.5.4", + "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==", "dependencies": { - "System.Runtime.CompilerServices.Unsafe": "6.1.2" + "System.Runtime.CompilerServices.Unsafe": "4.5.3" } }, "speckle.sdk.dependencies": { - "type": "Project" + "type": "Project", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "[9.0.4, )" + } + }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "CentralTransitive", + "requested": "[9.0.4, )", + "resolved": "9.0.4", + "contentHash": "9VGI5kxIvrNG2mqLQZnUR6y/3fcnygD8eNpHR+CqfbnIXvea6nehnYknDKQTxZVPMpzpNca+7DxLBmpdB3q0Bw==", + "dependencies": { + "System.Threading.Tasks.Extensions": "4.5.4" + } } }, "net8.0": { diff --git a/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json b/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json index 765c1137..efd29581 100644 --- a/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json +++ b/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json @@ -74,11 +74,6 @@ "resolved": "1.17.0", "contentHash": "8x+HCVTl/HHTGpscH3vMBhV8sknN/muZFw9s3TsI8SA6+c43cOTCi2+jE4KsU8pNLbJ++iF2ZFcpcXHXtDglnw==" }, - "Microsoft.Bcl.AsyncInterfaces": { - "type": "Transitive", - "resolved": "1.1.0", - "contentHash": "1Am6l4Vpn3/K32daEqZI+FFr96OlZkgwK2LcT3pZ2zWubR5zTPW3/FkO1Rat9kb7oQOa4rxgl9LJHc5tspCWfg==" - }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", "resolved": "8.0.0", @@ -385,6 +380,12 @@ "System.Reactive": "5.0.0" } }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "CentralTransitive", + "requested": "[9.0.4, )", + "resolved": "1.1.0", + "contentHash": "1Am6l4Vpn3/K32daEqZI+FFr96OlZkgwK2LcT3pZ2zWubR5zTPW3/FkO1Rat9kb7oQOa4rxgl9LJHc5tspCWfg==" + }, "Microsoft.Data.Sqlite": { "type": "CentralTransitive", "requested": "[7.0.5, )", diff --git a/tests/Speckle.Sdk.Tests.Performance/packages.lock.json b/tests/Speckle.Sdk.Tests.Performance/packages.lock.json index bff226e4..23a76189 100644 --- a/tests/Speckle.Sdk.Tests.Performance/packages.lock.json +++ b/tests/Speckle.Sdk.Tests.Performance/packages.lock.json @@ -92,11 +92,6 @@ "resolved": "1.17.0", "contentHash": "8x+HCVTl/HHTGpscH3vMBhV8sknN/muZFw9s3TsI8SA6+c43cOTCi2+jE4KsU8pNLbJ++iF2ZFcpcXHXtDglnw==" }, - "Microsoft.Bcl.AsyncInterfaces": { - "type": "Transitive", - "resolved": "1.1.0", - "contentHash": "1Am6l4Vpn3/K32daEqZI+FFr96OlZkgwK2LcT3pZ2zWubR5zTPW3/FkO1Rat9kb7oQOa4rxgl9LJHc5tspCWfg==" - }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", "resolved": "8.0.0", @@ -377,6 +372,12 @@ "System.Reactive": "5.0.0" } }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "CentralTransitive", + "requested": "[9.0.4, )", + "resolved": "1.1.0", + "contentHash": "1Am6l4Vpn3/K32daEqZI+FFr96OlZkgwK2LcT3pZ2zWubR5zTPW3/FkO1Rat9kb7oQOa4rxgl9LJHc5tspCWfg==" + }, "Microsoft.Data.Sqlite": { "type": "CentralTransitive", "requested": "[7.0.5, )", From f777050c102bc9f6e207142b3965bfc24a489146 Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Thu, 5 Mar 2026 13:40:18 +0000 Subject: [PATCH 11/16] change order of tab delimited json (#454) --- src/Speckle.Sdk/Pipelines/Send/DiskStore.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs b/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs index 140f9ad2..cc808450 100644 --- a/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs +++ b/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs @@ -56,7 +56,7 @@ public sealed class DiskStore await foreach (var item in _channel.ReadAllAsync(_cancellationToken).ConfigureAwait(false)) { - await writer.WriteLineAsync($"{item.Id}\t{item.Json}\t{item.SpeckleType}").ConfigureAwait(false); + await writer.WriteLineAsync($"{item.Id}\t{item.SpeckleType}\t{item.Json}").ConfigureAwait(false); } #if NET8_0_OR_GREATER await writer.FlushAsync(_cancellationToken).ConfigureAwait(false); From 515d45528d1b95e45336c151cc9eff303551ccea Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Wed, 11 Mar 2026 14:51:58 +0000 Subject: [PATCH 12/16] feat(otel): traces for new send pipeline (#457) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 😃 extra traces * default --- .../GraphQL/Inputs/ModelIngestionInputs.cs | 3 ++- src/Speckle.Sdk/Pipelines/Send/DiskStore.cs | 15 ++++++++++++--- src/Speckle.Sdk/Pipelines/Send/Uploader.cs | 19 ++++++++++++------- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/src/Speckle.Sdk/Api/GraphQL/Inputs/ModelIngestionInputs.cs b/src/Speckle.Sdk/Api/GraphQL/Inputs/ModelIngestionInputs.cs index 1215d034..87cbde66 100644 --- a/src/Speckle.Sdk/Api/GraphQL/Inputs/ModelIngestionInputs.cs +++ b/src/Speckle.Sdk/Api/GraphQL/Inputs/ModelIngestionInputs.cs @@ -14,7 +14,8 @@ public record ModelIngestionCreateInput( string modelId, string projectId, string progressMessage, - SourceDataInput sourceData + SourceDataInput sourceData, + int? maxIdleTimeoutSeconds = null ); public record ModelIngestionUpdateInput(string ingestionId, string projectId, string progressMessage, double? progress); diff --git a/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs b/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs index cc808450..88443633 100644 --- a/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs +++ b/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs @@ -3,13 +3,15 @@ using Microsoft.Extensions.Logging; using Speckle.InterfaceGenerator; using Speckle.Sdk.Dependencies; using Speckle.Sdk.Helpers; +using Speckle.Sdk.Logging; namespace Speckle.Sdk.Pipelines.Send; [GenerateAutoInterface] -public sealed class DiskStoreFactory(ILogger logger) : IDiskStoreFactory +public sealed class DiskStoreFactory(ILogger logger, ISdkActivityFactory activityFactory) : IDiskStoreFactory { - public DiskStore CreateInstance(CancellationToken cancellationToken) => new(logger, cancellationToken); + public DiskStore CreateInstance(CancellationToken cancellationToken) => + new(logger, activityFactory, cancellationToken); } public sealed class DiskStore @@ -17,11 +19,17 @@ public sealed class DiskStore private readonly RepackedChannel _channel; private readonly Task _writeToDiskTask; private readonly ILogger _logger; + private readonly ISdkActivityFactory _activityFactory; private readonly CancellationToken _cancellationToken; - internal DiskStore(ILogger logger, CancellationToken cancellationToken) + internal DiskStore( + ILogger logger, + ISdkActivityFactory activityFactory, + CancellationToken cancellationToken + ) { _logger = logger; + _activityFactory = activityFactory; _cancellationToken = cancellationToken; _channel = new RepackedChannel(1000, true, false); @@ -33,6 +41,7 @@ public sealed class DiskStore public async Task CompleteAsync() { + using var a = _activityFactory.Start("Waiting for DiskStore to complete"); _channel.CompleteWriter(); return await _writeToDiskTask.ConfigureAwait(false); } diff --git a/src/Speckle.Sdk/Pipelines/Send/Uploader.cs b/src/Speckle.Sdk/Pipelines/Send/Uploader.cs index 1cd4f8f4..0b7f2126 100644 --- a/src/Speckle.Sdk/Pipelines/Send/Uploader.cs +++ b/src/Speckle.Sdk/Pipelines/Send/Uploader.cs @@ -1,16 +1,17 @@ using System.Net.Http.Headers; using System.Text; -using Microsoft.Extensions.Logging; using Speckle.InterfaceGenerator; using Speckle.Newtonsoft.Json; using Speckle.Sdk.Credentials; using Speckle.Sdk.Helpers; +using Speckle.Sdk.Logging; using Speckle.Sdk.Pipelines.Progress; namespace Speckle.Sdk.Pipelines.Send; [GenerateAutoInterface] -public sealed class UploaderFactory(ISpeckleHttp httpClientFactory, ILogger logger) : IUploaderFactory +public sealed class UploaderFactory(ISpeckleHttp httpClientFactory, ISdkActivityFactory activityFactory) + : IUploaderFactory { public Uploader CreateInstance( string projectId, @@ -18,7 +19,7 @@ public sealed class UploaderFactory(ISpeckleHttp httpClientFactory, ILogger progress, CancellationToken cancellationToken - ) => new(projectId, ingestionId, logger, httpClientFactory, account, progress, cancellationToken); + ) => new(projectId, ingestionId, activityFactory, httpClientFactory, account, progress, cancellationToken); } public sealed class Uploader : IDisposable @@ -28,13 +29,13 @@ public sealed class Uploader : IDisposable private readonly CancellationToken _cancellationToken; private readonly HttpClient _speckleClient; private readonly HttpClient _s3Client; - private readonly ILogger _logger; + private readonly ISdkActivityFactory _activity; private readonly IProgress _progress; internal Uploader( string projectId, string ingestionId, - ILogger logger, + ISdkActivityFactory activity, ISpeckleHttp httpClientFactory, Account speckleAccount, IProgress progress, @@ -43,7 +44,7 @@ public sealed class Uploader : IDisposable { _projectId = projectId; _ingestionId = ingestionId; - _logger = logger; + _activity = activity; _cancellationToken = cancellationToken; _progress = progress; _speckleClient = httpClientFactory.CreateHttpClient(authorizationToken: speckleAccount.token); @@ -62,6 +63,8 @@ public sealed class Uploader : IDisposable private async Task GetPresignedUrl() { + using var a = _activity.Start("Get Presigned Url"); + var signUri = new Uri($"projects/{_projectId}/modelingestion/{_ingestionId}/uploads/sign", UriKind.Relative); using var signResponse = await _speckleClient.PostAsync(signUri, null, _cancellationToken).ConfigureAwait(false); @@ -80,7 +83,7 @@ public sealed class Uploader : IDisposable private async Task UploadToS3(Stream fileStream, PresignedUploadResponse presignedUploadResponse) { - _logger.LogInformation("Uploading file to pre-signed url"); + using var a = _activity.Start("Uploading file to pre-signed url"); Stream progressStream = new ProgressStream(fileStream, _progress); @@ -107,6 +110,8 @@ public sealed class Uploader : IDisposable private async Task TriggerProcessing(TriggerUploadRequest request) { + using var a = _activity.Start("Triggering Processing"); + Uri processUri = new($"projects/{_projectId}/modelingestion/{_ingestionId}/uploads/process", UriKind.Relative); string requestBody = JsonConvert.SerializeObject(request); using var content = new StringContent(requestBody, Encoding.UTF8, "application/json"); From 906ff9c3ff094f49a937c89ca292f32648a866e9 Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Wed, 11 Mar 2026 16:30:24 +0000 Subject: [PATCH 13/16] test: duck tests (#455) * lets get some testing action! * revert url * we'll try these fixes * we'll try this, if it doesn't work I'm deleting the test * try this * Add optional parent id to traces * how about now * and as zero * Fix build * fix code cov * FML subscription tests are so flaky --- .github/workflows/pr.yml | 6 +- .github/workflows/release.yml | 7 -- .../ISdkActivityFactory.cs | 6 +- .../Logging/NullActivityFactory.cs | 2 +- .../Progress/IngestionProgressManager.cs | 22 +++--- .../IngestionProgressManagerFactory.cs | 3 +- .../Progress/RenderedStreamProgress.cs | 4 +- .../Pipelines/Send/SendPipeline.cs | 6 -- .../ServerObjectManagerTests.cs | 8 +- .../Resources/SubscriptionResourceTests.cs | 2 +- .../Progress/IngestionProgressManagerTests.cs | 76 +++++++++++++++++++ .../Speckle.Sdk.Tests.Integration.csproj | 3 + .../Progress/AggregateProgressTests.cs | 21 +++++ .../Pipelines/Progress/ProgressStreamTests.cs | 72 ++++++++++++++++++ .../Progress/RenderedStreamProgressTests.cs | 46 +++++++++++ 15 files changed, 248 insertions(+), 36 deletions(-) create mode 100644 tests/Speckle.Sdk.Tests.Integration/Pipelines/Progress/IngestionProgressManagerTests.cs create mode 100644 tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/AggregateProgressTests.cs create mode 100644 tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/ProgressStreamTests.cs create mode 100644 tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/RenderedStreamProgressTests.cs diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 8068e7c2..637c7da8 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1,7 +1,11 @@ name: PR Test on: - pull_request: + pull_request: {} + push: + branches: + - "main" # Need to run for codecov to compare against the BASE + jobs: build: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 669d0541..9f47df08 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -46,13 +46,6 @@ jobs: SEMVER: ${{ steps.set-version.outputs.SEMVER }} FILE_VERSION: ${{ steps.set-version.outputs.FILE_VERSION }} - - name: Upload coverage reports to Codecov with GitHub Action - uses: codecov/codecov-action@v5 - continue-on-error: true - with: - fail_ci_if_error: true - files: tests/**/coverage.xml - token: ${{ secrets.CODECOV_TOKEN }} - name: NuGet login (OIDC → temp API key) uses: NuGet/login@v1 diff --git a/src/Speckle.Sdk.Dependencies/ISdkActivityFactory.cs b/src/Speckle.Sdk.Dependencies/ISdkActivityFactory.cs index 6db37c9a..2ed61881 100644 --- a/src/Speckle.Sdk.Dependencies/ISdkActivityFactory.cs +++ b/src/Speckle.Sdk.Dependencies/ISdkActivityFactory.cs @@ -4,5 +4,9 @@ namespace Speckle.Sdk.Logging; public interface ISdkActivityFactory : IDisposable { - ISdkActivity? Start(string? name = default, [CallerMemberName] string source = ""); + /// + /// + /// Only need to set if the parent is coming from an external source (e.g.to trace between client and server) + /// + ISdkActivity? Start(string? name = default, [CallerMemberName] string source = "", string? parentId = null); } diff --git a/src/Speckle.Sdk/Logging/NullActivityFactory.cs b/src/Speckle.Sdk/Logging/NullActivityFactory.cs index 580cc5ef..a8aeb098 100644 --- a/src/Speckle.Sdk/Logging/NullActivityFactory.cs +++ b/src/Speckle.Sdk/Logging/NullActivityFactory.cs @@ -4,5 +4,5 @@ public sealed class NullActivityFactory : ISdkActivityFactory { public void Dispose() { } - public ISdkActivity? Start(string? name = default, string source = "") => null; + public ISdkActivity? Start(string? name = default, string source = "", string? parentId = null) => null; } diff --git a/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManager.cs b/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManager.cs index be141c65..8e146ca4 100644 --- a/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManager.cs +++ b/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManager.cs @@ -14,20 +14,20 @@ public partial interface IIngestionProgressManager : IProgress; /// An implementation for the entire client side Ingestion progress update reporting /// Will throttles ingestion progress messages and reports their progress /// +/// +/// Normally we would pick quite a coarse updateInterval to try and spamming the server (1-5s) +/// [GenerateAutoInterface] public sealed class IngestionProgressManager( ILogger logger, IClient speckleClient, ModelIngestion ingestion, - string projectId, TimeSpan updateInterval, CancellationToken cancellationToken ) : IIngestionProgressManager { - /// - /// Normally we would pick quite a coarse throttle window to try and avoid over pressure (1-5s) - /// - private Task? _lastUpdate; + public Task? LastUpdate { get; private set; } + private long _lastUpdatedAt; private readonly object _lock = new(); @@ -48,15 +48,15 @@ public sealed class IngestionProgressManager( trimmedMessage = value.Status.TrimEnd('.'); - _lastUpdate = speckleClient + LastUpdate = speckleClient .Ingestion.UpdateProgress( - new ModelIngestionUpdateInput(ingestion.id, projectId, trimmedMessage, value.Progress), + new ModelIngestionUpdateInput(ingestion.id, ingestion.projectId, trimmedMessage, value.Progress), cancellationToken ) .ContinueWith( - HandleFaultedContinuation, + Continuation, CancellationToken.None, - TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously, + TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default ); } @@ -67,7 +67,7 @@ public sealed class IngestionProgressManager( /// if the update should be ignored, otherwise private bool ShouldIgnoreProgressUpdate() { - if (_lastUpdate is not null && !_lastUpdate.IsCompleted) + if (LastUpdate is not null && !LastUpdate.IsCompleted) { return true; } @@ -76,7 +76,7 @@ public sealed class IngestionProgressManager( return msSinceLastUpdate < updateInterval; } - private void HandleFaultedContinuation(Task updateTask) + private void Continuation(Task updateTask) { // The progress report failed... could be many reasons. // For now, we're not letting this fail the Ingestion in any way diff --git a/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManagerFactory.cs b/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManagerFactory.cs index 998431c2..522aeff0 100644 --- a/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManagerFactory.cs +++ b/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManagerFactory.cs @@ -12,11 +12,10 @@ public sealed class IngestionProgressManagerFactory(ILogger progress) : I ); } - private static readonly string[] s_suffixes = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"]; + private static readonly string[] s_suffixes = ["B", "KB", "MB", "GB", "TB", "PB"]; - private static (string suffix, double scaleFactor) GetFileSizeRendering(long value) + internal static (string suffix, double scaleFactor) GetFileSizeRendering(long value) { if (value <= 0) { diff --git a/src/Speckle.Sdk/Pipelines/Send/SendPipeline.cs b/src/Speckle.Sdk/Pipelines/Send/SendPipeline.cs index e00439ff..3ddd4249 100644 --- a/src/Speckle.Sdk/Pipelines/Send/SendPipeline.cs +++ b/src/Speckle.Sdk/Pipelines/Send/SendPipeline.cs @@ -70,11 +70,5 @@ public sealed class SendPipeline : IDisposable await _uploader.Send(fileStreamUpload).ConfigureAwait(false); } - public async Task WaitForUploadAndServerProcessing() - { - // TODO: in some way, wait for the server to process the upload and return the actual new version id - return await Task.FromResult("todo").ConfigureAwait(false); - } - public void Dispose() => _uploader.Dispose(); } diff --git a/tests/Speckle.Sdk.Serialization.Tests/ServerObjectManagerTests.cs b/tests/Speckle.Sdk.Serialization.Tests/ServerObjectManagerTests.cs index c81e734e..1ba692c9 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/ServerObjectManagerTests.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/ServerObjectManagerTests.cs @@ -52,7 +52,7 @@ public class ServerObjectManagerTests : MoqTest http.Setup(x => x.CreateHttpClient(It.IsAny(), timeout, token)).Returns(httpClient); var activityFactory = Create(); - activityFactory.Setup(x => x.Start(null, "DownloadObjects")).Returns((ISdkActivity?)null); + activityFactory.Setup(x => x.Start(null, "DownloadObjects", null)).Returns((ISdkActivity?)null); var serverObjectManager = new ServerObjectManager( http.Object, @@ -91,7 +91,7 @@ public class ServerObjectManagerTests : MoqTest http.Setup(x => x.CreateHttpClient(It.IsAny(), timeout, token)).Returns(httpClient); var activityFactory = Create(); - activityFactory.Setup(x => x.Start(null, "DownloadSingleObject")).Returns((ISdkActivity?)null); + activityFactory.Setup(x => x.Start(null, "DownloadSingleObject", null)).Returns((ISdkActivity?)null); var serverObjectManager = new ServerObjectManager( http.Object, @@ -132,7 +132,7 @@ public class ServerObjectManagerTests : MoqTest http.Setup(x => x.CreateHttpClient(It.IsAny(), timeout, token)).Returns(httpClient); var activityFactory = Create(); - activityFactory.Setup(x => x.Start(null, "HasObjects")).Returns((ISdkActivity?)null); + activityFactory.Setup(x => x.Start(null, "HasObjects", null)).Returns((ISdkActivity?)null); var serverObjectManager = new ServerObjectManager( http.Object, @@ -171,7 +171,7 @@ public class ServerObjectManagerTests : MoqTest http.Setup(x => x.CreateHttpClient(It.IsAny(), timeout, token)).Returns(httpClient); var activityFactory = Create(); - activityFactory.Setup(x => x.Start(null, "UploadObjects")).Returns((ISdkActivity?)null); + activityFactory.Setup(x => x.Start(null, "UploadObjects", null)).Returns((ISdkActivity?)null); var serverObjectManager = new ServerObjectManager( http.Object, diff --git a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/SubscriptionResourceTests.cs b/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/SubscriptionResourceTests.cs index dff86650..5ce49cb8 100644 --- a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/SubscriptionResourceTests.cs +++ b/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/SubscriptionResourceTests.cs @@ -13,7 +13,7 @@ public class SubscriptionResourceTests : IAsyncLifetime #if DEBUG private const int WAIT_PERIOD = 3000; // WSL is slow AF, so for local runs, we're being extra generous #else - private const int WAIT_PERIOD = 400; // For CI runs, a much smaller wait time is acceptable + private const int WAIT_PERIOD = 600; // For CI runs, a much smaller wait time is acceptable #endif private const int TIMEOUT = WAIT_PERIOD + WAIT_PERIOD + 600; private IClient _testUser; diff --git a/tests/Speckle.Sdk.Tests.Integration/Pipelines/Progress/IngestionProgressManagerTests.cs b/tests/Speckle.Sdk.Tests.Integration/Pipelines/Progress/IngestionProgressManagerTests.cs new file mode 100644 index 00000000..5ca8c8b5 --- /dev/null +++ b/tests/Speckle.Sdk.Tests.Integration/Pipelines/Progress/IngestionProgressManagerTests.cs @@ -0,0 +1,76 @@ +using Microsoft.Extensions.DependencyInjection; +using Speckle.Sdk.Api; +using Speckle.Sdk.Api.GraphQL.Models; +using Speckle.Sdk.Common; +using Speckle.Sdk.Pipelines.Progress; + +namespace Speckle.Sdk.Tests.Integration.Pipelines.Progress; + +[Trait("Server", "Internal")] +public class IngestionProgressManagerTests : IAsyncLifetime +{ + private IIngestionProgressManagerFactory _factory; + private IClient _client; + private Project _project; + private Model _model; + private ModelIngestion _ingestion; + + public async Task InitializeAsync() + { + var serviceProvider = TestServiceSetup.GetServiceProvider(); + _factory = serviceProvider.GetRequiredService(); + + _client = await Fixtures.SeedUserWithClient(); + _project = await _client.Project.Create(new("test", null, default)); + _model = await _client.Model.Create(new("test", null, _project.id)); + _ingestion = await _client.Ingestion.Create( + new(_model.id, _project.id, "Testing ingestion", new("integrationTests", "0.0.0", null, null)) + ); + } + + [Fact] + public async Task TestProgress_NoThrottle() + { + var sut = _factory.CreateInstance(_client, _ingestion, TimeSpan.Zero, CancellationToken.None); + const string FIRST_MESSAGE = "This is a test 123"; + const string SECOND_MESSAGE = "This is another test 321"; + + // first message (should go through) + sut.Report(new CardProgress(FIRST_MESSAGE, 0.123123123d)); + await sut.LastUpdate.NotNull(); + var res = await _client.Ingestion.Get(_ingestion.id, _project.id, CancellationToken.None); + + Assert.Equal(FIRST_MESSAGE, res.statusData.progressMessage); + + // second message (should also go through) + sut.Report(new CardProgress(SECOND_MESSAGE, 0.321321321d)); + await sut.LastUpdate.NotNull(); + res = await _client.Ingestion.Get(_ingestion.id, _project.id, CancellationToken.None); + + Assert.Equal(SECOND_MESSAGE, res.statusData.progressMessage); + } + + [Fact] + public async Task TestProgress_WithThrottle() + { + var sut = _factory.CreateInstance(_client, _ingestion, TimeSpan.FromMilliseconds(500), CancellationToken.None); + const string EXPECTED_MESSAGE = "First message should go through 123"; + + await Task.Delay(TimeSpan.FromMilliseconds(600)); + + // first message (should go through) + sut.Report(new CardProgress(EXPECTED_MESSAGE, 0.123123123d)); + // second message (should be dropped) + sut.Report(new CardProgress("Second message, should be dropped", 0.321321321d)); + await sut.LastUpdate.NotNull(); + var res = await _client.Ingestion.Get(_ingestion.id, _project.id, CancellationToken.None); + + Assert.Equal(EXPECTED_MESSAGE, res.statusData.progressMessage); + } + + public Task DisposeAsync() + { + _client.Dispose(); + return Task.CompletedTask; + } +} diff --git a/tests/Speckle.Sdk.Tests.Integration/Speckle.Sdk.Tests.Integration.csproj b/tests/Speckle.Sdk.Tests.Integration/Speckle.Sdk.Tests.Integration.csproj index 778355be..dfd7fa55 100644 --- a/tests/Speckle.Sdk.Tests.Integration/Speckle.Sdk.Tests.Integration.csproj +++ b/tests/Speckle.Sdk.Tests.Integration/Speckle.Sdk.Tests.Integration.csproj @@ -16,4 +16,7 @@ + + + diff --git a/tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/AggregateProgressTests.cs b/tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/AggregateProgressTests.cs new file mode 100644 index 00000000..50b0b878 --- /dev/null +++ b/tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/AggregateProgressTests.cs @@ -0,0 +1,21 @@ +using Moq; +using Speckle.Sdk.Pipelines.Progress; + +namespace Speckle.Sdk.Tests.Unit.Pipelines.Progress; + +public class AggregateProgressTests +{ + [Fact] + public void Report_InvokesReportOnAllInnerProgresses() + { + var mock1 = new Mock>(); + var mock2 = new Mock>(); + const int TEST_VALUE = 42; + var target = new AggregateProgress(mock1.Object, mock2.Object); + + target.Report(TEST_VALUE); + + mock1.Verify(x => x.Report(TEST_VALUE), Times.Once); + mock2.Verify(x => x.Report(TEST_VALUE), Times.Once); + } +} diff --git a/tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/ProgressStreamTests.cs b/tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/ProgressStreamTests.cs new file mode 100644 index 00000000..8563341c --- /dev/null +++ b/tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/ProgressStreamTests.cs @@ -0,0 +1,72 @@ +using System.Diagnostics.CodeAnalysis; +using Moq; +using Speckle.Sdk.Pipelines.Progress; + +namespace Speckle.Sdk.Tests.Unit.Pipelines.Progress; + +[SuppressMessage( + "Performance", + "CA1835:Prefer the \'Memory\'-based overloads for \'ReadAsync\' and \'WriteAsync\'", + Justification = "Need to test it" +)] +public class ProgressStreamTests : IDisposable +{ + private readonly Mock _innerStreamMock; + private readonly Mock> _progressMock; + private readonly ProgressStream _sut; + + public ProgressStreamTests() + { + // Setup the mocks + _innerStreamMock = new Mock(); + _innerStreamMock.Setup(s => s.Length).Returns(1024L); + + _progressMock = new Mock>(); + + // Inject mocks into the System Under Test + _sut = new ProgressStream(_innerStreamMock.Object, _progressMock.Object); + } + + [Fact] + public async Task ReadAsync_Should_CallInnerStreamAndReportProgress() + { + // Arrange + var buffer = new byte[10]; + _innerStreamMock + .Setup(s => s.ReadAsync(buffer, 0, buffer.Length, CancellationToken.None)) + .Returns(Task.FromResult(5)); + + // Act + await _sut.ReadAsync(buffer, 0, buffer.Length); + + // Assert - Inner Stream Read was called + _innerStreamMock.Verify(s => s.ReadAsync(buffer, 0, buffer.Length, CancellationToken.None), Times.Once); + + // Assert - Progress Report was called with the correct byte count + _progressMock.Verify(p => p.Report(It.IsAny()), Times.Once); + } + + [Fact] + public async Task WriteAsync_Should_CallInnerStreamAndReportProgress() + { + // Arrange + var buffer = new byte[10]; + _innerStreamMock + .Setup(s => s.WriteAsync(buffer, 0, buffer.Length, CancellationToken.None)) + .Returns(Task.FromResult(5)); + + // Act + await _sut.WriteAsync(buffer, 0, buffer.Length); + + // Assert - Inner Stream Write was called + _innerStreamMock.Verify(s => s.WriteAsync(buffer, 0, buffer.Length, CancellationToken.None), Times.Once); + + // Assert - Progress Report was called with the correct byte count + _progressMock.Verify(p => p.Report(It.IsAny()), Times.Once); + } + + public void Dispose() + { + _sut.Dispose(); + } +} diff --git a/tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/RenderedStreamProgressTests.cs b/tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/RenderedStreamProgressTests.cs new file mode 100644 index 00000000..33007f6c --- /dev/null +++ b/tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/RenderedStreamProgressTests.cs @@ -0,0 +1,46 @@ +using Speckle.Sdk.Pipelines.Progress; + +namespace Speckle.Sdk.Tests.Unit.Pipelines.Progress; + +public class RenderedStreamProgressTests +{ + [Theory] + [InlineData(1, "B", 1.0)] + [InlineData(1024, "B", 1.0)] + [InlineData(1024 + 1, "KB", 1.0 / 1024)] + [InlineData(1024 * 1024, "KB", 1.0 / 1024)] + [InlineData(1024 * 1024 + 1, "MB", 1.0 / (1024 * 1024))] + [InlineData(1024 * 1024 * 1024, "MB", 1.0 / (1024 * 1024))] + [InlineData(1024 * 1024 * 1024 + 1, "GB", 1.0 / (1024 * 1024 * 1024))] + [InlineData(1024L * 1024L * 1024L * 1024L, "GB", 1.0 / (1024L * 1024L * 1024L))] + public void GetFileSizeRendering_WithPositiveValue_ReturnsCorrectSuffix( + long value, + string expectedSuffix, + double expectedScaleFactor + ) + { + var result = RenderedStreamProgress.GetFileSizeRendering(value); + + Assert.Equal(expectedSuffix, result.suffix); + Assert.Equal(expectedScaleFactor, result.scaleFactor); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(-1000)] + public void GetFileSizeRendering_WithNonPositiveValue_ReturnsBytesSuffix(long value) + { + var result = RenderedStreamProgress.GetFileSizeRendering(value); + + Assert.Equal("B", result.suffix); + Assert.Equal(1d, result.scaleFactor); + } + + [Theory] + [InlineData(long.MaxValue)] + public void GetFileSizeRendering_WithVeryLargeValue_ThrowsArgumentOutOfRangeException(long value) + { + Assert.Throws(() => RenderedStreamProgress.GetFileSizeRendering(value)); + } +} From a2c99a537a43184020fde52ccfd46da71399e8fc Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Thu, 12 Mar 2026 12:52:10 +0000 Subject: [PATCH 14/16] feat(otel): Add proper support for remote spans (#458) * Add proper support for remote spans * Fix tests --- src/Speckle.Sdk.Dependencies/ISdkActivity.cs | 1 + src/Speckle.Sdk.Dependencies/ISdkActivityFactory.cs | 9 +++++++-- src/Speckle.Sdk/Logging/NullActivityFactory.cs | 5 ++++- .../ServerObjectManagerTests.cs | 8 ++++---- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/Speckle.Sdk.Dependencies/ISdkActivity.cs b/src/Speckle.Sdk.Dependencies/ISdkActivity.cs index d832c769..0b3d1cea 100644 --- a/src/Speckle.Sdk.Dependencies/ISdkActivity.cs +++ b/src/Speckle.Sdk.Dependencies/ISdkActivity.cs @@ -5,6 +5,7 @@ public interface ISdkActivity : IDisposable void SetTag(string key, object? value); void RecordException(Exception e); string TraceId { get; } + string SpanId { get; } void SetStatus(SdkActivityStatusCode code); void InjectHeaders(Action header); diff --git a/src/Speckle.Sdk.Dependencies/ISdkActivityFactory.cs b/src/Speckle.Sdk.Dependencies/ISdkActivityFactory.cs index 2ed61881..ce69c387 100644 --- a/src/Speckle.Sdk.Dependencies/ISdkActivityFactory.cs +++ b/src/Speckle.Sdk.Dependencies/ISdkActivityFactory.cs @@ -6,7 +6,12 @@ public interface ISdkActivityFactory : IDisposable { /// /// - /// Only need to set if the parent is coming from an external source (e.g.to trace between client and server) /// - ISdkActivity? Start(string? name = default, [CallerMemberName] string source = "", string? parentId = null); + ISdkActivity? Start(string? name = default, [CallerMemberName] string source = ""); + ISdkActivity? StartRemote( + string traceId, + string parentSpanId, + string? name = default, + [CallerMemberName] string source = "" + ); } diff --git a/src/Speckle.Sdk/Logging/NullActivityFactory.cs b/src/Speckle.Sdk/Logging/NullActivityFactory.cs index a8aeb098..e761a9bd 100644 --- a/src/Speckle.Sdk/Logging/NullActivityFactory.cs +++ b/src/Speckle.Sdk/Logging/NullActivityFactory.cs @@ -4,5 +4,8 @@ public sealed class NullActivityFactory : ISdkActivityFactory { public void Dispose() { } - public ISdkActivity? Start(string? name = default, string source = "", string? parentId = null) => null; + public ISdkActivity? Start(string? name = default, string source = "") => null; + + public ISdkActivity? StartRemote(string traceId, string parentSpanId, string? name = default, string source = "") => + null; } diff --git a/tests/Speckle.Sdk.Serialization.Tests/ServerObjectManagerTests.cs b/tests/Speckle.Sdk.Serialization.Tests/ServerObjectManagerTests.cs index 1ba692c9..c81e734e 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/ServerObjectManagerTests.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/ServerObjectManagerTests.cs @@ -52,7 +52,7 @@ public class ServerObjectManagerTests : MoqTest http.Setup(x => x.CreateHttpClient(It.IsAny(), timeout, token)).Returns(httpClient); var activityFactory = Create(); - activityFactory.Setup(x => x.Start(null, "DownloadObjects", null)).Returns((ISdkActivity?)null); + activityFactory.Setup(x => x.Start(null, "DownloadObjects")).Returns((ISdkActivity?)null); var serverObjectManager = new ServerObjectManager( http.Object, @@ -91,7 +91,7 @@ public class ServerObjectManagerTests : MoqTest http.Setup(x => x.CreateHttpClient(It.IsAny(), timeout, token)).Returns(httpClient); var activityFactory = Create(); - activityFactory.Setup(x => x.Start(null, "DownloadSingleObject", null)).Returns((ISdkActivity?)null); + activityFactory.Setup(x => x.Start(null, "DownloadSingleObject")).Returns((ISdkActivity?)null); var serverObjectManager = new ServerObjectManager( http.Object, @@ -132,7 +132,7 @@ public class ServerObjectManagerTests : MoqTest http.Setup(x => x.CreateHttpClient(It.IsAny(), timeout, token)).Returns(httpClient); var activityFactory = Create(); - activityFactory.Setup(x => x.Start(null, "HasObjects", null)).Returns((ISdkActivity?)null); + activityFactory.Setup(x => x.Start(null, "HasObjects")).Returns((ISdkActivity?)null); var serverObjectManager = new ServerObjectManager( http.Object, @@ -171,7 +171,7 @@ public class ServerObjectManagerTests : MoqTest http.Setup(x => x.CreateHttpClient(It.IsAny(), timeout, token)).Returns(httpClient); var activityFactory = Create(); - activityFactory.Setup(x => x.Start(null, "UploadObjects", null)).Returns((ISdkActivity?)null); + activityFactory.Setup(x => x.Start(null, "UploadObjects")).Returns((ISdkActivity?)null); var serverObjectManager = new ServerObjectManager( http.Object, From 70acc06f37ab9fe6d7e51075521cfc85432fecf0 Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Mon, 16 Mar 2026 15:32:33 +0000 Subject: [PATCH 15/16] feat(otel): Change sig for remote (#460) * Remote changes * Fix tests * Fix tests * ditto --- .../ISdkActivityFactory.cs | 17 ++++++----- .../SpeckleActivityKind.cs | 30 +++++++++++++++++++ .../Logging/NullActivityFactory.cs | 9 +++--- .../ServerObjectManagerTests.cs | 8 ++--- ...esourceTests.TestGetProjects.verified.json | 8 ----- ...sourceTests.TestGetWorkspace.verified.json | 8 ----- .../Resources/WorkspaceResourceTests.cs | 6 ++-- 7 files changed, 53 insertions(+), 33 deletions(-) create mode 100644 src/Speckle.Sdk.Dependencies/SpeckleActivityKind.cs delete mode 100644 tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.TestGetProjects.verified.json delete mode 100644 tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.TestGetWorkspace.verified.json diff --git a/src/Speckle.Sdk.Dependencies/ISdkActivityFactory.cs b/src/Speckle.Sdk.Dependencies/ISdkActivityFactory.cs index ce69c387..927d0f70 100644 --- a/src/Speckle.Sdk.Dependencies/ISdkActivityFactory.cs +++ b/src/Speckle.Sdk.Dependencies/ISdkActivityFactory.cs @@ -1,17 +1,20 @@ using System.Runtime.CompilerServices; +using Speckle.Connectors.Logging; namespace Speckle.Sdk.Logging; public interface ISdkActivityFactory : IDisposable { - /// - /// - /// - ISdkActivity? Start(string? name = default, [CallerMemberName] string source = ""); + ISdkActivity? Start( + string? name = null, + SdkActivityKind kind = SdkActivityKind.Internal, + [CallerMemberName] string source = "" + ); + ISdkActivity? StartRemote( - string traceId, - string parentSpanId, - string? name = default, + string traceContext, + SdkActivityKind kind, + string? name = null, [CallerMemberName] string source = "" ); } diff --git a/src/Speckle.Sdk.Dependencies/SpeckleActivityKind.cs b/src/Speckle.Sdk.Dependencies/SpeckleActivityKind.cs new file mode 100644 index 00000000..8ca93543 --- /dev/null +++ b/src/Speckle.Sdk.Dependencies/SpeckleActivityKind.cs @@ -0,0 +1,30 @@ +namespace Speckle.Connectors.Logging; + +public enum SdkActivityKind +{ + /// + /// Default value. + /// Indicates that the Activity represents an internal operation within an application, as opposed to an operations with remote parents or children. + /// + Internal = 0, + + /// + /// Server activity represents request incoming from external component. + /// + Server = 1, + + /// + /// Client activity represents outgoing request to the external component. + /// + Client = 2, + + /// + /// Producer activity represents output provided to external components. + /// + Producer = 3, + + /// + /// Consumer activity represents output received from an external component. + /// + Consumer = 4, +} diff --git a/src/Speckle.Sdk/Logging/NullActivityFactory.cs b/src/Speckle.Sdk/Logging/NullActivityFactory.cs index e761a9bd..039f91b8 100644 --- a/src/Speckle.Sdk/Logging/NullActivityFactory.cs +++ b/src/Speckle.Sdk/Logging/NullActivityFactory.cs @@ -1,11 +1,12 @@ -namespace Speckle.Sdk.Logging; +using Speckle.Connectors.Logging; + +namespace Speckle.Sdk.Logging; public sealed class NullActivityFactory : ISdkActivityFactory { public void Dispose() { } - public ISdkActivity? Start(string? name = default, string source = "") => null; + public ISdkActivity? Start(string? name, SdkActivityKind kind, string source) => null; - public ISdkActivity? StartRemote(string traceId, string parentSpanId, string? name = default, string source = "") => - null; + public ISdkActivity? StartRemote(string traceContext, SdkActivityKind kind, string? name, string source) => null; } diff --git a/tests/Speckle.Sdk.Serialization.Tests/ServerObjectManagerTests.cs b/tests/Speckle.Sdk.Serialization.Tests/ServerObjectManagerTests.cs index c81e734e..90455343 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/ServerObjectManagerTests.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/ServerObjectManagerTests.cs @@ -52,7 +52,7 @@ public class ServerObjectManagerTests : MoqTest http.Setup(x => x.CreateHttpClient(It.IsAny(), timeout, token)).Returns(httpClient); var activityFactory = Create(); - activityFactory.Setup(x => x.Start(null, "DownloadObjects")).Returns((ISdkActivity?)null); + activityFactory.Setup(x => x.Start(null, default, "DownloadObjects")).Returns((ISdkActivity?)null); var serverObjectManager = new ServerObjectManager( http.Object, @@ -91,7 +91,7 @@ public class ServerObjectManagerTests : MoqTest http.Setup(x => x.CreateHttpClient(It.IsAny(), timeout, token)).Returns(httpClient); var activityFactory = Create(); - activityFactory.Setup(x => x.Start(null, "DownloadSingleObject")).Returns((ISdkActivity?)null); + activityFactory.Setup(x => x.Start(null, default, "DownloadSingleObject")).Returns((ISdkActivity?)null); var serverObjectManager = new ServerObjectManager( http.Object, @@ -132,7 +132,7 @@ public class ServerObjectManagerTests : MoqTest http.Setup(x => x.CreateHttpClient(It.IsAny(), timeout, token)).Returns(httpClient); var activityFactory = Create(); - activityFactory.Setup(x => x.Start(null, "HasObjects")).Returns((ISdkActivity?)null); + activityFactory.Setup(x => x.Start(null, default, "HasObjects")).Returns((ISdkActivity?)null); var serverObjectManager = new ServerObjectManager( http.Object, @@ -171,7 +171,7 @@ public class ServerObjectManagerTests : MoqTest http.Setup(x => x.CreateHttpClient(It.IsAny(), timeout, token)).Returns(httpClient); var activityFactory = Create(); - activityFactory.Setup(x => x.Start(null, "UploadObjects")).Returns((ISdkActivity?)null); + activityFactory.Setup(x => x.Start(null, default, "UploadObjects")).Returns((ISdkActivity?)null); var serverObjectManager = new ServerObjectManager( http.Object, diff --git a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.TestGetProjects.verified.json b/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.TestGetProjects.verified.json deleted file mode 100644 index 781efcaf..00000000 --- a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.TestGetProjects.verified.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "Type": "AggregateException", - "InnerException": { - "Data": {}, - "Message": "FORBIDDEN: Your auth token does not have the required scope: workspace:read.", - "Type": "SpeckleGraphQLForbiddenException" - } -} diff --git a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.TestGetWorkspace.verified.json b/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.TestGetWorkspace.verified.json deleted file mode 100644 index 781efcaf..00000000 --- a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.TestGetWorkspace.verified.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "Type": "AggregateException", - "InnerException": { - "Data": {}, - "Message": "FORBIDDEN: Your auth token does not have the required scope: workspace:read.", - "Type": "SpeckleGraphQLForbiddenException" - } -} diff --git a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.cs b/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.cs index 8b73b43d..d597472d 100644 --- a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.cs +++ b/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.cs @@ -25,13 +25,15 @@ public class WorkspaceResourceTests public async Task TestGetWorkspace() { var ex = await Assert.ThrowsAsync(async () => _ = await Sut.Get("non-existent-id")); - await Verify(ex); + Assert.Single(ex.InnerExceptions); + Assert.All(ex.InnerExceptions, item => Assert.IsType(item)); } [Fact] public async Task TestGetProjects() { var ex = await Assert.ThrowsAsync(async () => _ = await Sut.GetProjects("non-existent-id")); - await Verify(ex); + Assert.Single(ex.InnerExceptions); + Assert.All(ex.InnerExceptions, item => Assert.IsType(item)); } } From edbc884d74c0838de8982d014694393acfc96cdb Mon Sep 17 00:00:00 2001 From: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com> Date: Mon, 23 Mar 2026 10:45:57 +0000 Subject: [PATCH 16/16] Clean diff --- src/Speckle.Sdk/Models/DynamicBaseMemberType.cs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/Speckle.Sdk/Models/DynamicBaseMemberType.cs b/src/Speckle.Sdk/Models/DynamicBaseMemberType.cs index b03c2579..4522b61e 100644 --- a/src/Speckle.Sdk/Models/DynamicBaseMemberType.cs +++ b/src/Speckle.Sdk/Models/DynamicBaseMemberType.cs @@ -7,32 +7,33 @@ namespace Speckle.Sdk.Models; public enum DynamicBaseMemberType { /// - /// The typed members of the DynamicBase object + /// The typed members of the object /// Instance = 1, /// - /// The dynamically added members of the DynamicBase object + /// The dynamically added members of the object /// Dynamic = 2, /// - /// The typed members flagged with ObsoleteAttribute attribute. + /// The typed members flagged with attribute. /// Obsolete = 4, /// - /// The typed methods flagged with TODO: + /// Old feature supported in v2 for grasshopper /// + [Obsolete("Feature no longer supported")] SchemaComputed = 16, /// - /// All the typed members, including ones with ObsoleteAttribute attributes. + /// All the typed members, including ones with attributes. /// InstanceAll = Instance + Obsolete, /// - /// All the members, including dynamic and instance members flagged with ObsoleteAttribute attributes + /// All the members, including dynamic and instance members flagged with attributes /// All = InstanceAll + Dynamic, }