Merge pull request #439 from specklesystems/duckdev

feat(packfile): duckdb packfile api (duckdev -> main)

.github/workflows/pr.yml

@@ -1,7 +1,11 @@
 name: PR Test
 
 on:
-  pull_request:
+  pull_request: {}
+  push: 
+    branches:
+      - "main" # Need to run for codecov to compare against the BASE
+
 
 jobs:
   build:

.github/workflows/release.yml

@@ -46,13 +46,6 @@ jobs:
           SEMVER: ${{ steps.set-version.outputs.SEMVER }}
           FILE_VERSION: ${{ steps.set-version.outputs.FILE_VERSION }}
 
-      - name: Upload coverage reports to Codecov with GitHub Action
-        uses: codecov/codecov-action@v5
-        continue-on-error: true
-        with:
-          fail_ci_if_error: true
-          files: tests/**/coverage.xml
-          token: ${{ secrets.CODECOV_TOKEN }}
 
       - name: NuGet login (OIDC → temp API key)
         uses: NuGet/login@v1

Directory.Packages.props

@@ -15,8 +15,8 @@
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.13.0" />
     <PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="[2.2.0,)" />
     <PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="[2.2.0,)" />
+    <PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="[9.0.4,)" />
     <PackageVersion Include="Microsoft.Extensions.Logging" Version="[2.2.0,)" />
-    <PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="[5.0.0,)" />
     <PackageVersion Include="Moq" Version="4.20.72" />
     <PackageVersion Include="Newtonsoft.Json.Schema" Version="4.0.1" />
     <PackageVersion Include="Open.ChannelExtensions" Version="9.1.0" />

src/Speckle.Automate.Sdk/packages.lock.json

@@ -281,7 +281,6 @@
         "type": "Project",
         "dependencies": {
           "GraphQL.Client": "[6.0.0, )",
-          "Microsoft.Bcl.AsyncInterfaces": "[5.0.0, )",
           "Microsoft.CSharp": "[4.7.0, )",
           "Microsoft.Data.Sqlite": "[7.0.5, )",
           "Microsoft.Extensions.DependencyInjection.Abstractions": "[2.2.0, )",
@@ -292,7 +291,10 @@
         }
       },
       "speckle.sdk.dependencies": {
-        "type": "Project"
+        "type": "Project",
+        "dependencies": {
+          "Microsoft.Bcl.AsyncInterfaces": "[9.0.4, )"
+        }
       },
       "GraphQL.Client": {
         "type": "CentralTransitive",
@@ -307,9 +309,9 @@
       },
       "Microsoft.Bcl.AsyncInterfaces": {
         "type": "CentralTransitive",
-        "requested": "[5.0.0, )",
-        "resolved": "8.0.0",
-        "contentHash": "3WA9q9yVqJp222P3x1wYIGDAkpjAku0TMUaaQV22g6L67AI0LdOIrVS7Ht2vJfLHGSPVuqN94vIr15qn+HEkHw==",
+        "requested": "[9.0.4, )",
+        "resolved": "9.0.4",
+        "contentHash": "9VGI5kxIvrNG2mqLQZnUR6y/3fcnygD8eNpHR+CqfbnIXvea6nehnYknDKQTxZVPMpzpNca+7DxLBmpdB3q0Bw==",
         "dependencies": {
           "System.Threading.Tasks.Extensions": "4.5.4"
         }

src/Speckle.Objects/packages.lock.json

@@ -228,7 +228,6 @@
         "type": "Project",
         "dependencies": {
           "GraphQL.Client": "[6.0.0, )",
-          "Microsoft.Bcl.AsyncInterfaces": "[5.0.0, )",
           "Microsoft.CSharp": "[4.7.0, )",
           "Microsoft.Data.Sqlite": "[7.0.5, )",
           "Microsoft.Extensions.DependencyInjection.Abstractions": "[2.2.0, )",
@@ -239,7 +238,10 @@
         }
       },
       "speckle.sdk.dependencies": {
-        "type": "Project"
+        "type": "Project",
+        "dependencies": {
+          "Microsoft.Bcl.AsyncInterfaces": "[9.0.4, )"
+        }
       },
       "GraphQL.Client": {
         "type": "CentralTransitive",
@@ -254,9 +256,9 @@
       },
       "Microsoft.Bcl.AsyncInterfaces": {
         "type": "CentralTransitive",
-        "requested": "[5.0.0, )",
-        "resolved": "5.0.0",
-        "contentHash": "W8DPQjkMScOMTtJbPwmPyj9c3zYSFGawDW3jwlBOOsnY+EzZFLgNQ/UMkK35JmkNOVPdCyPr2Tw7Vv9N+KA3ZQ==",
+        "requested": "[9.0.4, )",
+        "resolved": "9.0.4",
+        "contentHash": "9VGI5kxIvrNG2mqLQZnUR6y/3fcnygD8eNpHR+CqfbnIXvea6nehnYknDKQTxZVPMpzpNca+7DxLBmpdB3q0Bw==",
         "dependencies": {
           "System.Threading.Tasks.Extensions": "4.5.4"
         }

src/Speckle.Sdk.Dependencies/ISdkActivity.cs

@@ -5,6 +5,7 @@ public interface ISdkActivity : IDisposable
   void SetTag(string key, object? value);
   void RecordException(Exception e);
   string TraceId { get; }
+  string SpanId { get; }
   void SetStatus(SdkActivityStatusCode code);
 
   void InjectHeaders(Action<string, string> header);

src/Speckle.Sdk.Dependencies/ISdkActivityFactory.cs

@@ -1,8 +1,20 @@
 using System.Runtime.CompilerServices;
+using Speckle.Connectors.Logging;
 
 namespace Speckle.Sdk.Logging;
 
 public interface ISdkActivityFactory : IDisposable
 {
-  ISdkActivity? Start(string? name = default, [CallerMemberName] string source = "");
+  ISdkActivity? Start(
+    string? name = null,
+    SdkActivityKind kind = SdkActivityKind.Internal,
+    [CallerMemberName] string source = ""
+  );
+
+  ISdkActivity? StartRemote(
+    string traceContext,
+    SdkActivityKind kind,
+    string? name = null,
+    [CallerMemberName] string source = ""
+  );
 }

src/Speckle.Sdk.Dependencies/RepackedChannel.cs

@@ -0,0 +1,49 @@
+using System.Threading.Channels;
+
+namespace Speckle.Sdk.Dependencies;
+
+/// <summary>
+/// For various reasons related to our use of ILRepack.FullAuto,
+/// we cannot use Channels from the SDK project.
+/// We have to keep usage of it inside the Sdk.Dependencies project.
+///
+/// For the sake of quick development, I've wrapped the <see cref="Channel"/> class here in a type
+/// that is safe to use from the SDK project.
+///
+/// As and when we need more functions, we can add them here.
+///
+/// And yes... I'm not very happy about the way we've set this up
+/// </summary>
+/// <typeparam name="T"></typeparam>
+public sealed class RepackedChannel<T>
+{
+  private readonly Channel<T> _channel;
+
+  public RepackedChannel(int capacity, bool singleReader, bool singleWriter)
+  {
+    _channel = Channel.CreateBounded<T>(
+      new BoundedChannelOptions(capacity)
+      {
+        FullMode = BoundedChannelFullMode.Wait,
+        SingleReader = singleReader,
+        SingleWriter = singleWriter,
+      }
+    );
+  }
+
+  public void CompleteWriter() => _channel.Writer.Complete();
+
+  public ValueTask WriteAsync(T item, CancellationToken cancellationToken) =>
+    _channel.Writer.WriteAsync(item, cancellationToken);
+
+  public IAsyncEnumerable<T> ReadAllAsync(CancellationToken cancellationToken) =>
+    _channel.Reader.ReadAllAsync(cancellationToken);
+
+  // public async Task ReadAllAsync(Func<T, Task> callback, CancellationToken cancellationToken)
+  // {
+  //   await foreach (T item in _channel.Reader.ReadAllAsync(cancellationToken))
+  //   {
+  //     await callback.Invoke(item).ConfigureAwait(false);
+  //   }
+  // }
+}

src/Speckle.Sdk.Dependencies/Speckle.Sdk.Dependencies.csproj

@@ -28,4 +28,36 @@
     <PackageReference Include="Open.ChannelExtensions" PrivateAssets="all" />
     <PackageReference Include="System.Threading.Channels" PrivateAssets="all" />
   </ItemGroup>
+  <ItemGroup Condition=" '$(TargetFramework)' == 'netstandard2.0'">
+    <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" />
+  </ItemGroup>
+  <Target Name="BeforeILRepackPrepareBuild" BeforeTargets="ILRepackPrepareBuild">
+    <ItemGroup>
+      <!--
+      We're Being selective about which assemblies we're il-repacking
+      Avoiding repacling `Microsoft.Bcl.AsyncInterfaces.dll` because we need types like `ValueTask` and `IAsyncEnumerable` to be external
+      
+      Yes, this does beg the question, why are we using `IlRepack.FullAuto` instead of raw ILRepack. Well the truth is, I'd like to move away from FullAuto
+      since it's unmaintaned and is lagging behind ILRepack version.
+      -->
+      <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)System.Numerics.Vectors.dll" />
+      <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)System.Runtime.CompilerServices.Unsafe.dll" />
+      <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)System.Memory.dll" />
+      <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)Open.ChannelExtensions.dll" />
+      <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)System.Threading.Channels.dll" />
+      <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)System.Collections.Immutable.dll" />
+      <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)Polly.dll" />
+      <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)Polly.Contrib.WaitAndRetry.dll" />
+      <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)Polly.Extensions.Http.dll" />
+      <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)Microsoft.Extensions.ObjectPool.dll" />
+      <_ILRepackExcludeAssemblies_Items Include="$(OutputPath)*.dll" Exclude="@(_ILRepackIncludeAssemblies_Items)" />
+    </ItemGroup>
+    <Message
+      Text="These are the packages we are NOT ilrepacking '@(_ILRepackExcludeAssemblies_Items)'"
+      Importance="high"
+    />
+    <PropertyGroup>
+      <ILRepackExcludeAssemblies>@(_ILRepackExcludeAssemblies_Items)</ILRepackExcludeAssemblies>
+    </PropertyGroup>
+  </Target>
 </Project>

src/Speckle.Sdk.Dependencies/SpeckleActivityKind.cs

@@ -0,0 +1,30 @@
+namespace Speckle.Connectors.Logging;
+
+public enum SdkActivityKind
+{
+  /// <summary>
+  /// Default value.
+  /// Indicates that the Activity represents an internal operation within an application, as opposed to an operations with remote parents or children.
+  /// </summary>
+  Internal = 0,
+
+  /// <summary>
+  /// Server activity represents request incoming from external component.
+  /// </summary>
+  Server = 1,
+
+  /// <summary>
+  /// Client activity represents outgoing request to the external component.
+  /// </summary>
+  Client = 2,
+
+  /// <summary>
+  /// Producer activity represents output provided to external components.
+  /// </summary>
+  Producer = 3,
+
+  /// <summary>
+  /// Consumer activity represents output received from an external component.
+  /// </summary>
+  Consumer = 4,
+}

src/Speckle.Sdk.Dependencies/packages.lock.json

@@ -11,6 +11,15 @@
           "ILRepack": "2.0.33"
         }
       },
+      "Microsoft.Bcl.AsyncInterfaces": {
+        "type": "Direct",
+        "requested": "[9.0.4, )",
+        "resolved": "9.0.4",
+        "contentHash": "9VGI5kxIvrNG2mqLQZnUR6y/3fcnygD8eNpHR+CqfbnIXvea6nehnYknDKQTxZVPMpzpNca+7DxLBmpdB3q0Bw==",
+        "dependencies": {
+          "System.Threading.Tasks.Extensions": "4.5.4"
+        }
+      },
       "Microsoft.Extensions.ObjectPool": {
         "type": "Direct",
         "requested": "[9.0.4, )",
@@ -151,15 +160,6 @@
         "dependencies": {
           "System.Runtime.CompilerServices.Unsafe": "4.5.3"
         }
-      },
-      "Microsoft.Bcl.AsyncInterfaces": {
-        "type": "CentralTransitive",
-        "requested": "[5.0.0, )",
-        "resolved": "9.0.4",
-        "contentHash": "9VGI5kxIvrNG2mqLQZnUR6y/3fcnygD8eNpHR+CqfbnIXvea6nehnYknDKQTxZVPMpzpNca+7DxLBmpdB3q0Bw==",
-        "dependencies": {
-          "System.Threading.Tasks.Extensions": "4.5.4"
-        }
       }
     },
     "net8.0": {

src/Speckle.Sdk/Api/Blob/BlobApi.cs

@@ -193,30 +193,7 @@ public sealed class BlobApi : IBlobApi
     using var response = await _unauthedClient.SendAsync(requestMessage, cancellationToken).ConfigureAwait(false);
     response.EnsureSuccessStatusCode();
 
-    return ParseEtagHeader(response.Headers);
-  }
-
-  private static string ParseEtagHeader(HttpResponseHeaders headers)
-  {
-    if (!headers.TryGetValues("ETag", out var etagValues))
-    {
-      throw new ArgumentException(
-        "Response does not have an ETag attached to it, cannot use this as an upload",
-        nameof(headers)
-      );
-    }
-
-    var etagValuesArray = etagValues.ToArray();
-
-    if (etagValuesArray.Length != 1)
-    {
-      throw new ArgumentException(
-        $"Expected Etag header to have a single value but got {etagValuesArray.Length}",
-        nameof(headers)
-      );
-    }
-
-    return etagValuesArray[0];
+    return BlobApiHelpers.ParseEtagHeader(response.Headers);
   }
 
   /// <summary>

src/Speckle.Sdk/Api/GraphQL/Inputs/ModelIngestionInputs.cs

@@ -14,7 +14,8 @@ public record ModelIngestionCreateInput(
   string modelId,
   string projectId,
   string progressMessage,
-  SourceDataInput sourceData
+  SourceDataInput sourceData,
+  int? maxIdleTimeoutSeconds = null
 );
 
 public record ModelIngestionUpdateInput(string ingestionId, string projectId, string progressMessage, double? progress);

src/Speckle.Sdk/Api/GraphQL/Models/ModelIngestion.cs

@@ -6,7 +6,8 @@ public sealed class ModelIngestion
   public required DateTime createdAt { get; init; }
   public required DateTime updatedAt { get; init; }
   public required string modelId { get; init; }
+  public required string projectId { get; init; }
+  public required string userId { get; init; }
   public required bool cancellationRequested { get; init; }
   public required ModelIngestionStatusData statusData { get; init; }
-  // public required LimitedUser user { get; init; }
 }

src/Speckle.Sdk/Api/GraphQL/Models/ModelIngestionStatusData.cs

@@ -6,4 +6,5 @@ public sealed class ModelIngestionStatusData
 {
   public required ModelIngestionStatus status { get; init; }
   public required string? progressMessage { get; init; }
+  public required string? versionId { get; init; }
 }

src/Speckle.Sdk/Api/GraphQL/Resources/ModelIngestionResource.cs

@@ -44,6 +44,8 @@ public sealed class ModelIngestionResource
               createdAt
               updatedAt
               modelId
+              projectId
+              userId
               cancellationRequested
               statusData {
                 ... on HasModelIngestionStatus {
@@ -94,6 +96,8 @@ public sealed class ModelIngestionResource
             createdAt
             updatedAt
             modelId
+            projectId
+            userId
             cancellationRequested
             statusData {
               ... on HasModelIngestionStatus {
@@ -102,6 +106,10 @@ public sealed class ModelIngestionResource
               ... on HasProgressMessage {
                 progressMessage
               }
+              ... on ModelIngestionSuccessStatus
+              {
+                versionId
+              }
             }
           }
         }
@@ -142,6 +150,8 @@ public sealed class ModelIngestionResource
               createdAt
               updatedAt
               modelId
+              projectId
+              userId
               cancellationRequested
               statusData {
                 ... on HasModelIngestionStatus {
@@ -194,6 +204,8 @@ public sealed class ModelIngestionResource
               createdAt
               updatedAt
               modelId
+              projectId
+              userId
               cancellationRequested
               statusData {
                 ... on HasModelIngestionStatus {
@@ -245,6 +257,8 @@ public sealed class ModelIngestionResource
               createdAt
               updatedAt
               modelId
+              projectId
+              userId
               cancellationRequested
               statusData {
                 ... on HasModelIngestionStatus {
@@ -343,6 +357,8 @@ public sealed class ModelIngestionResource
               createdAt
               updatedAt
               modelId
+              projectId
+              userId
               cancellationRequested
               statusData {
                 ... on HasModelIngestionStatus {
@@ -398,6 +414,8 @@ public sealed class ModelIngestionResource
               createdAt
               updatedAt
               modelId
+              projectId
+              userId
               cancellationRequested
               statusData {
                 ... on HasModelIngestionStatus {
@@ -457,6 +475,8 @@ public sealed class ModelIngestionResource
               createdAt
               updatedAt
               modelId
+              projectId
+              userId
               cancellationRequested
               statusData {
                 ... on HasModelIngestionStatus {

src/Speckle.Sdk/Api/GraphQL/Resources/SubscriptionResource.cs

@@ -229,6 +229,8 @@ public sealed class SubscriptionResource : IDisposable
             createdAt
             updatedAt
             modelId
+            projectId
+            userId
             cancellationRequested
             statusData {
               ... on HasModelIngestionStatus {
@@ -237,6 +239,10 @@ public sealed class SubscriptionResource : IDisposable
               ... on HasProgressMessage {
                 progressMessage
               }
+              ... on ModelIngestionSuccessStatus
+              {
+                versionId
+              }
             }
           }
           type

src/Speckle.Sdk/Helpers/BlobApiHelpers.cs

@@ -0,0 +1,29 @@
+using System.Net.Http.Headers;
+
+namespace Speckle.Sdk.Helpers;
+
+public static class BlobApiHelpers
+{
+  public static string ParseEtagHeader(HttpResponseHeaders headers)
+  {
+    if (!headers.TryGetValues("ETag", out var etagValues))
+    {
+      throw new ArgumentException(
+        "Response does not have an ETag attached to it, cannot use this as an upload",
+        nameof(headers)
+      );
+    }
+
+    var etagValuesArray = etagValues.ToArray();
+
+    if (etagValuesArray.Length != 1)
+    {
+      throw new ArgumentException(
+        $"Expected Etag header to have a single value but got {etagValuesArray.Length}",
+        nameof(headers)
+      );
+    }
+
+    return etagValuesArray[0];
+  }
+}

src/Speckle.Sdk/Helpers/DisposableFile.cs

@@ -0,0 +1,34 @@
+using Microsoft.Extensions.Logging;
+
+namespace Speckle.Sdk.Helpers;
+
+/// <summary>
+/// <see cref="IDisposable"/> wrapper around the downloaded file to try and delete the file on Dispose
+/// </summary>
+/// <remarks>
+/// We're using a similar pattern in the Rhino File Importer codebase (see <c>ImportJobFile</c>)
+/// </remarks>
+/// <param name="logger"></param>
+/// <param name="file"></param>
+public sealed class DisposableFile(FileInfo file, ILogger logger, bool deleteOnDispose = true) : IDisposable
+{
+  public FileInfo FileInfo => file;
+
+  public void Dispose()
+  {
+    if (!deleteOnDispose)
+    {
+      return;
+    }
+
+    try
+    {
+      file.Delete();
+      logger.LogInformation("Cleaned up {File}", file);
+    }
+    catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
+    {
+      logger.LogWarning(ex, "Failed to clean up {File}", file);
+    }
+  }
+}

src/Speckle.Sdk/Helpers/StopwatchPollyfills.cs

@@ -0,0 +1,21 @@
+using System.Diagnostics;
+
+namespace Speckle.Sdk.Helpers;
+
+public static class StopwatchPolyfills
+{
+#if !NET7_0_OR_GREATER
+  private static readonly double s_tickFrequency = (double)TimeSpan.TicksPerSecond / Stopwatch.Frequency;
+#endif
+
+  public static TimeSpan GetElapsedTime(long startingTimestamp)
+  {
+#if NET7_0_OR_GREATER
+    return Stopwatch.GetElapsedTime(startingTimestamp);
+#else
+
+    long elapsedTicks = Stopwatch.GetTimestamp() - startingTimestamp;
+    return new TimeSpan((long)(elapsedTicks * s_tickFrequency));
+#endif
+  }
+}

src/Speckle.Sdk/Logging/NullActivityFactory.cs

@@ -1,8 +1,12 @@
-namespace Speckle.Sdk.Logging;
+using Speckle.Connectors.Logging;
+
+namespace Speckle.Sdk.Logging;
 
 public sealed class NullActivityFactory : ISdkActivityFactory
 {
   public void Dispose() { }
 
-  public ISdkActivity? Start(string? name = default, string source = "") => null;
+  public ISdkActivity? Start(string? name, SdkActivityKind kind, string source) => null;
+
+  public ISdkActivity? StartRemote(string traceContext, SdkActivityKind kind, string? name, string source) => null;
 }

src/Speckle.Sdk/Models/DynamicBaseMemberType.cs

@@ -7,12 +7,12 @@ namespace Speckle.Sdk.Models;
 public enum DynamicBaseMemberType
 {
   /// <summary>
-  /// The typed members of the DynamicBase object
+  /// The typed members of the <see cref="DynamicBase"/> object
   /// </summary>
   Instance = 1,
 
   /// <summary>
-  /// The dynamically added members of the DynamicBase object
+  /// The dynamically added members of the <see cref="DynamicBase"/> object
   /// </summary>
   Dynamic = 2,
 
@@ -22,8 +22,9 @@ public enum DynamicBaseMemberType
   Obsolete = 4,
 
   /// <summary>
-  /// The typed methods flagged with TODO:
+  /// Old feature supported in v2 for grasshopper
   /// </summary>
+  [Obsolete("Feature no longer supported")]
   SchemaComputed = 16,
 
   /// <summary>

src/Speckle.Sdk/Pipelines/Progress/AggregateProgress.cs

@@ -0,0 +1,12 @@
+namespace Speckle.Sdk.Pipelines.Progress;
+
+public sealed class AggregateProgress<T>(params IProgress<T>[] progresses) : IProgress<T>
+{
+  public void Report(T value)
+  {
+    foreach (var progress in progresses)
+    {
+      progress.Report(value);
+    }
+  }
+}

src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManager.cs

@@ -0,0 +1,89 @@
+using System.Diagnostics;
+using Microsoft.Extensions.Logging;
+using Speckle.InterfaceGenerator;
+using Speckle.Sdk.Api;
+using Speckle.Sdk.Api.GraphQL.Inputs;
+using Speckle.Sdk.Api.GraphQL.Models;
+using Speckle.Sdk.Helpers;
+
+namespace Speckle.Sdk.Pipelines.Progress;
+
+public partial interface IIngestionProgressManager : IProgress<CardProgress>;
+
+/// <summary>
+/// An <see langword="IProgress{IngestionProgressEventArgs}"/> implementation for the entire client side Ingestion progress update reporting
+/// Will throttles ingestion progress messages and reports their progress
+/// </summary>
+/// <remarks>
+/// Normally we would pick quite a coarse updateInterval to try and spamming the server (1-5s)
+/// </remarks>
+[GenerateAutoInterface]
+public sealed class IngestionProgressManager(
+  ILogger<IngestionProgressManager> logger,
+  IClient speckleClient,
+  ModelIngestion ingestion,
+  TimeSpan updateInterval,
+  CancellationToken cancellationToken
+) : IIngestionProgressManager
+{
+  public Task? LastUpdate { get; private set; }
+
+  private long _lastUpdatedAt;
+  private readonly object _lock = new();
+
+  [AutoInterfaceIgnore]
+  public void Report(CardProgress value)
+  {
+    cancellationToken.ThrowIfCancellationRequested();
+
+    string trimmedMessage;
+    lock (_lock)
+    {
+      if (ShouldIgnoreProgressUpdate())
+      {
+        return;
+      }
+
+      _lastUpdatedAt = Stopwatch.GetTimestamp();
+
+      trimmedMessage = value.Status.TrimEnd('.');
+
+      LastUpdate = speckleClient
+        .Ingestion.UpdateProgress(
+          new ModelIngestionUpdateInput(ingestion.id, ingestion.projectId, trimmedMessage, value.Progress),
+          cancellationToken
+        )
+        .ContinueWith(
+          Continuation,
+          CancellationToken.None,
+          TaskContinuationOptions.ExecuteSynchronously,
+          TaskScheduler.Default
+        );
+    }
+
+    logger.LogInformation("Progress update {Message} {Progress}", trimmedMessage, value.Progress);
+  }
+
+  /// <returns><see langword="true"/> if the update should be ignored, otherwise <see langword="false"/></returns>
+  private bool ShouldIgnoreProgressUpdate()
+  {
+    if (LastUpdate is not null && !LastUpdate.IsCompleted)
+    {
+      return true;
+    }
+
+    TimeSpan msSinceLastUpdate = StopwatchPolyfills.GetElapsedTime(_lastUpdatedAt);
+    return msSinceLastUpdate < updateInterval;
+  }
+
+  private void Continuation(Task updateTask)
+  {
+    // The progress report failed... could be many reasons.
+    // For now, we're not letting this fail the Ingestion in any way
+    // we'll log but otherwise let it slide while leaving no unobserved task exceptions
+    if (updateTask.IsFaulted)
+    {
+      logger.LogWarning(updateTask.Exception, "A progress update failed unexpectedly");
+    }
+  }
+}

src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManagerFactory.cs

@@ -0,0 +1,21 @@
+using Microsoft.Extensions.Logging;
+using Speckle.InterfaceGenerator;
+using Speckle.Sdk.Api;
+using Speckle.Sdk.Api.GraphQL.Models;
+
+namespace Speckle.Sdk.Pipelines.Progress;
+
+[GenerateAutoInterface]
+public sealed class IngestionProgressManagerFactory(ILogger<IngestionProgressManager> logger)
+  : IIngestionProgressManagerFactory
+{
+  public IIngestionProgressManager CreateInstance(
+    IClient speckleClient,
+    ModelIngestion ingestion,
+    TimeSpan updateInterval,
+    CancellationToken cancellationToken
+  )
+  {
+    return new IngestionProgressManager(logger, speckleClient, ingestion, updateInterval, cancellationToken);
+  }
+}

src/Speckle.Sdk/Pipelines/Progress/ProgressArgs.cs

@@ -0,0 +1,6 @@
+namespace Speckle.Sdk.Pipelines.Progress;
+
+//TODO: rename PipelineProgressArgs
+public readonly record struct CardProgress(string Status, double? Progress);
+
+public readonly record struct StreamProgressArgs(long BytesStreamed, long ExpectedTotalBytes);

src/Speckle.Sdk/Pipelines/Progress/ProgressStream.cs

@@ -0,0 +1,103 @@
+using System.Diagnostics.CodeAnalysis;
+
+namespace Speckle.Sdk.Pipelines.Progress;
+
+/// <summary>
+/// Wraps <paramref name="innerStream"/> to report streaming progress as bytes are read/written.
+/// </summary>
+public sealed class ProgressStream(Stream innerStream, IProgress<StreamProgressArgs>? progress = null) : Stream
+{
+  private long _bytesStreamed;
+
+  public override bool CanRead => innerStream.CanRead;
+  public override bool CanSeek => innerStream.CanSeek;
+  public override bool CanWrite => innerStream.CanWrite;
+  public override long Length => innerStream.Length;
+
+  public override long Position
+  {
+    get => innerStream.Position;
+    set => innerStream.Position = value;
+  }
+
+  public override int Read(byte[] buffer, int offset, int count)
+  {
+    int bytesRead = innerStream.Read(buffer, offset, count);
+    ReportProgress(bytesRead);
+    return bytesRead;
+  }
+
+  [SuppressMessage(
+    "Performance",
+    "CA1835:Prefer the \'Memory\'-based overloads for \'ReadAsync\' and \'WriteAsync\'",
+    Justification = "Analyser warning forwarded to caller"
+  )]
+  public override async Task<int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
+  {
+    int bytesRead = await innerStream.ReadAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false);
+    ReportProgress(bytesRead);
+    return bytesRead;
+  }
+
+#if NET8_0_OR_GREATER
+  public override async ValueTask<int> ReadAsync(Memory<byte> buffer, CancellationToken cancellationToken = default)
+  {
+    int bytesRead = await innerStream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false);
+    ReportProgress(bytesRead);
+    return bytesRead;
+  }
+#endif
+
+  private void ReportProgress(int newBytesProcessed)
+  {
+    _bytesStreamed += newBytesProcessed;
+    progress?.Report(new(_bytesStreamed, Length));
+  }
+
+  public override void Flush() => innerStream.Flush();
+
+  public override Task FlushAsync(CancellationToken cancellationToken) => innerStream.FlushAsync(cancellationToken);
+
+  public override long Seek(long offset, SeekOrigin origin) => innerStream.Seek(offset, origin);
+
+  public override void SetLength(long value) => throw new NotSupportedException(); //intentionally not supporting, as changing length of stream mid-flight will fuck up progress
+
+  public override void Write(byte[] buffer, int offset, int count)
+  {
+    innerStream.Write(buffer, offset, count);
+    ReportProgress(count);
+  }
+
+  [SuppressMessage(
+    "Performance",
+    "CA1835:Prefer the \'Memory\'-based overloads for \'ReadAsync\' and \'WriteAsync\'",
+    Justification = "Analyser warning forwarded to caller"
+  )]
+  public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
+  {
+    await innerStream.WriteAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false);
+    ReportProgress(count);
+  }
+
+#if NET6_0_OR_GREATER
+  public override async ValueTask WriteAsync(ReadOnlyMemory<byte> buffer, CancellationToken cancellationToken = default)
+  {
+    await innerStream.WriteAsync(buffer, cancellationToken).ConfigureAwait(false);
+    ReportProgress(buffer.Length);
+  }
+#endif
+
+  protected override void Dispose(bool disposing)
+  {
+    innerStream.Dispose();
+    base.Dispose(disposing);
+  }
+
+#if NET6_0_OR_GREATER
+  public override async ValueTask DisposeAsync()
+  {
+    await innerStream.DisposeAsync().ConfigureAwait(false);
+    await base.DisposeAsync().ConfigureAwait(false);
+  }
+#endif
+}

src/Speckle.Sdk/Pipelines/Progress/RenderedStreamProgress.cs

@@ -0,0 +1,40 @@
+namespace Speckle.Sdk.Pipelines.Progress;
+
+/// <summary>
+/// Renders "low level" data stream updates
+/// into "high level" <see cref="CardProgress"/> that is expected by Ingestion progress and DUI3
+/// </summary>
+/// <param name="progress"></param>
+public sealed class RenderedStreamProgress(IProgress<CardProgress> progress) : IProgress<StreamProgressArgs>
+{
+  public void Report(StreamProgressArgs value)
+  {
+    var (suffix, scaleFactor) = GetFileSizeRendering(value.ExpectedTotalBytes);
+    progress.Report(
+      new(
+        $"Uploading data... ({value.BytesStreamed * scaleFactor:F1}/{value.ExpectedTotalBytes * scaleFactor:F1} {suffix})",
+        (double)value.BytesStreamed / value.ExpectedTotalBytes
+      )
+    );
+  }
+
+  private static readonly string[] s_suffixes = ["B", "KB", "MB", "GB", "TB", "PB"];
+
+  internal static (string suffix, double scaleFactor) GetFileSizeRendering(long value)
+  {
+    if (value <= 0)
+    {
+      return (s_suffixes[0], 1d);
+    }
+
+    for (int i = 0; i < s_suffixes.Length; i++)
+    {
+      if (value <= Math.Pow(1024, i + 1))
+      {
+        return (s_suffixes[i], 1 / Math.Pow(1024, i));
+      }
+    }
+
+    throw new ArgumentOutOfRangeException(nameof(value), "Value is too large to convert to a file size");
+  }
+}

src/Speckle.Sdk/Pipelines/Send/DiskStore.cs

@@ -0,0 +1,85 @@
+using System.IO.Compression;
+using Microsoft.Extensions.Logging;
+using Speckle.InterfaceGenerator;
+using Speckle.Sdk.Dependencies;
+using Speckle.Sdk.Helpers;
+using Speckle.Sdk.Logging;
+
+namespace Speckle.Sdk.Pipelines.Send;
+
+[GenerateAutoInterface]
+public sealed class DiskStoreFactory(ILogger<DiskStore> logger, ISdkActivityFactory activityFactory) : IDiskStoreFactory
+{
+  public DiskStore CreateInstance(CancellationToken cancellationToken) =>
+    new(logger, activityFactory, cancellationToken);
+}
+
+public sealed class DiskStore
+{
+  private readonly RepackedChannel<UploadItem> _channel;
+  private readonly Task<DisposableFile> _writeToDiskTask;
+  private readonly ILogger<DiskStore> _logger;
+  private readonly ISdkActivityFactory _activityFactory;
+  private readonly CancellationToken _cancellationToken;
+
+  internal DiskStore(
+    ILogger<DiskStore> logger,
+    ISdkActivityFactory activityFactory,
+    CancellationToken cancellationToken
+  )
+  {
+    _logger = logger;
+    _activityFactory = activityFactory;
+    _cancellationToken = cancellationToken;
+
+    _channel = new RepackedChannel<UploadItem>(1000, true, false);
+    _writeToDiskTask = Task.Run(WriteFile, cancellationToken);
+  }
+
+  public async Task PushAsync(UploadItem item) =>
+    await _channel.WriteAsync(item, _cancellationToken).ConfigureAwait(false);
+
+  public async Task<DisposableFile> CompleteAsync()
+  {
+    using var a = _activityFactory.Start("Waiting for DiskStore to complete");
+    _channel.CompleteWriter();
+    return await _writeToDiskTask.ConfigureAwait(false);
+  }
+
+  /// <summary>
+  /// Reads from the Channel and streams the <see cref="UploadItem"/>s to a temporary file on disk.
+  /// Will keep reading until <see cref="CompleteAsync"/> is called.
+  /// </summary>
+  /// <returns>the file that was written</returns>
+  private async Task<DisposableFile> WriteFile()
+  {
+    string tempFilePath = Path.GetTempFileName();
+    var tempFile = new DisposableFile(new FileInfo(tempFilePath), _logger);
+    _logger.LogInformation("Writing temp file to {TempFilePath}", tempFilePath);
+
+    try
+    {
+      using var fileStream = new FileStream(tempFilePath, FileMode.Create, FileAccess.Write, FileShare.None);
+      using var gzip = new GZipStream(fileStream, CompressionLevel.Optimal);
+      using var writer = new StreamWriter(gzip);
+
+      await foreach (var item in _channel.ReadAllAsync(_cancellationToken).ConfigureAwait(false))
+      {
+        await writer.WriteLineAsync($"{item.Id}\t{item.SpeckleType}\t{item.Json}").ConfigureAwait(false);
+      }
+#if NET8_0_OR_GREATER
+      await writer.FlushAsync(_cancellationToken).ConfigureAwait(false);
+#else
+      await writer.FlushAsync().ConfigureAwait(false);
+#endif
+      tempFile.FileInfo.Refresh();
+
+      return tempFile;
+    }
+    catch
+    {
+      tempFile.Dispose();
+      throw;
+    }
+  }
+}

src/Speckle.Sdk/Pipelines/Send/SendPipeline.cs

@@ -0,0 +1,74 @@
+using Speckle.InterfaceGenerator;
+using Speckle.Sdk.Credentials;
+using Speckle.Sdk.Helpers;
+using Speckle.Sdk.Models;
+using Speckle.Sdk.Pipelines.Progress;
+
+namespace Speckle.Sdk.Pipelines.Send;
+
+[GenerateAutoInterface]
+public sealed class SendPipelineFactory(IUploaderFactory uploaderFactory, IDiskStoreFactory diskStoreFactory)
+  : ISendPipelineFactory
+{
+  public SendPipeline CreateInstance(
+    string projectId,
+    string ingestionId,
+    Account account,
+    IProgress<StreamProgressArgs> uploadProgress,
+    CancellationToken cancellationToken
+  )
+  {
+    var uploader = uploaderFactory.CreateInstance(projectId, ingestionId, account, uploadProgress, cancellationToken);
+    var diskStore = diskStoreFactory.CreateInstance(cancellationToken);
+    return new SendPipeline(uploader, diskStore);
+  }
+}
+
+public sealed class SendPipeline : IDisposable
+{
+  private readonly Serializer _serializer = new();
+  private readonly Uploader _uploader;
+  private readonly DiskStore _diskStore;
+
+  internal SendPipeline(Uploader uploader, DiskStore diskStore)
+  {
+    _uploader = uploader;
+    _diskStore = diskStore;
+  }
+
+  private UploadItem _lastItem;
+
+  public async Task<ObjectReference> Process(Base @base)
+  {
+    var results = _serializer.Serialize(@base).ToArray();
+    var first = results.First();
+    foreach (var item in results)
+    {
+      // we're not doing fire and forget here so that we get the backpressure from the uploader
+      await _diskStore.PushAsync(item).ConfigureAwait(false);
+    }
+
+    // NOTE: this is important to keep track of. When we serialze an object, we get back a list of objects, with the first one being the original root.
+    // In the case of the commit root object, this means the last object is not necessarily the root; we therefore need to manually track its existance here
+    // and ensure it's the last one through in the uploader's stream. See WaitForUpload down below.
+    _lastItem = first;
+    return first.Reference;
+  }
+
+  public async Task WaitForUpload()
+  {
+    await _diskStore.PushAsync(_lastItem).ConfigureAwait(false);
+    using DisposableFile tempFile = await _diskStore.CompleteAsync().ConfigureAwait(false);
+
+    using Stream fileStreamUpload = new FileStream(
+      tempFile.FileInfo.FullName,
+      FileMode.Open,
+      FileAccess.Read,
+      FileShare.Read
+    );
+
+    await _uploader.Send(fileStreamUpload).ConfigureAwait(false);
+  }
+
+  public void Dispose() => _uploader.Dispose();
+}

src/Speckle.Sdk/Pipelines/Send/Serializer.cs

@@ -0,0 +1,351 @@
+using System.Collections;
+using System.Drawing;
+using System.Globalization;
+using System.Reflection;
+using Speckle.DoubleNumerics;
+using Speckle.Newtonsoft.Json;
+using Speckle.Sdk.Dependencies;
+using Speckle.Sdk.Helpers;
+using Speckle.Sdk.Models;
+using Speckle.Sdk.Serialisation;
+
+namespace Speckle.Sdk.Pipelines.Send;
+
+/// <summary>
+/// Another serializer, cleaner and meaner. Provides methods for serializing Speckle objects into a format suitable for upload or storage.
+/// This class handles the conversion of <see cref="Speckle.Sdk.Models.Base"/> and its derivatives
+/// into serialized JSON structures along with associated metadata, closures, and references.
+/// <para>Any reference objects coming through are being "passed through" serialized - they do not get double encoded.</para>
+/// </summary>
+internal sealed class Serializer
+{
+  private readonly record struct PropertyInfo(string Name, object? Value, bool IsDetachable);
+
+  public IEnumerable<UploadItem> Serialize(Base root)
+  {
+    // Special case: if root is already an ObjectReference, serialize it verbatim
+    if (root is ObjectReference existingRef)
+    {
+      var uploadItem = ReferenceToUploadItem(existingRef);
+      yield return uploadItem;
+      yield break;
+    }
+
+    var detachedObjects = new List<(Id, Json, Dictionary<string, int>, Base, string)>();
+    var rootClosures = new Dictionary<string, int>();
+
+    var (rootId, rootJson) = SerializeBase(root, false, rootClosures, detachedObjects);
+
+    var rootReference = new ObjectReference
+    {
+      referencedId = rootId.Value,
+      applicationId = root.applicationId,
+      closure = rootClosures.Count > 0 ? rootClosures : null,
+    };
+
+    yield return new UploadItem(rootId.Value, rootJson, root.speckle_type, rootReference);
+
+    foreach (var (id, json, closures, baseObj, speckleType) in detachedObjects)
+    {
+      var reference = new ObjectReference
+      {
+        referencedId = id.Value,
+        applicationId = baseObj.applicationId,
+        closure = closures.Count > 0 ? closures : null,
+      };
+
+      yield return new UploadItem(id.Value, json, speckleType, reference);
+    }
+  }
+
+  private IEnumerable<PropertyInfo> ExtractProperties(Base baseObj)
+  {
+    var typedProperties = baseObj.GetInstanceMembers();
+    foreach (var prop in typedProperties)
+    {
+      if (prop.Name == "id" || prop.Name.StartsWith("__"))
+      {
+        continue;
+      }
+
+      if (prop.IsDefined(typeof(JsonIgnoreAttribute), false))
+      {
+        continue;
+      }
+
+      var value = prop.GetValue(baseObj);
+      var isDetachable = prop.GetCustomAttribute<DetachPropertyAttribute>(true)?.Detachable ?? false;
+
+      yield return new PropertyInfo(prop.Name, value, isDetachable);
+    }
+
+    foreach (var propName in baseObj.DynamicPropertyKeys)
+    {
+      if (propName.StartsWith("__"))
+      {
+        continue;
+      }
+
+      var value = baseObj[propName];
+
+#pragma warning disable CA1866
+      var isDetachable = propName.StartsWith("@");
+#pragma warning restore CA1866
+
+      yield return new PropertyInfo(propName, value, isDetachable);
+    }
+  }
+
+  private (Id, Json) SerializeBase(
+    Base baseObj,
+    bool forceDetach,
+    Dictionary<string, int> closures,
+    List<(Id, Json, Dictionary<string, int>, Base, string)> detachedObjects
+  )
+  {
+    var childClosures = new Dictionary<string, int>();
+
+    var sb = Pools.StringBuilders.Get();
+    try
+    {
+      using var stringWriter = new StringWriter(sb);
+      using var jsonWriter = new JsonTextWriter(stringWriter);
+      using var idWriter = new SerializerIdWriter(jsonWriter);
+
+      idWriter.WriteStartObject();
+
+      foreach (var prop in ExtractProperties(baseObj))
+      {
+        idWriter.WritePropertyName(prop.Name);
+        SerializeValue(prop.Value, idWriter, prop.IsDetachable, childClosures, detachedObjects);
+      }
+
+      var (jsonForId, finalWriter) = idWriter.FinishIdWriter();
+      var id = IdGenerator.ComputeId(jsonForId);
+
+      finalWriter.WritePropertyName("id");
+      finalWriter.WriteValue(id.Value);
+
+      baseObj.id = id.Value;
+
+      if ((forceDetach || childClosures.Count > 0) && childClosures.Count > 0)
+      {
+        finalWriter.WritePropertyName("__closure");
+        finalWriter.WriteStartObject();
+        foreach (var kvp in childClosures)
+        {
+          finalWriter.WritePropertyName(kvp.Key);
+          finalWriter.WriteValue(kvp.Value);
+        }
+        finalWriter.WriteEndObject();
+
+        foreach (var kvp in childClosures)
+        {
+          closures[kvp.Key] = closures.TryGetValue(kvp.Key, out var existing) ? existing + kvp.Value : kvp.Value;
+        }
+      }
+
+      finalWriter.WriteEndObject();
+      finalWriter.Flush();
+
+      var json = new Json(stringWriter.ToString());
+      return (id, json);
+    }
+    finally
+    {
+      Pools.StringBuilders.Return(sb);
+    }
+  }
+
+  private void SerializeValue(
+    object? value,
+    JsonWriter writer,
+    bool isDetachable,
+    Dictionary<string, int> closures,
+    List<(Id, Json, Dictionary<string, int>, Base, string)> detachedObjects
+  )
+  {
+    switch (value)
+    {
+      case Enum:
+        writer.WriteValue((int)value);
+        return;
+      case Guid g:
+        writer.WriteValue(g.ToString());
+        return;
+      case Color c:
+        writer.WriteValue(c.ToArgb());
+        return;
+      case DateTime dt:
+        writer.WriteValue(dt.ToString("o", CultureInfo.InvariantCulture));
+        return;
+      case Matrix4x4 md:
+        writer.WriteStartArray();
+        writer.WriteValue(md.M11);
+        writer.WriteValue(md.M12);
+        writer.WriteValue(md.M13);
+        writer.WriteValue(md.M14);
+        writer.WriteValue(md.M21);
+        writer.WriteValue(md.M22);
+        writer.WriteValue(md.M23);
+        writer.WriteValue(md.M24);
+        writer.WriteValue(md.M31);
+        writer.WriteValue(md.M32);
+        writer.WriteValue(md.M33);
+        writer.WriteValue(md.M34);
+        writer.WriteValue(md.M41);
+        writer.WriteValue(md.M42);
+        writer.WriteValue(md.M43);
+        writer.WriteValue(md.M44);
+        writer.WriteEndArray();
+        return;
+      // Handle ObjectReference before Base (since ObjectReference extends Base)
+      // This prevents double-serialization and properly propagates closures
+      case ObjectReference objRef:
+      {
+        writer.WriteStartObject();
+        writer.WritePropertyName("speckle_type");
+        writer.WriteValue("reference");
+        writer.WritePropertyName("referencedId");
+        writer.WriteValue(objRef.referencedId);
+        writer.WriteEndObject();
+
+        // Propagate closure: add the referenced ID
+        closures[objRef.referencedId] = closures.TryGetValue(objRef.referencedId, out var existing) ? existing + 1 : 1;
+
+        // Propagate nested closures from the ObjectReference.closure dictionary
+        if (objRef.closure != null)
+        {
+          foreach (var kvp in objRef.closure)
+          {
+            closures[kvp.Key] = closures.TryGetValue(kvp.Key, out var existingDepth)
+              ? existingDepth + kvp.Value
+              : kvp.Value;
+          }
+        }
+
+        return;
+      }
+      case Base baseObj:
+      {
+        if (isDetachable)
+        {
+          var childClosures = new Dictionary<string, int>();
+          var (childId, childJson) = SerializeBase(baseObj, true, childClosures, detachedObjects);
+
+          detachedObjects.Add((childId, childJson, childClosures, baseObj, baseObj.speckle_type));
+
+          writer.WriteStartObject();
+          writer.WritePropertyName("speckle_type");
+          writer.WriteValue("reference");
+          writer.WritePropertyName("referencedId");
+          writer.WriteValue(childId.Value);
+          writer.WriteEndObject();
+
+          closures[childId.Value] = closures.TryGetValue(childId.Value, out var existing) ? existing + 1 : 1;
+
+          foreach (var kvp in childClosures)
+          {
+            closures[kvp.Key] = closures.TryGetValue(kvp.Key, out var existingDepth)
+              ? existingDepth + kvp.Value
+              : kvp.Value;
+          }
+        }
+        else
+        {
+          var inlineClosures = new Dictionary<string, int>();
+          var (_, inlineJson) = SerializeBase(baseObj, false, inlineClosures, detachedObjects);
+
+          writer.WriteRawValue(inlineJson.Value);
+
+          foreach (var kvp in inlineClosures)
+          {
+            closures[kvp.Key] = closures.TryGetValue(kvp.Key, out var existingDepth)
+              ? existingDepth + kvp.Value
+              : kvp.Value;
+          }
+        }
+        return;
+      }
+      case IDictionary dict:
+      {
+        writer.WriteStartObject();
+        foreach (DictionaryEntry kvp in dict)
+        {
+          if (kvp.Key is not string key)
+          {
+            throw new ArgumentException("Dictionary keys must be strings", nameof(value));
+          }
+
+          writer.WritePropertyName(key);
+          SerializeValue(kvp.Value, writer, false, closures, detachedObjects);
+        }
+        writer.WriteEndObject();
+        return;
+      }
+      case ICollection collection:
+      {
+        writer.WriteStartArray();
+        foreach (var item in collection)
+        {
+          SerializeValue(item, writer, isDetachable, closures, detachedObjects);
+        }
+        writer.WriteEndArray();
+        return;
+      }
+      default:
+        // This case will handle primitives and `null`
+        // Will throw JsonWriterException if not supported
+        writer.WriteValue(value);
+        return;
+    }
+  }
+
+  private UploadItem ReferenceToUploadItem(ObjectReference existingRef)
+  {
+    var sb = Pools.StringBuilders.Get();
+    try
+    {
+      using var stringWriter = new StringWriter(sb);
+      using var jsonWriter = new JsonTextWriter(stringWriter);
+
+      jsonWriter.WriteStartObject();
+      jsonWriter.WritePropertyName("speckle_type");
+      jsonWriter.WriteValue("reference");
+      jsonWriter.WritePropertyName("referencedId");
+      jsonWriter.WriteValue(existingRef.referencedId);
+      jsonWriter.WritePropertyName("__closure");
+
+      if (existingRef.closure != null && existingRef.closure.Count > 0)
+      {
+        jsonWriter.WriteStartObject();
+        foreach (var kvp in existingRef.closure)
+        {
+          jsonWriter.WritePropertyName(kvp.Key);
+          jsonWriter.WriteValue(kvp.Value);
+        }
+        jsonWriter.WriteEndObject();
+      }
+      else
+      {
+        jsonWriter.WriteNull();
+      }
+
+      jsonWriter.WriteEndObject();
+      jsonWriter.Flush();
+
+      var refJson = new Json(stringWriter.ToString());
+
+      return new UploadItem(
+        existingRef.referencedId,
+        refJson,
+        existingRef.speckle_type,
+        existingRef // Pass through the original ObjectReference
+      );
+    }
+    finally
+    {
+      Pools.StringBuilders.Return(sb);
+    }
+  }
+}

src/Speckle.Sdk/Pipelines/Send/Uploader.cs

@@ -0,0 +1,132 @@
+using System.Net.Http.Headers;
+using System.Text;
+using Speckle.InterfaceGenerator;
+using Speckle.Newtonsoft.Json;
+using Speckle.Sdk.Credentials;
+using Speckle.Sdk.Helpers;
+using Speckle.Sdk.Logging;
+using Speckle.Sdk.Pipelines.Progress;
+
+namespace Speckle.Sdk.Pipelines.Send;
+
+[GenerateAutoInterface]
+public sealed class UploaderFactory(ISpeckleHttp httpClientFactory, ISdkActivityFactory activityFactory)
+  : IUploaderFactory
+{
+  public Uploader CreateInstance(
+    string projectId,
+    string ingestionId,
+    Account account,
+    IProgress<StreamProgressArgs> progress,
+    CancellationToken cancellationToken
+  ) => new(projectId, ingestionId, activityFactory, httpClientFactory, account, progress, cancellationToken);
+}
+
+public sealed class Uploader : IDisposable
+{
+  private readonly string _projectId;
+  private readonly string _ingestionId;
+  private readonly CancellationToken _cancellationToken;
+  private readonly HttpClient _speckleClient;
+  private readonly HttpClient _s3Client;
+  private readonly ISdkActivityFactory _activity;
+  private readonly IProgress<StreamProgressArgs> _progress;
+
+  internal Uploader(
+    string projectId,
+    string ingestionId,
+    ISdkActivityFactory activity,
+    ISpeckleHttp httpClientFactory,
+    Account speckleAccount,
+    IProgress<StreamProgressArgs> progress,
+    CancellationToken cancellationToken
+  )
+  {
+    _projectId = projectId;
+    _ingestionId = ingestionId;
+    _activity = activity;
+    _cancellationToken = cancellationToken;
+    _progress = progress;
+    _speckleClient = httpClientFactory.CreateHttpClient(authorizationToken: speckleAccount.token);
+    _speckleClient.BaseAddress = new(new(speckleAccount.serverInfo.url), "/api/v1/");
+
+    _s3Client = httpClientFactory.CreateHttpClient();
+  }
+
+  public async Task Send(Stream fileStream)
+  {
+    PresignedUploadResponse presignedUploadResponse = await GetPresignedUrl().ConfigureAwait(false);
+    var etag = await UploadToS3(fileStream, presignedUploadResponse).ConfigureAwait(false);
+
+    await TriggerProcessing(new() { Etag = etag }).ConfigureAwait(false);
+  }
+
+  private async Task<PresignedUploadResponse> GetPresignedUrl()
+  {
+    using var a = _activity.Start("Get Presigned Url");
+
+    var signUri = new Uri($"projects/{_projectId}/modelingestion/{_ingestionId}/uploads/sign", UriKind.Relative);
+
+    using var signResponse = await _speckleClient.PostAsync(signUri, null, _cancellationToken).ConfigureAwait(false);
+    signResponse.EnsureSuccessStatusCode();
+
+#if NET5_0_OR_GREATER
+    string signResponseString = await signResponse.Content.ReadAsStringAsync(_cancellationToken).ConfigureAwait(false);
+#else
+    string signResponseString = await signResponse.Content.ReadAsStringAsync().ConfigureAwait(false);
+#endif
+    PresignedUploadResponse presignedUpload =
+      JsonConvert.DeserializeObject<PresignedUploadResponse>(signResponseString)
+      ?? throw new InvalidOperationException("Failed to get presigned upload URL");
+    return presignedUpload;
+  }
+
+  private async Task<string> UploadToS3(Stream fileStream, PresignedUploadResponse presignedUploadResponse)
+  {
+    using var a = _activity.Start("Uploading file to pre-signed url");
+
+    Stream progressStream = new ProgressStream(fileStream, _progress);
+
+    using var streamContent = new StreamContent(progressStream);
+    streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
+    streamContent.Headers.ContentLength = fileStream.Length;
+
+    using var uploadRequest = new HttpRequestMessage(HttpMethod.Put, presignedUploadResponse.Url);
+    foreach (var kvp in presignedUploadResponse.AdditionalRequestHeaders)
+    {
+      uploadRequest.Headers.Add(kvp.Key, kvp.Value);
+    }
+
+    uploadRequest.Content = streamContent;
+
+    using var uploadResponse = await _s3Client
+      .SendAsync(uploadRequest, HttpCompletionOption.ResponseHeadersRead, _cancellationToken)
+      .ConfigureAwait(false);
+
+    uploadResponse.EnsureSuccessStatusCode();
+
+    return BlobApiHelpers.ParseEtagHeader(uploadResponse.Headers);
+  }
+
+  private async Task TriggerProcessing(TriggerUploadRequest request)
+  {
+    using var a = _activity.Start("Triggering Processing");
+
+    Uri processUri = new($"projects/{_projectId}/modelingestion/{_ingestionId}/uploads/process", UriKind.Relative);
+    string requestBody = JsonConvert.SerializeObject(request);
+    using var content = new StringContent(requestBody, Encoding.UTF8, "application/json");
+
+    using HttpResponseMessage processResponse = await _speckleClient
+      .PostAsync(processUri, content, _cancellationToken)
+      .ConfigureAwait(false);
+
+    string body = await processResponse.Content.ReadAsStringAsync().ConfigureAwait(false);
+    processResponse.EnsureSuccessStatusCode();
+  }
+
+  public void Dispose()
+  {
+    _speckleClient.Dispose();
+    _s3Client.Dispose();
+  }
+}

src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs

@@ -0,0 +1,20 @@
+using Speckle.Newtonsoft.Json;
+using Speckle.Sdk.Models;
+using Speckle.Sdk.Serialisation;
+
+namespace Speckle.Sdk.Pipelines.Send;
+
+public record UploadItem(string Id, Json Json, string SpeckleType, ObjectReference Reference);
+
+internal record PresignedUploadResponse
+{
+  public required Uri Url { get; init; }
+  public required string Key { get; init; }
+  public Dictionary<string, string> AdditionalRequestHeaders { get; init; } = new();
+}
+
+internal readonly struct TriggerUploadRequest
+{
+  [JsonProperty("etag")]
+  public required string Etag { get; init; }
+}

src/Speckle.Sdk/Serialisation/V2/Send/BaseItem.cs

@@ -2,7 +2,13 @@ using System.Text;
 
 namespace Speckle.Sdk.Serialisation.V2.Send;
 
-public sealed record BaseItem(Id Id, Json Json, bool NeedsStorage, Dictionary<Id, int>? Closures) : IHasByteSize
+public sealed record BaseItem(
+  Id Id,
+  Json Json,
+  bool NeedsStorage,
+  Dictionary<Id, int>? Closures,
+  bool? IsReference = false
+) : IHasByteSize
 {
   public int ByteSize { get; } = Encoding.UTF8.GetByteCount(Json.Value);
 

src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs

@@ -113,16 +113,6 @@ public sealed class SerializeProcess(
         _processSource.Token
       );
       var findTotalObjectsTask = Task.CompletedTask;
-      if (!options.SkipFindTotalObjects)
-      {
-        ThrowIfFailed();
-        findTotalObjectsTask = Task.Factory.StartNew(
-          () => TraverseTotal(root),
-          _processSource.Token,
-          TaskCreationOptions.AttachedToParent | TaskCreationOptions.PreferFairness,
-          _highest
-        );
-      }
 
       await Traverse(root).ConfigureAwait(false);
       ThrowIfFailed();
@@ -133,6 +123,7 @@ public sealed class SerializeProcess(
       ThrowIfFailed();
       await WaitForSchedulerCompletion().ConfigureAwait(false);
       ThrowIfFailed();
+
       return new(root.id.NotNull(), baseSerializer.ObjectReferences.Freeze());
     }
     catch (OperationCanceledException)

src/Speckle.Sdk/ServiceRegistration.cs

@@ -8,6 +8,7 @@ using Speckle.Sdk.Dependencies;
 using Speckle.Sdk.Host;
 using Speckle.Sdk.Logging;
 using Speckle.Sdk.Models.GraphTraversal;
+using Speckle.Sdk.Pipelines.Progress;
 using Speckle.Sdk.Serialisation.V2;
 using Speckle.Sdk.Serialisation.V2.Receive;
 using Speckle.Sdk.Serialisation.V2.Send;
@@ -96,7 +97,8 @@ public static class ServiceRegistration
       typeof(DeserializeProcess),
       typeof(ObjectLoader),
       typeof(TraversalRule),
-      typeof(Client)
+      typeof(Client),
+      typeof(IngestionProgressManager)
     );
     serviceCollection.AddMatchingInterfacesAsTransient(typeof(GraphQLRetry).Assembly);
     return serviceCollection;

src/Speckle.Sdk/Speckle.Sdk.csproj

@@ -35,7 +35,6 @@
     <PackageReference Include="Microsoft.CSharp" />
     <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
     <PackageReference Include="Microsoft.Extensions.Logging" />
-    <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\Speckle.Sdk.Dependencies\Speckle.Sdk.Dependencies.csproj" />

src/Speckle.Sdk/packages.lock.json

@@ -13,15 +13,6 @@
           "System.Reactive": "5.0.0"
         }
       },
-      "Microsoft.Bcl.AsyncInterfaces": {
-        "type": "Direct",
-        "requested": "[5.0.0, )",
-        "resolved": "5.0.0",
-        "contentHash": "W8DPQjkMScOMTtJbPwmPyj9c3zYSFGawDW3jwlBOOsnY+EzZFLgNQ/UMkK35JmkNOVPdCyPr2Tw7Vv9N+KA3ZQ==",
-        "dependencies": {
-          "System.Threading.Tasks.Extensions": "4.5.4"
-        }
-      },
       "Microsoft.CSharp": {
         "type": "Direct",
         "requested": "[4.7.0, )",
@@ -291,7 +282,19 @@
         }
       },
       "speckle.sdk.dependencies": {
-        "type": "Project"
+        "type": "Project",
+        "dependencies": {
+          "Microsoft.Bcl.AsyncInterfaces": "[9.0.4, )"
+        }
+      },
+      "Microsoft.Bcl.AsyncInterfaces": {
+        "type": "CentralTransitive",
+        "requested": "[9.0.4, )",
+        "resolved": "9.0.4",
+        "contentHash": "9VGI5kxIvrNG2mqLQZnUR6y/3fcnygD8eNpHR+CqfbnIXvea6nehnYknDKQTxZVPMpzpNca+7DxLBmpdB3q0Bw==",
+        "dependencies": {
+          "System.Threading.Tasks.Extensions": "4.5.4"
+        }
       }
     },
     "net8.0": {

tests/Speckle.Sdk.Serialization.Testing/packages.lock.json

@@ -382,7 +382,7 @@
       },
       "Microsoft.Bcl.AsyncInterfaces": {
         "type": "CentralTransitive",
-        "requested": "[5.0.0, )",
+        "requested": "[9.0.4, )",
         "resolved": "1.1.0",
         "contentHash": "1Am6l4Vpn3/K32daEqZI+FFr96OlZkgwK2LcT3pZ2zWubR5zTPW3/FkO1Rat9kb7oQOa4rxgl9LJHc5tspCWfg=="
       },

tests/Speckle.Sdk.Serialization.Tests/ServerObjectManagerTests.cs

@@ -52,7 +52,7 @@ public class ServerObjectManagerTests : MoqTest
     http.Setup(x => x.CreateHttpClient(It.IsAny<HttpClientHandler>(), timeout, token)).Returns(httpClient);
 
     var activityFactory = Create<ISdkActivityFactory>();
-    activityFactory.Setup(x => x.Start(null, "DownloadObjects")).Returns((ISdkActivity?)null);
+    activityFactory.Setup(x => x.Start(null, default, "DownloadObjects")).Returns((ISdkActivity?)null);
 
     var serverObjectManager = new ServerObjectManager(
       http.Object,
@@ -91,7 +91,7 @@ public class ServerObjectManagerTests : MoqTest
     http.Setup(x => x.CreateHttpClient(It.IsAny<HttpClientHandler>(), timeout, token)).Returns(httpClient);
 
     var activityFactory = Create<ISdkActivityFactory>();
-    activityFactory.Setup(x => x.Start(null, "DownloadSingleObject")).Returns((ISdkActivity?)null);
+    activityFactory.Setup(x => x.Start(null, default, "DownloadSingleObject")).Returns((ISdkActivity?)null);
 
     var serverObjectManager = new ServerObjectManager(
       http.Object,
@@ -132,7 +132,7 @@ public class ServerObjectManagerTests : MoqTest
     http.Setup(x => x.CreateHttpClient(It.IsAny<HttpClientHandler>(), timeout, token)).Returns(httpClient);
 
     var activityFactory = Create<ISdkActivityFactory>();
-    activityFactory.Setup(x => x.Start(null, "HasObjects")).Returns((ISdkActivity?)null);
+    activityFactory.Setup(x => x.Start(null, default, "HasObjects")).Returns((ISdkActivity?)null);
 
     var serverObjectManager = new ServerObjectManager(
       http.Object,
@@ -171,7 +171,7 @@ public class ServerObjectManagerTests : MoqTest
     http.Setup(x => x.CreateHttpClient(It.IsAny<HttpClientHandler>(), timeout, token)).Returns(httpClient);
 
     var activityFactory = Create<ISdkActivityFactory>();
-    activityFactory.Setup(x => x.Start(null, "UploadObjects")).Returns((ISdkActivity?)null);
+    activityFactory.Setup(x => x.Start(null, default, "UploadObjects")).Returns((ISdkActivity?)null);
 
     var serverObjectManager = new ServerObjectManager(
       http.Object,

tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/ModelIngestionResourceTests.cs

@@ -129,8 +129,10 @@ public sealed class ModelIngestionResourceTests : IAsyncLifetime
     ModelIngestionSuccessInput finish = new(ingest.id, _project.id, sendResult.RootId, "yay!");
     string versionId = await Sut.Complete(finish);
     Version version = await _testUser.Version.Get(versionId, _project.id);
+    ModelIngestion finalIngestion = await _testUser.Ingestion.Get(ingest.id, _project.id);
     Assert.Equal(version.id, versionId);
     Assert.Equal(sendResult.RootId, version.referencedObject);
+    Assert.Equal(finalIngestion.statusData.versionId, versionId);
   }
 
   [Fact]
@@ -147,6 +149,11 @@ public sealed class ModelIngestionResourceTests : IAsyncLifetime
     ModelIngestion res = await Sut.Get(ingest.id, _project.id);
     Assert.Equal(ingest.id, res.id);
     Assert.Equal(ingest.statusData.status, res.statusData.status);
+    Assert.Equal(ingest.statusData.versionId, res.statusData.versionId);
+    Assert.Null(res.statusData.versionId);
+    Assert.Equal(_model.id, res.modelId);
+    Assert.Equal(_project.id, res.projectId);
+    Assert.Equal(_testUser.Account.userInfo.id, res.userId);
   }
 
   [Fact]

tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/SubscriptionResourceTests.cs

@@ -13,7 +13,7 @@ public class SubscriptionResourceTests : IAsyncLifetime
 #if DEBUG
   private const int WAIT_PERIOD = 3000; // WSL is slow AF, so for local runs, we're being extra generous
 #else
-  private const int WAIT_PERIOD = 400; // For CI runs, a much smaller wait time is acceptable
+  private const int WAIT_PERIOD = 600; // For CI runs, a much smaller wait time is acceptable
 #endif
   private const int TIMEOUT = WAIT_PERIOD + WAIT_PERIOD + 600;
   private IClient _testUser;

tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.TestGetProjects.verified.json

@@ -1,8 +0,0 @@
-{
-  "Type": "AggregateException",
-  "InnerException": {
-    "Data": {},
-    "Message": "FORBIDDEN: Your auth token does not have the required scope: workspace:read.",
-    "Type": "SpeckleGraphQLForbiddenException"
-  }
-}

tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.TestGetWorkspace.verified.json

@@ -1,8 +0,0 @@
-{
-  "Type": "AggregateException",
-  "InnerException": {
-    "Data": {},
-    "Message": "FORBIDDEN: Your auth token does not have the required scope: workspace:read.",
-    "Type": "SpeckleGraphQLForbiddenException"
-  }
-}

tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.cs

@@ -25,13 +25,15 @@ public class WorkspaceResourceTests
   public async Task TestGetWorkspace()
   {
     var ex = await Assert.ThrowsAsync<AggregateException>(async () => _ = await Sut.Get("non-existent-id"));
-    await Verify(ex);
+    Assert.Single(ex.InnerExceptions);
+    Assert.All(ex.InnerExceptions, item => Assert.IsType<SpeckleGraphQLForbiddenException>(item));
   }
 
   [Fact]
   public async Task TestGetProjects()
   {
     var ex = await Assert.ThrowsAsync<AggregateException>(async () => _ = await Sut.GetProjects("non-existent-id"));
-    await Verify(ex);
+    Assert.Single(ex.InnerExceptions);
+    Assert.All(ex.InnerExceptions, item => Assert.IsType<SpeckleGraphQLForbiddenException>(item));
   }
 }

tests/Speckle.Sdk.Tests.Integration/Pipelines/Progress/IngestionProgressManagerTests.cs

@@ -0,0 +1,76 @@
+using Microsoft.Extensions.DependencyInjection;
+using Speckle.Sdk.Api;
+using Speckle.Sdk.Api.GraphQL.Models;
+using Speckle.Sdk.Common;
+using Speckle.Sdk.Pipelines.Progress;
+
+namespace Speckle.Sdk.Tests.Integration.Pipelines.Progress;
+
+[Trait("Server", "Internal")]
+public class IngestionProgressManagerTests : IAsyncLifetime
+{
+  private IIngestionProgressManagerFactory _factory;
+  private IClient _client;
+  private Project _project;
+  private Model _model;
+  private ModelIngestion _ingestion;
+
+  public async Task InitializeAsync()
+  {
+    var serviceProvider = TestServiceSetup.GetServiceProvider();
+    _factory = serviceProvider.GetRequiredService<IIngestionProgressManagerFactory>();
+
+    _client = await Fixtures.SeedUserWithClient();
+    _project = await _client.Project.Create(new("test", null, default));
+    _model = await _client.Model.Create(new("test", null, _project.id));
+    _ingestion = await _client.Ingestion.Create(
+      new(_model.id, _project.id, "Testing ingestion", new("integrationTests", "0.0.0", null, null))
+    );
+  }
+
+  [Fact]
+  public async Task TestProgress_NoThrottle()
+  {
+    var sut = _factory.CreateInstance(_client, _ingestion, TimeSpan.Zero, CancellationToken.None);
+    const string FIRST_MESSAGE = "This is a test 123";
+    const string SECOND_MESSAGE = "This is another test 321";
+
+    // first message (should go through)
+    sut.Report(new CardProgress(FIRST_MESSAGE, 0.123123123d));
+    await sut.LastUpdate.NotNull();
+    var res = await _client.Ingestion.Get(_ingestion.id, _project.id, CancellationToken.None);
+
+    Assert.Equal(FIRST_MESSAGE, res.statusData.progressMessage);
+
+    // second message (should also go through)
+    sut.Report(new CardProgress(SECOND_MESSAGE, 0.321321321d));
+    await sut.LastUpdate.NotNull();
+    res = await _client.Ingestion.Get(_ingestion.id, _project.id, CancellationToken.None);
+
+    Assert.Equal(SECOND_MESSAGE, res.statusData.progressMessage);
+  }
+
+  [Fact]
+  public async Task TestProgress_WithThrottle()
+  {
+    var sut = _factory.CreateInstance(_client, _ingestion, TimeSpan.FromMilliseconds(500), CancellationToken.None);
+    const string EXPECTED_MESSAGE = "First message should go through 123";
+
+    await Task.Delay(TimeSpan.FromMilliseconds(600));
+
+    // first message (should go through)
+    sut.Report(new CardProgress(EXPECTED_MESSAGE, 0.123123123d));
+    // second message (should be dropped)
+    sut.Report(new CardProgress("Second message, should be dropped", 0.321321321d));
+    await sut.LastUpdate.NotNull();
+    var res = await _client.Ingestion.Get(_ingestion.id, _project.id, CancellationToken.None);
+
+    Assert.Equal(EXPECTED_MESSAGE, res.statusData.progressMessage);
+  }
+
+  public Task DisposeAsync()
+  {
+    _client.Dispose();
+    return Task.CompletedTask;
+  }
+}

tests/Speckle.Sdk.Tests.Integration/Speckle.Sdk.Tests.Integration.csproj

@@ -16,4 +16,7 @@
     <ProjectReference Include="..\..\src\Speckle.Sdk\Speckle.Sdk.csproj" />
     <ProjectReference Include="..\Speckle.Sdk.Testing\Speckle.Sdk.Testing.csproj" />
   </ItemGroup>
+  <ItemGroup>
+    <Folder Include="Pipelines\Send\" />
+  </ItemGroup>
 </Project>

tests/Speckle.Sdk.Tests.Performance/packages.lock.json

@@ -374,7 +374,7 @@
       },
       "Microsoft.Bcl.AsyncInterfaces": {
         "type": "CentralTransitive",
-        "requested": "[5.0.0, )",
+        "requested": "[9.0.4, )",
         "resolved": "1.1.0",
         "contentHash": "1Am6l4Vpn3/K32daEqZI+FFr96OlZkgwK2LcT3pZ2zWubR5zTPW3/FkO1Rat9kb7oQOa4rxgl9LJHc5tspCWfg=="
       },

tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/AggregateProgressTests.cs

@@ -0,0 +1,21 @@
+using Moq;
+using Speckle.Sdk.Pipelines.Progress;
+
+namespace Speckle.Sdk.Tests.Unit.Pipelines.Progress;
+
+public class AggregateProgressTests
+{
+  [Fact]
+  public void Report_InvokesReportOnAllInnerProgresses()
+  {
+    var mock1 = new Mock<IProgress<int>>();
+    var mock2 = new Mock<IProgress<int>>();
+    const int TEST_VALUE = 42;
+    var target = new AggregateProgress<int>(mock1.Object, mock2.Object);
+
+    target.Report(TEST_VALUE);
+
+    mock1.Verify(x => x.Report(TEST_VALUE), Times.Once);
+    mock2.Verify(x => x.Report(TEST_VALUE), Times.Once);
+  }
+}

tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/ProgressStreamTests.cs

@@ -0,0 +1,72 @@
+using System.Diagnostics.CodeAnalysis;
+using Moq;
+using Speckle.Sdk.Pipelines.Progress;
+
+namespace Speckle.Sdk.Tests.Unit.Pipelines.Progress;
+
+[SuppressMessage(
+  "Performance",
+  "CA1835:Prefer the \'Memory\'-based overloads for \'ReadAsync\' and \'WriteAsync\'",
+  Justification = "Need to test it"
+)]
+public class ProgressStreamTests : IDisposable
+{
+  private readonly Mock<Stream> _innerStreamMock;
+  private readonly Mock<IProgress<StreamProgressArgs>> _progressMock;
+  private readonly ProgressStream _sut;
+
+  public ProgressStreamTests()
+  {
+    // Setup the mocks
+    _innerStreamMock = new Mock<Stream>();
+    _innerStreamMock.Setup(s => s.Length).Returns(1024L);
+
+    _progressMock = new Mock<IProgress<StreamProgressArgs>>();
+
+    // Inject mocks into the System Under Test
+    _sut = new ProgressStream(_innerStreamMock.Object, _progressMock.Object);
+  }
+
+  [Fact]
+  public async Task ReadAsync_Should_CallInnerStreamAndReportProgress()
+  {
+    // Arrange
+    var buffer = new byte[10];
+    _innerStreamMock
+      .Setup(s => s.ReadAsync(buffer, 0, buffer.Length, CancellationToken.None))
+      .Returns(Task.FromResult(5));
+
+    // Act
+    await _sut.ReadAsync(buffer, 0, buffer.Length);
+
+    // Assert - Inner Stream Read was called
+    _innerStreamMock.Verify(s => s.ReadAsync(buffer, 0, buffer.Length, CancellationToken.None), Times.Once);
+
+    // Assert - Progress Report was called with the correct byte count
+    _progressMock.Verify(p => p.Report(It.IsAny<StreamProgressArgs>()), Times.Once);
+  }
+
+  [Fact]
+  public async Task WriteAsync_Should_CallInnerStreamAndReportProgress()
+  {
+    // Arrange
+    var buffer = new byte[10];
+    _innerStreamMock
+      .Setup(s => s.WriteAsync(buffer, 0, buffer.Length, CancellationToken.None))
+      .Returns(Task.FromResult(5));
+
+    // Act
+    await _sut.WriteAsync(buffer, 0, buffer.Length);
+
+    // Assert - Inner Stream Write was called
+    _innerStreamMock.Verify(s => s.WriteAsync(buffer, 0, buffer.Length, CancellationToken.None), Times.Once);
+
+    // Assert - Progress Report was called with the correct byte count
+    _progressMock.Verify(p => p.Report(It.IsAny<StreamProgressArgs>()), Times.Once);
+  }
+
+  public void Dispose()
+  {
+    _sut.Dispose();
+  }
+}

tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/RenderedStreamProgressTests.cs

@@ -0,0 +1,46 @@
+using Speckle.Sdk.Pipelines.Progress;
+
+namespace Speckle.Sdk.Tests.Unit.Pipelines.Progress;
+
+public class RenderedStreamProgressTests
+{
+  [Theory]
+  [InlineData(1, "B", 1.0)]
+  [InlineData(1024, "B", 1.0)]
+  [InlineData(1024 + 1, "KB", 1.0 / 1024)]
+  [InlineData(1024 * 1024, "KB", 1.0 / 1024)]
+  [InlineData(1024 * 1024 + 1, "MB", 1.0 / (1024 * 1024))]
+  [InlineData(1024 * 1024 * 1024, "MB", 1.0 / (1024 * 1024))]
+  [InlineData(1024 * 1024 * 1024 + 1, "GB", 1.0 / (1024 * 1024 * 1024))]
+  [InlineData(1024L * 1024L * 1024L * 1024L, "GB", 1.0 / (1024L * 1024L * 1024L))]
+  public void GetFileSizeRendering_WithPositiveValue_ReturnsCorrectSuffix(
+    long value,
+    string expectedSuffix,
+    double expectedScaleFactor
+  )
+  {
+    var result = RenderedStreamProgress.GetFileSizeRendering(value);
+
+    Assert.Equal(expectedSuffix, result.suffix);
+    Assert.Equal(expectedScaleFactor, result.scaleFactor);
+  }
+
+  [Theory]
+  [InlineData(0)]
+  [InlineData(-1)]
+  [InlineData(-1000)]
+  public void GetFileSizeRendering_WithNonPositiveValue_ReturnsBytesSuffix(long value)
+  {
+    var result = RenderedStreamProgress.GetFileSizeRendering(value);
+
+    Assert.Equal("B", result.suffix);
+    Assert.Equal(1d, result.scaleFactor);
+  }
+
+  [Theory]
+  [InlineData(long.MaxValue)]
+  public void GetFileSizeRendering_WithVeryLargeValue_ThrowsArgumentOutOfRangeException(long value)
+  {
+    Assert.Throws<ArgumentOutOfRangeException>(() => RenderedStreamProgress.GetFileSizeRendering(value));
+  }
+}