Serialize using a Channel (#146)

* Use a stack channel for deserialization

* multi-threaded

* add object dictionary pool

* more pooling

* adjust sqlite transport

* format

* Optimize IsPropNameValid

* object loader first pass

* save test

* add cache pre check

* save better deserialize

* mostly works

* uses tasks but slower at end

* rework to make more sense

* add check to avoid multi-deserialize

* modify max parallelism

* async enqueuing of tasks

* switch to more asyncenumerable

* fmt

* fmt

* cleanup sqlite

* make ServerObjectManager

* revert change

* add ability to skip cache check

* cache json to know what is loaded

* testing

* clean up usage

* clean up and added new op

* Fix exception handling

* fixing progress

* remove codejam

* Hides ObjectPool dependency

* fmt

* Use the 1.0 BCL async to try to be more compatible

* rename to dependencies

* Move Polly to internal dependencies

* format

* remove more old references

* remove stackchannel

* fixes for registration

* remove console writeline

* add cache check shortcut for root object

* start refactoring send

* recevie2 benchmark

* add test for deserialize new

* use channels for sending

* test and fixes

* Use same asyncinterfaces as Dynamo.  Merge fixes

* clean up

* fix download object progress

* put back from bad merge

* intermediate commit: separating get child function from serializer

* send didn't error

* add channels

* Use net48, netstandard2.1 and net8

* remove collection special case

* have to make a tree of tasks even though it may serialize things twice

* pre-id changing during serialize

* need AsyncInterfaces for net48 :(

* options changes

* revert to netstandard2.0 and net8.0

* fix totals

* revert httpcontext changes

* format

* clean up

* active tasks works when accounting for id not being stable

* add id tests

* more fixes

* works

* format

* Convert to BaseItem and use single SQLite checks to avoid locks

* use locks and batch sqlite operations

* hook up and handle null ids

* remove unused parameter

* remove progress from serializer itself

* invert has objects call

* readd object references

* format

* fix tests

* remove active tasks check

* bug fix for json cache

* remove locks from sqlite

* General Send test

* add childclosures

* redo extract all to be enumerable

* group tests in projects

* caching json does matter

* cache checking should be managed by channels

* format

* Merge pull request #152 from specklesystems/new-json-test

Uses a new objects test in Revit for serialization tests

* add skip

* add new roundtrip test

* fix finish

* clean up tests

* check happens in serialize...don't do it twice

* better progress reporting

* fix progress reporting

* only use detached properties when children gathering

* move detached tests

* add detached tests

* fix merge

* Fix progress change

* fix more tests

---------

Co-authored-by: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com>
Co-authored-by: Claire Kuang <kuang.claire@gmail.com>

src/Speckle.Sdk.Dependencies/Serialization/ChannelLoader.cs

@@ -1,33 +1,29 @@
 using Open.ChannelExtensions;
 
-namespace Speckle.Sdk.Serialisation.V2.Receive;
+namespace Speckle.Sdk.Dependencies.Serialization;
 
 public abstract class ChannelLoader
 {
-  private const int HTTP_ID_CHUNK_SIZE = 500;
+  private const int HTTP_GET_CHUNK_SIZE = 500;
   private const int MAX_PARALLELISM_HTTP = 4;
+  private static readonly TimeSpan HTTP_BATCH_TIMEOUT = TimeSpan.FromSeconds(2);
+  private static readonly int MAX_CACHE_PARALLELISM = Environment.ProcessorCount;
 
   protected async Task GetAndCache(IEnumerable<string> allChildrenIds, CancellationToken cancellationToken = default) =>
     await allChildrenIds
       .ToChannel(cancellationToken: cancellationToken)
-      .Pipe(Environment.ProcessorCount, CheckCache, cancellationToken: cancellationToken)
+      .Pipe(MAX_CACHE_PARALLELISM, CheckCache, cancellationToken: cancellationToken)
       .Filter(x => x is not null)
-      .Batch(HTTP_ID_CHUNK_SIZE)
-      .WithTimeout(TimeSpan.FromSeconds(2))
-      .PipeAsync(
-        MAX_PARALLELISM_HTTP,
-        async x => await DownloadAndCache(x).ConfigureAwait(false),
-        -1,
-        false,
-        cancellationToken
-      )
+      .Batch(HTTP_GET_CHUNK_SIZE)
+      .WithTimeout(HTTP_BATCH_TIMEOUT)
+      .PipeAsync(MAX_PARALLELISM_HTTP, async x => await Download(x).ConfigureAwait(false), -1, false, cancellationToken)
       .Join()
-      .ReadAllConcurrently(Environment.ProcessorCount, SaveToCache, cancellationToken)
+      .ReadAllConcurrently(MAX_CACHE_PARALLELISM, SaveToCache, cancellationToken)
       .ConfigureAwait(false);
 
   public abstract string? CheckCache(string id);
 
-  public abstract Task<List<(string, string)>> DownloadAndCache(List<string?> ids);
+  public abstract Task<List<BaseItem>> Download(List<string?> ids);
 
-  public abstract void SaveToCache((string, string) x);
+  public abstract void SaveToCache(BaseItem x);
 }

src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs

@@ -0,0 +1,46 @@
+using System.Threading.Channels;
+using Open.ChannelExtensions;
+
+namespace Speckle.Sdk.Dependencies.Serialization;
+
+public readonly record struct BaseItem(string Id, string Json, bool NeedsStorage);
+
+public abstract class ChannelSaver
+{
+  private const int HTTP_SEND_CHUNK_SIZE = 500;
+  private static readonly TimeSpan HTTP_BATCH_TIMEOUT = TimeSpan.FromSeconds(2);
+  private const int MAX_PARALLELISM_HTTP = 4;
+  private const int MAX_CACHE_WRITE_PARALLELISM = 1;
+  private const int MAX_CACHE_BATCH = 100;
+
+  private readonly Channel<BaseItem> _checkCacheChannel = Channel.CreateUnbounded<BaseItem>();
+
+  public Task Start(string streamId, CancellationToken cancellationToken = default) =>
+    _checkCacheChannel
+      .Reader.Batch(HTTP_SEND_CHUNK_SIZE)
+      .WithTimeout(HTTP_BATCH_TIMEOUT)
+      .PipeAsync(
+        MAX_PARALLELISM_HTTP,
+        async x => await SendToServer(streamId, x, cancellationToken).ConfigureAwait(false),
+        -1,
+        false,
+        cancellationToken
+      )
+      .Join()
+      .Batch(MAX_CACHE_BATCH)
+      .WithTimeout(HTTP_BATCH_TIMEOUT)
+      .ReadAllConcurrently(MAX_CACHE_WRITE_PARALLELISM, SaveToCache, cancellationToken);
+
+  public async Task Save(BaseItem item, CancellationToken cancellationToken = default) =>
+    await _checkCacheChannel.Writer.WriteAsync(item, cancellationToken).ConfigureAwait(false);
+
+  public void Done() => _checkCacheChannel.Writer.TryComplete();
+
+  public abstract Task<List<BaseItem>> SendToServer(
+    string streamId,
+    List<BaseItem> batch,
+    CancellationToken cancellationToken
+  );
+
+  public abstract void SaveToCache(List<BaseItem> item);
+}

src/Speckle.Sdk/Api/Operations/Operations.Receive.cs

@@ -26,7 +26,7 @@ public partial class Operations
 
     try
     {
-      var sqliteTransport = new SQLiteCacheManager(streamId);
+      var sqliteTransport = new SQLiteReceiveCacheManager(streamId);
       var serverObjects = new ServerObjectManager(speckleHttp, activityFactory, url, authorizationToken);
       var o = new ObjectLoader(sqliteTransport, serverObjects, streamId, onProgressAction);
       var process = new DeserializeProcess(onProgressAction, o);

src/Speckle.Sdk/Api/Operations/Operations.Send.cs

@@ -4,12 +4,52 @@ using Speckle.Newtonsoft.Json.Linq;
 using Speckle.Sdk.Logging;
 using Speckle.Sdk.Models;
 using Speckle.Sdk.Serialisation;
+using Speckle.Sdk.Serialisation.V2;
+using Speckle.Sdk.Serialisation.V2.Send;
 using Speckle.Sdk.Transports;
 
 namespace Speckle.Sdk.Api;
 
 public partial class Operations
 {
+  public async Task<(string rootObjId, IReadOnlyDictionary<string, ObjectReference> convertedReferences)> Send2(
+    Uri url,
+    string streamId,
+    string? authorizationToken,
+    Base value,
+    IProgress<ProgressArgs>? onProgressAction = null,
+    CancellationToken cancellationToken = default
+  )
+  {
+    using var receiveActivity = activityFactory.Start("Operations.Send");
+    metricsFactory.CreateCounter<long>("Send").Add(1);
+
+    try
+    {
+      var sqliteTransport = new SQLiteSendCacheManager(streamId);
+      var serverObjects = new ServerObjectManager(speckleHttp, activityFactory, url, authorizationToken);
+      var process = new SerializeProcess(
+        onProgressAction,
+        sqliteTransport,
+        serverObjects,
+        speckleBaseChildFinder,
+        speckleBasePropertyGatherer
+      );
+      var (rootObjId, convertedReferences) = await process
+        .Serialize(streamId, value, cancellationToken)
+        .ConfigureAwait(false);
+
+      receiveActivity?.SetStatus(SdkActivityStatusCode.Ok);
+      return new(rootObjId, convertedReferences);
+    }
+    catch (Exception ex)
+    {
+      receiveActivity?.SetStatus(SdkActivityStatusCode.Error);
+      receiveActivity?.RecordException(ex);
+      throw;
+    }
+  }
+
   /// <summary>
   /// Sends a Speckle Object to the provided <paramref name="transport"/> and (optionally) the default local cache
   /// </summary>

src/Speckle.Sdk/Api/Operations/Operations.cs

@@ -2,6 +2,7 @@ using Microsoft.Extensions.Logging;
 using Speckle.InterfaceGenerator;
 using Speckle.Sdk.Helpers;
 using Speckle.Sdk.Logging;
+using Speckle.Sdk.Serialisation.V2.Send;
 
 namespace Speckle.Sdk.Api;
 
@@ -15,5 +16,7 @@ public partial class Operations(
   ILogger<Operations> logger,
   ISpeckleHttp speckleHttp,
   ISdkActivityFactory activityFactory,
-  ISdkMetricsFactory metricsFactory
+  ISdkMetricsFactory metricsFactory,
+  ISpeckleBaseChildFinder speckleBaseChildFinder,
+  ISpeckleBasePropertyGatherer speckleBasePropertyGatherer
 ) : IOperations;

src/Speckle.Sdk/Credentials/AccountManager.cs

@@ -807,7 +807,7 @@ public class AccountManager(ISpeckleApplication application, ILogger<AccountMana
   /// <param name="server">Server endpoint to get header</param>
   /// <returns><see langword="true"/> if response contains FE2 header and the value was <see langword="true"/></returns>
   /// <exception cref="SpeckleException">response contained FE2 header, but the value was <see langword="null"/>, empty, or not parseable to a <see cref="Boolean"/></exception>
-  /// <exception cref="HttpRequestException">Request to <paramref name="server"/> failed to send or response was not successful</exception>
+  /// <exception cref="System.Net.Http.HttpRequestException">Request to <paramref name="server"/> failed to send or response was not successful</exception>
   private async Task<bool> IsFrontend2Server(Uri server)
   {
     using var httpClient = speckleHttp.CreateHttpClient();

src/Speckle.Sdk/Credentials/AuthFlowException.cs

@@ -1,6 +1,8 @@
 namespace Speckle.Sdk.Credentials;
 
+#pragma warning disable CA2237
 public sealed class AuthFlowException : Exception
+#pragma warning restore CA2237
 {
   public AuthFlowException(string? message, Exception? innerException)
     : base(message, innerException) { }

src/Speckle.Sdk/Helpers/Http.cs

@@ -13,7 +13,7 @@ public class SpeckleHttp(ILogger<SpeckleHttp> logger, ISpeckleHttpClientHandlerF
   /// Sends a <c>GET</c> request to the provided <paramref name="uri"/>
   /// </summary>
   /// <param name="uri">The URI that should be pinged</param>
-  /// <exception cref="HttpRequestException">Request to <paramref name="uri"/> failed</exception>
+  /// <exception cref="System.Net.Http.HttpRequestException">Request to <paramref name="uri"/> failed</exception>
   public async Task<HttpResponseMessage> HttpPing(Uri uri)
   {
     try

src/Speckle.Sdk/Serialisation/IdGenerator.cs

@@ -0,0 +1,19 @@
+using System.Diagnostics.Contracts;
+using Speckle.Sdk.Helpers;
+using Speckle.Sdk.Models;
+
+namespace Speckle.Sdk.Serialisation;
+
+public static class IdGenerator
+{
+  [Pure]
+  public static string ComputeId(string serialized)
+  {
+#if NET6_0_OR_GREATER
+    string hash = Crypt.Sha256(serialized.AsSpan(), length: HashUtility.HASH_LENGTH);
+#else
+    string hash = Crypt.Sha256(serialized, length: HashUtility.HASH_LENGTH);
+#endif
+    return hash;
+  }
+}

src/Speckle.Sdk/Serialisation/SpeckleObjectDeserializer.cs

@@ -135,7 +135,7 @@ public sealed class SpeckleObjectDeserializer
               var closures = ClosureParser.GetClosures(reader);
               if (closures.Any())
               {
-                _total = closures.Select(x => x.Item1).Concat(_deserializedObjects.Keys).Distinct().Count();
+                _total = 0;
                 foreach (var closure in closures)
                 {
                   string objId = closure.Item1;

src/Speckle.Sdk/Serialisation/SpeckleObjectSerializer.cs

@@ -1,6 +1,5 @@
 using System.Collections;
 using System.Diagnostics;
-using System.Diagnostics.Contracts;
 using System.Drawing;
 using System.Globalization;
 using System.Reflection;
@@ -359,7 +358,7 @@ public class SpeckleObjectSerializer
     if (writer is SerializerIdWriter serializerIdWriter)
     {
       (var json, writer) = serializerIdWriter.FinishIdWriter();
-      id = ComputeId(json);
+      id = IdGenerator.ComputeId(json);
     }
     else
     {
@@ -434,17 +433,6 @@ public class SpeckleObjectSerializer
     }
   }
 
-  [Pure]
-  private static string ComputeId(string serialized)
-  {
-#if NET6_0_OR_GREATER
-    string hash = Crypt.Sha256(serialized.AsSpan(), length: HashUtility.HASH_LENGTH);
-#else
-    string hash = Crypt.Sha256(serialized, length: HashUtility.HASH_LENGTH);
-#endif
-    return hash;
-  }
-
   private void StoreObject(string objectId, string objectJson)
   {
     _stopwatch.Stop();

src/Speckle.Sdk/Serialisation/V2/Receive/DeserializeProcess.cs

@@ -1,4 +1,5 @@
 using System.Collections.Concurrent;
+using System.Data;
 using Speckle.Sdk.Models;
 using Speckle.Sdk.Serialisation.Utilities;
 using Speckle.Sdk.Transports;
@@ -89,7 +90,7 @@ public sealed class DeserializeProcess(IProgress<ProgressArgs>? progress, IObjec
       var json = objectLoader.LoadId(id);
       if (json == null)
       {
-        throw new InvalidOperationException();
+        throw new MissingPrimaryKeyException($"Missing object id in SQLite cache: {id}");
       }
       var childrenIds = ClosureParser.GetClosures(json).OrderByDescending(x => x.Item2).Select(x => x.Item1).ToList();
       closures = (json, childrenIds);

src/Speckle.Sdk/Serialisation/V2/Receive/ObjectLoader.cs

@@ -1,5 +1,6 @@
 using Speckle.InterfaceGenerator;
 using Speckle.Sdk.Common;
+using Speckle.Sdk.Dependencies.Serialization;
 using Speckle.Sdk.Serialisation.Utilities;
 using Speckle.Sdk.Transports;
 
@@ -7,7 +8,7 @@ namespace Speckle.Sdk.Serialisation.V2.Receive;
 
 [GenerateAutoInterface]
 public sealed class ObjectLoader(
-  ISQLiteCacheManager sqLiteCacheManager,
+  ISQLiteReceiveCacheManager sqliteReceiveCacheManager,
   IServerObjectManager serverObjectManager,
   string streamId,
   IProgress<ProgressArgs>? progress
@@ -28,7 +29,7 @@ public sealed class ObjectLoader(
     string? rootJson;
     if (!options.SkipCache)
     {
-      rootJson = sqLiteCacheManager.GetObject(rootId);
+      rootJson = sqliteReceiveCacheManager.GetObject(rootId);
       if (rootJson != null)
       {
         //assume everything exists as the root is there.
@@ -50,7 +51,10 @@ public sealed class ObjectLoader(
     await GetAndCache(allChildrenIds, cancellationToken).ConfigureAwait(false);
 
     //save the root last to shortcut later
-    sqLiteCacheManager.SaveObjectSync(rootId, rootJson);
+    if (!options.SkipCache)
+    {
+      sqliteReceiveCacheManager.SaveObject(new(rootId, rootJson, true));
+    }
     return (rootJson, allChildrenIds);
   }
 
@@ -59,7 +63,7 @@ public sealed class ObjectLoader(
   {
     _checkCache++;
     progress?.Report(new(ProgressEvent.CacheCheck, _checkCache, _allChildrenCount));
-    if (!_options.SkipCache && !sqLiteCacheManager.HasObject(id))
+    if (!_options.SkipCache && !sqliteReceiveCacheManager.HasObject(id))
     {
       return id;
     }
@@ -68,11 +72,9 @@ public sealed class ObjectLoader(
   }
 
   [AutoInterfaceIgnore]
-  public override async Task<List<(string, string)>> DownloadAndCache(List<string?> ids)
+  public override async Task<List<BaseItem>> Download(List<string?> ids)
   {
-    var count = 0L;
-    progress?.Report(new(ProgressEvent.DownloadObject, count, _allChildrenCount));
-    var toCache = new List<(string, string)>();
+    var toCache = new List<BaseItem>();
     await foreach (
       var (id, json) in serverObjectManager.DownloadObjects(
         streamId,
@@ -82,25 +84,22 @@ public sealed class ObjectLoader(
       )
     )
     {
-      count++;
-      progress?.Report(new(ProgressEvent.DownloadObject, count, _allChildrenCount));
-      toCache.Add((id, json));
+      toCache.Add(new(id, json, true));
     }
 
     return toCache;
   }
 
   [AutoInterfaceIgnore]
-  public override void SaveToCache((string, string) x)
+  public override void SaveToCache(BaseItem x)
   {
     if (!_options.SkipCache)
     {
-      sqLiteCacheManager.SaveObjectSync(x.Item1, x.Item2);
+      sqliteReceiveCacheManager.SaveObject(x);
+      _cached++;
+      progress?.Report(new(ProgressEvent.CachedToLocal, _cached, _allChildrenCount));
     }
-
-    _cached++;
-    progress?.Report(new(ProgressEvent.Cached, _cached, _allChildrenCount));
   }
 
-  public string? LoadId(string id) => sqLiteCacheManager.GetObject(id);
+  public string? LoadId(string id) => sqliteReceiveCacheManager.GetObject(id);
 }

src/Speckle.Sdk/Serialisation/V2/SQLiteCacheManager.cs

@@ -1,19 +1,16 @@
-using Microsoft.Data.Sqlite;
-using Speckle.InterfaceGenerator;
+using Microsoft.Data.Sqlite;
 using Speckle.Sdk.Logging;
 using Speckle.Sdk.Transports;
 
 namespace Speckle.Sdk.Serialisation.V2;
 
-[GenerateAutoInterface]
-public class SQLiteCacheManager : ISQLiteCacheManager
+public abstract class SQLiteCacheManager
 {
   private readonly string _rootPath;
-  private readonly string _connectionString;
   private const string APPLICATION_NAME = "Speckle";
   private const string DATA_FOLDER = "Projects";
 
-  public SQLiteCacheManager(string streamId)
+  protected SQLiteCacheManager(string streamId)
   {
     var basePath = SpecklePathProvider.UserApplicationDataPath();
 
@@ -30,7 +27,7 @@ public class SQLiteCacheManager : ISQLiteCacheManager
       throw new TransportException($"Path was invalid or could not be created {_rootPath}", ex);
     }
 
-    _connectionString = $"Data Source={_rootPath};";
+    ConnectionString = $"Data Source={_rootPath};";
     Initialize();
   }
 
@@ -43,7 +40,7 @@ public class SQLiteCacheManager : ISQLiteCacheManager
     //  foreach (var str2 in HexChars)
     //    cart.Add(str + str2);
 
-    using var c = new SqliteConnection(_connectionString);
+    using var c = new SqliteConnection(ConnectionString);
     c.Open();
     const string COMMAND_TEXT =
       @"
@@ -71,44 +68,13 @@ public class SQLiteCacheManager : ISQLiteCacheManager
 
     using SqliteCommand cmd2 = new("PRAGMA temp_store=MEMORY;", c);
     cmd2.ExecuteNonQuery();
+
+    using SqliteCommand cmd3 = new("PRAGMA mmap_size = 30000000000;", c);
+    cmd3.ExecuteNonQuery();
+
+    using SqliteCommand cmd4 = new("PRAGMA page_size = 32768;", c);
+    cmd4.ExecuteNonQuery();
   }
 
-  public string? GetObject(string id)
-  {
-    using var c = new SqliteConnection(_connectionString);
-    c.Open();
-    using var command = new SqliteCommand("SELECT * FROM objects WHERE hash = @hash LIMIT 1 ", c);
-    command.Parameters.AddWithValue("@hash", id);
-    using var reader = command.ExecuteReader();
-    if (reader.Read())
-    {
-      return reader.GetString(1);
-    }
-    return null; // pass on the duty of null checks to consumers
-  }
-
-  public bool HasObject(string objectId)
-  {
-    using var c = new SqliteConnection(_connectionString);
-    c.Open();
-    const string COMMAND_TEXT = "SELECT 1 FROM objects WHERE hash = @hash LIMIT 1 ";
-    using var command = new SqliteCommand(COMMAND_TEXT, c);
-    command.Parameters.AddWithValue("@hash", objectId);
-
-    using var reader = command.ExecuteReader();
-    bool rowFound = reader.Read();
-    return rowFound;
-  }
-
-  public void SaveObjectSync(string hash, string serializedObject)
-  {
-    using var c = new SqliteConnection(_connectionString);
-    c.Open();
-    const string COMMAND_TEXT = "INSERT OR IGNORE INTO objects(hash, content) VALUES(@hash, @content)";
-
-    using var command = new SqliteCommand(COMMAND_TEXT, c);
-    command.Parameters.AddWithValue("@hash", hash);
-    command.Parameters.AddWithValue("@content", serializedObject);
-    command.ExecuteNonQuery();
-  }
+  protected string ConnectionString { get; }
 }

src/Speckle.Sdk/Serialisation/V2/SQLiteReceiveCacheManager.cs

@@ -0,0 +1,49 @@
+using Microsoft.Data.Sqlite;
+using Speckle.InterfaceGenerator;
+using Speckle.Sdk.Dependencies.Serialization;
+
+namespace Speckle.Sdk.Serialisation.V2;
+
+[GenerateAutoInterface]
+public class SQLiteReceiveCacheManager(string streamId) : SQLiteCacheManager(streamId), ISQLiteReceiveCacheManager
+{
+  public string? GetObject(string id)
+  {
+    using var c = new SqliteConnection(ConnectionString);
+    c.Open();
+    using var command = new SqliteCommand("SELECT * FROM objects WHERE hash = @hash LIMIT 1 ", c);
+    command.Parameters.AddWithValue("@hash", id);
+    using var reader = command.ExecuteReader();
+    if (reader.Read())
+    {
+      return reader.GetString(1);
+    }
+
+    return null; // pass on the duty of null checks to consumers
+  }
+
+  public void SaveObject(BaseItem item)
+  {
+    using var c = new SqliteConnection(ConnectionString);
+    c.Open();
+    const string COMMAND_TEXT = "INSERT OR IGNORE INTO objects(hash, content) VALUES(@hash, @content)";
+
+    using var command = new SqliteCommand(COMMAND_TEXT, c);
+    command.Parameters.AddWithValue("@hash", item.Id);
+    command.Parameters.AddWithValue("@content", item.Json);
+    command.ExecuteNonQuery();
+  }
+
+  public bool HasObject(string objectId)
+  {
+    using var c = new SqliteConnection(ConnectionString);
+    c.Open();
+    const string COMMAND_TEXT = "SELECT 1 FROM objects WHERE hash = @hash LIMIT 1 ";
+    using var command = new SqliteCommand(COMMAND_TEXT, c);
+    command.Parameters.AddWithValue("@hash", objectId);
+
+    using var reader = command.ExecuteReader();
+    bool rowFound = reader.Read();
+    return rowFound;
+  }
+}

src/Speckle.Sdk/Serialisation/V2/SQLiteSendCacheManager.cs

@@ -0,0 +1,57 @@
+using Microsoft.Data.Sqlite;
+using Speckle.InterfaceGenerator;
+using Speckle.Sdk.Dependencies.Serialization;
+
+namespace Speckle.Sdk.Serialisation.V2;
+
+[GenerateAutoInterface]
+public class SQLiteSendCacheManager(string streamId) : SQLiteCacheManager(streamId), ISQLiteSendCacheManager
+{
+  public string? GetObject(string id)
+  {
+    using var c = new SqliteConnection(ConnectionString);
+    c.Open();
+    using var command = new SqliteCommand("SELECT * FROM objects WHERE hash = @hash LIMIT 1 ", c);
+    command.Parameters.AddWithValue("@hash", id);
+    using var reader = command.ExecuteReader();
+    if (reader.Read())
+    {
+      return reader.GetString(1);
+    }
+
+    return null; // pass on the duty of null checks to consumers
+  }
+
+  public bool HasObject(string objectId)
+  {
+    using var c = new SqliteConnection(ConnectionString);
+    c.Open();
+    const string COMMAND_TEXT = "SELECT 1 FROM objects WHERE hash = @hash LIMIT 1 ";
+    using var command = new SqliteCommand(COMMAND_TEXT, c);
+    command.Parameters.AddWithValue("@hash", objectId);
+
+    using var reader = command.ExecuteReader();
+    bool rowFound = reader.Read();
+    return rowFound;
+  }
+
+  public void SaveObjects(List<BaseItem> items)
+  {
+    using var c = new SqliteConnection(ConnectionString);
+    c.Open();
+    using var t = c.BeginTransaction();
+    const string COMMAND_TEXT = "INSERT OR IGNORE INTO objects(hash, content) VALUES(@hash, @content)";
+
+    using var command = new SqliteCommand(COMMAND_TEXT, c);
+    command.Transaction = t;
+    var idParam = command.Parameters.Add("@hash", SqliteType.Text);
+    var jsonParam = command.Parameters.Add("@content", SqliteType.Text);
+    foreach (var item in items)
+    {
+      idParam.Value = item.Id;
+      jsonParam.Value = item.Json;
+      command.ExecuteNonQuery();
+    }
+    t.Commit();
+  }
+}

src/Speckle.Sdk/Serialisation/V2/Send/PropertyAttributeInfo.cs

@@ -0,0 +1,24 @@
+using Speckle.Newtonsoft.Json;
+
+namespace Speckle.Sdk.Serialisation.V2.Send;
+
+public readonly struct PropertyAttributeInfo
+{
+  public PropertyAttributeInfo(
+    bool isDetachable,
+    bool isChunkable,
+    int chunkSize,
+    JsonPropertyAttribute? jsonPropertyAttribute
+  )
+  {
+    IsDetachable = isDetachable || isChunkable;
+    IsChunkable = isChunkable;
+    ChunkSize = chunkSize;
+    JsonPropertyInfo = jsonPropertyAttribute;
+  }
+
+  public readonly bool IsDetachable;
+  public readonly bool IsChunkable;
+  public readonly int ChunkSize;
+  public readonly JsonPropertyAttribute? JsonPropertyInfo;
+}

src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs

@@ -0,0 +1,156 @@
+using System.Collections.Concurrent;
+using Speckle.Sdk.Common;
+using Speckle.Sdk.Dependencies.Serialization;
+using Speckle.Sdk.Models;
+using Speckle.Sdk.Transports;
+
+namespace Speckle.Sdk.Serialisation.V2.Send;
+
+public record SerializeProcessOptions(bool SkipCache, bool SkipServer);
+
+public class SerializeProcess(
+  IProgress<ProgressArgs>? progress,
+  ISQLiteSendCacheManager sqliteSendCacheManager,
+  IServerObjectManager serverObjectManager,
+  ISpeckleBaseChildFinder speckleBaseChildFinder,
+  ISpeckleBasePropertyGatherer speckleBasePropertyGatherer
+) : ChannelSaver
+{
+  private readonly ConcurrentDictionary<string, string> _jsonCache = new();
+  private readonly ConcurrentDictionary<string, ObjectReference> _objectReferences = new();
+
+  private long _total;
+  private long _cached;
+  private long _serialized;
+
+  private SerializeProcessOptions _options = new(false, false);
+
+  public async Task<(string rootObjId, IReadOnlyDictionary<string, ObjectReference> convertedReferences)> Serialize(
+    string streamId,
+    Base root,
+    CancellationToken cancellationToken,
+    SerializeProcessOptions? options = null
+  )
+  {
+    _options = options ?? _options;
+    var channelTask = Start(streamId, cancellationToken);
+    await Traverse(root, true, cancellationToken).ConfigureAwait(false);
+    await channelTask.ConfigureAwait(false);
+    return (root.id, _objectReferences);
+  }
+
+  private async Task<List<Dictionary<string, int>>> Traverse(Base obj, bool isEnd, CancellationToken cancellationToken)
+  {
+    var tasks = new List<Task<List<Dictionary<string, int>>>>();
+    foreach (var child in speckleBaseChildFinder.GetChildren(obj))
+    {
+      Interlocked.Increment(ref _total);
+      // tmp is necessary because of the way closures close over loop variables
+      var tmp = child;
+      var t = Task
+        .Factory.StartNew(
+          () => Traverse(tmp, false, cancellationToken),
+          cancellationToken,
+          TaskCreationOptions.AttachedToParent,
+          TaskScheduler.Default
+        )
+        .Unwrap();
+      tasks.Add(t);
+    }
+
+    if (tasks.Count > 0)
+    {
+      await Task.WhenAll(tasks).ConfigureAwait(false);
+    }
+    var closures = tasks
+      .Select(t => t.Result)
+      .Aggregate(
+        new List<Dictionary<string, int>>(),
+        (a, s) =>
+        {
+          a.AddRange(s);
+          return a;
+        }
+      )
+      .ToList();
+
+    var item = Serialise(obj, closures);
+    Interlocked.Increment(ref _serialized);
+    progress?.Report(new(ProgressEvent.FromCacheOrSerialized, _serialized, _total));
+    if (item?.NeedsStorage ?? false)
+    {
+      await Save(item.Value, cancellationToken).ConfigureAwait(false);
+    }
+    if (isEnd)
+    {
+      Done();
+    }
+    return closures;
+  }
+
+  //leave this sync
+  private BaseItem? Serialise(Base obj, List<Dictionary<string, int>> childClosures)
+  {
+    if (obj.id != null && _jsonCache.ContainsKey(obj.id))
+    {
+      return null;
+    }
+
+    string? json = null;
+    if (!_options.SkipCache && obj.id != null)
+    {
+      json = sqliteSendCacheManager.GetObject(obj.id);
+    }
+    if (json == null)
+    {
+      var id = obj.id;
+      if (id is null || !_jsonCache.TryGetValue(id, out json))
+      {
+        SpeckleObjectSerializer2 serializer2 = new(speckleBasePropertyGatherer, childClosures);
+        json = serializer2.Serialize(obj);
+        obj.id.NotNull();
+        foreach (var kvp in serializer2.ObjectReferences)
+        {
+          _objectReferences.TryAdd(kvp.Key, kvp.Value);
+        }
+
+        _jsonCache.TryAdd(obj.id, json);
+        if (id is not null && id != obj.id)
+        {
+          //in case the ids changes which is due to id hash algorithm changing
+          _jsonCache.TryAdd(id, json);
+        }
+      }
+      return new BaseItem(obj.id.NotNull(), json, true);
+    }
+    return new BaseItem(obj.id.NotNull(), json.NotNull(), false);
+  }
+
+  public override async Task<List<BaseItem>> SendToServer(
+    string streamId,
+    List<BaseItem> batch,
+    CancellationToken cancellationToken
+  )
+  {
+    if (batch.Count == 0)
+    {
+      return batch;
+    }
+
+    if (!_options.SkipServer)
+    {
+      await serverObjectManager.UploadObjects(streamId, batch, true, progress, cancellationToken).ConfigureAwait(false);
+    }
+    return batch;
+  }
+
+  public override void SaveToCache(List<BaseItem> items)
+  {
+    if (!_options.SkipCache)
+    {
+      sqliteSendCacheManager.SaveObjects(items);
+      Interlocked.Exchange(ref _cached, _cached + items.Count);
+      progress?.Report(new(ProgressEvent.CachedToLocal, _cached, null));
+    }
+  }
+}

src/Speckle.Sdk/Serialisation/V2/Send/SpeckleBaseChildFinder.cs

@@ -0,0 +1,41 @@
+using System.Collections;
+using Speckle.InterfaceGenerator;
+using Speckle.Sdk.Models;
+
+namespace Speckle.Sdk.Serialisation.V2.Send;
+
+[GenerateAutoInterface]
+public class SpeckleBaseChildFinder(ISpeckleBasePropertyGatherer propertyGatherer) : ISpeckleBaseChildFinder
+{
+  public IEnumerable<Base> GetChildren(Base obj)
+  {
+    var props = propertyGatherer.ExtractAllProperties(obj);
+    foreach (var kvp in props.Where(x => x.PropertyAttributeInfo.IsDetachable))
+    {
+      if (kvp.Value is Base child)
+      {
+        yield return child;
+      }
+      if (kvp.Value is ICollection c)
+      {
+        foreach (var childC in c)
+        {
+          if (childC is Base b)
+          {
+            yield return b;
+          }
+        }
+      }
+      if (kvp.Value is IDictionary d)
+      {
+        foreach (DictionaryEntry de in d)
+        {
+          if (de.Value is Base b)
+          {
+            yield return b;
+          }
+        }
+      }
+    }
+  }
+}

src/Speckle.Sdk/Serialisation/V2/Send/SpeckleBasePropertyGatherer.cs

@@ -0,0 +1,99 @@
+using System.Collections.Concurrent;
+using System.Reflection;
+using Speckle.InterfaceGenerator;
+using Speckle.Newtonsoft.Json;
+using Speckle.Sdk.Common;
+using Speckle.Sdk.Helpers;
+using Speckle.Sdk.Models;
+
+namespace Speckle.Sdk.Serialisation.V2.Send;
+
+public readonly record struct Property(string Name, object? Value, PropertyAttributeInfo PropertyAttributeInfo);
+
+[GenerateAutoInterface]
+public class SpeckleBasePropertyGatherer : ISpeckleBasePropertyGatherer
+{
+  private readonly ConcurrentDictionary<string, List<(PropertyInfo, PropertyAttributeInfo)>> _typedPropertiesCache =
+    new();
+
+  public IEnumerable<Property> ExtractAllProperties(Base baseObj)
+  {
+    IReadOnlyList<(PropertyInfo, PropertyAttributeInfo)> typedProperties = GetTypedPropertiesWithCache(baseObj);
+    IReadOnlyCollection<string> dynamicProperties = baseObj.DynamicPropertyKeys;
+
+    // Construct `allProperties`: Add typed properties
+    foreach ((PropertyInfo propertyInfo, PropertyAttributeInfo detachInfo) in typedProperties)
+    {
+      object? baseValue = propertyInfo.GetValue(baseObj);
+      yield return new(propertyInfo.Name, baseValue, detachInfo);
+    }
+
+    // Construct `allProperties`: Add dynamic properties
+    foreach (string propName in dynamicProperties)
+    {
+      if (propName.StartsWith("__"))
+      {
+        continue;
+      }
+
+      object? baseValue = baseObj[propName];
+
+      bool isDetachable = PropNameValidator.IsDetached(propName);
+
+      int chunkSize = 1000;
+      bool isChunkable = isDetachable && PropNameValidator.IsChunkable(propName, out chunkSize);
+
+      yield return new(propName, baseValue, new PropertyAttributeInfo(isDetachable, isChunkable, chunkSize, null));
+    }
+  }
+
+  // (propertyInfo, isDetachable, isChunkable, chunkSize, JsonPropertyAttribute)
+  private IReadOnlyList<(PropertyInfo, PropertyAttributeInfo)> GetTypedPropertiesWithCache(Base baseObj)
+  {
+    Type type = baseObj.GetType();
+
+    if (
+      _typedPropertiesCache.TryGetValue(
+        type.FullName.NotNull(),
+        out List<(PropertyInfo, PropertyAttributeInfo)>? cached
+      )
+    )
+    {
+      return cached;
+    }
+
+    var typedProperties = baseObj.GetInstanceMembers().ToList();
+    List<(PropertyInfo, PropertyAttributeInfo)> ret = new(typedProperties.Count);
+
+    foreach (PropertyInfo typedProperty in typedProperties)
+    {
+      if (typedProperty.Name.StartsWith("__") || typedProperty.Name == "id")
+      {
+        continue;
+      }
+
+      bool jsonIgnore = typedProperty.IsDefined(typeof(JsonIgnoreAttribute), false);
+      if (jsonIgnore)
+      {
+        continue;
+      }
+
+      _ = typedProperty.GetValue(baseObj);
+
+      List<DetachPropertyAttribute> detachableAttributes = typedProperty
+        .GetCustomAttributes<DetachPropertyAttribute>(true)
+        .ToList();
+      List<ChunkableAttribute> chunkableAttributes = typedProperty
+        .GetCustomAttributes<ChunkableAttribute>(true)
+        .ToList();
+      bool isDetachable = detachableAttributes.Count > 0 && detachableAttributes[0].Detachable;
+      bool isChunkable = chunkableAttributes.Count > 0;
+      int chunkSize = isChunkable ? chunkableAttributes[0].MaxObjCountPerChunk : 1000;
+      JsonPropertyAttribute? jsonPropertyAttribute = typedProperty.GetCustomAttribute<JsonPropertyAttribute>();
+      ret.Add((typedProperty, new PropertyAttributeInfo(isDetachable, isChunkable, chunkSize, jsonPropertyAttribute)));
+    }
+
+    _typedPropertiesCache[type.FullName] = ret;
+    return ret;
+  }
+}

src/Speckle.Sdk/Serialisation/V2/Send/SpeckleObjectSerializer.cs

@@ -0,0 +1,361 @@
+using System.Collections;
+using System.Drawing;
+using System.Globalization;
+using Speckle.DoubleNumerics;
+using Speckle.Newtonsoft.Json;
+using Speckle.Sdk.Common;
+using Speckle.Sdk.Helpers;
+using Speckle.Sdk.Models;
+
+namespace Speckle.Sdk.Serialisation.V2.Send;
+
+public class SpeckleObjectSerializer2
+{
+  private HashSet<object> _parentObjects = new();
+  private readonly List<Dictionary<string, int>> _childclosures;
+
+  private readonly bool _trackDetachedChildren;
+  private readonly ISpeckleBasePropertyGatherer _propertyGatherer;
+  private readonly CancellationToken _cancellationToken;
+
+  /// <summary>
+  /// Keeps track of all detached children created during serialisation that have an applicationId (provided this serializer instance has been told to track detached children).
+  /// This is currently used to cache previously converted objects and avoid their conversion if they haven't changed. See the DUI3 send bindings in rhino or another host app.
+  /// </summary>
+  public Dictionary<string, ObjectReference> ObjectReferences { get; } = new();
+
+  /// <summary>
+  /// Creates a new Serializer instance.
+  /// </summary>
+  /// <param name="trackDetachedChildren">Whether to store all detachable objects while serializing. They can be retrieved via <see cref="ObjectReferences"/> post serialization.</param>
+  /// <param name="cancellationToken"></param>
+  public SpeckleObjectSerializer2(
+    ISpeckleBasePropertyGatherer propertyGatherer,
+    List<Dictionary<string, int>> childclosures,
+    bool trackDetachedChildren = false,
+    CancellationToken cancellationToken = default
+  )
+  {
+    _childclosures = childclosures;
+    _propertyGatherer = propertyGatherer;
+    _cancellationToken = cancellationToken;
+    _trackDetachedChildren = trackDetachedChildren;
+  }
+
+  /// <param name="baseObj">The object to serialize</param>
+  /// <returns>The serialized JSON</returns>
+  /// <exception cref="InvalidOperationException">The serializer is busy (already serializing an object)</exception>
+  /// <exception cref="SpeckleSerializeException">Failed to extract (pre-serialize) properties from the <paramref name="baseObj"/></exception>
+  public string Serialize(Base baseObj)
+  {
+    try
+    {
+      try
+      {
+        var result = SerializeBase(baseObj, true).NotNull();
+        return result.Json;
+      }
+      catch (Exception ex) when (!ex.IsFatal() && ex is not OperationCanceledException)
+      {
+        throw new SpeckleSerializeException($"Failed to extract (pre-serialize) properties from the {baseObj}", ex);
+      }
+    }
+    finally
+    {
+      _parentObjects = new HashSet<object>();
+    }
+  }
+
+  // `Preserialize` means transforming all objects into the final form that will appear in json, with basic .net objects
+  // (primitives, lists and dictionaries with string keys)
+  private void SerializeProperty(
+    object? obj,
+    JsonWriter writer,
+    bool computeClosures = false,
+    PropertyAttributeInfo inheritedDetachInfo = default
+  )
+  {
+    _cancellationToken.ThrowIfCancellationRequested();
+
+    if (obj == null)
+    {
+      writer.WriteNull();
+      return;
+    }
+
+    if (obj.GetType().IsPrimitive || obj is string)
+    {
+      writer.WriteValue(obj);
+      return;
+    }
+
+    switch (obj)
+    {
+      // Start with object references so they're not captured by the Base class case below
+      // Note: this change was needed as we've made the ObjectReference type inherit from Base for
+      // the purpose of the "do not convert unchanged previously converted objects" POC.
+      case ObjectReference r:
+        Dictionary<string, object?> ret =
+          new()
+          {
+            ["speckle_type"] = r.speckle_type,
+            ["referencedId"] = r.referencedId,
+            ["__closure"] = r.closure,
+          };
+        if (r.closure is not null)
+        {
+          foreach (var kvp in r.closure)
+          {
+            UpdateChildClosures(kvp.Key);
+          }
+        }
+        UpdateChildClosures(r.referencedId);
+        SerializeProperty(ret, writer);
+        break;
+      case Base b:
+        var result = SerializeBase(b, computeClosures, inheritedDetachInfo);
+        if (result is not null)
+        {
+          writer.WriteRawValue(result.Json);
+        }
+        else
+        {
+          writer.WriteNull();
+        }
+        break;
+      case IDictionary d:
+        {
+          writer.WriteStartObject();
+
+          foreach (DictionaryEntry kvp in d)
+          {
+            if (kvp.Key is not string key)
+            {
+              throw new ArgumentException(
+                "Serializing dictionaries that are not string based keys is not supported",
+                nameof(obj)
+              );
+            }
+
+            writer.WritePropertyName(key);
+            SerializeProperty(kvp.Value, writer, inheritedDetachInfo: inheritedDetachInfo);
+          }
+          writer.WriteEndObject();
+        }
+        break;
+      case ICollection e:
+        {
+          writer.WriteStartArray();
+          foreach (object? element in e)
+          {
+            SerializeProperty(element, writer, inheritedDetachInfo: inheritedDetachInfo);
+          }
+          writer.WriteEndArray();
+        }
+        break;
+      case Enum:
+        writer.WriteValue((int)obj);
+        break;
+      // Support for simple types
+      case Guid g:
+        writer.WriteValue(g.ToString());
+        break;
+      case Color c:
+        writer.WriteValue(c.ToArgb());
+        break;
+      case DateTime t:
+        writer.WriteValue(t.ToString("o", CultureInfo.InvariantCulture));
+        break;
+      case Matrix4x4 md:
+        writer.WriteStartArray();
+
+        writer.WriteValue(md.M11);
+        writer.WriteValue(md.M12);
+        writer.WriteValue(md.M13);
+        writer.WriteValue(md.M14);
+        writer.WriteValue(md.M21);
+        writer.WriteValue(md.M22);
+        writer.WriteValue(md.M23);
+        writer.WriteValue(md.M24);
+        writer.WriteValue(md.M31);
+        writer.WriteValue(md.M32);
+        writer.WriteValue(md.M33);
+        writer.WriteValue(md.M34);
+        writer.WriteValue(md.M41);
+        writer.WriteValue(md.M42);
+        writer.WriteValue(md.M43);
+        writer.WriteValue(md.M44);
+        writer.WriteEndArray();
+        break;
+      //BACKWARDS COMPATIBILITY: matrix4x4 changed from System.Numerics float to System.DoubleNumerics double in release 2.16
+      case System.Numerics.Matrix4x4:
+        throw new ArgumentException("Please use Speckle.DoubleNumerics.Matrix4x4 instead", nameof(obj));
+      default:
+        throw new ArgumentException($"Unsupported value in serialization: {obj.GetType()}", nameof(obj));
+    }
+  }
+
+  private SerializationResult? SerializeBase(
+    Base baseObj,
+    bool computeClosures = false,
+    PropertyAttributeInfo inheritedDetachInfo = default
+  )
+  {
+    // handle circular references
+    bool alreadySerialized = !_parentObjects.Add(baseObj);
+    if (alreadySerialized)
+    {
+      return null;
+    }
+
+    Dictionary<string, int> closure = new();
+    string id;
+    string json;
+    lock (_childclosures)
+    {
+      if (computeClosures || inheritedDetachInfo.IsDetachable || baseObj is Blob)
+      {
+        _childclosures.Add(closure);
+      }
+
+      using var writer = new StringWriter();
+      using var jsonWriter = SpeckleObjectSerializerPool.Instance.GetJsonTextWriter(writer);
+      id = SerializeBaseObject(baseObj, jsonWriter, closure);
+      json = writer.ToString();
+
+      if (computeClosures || inheritedDetachInfo.IsDetachable || baseObj is Blob)
+      {
+        _childclosures.RemoveAt(_childclosures.Count - 1);
+      }
+    }
+
+    _parentObjects.Remove(baseObj);
+
+    if (baseObj is Blob)
+    {
+      throw new NotSupportedException();
+      /*StoreBlob(myBlob);
+      UpdateParentClosures($"blob:{id}");
+      return new(json, id);*/
+    }
+
+    if (inheritedDetachInfo.IsDetachable)
+    {
+      ObjectReference objRef = new() { referencedId = id.NotNull() };
+      using var writer2 = new StringWriter();
+      using var jsonWriter2 = SpeckleObjectSerializerPool.Instance.GetJsonTextWriter(writer2);
+      SerializeProperty(objRef, jsonWriter2);
+      var json2 = writer2.ToString();
+      UpdateChildClosures(id);
+
+      // add to obj refs to return
+      if (baseObj.applicationId != null && _trackDetachedChildren) // && baseObj is not DataChunk && baseObj is not Abstract) // not needed, as data chunks will never have application ids, and abstract objs are not really used.
+      {
+        ObjectReferences[baseObj.applicationId] = new ObjectReference()
+        {
+          referencedId = id,
+          applicationId = baseObj.applicationId,
+          closure = closure,
+        };
+      }
+      return new(json2, null);
+    }
+    return new(json.NotNull(), id);
+  }
+
+  private string SerializeBaseObject(Base baseObj, JsonWriter writer, IReadOnlyDictionary<string, int> closure)
+  {
+    if (baseObj is not Blob)
+    {
+      writer = new SerializerIdWriter(writer);
+    }
+
+    writer.WriteStartObject();
+    // Convert all properties
+    foreach (var prop in _propertyGatherer.ExtractAllProperties(baseObj))
+    {
+      if (prop.PropertyAttributeInfo.JsonPropertyInfo is { NullValueHandling: NullValueHandling.Ignore })
+      {
+        continue;
+      }
+
+      writer.WritePropertyName(prop.Name);
+      SerializeProperty(prop.Value, writer, prop.PropertyAttributeInfo);
+    }
+
+    string id;
+    if (writer is SerializerIdWriter serializerIdWriter)
+    {
+      (var json, writer) = serializerIdWriter.FinishIdWriter();
+      id = IdGenerator.ComputeId(json);
+    }
+    else
+    {
+      id = ((Blob)baseObj).id;
+    }
+    writer.WritePropertyName("id");
+    writer.WriteValue(id);
+    baseObj.id = id;
+
+    if (closure.Count > 0)
+    {
+      writer.WritePropertyName("__closure");
+      writer.WriteStartObject();
+      foreach (var c in closure)
+      {
+        writer.WritePropertyName(c.Key);
+        writer.WriteValue(c.Value);
+      }
+      writer.WriteEndObject();
+    }
+
+    writer.WriteEndObject();
+    return id;
+  }
+
+  private void SerializeProperty(object? baseValue, JsonWriter jsonWriter, PropertyAttributeInfo detachInfo)
+  {
+    if (baseValue is IEnumerable chunkableCollection && detachInfo.IsChunkable)
+    {
+      List<DataChunk> chunks = new();
+      DataChunk crtChunk = new() { data = new List<object?>(detachInfo.ChunkSize) };
+
+      foreach (object element in chunkableCollection)
+      {
+        crtChunk.data.Add(element);
+        if (crtChunk.data.Count >= detachInfo.ChunkSize)
+        {
+          chunks.Add(crtChunk);
+          crtChunk = new DataChunk { data = new List<object?>(detachInfo.ChunkSize) };
+        }
+      }
+
+      if (crtChunk.data.Count > 0)
+      {
+        chunks.Add(crtChunk);
+      }
+
+      SerializeProperty(chunks, jsonWriter, inheritedDetachInfo: new PropertyAttributeInfo(true, false, 0, null));
+      return;
+    }
+
+    SerializeProperty(baseValue, jsonWriter, inheritedDetachInfo: detachInfo);
+  }
+
+  private void UpdateChildClosures(string objectId)
+  {
+    lock (_childclosures)
+    {
+      for (int i = 0; i < _childclosures.Count; i++)
+      {
+        int childDepth = _childclosures.Count - i;
+        if (!_childclosures[i].TryGetValue(objectId, out int currentValue))
+        {
+          currentValue = childDepth;
+        }
+
+        _childclosures[i][objectId] = Math.Min(currentValue, childDepth);
+      }
+    }
+  }
+}

src/Speckle.Sdk/Serialisation/V2/ServerObjectManager.cs

@@ -1,11 +1,15 @@
 using System.Net;
+using System.Net.Http.Headers;
 using System.Runtime.CompilerServices;
 using System.Text;
 using Speckle.InterfaceGenerator;
 using Speckle.Newtonsoft.Json;
+using Speckle.Sdk.Common;
+using Speckle.Sdk.Dependencies.Serialization;
 using Speckle.Sdk.Helpers;
 using Speckle.Sdk.Logging;
 using Speckle.Sdk.Transports;
+using Speckle.Sdk.Transports.ServerUtils;
 
 namespace Speckle.Sdk.Serialisation.V2;
 
@@ -132,4 +136,79 @@ public class ServerObjectManager : IServerObjectManager
       }
     }
   }
+
+  public async Task<Dictionary<string, bool>> HasObjects(
+    string streamId,
+    IReadOnlyList<string> objectIds,
+    CancellationToken cancellationToken
+  )
+  {
+    cancellationToken.ThrowIfCancellationRequested();
+
+    // Stopwatch sw = new Stopwatch(); sw.Start();
+
+    string objectsPostParameter = JsonConvert.SerializeObject(objectIds);
+    var payload = new Dictionary<string, string> { { "objects", objectsPostParameter } };
+    string serializedPayload = JsonConvert.SerializeObject(payload);
+    var uri = new Uri($"/api/diff/{streamId}", UriKind.Relative);
+
+    using StringContent stringContent = new(serializedPayload, Encoding.UTF8, "application/json");
+    using HttpResponseMessage response = await _client
+      .PostAsync(uri, stringContent, cancellationToken)
+      .ConfigureAwait(false);
+
+    response.EnsureSuccessStatusCode();
+#if NET8_0_OR_GREATER
+    var hasObjects = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
+#else
+    var hasObjects = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
+#endif
+    return JsonConvert.DeserializeObject<Dictionary<string, bool>>(hasObjects).NotNull();
+  }
+
+  public async Task UploadObjects(
+    string streamId,
+    IReadOnlyList<BaseItem> objects,
+    bool compressPayloads,
+    IProgress<ProgressArgs>? progress,
+    CancellationToken cancellationToken
+  )
+  {
+    cancellationToken.ThrowIfCancellationRequested();
+
+    using HttpRequestMessage message =
+      new() { RequestUri = new Uri($"/objects/{streamId}", UriKind.Relative), Method = HttpMethod.Post };
+
+    MultipartFormDataContent multipart = new();
+
+    int mpId = 0;
+    var ctBuilder = new StringBuilder("[");
+    for (int i = 0; i < objects.Count; i++)
+    {
+      if (i > 0)
+      {
+        ctBuilder.Append(',');
+      }
+
+      ctBuilder.Append(objects[i].Json);
+    }
+    ctBuilder.Append(']');
+    string ct = ctBuilder.ToString();
+
+    if (compressPayloads)
+    {
+      var content = new GzipContent(new StringContent(ct, Encoding.UTF8));
+      content.Headers.ContentType = new MediaTypeHeaderValue("application/gzip");
+      multipart.Add(content, $"batch-{mpId}", $"batch-{mpId}");
+    }
+    else
+    {
+      multipart.Add(new StringContent(ct, Encoding.UTF8), $"batch-{mpId}", $"batch-{mpId}");
+    }
+
+    message.Content = new ProgressContent(multipart, progress);
+    HttpResponseMessage response = await _client.SendAsync(message, cancellationToken).ConfigureAwait(false);
+
+    response.EnsureSuccessStatusCode();
+  }
 }

src/Speckle.Sdk/SpeckleException.cs

@@ -1,6 +1,8 @@
 namespace Speckle.Sdk;
 
+#pragma warning disable CA2237
 public class SpeckleException : Exception
+#pragma warning restore CA2237
 {
   public SpeckleException() { }
 

src/Speckle.Sdk/Transports/DiskTransport.cs

@@ -93,7 +93,6 @@ public class DiskTransport : ICloneable, ITransport
     }
 
     SavedObjectCount++;
-    OnProgressAction?.Report(new(ProgressEvent.DownloadObject, SavedObjectCount, null));
     stopwatch.Stop();
     Elapsed += stopwatch.Elapsed;
   }

src/Speckle.Sdk/Transports/MemoryTransport.cs

@@ -85,7 +85,6 @@ public sealed class MemoryTransport : ITransport, ICloneable, IBlobCapableTransp
     _objects[id] = serializedObject;
 
     SavedObjectCount++;
-    OnProgressAction?.Report(new(ProgressEvent.UploadObject, 1, 1));
     stopwatch.Stop();
     Elapsed += stopwatch.Elapsed;
   }

src/Speckle.Sdk/Transports/ProgressArgs.cs

@@ -4,12 +4,14 @@ public readonly record struct ProgressArgs(ProgressEvent ProgressEvent, long Cou
 
 public enum ProgressEvent
 {
-  CacheCheck,
-  Cached,
-  DownloadBytes,
+  CachedToLocal, //send and receive
+
+  FromCacheOrSerialized,
   UploadBytes,
-  DownloadObject,
-  UploadObject,
+
+  CacheCheck,
+  DownloadBytes,
   DeserializeObject,
-  SerializeObject,
+
+  SerializeObject, // old
 }

src/Speckle.Sdk/Transports/SQLiteTransport.cs

@@ -330,8 +330,6 @@ public sealed class SQLiteTransport : IDisposable, ICloneable, ITransport, IBlob
         CancellationToken.ThrowIfCancellationRequested();
       }
 
-      OnProgressAction?.Report(new(ProgressEvent.DownloadObject, saved, _queue.Count + 1));
-
       CancellationToken.ThrowIfCancellationRequested();
 
       if (!_queue.IsEmpty)

src/Speckle.Sdk/Transports/TransportHelpers.cs

@@ -31,7 +31,6 @@ public static class TransportHelpers
 
     var closures = ClosureParser.GetChildrenIds(parent).ToList();
 
-    int i = 0;
     foreach (var closure in closures)
     {
       cancellationToken.ThrowIfCancellationRequested();
@@ -50,8 +49,6 @@ public static class TransportHelpers
       }
 
       targetTransport.SaveObject(closure, child);
-      var count = i++;
-      sourceTransport.OnProgressAction?.Report(new ProgressArgs(ProgressEvent.UploadObject, count, closures.Count));
     }
 
     return parent;

tests/Speckle.Sdk.Serialization.Testing/DummyServerObjectManager.cs

@@ -0,0 +1,47 @@
+using System.Text;
+using Speckle.Sdk.Dependencies.Serialization;
+using Speckle.Sdk.Serialisation.V2;
+using Speckle.Sdk.Transports;
+
+namespace Speckle.Sdk.Serialization.Testing;
+
+public class DummyServerObjectManager : IServerObjectManager
+{
+  public IAsyncEnumerable<(string, string)> DownloadObjects(
+    string streamId,
+    IReadOnlyList<string> objectIds,
+    IProgress<ProgressArgs>? progress,
+    CancellationToken cancellationToken
+  ) => throw new NotImplementedException();
+
+  public Task<string?> DownloadSingleObject(
+    string streamId,
+    string objectId,
+    IProgress<ProgressArgs>? progress,
+    CancellationToken cancellationToken
+  ) => throw new NotImplementedException();
+
+  public Task<Dictionary<string, bool>> HasObjects(
+    string streamId,
+    IReadOnlyList<string> objectIds,
+    CancellationToken cancellationToken
+  ) => throw new NotImplementedException();
+
+  public Task UploadObjects(
+    string streamId,
+    IReadOnlyList<BaseItem> objects,
+    bool compressPayloads,
+    IProgress<ProgressArgs>? progress,
+    CancellationToken cancellationToken
+  )
+  {
+    long totalBytes = 0;
+    foreach (var item in objects)
+    {
+      totalBytes += Encoding.Default.GetByteCount(item.Json);
+    }
+
+    progress?.Report(new(ProgressEvent.UploadBytes, totalBytes, totalBytes));
+    return Task.CompletedTask;
+  }
+}

tests/Speckle.Sdk.Serialization.Testing/Program.cs

@@ -1,12 +1,14 @@
 using System.Reflection;
 using Microsoft.Extensions.DependencyInjection;
 using Speckle.Sdk;
+using Speckle.Sdk.Credentials;
 using Speckle.Sdk.Helpers;
 using Speckle.Sdk.Host;
 using Speckle.Sdk.Logging;
 using Speckle.Sdk.Models;
 using Speckle.Sdk.Serialisation.V2;
 using Speckle.Sdk.Serialisation.V2.Receive;
+using Speckle.Sdk.Serialisation.V2.Send;
 using Speckle.Sdk.Serialization.Testing;
 
 const bool skipCache = false;
@@ -17,6 +19,10 @@ TypeLoader.Initialize(typeof(Base).Assembly, Assembly.GetExecutingAssembly());
 var url = "https://latest.speckle.systems/projects/a3ac1b2706/models/59d3b0f3c6"; //small?
 var streamId = "a3ac1b2706";
 var rootId = "7d53bcf28c6696ecac8781684a0aa006";*/
+/*
+var url = "https://latest.speckle.systems/"; //other?
+var streamId = "368f598929";
+var rootId = "67374cfe689c43ff8be12090af122244";*/
 
 
 var url = "https://latest.speckle.systems/projects/2099ac4b5f/models/da511c4d1e"; //perf?
@@ -28,19 +34,30 @@ serviceCollection.AddSpeckleSdk(HostApplications.Navisworks, HostAppVersion.v202
 var serviceProvider = serviceCollection.BuildServiceProvider();
 
 Console.WriteLine("Attach");
-Console.ReadLine();
-Console.WriteLine("Executing");
 
+var token = serviceProvider.GetRequiredService<IAccountManager>().GetDefaultAccount()?.token;
 var progress = new Progress(true);
-var sqliteTransport = new SQLiteCacheManager(streamId);
+var sqliteTransport = new SQLiteReceiveCacheManager(streamId);
 var serverObjects = new ServerObjectManager(
   serviceProvider.GetRequiredService<ISpeckleHttp>(),
   serviceProvider.GetRequiredService<ISdkActivityFactory>(),
   new Uri(url),
-  null
+  token
 );
 var o = new ObjectLoader(sqliteTransport, serverObjects, streamId, progress);
 var process = new DeserializeProcess(progress, o);
-await process.Deserialize(rootId, default, new(skipCache)).ConfigureAwait(false);
+var @base = await process.Deserialize(rootId, default, new(skipCache)).ConfigureAwait(false);
+Console.WriteLine("Deserialized");
+Console.ReadLine();
+Console.WriteLine("Executing");
+
+var process2 = new SerializeProcess(
+  progress,
+  new SQLiteSendCacheManager(streamId),
+  new DummyServerObjectManager(),
+  new SpeckleBaseChildFinder(new SpeckleBasePropertyGatherer()),
+  new SpeckleBasePropertyGatherer()
+);
+await process2.Serialize(streamId, @base, default, new SerializeProcessOptions(skipCache, true)).ConfigureAwait(false);
 Console.WriteLine("Detach");
 Console.ReadLine();

tests/Speckle.Sdk.Serialization.Testing/Progress.cs

@@ -4,7 +4,7 @@ namespace Speckle.Sdk.Serialization.Testing;
 
 public class Progress(bool write) : IProgress<ProgressArgs>
 {
-  private readonly TimeSpan DEBOUNCE = TimeSpan.FromMilliseconds(500);
+  private readonly TimeSpan DEBOUNCE = TimeSpan.FromSeconds(1);
   private DateTime _lastTime = DateTime.UtcNow;
 
   private long _totalBytes;

tests/Speckle.Sdk.Serialization.Tests/BaseComparer.cs

@@ -0,0 +1,40 @@
+using Speckle.Sdk.Models;
+
+namespace Speckle.Sdk.Serialization.Tests;
+
+public class BaseComparer : IEqualityComparer<Base>
+{
+  public bool Equals(Base? x, Base? y)
+  {
+    if (ReferenceEquals(x, y))
+      return true;
+    if (x is null)
+      return false;
+    if (y is null)
+      return false;
+    Type type = x.GetType();
+    if (type != y.GetType())
+      return false;
+    var types = DynamicBaseMemberType.Instance | DynamicBaseMemberType.Dynamic | DynamicBaseMemberType.SchemaIgnored;
+    var membersX = x.GetMembers(types);
+    var membersY = y.GetMembers(types);
+    if (membersX.Count != membersY.Count)
+      return false;
+    foreach (var kvp in membersX)
+    {
+      var propertyInfo = type.GetProperty(kvp.Key);
+      if (propertyInfo is not null && !propertyInfo.CanWrite)
+      {
+        continue;
+      }
+      if (y[kvp.Key] != kvp.Value)
+        return false;
+    }
+    return x.id == y.id && x.applicationId == y.applicationId;
+  }
+
+  public int GetHashCode(Base obj)
+  {
+    return HashCode.Combine(obj.id, obj.applicationId);
+  }
+}

tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs

@@ -0,0 +1,250 @@
+using System.Collections.Concurrent;
+using System.Text;
+using NUnit.Framework;
+using Shouldly;
+using Speckle.Newtonsoft.Json.Linq;
+using Speckle.Sdk.Dependencies.Serialization;
+using Speckle.Sdk.Host;
+using Speckle.Sdk.Models;
+using Speckle.Sdk.Serialisation;
+using Speckle.Sdk.Serialisation.V2;
+using Speckle.Sdk.Serialisation.V2.Send;
+using Speckle.Sdk.Transports;
+
+namespace Speckle.Sdk.Serialization.Tests;
+
+public class DetachedTests
+{
+  [SetUp]
+  public void Setup()
+  {
+    TypeLoader.Reset();
+    TypeLoader.Initialize(typeof(Base).Assembly, typeof(DetachedTests).Assembly);
+  }
+
+  [Test(Description = "Checks that all typed properties (including obsolete ones) are returned")]
+  public async Task CanSerialize_New_Detached()
+  {
+    var expectedJson = """
+      {
+          "list": [],
+          "arr": null,
+          "detachedProp": {
+              "speckle_type": "reference",
+              "referencedId": "d3dd4621b2f68c3058c2b9c023a9de19",
+              "__closure": null
+          },
+          "attachedProp": {
+              "name": "attachedProp",
+              "applicationId": null,
+              "speckle_type": "Speckle.Core.Tests.Unit.Models.BaseTests+SamplePropBase",
+              "id": "90d58b65c9036a8bc50743f4c71c1c2e"
+          },
+          "crazyProp": null,
+          "applicationId": null,
+          "speckle_type": "Speckle.Core.Tests.Unit.Models.BaseTests+SampleObjectBase",
+          "dynamicProp": 123,
+          "id": "9ff8efb13c62fa80f3d1c4519376ba13",
+          "__closure": {
+              "d3dd4621b2f68c3058c2b9c023a9de19": 1
+          }
+      }
+      """;
+    var detachedJson = """
+      {
+          "name": "detachedProp",
+          "applicationId": null,
+          "speckle_type": "Speckle.Core.Tests.Unit.Models.BaseTests+SamplePropBase",
+          "id": "d3dd4621b2f68c3058c2b9c023a9de19"
+      }
+      """;
+    var @base = new SampleObjectBase();
+    @base["dynamicProp"] = 123;
+    @base.detachedProp = new SamplePropBase() { name = "detachedProp" };
+    @base.attachedProp = new SamplePropBase() { name = "attachedProp" };
+
+    var objects = new Dictionary<string, string>();
+
+    var process2 = new SerializeProcess(
+      null,
+      new DummySendCacheManager(objects),
+      new DummyServerObjectManager(),
+      new SpeckleBaseChildFinder(new SpeckleBasePropertyGatherer()),
+      new SpeckleBasePropertyGatherer()
+    );
+    await process2
+      .Serialize(string.Empty, @base, default, new SerializeProcessOptions(false, true))
+      .ConfigureAwait(false);
+
+    objects.Count.ShouldBe(2);
+    objects.ContainsKey("9ff8efb13c62fa80f3d1c4519376ba13").ShouldBeTrue();
+    objects.ContainsKey("d3dd4621b2f68c3058c2b9c023a9de19").ShouldBeTrue();
+    JToken
+      .DeepEquals(JObject.Parse(expectedJson), JObject.Parse(objects["9ff8efb13c62fa80f3d1c4519376ba13"]))
+      .ShouldBeTrue();
+    JToken
+      .DeepEquals(JObject.Parse(detachedJson), JObject.Parse(objects["d3dd4621b2f68c3058c2b9c023a9de19"]))
+      .ShouldBeTrue();
+  }
+
+  [Test(Description = "Checks that all typed properties (including obsolete ones) are returned")]
+  public void CanSerialize_Old_Detached()
+  {
+    var expectedJson = """
+      {
+          "list": [],
+          "arr": null,
+          "detachedProp": {
+              "speckle_type": "reference",
+              "referencedId": "d3dd4621b2f68c3058c2b9c023a9de19",
+              "__closure": null
+          },
+          "attachedProp": {
+              "name": "attachedProp",
+              "applicationId": null,
+              "speckle_type": "Speckle.Core.Tests.Unit.Models.BaseTests+SamplePropBase",
+              "id": "90d58b65c9036a8bc50743f4c71c1c2e"
+          },
+          "crazyProp": null,
+          "applicationId": null,
+          "speckle_type": "Speckle.Core.Tests.Unit.Models.BaseTests+SampleObjectBase",
+          "dynamicProp": 123,
+          "id": "9ff8efb13c62fa80f3d1c4519376ba13",
+          "__closure": {
+              "d3dd4621b2f68c3058c2b9c023a9de19": 1
+          }
+      }
+      """;
+    var detachedJson = """
+      {
+          "name": "detachedProp",
+          "applicationId": null,
+          "speckle_type": "Speckle.Core.Tests.Unit.Models.BaseTests+SamplePropBase",
+          "id": "d3dd4621b2f68c3058c2b9c023a9de19"
+      }
+      """;
+    var @base = new SampleObjectBase();
+    @base["dynamicProp"] = 123;
+    @base.detachedProp = new SamplePropBase() { name = "detachedProp" };
+    @base.attachedProp = new SamplePropBase() { name = "attachedProp" };
+
+    var objects = new ConcurrentDictionary<string, string>();
+    var serializer = new SpeckleObjectSerializer(new[] { new MemoryTransport(objects) });
+    var json = serializer.Serialize(@base);
+
+    objects.Count.ShouldBe(2);
+    objects.ContainsKey("9ff8efb13c62fa80f3d1c4519376ba13").ShouldBeTrue();
+    objects.ContainsKey("d3dd4621b2f68c3058c2b9c023a9de19").ShouldBeTrue();
+    JToken.DeepEquals(JObject.Parse(json), JObject.Parse(objects["9ff8efb13c62fa80f3d1c4519376ba13"])).ShouldBeTrue();
+    JToken
+      .DeepEquals(JObject.Parse(expectedJson), JObject.Parse(objects["9ff8efb13c62fa80f3d1c4519376ba13"]))
+      .ShouldBeTrue();
+    JToken
+      .DeepEquals(JObject.Parse(detachedJson), JObject.Parse(objects["d3dd4621b2f68c3058c2b9c023a9de19"]))
+      .ShouldBeTrue();
+  }
+
+  [Test]
+  public void GetPropertiesExpected()
+  {
+    var @base = new SampleObjectBase();
+    @base["dynamicProp"] = 123;
+    @base["@prop2"] = 2;
+    @base["__prop3"] = 3;
+    @base.detachedProp = new SamplePropBase() { name = "detachedProp" };
+    @base.attachedProp = new SamplePropBase() { name = "attachedProp" };
+
+    var children = new SpeckleBasePropertyGatherer().ExtractAllProperties(@base).ToList();
+
+    children.Count.ShouldBe(9);
+    children.First(x => x.Name == "dynamicProp").PropertyAttributeInfo.IsDetachable.ShouldBeFalse();
+    children.First(x => x.Name == "attachedProp").PropertyAttributeInfo.IsDetachable.ShouldBeFalse();
+    children.First(x => x.Name == "crazyProp").PropertyAttributeInfo.IsDetachable.ShouldBeFalse();
+    children.First(x => x.Name == "speckle_type").PropertyAttributeInfo.IsDetachable.ShouldBeFalse();
+    children.First(x => x.Name == "applicationId").PropertyAttributeInfo.IsDetachable.ShouldBeFalse();
+
+    children.First(x => x.Name == "detachedProp").PropertyAttributeInfo.IsDetachable.ShouldBeTrue();
+    children.First(x => x.Name == "list").PropertyAttributeInfo.IsDetachable.ShouldBeTrue();
+    children.First(x => x.Name == "arr").PropertyAttributeInfo.IsDetachable.ShouldBeTrue();
+    children.First(x => x.Name == "@prop2").PropertyAttributeInfo.IsDetachable.ShouldBeTrue();
+  }
+}
+
+[SpeckleType("Speckle.Core.Tests.Unit.Models.BaseTests+SampleObjectBase")]
+public class SampleObjectBase : Base
+{
+  [Chunkable, DetachProperty]
+  public List<double> list { get; set; } = new();
+
+  [Chunkable(300), DetachProperty]
+  public double[] arr { get; set; }
+
+  [DetachProperty]
+  public SamplePropBase detachedProp { get; set; }
+
+  public SamplePropBase attachedProp { get; set; }
+
+  public string crazyProp { get; set; }
+}
+
+[SpeckleType("Speckle.Core.Tests.Unit.Models.BaseTests+SamplePropBase")]
+public class SamplePropBase : Base
+{
+  public string name { get; set; }
+}
+
+public class DummyServerObjectManager : IServerObjectManager
+{
+  public IAsyncEnumerable<(string, string)> DownloadObjects(
+    string streamId,
+    IReadOnlyList<string> objectIds,
+    IProgress<ProgressArgs>? progress,
+    CancellationToken cancellationToken
+  ) => throw new NotImplementedException();
+
+  public Task<string?> DownloadSingleObject(
+    string streamId,
+    string objectId,
+    IProgress<ProgressArgs>? progress,
+    CancellationToken cancellationToken
+  ) => throw new NotImplementedException();
+
+  public Task<Dictionary<string, bool>> HasObjects(
+    string streamId,
+    IReadOnlyList<string> objectIds,
+    CancellationToken cancellationToken
+  ) => throw new NotImplementedException();
+
+  public Task UploadObjects(
+    string streamId,
+    IReadOnlyList<BaseItem> objects,
+    bool compressPayloads,
+    IProgress<ProgressArgs>? progress,
+    CancellationToken cancellationToken
+  )
+  {
+    long totalBytes = 0;
+    foreach (var item in objects)
+    {
+      totalBytes += Encoding.Default.GetByteCount(item.Json);
+    }
+
+    progress?.Report(new(ProgressEvent.UploadBytes, totalBytes, totalBytes));
+    return Task.CompletedTask;
+  }
+}
+
+public class DummySendCacheManager(Dictionary<string, string> objects) : ISQLiteSendCacheManager
+{
+  public string? GetObject(string id) => null;
+
+  public bool HasObject(string objectId) => false;
+
+  public void SaveObjects(List<BaseItem> items)
+  {
+    foreach (var item in items)
+    {
+      objects.Add(item.Id, item.Json);
+    }
+  }
+}

tests/Speckle.Sdk.Serialization.Tests/DummyReceiveServerObjectManager.cs

@@ -0,0 +1,59 @@
+using System.Runtime.CompilerServices;
+using System.Text;
+using Speckle.Sdk.Dependencies.Serialization;
+using Speckle.Sdk.Serialisation.V2;
+using Speckle.Sdk.Transports;
+
+namespace Speckle.Sdk.Serialization.Tests;
+
+public class DummyReceiveServerObjectManager(Dictionary<string, string> objects) : IServerObjectManager
+{
+  public async IAsyncEnumerable<(string, string)> DownloadObjects(
+    string streamId,
+    IReadOnlyList<string> objectIds,
+    IProgress<ProgressArgs>? progress,
+    [EnumeratorCancellation] CancellationToken cancellationToken
+  )
+  {
+    await Task.CompletedTask;
+    foreach (var id in objectIds)
+    {
+      yield return (id, objects[id]);
+    }
+  }
+
+  public async Task<string?> DownloadSingleObject(
+    string streamId,
+    string objectId,
+    IProgress<ProgressArgs>? progress,
+    CancellationToken cancellationToken
+  )
+  {
+    await Task.CompletedTask;
+    return objects[objectId];
+  }
+
+  public Task<Dictionary<string, bool>> HasObjects(
+    string streamId,
+    IReadOnlyList<string> objectIds,
+    CancellationToken cancellationToken
+  ) => throw new NotImplementedException();
+
+  public Task UploadObjects(
+    string streamId,
+    IReadOnlyList<BaseItem> objects,
+    bool compressPayloads,
+    IProgress<ProgressArgs>? progress,
+    CancellationToken cancellationToken
+  )
+  {
+    long totalBytes = 0;
+    foreach (var item in objects)
+    {
+      totalBytes += Encoding.Default.GetByteCount(item.Json);
+    }
+
+    progress?.Report(new(ProgressEvent.UploadBytes, totalBytes, totalBytes));
+    return Task.CompletedTask;
+  }
+}

tests/Speckle.Sdk.Serialization.Tests/DummySendServerObjectManager.cs

@@ -0,0 +1,58 @@
+using System.Collections.Concurrent;
+using Shouldly;
+using Speckle.Newtonsoft.Json.Linq;
+using Speckle.Sdk.Common;
+using Speckle.Sdk.Dependencies.Serialization;
+using Speckle.Sdk.Serialisation.V2;
+using Speckle.Sdk.Transports;
+
+namespace Speckle.Sdk.Serialization.Tests;
+
+public class DummySendServerObjectManager(ConcurrentDictionary<string, string> savedObjects) : IServerObjectManager
+{
+  public IAsyncEnumerable<(string, string)> DownloadObjects(
+    string streamId,
+    IReadOnlyList<string> objectIds,
+    IProgress<ProgressArgs>? progress,
+    CancellationToken cancellationToken
+  ) => throw new NotImplementedException();
+
+  public Task<string?> DownloadSingleObject(
+    string streamId,
+    string objectId,
+    IProgress<ProgressArgs>? progress,
+    CancellationToken cancellationToken
+  ) => throw new NotImplementedException();
+
+  public Task<Dictionary<string, bool>> HasObjects(
+    string streamId,
+    IReadOnlyList<string> objectIds,
+    CancellationToken cancellationToken
+  )
+  {
+    return Task.FromResult(objectIds.ToDictionary(x => x, x => false));
+  }
+
+  public Task UploadObjects(
+    string streamId,
+    IReadOnlyList<BaseItem> objects,
+    bool compressPayloads,
+    IProgress<ProgressArgs>? progress,
+    CancellationToken cancellationToken
+  )
+  {
+    foreach (var obj in objects)
+    {
+      obj.Id.ShouldBe(JObject.Parse(obj.Json)["id"].NotNull().Value<string>());
+      if (savedObjects.TryGetValue(obj.Id, out var j))
+      {
+        j.ShouldBe(obj.Json);
+      }
+      else
+      {
+        savedObjects.TryAdd(obj.Id, obj.Json);
+      }
+    }
+    return Task.CompletedTask;
+  }
+}

tests/Speckle.Sdk.Serialization.Tests/DummySqLiteReceiveManager.cs

@@ -0,0 +1,13 @@
+using Speckle.Sdk.Dependencies.Serialization;
+using Speckle.Sdk.Serialisation.V2;
+
+namespace Speckle.Sdk.Serialization.Tests;
+
+public class DummySqLiteReceiveManager(Dictionary<string, string> savedObjects) : ISQLiteReceiveCacheManager
+{
+  public string? GetObject(string id) => savedObjects.GetValueOrDefault(id);
+
+  public void SaveObject(BaseItem item) => throw new NotImplementedException();
+
+  public bool HasObject(string objectId) => throw new NotImplementedException();
+}

tests/Speckle.Sdk.Serialization.Tests/DummySqLiteSendManager.cs

@@ -0,0 +1,13 @@
+using Speckle.Sdk.Dependencies.Serialization;
+using Speckle.Sdk.Serialisation.V2;
+
+namespace Speckle.Sdk.Serialization.Tests;
+
+public class DummySqLiteSendManager : ISQLiteSendCacheManager
+{
+  public string? GetObject(string id) => throw new NotImplementedException();
+
+  public bool HasObject(string objectId) => throw new NotImplementedException();
+
+  public void SaveObjects(List<BaseItem> items) => throw new NotImplementedException();
+}

tests/Speckle.Sdk.Tests.Performance/Benchmarks/GeneralDeserializerTest.cs

@@ -46,7 +46,7 @@ public class GeneralDeserializer : IDisposable
   [Benchmark]
   public async Task<Base> RunTest_New()
   {
-    var sqlite = new SQLiteCacheManager(streamId);
+    var sqlite = new SQLiteReceiveCacheManager(streamId);
     var serverObjects = new ServerObjectManager(
       TestDataHelper.ServiceProvider.GetRequiredService<ISpeckleHttp>(),
       TestDataHelper.ServiceProvider.GetRequiredService<ISdkActivityFactory>(),

tests/Speckle.Sdk.Tests.Performance/Benchmarks/GeneralSendTest.cs

@@ -0,0 +1,84 @@
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Engines;
+using Microsoft.Extensions.DependencyInjection;
+using Speckle.Objects.Geometry;
+using Speckle.Sdk.Api;
+using Speckle.Sdk.Api.GraphQL.Enums;
+using Speckle.Sdk.Api.GraphQL.Models;
+using Speckle.Sdk.Common;
+using Speckle.Sdk.Credentials;
+using Speckle.Sdk.Host;
+using Speckle.Sdk.Models;
+using Speckle.Sdk.Serialisation;
+using Speckle.Sdk.Transports;
+
+namespace Speckle.Sdk.Tests.Performance.Benchmarks;
+
+/// <summary>
+/// How many threads on our Deserializer is optimal
+/// </summary>
+[MemoryDiagnoser]
+[SimpleJob(RunStrategy.Monitoring, iterationCount: 1)]
+public class GeneralSendTest
+{
+  private Base _testData;
+  private IOperations _operations;
+  private ServerTransport _remote;
+  private Account acc;
+  private Client client;
+
+  private Project _project;
+
+  [GlobalSetup]
+  public async Task Setup()
+  {
+    TypeLoader.Initialize(typeof(Base).Assembly, typeof(Point).Assembly);
+    using var dataSource = new TestDataHelper();
+    await dataSource
+      .SeedTransport(
+        new Account() { serverInfo = new() { url = "https://latest.speckle.systems/" } },
+        "2099ac4b5f",
+        "30fb4cbe6eb2202b9e7b4a4fcc3dd2b6",
+        false
+      )
+      .ConfigureAwait(false);
+
+    SpeckleObjectDeserializer deserializer = new() { ReadTransport = dataSource.Transport };
+    string data = await dataSource.Transport.GetObject(dataSource.ObjectId).NotNull();
+    _testData = await deserializer.DeserializeAsync(data).NotNull();
+    _operations = TestDataHelper.ServiceProvider.GetRequiredService<IOperations>();
+
+    acc = TestDataHelper
+      .ServiceProvider.GetRequiredService<IAccountManager>()
+      .GetAccounts("https://latest.speckle.systems")
+      .First();
+
+    client = TestDataHelper.ServiceProvider.GetRequiredService<IClientFactory>().Create(acc);
+
+    _project = await client.Project.Create(
+      new($"General Send Test run {Guid.NewGuid()}", null, ProjectVisibility.Public)
+    );
+    _remote = TestDataHelper.ServiceProvider.GetRequiredService<IServerTransportFactory>().Create(acc, _project.id);
+  }
+
+  [Benchmark(Baseline = true)]
+  public async Task<string> Send_old()
+  {
+    using SQLiteTransport local = new();
+    var res = await _operations.Send(_testData, [_remote, local]);
+    return await TagVersion($"Send_old {Guid.NewGuid()}", res.rootObjId);
+  }
+
+  [Benchmark]
+  public async Task<string> Send_new()
+  {
+    var res = await _operations.Send2(new(acc.serverInfo.url), _project.id, acc.token, _testData);
+    return await TagVersion($"Send_new {Guid.NewGuid()}", res.rootObjId);
+  }
+
+  private async Task<string> TagVersion(string name, string objectId)
+  {
+    var model = await client.Model.Create(new(name, null, _project.id));
+    return await client.Version.Create(new(objectId, model.id, _project.id));
+  }
+}

tests/Speckle.Sdk.Tests.Performance/Program.cs

@@ -1,5 +1,9 @@
 // See https://aka.ms/new-console-template for more information
 
 using BenchmarkDotNet.Running;
+using Speckle.Sdk.Tests.Performance.Benchmarks;
 
 BenchmarkSwitcher.FromAssemblies([typeof(Program).Assembly]).Run(args);
+// var sut = new GeneralSendTest();
+// await sut.Setup();
+// await sut.Send2();

tests/Speckle.Sdk.Tests.Unit/Api/Operations/SendReceiveLocal.cs

@@ -203,17 +203,7 @@ public sealed class SendReceiveLocal : IDisposable
       );
     }
 
-    ProgressArgs? progress = null;
-    (_commitId02, _) = await _operations.Send(
-      myObject,
-      _sut,
-      false,
-      onProgressAction: new UnitTestProgress<ProgressArgs>(x =>
-      {
-        progress = x;
-      })
-    );
-    progress.ShouldNotBeNull();
+    (_commitId02, _) = await _operations.Send(myObject, _sut, false);
   }
 
   [Test(Description = "Should show progress!"), Order(5)]

tests/Speckle.Sdk.Tests.Unit/Models/BaseTests.cs

@@ -1,7 +1,16 @@
+using System.Collections.Concurrent;
+using System.Text;
 using NUnit.Framework;
+using Shouldly;
+using Speckle.Newtonsoft.Json.Linq;
 using Speckle.Sdk.Common;
+using Speckle.Sdk.Dependencies.Serialization;
 using Speckle.Sdk.Host;
 using Speckle.Sdk.Models;
+using Speckle.Sdk.Serialisation;
+using Speckle.Sdk.Serialisation.V2;
+using Speckle.Sdk.Serialisation.V2.Send;
+using Speckle.Sdk.Transports;
 
 namespace Speckle.Sdk.Tests.Unit.Models;
 

tests/Speckle.Sdk.Tests.Unit/Transports/TransportTests.cs

@@ -93,19 +93,6 @@ public abstract class TransportTests
     }
   }
 
-  [Test]
-  public async Task ProgressAction_Called_OnSaveObject()
-  {
-    bool wasCalled = false;
-    Sut.NotNull().OnProgressAction = new UnitTestProgress<ProgressArgs>(_ => wasCalled = true);
-
-    Sut.SaveObject("12345", "fake payload data");
-
-    await Sut.WriteComplete();
-
-    Assert.That(wasCalled, Is.True);
-  }
-
   [Test]
   public void ToString_IsNotEmpty()
   {