Batch by size, Closures and Detached items are computed differently (#164)

* Can debug dependencies

* Different exceptions

* Uses root id only after we found it to signal the end

* DataChunks are created later and need to be accounted for

* format

* use app ids in tests and references

* check sqlite cache after serialize

* use dummy to go through channels to end

* fmt

* Extend channel lib to batch by size

* fmt

* build fix

* adjust limits

* FIx sending

* Optimize reference generation

* more

* remove tolist

* rework closures to be constant and serializer only deals with current....references bases are cached

* fix chunk creation

* another bug fix

* clean up with factories

* add deserializer factory

* Needed to reference interface

* move around streamId

* some clean up

* Use StringBuilder pool on serialization to reduce memory pressure

* remove extra

* remove extra clears

* Fix a flaw in batchsize

* use default complete

* format

* loader should use 1 writer that is batched

* remove redundant ref gen

* Fix graphql commands by adding project id

src/Speckle.Sdk.Dependencies/Pools.cs

@@ -1,4 +1,5 @@
-using Microsoft.Extensions.ObjectPool;
+using System.Text;
+using Microsoft.Extensions.ObjectPool;
 
 namespace Speckle.Sdk.Dependencies;
 
@@ -17,16 +18,6 @@ public static class Pools
     }
   }
 
-  public static Pool<List<string>> ListString { get; } = new(new ListStringPolicy());
-
-  private sealed class ListStringPolicy : IPooledObjectPolicy<List<string>>
-  {
-    public List<string> Create() => new(20);
-
-    public bool Return(List<string> obj)
-    {
-      obj.Clear();
-      return true;
-    }
-  }
+  public static Pool<StringBuilder> StringBuilders { get; } =
+    new(new StringBuilderPooledObjectPolicy() { MaximumRetainedCapacity = 100 * 1024 * 1024 });
 }

src/Speckle.Sdk.Dependencies/Serialization/ChannelExtensions.cs

@@ -0,0 +1,21 @@
+using System.Threading.Channels;
+using Open.ChannelExtensions;
+using Speckle.Sdk.Dependencies.Serialization;
+
+namespace Speckle.Sdk.Serialisation.V2.Send;
+
+public static class ChannelExtensions
+{
+  public static BatchingChannelReader<BaseItem, List<BaseItem>> BatchBySize(
+    this ChannelReader<BaseItem> source,
+    int batchSize,
+    bool singleReader = false,
+    bool allowSynchronousContinuations = false
+  ) =>
+    new SizeBatchingChannelReader(
+      source ?? throw new ArgumentNullException(nameof(source)),
+      batchSize,
+      singleReader,
+      allowSynchronousContinuations
+    );
+}

src/Speckle.Sdk.Dependencies/Serialization/ChannelLoader.cs

@@ -7,23 +7,27 @@ public abstract class ChannelLoader
   private const int HTTP_GET_CHUNK_SIZE = 500;
   private const int MAX_PARALLELISM_HTTP = 4;
   private static readonly TimeSpan HTTP_BATCH_TIMEOUT = TimeSpan.FromSeconds(2);
-  private static readonly int MAX_CACHE_PARALLELISM = Environment.ProcessorCount;
+  private static readonly int MAX_READ_CACHE_PARALLELISM = Environment.ProcessorCount;
+  private const int MAX_SAVE_CACHE_BATCH = 200;
+  private const int MAX_SAVE_CACHE_PARALLELISM = 1;
 
   protected async Task GetAndCache(IEnumerable<string> allChildrenIds, CancellationToken cancellationToken = default) =>
     await allChildrenIds
       .ToChannel(cancellationToken: cancellationToken)
-      .Pipe(MAX_CACHE_PARALLELISM, CheckCache, cancellationToken: cancellationToken)
+      .Pipe(MAX_READ_CACHE_PARALLELISM, CheckCache, cancellationToken: cancellationToken)
       .Filter(x => x is not null)
       .Batch(HTTP_GET_CHUNK_SIZE)
       .WithTimeout(HTTP_BATCH_TIMEOUT)
       .PipeAsync(MAX_PARALLELISM_HTTP, async x => await Download(x).ConfigureAwait(false), -1, false, cancellationToken)
       .Join()
-      .ReadAllConcurrently(MAX_CACHE_PARALLELISM, SaveToCache, cancellationToken)
+      .Batch(MAX_SAVE_CACHE_BATCH)
+      .WithTimeout(HTTP_BATCH_TIMEOUT)
+      .ReadAllConcurrently(MAX_SAVE_CACHE_PARALLELISM, SaveToCache, cancellationToken)
       .ConfigureAwait(false);
 
   public abstract string? CheckCache(string id);
 
   public abstract Task<List<BaseItem>> Download(List<string?> ids);
 
-  public abstract void SaveToCache(BaseItem x);
+  public abstract void SaveToCache(List<BaseItem> x);
 }

src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs

@@ -1,29 +1,33 @@
-using System.Threading.Channels;
+using System.Text;
+using System.Threading.Channels;
 using Open.ChannelExtensions;
+using Speckle.Sdk.Serialisation.V2.Send;
 
 namespace Speckle.Sdk.Dependencies.Serialization;
 
-public readonly record struct BaseItem(string Id, string Json, bool NeedsStorage);
+public readonly record struct BaseItem(string Id, string Json, bool NeedsStorage)
+{
+  public int Size { get; } = Encoding.UTF8.GetByteCount(Json);
+}
 
 public abstract class ChannelSaver
 {
-  private const int HTTP_SEND_CHUNK_SIZE = 500;
+  private const int HTTP_SEND_CHUNK_SIZE = 25_000_000; //bytes
   private static readonly TimeSpan HTTP_BATCH_TIMEOUT = TimeSpan.FromSeconds(2);
   private const int MAX_PARALLELISM_HTTP = 4;
   private const int MAX_CACHE_WRITE_PARALLELISM = 1;
-  private const int MAX_CACHE_BATCH = 100;
-  private const string DUMMY = "dummy";
+  private const int MAX_CACHE_BATCH = 200;
 
   private readonly Channel<BaseItem> _checkCacheChannel = Channel.CreateUnbounded<BaseItem>();
 
-  public Task Start(string streamId, CancellationToken cancellationToken = default)
+  public Task Start(CancellationToken cancellationToken = default)
   {
     var t = _checkCacheChannel
-      .Reader.Batch(HTTP_SEND_CHUNK_SIZE)
+      .Reader.BatchBySize(HTTP_SEND_CHUNK_SIZE)
       .WithTimeout(HTTP_BATCH_TIMEOUT)
       .PipeAsync(
         MAX_PARALLELISM_HTTP,
-        async x => await SendToServerInternal(streamId, x, cancellationToken).ConfigureAwait(false),
+        async x => await SendToServer(x, cancellationToken).ConfigureAwait(false),
         -1,
         false,
         cancellationToken
@@ -31,52 +35,19 @@ public abstract class ChannelSaver
       .Join()
       .Batch(MAX_CACHE_BATCH)
       .WithTimeout(HTTP_BATCH_TIMEOUT)
-      .ReadAllConcurrently(MAX_CACHE_WRITE_PARALLELISM, SaveToCacheInternal, cancellationToken);
+      .ReadAllConcurrently(MAX_CACHE_WRITE_PARALLELISM, SaveToCache, cancellationToken);
     return t;
   }
 
   public async Task Save(BaseItem item, CancellationToken cancellationToken = default) =>
     await _checkCacheChannel.Writer.WriteAsync(item, cancellationToken).ConfigureAwait(false);
 
-  private async Task<List<BaseItem>> SendToServerInternal(
-    string streamId,
-    List<BaseItem> batch,
-    CancellationToken cancellationToken = default
-  )
+  public abstract Task<List<BaseItem>> SendToServer(List<BaseItem> batch, CancellationToken cancellationToken);
+
+  public Task Done()
   {
-    var ending = batch.Select(x => x.Id).Contains(DUMMY);
-    if (ending)
-    {
-      batch.RemoveAll(x => x.Id == DUMMY);
-    }
-    var results = await SendToServer(streamId, batch, cancellationToken).ConfigureAwait(false);
-    if (ending)
-    {
-      results.Add(new BaseItem(DUMMY, DUMMY, false));
-    }
-    return results;
-  }
-
-  public abstract Task<List<BaseItem>> SendToServer(
-    string streamId,
-    List<BaseItem> batch,
-    CancellationToken cancellationToken
-  );
-
-  public async Task Done() => await Save(new BaseItem(DUMMY, DUMMY, false)).ConfigureAwait(false);
-
-  private void SaveToCacheInternal(List<BaseItem> batch)
-  {
-    var ending = batch.Select(x => x.Id).Contains(DUMMY);
-    if (ending)
-    {
-      batch.RemoveAll(x => x.Id == DUMMY);
-    }
-    SaveToCache(batch);
-    if (ending)
-    {
-      _checkCacheChannel.Writer.Complete();
-    }
+    _checkCacheChannel.Writer.Complete();
+    return Task.CompletedTask;
   }
 
   public abstract void SaveToCache(List<BaseItem> item);

src/Speckle.Sdk.Dependencies/Serialization/SizeBatchingChannelReader.cs

@@ -0,0 +1,36 @@
+using System.Threading.Channels;
+using Open.ChannelExtensions;
+using Speckle.Sdk.Dependencies.Serialization;
+
+namespace Speckle.Sdk.Serialisation.V2.Send;
+
+public class SizeBatchingChannelReader(
+  ChannelReader<BaseItem> source,
+  int batchSize,
+  bool singleReader,
+  bool syncCont = false
+) : BatchingChannelReader<BaseItem, List<BaseItem>>(source, batchSize, singleReader, syncCont)
+{
+  private readonly int _batchSize = batchSize;
+
+  protected override List<BaseItem> CreateBatch(int capacity) => new();
+
+  protected override void TrimBatch(List<BaseItem> batch) => batch.TrimExcess();
+
+  protected override void AddBatchItem(List<BaseItem> batch, BaseItem item) => batch.Add(item);
+
+  protected override int GetBatchSize(List<BaseItem> batch)
+  {
+    int size = 0;
+    foreach (BaseItem item in batch)
+    {
+      size += item.Size;
+    }
+
+    if (size >= _batchSize)
+    {
+      return _batchSize;
+    }
+    return size;
+  }
+}

src/Speckle.Sdk/Api/GraphQL/Inputs/VersionInputs.cs

@@ -1,10 +1,10 @@
 namespace Speckle.Sdk.Api.GraphQL.Inputs;
 
-public sealed record UpdateVersionInput(string versionId, string? message);
+public sealed record UpdateVersionInput(string versionId, string projectId, string? message);
 
-public sealed record MoveVersionsInput(string targetModelName, IReadOnlyList<string> versionIds);
+public sealed record MoveVersionsInput(string projectId, string targetModelName, IReadOnlyList<string> versionIds);
 
-public sealed record DeleteVersionsInput(IReadOnlyList<string> versionIds);
+public sealed record DeleteVersionsInput(IReadOnlyList<string> versionIds, string projectId);
 
 public sealed record CreateVersionInput(
   string objectId,

src/Speckle.Sdk/Api/Operations/Operations.Receive.cs

@@ -2,8 +2,6 @@ using Microsoft.Extensions.Logging;
 using Speckle.Sdk.Logging;
 using Speckle.Sdk.Models;
 using Speckle.Sdk.Serialisation;
-using Speckle.Sdk.Serialisation.V2;
-using Speckle.Sdk.Serialisation.V2.Receive;
 using Speckle.Sdk.Transports;
 
 namespace Speckle.Sdk.Api;
@@ -26,10 +24,12 @@ public partial class Operations
 
     try
     {
-      var sqliteTransport = new SQLiteReceiveCacheManager(streamId);
-      var serverObjects = new ServerObjectManager(speckleHttp, activityFactory, url, authorizationToken);
-      var o = new ObjectLoader(sqliteTransport, serverObjects, streamId, onProgressAction);
-      var process = new DeserializeProcess(onProgressAction, o);
+      var process = serializeProcessFactory.CreateDeserializeProcess(
+        url,
+        streamId,
+        authorizationToken,
+        onProgressAction
+      );
       var result = await process.Deserialize(objectId, cancellationToken).ConfigureAwait(false);
       receiveActivity?.SetStatus(SdkActivityStatusCode.Ok);
       return result;

src/Speckle.Sdk/Api/Operations/Operations.Send.cs

@@ -4,8 +4,6 @@ using Speckle.Newtonsoft.Json.Linq;
 using Speckle.Sdk.Logging;
 using Speckle.Sdk.Models;
 using Speckle.Sdk.Serialisation;
-using Speckle.Sdk.Serialisation.V2;
-using Speckle.Sdk.Serialisation.V2.Send;
 using Speckle.Sdk.Transports;
 
 namespace Speckle.Sdk.Api;
@@ -26,18 +24,8 @@ public partial class Operations
 
     try
     {
-      var sqliteTransport = new SQLiteSendCacheManager(streamId);
-      var serverObjects = new ServerObjectManager(speckleHttp, activityFactory, url, authorizationToken);
-      var process = new SerializeProcess(
-        onProgressAction,
-        sqliteTransport,
-        serverObjects,
-        speckleBaseChildFinder,
-        speckleBasePropertyGatherer
-      );
-      var (rootObjId, convertedReferences) = await process
-        .Serialize(streamId, value, cancellationToken)
-        .ConfigureAwait(false);
+      var process = serializeProcessFactory.CreateSerializeProcess(url, streamId, authorizationToken, onProgressAction);
+      var (rootObjId, convertedReferences) = await process.Serialize(value, cancellationToken).ConfigureAwait(false);
 
       receiveActivity?.SetStatus(SdkActivityStatusCode.Ok);
       return new(rootObjId, convertedReferences);

src/Speckle.Sdk/Api/Operations/Operations.cs

@@ -1,8 +1,7 @@
 using Microsoft.Extensions.Logging;
 using Speckle.InterfaceGenerator;
-using Speckle.Sdk.Helpers;
 using Speckle.Sdk.Logging;
-using Speckle.Sdk.Serialisation.V2.Send;
+using Speckle.Sdk.Serialisation.V2;
 
 namespace Speckle.Sdk.Api;
 
@@ -14,9 +13,7 @@ namespace Speckle.Sdk.Api;
 [GenerateAutoInterface]
 public partial class Operations(
   ILogger<Operations> logger,
-  ISpeckleHttp speckleHttp,
   ISdkActivityFactory activityFactory,
   ISdkMetricsFactory metricsFactory,
-  ISpeckleBaseChildFinder speckleBaseChildFinder,
-  ISpeckleBasePropertyGatherer speckleBasePropertyGatherer
+  ISerializeProcessFactory serializeProcessFactory
 ) : IOperations;

src/Speckle.Sdk/Helpers/SerializerIdWriter.cs

@@ -1,4 +1,6 @@
-using Speckle.Newtonsoft.Json;
+using System.Text;
+using Speckle.Newtonsoft.Json;
+using Speckle.Sdk.Dependencies;
 using Speckle.Sdk.Serialisation;
 
 namespace Speckle.Sdk.Helpers;
@@ -9,12 +11,14 @@ public sealed class SerializerIdWriter : JsonWriter
 #pragma warning disable CA2213
   private readonly JsonWriter _jsonIdWriter;
   private readonly StringWriter _idWriter;
+  private readonly StringBuilder _stringBuilder;
 #pragma warning restore CA2213
 
   public SerializerIdWriter(JsonWriter jsonWriter)
   {
     _jsonWriter = jsonWriter;
-    _idWriter = new StringWriter();
+    _stringBuilder = Pools.StringBuilders.Get();
+    _idWriter = new StringWriter(_stringBuilder);
     _jsonIdWriter = SpeckleObjectSerializerPool.Instance.GetJsonTextWriter(_idWriter);
   }
 
@@ -23,6 +27,7 @@ public sealed class SerializerIdWriter : JsonWriter
     _jsonIdWriter.WriteEndObject();
     _jsonIdWriter.Flush();
     var json = _idWriter.ToString();
+    Pools.StringBuilders.Return(_stringBuilder);
     return (json, _jsonWriter);
   }
 

src/Speckle.Sdk/Serialisation/SpeckleObjectSerializer.cs

@@ -6,6 +6,7 @@ using System.Reflection;
 using Speckle.DoubleNumerics;
 using Speckle.Newtonsoft.Json;
 using Speckle.Sdk.Common;
+using Speckle.Sdk.Dependencies;
 using Speckle.Sdk.Helpers;
 using Speckle.Sdk.Models;
 using Speckle.Sdk.Serialisation.Utilities;
@@ -249,10 +250,12 @@ public class SpeckleObjectSerializer
       _parentClosures.Add(closure);
     }
 
+    var stringBuilder = Pools.StringBuilders.Get();
     using var writer = new StringWriter();
     using var jsonWriter = SpeckleObjectSerializerPool.Instance.GetJsonTextWriter(writer);
     string id = SerializeBaseObject(baseObj, jsonWriter, closure);
     var json = writer.ToString();
+    Pools.StringBuilders.Return(stringBuilder);
 
     if (computeClosures || inheritedDetachInfo.IsDetachable || baseObj is Blob)
     {

src/Speckle.Sdk/Serialisation/Utilities/ReferenceGenerator.cs

@@ -3,7 +3,7 @@
 public static class ReferenceGenerator
 {
   private const string REFERENCE_JSON_START = "{\"speckle_type\":\"reference\",\"referencedId\":\"";
-  private const string REFERENCE_JSON_END = "\",\"__closure\":null}";
+  private const string REFERENCE_JSON_END = "\",\"__closure\":null}"; //TODO: remove null but ID calculation changes
 
   public static string CreateReference(string id) => REFERENCE_JSON_START + id + REFERENCE_JSON_END;
 }

src/Speckle.Sdk/Serialisation/V2/Receive/DeserializeProcess.cs

@@ -1,21 +1,33 @@
 using System.Collections.Concurrent;
+using Speckle.InterfaceGenerator;
 using Speckle.Sdk.Models;
 using Speckle.Sdk.Serialisation.Utilities;
 using Speckle.Sdk.Transports;
 
 namespace Speckle.Sdk.Serialisation.V2.Receive;
 
-public record DeserializeOptions(bool SkipCache);
+public record DeserializeOptions(
+  bool SkipCache,
+  bool ThrowOnMissingReferences = true,
+  bool SkipInvalidConverts = false
+);
 
-public sealed class DeserializeProcess(IProgress<ProgressArgs>? progress, IObjectLoader objectLoader)
+[GenerateAutoInterface]
+public sealed class DeserializeProcess(
+  IProgress<ProgressArgs>? progress,
+  IObjectLoader objectLoader,
+  IObjectDeserializerFactory objectDeserializerFactory
+) : IDeserializeProcess
 {
-  private readonly ConcurrentDictionary<string, (string, IReadOnlyList<string>)> _closures = new();
   private long _total;
   private DeserializeOptions _options = new(false);
 
-  public ConcurrentDictionary<string, Base> BaseCache { get; } = new();
+  private readonly ConcurrentDictionary<string, (string, IReadOnlyList<string>)> _closures = new();
+  private readonly ConcurrentDictionary<string, Base> _baseCache = new();
   private readonly ConcurrentDictionary<string, Task> _activeTasks = new();
 
+  public IReadOnlyDictionary<string, Base> BaseCache => _baseCache;
+
   public async Task<Base> Deserialize(
     string rootId,
     CancellationToken cancellationToken,
@@ -28,14 +40,14 @@ public sealed class DeserializeProcess(IProgress<ProgressArgs>? progress, IObjec
       .ConfigureAwait(false);
     _total = childrenIds.Count;
     _closures.TryAdd(rootId, (rootJson, childrenIds));
-    progress?.Report(new(ProgressEvent.DeserializeObject, BaseCache.Count, childrenIds.Count));
+    progress?.Report(new(ProgressEvent.DeserializeObject, _baseCache.Count, childrenIds.Count));
     await Traverse(rootId, cancellationToken).ConfigureAwait(false);
-    return BaseCache[rootId];
+    return _baseCache[rootId];
   }
 
   private async Task Traverse(string id, CancellationToken cancellationToken)
   {
-    if (BaseCache.ContainsKey(id))
+    if (_baseCache.ContainsKey(id))
     {
       return;
     }
@@ -43,7 +55,7 @@ public sealed class DeserializeProcess(IProgress<ProgressArgs>? progress, IObjec
     var tasks = new List<Task>();
     foreach (var childId in childIds)
     {
-      if (BaseCache.ContainsKey(childId))
+      if (_baseCache.ContainsKey(childId))
       {
         continue;
       }
@@ -75,10 +87,10 @@ public sealed class DeserializeProcess(IProgress<ProgressArgs>? progress, IObjec
     }
 
     //don't redo things if the id is decoded already in the cache
-    if (!BaseCache.ContainsKey(id))
+    if (!_baseCache.ContainsKey(id))
     {
       DecodeOrEnqueueChildren(id);
-      progress?.Report(new(ProgressEvent.DeserializeObject, BaseCache.Count, _total));
+      progress?.Report(new(ProgressEvent.DeserializeObject, _baseCache.Count, _total));
     }
   }
 
@@ -101,13 +113,13 @@ public sealed class DeserializeProcess(IProgress<ProgressArgs>? progress, IObjec
 
   public void DecodeOrEnqueueChildren(string id)
   {
-    if (BaseCache.ContainsKey(id))
+    if (_baseCache.ContainsKey(id))
     {
       return;
     }
     (string json, _) = GetClosures(id);
     var @base = Deserialise(id, json);
-    BaseCache.TryAdd(id, @base);
+    _baseCache.TryAdd(id, @base);
     //remove from JSON cache because we've finally made the Base
     _closures.TryRemove(id, out _);
     _activeTasks.TryRemove(id, out _);
@@ -115,11 +127,12 @@ public sealed class DeserializeProcess(IProgress<ProgressArgs>? progress, IObjec
 
   private Base Deserialise(string id, string json)
   {
-    if (BaseCache.TryGetValue(id, out var baseObject))
+    if (_baseCache.TryGetValue(id, out var baseObject))
     {
       return baseObject;
     }
-    SpeckleObjectDeserializer2 deserializer = new(BaseCache, SpeckleObjectSerializerPool.Instance);
+
+    var deserializer = objectDeserializerFactory.Create(_baseCache);
     return deserializer.Deserialize(json);
   }
 }

src/Speckle.Sdk/Serialisation/V2/Receive/ObjectDeserializer.cs

@@ -1,4 +1,5 @@
 using System.Numerics;
+using Speckle.InterfaceGenerator;
 using Speckle.Newtonsoft.Json;
 using Speckle.Sdk.Common;
 using Speckle.Sdk.Dependencies;
@@ -6,13 +7,12 @@ using Speckle.Sdk.Models;
 
 namespace Speckle.Sdk.Serialisation.V2.Receive;
 
-public record DeserializedOptions(bool ThrowOnMissingReferences = true, bool SkipInvalidConverts = false);
-
-public sealed class SpeckleObjectDeserializer2(
+[GenerateAutoInterface]
+public sealed class ObjectDeserializer(
   IReadOnlyDictionary<string, Base> references,
   SpeckleObjectSerializerPool pool,
-  DeserializedOptions? options = null
-)
+  DeserializeOptions? options = null
+) : IObjectDeserializer
 {
   /// <param name="objectJson">The JSON string of the object to be deserialized <see cref="Base"/></param>
   /// <returns>A <see cref="Base"/> typed object deserialized from the <paramref name="objectJson"/></returns>

src/Speckle.Sdk/Serialisation/V2/Receive/ObjectDeserializerFactory.cs

@@ -0,0 +1,11 @@
+using Speckle.InterfaceGenerator;
+using Speckle.Sdk.Models;
+
+namespace Speckle.Sdk.Serialisation.V2.Receive;
+
+[GenerateAutoInterface]
+public class ObjectDeserializerFactory : IObjectDeserializerFactory
+{
+  public IObjectDeserializer Create(IReadOnlyDictionary<string, Base> references, DeserializeOptions? options = null) =>
+    new ObjectDeserializer(references, SpeckleObjectSerializerPool.Instance, options);
+}

src/Speckle.Sdk/Serialisation/V2/Receive/ObjectLoader.cs

@@ -10,7 +10,6 @@ namespace Speckle.Sdk.Serialisation.V2.Receive;
 public sealed class ObjectLoader(
   ISQLiteReceiveCacheManager sqliteReceiveCacheManager,
   IServerObjectManager serverObjectManager,
-  string streamId,
   IProgress<ProgressArgs>? progress
 ) : ChannelLoader, IObjectLoader
 {
@@ -38,7 +37,7 @@ public sealed class ObjectLoader(
       }
     }
     rootJson = await serverObjectManager
-      .DownloadSingleObject(streamId, rootId, progress, cancellationToken)
+      .DownloadSingleObject(rootId, progress, cancellationToken)
       .NotNull()
       .ConfigureAwait(false);
     List<string> allChildrenIds = ClosureParser
@@ -76,12 +75,7 @@ public sealed class ObjectLoader(
   {
     var toCache = new List<BaseItem>();
     await foreach (
-      var (id, json) in serverObjectManager.DownloadObjects(
-        streamId,
-        ids.Select(x => x.NotNull()).ToList(),
-        progress,
-        default
-      )
+      var (id, json) in serverObjectManager.DownloadObjects(ids.Select(x => x.NotNull()).ToList(), progress, default)
     )
     {
       toCache.Add(new(id, json, true));
@@ -97,12 +91,12 @@ public sealed class ObjectLoader(
   }
 
   [AutoInterfaceIgnore]
-  public override void SaveToCache(BaseItem x)
+  public override void SaveToCache(List<BaseItem> batch)
   {
     if (!_options.SkipCache)
     {
-      sqliteReceiveCacheManager.SaveObject(x);
-      _cached++;
+      sqliteReceiveCacheManager.SaveObjects(batch);
+      Interlocked.Exchange(ref _cached, _cached + batch.Count);
       progress?.Report(new(ProgressEvent.CachedToLocal, _cached, _allChildrenCount));
     }
   }

src/Speckle.Sdk/Serialisation/V2/SQLiteReceiveCacheManager.cs

@@ -34,6 +34,26 @@ public class SQLiteReceiveCacheManager(string streamId) : SQLiteCacheManager(str
     command.ExecuteNonQuery();
   }
 
+  public void SaveObjects(List<BaseItem> items)
+  {
+    using var c = new SqliteConnection(ConnectionString);
+    c.Open();
+    using var t = c.BeginTransaction();
+    const string COMMAND_TEXT = "INSERT OR IGNORE INTO objects(hash, content) VALUES(@hash, @content)";
+
+    using var command = new SqliteCommand(COMMAND_TEXT, c);
+    command.Transaction = t;
+    var idParam = command.Parameters.Add("@hash", SqliteType.Text);
+    var jsonParam = command.Parameters.Add("@content", SqliteType.Text);
+    foreach (var item in items)
+    {
+      idParam.Value = item.Id;
+      jsonParam.Value = item.Json;
+      command.ExecuteNonQuery();
+    }
+    t.Commit();
+  }
+
   public bool HasObject(string objectId)
   {
     using var c = new SqliteConnection(ConnectionString);

src/Speckle.Sdk/Serialisation/V2/Send/BaseChildFinder.cs

@@ -5,14 +5,14 @@ using Speckle.Sdk.Models;
 namespace Speckle.Sdk.Serialisation.V2.Send;
 
 [GenerateAutoInterface]
-public class SpeckleBaseChildFinder(ISpeckleBasePropertyGatherer propertyGatherer) : ISpeckleBaseChildFinder
+public class BaseChildFinder(IBasePropertyGatherer propertyGatherer) : IBaseChildFinder
 {
   public IEnumerable<Property> GetChildProperties(Base obj) =>
     propertyGatherer.ExtractAllProperties(obj).Where(x => x.PropertyAttributeInfo.IsDetachable);
 
   public IEnumerable<Base> GetChildren(Base obj)
   {
-    var props = GetChildProperties(obj).ToList();
+    var props = GetChildProperties(obj);
     foreach (var kvp in props)
     {
       if (kvp.Value is Base child)

src/Speckle.Sdk/Serialisation/V2/Send/BasePropertyGatherer.cs

@@ -1,4 +1,4 @@
-using System.Collections.Concurrent;
+using System.Collections.Concurrent;
 using System.Reflection;
 using Speckle.InterfaceGenerator;
 using Speckle.Newtonsoft.Json;
@@ -11,7 +11,7 @@ namespace Speckle.Sdk.Serialisation.V2.Send;
 public readonly record struct Property(string Name, object? Value, PropertyAttributeInfo PropertyAttributeInfo);
 
 [GenerateAutoInterface]
-public class SpeckleBasePropertyGatherer : ISpeckleBasePropertyGatherer
+public class BasePropertyGatherer : IBasePropertyGatherer
 {
   private readonly ConcurrentDictionary<string, List<(PropertyInfo, PropertyAttributeInfo)>> _typedPropertiesCache =
     new();

src/Speckle.Sdk/Serialisation/V2/Send/ObjectSerializer.cs

@@ -1,22 +1,28 @@
 using System.Collections;
+using System.Collections.Concurrent;
 using System.Drawing;
 using System.Globalization;
 using Speckle.DoubleNumerics;
+using Speckle.InterfaceGenerator;
 using Speckle.Newtonsoft.Json;
 using Speckle.Sdk.Common;
+using Speckle.Sdk.Dependencies;
+using Speckle.Sdk.Dependencies.Serialization;
 using Speckle.Sdk.Helpers;
 using Speckle.Sdk.Models;
 using Speckle.Sdk.Serialisation.Utilities;
 
 namespace Speckle.Sdk.Serialisation.V2.Send;
 
-public class SpeckleObjectSerializer2
+[GenerateAutoInterface]
+public class ObjectSerializer : IObjectSerializer
 {
   private HashSet<object> _parentObjects = new();
-  private readonly List<Dictionary<string, int>> _childclosures;
+  private readonly Dictionary<string, int> _currentClosures = new();
+  private readonly ConcurrentDictionary<Base, (string, Dictionary<string, int>)> _baseCache;
 
   private readonly bool _trackDetachedChildren;
-  private readonly ISpeckleBasePropertyGatherer _propertyGatherer;
+  private readonly IBasePropertyGatherer _propertyGatherer;
   private readonly CancellationToken _cancellationToken;
 
   /// <summary>
@@ -32,14 +38,14 @@ public class SpeckleObjectSerializer2
   /// </summary>
   /// <param name="trackDetachedChildren">Whether to store all detachable objects while serializing. They can be retrieved via <see cref="ObjectReferences"/> post serialization.</param>
   /// <param name="cancellationToken"></param>
-  public SpeckleObjectSerializer2(
-    ISpeckleBasePropertyGatherer propertyGatherer,
-    List<Dictionary<string, int>> childclosures,
+  public ObjectSerializer(
+    IBasePropertyGatherer propertyGatherer,
+    ConcurrentDictionary<Base, (string, Dictionary<string, int>)> baseCache,
     bool trackDetachedChildren = false,
     CancellationToken cancellationToken = default
   )
   {
-    _childclosures = childclosures;
+    _baseCache = baseCache;
     _propertyGatherer = propertyGatherer;
     _cancellationToken = cancellationToken;
     _trackDetachedChildren = trackDetachedChildren;
@@ -55,8 +61,9 @@ public class SpeckleObjectSerializer2
     {
       try
       {
-        var result = SerializeBase(baseObj, true).NotNull();
-        return [(result.Id.NotNull(), result.Json), .. _chunks];
+        var item = SerializeBase(baseObj, true).NotNull();
+        _baseCache.TryAdd(baseObj, (item.Json, _currentClosures));
+        return [new(item.Id, item.Json), .. _chunks];
       }
       catch (Exception ex) when (!ex.IsFatal() && ex is not OperationCanceledException)
       {
@@ -71,12 +78,7 @@ public class SpeckleObjectSerializer2
 
   // `Preserialize` means transforming all objects into the final form that will appear in json, with basic .net objects
   // (primitives, lists and dictionaries with string keys)
-  private void SerializeProperty(
-    object? obj,
-    JsonWriter writer,
-    bool computeClosures = false,
-    PropertyAttributeInfo inheritedDetachInfo = default
-  )
+  private void SerializeProperty(object? obj, JsonWriter writer, PropertyAttributeInfo inheritedDetachInfo = default)
   {
     _cancellationToken.ThrowIfCancellationRequested();
 
@@ -105,21 +107,13 @@ public class SpeckleObjectSerializer2
             ["referencedId"] = r.referencedId,
             ["__closure"] = r.closure,
           };
-        if (r.closure is not null)
-        {
-          foreach (var kvp in r.closure)
-          {
-            UpdateChildClosures(kvp.Key);
-          }
-        }
-        UpdateChildClosures(r.referencedId);
         SerializeProperty(ret, writer);
         break;
       case Base b:
-        var result = SerializeBase(b, computeClosures, inheritedDetachInfo);
+        var result = SerializeBase(b, false, inheritedDetachInfo);
         if (result is not null)
         {
-          writer.WriteRawValue(result.Json);
+          writer.WriteRawValue(result.Value.Json);
         }
         else
         {
@@ -198,11 +192,7 @@ public class SpeckleObjectSerializer2
     }
   }
 
-  private SerializationResult? SerializeBase(
-    Base baseObj,
-    bool computeClosures = false,
-    PropertyAttributeInfo inheritedDetachInfo = default
-  )
+  private BaseItem? SerializeBase(Base baseObj, bool isRoot, PropertyAttributeInfo inheritedDetachInfo = default)
   {
     // handle circular references
     bool alreadySerialized = !_parentObjects.Add(baseObj);
@@ -211,25 +201,25 @@ public class SpeckleObjectSerializer2
       return null;
     }
 
-    Dictionary<string, int> closure = new();
+    Dictionary<string, int> childClosures;
     string id;
     string json;
-    lock (_childclosures)
+    if (_baseCache.TryGetValue(baseObj, out var info))
     {
-      if (computeClosures || inheritedDetachInfo.IsDetachable || baseObj is Blob)
-      {
-        _childclosures.Add(closure);
-      }
-
-      using var writer = new StringWriter();
+      id = baseObj.id;
+      childClosures = info.Item2;
+      json = info.Item1;
+      MergeClosures(_currentClosures, childClosures);
+    }
+    else
+    {
+      childClosures = isRoot ? _currentClosures : new();
+      var sb = Pools.StringBuilders.Get();
+      using var writer = new StringWriter(sb);
       using var jsonWriter = SpeckleObjectSerializerPool.Instance.GetJsonTextWriter(writer);
-      id = SerializeBaseObject(baseObj, jsonWriter, closure);
+      id = SerializeBaseObject(baseObj, jsonWriter, childClosures);
       json = writer.ToString();
-
-      if (computeClosures || inheritedDetachInfo.IsDetachable || baseObj is Blob)
-      {
-        _childclosures.RemoveAt(_childclosures.Count - 1);
-      }
+      Pools.StringBuilders.Return(sb);
     }
 
     _parentObjects.Remove(baseObj);
@@ -245,8 +235,7 @@ public class SpeckleObjectSerializer2
     if (inheritedDetachInfo.IsDetachable)
     {
       var json2 = ReferenceGenerator.CreateReference(id);
-      UpdateChildClosures(id);
-
+      AddClosure(id);
       // add to obj refs to return
       if (baseObj.applicationId != null && _trackDetachedChildren) // && baseObj is not DataChunk && baseObj is not Abstract) // not needed, as data chunks will never have application ids, and abstract objs are not really used.
       {
@@ -254,16 +243,16 @@ public class SpeckleObjectSerializer2
         {
           referencedId = id,
           applicationId = baseObj.applicationId,
-          closure = closure,
+          closure = childClosures,
         };
       }
-      _chunks.Add((id, json));
-      return new(json2, null);
+      _chunks.Add(new(id, json));
+      return new(id, json2, true);
     }
-    return new(json.NotNull(), id);
+    return new(id, json, true);
   }
 
-  private string SerializeBaseObject(Base baseObj, JsonWriter writer, IReadOnlyDictionary<string, int> closure)
+  private string SerializeBaseObject(Base baseObj, JsonWriter writer, Dictionary<string, int> closure)
   {
     if (baseObj is not Blob)
     {
@@ -280,7 +269,7 @@ public class SpeckleObjectSerializer2
       }
 
       writer.WritePropertyName(prop.Name);
-      SerializeProperty(prop.Value, writer, prop.PropertyAttributeInfo);
+      SerializeOrChunkProperty(prop.Value, writer, prop.PropertyAttributeInfo);
     }
 
     string id;
@@ -313,7 +302,7 @@ public class SpeckleObjectSerializer2
     return id;
   }
 
-  private void SerializeProperty(object? baseValue, JsonWriter jsonWriter, PropertyAttributeInfo detachInfo)
+  private void SerializeOrChunkProperty(object? baseValue, JsonWriter jsonWriter, PropertyAttributeInfo detachInfo)
   {
     if (baseValue is IEnumerable chunkableCollection && detachInfo.IsChunkable)
     {
@@ -342,20 +331,13 @@ public class SpeckleObjectSerializer2
     SerializeProperty(baseValue, jsonWriter, inheritedDetachInfo: detachInfo);
   }
 
-  private void UpdateChildClosures(string objectId)
+  private static void MergeClosures(Dictionary<string, int> current, Dictionary<string, int> child)
   {
-    lock (_childclosures)
+    foreach (var closure in child)
     {
-      for (int i = 0; i < _childclosures.Count; i++)
-      {
-        int childDepth = _childclosures.Count - i;
-        if (!_childclosures[i].TryGetValue(objectId, out int currentValue))
-        {
-          currentValue = childDepth;
-        }
-
-        _childclosures[i][objectId] = Math.Min(currentValue, childDepth);
-      }
+      current[closure.Key] = 100;
     }
   }
+
+  private void AddClosure(string id) => _currentClosures[id] = 100;
 }

src/Speckle.Sdk/Serialisation/V2/Send/ObjectSerializerFactory.cs

@@ -0,0 +1,14 @@
+using System.Collections.Concurrent;
+using Speckle.InterfaceGenerator;
+using Speckle.Sdk.Models;
+
+namespace Speckle.Sdk.Serialisation.V2.Send;
+
+[GenerateAutoInterface]
+public class ObjectSerializerFactory(IBasePropertyGatherer propertyGatherer) : IObjectSerializerFactory
+{
+  public IObjectSerializer Create(
+    ConcurrentDictionary<Base, (string, Dictionary<string, int>)> baseCache,
+    CancellationToken cancellationToken
+  ) => new ObjectSerializer(propertyGatherer, baseCache, true, cancellationToken);
+}

src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs

@@ -1,4 +1,5 @@
 using System.Collections.Concurrent;
+using Speckle.InterfaceGenerator;
 using Speckle.Sdk.Common;
 using Speckle.Sdk.Dependencies.Serialization;
 using Speckle.Sdk.Models;
@@ -6,17 +7,19 @@ using Speckle.Sdk.Transports;
 
 namespace Speckle.Sdk.Serialisation.V2.Send;
 
-public record SerializeProcessOptions(bool SkipCache, bool SkipServer);
+public record SerializeProcessOptions(bool SkipCacheRead, bool SkipCacheWrite, bool SkipServer);
 
+[GenerateAutoInterface]
 public class SerializeProcess(
   IProgress<ProgressArgs>? progress,
   ISQLiteSendCacheManager sqliteSendCacheManager,
   IServerObjectManager serverObjectManager,
-  ISpeckleBaseChildFinder speckleBaseChildFinder,
-  ISpeckleBasePropertyGatherer speckleBasePropertyGatherer
-) : ChannelSaver
+  IBaseChildFinder baseChildFinder,
+  IObjectSerializerFactory objectSerializerFactory
+) : ChannelSaver, ISerializeProcess
 {
   private readonly ConcurrentDictionary<string, string> _jsonCache = new();
+  private readonly ConcurrentDictionary<Base, (string, Dictionary<string, int>)> _baseCache = new();
   private readonly ConcurrentDictionary<string, ObjectReference> _objectReferences = new();
 
   private long _totalFound;
@@ -25,26 +28,25 @@ public class SerializeProcess(
   private long _cached;
   private long _serialized;
 
-  private SerializeProcessOptions _options = new(false, false);
+  private SerializeProcessOptions _options = new(false, false, false);
 
   public async Task<(string rootObjId, IReadOnlyDictionary<string, ObjectReference> convertedReferences)> Serialize(
-    string streamId,
     Base root,
     CancellationToken cancellationToken,
     SerializeProcessOptions? options = null
   )
   {
     _options = options ?? _options;
-    var channelTask = Start(streamId, cancellationToken);
+    var channelTask = Start(cancellationToken);
     await Traverse(root, true, cancellationToken).ConfigureAwait(false);
     await channelTask.ConfigureAwait(false);
     return (root.id, _objectReferences);
   }
 
-  private async Task<List<Dictionary<string, int>>> Traverse(Base obj, bool isEnd, CancellationToken cancellationToken)
+  private async Task Traverse(Base obj, bool isEnd, CancellationToken cancellationToken)
   {
-    var tasks = new List<Task<List<Dictionary<string, int>>>>();
-    foreach (var child in speckleBaseChildFinder.GetChildren(obj))
+    var tasks = new List<Task>();
+    foreach (var child in baseChildFinder.GetChildren(obj))
     {
       Interlocked.Increment(ref _totalFound);
       progress?.Report(new(ProgressEvent.FindingChildren, _totalFound, null));
@@ -65,19 +67,8 @@ public class SerializeProcess(
     {
       await Task.WhenAll(tasks).ConfigureAwait(false);
     }
-    var closures = tasks
-      .Select(t => t.Result)
-      .Aggregate(
-        new List<Dictionary<string, int>>(),
-        (a, s) =>
-        {
-          a.AddRange(s);
-          return a;
-        }
-      )
-      .ToList();
 
-    var items = Serialise(obj, closures);
+    var items = Serialise(obj, cancellationToken);
     foreach (var item in items)
     {
       Interlocked.Increment(ref _serialized);
@@ -93,18 +84,17 @@ public class SerializeProcess(
     {
       await Done().ConfigureAwait(false);
     }
-    return closures;
   }
 
   //leave this sync
-  private IEnumerable<BaseItem> Serialise(Base obj, List<Dictionary<string, int>> childClosures)
+  private IEnumerable<BaseItem> Serialise(Base obj, CancellationToken cancellationToken)
   {
     if (obj.id != null && _jsonCache.ContainsKey(obj.id))
     {
       yield break;
     }
 
-    if (!_options.SkipCache && obj.id != null)
+    if (!_options.SkipCacheRead && obj.id != null)
     {
       var cachedJson = sqliteSendCacheManager.GetObject(obj.id);
       if (cachedJson != null)
@@ -116,7 +106,7 @@ public class SerializeProcess(
     var id = obj.id;
     if (id is null || !_jsonCache.TryGetValue(id, out var json))
     {
-      SpeckleObjectSerializer2 serializer2 = new(speckleBasePropertyGatherer, childClosures, true);
+      var serializer2 = objectSerializerFactory.Create(_baseCache, cancellationToken);
       var items = serializer2.Serialize(obj).ToList();
       foreach (var kvp in serializer2.ObjectReferences)
       {
@@ -149,7 +139,7 @@ public class SerializeProcess(
 
   private BaseItem CheckCache(string id, string json)
   {
-    if (!_options.SkipCache)
+    if (!_options.SkipCacheRead)
     {
       var cachedJson = sqliteSendCacheManager.GetObject(id);
       if (cachedJson != null)
@@ -160,38 +150,23 @@ public class SerializeProcess(
     return new BaseItem(id, json, true);
   }
 
-  public override async Task<List<BaseItem>> SendToServer(
-    string streamId,
-    List<BaseItem> batch,
-    CancellationToken cancellationToken
-  )
+  public override async Task<List<BaseItem>> SendToServer(List<BaseItem> batch, CancellationToken cancellationToken)
   {
-    if (batch.Count == 0)
+    if (!_options.SkipServer && batch.Count != 0)
     {
-      progress?.Report(new(ProgressEvent.UploadedObjects, _uploaded, _totalToUpload));
-      return batch;
-    }
-
-    if (!_options.SkipServer)
-    {
-      await serverObjectManager.UploadObjects(streamId, batch, true, progress, cancellationToken).ConfigureAwait(false);
+      await serverObjectManager.UploadObjects(batch, true, progress, cancellationToken).ConfigureAwait(false);
       Interlocked.Exchange(ref _uploaded, _uploaded + batch.Count);
       progress?.Report(new(ProgressEvent.UploadedObjects, _uploaded, _totalToUpload));
     }
     return batch;
   }
 
-  public override void SaveToCache(List<BaseItem> items)
+  public override void SaveToCache(List<BaseItem> batch)
   {
-    if (!_options.SkipCache)
+    if (!_options.SkipCacheWrite && batch.Count != 0)
     {
-      if (items.Count == 0)
-      {
-        progress?.Report(new(ProgressEvent.CachedToLocal, _cached, null));
-        return;
-      }
-      sqliteSendCacheManager.SaveObjects(items);
-      Interlocked.Exchange(ref _cached, _cached + items.Count);
+      sqliteSendCacheManager.SaveObjects(batch);
+      Interlocked.Exchange(ref _cached, _cached + batch.Count);
       progress?.Report(new(ProgressEvent.CachedToLocal, _cached, null));
     }
   }

src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs

@@ -0,0 +1,64 @@
+using Speckle.Sdk.Helpers;
+using Speckle.Sdk.Logging;
+using Speckle.Sdk.Serialisation.V2.Receive;
+using Speckle.Sdk.Serialisation.V2.Send;
+using Speckle.Sdk.Transports;
+
+namespace Speckle.Sdk.Serialisation.V2;
+
+public interface ISerializeProcessFactory
+{
+  ISerializeProcess CreateSerializeProcess(
+    Uri url,
+    string streamId,
+    string? authorizationToken,
+    IProgress<ProgressArgs>? progress
+  );
+  IDeserializeProcess CreateDeserializeProcess(
+    Uri url,
+    string streamId,
+    string? authorizationToken,
+    IProgress<ProgressArgs>? progress
+  );
+}
+
+public class SerializeProcessFactory(
+  ISpeckleHttp speckleHttp,
+  ISdkActivityFactory activityFactory,
+  IBaseChildFinder baseChildFinder,
+  IObjectSerializerFactory objectSerializerFactory,
+  IObjectDeserializerFactory objectDeserializerFactory
+) : ISerializeProcessFactory
+{
+  public ISerializeProcess CreateSerializeProcess(
+    Uri url,
+    string streamId,
+    string? authorizationToken,
+    IProgress<ProgressArgs>? progress
+  )
+  {
+    var sqliteSendCacheManager = new SQLiteSendCacheManager(streamId);
+    var serverObjectManager = new ServerObjectManager(speckleHttp, activityFactory, url, streamId, authorizationToken);
+    return new SerializeProcess(
+      progress,
+      sqliteSendCacheManager,
+      serverObjectManager,
+      baseChildFinder,
+      objectSerializerFactory
+    );
+  }
+
+  public IDeserializeProcess CreateDeserializeProcess(
+    Uri url,
+    string streamId,
+    string? authorizationToken,
+    IProgress<ProgressArgs>? progress
+  )
+  {
+    var sqliteSendCacheManager = new SQLiteReceiveCacheManager(streamId);
+    var serverObjectManager = new ServerObjectManager(speckleHttp, activityFactory, url, streamId, authorizationToken);
+
+    var objectLoader = new ObjectLoader(sqliteSendCacheManager, serverObjectManager, progress);
+    return new DeserializeProcess(progress, objectLoader, objectDeserializerFactory);
+  }
+}

src/Speckle.Sdk/Serialisation/V2/ServerObjectManager.cs

@@ -16,15 +16,17 @@ namespace Speckle.Sdk.Serialisation.V2;
 [GenerateAutoInterface]
 public class ServerObjectManager : IServerObjectManager
 {
-  private static readonly char[] s_separator = { '\t' };
+  private static readonly char[] s_separator = ['\t'];
 
   private readonly ISdkActivityFactory _activityFactory;
   private readonly HttpClient _client;
+  private readonly string _streamId;
 
   public ServerObjectManager(
     ISpeckleHttp speckleHttp,
     ISdkActivityFactory activityFactory,
     Uri baseUri,
+    string streamId,
     string? authorizationToken,
     int timeoutSeconds = 120
   )
@@ -36,10 +38,10 @@ public class ServerObjectManager : IServerObjectManager
       authorizationToken: authorizationToken
     );
     _client.BaseAddress = baseUri;
+    _streamId = streamId;
   }
 
   public async IAsyncEnumerable<(string, string)> DownloadObjects(
-    string streamId,
     IReadOnlyList<string> objectIds,
     IProgress<ProgressArgs>? progress,
     [EnumeratorCancellation] CancellationToken cancellationToken
@@ -50,7 +52,7 @@ public class ServerObjectManager : IServerObjectManager
 
     using var childrenHttpMessage = new HttpRequestMessage
     {
-      RequestUri = new Uri($"/api/getobjects/{streamId}", UriKind.Relative),
+      RequestUri = new Uri($"/api/getobjects/{_streamId}", UriKind.Relative),
       Method = HttpMethod.Post,
     };
 
@@ -73,7 +75,6 @@ public class ServerObjectManager : IServerObjectManager
   }
 
   public async Task<string?> DownloadSingleObject(
-    string streamId,
     string objectId,
     IProgress<ProgressArgs>? progress,
     CancellationToken cancellationToken
@@ -85,7 +86,7 @@ public class ServerObjectManager : IServerObjectManager
     // Get root object
     using var rootHttpMessage = new HttpRequestMessage
     {
-      RequestUri = new Uri($"/objects/{streamId}/{objectId}/single", UriKind.Relative),
+      RequestUri = new Uri($"/objects/{_streamId}/{objectId}/single", UriKind.Relative),
       Method = HttpMethod.Get,
     };
 
@@ -138,7 +139,6 @@ public class ServerObjectManager : IServerObjectManager
   }
 
   public async Task<Dictionary<string, bool>> HasObjects(
-    string streamId,
     IReadOnlyList<string> objectIds,
     CancellationToken cancellationToken
   )
@@ -150,7 +150,7 @@ public class ServerObjectManager : IServerObjectManager
     string objectsPostParameter = JsonConvert.SerializeObject(objectIds);
     var payload = new Dictionary<string, string> { { "objects", objectsPostParameter } };
     string serializedPayload = JsonConvert.SerializeObject(payload);
-    var uri = new Uri($"/api/diff/{streamId}", UriKind.Relative);
+    var uri = new Uri($"/api/diff/{_streamId}", UriKind.Relative);
 
     using StringContent stringContent = new(serializedPayload, Encoding.UTF8, "application/json");
     using HttpResponseMessage response = await _client
@@ -167,7 +167,6 @@ public class ServerObjectManager : IServerObjectManager
   }
 
   public async Task UploadObjects(
-    string streamId,
     IReadOnlyList<BaseItem> objects,
     bool compressPayloads,
     IProgress<ProgressArgs>? progress,
@@ -177,7 +176,7 @@ public class ServerObjectManager : IServerObjectManager
     cancellationToken.ThrowIfCancellationRequested();
 
     using HttpRequestMessage message =
-      new() { RequestUri = new Uri($"/objects/{streamId}", UriKind.Relative), Method = HttpMethod.Post };
+      new() { RequestUri = new Uri($"/objects/{_streamId}", UriKind.Relative), Method = HttpMethod.Post };
 
     MultipartFormDataContent multipart = new();
 

tests/Speckle.Sdk.Serialization.Testing/DummyServerObjectManager.cs

@@ -8,27 +8,23 @@ namespace Speckle.Sdk.Serialization.Testing;
 public class DummyServerObjectManager : IServerObjectManager
 {
   public IAsyncEnumerable<(string, string)> DownloadObjects(
-    string streamId,
     IReadOnlyList<string> objectIds,
     IProgress<ProgressArgs>? progress,
     CancellationToken cancellationToken
   ) => throw new NotImplementedException();
 
   public Task<string?> DownloadSingleObject(
-    string streamId,
     string objectId,
     IProgress<ProgressArgs>? progress,
     CancellationToken cancellationToken
   ) => throw new NotImplementedException();
 
   public Task<Dictionary<string, bool>> HasObjects(
-    string streamId,
     IReadOnlyList<string> objectIds,
     CancellationToken cancellationToken
   ) => throw new NotImplementedException();
 
   public Task UploadObjects(
-    string streamId,
     IReadOnlyList<BaseItem> objects,
     bool compressPayloads,
     IProgress<ProgressArgs>? progress,

tests/Speckle.Sdk.Serialization.Testing/Program.cs

@@ -1,3 +1,4 @@
+#pragma warning disable CA1506
 using System.Reflection;
 using Microsoft.Extensions.DependencyInjection;
 using Speckle.Sdk;
@@ -11,23 +12,25 @@ using Speckle.Sdk.Serialisation.V2.Receive;
 using Speckle.Sdk.Serialisation.V2.Send;
 using Speckle.Sdk.Serialization.Testing;
 
-const bool skipCache = false;
+const bool skipCacheReceive = false;
+const bool skipCacheSendCheck = true;
+const bool skipCacheSendSave = false;
 TypeLoader.Reset();
 TypeLoader.Initialize(typeof(Base).Assembly, Assembly.GetExecutingAssembly());
 
-/*
 var url = "https://latest.speckle.systems/projects/a3ac1b2706/models/59d3b0f3c6"; //small?
 var streamId = "a3ac1b2706";
-var rootId = "7d53bcf28c6696ecac8781684a0aa006";*/
+var rootId = "7d53bcf28c6696ecac8781684a0aa006";
+
 /*
 var url = "https://latest.speckle.systems/"; //other?
 var streamId = "368f598929";
 var rootId = "67374cfe689c43ff8be12090af122244";*/
 
-
+/*
 var url = "https://latest.speckle.systems/projects/2099ac4b5f/models/da511c4d1e"; //perf?
 var streamId = "2099ac4b5f";
-var rootId = "30fb4cbe6eb2202b9e7b4a4fcc3dd2b6";
+var rootId = "30fb4cbe6eb2202b9e7b4a4fcc3dd2b6";*/
 
 var serviceCollection = new ServiceCollection();
 serviceCollection.AddSpeckleSdk(HostApplications.Navisworks, HostAppVersion.v2023, "Test");
@@ -37,27 +40,25 @@ Console.WriteLine("Attach");
 
 var token = serviceProvider.GetRequiredService<IAccountManager>().GetDefaultAccount()?.token;
 var progress = new Progress(true);
-var sqliteTransport = new SQLiteReceiveCacheManager(streamId);
-var serverObjects = new ServerObjectManager(
+
+var factory = new SerializeProcessFactory(
   serviceProvider.GetRequiredService<ISpeckleHttp>(),
   serviceProvider.GetRequiredService<ISdkActivityFactory>(),
-  new Uri(url),
-  token
+  new BaseChildFinder(new BasePropertyGatherer()),
+  new ObjectSerializerFactory(new BasePropertyGatherer()),
+  new ObjectDeserializerFactory()
 );
-var o = new ObjectLoader(sqliteTransport, serverObjects, streamId, progress);
-var process = new DeserializeProcess(progress, o);
-var @base = await process.Deserialize(rootId, default, new(skipCache)).ConfigureAwait(false);
+var process = factory.CreateDeserializeProcess(new Uri(url), streamId, token, progress);
+var @base = await process.Deserialize(rootId, default, new(skipCacheReceive)).ConfigureAwait(false);
+
 Console.WriteLine("Deserialized");
 Console.ReadLine();
 Console.WriteLine("Executing");
 
-var process2 = new SerializeProcess(
-  progress,
-  new SQLiteSendCacheManager(streamId),
-  new DummyServerObjectManager(),
-  new SpeckleBaseChildFinder(new SpeckleBasePropertyGatherer()),
-  new SpeckleBasePropertyGatherer()
-);
-await process2.Serialize(streamId, @base, default, new SerializeProcessOptions(skipCache, true)).ConfigureAwait(false);
+var process2 = factory.CreateSerializeProcess(new Uri(url), streamId, token, progress);
+await process2
+  .Serialize(@base, default, new SerializeProcessOptions(skipCacheSendCheck, skipCacheSendSave, true))
+  .ConfigureAwait(false);
 Console.WriteLine("Detach");
 Console.ReadLine();
+#pragma warning restore CA1506

tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs

@@ -2,6 +2,7 @@
 using System.Text;
 using NUnit.Framework;
 using Shouldly;
+using Speckle.Newtonsoft.Json;
 using Speckle.Newtonsoft.Json.Linq;
 using Speckle.Objects.Geometry;
 using Speckle.Sdk.Dependencies.Serialization;
@@ -47,7 +48,7 @@ public class DetachedTests
           "dynamicProp": 123,
           "id": "9ff8efb13c62fa80f3d1c4519376ba13",
           "__closure": {
-              "d3dd4621b2f68c3058c2b9c023a9de19": 1
+              "d3dd4621b2f68c3058c2b9c023a9de19": 100
           }
       }
       """;
@@ -70,12 +71,10 @@ public class DetachedTests
       null,
       new DummySendCacheManager(objects),
       new DummyServerObjectManager(),
-      new SpeckleBaseChildFinder(new SpeckleBasePropertyGatherer()),
-      new SpeckleBasePropertyGatherer()
+      new BaseChildFinder(new BasePropertyGatherer()),
+      new ObjectSerializerFactory(new BasePropertyGatherer())
     );
-    await process2
-      .Serialize(string.Empty, @base, default, new SerializeProcessOptions(false, true))
-      .ConfigureAwait(false);
+    await process2.Serialize(@base, default, new SerializeProcessOptions(false, false, true)).ConfigureAwait(false);
 
     objects.Count.ShouldBe(2);
     objects.ContainsKey("9ff8efb13c62fa80f3d1c4519376ba13").ShouldBeTrue();
@@ -155,7 +154,7 @@ public class DetachedTests
     @base.detachedProp = new SamplePropBase() { name = "detachedProp" };
     @base.attachedProp = new SamplePropBase() { name = "attachedProp" };
 
-    var children = new SpeckleBaseChildFinder(new SpeckleBasePropertyGatherer()).GetChildProperties(@base).ToList();
+    var children = new BaseChildFinder(new BasePropertyGatherer()).GetChildProperties(@base).ToList();
 
     children.Count.ShouldBe(4);
     children.First(x => x.Name == "detachedProp").PropertyAttributeInfo.IsDetachable.ShouldBeTrue();
@@ -174,7 +173,7 @@ public class DetachedTests
     @base.detachedProp = new SamplePropBase() { name = "detachedProp" };
     @base.attachedProp = new SamplePropBase() { name = "attachedProp" };
 
-    var children = new SpeckleBasePropertyGatherer().ExtractAllProperties(@base).ToList();
+    var children = new BasePropertyGatherer().ExtractAllProperties(@base).ToList();
 
     children.Count.ShouldBe(9);
     children.First(x => x.Name == "dynamicProp").PropertyAttributeInfo.IsDetachable.ShouldBeFalse();
@@ -224,14 +223,14 @@ public class DetachedTests
           "dynamicProp": 123,
           "id": "fd4efeb8a036838c53ad1cf9e82b8992",
           "__closure": {
-              "8d27f5c7fac36d985d89bb6d6d8acddc": 3,
-              "4ba53b5e84e956fb076bc8b0a03ca879": 2,
-              "32a385e7ddeda810e037b21ab26381b7": 1,
-              "1afc694774efa5913d0077302cd37888": 3,
-              "045cbee36837d589b17f9d8483c90763": 2,
-              "c3858f47dd3e7a308a1b465375f1645f": 1,
-              "5b86b66b61c556ead500915b05852875": 2,
-              "027a7c5ffcf8d8efe432899c729a954c": 1
+              "8d27f5c7fac36d985d89bb6d6d8acddc": 100,
+              "4ba53b5e84e956fb076bc8b0a03ca879": 100,
+              "32a385e7ddeda810e037b21ab26381b7": 100,
+              "1afc694774efa5913d0077302cd37888": 100,
+              "045cbee36837d589b17f9d8483c90763": 100,
+              "c3858f47dd3e7a308a1b465375f1645f": 100,
+              "5b86b66b61c556ead500915b05852875": 100,
+              "027a7c5ffcf8d8efe432899c729a954c": 100
           }
       }
       """;
@@ -263,15 +262,18 @@ public class DetachedTests
       null,
       new DummySendCacheManager(objects),
       new DummyServerObjectManager(),
-      new SpeckleBaseChildFinder(new SpeckleBasePropertyGatherer()),
-      new SpeckleBasePropertyGatherer()
+      new BaseChildFinder(new BasePropertyGatherer()),
+      new ObjectSerializerFactory(new BasePropertyGatherer())
     );
     var results = await process2
-      .Serialize(string.Empty, @base, default, new SerializeProcessOptions(false, true))
+      .Serialize(@base, default, new SerializeProcessOptions(false, false, true))
       .ConfigureAwait(false);
 
     objects.Count.ShouldBe(9);
-    JToken.DeepEquals(JObject.Parse(root), JObject.Parse(objects["fd4efeb8a036838c53ad1cf9e82b8992"])).ShouldBeTrue();
+    var x = JObject.Parse(objects["fd4efeb8a036838c53ad1cf9e82b8992"]);
+    var y = x.ToString(Formatting.Indented);
+    Console.WriteLine(y);
+    JToken.DeepEquals(JObject.Parse(root), x).ShouldBeTrue();
 
     results.rootObjId.ShouldBe(@base.id);
     results.convertedReferences.Count.ShouldBe(2);
@@ -333,27 +335,23 @@ public class SamplePropBase2 : Base
 public class DummyServerObjectManager : IServerObjectManager
 {
   public IAsyncEnumerable<(string, string)> DownloadObjects(
-    string streamId,
     IReadOnlyList<string> objectIds,
     IProgress<ProgressArgs>? progress,
     CancellationToken cancellationToken
   ) => throw new NotImplementedException();
 
   public Task<string?> DownloadSingleObject(
-    string streamId,
     string objectId,
     IProgress<ProgressArgs>? progress,
     CancellationToken cancellationToken
   ) => throw new NotImplementedException();
 
   public Task<Dictionary<string, bool>> HasObjects(
-    string streamId,
     IReadOnlyList<string> objectIds,
     CancellationToken cancellationToken
   ) => throw new NotImplementedException();
 
   public Task UploadObjects(
-    string streamId,
     IReadOnlyList<BaseItem> objects,
     bool compressPayloads,
     IProgress<ProgressArgs>? progress,

tests/Speckle.Sdk.Serialization.Tests/DummyReceiveServerObjectManager.cs

@@ -9,7 +9,6 @@ namespace Speckle.Sdk.Serialization.Tests;
 public class DummyReceiveServerObjectManager(Dictionary<string, string> objects) : IServerObjectManager
 {
   public async IAsyncEnumerable<(string, string)> DownloadObjects(
-    string streamId,
     IReadOnlyList<string> objectIds,
     IProgress<ProgressArgs>? progress,
     [EnumeratorCancellation] CancellationToken cancellationToken
@@ -23,7 +22,6 @@ public class DummyReceiveServerObjectManager(Dictionary<string, string> objects)
   }
 
   public async Task<string?> DownloadSingleObject(
-    string streamId,
     string objectId,
     IProgress<ProgressArgs>? progress,
     CancellationToken cancellationToken
@@ -34,13 +32,11 @@ public class DummyReceiveServerObjectManager(Dictionary<string, string> objects)
   }
 
   public Task<Dictionary<string, bool>> HasObjects(
-    string streamId,
     IReadOnlyList<string> objectIds,
     CancellationToken cancellationToken
   ) => throw new NotImplementedException();
 
   public Task UploadObjects(
-    string streamId,
     IReadOnlyList<BaseItem> objects,
     bool compressPayloads,
     IProgress<ProgressArgs>? progress,

tests/Speckle.Sdk.Serialization.Tests/DummySendServerObjectManager.cs

@@ -11,30 +11,23 @@ namespace Speckle.Sdk.Serialization.Tests;
 public class DummySendServerObjectManager(ConcurrentDictionary<string, string> savedObjects) : IServerObjectManager
 {
   public IAsyncEnumerable<(string, string)> DownloadObjects(
-    string streamId,
     IReadOnlyList<string> objectIds,
     IProgress<ProgressArgs>? progress,
     CancellationToken cancellationToken
   ) => throw new NotImplementedException();
 
   public Task<string?> DownloadSingleObject(
-    string streamId,
     string objectId,
     IProgress<ProgressArgs>? progress,
     CancellationToken cancellationToken
   ) => throw new NotImplementedException();
 
-  public Task<Dictionary<string, bool>> HasObjects(
-    string streamId,
-    IReadOnlyList<string> objectIds,
-    CancellationToken cancellationToken
-  )
+  public Task<Dictionary<string, bool>> HasObjects(IReadOnlyList<string> objectIds, CancellationToken cancellationToken)
   {
     return Task.FromResult(objectIds.ToDictionary(x => x, x => false));
   }
 
   public Task UploadObjects(
-    string streamId,
     IReadOnlyList<BaseItem> objects,
     bool compressPayloads,
     IProgress<ProgressArgs>? progress,

tests/Speckle.Sdk.Serialization.Tests/DummySqLiteReceiveManager.cs

@@ -9,5 +9,7 @@ public class DummySqLiteReceiveManager(Dictionary<string, string> savedObjects)
 
   public void SaveObject(BaseItem item) => throw new NotImplementedException();
 
+  public void SaveObjects(List<BaseItem> item) => throw new NotImplementedException();
+
   public bool HasObject(string objectId) => throw new NotImplementedException();
 }

tests/Speckle.Sdk.Serialization.Tests/SerializationTests.cs

@@ -146,7 +146,7 @@ public class SerializationTests
     var fullName = _assembly.GetManifestResourceNames().Single(x => x.EndsWith(fileName));
     var json = await ReadJson(fullName);
     var closures = ReadAsObjects(json);
-    var process = new DeserializeProcess(null, new TestObjectLoader(closures));
+    var process = new DeserializeProcess(null, new TestObjectLoader(closures), new ObjectDeserializerFactory());
     await process.Deserialize("3416d3fe01c9196115514c4a2f41617b", default);
     foreach (var (id, objJson) in closures)
     {
@@ -228,10 +228,9 @@ public class SerializationTests
     var o = new ObjectLoader(
       new DummySqLiteReceiveManager(closure),
       new DummyReceiveServerObjectManager(closure),
-      string.Empty,
       null
     );
-    var process = new DeserializeProcess(null, o);
+    var process = new DeserializeProcess(null, o, new ObjectDeserializerFactory());
     var root = await process.Deserialize(rootId, default, new DeserializeOptions(true));
 
     var newIdToJson = new ConcurrentDictionary<string, string>();
@@ -239,15 +238,10 @@ public class SerializationTests
       null,
       new DummySqLiteSendManager(),
       new DummySendServerObjectManager(newIdToJson),
-      new SpeckleBaseChildFinder(new SpeckleBasePropertyGatherer()),
-      new SpeckleBasePropertyGatherer()
-    );
-    var (rootId2, _) = await serializeProcess.Serialize(
-      string.Empty,
-      root,
-      default,
-      new SerializeProcessOptions(true, false)
+      new BaseChildFinder(new BasePropertyGatherer()),
+      new ObjectSerializerFactory(new BasePropertyGatherer())
     );
+    var (rootId2, _) = await serializeProcess.Serialize(root, default, new SerializeProcessOptions(true, true, false));
 
     rootId2.ShouldBe(root.id);
     newIdToJson.Count.ShouldBe(count);

tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/VersionResourceTests.cs

@@ -73,7 +73,7 @@ public class VersionResourceTests
   {
     const string NEW_MESSAGE = "MY new version message";
 
-    UpdateVersionInput input = new(_version.id, NEW_MESSAGE);
+    UpdateVersionInput input = new(_version.id, _project.id, NEW_MESSAGE);
     Version updatedVersion = await Sut.Update(input);
 
     Assert.That(updatedVersion, Has.Property(nameof(Version.id)).EqualTo(_version.id));
@@ -84,7 +84,7 @@ public class VersionResourceTests
   [Test]
   public async Task VersionMoveToModel()
   {
-    MoveVersionsInput input = new(_model2.name, new[] { _version.id });
+    MoveVersionsInput input = new(_project.id, _model2.name, [_version.id]);
     string id = await Sut.MoveToModel(input);
     Assert.That(id, Is.EqualTo(_model2.id));
     Version movedVersion = await Sut.Get(_version.id, _project.id);
@@ -97,7 +97,7 @@ public class VersionResourceTests
   [Test]
   public async Task VersionDelete()
   {
-    DeleteVersionsInput input = new(new[] { _version.id });
+    DeleteVersionsInput input = new([_version.id], _project.id);
 
     bool response = await Sut.Delete(input);
     Assert.That(response, Is.True);

tests/Speckle.Sdk.Tests.Performance/Benchmarks/GeneralDeserializerTest.cs

@@ -51,10 +51,11 @@ public class GeneralDeserializer : IDisposable
       TestDataHelper.ServiceProvider.GetRequiredService<ISpeckleHttp>(),
       TestDataHelper.ServiceProvider.GetRequiredService<ISdkActivityFactory>(),
       new Uri(url),
+      streamId,
       null
     );
-    var o = new ObjectLoader(sqlite, serverObjects, streamId, null);
-    var process = new DeserializeProcess(null, o);
+    var o = new ObjectLoader(sqlite, serverObjects, null);
+    var process = new DeserializeProcess(null, o, new ObjectDeserializerFactory());
     return await process.Deserialize(rootId, default, new(skipCache)).ConfigureAwait(false);
   }