Merge pull request #358 from specklesystems/dev

(NO SQUASH) Main to dev for release

src/Speckle.Objects/Other/LevelProxy.cs

@@ -0,0 +1,20 @@
+using Speckle.Objects.Data;
+using Speckle.Sdk.Models;
+using Speckle.Sdk.Models.Proxies;
+
+namespace Speckle.Objects.Other;
+
+/// <summary>
+/// Proxy for levels as DataObject value.
+/// <remarks> These proxy lives in Objects library because it depends on DataObject</remarks>
+/// </summary>
+[SpeckleType("Objects.Other.LevelProxy")]
+public class LevelProxy : Base, IProxyCollection
+{
+  /// <summary>
+  /// The list of application ids of objects that use this level
+  /// </summary>
+  public required List<string> objects { get; set; }
+
+  public required DataObject value { get; set; }
+}

src/Speckle.Objects/Other/RenderMaterial.cs

@@ -1,7 +1,6 @@
 using System.Drawing;
 using Speckle.Newtonsoft.Json;
 using Speckle.Sdk.Models;
-using Speckle.Sdk.Models.Proxies;
 
 namespace Speckle.Objects.Other;
 
@@ -39,20 +38,3 @@ public class RenderMaterial : Base
     set => diffuse = value.ToArgb();
   }
 }
-
-/// <summary>
-/// Used to store render material to object relationships in root collections
-/// </summary>
-[SpeckleType("Objects.Other.RenderMaterialProxy")]
-public class RenderMaterialProxy : Base, IProxyCollection
-{
-  /// <summary>
-  /// The list of application ids of objects that use this render material
-  /// </summary>
-  public required List<string> objects { get; set; }
-
-  /// <summary>
-  /// The render material used by <see cref="objects"/>
-  /// </summary>
-  public required RenderMaterial value { get; set; }
-}

src/Speckle.Objects/Other/RenderMaterialProxy.cs

@@ -0,0 +1,22 @@
+using Speckle.Sdk.Models;
+using Speckle.Sdk.Models.Proxies;
+
+namespace Speckle.Objects.Other;
+
+/// <summary>
+/// Used to store render material to object relationships in root collections
+/// <remarks> These proxy lives in Objects library because it depends on RenderMaterial</remarks>
+/// </summary>
+[SpeckleType("Objects.Other.RenderMaterialProxy")]
+public class RenderMaterialProxy : Base, IProxyCollection
+{
+  /// <summary>
+  /// The list of application ids of objects that use this render material
+  /// </summary>
+  public required List<string> objects { get; set; }
+
+  /// <summary>
+  /// The render material used by <see cref="objects"/>
+  /// </summary>
+  public required RenderMaterial value { get; set; }
+}

src/Speckle.Sdk.Dependencies/Collections.cs

@@ -22,23 +22,4 @@ public static class Collections
 public static class EnumerableExtensions
 {
   public static IEnumerable<int> RangeFrom(int from, int to) => Enumerable.Range(from, to - from + 1);
-
-#if NETSTANDARD2_0
-  public static IEnumerable<TSource> DistinctBy<TSource, TKey>(
-    this IEnumerable<TSource> source,
-    Func<TSource, TKey> keySelector
-  )
-  {
-    var keys = new HashSet<TKey>();
-    foreach (var element in source)
-    {
-      if (keys.Contains(keySelector(element)))
-      {
-        continue;
-      }
-      keys.Add(keySelector(element));
-      yield return element;
-    }
-  }
-#endif
 }

src/Speckle.Sdk.Dependencies/Serialization/ChannelLoader.cs

@@ -5,7 +5,7 @@ namespace Speckle.Sdk.Dependencies.Serialization;
 
 public abstract class ChannelLoader<T>(CancellationToken cancellationToken)
 {
-  private const int RECEIVE_CAPACITY = 5000;
+  private const int RECEIVE_CAPACITY = 10000;
 
   private const int HTTP_GET_CHUNK_SIZE = 500;
   private const int MAX_PARALLELISM_HTTP = 4;
@@ -109,6 +109,9 @@ public abstract class ChannelLoader<T>(CancellationToken cancellationToken)
     Exception = ex;
     _channel.Writer.TryComplete(ex);
     //cancel everything!
-    _cts.Cancel();
+    if (!_cts.IsCancellationRequested)
+    {
+      _cts.Cancel();
+    }
   }
 }

src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs

@@ -8,13 +8,13 @@ namespace Speckle.Sdk.Dependencies.Serialization;
 public abstract class ChannelSaver<T>
   where T : IHasByteSize
 {
-  private const int SEND_CAPACITY = 1000;
+  private const int SEND_CAPACITY = 10000;
   private const int HTTP_SEND_CHUNK_SIZE = 25_000_000; //bytes
   private static readonly TimeSpan HTTP_BATCH_TIMEOUT = TimeSpan.FromSeconds(2);
   private const int MAX_PARALLELISM_HTTP = 4;
   private const int HTTP_CAPACITY = 500;
-  private const int MAX_CACHE_WRITE_PARALLELISM = 4;
-  private const int MAX_CACHE_BATCH = 500;
+  private const int MAX_CACHE_WRITE_PARALLELISM = 1;
+  private const int MAX_CACHE_BATCH = 1000;
 
   private readonly Channel<T> _checkCacheChannel = Channel.CreateBounded<T>(
     new BoundedChannelOptions(SEND_CAPACITY)
@@ -45,9 +45,9 @@ public abstract class ChannelSaver<T>
         cancellationToken
       )
       .Join()
-      .Batch(cacheBatchSize ?? MAX_CACHE_BATCH)
+      .Batch(cacheBatchSize ?? MAX_CACHE_BATCH, singleReader: true)
       .WithTimeout(HTTP_BATCH_TIMEOUT)
-      .ReadAllConcurrently(maxParallelism ?? MAX_CACHE_WRITE_PARALLELISM, SaveToCache, cancellationToken)
+      .ReadAllConcurrently(MAX_CACHE_WRITE_PARALLELISM, SaveToCache, cancellationToken)
       .ContinueWith(
         t =>
         {

src/Speckle.Sdk.Dependencies/SpeckleHttpClientHandler.cs

@@ -41,13 +41,7 @@ internal sealed class SpeckleHttpClientHandler : DelegatingHandler
       activity?.InjectHeaders((k, v) => request.Headers.TryAddWithoutValidation(k, v));
 
       var policyResult = await _resiliencePolicy
-        .ExecuteAndCaptureAsync(
-          ctx =>
-          {
-            return base.SendAsync(request, cancellationToken);
-          },
-          context
-        )
+        .ExecuteAndCaptureAsync(ctx => base.SendAsync(request, cancellationToken), context)
         .ConfigureAwait(false);
       context.TryGetValue("retryCount", out var retryCount);
       activity?.SetTag("retryCount", retryCount);

src/Speckle.Sdk/Api/Exceptions.cs

@@ -126,3 +126,14 @@ public sealed class WorkspacePermissionException : SpeckleGraphQLException
   public WorkspacePermissionException(string? message, Exception? innerException)
     : base(message, innerException) { }
 }
+
+public sealed class CannotCreateCommitException : SpeckleGraphQLException
+{
+  public CannotCreateCommitException() { }
+
+  public CannotCreateCommitException(string? message)
+    : base(message) { }
+
+  public CannotCreateCommitException(string? message, Exception? innerException)
+    : base(message, innerException) { }
+}

src/Speckle.Sdk/Api/GraphQL/Client.cs

@@ -127,10 +127,10 @@ public sealed class Client : ISpeckleGraphQLClient, IClient
       activity?.SetStatus(SdkActivityStatusCode.Ok);
       return ret;
     }
-    catch (Exception ex)
+    catch (Exception)
     {
       activity?.SetStatus(SdkActivityStatusCode.Error);
-      activity?.RecordException(ex);
+      // Don't record exception as it's rethrown.
       throw;
     }
   }

src/Speckle.Sdk/Api/GraphQL/GraphQLErrorHandler.cs

@@ -33,6 +33,7 @@ internal static class GraphQLErrorHandler
         "BAD_USER_INPUT" => new SpeckleGraphQLBadInputException(message),
         "INTERNAL_SERVER_ERROR" => new SpeckleGraphQLInternalErrorException(message),
         "WORKSPACES_MODULE_DISABLED_ERROR" => new SpeckleGraphQLWorkspaceNotEnabledException(message),
+        "COMMIT_CREATE_ERROR" => new CannotCreateCommitException(message),
         _ => new SpeckleGraphQLException(message),
       };
       exceptions.Add(ex);

src/Speckle.Sdk/Api/GraphQL/GraphQLHttpClientExtensions.cs

@@ -40,7 +40,8 @@ public static class GraphQLHttpClientExtensions
     response.EnsureGraphQLSuccess();
 
     string versionString = response.Data.data.data;
-    if (versionString == "dev")
+    //Local server builds will have a non-numerical version string
+    if (versionString == "dev" || versionString == "custom")
     {
       return new Version(999, 999, 999);
     }

src/Speckle.Sdk/Api/Operations/Operations.Send.cs

@@ -47,6 +47,11 @@ public partial class Operations
       receiveActivity?.SetStatus(SdkActivityStatusCode.Ok);
       return results;
     }
+    catch (OperationCanceledException)
+    {
+      //this is handled by the caller
+      throw;
+    }
     catch (Exception ex)
     {
       receiveActivity?.SetStatus(SdkActivityStatusCode.Error);

src/Speckle.Sdk/Host/TypeLoader.cs

@@ -15,10 +15,14 @@ internal static class TypeLoader
   private static ConcurrentDictionary<string, Type> s_cachedTypes = new();
   private static ConcurrentDictionary<Type, string> s_fullTypeStrings = new();
   private static ConcurrentDictionary<PropertyInfo, JsonPropertyAttribute?> s_jsonPropertyAttribute = new();
+  private static readonly ConcurrentDictionary<PropertyInfo, bool> s_obsolete = new();
   private static ConcurrentDictionary<Type, IReadOnlyList<PropertyInfo>> s_propInfoCache = new();
 
   public static IEnumerable<LoadedType> Types => s_availableTypes;
 
+  public static bool IsObsolete(PropertyInfo property) =>
+    s_obsolete.GetOrAdd(property, p => p.IsDefined(typeof(ObsoleteAttribute), true));
+
   public static JsonPropertyAttribute? GetJsonPropertyAttribute(PropertyInfo property) =>
     s_jsonPropertyAttribute.GetOrAdd(property, p => p.GetCustomAttribute<JsonPropertyAttribute>(true));
 

src/Speckle.Sdk/Models/Base.cs

@@ -3,7 +3,6 @@ using System.Diagnostics.CodeAnalysis;
 using System.Reflection;
 using Speckle.Newtonsoft.Json;
 using Speckle.Newtonsoft.Json.Linq;
-using Speckle.Sdk.Common;
 using Speckle.Sdk.Helpers;
 using Speckle.Sdk.Host;
 using Speckle.Sdk.Serialisation;
@@ -92,8 +91,7 @@ public class Base : DynamicBase, ISpeckleObject
     var typedProps = @base.GetInstanceMembers();
     foreach (var prop in typedProps.Where(p => p.CanRead))
     {
-      bool isIgnored =
-        prop.IsDefined(typeof(ObsoleteAttribute), true) || prop.IsDefined(typeof(JsonIgnoreAttribute), true);
+      bool isIgnored = TypeLoader.IsObsolete(prop) || prop.IsDefined(typeof(JsonIgnoreAttribute), true);
       if (isIgnored)
       {
         continue;
@@ -193,30 +191,4 @@ public class Base : DynamicBase, ISpeckleObject
         return count;
     }
   }
-
-  /// <summary>
-  /// Creates a shallow copy of the current base object.
-  /// This operation does NOT copy/duplicate the data inside each prop.
-  /// The new object's property values will be pointers to the original object's property value.
-  /// </summary>
-  /// <returns>A shallow copy of the original object.</returns>
-  public Base ShallowCopy()
-  {
-    Type type = GetType();
-    Base myDuplicate = (Base)Activator.CreateInstance(type).NotNull();
-    myDuplicate.id = id;
-    myDuplicate.applicationId = applicationId;
-
-    foreach (var kvp in GetMembers())
-    {
-      var propertyInfo = type.GetProperty(kvp.Key);
-      if (propertyInfo is not null && !propertyInfo.CanWrite)
-      {
-        continue;
-      }
-      myDuplicate[kvp.Key] = kvp.Value;
-    }
-
-    return myDuplicate;
-  }
 }

src/Speckle.Sdk/Models/DynamicBase.cs

@@ -7,7 +7,7 @@ namespace Speckle.Sdk.Models;
 
 /// <summary>
 /// Base class implementing a bunch of nice dynamic object methods, like adding and removing props dynamically. Makes c# feel like json.
-/// <para>Orginally adapted from Rick Strahl 🤘</para>
+/// <para>Originally adapted from Rick Strahl 🤘</para>
 /// <para>https://weblog.west-wind.com/posts/2012/feb/08/creating-a-dynamic-extensible-c-expando-object</para>
 /// </summary>
 public class DynamicBase : DynamicObject, IDynamicMetaObjectProvider
@@ -84,6 +84,44 @@ public class DynamicBase : DynamicObject, IDynamicMetaObjectProvider
     }
   }
 
+  /// <summary>
+  /// Creates a shallow copy of the current base object.
+  /// This operation does NOT copy/duplicate the data inside each prop.
+  /// The new object's property values will be pointers to the original object's property value.
+  /// </summary>
+  /// <returns>A shallow copy of the original object.</returns>
+  public DynamicBase ShallowCopy()
+  {
+    Type type = GetType();
+    DynamicBase myDuplicate = (DynamicBase)(
+      Activator.CreateInstance(type) ?? throw new SpeckleException($"Failed to create instance of {type.Name}")
+    );
+
+    // Add dynamic members
+    foreach (var kvp in _properties)
+    {
+      myDuplicate._properties[kvp.Key] = kvp.Value;
+    }
+
+    var pinfos = TypeLoader.GetBaseProperties(type).Where(x => !TypeLoader.IsObsolete(x));
+    foreach (var pi in pinfos)
+    {
+      if (pi.CanWrite)
+      {
+        try
+        {
+          pi.SetValue(myDuplicate, pi.GetValue(this));
+        }
+        catch (Exception ex) when (!ex.IsFatal())
+        {
+          throw new SpeckleException($"Failed to set value for {type.Name}.{pi.Name}", ex);
+        }
+      }
+    }
+
+    return myDuplicate;
+  }
+
   /// <inheritdoc />
   /// <summary>
   /// Gets properties via the dot syntax.
@@ -232,7 +270,7 @@ public class DynamicBase : DynamicObject, IDynamicMetaObjectProvider
         .GetBaseProperties(GetType())
         .Where(x =>
         {
-          var hasObsolete = x.IsDefined(typeof(ObsoleteAttribute), true);
+          var hasObsolete = TypeLoader.IsObsolete(x);
 
           // If obsolete is false and prop has obsolete attr
           // OR

src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs

@@ -13,7 +13,26 @@ public sealed class SqLiteJsonCacheManager : ISqLiteJsonCacheManager
 {
   private readonly CacheDbCommandPool _pool;
 
-  public SqLiteJsonCacheManager(string path, int concurrency)
+  public static ISqLiteJsonCacheManager FromMemory(int concurrency) => new SqLiteJsonCacheManager(concurrency);
+
+  private SqLiteJsonCacheManager(int concurrency)
+  {
+    //disable pooling as we pool ourselves
+    var builder = new SqliteConnectionStringBuilder
+    {
+      Pooling = false,
+      DataSource = ":memory:",
+      Cache = SqliteCacheMode.Shared,
+      Mode = SqliteOpenMode.Memory,
+    };
+    _pool = new CacheDbCommandPool(builder.ToString(), concurrency);
+    Initialize();
+  }
+
+  public static ISqLiteJsonCacheManager FromFilePath(string path, int concurrency) =>
+    new SqLiteJsonCacheManager(path, concurrency);
+
+  private SqLiteJsonCacheManager(string path, int concurrency)
   {
     //disable pooling as we pool ourselves
     var builder = new SqliteConnectionStringBuilder { Pooling = false, DataSource = path };
@@ -47,12 +66,6 @@ public sealed class SqLiteJsonCacheManager : ISqLiteJsonCacheManager
         command.ExecuteNonQuery();
       }
 
-      // Insert Optimisations
-
-      //Note / Hack: This setting has the potential to corrupt the db.
-      //cmd = new SqliteCommand("PRAGMA synchronous=OFF;", Connection);
-      //cmd.ExecuteNonQuery();
-
       using (SqliteCommand cmd1 = new("PRAGMA count_changes=OFF;", c))
       {
         cmd1.ExecuteNonQuery();

src/Speckle.Sdk/SQLite/SqLiteJsonCacheManagerFactory.cs

@@ -9,7 +9,8 @@ public class SqLiteJsonCacheManagerFactory : ISqLiteJsonCacheManagerFactory
 {
   public const int INITIAL_CONCURRENCY = 4;
 
-  private ISqLiteJsonCacheManager Create(string path, int concurrency) => new SqLiteJsonCacheManager(path, concurrency);
+  private ISqLiteJsonCacheManager Create(string path, int concurrency) =>
+    SqLiteJsonCacheManager.FromFilePath(path, concurrency);
 
   public ISqLiteJsonCacheManager CreateForUser(string scope) =>
     Create(Path.Combine(SpecklePathProvider.UserApplicationDataPath(), "Speckle", $"{scope}.db"), 1);

src/Speckle.Sdk/Serialisation/V2/Receive/ObjectLoader.cs

@@ -171,7 +171,7 @@ public sealed class ObjectLoader(
     cancellationToken.ThrowIfCancellationRequested();
     if (Exception is not null)
     {
-      throw new SpeckleException("Error while loading", Exception);
+      throw new SpeckleException($"Error while loading: {Exception.Message}", Exception);
     }
   }
 }

src/Speckle.Sdk/Serialisation/V2/Send/ObjectSaver.cs

@@ -38,10 +38,11 @@ public sealed class ObjectSaver(
   private long _cached;
 
   private long _objectsSerialized;
+  private bool _disposed;
 
   protected override async Task SendToServerInternal(Batch<BaseItem> batch)
   {
-    if (_cancellationTokenSource.IsCancellationRequested)
+    if (IsCancelled())
     {
       return;
     }
@@ -66,7 +67,7 @@ public sealed class ObjectSaver(
     }
     catch (OperationCanceledException)
     {
-      _cancellationTokenSource.Cancel();
+      CancelSaving();
     }
 #pragma warning disable CA1031
     catch (Exception e)
@@ -91,7 +92,7 @@ public sealed class ObjectSaver(
 
   public override void SaveToCache(List<BaseItem> batch)
   {
-    if (_cancellationTokenSource.IsCancellationRequested)
+    if (IsCancelled())
     {
       return;
     }
@@ -106,7 +107,7 @@ public sealed class ObjectSaver(
     }
     catch (OperationCanceledException)
     {
-      _cancellationTokenSource.Cancel();
+      CancelSaving();
     }
 #pragma warning disable CA1031
     catch (Exception e)
@@ -123,8 +124,23 @@ public sealed class ObjectSaver(
     }
   }
 
+  private bool IsCancelled() => _disposed || _cancellationTokenSource.IsCancellationRequested;
+
+  private void CancelSaving()
+  {
+    if (IsCancelled())
+    {
+      return;
+    }
+    _cancellationTokenSource.Cancel();
+  }
+
   private void RecordException(Exception e)
   {
+    if (IsCancelled())
+    {
+      return;
+    }
     //order here matters
     logger.LogError(e, "Error in SDK: {message}", e.Message);
     Exception = e;
@@ -133,6 +149,7 @@ public sealed class ObjectSaver(
 
   public void Dispose()
   {
+    _disposed = true;
     _cancellationTokenSource.Dispose();
     sqLiteJsonCacheManager.Dispose();
   }

src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs

@@ -16,8 +16,8 @@ public record SerializeProcessOptions(
   bool SkipFindTotalObjects = false
 )
 {
-  public int? MaxHttpSendSize { get; set; }
-  public int? MaxCacheSize { get; set; }
+  public int? MaxHttpSendBatchSize { get; set; }
+  public int? MaxCacheBatchSize { get; set; }
   public int? MaxParallelism { get; set; }
 }
 
@@ -47,6 +47,7 @@ public sealed class SerializeProcess(
     cancellationToken
   );
   private readonly ILogger<SerializeProcess> _logger = loggerFactory.CreateLogger<SerializeProcess>();
+  private bool _disposed;
 
   //async dispose
   [SuppressMessage("Usage", "CA2213:Disposable fields should be disposed")]
@@ -83,6 +84,7 @@ public sealed class SerializeProcess(
   [AutoInterfaceIgnore]
   public async ValueTask DisposeAsync()
   {
+    _disposed = true;
     await WaitForSchedulerCompletion().ConfigureAwait(false);
     await _highest.DisposeAsync().ConfigureAwait(false);
     await _belowNormal.DisposeAsync().ConfigureAwait(false);
@@ -95,7 +97,7 @@ public sealed class SerializeProcess(
     //order here matters...null with cancellation means a user did it, otherwise it's a real Exception
     if (objectSaver.Exception is not null)
     {
-      throw new SpeckleException("Error while sending", objectSaver.Exception);
+      throw new SpeckleException($"Error while sending: {objectSaver.Exception.Message}", objectSaver.Exception);
     }
     _processSource.Token.ThrowIfCancellationRequested();
   }
@@ -112,8 +114,8 @@ public sealed class SerializeProcess(
     {
       var channelTask = objectSaver.Start(
         options?.MaxParallelism,
-        options?.MaxHttpSendSize,
-        options?.MaxCacheSize,
+        options?.MaxHttpSendBatchSize,
+        options?.MaxCacheBatchSize,
         _processSource.Token
       );
       var findTotalObjectsTask = Task.CompletedTask;
@@ -148,7 +150,7 @@ public sealed class SerializeProcess(
 
   private void TraverseTotal(Base obj)
   {
-    if (_processSource.Token.IsCancellationRequested)
+    if (IsCancelled())
     {
       return;
     }
@@ -162,7 +164,7 @@ public sealed class SerializeProcess(
 
   private async Task<Dictionary<Id, NodeInfo>> Traverse(Base obj)
   {
-    if (_processSource.Token.IsCancellationRequested)
+    if (IsCancelled())
     {
       return EMPTY_CLOSURES;
     }
@@ -174,7 +176,7 @@ public sealed class SerializeProcess(
       {
         // tmp is necessary because of the way closures close over loop variables
         var tmp = child;
-        if (_processSource.Token.IsCancellationRequested)
+        if (IsCancelled())
         {
           return EMPTY_CLOSURES;
         }
@@ -191,7 +193,7 @@ public sealed class SerializeProcess(
         tasks.Add(t);
       }
 
-      if (_processSource.Token.IsCancellationRequested)
+      if (IsCancelled())
       {
         return EMPTY_CLOSURES;
       }
@@ -218,7 +220,7 @@ public sealed class SerializeProcess(
       }
       _taskResultPool.Return(tasks);
 
-      if (_processSource.Token.IsCancellationRequested)
+      if (IsCancelled())
       {
         return EMPTY_CLOSURES;
       }
@@ -226,22 +228,30 @@ public sealed class SerializeProcess(
       var childClosures = _childClosurePool.Get();
       foreach (var childClosure in taskClosures)
       {
+        if (IsCancelled())
+        {
+          return EMPTY_CLOSURES;
+        }
         foreach (var kvp in childClosure)
         {
           childClosures[kvp.Key] = kvp.Value;
+          if (IsCancelled())
+          {
+            return EMPTY_CLOSURES;
+          }
         }
 
         _currentClosurePool.Return(childClosure);
       }
 
-      if (_processSource.Token.IsCancellationRequested)
+      if (IsCancelled())
       {
         return EMPTY_CLOSURES;
       }
 
       var items = baseSerializer.Serialise(obj, childClosures, _options.SkipCacheRead, _processSource.Token);
 
-      if (_processSource.Token.IsCancellationRequested)
+      if (IsCancelled())
       {
         return EMPTY_CLOSURES;
       }
@@ -253,13 +263,13 @@ public sealed class SerializeProcess(
         progress?.Report(new(ProgressEvent.FromCacheOrSerialized, _objectCount, Math.Max(_objectCount, _objectsFound)));
         foreach (var item in items)
         {
+          if (IsCancelled())
+          {
+            return EMPTY_CLOSURES;
+          }
+
           if (item.NeedsStorage)
           {
-            if (_processSource.Token.IsCancellationRequested)
-            {
-              return EMPTY_CLOSURES;
-            }
-
             Interlocked.Increment(ref _objectsSerialized);
             await objectSaver.SaveAsync(item).ConfigureAwait(false);
           }
@@ -290,6 +300,8 @@ public sealed class SerializeProcess(
     }
   }
 
+  public bool IsCancelled() => _disposed || _processSource.IsCancellationRequested;
+
   public void RecordException(Exception e)
   {
     if (e is OperationCanceledException)
@@ -304,6 +316,11 @@ public sealed class SerializeProcess(
     {
       return;
     }
+    if (IsCancelled())
+    {
+      //if we are already cancelled, don't log or save the exceptions
+      return;
+    }
     //order here matters
     _logger.LogError(e, "Error in SDK: {message}", e.Message);
     objectSaver.Exception = e;

src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs

@@ -64,18 +64,10 @@ public class SerializeProcessFactory(
 #pragma warning disable CA2000
     var memoryJsonCacheManager = new MemoryJsonCacheManager(jsonCache);
 #pragma warning restore CA2000
-    return new SerializeProcess(
+    return CreateSerializeProcess(
+      memoryJsonCacheManager,
+      new MemoryServerObjectManager(objects),
       progress,
-      new ObjectSaver(
-        progress,
-        memoryJsonCacheManager,
-        new MemoryServerObjectManager(objects),
-        loggerFactory.CreateLogger<ObjectSaver>(),
-        cancellationToken
-      ),
-      baseChildFinder,
-      new BaseSerializer(memoryJsonCacheManager, objectSerializerFactory),
-      loggerFactory,
       cancellationToken,
       options
     );

tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs

@@ -123,7 +123,7 @@ public class DetachedTests
       objects,
       null,
       default,
-      new SerializeProcessOptions(false, false, true, true) { MaxParallelism = 1, MaxHttpSendSize = 1 }
+      new SerializeProcessOptions(false, false, true, true) { MaxParallelism = 1, MaxHttpSendBatchSize = 1 }
     );
     var results = await serializeProcess.Serialize(@base);
 
@@ -150,7 +150,7 @@ public class DetachedTests
       objects,
       null,
       default,
-      new SerializeProcessOptions(false, false, true, true) { MaxParallelism = 1, MaxHttpSendSize = 1 }
+      new SerializeProcessOptions(false, false, true, true) { MaxParallelism = 1, MaxHttpSendBatchSize = 1 }
     );
     var results = await serializeProcess.Serialize(@base);
 
@@ -172,7 +172,7 @@ public class DetachedTests
       objects,
       null,
       default,
-      new SerializeProcessOptions(false, false, true, true) { MaxParallelism = 1, MaxHttpSendSize = 1 }
+      new SerializeProcessOptions(false, false, true, true) { MaxParallelism = 1, MaxHttpSendBatchSize = 1 }
     );
     var results = await serializeProcess.Serialize(@base);
 

tests/Speckle.Sdk.Serialization.Tests/ExceptionTests.Test_Exceptions_Cache.verified.json

@@ -6,6 +6,6 @@
     "Message": "The method or operation is not implemented.",
     "Type": "NotImplementedException"
   },
-  "Message": "Error while sending",
+  "Message": "Error while sending: The method or operation is not implemented.",
   "Type": "SpeckleException"
 }

tests/Speckle.Sdk.Serialization.Tests/ExceptionTests.Test_Exceptions_Cache_ExceptionsAfter_10.verified.json

@@ -6,6 +6,6 @@
     "Message": "Count exceeded",
     "Type": "Exception"
   },
-  "Message": "Error while sending",
+  "Message": "Error while sending: Count exceeded",
   "Type": "SpeckleException"
 }

tests/Speckle.Sdk.Serialization.Tests/ExceptionTests.Test_Exceptions_Receive_Cache_fileName=False.verified.json

@@ -6,6 +6,6 @@
     "Message": "The method or operation is not implemented.",
     "Type": "NotImplementedException"
   },
-  "Message": "Error while loading",
+  "Message": "Error while loading: The method or operation is not implemented.",
   "Type": "SpeckleException"
 }

tests/Speckle.Sdk.Serialization.Tests/ExceptionTests.Test_Exceptions_Upload.verified.json

@@ -6,6 +6,6 @@
     "Message": "The method or operation is not implemented.",
     "Type": "NotImplementedException"
   },
-  "Message": "Error while sending",
+  "Message": "Error while sending: The method or operation is not implemented.",
   "Type": "SpeckleException"
 }

tests/Speckle.Sdk.Serialization.Tests/ExceptionTests.cs

@@ -95,8 +95,8 @@ public class ExceptionTests
       default,
       new SerializeProcessOptions(false, false, false, true)
       {
-        MaxHttpSendSize = 1,
-        MaxCacheSize = 1,
+        MaxHttpSendBatchSize = 1,
+        MaxCacheBatchSize = 1,
         MaxParallelism = 1,
       }
     );

tests/Speckle.Sdk.Serialization.Tests/SerializationTests.cs

@@ -14,6 +14,7 @@ using Speckle.Sdk.Serialisation.V2;
 using Speckle.Sdk.Serialisation.V2.Receive;
 using Speckle.Sdk.Serialisation.V2.Send;
 using Speckle.Sdk.Serialization.Tests.Framework;
+using Speckle.Sdk.SQLite;
 using Speckle.Sdk.Testing.Framework;
 
 namespace Speckle.Sdk.Serialization.Tests;
@@ -50,45 +51,45 @@ public class SerializationTests
     public void Dispose() { }
   }
 
-  [Theory]
-  [InlineData("RevitObject.json.gz")]
-  public async Task Basic_Namespace_Validation(string fileName)
-  {
-    var closures = TestFileManager.GetFileAsClosures(fileName);
-    var deserializer = new SpeckleObjectDeserializer
+  /*  [Theory]
+    [InlineData("RevitObject.json.gz")]
+    public async Task Basic_Namespace_Validation(string fileName)
     {
-      ReadTransport = new TestTransport(closures),
-      CancellationToken = default,
-    };
-
-    foreach (var (id, objJson) in closures)
-    {
-      var jObject = JObject.Parse(objJson);
-      var oldSpeckleType = jObject["speckle_type"].NotNull().Value<string>().NotNull();
-      var starts = oldSpeckleType.StartsWith("Speckle.Core.") || oldSpeckleType.StartsWith("Objects.");
-      starts.Should().BeTrue($"{oldSpeckleType} isn't expected");
-
-      var baseType = await deserializer.DeserializeAsync(objJson);
-      baseType.id.Should().Be(id);
-
-      var oldType = TypeLoader.GetAtomicType(oldSpeckleType);
-      if (oldType == typeof(Base))
+      var closures = TestFileManager.GetFileAsClosures(fileName);
+      var deserializer = new SpeckleObjectDeserializer
       {
-        oldSpeckleType.Should().NotContain("Base");
-      }
-      else
+        ReadTransport = new TestTransport(closures),
+        CancellationToken = default,
+      };
+  
+      foreach (var (id, objJson) in closures)
       {
-        starts = baseType.speckle_type.StartsWith("Speckle.Core.") || baseType.speckle_type.StartsWith("Objects.");
-        starts.Should().BeTrue($"{baseType.speckle_type} isn't expected");
-
-        var type = TypeLoader.GetAtomicType(baseType.speckle_type);
-        type.Should().NotBeNull();
-        var name = TypeLoader.GetTypeString(type) ?? throw new ArgumentNullException($"Could not find: {type}");
-        starts = name.StartsWith("Speckle.Core") || name.StartsWith("Objects");
-        starts.Should().BeTrue($"{name} isn't expected");
+        var jObject = JObject.Parse(objJson);
+        var oldSpeckleType = jObject["speckle_type"].NotNull().Value<string>().NotNull();
+        var starts = oldSpeckleType.StartsWith("Speckle.Core.") || oldSpeckleType.StartsWith("Objects.");
+        starts.Should().BeTrue($"{oldSpeckleType} isn't expected");
+  
+        var baseType = await deserializer.DeserializeAsync(objJson);
+        baseType.id.Should().Be(id);
+  
+        var oldType = TypeLoader.GetAtomicType(oldSpeckleType);
+        if (oldType == typeof(Base))
+        {
+          oldSpeckleType.Should().NotContain("Base");
+        }
+        else
+        {
+          starts = baseType.speckle_type.StartsWith("Speckle.Core.") || baseType.speckle_type.StartsWith("Objects.");
+          starts.Should().BeTrue($"{baseType.speckle_type} isn't expected");
+  
+          var type = TypeLoader.GetAtomicType(baseType.speckle_type);
+          type.Should().NotBeNull();
+          var name = TypeLoader.GetTypeString(type) ?? throw new ArgumentNullException($"Could not find: {type}");
+          starts = name.StartsWith("Speckle.Core") || name.StartsWith("Objects");
+          starts.Should().BeTrue($"{name} isn't expected");
+        }
       }
-    }
-  }
+    }*/
 
   [Theory]
   [InlineData("RevitObject.json.gz")]
@@ -184,9 +185,16 @@ public class SerializationTests
   }
 
   [Theory]
-  [InlineData("RevitObject.json.gz", "3416d3fe01c9196115514c4a2f41617b", 7818, 4674)]
-  public async Task Roundtrip_Test_New(string fileName, string rootId, int oldCount, int newCount)
+  [InlineData(1)]
+  [InlineData(2)]
+  [InlineData(3)]
+  [InlineData(4)]
+  public async Task Roundtrip_Test_New(int concurrency)
   {
+    string fileName = "RevitObject.json.gz";
+    string rootId = "3416d3fe01c9196115514c4a2f41617b";
+    int oldCount = 7818;
+    int newCount = 4674;
     var closures = TestFileManager.GetFileAsClosures(fileName);
     closures.Count.Should().Be(oldCount);
 
@@ -218,11 +226,11 @@ public class SerializationTests
 
     await using (
       var serializeProcess = _factory.CreateSerializeProcess(
-        new ConcurrentDictionary<Id, Json>(),
-        newIdToJson,
+        SqLiteJsonCacheManager.FromMemory(1),
+        new MemoryServerObjectManager(newIdToJson),
         null,
         default,
-        new SerializeProcessOptions(true, true, false, true)
+        new SerializeProcessOptions(false, false, false, true) { MaxCacheBatchSize = 1, MaxParallelism = concurrency }
       )
     )
     {

tests/Speckle.Sdk.Tests.Unit/Api/GraphQLErrorHandler.cs

@@ -23,6 +23,7 @@ public class GraphQLErrorHandlerTests
     ];
     yield return [typeof(SpeckleGraphQLException), new Map { { "foo", "bar" } }];
     yield return [typeof(SpeckleGraphQLException), new Map { { "code", "CUSTOM_THING" } }];
+    yield return [typeof(CannotCreateCommitException), new Map { { "code", "COMMIT_CREATE_ERROR" } }];
   }
 
   [Theory]

tests/Speckle.Sdk.Tests.Unit/Helpers/Path.cs

@@ -18,7 +18,7 @@ public class SpecklePathTests
     }
     else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
     {
-      pattern = @"\/Users\/.*\/\.config";
+      pattern = @"\/Users\/.*\/Library\/Application Support";
     }
     else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
     {
@@ -57,7 +57,7 @@ public class SpecklePathTests
     }
     else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
     {
-      pattern = @"\/Users\/.*\/\.config";
+      pattern = @"\/Users\/.*\/Library\/Application Support";
     }
     else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
     {

tests/Speckle.Sdk.Tests.Unit/Models/BaseTests.cs

@@ -213,7 +213,9 @@ public class BaseTests
   [Fact]
   public void CanShallowCopy()
   {
-    var sample = new SampleObject();
+    var sample = new SampleObject { id = "sampleId" };
+    dynamic x = sample;
+    x.test = "test";
     var copy = sample.ShallowCopy();
 
     var selectedMembers = DynamicBaseMemberType.Dynamic | DynamicBaseMemberType.Instance;

tests/Speckle.Sdk.Tests.Unit/Models/DynamicBaseTests.cs

@@ -0,0 +1,141 @@
+using FluentAssertions;
+using Microsoft.CSharp.RuntimeBinder;
+using Speckle.Sdk.Host;
+using Speckle.Sdk.Models;
+
+namespace Speckle.Sdk.Tests.Unit.Models;
+
+public class DynamicBaseTests
+{
+  public DynamicBaseTests()
+  {
+    TypeLoader.Reset();
+    TypeLoader.Initialize(typeof(Base).Assembly, typeof(BaseTests).Assembly);
+  }
+
+  [Fact]
+  public void Indexer_SetAndGet()
+  {
+    // Arrange
+    var dynamicBase = new DynamicBase();
+    var key = "testProperty";
+    var value = "testValue";
+
+    // Act
+    dynamicBase[key] = value;
+    var result = dynamicBase[key];
+
+    // Assert
+    result.Should().Be(value);
+  }
+
+  [Fact]
+  public void DynamicProperty_SetAndGet()
+  {
+    // Arrange
+    dynamic dynamicBase = new DynamicBase();
+    var value = "dynamicValue";
+
+    // Act
+    dynamicBase.dynamicProperty = value;
+    object result = dynamicBase.dynamicProperty;
+
+    // Assert
+    result.Should().Be(value);
+  }
+
+  [Fact]
+  public void GetMembers_Default()
+  {
+    // Arrange
+    dynamic dynamicBase = new DynamicBase();
+    dynamicBase.dynamicProp = "hello";
+
+    // Act
+    IDictionary<string, object?> members = dynamicBase.GetMembers();
+
+    // Assert
+    members.Should().ContainKey("dynamicProp");
+  }
+
+  [Fact]
+  public void GetMembers_Instance()
+  {
+    // Arrange
+    var dynamicBase = new TestDynamicBase();
+
+    // Act
+    var members = dynamicBase.GetMembers(DynamicBaseMemberType.Instance);
+
+    // Assert
+    members.Should().ContainKey(nameof(TestDynamicBase.InstanceProperty));
+    members.Should().NotContainKey("dynamicProp");
+  }
+
+  [Fact]
+  public void GetDynamicMemberNames()
+  {
+    // Arrange
+    dynamic dynamicBase = new DynamicBase();
+    dynamicBase.prop1 = 1;
+    dynamicBase.prop2 = "test";
+
+    // Act
+    IEnumerable<string> memberNames = dynamicBase.GetDynamicMemberNames();
+
+    // Assert
+    memberNames.Should().BeEquivalentTo(["DynamicPropertyKeys", "prop1", "prop2"]);
+  }
+
+  [Fact]
+  public void TryGetMember_Existing()
+  {
+    // Arrange
+    dynamic dynamicBase = new DynamicBase();
+    dynamicBase.existingProp = "I exist";
+
+    // Act
+    var result = dynamicBase.existingProp;
+
+    // Assert
+    ((object)result)
+      .Should()
+      .Be("I exist");
+  }
+
+  [Fact]
+  public void TryGetMember_NonExisting()
+  {
+    // Arrange
+    dynamic dynamicBase = new DynamicBase();
+
+    // Act
+    Action act = () =>
+    {
+      var result = dynamicBase.nonExistingProp;
+    };
+
+    // Assert
+    act.Should().Throw<RuntimeBinderException>();
+  }
+
+  [Fact]
+  public void TrySetMember()
+  {
+    // Arrange
+    dynamic dynamicBase = new DynamicBase();
+
+    // Act
+    dynamicBase.newProp = "newValue";
+
+    // Assert
+    ((object)dynamicBase.newProp)
+      .Should()
+      .Be("newValue");
+  }
+
+  private class TestDynamicBase : DynamicBase
+  {
+    public string InstanceProperty { get; set; } = "instance";
+  }
+}

tests/Speckle.Sdk.Tests.Unit/SQLite/SQLiteJsonCacheManagerTests.cs

@@ -22,7 +22,7 @@ public class SQLiteJsonCacheManagerTests : IDisposable
   public void TestGetAll()
   {
     var data = new List<(string id, string json)>() { ("id1", "1"), ("id2", "2") };
-    using var manager = new SqLiteJsonCacheManager(_basePath, 2);
+    using var manager = SqLiteJsonCacheManager.FromFilePath(_basePath, 2);
     manager.SaveObjects(data);
     var items = manager.GetAllObjects();
     items.Count.Should().Be(data.Count);
@@ -38,7 +38,7 @@ public class SQLiteJsonCacheManagerTests : IDisposable
   public void TestGet()
   {
     var data = new List<(string id, string json)>() { ("id1", "1"), ("id2", "2") };
-    using var manager = new SqLiteJsonCacheManager(_basePath, 2);
+    using var manager = SqLiteJsonCacheManager.FromFilePath(_basePath, 2);
     foreach (var d in data)
     {
       manager.SaveObject(d.id, d.json);
@@ -84,7 +84,7 @@ public class SQLiteJsonCacheManagerTests : IDisposable
   public void TestLargeJsonPayload()
   {
     var largeJson = new string('a', 100_000);
-    using var manager = new SqLiteJsonCacheManager(_basePath, 2);
+    using var manager = SqLiteJsonCacheManager.FromFilePath(_basePath, 2);
     manager.SaveObject("large", largeJson);
     var result = manager.GetObject("large");
     result.Should().Be(largeJson);
@@ -96,7 +96,7 @@ public class SQLiteJsonCacheManagerTests : IDisposable
     var id = "spécial_字符_!@#$%^&*()";
     var json = /*lang=json,strict*/
       "{\"value\": \"特殊字符!@#$%^&*()\"}";
-    using var manager = new SqLiteJsonCacheManager(_basePath, 2);
+    using var manager = SqLiteJsonCacheManager.FromFilePath(_basePath, 2);
     manager.SaveObject(id, json);
     var result = manager.GetObject(id);
     result.Should().Be(json);
@@ -108,7 +108,7 @@ public class SQLiteJsonCacheManagerTests : IDisposable
   [Fact]
   public void TestBulkInsertEmptyCollection()
   {
-    using var manager = new SqLiteJsonCacheManager(_basePath, 2);
+    using var manager = SqLiteJsonCacheManager.FromFilePath(_basePath, 2);
     manager.SaveObjects(new List<(string, string)>());
     manager.GetAllObjects().Count.Should().Be(0);
   }
@@ -116,7 +116,7 @@ public class SQLiteJsonCacheManagerTests : IDisposable
   [Fact]
   public void TestRepeatedUpdateAndDelete()
   {
-    using var manager = new SqLiteJsonCacheManager(_basePath, 2);
+    using var manager = SqLiteJsonCacheManager.FromFilePath(_basePath, 2);
     manager.SaveObject("id", "1");
     manager.UpdateObject("id", "2");
     manager.UpdateObject("id", "3");
@@ -129,7 +129,7 @@ public class SQLiteJsonCacheManagerTests : IDisposable
   [Fact]
   public void TestGetAndDeleteNonExistentId()
   {
-    using var manager = new SqLiteJsonCacheManager(_basePath, 2);
+    using var manager = SqLiteJsonCacheManager.FromFilePath(_basePath, 2);
     manager.GetObject("doesnotexist").Should().BeNull();
     manager.HasObject("doesnotexist").Should().BeFalse();
     manager.DeleteObject("doesnotexist"); // Should not throw
@@ -138,7 +138,7 @@ public class SQLiteJsonCacheManagerTests : IDisposable
   [Fact]
   public void TestNullOrEmptyInput()
   {
-    using var manager = new SqLiteJsonCacheManager(_basePath, 2);
+    using var manager = SqLiteJsonCacheManager.FromFilePath(_basePath, 2);
     // Empty id
     Assert.Throws<ArgumentException>(() => manager.SaveObject("", "emptyid"));
     // Empty json