Merge pull request #348 from specklesystems/dev

New release for connectors

src/Speckle.Objects/Other/LevelProxy.cs

@@ -0,0 +1,20 @@
+using Speckle.Objects.Data;
+using Speckle.Sdk.Models;
+using Speckle.Sdk.Models.Proxies;
+
+namespace Speckle.Objects.Other;
+
+/// <summary>
+/// Proxy for levels as DataObject value.
+/// <remarks> These proxy lives in Objects library because it depends on DataObject</remarks>
+/// </summary>
+[SpeckleType("Objects.Other.LevelProxy")]
+public class LevelProxy : Base, IProxyCollection
+{
+  /// <summary>
+  /// The list of application ids of objects that use this level
+  /// </summary>
+  public required List<string> objects { get; set; }
+
+  public required DataObject value { get; set; }
+}

src/Speckle.Objects/Other/RenderMaterial.cs

@@ -1,7 +1,6 @@
 using System.Drawing;
 using Speckle.Newtonsoft.Json;
 using Speckle.Sdk.Models;
-using Speckle.Sdk.Models.Proxies;
 
 namespace Speckle.Objects.Other;
 
@@ -39,20 +38,3 @@ public class RenderMaterial : Base
     set => diffuse = value.ToArgb();
   }
 }
-
-/// <summary>
-/// Used to store render material to object relationships in root collections
-/// </summary>
-[SpeckleType("Objects.Other.RenderMaterialProxy")]
-public class RenderMaterialProxy : Base, IProxyCollection
-{
-  /// <summary>
-  /// The list of application ids of objects that use this render material
-  /// </summary>
-  public required List<string> objects { get; set; }
-
-  /// <summary>
-  /// The render material used by <see cref="objects"/>
-  /// </summary>
-  public required RenderMaterial value { get; set; }
-}

src/Speckle.Objects/Other/RenderMaterialProxy.cs

@@ -0,0 +1,22 @@
+using Speckle.Sdk.Models;
+using Speckle.Sdk.Models.Proxies;
+
+namespace Speckle.Objects.Other;
+
+/// <summary>
+/// Used to store render material to object relationships in root collections
+/// <remarks> These proxy lives in Objects library because it depends on RenderMaterial</remarks>
+/// </summary>
+[SpeckleType("Objects.Other.RenderMaterialProxy")]
+public class RenderMaterialProxy : Base, IProxyCollection
+{
+  /// <summary>
+  /// The list of application ids of objects that use this render material
+  /// </summary>
+  public required List<string> objects { get; set; }
+
+  /// <summary>
+  /// The render material used by <see cref="objects"/>
+  /// </summary>
+  public required RenderMaterial value { get; set; }
+}

src/Speckle.Sdk.Dependencies/Collections.cs

@@ -22,23 +22,4 @@ public static class Collections
 public static class EnumerableExtensions
 {
   public static IEnumerable<int> RangeFrom(int from, int to) => Enumerable.Range(from, to - from + 1);
-
-#if NETSTANDARD2_0
-  public static IEnumerable<TSource> DistinctBy<TSource, TKey>(
-    this IEnumerable<TSource> source,
-    Func<TSource, TKey> keySelector
-  )
-  {
-    var keys = new HashSet<TKey>();
-    foreach (var element in source)
-    {
-      if (keys.Contains(keySelector(element)))
-      {
-        continue;
-      }
-      keys.Add(keySelector(element));
-      yield return element;
-    }
-  }
-#endif
 }

src/Speckle.Sdk.Dependencies/Serialization/ChannelLoader.cs

@@ -5,7 +5,7 @@ namespace Speckle.Sdk.Dependencies.Serialization;
 
 public abstract class ChannelLoader<T>(CancellationToken cancellationToken)
 {
-  private const int RECEIVE_CAPACITY = 5000;
+  private const int RECEIVE_CAPACITY = 10000;
 
   private const int HTTP_GET_CHUNK_SIZE = 500;
   private const int MAX_PARALLELISM_HTTP = 4;

src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs

@@ -8,7 +8,7 @@ namespace Speckle.Sdk.Dependencies.Serialization;
 public abstract class ChannelSaver<T>
   where T : IHasByteSize
 {
-  private const int SEND_CAPACITY = 1000;
+  private const int SEND_CAPACITY = 10000;
   private const int HTTP_SEND_CHUNK_SIZE = 25_000_000; //bytes
   private static readonly TimeSpan HTTP_BATCH_TIMEOUT = TimeSpan.FromSeconds(2);
   private const int MAX_PARALLELISM_HTTP = 4;

src/Speckle.Sdk/Api/Exceptions.cs

@@ -126,3 +126,14 @@ public sealed class WorkspacePermissionException : SpeckleGraphQLException
   public WorkspacePermissionException(string? message, Exception? innerException)
     : base(message, innerException) { }
 }
+
+public sealed class CannotCreateCommitException : SpeckleGraphQLException
+{
+  public CannotCreateCommitException() { }
+
+  public CannotCreateCommitException(string? message)
+    : base(message) { }
+
+  public CannotCreateCommitException(string? message, Exception? innerException)
+    : base(message, innerException) { }
+}

src/Speckle.Sdk/Api/GraphQL/GraphQLErrorHandler.cs

@@ -33,6 +33,7 @@ internal static class GraphQLErrorHandler
         "BAD_USER_INPUT" => new SpeckleGraphQLBadInputException(message),
         "INTERNAL_SERVER_ERROR" => new SpeckleGraphQLInternalErrorException(message),
         "WORKSPACES_MODULE_DISABLED_ERROR" => new SpeckleGraphQLWorkspaceNotEnabledException(message),
+        "COMMIT_CREATE_ERROR" => new CannotCreateCommitException(message),
         _ => new SpeckleGraphQLException(message),
       };
       exceptions.Add(ex);

src/Speckle.Sdk/Api/Operations/Operations.Send.cs

@@ -47,6 +47,11 @@ public partial class Operations
       receiveActivity?.SetStatus(SdkActivityStatusCode.Ok);
       return results;
     }
+    catch (OperationCanceledException)
+    {
+      //this is handled by the caller
+      throw;
+    }
     catch (Exception ex)
     {
       receiveActivity?.SetStatus(SdkActivityStatusCode.Error);

src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs

@@ -13,18 +13,15 @@ public sealed class SqLiteJsonCacheManager : ISqLiteJsonCacheManager
 {
   private readonly CacheDbCommandPool _pool;
 
-  public string Path { get; }
-
   public static ISqLiteJsonCacheManager FromMemory(int concurrency) => new SqLiteJsonCacheManager(concurrency);
 
   private SqLiteJsonCacheManager(int concurrency)
   {
-    Path = ":memory:";
     //disable pooling as we pool ourselves
     var builder = new SqliteConnectionStringBuilder
     {
       Pooling = false,
-      DataSource = Path,
+      DataSource = ":memory:",
       Cache = SqliteCacheMode.Shared,
       Mode = SqliteOpenMode.Memory,
     };
@@ -37,9 +34,8 @@ public sealed class SqLiteJsonCacheManager : ISqLiteJsonCacheManager
 
   private SqLiteJsonCacheManager(string path, int concurrency)
   {
-    Path = path;
     //disable pooling as we pool ourselves
-    var builder = new SqliteConnectionStringBuilder { Pooling = false, DataSource = Path };
+    var builder = new SqliteConnectionStringBuilder { Pooling = false, DataSource = path };
     _pool = new CacheDbCommandPool(builder.ToString(), concurrency);
     Initialize();
   }

src/Speckle.Sdk/Serialisation/V2/MemoryJsonCacheManager.cs

@@ -7,10 +7,6 @@ namespace Speckle.Sdk.Serialisation.V2;
 public class MemoryJsonCacheManager(ConcurrentDictionary<Id, Json> jsonCache) : ISqLiteJsonCacheManager
 #pragma warning restore CA1063
 {
-#pragma warning disable CA1065
-  public string Path => "MemoryJsonCacheManager";
-#pragma warning restore CA1065
-
   public IReadOnlyCollection<(string Id, string Json)> GetAllObjects() =>
     jsonCache.Select(x => (x.Key.Value, x.Value.Value)).ToList();
 

src/Speckle.Sdk/Serialisation/V2/Send/ObjectSaverFactory.cs

@@ -1,50 +0,0 @@
-using System.Collections.Concurrent;
-using Microsoft.Extensions.Logging;
-using Speckle.InterfaceGenerator;
-using Speckle.Sdk.SQLite;
-using Speckle.Sdk.Transports;
-
-namespace Speckle.Sdk.Serialisation.V2.Send;
-
-public partial interface IObjectSaverFactory : IDisposable;
-
-[GenerateAutoInterface]
-public sealed class ObjectSaverFactory(ILoggerFactory loggerFactory) : IObjectSaverFactory
-{
-  private readonly ConcurrentDictionary<string, IObjectSaver> _savers = new();
-
-  public IObjectSaver Create(
-    IServerObjectManager serverObjectManager,
-    ISqLiteJsonCacheManager sqLiteJsonCacheManager,
-    IProgress<ProgressArgs>? progress,
-    CancellationToken cancellationToken,
-    SerializeProcessOptions? options = null
-  )
-  {
-    if (!_savers.TryGetValue(sqLiteJsonCacheManager.Path, out var saver))
-    {
-      saver = new ObjectSaver(
-        progress,
-        sqLiteJsonCacheManager,
-        serverObjectManager,
-        loggerFactory.CreateLogger<ObjectSaver>(),
-        cancellationToken,
-        options
-      );
-      _savers.TryAdd(sqLiteJsonCacheManager.Path, saver);
-    }
-
-    return saver;
-  }
-
-  [AutoInterfaceIgnore]
-  public void Dispose()
-  {
-    foreach (var pool in _savers)
-    {
-      pool.Value.Dispose();
-    }
-
-    _savers.Clear();
-  }
-}

src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs

@@ -86,6 +86,7 @@ public sealed class SerializeProcess(
     await WaitForSchedulerCompletion().ConfigureAwait(false);
     await _highest.DisposeAsync().ConfigureAwait(false);
     await _belowNormal.DisposeAsync().ConfigureAwait(false);
+    objectSaver.Dispose();
     _processSource.Dispose();
   }
 
@@ -225,9 +226,17 @@ public sealed class SerializeProcess(
       var childClosures = _childClosurePool.Get();
       foreach (var childClosure in taskClosures)
       {
+        if (_processSource.Token.IsCancellationRequested)
+        {
+          return EMPTY_CLOSURES;
+        }
         foreach (var kvp in childClosure)
         {
           childClosures[kvp.Key] = kvp.Value;
+          if (_processSource.Token.IsCancellationRequested)
+          {
+            return EMPTY_CLOSURES;
+          }
         }
 
         _currentClosurePool.Return(childClosure);
@@ -252,13 +261,13 @@ public sealed class SerializeProcess(
         progress?.Report(new(ProgressEvent.FromCacheOrSerialized, _objectCount, Math.Max(_objectCount, _objectsFound)));
         foreach (var item in items)
         {
+          if (_processSource.Token.IsCancellationRequested)
+          {
+            return EMPTY_CLOSURES;
+          }
+
           if (item.NeedsStorage)
           {
-            if (_processSource.Token.IsCancellationRequested)
-            {
-              return EMPTY_CLOSURES;
-            }
-
             Interlocked.Increment(ref _objectsSerialized);
             await objectSaver.SaveAsync(item).ConfigureAwait(false);
           }

src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs

@@ -13,7 +13,6 @@ public class SerializeProcessFactory(
   IObjectSerializerFactory objectSerializerFactory,
   ISqLiteJsonCacheManagerFactory sqLiteJsonCacheManagerFactory,
   IServerObjectManagerFactory serverObjectManagerFactory,
-  IObjectSaverFactory objectSaverFactory,
   ILoggerFactory loggerFactory
 ) : ISerializeProcessFactory
 {
@@ -40,7 +39,13 @@ public class SerializeProcessFactory(
   ) =>
     new SerializeProcess(
       progress,
-      objectSaverFactory.Create(serverObjectManager, sqLiteJsonCacheManager, progress, cancellationToken, options),
+      new ObjectSaver(
+        progress,
+        sqLiteJsonCacheManager,
+        serverObjectManager,
+        loggerFactory.CreateLogger<ObjectSaver>(),
+        cancellationToken
+      ),
       baseChildFinder,
       new BaseSerializer(sqLiteJsonCacheManager, objectSerializerFactory),
       loggerFactory,

src/Speckle.Sdk/ServiceRegistration.cs

@@ -97,8 +97,6 @@ public static class ServiceRegistration
       typeof(Client)
     );
     serviceCollection.AddMatchingInterfacesAsTransient(typeof(GraphQLRetry).Assembly);
-    //we want to make object savers be singletons per stream so needs a singleton factory
-    serviceCollection.AddSingleton<IObjectSaverFactory, ObjectSaverFactory>();
     return serviceCollection;
   }
 

tests/Speckle.Sdk.Serialization.Tests/DataObjectTests.cs

@@ -41,7 +41,7 @@ public class DataObjectTests
       new DummyServerObjectManager(),
       null,
       default,
-      new SerializeProcessOptions(false, false, false, true)
+      new SerializeProcessOptions(true, true, false, true)
     );
     await serializeProcess.Serialize(x);
     await VerifyJson(json.Single().Value.Value).UseParameters(type);

tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs

@@ -41,7 +41,7 @@ public class DetachedTests
       objects,
       null,
       default,
-      new SerializeProcessOptions(false, false, false, true)
+      new SerializeProcessOptions(false, false, true, true)
     );
     await serializeProcess.Serialize(@base);
 
@@ -123,7 +123,7 @@ public class DetachedTests
       objects,
       null,
       default,
-      new SerializeProcessOptions(false, false, false, true) { MaxParallelism = 1, MaxHttpSendBatchSize = 1 }
+      new SerializeProcessOptions(false, false, true, true) { MaxParallelism = 1, MaxHttpSendBatchSize = 1 }
     );
     var results = await serializeProcess.Serialize(@base);
 
@@ -150,7 +150,7 @@ public class DetachedTests
       objects,
       null,
       default,
-      new SerializeProcessOptions(false, false, false, true) { MaxParallelism = 1, MaxHttpSendBatchSize = 1 }
+      new SerializeProcessOptions(false, false, true, true) { MaxParallelism = 1, MaxHttpSendBatchSize = 1 }
     );
     var results = await serializeProcess.Serialize(@base);
 
@@ -172,7 +172,7 @@ public class DetachedTests
       objects,
       null,
       default,
-      new SerializeProcessOptions(false, false, false, true) { MaxParallelism = 1, MaxHttpSendBatchSize = 1 }
+      new SerializeProcessOptions(false, false, true, true) { MaxParallelism = 1, MaxHttpSendBatchSize = 1 }
     );
     var results = await serializeProcess.Serialize(@base);
 
@@ -239,7 +239,7 @@ public class DetachedTests
       objects,
       null,
       default,
-      new SerializeProcessOptions(false, false, false, true)
+      new SerializeProcessOptions(false, false, true, true)
     );
 
     var results = await serializeProcess.Serialize(@base);
@@ -272,7 +272,7 @@ public class DetachedTests
       objects,
       null,
       default,
-      new SerializeProcessOptions(false, false, false, true)
+      new SerializeProcessOptions(false, false, true, true)
     );
     var results = await serializeProcess.Serialize(@base);
     await VerifyJsonDictionary(objects);
@@ -373,9 +373,6 @@ public class DummyServerObjectManager : IServerObjectManager
 
 public class DummySendCacheManager(Dictionary<string, string> objects) : ISqLiteJsonCacheManager
 {
-#pragma warning disable CA1065
-  public string Path => throw new NotImplementedException();
-#pragma warning restore CA1065
   public void Dispose() { }
 
   public IReadOnlyCollection<(string, string)> GetAllObjects() => throw new NotImplementedException();

tests/Speckle.Sdk.Serialization.Tests/DummyCancellationSqLiteSendManager.cs

@@ -4,9 +4,6 @@ namespace Speckle.Sdk.Serialization.Tests;
 
 public class DummyCancellationSqLiteSendManager : ISqLiteJsonCacheManager
 {
-#pragma warning disable CA1065
-  public string Path => throw new NotImplementedException();
-#pragma warning restore CA1065
   public string? GetObject(string id) => null;
 
   public void SaveObject(string id, string json) => throw new NotImplementedException();

tests/Speckle.Sdk.Serialization.Tests/Framework/ExceptionSendCacheManager.cs

@@ -4,7 +4,6 @@ namespace Speckle.Sdk.Serialization.Tests.Framework;
 
 public class ExceptionSendCacheManager(bool? hasObject = null, int? exceptionsAfter = null) : ISqLiteJsonCacheManager
 {
-  public string Path => "ExceptionSendCacheManager";
   private readonly object _lock = new();
   private int _count;
 

tests/Speckle.Sdk.Serialization.Tests/ObjectSaverFactoryTests.cs

@@ -1,90 +0,0 @@
-using FluentAssertions;
-using Microsoft.Extensions.Logging;
-using Moq;
-using Speckle.Sdk.SQLite;
-using Speckle.Sdk.Testing;
-
-namespace Speckle.Sdk.Serialisation.V2.Send.Tests;
-
-public class ObjectSaverFactoryTests : MoqTest
-{
-  private readonly Mock<ILoggerFactory> _loggerFactoryMock;
-  private readonly Mock<ILogger<ObjectSaver>> _loggerMock;
-  private readonly ObjectSaverFactory _factory;
-
-  public ObjectSaverFactoryTests()
-  {
-    _loggerFactoryMock = Create<ILoggerFactory>();
-    _loggerMock = Create<ILogger<ObjectSaver>>();
-    _factory = new ObjectSaverFactory(_loggerFactoryMock.Object);
-  }
-
-  public override void Dispose()
-  {
-    _factory.Dispose();
-    base.Dispose();
-  }
-
-  [Fact]
-  public void Create_ShouldReturnObjectSaverInstance()
-  {
-    _loggerFactoryMock.Setup(f => f.CreateLogger(typeof(ObjectSaver).FullName)).Returns(_loggerMock.Object);
-    var cacheManagerMock = Create<ISqLiteJsonCacheManager>();
-    cacheManagerMock.Setup(x => x.Dispose());
-    cacheManagerMock.SetupGet(c => c.Path).Returns("/tmp/test1.db");
-
-    var saver = _factory.Create(
-      Create<IServerObjectManager>().Object,
-      cacheManagerMock.Object,
-      null,
-      CancellationToken.None
-    );
-
-    saver.Should().NotBeNull();
-  }
-
-  [Fact]
-  public void Create_ShouldReturnSameInstanceForSamePath()
-  {
-    _loggerFactoryMock.Setup(f => f.CreateLogger(typeof(ObjectSaver).FullName)).Returns(_loggerMock.Object);
-    var cacheManagerMock = Create<ISqLiteJsonCacheManager>();
-    cacheManagerMock.Setup(x => x.Dispose());
-    cacheManagerMock.SetupGet(c => c.Path).Returns("/tmp/test2.db");
-
-    var saver1 = _factory.Create(
-      Create<IServerObjectManager>().Object,
-      cacheManagerMock.Object,
-      null,
-      CancellationToken.None
-    );
-    var saver2 = _factory.Create(
-      Create<IServerObjectManager>().Object,
-      cacheManagerMock.Object,
-      null,
-      CancellationToken.None
-    );
-
-    saver1.Should().BeSameAs(saver2);
-  }
-
-  [Fact]
-  public void Dispose_ShouldDisposeAllSavers()
-  {
-    var saverMock1 = Create<IObjectSaver>();
-    _factory
-      .GetType()
-      .GetField("_savers", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)
-      ?.SetValue(
-        _factory,
-        new System.Collections.Concurrent.ConcurrentDictionary<string, IObjectSaver>(
-          new[]
-          {
-            new System.Collections.Generic.KeyValuePair<string, IObjectSaver>("/tmp/test3.db", saverMock1.Object),
-          }
-        )
-      );
-    saverMock1.Setup(x => x.Dispose());
-
-    _factory.Dispose();
-  }
-}

tests/Speckle.Sdk.Testing/Framework/DummySqLiteReceiveManager.cs

@@ -5,9 +5,6 @@ namespace Speckle.Sdk.Testing.Framework;
 public sealed class DummySqLiteReceiveManager(IReadOnlyDictionary<string, string> savedObjects)
   : ISqLiteJsonCacheManager
 {
-#pragma warning disable CA1065
-  public string Path => throw new NotImplementedException();
-#pragma warning restore CA1065
   public void Dispose() { }
 
   public IReadOnlyCollection<(string, string)> GetAllObjects() => throw new NotImplementedException();

tests/Speckle.Sdk.Testing/Framework/DummySqLiteSendManager.cs

@@ -4,8 +4,6 @@ namespace Speckle.Sdk.Testing.Framework;
 
 public class DummySqLiteSendManager : ISqLiteJsonCacheManager
 {
-  public string Path => "DummySqLiteSendManager";
-
   public string? GetObject(string id) => throw new NotImplementedException();
 
   public void SaveObject(string id, string json) => throw new NotImplementedException();

tests/Speckle.Sdk.Testing/MoqTest.cs

@@ -8,7 +8,7 @@ public abstract class MoqTest : IDisposable
 {
   protected MoqTest() => Repository = new(MockBehavior.Strict);
 
-  public virtual void Dispose() => Repository.VerifyAll();
+  public void Dispose() => Repository.VerifyAll();
 
   protected MockRepository Repository { get; private set; } = new(MockBehavior.Strict);
 

tests/Speckle.Sdk.Tests.Unit/Api/GraphQLErrorHandler.cs

@@ -23,6 +23,7 @@ public class GraphQLErrorHandlerTests
     ];
     yield return [typeof(SpeckleGraphQLException), new Map { { "foo", "bar" } }];
     yield return [typeof(SpeckleGraphQLException), new Map { { "code", "CUSTOM_THING" } }];
+    yield return [typeof(CannotCreateCommitException), new Map { { "code", "COMMIT_CREATE_ERROR" } }];
   }
 
   [Theory]

tests/Speckle.Sdk.Tests.Unit/Serialisation/SerializeProcessRecordExceptionTests.cs

@@ -21,6 +21,7 @@ public class SerializeProcessRecordExceptionTests : MoqTest
       .Setup(f => f.CreateLogger("Speckle.Sdk.Serialisation.V2.PriorityScheduler"))
       .Returns(Create<ILogger<PriorityScheduler>>().Object);
     var objectSaverMock = Create<IObjectSaver>();
+    objectSaverMock.Setup(x => x.Dispose());
     var baseChildFinderMock = Create<IBaseChildFinder>();
     var baseSerializerMock = Create<IBaseSerializer>();
     using var cts = new CancellationTokenSource();
@@ -56,6 +57,7 @@ public class SerializeProcessRecordExceptionTests : MoqTest
       .Setup(f => f.CreateLogger("Speckle.Sdk.Serialisation.V2.PriorityScheduler"))
       .Returns(Create<ILogger<PriorityScheduler>>().Object);
     var objectSaverMock = Create<IObjectSaver>();
+    objectSaverMock.Setup(x => x.Dispose());
     var baseChildFinderMock = Create<IBaseChildFinder>();
     var baseSerializerMock = Create<IBaseSerializer>();
     using var cts = new CancellationTokenSource();
@@ -86,6 +88,7 @@ public class SerializeProcessRecordExceptionTests : MoqTest
       .Setup(f => f.CreateLogger("Speckle.Sdk.Serialisation.V2.PriorityScheduler"))
       .Returns(Create<ILogger<PriorityScheduler>>().Object);
     var objectSaverMock = Create<IObjectSaver>();
+    objectSaverMock.Setup(x => x.Dispose());
     var baseChildFinderMock = Create<IBaseChildFinder>();
     var baseSerializerMock = Create<IBaseSerializer>();
     using var cts = new CancellationTokenSource();