Merge pull request #335 from specklesystems/adam/cnx-1786-allow-multiple-sends-to-access-sqlite-in-a-non-locking-2

Use an object saver per stream instead of sqlite manager per stream

src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs

@@ -13,15 +13,18 @@ public sealed class SqLiteJsonCacheManager : ISqLiteJsonCacheManager
 {
   private readonly CacheDbCommandPool _pool;
 
+  public string Path { get; }
+
   public static ISqLiteJsonCacheManager FromMemory(int concurrency) => new SqLiteJsonCacheManager(concurrency);
 
   private SqLiteJsonCacheManager(int concurrency)
   {
+    Path = ":memory:";
     //disable pooling as we pool ourselves
     var builder = new SqliteConnectionStringBuilder
     {
       Pooling = false,
-      DataSource = ":memory:",
+      DataSource = Path,
       Cache = SqliteCacheMode.Shared,
       Mode = SqliteOpenMode.Memory,
     };
@@ -34,8 +37,9 @@ public sealed class SqLiteJsonCacheManager : ISqLiteJsonCacheManager
 
   private SqLiteJsonCacheManager(string path, int concurrency)
   {
+    Path = path;
     //disable pooling as we pool ourselves
-    var builder = new SqliteConnectionStringBuilder { Pooling = false, DataSource = path };
+    var builder = new SqliteConnectionStringBuilder { Pooling = false, DataSource = Path };
     _pool = new CacheDbCommandPool(builder.ToString(), concurrency);
     Initialize();
   }

src/Speckle.Sdk/Serialisation/V2/MemoryJsonCacheManager.cs

@@ -7,6 +7,10 @@ namespace Speckle.Sdk.Serialisation.V2;
 public class MemoryJsonCacheManager(ConcurrentDictionary<Id, Json> jsonCache) : ISqLiteJsonCacheManager
 #pragma warning restore CA1063
 {
+#pragma warning disable CA1065
+  public string Path => "MemoryJsonCacheManager";
+#pragma warning restore CA1065
+
   public IReadOnlyCollection<(string Id, string Json)> GetAllObjects() =>
     jsonCache.Select(x => (x.Key.Value, x.Value.Value)).ToList();
 

src/Speckle.Sdk/Serialisation/V2/Send/ObjectSaverFactory.cs

@@ -0,0 +1,50 @@
+using System.Collections.Concurrent;
+using Microsoft.Extensions.Logging;
+using Speckle.InterfaceGenerator;
+using Speckle.Sdk.SQLite;
+using Speckle.Sdk.Transports;
+
+namespace Speckle.Sdk.Serialisation.V2.Send;
+
+public partial interface IObjectSaverFactory : IDisposable;
+
+[GenerateAutoInterface]
+public sealed class ObjectSaverFactory(ILoggerFactory loggerFactory) : IObjectSaverFactory
+{
+  private readonly ConcurrentDictionary<string, IObjectSaver> _savers = new();
+
+  public IObjectSaver Create(
+    IServerObjectManager serverObjectManager,
+    ISqLiteJsonCacheManager sqLiteJsonCacheManager,
+    IProgress<ProgressArgs>? progress,
+    CancellationToken cancellationToken,
+    SerializeProcessOptions? options = null
+  )
+  {
+    if (!_savers.TryGetValue(sqLiteJsonCacheManager.Path, out var saver))
+    {
+      saver = new ObjectSaver(
+        progress,
+        sqLiteJsonCacheManager,
+        serverObjectManager,
+        loggerFactory.CreateLogger<ObjectSaver>(),
+        cancellationToken,
+        options
+      );
+      _savers.TryAdd(sqLiteJsonCacheManager.Path, saver);
+    }
+
+    return saver;
+  }
+
+  [AutoInterfaceIgnore]
+  public void Dispose()
+  {
+    foreach (var pool in _savers)
+    {
+      pool.Value.Dispose();
+    }
+
+    _savers.Clear();
+  }
+}

src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs

@@ -86,7 +86,6 @@ public sealed class SerializeProcess(
     await WaitForSchedulerCompletion().ConfigureAwait(false);
     await _highest.DisposeAsync().ConfigureAwait(false);
     await _belowNormal.DisposeAsync().ConfigureAwait(false);
-    objectSaver.Dispose();
     _processSource.Dispose();
   }
 

src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs

@@ -13,6 +13,7 @@ public class SerializeProcessFactory(
   IObjectSerializerFactory objectSerializerFactory,
   ISqLiteJsonCacheManagerFactory sqLiteJsonCacheManagerFactory,
   IServerObjectManagerFactory serverObjectManagerFactory,
+  IObjectSaverFactory objectSaverFactory,
   ILoggerFactory loggerFactory
 ) : ISerializeProcessFactory
 {
@@ -39,13 +40,7 @@ public class SerializeProcessFactory(
   ) =>
     new SerializeProcess(
       progress,
-      new ObjectSaver(
-        progress,
-        sqLiteJsonCacheManager,
-        serverObjectManager,
-        loggerFactory.CreateLogger<ObjectSaver>(),
-        cancellationToken
-      ),
+      objectSaverFactory.Create(serverObjectManager, sqLiteJsonCacheManager, progress, cancellationToken, options),
       baseChildFinder,
       new BaseSerializer(sqLiteJsonCacheManager, objectSerializerFactory),
       loggerFactory,

src/Speckle.Sdk/ServiceRegistration.cs

@@ -97,6 +97,8 @@ public static class ServiceRegistration
       typeof(Client)
     );
     serviceCollection.AddMatchingInterfacesAsTransient(typeof(GraphQLRetry).Assembly);
+    //we want to make object savers be singletons per stream so needs a singleton factory
+    serviceCollection.AddSingleton<IObjectSaverFactory, ObjectSaverFactory>();
     return serviceCollection;
   }
 

tests/Speckle.Sdk.Serialization.Tests/DataObjectTests.cs

@@ -41,7 +41,7 @@ public class DataObjectTests
       new DummyServerObjectManager(),
       null,
       default,
-      new SerializeProcessOptions(true, true, false, true)
+      new SerializeProcessOptions(false, false, false, true)
     );
     await serializeProcess.Serialize(x);
     await VerifyJson(json.Single().Value.Value).UseParameters(type);

tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs

@@ -41,7 +41,7 @@ public class DetachedTests
       objects,
       null,
       default,
-      new SerializeProcessOptions(false, false, true, true)
+      new SerializeProcessOptions(false, false, false, true)
     );
     await serializeProcess.Serialize(@base);
 
@@ -123,7 +123,7 @@ public class DetachedTests
       objects,
       null,
       default,
-      new SerializeProcessOptions(false, false, true, true) { MaxParallelism = 1, MaxHttpSendBatchSize = 1 }
+      new SerializeProcessOptions(false, false, false, true) { MaxParallelism = 1, MaxHttpSendBatchSize = 1 }
     );
     var results = await serializeProcess.Serialize(@base);
 
@@ -150,7 +150,7 @@ public class DetachedTests
       objects,
       null,
       default,
-      new SerializeProcessOptions(false, false, true, true) { MaxParallelism = 1, MaxHttpSendBatchSize = 1 }
+      new SerializeProcessOptions(false, false, false, true) { MaxParallelism = 1, MaxHttpSendBatchSize = 1 }
     );
     var results = await serializeProcess.Serialize(@base);
 
@@ -172,7 +172,7 @@ public class DetachedTests
       objects,
       null,
       default,
-      new SerializeProcessOptions(false, false, true, true) { MaxParallelism = 1, MaxHttpSendBatchSize = 1 }
+      new SerializeProcessOptions(false, false, false, true) { MaxParallelism = 1, MaxHttpSendBatchSize = 1 }
     );
     var results = await serializeProcess.Serialize(@base);
 
@@ -239,7 +239,7 @@ public class DetachedTests
       objects,
       null,
       default,
-      new SerializeProcessOptions(false, false, true, true)
+      new SerializeProcessOptions(false, false, false, true)
     );
 
     var results = await serializeProcess.Serialize(@base);
@@ -272,7 +272,7 @@ public class DetachedTests
       objects,
       null,
       default,
-      new SerializeProcessOptions(false, false, true, true)
+      new SerializeProcessOptions(false, false, false, true)
     );
     var results = await serializeProcess.Serialize(@base);
     await VerifyJsonDictionary(objects);
@@ -373,6 +373,9 @@ public class DummyServerObjectManager : IServerObjectManager
 
 public class DummySendCacheManager(Dictionary<string, string> objects) : ISqLiteJsonCacheManager
 {
+#pragma warning disable CA1065
+  public string Path => throw new NotImplementedException();
+#pragma warning restore CA1065
   public void Dispose() { }
 
   public IReadOnlyCollection<(string, string)> GetAllObjects() => throw new NotImplementedException();

tests/Speckle.Sdk.Serialization.Tests/DummyCancellationSqLiteSendManager.cs

@@ -4,6 +4,9 @@ namespace Speckle.Sdk.Serialization.Tests;
 
 public class DummyCancellationSqLiteSendManager : ISqLiteJsonCacheManager
 {
+#pragma warning disable CA1065
+  public string Path => throw new NotImplementedException();
+#pragma warning restore CA1065
   public string? GetObject(string id) => null;
 
   public void SaveObject(string id, string json) => throw new NotImplementedException();

tests/Speckle.Sdk.Serialization.Tests/Framework/ExceptionSendCacheManager.cs

@@ -4,6 +4,7 @@ namespace Speckle.Sdk.Serialization.Tests.Framework;
 
 public class ExceptionSendCacheManager(bool? hasObject = null, int? exceptionsAfter = null) : ISqLiteJsonCacheManager
 {
+  public string Path => "ExceptionSendCacheManager";
   private readonly object _lock = new();
   private int _count;
 

tests/Speckle.Sdk.Serialization.Tests/ObjectSaverFactoryTests.cs

@@ -0,0 +1,90 @@
+using FluentAssertions;
+using Microsoft.Extensions.Logging;
+using Moq;
+using Speckle.Sdk.SQLite;
+using Speckle.Sdk.Testing;
+
+namespace Speckle.Sdk.Serialisation.V2.Send.Tests;
+
+public class ObjectSaverFactoryTests : MoqTest
+{
+  private readonly Mock<ILoggerFactory> _loggerFactoryMock;
+  private readonly Mock<ILogger<ObjectSaver>> _loggerMock;
+  private readonly ObjectSaverFactory _factory;
+
+  public ObjectSaverFactoryTests()
+  {
+    _loggerFactoryMock = Create<ILoggerFactory>();
+    _loggerMock = Create<ILogger<ObjectSaver>>();
+    _factory = new ObjectSaverFactory(_loggerFactoryMock.Object);
+  }
+
+  public override void Dispose()
+  {
+    _factory.Dispose();
+    base.Dispose();
+  }
+
+  [Fact]
+  public void Create_ShouldReturnObjectSaverInstance()
+  {
+    _loggerFactoryMock.Setup(f => f.CreateLogger(typeof(ObjectSaver).FullName)).Returns(_loggerMock.Object);
+    var cacheManagerMock = Create<ISqLiteJsonCacheManager>();
+    cacheManagerMock.Setup(x => x.Dispose());
+    cacheManagerMock.SetupGet(c => c.Path).Returns("/tmp/test1.db");
+
+    var saver = _factory.Create(
+      Create<IServerObjectManager>().Object,
+      cacheManagerMock.Object,
+      null,
+      CancellationToken.None
+    );
+
+    saver.Should().NotBeNull();
+  }
+
+  [Fact]
+  public void Create_ShouldReturnSameInstanceForSamePath()
+  {
+    _loggerFactoryMock.Setup(f => f.CreateLogger(typeof(ObjectSaver).FullName)).Returns(_loggerMock.Object);
+    var cacheManagerMock = Create<ISqLiteJsonCacheManager>();
+    cacheManagerMock.Setup(x => x.Dispose());
+    cacheManagerMock.SetupGet(c => c.Path).Returns("/tmp/test2.db");
+
+    var saver1 = _factory.Create(
+      Create<IServerObjectManager>().Object,
+      cacheManagerMock.Object,
+      null,
+      CancellationToken.None
+    );
+    var saver2 = _factory.Create(
+      Create<IServerObjectManager>().Object,
+      cacheManagerMock.Object,
+      null,
+      CancellationToken.None
+    );
+
+    saver1.Should().BeSameAs(saver2);
+  }
+
+  [Fact]
+  public void Dispose_ShouldDisposeAllSavers()
+  {
+    var saverMock1 = Create<IObjectSaver>();
+    _factory
+      .GetType()
+      .GetField("_savers", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)
+      ?.SetValue(
+        _factory,
+        new System.Collections.Concurrent.ConcurrentDictionary<string, IObjectSaver>(
+          new[]
+          {
+            new System.Collections.Generic.KeyValuePair<string, IObjectSaver>("/tmp/test3.db", saverMock1.Object),
+          }
+        )
+      );
+    saverMock1.Setup(x => x.Dispose());
+
+    _factory.Dispose();
+  }
+}

tests/Speckle.Sdk.Testing/Framework/DummySqLiteReceiveManager.cs

@@ -5,6 +5,9 @@ namespace Speckle.Sdk.Testing.Framework;
 public sealed class DummySqLiteReceiveManager(IReadOnlyDictionary<string, string> savedObjects)
   : ISqLiteJsonCacheManager
 {
+#pragma warning disable CA1065
+  public string Path => throw new NotImplementedException();
+#pragma warning restore CA1065
   public void Dispose() { }
 
   public IReadOnlyCollection<(string, string)> GetAllObjects() => throw new NotImplementedException();

tests/Speckle.Sdk.Testing/Framework/DummySqLiteSendManager.cs

@@ -4,6 +4,8 @@ namespace Speckle.Sdk.Testing.Framework;
 
 public class DummySqLiteSendManager : ISqLiteJsonCacheManager
 {
+  public string Path => "DummySqLiteSendManager";
+
   public string? GetObject(string id) => throw new NotImplementedException();
 
   public void SaveObject(string id, string json) => throw new NotImplementedException();

tests/Speckle.Sdk.Testing/MoqTest.cs

@@ -8,7 +8,7 @@ public abstract class MoqTest : IDisposable
 {
   protected MoqTest() => Repository = new(MockBehavior.Strict);
 
-  public void Dispose() => Repository.VerifyAll();
+  public virtual void Dispose() => Repository.VerifyAll();
 
   protected MockRepository Repository { get; private set; } = new(MockBehavior.Strict);
 

tests/Speckle.Sdk.Tests.Unit/Serialisation/SerializeProcessRecordExceptionTests.cs

@@ -21,7 +21,6 @@ public class SerializeProcessRecordExceptionTests : MoqTest
       .Setup(f => f.CreateLogger("Speckle.Sdk.Serialisation.V2.PriorityScheduler"))
       .Returns(Create<ILogger<PriorityScheduler>>().Object);
     var objectSaverMock = Create<IObjectSaver>();
-    objectSaverMock.Setup(x => x.Dispose());
     var baseChildFinderMock = Create<IBaseChildFinder>();
     var baseSerializerMock = Create<IBaseSerializer>();
     using var cts = new CancellationTokenSource();
@@ -57,7 +56,6 @@ public class SerializeProcessRecordExceptionTests : MoqTest
       .Setup(f => f.CreateLogger("Speckle.Sdk.Serialisation.V2.PriorityScheduler"))
       .Returns(Create<ILogger<PriorityScheduler>>().Object);
     var objectSaverMock = Create<IObjectSaver>();
-    objectSaverMock.Setup(x => x.Dispose());
     var baseChildFinderMock = Create<IBaseChildFinder>();
     var baseSerializerMock = Create<IBaseSerializer>();
     using var cts = new CancellationTokenSource();
@@ -88,7 +86,6 @@ public class SerializeProcessRecordExceptionTests : MoqTest
       .Setup(f => f.CreateLogger("Speckle.Sdk.Serialisation.V2.PriorityScheduler"))
       .Returns(Create<ILogger<PriorityScheduler>>().Object);
     var objectSaverMock = Create<IObjectSaver>();
-    objectSaverMock.Setup(x => x.Dispose());
     var baseChildFinderMock = Create<IBaseChildFinder>();
     var baseSerializerMock = Create<IBaseSerializer>();
     using var cts = new CancellationTokenSource();