From 7f8b59d348da4c5f6d1ae88f7a252b5dc87f5c37 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Tue, 10 Jun 2025 11:15:01 +0100 Subject: [PATCH] Pool object savers instead of sqlite --- .../SQLite/SQLiteJsonCacheManager.cs | 8 +++-- .../V2/MemoryJsonCacheManager.cs | 4 +++ .../V2/Send/ObjectSaverFactory.cs | 36 +++++++++++++++++++ .../Serialisation/V2/Send/SerializeProcess.cs | 1 - .../V2/SerializeProcessFactory.cs | 11 ++---- src/Speckle.Sdk/ServiceRegistration.cs | 2 ++ .../DetachedTests.cs | 3 ++ .../DummyCancellationSqLiteSendManager.cs | 3 ++ .../Framework/ExceptionSendCacheManager.cs | 3 ++ .../Framework/DummySqLiteReceiveManager.cs | 3 ++ .../Framework/DummySqLiteSendManager.cs | 3 ++ 11 files changed, 66 insertions(+), 11 deletions(-) create mode 100644 src/Speckle.Sdk/Serialisation/V2/Send/ObjectSaverFactory.cs diff --git a/src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs b/src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs index 0c3d89f7..dfb4322a 100644 --- a/src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs +++ b/src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs @@ -12,16 +12,19 @@ public partial interface ISqLiteJsonCacheManager : IDisposable; public sealed class SqLiteJsonCacheManager : ISqLiteJsonCacheManager { private readonly CacheDbCommandPool _pool; + + public string Path {get;} public static ISqLiteJsonCacheManager FromMemory(int concurrency) => new SqLiteJsonCacheManager(concurrency); private SqLiteJsonCacheManager(int concurrency) { + Path = ":memory:"; //disable pooling as we pool ourselves var builder = new SqliteConnectionStringBuilder { Pooling = false, - DataSource = ":memory:", + DataSource = Path, Cache = SqliteCacheMode.Shared, Mode = SqliteOpenMode.Memory, }; @@ -34,8 +37,9 @@ public sealed class SqLiteJsonCacheManager : ISqLiteJsonCacheManager private SqLiteJsonCacheManager(string path, int concurrency) { + Path = path; //disable pooling as we pool ourselves - var builder = new SqliteConnectionStringBuilder { Pooling = false, DataSource = path }; + var builder = new SqliteConnectionStringBuilder { Pooling = false, DataSource = Path }; _pool = new CacheDbCommandPool(builder.ToString(), concurrency); Initialize(); } diff --git a/src/Speckle.Sdk/Serialisation/V2/MemoryJsonCacheManager.cs b/src/Speckle.Sdk/Serialisation/V2/MemoryJsonCacheManager.cs index 26e142be..447c3ce9 100644 --- a/src/Speckle.Sdk/Serialisation/V2/MemoryJsonCacheManager.cs +++ b/src/Speckle.Sdk/Serialisation/V2/MemoryJsonCacheManager.cs @@ -7,6 +7,10 @@ namespace Speckle.Sdk.Serialisation.V2; public class MemoryJsonCacheManager(ConcurrentDictionary jsonCache) : ISqLiteJsonCacheManager #pragma warning restore CA1063 { +#pragma warning disable CA1065 + public string Path => throw new NotImplementedException(); +#pragma warning restore CA1065 + public IReadOnlyCollection<(string Id, string Json)> GetAllObjects() => jsonCache.Select(x => (x.Key.Value, x.Value.Value)).ToList(); diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/ObjectSaverFactory.cs b/src/Speckle.Sdk/Serialisation/V2/Send/ObjectSaverFactory.cs new file mode 100644 index 00000000..8988f492 --- /dev/null +++ b/src/Speckle.Sdk/Serialisation/V2/Send/ObjectSaverFactory.cs @@ -0,0 +1,36 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging; +using Speckle.InterfaceGenerator; +using Speckle.Sdk.SQLite; +using Speckle.Sdk.Transports; + +namespace Speckle.Sdk.Serialisation.V2.Send; + +public partial interface IObjectSaverFactory : IDisposable; +[GenerateAutoInterface] +public sealed class ObjectSaverFactory( IServerObjectManager serverObjectManager, ILoggerFactory loggerFactory) : IObjectSaverFactory +{private readonly ConcurrentDictionary _savers = new(); + public IObjectSaver Create( + ISqLiteJsonCacheManager sqLiteJsonCacheManager, IProgress? progress, CancellationToken cancellationToken, + SerializeProcessOptions? options = null) + { + if (!_savers.TryGetValue(sqLiteJsonCacheManager.Path, out var saver)) + { + saver = new ObjectSaver(progress,sqLiteJsonCacheManager, serverObjectManager, loggerFactory.CreateLogger(), + cancellationToken, options); + _savers.TryAdd(sqLiteJsonCacheManager.Path, saver); + } + + return saver; + } + [AutoInterfaceIgnore] + public void Dispose() + { + foreach (var pool in _savers) + { + pool.Value.Dispose(); + } + + _savers.Clear(); + } +} diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs index 0254921d..35cb3731 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs @@ -86,7 +86,6 @@ public sealed class SerializeProcess( await WaitForSchedulerCompletion().ConfigureAwait(false); await _highest.DisposeAsync().ConfigureAwait(false); await _belowNormal.DisposeAsync().ConfigureAwait(false); - objectSaver.Dispose(); _processSource.Dispose(); } diff --git a/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs b/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs index c1428bf9..3e0a489e 100644 --- a/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs +++ b/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs @@ -13,6 +13,7 @@ public class SerializeProcessFactory( IObjectSerializerFactory objectSerializerFactory, ISqLiteJsonCacheManagerFactory sqLiteJsonCacheManagerFactory, IServerObjectManagerFactory serverObjectManagerFactory, + IObjectSaverFactory objectSaverFactory, ILoggerFactory loggerFactory ) : ISerializeProcessFactory { @@ -27,7 +28,7 @@ public class SerializeProcessFactory( { var sqLiteJsonCacheManager = sqLiteJsonCacheManagerFactory.CreateFromStream(streamId); var serverObjectManager = serverObjectManagerFactory.Create(url, streamId, authorizationToken); - return CreateSerializeProcess(sqLiteJsonCacheManager, serverObjectManager, progress, cancellationToken, options); + return CreateSerializeProcess(sqLiteJsonCacheManager, serverObjectManager, progress, cancellationToken, options); } public ISerializeProcess CreateSerializeProcess( @@ -39,13 +40,7 @@ public class SerializeProcessFactory( ) => new SerializeProcess( progress, - new ObjectSaver( - progress, - sqLiteJsonCacheManager, - serverObjectManager, - loggerFactory.CreateLogger(), - cancellationToken - ), + objectSaverFactory.Create(sqLiteJsonCacheManager, progress, cancellationToken, options), baseChildFinder, new BaseSerializer(sqLiteJsonCacheManager, objectSerializerFactory), loggerFactory, diff --git a/src/Speckle.Sdk/ServiceRegistration.cs b/src/Speckle.Sdk/ServiceRegistration.cs index 8f8066ae..106be8bd 100644 --- a/src/Speckle.Sdk/ServiceRegistration.cs +++ b/src/Speckle.Sdk/ServiceRegistration.cs @@ -97,6 +97,8 @@ public static class ServiceRegistration typeof(Client) ); serviceCollection.AddMatchingInterfacesAsTransient(typeof(GraphQLRetry).Assembly); + //we want to make object savers be singletons per stream so needs a singleton factory + serviceCollection.AddSingleton(); return serviceCollection; } diff --git a/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs b/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs index a8664172..6df010aa 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs @@ -373,6 +373,9 @@ public class DummyServerObjectManager : IServerObjectManager public class DummySendCacheManager(Dictionary objects) : ISqLiteJsonCacheManager { +#pragma warning disable CA1065 + public string Path => throw new NotImplementedException(); +#pragma warning restore CA1065 public void Dispose() { } public IReadOnlyCollection<(string, string)> GetAllObjects() => throw new NotImplementedException(); diff --git a/tests/Speckle.Sdk.Serialization.Tests/DummyCancellationSqLiteSendManager.cs b/tests/Speckle.Sdk.Serialization.Tests/DummyCancellationSqLiteSendManager.cs index 2d6ca588..1244b303 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/DummyCancellationSqLiteSendManager.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/DummyCancellationSqLiteSendManager.cs @@ -4,6 +4,9 @@ namespace Speckle.Sdk.Serialization.Tests; public class DummyCancellationSqLiteSendManager : ISqLiteJsonCacheManager { +#pragma warning disable CA1065 + public string Path => throw new NotImplementedException(); +#pragma warning restore CA1065 public string? GetObject(string id) => null; public void SaveObject(string id, string json) => throw new NotImplementedException(); diff --git a/tests/Speckle.Sdk.Serialization.Tests/Framework/ExceptionSendCacheManager.cs b/tests/Speckle.Sdk.Serialization.Tests/Framework/ExceptionSendCacheManager.cs index 5572aa02..22e5372e 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/Framework/ExceptionSendCacheManager.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/Framework/ExceptionSendCacheManager.cs @@ -4,6 +4,9 @@ namespace Speckle.Sdk.Serialization.Tests.Framework; public class ExceptionSendCacheManager(bool? hasObject = null, int? exceptionsAfter = null) : ISqLiteJsonCacheManager { +#pragma warning disable CA1065 + public string Path => throw new NotImplementedException(); +#pragma warning restore CA1065 private readonly object _lock = new(); private int _count; diff --git a/tests/Speckle.Sdk.Testing/Framework/DummySqLiteReceiveManager.cs b/tests/Speckle.Sdk.Testing/Framework/DummySqLiteReceiveManager.cs index fcfe6832..01109410 100644 --- a/tests/Speckle.Sdk.Testing/Framework/DummySqLiteReceiveManager.cs +++ b/tests/Speckle.Sdk.Testing/Framework/DummySqLiteReceiveManager.cs @@ -5,6 +5,9 @@ namespace Speckle.Sdk.Testing.Framework; public sealed class DummySqLiteReceiveManager(IReadOnlyDictionary savedObjects) : ISqLiteJsonCacheManager { +#pragma warning disable CA1065 + public string Path => throw new NotImplementedException(); +#pragma warning restore CA1065 public void Dispose() { } public IReadOnlyCollection<(string, string)> GetAllObjects() => throw new NotImplementedException(); diff --git a/tests/Speckle.Sdk.Testing/Framework/DummySqLiteSendManager.cs b/tests/Speckle.Sdk.Testing/Framework/DummySqLiteSendManager.cs index b1425dd6..63983aa6 100644 --- a/tests/Speckle.Sdk.Testing/Framework/DummySqLiteSendManager.cs +++ b/tests/Speckle.Sdk.Testing/Framework/DummySqLiteSendManager.cs @@ -4,6 +4,9 @@ namespace Speckle.Sdk.Testing.Framework; public class DummySqLiteSendManager : ISqLiteJsonCacheManager { +#pragma warning disable CA1065 + public string Path => throw new NotImplementedException(); +#pragma warning restore CA1065 public string? GetObject(string id) => throw new NotImplementedException(); public void SaveObject(string id, string json) => throw new NotImplementedException();