From 9e7f26f7a6b69a016b16715ca163b7655c37fe1d Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Wed, 23 Jul 2025 10:07:57 +0100 Subject: [PATCH] Add ModelCacheManager class and use it (#356) * Introduce ModelCacheManager to manage cache and sizes and deletions * move and abstract * add tests and format * Update src/Speckle.Sdk/Caching/ModelCacheManager.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Clean up --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/Speckle.Sdk/Caching/FileSystem.cs | 22 +++++ src/Speckle.Sdk/Caching/ModelCacheManager.cs | 93 +++++++++++++++++++ .../SQLite/SqLiteJsonCacheManagerFactory.cs | 6 +- .../Serialisation/Utilities/SqlitePaths.cs | 30 ------ .../Transports/SQLiteTransport2.cs | 6 +- .../ModelCacheManagerMockTests.cs | 83 +++++++++++++++++ .../ModelCacheManagerTests.cs | 0 .../Transports/SQLiteTransport2Tests.cs | 4 +- 8 files changed, 206 insertions(+), 38 deletions(-) create mode 100644 src/Speckle.Sdk/Caching/FileSystem.cs create mode 100644 src/Speckle.Sdk/Caching/ModelCacheManager.cs delete mode 100644 src/Speckle.Sdk/Serialisation/Utilities/SqlitePaths.cs create mode 100644 tests/Speckle.Sdk.Tests.Unit/ModelCacheManagerMockTests.cs create mode 100644 tests/Speckle.Sdk.Tests.Unit/ModelCacheManagerTests.cs diff --git a/src/Speckle.Sdk/Caching/FileSystem.cs b/src/Speckle.Sdk/Caching/FileSystem.cs new file mode 100644 index 00000000..261d47ca --- /dev/null +++ b/src/Speckle.Sdk/Caching/FileSystem.cs @@ -0,0 +1,22 @@ +using Speckle.InterfaceGenerator; + +namespace Speckle.Sdk.Caching; + +/// +/// This mocks away the file system operations for testing purposes. +/// +[GenerateAutoInterface] +public class FileSystem : IFileSystem +{ + public bool DirectoryExists(string path) => Directory.Exists(path); + + public void CreateDirectory(string path) => Directory.CreateDirectory(path); + + public IEnumerable EnumerateFiles(string path) => Directory.EnumerateFiles(path); + + public void DeleteFile(string path) => File.Delete(path); + + public long GetFileSize(string path) => new FileInfo(path).Length; + + public string Combine(params string[] paths) => Path.Combine(paths); +} diff --git a/src/Speckle.Sdk/Caching/ModelCacheManager.cs b/src/Speckle.Sdk/Caching/ModelCacheManager.cs new file mode 100644 index 00000000..fbbe0688 --- /dev/null +++ b/src/Speckle.Sdk/Caching/ModelCacheManager.cs @@ -0,0 +1,93 @@ +using Microsoft.Extensions.Logging; +using Speckle.InterfaceGenerator; +using Speckle.Sdk.Logging; +using Speckle.Sdk.Transports; + +namespace Speckle.Sdk.Caching; + +/// +/// This class manages the cache for model data, providing methods to get stream paths, clear the cache, and calculate cache size. +/// +[GenerateAutoInterface] +public class ModelCacheManager(ILogger logger, IFileSystem fileSystem) : IModelCacheManager +{ + private const string DATA_FOLDER = "Projects"; + private static readonly string s_basePath = SpecklePathProvider.UserSpeckleFolderPath; + + private static string CacheFolder => Path.Combine(s_basePath, DATA_FOLDER); + + public string GetStreamPath(string streamId) => GetDbPath(streamId); + + public static string GetDbPath(string streamId) + { + var db = Path.Combine(CacheFolder, $"{streamId}.db"); + try + { + Directory.CreateDirectory(CacheFolder); //ensure dir is there + return db; + } + catch (Exception ex) + when (ex is ArgumentException or IOException or UnauthorizedAccessException or NotSupportedException) + { + throw new TransportException($"Path was invalid or could not be created {db}", ex); + } + } + + public void ClearCache() + { + try + { + if (!fileSystem.DirectoryExists(CacheFolder)) + { + return; + } + + foreach (var db in fileSystem.EnumerateFiles(CacheFolder)) + { + try + { + fileSystem.DeleteFile(db); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or NotSupportedException) + { + logger.LogWarning(ex, "Failed to delete cache file {filePath}", db); + } + } + } + catch (Exception ex) + when (ex is ArgumentException or IOException or UnauthorizedAccessException or NotSupportedException) + { + throw new TransportException($"Cache folder could not be cleared: {CacheFolder}", ex); + } + } + + public long GetCacheSize() + { + try + { + if (!fileSystem.DirectoryExists(CacheFolder)) + { + return 0; + } + + long size = 0; + foreach (var file in fileSystem.EnumerateFiles(CacheFolder)) + { + try + { + size += fileSystem.GetFileSize(file); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or NotSupportedException) + { + logger.LogWarning(ex, "Failed to get size for cache file {a}", file); + } + } + return size; + } + catch (Exception ex) + when (ex is ArgumentException or IOException or UnauthorizedAccessException or NotSupportedException) + { + throw new TransportException($"Cache folder size could not be determined: {CacheFolder}", ex); + } + } +} diff --git a/src/Speckle.Sdk/SQLite/SqLiteJsonCacheManagerFactory.cs b/src/Speckle.Sdk/SQLite/SqLiteJsonCacheManagerFactory.cs index 9eddb021..75147027 100644 --- a/src/Speckle.Sdk/SQLite/SqLiteJsonCacheManagerFactory.cs +++ b/src/Speckle.Sdk/SQLite/SqLiteJsonCacheManagerFactory.cs @@ -1,11 +1,11 @@ using Speckle.InterfaceGenerator; +using Speckle.Sdk.Caching; using Speckle.Sdk.Logging; -using Speckle.Sdk.Serialisation.Utilities; namespace Speckle.Sdk.SQLite; [GenerateAutoInterface] -public class SqLiteJsonCacheManagerFactory : ISqLiteJsonCacheManagerFactory +public class SqLiteJsonCacheManagerFactory(IModelCacheManager modelCacheManager) : ISqLiteJsonCacheManagerFactory { public const int INITIAL_CONCURRENCY = 4; @@ -16,5 +16,5 @@ public class SqLiteJsonCacheManagerFactory : ISqLiteJsonCacheManagerFactory Create(Path.Combine(SpecklePathProvider.UserApplicationDataPath(), "Speckle", $"{scope}.db"), 1); public ISqLiteJsonCacheManager CreateFromStream(string streamId) => - Create(SqlitePaths.GetDBPath(streamId), INITIAL_CONCURRENCY); + Create(modelCacheManager.GetStreamPath(streamId), INITIAL_CONCURRENCY); } diff --git a/src/Speckle.Sdk/Serialisation/Utilities/SqlitePaths.cs b/src/Speckle.Sdk/Serialisation/Utilities/SqlitePaths.cs deleted file mode 100644 index 3d0b0f37..00000000 --- a/src/Speckle.Sdk/Serialisation/Utilities/SqlitePaths.cs +++ /dev/null @@ -1,30 +0,0 @@ -using Speckle.Sdk.Logging; -using Speckle.Sdk.Transports; - -namespace Speckle.Sdk.Serialisation.Utilities; - -public static class SqlitePaths -{ - private const string APPLICATION_NAME = "Speckle"; - private const string DATA_FOLDER = "Projects"; - private static readonly string basePath = SpecklePathProvider.UserApplicationDataPath(); - - public static string BlobStorageFolder => - SpecklePathProvider.BlobStoragePath(Path.Combine(basePath, APPLICATION_NAME)); - - public static string GetDBPath(string streamId) - { - var dir = Path.Combine(basePath, APPLICATION_NAME, DATA_FOLDER); - var db = Path.Combine(dir, $"{streamId}.db"); - try - { - Directory.CreateDirectory(dir); //ensure dir is there - return db; - } - catch (Exception ex) - when (ex is ArgumentException or IOException or UnauthorizedAccessException or NotSupportedException) - { - throw new TransportException($"Path was invalid or could not be created {db}", ex); - } - } -} diff --git a/src/Speckle.Sdk/Transports/SQLiteTransport2.cs b/src/Speckle.Sdk/Transports/SQLiteTransport2.cs index 4dbaccd8..3009d3cb 100644 --- a/src/Speckle.Sdk/Transports/SQLiteTransport2.cs +++ b/src/Speckle.Sdk/Transports/SQLiteTransport2.cs @@ -3,9 +3,9 @@ using System.Diagnostics; using System.Text; using System.Timers; using Microsoft.Data.Sqlite; +using Speckle.Sdk.Caching; using Speckle.Sdk.Logging; using Speckle.Sdk.Models; -using Speckle.Sdk.Serialisation.Utilities; using Timer = System.Timers.Timer; namespace Speckle.Sdk.Transports; @@ -28,7 +28,7 @@ public sealed class SQLiteTransport2 : IDisposable, ICloneable, ITransport, IBlo { _streamId = streamId; - _rootPath = SqlitePaths.GetDBPath(streamId); + _rootPath = ModelCacheManager.GetDbPath(streamId); _connectionString = $"Data Source={_rootPath};"; @@ -50,7 +50,7 @@ public sealed class SQLiteTransport2 : IDisposable, ICloneable, ITransport, IBlo private SqliteConnection Connection { get; set; } private readonly SemaphoreSlim _connectionLock = new(1, 1); - public string BlobStorageFolder => SqlitePaths.BlobStorageFolder; + public string BlobStorageFolder => SpecklePathProvider.UserSpeckleFolderPath; public void SaveBlob(Blob obj) { diff --git a/tests/Speckle.Sdk.Tests.Unit/ModelCacheManagerMockTests.cs b/tests/Speckle.Sdk.Tests.Unit/ModelCacheManagerMockTests.cs new file mode 100644 index 00000000..0c41a552 --- /dev/null +++ b/tests/Speckle.Sdk.Tests.Unit/ModelCacheManagerMockTests.cs @@ -0,0 +1,83 @@ +using FluentAssertions; +using Microsoft.Extensions.Logging; +using Moq; +using Speckle.Sdk.Caching; +using Speckle.Sdk.Testing; + +namespace Speckle.Sdk.Tests.Unit; + +public class ModelCacheManagerMockTests : MoqTest +{ + private readonly Mock _fileSystemMock; + private readonly ModelCacheManager _manager; + + public ModelCacheManagerMockTests() + { + Mock> loggerMock = Create>(MockBehavior.Loose); + _fileSystemMock = Create(); + _manager = new ModelCacheManager(loggerMock.Object, _fileSystemMock.Object); + } + + [Fact] + public void ClearCache_ShouldNotDeleteFiles_WhenDirectoryDoesNotExist() + { + _fileSystemMock.Setup(fs => fs.DirectoryExists(It.IsAny())).Returns(false); + _manager.ClearCache(); + _fileSystemMock.Verify(fs => fs.EnumerateFiles(It.IsAny()), Times.Never); + _fileSystemMock.Verify(fs => fs.DeleteFile(It.IsAny()), Times.Never); + } + + [Fact] + public void ClearCache_ShouldDeleteFiles_WhenDirectoryExists() + { + var files = new List { "file1.db", "file2.db" }; + _fileSystemMock.Setup(fs => fs.DirectoryExists(It.IsAny())).Returns(true); + _fileSystemMock.Setup(fs => fs.EnumerateFiles(It.IsAny())).Returns(files); + foreach (var file in files) + { + _fileSystemMock.Setup(fs => fs.DeleteFile(file)); + } + _manager.ClearCache(); + } + + [Fact] + public void ClearCache_ShouldLogWarning_WhenDeleteFileThrows() + { + var files = new List { "file1.db" }; + _fileSystemMock.Setup(fs => fs.DirectoryExists(It.IsAny())).Returns(true); + _fileSystemMock.Setup(fs => fs.EnumerateFiles(It.IsAny())).Returns(files); + _fileSystemMock.Setup(fs => fs.DeleteFile(It.IsAny())).Throws(); + _manager.ClearCache(); + } + + [Fact] + public void GetCacheSize_ShouldReturnZero_WhenDirectoryDoesNotExist() + { + _fileSystemMock.Setup(fs => fs.DirectoryExists(It.IsAny())).Returns(false); + var size = _manager.GetCacheSize(); + size.Should().Be(0); + } + + [Fact] + public void GetCacheSize_ShouldSumFileSizes() + { + var files = new List { "file1.db", "file2.db" }; + _fileSystemMock.Setup(fs => fs.DirectoryExists(It.IsAny())).Returns(true); + _fileSystemMock.Setup(fs => fs.EnumerateFiles(It.IsAny())).Returns(files); + _fileSystemMock.Setup(fs => fs.GetFileSize("file1.db")).Returns(10); + _fileSystemMock.Setup(fs => fs.GetFileSize("file2.db")).Returns(20); + var size = _manager.GetCacheSize(); + size.Should().Be(30); + } + + [Fact] + public void GetCacheSize_ShouldLogWarning_WhenGetFileSizeThrows() + { + var files = new List { "file1.db" }; + _fileSystemMock.Setup(fs => fs.DirectoryExists(It.IsAny())).Returns(true); + _fileSystemMock.Setup(fs => fs.EnumerateFiles(It.IsAny())).Returns(files); + _fileSystemMock.Setup(fs => fs.GetFileSize(It.IsAny())).Throws(); + var size = _manager.GetCacheSize(); + size.Should().Be(0); + } +} diff --git a/tests/Speckle.Sdk.Tests.Unit/ModelCacheManagerTests.cs b/tests/Speckle.Sdk.Tests.Unit/ModelCacheManagerTests.cs new file mode 100644 index 00000000..e69de29b diff --git a/tests/Speckle.Sdk.Tests.Unit/Transports/SQLiteTransport2Tests.cs b/tests/Speckle.Sdk.Tests.Unit/Transports/SQLiteTransport2Tests.cs index 5b82d283..55b46392 100644 --- a/tests/Speckle.Sdk.Tests.Unit/Transports/SQLiteTransport2Tests.cs +++ b/tests/Speckle.Sdk.Tests.Unit/Transports/SQLiteTransport2Tests.cs @@ -1,7 +1,7 @@ using FluentAssertions; using Microsoft.Data.Sqlite; +using Speckle.Sdk.Caching; using Speckle.Sdk.Common; -using Speckle.Sdk.Serialisation.Utilities; using Speckle.Sdk.Transports; namespace Speckle.Sdk.Tests.Unit.Transports; @@ -13,7 +13,7 @@ public sealed class SQLiteTransport2Tests : TransportTests, IDisposable private SQLiteTransport2? _sqlite; private static readonly string s_name = $"test-{Guid.NewGuid()}"; - private static readonly string s_basePath = SqlitePaths.GetDBPath(s_name); + private static readonly string s_basePath = ModelCacheManager.GetDbPath(s_name); public SQLiteTransport2Tests() {