diff --git a/Directory.Packages.props b/Directory.Packages.props index a6f4d7cb..f52f1e0e 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -8,10 +8,12 @@ + + diff --git a/Speckle.Sdk.sln b/Speckle.Sdk.sln index bb0e5bbd..48e0047c 100644 --- a/Speckle.Sdk.sln +++ b/Speckle.Sdk.sln @@ -42,6 +42,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "workflows", "workflows", "{ EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Speckle.Sdk.Tests.Performance", "tests\Speckle.Sdk.Tests.Performance\Speckle.Sdk.Tests.Performance.csproj", "{870E3396-E6F7-43AE-B120-E651FA4F46BD}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Speckle.Sdk.Serialization.Testing", "tests\Speckle.Sdk.Serialization.Testing\Speckle.Sdk.Serialization.Testing.csproj", "{FF922B6D-D416-4348-8CB8-0C8B28691070}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -84,6 +86,10 @@ Global {870E3396-E6F7-43AE-B120-E651FA4F46BD}.Debug|Any CPU.Build.0 = Debug|Any CPU {870E3396-E6F7-43AE-B120-E651FA4F46BD}.Release|Any CPU.ActiveCfg = Release|Any CPU {870E3396-E6F7-43AE-B120-E651FA4F46BD}.Release|Any CPU.Build.0 = Release|Any CPU + {FF922B6D-D416-4348-8CB8-0C8B28691070}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FF922B6D-D416-4348-8CB8-0C8B28691070}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FF922B6D-D416-4348-8CB8-0C8B28691070}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FF922B6D-D416-4348-8CB8-0C8B28691070}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(NestedProjects) = preSolution {A413E196-3696-4F48-B635-04B5F76BF9C9} = {5CB96C27-FC5B-4A41-86B6-951AF99B8116} @@ -95,5 +101,6 @@ Global {4FB41A6D-D139-4111-8115-E3F9F6BEAF24} = {35047EE7-AD1D-4741-80A7-8F0E874718E9} {B623BD21-5CAA-43F9-A539-1835276C220E} = {DA2AED52-58F9-471E-8AD8-102FD36129E3} {870E3396-E6F7-43AE-B120-E651FA4F46BD} = {35047EE7-AD1D-4741-80A7-8F0E874718E9} + {FF922B6D-D416-4348-8CB8-0C8B28691070} = {35047EE7-AD1D-4741-80A7-8F0E874718E9} EndGlobalSection EndGlobal diff --git a/src/Speckle.Objects/packages.lock.json b/src/Speckle.Objects/packages.lock.json index bf429349..e5c32636 100644 --- a/src/Speckle.Objects/packages.lock.json +++ b/src/Speckle.Objects/packages.lock.json @@ -234,10 +234,12 @@ "type": "Project", "dependencies": { "GraphQL.Client": "[6.0.0, )", + "Microsoft.Bcl.AsyncInterfaces": "[8.0.0, )", "Microsoft.CSharp": "[4.7.0, )", "Microsoft.Data.Sqlite": "[7.0.7, )", "Microsoft.Extensions.DependencyInjection.Abstractions": "[2.2.0, )", "Microsoft.Extensions.Logging": "[2.2.0, )", + "Microsoft.Extensions.ObjectPool": "[8.0.10, )", "Polly": "[7.2.3, )", "Polly.Contrib.WaitAndRetry": "[1.1.1, )", "Polly.Extensions.Http": "[3.0.0, )", @@ -256,6 +258,15 @@ "System.Reactive": "5.0.0" } }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "CentralTransitive", + "requested": "[8.0.0, )", + "resolved": "8.0.0", + "contentHash": "3WA9q9yVqJp222P3x1wYIGDAkpjAku0TMUaaQV22g6L67AI0LdOIrVS7Ht2vJfLHGSPVuqN94vIr15qn+HEkHw==", + "dependencies": { + "System.Threading.Tasks.Extensions": "4.5.4" + } + }, "Microsoft.CSharp": { "type": "CentralTransitive", "requested": "[4.7.0, )", @@ -290,6 +301,12 @@ "Microsoft.Extensions.Options": "2.2.0" } }, + "Microsoft.Extensions.ObjectPool": { + "type": "CentralTransitive", + "requested": "[8.0.10, )", + "resolved": "8.0.10", + "contentHash": "u7gAG7JgxF8VSJUGPSudAcPxOt+ymJKQCSxNRxiuKV+klCQbHljQR75SilpedCTfhPWDhtUwIJpnDVtspr9nMg==" + }, "Polly": { "type": "CentralTransitive", "requested": "[7.2.3, )", @@ -494,10 +511,12 @@ "type": "Project", "dependencies": { "GraphQL.Client": "[6.0.0, )", + "Microsoft.Bcl.AsyncInterfaces": "[8.0.0, )", "Microsoft.CSharp": "[4.7.0, )", "Microsoft.Data.Sqlite": "[7.0.7, )", "Microsoft.Extensions.DependencyInjection.Abstractions": "[2.2.0, )", "Microsoft.Extensions.Logging": "[2.2.0, )", + "Microsoft.Extensions.ObjectPool": "[8.0.10, )", "Polly": "[7.2.3, )", "Polly.Contrib.WaitAndRetry": "[1.1.1, )", "Polly.Extensions.Http": "[3.0.0, )", @@ -516,6 +535,12 @@ "System.Reactive": "5.0.0" } }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "CentralTransitive", + "requested": "[8.0.0, )", + "resolved": "8.0.0", + "contentHash": "3WA9q9yVqJp222P3x1wYIGDAkpjAku0TMUaaQV22g6L67AI0LdOIrVS7Ht2vJfLHGSPVuqN94vIr15qn+HEkHw==" + }, "Microsoft.CSharp": { "type": "CentralTransitive", "requested": "[4.7.0, )", @@ -550,6 +575,12 @@ "Microsoft.Extensions.Options": "2.2.0" } }, + "Microsoft.Extensions.ObjectPool": { + "type": "CentralTransitive", + "requested": "[8.0.10, )", + "resolved": "8.0.10", + "contentHash": "u7gAG7JgxF8VSJUGPSudAcPxOt+ymJKQCSxNRxiuKV+klCQbHljQR75SilpedCTfhPWDhtUwIJpnDVtspr9nMg==" + }, "Polly": { "type": "CentralTransitive", "requested": "[7.2.3, )", diff --git a/src/Speckle.Sdk/Api/Operations/Operations.Receive.cs b/src/Speckle.Sdk/Api/Operations/Operations.Receive.cs index 27e3da7a..60d0c296 100644 --- a/src/Speckle.Sdk/Api/Operations/Operations.Receive.cs +++ b/src/Speckle.Sdk/Api/Operations/Operations.Receive.cs @@ -2,12 +2,46 @@ using Microsoft.Extensions.Logging; using Speckle.Sdk.Logging; using Speckle.Sdk.Models; using Speckle.Sdk.Serialisation; +using Speckle.Sdk.Serialisation.V2; +using Speckle.Sdk.Serialisation.V2.Receive; using Speckle.Sdk.Transports; namespace Speckle.Sdk.Api; public partial class Operations { + public async Task Receive2( + Uri url, + string streamId, + string objectId, + string? authorizationToken = null, + IProgress? onProgressAction = null, + CancellationToken cancellationToken = default + ) + { + using var receiveActivity = activityFactory.Start("Operations.Receive"); + metricsFactory.CreateCounter("Receive").Add(1); + + receiveActivity?.SetTag("objectId", objectId); + + try + { + var sqliteTransport = new SQLiteCacheManager(streamId); + var serverObjects = new ServerObjectManager(speckleHttp, activityFactory, url, authorizationToken); + var o = new ObjectLoader(sqliteTransport, serverObjects, streamId, onProgressAction); + var process = new DeserializeProcess(onProgressAction, o); + var result = await process.Deserialize(objectId, cancellationToken).ConfigureAwait(false); + receiveActivity?.SetStatus(SdkActivityStatusCode.Ok); + return result; + } + catch (Exception ex) + { + receiveActivity?.SetStatus(SdkActivityStatusCode.Error); + receiveActivity?.RecordException(ex); + throw; + } + } + /// /// Receives an object (and all its sub-children) from the two provided s. ///
diff --git a/src/Speckle.Sdk/Api/Operations/Operations.cs b/src/Speckle.Sdk/Api/Operations/Operations.cs index dd050250..b3bf342c 100644 --- a/src/Speckle.Sdk/Api/Operations/Operations.cs +++ b/src/Speckle.Sdk/Api/Operations/Operations.cs @@ -1,5 +1,6 @@ using Microsoft.Extensions.Logging; using Speckle.InterfaceGenerator; +using Speckle.Sdk.Helpers; using Speckle.Sdk.Logging; namespace Speckle.Sdk.Api; @@ -12,6 +13,7 @@ namespace Speckle.Sdk.Api; [GenerateAutoInterface] public partial class Operations( ILogger logger, + ISpeckleHttp speckleHttp, ISdkActivityFactory activityFactory, ISdkMetricsFactory metricsFactory ) : IOperations; diff --git a/src/Speckle.Sdk/Helpers/SpeckleHttpClientHandler.cs b/src/Speckle.Sdk/Helpers/SpeckleHttpClientHandler.cs index 58959fc6..f2fbea93 100644 --- a/src/Speckle.Sdk/Helpers/SpeckleHttpClientHandler.cs +++ b/src/Speckle.Sdk/Helpers/SpeckleHttpClientHandler.cs @@ -66,8 +66,14 @@ public sealed class SpeckleHttpClientHandler : DelegatingHandler if (policyResult.Outcome == OutcomeType.Successful) { + activity?.SetStatus(SdkActivityStatusCode.Ok); return policyResult.Result.NotNull(); } + activity?.SetStatus(SdkActivityStatusCode.Error); + if (policyResult.FinalException != null) + { + activity?.RecordException(policyResult.FinalException); + } // if the policy failed due to a cancellation, AND it was our cancellation token, then don't wrap the exception, and rethrow an new cancellation if (policyResult.FinalException is OperationCanceledException) @@ -75,7 +81,10 @@ public sealed class SpeckleHttpClientHandler : DelegatingHandler cancellationToken.ThrowIfCancellationRequested(); } - throw new HttpRequestException("Policy Failed", policyResult.FinalException); + throw new HttpRequestException( + "Policy Failed: " + policyResult.FinalHandledResult?.StatusCode ?? "Unknown", + policyResult.FinalException + ); } } } diff --git a/src/Speckle.Sdk/Serialisation/SpeckleObjectSerializerPool.cs b/src/Speckle.Sdk/Serialisation/SpeckleObjectSerializerPool.cs index cedba016..0f881065 100644 --- a/src/Speckle.Sdk/Serialisation/SpeckleObjectSerializerPool.cs +++ b/src/Speckle.Sdk/Serialisation/SpeckleObjectSerializerPool.cs @@ -1,4 +1,5 @@ using System.Buffers; +using Microsoft.Extensions.ObjectPool; using Speckle.Newtonsoft.Json; using Speckle.Sdk.Common; @@ -22,4 +23,31 @@ public class SpeckleObjectSerializerPool public void Return(T[]? array) => pool.Return(array.NotNull()); } + + public ObjectPool> ObjectDictionaries { get; } = + ObjectPool.Create(new ObjectDictionaryPolicy()); + + private class ObjectDictionaryPolicy : IPooledObjectPolicy> + { + public Dictionary Create() => new(50, StringComparer.OrdinalIgnoreCase); + + public bool Return(Dictionary obj) + { + obj.Clear(); + return true; + } + } + + public ObjectPool> ListString { get; } = ObjectPool.Create(new ListStringPolicy()); + + private class ListStringPolicy : IPooledObjectPolicy> + { + public List Create() => new(20); + + public bool Return(List obj) + { + obj.Clear(); + return true; + } + } } diff --git a/src/Speckle.Sdk/Serialisation/V2/AsyncExtensions.cs b/src/Speckle.Sdk/Serialisation/V2/AsyncExtensions.cs new file mode 100644 index 00000000..849ef5af --- /dev/null +++ b/src/Speckle.Sdk/Serialisation/V2/AsyncExtensions.cs @@ -0,0 +1,212 @@ +using System.Diagnostics; + +namespace Speckle.Sdk.Serialisation.V2; + +public static class AsyncExtensions +{ + public static async ValueTask FirstAsync(this IAsyncEnumerable source) + { + var e = source.GetAsyncEnumerator(); + if (await e.MoveNextAsync().ConfigureAwait(false)) + { + return e.Current; + } + throw new InvalidOperationException("Sequence contains no elements"); + } + + public static async IAsyncEnumerable SelectManyAsync(this IEnumerable> source) + { + // get enumerators from all inner IAsyncEnumerable + var enumerators = source.Select(x => x.GetAsyncEnumerator()).ToList(); + + List, bool)>> runningTasks = new(); + + // start all inner IAsyncEnumerable + foreach (var asyncEnumerator in enumerators) + { + runningTasks.Add(MoveNextWrapped(asyncEnumerator)); + } + + // while there are any running tasks + while (runningTasks.Count != 0) + { + // get next finished task and remove it from list + var finishedTask = await Task.WhenAny(runningTasks).ConfigureAwait(false); + runningTasks.Remove(finishedTask); + + // get result from finished IAsyncEnumerable + var result = await finishedTask.ConfigureAwait(false); + var asyncEnumerator = result.Item1; + var hasItem = result.Item2; + + // if IAsyncEnumerable has item, return it and put it back as running for next item + if (hasItem) + { + yield return asyncEnumerator.Current; + + runningTasks.Add(MoveNextWrapped(asyncEnumerator)); + } + } + + // don't forget to dispose, should be in finally + foreach (var asyncEnumerator in enumerators) + { + await asyncEnumerator.DisposeAsync().ConfigureAwait(false); + } + } + + /// + /// Helper method that returns Task with tuple of IAsyncEnumerable and it's result of MoveNextAsync. + /// + private static async Task<(IAsyncEnumerator, bool)> MoveNextWrapped( + IAsyncEnumerator asyncEnumerator + ) + { + var res = await asyncEnumerator.MoveNextAsync().ConfigureAwait(false); + return (asyncEnumerator, res); + } + + public static IAsyncEnumerable BatchAsync(this IAsyncEnumerable source, int size) => + AsyncEnumerableChunkIterator(source, size); + + private static async IAsyncEnumerable AsyncEnumerableChunkIterator( + IAsyncEnumerable source, + int size + ) + { +#pragma warning disable CA2007 + await using IAsyncEnumerator e = source.GetAsyncEnumerator(); +#pragma warning restore CA2007 + + // Before allocating anything, make sure there's at least one element. + if (await e.MoveNextAsync().ConfigureAwait(false)) + { + // Now that we know we have at least one item, allocate an initial storage array. This is not + // the array we'll yield. It starts out small in order to avoid significantly overallocating + // when the source has many fewer elements than the chunk size. + int arraySize = Math.Min(size, 4); + int i; + do + { + var array = new TSource[arraySize]; + + // Store the first item. + array[0] = e.Current; + i = 1; + + if (size != array.Length) + { + // This is the first chunk. As we fill the array, grow it as needed. + for (; i < size && await e.MoveNextAsync().ConfigureAwait(false); i++) + { + if (i >= array.Length) + { + arraySize = (int)Math.Min((uint)size, 2 * (uint)array.Length); + Array.Resize(ref array, arraySize); + } + + array[i] = e.Current; + } + } + else + { + // For all but the first chunk, the array will already be correctly sized. + // We can just store into it until either it's full or MoveNext returns false. + TSource[] local = array; // avoid bounds checks by using cached local (`array` is lifted to iterator object as a field) + Debug.Assert(local.Length == size); + for (; (uint)i < (uint)local.Length && await e.MoveNextAsync().ConfigureAwait(false); i++) + { + local[i] = e.Current; + } + } + + if (i != array.Length) + { + Array.Resize(ref array, i); + } + + yield return array; + } while (i >= size && await e.MoveNextAsync().ConfigureAwait(false)); + } + } + + public static IEnumerable Batch(this IEnumerable source, int size) + { + if (source is TSource[] array) + { + // Special-case arrays, which have an immutable length. This enables us to not only do an + // empty check and avoid allocating an iterator object when empty, it enables us to have a + // much more efficient (and simpler) implementation for chunking up the array. + return array.Length != 0 ? ArrayChunkIterator(array, size) : []; + } + + return EnumerableChunkIterator(source, size); + } + + private static IEnumerable ArrayChunkIterator(TSource[] source, int size) + { + int index = 0; + while (index < source.Length) + { + TSource[] chunk = new ReadOnlySpan(source, index, Math.Min(size, source.Length - index)).ToArray(); + index += chunk.Length; + yield return chunk; + } + } + + private static IEnumerable EnumerableChunkIterator(IEnumerable source, int size) + { + using IEnumerator e = source.GetEnumerator(); + + // Before allocating anything, make sure there's at least one element. + if (e.MoveNext()) + { + // Now that we know we have at least one item, allocate an initial storage array. This is not + // the array we'll yield. It starts out small in order to avoid significantly overallocating + // when the source has many fewer elements than the chunk size. + int arraySize = Math.Min(size, 4); + int i; + do + { + var array = new TSource[arraySize]; + + // Store the first item. + array[0] = e.Current; + i = 1; + + if (size != array.Length) + { + // This is the first chunk. As we fill the array, grow it as needed. + for (; i < size && e.MoveNext(); i++) + { + if (i >= array.Length) + { + arraySize = (int)Math.Min((uint)size, 2 * (uint)array.Length); + Array.Resize(ref array, arraySize); + } + + array[i] = e.Current; + } + } + else + { + // For all but the first chunk, the array will already be correctly sized. + // We can just store into it until either it's full or MoveNext returns false. + TSource[] local = array; // avoid bounds checks by using cached local (`array` is lifted to iterator object as a field) + Debug.Assert(local.Length == size); + for (; (uint)i < (uint)local.Length && e.MoveNext(); i++) + { + local[i] = e.Current; + } + } + + if (i != array.Length) + { + Array.Resize(ref array, i); + } + + yield return array; + } while (i >= size && e.MoveNext()); + } + } +} diff --git a/src/Speckle.Sdk/Serialisation/V2/Receive/DeserializeProcess.cs b/src/Speckle.Sdk/Serialisation/V2/Receive/DeserializeProcess.cs new file mode 100644 index 00000000..997343db --- /dev/null +++ b/src/Speckle.Sdk/Serialisation/V2/Receive/DeserializeProcess.cs @@ -0,0 +1,116 @@ +using System.Collections.Concurrent; +using Speckle.Sdk.Models; +using Speckle.Sdk.Serialisation.Utilities; +using Speckle.Sdk.Transports; + +namespace Speckle.Sdk.Serialisation.V2.Receive; + +public record DeserializeOptions(bool? SkipCacheCheck = null); + +public sealed class DeserializeProcess(IProgress? progress, IObjectLoader objectLoader) +{ + private readonly ConcurrentDictionary)> _closures = new(); + private long _total; + + public ConcurrentDictionary BaseCache { get; } = new(); + + public async Task Deserialize( + string rootId, + CancellationToken cancellationToken, + DeserializeOptions? options = null + ) + { + var (rootJson, childrenIds) = await objectLoader + .GetAndCache(rootId, cancellationToken, options) + .ConfigureAwait(false); + _total = childrenIds.Count; + _closures.TryAdd(rootId, (rootJson, childrenIds)); + progress?.Report(new(ProgressEvent.DeserializeObject, BaseCache.Count, childrenIds.Count)); + await Traverse(rootId, cancellationToken).ConfigureAwait(false); + return BaseCache[rootId]; + } + + private async Task Traverse(string id, CancellationToken cancellationToken) + { + if (BaseCache.ContainsKey(id)) + { + return; + } + var (_, childIds) = GetClosures(id); + var tasks = new List(); + foreach (var childId in childIds) + { + lock (BaseCache) + { + if (BaseCache.ContainsKey(childId)) + { + continue; + } + + // tmp is necessary because of the way closures close over loop variables + var tmpId = childId; + Task t = Task + .Factory.StartNew( + () => Traverse(tmpId, cancellationToken), + cancellationToken, + TaskCreationOptions.AttachedToParent, + TaskScheduler.Default + ) + .Unwrap(); + tasks.Add(t); + } + } + + if (tasks.Count > 0) + { + await Task.WhenAll(tasks).ConfigureAwait(false); + } + + //don't redo things if the id is decoded already in the cache + if (!BaseCache.ContainsKey(id)) + { + DecodeOrEnqueueChildren(id); + progress?.Report(new(ProgressEvent.DeserializeObject, BaseCache.Count, _total)); + } + } + + private (string, IReadOnlyList) GetClosures(string id) + { + if (!_closures.TryGetValue(id, out var closures)) + { + var json = objectLoader.LoadId(id); + if (json == null) + { + throw new InvalidOperationException(); + } + var childrenIds = ClosureParser.GetClosures(json).OrderByDescending(x => x.Item2).Select(x => x.Item1).ToList(); + closures = (json, childrenIds); + _closures.TryAdd(id, closures); + } + + return closures; + } + + public void DecodeOrEnqueueChildren(string id) + { + if (BaseCache.ContainsKey(id)) + { + return; + } + (string json, _) = GetClosures(id); + var @base = Deserialise(id, json); + BaseCache.TryAdd(id, @base); + //remove from JSON cache because we've finally made the Base + _closures.TryRemove(id, out _); + } + + private Base Deserialise(string id, string json) + { + if (BaseCache.TryGetValue(id, out var baseObject)) + { + return baseObject; + } + SpeckleObjectDeserializer2 deserializer = new(BaseCache, SpeckleObjectSerializerPool.Instance); + return deserializer.Deserialize(json); + } +} diff --git a/src/Speckle.Sdk/Serialisation/V2/Receive/DictionaryConverter.cs b/src/Speckle.Sdk/Serialisation/V2/Receive/DictionaryConverter.cs new file mode 100644 index 00000000..f1b1e07d --- /dev/null +++ b/src/Speckle.Sdk/Serialisation/V2/Receive/DictionaryConverter.cs @@ -0,0 +1,83 @@ +using System.Reflection; +using Speckle.Newtonsoft.Json; +using Speckle.Sdk.Common; +using Speckle.Sdk.Host; +using Speckle.Sdk.Models; +using Speckle.Sdk.Serialisation.Utilities; + +namespace Speckle.Sdk.Serialisation.V2.Receive; + +public static class DictionaryConverter +{ + /// + /// Property that describes the type of the object. + /// + public const string TYPE_DISCRIMINATOR = nameof(Base.speckle_type); + private static readonly object?[] s_invokeNull = [null]; + + public static string? BlobStorageFolder { get; set; } + + public static Base Dict2Base(Dictionary dictObj, bool skipInvalidConverts) + { + string typeName = (string)dictObj[TYPE_DISCRIMINATOR].NotNull(); + Type type = TypeLoader.GetType(typeName); + Base baseObj = (Base)Activator.CreateInstance(type).NotNull(); + + dictObj.Remove(TYPE_DISCRIMINATOR); + dictObj.Remove("__closure"); + + var staticProperties = TypeCache.GetTypeProperties(typeName); + foreach (var entry in dictObj) + { + if (staticProperties.TryGetValue(entry.Key, out PropertyInfo? value) && value.CanWrite) + { + if (entry.Value == null) + { + // Check for JsonProperty(NullValueHandling = NullValueHandling.Ignore) attribute + JsonPropertyAttribute? attr = TypeLoader.GetJsonPropertyAttribute(value); + if (attr is { NullValueHandling: NullValueHandling.Ignore }) + { + continue; + } + } + + Type targetValueType = value.PropertyType; + bool conversionOk = ValueConverter.ConvertValue( + targetValueType, + entry.Value, + skipInvalidConverts, + out object? convertedValue + ); + if (conversionOk) + { + value.SetValue(baseObj, convertedValue); + } + else + { + // Cannot convert the value in the json to the static property type + throw new SpeckleDeserializeException( + $"Cannot deserialize {entry.Value?.GetType().FullName} to {targetValueType.FullName}" + ); + } + } + else + { + // No writable property with this name + CallSiteCache.SetValue(entry.Key, baseObj, entry.Value); + } + } + + if (baseObj is Blob bb && BlobStorageFolder != null) + { + bb.filePath = bb.GetLocalDestinationPath(BlobStorageFolder); + } + + var onDeserializedCallbacks = TypeCache.GetOnDeserializedCallbacks(typeName); + foreach (MethodInfo onDeserialized in onDeserializedCallbacks) + { + onDeserialized.Invoke(baseObj, s_invokeNull); + } + + return baseObj; + } +} diff --git a/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectLoader.cs b/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectLoader.cs new file mode 100644 index 00000000..3c0916da --- /dev/null +++ b/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectLoader.cs @@ -0,0 +1,124 @@ +using System.Collections.Concurrent; +using Speckle.InterfaceGenerator; +using Speckle.Sdk.Common; +using Speckle.Sdk.Serialisation.Utilities; +using Speckle.Sdk.Transports; + +namespace Speckle.Sdk.Serialisation.V2.Receive; + +[GenerateAutoInterface] +public sealed class ObjectLoader( + ISQLiteCacheManager sqLiteCacheManager, + IServerObjectManager serverObjectManager, + string streamId, + IProgress? progress +) : IObjectLoader +{ + private const int HTTP_ID_CHUNK_SIZE = 500; + private const int CACHE_CHUNK_SIZE = 3000; + private const int MAX_PARALLELISM_HTTP = 4; + + public async Task<(string, IReadOnlyList)> GetAndCache( + string rootId, + CancellationToken cancellationToken, + DeserializeOptions? options = null + ) + { + var rootJson = sqLiteCacheManager.GetObject(rootId); + if (rootJson != null) + { + //assume everything exists as the root is there. + var allChildren = ClosureParser.GetChildrenIds(rootJson).ToList(); + return (rootJson, allChildren); + } + rootJson = await serverObjectManager + .DownloadSingleObject(streamId, rootId, progress, cancellationToken) + .NotNull() + .ConfigureAwait(false); + var allChildrenIds = ClosureParser + .GetClosures(rootJson) + .OrderByDescending(x => x.Item2) + .Select(x => x.Item1) + .Where(x => !x.StartsWith("blob", StringComparison.Ordinal)) + .ToList(); + if (!(options?.SkipCacheCheck ?? false)) + { + var idsToDownload = CheckCache(allChildrenIds); + await DownloadAndCache(idsToDownload, cancellationToken).ConfigureAwait(false); + } + //save the root last to shortcut later + sqLiteCacheManager.SaveObjectSync(rootId, rootJson); + return (rootJson, allChildrenIds); + } + + private async IAsyncEnumerable CheckCache(IReadOnlyList childrenIds) + { + var count = 0L; + progress?.Report(new(ProgressEvent.CacheCheck, count, childrenIds.Count)); + await foreach ( + var (id, result) in childrenIds + .Batch(CACHE_CHUNK_SIZE) + .Select(x => sqLiteCacheManager.HasObjects2(x)) // there needs to be a Task somewhere here + .SelectManyAsync() + ) + { + count++; + progress?.Report(new(ProgressEvent.CacheCheck, count, childrenIds.Count)); + if (!result) + { + yield return id; + } + } + } + + private async Task DownloadAndCache(IAsyncEnumerable ids, CancellationToken cancellationToken) + { + var count = 0L; + progress?.Report(new(ProgressEvent.DownloadObject, count, null)); + var toCache = new List<(string, string)>(); + var tasks = new ConcurrentBag(); + using SemaphoreSlim ss = new(MAX_PARALLELISM_HTTP, MAX_PARALLELISM_HTTP); + await foreach (var idBatch in ids.BatchAsync(HTTP_ID_CHUNK_SIZE).WithCancellation(cancellationToken)) + { + await ss.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + await foreach ( + var (id, json) in serverObjectManager.DownloadObjects(streamId, idBatch, progress, cancellationToken) + ) + { + count++; + progress?.Report(new(ProgressEvent.DownloadObject, count, null)); + toCache.Add((id, json)); + if (toCache.Count >= CACHE_CHUNK_SIZE) + { + var toSave = toCache; + toCache = new List<(string, string)>(); +#pragma warning disable CA2008 + tasks.Add( + Task.Factory.StartNew(() => sqLiteCacheManager.SaveObjects(toSave, cancellationToken), cancellationToken) + ); +#pragma warning restore CA2008 + } + } + } + finally + { + ss.Release(); + } + } + + if (toCache.Count > 0) + { +#pragma warning disable CA2008 + tasks.Add( + Task.Factory.StartNew(() => sqLiteCacheManager.SaveObjects(toCache, cancellationToken), cancellationToken) + ); +#pragma warning restore CA2008 + } + + await Task.WhenAll(tasks).ConfigureAwait(false); + } + + public string? LoadId(string id) => sqLiteCacheManager.GetObject(id); +} diff --git a/src/Speckle.Sdk/Serialisation/V2/Receive/SpeckleObjectDeserializer2.cs b/src/Speckle.Sdk/Serialisation/V2/Receive/SpeckleObjectDeserializer2.cs new file mode 100644 index 00000000..fb137d84 --- /dev/null +++ b/src/Speckle.Sdk/Serialisation/V2/Receive/SpeckleObjectDeserializer2.cs @@ -0,0 +1,160 @@ +using System.Numerics; +using Speckle.Newtonsoft.Json; +using Speckle.Sdk.Common; +using Speckle.Sdk.Models; + +namespace Speckle.Sdk.Serialisation.V2.Receive; + +public record DeserializedOptions(bool ThrowOnMissingReferences = true, bool SkipInvalidConverts = false); + +public sealed class SpeckleObjectDeserializer2( + IReadOnlyDictionary references, + SpeckleObjectSerializerPool pool, + DeserializedOptions? options = null +) +{ + /// The JSON string of the object to be deserialized + /// A typed object deserialized from the + /// was null + /// cannot be deserialised to type + // /// did not contain the required json objects (closures) + public Base Deserialize(string objectJson) + { + if (objectJson is null) + { + throw new ArgumentNullException(nameof(objectJson), $"Cannot deserialize {nameof(objectJson)}, value was null"); + } + // Apparently this automatically parses DateTimes in strings if it matches the format: + // JObject doc1 = JObject.Parse(objectJson); + + // This is equivalent code that doesn't parse datetimes: + using var stringReader = new StringReader(objectJson); + using JsonTextReader reader = pool.GetJsonTextReader(stringReader); + + reader.DateParseHandling = DateParseHandling.None; + + Base? converted; + try + { + reader.Read(); + converted = (Base)ReadObject(reader).NotNull(); + } + catch (Exception ex) when (!ex.IsFatal() && ex is not OperationCanceledException) + { + throw new SpeckleDeserializeException("Failed to deserialize", ex); + } + + return converted; + } + + private List ReadArrayAsync(JsonReader reader) + { + reader.Read(); + List retList = new(); + while (reader.TokenType != JsonToken.EndArray) + { + object? convertedValue = ReadProperty(reader); + if (convertedValue is DataChunk chunk) + { + retList.AddRange(chunk.data); + } + else + { + retList.Add(convertedValue); + } + reader.Read(); //goes to next + } + return retList; + } + + private object? ReadObject(JsonReader reader) + { + reader.Read(); + Dictionary dict = pool.ObjectDictionaries.Get(); + while (reader.TokenType != JsonToken.EndObject) + { + switch (reader.TokenType) + { + case JsonToken.PropertyName: + { + var propName = reader.Value.NotNull().ToString().NotNull(); + reader.Read(); //goes prop value + object? convertedValue = ReadProperty(reader); + dict[propName] = convertedValue; + reader.Read(); //goes to next + } + break; + default: + throw new InvalidOperationException($"Unknown {reader.ValueType} with {reader.Value}"); + } + } + + if (!dict.TryGetValue(DictionaryConverter.TYPE_DISCRIMINATOR, out object? speckleType)) + { + return dict; + } + + if (speckleType as string == "reference" && dict.TryGetValue("referencedId", out object? referencedId)) + { + var objId = (string)referencedId.NotNull(); + if (references.TryGetValue(objId, out Base? closure)) + { + return closure; + } + + if (options is null || options.ThrowOnMissingReferences) + { + throw new InvalidOperationException($"missing reference: {objId}"); + } + //since we don't throw on missing references, return null + return null; + } + + var b = DictionaryConverter.Dict2Base(dict, options?.SkipInvalidConverts ?? false); + pool.ObjectDictionaries.Return(dict); + return b; + } + + private object? ReadProperty(JsonReader reader) + { + switch (reader.TokenType) + { + case JsonToken.Undefined: + case JsonToken.Null: + case JsonToken.None: + return null; + case JsonToken.Boolean: + return (bool)reader.Value.NotNull(); + case JsonToken.Integer: + if (reader.Value is long longValue) + { + return longValue; + } + if (reader.Value is BigInteger bitInt) + { + // This is behaviour carried over from v2 to facilitate large numbers from Python + // This is quite hacky, as it's a bit questionable exactly what numbers are supported, and with what tolerance + // For this reason, this can be considered undocumented behaviour, and is only for values within the range of a 64bit integer. + return (double)bitInt; + } + + throw new ArgumentException( + $"Found an unsupported integer type {reader.Value?.GetType()} with value {reader.Value}" + ); + case JsonToken.Float: + return (double)reader.Value.NotNull(); + case JsonToken.String: + return (string?)reader.Value.NotNull(); + case JsonToken.Date: + return (DateTime)reader.Value.NotNull(); + case JsonToken.StartArray: + return ReadArrayAsync(reader); + case JsonToken.StartObject: + var dict = ReadObject(reader); + return dict; + + default: + throw new ArgumentException("Json value not supported: " + reader.ValueType); + } + } +} diff --git a/src/Speckle.Sdk/Serialisation/V2/Receive/ValueConverter.cs b/src/Speckle.Sdk/Serialisation/V2/Receive/ValueConverter.cs new file mode 100644 index 00000000..3269b2bc --- /dev/null +++ b/src/Speckle.Sdk/Serialisation/V2/Receive/ValueConverter.cs @@ -0,0 +1,330 @@ +using System.Collections; +using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.Contracts; +using System.Drawing; +using System.Globalization; +using Speckle.DoubleNumerics; +using Speckle.Sdk.Common; + +namespace Speckle.Sdk.Serialisation.V2.Receive; + +internal static class ValueConverter +{ + private static readonly object[] s_singleValue = new object[1]; + + [SuppressMessage( + "Maintainability", + "CA1502:Avoid excessive complexity", + Justification = "To fix this requires rewrite of serializaiton" + )] + public static bool ConvertValue(Type type, object? value, bool skipInvalidConverts, out object? convertedValue) + { + // TODO: Document list of supported values in the SDK. (and grow it as needed) + + convertedValue = null; + if (value == null) + { + return true; + } + + Type valueType = value.GetType(); + + if (type.IsAssignableFrom(valueType)) + { + convertedValue = value; + return true; + } + + //strings + if (type == typeof(string)) + { + convertedValue = Convert.ToString(value); + return true; + } + + #region Enum + if (type.IsEnum) + { + if (valueType != typeof(long)) + { + return false; + } + + convertedValue = Enum.ToObject(type, (long)value); + return true; + } + #endregion + + switch (type.Name) + { + case "Nullable`1": + return ConvertValue(type.GenericTypeArguments[0], value, skipInvalidConverts, out convertedValue); + #region Numbers + case "Int64": + if (valueType == typeof(long)) + { + convertedValue = (long)value; + return true; + } + + return false; + case "Int32": + if (valueType == typeof(long)) + { + convertedValue = (int)(long)value; + return true; + } + + return false; + case "Int16": + if (valueType == typeof(long)) + { + convertedValue = (short)(long)value; + return true; + } + + return false; + case "UInt64": + if (valueType == typeof(long)) + { + convertedValue = (ulong)(long)value; + return true; + } + + return false; + case "UInt32": + if (valueType == typeof(long)) + { + convertedValue = (uint)(long)value; + return true; + } + + return false; + case "UInt16": + if (valueType == typeof(long)) + { + convertedValue = (ushort)(long)value; + return true; + } + + return false; + case "Double": + if (valueType == typeof(double)) + { + convertedValue = (double)value; + return true; + } + if (valueType == typeof(long)) + { + convertedValue = (double)(long)value; + return true; + } + switch (value) + { + case "NaN": + convertedValue = double.NaN; + return true; + case "Infinity": + convertedValue = double.PositiveInfinity; + return true; + case "-Infinity": + convertedValue = double.NegativeInfinity; + return true; + default: + return false; + } + + case "Single": + if (valueType == typeof(double)) + { + convertedValue = (float)(double)value; + return true; + } + if (valueType == typeof(long)) + { + convertedValue = (float)(long)value; + return true; + } + switch (value) + { + case "NaN": + convertedValue = float.NaN; + return true; + case "Infinity": + convertedValue = float.PositiveInfinity; + return true; + case "-Infinity": + convertedValue = float.NegativeInfinity; + return true; + default: + return false; + } + + #endregion + } + + // Handle List<>, IList<>, and IReadOnlyList<> + if (type.IsGenericType && IsGenericList(type)) + { + if (value is not List valueList) + { + return false; + } + + var targetType = typeof(List<>).MakeGenericType(type.GenericTypeArguments); + Type listElementType = type.GenericTypeArguments[0]; + + s_singleValue[0] = valueList.Count; + //reuse array to avoid params array allocation + IList ret = (IList)Activator.CreateInstance(targetType, s_singleValue).NotNull(); + + foreach (object inputListElement in valueList) + { + if (!ConvertValue(listElementType, inputListElement, skipInvalidConverts, out object? convertedListElement)) + { + if (skipInvalidConverts) + { + continue; + } + return false; + } + + ret.Add(convertedListElement); + } + convertedValue = ret; + return true; + } + + // Handle Dictionary + if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Dictionary<,>)) + { + if (value is not Dictionary valueDict) + { + return false; + } + + if (type.GenericTypeArguments[0] != typeof(string)) + { + throw new ArgumentException("Dictionaries with non-string keys are not supported", nameof(type)); + } + + Type dictValueType = type.GenericTypeArguments[1]; + IDictionary ret = (IDictionary)Activator.CreateInstance(type).NotNull(); + + foreach (KeyValuePair kv in valueDict) + { + if (!ConvertValue(dictValueType, kv.Value, skipInvalidConverts, out object? convertedDictValue)) + { + if (skipInvalidConverts) + { + continue; + } + return false; + } + + ret[kv.Key] = convertedDictValue; + } + convertedValue = ret; + return true; + } + + // Handle arrays + if (type.IsArray) + { + if (value is not List valueList) + { + return false; + } + + Type arrayElementType = + type.GetElementType() ?? throw new ArgumentException("IsArray yet not valid element type", nameof(type)); + + Array ret = Array.CreateInstance(arrayElementType, valueList.Count); + for (int i = 0; i < valueList.Count; i++) + { + object inputListElement = valueList[i]; + if (!ConvertValue(arrayElementType, inputListElement, skipInvalidConverts, out object? convertedListElement)) + { + if (skipInvalidConverts) + { + continue; + } + return false; + } + + ret.SetValue(convertedListElement, i); + } + convertedValue = ret; + return true; + } + + // Handle simple classes/structs + if (type == typeof(Guid) && value is string str) + { + convertedValue = Guid.Parse(str); + return true; + } + + if (type == typeof(Color) && value is long integer) + { + convertedValue = Color.FromArgb((int)integer); + return true; + } + + if (type == typeof(DateTime) && value is string s) + { + convertedValue = DateTime.ParseExact(s, "o", CultureInfo.InvariantCulture); + return true; + } + + #region BACKWARDS COMPATIBILITY: matrix4x4 changed from System.Numerics float to System.DoubleNumerics double in release 2.16 + if (type == typeof(System.Numerics.Matrix4x4) && value is IReadOnlyList) + { + throw new ArgumentException("Only Speckle.DoubleNumerics.Matrix4x4 is supported.", nameof(type)); + } + #endregion + + if (type == typeof(Matrix4x4) && value is IReadOnlyList l) + { + double I(int index) => Convert.ToDouble(l[index]); + convertedValue = new Matrix4x4( + I(0), + I(1), + I(2), + I(3), + I(4), + I(5), + I(6), + I(7), + I(8), + I(9), + I(10), + I(11), + I(12), + I(13), + I(14), + I(15) + ); + return true; + } + + return false; + } + + /// + /// Tests that the given is assignable from a generic type def + /// + /// + /// + [Pure] + private static bool IsGenericList(Type type) + { + if (!type.IsGenericType) + { + return false; + } + + Type typeDef = type.GetGenericTypeDefinition(); + return typeDef == typeof(List<>) || typeDef == typeof(IList<>) || typeDef == typeof(IReadOnlyList<>); + } +} diff --git a/src/Speckle.Sdk/Serialisation/V2/SQLiteCacheManager.cs b/src/Speckle.Sdk/Serialisation/V2/SQLiteCacheManager.cs new file mode 100644 index 00000000..8bda8973 --- /dev/null +++ b/src/Speckle.Sdk/Serialisation/V2/SQLiteCacheManager.cs @@ -0,0 +1,156 @@ +using Microsoft.Data.Sqlite; +using Speckle.InterfaceGenerator; +using Speckle.Sdk.Logging; +using Speckle.Sdk.Transports; + +namespace Speckle.Sdk.Serialisation.V2; + +[GenerateAutoInterface] +public class SQLiteCacheManager : ISQLiteCacheManager +{ + private readonly string _rootPath; + private readonly string _connectionString; + private const string APPLICATION_NAME = "Speckle"; + private const string DATA_FOLDER = "Projects"; + + public SQLiteCacheManager(string streamId) + { + var basePath = SpecklePathProvider.UserApplicationDataPath(); + + try + { + var dir = Path.Combine(basePath, APPLICATION_NAME, DATA_FOLDER); + _rootPath = Path.Combine(dir, $"{streamId}.db"); + + Directory.CreateDirectory(dir); //ensure dir is there + } + catch (Exception ex) + when (ex is ArgumentException or IOException or UnauthorizedAccessException or NotSupportedException) + { + throw new TransportException($"Path was invalid or could not be created {_rootPath}", ex); + } + + _connectionString = $"Data Source={_rootPath};"; + Initialize(); + } + + private void Initialize() + { + // NOTE: used for creating partioned object tables. + //string[] HexChars = new string[] { "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f" }; + //var cart = new List(); + //foreach (var str in HexChars) + // foreach (var str2 in HexChars) + // cart.Add(str + str2); + + using var c = new SqliteConnection(_connectionString); + c.Open(); + const string COMMAND_TEXT = + @" + CREATE TABLE IF NOT EXISTS objects( + hash TEXT PRIMARY KEY, + content TEXT + ) WITHOUT ROWID; + "; + using (var command = new SqliteCommand(COMMAND_TEXT, c)) + { + command.ExecuteNonQuery(); + } + + // Insert Optimisations + + using SqliteCommand cmd0 = new("PRAGMA journal_mode='wal';", c); + cmd0.ExecuteNonQuery(); + + //Note / Hack: This setting has the potential to corrupt the db. + //cmd = new SqliteCommand("PRAGMA synchronous=OFF;", Connection); + //cmd.ExecuteNonQuery(); + + using SqliteCommand cmd1 = new("PRAGMA count_changes=OFF;", c); + cmd1.ExecuteNonQuery(); + + using SqliteCommand cmd2 = new("PRAGMA temp_store=MEMORY;", c); + cmd2.ExecuteNonQuery(); + } + + public IEnumerable<(string, string)> GetObjects(IEnumerable ids, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + using var c = new SqliteConnection(_connectionString); + c.Open(); + using var command = new SqliteCommand("SELECT content FROM objects WHERE hash = @hash LIMIT 1 ", c); + foreach (var id in ids) + { + command.Parameters.Clear(); + command.Parameters.AddWithValue("@hash", id); + using var reader = command.ExecuteReader(); + if (reader.Read()) + { + yield return (id, reader.GetString(0)); + } + } + } + + public void SaveObjects(IEnumerable<(string, string)> objects, CancellationToken cancellationToken) + { + using var c = new SqliteConnection(_connectionString); + c.Open(); + using var t = c.BeginTransaction(); + const string COMMAND_TEXT = "INSERT OR IGNORE INTO objects(hash, content) VALUES(@hash, @content)"; + + foreach (var (id, content) in objects) + { + using var command = new SqliteCommand(COMMAND_TEXT, c, t); + command.Parameters.AddWithValue("@hash", id); + command.Parameters.AddWithValue("@content", content); + command.ExecuteNonQuery(); + cancellationToken.ThrowIfCancellationRequested(); + } + + t.Commit(); + } + + public string? GetObject(string id) + { + using var c = new SqliteConnection(_connectionString); + c.Open(); + using var command = new SqliteCommand("SELECT * FROM objects WHERE hash = @hash LIMIT 1 ", c); + command.Parameters.AddWithValue("@hash", id); + using var reader = command.ExecuteReader(); + if (reader.Read()) + { + return reader.GetString(1); + } + return null; // pass on the duty of null checks to consumers + } + + public async IAsyncEnumerable<(string, bool)> HasObjects2(IEnumerable objectIds) + { + await Task.Delay(10).ConfigureAwait(false); + using var c = new SqliteConnection(_connectionString); + c.Open(); + const string COMMAND_TEXT = "SELECT 1 FROM objects WHERE hash = @hash LIMIT 1 "; + using var command = new SqliteCommand(COMMAND_TEXT, c); + foreach (string objectId in objectIds) + { + command.Parameters.Clear(); + command.Parameters.AddWithValue("@hash", objectId); + + using var reader = command.ExecuteReader(); + bool rowFound = reader.Read(); + yield return (objectId, rowFound); + } + } + + public void SaveObjectSync(string hash, string serializedObject) + { + using var c = new SqliteConnection(_connectionString); + c.Open(); + const string COMMAND_TEXT = "INSERT OR IGNORE INTO objects(hash, content) VALUES(@hash, @content)"; + + using var command = new SqliteCommand(COMMAND_TEXT, c); + command.Parameters.AddWithValue("@hash", hash); + command.Parameters.AddWithValue("@content", serializedObject); + command.ExecuteNonQuery(); + } +} diff --git a/src/Speckle.Sdk/Serialisation/V2/ServerObjectManager.cs b/src/Speckle.Sdk/Serialisation/V2/ServerObjectManager.cs new file mode 100644 index 00000000..b20ed9b2 --- /dev/null +++ b/src/Speckle.Sdk/Serialisation/V2/ServerObjectManager.cs @@ -0,0 +1,135 @@ +using System.Net; +using System.Runtime.CompilerServices; +using System.Text; +using Speckle.InterfaceGenerator; +using Speckle.Newtonsoft.Json; +using Speckle.Sdk.Helpers; +using Speckle.Sdk.Logging; +using Speckle.Sdk.Transports; + +namespace Speckle.Sdk.Serialisation.V2; + +[GenerateAutoInterface] +public class ServerObjectManager : IServerObjectManager +{ + private static readonly char[] s_separator = { '\t' }; + + private readonly ISdkActivityFactory _activityFactory; + private readonly HttpClient _client; + + public ServerObjectManager( + ISpeckleHttp speckleHttp, + ISdkActivityFactory activityFactory, + Uri baseUri, + string? authorizationToken, + int timeoutSeconds = 120 + ) + { + _activityFactory = activityFactory; + _client = speckleHttp.CreateHttpClient( + new HttpClientHandler { AutomaticDecompression = DecompressionMethods.GZip }, + timeoutSeconds: timeoutSeconds, + authorizationToken: authorizationToken + ); + _client.BaseAddress = baseUri; + } + + public async IAsyncEnumerable<(string, string)> DownloadObjects( + string streamId, + IReadOnlyList objectIds, + IProgress? progress, + [EnumeratorCancellation] CancellationToken cancellationToken + ) + { + using var _ = _activityFactory.Start(); + cancellationToken.ThrowIfCancellationRequested(); + + using var childrenHttpMessage = new HttpRequestMessage + { + RequestUri = new Uri($"/api/getobjects/{streamId}", UriKind.Relative), + Method = HttpMethod.Post, + }; + + Dictionary postParameters = new() { { "objects", JsonConvert.SerializeObject(objectIds) } }; + string serializedPayload = JsonConvert.SerializeObject(postParameters); + childrenHttpMessage.Content = new StringContent(serializedPayload, Encoding.UTF8, "application/json"); + childrenHttpMessage.Headers.Add("Accept", "text/plain"); + + HttpResponseMessage childrenHttpResponse = await _client + .SendAsync(childrenHttpMessage, cancellationToken) + .ConfigureAwait(false); + + await foreach (var (id, json) in ResponseProgress(childrenHttpResponse, progress, false, cancellationToken)) + { + if (id is not null) + { + yield return (id, json); + } + } + } + + public async Task DownloadSingleObject( + string streamId, + string objectId, + IProgress? progress, + CancellationToken cancellationToken + ) + { + using var _ = _activityFactory.Start(); + cancellationToken.ThrowIfCancellationRequested(); + + // Get root object + using var rootHttpMessage = new HttpRequestMessage + { + RequestUri = new Uri($"/objects/{streamId}/{objectId}/single", UriKind.Relative), + Method = HttpMethod.Get, + }; + + HttpResponseMessage rootHttpResponse = await _client + .SendAsync(rootHttpMessage, HttpCompletionOption.ResponseContentRead, cancellationToken) + .ConfigureAwait(false); + + var (_, json) = await ResponseProgress(rootHttpResponse, progress, true, cancellationToken) + .FirstAsync() + .ConfigureAwait(false); + return json; + } + + private async IAsyncEnumerable<(string?, string)> ResponseProgress( + HttpResponseMessage childrenHttpResponse, + IProgress? progress, + bool isSingle, + [EnumeratorCancellation] CancellationToken cancellationToken + ) + { + childrenHttpResponse.EnsureSuccessStatusCode(); + var length = childrenHttpResponse.Content.Headers.ContentLength; +#if NET8_0_OR_GREATER + using Stream childrenStream = await childrenHttpResponse + .Content.ReadAsStreamAsync(cancellationToken) + .ConfigureAwait(false); +#else + using Stream childrenStream = await childrenHttpResponse.Content.ReadAsStreamAsync().ConfigureAwait(false); +#endif + + using var reader = new StreamReader(new ProgressStream(childrenStream, length, progress, true), Encoding.UTF8); + +#if NET8_0_OR_GREATER + while (await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false) is { } line) +#else + while (await reader.ReadLineAsync().ConfigureAwait(false) is { } line) +#endif + { + cancellationToken.ThrowIfCancellationRequested(); + if (!isSingle) + { + var pcs = line.Split(s_separator, 2); + yield return (pcs[0], pcs[1]); + } + else + { + yield return (string.Empty, line); + } + } + } +} diff --git a/src/Speckle.Sdk/Speckle.Sdk.csproj b/src/Speckle.Sdk/Speckle.Sdk.csproj index fb32c1ba..68626500 100644 --- a/src/Speckle.Sdk/Speckle.Sdk.csproj +++ b/src/Speckle.Sdk/Speckle.Sdk.csproj @@ -30,6 +30,7 @@ + @@ -44,6 +45,7 @@ + diff --git a/src/Speckle.Sdk/Transports/ITransport.cs b/src/Speckle.Sdk/Transports/ITransport.cs index d6887c89..6d315966 100644 --- a/src/Speckle.Sdk/Transports/ITransport.cs +++ b/src/Speckle.Sdk/Transports/ITransport.cs @@ -2,23 +2,6 @@ using Speckle.Sdk.Models; namespace Speckle.Sdk.Transports; -public enum ProgressEvent -{ - DownloadBytes, - UploadBytes, - DownloadObject, - UploadObject, - DeserializeObject, - SerializeObject, -} - -public readonly record struct ProgressArgs( - ProgressEvent ProgressEvent, - long? Count, - long? Total, - long? ProcessedTotal = null -); - /// /// Interface defining the contract for transport implementations. /// diff --git a/src/Speckle.Sdk/Transports/ProgressArgs.cs b/src/Speckle.Sdk/Transports/ProgressArgs.cs new file mode 100644 index 00000000..4948cbca --- /dev/null +++ b/src/Speckle.Sdk/Transports/ProgressArgs.cs @@ -0,0 +1,14 @@ +namespace Speckle.Sdk.Transports; + +public readonly record struct ProgressArgs(ProgressEvent ProgressEvent, long Count, long? Total); + +public enum ProgressEvent +{ + CacheCheck, + DownloadBytes, + UploadBytes, + DownloadObject, + UploadObject, + DeserializeObject, + SerializeObject, +} diff --git a/src/Speckle.Sdk/Transports/SQLiteTransport.cs b/src/Speckle.Sdk/Transports/SQLiteTransport.cs index b9c949cb..e9742e83 100644 --- a/src/Speckle.Sdk/Transports/SQLiteTransport.cs +++ b/src/Speckle.Sdk/Transports/SQLiteTransport.cs @@ -404,24 +404,20 @@ public sealed class SQLiteTransport : IDisposable, ICloneable, ITransport, IBlob { CancellationToken.ThrowIfCancellationRequested(); await _connectionLock.WaitAsync(CancellationToken).ConfigureAwait(false); + var startTime = Stopwatch.GetTimestamp(); try { - var startTime = Stopwatch.GetTimestamp(); - using (var command = new SqliteCommand("SELECT * FROM objects WHERE hash = @hash LIMIT 1 ", Connection)) + using var command = new SqliteCommand("SELECT * FROM objects WHERE hash = @hash LIMIT 1 ", Connection); + command.Parameters.AddWithValue("@hash", id); + using var reader = command.ExecuteReader(); + if (reader.Read()) { - command.Parameters.AddWithValue("@hash", id); - using var reader = await command.ExecuteReaderAsync(CancellationToken).ConfigureAwait(false); - - while (await reader.ReadAsync(CancellationToken).ConfigureAwait(false)) - { - return reader.GetString(1); - } + return reader.GetString(1); } - - Elapsed += LoggingHelpers.GetElapsedTime(startTime, Stopwatch.GetTimestamp()); } finally { + Elapsed += LoggingHelpers.GetElapsedTime(startTime, Stopwatch.GetTimestamp()); _connectionLock.Release(); } return null; // pass on the duty of null checks to consumers diff --git a/src/Speckle.Sdk/Transports/ServerUtils/ServerAPI.cs b/src/Speckle.Sdk/Transports/ServerUtils/ServerAPI.cs index a2f4a309..78b268fc 100644 --- a/src/Speckle.Sdk/Transports/ServerUtils/ServerAPI.cs +++ b/src/Speckle.Sdk/Transports/ServerUtils/ServerAPI.cs @@ -381,9 +381,6 @@ public sealed class ServerApi : IDisposable, IServerApi { hasObjects[prop.Key] = (bool)prop.Value.NotNull(); } - - // Console.WriteLine($"ServerApi::HasObjects({objectIds.Count}) request in {sw.ElapsedMilliseconds / 1000.0} sec"); - return hasObjects; } diff --git a/src/Speckle.Sdk/packages.lock.json b/src/Speckle.Sdk/packages.lock.json index d7266b92..ff8d5db6 100644 --- a/src/Speckle.Sdk/packages.lock.json +++ b/src/Speckle.Sdk/packages.lock.json @@ -19,6 +19,15 @@ "System.Reactive": "5.0.0" } }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "Direct", + "requested": "[8.0.0, )", + "resolved": "8.0.0", + "contentHash": "3WA9q9yVqJp222P3x1wYIGDAkpjAku0TMUaaQV22g6L67AI0LdOIrVS7Ht2vJfLHGSPVuqN94vIr15qn+HEkHw==", + "dependencies": { + "System.Threading.Tasks.Extensions": "4.5.4" + } + }, "Microsoft.CSharp": { "type": "Direct", "requested": "[4.7.0, )", @@ -53,6 +62,12 @@ "Microsoft.Extensions.Options": "2.2.0" } }, + "Microsoft.Extensions.ObjectPool": { + "type": "Direct", + "requested": "[8.0.10, )", + "resolved": "8.0.10", + "contentHash": "u7gAG7JgxF8VSJUGPSudAcPxOt+ymJKQCSxNRxiuKV+klCQbHljQR75SilpedCTfhPWDhtUwIJpnDVtspr9nMg==" + }, "Microsoft.SourceLink.GitHub": { "type": "Direct", "requested": "[8.0.0, )", @@ -327,6 +342,12 @@ "System.Reactive": "5.0.0" } }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "Direct", + "requested": "[8.0.0, )", + "resolved": "8.0.0", + "contentHash": "3WA9q9yVqJp222P3x1wYIGDAkpjAku0TMUaaQV22g6L67AI0LdOIrVS7Ht2vJfLHGSPVuqN94vIr15qn+HEkHw==" + }, "Microsoft.CSharp": { "type": "Direct", "requested": "[4.7.0, )", @@ -361,6 +382,12 @@ "Microsoft.Extensions.Options": "2.2.0" } }, + "Microsoft.Extensions.ObjectPool": { + "type": "Direct", + "requested": "[8.0.10, )", + "resolved": "8.0.10", + "contentHash": "u7gAG7JgxF8VSJUGPSudAcPxOt+ymJKQCSxNRxiuKV+klCQbHljQR75SilpedCTfhPWDhtUwIJpnDVtspr9nMg==" + }, "Microsoft.SourceLink.GitHub": { "type": "Direct", "requested": "[8.0.0, )", diff --git a/tests/Speckle.Objects.Tests.Unit/packages.lock.json b/tests/Speckle.Objects.Tests.Unit/packages.lock.json index 74482024..68cc315a 100644 --- a/tests/Speckle.Objects.Tests.Unit/packages.lock.json +++ b/tests/Speckle.Objects.Tests.Unit/packages.lock.json @@ -274,10 +274,12 @@ "type": "Project", "dependencies": { "GraphQL.Client": "[6.0.0, )", + "Microsoft.Bcl.AsyncInterfaces": "[8.0.0, )", "Microsoft.CSharp": "[4.7.0, )", "Microsoft.Data.Sqlite": "[7.0.7, )", "Microsoft.Extensions.DependencyInjection.Abstractions": "[2.2.0, )", "Microsoft.Extensions.Logging": "[2.2.0, )", + "Microsoft.Extensions.ObjectPool": "[8.0.10, )", "Polly": "[7.2.3, )", "Polly.Contrib.WaitAndRetry": "[1.1.1, )", "Polly.Extensions.Http": "[3.0.0, )", @@ -296,6 +298,12 @@ "System.Reactive": "5.0.0" } }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "CentralTransitive", + "requested": "[8.0.0, )", + "resolved": "8.0.0", + "contentHash": "3WA9q9yVqJp222P3x1wYIGDAkpjAku0TMUaaQV22g6L67AI0LdOIrVS7Ht2vJfLHGSPVuqN94vIr15qn+HEkHw==" + }, "Microsoft.CSharp": { "type": "CentralTransitive", "requested": "[4.7.0, )", @@ -330,6 +338,12 @@ "Microsoft.Extensions.Options": "2.2.0" } }, + "Microsoft.Extensions.ObjectPool": { + "type": "CentralTransitive", + "requested": "[8.0.10, )", + "resolved": "8.0.10", + "contentHash": "u7gAG7JgxF8VSJUGPSudAcPxOt+ymJKQCSxNRxiuKV+klCQbHljQR75SilpedCTfhPWDhtUwIJpnDVtspr9nMg==" + }, "Polly": { "type": "CentralTransitive", "requested": "[7.2.3, )", diff --git a/tests/Speckle.Sdk.Serialization.Testing/Program.cs b/tests/Speckle.Sdk.Serialization.Testing/Program.cs new file mode 100644 index 00000000..8914c981 --- /dev/null +++ b/tests/Speckle.Sdk.Serialization.Testing/Program.cs @@ -0,0 +1,46 @@ +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Speckle.Sdk; +using Speckle.Sdk.Helpers; +using Speckle.Sdk.Host; +using Speckle.Sdk.Logging; +using Speckle.Sdk.Models; +using Speckle.Sdk.Serialisation.V2; +using Speckle.Sdk.Serialisation.V2.Receive; +using Speckle.Sdk.Serialization.Testing; + +const bool skipCache = false; +TypeLoader.Reset(); +TypeLoader.Initialize(typeof(Base).Assembly, Assembly.GetExecutingAssembly()); + +/* +var url = "https://latest.speckle.systems/projects/a3ac1b2706/models/59d3b0f3c6"; //small? +var streamId = "a3ac1b2706"; +var rootId = "7d53bcf28c6696ecac8781684a0aa006";*/ + + +var url = "https://latest.speckle.systems/projects/2099ac4b5f/models/da511c4d1e"; //perf? +var streamId = "2099ac4b5f"; +var rootId = "30fb4cbe6eb2202b9e7b4a4fcc3dd2b6"; + +var serviceCollection = new ServiceCollection(); +serviceCollection.AddSpeckleSdk(HostApplications.Navisworks, HostAppVersion.v2023, "Test"); +var serviceProvider = serviceCollection.BuildServiceProvider(); + +Console.WriteLine("Attach"); +Console.ReadLine(); +Console.WriteLine("Executing"); + +var progress = new Progress(true); +var sqliteTransport = new SQLiteCacheManager(streamId); +var serverObjects = new ServerObjectManager( + serviceProvider.GetRequiredService(), + serviceProvider.GetRequiredService(), + new Uri(url), + null +); +var o = new ObjectLoader(sqliteTransport, serverObjects, streamId, progress); +var process = new DeserializeProcess(progress, o); +await process.Deserialize(rootId, default, new(skipCache)).ConfigureAwait(false); +Console.WriteLine("Detach"); +Console.ReadLine(); diff --git a/tests/Speckle.Sdk.Serialization.Testing/Progress.cs b/tests/Speckle.Sdk.Serialization.Testing/Progress.cs new file mode 100644 index 00000000..b0a4f663 --- /dev/null +++ b/tests/Speckle.Sdk.Serialization.Testing/Progress.cs @@ -0,0 +1,36 @@ +using Speckle.Sdk.Transports; + +namespace Speckle.Sdk.Serialization.Testing; + +public class Progress(bool write) : IProgress +{ + private readonly TimeSpan DEBOUNCE = TimeSpan.FromMilliseconds(500); + private DateTime _lastTime = DateTime.UtcNow; + + private long _totalBytes; + + public void Report(ProgressArgs value) + { + if (write) + { + if (value.ProgressEvent == ProgressEvent.DownloadBytes) + { + Interlocked.Add(ref _totalBytes, value.Count); + } + var now = DateTime.UtcNow; + if (now - _lastTime >= DEBOUNCE) + { + if (value.ProgressEvent == ProgressEvent.DownloadBytes) + { + Console.WriteLine(value.ProgressEvent + " t " + _totalBytes); + } + else + { + Console.WriteLine(value.ProgressEvent + " c " + value.Count + " t " + value.Total); + } + + _lastTime = now; + } + } + } +} diff --git a/tests/Speckle.Sdk.Serialization.Testing/Speckle.Sdk.Serialization.Testing.csproj b/tests/Speckle.Sdk.Serialization.Testing/Speckle.Sdk.Serialization.Testing.csproj new file mode 100644 index 00000000..3d97bc78 --- /dev/null +++ b/tests/Speckle.Sdk.Serialization.Testing/Speckle.Sdk.Serialization.Testing.csproj @@ -0,0 +1,17 @@ + + + + Exe + net8.0 + enable + enable + + + + + + + + + + diff --git a/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json b/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json new file mode 100644 index 00000000..825f13db --- /dev/null +++ b/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json @@ -0,0 +1,472 @@ +{ + "version": 2, + "dependencies": { + "net8.0": { + "GitVersion.MsBuild": { + "type": "Direct", + "requested": "[5.12.0, )", + "resolved": "5.12.0", + "contentHash": "dJuigXycpJNOiLT9or7mkHSkGFHgGW3/p6cNNYEKZBa7Hhp1FdX/cvqYWWYhRLpfoZOedeA7aRbYiOB3vW/dvA==" + }, + "Microsoft.Extensions.DependencyInjection": { + "type": "Direct", + "requested": "[2.2.0, )", + "resolved": "2.2.0", + "contentHash": "MZtBIwfDFork5vfjpJdG5g8wuJFt7d/y3LOSVVtDK/76wlbtz6cjltfKHqLx2TKVqTj5/c41t77m1+h20zqtPA==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0" + } + }, + "Microsoft.SourceLink.GitHub": { + "type": "Direct", + "requested": "[8.0.0, )", + "resolved": "8.0.0", + "contentHash": "G5q7OqtwIyGTkeIOAc3u2ZuV/kicQaec5EaRnc0pIeSnh9LUjj+PYQrJYBURvDt7twGl2PKA7nSN0kz1Zw5bnQ==", + "dependencies": { + "Microsoft.Build.Tasks.Git": "8.0.0", + "Microsoft.SourceLink.Common": "8.0.0" + } + }, + "PolySharp": { + "type": "Direct", + "requested": "[1.14.1, )", + "resolved": "1.14.1", + "contentHash": "mOOmFYwad3MIOL14VCjj02LljyF1GNw1wP0YVlxtcPvqdxjGGMNdNJJxHptlry3MOd8b40Flm8RPOM8JOlN2sQ==" + }, + "Speckle.InterfaceGenerator": { + "type": "Direct", + "requested": "[0.9.6, )", + "resolved": "0.9.6", + "contentHash": "HKH7tYrYYlCK1ct483hgxERAdVdMtl7gUKW9ijWXxA1UsYR4Z+TrRHYmzZ9qmpu1NnTycSrp005NYM78GDKV1w==" + }, + "BenchmarkDotNet.Annotations": { + "type": "Transitive", + "resolved": "0.14.0", + "contentHash": "CUDCg6bgHrDzhjnA+IOBl5gAo8Y5hZ2YSs7MBXrYMlMKpBZqrD5ez0537uDveOkcf+YWAoK+S4sMcuWPbIz8bw==" + }, + "CommandLineParser": { + "type": "Transitive", + "resolved": "2.9.1", + "contentHash": "OE0sl1/sQ37bjVsPKKtwQlWDgqaxWgtme3xZz7JssWUzg5JpMIyHgCTY9MVMxOg48fJ1AgGT3tgdH5m/kQ5xhA==" + }, + "Gee.External.Capstone": { + "type": "Transitive", + "resolved": "2.3.0", + "contentHash": "2ap/rYmjtzCOT8hxrnEW/QeiOt+paD8iRrIcdKX0cxVwWLFa1e+JDBNeECakmccXrSFeBQuu5AV8SNkipFMMMw==" + }, + "GraphQL.Client.Abstractions": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "h7uzWFORHZ+CCjwr/ThAyXMr0DPpzEANDa4Uo54wqCQ+j7qUKwqYTgOrb1W40sqbvNaZm9v/X7It31SUw0maHA==", + "dependencies": { + "GraphQL.Primitives": "6.0.0" + } + }, + "GraphQL.Client.Abstractions.Websocket": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "Nr9bPf8gIOvLuXpqEpqr9z9jslYFJOvd0feHth3/kPqeR3uMbjF5pjiwh4jxyMcxHdr8Pb6QiXkV3hsSyt0v7A==", + "dependencies": { + "GraphQL.Client.Abstractions": "6.0.0" + } + }, + "GraphQL.Primitives": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "yg72rrYDapfsIUrul7aF6wwNnTJBOFvuA9VdDTQpPa8AlAriHbufeXYLBcodKjfUdkCnaiggX1U/nEP08Zb5GA==" + }, + "Iced": { + "type": "Transitive", + "resolved": "1.17.0", + "contentHash": "8x+HCVTl/HHTGpscH3vMBhV8sknN/muZFw9s3TsI8SA6+c43cOTCi2+jE4KsU8pNLbJ++iF2ZFcpcXHXtDglnw==" + }, + "Microsoft.Build.Tasks.Git": { + "type": "Transitive", + "resolved": "8.0.0", + "contentHash": "bZKfSIKJRXLTuSzLudMFte/8CempWjVamNUR5eHJizsy+iuOuO/k2gnh7W0dHJmYY0tBf+gUErfluCv5mySAOQ==" + }, + "Microsoft.CodeAnalysis.Analyzers": { + "type": "Transitive", + "resolved": "3.3.3", + "contentHash": "j/rOZtLMVJjrfLRlAMckJLPW/1rze9MT1yfWqSIbUPGRu1m1P0fuo9PmqapwsmePfGB5PJrudQLvmUOAMF0DqQ==" + }, + "Microsoft.CodeAnalysis.Common": { + "type": "Transitive", + "resolved": "4.1.0", + "contentHash": "bNzTyxP3iD5FPFHfVDl15Y6/wSoI7e3MeV0lOaj9igbIKTjgrmuw6LoVJ06jUNFA7+KaDC/OIsStWl/FQJz6sQ==", + "dependencies": { + "Microsoft.CodeAnalysis.Analyzers": "3.3.3", + "System.Collections.Immutable": "5.0.0", + "System.Memory": "4.5.4", + "System.Reflection.Metadata": "5.0.0", + "System.Runtime.CompilerServices.Unsafe": "5.0.0", + "System.Text.Encoding.CodePages": "4.5.1", + "System.Threading.Tasks.Extensions": "4.5.4" + } + }, + "Microsoft.CodeAnalysis.CSharp": { + "type": "Transitive", + "resolved": "4.1.0", + "contentHash": "sbu6kDGzo9bfQxuqWpeEE7I9P30bSuZEnpDz9/qz20OU6pm79Z63+/BsAzO2e/R/Q97kBrpj647wokZnEVr97w==", + "dependencies": { + "Microsoft.CodeAnalysis.Common": "[4.1.0]" + } + }, + "Microsoft.Data.Sqlite.Core": { + "type": "Transitive", + "resolved": "7.0.7", + "contentHash": "21FRzcJhaTrlv7kTrqr/ltFcSQM2TyuTTPhUcjO8H73od7Bb3QraNW90c7lUucNI/245XPkKZG4fp7/7OsKCSg==", + "dependencies": { + "SQLitePCLRaw.core": "2.1.4" + } + }, + "Microsoft.Diagnostics.NETCore.Client": { + "type": "Transitive", + "resolved": "0.2.251802", + "contentHash": "bqnYl6AdSeboeN4v25hSukK6Odm6/54E3Y2B8rBvgqvAW0mF8fo7XNRVE2DMOG7Rk0fiuA079QIH28+V+W1Zdg==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "1.1.0", + "Microsoft.Extensions.Logging": "2.1.1" + } + }, + "Microsoft.Diagnostics.Runtime": { + "type": "Transitive", + "resolved": "2.2.332302", + "contentHash": "Hp84ivxSKIMTBzYSATxmUsm3YSXHWivcwiRRbsydGmqujMUK8BAueLN0ssAVEOkOBmh0vjUBhrq7YcroT7VCug==", + "dependencies": { + "Microsoft.Diagnostics.NETCore.Client": "0.2.251802", + "System.Collections.Immutable": "5.0.0", + "System.Runtime.CompilerServices.Unsafe": "5.0.0" + } + }, + "Microsoft.Diagnostics.Tracing.TraceEvent": { + "type": "Transitive", + "resolved": "3.1.8", + "contentHash": "kl3UMrZKSeSEYZ8rt/GjLUQToREjgQABqfg6PzQBmSlYHTZOKE9ePEOS2xptROQ9SVvngg3QGX51TIT11iZ0wA==", + "dependencies": { + "Microsoft.Win32.Registry": "4.4.0", + "System.Runtime.CompilerServices.Unsafe": "5.0.0" + } + }, + "Microsoft.DotNet.PlatformAbstractions": { + "type": "Transitive", + "resolved": "3.1.6", + "contentHash": "jek4XYaQ/PGUwDKKhwR8K47Uh1189PFzMeLqO83mXrXQVIpARZCcfuDedH50YDTepBkfijCZN5U/vZi++erxtg==" + }, + "Microsoft.Extensions.Configuration": { + "type": "Transitive", + "resolved": "2.2.0", + "contentHash": "nOP8R1mVb/6mZtm2qgAJXn/LFm/2kMjHDAg/QJLFG6CuWYJtaD3p1BwQhufBVvRzL9ceJ/xF0SQ0qsI2GkDQAA==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "2.2.0" + } + }, + "Microsoft.Extensions.Configuration.Abstractions": { + "type": "Transitive", + "resolved": "2.2.0", + "contentHash": "65MrmXCziWaQFrI0UHkQbesrX5wTwf9XPjY5yFm/VkgJKFJ5gqvXRoXjIZcf2wLi5ZlwGz/oMYfyURVCWbM5iw==", + "dependencies": { + "Microsoft.Extensions.Primitives": "2.2.0" + } + }, + "Microsoft.Extensions.Configuration.Binder": { + "type": "Transitive", + "resolved": "2.2.0", + "contentHash": "vJ9xvOZCnUAIHcGC3SU35r3HKmHTVIeHzo6u/qzlHAqD8m6xv92MLin4oJntTvkpKxVX3vI1GFFkIQtU3AdlsQ==", + "dependencies": { + "Microsoft.Extensions.Configuration": "2.2.0" + } + }, + "Microsoft.Extensions.Logging.Abstractions": { + "type": "Transitive", + "resolved": "2.2.0", + "contentHash": "B2WqEox8o+4KUOpL7rZPyh6qYjik8tHi2tN8Z9jZkHzED8ElYgZa/h6K+xliB435SqUcWT290Fr2aa8BtZjn8A==" + }, + "Microsoft.Extensions.Options": { + "type": "Transitive", + "resolved": "2.2.0", + "contentHash": "UpZLNLBpIZ0GTebShui7xXYh6DmBHjWM8NxGxZbdQh/bPZ5e6YswqI+bru6BnEL5eWiOdodsXtEz3FROcgi/qg==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0", + "Microsoft.Extensions.Primitives": "2.2.0", + "System.ComponentModel.Annotations": "4.5.0" + } + }, + "Microsoft.Extensions.Primitives": { + "type": "Transitive", + "resolved": "2.2.0", + "contentHash": "azyQtqbm4fSaDzZHD/J+V6oWMFaf2tWP4WEGIYePLCMw3+b2RQdj9ybgbQyjCshcitQKQ4lEDOZjmSlTTrHxUg==", + "dependencies": { + "System.Memory": "4.5.1", + "System.Runtime.CompilerServices.Unsafe": "4.5.1" + } + }, + "Microsoft.NETCore.Platforms": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "VyPlqzH2wavqquTcYpkIIAQ6WdenuKoFN0BdYBbCWsclXacSOHNQn66Gt4z5NBqEYW0FAPm5rlvki9ZiCij5xQ==" + }, + "Microsoft.SourceLink.Common": { + "type": "Transitive", + "resolved": "8.0.0", + "contentHash": "dk9JPxTCIevS75HyEQ0E4OVAFhB2N+V9ShCXf8Q6FkUQZDkgLI12y679Nym1YqsiSysuQskT7Z+6nUf3yab6Vw==" + }, + "Microsoft.Win32.Registry": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "dDoKi0PnDz31yAyETfRntsLArTlVAVzUzCIvvEDsDsucrl33Dl8pIJG06ePTJTI3tGpeyHS9Cq7Foc/s4EeKcg==", + "dependencies": { + "System.Security.AccessControl": "5.0.0", + "System.Security.Principal.Windows": "5.0.0" + } + }, + "Perfolizer": { + "type": "Transitive", + "resolved": "0.3.17", + "contentHash": "FQgtCoF2HFwvzKWulAwBS5BGLlh8pgbrJtOp47jyBwh2CW16juVtacN1azOA2BqdrJXkXTNLNRMo7ZlHHiuAnA==" + }, + "SQLitePCLRaw.bundle_e_sqlite3": { + "type": "Transitive", + "resolved": "2.1.4", + "contentHash": "EWI1olKDjFEBMJu0+3wuxwziIAdWDVMYLhuZ3Qs84rrz+DHwD00RzWPZCa+bLnHCf3oJwuFZIRsHT5p236QXww==", + "dependencies": { + "SQLitePCLRaw.lib.e_sqlite3": "2.1.4", + "SQLitePCLRaw.provider.e_sqlite3": "2.1.4" + } + }, + "SQLitePCLRaw.core": { + "type": "Transitive", + "resolved": "2.1.4", + "contentHash": "inBjvSHo9UDKneGNzfUfDjK08JzlcIhn1+SP5Y3m6cgXpCxXKCJDy6Mka7LpgSV+UZmKSnC8rTwB0SQ0xKu5pA==", + "dependencies": { + "System.Memory": "4.5.3" + } + }, + "SQLitePCLRaw.lib.e_sqlite3": { + "type": "Transitive", + "resolved": "2.1.4", + "contentHash": "2C9Q9eX7CPLveJA0rIhf9RXAvu+7nWZu1A2MdG6SD/NOu26TakGgL1nsbc0JAspGijFOo3HoN79xrx8a368fBg==" + }, + "SQLitePCLRaw.provider.e_sqlite3": { + "type": "Transitive", + "resolved": "2.1.4", + "contentHash": "CSlb5dUp1FMIkez9Iv5EXzpeq7rHryVNqwJMWnpq87j9zWZexaEMdisDktMsnnrzKM6ahNrsTkjqNodTBPBxtQ==", + "dependencies": { + "SQLitePCLRaw.core": "2.1.4" + } + }, + "System.CodeDom": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "JPJArwA1kdj8qDAkY2XGjSWoYnqiM7q/3yRNkt6n28Mnn95MuEGkZXUbPBf7qc3IjwrGY5ttQon7yqHZyQJmOQ==" + }, + "System.Collections.Immutable": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "FXkLXiK0sVVewcso0imKQoOxjoPAj42R8HtjjbSjVPAzwDfzoyoznWxgA3c38LDbN9SJux1xXoXYAhz98j7r2g==" + }, + "System.ComponentModel.Annotations": { + "type": "Transitive", + "resolved": "4.5.0", + "contentHash": "UxYQ3FGUOtzJ7LfSdnYSFd7+oEv6M8NgUatatIN2HxNtDdlcvFAf+VIq4Of9cDMJEJC0aSRv/x898RYhB4Yppg==" + }, + "System.Management": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "MF1CHaRcC+MLFdnDthv4/bKWBZnlnSpkGqa87pKukQefgEdwtb9zFW6zs0GjPp73qtpYYg4q6PEKbzJbxCpKfw==", + "dependencies": { + "Microsoft.NETCore.Platforms": "5.0.0", + "Microsoft.Win32.Registry": "5.0.0", + "System.CodeDom": "5.0.0" + } + }, + "System.Memory": { + "type": "Transitive", + "resolved": "4.5.4", + "contentHash": "1MbJTHS1lZ4bS4FmsJjnuGJOu88ZzTT2rLvrhW7Ygic+pC0NWA+3hgAen0HRdsocuQXCkUTdFn9yHJJhsijDXw==" + }, + "System.Reactive": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "erBZjkQHWL9jpasCE/0qKAryzVBJFxGHVBAvgRN1bzM0q2s1S4oYREEEL0Vb+1kA/6BKb5FjUZMp5VXmy+gzkQ==" + }, + "System.Reflection.Metadata": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "5NecZgXktdGg34rh1OenY1rFNDCI8xSjFr+Z4OU4cU06AQHUdRnIIEeWENu3Wl4YowbzkymAIMvi3WyK9U53pQ==" + }, + "System.Runtime.CompilerServices.Unsafe": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "ZD9TMpsmYJLrxbbmdvhwt9YEgG5WntEnZ/d1eH8JBX9LBp+Ju8BSBhUGbZMNVHHomWo2KVImJhTDl2hIgw/6MA==" + }, + "System.Security.AccessControl": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "dagJ1mHZO3Ani8GH0PHpPEe/oYO+rVdbQjvjJkBRNQkX4t0r1iaeGn8+/ybkSLEan3/slM0t59SVdHzuHf2jmw==", + "dependencies": { + "Microsoft.NETCore.Platforms": "5.0.0", + "System.Security.Principal.Windows": "5.0.0" + } + }, + "System.Security.Principal.Windows": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "t0MGLukB5WAVU9bO3MGzvlGnyJPgUlcwerXn1kzBRjwLKixT96XV0Uza41W49gVd8zEMFu9vQEFlv0IOrytICA==" + }, + "System.Text.Encoding.CodePages": { + "type": "Transitive", + "resolved": "4.5.1", + "contentHash": "4J2JQXbftjPMppIHJ7IC+VXQ9XfEagN92vZZNoG12i+zReYlim5dMoXFC1Zzg7tsnKDM7JPo5bYfFK4Jheq44w==", + "dependencies": { + "Microsoft.NETCore.Platforms": "2.1.2", + "System.Runtime.CompilerServices.Unsafe": "4.5.2" + } + }, + "System.Threading.Tasks.Extensions": { + "type": "Transitive", + "resolved": "4.5.4", + "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==" + }, + "speckle.objects": { + "type": "Project", + "dependencies": { + "Speckle.Sdk": "[1.0.0, )" + } + }, + "speckle.sdk": { + "type": "Project", + "dependencies": { + "GraphQL.Client": "[6.0.0, )", + "Microsoft.Bcl.AsyncInterfaces": "[8.0.0, )", + "Microsoft.CSharp": "[4.7.0, )", + "Microsoft.Data.Sqlite": "[7.0.7, )", + "Microsoft.Extensions.DependencyInjection.Abstractions": "[2.2.0, )", + "Microsoft.Extensions.Logging": "[2.2.0, )", + "Microsoft.Extensions.ObjectPool": "[8.0.10, )", + "Polly": "[7.2.3, )", + "Polly.Contrib.WaitAndRetry": "[1.1.1, )", + "Polly.Extensions.Http": "[3.0.0, )", + "Speckle.DoubleNumerics": "[4.0.1, )", + "Speckle.Newtonsoft.Json": "[13.0.2, )" + } + }, + "speckle.sdk.tests.performance": { + "type": "Project", + "dependencies": { + "BenchmarkDotNet": "[0.14.0, )", + "Microsoft.Extensions.DependencyInjection": "[2.2.0, )", + "Speckle.Objects": "[1.0.0, )" + } + }, + "BenchmarkDotNet": { + "type": "CentralTransitive", + "requested": "[0.14.0, )", + "resolved": "0.14.0", + "contentHash": "eIPSDKi3oni734M1rt/XJAwGQQOIf9gLjRRKKJ0HuVy3vYd7gnmAIX1bTjzI9ZbAY/nPddgqqgM/TeBYitMCIg==", + "dependencies": { + "BenchmarkDotNet.Annotations": "0.14.0", + "CommandLineParser": "2.9.1", + "Gee.External.Capstone": "2.3.0", + "Iced": "1.17.0", + "Microsoft.CodeAnalysis.CSharp": "4.1.0", + "Microsoft.Diagnostics.Runtime": "2.2.332302", + "Microsoft.Diagnostics.Tracing.TraceEvent": "3.1.8", + "Microsoft.DotNet.PlatformAbstractions": "3.1.6", + "Perfolizer": "[0.3.17]", + "System.Management": "5.0.0" + } + }, + "GraphQL.Client": { + "type": "CentralTransitive", + "requested": "[6.0.0, )", + "resolved": "6.0.0", + "contentHash": "8yPNBbuVBpTptivyAlak4GZvbwbUcjeQTL4vN1HKHRuOykZ4r7l5fcLS6vpyPyLn0x8FsL31xbOIKyxbmR9rbA==", + "dependencies": { + "GraphQL.Client.Abstractions": "6.0.0", + "GraphQL.Client.Abstractions.Websocket": "6.0.0", + "System.Reactive": "5.0.0" + } + }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "CentralTransitive", + "requested": "[8.0.0, )", + "resolved": "8.0.0", + "contentHash": "3WA9q9yVqJp222P3x1wYIGDAkpjAku0TMUaaQV22g6L67AI0LdOIrVS7Ht2vJfLHGSPVuqN94vIr15qn+HEkHw==" + }, + "Microsoft.CSharp": { + "type": "CentralTransitive", + "requested": "[4.7.0, )", + "resolved": "4.7.0", + "contentHash": "pTj+D3uJWyN3My70i2Hqo+OXixq3Os2D1nJ2x92FFo6sk8fYS1m1WLNTs0Dc1uPaViH0YvEEwvzddQ7y4rhXmA==" + }, + "Microsoft.Data.Sqlite": { + "type": "CentralTransitive", + "requested": "[7.0.7, )", + "resolved": "7.0.7", + "contentHash": "tiNmV1oPy+Z2R7Wd0bPB/FxCr8B+/5q11OpDMG751GA/YuOL7MZrBFfzv5oFRlFe08K6sjrnbrauzzGIeNrzLQ==", + "dependencies": { + "Microsoft.Data.Sqlite.Core": "7.0.7", + "SQLitePCLRaw.bundle_e_sqlite3": "2.1.4" + } + }, + "Microsoft.Extensions.DependencyInjection.Abstractions": { + "type": "CentralTransitive", + "requested": "[2.2.0, )", + "resolved": "2.2.0", + "contentHash": "f9hstgjVmr6rmrfGSpfsVOl2irKAgr1QjrSi3FgnS7kulxband50f2brRLwySAQTADPZeTdow0mpSMcoAdadCw==" + }, + "Microsoft.Extensions.Logging": { + "type": "CentralTransitive", + "requested": "[2.2.0, )", + "resolved": "2.2.0", + "contentHash": "Nxqhadc9FCmFHzU+fz3oc8sFlE6IadViYg8dfUdGzJZ2JUxnCsRghBhhOWdM4B2zSZqEc+0BjliBh/oNdRZuig==", + "dependencies": { + "Microsoft.Extensions.Configuration.Binder": "2.2.0", + "Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0", + "Microsoft.Extensions.Logging.Abstractions": "2.2.0", + "Microsoft.Extensions.Options": "2.2.0" + } + }, + "Microsoft.Extensions.ObjectPool": { + "type": "CentralTransitive", + "requested": "[8.0.10, )", + "resolved": "8.0.10", + "contentHash": "u7gAG7JgxF8VSJUGPSudAcPxOt+ymJKQCSxNRxiuKV+klCQbHljQR75SilpedCTfhPWDhtUwIJpnDVtspr9nMg==" + }, + "Polly": { + "type": "CentralTransitive", + "requested": "[7.2.3, )", + "resolved": "7.2.3", + "contentHash": "DeCY0OFbNdNxsjntr1gTXHJ5pKUwYzp04Er2LLeN3g6pWhffsGuKVfMBLe1lw7x76HrPkLxKEFxBlpRxS2nDEQ==" + }, + "Polly.Contrib.WaitAndRetry": { + "type": "CentralTransitive", + "requested": "[1.1.1, )", + "resolved": "1.1.1", + "contentHash": "1MUQLiSo4KDkQe6nzQRhIU05lm9jlexX5BVsbuw0SL82ynZ+GzAHQxJVDPVBboxV37Po3SG077aX8DuSy8TkaA==" + }, + "Polly.Extensions.Http": { + "type": "CentralTransitive", + "requested": "[3.0.0, )", + "resolved": "3.0.0", + "contentHash": "drrG+hB3pYFY7w1c3BD+lSGYvH2oIclH8GRSehgfyP5kjnFnHKQuuBhuHLv+PWyFuaTDyk/vfRpnxOzd11+J8g==", + "dependencies": { + "Polly": "7.1.0" + } + }, + "Speckle.DoubleNumerics": { + "type": "CentralTransitive", + "requested": "[4.0.1, )", + "resolved": "4.0.1", + "contentHash": "MzEQ1Im0zTja+tEsdRIk/WlPiKqb22NmTOJcR1ZKm/mz46pezyyID3/wRz6vJUELMpSLnG7LhsxBL+nxbr7V0w==" + }, + "Speckle.Newtonsoft.Json": { + "type": "CentralTransitive", + "requested": "[13.0.2, )", + "resolved": "13.0.2", + "contentHash": "g1BejUZwax5PRfL6xHgLEK23sqHWOgOj9hE7RvfRRlN00AGt8GnPYt8HedSK7UB3HiRW8zCA9Pn0iiYxCK24BA==" + } + } + } +} \ No newline at end of file diff --git a/tests/Speckle.Sdk.Serialization.Tests/SerializationTests.cs b/tests/Speckle.Sdk.Serialization.Tests/SerializationTests.cs index 90c294c0..ee3f3379 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/SerializationTests.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/SerializationTests.cs @@ -1,5 +1,4 @@ -using System.Reflection; -using Microsoft.Extensions.Logging.Abstractions; +using System.Reflection; using NUnit.Framework; using Shouldly; using Speckle.Newtonsoft.Json.Linq; @@ -8,6 +7,8 @@ using Speckle.Sdk.Common; using Speckle.Sdk.Host; using Speckle.Sdk.Models; using Speckle.Sdk.Serialisation; +using Speckle.Sdk.Serialisation.Utilities; +using Speckle.Sdk.Serialisation.V2.Receive; namespace Speckle.Sdk.Serialization.Tests; @@ -15,6 +16,21 @@ namespace Speckle.Sdk.Serialization.Tests; [Description("For certain types, changing property from one type to another should be implicitly backwards compatible")] public class SerializationTests { + private class TestLoader(string json) : IObjectLoader + { + public Task<(string, IReadOnlyList)> GetAndCache( + string rootId, + CancellationToken cancellationToken, + DeserializeOptions? options = null + ) + { + var childrenIds = ClosureParser.GetChildrenIds(json).ToList(); + return Task.FromResult<(string, IReadOnlyList)>((json, childrenIds)); + } + + public string? LoadId(string id) => null; + } + private readonly Assembly _assembly = Assembly.GetExecutingAssembly(); [SetUp] @@ -31,10 +47,9 @@ public class SerializationTests return await reader.ReadToEndAsync(); } - private async Task> ReadAsObjects(string fullName) + private Dictionary ReadAsObjects(string json) { var jsonObjects = new Dictionary(); - var json = await ReadJson(fullName); var array = JArray.Parse(json); foreach (var obj in array) { @@ -46,17 +61,53 @@ public class SerializationTests return jsonObjects; } + /* + [Test] + [TestCase("RevitObject.json")] + public async Task RunTest2(string fileName) + { + var fullName = _assembly.GetManifestResourceNames().Single(x => x.EndsWith(fileName)); + var json = await ReadJson(fullName); + var closure = await ReadAsObjects(json); + using DeserializeProcess sut = new(null, new TestLoader(json), new TestTransport(closure)); + var @base = await sut.Deserialize("551513ff4f3596024547fc818f1f3f70"); + @base.ShouldNotBeNull(); + }*/ + + public class TestObjectLoader(Dictionary idToObject) : IObjectLoader + { + public Task<(string, IReadOnlyList)> GetAndCache( + string rootId, + CancellationToken cancellationToken, + DeserializeOptions? options = default + ) + { + var json = idToObject.GetValueOrDefault(rootId); + if (json == null) + { + throw new KeyNotFoundException("Root not found"); + } + + var allChildren = ClosureParser.GetChildrenIds(json).ToList(); + return Task.FromResult<(string, IReadOnlyList)>((json, allChildren)); + } + + public string? LoadId(string id) => idToObject.GetValueOrDefault(id); + } + [Test] [TestCase("RevitObject.json")] public async Task Basic_Namespace_Validation(string fileName) { var fullName = _assembly.GetManifestResourceNames().Single(x => x.EndsWith(fileName)); - var closure = await ReadAsObjects(fullName); + var json = await ReadJson(fullName); + var closure = ReadAsObjects(json); var deserializer = new SpeckleObjectDeserializer { ReadTransport = new TestTransport(closure), CancellationToken = default, }; + foreach (var (id, objJson) in closure) { var jObject = JObject.Parse(objJson); @@ -77,4 +128,33 @@ public class SerializationTests starts.ShouldBeTrue($"{name} isn't expected"); } } + + [Test] + [TestCase("RevitObject.json")] + public async Task Basic_Namespace_Validation_New(string fileName) + { + var fullName = _assembly.GetManifestResourceNames().Single(x => x.EndsWith(fileName)); + var json = await ReadJson(fullName); + var closures = ReadAsObjects(json); + var process = new DeserializeProcess(null, new TestObjectLoader(closures)); + await process.Deserialize("551513ff4f3596024547fc818f1f3f70", default); + foreach (var (id, objJson) in closures) + { + var jObject = JObject.Parse(objJson); + var oldSpeckleType = jObject["speckle_type"].NotNull().Value().NotNull(); + var starts = oldSpeckleType.StartsWith("Speckle.Core.") || oldSpeckleType.StartsWith("Objects."); + starts.ShouldBeTrue($"{oldSpeckleType} isn't expected"); + + var baseType = process.BaseCache[id]; + + starts = baseType.speckle_type.StartsWith("Speckle.Core.") || baseType.speckle_type.StartsWith("Objects."); + starts.ShouldBeTrue($"{baseType.speckle_type} isn't expected"); + + var type = TypeLoader.GetAtomicType(baseType.speckle_type); + type.ShouldNotBeNull(); + var name = TypeLoader.GetTypeString(type) ?? throw new ArgumentNullException(); + starts = name.StartsWith("Speckle.Core") || name.StartsWith("Objects"); + starts.ShouldBeTrue($"{name} isn't expected"); + } + } } diff --git a/tests/Speckle.Sdk.Serialization.Tests/packages.lock.json b/tests/Speckle.Sdk.Serialization.Tests/packages.lock.json index 74482024..68cc315a 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/packages.lock.json +++ b/tests/Speckle.Sdk.Serialization.Tests/packages.lock.json @@ -274,10 +274,12 @@ "type": "Project", "dependencies": { "GraphQL.Client": "[6.0.0, )", + "Microsoft.Bcl.AsyncInterfaces": "[8.0.0, )", "Microsoft.CSharp": "[4.7.0, )", "Microsoft.Data.Sqlite": "[7.0.7, )", "Microsoft.Extensions.DependencyInjection.Abstractions": "[2.2.0, )", "Microsoft.Extensions.Logging": "[2.2.0, )", + "Microsoft.Extensions.ObjectPool": "[8.0.10, )", "Polly": "[7.2.3, )", "Polly.Contrib.WaitAndRetry": "[1.1.1, )", "Polly.Extensions.Http": "[3.0.0, )", @@ -296,6 +298,12 @@ "System.Reactive": "5.0.0" } }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "CentralTransitive", + "requested": "[8.0.0, )", + "resolved": "8.0.0", + "contentHash": "3WA9q9yVqJp222P3x1wYIGDAkpjAku0TMUaaQV22g6L67AI0LdOIrVS7Ht2vJfLHGSPVuqN94vIr15qn+HEkHw==" + }, "Microsoft.CSharp": { "type": "CentralTransitive", "requested": "[4.7.0, )", @@ -330,6 +338,12 @@ "Microsoft.Extensions.Options": "2.2.0" } }, + "Microsoft.Extensions.ObjectPool": { + "type": "CentralTransitive", + "requested": "[8.0.10, )", + "resolved": "8.0.10", + "contentHash": "u7gAG7JgxF8VSJUGPSudAcPxOt+ymJKQCSxNRxiuKV+klCQbHljQR75SilpedCTfhPWDhtUwIJpnDVtspr9nMg==" + }, "Polly": { "type": "CentralTransitive", "requested": "[7.2.3, )", diff --git a/tests/Speckle.Sdk.Tests.Integration/packages.lock.json b/tests/Speckle.Sdk.Tests.Integration/packages.lock.json index c7a8859c..e220ce73 100644 --- a/tests/Speckle.Sdk.Tests.Integration/packages.lock.json +++ b/tests/Speckle.Sdk.Tests.Integration/packages.lock.json @@ -268,10 +268,12 @@ "type": "Project", "dependencies": { "GraphQL.Client": "[6.0.0, )", + "Microsoft.Bcl.AsyncInterfaces": "[8.0.0, )", "Microsoft.CSharp": "[4.7.0, )", "Microsoft.Data.Sqlite": "[7.0.7, )", "Microsoft.Extensions.DependencyInjection.Abstractions": "[2.2.0, )", "Microsoft.Extensions.Logging": "[2.2.0, )", + "Microsoft.Extensions.ObjectPool": "[8.0.10, )", "Polly": "[7.2.3, )", "Polly.Contrib.WaitAndRetry": "[1.1.1, )", "Polly.Extensions.Http": "[3.0.0, )", @@ -303,6 +305,12 @@ "System.Reactive": "5.0.0" } }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "CentralTransitive", + "requested": "[8.0.0, )", + "resolved": "8.0.0", + "contentHash": "3WA9q9yVqJp222P3x1wYIGDAkpjAku0TMUaaQV22g6L67AI0LdOIrVS7Ht2vJfLHGSPVuqN94vIr15qn+HEkHw==" + }, "Microsoft.CSharp": { "type": "CentralTransitive", "requested": "[4.7.0, )", @@ -346,6 +354,12 @@ "Microsoft.Extensions.Options": "2.2.0" } }, + "Microsoft.Extensions.ObjectPool": { + "type": "CentralTransitive", + "requested": "[8.0.10, )", + "resolved": "8.0.10", + "contentHash": "u7gAG7JgxF8VSJUGPSudAcPxOt+ymJKQCSxNRxiuKV+klCQbHljQR75SilpedCTfhPWDhtUwIJpnDVtspr9nMg==" + }, "Polly": { "type": "CentralTransitive", "requested": "[7.2.3, )", diff --git a/tests/Speckle.Sdk.Tests.Performance/Benchmarks/GeneralDeserializerTest.cs b/tests/Speckle.Sdk.Tests.Performance/Benchmarks/GeneralDeserializerTest.cs index 5370097b..5b1048d5 100644 --- a/tests/Speckle.Sdk.Tests.Performance/Benchmarks/GeneralDeserializerTest.cs +++ b/tests/Speckle.Sdk.Tests.Performance/Benchmarks/GeneralDeserializerTest.cs @@ -1,11 +1,15 @@ -using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Engines; -using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.DependencyInjection; using Speckle.Objects.Geometry; using Speckle.Sdk.Credentials; +using Speckle.Sdk.Helpers; using Speckle.Sdk.Host; +using Speckle.Sdk.Logging; using Speckle.Sdk.Models; using Speckle.Sdk.Serialisation; +using Speckle.Sdk.Serialisation.V2; +using Speckle.Sdk.Serialisation.V2.Receive; namespace Speckle.Sdk.Tests.Performance.Benchmarks; @@ -13,9 +17,20 @@ namespace Speckle.Sdk.Tests.Performance.Benchmarks; /// How many threads on our Deserializer is optimal /// [MemoryDiagnoser] -[SimpleJob(RunStrategy.Monitoring)] +[SimpleJob(RunStrategy.Monitoring, 0, 0, 2)] public class GeneralDeserializer : IDisposable { + private const bool skipCache = true; + + /* + private const string url = "https://latest.speckle.systems/projects/a3ac1b2706/models/59d3b0f3c6"; //small? + private const string streamId = "a3ac1b2706"; + private const string rootId = "7d53bcf28c6696ecac8781684a0aa006";*/ + + + private const string url = "https://latest.speckle.systems/projects/2099ac4b5f/models/da511c4d1e"; //perf? + private const string streamId = "2099ac4b5f"; + private const string rootId = "30fb4cbe6eb2202b9e7b4a4fcc3dd2b6"; private TestDataHelper _dataSource; [GlobalSetup] @@ -24,33 +39,36 @@ public class GeneralDeserializer : IDisposable TypeLoader.Initialize(typeof(Base).Assembly, typeof(Point).Assembly); _dataSource = new TestDataHelper(); await _dataSource - .SeedTransport( - new Account() - { - serverInfo = new() { url = "https://latest.speckle.systems/projects/2099ac4b5f/models/da511c4d1e" }, - }, - "2099ac4b5f", - "30fb4cbe6eb2202b9e7b4a4fcc3dd2b6" - ) + .SeedTransport(new Account() { serverInfo = new() { url = url } }, streamId, rootId, skipCache) .ConfigureAwait(false); } [Benchmark] - public async Task RunTest() + public async Task RunTest_New() { - SpeckleObjectDeserializer sut = new() { ReadTransport = _dataSource.Transport }; - string data = await _dataSource.Transport.GetObject(_dataSource.ObjectId)!; - return await sut.DeserializeAsync(data); + var sqlite = new SQLiteCacheManager(streamId); + var serverObjects = new ServerObjectManager( + TestDataHelper.ServiceProvider.GetRequiredService(), + TestDataHelper.ServiceProvider.GetRequiredService(), + new Uri(url), + null + ); + var o = new ObjectLoader(sqlite, serverObjects, streamId, null); + var process = new DeserializeProcess(null, o); + return await process.Deserialize(rootId, default, new(skipCache)).ConfigureAwait(false); } + /* + [Benchmark] + public async Task RunTest_Old() + { + SpeckleObjectDeserializer sut = new() { ReadTransport = _dataSource.Transport }; + string data = await _dataSource.Transport.GetObject(_dataSource.ObjectId)!; + return await sut.DeserializeAsync(data); + } + */ [GlobalCleanup] - public void Cleanup() - { - Dispose(); - } + public void Cleanup() => Dispose(); - public void Dispose() - { - _dataSource.Dispose(); - } + public void Dispose() => _dataSource.Dispose(); } diff --git a/tests/Speckle.Sdk.Tests.Performance/Benchmarks/GeneralReceiveTest.cs b/tests/Speckle.Sdk.Tests.Performance/Benchmarks/GeneralReceiveTest.cs new file mode 100644 index 00000000..60c2ee39 --- /dev/null +++ b/tests/Speckle.Sdk.Tests.Performance/Benchmarks/GeneralReceiveTest.cs @@ -0,0 +1,70 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Engines; +using Microsoft.Extensions.DependencyInjection; +using Speckle.Objects.Geometry; +using Speckle.Sdk.Api; +using Speckle.Sdk.Credentials; +using Speckle.Sdk.Helpers; +using Speckle.Sdk.Host; +using Speckle.Sdk.Logging; +using Speckle.Sdk.Models; +using Speckle.Sdk.Serialisation; +using Speckle.Sdk.Serialisation.V2; +using Speckle.Sdk.Serialisation.V2.Receive; +using Speckle.Sdk.Transports; + +namespace Speckle.Sdk.Tests.Performance.Benchmarks; + +/// +/// How many threads on our Deserializer is optimal +/// +[MemoryDiagnoser] +[SimpleJob(RunStrategy.Monitoring, 0, 0, 1)] +public class GeneralReceiveTest : IDisposable +{ + /* + private const string url = "https://latest.speckle.systems/projects/a3ac1b2706/models/59d3b0f3c6"; //small? + private const string streamId = "a3ac1b2706";S + private const string rootId = "7d53bcf28c6696ecac8781684a0aa006";*/ + + + private const string url = "https://latest.speckle.systems/projects/2099ac4b5f/models/da511c4d1e"; //perf? + private readonly Uri _baseUrl = new("https://latest.speckle.systems"); + private const string streamId = "2099ac4b5f"; + private const string rootId = "30fb4cbe6eb2202b9e7b4a4fcc3dd2b6"; + private TestDataHelper _dataSource; + private IOperations _operations; + private ITransport remoteTransport; + + [GlobalSetup] + public async Task Setup() + { + TypeLoader.Initialize(typeof(Base).Assembly, typeof(Point).Assembly); + _dataSource = new TestDataHelper(); + var acc = new Account() { serverInfo = new() { url = url } }; + await _dataSource.SeedTransport(acc, streamId, rootId, true).ConfigureAwait(false); + _operations = TestDataHelper.ServiceProvider.GetRequiredService(); + // await _operations.Receive2(_baseUrl, streamId, rootId, null); + + remoteTransport = TestDataHelper + .ServiceProvider.GetRequiredService() + .Create(acc, streamId); + } + + [Benchmark] + public async Task RunTest_Receive() + { + return await _operations.Receive(rootId, remoteTransport, _dataSource.Transport); + } + + [Benchmark] + public async Task RunTest_Receive2() + { + return await _operations.Receive2(_baseUrl, streamId, rootId, null); + } + + [GlobalCleanup] + public void Cleanup() => Dispose(); + + public void Dispose() => _dataSource.Dispose(); +} diff --git a/tests/Speckle.Sdk.Tests.Performance/Benchmarks/GeneralSerializerTest.cs b/tests/Speckle.Sdk.Tests.Performance/Benchmarks/GeneralSerializerTest.cs index d18c6d2e..deb8a27d 100644 --- a/tests/Speckle.Sdk.Tests.Performance/Benchmarks/GeneralSerializerTest.cs +++ b/tests/Speckle.Sdk.Tests.Performance/Benchmarks/GeneralSerializerTest.cs @@ -32,7 +32,8 @@ public class GeneralSerializerTest serverInfo = new() { url = "https://latest.speckle.systems/projects/2099ac4b5f/models/da511c4d1e" }, }, "2099ac4b5f", - "30fb4cbe6eb2202b9e7b4a4fcc3dd2b6" + "30fb4cbe6eb2202b9e7b4a4fcc3dd2b6", + false ) .ConfigureAwait(false); diff --git a/tests/Speckle.Sdk.Tests.Performance/TestDataHelper.cs b/tests/Speckle.Sdk.Tests.Performance/TestDataHelper.cs index 2420efdd..9ca4a397 100644 --- a/tests/Speckle.Sdk.Tests.Performance/TestDataHelper.cs +++ b/tests/Speckle.Sdk.Tests.Performance/TestDataHelper.cs @@ -23,24 +23,33 @@ public sealed class TestDataHelper : IDisposable ServiceProvider = serviceCollection.BuildServiceProvider(); } - public async Task SeedTransport(Account account, string streamId, string objectId) + public async Task SeedTransport(Account account, string streamId, string objectId, bool skipCache) { // Transport = new SQLiteTransport(s_basePath, APPLICATION_NAME); Transport = new SQLiteTransport(); //seed SQLite transport with test data - ObjectId = await SeedTransport(account, streamId, objectId, Transport).ConfigureAwait(false); + ObjectId = await SeedTransport(account, streamId, objectId, Transport, skipCache).ConfigureAwait(false); } - public async Task SeedTransport(Account account, string streamId, string objectId, ITransport transport) + public async Task SeedTransport( + Account account, + string streamId, + string objectId, + ITransport transport, + bool skipCache + ) { - using ServerTransport remoteTransport = ServiceProvider - .GetRequiredService() - .Create(account, streamId); - transport.BeginWrite(); - await remoteTransport.CopyObjectAndChildren(objectId, transport).ConfigureAwait(false); - transport.EndWrite(); - await transport.WriteComplete().ConfigureAwait(false); + if (!skipCache) + { + using ServerTransport remoteTransport = ServiceProvider + .GetRequiredService() + .Create(account, streamId); + transport.BeginWrite(); + await remoteTransport.CopyObjectAndChildren(objectId, transport).ConfigureAwait(false); + transport.EndWrite(); + await transport.WriteComplete().ConfigureAwait(false); + } return objectId; } diff --git a/tests/Speckle.Sdk.Tests.Performance/packages.lock.json b/tests/Speckle.Sdk.Tests.Performance/packages.lock.json index e9528b66..703abca5 100644 --- a/tests/Speckle.Sdk.Tests.Performance/packages.lock.json +++ b/tests/Speckle.Sdk.Tests.Performance/packages.lock.json @@ -98,11 +98,6 @@ "resolved": "1.17.0", "contentHash": "8x+HCVTl/HHTGpscH3vMBhV8sknN/muZFw9s3TsI8SA6+c43cOTCi2+jE4KsU8pNLbJ++iF2ZFcpcXHXtDglnw==" }, - "Microsoft.Bcl.AsyncInterfaces": { - "type": "Transitive", - "resolved": "1.1.0", - "contentHash": "1Am6l4Vpn3/K32daEqZI+FFr96OlZkgwK2LcT3pZ2zWubR5zTPW3/FkO1Rat9kb7oQOa4rxgl9LJHc5tspCWfg==" - }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", "resolved": "8.0.0", @@ -361,10 +356,12 @@ "type": "Project", "dependencies": { "GraphQL.Client": "[6.0.0, )", + "Microsoft.Bcl.AsyncInterfaces": "[8.0.0, )", "Microsoft.CSharp": "[4.7.0, )", "Microsoft.Data.Sqlite": "[7.0.7, )", "Microsoft.Extensions.DependencyInjection.Abstractions": "[2.2.0, )", "Microsoft.Extensions.Logging": "[2.2.0, )", + "Microsoft.Extensions.ObjectPool": "[8.0.10, )", "Polly": "[7.2.3, )", "Polly.Contrib.WaitAndRetry": "[1.1.1, )", "Polly.Extensions.Http": "[3.0.0, )", @@ -383,6 +380,12 @@ "System.Reactive": "5.0.0" } }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "CentralTransitive", + "requested": "[8.0.0, )", + "resolved": "8.0.0", + "contentHash": "3WA9q9yVqJp222P3x1wYIGDAkpjAku0TMUaaQV22g6L67AI0LdOIrVS7Ht2vJfLHGSPVuqN94vIr15qn+HEkHw==" + }, "Microsoft.CSharp": { "type": "CentralTransitive", "requested": "[4.7.0, )", @@ -417,6 +420,12 @@ "Microsoft.Extensions.Options": "2.2.0" } }, + "Microsoft.Extensions.ObjectPool": { + "type": "CentralTransitive", + "requested": "[8.0.10, )", + "resolved": "8.0.10", + "contentHash": "u7gAG7JgxF8VSJUGPSudAcPxOt+ymJKQCSxNRxiuKV+klCQbHljQR75SilpedCTfhPWDhtUwIJpnDVtspr9nMg==" + }, "Polly": { "type": "CentralTransitive", "requested": "[7.2.3, )", diff --git a/tests/Speckle.Sdk.Tests.Unit/packages.lock.json b/tests/Speckle.Sdk.Tests.Unit/packages.lock.json index 68ea50dd..ca08e1f8 100644 --- a/tests/Speckle.Sdk.Tests.Unit/packages.lock.json +++ b/tests/Speckle.Sdk.Tests.Unit/packages.lock.json @@ -283,10 +283,12 @@ "type": "Project", "dependencies": { "GraphQL.Client": "[6.0.0, )", + "Microsoft.Bcl.AsyncInterfaces": "[8.0.0, )", "Microsoft.CSharp": "[4.7.0, )", "Microsoft.Data.Sqlite": "[7.0.7, )", "Microsoft.Extensions.DependencyInjection.Abstractions": "[2.2.0, )", "Microsoft.Extensions.Logging": "[2.2.0, )", + "Microsoft.Extensions.ObjectPool": "[8.0.10, )", "Polly": "[7.2.3, )", "Polly.Contrib.WaitAndRetry": "[1.1.1, )", "Polly.Extensions.Http": "[3.0.0, )", @@ -305,6 +307,12 @@ "System.Reactive": "5.0.0" } }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "CentralTransitive", + "requested": "[8.0.0, )", + "resolved": "8.0.0", + "contentHash": "3WA9q9yVqJp222P3x1wYIGDAkpjAku0TMUaaQV22g6L67AI0LdOIrVS7Ht2vJfLHGSPVuqN94vIr15qn+HEkHw==" + }, "Microsoft.CSharp": { "type": "CentralTransitive", "requested": "[4.7.0, )", @@ -339,6 +347,12 @@ "Microsoft.Extensions.Options": "2.2.0" } }, + "Microsoft.Extensions.ObjectPool": { + "type": "CentralTransitive", + "requested": "[8.0.10, )", + "resolved": "8.0.10", + "contentHash": "u7gAG7JgxF8VSJUGPSudAcPxOt+ymJKQCSxNRxiuKV+klCQbHljQR75SilpedCTfhPWDhtUwIJpnDVtspr9nMg==" + }, "Polly": { "type": "CentralTransitive", "requested": "[7.2.3, )",