clean up from progress (#78)

* progress intermediate commit

* add progress for download

* remove unused code

* remove batch sent callbacks

* multi-threaded deserialize works

* Progress for download and deserialization

* Fix tests

* Have less indeterminate deserialization

* fix deserialization

* make download faster with buffered stream

* put local receive back

* remove unused callback

* fmt

* Progress for serialization and upload

* fix uploading

* clean up from progress

* merge fixes and fmt

src/Speckle.Sdk/Models/Blob.cs

@@ -49,7 +49,7 @@ public class Blob : Base
   {
     if ((_isHashExpired || _hash == null) && filePath != null)
     {
-      _hash = Utilities.HashFile(filePath);
+      _hash = HashUtility.HashFile(filePath);
     }
 
     return _hash;

src/Speckle.Sdk/Models/HashUtility.cs

@@ -0,0 +1,26 @@
+using System.Diagnostics.CodeAnalysis;
+using System.Security.Cryptography;
+
+namespace Speckle.Sdk.Models;
+
+public static class HashUtility
+{
+  public enum HashingFunctions
+  {
+    SHA256,
+    MD5
+  }
+
+  public const int HASH_LENGTH = 32;
+
+  [SuppressMessage("Security", "CA5351:Do Not Use Broken Cryptographic Algorithms")]
+  public static string HashFile(string filePath, HashingFunctions func = HashingFunctions.SHA256)
+  {
+    using HashAlgorithm hashAlgorithm = func == HashingFunctions.MD5 ? MD5.Create() : SHA256.Create();
+
+    using var stream = File.OpenRead(filePath);
+
+    var hash = hashAlgorithm.ComputeHash(stream);
+    return BitConverter.ToString(hash, 0, HASH_LENGTH).Replace("-", "").ToLowerInvariant();
+  }
+}

src/Speckle.Sdk/Models/Utilities.cs

@@ -1,61 +0,0 @@
-using System.Collections;
-using System.Diagnostics.CodeAnalysis;
-using System.Security.Cryptography;
-
-namespace Speckle.Sdk.Models;
-
-public static class Utilities
-{
-  public enum HashingFunctions
-  {
-    SHA256,
-    MD5
-  }
-
-  public const int HASH_LENGTH = 32;
-
-  [SuppressMessage("Security", "CA5351:Do Not Use Broken Cryptographic Algorithms")]
-  public static string HashFile(string filePath, HashingFunctions func = HashingFunctions.SHA256)
-  {
-    using HashAlgorithm hashAlgorithm = func == HashingFunctions.MD5 ? MD5.Create() : SHA256.Create();
-
-    using var stream = File.OpenRead(filePath);
-
-    var hash = hashAlgorithm.ComputeHash(stream);
-    return BitConverter.ToString(hash, 0, HASH_LENGTH).Replace("-", "").ToLowerInvariant();
-  }
-
-  /// <summary>
-  /// Utility function to flatten a conversion result that might have nested lists of objects.
-  /// This happens, for example, in the case of multiple display value fallbacks for a given object.
-  /// </summary>
-  /// <remarks>
-  ///   Assuming native objects are not inherited from IList.
-  /// </remarks>
-  /// <param name="item"> Object to flatten</param>
-  /// <returns> Flattened objects after to host.</returns>
-  public static List<object?> FlattenToHostConversionResult(object? item)
-  {
-    List<object?> convertedList = new();
-    Stack<object?> stack = new();
-    stack.Push(item);
-
-    while (stack.Count > 0)
-    {
-      object? current = stack.Pop();
-      if (current is IList list)
-      {
-        foreach (object? subItem in list)
-        {
-          stack.Push(subItem);
-        }
-      }
-      else
-      {
-        convertedList.Add(current);
-      }
-    }
-
-    return convertedList;
-  }
-}

src/Speckle.Sdk/Serialisation/BaseObjectDeserializerV2.cs

@@ -6,7 +6,7 @@ using Speckle.Sdk.Common;
 using Speckle.Sdk.Host;
 using Speckle.Sdk.Logging;
 using Speckle.Sdk.Models;
-using Speckle.Sdk.Serialisation.SerializationUtilities;
+using Speckle.Sdk.Serialisation.Utilities;
 using Speckle.Sdk.Transports;
 
 namespace Speckle.Sdk.Serialisation;
@@ -15,6 +15,7 @@ public sealed class BaseObjectDeserializerV2
 {
   private bool _isBusy;
   private readonly object _callbackLock = new();
+  private readonly object?[] _invokeNull = [null];
 
   // id -> Base if already deserialized or id -> Task<object> if was handled by a bg thread
   private Dictionary<string, object?>? _deserializedObjects;
@@ -37,9 +38,7 @@ public sealed class BaseObjectDeserializerV2
 
   public string? BlobStorageFolder { get; set; }
   public TimeSpan Elapsed { get; private set; }
-
-  public static int DefaultNumberThreads => Math.Min(Environment.ProcessorCount, 6); //6 threads seems the sweet spot, see performance test project
-  public int WorkerThreadCount { get; set; } = DefaultNumberThreads;
+  public int WorkerThreadCount { get; set; } = Math.Min(Environment.ProcessorCount, 6); //6 threads seems the sweet spot, see performance test project;
 
   /// <param name="rootObjectJson">The JSON string of the object to be deserialized <see cref="Base"/></param>
   /// <returns>A <see cref="Base"/> typed object deserialized from the <paramref name="rootObjectJson"/></returns>
@@ -318,7 +317,7 @@ public sealed class BaseObjectDeserializerV2
     dictObj.Remove(TYPE_DISCRIMINATOR);
     dictObj.Remove("__closure");
 
-    var staticProperties = BaseObjectSerializationUtilities.GetTypeProperties(typeName);
+    var staticProperties = TypeCache.GetTypeProperties(typeName);
     foreach (var entry in dictObj)
     {
       if (staticProperties.TryGetValue(entry.Key, out PropertyInfo? value) && value.CanWrite)
@@ -359,10 +358,10 @@ public sealed class BaseObjectDeserializerV2
       bb.filePath = bb.GetLocalDestinationPath(BlobStorageFolder);
     }
 
-    var onDeserializedCallbacks = BaseObjectSerializationUtilities.GetOnDeserializedCallbacks(typeName);
+    var onDeserializedCallbacks = TypeCache.GetOnDeserializedCallbacks(typeName);
     foreach (MethodInfo onDeserialized in onDeserializedCallbacks)
     {
-      onDeserialized.Invoke(baseObj, new object?[] { null });
+      onDeserialized.Invoke(baseObj, _invokeNull);
     }
 
     return baseObj;

src/Speckle.Sdk/Serialisation/BaseObjectSerializerV2.cs

@@ -12,7 +12,6 @@ using Speckle.Sdk.Logging;
 using Speckle.Sdk.Models;
 using Speckle.Sdk.Transports;
 using Constants = Speckle.Sdk.Helpers.Constants;
-using Utilities = Speckle.Sdk.Models.Utilities;
 
 namespace Speckle.Sdk.Serialisation;
 
@@ -438,7 +437,7 @@ public class BaseObjectSerializerV2
   private static string ComputeId(IReadOnlyDictionary<string, object?> obj)
   {
     string serialized = JsonConvert.SerializeObject(obj);
-    string hash = Crypt.Sha256(serialized, length: Utilities.HASH_LENGTH);
+    string hash = Crypt.Sha256(serialized, length: HashUtility.HASH_LENGTH);
     return hash;
   }
 

src/Speckle.Sdk/Serialisation/SpeckleDeserializeException.cs

@@ -0,0 +1,10 @@
+namespace Speckle.Sdk.Serialisation;
+
+public class SpeckleDeserializeException : SpeckleException
+{
+  public SpeckleDeserializeException(string message, Exception? inner = null)
+    : base(message, inner) { }
+
+  public SpeckleDeserializeException(string message)
+    : base(message) { }
+}

src/Speckle.Sdk/Serialisation/SpeckleSerializerException.cs

@@ -1,25 +1,10 @@
-using Speckle.Sdk.Logging;
-
-namespace Speckle.Sdk.Serialisation;
+namespace Speckle.Sdk.Serialisation;
 
 public class SpeckleSerializeException : SpeckleException
 {
-  public SpeckleSerializeException() { }
-
   public SpeckleSerializeException(string message, Exception? inner = null)
     : base(message, inner) { }
 
   public SpeckleSerializeException(string message)
     : base(message) { }
 }
-
-public class SpeckleDeserializeException : SpeckleException
-{
-  public SpeckleDeserializeException() { }
-
-  public SpeckleDeserializeException(string message, Exception? inner = null)
-    : base(message, inner) { }
-
-  public SpeckleDeserializeException(string message)
-    : base(message) { }
-}

src/Speckle.Sdk/Serialisation/Utilities/CallsiteCache.cs

@@ -2,7 +2,7 @@
 using System.Runtime.CompilerServices;
 using Microsoft.CSharp.RuntimeBinder;
 
-namespace Speckle.Sdk.Serialisation.SerializationUtilities;
+namespace Speckle.Sdk.Serialisation.Utilities;
 
 internal static class CallSiteCache
 {

src/Speckle.Sdk/Serialisation/Utilities/ClosureParser.cs

@@ -1,7 +1,7 @@
 using Speckle.Newtonsoft.Json;
 using Speckle.Sdk.Common;
 
-namespace Speckle.Sdk.Serialisation.SerializationUtilities;
+namespace Speckle.Sdk.Serialisation.Utilities;
 
 public static class ClosureParser
 {

src/Speckle.Sdk/Serialisation/Utilities/DeserializationWorkerThreads.cs

@@ -1,6 +1,6 @@
-using Speckle.Sdk.Logging;
+using Speckle.Sdk.Common;
 
-namespace Speckle.Sdk.Serialisation.SerializationUtilities;
+namespace Speckle.Sdk.Serialisation.Utilities;
 
 internal enum WorkerThreadTaskType
 {
@@ -48,7 +48,7 @@ internal sealed class DeserializationWorkerThreads : ParallelOperationExecutor<W
 
       try
       {
-        (string objectJson, long? current, long? total) = ((string, long?, long?))inputValue!;
+        (string objectJson, long? current, long? total) = ((string, long?, long?))inputValue.NotNull();
         var result = RunOperation(taskType, objectJson, current, total, _serializer);
         tcs.SetResult(result);
       }

src/Speckle.Sdk/Serialisation/Utilities/OperationTask.cs

@@ -0,0 +1,23 @@
+namespace Speckle.Sdk.Serialisation.Utilities;
+
+internal readonly struct OperationTask<T>
+  where T : struct
+{
+  public readonly T OperationType;
+  public readonly object? InputValue;
+  public readonly TaskCompletionSource<object?>? Tcs;
+
+  public OperationTask(T operationType, object? inputValue = null, TaskCompletionSource<object?>? tcs = null)
+  {
+    OperationType = operationType;
+    InputValue = inputValue;
+    Tcs = tcs;
+  }
+
+  public void Deconstruct(out T operationType, out object? inputValue, out TaskCompletionSource<object?>? tcs)
+  {
+    operationType = OperationType;
+    inputValue = InputValue;
+    tcs = Tcs;
+  }
+}

src/Speckle.Sdk/Serialisation/Utilities/ParallelOperationExecutor.cs

@@ -1,28 +1,6 @@
-using System.Collections.Concurrent;
+using System.Collections.Concurrent;
 
-namespace Speckle.Sdk.Serialisation.SerializationUtilities;
-
-internal readonly struct OperationTask<T>
-  where T : struct
-{
-  public readonly T OperationType;
-  public readonly object? InputValue;
-  public readonly TaskCompletionSource<object?>? Tcs;
-
-  public OperationTask(T operationType, object? inputValue = null, TaskCompletionSource<object?>? tcs = null)
-  {
-    OperationType = operationType;
-    InputValue = inputValue;
-    Tcs = tcs;
-  }
-
-  public void Deconstruct(out T operationType, out object? inputValue, out TaskCompletionSource<object?>? tcs)
-  {
-    operationType = OperationType;
-    inputValue = InputValue;
-    tcs = Tcs;
-  }
-}
+namespace Speckle.Sdk.Serialisation.Utilities;
 
 internal abstract class ParallelOperationExecutor<TOperation> : IDisposable
   where TOperation : struct

src/Speckle.Sdk/Serialisation/Utilities/TypeCache.cs

@@ -3,9 +3,9 @@ using System.Reflection;
 using System.Runtime.Serialization;
 using Speckle.Sdk.Host;
 
-namespace Speckle.Sdk.Serialisation.SerializationUtilities;
+namespace Speckle.Sdk.Serialisation.Utilities;
 
-internal static class BaseObjectSerializationUtilities
+internal static class TypeCache
 {
   #region Getting Types
   private static ConcurrentDictionary<string, IReadOnlyDictionary<string, PropertyInfo>> s_typeProperties = new();

src/Speckle.Sdk/Serialisation/Utilities/ValueConverter.cs

@@ -6,7 +6,7 @@ using System.Globalization;
 using Speckle.DoubleNumerics;
 using Speckle.Sdk.Logging;
 
-namespace Speckle.Sdk.Serialisation.SerializationUtilities;
+namespace Speckle.Sdk.Serialisation.Utilities;
 
 internal static class ValueConverter
 {

src/Speckle.Sdk/Transports/ServerUtils/ParallelServerAPI.cs

@@ -3,7 +3,7 @@ using System.Diagnostics;
 using Speckle.Sdk.Common;
 using Speckle.Sdk.Helpers;
 using Speckle.Sdk.Logging;
-using Speckle.Sdk.Serialisation.SerializationUtilities;
+using Speckle.Sdk.Serialisation.Utilities;
 
 namespace Speckle.Sdk.Transports.ServerUtils;
 

tests/Speckle.Objects.Tests.Unit/ModelPropertySupportedTypes.cs

@@ -34,7 +34,7 @@ public class ModelPropertySupportedTypes
   /// </summary>
   /// <remarks>
   /// If you're tempted to add to this list, please ensure both our serializer and deserializer support properties of this type
-  /// Check the <see cref="Speckle.Sdk.Serialisation.SerializationUtilities.ValueConverter"/>
+  /// Check the <see cref="Speckle.Sdk.Serialisation.Utilities.ValueConverter"/>
   /// Check the <see cref="BaseObjectSerializerV2"/>
   /// (or is an interface where all concrete types are supported)
   /// You should also consider adding a test in SerializerNonBreakingChanges

tests/Speckle.Sdk.Serialization.Tests/SerializationTests.cs

@@ -7,7 +7,6 @@ using Speckle.Sdk.Common;
 using Speckle.Sdk.Host;
 using Speckle.Sdk.Models;
 using Speckle.Sdk.Serialisation;
-using Speckle.Sdk.Serialisation.SerializationUtilities;
 
 namespace Speckle.Sdk.Serialization.Tests;
 

tests/Speckle.Sdk.Tests.Unit/Models/UtilitiesTests.cs

@@ -5,7 +5,7 @@ using Speckle.Sdk.Models;
 namespace Speckle.Sdk.Tests.Unit.Models;
 
 [TestFixture(TestOf = typeof(Crypt))]
-public sealed class UtilitiesTests
+public sealed class HashUtilityTests
 {
   [Test]
   [TestOf(nameof(Crypt.Md5))]
@@ -29,44 +29,4 @@ public sealed class UtilitiesTests
     Assert.That(lower, Is.EqualTo(expected.ToLower()));
     Assert.That(upper, Is.EqualTo(expected.ToUpper()));
   }
-
-  [Test]
-  public void FlattenToNativeConversion()
-  {
-    var singleObject = new object();
-    var nestedObjects = new List<object>()
-    {
-      new List<object>()
-      {
-        new(), // obj 1
-        new() // obj 2
-      },
-      new() // obj 3
-    };
-
-    var testEnum = new List<object>() { new(), new() }.Select(o => o);
-
-    var nestedObjectsWithEnumerableInherited = new List<object>()
-    {
-      new List<object>()
-      {
-        new(), // obj 1
-        new(), // obj 2
-        testEnum // obj 3
-      },
-      new() // obj 4
-    };
-
-    var parentTestEnumFlattened = Utilities.FlattenToHostConversionResult(testEnum);
-    var singleObjectFlattened = Utilities.FlattenToHostConversionResult(singleObject);
-    var nestedObjectsFlattened = Utilities.FlattenToHostConversionResult(nestedObjects);
-    var nestedObjectsWithEnumerableInheritedFlattened = Utilities.FlattenToHostConversionResult(
-      nestedObjectsWithEnumerableInherited
-    );
-
-    Assert.That(parentTestEnumFlattened.Count, Is.EqualTo(1));
-    Assert.That(singleObjectFlattened.Count, Is.EqualTo(1));
-    Assert.That(nestedObjectsFlattened.Count, Is.EqualTo(3));
-    Assert.That(nestedObjectsWithEnumerableInheritedFlattened.Count, Is.EqualTo(4));
-  }
 }

tests/Speckle.Sdk.Tests.Unit/Serialisation/ObjectModelDeprecationTests.cs

@@ -3,13 +3,12 @@ using Shouldly;
 using Speckle.Sdk.Host;
 using Speckle.Sdk.Models;
 using Speckle.Sdk.Serialisation.Deprecated;
-using Speckle.Sdk.Serialisation.SerializationUtilities;
 
 namespace Speckle.Sdk.Tests.Unit.Serialisation
 {
   [TestFixture]
-  [TestOf(typeof(BaseObjectSerializationUtilities))]
-  public class ObjectModelDeprecationTests
+  [TestOf(typeof(TypeLoader))]
+  public class TypeLoaderTests
   {
     [SetUp]
     public void Setup()