Merge branch 'dev' into main-dev

This commit is contained in:
Adam Hathcock
2025-04-09 11:22:51 +01:00
34 changed files with 562 additions and 678 deletions
@@ -22,4 +22,23 @@ public static class Collections
public static class EnumerableExtensions
{
public static IEnumerable<int> RangeFrom(int from, int to) => Enumerable.Range(from, to - from + 1);
#if NETSTANDARD2_0
public static IEnumerable<TSource> DistinctBy<TSource, TKey>(
this IEnumerable<TSource> source,
Func<TSource, TKey> keySelector
)
{
var keys = new HashSet<TKey>();
foreach (var element in source)
{
if (keys.Contains(keySelector(element)))
{
continue;
}
keys.Add(keySelector(element));
yield return element;
}
}
#endif
}
@@ -77,7 +77,7 @@ public abstract class ChannelSaver<T>
{
try
{
await SendToServer((Batch<T>)batch).ConfigureAwait(false);
await SendToServerInternal((Batch<T>)batch).ConfigureAwait(false);
return batch;
}
#pragma warning disable CA1031
@@ -89,20 +89,6 @@ public abstract class ChannelSaver<T>
}
}
public async Task SendToServer(Batch<T> batch)
{
try
{
await SendToServerInternal(batch).ConfigureAwait(false);
}
#pragma warning disable CA1031
catch (Exception ex)
#pragma warning restore CA1031
{
RecordException(ex);
}
}
protected abstract Task SendToServerInternal(Batch<T> batch);
public abstract void SaveToCache(List<T> item);
@@ -117,7 +103,7 @@ public abstract class ChannelSaver<T>
}
}
protected Exception? Exception { get; set; }
public Exception? Exception { get; set; }
private void RecordException(Exception ex)
{
@@ -29,7 +29,7 @@ public partial class Operations
metricsFactory.CreateCounter<long>("Receive").Add(1);
receiveActivity?.SetTag("objectId", objectId);
var process = serializeProcessFactory.CreateDeserializeProcess(
var process = deserializeProcessFactory.CreateDeserializeProcess(
url,
streamId,
authorizationToken,
+2 -1
View File
@@ -15,5 +15,6 @@ public partial class Operations(
ILogger<Operations> logger,
ISdkActivityFactory activityFactory,
ISdkMetricsFactory metricsFactory,
ISerializeProcessFactory serializeProcessFactory
ISerializeProcessFactory serializeProcessFactory,
IDeserializeProcessFactory deserializeProcessFactory
) : IOperations;
+2 -1
View File
@@ -1,3 +1,4 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("Speckle.Sdk.Tests.Unit")]
[assembly: InternalsVisibleTo("Speckle.Objects.Tests.Unit")]
[assembly: InternalsVisibleTo("Speckle.Sdk.Tests.Performance")]
-24
View File
@@ -1,24 +0,0 @@
namespace Speckle.Sdk.Host;
public enum HostAppVersion
{
v3,
v6,
v7,
v8,
v2019,
v2020,
v2021,
v2022,
v2023,
v2024,
v2025,
v2026,
v21,
v22,
v25,
v26,
v715,
v716,
v717,
}
-13
View File
@@ -1,13 +0,0 @@
namespace Speckle.Sdk.Host;
public readonly struct HostApplication
{
public string Name { get; }
public string Slug { get; }
public HostApplication(string name, string slug)
{
Name = name;
Slug = slug;
}
}
-43
View File
@@ -1,43 +0,0 @@
namespace Speckle.Sdk.Host;
/// <summary>
/// List of Host Applications - their slugs should match our ghost tags and ci/cd slugs
/// </summary>
public static class HostApplications
{
public static string GetVersion(HostAppVersion version) => version.ToString().TrimStart('v');
public static readonly HostApplication Rhino = new("Rhino", "rhino"),
Grasshopper = new("Grasshopper", "grasshopper"),
Revit = new("Revit", "revit"),
Dynamo = new("Dynamo", "dynamo"),
Unity = new("Unity", "unity"),
GSA = new("GSA", "gsa"),
Civil = new("Civil 3D", "civil3d"),
Civil3D = new("Civil 3D", "civil3d"),
AutoCAD = new("AutoCAD", "autocad"),
MicroStation = new("MicroStation", "microstation"),
OpenRoads = new("OpenRoads", "openroads"),
OpenRail = new("OpenRail", "openrail"),
OpenBuildings = new("OpenBuildings", "openbuildings"),
ETABS = new("ETABS", "etabs"),
SAP2000 = new("SAP2000", "sap2000"),
CSiBridge = new("CSiBridge", "csibridge"),
SAFE = new("SAFE", "safe"),
TeklaStructures = new("Tekla Structures", "teklastructures"),
Dxf = new("DXF Converter", "dxf"),
Excel = new("Excel", "excel"),
Unreal = new("Unreal", "unreal"),
PowerBI = new("Power BI", "powerbi"),
Blender = new("Blender", "blender"),
QGIS = new("QGIS", "qgis"),
ArcGIS = new("ArcGIS", "arcgis"),
SketchUp = new("SketchUp", "sketchup"),
Archicad = new("Archicad", "archicad"),
TopSolid = new("TopSolid", "topsolid"),
Python = new("Python", "python"),
NET = new(".NET", "net"),
Navisworks = new("Navisworks", "navisworks"),
AdvanceSteel = new("Advance Steel", "advancesteel"),
Other = new("Other", "other");
}
+2 -2
View File
@@ -5,9 +5,9 @@ using Speckle.Sdk.Models;
namespace Speckle.Sdk.Host;
public record LoadedType(string Name, Type Type, List<string> DeprecatedNames);
internal record LoadedType(string Name, Type Type, List<string> DeprecatedNames);
public static class TypeLoader
internal static class TypeLoader
{
private static bool s_initialized;
private static List<LoadedType> s_availableTypes = new();
@@ -0,0 +1,56 @@
using System.Collections.Concurrent;
using Microsoft.Extensions.Logging;
using Speckle.InterfaceGenerator;
using Speckle.Sdk.Serialisation.V2.Receive;
using Speckle.Sdk.SQLite;
using Speckle.Sdk.Transports;
namespace Speckle.Sdk.Serialisation.V2;
[GenerateAutoInterface]
public class DeserializeProcessFactory(
IBaseDeserializer baseDeserializer,
ISqLiteJsonCacheManagerFactory sqLiteJsonCacheManagerFactory,
IServerObjectManagerFactory serverObjectManagerFactory,
ILoggerFactory loggerFactory
) : IDeserializeProcessFactory
{
public IDeserializeProcess CreateDeserializeProcess(
Uri url,
string streamId,
string? authorizationToken,
IProgress<ProgressArgs>? progress,
CancellationToken cancellationToken,
DeserializeProcessOptions? options = null
)
{
var sqLiteJsonCacheManager = sqLiteJsonCacheManagerFactory.CreateFromStream(streamId);
var serverObjectManager = serverObjectManagerFactory.Create(url, streamId, authorizationToken);
return new DeserializeProcess(
sqLiteJsonCacheManager,
serverObjectManager,
progress,
baseDeserializer,
loggerFactory,
cancellationToken,
options
);
}
public IDeserializeProcess CreateDeserializeProcess(
ConcurrentDictionary<Id, Json> jsonCache,
ConcurrentDictionary<string, string> objects,
IProgress<ProgressArgs>? progress,
CancellationToken cancellationToken,
DeserializeProcessOptions? options = null
) =>
new DeserializeProcess(
new MemoryJsonCacheManager(jsonCache),
new MemoryServerObjectManager(objects),
progress,
baseDeserializer,
loggerFactory,
cancellationToken,
options
);
}
@@ -1,68 +0,0 @@
using System.Text;
using Speckle.Sdk.Serialisation.V2.Send;
using Speckle.Sdk.SQLite;
using Speckle.Sdk.Transports;
namespace Speckle.Sdk.Serialisation.V2;
#pragma warning disable CA1063
public class DummySqLiteJsonCacheManager : ISqLiteJsonCacheManager
#pragma warning restore CA1063
{
#pragma warning disable CA1816
#pragma warning disable CA1063
public void Dispose() { }
#pragma warning restore CA1063
#pragma warning restore CA1816
public IReadOnlyCollection<(string, string)> GetAllObjects() => throw new NotImplementedException();
public void DeleteObject(string id) => throw new NotImplementedException();
public string? GetObject(string id) => null;
public void SaveObject(string id, string json) => throw new NotImplementedException();
public void UpdateObject(string id, string json) => throw new NotImplementedException();
public virtual void SaveObjects(IEnumerable<(string id, string json)> items) => throw new NotImplementedException();
public bool HasObject(string objectId) => false;
}
public class DummySendServerObjectManager : IServerObjectManager
{
public IAsyncEnumerable<(string, string)> DownloadObjects(
IReadOnlyCollection<string> objectIds,
IProgress<ProgressArgs>? progress,
CancellationToken cancellationToken
) => throw new NotImplementedException();
public Task<string?> DownloadSingleObject(
string objectId,
IProgress<ProgressArgs>? progress,
CancellationToken cancellationToken
) => throw new NotImplementedException();
public Task<Dictionary<string, bool>> HasObjects(
IReadOnlyCollection<string> objectIds,
CancellationToken cancellationToken
) => throw new NotImplementedException();
public Task UploadObjects(
IReadOnlyList<BaseItem> objects,
bool compressPayloads,
IProgress<ProgressArgs>? progress,
CancellationToken cancellationToken
)
{
long totalBytes = 0;
foreach (var item in objects)
{
totalBytes += Encoding.Default.GetByteCount(item.Json.Value);
}
progress?.Report(new(ProgressEvent.UploadBytes, totalBytes, totalBytes));
return Task.CompletedTask;
}
}
@@ -0,0 +1,37 @@
using System.Collections.Concurrent;
using Speckle.Sdk.SQLite;
namespace Speckle.Sdk.Serialisation.V2;
#pragma warning disable CA1063
public class MemoryJsonCacheManager(ConcurrentDictionary<Id, Json> jsonCache) : ISqLiteJsonCacheManager
#pragma warning restore CA1063
{
public IReadOnlyCollection<(string Id, string Json)> GetAllObjects() =>
jsonCache.Select(x => (x.Key.Value, x.Value.Value)).ToList();
public void DeleteObject(string id) => jsonCache.TryRemove(new Id(id), out _);
public string? GetObject(string id) => jsonCache.TryGetValue(new Id(id), out var json) ? json.Value : null;
public void SaveObject(string id, string json) => jsonCache.TryAdd(new Id(id), new Json(json));
public void UpdateObject(string id, string json) => jsonCache[new Id(id)] = new Json(json);
public virtual void SaveObjects(IEnumerable<(string id, string json)> items)
{
foreach (var (id, json) in items)
{
SaveObject(id, json);
}
}
public bool HasObject(string objectId) => jsonCache.ContainsKey(new Id(objectId));
#pragma warning disable CA1063
#pragma warning disable CA1816
public void Dispose()
#pragma warning restore CA1816
#pragma warning restore CA1063
{ }
}
@@ -0,0 +1,48 @@
using System.Collections.Concurrent;
using System.Runtime.CompilerServices;
using Speckle.Sdk.Serialisation.V2.Send;
using Speckle.Sdk.Transports;
namespace Speckle.Sdk.Serialisation.V2;
public class MemoryServerObjectManager(ConcurrentDictionary<string, string> objects) : IServerObjectManager
{
public virtual async IAsyncEnumerable<(string, string)> DownloadObjects(
IReadOnlyCollection<string> objectIds,
IProgress<ProgressArgs>? progress,
[EnumeratorCancellation] CancellationToken cancellationToken
)
{
foreach (var item in objects.Where(x => objectIds.Contains(x.Key)))
{
cancellationToken.ThrowIfCancellationRequested();
yield return (item.Key, item.Value);
}
await Task.CompletedTask.ConfigureAwait(false);
}
public virtual Task<string?> DownloadSingleObject(
string objectId,
IProgress<ProgressArgs>? progress,
CancellationToken cancellationToken
) => Task.FromResult(objects.TryGetValue(objectId, out var json) ? json : null);
public virtual Task<Dictionary<string, bool>> HasObjects(
IReadOnlyCollection<string> objectIds,
CancellationToken cancellationToken
) => Task.FromResult(objectIds.ToDictionary(x => x, objects.ContainsKey));
public virtual Task UploadObjects(
IReadOnlyList<BaseItem> objectToUpload,
bool compressPayloads,
IProgress<ProgressArgs>? progress,
CancellationToken cancellationToken
)
{
foreach (BaseItem baseItem in objectToUpload)
{
objects.TryAdd(baseItem.Id.Value, baseItem.Json.Value);
}
return Task.CompletedTask;
}
}
@@ -0,0 +1,126 @@
using Microsoft.Extensions.Logging;
using Speckle.Sdk.Dependencies;
using Speckle.Sdk.Dependencies.Serialization;
using Speckle.Sdk.SQLite;
using Speckle.Sdk.Transports;
namespace Speckle.Sdk.Serialisation.V2.Send;
public interface IObjectSaver : IDisposable
{
Exception? Exception { get; set; }
Task Start(CancellationToken cancellationToken);
void DoneTraversing();
Task DoneSaving();
void SaveItem(BaseItem item, CancellationToken cancellationToken);
}
public sealed class ObjectSaver(
IProgress<ProgressArgs>? progress,
ISqLiteJsonCacheManager sqLiteJsonCacheManager,
IServerObjectManager serverObjectManager,
ILogger<ObjectSaver> logger,
CancellationToken cancellationToken,
#pragma warning disable CS9107
#pragma warning disable CA2254
SerializeProcessOptions? options = null
) : ChannelSaver<BaseItem>, IObjectSaver
#pragma warning restore CA2254
#pragma warning restore CS9107
{
private readonly CancellationTokenSource _cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(
cancellationToken
);
private readonly SerializeProcessOptions _options = options ?? new();
private long _uploaded;
private long _cached;
private long _objectsSerialized;
protected override async Task SendToServerInternal(Batch<BaseItem> batch)
{
if (_cancellationTokenSource.IsCancellationRequested)
{
return;
}
try
{
if (!_options.SkipServer && batch.Items.Count != 0)
{
var objectBatch = batch.Items.Distinct().ToList();
var hasObjects = await serverObjectManager
.HasObjects(objectBatch.Select(x => x.Id.Value).Freeze(), _cancellationTokenSource.Token)
.ConfigureAwait(false);
objectBatch = batch.Items.Where(x => !hasObjects[x.Id.Value]).ToList();
if (objectBatch.Count != 0)
{
await serverObjectManager
.UploadObjects(objectBatch, true, progress, _cancellationTokenSource.Token)
.ConfigureAwait(false);
Interlocked.Add(ref _uploaded, batch.Items.Count);
}
progress?.Report(new(ProgressEvent.UploadedObjects, _uploaded, null));
}
}
catch (OperationCanceledException)
{
_cancellationTokenSource.Cancel();
}
#pragma warning disable CA1031
catch (Exception e)
#pragma warning restore CA1031
{
RecordException(e);
}
}
public void SaveItem(BaseItem item, CancellationToken cancellationToken)
{
Interlocked.Increment(ref _objectsSerialized);
Save(item, cancellationToken);
}
public override void SaveToCache(List<BaseItem> batch)
{
if (_cancellationTokenSource.IsCancellationRequested)
{
return;
}
try
{
if (!_options.SkipCacheWrite && batch.Count != 0)
{
sqLiteJsonCacheManager.SaveObjects(batch.Select(x => (x.Id.Value, x.Json.Value)));
Interlocked.Add(ref _cached, batch.Count);
progress?.Report(new(ProgressEvent.CachedToLocal, _cached, _objectsSerialized));
}
}
catch (OperationCanceledException)
{
_cancellationTokenSource.Cancel();
}
#pragma warning disable CA1031
catch (Exception e)
#pragma warning restore CA1031
{
RecordException(e);
}
}
private void RecordException(Exception e)
{
//order here matters
logger.LogError(e, "Error in SDK");
Exception = e;
_cancellationTokenSource.Cancel();
}
public void Dispose()
{
_cancellationTokenSource.Dispose();
sqLiteJsonCacheManager.Dispose();
}
}
@@ -4,9 +4,7 @@ using Microsoft.Extensions.Logging;
using Speckle.InterfaceGenerator;
using Speckle.Sdk.Common;
using Speckle.Sdk.Dependencies;
using Speckle.Sdk.Dependencies.Serialization;
using Speckle.Sdk.Models;
using Speckle.Sdk.SQLite;
using Speckle.Sdk.Transports;
namespace Speckle.Sdk.Serialisation.V2.Send;
@@ -28,20 +26,20 @@ public partial interface ISerializeProcess : IAsyncDisposable;
[GenerateAutoInterface]
public sealed class SerializeProcess(
IProgress<ProgressArgs>? progress,
ISqLiteJsonCacheManager sqLiteJsonCacheManager,
IServerObjectManager serverObjectManager,
IObjectSaver objectSaver,
IBaseChildFinder baseChildFinder,
IBaseSerializer baseSerializer,
ILoggerFactory loggerFactory,
CancellationToken cancellationToken,
SerializeProcessOptions? options = null
) : ChannelSaver<BaseItem>, ISerializeProcess
) : ISerializeProcess
{
private static readonly Dictionary<Id, NodeInfo> EMPTY_CLOSURES = new();
private readonly CancellationTokenSource _processSource = CancellationTokenSource.CreateLinkedTokenSource(
cancellationToken
);
private readonly ILogger<SerializeProcess> _logger = loggerFactory.CreateLogger<SerializeProcess>();
//async dispose
[SuppressMessage("Usage", "CA2213:Disposable fields should be disposed")]
@@ -59,7 +57,6 @@ public sealed class SerializeProcess(
Environment.ProcessorCount * 2
);
private readonly SerializeProcessOptions _options = options ?? new();
private readonly ILogger<SerializeProcess> _logger = loggerFactory.CreateLogger<SerializeProcess>();
private readonly Pool<Dictionary<Id, NodeInfo>> _currentClosurePool = Pools.CreateDictionaryPool<Id, NodeInfo>();
private readonly Pool<ConcurrentDictionary<Id, NodeInfo>> _childClosurePool = Pools.CreateConcurrentDictionaryPool<
@@ -72,25 +69,22 @@ public sealed class SerializeProcess(
private long _objectsSerialized;
private long _uploaded;
private long _cached;
[AutoInterfaceIgnore]
public async ValueTask DisposeAsync()
{
await WaitForSchedulerCompletion().ConfigureAwait(false);
await _highest.DisposeAsync().ConfigureAwait(false);
await _belowNormal.DisposeAsync().ConfigureAwait(false);
sqLiteJsonCacheManager.Dispose();
objectSaver.Dispose();
_processSource.Dispose();
}
private void ThrowIfFailed()
{
//order here matters...null with cancellation means a user did it, otherwise it's a real Exception
if (Exception is not null)
if (objectSaver.Exception is not null)
{
throw new SpeckleException("Error while sending", Exception);
throw new SpeckleException("Error while sending", objectSaver.Exception);
}
_processSource.Token.ThrowIfCancellationRequested();
}
@@ -105,7 +99,7 @@ public sealed class SerializeProcess(
{
try
{
var channelTask = Start(_processSource.Token);
var channelTask = objectSaver.Start(_processSource.Token);
var findTotalObjectsTask = Task.CompletedTask;
if (!_options.SkipFindTotalObjects)
{
@@ -120,10 +114,10 @@ public sealed class SerializeProcess(
await Traverse(root, _processSource.Token).ConfigureAwait(false);
ThrowIfFailed();
DoneTraversing();
objectSaver.DoneTraversing();
await Task.WhenAll(findTotalObjectsTask, channelTask).ConfigureAwait(false);
ThrowIfFailed();
await DoneSaving().ConfigureAwait(false);
await objectSaver.DoneSaving().ConfigureAwait(false);
ThrowIfFailed();
await WaitForSchedulerCompletion().ConfigureAwait(false);
ThrowIfFailed();
@@ -254,7 +248,7 @@ public sealed class SerializeProcess(
}
Interlocked.Increment(ref _objectsSerialized);
Save(item, childCancellationTokenSource.Token);
objectSaver.SaveItem(item, childCancellationTokenSource.Token);
}
if (!currentClosures.ContainsKey(item.Id))
@@ -283,76 +277,11 @@ public sealed class SerializeProcess(
}
}
protected override async Task SendToServerInternal(Batch<BaseItem> batch)
{
if (_processSource.IsCancellationRequested)
{
return;
}
try
{
if (!_options.SkipServer && batch.Items.Count != 0)
{
var objectBatch = batch.Items.Distinct().ToList();
var hasObjects = await serverObjectManager
.HasObjects(objectBatch.Select(x => x.Id.Value).Freeze(), _processSource.Token)
.ConfigureAwait(false);
objectBatch = batch.Items.Where(x => !hasObjects[x.Id.Value]).ToList();
if (objectBatch.Count != 0)
{
await serverObjectManager
.UploadObjects(objectBatch, true, progress, _processSource.Token)
.ConfigureAwait(false);
Interlocked.Exchange(ref _uploaded, _uploaded + batch.Items.Count);
}
progress?.Report(new(ProgressEvent.UploadedObjects, _uploaded, null));
}
}
catch (OperationCanceledException)
{
_processSource.Cancel();
}
#pragma warning disable CA1031
catch (Exception e)
#pragma warning restore CA1031
{
RecordException(e);
}
}
public override void SaveToCache(List<BaseItem> batch)
{
if (_processSource.IsCancellationRequested)
{
return;
}
try
{
if (!_options.SkipCacheWrite && batch.Count != 0)
{
sqLiteJsonCacheManager.SaveObjects(batch.Select(x => (x.Id.Value, x.Json.Value)));
Interlocked.Exchange(ref _cached, _cached + batch.Count);
progress?.Report(new(ProgressEvent.CachedToLocal, _cached, _objectsSerialized));
}
}
catch (OperationCanceledException)
{
_processSource.Cancel();
}
#pragma warning disable CA1031
catch (Exception e)
#pragma warning restore CA1031
{
RecordException(e);
}
}
private void RecordException(Exception e)
{
//order here matters
_logger.LogError(e, "Error in SDK");
Exception = e;
objectSaver.Exception = e;
_processSource.Cancel();
}
}
@@ -1,35 +1,16 @@
using System.Collections.Concurrent;
using Microsoft.Extensions.Logging;
using Speckle.Sdk.Serialisation.V2.Receive;
using Speckle.InterfaceGenerator;
using Speckle.Sdk.Serialisation.V2.Send;
using Speckle.Sdk.SQLite;
using Speckle.Sdk.Transports;
namespace Speckle.Sdk.Serialisation.V2;
public interface ISerializeProcessFactory
{
ISerializeProcess CreateSerializeProcess(
Uri url,
string streamId,
string? authorizationToken,
IProgress<ProgressArgs>? progress,
CancellationToken cancellationToken,
SerializeProcessOptions? options = null
);
IDeserializeProcess CreateDeserializeProcess(
Uri url,
string streamId,
string? authorizationToken,
IProgress<ProgressArgs>? progress,
CancellationToken cancellationToken,
DeserializeProcessOptions? options = null
);
}
[GenerateAutoInterface]
public class SerializeProcessFactory(
IBaseChildFinder baseChildFinder,
IObjectSerializerFactory objectSerializerFactory,
IBaseDeserializer baseDeserializer,
ISqLiteJsonCacheManagerFactory sqLiteJsonCacheManagerFactory,
IServerObjectManagerFactory serverObjectManagerFactory,
ILoggerFactory loggerFactory
@@ -46,34 +27,54 @@ public class SerializeProcessFactory(
{
var sqLiteJsonCacheManager = sqLiteJsonCacheManagerFactory.CreateFromStream(streamId);
var serverObjectManager = serverObjectManagerFactory.Create(url, streamId, authorizationToken);
return new SerializeProcess(
return CreateSerializeProcess(sqLiteJsonCacheManager, serverObjectManager, progress, cancellationToken, options);
}
public ISerializeProcess CreateSerializeProcess(
ISqLiteJsonCacheManager sqLiteJsonCacheManager,
IServerObjectManager serverObjectManager,
IProgress<ProgressArgs>? progress,
CancellationToken cancellationToken,
SerializeProcessOptions? options = null
) =>
new SerializeProcess(
progress,
sqLiteJsonCacheManager,
serverObjectManager,
new ObjectSaver(
progress,
sqLiteJsonCacheManager,
serverObjectManager,
loggerFactory.CreateLogger<ObjectSaver>(),
cancellationToken
),
baseChildFinder,
new BaseSerializer(sqLiteJsonCacheManager, objectSerializerFactory),
loggerFactory,
cancellationToken,
options
);
}
public IDeserializeProcess CreateDeserializeProcess(
Uri url,
string streamId,
string? authorizationToken,
public ISerializeProcess CreateSerializeProcess(
ConcurrentDictionary<Id, Json> jsonCache,
ConcurrentDictionary<string, string> objects,
IProgress<ProgressArgs>? progress,
CancellationToken cancellationToken,
DeserializeProcessOptions? options = null
SerializeProcessOptions? options = null
)
{
var sqLiteJsonCacheManager = sqLiteJsonCacheManagerFactory.CreateFromStream(streamId);
var serverObjectManager = serverObjectManagerFactory.Create(url, streamId, authorizationToken);
return new DeserializeProcess(
sqLiteJsonCacheManager,
serverObjectManager,
#pragma warning disable CA2000
var memoryJsonCacheManager = new MemoryJsonCacheManager(jsonCache);
#pragma warning restore CA2000
return new SerializeProcess(
progress,
baseDeserializer,
new ObjectSaver(
progress,
memoryJsonCacheManager,
new MemoryServerObjectManager(objects),
loggerFactory.CreateLogger<ObjectSaver>(),
cancellationToken
),
baseChildFinder,
new BaseSerializer(memoryJsonCacheManager, objectSerializerFactory),
loggerFactory,
cancellationToken,
options
+48 -8
View File
@@ -7,25 +7,65 @@ using Speckle.Sdk.Logging;
namespace Speckle.Sdk;
public record Application(string Name, string Slug);
public record SpeckleSdkOptions(
Application Application,
string ApplicationVersion,
string? SpeckleVersion,
IEnumerable<Assembly>? Assemblies
);
public static class ServiceRegistration
{
private static string GetAssemblyVersion() =>
Assembly.GetExecutingAssembly().GetName().Version?.ToString() ?? "Unknown";
public static IServiceCollection AddSpeckleSdk(
this IServiceCollection serviceCollection,
HostApplication application,
HostAppVersion version,
string speckleVersion
Application application,
string applicationVersion,
string? speckleVersion = null,
IEnumerable<Assembly>? assemblies = null
) => serviceCollection.AddSpeckleSdk(new(application, applicationVersion, speckleVersion, assemblies));
public static IServiceCollection AddSpeckleSdk(
this IServiceCollection serviceCollection,
Application application,
string applicationVersion,
string? speckleVersion,
params Assembly[] assemblies
) => serviceCollection.AddSpeckleSdk(new(application, applicationVersion, speckleVersion, assemblies));
public static IServiceCollection AddSpeckleSdk(
this IServiceCollection serviceCollection,
Application application,
string applicationVersion,
params Assembly[] assemblies
) => serviceCollection.AddSpeckleSdk(new(application, applicationVersion, null, assemblies));
public static IServiceCollection AddSpeckleSdk(
this IServiceCollection serviceCollection,
SpeckleSdkOptions speckleSdkOptions
)
{
var currentAssembly = Assembly.GetExecutingAssembly();
var allAssembles = speckleSdkOptions.Assemblies?.ToList() ?? [];
if (!allAssembles.Contains(currentAssembly))
{
allAssembles.Add(currentAssembly);
}
TypeLoader.Reset();
TypeLoader.Initialize(allAssembles.ToArray());
serviceCollection.AddLogging();
string name = application.Name;
serviceCollection.AddSingleton<ISpeckleApplication>(
new SpeckleApplication
{
HostApplication = name,
SpeckleVersion = speckleVersion,
HostApplicationVersion = HostApplications.GetVersion(version),
Slug = application.Slug,
HostApplication = speckleSdkOptions.Application.Name,
HostApplicationVersion = speckleSdkOptions.ApplicationVersion,
Slug = speckleSdkOptions.Application.Slug,
SpeckleVersion = speckleSdkOptions.SpeckleVersion ?? GetAssemblyVersion(),
}
);
serviceCollection.TryAddSingleton<ISdkActivityFactory, NullActivityFactory>();
-128
View File
@@ -1,128 +0,0 @@
using System.Diagnostics;
using System.Text;
using Speckle.Sdk.Logging;
namespace Speckle.Sdk.Transports;
/// <summary>
/// Writes speckle objects to disk.
/// </summary>
public class DiskTransport : ICloneable, ITransport
{
public DiskTransport(string? basePath = null)
{
basePath ??= Path.Combine(SpecklePathProvider.UserSpeckleFolderPath, "DiskTransportFiles");
RootPath = Path.Combine(basePath);
Directory.CreateDirectory(RootPath);
}
public string RootPath { get; set; }
public object Clone()
{
return new DiskTransport
{
RootPath = RootPath,
CancellationToken = CancellationToken,
OnErrorAction = OnErrorAction,
OnProgressAction = OnProgressAction,
TransportName = TransportName,
};
}
public string TransportName { get; set; } = "Disk";
public Dictionary<string, object> TransportContext =>
new()
{
{ "name", TransportName },
{ "type", GetType().Name },
{ "basePath", RootPath },
};
public CancellationToken CancellationToken { get; set; }
public IProgress<ProgressArgs>? OnProgressAction { get; set; }
public Action<string, Exception>? OnErrorAction { get; set; }
public int SavedObjectCount { get; private set; }
public TimeSpan Elapsed { get; set; } = TimeSpan.Zero;
public void BeginWrite()
{
SavedObjectCount = 0;
}
public void EndWrite() { }
public Task<string?> GetObject(string id)
{
CancellationToken.ThrowIfCancellationRequested();
var filePath = Path.Combine(RootPath, id);
if (File.Exists(filePath))
{
return Task.FromResult<string?>(File.ReadAllText(filePath, Encoding.UTF8));
}
return Task.FromResult<string?>(null);
}
public void SaveObject(string id, string serializedObject)
{
var stopwatch = Stopwatch.StartNew();
CancellationToken.ThrowIfCancellationRequested();
var filePath = Path.Combine(RootPath, id);
if (File.Exists(filePath))
{
return;
}
try
{
File.WriteAllText(filePath, serializedObject, Encoding.UTF8);
}
catch (Exception ex)
{
throw new TransportException(this, $"Failed to write object {id} to disk", ex);
}
SavedObjectCount++;
stopwatch.Stop();
Elapsed += stopwatch.Elapsed;
}
public Task WriteComplete()
{
return Task.CompletedTask;
}
public async Task<string> CopyObjectAndChildren(string id, ITransport targetTransport)
{
string res = await TransportHelpers
.CopyObjectAndChildrenAsync(id, this, targetTransport, CancellationToken)
.ConfigureAwait(false);
return res;
}
public Task<Dictionary<string, bool>> HasObjects(IReadOnlyList<string> objectIds)
{
Dictionary<string, bool> ret = new();
foreach (string objectId in objectIds)
{
var filePath = Path.Combine(RootPath, objectId);
ret[objectId] = File.Exists(filePath);
}
return Task.FromResult(ret);
}
public override string ToString()
{
return $"Disk Transport @{RootPath}";
}
}
@@ -1,22 +1,14 @@
#pragma warning disable CA1506
using System.Reflection;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Speckle.Sdk;
using Speckle.Sdk.Credentials;
using Speckle.Sdk.Host;
using Speckle.Sdk.Models;
using Speckle.Sdk.Serialisation.V2;
using Speckle.Sdk.Serialisation.V2.Receive;
using Speckle.Sdk.Serialisation.V2.Send;
using Speckle.Sdk.Serialization.Testing;
using Speckle.Sdk.SQLite;
const bool skipCacheReceive = false;
const bool skipCacheSendCheck = true;
const bool skipCacheSendSave = false;
TypeLoader.Reset();
TypeLoader.Initialize(typeof(Base).Assembly, Assembly.GetExecutingAssembly());
var url = "https://latest.speckle.systems/projects/a3ac1b2706/models/59d3b0f3c6"; //small?
var streamId = "a3ac1b2706";
@@ -33,7 +25,7 @@ var streamId = "2099ac4b5f";
var rootId = "30fb4cbe6eb2202b9e7b4a4fcc3dd2b6";*/
var serviceCollection = new ServiceCollection();
serviceCollection.AddSpeckleSdk(HostApplications.Navisworks, HostAppVersion.v2023, "Test");
serviceCollection.AddSpeckleSdk(new("Tests", "test"), "v3");
var serviceProvider = serviceCollection.BuildServiceProvider();
Console.WriteLine("Attach");
@@ -41,21 +33,15 @@ Console.WriteLine("Attach");
var token = serviceProvider.GetRequiredService<IAccountManager>().GetDefaultAccount()?.token;
var progress = new Progress(true);
var factory = new SerializeProcessFactory(
new BaseChildFinder(new BasePropertyGatherer()),
new ObjectSerializerFactory(new BasePropertyGatherer()),
new BaseDeserializer(new ObjectDeserializerFactory()),
serviceProvider.GetRequiredService<ISqLiteJsonCacheManagerFactory>(),
serviceProvider.GetRequiredService<IServerObjectManagerFactory>(),
new NullLoggerFactory()
);
var factory = serviceProvider.GetRequiredService<IDeserializeProcessFactory>();
var process = factory.CreateDeserializeProcess(new Uri(url), streamId, token, progress, default, new(skipCacheReceive));
var @base = await process.Deserialize(rootId).ConfigureAwait(false);
Console.WriteLine("Deserialized");
Console.ReadLine();
Console.WriteLine("Executing");
var process2 = factory.CreateSerializeProcess(
var serializeProcessFactory = serviceProvider.GetRequiredService<ISerializeProcessFactory>();
var serializeProcess = serializeProcessFactory.CreateSerializeProcess(
new Uri(url),
streamId,
token,
@@ -63,8 +49,8 @@ var process2 = factory.CreateSerializeProcess(
default,
new SerializeProcessOptions(skipCacheSendCheck, skipCacheSendSave, true, true)
);
await process2.Serialize(@base).ConfigureAwait(false);
await serializeProcess.Serialize(@base).ConfigureAwait(false);
Console.WriteLine("Detach");
Console.ReadLine();
await process2.DisposeAsync().ConfigureAwait(false);
await serializeProcess.DisposeAsync().ConfigureAwait(false);
#pragma warning restore CA1506
@@ -1,4 +1,6 @@
using Speckle.Sdk.Serialisation.V2;
using System.Collections.Concurrent;
using Speckle.Sdk.Serialisation;
using Speckle.Sdk.Serialisation.V2;
using Speckle.Sdk.Serialisation.V2.Send;
using Speckle.Sdk.Testing.Framework;
using Speckle.Sdk.Transports;
@@ -6,7 +8,7 @@ using Speckle.Sdk.Transports;
namespace Speckle.Sdk.Serialization.Tests;
public sealed class CancellationSqLiteJsonCacheManager(CancellationTokenSource cancellationTokenSource)
: DummySqLiteJsonCacheManager
: MemoryJsonCacheManager(new ConcurrentDictionary<Id, Json>())
{
public override void SaveObjects(IEnumerable<(string id, string json)> items)
{
@@ -24,7 +26,8 @@ public class CancellationSqLiteSendManager(CancellationTokenSource cancellationT
}
}
public class CancellationServerObjectManager(CancellationTokenSource cancellationTokenSource) : DummyServerObjectManager
public class CancellationServerObjectManager(CancellationTokenSource cancellationTokenSource)
: MemoryServerObjectManager(new ConcurrentDictionary<string, string>())
{
public override Task UploadObjects(
IReadOnlyList<BaseItem> objects,
@@ -1,10 +1,11 @@
using System.Collections.Concurrent;
using FluentAssertions;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Speckle.Objects.Geometry;
using Speckle.Sdk.Host;
using Speckle.Sdk.Models;
using Speckle.Sdk.Serialisation;
using Speckle.Sdk.Serialisation.V2;
using Speckle.Sdk.Serialisation.V2.Receive;
using Speckle.Sdk.Serialisation.V2.Send;
using Speckle.Sdk.Serialization.Tests.Framework;
@@ -14,10 +15,15 @@ namespace Speckle.Sdk.Serialization.Tests;
public class CancellationTests
{
private readonly ISerializeProcessFactory _factory;
public CancellationTests()
{
TypeLoader.Reset();
TypeLoader.Initialize(typeof(Base).Assembly, typeof(DetachedTests).Assembly, typeof(Polyline).Assembly);
var serviceCollection = new ServiceCollection();
serviceCollection.AddSpeckleSdk(new("Tests", "test"), "v3", typeof(TestClass).Assembly, typeof(Polyline).Assembly);
var serviceProvider = serviceCollection.BuildServiceProvider();
_factory = serviceProvider.GetRequiredService<ISerializeProcessFactory>();
}
[Fact]
@@ -26,13 +32,11 @@ public class CancellationTests
var testClass = new TestClass() { RegularProperty = "Hello" };
using var cancellationSource = new CancellationTokenSource();
await using var serializeProcess = new SerializeProcess(
await using var serializeProcess = _factory.CreateSerializeProcess(
new ConcurrentDictionary<Id, Json>(),
new ConcurrentDictionary<string, string>(),
null,
new DummySqLiteSendManager(),
new DummyServerObjectManager(),
new BaseChildFinder(new BasePropertyGatherer()),
new BaseSerializer(new DummySqLiteSendManager(), new ObjectSerializerFactory(new BasePropertyGatherer())),
new NullLoggerFactory(),
cancellationSource.Token,
new SerializeProcessOptions(true, true, false, true)
);
@@ -50,15 +54,13 @@ public class CancellationTests
var testClass = new TestClass() { RegularProperty = "Hello" };
using var cancellationSource = new CancellationTokenSource();
await using var serializeProcess = new SerializeProcess(
null,
await using var serializeProcess = _factory.CreateSerializeProcess(
new DummySqLiteSendManager(),
new CancellationServerObjectManager(cancellationSource),
new BaseChildFinder(new BasePropertyGatherer()),
new BaseSerializer(new DummySqLiteSendManager(), new ObjectSerializerFactory(new BasePropertyGatherer())),
new NullLoggerFactory(),
null,
cancellationSource.Token,
new SerializeProcessOptions(true, false, false, true)
new SerializeProcessOptions(true, true, false, true)
);
var ex = await Assert.ThrowsAsync<OperationCanceledException>(
async () => await serializeProcess.Serialize(testClass)
@@ -73,16 +75,14 @@ public class CancellationTests
var testClass = new TestClass() { RegularProperty = "Hello" };
using var cancellationSource = new CancellationTokenSource();
await using var serializeProcess = new SerializeProcess(
await using var serializeProcess = _factory.CreateSerializeProcess(
new DummySqLiteSendManager(),
new CancellationServerObjectManager(cancellationSource),
null,
new CancellationSqLiteSendManager(cancellationSource),
new DummyServerObjectManager(),
new BaseChildFinder(new BasePropertyGatherer()),
new BaseSerializer(new DummySqLiteSendManager(), new ObjectSerializerFactory(new BasePropertyGatherer())),
new NullLoggerFactory(),
cancellationSource.Token,
new SerializeProcessOptions(true, false, false, true)
new SerializeProcessOptions(true, true, false, true)
);
var ex = await Assert.ThrowsAsync<OperationCanceledException>(
async () => await serializeProcess.Serialize(testClass)
);
@@ -94,7 +94,7 @@ public class CancellationTests
[InlineData("RevitObject.json.gz", "3416d3fe01c9196115514c4a2f41617b", 7818)]
public async Task Cancellation_Receive_Cache(string fileName, string rootId, int oldCount)
{
var closures = await TestFileManager.GetFileAsClosures(fileName);
var closures = TestFileManager.GetFileAsClosures(fileName);
closures.Count.Should().Be(oldCount);
using var cancellationSource = new CancellationTokenSource();
@@ -121,7 +121,7 @@ public class CancellationTests
[InlineData("RevitObject.json.gz", "3416d3fe01c9196115514c4a2f41617b", 7818)]
public async Task Cancellation_Receive_Server(string fileName, string rootId, int oldCount)
{
var closures = await TestFileManager.GetFileAsClosures(fileName);
var closures = TestFileManager.GetFileAsClosures(fileName);
closures.Count.Should().Be(oldCount);
using var cancellationSource = new CancellationTokenSource();
@@ -148,7 +148,7 @@ public class CancellationTests
[InlineData("RevitObject.json.gz", "3416d3fe01c9196115514c4a2f41617b", 7818)]
public async Task Cancellation_Receive_Deserialize(string fileName, string rootId, int oldCount)
{
var closures = await TestFileManager.GetFileAsClosures(fileName);
var closures = TestFileManager.GetFileAsClosures(fileName);
closures.Count.Should().Be(oldCount);
using var cancellationSource = new CancellationTokenSource();
@@ -1,10 +1,9 @@
using System.Collections.Concurrent;
using System.Text;
using FluentAssertions;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.DependencyInjection;
using Speckle.Newtonsoft.Json.Linq;
using Speckle.Objects.Geometry;
using Speckle.Sdk.Host;
using Speckle.Sdk.Models;
using Speckle.Sdk.Serialisation;
using Speckle.Sdk.Serialisation.V2;
@@ -16,10 +15,15 @@ namespace Speckle.Sdk.Serialization.Tests;
public class DetachedTests
{
private readonly ISerializeProcessFactory _factory;
public DetachedTests()
{
TypeLoader.Reset();
TypeLoader.Initialize(typeof(Base).Assembly, typeof(DetachedTests).Assembly, typeof(Polyline).Assembly);
var serviceCollection = new ServiceCollection();
serviceCollection.AddSpeckleSdk(new("Tests", "test"), "v3", typeof(TestClass).Assembly, typeof(Polyline).Assembly);
var serviceProvider = serviceCollection.BuildServiceProvider();
_factory = serviceProvider.GetRequiredService<ISerializeProcessFactory>();
}
[Fact]
@@ -30,19 +34,16 @@ public class DetachedTests
@base.detachedProp = new SamplePropBase() { name = "detachedProp" };
@base.attachedProp = new SamplePropBase() { name = "attachedProp" };
var objects = new Dictionary<string, string>();
var objects = new ConcurrentDictionary<string, string>();
await using var process2 = new SerializeProcess(
await using var serializeProcess = _factory.CreateSerializeProcess(
new ConcurrentDictionary<Id, Json>(),
objects,
null,
new DummySendCacheManager(objects),
new DummyServerObjectManager(),
new BaseChildFinder(new BasePropertyGatherer()),
new BaseSerializer(new DummySendCacheManager(objects), new ObjectSerializerFactory(new BasePropertyGatherer())),
new NullLoggerFactory(),
default,
new SerializeProcessOptions(false, false, true, true)
);
await process2.Serialize(@base);
await serializeProcess.Serialize(@base);
await VerifyJsonDictionary(objects);
}
@@ -115,19 +116,16 @@ public class DetachedTests
line = new Polyline() { units = "test", value = [3.0, 4.0] },
};
var objects = new Dictionary<string, string>();
var objects = new ConcurrentDictionary<string, string>();
await using var process2 = new SerializeProcess(
await using var serializeProcess = _factory.CreateSerializeProcess(
new ConcurrentDictionary<Id, Json>(),
objects,
null,
new DummySendCacheManager(objects),
new DummyServerObjectManager(),
new BaseChildFinder(new BasePropertyGatherer()),
new BaseSerializer(new DummySendCacheManager(objects), new ObjectSerializerFactory(new BasePropertyGatherer())),
new NullLoggerFactory(),
default,
new SerializeProcessOptions(false, false, true, true)
);
var results = await process2.Serialize(@base);
var results = await serializeProcess.Serialize(@base);
await VerifyJsonDictionary(objects);
}
@@ -185,19 +183,17 @@ public class DetachedTests
@base.list = new List<double>() { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
@base.list2 = new List<double>() { 1, 10 };
var objects = new Dictionary<string, string>();
var objects = new ConcurrentDictionary<string, string>();
await using var process2 = new SerializeProcess(
await using var serializeProcess = _factory.CreateSerializeProcess(
new ConcurrentDictionary<Id, Json>(),
objects,
null,
new DummySendCacheManager(objects),
new DummyServerObjectManager(),
new BaseChildFinder(new BasePropertyGatherer()),
new BaseSerializer(new DummySendCacheManager(objects), new ObjectSerializerFactory(new BasePropertyGatherer())),
new NullLoggerFactory(),
default,
new SerializeProcessOptions(false, false, true, true)
);
var results = await process2.Serialize(@base);
var results = await serializeProcess.Serialize(@base);
objects.Count.Should().Be(3);
var x = JObject.Parse(objects["efeadaca70a85ae6d3acfc93a8b380db"]);
@@ -220,19 +216,16 @@ public class DetachedTests
@base.list2 = new List<double>() { 1, 10 };
@base.arr = [1, 10];
var objects = new Dictionary<string, string>();
var objects = new ConcurrentDictionary<string, string>();
await using var process2 = new SerializeProcess(
await using var serializeProcess = _factory.CreateSerializeProcess(
new ConcurrentDictionary<Id, Json>(),
objects,
null,
new DummySendCacheManager(objects),
new DummyServerObjectManager(),
new BaseChildFinder(new BasePropertyGatherer()),
new BaseSerializer(new DummySendCacheManager(objects), new ObjectSerializerFactory(new BasePropertyGatherer())),
new NullLoggerFactory(),
default,
new SerializeProcessOptions(false, false, true, true)
);
var results = await process2.Serialize(@base);
var results = await serializeProcess.Serialize(@base);
await VerifyJsonDictionary(objects);
}
}
@@ -1,9 +1,11 @@
using System.Collections.Concurrent;
using FluentAssertions;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Speckle.Objects.Geometry;
using Speckle.Sdk.Host;
using Speckle.Sdk.Models;
using Speckle.Sdk.Serialisation;
using Speckle.Sdk.Serialisation.V2;
using Speckle.Sdk.Serialisation.V2.Receive;
using Speckle.Sdk.Serialisation.V2.Send;
using Speckle.Sdk.Serialization.Tests.Framework;
@@ -13,10 +15,15 @@ namespace Speckle.Sdk.Serialization.Tests;
public class ExceptionTests
{
private readonly ISerializeProcessFactory _factory;
public ExceptionTests()
{
TypeLoader.Reset();
TypeLoader.Initialize(typeof(Base).Assembly, typeof(DetachedTests).Assembly, typeof(Polyline).Assembly);
var serviceCollection = new ServiceCollection();
serviceCollection.AddSpeckleSdk(new("Tests", "test"), "v3", typeof(TestClass).Assembly, typeof(Polyline).Assembly);
var serviceProvider = serviceCollection.BuildServiceProvider();
_factory = serviceProvider.GetRequiredService<ISerializeProcessFactory>();
}
[Fact]
@@ -24,20 +31,18 @@ public class ExceptionTests
{
var testClass = new TestClass() { RegularProperty = "Hello" };
var objects = new Dictionary<string, string>();
await using var process2 = new SerializeProcess(
null,
new DummySendCacheManager(objects),
var objects = new ConcurrentDictionary<Id, Json>();
await using var serializeProcess = _factory.CreateSerializeProcess(
new MemoryJsonCacheManager(objects),
new ExceptionServerObjectManager(),
new BaseChildFinder(new BasePropertyGatherer()),
new BaseSerializer(new DummySendCacheManager(objects), new ObjectSerializerFactory(new BasePropertyGatherer())),
new NullLoggerFactory(),
null,
default,
new SerializeProcessOptions(false, false, false, true)
);
//4 exceptions are fine because we use 4 threads for saving cache
var ex = await Assert.ThrowsAsync<SpeckleException>(async () => await process2.Serialize(testClass));
var ex = await Assert.ThrowsAsync<SpeckleException>(async () => await serializeProcess.Serialize(testClass));
await Verify(ex);
}
@@ -46,18 +51,15 @@ public class ExceptionTests
{
var testClass = new TestClass() { RegularProperty = "Hello" };
await using var process2 = new SerializeProcess(
null,
await using var serializeProcess = _factory.CreateSerializeProcess(
new ExceptionSendCacheManager(),
new DummyServerObjectManager(),
new BaseChildFinder(new BasePropertyGatherer()),
new BaseSerializer(new ExceptionSendCacheManager(), new ObjectSerializerFactory(new BasePropertyGatherer())),
new NullLoggerFactory(),
new MemoryServerObjectManager(new()),
null,
default,
new SerializeProcessOptions(false, false, false, true)
);
var ex = await Assert.ThrowsAsync<SpeckleException>(async () => await process2.Serialize(testClass));
var ex = await Assert.ThrowsAsync<SpeckleException>(async () => await serializeProcess.Serialize(testClass));
await Verify(ex);
}
@@ -66,19 +68,15 @@ public class ExceptionTests
{
var testClass = new TestClass() { RegularProperty = "Hello" };
var jsonManager = new ExceptionSendCacheManager(exceptionsAfter: 10);
await using var process2 = new SerializeProcess(
await using var serializeProcess = _factory.CreateSerializeProcess(
new ExceptionSendCacheManager(exceptionsAfter: 10),
new MemoryServerObjectManager(new()),
null,
jsonManager,
new DummyServerObjectManager(),
new BaseChildFinder(new BasePropertyGatherer()),
new BaseSerializer(jsonManager, new ObjectSerializerFactory(new BasePropertyGatherer())),
new NullLoggerFactory(),
default,
new SerializeProcessOptions(false, false, false, true)
);
var ex = await Assert.ThrowsAsync<SpeckleException>(async () => await process2.Serialize(testClass));
var ex = await Assert.ThrowsAsync<SpeckleException>(async () => await serializeProcess.Serialize(testClass));
await Verify(ex);
}
@@ -111,7 +109,7 @@ public class ExceptionTests
[InlineData("RevitObject.json.gz", "3416d3fe01c9196115514c4a2f41617b", 7818)]
public async Task Test_Exceptions_Receive_Server(string fileName, string rootId, int oldCount)
{
var closures = await TestFileManager.GetFileAsClosures(fileName);
var closures = TestFileManager.GetFileAsClosures(fileName);
closures.Count.Should().Be(oldCount);
await using var process = new DeserializeProcess(
@@ -136,7 +134,7 @@ public class ExceptionTests
[InlineData("RevitObject.json.gz", "3416d3fe01c9196115514c4a2f41617b", 7818, true)]
public async Task Test_Exceptions_Receive_Cache(string fileName, string rootId, int oldCount, bool? hasObject)
{
var closures = await TestFileManager.GetFileAsClosures(fileName);
var closures = TestFileManager.GetFileAsClosures(fileName);
closures.Count.Should().Be(oldCount);
await using var process = new DeserializeProcess(
@@ -1,16 +1,23 @@
using Microsoft.Extensions.Logging.Abstractions;
using Speckle.Sdk.Host;
using System.Collections.Concurrent;
using Microsoft.Extensions.DependencyInjection;
using Speckle.Sdk.Models;
using Speckle.Sdk.Serialisation;
using Speckle.Sdk.Serialisation.V2;
using Speckle.Sdk.Serialisation.V2.Send;
namespace Speckle.Sdk.Serialization.Tests;
public class ExplicitInterfaceTests
{
private readonly ISerializeProcessFactory _factory;
public ExplicitInterfaceTests()
{
TypeLoader.Reset();
TypeLoader.Initialize(typeof(Base).Assembly, typeof(TestClass).Assembly);
var serviceCollection = new ServiceCollection();
serviceCollection.AddSpeckleSdk(new("Tests", "test"), "v3", typeof(TestClass).Assembly);
var serviceProvider = serviceCollection.BuildServiceProvider();
_factory = serviceProvider.GetRequiredService<ISerializeProcessFactory>();
}
[Fact]
@@ -18,19 +25,16 @@ public class ExplicitInterfaceTests
{
var testClass = new TestClass() { RegularProperty = "Hello" };
var objects = new Dictionary<string, string>();
await using var process2 = new SerializeProcess(
var objects = new ConcurrentDictionary<string, string>();
await using var serializeProcess = _factory.CreateSerializeProcess(
new ConcurrentDictionary<Id, Json>(),
objects,
null,
new DummySendCacheManager(objects),
new DummyServerObjectManager(),
new BaseChildFinder(new BasePropertyGatherer()),
new BaseSerializer(new DummySendCacheManager(objects), new ObjectSerializerFactory(new BasePropertyGatherer())),
new NullLoggerFactory(),
default,
new SerializeProcessOptions(false, false, true, true)
new SerializeProcessOptions(true, true, false, true)
);
await process2.Serialize(testClass);
await serializeProcess.Serialize(testClass);
await VerifyJsonDictionary(objects);
}
@@ -1,7 +1,8 @@
using System.IO.Compression;
using System.Collections.Concurrent;
using System.IO.Compression;
using System.Reflection;
using Speckle.Newtonsoft.Json.Linq;
using Speckle.Objects.Data;
using Speckle.Objects.Geometry;
using Speckle.Sdk.Common;
using Speckle.Sdk.Host;
using Speckle.Sdk.Models;
@@ -10,49 +11,50 @@ namespace Speckle.Sdk.Serialization.Tests.Framework;
public static class TestFileManager
{
static TestFileManager()
{
TypeLoader.Reset();
TypeLoader.Initialize(typeof(Base).Assembly, typeof(DataObject).Assembly, _assembly);
}
private static readonly Assembly s_assembly = Assembly.GetExecutingAssembly(); //test
private static readonly Assembly s_speckleAssembly = typeof(Base).Assembly; //speckle.sdk
private static readonly Assembly s_speckleObjectsAssembly = typeof(Polyline).Assembly; //speckle.sdk
private static readonly Dictionary<string, IReadOnlyDictionary<string, string>> s_objects = new();
private static readonly Assembly _assembly = Assembly.GetExecutingAssembly();
private static readonly Dictionary<string, IReadOnlyDictionary<string, string>> _objects = new();
public static async Task<IReadOnlyDictionary<string, string>> GetFileAsClosures(string fileName)
public static IReadOnlyDictionary<string, string> GetFileAsClosures(string fileName)
{
if (!_objects.TryGetValue(fileName, out var closure))
lock (s_objects)
{
var fullName = _assembly.GetManifestResourceNames().Single(x => x.EndsWith(fileName));
var json = await ReadJson(fullName);
closure = ReadAsObjects(json);
_objects.Add(fileName, closure);
if (!s_objects.TryGetValue(fileName, out var closure))
{
TypeLoader.Reset();
TypeLoader.Initialize(s_assembly, s_speckleAssembly, s_speckleObjectsAssembly);
var fullName = s_assembly.GetManifestResourceNames().Single(x => x.EndsWith(fileName));
var json = ReadJson(fullName);
closure = ReadAsObjects(json);
s_objects.Add(fileName, closure);
}
return closure;
}
return closure;
}
private static async Task<string> ReadJson(string fullName)
private static string ReadJson(string fullName)
{
await using var stream = _assembly.GetManifestResourceStream(fullName).NotNull();
using var stream = s_assembly.GetManifestResourceStream(fullName).NotNull();
if (fullName.EndsWith(".gz"))
{
await using var z = new GZipStream(stream, CompressionMode.Decompress);
using var z = new GZipStream(stream, CompressionMode.Decompress);
using var reader2 = new StreamReader(z);
return await reader2.ReadToEndAsync();
return reader2.ReadToEnd();
}
using var reader = new StreamReader(stream);
return await reader.ReadToEndAsync();
return reader.ReadToEnd();
}
private static Dictionary<string, string> ReadAsObjects(string json)
private static ConcurrentDictionary<string, string> ReadAsObjects(string json)
{
var jsonObjects = new Dictionary<string, string>();
var jsonObjects = new ConcurrentDictionary<string, string>();
var array = JArray.Parse(json);
foreach (var obj in array)
{
if (obj is JObject jobj)
{
jsonObjects.Add(jobj["id"].NotNull().Value<string>().NotNull(), jobj.ToString());
jsonObjects.TryAdd(jobj["id"].NotNull().Value<string>().NotNull(), jobj.ToString());
}
}
return jsonObjects;
@@ -1,13 +1,16 @@
using System.Collections.Concurrent;
using FluentAssertions;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Speckle.Newtonsoft.Json;
using Speckle.Newtonsoft.Json.Linq;
using Speckle.Objects.Geometry;
using Speckle.Sdk.Common;
using Speckle.Sdk.Host;
using Speckle.Sdk.Models;
using Speckle.Sdk.Serialisation;
using Speckle.Sdk.Serialisation.Utilities;
using Speckle.Sdk.Serialisation.V2;
using Speckle.Sdk.Serialisation.V2.Receive;
using Speckle.Sdk.Serialisation.V2.Send;
using Speckle.Sdk.Serialization.Tests.Framework;
@@ -17,32 +20,17 @@ namespace Speckle.Sdk.Serialization.Tests;
public class SerializationTests
{
private class TestLoader(string json) : IObjectLoader
private readonly ISerializeProcessFactory _factory;
public SerializationTests()
{
public Task<(Json, IReadOnlyCollection<Id>)> GetAndCache(string rootId, DeserializeProcessOptions? options)
{
var childrenIds = ClosureParser.GetChildrenIds(new(json), default).Select(x => new Id(x)).ToList();
return Task.FromResult<(Json, IReadOnlyCollection<Id>)>((new(json), childrenIds));
}
var serviceCollection = new ServiceCollection();
serviceCollection.AddSpeckleSdk(new("Tests", "test"), "v3", typeof(TestClass).Assembly, typeof(Polyline).Assembly);
var serviceProvider = serviceCollection.BuildServiceProvider();
public string? LoadId(string id) => null;
public void Dispose() { }
_factory = serviceProvider.GetRequiredService<ISerializeProcessFactory>();
}
/*
[Test]
[TestCase("RevitObject.json")]
public async Task RunTest2(string fileName)
{
var fullName = _assembly.GetManifestResourceNames().Single(x => x.EndsWith(fileName));
var json = await ReadJson(fullName);
var closure = await ReadAsObjects(json);
using DeserializeProcess sut = new(null, new TestLoader(json), new TestTransport(closure));
var @base = await sut.Deserialize("551513ff4f3596024547fc818f1f3f70");
@base.Should().NotBeNull();
}*/
public class TestObjectLoader(IReadOnlyDictionary<string, string> idToObject) : IObjectLoader
{
public Task<(Json, IReadOnlyCollection<Id>)> GetAndCache(string rootId, DeserializeProcessOptions? options)
@@ -66,7 +54,7 @@ public class SerializationTests
[InlineData("RevitObject.json.gz")]
public async Task Basic_Namespace_Validation(string fileName)
{
var closures = await TestFileManager.GetFileAsClosures(fileName);
var closures = TestFileManager.GetFileAsClosures(fileName);
var deserializer = new SpeckleObjectDeserializer
{
ReadTransport = new TestTransport(closures),
@@ -106,7 +94,7 @@ public class SerializationTests
[InlineData("RevitObject.json.gz")]
public async Task Basic_Namespace_Validation_New(string fileName)
{
var closures = await TestFileManager.GetFileAsClosures(fileName);
var closures = TestFileManager.GetFileAsClosures(fileName);
await using var process = new DeserializeProcess(
new TestObjectLoader(closures),
null,
@@ -165,7 +153,7 @@ public class SerializationTests
[InlineData("RevitObject.json.gz", "3416d3fe01c9196115514c4a2f41617b", 7818)]
public async Task Roundtrip_Test_Old(string fileName, string _, int count)
{
var closures = await TestFileManager.GetFileAsClosures(fileName);
var closures = TestFileManager.GetFileAsClosures(fileName);
var deserializer = new SpeckleObjectDeserializer
{
ReadTransport = new TestTransport(closures),
@@ -199,7 +187,7 @@ public class SerializationTests
[InlineData("RevitObject.json.gz", "3416d3fe01c9196115514c4a2f41617b", 7818, 4674)]
public async Task Roundtrip_Test_New(string fileName, string rootId, int oldCount, int newCount)
{
var closures = await TestFileManager.GetFileAsClosures(fileName);
var closures = TestFileManager.GetFileAsClosures(fileName);
closures.Count.Should().Be(oldCount);
Base root;
@@ -226,14 +214,12 @@ public class SerializationTests
}
var newIdToJson = new ConcurrentDictionary<string, string>();
await using (
var serializeProcess = new SerializeProcess(
var serializeProcess = _factory.CreateSerializeProcess(
new ConcurrentDictionary<Id, Json>(),
newIdToJson,
null,
new DummySqLiteSendManager(),
new DummySendServerObjectManager(newIdToJson),
new BaseChildFinder(new BasePropertyGatherer()),
new BaseSerializer(new DummySqLiteSendManager(), new ObjectSerializerFactory(new BasePropertyGatherer())),
new NullLoggerFactory(),
default,
new SerializeProcessOptions(true, true, false, true)
)
@@ -10,6 +10,7 @@
<PackageReference Include="altcover" />
<PackageReference Include="AwesomeAssertions" />
<PackageReference Include="HttpMultipartParser" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" />
<PackageReference Include="Microsoft.NET.Test.Sdk" />
<PackageReference Include="Moq" />
<PackageReference Include="RichardSzalay.MockHttp" />
@@ -25,6 +25,15 @@
"System.Buffers": "4.6.0"
}
},
"Microsoft.Extensions.DependencyInjection": {
"type": "Direct",
"requested": "[2.2.0, )",
"resolved": "2.2.0",
"contentHash": "MZtBIwfDFork5vfjpJdG5g8wuJFt7d/y3LOSVVtDK/76wlbtz6cjltfKHqLx2TKVqTj5/c41t77m1+h20zqtPA==",
"dependencies": {
"Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0"
}
},
"Microsoft.NET.Test.Sdk": {
"type": "Direct",
"requested": "[17.13.0, )",
@@ -1,5 +1,4 @@
using Microsoft.Extensions.DependencyInjection;
using Speckle.Sdk.Host;
namespace Speckle.Sdk.Tests.Integration;
@@ -8,7 +7,7 @@ public static class TestServiceSetup
public static IServiceProvider GetServiceProvider()
{
var serviceCollection = new ServiceCollection();
serviceCollection.AddSpeckleSdk(HostApplications.Navisworks, HostAppVersion.v2023, "Test");
serviceCollection.AddSpeckleSdk(new("Tests", "test"), "v3");
return serviceCollection.BuildServiceProvider();
}
}
@@ -2,7 +2,6 @@
using Microsoft.Extensions.DependencyInjection;
using Speckle.Sdk.Api;
using Speckle.Sdk.Credentials;
using Speckle.Sdk.Host;
using Speckle.Sdk.Models;
using Speckle.Sdk.Transports;
@@ -19,7 +18,7 @@ public sealed class TestDataHelper : IDisposable
public TestDataHelper()
{
var serviceCollection = new ServiceCollection();
serviceCollection.AddSpeckleSdk(HostApplications.Navisworks, HostAppVersion.v2023, "Test");
serviceCollection.AddSpeckleSdk(new("Tests", "test"), "v3");
ServiceProvider = serviceCollection.BuildServiceProvider();
}
@@ -1,21 +0,0 @@
using FluentAssertions;
using Speckle.Sdk.Host;
namespace Speckle.Sdk.Tests.Unit.Host;
public class HostApplicationTests
{
public static TheoryData<HostAppVersion> HostAppVersionData => new(Enum.GetValues<HostAppVersion>().ToList());
[Theory]
[MemberData(nameof(HostAppVersionData))]
public void HostAppVersionParsingTests(HostAppVersion appVersion)
{
// Assert that the string representation starts with 'v'
appVersion.ToString().StartsWith('v').Should().BeTrue();
// Assert that the parsed version is a positive integer
var version = HostApplications.GetVersion(appVersion);
int.Parse(version).Should().BePositive();
}
}
@@ -1,8 +1,6 @@
using System.Reflection;
using FluentAssertions;
using FluentAssertions;
using Microsoft.Extensions.DependencyInjection;
using Speckle.Sdk.Api;
using Speckle.Sdk.Host;
using Speckle.Sdk.Models;
using Speckle.Sdk.Tests.Unit.Host;
@@ -14,8 +12,6 @@ public class SimpleRoundTripTests
public SimpleRoundTripTests()
{
TypeLoader.Reset();
TypeLoader.Initialize(typeof(Base).Assembly, Assembly.GetExecutingAssembly());
var serviceProvider = TestServiceSetup.GetServiceProvider();
_operations = serviceProvider.GetRequiredService<IOperations>();
}
@@ -1,5 +1,5 @@
using Microsoft.Extensions.DependencyInjection;
using Speckle.Sdk.Host;
using System.Reflection;
using Microsoft.Extensions.DependencyInjection;
namespace Speckle.Sdk.Tests.Unit;
@@ -8,7 +8,7 @@ public static class TestServiceSetup
public static IServiceProvider GetServiceProvider()
{
var serviceCollection = new ServiceCollection();
serviceCollection.AddSpeckleSdk(HostApplications.Navisworks, HostAppVersion.v2023, "Test");
serviceCollection.AddSpeckleSdk(new("Tests", "test"), "v3", Assembly.GetExecutingAssembly());
return serviceCollection.BuildServiceProvider();
}
}
@@ -1,39 +0,0 @@
using FluentAssertions;
using Speckle.Sdk.Common;
using Speckle.Sdk.Transports;
namespace Speckle.Sdk.Tests.Unit.Transports;
public sealed class DiskTransportTests : TransportTests, IDisposable
{
private readonly DiskTransport _diskTransport;
private readonly string _basePath = $"./temp_{Guid.NewGuid()}";
private const string ApplicationName = "Speckle Integration Tests";
private readonly string _fullPath;
protected override ITransport Sut => _diskTransport.NotNull();
public DiskTransportTests()
{
_fullPath = Path.Combine(_basePath, ApplicationName);
_diskTransport = new DiskTransport(_fullPath);
}
[Fact]
public void DirectoryCreated_AfterInitialization()
{
// Act
var directoryExists = Directory.Exists(_fullPath);
// Assert
directoryExists.Should().BeTrue();
}
public void Dispose()
{
if (Directory.Exists(_basePath))
{
Directory.Delete(_basePath, true);
}
}
}