diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 8068e7c2..637c7da8 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1,7 +1,11 @@ name: PR Test on: - pull_request: + pull_request: {} + push: + branches: + - "main" # Need to run for codecov to compare against the BASE + jobs: build: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 669d0541..9f47df08 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -46,13 +46,6 @@ jobs: SEMVER: ${{ steps.set-version.outputs.SEMVER }} FILE_VERSION: ${{ steps.set-version.outputs.FILE_VERSION }} - - name: Upload coverage reports to Codecov with GitHub Action - uses: codecov/codecov-action@v5 - continue-on-error: true - with: - fail_ci_if_error: true - files: tests/**/coverage.xml - token: ${{ secrets.CODECOV_TOKEN }} - name: NuGet login (OIDC → temp API key) uses: NuGet/login@v1 diff --git a/Directory.Packages.props b/Directory.Packages.props index 86c51244..8c51176a 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -15,8 +15,8 @@ + - diff --git a/src/Speckle.Automate.Sdk/packages.lock.json b/src/Speckle.Automate.Sdk/packages.lock.json index 5e6185c6..f7174f7c 100644 --- a/src/Speckle.Automate.Sdk/packages.lock.json +++ b/src/Speckle.Automate.Sdk/packages.lock.json @@ -281,7 +281,6 @@ "type": "Project", "dependencies": { "GraphQL.Client": "[6.0.0, )", - "Microsoft.Bcl.AsyncInterfaces": "[5.0.0, )", "Microsoft.CSharp": "[4.7.0, )", "Microsoft.Data.Sqlite": "[7.0.5, )", "Microsoft.Extensions.DependencyInjection.Abstractions": "[2.2.0, )", @@ -292,7 +291,10 @@ } }, "speckle.sdk.dependencies": { - "type": "Project" + "type": "Project", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "[9.0.4, )" + } }, "GraphQL.Client": { "type": "CentralTransitive", @@ -307,9 +309,9 @@ }, "Microsoft.Bcl.AsyncInterfaces": { "type": "CentralTransitive", - "requested": "[5.0.0, )", - "resolved": "8.0.0", - "contentHash": "3WA9q9yVqJp222P3x1wYIGDAkpjAku0TMUaaQV22g6L67AI0LdOIrVS7Ht2vJfLHGSPVuqN94vIr15qn+HEkHw==", + "requested": "[9.0.4, )", + "resolved": "9.0.4", + "contentHash": "9VGI5kxIvrNG2mqLQZnUR6y/3fcnygD8eNpHR+CqfbnIXvea6nehnYknDKQTxZVPMpzpNca+7DxLBmpdB3q0Bw==", "dependencies": { "System.Threading.Tasks.Extensions": "4.5.4" } diff --git a/src/Speckle.Objects/packages.lock.json b/src/Speckle.Objects/packages.lock.json index 17abca99..1fe1cf2d 100644 --- a/src/Speckle.Objects/packages.lock.json +++ b/src/Speckle.Objects/packages.lock.json @@ -228,7 +228,6 @@ "type": "Project", "dependencies": { "GraphQL.Client": "[6.0.0, )", - "Microsoft.Bcl.AsyncInterfaces": "[5.0.0, )", "Microsoft.CSharp": "[4.7.0, )", "Microsoft.Data.Sqlite": "[7.0.5, )", "Microsoft.Extensions.DependencyInjection.Abstractions": "[2.2.0, )", @@ -239,7 +238,10 @@ } }, "speckle.sdk.dependencies": { - "type": "Project" + "type": "Project", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "[9.0.4, )" + } }, "GraphQL.Client": { "type": "CentralTransitive", @@ -254,9 +256,9 @@ }, "Microsoft.Bcl.AsyncInterfaces": { "type": "CentralTransitive", - "requested": "[5.0.0, )", - "resolved": "5.0.0", - "contentHash": "W8DPQjkMScOMTtJbPwmPyj9c3zYSFGawDW3jwlBOOsnY+EzZFLgNQ/UMkK35JmkNOVPdCyPr2Tw7Vv9N+KA3ZQ==", + "requested": "[9.0.4, )", + "resolved": "9.0.4", + "contentHash": "9VGI5kxIvrNG2mqLQZnUR6y/3fcnygD8eNpHR+CqfbnIXvea6nehnYknDKQTxZVPMpzpNca+7DxLBmpdB3q0Bw==", "dependencies": { "System.Threading.Tasks.Extensions": "4.5.4" } diff --git a/src/Speckle.Sdk.Dependencies/ISdkActivity.cs b/src/Speckle.Sdk.Dependencies/ISdkActivity.cs index d832c769..0b3d1cea 100644 --- a/src/Speckle.Sdk.Dependencies/ISdkActivity.cs +++ b/src/Speckle.Sdk.Dependencies/ISdkActivity.cs @@ -5,6 +5,7 @@ public interface ISdkActivity : IDisposable void SetTag(string key, object? value); void RecordException(Exception e); string TraceId { get; } + string SpanId { get; } void SetStatus(SdkActivityStatusCode code); void InjectHeaders(Action header); diff --git a/src/Speckle.Sdk.Dependencies/ISdkActivityFactory.cs b/src/Speckle.Sdk.Dependencies/ISdkActivityFactory.cs index 6db37c9a..927d0f70 100644 --- a/src/Speckle.Sdk.Dependencies/ISdkActivityFactory.cs +++ b/src/Speckle.Sdk.Dependencies/ISdkActivityFactory.cs @@ -1,8 +1,20 @@ using System.Runtime.CompilerServices; +using Speckle.Connectors.Logging; namespace Speckle.Sdk.Logging; public interface ISdkActivityFactory : IDisposable { - ISdkActivity? Start(string? name = default, [CallerMemberName] string source = ""); + ISdkActivity? Start( + string? name = null, + SdkActivityKind kind = SdkActivityKind.Internal, + [CallerMemberName] string source = "" + ); + + ISdkActivity? StartRemote( + string traceContext, + SdkActivityKind kind, + string? name = null, + [CallerMemberName] string source = "" + ); } diff --git a/src/Speckle.Sdk.Dependencies/RepackedChannel.cs b/src/Speckle.Sdk.Dependencies/RepackedChannel.cs new file mode 100644 index 00000000..a5f2e73b --- /dev/null +++ b/src/Speckle.Sdk.Dependencies/RepackedChannel.cs @@ -0,0 +1,49 @@ +using System.Threading.Channels; + +namespace Speckle.Sdk.Dependencies; + +/// +/// For various reasons related to our use of ILRepack.FullAuto, +/// we cannot use Channels from the SDK project. +/// We have to keep usage of it inside the Sdk.Dependencies project. +/// +/// For the sake of quick development, I've wrapped the class here in a type +/// that is safe to use from the SDK project. +/// +/// As and when we need more functions, we can add them here. +/// +/// And yes... I'm not very happy about the way we've set this up +/// +/// +public sealed class RepackedChannel +{ + private readonly Channel _channel; + + public RepackedChannel(int capacity, bool singleReader, bool singleWriter) + { + _channel = Channel.CreateBounded( + new BoundedChannelOptions(capacity) + { + FullMode = BoundedChannelFullMode.Wait, + SingleReader = singleReader, + SingleWriter = singleWriter, + } + ); + } + + public void CompleteWriter() => _channel.Writer.Complete(); + + public ValueTask WriteAsync(T item, CancellationToken cancellationToken) => + _channel.Writer.WriteAsync(item, cancellationToken); + + public IAsyncEnumerable ReadAllAsync(CancellationToken cancellationToken) => + _channel.Reader.ReadAllAsync(cancellationToken); + + // public async Task ReadAllAsync(Func callback, CancellationToken cancellationToken) + // { + // await foreach (T item in _channel.Reader.ReadAllAsync(cancellationToken)) + // { + // await callback.Invoke(item).ConfigureAwait(false); + // } + // } +} diff --git a/src/Speckle.Sdk.Dependencies/Speckle.Sdk.Dependencies.csproj b/src/Speckle.Sdk.Dependencies/Speckle.Sdk.Dependencies.csproj index 043b7876..fe91aa91 100644 --- a/src/Speckle.Sdk.Dependencies/Speckle.Sdk.Dependencies.csproj +++ b/src/Speckle.Sdk.Dependencies/Speckle.Sdk.Dependencies.csproj @@ -28,4 +28,36 @@ + + + + + + + <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)System.Numerics.Vectors.dll" /> + <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)System.Runtime.CompilerServices.Unsafe.dll" /> + <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)System.Memory.dll" /> + <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)Open.ChannelExtensions.dll" /> + <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)System.Threading.Channels.dll" /> + <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)System.Collections.Immutable.dll" /> + <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)Polly.dll" /> + <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)Polly.Contrib.WaitAndRetry.dll" /> + <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)Polly.Extensions.Http.dll" /> + <_ILRepackIncludeAssemblies_Items Include="$(OutputPath)Microsoft.Extensions.ObjectPool.dll" /> + <_ILRepackExcludeAssemblies_Items Include="$(OutputPath)*.dll" Exclude="@(_ILRepackIncludeAssemblies_Items)" /> + + + + @(_ILRepackExcludeAssemblies_Items) + + diff --git a/src/Speckle.Sdk.Dependencies/SpeckleActivityKind.cs b/src/Speckle.Sdk.Dependencies/SpeckleActivityKind.cs new file mode 100644 index 00000000..8ca93543 --- /dev/null +++ b/src/Speckle.Sdk.Dependencies/SpeckleActivityKind.cs @@ -0,0 +1,30 @@ +namespace Speckle.Connectors.Logging; + +public enum SdkActivityKind +{ + /// + /// Default value. + /// Indicates that the Activity represents an internal operation within an application, as opposed to an operations with remote parents or children. + /// + Internal = 0, + + /// + /// Server activity represents request incoming from external component. + /// + Server = 1, + + /// + /// Client activity represents outgoing request to the external component. + /// + Client = 2, + + /// + /// Producer activity represents output provided to external components. + /// + Producer = 3, + + /// + /// Consumer activity represents output received from an external component. + /// + Consumer = 4, +} diff --git a/src/Speckle.Sdk.Dependencies/packages.lock.json b/src/Speckle.Sdk.Dependencies/packages.lock.json index c664417c..ad39f5df 100644 --- a/src/Speckle.Sdk.Dependencies/packages.lock.json +++ b/src/Speckle.Sdk.Dependencies/packages.lock.json @@ -11,6 +11,15 @@ "ILRepack": "2.0.33" } }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "Direct", + "requested": "[9.0.4, )", + "resolved": "9.0.4", + "contentHash": "9VGI5kxIvrNG2mqLQZnUR6y/3fcnygD8eNpHR+CqfbnIXvea6nehnYknDKQTxZVPMpzpNca+7DxLBmpdB3q0Bw==", + "dependencies": { + "System.Threading.Tasks.Extensions": "4.5.4" + } + }, "Microsoft.Extensions.ObjectPool": { "type": "Direct", "requested": "[9.0.4, )", @@ -151,15 +160,6 @@ "dependencies": { "System.Runtime.CompilerServices.Unsafe": "4.5.3" } - }, - "Microsoft.Bcl.AsyncInterfaces": { - "type": "CentralTransitive", - "requested": "[5.0.0, )", - "resolved": "9.0.4", - "contentHash": "9VGI5kxIvrNG2mqLQZnUR6y/3fcnygD8eNpHR+CqfbnIXvea6nehnYknDKQTxZVPMpzpNca+7DxLBmpdB3q0Bw==", - "dependencies": { - "System.Threading.Tasks.Extensions": "4.5.4" - } } }, "net8.0": { diff --git a/src/Speckle.Sdk/Api/Blob/BlobApi.cs b/src/Speckle.Sdk/Api/Blob/BlobApi.cs index 1dd68eb5..10939f6d 100644 --- a/src/Speckle.Sdk/Api/Blob/BlobApi.cs +++ b/src/Speckle.Sdk/Api/Blob/BlobApi.cs @@ -193,30 +193,7 @@ public sealed class BlobApi : IBlobApi using var response = await _unauthedClient.SendAsync(requestMessage, cancellationToken).ConfigureAwait(false); response.EnsureSuccessStatusCode(); - return ParseEtagHeader(response.Headers); - } - - private static string ParseEtagHeader(HttpResponseHeaders headers) - { - if (!headers.TryGetValues("ETag", out var etagValues)) - { - throw new ArgumentException( - "Response does not have an ETag attached to it, cannot use this as an upload", - nameof(headers) - ); - } - - var etagValuesArray = etagValues.ToArray(); - - if (etagValuesArray.Length != 1) - { - throw new ArgumentException( - $"Expected Etag header to have a single value but got {etagValuesArray.Length}", - nameof(headers) - ); - } - - return etagValuesArray[0]; + return BlobApiHelpers.ParseEtagHeader(response.Headers); } /// diff --git a/src/Speckle.Sdk/Api/GraphQL/Inputs/ModelIngestionInputs.cs b/src/Speckle.Sdk/Api/GraphQL/Inputs/ModelIngestionInputs.cs index 1215d034..87cbde66 100644 --- a/src/Speckle.Sdk/Api/GraphQL/Inputs/ModelIngestionInputs.cs +++ b/src/Speckle.Sdk/Api/GraphQL/Inputs/ModelIngestionInputs.cs @@ -14,7 +14,8 @@ public record ModelIngestionCreateInput( string modelId, string projectId, string progressMessage, - SourceDataInput sourceData + SourceDataInput sourceData, + int? maxIdleTimeoutSeconds = null ); public record ModelIngestionUpdateInput(string ingestionId, string projectId, string progressMessage, double? progress); diff --git a/src/Speckle.Sdk/Api/GraphQL/Models/ModelIngestion.cs b/src/Speckle.Sdk/Api/GraphQL/Models/ModelIngestion.cs index 58d0dc67..e5c08dfc 100644 --- a/src/Speckle.Sdk/Api/GraphQL/Models/ModelIngestion.cs +++ b/src/Speckle.Sdk/Api/GraphQL/Models/ModelIngestion.cs @@ -6,7 +6,8 @@ public sealed class ModelIngestion public required DateTime createdAt { get; init; } public required DateTime updatedAt { get; init; } public required string modelId { get; init; } + public required string projectId { get; init; } + public required string userId { get; init; } public required bool cancellationRequested { get; init; } public required ModelIngestionStatusData statusData { get; init; } - // public required LimitedUser user { get; init; } } diff --git a/src/Speckle.Sdk/Api/GraphQL/Models/ModelIngestionStatusData.cs b/src/Speckle.Sdk/Api/GraphQL/Models/ModelIngestionStatusData.cs index a195bdfc..8e98d1e6 100644 --- a/src/Speckle.Sdk/Api/GraphQL/Models/ModelIngestionStatusData.cs +++ b/src/Speckle.Sdk/Api/GraphQL/Models/ModelIngestionStatusData.cs @@ -6,4 +6,5 @@ public sealed class ModelIngestionStatusData { public required ModelIngestionStatus status { get; init; } public required string? progressMessage { get; init; } + public required string? versionId { get; init; } } diff --git a/src/Speckle.Sdk/Api/GraphQL/Resources/ModelIngestionResource.cs b/src/Speckle.Sdk/Api/GraphQL/Resources/ModelIngestionResource.cs index d029a61d..d01a6a10 100644 --- a/src/Speckle.Sdk/Api/GraphQL/Resources/ModelIngestionResource.cs +++ b/src/Speckle.Sdk/Api/GraphQL/Resources/ModelIngestionResource.cs @@ -44,6 +44,8 @@ public sealed class ModelIngestionResource createdAt updatedAt modelId + projectId + userId cancellationRequested statusData { ... on HasModelIngestionStatus { @@ -94,6 +96,8 @@ public sealed class ModelIngestionResource createdAt updatedAt modelId + projectId + userId cancellationRequested statusData { ... on HasModelIngestionStatus { @@ -102,6 +106,10 @@ public sealed class ModelIngestionResource ... on HasProgressMessage { progressMessage } + ... on ModelIngestionSuccessStatus + { + versionId + } } } } @@ -142,6 +150,8 @@ public sealed class ModelIngestionResource createdAt updatedAt modelId + projectId + userId cancellationRequested statusData { ... on HasModelIngestionStatus { @@ -194,6 +204,8 @@ public sealed class ModelIngestionResource createdAt updatedAt modelId + projectId + userId cancellationRequested statusData { ... on HasModelIngestionStatus { @@ -245,6 +257,8 @@ public sealed class ModelIngestionResource createdAt updatedAt modelId + projectId + userId cancellationRequested statusData { ... on HasModelIngestionStatus { @@ -343,6 +357,8 @@ public sealed class ModelIngestionResource createdAt updatedAt modelId + projectId + userId cancellationRequested statusData { ... on HasModelIngestionStatus { @@ -398,6 +414,8 @@ public sealed class ModelIngestionResource createdAt updatedAt modelId + projectId + userId cancellationRequested statusData { ... on HasModelIngestionStatus { @@ -457,6 +475,8 @@ public sealed class ModelIngestionResource createdAt updatedAt modelId + projectId + userId cancellationRequested statusData { ... on HasModelIngestionStatus { diff --git a/src/Speckle.Sdk/Api/GraphQL/Resources/SubscriptionResource.cs b/src/Speckle.Sdk/Api/GraphQL/Resources/SubscriptionResource.cs index 9a014cff..8ba201d1 100644 --- a/src/Speckle.Sdk/Api/GraphQL/Resources/SubscriptionResource.cs +++ b/src/Speckle.Sdk/Api/GraphQL/Resources/SubscriptionResource.cs @@ -229,6 +229,8 @@ public sealed class SubscriptionResource : IDisposable createdAt updatedAt modelId + projectId + userId cancellationRequested statusData { ... on HasModelIngestionStatus { @@ -237,6 +239,10 @@ public sealed class SubscriptionResource : IDisposable ... on HasProgressMessage { progressMessage } + ... on ModelIngestionSuccessStatus + { + versionId + } } } type diff --git a/src/Speckle.Sdk/Helpers/BlobApiHelpers.cs b/src/Speckle.Sdk/Helpers/BlobApiHelpers.cs new file mode 100644 index 00000000..c7979567 --- /dev/null +++ b/src/Speckle.Sdk/Helpers/BlobApiHelpers.cs @@ -0,0 +1,29 @@ +using System.Net.Http.Headers; + +namespace Speckle.Sdk.Helpers; + +public static class BlobApiHelpers +{ + public static string ParseEtagHeader(HttpResponseHeaders headers) + { + if (!headers.TryGetValues("ETag", out var etagValues)) + { + throw new ArgumentException( + "Response does not have an ETag attached to it, cannot use this as an upload", + nameof(headers) + ); + } + + var etagValuesArray = etagValues.ToArray(); + + if (etagValuesArray.Length != 1) + { + throw new ArgumentException( + $"Expected Etag header to have a single value but got {etagValuesArray.Length}", + nameof(headers) + ); + } + + return etagValuesArray[0]; + } +} diff --git a/src/Speckle.Sdk/Helpers/DisposableFile.cs b/src/Speckle.Sdk/Helpers/DisposableFile.cs new file mode 100644 index 00000000..b4200678 --- /dev/null +++ b/src/Speckle.Sdk/Helpers/DisposableFile.cs @@ -0,0 +1,34 @@ +using Microsoft.Extensions.Logging; + +namespace Speckle.Sdk.Helpers; + +/// +/// wrapper around the downloaded file to try and delete the file on Dispose +/// +/// +/// We're using a similar pattern in the Rhino File Importer codebase (see ImportJobFile) +/// +/// +/// +public sealed class DisposableFile(FileInfo file, ILogger logger, bool deleteOnDispose = true) : IDisposable +{ + public FileInfo FileInfo => file; + + public void Dispose() + { + if (!deleteOnDispose) + { + return; + } + + try + { + file.Delete(); + logger.LogInformation("Cleaned up {File}", file); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + logger.LogWarning(ex, "Failed to clean up {File}", file); + } + } +} diff --git a/src/Speckle.Sdk/Helpers/StopwatchPollyfills.cs b/src/Speckle.Sdk/Helpers/StopwatchPollyfills.cs new file mode 100644 index 00000000..83dd0e05 --- /dev/null +++ b/src/Speckle.Sdk/Helpers/StopwatchPollyfills.cs @@ -0,0 +1,21 @@ +using System.Diagnostics; + +namespace Speckle.Sdk.Helpers; + +public static class StopwatchPolyfills +{ +#if !NET7_0_OR_GREATER + private static readonly double s_tickFrequency = (double)TimeSpan.TicksPerSecond / Stopwatch.Frequency; +#endif + + public static TimeSpan GetElapsedTime(long startingTimestamp) + { +#if NET7_0_OR_GREATER + return Stopwatch.GetElapsedTime(startingTimestamp); +#else + + long elapsedTicks = Stopwatch.GetTimestamp() - startingTimestamp; + return new TimeSpan((long)(elapsedTicks * s_tickFrequency)); +#endif + } +} diff --git a/src/Speckle.Sdk/Logging/NullActivityFactory.cs b/src/Speckle.Sdk/Logging/NullActivityFactory.cs index 580cc5ef..039f91b8 100644 --- a/src/Speckle.Sdk/Logging/NullActivityFactory.cs +++ b/src/Speckle.Sdk/Logging/NullActivityFactory.cs @@ -1,8 +1,12 @@ -namespace Speckle.Sdk.Logging; +using Speckle.Connectors.Logging; + +namespace Speckle.Sdk.Logging; public sealed class NullActivityFactory : ISdkActivityFactory { public void Dispose() { } - public ISdkActivity? Start(string? name = default, string source = "") => null; + public ISdkActivity? Start(string? name, SdkActivityKind kind, string source) => null; + + public ISdkActivity? StartRemote(string traceContext, SdkActivityKind kind, string? name, string source) => null; } diff --git a/src/Speckle.Sdk/Models/DynamicBaseMemberType.cs b/src/Speckle.Sdk/Models/DynamicBaseMemberType.cs index fc114380..4522b61e 100644 --- a/src/Speckle.Sdk/Models/DynamicBaseMemberType.cs +++ b/src/Speckle.Sdk/Models/DynamicBaseMemberType.cs @@ -7,12 +7,12 @@ namespace Speckle.Sdk.Models; public enum DynamicBaseMemberType { /// - /// The typed members of the DynamicBase object + /// The typed members of the object /// Instance = 1, /// - /// The dynamically added members of the DynamicBase object + /// The dynamically added members of the object /// Dynamic = 2, @@ -22,8 +22,9 @@ public enum DynamicBaseMemberType Obsolete = 4, /// - /// The typed methods flagged with TODO: + /// Old feature supported in v2 for grasshopper /// + [Obsolete("Feature no longer supported")] SchemaComputed = 16, /// diff --git a/src/Speckle.Sdk/Pipelines/Progress/AggregateProgress.cs b/src/Speckle.Sdk/Pipelines/Progress/AggregateProgress.cs new file mode 100644 index 00000000..74dfa327 --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Progress/AggregateProgress.cs @@ -0,0 +1,12 @@ +namespace Speckle.Sdk.Pipelines.Progress; + +public sealed class AggregateProgress(params IProgress[] progresses) : IProgress +{ + public void Report(T value) + { + foreach (var progress in progresses) + { + progress.Report(value); + } + } +} diff --git a/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManager.cs b/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManager.cs new file mode 100644 index 00000000..8e146ca4 --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManager.cs @@ -0,0 +1,89 @@ +using System.Diagnostics; +using Microsoft.Extensions.Logging; +using Speckle.InterfaceGenerator; +using Speckle.Sdk.Api; +using Speckle.Sdk.Api.GraphQL.Inputs; +using Speckle.Sdk.Api.GraphQL.Models; +using Speckle.Sdk.Helpers; + +namespace Speckle.Sdk.Pipelines.Progress; + +public partial interface IIngestionProgressManager : IProgress; + +/// +/// An implementation for the entire client side Ingestion progress update reporting +/// Will throttles ingestion progress messages and reports their progress +/// +/// +/// Normally we would pick quite a coarse updateInterval to try and spamming the server (1-5s) +/// +[GenerateAutoInterface] +public sealed class IngestionProgressManager( + ILogger logger, + IClient speckleClient, + ModelIngestion ingestion, + TimeSpan updateInterval, + CancellationToken cancellationToken +) : IIngestionProgressManager +{ + public Task? LastUpdate { get; private set; } + + private long _lastUpdatedAt; + private readonly object _lock = new(); + + [AutoInterfaceIgnore] + public void Report(CardProgress value) + { + cancellationToken.ThrowIfCancellationRequested(); + + string trimmedMessage; + lock (_lock) + { + if (ShouldIgnoreProgressUpdate()) + { + return; + } + + _lastUpdatedAt = Stopwatch.GetTimestamp(); + + trimmedMessage = value.Status.TrimEnd('.'); + + LastUpdate = speckleClient + .Ingestion.UpdateProgress( + new ModelIngestionUpdateInput(ingestion.id, ingestion.projectId, trimmedMessage, value.Progress), + cancellationToken + ) + .ContinueWith( + Continuation, + CancellationToken.None, + TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default + ); + } + + logger.LogInformation("Progress update {Message} {Progress}", trimmedMessage, value.Progress); + } + + /// if the update should be ignored, otherwise + private bool ShouldIgnoreProgressUpdate() + { + if (LastUpdate is not null && !LastUpdate.IsCompleted) + { + return true; + } + + TimeSpan msSinceLastUpdate = StopwatchPolyfills.GetElapsedTime(_lastUpdatedAt); + return msSinceLastUpdate < updateInterval; + } + + private void Continuation(Task updateTask) + { + // The progress report failed... could be many reasons. + // For now, we're not letting this fail the Ingestion in any way + // we'll log but otherwise let it slide while leaving no unobserved task exceptions + if (updateTask.IsFaulted) + { + logger.LogWarning(updateTask.Exception, "A progress update failed unexpectedly"); + } + } +} diff --git a/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManagerFactory.cs b/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManagerFactory.cs new file mode 100644 index 00000000..522aeff0 --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManagerFactory.cs @@ -0,0 +1,21 @@ +using Microsoft.Extensions.Logging; +using Speckle.InterfaceGenerator; +using Speckle.Sdk.Api; +using Speckle.Sdk.Api.GraphQL.Models; + +namespace Speckle.Sdk.Pipelines.Progress; + +[GenerateAutoInterface] +public sealed class IngestionProgressManagerFactory(ILogger logger) + : IIngestionProgressManagerFactory +{ + public IIngestionProgressManager CreateInstance( + IClient speckleClient, + ModelIngestion ingestion, + TimeSpan updateInterval, + CancellationToken cancellationToken + ) + { + return new IngestionProgressManager(logger, speckleClient, ingestion, updateInterval, cancellationToken); + } +} diff --git a/src/Speckle.Sdk/Pipelines/Progress/ProgressArgs.cs b/src/Speckle.Sdk/Pipelines/Progress/ProgressArgs.cs new file mode 100644 index 00000000..bfc47dbc --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Progress/ProgressArgs.cs @@ -0,0 +1,6 @@ +namespace Speckle.Sdk.Pipelines.Progress; + +//TODO: rename PipelineProgressArgs +public readonly record struct CardProgress(string Status, double? Progress); + +public readonly record struct StreamProgressArgs(long BytesStreamed, long ExpectedTotalBytes); diff --git a/src/Speckle.Sdk/Pipelines/Progress/ProgressStream.cs b/src/Speckle.Sdk/Pipelines/Progress/ProgressStream.cs new file mode 100644 index 00000000..f75c7837 --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Progress/ProgressStream.cs @@ -0,0 +1,103 @@ +using System.Diagnostics.CodeAnalysis; + +namespace Speckle.Sdk.Pipelines.Progress; + +/// +/// Wraps to report streaming progress as bytes are read/written. +/// +public sealed class ProgressStream(Stream innerStream, IProgress? progress = null) : Stream +{ + private long _bytesStreamed; + + public override bool CanRead => innerStream.CanRead; + public override bool CanSeek => innerStream.CanSeek; + public override bool CanWrite => innerStream.CanWrite; + public override long Length => innerStream.Length; + + public override long Position + { + get => innerStream.Position; + set => innerStream.Position = value; + } + + public override int Read(byte[] buffer, int offset, int count) + { + int bytesRead = innerStream.Read(buffer, offset, count); + ReportProgress(bytesRead); + return bytesRead; + } + + [SuppressMessage( + "Performance", + "CA1835:Prefer the \'Memory\'-based overloads for \'ReadAsync\' and \'WriteAsync\'", + Justification = "Analyser warning forwarded to caller" + )] + public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + int bytesRead = await innerStream.ReadAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); + ReportProgress(bytesRead); + return bytesRead; + } + +#if NET8_0_OR_GREATER + public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + { + int bytesRead = await innerStream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + ReportProgress(bytesRead); + return bytesRead; + } +#endif + + private void ReportProgress(int newBytesProcessed) + { + _bytesStreamed += newBytesProcessed; + progress?.Report(new(_bytesStreamed, Length)); + } + + public override void Flush() => innerStream.Flush(); + + public override Task FlushAsync(CancellationToken cancellationToken) => innerStream.FlushAsync(cancellationToken); + + public override long Seek(long offset, SeekOrigin origin) => innerStream.Seek(offset, origin); + + public override void SetLength(long value) => throw new NotSupportedException(); //intentionally not supporting, as changing length of stream mid-flight will fuck up progress + + public override void Write(byte[] buffer, int offset, int count) + { + innerStream.Write(buffer, offset, count); + ReportProgress(count); + } + + [SuppressMessage( + "Performance", + "CA1835:Prefer the \'Memory\'-based overloads for \'ReadAsync\' and \'WriteAsync\'", + Justification = "Analyser warning forwarded to caller" + )] + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + await innerStream.WriteAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); + ReportProgress(count); + } + +#if NET6_0_OR_GREATER + public override async ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) + { + await innerStream.WriteAsync(buffer, cancellationToken).ConfigureAwait(false); + ReportProgress(buffer.Length); + } +#endif + + protected override void Dispose(bool disposing) + { + innerStream.Dispose(); + base.Dispose(disposing); + } + +#if NET6_0_OR_GREATER + public override async ValueTask DisposeAsync() + { + await innerStream.DisposeAsync().ConfigureAwait(false); + await base.DisposeAsync().ConfigureAwait(false); + } +#endif +} diff --git a/src/Speckle.Sdk/Pipelines/Progress/RenderedStreamProgress.cs b/src/Speckle.Sdk/Pipelines/Progress/RenderedStreamProgress.cs new file mode 100644 index 00000000..b4f8d391 --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Progress/RenderedStreamProgress.cs @@ -0,0 +1,40 @@ +namespace Speckle.Sdk.Pipelines.Progress; + +/// +/// Renders "low level" data stream updates +/// into "high level" that is expected by Ingestion progress and DUI3 +/// +/// +public sealed class RenderedStreamProgress(IProgress progress) : IProgress +{ + public void Report(StreamProgressArgs value) + { + var (suffix, scaleFactor) = GetFileSizeRendering(value.ExpectedTotalBytes); + progress.Report( + new( + $"Uploading data... ({value.BytesStreamed * scaleFactor:F1}/{value.ExpectedTotalBytes * scaleFactor:F1} {suffix})", + (double)value.BytesStreamed / value.ExpectedTotalBytes + ) + ); + } + + private static readonly string[] s_suffixes = ["B", "KB", "MB", "GB", "TB", "PB"]; + + internal static (string suffix, double scaleFactor) GetFileSizeRendering(long value) + { + if (value <= 0) + { + return (s_suffixes[0], 1d); + } + + for (int i = 0; i < s_suffixes.Length; i++) + { + if (value <= Math.Pow(1024, i + 1)) + { + return (s_suffixes[i], 1 / Math.Pow(1024, i)); + } + } + + throw new ArgumentOutOfRangeException(nameof(value), "Value is too large to convert to a file size"); + } +} diff --git a/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs b/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs new file mode 100644 index 00000000..88443633 --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Send/DiskStore.cs @@ -0,0 +1,85 @@ +using System.IO.Compression; +using Microsoft.Extensions.Logging; +using Speckle.InterfaceGenerator; +using Speckle.Sdk.Dependencies; +using Speckle.Sdk.Helpers; +using Speckle.Sdk.Logging; + +namespace Speckle.Sdk.Pipelines.Send; + +[GenerateAutoInterface] +public sealed class DiskStoreFactory(ILogger logger, ISdkActivityFactory activityFactory) : IDiskStoreFactory +{ + public DiskStore CreateInstance(CancellationToken cancellationToken) => + new(logger, activityFactory, cancellationToken); +} + +public sealed class DiskStore +{ + private readonly RepackedChannel _channel; + private readonly Task _writeToDiskTask; + private readonly ILogger _logger; + private readonly ISdkActivityFactory _activityFactory; + private readonly CancellationToken _cancellationToken; + + internal DiskStore( + ILogger logger, + ISdkActivityFactory activityFactory, + CancellationToken cancellationToken + ) + { + _logger = logger; + _activityFactory = activityFactory; + _cancellationToken = cancellationToken; + + _channel = new RepackedChannel(1000, true, false); + _writeToDiskTask = Task.Run(WriteFile, cancellationToken); + } + + public async Task PushAsync(UploadItem item) => + await _channel.WriteAsync(item, _cancellationToken).ConfigureAwait(false); + + public async Task CompleteAsync() + { + using var a = _activityFactory.Start("Waiting for DiskStore to complete"); + _channel.CompleteWriter(); + return await _writeToDiskTask.ConfigureAwait(false); + } + + /// + /// Reads from the Channel and streams the s to a temporary file on disk. + /// Will keep reading until is called. + /// + /// the file that was written + private async Task WriteFile() + { + string tempFilePath = Path.GetTempFileName(); + var tempFile = new DisposableFile(new FileInfo(tempFilePath), _logger); + _logger.LogInformation("Writing temp file to {TempFilePath}", tempFilePath); + + try + { + using var fileStream = new FileStream(tempFilePath, FileMode.Create, FileAccess.Write, FileShare.None); + using var gzip = new GZipStream(fileStream, CompressionLevel.Optimal); + using var writer = new StreamWriter(gzip); + + await foreach (var item in _channel.ReadAllAsync(_cancellationToken).ConfigureAwait(false)) + { + await writer.WriteLineAsync($"{item.Id}\t{item.SpeckleType}\t{item.Json}").ConfigureAwait(false); + } +#if NET8_0_OR_GREATER + await writer.FlushAsync(_cancellationToken).ConfigureAwait(false); +#else + await writer.FlushAsync().ConfigureAwait(false); +#endif + tempFile.FileInfo.Refresh(); + + return tempFile; + } + catch + { + tempFile.Dispose(); + throw; + } + } +} diff --git a/src/Speckle.Sdk/Pipelines/Send/SendPipeline.cs b/src/Speckle.Sdk/Pipelines/Send/SendPipeline.cs new file mode 100644 index 00000000..3ddd4249 --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Send/SendPipeline.cs @@ -0,0 +1,74 @@ +using Speckle.InterfaceGenerator; +using Speckle.Sdk.Credentials; +using Speckle.Sdk.Helpers; +using Speckle.Sdk.Models; +using Speckle.Sdk.Pipelines.Progress; + +namespace Speckle.Sdk.Pipelines.Send; + +[GenerateAutoInterface] +public sealed class SendPipelineFactory(IUploaderFactory uploaderFactory, IDiskStoreFactory diskStoreFactory) + : ISendPipelineFactory +{ + public SendPipeline CreateInstance( + string projectId, + string ingestionId, + Account account, + IProgress uploadProgress, + CancellationToken cancellationToken + ) + { + var uploader = uploaderFactory.CreateInstance(projectId, ingestionId, account, uploadProgress, cancellationToken); + var diskStore = diskStoreFactory.CreateInstance(cancellationToken); + return new SendPipeline(uploader, diskStore); + } +} + +public sealed class SendPipeline : IDisposable +{ + private readonly Serializer _serializer = new(); + private readonly Uploader _uploader; + private readonly DiskStore _diskStore; + + internal SendPipeline(Uploader uploader, DiskStore diskStore) + { + _uploader = uploader; + _diskStore = diskStore; + } + + private UploadItem _lastItem; + + public async Task Process(Base @base) + { + var results = _serializer.Serialize(@base).ToArray(); + var first = results.First(); + foreach (var item in results) + { + // we're not doing fire and forget here so that we get the backpressure from the uploader + await _diskStore.PushAsync(item).ConfigureAwait(false); + } + + // NOTE: this is important to keep track of. When we serialze an object, we get back a list of objects, with the first one being the original root. + // In the case of the commit root object, this means the last object is not necessarily the root; we therefore need to manually track its existance here + // and ensure it's the last one through in the uploader's stream. See WaitForUpload down below. + _lastItem = first; + return first.Reference; + } + + public async Task WaitForUpload() + { + await _diskStore.PushAsync(_lastItem).ConfigureAwait(false); + using DisposableFile tempFile = await _diskStore.CompleteAsync().ConfigureAwait(false); + + using Stream fileStreamUpload = new FileStream( + tempFile.FileInfo.FullName, + FileMode.Open, + FileAccess.Read, + FileShare.Read + ); + + await _uploader.Send(fileStreamUpload).ConfigureAwait(false); + } + + public void Dispose() => _uploader.Dispose(); +} diff --git a/src/Speckle.Sdk/Pipelines/Send/Serializer.cs b/src/Speckle.Sdk/Pipelines/Send/Serializer.cs new file mode 100644 index 00000000..ed78ce73 --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Send/Serializer.cs @@ -0,0 +1,351 @@ +using System.Collections; +using System.Drawing; +using System.Globalization; +using System.Reflection; +using Speckle.DoubleNumerics; +using Speckle.Newtonsoft.Json; +using Speckle.Sdk.Dependencies; +using Speckle.Sdk.Helpers; +using Speckle.Sdk.Models; +using Speckle.Sdk.Serialisation; + +namespace Speckle.Sdk.Pipelines.Send; + +/// +/// Another serializer, cleaner and meaner. Provides methods for serializing Speckle objects into a format suitable for upload or storage. +/// This class handles the conversion of and its derivatives +/// into serialized JSON structures along with associated metadata, closures, and references. +/// Any reference objects coming through are being "passed through" serialized - they do not get double encoded. +/// +internal sealed class Serializer +{ + private readonly record struct PropertyInfo(string Name, object? Value, bool IsDetachable); + + public IEnumerable Serialize(Base root) + { + // Special case: if root is already an ObjectReference, serialize it verbatim + if (root is ObjectReference existingRef) + { + var uploadItem = ReferenceToUploadItem(existingRef); + yield return uploadItem; + yield break; + } + + var detachedObjects = new List<(Id, Json, Dictionary, Base, string)>(); + var rootClosures = new Dictionary(); + + var (rootId, rootJson) = SerializeBase(root, false, rootClosures, detachedObjects); + + var rootReference = new ObjectReference + { + referencedId = rootId.Value, + applicationId = root.applicationId, + closure = rootClosures.Count > 0 ? rootClosures : null, + }; + + yield return new UploadItem(rootId.Value, rootJson, root.speckle_type, rootReference); + + foreach (var (id, json, closures, baseObj, speckleType) in detachedObjects) + { + var reference = new ObjectReference + { + referencedId = id.Value, + applicationId = baseObj.applicationId, + closure = closures.Count > 0 ? closures : null, + }; + + yield return new UploadItem(id.Value, json, speckleType, reference); + } + } + + private IEnumerable ExtractProperties(Base baseObj) + { + var typedProperties = baseObj.GetInstanceMembers(); + foreach (var prop in typedProperties) + { + if (prop.Name == "id" || prop.Name.StartsWith("__")) + { + continue; + } + + if (prop.IsDefined(typeof(JsonIgnoreAttribute), false)) + { + continue; + } + + var value = prop.GetValue(baseObj); + var isDetachable = prop.GetCustomAttribute(true)?.Detachable ?? false; + + yield return new PropertyInfo(prop.Name, value, isDetachable); + } + + foreach (var propName in baseObj.DynamicPropertyKeys) + { + if (propName.StartsWith("__")) + { + continue; + } + + var value = baseObj[propName]; + +#pragma warning disable CA1866 + var isDetachable = propName.StartsWith("@"); +#pragma warning restore CA1866 + + yield return new PropertyInfo(propName, value, isDetachable); + } + } + + private (Id, Json) SerializeBase( + Base baseObj, + bool forceDetach, + Dictionary closures, + List<(Id, Json, Dictionary, Base, string)> detachedObjects + ) + { + var childClosures = new Dictionary(); + + var sb = Pools.StringBuilders.Get(); + try + { + using var stringWriter = new StringWriter(sb); + using var jsonWriter = new JsonTextWriter(stringWriter); + using var idWriter = new SerializerIdWriter(jsonWriter); + + idWriter.WriteStartObject(); + + foreach (var prop in ExtractProperties(baseObj)) + { + idWriter.WritePropertyName(prop.Name); + SerializeValue(prop.Value, idWriter, prop.IsDetachable, childClosures, detachedObjects); + } + + var (jsonForId, finalWriter) = idWriter.FinishIdWriter(); + var id = IdGenerator.ComputeId(jsonForId); + + finalWriter.WritePropertyName("id"); + finalWriter.WriteValue(id.Value); + + baseObj.id = id.Value; + + if ((forceDetach || childClosures.Count > 0) && childClosures.Count > 0) + { + finalWriter.WritePropertyName("__closure"); + finalWriter.WriteStartObject(); + foreach (var kvp in childClosures) + { + finalWriter.WritePropertyName(kvp.Key); + finalWriter.WriteValue(kvp.Value); + } + finalWriter.WriteEndObject(); + + foreach (var kvp in childClosures) + { + closures[kvp.Key] = closures.TryGetValue(kvp.Key, out var existing) ? existing + kvp.Value : kvp.Value; + } + } + + finalWriter.WriteEndObject(); + finalWriter.Flush(); + + var json = new Json(stringWriter.ToString()); + return (id, json); + } + finally + { + Pools.StringBuilders.Return(sb); + } + } + + private void SerializeValue( + object? value, + JsonWriter writer, + bool isDetachable, + Dictionary closures, + List<(Id, Json, Dictionary, Base, string)> detachedObjects + ) + { + switch (value) + { + case Enum: + writer.WriteValue((int)value); + return; + case Guid g: + writer.WriteValue(g.ToString()); + return; + case Color c: + writer.WriteValue(c.ToArgb()); + return; + case DateTime dt: + writer.WriteValue(dt.ToString("o", CultureInfo.InvariantCulture)); + return; + case Matrix4x4 md: + writer.WriteStartArray(); + writer.WriteValue(md.M11); + writer.WriteValue(md.M12); + writer.WriteValue(md.M13); + writer.WriteValue(md.M14); + writer.WriteValue(md.M21); + writer.WriteValue(md.M22); + writer.WriteValue(md.M23); + writer.WriteValue(md.M24); + writer.WriteValue(md.M31); + writer.WriteValue(md.M32); + writer.WriteValue(md.M33); + writer.WriteValue(md.M34); + writer.WriteValue(md.M41); + writer.WriteValue(md.M42); + writer.WriteValue(md.M43); + writer.WriteValue(md.M44); + writer.WriteEndArray(); + return; + // Handle ObjectReference before Base (since ObjectReference extends Base) + // This prevents double-serialization and properly propagates closures + case ObjectReference objRef: + { + writer.WriteStartObject(); + writer.WritePropertyName("speckle_type"); + writer.WriteValue("reference"); + writer.WritePropertyName("referencedId"); + writer.WriteValue(objRef.referencedId); + writer.WriteEndObject(); + + // Propagate closure: add the referenced ID + closures[objRef.referencedId] = closures.TryGetValue(objRef.referencedId, out var existing) ? existing + 1 : 1; + + // Propagate nested closures from the ObjectReference.closure dictionary + if (objRef.closure != null) + { + foreach (var kvp in objRef.closure) + { + closures[kvp.Key] = closures.TryGetValue(kvp.Key, out var existingDepth) + ? existingDepth + kvp.Value + : kvp.Value; + } + } + + return; + } + case Base baseObj: + { + if (isDetachable) + { + var childClosures = new Dictionary(); + var (childId, childJson) = SerializeBase(baseObj, true, childClosures, detachedObjects); + + detachedObjects.Add((childId, childJson, childClosures, baseObj, baseObj.speckle_type)); + + writer.WriteStartObject(); + writer.WritePropertyName("speckle_type"); + writer.WriteValue("reference"); + writer.WritePropertyName("referencedId"); + writer.WriteValue(childId.Value); + writer.WriteEndObject(); + + closures[childId.Value] = closures.TryGetValue(childId.Value, out var existing) ? existing + 1 : 1; + + foreach (var kvp in childClosures) + { + closures[kvp.Key] = closures.TryGetValue(kvp.Key, out var existingDepth) + ? existingDepth + kvp.Value + : kvp.Value; + } + } + else + { + var inlineClosures = new Dictionary(); + var (_, inlineJson) = SerializeBase(baseObj, false, inlineClosures, detachedObjects); + + writer.WriteRawValue(inlineJson.Value); + + foreach (var kvp in inlineClosures) + { + closures[kvp.Key] = closures.TryGetValue(kvp.Key, out var existingDepth) + ? existingDepth + kvp.Value + : kvp.Value; + } + } + return; + } + case IDictionary dict: + { + writer.WriteStartObject(); + foreach (DictionaryEntry kvp in dict) + { + if (kvp.Key is not string key) + { + throw new ArgumentException("Dictionary keys must be strings", nameof(value)); + } + + writer.WritePropertyName(key); + SerializeValue(kvp.Value, writer, false, closures, detachedObjects); + } + writer.WriteEndObject(); + return; + } + case ICollection collection: + { + writer.WriteStartArray(); + foreach (var item in collection) + { + SerializeValue(item, writer, isDetachable, closures, detachedObjects); + } + writer.WriteEndArray(); + return; + } + default: + // This case will handle primitives and `null` + // Will throw JsonWriterException if not supported + writer.WriteValue(value); + return; + } + } + + private UploadItem ReferenceToUploadItem(ObjectReference existingRef) + { + var sb = Pools.StringBuilders.Get(); + try + { + using var stringWriter = new StringWriter(sb); + using var jsonWriter = new JsonTextWriter(stringWriter); + + jsonWriter.WriteStartObject(); + jsonWriter.WritePropertyName("speckle_type"); + jsonWriter.WriteValue("reference"); + jsonWriter.WritePropertyName("referencedId"); + jsonWriter.WriteValue(existingRef.referencedId); + jsonWriter.WritePropertyName("__closure"); + + if (existingRef.closure != null && existingRef.closure.Count > 0) + { + jsonWriter.WriteStartObject(); + foreach (var kvp in existingRef.closure) + { + jsonWriter.WritePropertyName(kvp.Key); + jsonWriter.WriteValue(kvp.Value); + } + jsonWriter.WriteEndObject(); + } + else + { + jsonWriter.WriteNull(); + } + + jsonWriter.WriteEndObject(); + jsonWriter.Flush(); + + var refJson = new Json(stringWriter.ToString()); + + return new UploadItem( + existingRef.referencedId, + refJson, + existingRef.speckle_type, + existingRef // Pass through the original ObjectReference + ); + } + finally + { + Pools.StringBuilders.Return(sb); + } + } +} diff --git a/src/Speckle.Sdk/Pipelines/Send/Uploader.cs b/src/Speckle.Sdk/Pipelines/Send/Uploader.cs new file mode 100644 index 00000000..0b7f2126 --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Send/Uploader.cs @@ -0,0 +1,132 @@ +using System.Net.Http.Headers; +using System.Text; +using Speckle.InterfaceGenerator; +using Speckle.Newtonsoft.Json; +using Speckle.Sdk.Credentials; +using Speckle.Sdk.Helpers; +using Speckle.Sdk.Logging; +using Speckle.Sdk.Pipelines.Progress; + +namespace Speckle.Sdk.Pipelines.Send; + +[GenerateAutoInterface] +public sealed class UploaderFactory(ISpeckleHttp httpClientFactory, ISdkActivityFactory activityFactory) + : IUploaderFactory +{ + public Uploader CreateInstance( + string projectId, + string ingestionId, + Account account, + IProgress progress, + CancellationToken cancellationToken + ) => new(projectId, ingestionId, activityFactory, httpClientFactory, account, progress, cancellationToken); +} + +public sealed class Uploader : IDisposable +{ + private readonly string _projectId; + private readonly string _ingestionId; + private readonly CancellationToken _cancellationToken; + private readonly HttpClient _speckleClient; + private readonly HttpClient _s3Client; + private readonly ISdkActivityFactory _activity; + private readonly IProgress _progress; + + internal Uploader( + string projectId, + string ingestionId, + ISdkActivityFactory activity, + ISpeckleHttp httpClientFactory, + Account speckleAccount, + IProgress progress, + CancellationToken cancellationToken + ) + { + _projectId = projectId; + _ingestionId = ingestionId; + _activity = activity; + _cancellationToken = cancellationToken; + _progress = progress; + _speckleClient = httpClientFactory.CreateHttpClient(authorizationToken: speckleAccount.token); + _speckleClient.BaseAddress = new(new(speckleAccount.serverInfo.url), "/api/v1/"); + + _s3Client = httpClientFactory.CreateHttpClient(); + } + + public async Task Send(Stream fileStream) + { + PresignedUploadResponse presignedUploadResponse = await GetPresignedUrl().ConfigureAwait(false); + var etag = await UploadToS3(fileStream, presignedUploadResponse).ConfigureAwait(false); + + await TriggerProcessing(new() { Etag = etag }).ConfigureAwait(false); + } + + private async Task GetPresignedUrl() + { + using var a = _activity.Start("Get Presigned Url"); + + var signUri = new Uri($"projects/{_projectId}/modelingestion/{_ingestionId}/uploads/sign", UriKind.Relative); + + using var signResponse = await _speckleClient.PostAsync(signUri, null, _cancellationToken).ConfigureAwait(false); + signResponse.EnsureSuccessStatusCode(); + +#if NET5_0_OR_GREATER + string signResponseString = await signResponse.Content.ReadAsStringAsync(_cancellationToken).ConfigureAwait(false); +#else + string signResponseString = await signResponse.Content.ReadAsStringAsync().ConfigureAwait(false); +#endif + PresignedUploadResponse presignedUpload = + JsonConvert.DeserializeObject(signResponseString) + ?? throw new InvalidOperationException("Failed to get presigned upload URL"); + return presignedUpload; + } + + private async Task UploadToS3(Stream fileStream, PresignedUploadResponse presignedUploadResponse) + { + using var a = _activity.Start("Uploading file to pre-signed url"); + + Stream progressStream = new ProgressStream(fileStream, _progress); + + using var streamContent = new StreamContent(progressStream); + streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); + streamContent.Headers.ContentLength = fileStream.Length; + + using var uploadRequest = new HttpRequestMessage(HttpMethod.Put, presignedUploadResponse.Url); + foreach (var kvp in presignedUploadResponse.AdditionalRequestHeaders) + { + uploadRequest.Headers.Add(kvp.Key, kvp.Value); + } + + uploadRequest.Content = streamContent; + + using var uploadResponse = await _s3Client + .SendAsync(uploadRequest, HttpCompletionOption.ResponseHeadersRead, _cancellationToken) + .ConfigureAwait(false); + + uploadResponse.EnsureSuccessStatusCode(); + + return BlobApiHelpers.ParseEtagHeader(uploadResponse.Headers); + } + + private async Task TriggerProcessing(TriggerUploadRequest request) + { + using var a = _activity.Start("Triggering Processing"); + + Uri processUri = new($"projects/{_projectId}/modelingestion/{_ingestionId}/uploads/process", UriKind.Relative); + string requestBody = JsonConvert.SerializeObject(request); + using var content = new StringContent(requestBody, Encoding.UTF8, "application/json"); + + using HttpResponseMessage processResponse = await _speckleClient + .PostAsync(processUri, content, _cancellationToken) + .ConfigureAwait(false); + + string body = await processResponse.Content.ReadAsStringAsync().ConfigureAwait(false); + processResponse.EnsureSuccessStatusCode(); + } + + public void Dispose() + { + _speckleClient.Dispose(); + _s3Client.Dispose(); + } +} diff --git a/src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs b/src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs new file mode 100644 index 00000000..11df1420 --- /dev/null +++ b/src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs @@ -0,0 +1,20 @@ +using Speckle.Newtonsoft.Json; +using Speckle.Sdk.Models; +using Speckle.Sdk.Serialisation; + +namespace Speckle.Sdk.Pipelines.Send; + +public record UploadItem(string Id, Json Json, string SpeckleType, ObjectReference Reference); + +internal record PresignedUploadResponse +{ + public required Uri Url { get; init; } + public required string Key { get; init; } + public Dictionary AdditionalRequestHeaders { get; init; } = new(); +} + +internal readonly struct TriggerUploadRequest +{ + [JsonProperty("etag")] + public required string Etag { get; init; } +} diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/BaseItem.cs b/src/Speckle.Sdk/Serialisation/V2/Send/BaseItem.cs index 2a88e3db..9c5b02bf 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Send/BaseItem.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Send/BaseItem.cs @@ -2,7 +2,13 @@ using System.Text; namespace Speckle.Sdk.Serialisation.V2.Send; -public sealed record BaseItem(Id Id, Json Json, bool NeedsStorage, Dictionary? Closures) : IHasByteSize +public sealed record BaseItem( + Id Id, + Json Json, + bool NeedsStorage, + Dictionary? Closures, + bool? IsReference = false +) : IHasByteSize { public int ByteSize { get; } = Encoding.UTF8.GetByteCount(Json.Value); diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs index 6dac6f6f..29965525 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs @@ -113,16 +113,6 @@ public sealed class SerializeProcess( _processSource.Token ); var findTotalObjectsTask = Task.CompletedTask; - if (!options.SkipFindTotalObjects) - { - ThrowIfFailed(); - findTotalObjectsTask = Task.Factory.StartNew( - () => TraverseTotal(root), - _processSource.Token, - TaskCreationOptions.AttachedToParent | TaskCreationOptions.PreferFairness, - _highest - ); - } await Traverse(root).ConfigureAwait(false); ThrowIfFailed(); @@ -133,6 +123,7 @@ public sealed class SerializeProcess( ThrowIfFailed(); await WaitForSchedulerCompletion().ConfigureAwait(false); ThrowIfFailed(); + return new(root.id.NotNull(), baseSerializer.ObjectReferences.Freeze()); } catch (OperationCanceledException) diff --git a/src/Speckle.Sdk/ServiceRegistration.cs b/src/Speckle.Sdk/ServiceRegistration.cs index 94e147cc..9028ae7c 100644 --- a/src/Speckle.Sdk/ServiceRegistration.cs +++ b/src/Speckle.Sdk/ServiceRegistration.cs @@ -8,6 +8,7 @@ using Speckle.Sdk.Dependencies; using Speckle.Sdk.Host; using Speckle.Sdk.Logging; using Speckle.Sdk.Models.GraphTraversal; +using Speckle.Sdk.Pipelines.Progress; using Speckle.Sdk.Serialisation.V2; using Speckle.Sdk.Serialisation.V2.Receive; using Speckle.Sdk.Serialisation.V2.Send; @@ -96,7 +97,8 @@ public static class ServiceRegistration typeof(DeserializeProcess), typeof(ObjectLoader), typeof(TraversalRule), - typeof(Client) + typeof(Client), + typeof(IngestionProgressManager) ); serviceCollection.AddMatchingInterfacesAsTransient(typeof(GraphQLRetry).Assembly); return serviceCollection; diff --git a/src/Speckle.Sdk/Speckle.Sdk.csproj b/src/Speckle.Sdk/Speckle.Sdk.csproj index 5dbc5924..000ee2bf 100644 --- a/src/Speckle.Sdk/Speckle.Sdk.csproj +++ b/src/Speckle.Sdk/Speckle.Sdk.csproj @@ -35,7 +35,6 @@ - diff --git a/src/Speckle.Sdk/packages.lock.json b/src/Speckle.Sdk/packages.lock.json index 5ea95453..1426afb3 100644 --- a/src/Speckle.Sdk/packages.lock.json +++ b/src/Speckle.Sdk/packages.lock.json @@ -13,15 +13,6 @@ "System.Reactive": "5.0.0" } }, - "Microsoft.Bcl.AsyncInterfaces": { - "type": "Direct", - "requested": "[5.0.0, )", - "resolved": "5.0.0", - "contentHash": "W8DPQjkMScOMTtJbPwmPyj9c3zYSFGawDW3jwlBOOsnY+EzZFLgNQ/UMkK35JmkNOVPdCyPr2Tw7Vv9N+KA3ZQ==", - "dependencies": { - "System.Threading.Tasks.Extensions": "4.5.4" - } - }, "Microsoft.CSharp": { "type": "Direct", "requested": "[4.7.0, )", @@ -291,7 +282,19 @@ } }, "speckle.sdk.dependencies": { - "type": "Project" + "type": "Project", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "[9.0.4, )" + } + }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "CentralTransitive", + "requested": "[9.0.4, )", + "resolved": "9.0.4", + "contentHash": "9VGI5kxIvrNG2mqLQZnUR6y/3fcnygD8eNpHR+CqfbnIXvea6nehnYknDKQTxZVPMpzpNca+7DxLBmpdB3q0Bw==", + "dependencies": { + "System.Threading.Tasks.Extensions": "4.5.4" + } } }, "net8.0": { diff --git a/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json b/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json index 046c2c3c..efd29581 100644 --- a/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json +++ b/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json @@ -382,7 +382,7 @@ }, "Microsoft.Bcl.AsyncInterfaces": { "type": "CentralTransitive", - "requested": "[5.0.0, )", + "requested": "[9.0.4, )", "resolved": "1.1.0", "contentHash": "1Am6l4Vpn3/K32daEqZI+FFr96OlZkgwK2LcT3pZ2zWubR5zTPW3/FkO1Rat9kb7oQOa4rxgl9LJHc5tspCWfg==" }, diff --git a/tests/Speckle.Sdk.Serialization.Tests/ServerObjectManagerTests.cs b/tests/Speckle.Sdk.Serialization.Tests/ServerObjectManagerTests.cs index c81e734e..90455343 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/ServerObjectManagerTests.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/ServerObjectManagerTests.cs @@ -52,7 +52,7 @@ public class ServerObjectManagerTests : MoqTest http.Setup(x => x.CreateHttpClient(It.IsAny(), timeout, token)).Returns(httpClient); var activityFactory = Create(); - activityFactory.Setup(x => x.Start(null, "DownloadObjects")).Returns((ISdkActivity?)null); + activityFactory.Setup(x => x.Start(null, default, "DownloadObjects")).Returns((ISdkActivity?)null); var serverObjectManager = new ServerObjectManager( http.Object, @@ -91,7 +91,7 @@ public class ServerObjectManagerTests : MoqTest http.Setup(x => x.CreateHttpClient(It.IsAny(), timeout, token)).Returns(httpClient); var activityFactory = Create(); - activityFactory.Setup(x => x.Start(null, "DownloadSingleObject")).Returns((ISdkActivity?)null); + activityFactory.Setup(x => x.Start(null, default, "DownloadSingleObject")).Returns((ISdkActivity?)null); var serverObjectManager = new ServerObjectManager( http.Object, @@ -132,7 +132,7 @@ public class ServerObjectManagerTests : MoqTest http.Setup(x => x.CreateHttpClient(It.IsAny(), timeout, token)).Returns(httpClient); var activityFactory = Create(); - activityFactory.Setup(x => x.Start(null, "HasObjects")).Returns((ISdkActivity?)null); + activityFactory.Setup(x => x.Start(null, default, "HasObjects")).Returns((ISdkActivity?)null); var serverObjectManager = new ServerObjectManager( http.Object, @@ -171,7 +171,7 @@ public class ServerObjectManagerTests : MoqTest http.Setup(x => x.CreateHttpClient(It.IsAny(), timeout, token)).Returns(httpClient); var activityFactory = Create(); - activityFactory.Setup(x => x.Start(null, "UploadObjects")).Returns((ISdkActivity?)null); + activityFactory.Setup(x => x.Start(null, default, "UploadObjects")).Returns((ISdkActivity?)null); var serverObjectManager = new ServerObjectManager( http.Object, diff --git a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/ModelIngestionResourceTests.cs b/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/ModelIngestionResourceTests.cs index 88ff78d2..7b8e1552 100644 --- a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/ModelIngestionResourceTests.cs +++ b/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/ModelIngestionResourceTests.cs @@ -129,8 +129,10 @@ public sealed class ModelIngestionResourceTests : IAsyncLifetime ModelIngestionSuccessInput finish = new(ingest.id, _project.id, sendResult.RootId, "yay!"); string versionId = await Sut.Complete(finish); Version version = await _testUser.Version.Get(versionId, _project.id); + ModelIngestion finalIngestion = await _testUser.Ingestion.Get(ingest.id, _project.id); Assert.Equal(version.id, versionId); Assert.Equal(sendResult.RootId, version.referencedObject); + Assert.Equal(finalIngestion.statusData.versionId, versionId); } [Fact] @@ -147,6 +149,11 @@ public sealed class ModelIngestionResourceTests : IAsyncLifetime ModelIngestion res = await Sut.Get(ingest.id, _project.id); Assert.Equal(ingest.id, res.id); Assert.Equal(ingest.statusData.status, res.statusData.status); + Assert.Equal(ingest.statusData.versionId, res.statusData.versionId); + Assert.Null(res.statusData.versionId); + Assert.Equal(_model.id, res.modelId); + Assert.Equal(_project.id, res.projectId); + Assert.Equal(_testUser.Account.userInfo.id, res.userId); } [Fact] diff --git a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/SubscriptionResourceTests.cs b/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/SubscriptionResourceTests.cs index dff86650..5ce49cb8 100644 --- a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/SubscriptionResourceTests.cs +++ b/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/SubscriptionResourceTests.cs @@ -13,7 +13,7 @@ public class SubscriptionResourceTests : IAsyncLifetime #if DEBUG private const int WAIT_PERIOD = 3000; // WSL is slow AF, so for local runs, we're being extra generous #else - private const int WAIT_PERIOD = 400; // For CI runs, a much smaller wait time is acceptable + private const int WAIT_PERIOD = 600; // For CI runs, a much smaller wait time is acceptable #endif private const int TIMEOUT = WAIT_PERIOD + WAIT_PERIOD + 600; private IClient _testUser; diff --git a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.TestGetProjects.verified.json b/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.TestGetProjects.verified.json deleted file mode 100644 index 781efcaf..00000000 --- a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.TestGetProjects.verified.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "Type": "AggregateException", - "InnerException": { - "Data": {}, - "Message": "FORBIDDEN: Your auth token does not have the required scope: workspace:read.", - "Type": "SpeckleGraphQLForbiddenException" - } -} diff --git a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.TestGetWorkspace.verified.json b/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.TestGetWorkspace.verified.json deleted file mode 100644 index 781efcaf..00000000 --- a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.TestGetWorkspace.verified.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "Type": "AggregateException", - "InnerException": { - "Data": {}, - "Message": "FORBIDDEN: Your auth token does not have the required scope: workspace:read.", - "Type": "SpeckleGraphQLForbiddenException" - } -} diff --git a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.cs b/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.cs index 8b73b43d..d597472d 100644 --- a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.cs +++ b/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/WorkspaceResourceTests.cs @@ -25,13 +25,15 @@ public class WorkspaceResourceTests public async Task TestGetWorkspace() { var ex = await Assert.ThrowsAsync(async () => _ = await Sut.Get("non-existent-id")); - await Verify(ex); + Assert.Single(ex.InnerExceptions); + Assert.All(ex.InnerExceptions, item => Assert.IsType(item)); } [Fact] public async Task TestGetProjects() { var ex = await Assert.ThrowsAsync(async () => _ = await Sut.GetProjects("non-existent-id")); - await Verify(ex); + Assert.Single(ex.InnerExceptions); + Assert.All(ex.InnerExceptions, item => Assert.IsType(item)); } } diff --git a/tests/Speckle.Sdk.Tests.Integration/Pipelines/Progress/IngestionProgressManagerTests.cs b/tests/Speckle.Sdk.Tests.Integration/Pipelines/Progress/IngestionProgressManagerTests.cs new file mode 100644 index 00000000..5ca8c8b5 --- /dev/null +++ b/tests/Speckle.Sdk.Tests.Integration/Pipelines/Progress/IngestionProgressManagerTests.cs @@ -0,0 +1,76 @@ +using Microsoft.Extensions.DependencyInjection; +using Speckle.Sdk.Api; +using Speckle.Sdk.Api.GraphQL.Models; +using Speckle.Sdk.Common; +using Speckle.Sdk.Pipelines.Progress; + +namespace Speckle.Sdk.Tests.Integration.Pipelines.Progress; + +[Trait("Server", "Internal")] +public class IngestionProgressManagerTests : IAsyncLifetime +{ + private IIngestionProgressManagerFactory _factory; + private IClient _client; + private Project _project; + private Model _model; + private ModelIngestion _ingestion; + + public async Task InitializeAsync() + { + var serviceProvider = TestServiceSetup.GetServiceProvider(); + _factory = serviceProvider.GetRequiredService(); + + _client = await Fixtures.SeedUserWithClient(); + _project = await _client.Project.Create(new("test", null, default)); + _model = await _client.Model.Create(new("test", null, _project.id)); + _ingestion = await _client.Ingestion.Create( + new(_model.id, _project.id, "Testing ingestion", new("integrationTests", "0.0.0", null, null)) + ); + } + + [Fact] + public async Task TestProgress_NoThrottle() + { + var sut = _factory.CreateInstance(_client, _ingestion, TimeSpan.Zero, CancellationToken.None); + const string FIRST_MESSAGE = "This is a test 123"; + const string SECOND_MESSAGE = "This is another test 321"; + + // first message (should go through) + sut.Report(new CardProgress(FIRST_MESSAGE, 0.123123123d)); + await sut.LastUpdate.NotNull(); + var res = await _client.Ingestion.Get(_ingestion.id, _project.id, CancellationToken.None); + + Assert.Equal(FIRST_MESSAGE, res.statusData.progressMessage); + + // second message (should also go through) + sut.Report(new CardProgress(SECOND_MESSAGE, 0.321321321d)); + await sut.LastUpdate.NotNull(); + res = await _client.Ingestion.Get(_ingestion.id, _project.id, CancellationToken.None); + + Assert.Equal(SECOND_MESSAGE, res.statusData.progressMessage); + } + + [Fact] + public async Task TestProgress_WithThrottle() + { + var sut = _factory.CreateInstance(_client, _ingestion, TimeSpan.FromMilliseconds(500), CancellationToken.None); + const string EXPECTED_MESSAGE = "First message should go through 123"; + + await Task.Delay(TimeSpan.FromMilliseconds(600)); + + // first message (should go through) + sut.Report(new CardProgress(EXPECTED_MESSAGE, 0.123123123d)); + // second message (should be dropped) + sut.Report(new CardProgress("Second message, should be dropped", 0.321321321d)); + await sut.LastUpdate.NotNull(); + var res = await _client.Ingestion.Get(_ingestion.id, _project.id, CancellationToken.None); + + Assert.Equal(EXPECTED_MESSAGE, res.statusData.progressMessage); + } + + public Task DisposeAsync() + { + _client.Dispose(); + return Task.CompletedTask; + } +} diff --git a/tests/Speckle.Sdk.Tests.Integration/Speckle.Sdk.Tests.Integration.csproj b/tests/Speckle.Sdk.Tests.Integration/Speckle.Sdk.Tests.Integration.csproj index 778355be..dfd7fa55 100644 --- a/tests/Speckle.Sdk.Tests.Integration/Speckle.Sdk.Tests.Integration.csproj +++ b/tests/Speckle.Sdk.Tests.Integration/Speckle.Sdk.Tests.Integration.csproj @@ -16,4 +16,7 @@ + + + diff --git a/tests/Speckle.Sdk.Tests.Performance/packages.lock.json b/tests/Speckle.Sdk.Tests.Performance/packages.lock.json index 16d806d9..23a76189 100644 --- a/tests/Speckle.Sdk.Tests.Performance/packages.lock.json +++ b/tests/Speckle.Sdk.Tests.Performance/packages.lock.json @@ -374,7 +374,7 @@ }, "Microsoft.Bcl.AsyncInterfaces": { "type": "CentralTransitive", - "requested": "[5.0.0, )", + "requested": "[9.0.4, )", "resolved": "1.1.0", "contentHash": "1Am6l4Vpn3/K32daEqZI+FFr96OlZkgwK2LcT3pZ2zWubR5zTPW3/FkO1Rat9kb7oQOa4rxgl9LJHc5tspCWfg==" }, diff --git a/tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/AggregateProgressTests.cs b/tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/AggregateProgressTests.cs new file mode 100644 index 00000000..50b0b878 --- /dev/null +++ b/tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/AggregateProgressTests.cs @@ -0,0 +1,21 @@ +using Moq; +using Speckle.Sdk.Pipelines.Progress; + +namespace Speckle.Sdk.Tests.Unit.Pipelines.Progress; + +public class AggregateProgressTests +{ + [Fact] + public void Report_InvokesReportOnAllInnerProgresses() + { + var mock1 = new Mock>(); + var mock2 = new Mock>(); + const int TEST_VALUE = 42; + var target = new AggregateProgress(mock1.Object, mock2.Object); + + target.Report(TEST_VALUE); + + mock1.Verify(x => x.Report(TEST_VALUE), Times.Once); + mock2.Verify(x => x.Report(TEST_VALUE), Times.Once); + } +} diff --git a/tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/ProgressStreamTests.cs b/tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/ProgressStreamTests.cs new file mode 100644 index 00000000..8563341c --- /dev/null +++ b/tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/ProgressStreamTests.cs @@ -0,0 +1,72 @@ +using System.Diagnostics.CodeAnalysis; +using Moq; +using Speckle.Sdk.Pipelines.Progress; + +namespace Speckle.Sdk.Tests.Unit.Pipelines.Progress; + +[SuppressMessage( + "Performance", + "CA1835:Prefer the \'Memory\'-based overloads for \'ReadAsync\' and \'WriteAsync\'", + Justification = "Need to test it" +)] +public class ProgressStreamTests : IDisposable +{ + private readonly Mock _innerStreamMock; + private readonly Mock> _progressMock; + private readonly ProgressStream _sut; + + public ProgressStreamTests() + { + // Setup the mocks + _innerStreamMock = new Mock(); + _innerStreamMock.Setup(s => s.Length).Returns(1024L); + + _progressMock = new Mock>(); + + // Inject mocks into the System Under Test + _sut = new ProgressStream(_innerStreamMock.Object, _progressMock.Object); + } + + [Fact] + public async Task ReadAsync_Should_CallInnerStreamAndReportProgress() + { + // Arrange + var buffer = new byte[10]; + _innerStreamMock + .Setup(s => s.ReadAsync(buffer, 0, buffer.Length, CancellationToken.None)) + .Returns(Task.FromResult(5)); + + // Act + await _sut.ReadAsync(buffer, 0, buffer.Length); + + // Assert - Inner Stream Read was called + _innerStreamMock.Verify(s => s.ReadAsync(buffer, 0, buffer.Length, CancellationToken.None), Times.Once); + + // Assert - Progress Report was called with the correct byte count + _progressMock.Verify(p => p.Report(It.IsAny()), Times.Once); + } + + [Fact] + public async Task WriteAsync_Should_CallInnerStreamAndReportProgress() + { + // Arrange + var buffer = new byte[10]; + _innerStreamMock + .Setup(s => s.WriteAsync(buffer, 0, buffer.Length, CancellationToken.None)) + .Returns(Task.FromResult(5)); + + // Act + await _sut.WriteAsync(buffer, 0, buffer.Length); + + // Assert - Inner Stream Write was called + _innerStreamMock.Verify(s => s.WriteAsync(buffer, 0, buffer.Length, CancellationToken.None), Times.Once); + + // Assert - Progress Report was called with the correct byte count + _progressMock.Verify(p => p.Report(It.IsAny()), Times.Once); + } + + public void Dispose() + { + _sut.Dispose(); + } +} diff --git a/tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/RenderedStreamProgressTests.cs b/tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/RenderedStreamProgressTests.cs new file mode 100644 index 00000000..33007f6c --- /dev/null +++ b/tests/Speckle.Sdk.Tests.Unit/Pipelines/Progress/RenderedStreamProgressTests.cs @@ -0,0 +1,46 @@ +using Speckle.Sdk.Pipelines.Progress; + +namespace Speckle.Sdk.Tests.Unit.Pipelines.Progress; + +public class RenderedStreamProgressTests +{ + [Theory] + [InlineData(1, "B", 1.0)] + [InlineData(1024, "B", 1.0)] + [InlineData(1024 + 1, "KB", 1.0 / 1024)] + [InlineData(1024 * 1024, "KB", 1.0 / 1024)] + [InlineData(1024 * 1024 + 1, "MB", 1.0 / (1024 * 1024))] + [InlineData(1024 * 1024 * 1024, "MB", 1.0 / (1024 * 1024))] + [InlineData(1024 * 1024 * 1024 + 1, "GB", 1.0 / (1024 * 1024 * 1024))] + [InlineData(1024L * 1024L * 1024L * 1024L, "GB", 1.0 / (1024L * 1024L * 1024L))] + public void GetFileSizeRendering_WithPositiveValue_ReturnsCorrectSuffix( + long value, + string expectedSuffix, + double expectedScaleFactor + ) + { + var result = RenderedStreamProgress.GetFileSizeRendering(value); + + Assert.Equal(expectedSuffix, result.suffix); + Assert.Equal(expectedScaleFactor, result.scaleFactor); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(-1000)] + public void GetFileSizeRendering_WithNonPositiveValue_ReturnsBytesSuffix(long value) + { + var result = RenderedStreamProgress.GetFileSizeRendering(value); + + Assert.Equal("B", result.suffix); + Assert.Equal(1d, result.scaleFactor); + } + + [Theory] + [InlineData(long.MaxValue)] + public void GetFileSizeRendering_WithVeryLargeValue_ThrowsArgumentOutOfRangeException(long value) + { + Assert.Throws(() => RenderedStreamProgress.GetFileSizeRendering(value)); + } +}