Compare commits

..

4 Commits

Author SHA1 Message Date
Jedd Morgan 81ef81eef1 comment tweaks
.NET Build and Publish / build (push) Has been cancelled
2026-03-02 11:48:00 +00:00
Jedd Morgan fa03714c11 restore solution 2026-03-02 11:47:14 +00:00
Jedd Morgan ca04c9836d transitive for sdk 2026-03-02 11:12:36 +00:00
Jedd Morgan ecf90336ee Repacked channel 2026-03-02 11:09:34 +00:00
19 changed files with 49 additions and 276 deletions
+1 -5
View File
@@ -1,11 +1,7 @@
name: PR Test
on:
pull_request: {}
push:
branches:
- "main" # Need to run for codecov to compare against the BASE
pull_request:
jobs:
build:
+7
View File
@@ -46,6 +46,13 @@ jobs:
SEMVER: ${{ steps.set-version.outputs.SEMVER }}
FILE_VERSION: ${{ steps.set-version.outputs.FILE_VERSION }}
- name: Upload coverage reports to Codecov with GitHub Action
uses: codecov/codecov-action@v5
continue-on-error: true
with:
fail_ci_if_error: true
files: tests/**/coverage.xml
token: ${{ secrets.CODECOV_TOKEN }}
- name: NuGet login (OIDC → temp API key)
uses: NuGet/login@v1
@@ -4,9 +4,5 @@ namespace Speckle.Sdk.Logging;
public interface ISdkActivityFactory : IDisposable
{
/// <param name="name"></param>
/// <param name="source"></param>
/// <param name="parentId">Only need to set if the parent is coming from an external source (e.g.to trace between client and server)</param>
/// <returns></returns>
ISdkActivity? Start(string? name = default, [CallerMemberName] string source = "", string? parentId = null);
ISdkActivity? Start(string? name = default, [CallerMemberName] string source = "");
}
@@ -53,7 +53,7 @@
<_ILRepackExcludeAssemblies_Items Include="$(OutputPath)*.dll" Exclude="@(_ILRepackIncludeAssemblies_Items)" />
</ItemGroup>
<Message
Text="These are the packages we are NOT ilrepacking '@(_ILRepackExcludeAssemblies_Items)'"
Text="These are the packages we are NOT ilrepacking '$(_ILRepackExcludeAssemblies_Items)'"
Importance="high"
/>
<PropertyGroup>
@@ -14,8 +14,7 @@ public record ModelIngestionCreateInput(
string modelId,
string projectId,
string progressMessage,
SourceDataInput sourceData,
int? maxIdleTimeoutSeconds = null
SourceDataInput sourceData
);
public record ModelIngestionUpdateInput(string ingestionId, string projectId, string progressMessage, double? progress);
@@ -4,5 +4,5 @@ public sealed class NullActivityFactory : ISdkActivityFactory
{
public void Dispose() { }
public ISdkActivity? Start(string? name = default, string source = "", string? parentId = null) => null;
public ISdkActivity? Start(string? name = default, string source = "") => null;
}
@@ -14,20 +14,20 @@ public partial interface IIngestionProgressManager : IProgress<CardProgress>;
/// An <see langword="IProgress{IngestionProgressEventArgs}"/> implementation for the entire client side Ingestion progress update reporting
/// Will throttles ingestion progress messages and reports their progress
/// </summary>
/// <remarks>
/// Normally we would pick quite a coarse updateInterval to try and spamming the server (1-5s)
/// </remarks>
[GenerateAutoInterface]
public sealed class IngestionProgressManager(
ILogger<IngestionProgressManager> logger,
IClient speckleClient,
ModelIngestion ingestion,
string projectId,
TimeSpan updateInterval,
CancellationToken cancellationToken
) : IIngestionProgressManager
{
public Task? LastUpdate { get; private set; }
/// <remarks>
/// Normally we would pick quite a coarse throttle window to try and avoid over pressure (1-5s)
/// </remarks>
private Task? _lastUpdate;
private long _lastUpdatedAt;
private readonly object _lock = new();
@@ -48,15 +48,15 @@ public sealed class IngestionProgressManager(
trimmedMessage = value.Status.TrimEnd('.');
LastUpdate = speckleClient
_lastUpdate = speckleClient
.Ingestion.UpdateProgress(
new ModelIngestionUpdateInput(ingestion.id, ingestion.projectId, trimmedMessage, value.Progress),
new ModelIngestionUpdateInput(ingestion.id, projectId, trimmedMessage, value.Progress),
cancellationToken
)
.ContinueWith(
Continuation,
HandleFaultedContinuation,
CancellationToken.None,
TaskContinuationOptions.ExecuteSynchronously,
TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously,
TaskScheduler.Default
);
}
@@ -67,7 +67,7 @@ public sealed class IngestionProgressManager(
/// <returns><see langword="true"/> if the update should be ignored, otherwise <see langword="false"/></returns>
private bool ShouldIgnoreProgressUpdate()
{
if (LastUpdate is not null && !LastUpdate.IsCompleted)
if (_lastUpdate is not null && !_lastUpdate.IsCompleted)
{
return true;
}
@@ -76,7 +76,7 @@ public sealed class IngestionProgressManager(
return msSinceLastUpdate < updateInterval;
}
private void Continuation(Task updateTask)
private void HandleFaultedContinuation(Task updateTask)
{
// The progress report failed... could be many reasons.
// For now, we're not letting this fail the Ingestion in any way
@@ -12,10 +12,11 @@ public sealed class IngestionProgressManagerFactory(ILogger<IngestionProgressMan
public IIngestionProgressManager CreateInstance(
IClient speckleClient,
ModelIngestion ingestion,
string projectId,
TimeSpan updateInterval,
CancellationToken cancellationToken
)
{
return new IngestionProgressManager(logger, speckleClient, ingestion, updateInterval, cancellationToken);
return new IngestionProgressManager(logger, speckleClient, ingestion, projectId, updateInterval, cancellationToken);
}
}
@@ -18,9 +18,9 @@ public sealed class RenderedStreamProgress(IProgress<CardProgress> progress) : I
);
}
private static readonly string[] s_suffixes = ["B", "KB", "MB", "GB", "TB", "PB"];
private static readonly string[] s_suffixes = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"];
internal static (string suffix, double scaleFactor) GetFileSizeRendering(long value)
private static (string suffix, double scaleFactor) GetFileSizeRendering(long value)
{
if (value <= 0)
{
+4 -13
View File
@@ -3,15 +3,13 @@ using Microsoft.Extensions.Logging;
using Speckle.InterfaceGenerator;
using Speckle.Sdk.Dependencies;
using Speckle.Sdk.Helpers;
using Speckle.Sdk.Logging;
namespace Speckle.Sdk.Pipelines.Send;
[GenerateAutoInterface]
public sealed class DiskStoreFactory(ILogger<DiskStore> logger, ISdkActivityFactory activityFactory) : IDiskStoreFactory
public sealed class DiskStoreFactory(ILogger<DiskStore> logger) : IDiskStoreFactory
{
public DiskStore CreateInstance(CancellationToken cancellationToken) =>
new(logger, activityFactory, cancellationToken);
public DiskStore CreateInstance(CancellationToken cancellationToken) => new(logger, cancellationToken);
}
public sealed class DiskStore
@@ -19,17 +17,11 @@ public sealed class DiskStore
private readonly RepackedChannel<UploadItem> _channel;
private readonly Task<DisposableFile> _writeToDiskTask;
private readonly ILogger<DiskStore> _logger;
private readonly ISdkActivityFactory _activityFactory;
private readonly CancellationToken _cancellationToken;
internal DiskStore(
ILogger<DiskStore> logger,
ISdkActivityFactory activityFactory,
CancellationToken cancellationToken
)
internal DiskStore(ILogger<DiskStore> logger, CancellationToken cancellationToken)
{
_logger = logger;
_activityFactory = activityFactory;
_cancellationToken = cancellationToken;
_channel = new RepackedChannel<UploadItem>(1000, true, false);
@@ -41,7 +33,6 @@ public sealed class DiskStore
public async Task<DisposableFile> CompleteAsync()
{
using var a = _activityFactory.Start("Waiting for DiskStore to complete");
_channel.CompleteWriter();
return await _writeToDiskTask.ConfigureAwait(false);
}
@@ -65,7 +56,7 @@ public sealed class DiskStore
await foreach (var item in _channel.ReadAllAsync(_cancellationToken).ConfigureAwait(false))
{
await writer.WriteLineAsync($"{item.Id}\t{item.SpeckleType}\t{item.Json}").ConfigureAwait(false);
await writer.WriteLineAsync($"{item.Id}\t{item.Json}\t{item.SpeckleType}").ConfigureAwait(false);
}
#if NET8_0_OR_GREATER
await writer.FlushAsync(_cancellationToken).ConfigureAwait(false);
@@ -70,5 +70,11 @@ public sealed class SendPipeline : IDisposable
await _uploader.Send(fileStreamUpload).ConfigureAwait(false);
}
public async Task<string> WaitForUploadAndServerProcessing()
{
// TODO: in some way, wait for the server to process the upload and return the actual new version id
return await Task.FromResult("todo").ConfigureAwait(false);
}
public void Dispose() => _uploader.Dispose();
}
+7 -12
View File
@@ -1,17 +1,16 @@
using System.Net.Http.Headers;
using System.Text;
using Microsoft.Extensions.Logging;
using Speckle.InterfaceGenerator;
using Speckle.Newtonsoft.Json;
using Speckle.Sdk.Credentials;
using Speckle.Sdk.Helpers;
using Speckle.Sdk.Logging;
using Speckle.Sdk.Pipelines.Progress;
namespace Speckle.Sdk.Pipelines.Send;
[GenerateAutoInterface]
public sealed class UploaderFactory(ISpeckleHttp httpClientFactory, ISdkActivityFactory activityFactory)
: IUploaderFactory
public sealed class UploaderFactory(ISpeckleHttp httpClientFactory, ILogger<Uploader> logger) : IUploaderFactory
{
public Uploader CreateInstance(
string projectId,
@@ -19,7 +18,7 @@ public sealed class UploaderFactory(ISpeckleHttp httpClientFactory, ISdkActivity
Account account,
IProgress<StreamProgressArgs> progress,
CancellationToken cancellationToken
) => new(projectId, ingestionId, activityFactory, httpClientFactory, account, progress, cancellationToken);
) => new(projectId, ingestionId, logger, httpClientFactory, account, progress, cancellationToken);
}
public sealed class Uploader : IDisposable
@@ -29,13 +28,13 @@ public sealed class Uploader : IDisposable
private readonly CancellationToken _cancellationToken;
private readonly HttpClient _speckleClient;
private readonly HttpClient _s3Client;
private readonly ISdkActivityFactory _activity;
private readonly ILogger<Uploader> _logger;
private readonly IProgress<StreamProgressArgs> _progress;
internal Uploader(
string projectId,
string ingestionId,
ISdkActivityFactory activity,
ILogger<Uploader> logger,
ISpeckleHttp httpClientFactory,
Account speckleAccount,
IProgress<StreamProgressArgs> progress,
@@ -44,7 +43,7 @@ public sealed class Uploader : IDisposable
{
_projectId = projectId;
_ingestionId = ingestionId;
_activity = activity;
_logger = logger;
_cancellationToken = cancellationToken;
_progress = progress;
_speckleClient = httpClientFactory.CreateHttpClient(authorizationToken: speckleAccount.token);
@@ -63,8 +62,6 @@ public sealed class Uploader : IDisposable
private async Task<PresignedUploadResponse> GetPresignedUrl()
{
using var a = _activity.Start("Get Presigned Url");
var signUri = new Uri($"projects/{_projectId}/modelingestion/{_ingestionId}/uploads/sign", UriKind.Relative);
using var signResponse = await _speckleClient.PostAsync(signUri, null, _cancellationToken).ConfigureAwait(false);
@@ -83,7 +80,7 @@ public sealed class Uploader : IDisposable
private async Task<string> UploadToS3(Stream fileStream, PresignedUploadResponse presignedUploadResponse)
{
using var a = _activity.Start("Uploading file to pre-signed url");
_logger.LogInformation("Uploading file to pre-signed url");
Stream progressStream = new ProgressStream(fileStream, _progress);
@@ -110,8 +107,6 @@ public sealed class Uploader : IDisposable
private async Task TriggerProcessing(TriggerUploadRequest request)
{
using var a = _activity.Start("Triggering Processing");
Uri processUri = new($"projects/{_projectId}/modelingestion/{_ingestionId}/uploads/process", UriKind.Relative);
string requestBody = JsonConvert.SerializeObject(request);
using var content = new StringContent(requestBody, Encoding.UTF8, "application/json");
@@ -52,7 +52,7 @@ public class ServerObjectManagerTests : MoqTest
http.Setup(x => x.CreateHttpClient(It.IsAny<HttpClientHandler>(), timeout, token)).Returns(httpClient);
var activityFactory = Create<ISdkActivityFactory>();
activityFactory.Setup(x => x.Start(null, "DownloadObjects", null)).Returns((ISdkActivity?)null);
activityFactory.Setup(x => x.Start(null, "DownloadObjects")).Returns((ISdkActivity?)null);
var serverObjectManager = new ServerObjectManager(
http.Object,
@@ -91,7 +91,7 @@ public class ServerObjectManagerTests : MoqTest
http.Setup(x => x.CreateHttpClient(It.IsAny<HttpClientHandler>(), timeout, token)).Returns(httpClient);
var activityFactory = Create<ISdkActivityFactory>();
activityFactory.Setup(x => x.Start(null, "DownloadSingleObject", null)).Returns((ISdkActivity?)null);
activityFactory.Setup(x => x.Start(null, "DownloadSingleObject")).Returns((ISdkActivity?)null);
var serverObjectManager = new ServerObjectManager(
http.Object,
@@ -132,7 +132,7 @@ public class ServerObjectManagerTests : MoqTest
http.Setup(x => x.CreateHttpClient(It.IsAny<HttpClientHandler>(), timeout, token)).Returns(httpClient);
var activityFactory = Create<ISdkActivityFactory>();
activityFactory.Setup(x => x.Start(null, "HasObjects", null)).Returns((ISdkActivity?)null);
activityFactory.Setup(x => x.Start(null, "HasObjects")).Returns((ISdkActivity?)null);
var serverObjectManager = new ServerObjectManager(
http.Object,
@@ -171,7 +171,7 @@ public class ServerObjectManagerTests : MoqTest
http.Setup(x => x.CreateHttpClient(It.IsAny<HttpClientHandler>(), timeout, token)).Returns(httpClient);
var activityFactory = Create<ISdkActivityFactory>();
activityFactory.Setup(x => x.Start(null, "UploadObjects", null)).Returns((ISdkActivity?)null);
activityFactory.Setup(x => x.Start(null, "UploadObjects")).Returns((ISdkActivity?)null);
var serverObjectManager = new ServerObjectManager(
http.Object,
@@ -13,7 +13,7 @@ public class SubscriptionResourceTests : IAsyncLifetime
#if DEBUG
private const int WAIT_PERIOD = 3000; // WSL is slow AF, so for local runs, we're being extra generous
#else
private const int WAIT_PERIOD = 600; // For CI runs, a much smaller wait time is acceptable
private const int WAIT_PERIOD = 400; // For CI runs, a much smaller wait time is acceptable
#endif
private const int TIMEOUT = WAIT_PERIOD + WAIT_PERIOD + 600;
private IClient _testUser;
@@ -1,76 +0,0 @@
using Microsoft.Extensions.DependencyInjection;
using Speckle.Sdk.Api;
using Speckle.Sdk.Api.GraphQL.Models;
using Speckle.Sdk.Common;
using Speckle.Sdk.Pipelines.Progress;
namespace Speckle.Sdk.Tests.Integration.Pipelines.Progress;
[Trait("Server", "Internal")]
public class IngestionProgressManagerTests : IAsyncLifetime
{
private IIngestionProgressManagerFactory _factory;
private IClient _client;
private Project _project;
private Model _model;
private ModelIngestion _ingestion;
public async Task InitializeAsync()
{
var serviceProvider = TestServiceSetup.GetServiceProvider();
_factory = serviceProvider.GetRequiredService<IIngestionProgressManagerFactory>();
_client = await Fixtures.SeedUserWithClient();
_project = await _client.Project.Create(new("test", null, default));
_model = await _client.Model.Create(new("test", null, _project.id));
_ingestion = await _client.Ingestion.Create(
new(_model.id, _project.id, "Testing ingestion", new("integrationTests", "0.0.0", null, null))
);
}
[Fact]
public async Task TestProgress_NoThrottle()
{
var sut = _factory.CreateInstance(_client, _ingestion, TimeSpan.Zero, CancellationToken.None);
const string FIRST_MESSAGE = "This is a test 123";
const string SECOND_MESSAGE = "This is another test 321";
// first message (should go through)
sut.Report(new CardProgress(FIRST_MESSAGE, 0.123123123d));
await sut.LastUpdate.NotNull();
var res = await _client.Ingestion.Get(_ingestion.id, _project.id, CancellationToken.None);
Assert.Equal(FIRST_MESSAGE, res.statusData.progressMessage);
// second message (should also go through)
sut.Report(new CardProgress(SECOND_MESSAGE, 0.321321321d));
await sut.LastUpdate.NotNull();
res = await _client.Ingestion.Get(_ingestion.id, _project.id, CancellationToken.None);
Assert.Equal(SECOND_MESSAGE, res.statusData.progressMessage);
}
[Fact]
public async Task TestProgress_WithThrottle()
{
var sut = _factory.CreateInstance(_client, _ingestion, TimeSpan.FromMilliseconds(500), CancellationToken.None);
const string EXPECTED_MESSAGE = "First message should go through 123";
await Task.Delay(TimeSpan.FromMilliseconds(600));
// first message (should go through)
sut.Report(new CardProgress(EXPECTED_MESSAGE, 0.123123123d));
// second message (should be dropped)
sut.Report(new CardProgress("Second message, should be dropped", 0.321321321d));
await sut.LastUpdate.NotNull();
var res = await _client.Ingestion.Get(_ingestion.id, _project.id, CancellationToken.None);
Assert.Equal(EXPECTED_MESSAGE, res.statusData.progressMessage);
}
public Task DisposeAsync()
{
_client.Dispose();
return Task.CompletedTask;
}
}
@@ -16,7 +16,4 @@
<ProjectReference Include="..\..\src\Speckle.Sdk\Speckle.Sdk.csproj" />
<ProjectReference Include="..\Speckle.Sdk.Testing\Speckle.Sdk.Testing.csproj" />
</ItemGroup>
<ItemGroup>
<Folder Include="Pipelines\Send\" />
</ItemGroup>
</Project>
@@ -1,21 +0,0 @@
using Moq;
using Speckle.Sdk.Pipelines.Progress;
namespace Speckle.Sdk.Tests.Unit.Pipelines.Progress;
public class AggregateProgressTests
{
[Fact]
public void Report_InvokesReportOnAllInnerProgresses()
{
var mock1 = new Mock<IProgress<int>>();
var mock2 = new Mock<IProgress<int>>();
const int TEST_VALUE = 42;
var target = new AggregateProgress<int>(mock1.Object, mock2.Object);
target.Report(TEST_VALUE);
mock1.Verify(x => x.Report(TEST_VALUE), Times.Once);
mock2.Verify(x => x.Report(TEST_VALUE), Times.Once);
}
}
@@ -1,72 +0,0 @@
using System.Diagnostics.CodeAnalysis;
using Moq;
using Speckle.Sdk.Pipelines.Progress;
namespace Speckle.Sdk.Tests.Unit.Pipelines.Progress;
[SuppressMessage(
"Performance",
"CA1835:Prefer the \'Memory\'-based overloads for \'ReadAsync\' and \'WriteAsync\'",
Justification = "Need to test it"
)]
public class ProgressStreamTests : IDisposable
{
private readonly Mock<Stream> _innerStreamMock;
private readonly Mock<IProgress<StreamProgressArgs>> _progressMock;
private readonly ProgressStream _sut;
public ProgressStreamTests()
{
// Setup the mocks
_innerStreamMock = new Mock<Stream>();
_innerStreamMock.Setup(s => s.Length).Returns(1024L);
_progressMock = new Mock<IProgress<StreamProgressArgs>>();
// Inject mocks into the System Under Test
_sut = new ProgressStream(_innerStreamMock.Object, _progressMock.Object);
}
[Fact]
public async Task ReadAsync_Should_CallInnerStreamAndReportProgress()
{
// Arrange
var buffer = new byte[10];
_innerStreamMock
.Setup(s => s.ReadAsync(buffer, 0, buffer.Length, CancellationToken.None))
.Returns(Task.FromResult(5));
// Act
await _sut.ReadAsync(buffer, 0, buffer.Length);
// Assert - Inner Stream Read was called
_innerStreamMock.Verify(s => s.ReadAsync(buffer, 0, buffer.Length, CancellationToken.None), Times.Once);
// Assert - Progress Report was called with the correct byte count
_progressMock.Verify(p => p.Report(It.IsAny<StreamProgressArgs>()), Times.Once);
}
[Fact]
public async Task WriteAsync_Should_CallInnerStreamAndReportProgress()
{
// Arrange
var buffer = new byte[10];
_innerStreamMock
.Setup(s => s.WriteAsync(buffer, 0, buffer.Length, CancellationToken.None))
.Returns(Task.FromResult(5));
// Act
await _sut.WriteAsync(buffer, 0, buffer.Length);
// Assert - Inner Stream Write was called
_innerStreamMock.Verify(s => s.WriteAsync(buffer, 0, buffer.Length, CancellationToken.None), Times.Once);
// Assert - Progress Report was called with the correct byte count
_progressMock.Verify(p => p.Report(It.IsAny<StreamProgressArgs>()), Times.Once);
}
public void Dispose()
{
_sut.Dispose();
}
}
@@ -1,46 +0,0 @@
using Speckle.Sdk.Pipelines.Progress;
namespace Speckle.Sdk.Tests.Unit.Pipelines.Progress;
public class RenderedStreamProgressTests
{
[Theory]
[InlineData(1, "B", 1.0)]
[InlineData(1024, "B", 1.0)]
[InlineData(1024 + 1, "KB", 1.0 / 1024)]
[InlineData(1024 * 1024, "KB", 1.0 / 1024)]
[InlineData(1024 * 1024 + 1, "MB", 1.0 / (1024 * 1024))]
[InlineData(1024 * 1024 * 1024, "MB", 1.0 / (1024 * 1024))]
[InlineData(1024 * 1024 * 1024 + 1, "GB", 1.0 / (1024 * 1024 * 1024))]
[InlineData(1024L * 1024L * 1024L * 1024L, "GB", 1.0 / (1024L * 1024L * 1024L))]
public void GetFileSizeRendering_WithPositiveValue_ReturnsCorrectSuffix(
long value,
string expectedSuffix,
double expectedScaleFactor
)
{
var result = RenderedStreamProgress.GetFileSizeRendering(value);
Assert.Equal(expectedSuffix, result.suffix);
Assert.Equal(expectedScaleFactor, result.scaleFactor);
}
[Theory]
[InlineData(0)]
[InlineData(-1)]
[InlineData(-1000)]
public void GetFileSizeRendering_WithNonPositiveValue_ReturnsBytesSuffix(long value)
{
var result = RenderedStreamProgress.GetFileSizeRendering(value);
Assert.Equal("B", result.suffix);
Assert.Equal(1d, result.scaleFactor);
}
[Theory]
[InlineData(long.MaxValue)]
public void GetFileSizeRendering_WithVeryLargeValue_ThrowsArgumentOutOfRangeException(long value)
{
Assert.Throws<ArgumentOutOfRangeException>(() => RenderedStreamProgress.GetFileSizeRendering(value));
}
}