diff --git a/.gitignore b/.gitignore index 5b1cd032a..af18aab41 100644 --- a/.gitignore +++ b/.gitignore @@ -22,4 +22,4 @@ coverage.xml output/ Images/Thumbs.db -.claude +.claude/ diff --git a/CodeMetricsConfig.txt b/CodeMetricsConfig.txt index e69bf2348..898cf21de 100644 --- a/CodeMetricsConfig.txt +++ b/CodeMetricsConfig.txt @@ -1,4 +1,4 @@ CA1502: 25 CA1501: 5 -CA1506(Method): 50 +CA1506(Method): 60 CA1506(Type): 95 diff --git a/Connectors/Autocad/Speckle.Connectors.AutocadShared/DependencyInjection/AutocadConnectorModule.cs b/Connectors/Autocad/Speckle.Connectors.AutocadShared/DependencyInjection/AutocadConnectorModule.cs index 6e199726f..656e65b96 100644 --- a/Connectors/Autocad/Speckle.Connectors.AutocadShared/DependencyInjection/AutocadConnectorModule.cs +++ b/Connectors/Autocad/Speckle.Connectors.AutocadShared/DependencyInjection/AutocadConnectorModule.cs @@ -18,6 +18,10 @@ public static class AutocadConnectorModule // Send serviceCollection.LoadSend(); serviceCollection.AddScoped, AutocadRootObjectBuilder>(); + serviceCollection.AddScoped< + IRootContinuousTraversalBuilder, + AutocadContinuousTraversalBuilder + >(); // Receive serviceCollection.LoadReceive(); diff --git a/Connectors/Autocad/Speckle.Connectors.AutocadShared/HostApp/AutocadColorBaker.cs b/Connectors/Autocad/Speckle.Connectors.AutocadShared/HostApp/AutocadColorBaker.cs index c02e1d839..4c74c1194 100644 --- a/Connectors/Autocad/Speckle.Connectors.AutocadShared/HostApp/AutocadColorBaker.cs +++ b/Connectors/Autocad/Speckle.Connectors.AutocadShared/HostApp/AutocadColorBaker.cs @@ -1,9 +1,9 @@ using Autodesk.AutoCAD.Colors; using Microsoft.Extensions.Logging; -using Speckle.Connectors.Common.Operations; using Speckle.InterfaceGenerator; using Speckle.Sdk; using Speckle.Sdk.Models.Proxies; +using Speckle.Sdk.Pipelines.Progress; using AutocadColor = Autodesk.AutoCAD.Colors.Color; namespace Speckle.Connectors.Autocad.HostApp; diff --git a/Connectors/Autocad/Speckle.Connectors.AutocadShared/HostApp/AutocadInstanceBaker.cs b/Connectors/Autocad/Speckle.Connectors.AutocadShared/HostApp/AutocadInstanceBaker.cs index e1373f54f..3d7782c99 100644 --- a/Connectors/Autocad/Speckle.Connectors.AutocadShared/HostApp/AutocadInstanceBaker.cs +++ b/Connectors/Autocad/Speckle.Connectors.AutocadShared/HostApp/AutocadInstanceBaker.cs @@ -5,7 +5,6 @@ using Microsoft.Extensions.Logging; using Speckle.Connectors.Autocad.HostApp.Extensions; using Speckle.Connectors.Common.Conversion; using Speckle.Connectors.Common.Instances; -using Speckle.Connectors.Common.Operations; using Speckle.Converters.Autocad; using Speckle.Converters.Common; using Speckle.DoubleNumerics; @@ -16,6 +15,7 @@ using Speckle.Sdk.Dependencies; using Speckle.Sdk.Models; using Speckle.Sdk.Models.Collections; using Speckle.Sdk.Models.Instances; +using Speckle.Sdk.Pipelines.Progress; using AutocadColor = Autodesk.AutoCAD.Colors.Color; namespace Speckle.Connectors.Autocad.HostApp; diff --git a/Connectors/Autocad/Speckle.Connectors.AutocadShared/HostApp/AutocadMaterialBaker.cs b/Connectors/Autocad/Speckle.Connectors.AutocadShared/HostApp/AutocadMaterialBaker.cs index 1e09a7f2b..1265ee081 100644 --- a/Connectors/Autocad/Speckle.Connectors.AutocadShared/HostApp/AutocadMaterialBaker.cs +++ b/Connectors/Autocad/Speckle.Connectors.AutocadShared/HostApp/AutocadMaterialBaker.cs @@ -3,12 +3,12 @@ using Autodesk.AutoCAD.DatabaseServices; using Autodesk.AutoCAD.GraphicsInterface; using Microsoft.Extensions.Logging; using Speckle.Connectors.Common.Conversion; -using Speckle.Connectors.Common.Operations; using Speckle.InterfaceGenerator; using Speckle.Objects.Other; using Speckle.Sdk; using Speckle.Sdk.Common; using Speckle.Sdk.Models; +using Speckle.Sdk.Pipelines.Progress; using Material = Autodesk.AutoCAD.DatabaseServices.Material; using RenderMaterial = Speckle.Objects.Other.RenderMaterial; diff --git a/Connectors/Autocad/Speckle.Connectors.AutocadShared/Operations/Receive/AutocadHostObjectBaseBuilder.cs b/Connectors/Autocad/Speckle.Connectors.AutocadShared/Operations/Receive/AutocadHostObjectBaseBuilder.cs index fab625077..b06fa98e2 100644 --- a/Connectors/Autocad/Speckle.Connectors.AutocadShared/Operations/Receive/AutocadHostObjectBaseBuilder.cs +++ b/Connectors/Autocad/Speckle.Connectors.AutocadShared/Operations/Receive/AutocadHostObjectBaseBuilder.cs @@ -12,6 +12,7 @@ using Speckle.Sdk.Dependencies; using Speckle.Sdk.Models; using Speckle.Sdk.Models.Collections; using Speckle.Sdk.Models.Instances; +using Speckle.Sdk.Pipelines.Progress; using AutocadColor = Autodesk.AutoCAD.Colors.Color; namespace Speckle.Connectors.Autocad.Operations.Receive; diff --git a/Connectors/Autocad/Speckle.Connectors.AutocadShared/Operations/Send/AutocadContinuousTraversalBaseBuilder.cs b/Connectors/Autocad/Speckle.Connectors.AutocadShared/Operations/Send/AutocadContinuousTraversalBaseBuilder.cs new file mode 100644 index 000000000..67ae43e14 --- /dev/null +++ b/Connectors/Autocad/Speckle.Connectors.AutocadShared/Operations/Send/AutocadContinuousTraversalBaseBuilder.cs @@ -0,0 +1,197 @@ +using System.Diagnostics.CodeAnalysis; +using Autodesk.AutoCAD.DatabaseServices; +using Microsoft.Extensions.Logging; +using Speckle.Connectors.Autocad.HostApp; +using Speckle.Connectors.Common.Builders; +using Speckle.Connectors.Common.Caching; +using Speckle.Connectors.Common.Conversion; +using Speckle.Connectors.Common.Extensions; +using Speckle.Connectors.Common.Operations; +using Speckle.Converters.Common; +using Speckle.Sdk; +using Speckle.Sdk.Logging; +using Speckle.Sdk.Models; +using Speckle.Sdk.Models.Collections; +using Speckle.Sdk.Models.Instances; +using Speckle.Sdk.Pipelines.Progress; +using Speckle.Sdk.Pipelines.Send; + +namespace Speckle.Connectors.Autocad.Operations.Send; + +/// +/// Abstract base class for AutoCAD continuous traversal builders that stream objects through a +/// for packfile-based uploads. Same conversion logic as +/// , but processes elements through the pipeline. +/// +public abstract class AutocadContinuousTraversalBaseBuilder : IRootContinuousTraversalBuilder +{ + private readonly IRootToSpeckleConverter _converter; + private readonly string[] _documentPathSeparator = ["\\"]; + private readonly ISendConversionCache _sendConversionCache; + private readonly AutocadInstanceUnpacker _instanceUnpacker; + private readonly AutocadMaterialUnpacker _materialUnpacker; + private readonly AutocadColorUnpacker _colorUnpacker; + private readonly AutocadGroupUnpacker _groupUnpacker; + private readonly ILogger _logger; + private readonly ISdkActivityFactory _activityFactory; + + protected AutocadContinuousTraversalBaseBuilder( + IRootToSpeckleConverter converter, + ISendConversionCache sendConversionCache, + AutocadInstanceUnpacker instanceObjectManager, + AutocadMaterialUnpacker materialUnpacker, + AutocadColorUnpacker colorUnpacker, + AutocadGroupUnpacker groupUnpacker, + ILogger logger, + ISdkActivityFactory activityFactory + ) + { + _converter = converter; + _sendConversionCache = sendConversionCache; + _instanceUnpacker = instanceObjectManager; + _materialUnpacker = materialUnpacker; + _colorUnpacker = colorUnpacker; + _groupUnpacker = groupUnpacker; + _logger = logger; + _activityFactory = activityFactory; + } + + [SuppressMessage( + "Maintainability", + "CA1506:Avoid excessive class coupling", + Justification = """ + It is already simplified but has many different references since it is a builder. Do not know can we simplify it now. + Later we might consider to refactor proxies from one proxy manager? but we do not know the shape of it all potential + proxy classes yet. So I'm supressing this one now!!! + """ + )] + public async Task Build( + IReadOnlyList objects, + string projectId, + SendPipeline sendPipeline, + IProgress onOperationProgressed, + CancellationToken cancellationToken + ) + { + // 0 - Init the root + Collection root = + new() + { + name = Application + .DocumentManager.CurrentDocument.Name.Split(_documentPathSeparator, StringSplitOptions.None) + .Reverse() + .First() + }; + + Document doc = Application.DocumentManager.CurrentDocument; + using Transaction tr = doc.Database.TransactionManager.StartTransaction(); + + // 1 - Unpack the instances + var (atomicObjects, _, instanceProxies, instanceDefinitionProxies) = _instanceUnpacker.UnpackSelection(objects); + root[ProxyKeys.INSTANCE_DEFINITION] = instanceDefinitionProxies; + + // 2 - Unpack the groups + root[ProxyKeys.GROUP] = _groupUnpacker.UnpackGroups(atomicObjects); + using (var _ = _activityFactory.Start("Converting objects")) + { + // 3 - Convert atomic objects and process through pipeline + List usedAcadLayers = new(); + List results = new(); + int count = 0; + foreach (var (entity, applicationId) in atomicObjects) + { + cancellationToken.ThrowIfCancellationRequested(); + (Collection objectCollection, LayerTableRecord? autocadLayer) = CreateObjectCollection(entity, tr); + + if (autocadLayer is not null) + { + usedAcadLayers.Add(autocadLayer); + root.elements.Add(objectCollection); + } + + var result = await ConvertAutocadEntity( + entity, + applicationId, + objectCollection, + instanceProxies, + projectId, + sendPipeline + ); + results.Add(result); + + onOperationProgressed.Report( + new($"Converting objects... ({count:N0} / {atomicObjects.Count:N0})", (double)++count / atomicObjects.Count) + ); + } + + if (results.All(x => x.Status == Status.ERROR)) + { + throw new SpeckleException("Failed to convert all objects."); + } + + // 4 - Unpack the render material proxies + root[ProxyKeys.RENDER_MATERIAL] = _materialUnpacker.UnpackMaterials(atomicObjects, usedAcadLayers); + + // 5 - Unpack the color proxies + root[ProxyKeys.COLOR] = _colorUnpacker.UnpackColors(atomicObjects, usedAcadLayers); + + // add any additional properties (most likely from verticals) + AddAdditionalProxiesToRoot(root); + + // Process root collection and wait for all uploads + await sendPipeline.Process(root); + await sendPipeline.WaitForUpload(); + + return new RootObjectBuilderResult(root, results); + } + } + + public virtual (Collection, LayerTableRecord?) CreateObjectCollection(Entity entity, Transaction tr) + { + return (new(), null); + } + + public virtual void AddAdditionalProxiesToRoot(Collection rootCollection) + { + return; + } + + private async Task ConvertAutocadEntity( + Entity entity, + string applicationId, + Collection collectionHost, + IReadOnlyDictionary instanceProxies, + string projectId, + SendPipeline sendPipeline + ) + { + string sourceType = entity.GetType().ToString(); + try + { + Base converted; + if (entity is BlockReference && instanceProxies.TryGetValue(applicationId, out InstanceProxy? instanceProxy)) + { + converted = instanceProxy; + } + else if (_sendConversionCache.TryGetValue(projectId, applicationId, out ObjectReference? value)) + { + converted = value; + } + else + { + converted = _converter.Convert(entity); + converted.applicationId = applicationId; + } + + // NOTE: this is the main part that differentiate from the main root object builder + var reference = await sendPipeline.Process(converted).ConfigureAwait(false); + collectionHost.elements.Add(reference); + return new(Status.SUCCESS, applicationId, sourceType, reference); + } + catch (Exception ex) when (!ex.IsFatal()) + { + _logger.LogSendConversionError(ex, sourceType); + return new(Status.ERROR, applicationId, sourceType, null, ex); + } + } +} diff --git a/Connectors/Autocad/Speckle.Connectors.AutocadShared/Operations/Send/AutocadContinuousTraversalBuilder.cs b/Connectors/Autocad/Speckle.Connectors.AutocadShared/Operations/Send/AutocadContinuousTraversalBuilder.cs new file mode 100644 index 000000000..eb7e6b893 --- /dev/null +++ b/Connectors/Autocad/Speckle.Connectors.AutocadShared/Operations/Send/AutocadContinuousTraversalBuilder.cs @@ -0,0 +1,46 @@ +using Autodesk.AutoCAD.DatabaseServices; +using Microsoft.Extensions.Logging; +using Speckle.Connectors.Autocad.HostApp; +using Speckle.Connectors.Common.Caching; +using Speckle.Converters.Common; +using Speckle.Sdk.Logging; +using Speckle.Sdk.Models.Collections; + +namespace Speckle.Connectors.Autocad.Operations.Send; + +public sealed class AutocadContinuousTraversalBuilder : AutocadContinuousTraversalBaseBuilder +{ + private readonly AutocadLayerUnpacker _layerUnpacker; + + public AutocadContinuousTraversalBuilder( + AutocadLayerUnpacker layerUnpacker, + IRootToSpeckleConverter converter, + ISendConversionCache sendConversionCache, + AutocadInstanceUnpacker instanceObjectManager, + AutocadMaterialUnpacker materialUnpacker, + AutocadColorUnpacker colorUnpacker, + AutocadGroupUnpacker groupUnpacker, + ILogger logger, + ISdkActivityFactory activityFactory + ) + : base( + converter, + sendConversionCache, + instanceObjectManager, + materialUnpacker, + colorUnpacker, + groupUnpacker, + logger, + activityFactory + ) + { + _layerUnpacker = layerUnpacker; + } + + public override (Collection, LayerTableRecord?) CreateObjectCollection(Entity entity, Transaction tr) + { + Layer layer = _layerUnpacker.GetOrCreateSpeckleLayer(entity, tr, out LayerTableRecord? autocadLayer); + + return (layer, autocadLayer); + } +} diff --git a/Connectors/Autocad/Speckle.Connectors.AutocadShared/Operations/Send/AutocadRootObjectBaseBuilder.cs b/Connectors/Autocad/Speckle.Connectors.AutocadShared/Operations/Send/AutocadRootObjectBaseBuilder.cs index 20b5660d2..ba3befb4c 100644 --- a/Connectors/Autocad/Speckle.Connectors.AutocadShared/Operations/Send/AutocadRootObjectBaseBuilder.cs +++ b/Connectors/Autocad/Speckle.Connectors.AutocadShared/Operations/Send/AutocadRootObjectBaseBuilder.cs @@ -16,6 +16,7 @@ using Speckle.Sdk.Logging; using Speckle.Sdk.Models; using Speckle.Sdk.Models.Collections; using Speckle.Sdk.Models.Instances; +using Speckle.Sdk.Pipelines.Progress; namespace Speckle.Connectors.Autocad.Operations.Send; diff --git a/Connectors/Autocad/Speckle.Connectors.AutocadShared/Speckle.Connectors.AutocadShared.projitems b/Connectors/Autocad/Speckle.Connectors.AutocadShared/Speckle.Connectors.AutocadShared.projitems index c393079d5..bac2b770e 100644 --- a/Connectors/Autocad/Speckle.Connectors.AutocadShared/Speckle.Connectors.AutocadShared.projitems +++ b/Connectors/Autocad/Speckle.Connectors.AutocadShared/Speckle.Connectors.AutocadShared.projitems @@ -41,6 +41,8 @@ + + diff --git a/Connectors/Autocad/Speckle.Connectors.Civil3dShared/DependencyInjection/Civil3dConnectorModule.cs b/Connectors/Autocad/Speckle.Connectors.Civil3dShared/DependencyInjection/Civil3dConnectorModule.cs index 4ac057e8d..4cc3ae659 100644 --- a/Connectors/Autocad/Speckle.Connectors.Civil3dShared/DependencyInjection/Civil3dConnectorModule.cs +++ b/Connectors/Autocad/Speckle.Connectors.Civil3dShared/DependencyInjection/Civil3dConnectorModule.cs @@ -22,6 +22,10 @@ public static class Civil3dConnectorModule // add send serviceCollection.LoadSend(); serviceCollection.AddScoped, Civil3dRootObjectBuilder>(); + serviceCollection.AddScoped< + IRootContinuousTraversalBuilder, + Civil3dContinuousTraversalBuilder + >(); serviceCollection.AddSingleton(); // add receive diff --git a/Connectors/Autocad/Speckle.Connectors.Civil3dShared/Operations/Send/Civil3dContinuousTraversalBuilder.cs b/Connectors/Autocad/Speckle.Connectors.Civil3dShared/Operations/Send/Civil3dContinuousTraversalBuilder.cs new file mode 100644 index 000000000..cc3379a15 --- /dev/null +++ b/Connectors/Autocad/Speckle.Connectors.Civil3dShared/Operations/Send/Civil3dContinuousTraversalBuilder.cs @@ -0,0 +1,57 @@ +using Autodesk.AutoCAD.DatabaseServices; +using Microsoft.Extensions.Logging; +using Speckle.Connectors.Autocad.HostApp; +using Speckle.Connectors.Autocad.Operations.Send; +using Speckle.Connectors.Common.Caching; +using Speckle.Connectors.Common.Operations; +using Speckle.Converters.Civil3dShared.ToSpeckle; +using Speckle.Converters.Common; +using Speckle.Sdk.Logging; +using Speckle.Sdk.Models.Collections; + +namespace Speckle.Connectors.Civil3dShared.Operations.Send; + +public sealed class Civil3dContinuousTraversalBuilder : AutocadContinuousTraversalBaseBuilder +{ + private readonly AutocadLayerUnpacker _layerUnpacker; + private readonly PropertySetDefinitionHandler _propertySetDefinitionHandler; + + public Civil3dContinuousTraversalBuilder( + AutocadLayerUnpacker layerUnpacker, + PropertySetDefinitionHandler propertySetDefinitionHandler, + IRootToSpeckleConverter converter, + ISendConversionCache sendConversionCache, + AutocadInstanceUnpacker instanceObjectManager, + AutocadMaterialUnpacker materialUnpacker, + AutocadColorUnpacker colorUnpacker, + AutocadGroupUnpacker groupUnpacker, + ILogger logger, + ISdkActivityFactory activityFactory + ) + : base( + converter, + sendConversionCache, + instanceObjectManager, + materialUnpacker, + colorUnpacker, + groupUnpacker, + logger, + activityFactory + ) + { + _layerUnpacker = layerUnpacker; + _propertySetDefinitionHandler = propertySetDefinitionHandler; + } + + public override (Collection, LayerTableRecord?) CreateObjectCollection(Entity entity, Transaction tr) + { + Layer layer = _layerUnpacker.GetOrCreateSpeckleLayer(entity, tr, out LayerTableRecord? autocadLayer); + + return (layer, autocadLayer); + } + + public override void AddAdditionalProxiesToRoot(Collection rootObject) + { + rootObject[ProxyKeys.PROPERTYSET_DEFINITIONS] = _propertySetDefinitionHandler.Definitions; + } +} diff --git a/Connectors/Autocad/Speckle.Connectors.Civil3dShared/Speckle.Connectors.Civil3dShared.projitems b/Connectors/Autocad/Speckle.Connectors.Civil3dShared/Speckle.Connectors.Civil3dShared.projitems index 578a9fad0..ed43904aa 100644 --- a/Connectors/Autocad/Speckle.Connectors.Civil3dShared/Speckle.Connectors.Civil3dShared.projitems +++ b/Connectors/Autocad/Speckle.Connectors.Civil3dShared/Speckle.Connectors.Civil3dShared.projitems @@ -13,6 +13,7 @@ + diff --git a/Connectors/CSi/Speckle.Connectors.CSiShared/Operations/Send/CsiContinuousTraversalBuilder.cs b/Connectors/CSi/Speckle.Connectors.CSiShared/Operations/Send/CsiContinuousTraversalBuilder.cs new file mode 100644 index 000000000..01a320058 --- /dev/null +++ b/Connectors/CSi/Speckle.Connectors.CSiShared/Operations/Send/CsiContinuousTraversalBuilder.cs @@ -0,0 +1,211 @@ +using Microsoft.Extensions.Logging; +using Speckle.Connectors.Common.Builders; +using Speckle.Connectors.Common.Conversion; +using Speckle.Connectors.Common.Operations; +using Speckle.Connectors.CSiShared.HostApp; +using Speckle.Connectors.CSiShared.HostApp.Helpers; +using Speckle.Connectors.CSiShared.Utils; +using Speckle.Converters.Common; +using Speckle.Converters.CSiShared; +using Speckle.Converters.CSiShared.Extensions; +using Speckle.Converters.CSiShared.Utils; +using Speckle.Sdk; +using Speckle.Sdk.Logging; +using Speckle.Sdk.Models; +using Speckle.Sdk.Models.Collections; +using Speckle.Sdk.Pipelines.Progress; +using Speckle.Sdk.Pipelines.Send; + +namespace Speckle.Connectors.CSiShared.Builders; + +/// +/// Continuous traversal builder for CSi that streams objects through a +/// for packfile-based uploads. Same conversion logic as . +/// +public class CsiContinuousTraversalBuilder : IRootContinuousTraversalBuilder +{ + private readonly IRootToSpeckleConverter _rootToSpeckleConverter; + private readonly IConverterSettingsStore _converterSettings; + private readonly CsiSendCollectionManager _sendCollectionManager; + private readonly IMaterialUnpacker _materialUnpacker; + private readonly ISectionUnpacker _sectionUnpacker; + private readonly ILogger _logger; + private readonly ISdkActivityFactory _activityFactory; + private readonly ICsiApplicationService _csiApplicationService; + private readonly AnalysisResultsExtractor _analysisResultsExtractor; + + public CsiContinuousTraversalBuilder( + IRootToSpeckleConverter rootToSpeckleConverter, + IConverterSettingsStore converterSettings, + CsiSendCollectionManager sendCollectionManager, + IMaterialUnpacker materialUnpacker, + ISectionUnpacker sectionUnpacker, + ILogger logger, + ISdkActivityFactory activityFactory, + ICsiApplicationService csiApplicationService, + AnalysisResultsExtractor analysisResultsExtractor + ) + { + _converterSettings = converterSettings; + _sendCollectionManager = sendCollectionManager; + _materialUnpacker = materialUnpacker; + _sectionUnpacker = sectionUnpacker; + _rootToSpeckleConverter = rootToSpeckleConverter; + _logger = logger; + _activityFactory = activityFactory; + _csiApplicationService = csiApplicationService; + _analysisResultsExtractor = analysisResultsExtractor; + } + + public async Task Build( + IReadOnlyList csiObjects, + string projectId, + SendPipeline sendPipeline, + IProgress onOperationProgressed, + CancellationToken cancellationToken + ) + { + using var activity = _activityFactory.Start("Build"); + + string modelFileName = _csiApplicationService.SapModel.GetModelFilename(false) ?? "Unnamed model"; + (string forceUnit, string tempUnit) = GetForceAndTemperatureUnits(); + + Collection rootObjectCollection = + new() + { + name = modelFileName, + ["units"] = _converterSettings.Current.SpeckleUnits, + ["forceUnits"] = forceUnit, + ["temperatureUnits"] = tempUnit + }; + + List results = new(csiObjects.Count); + int count = 0; + + using (var _ = _activityFactory.Start("Convert all")) + { + foreach (ICsiWrapper csiObject in csiObjects) + { + cancellationToken.ThrowIfCancellationRequested(); + var result = await ConvertCsiObject(csiObject, rootObjectCollection, sendPipeline); + results.Add(result); + + count++; + onOperationProgressed.Report( + new($"Converting objects... ({count:N0} / {csiObjects.Count:N0})", (double)count / csiObjects.Count) + ); + await Task.Yield(); + } + } + + if (results.All(x => x.Status == Status.ERROR)) + { + throw new SpeckleException("Failed to convert all objects"); + } + + using (var _ = _activityFactory.Start("Process Proxies")) + { + rootObjectCollection[ProxyKeys.MATERIAL] = _materialUnpacker.UnpackMaterials().ToList(); + rootObjectCollection[ProxyKeys.SECTION] = _sectionUnpacker.UnpackSections().ToList(); + } + + // Extract analysis results (if applicable) + var objectSelectionSummary = GetObjectSummary(csiObjects); + var selectedCasesAndCombinations = _converterSettings.Current.SelectedLoadCasesAndCombinations; + var requestedResultTypes = _converterSettings.Current.SelectedResultTypes; + + if (selectedCasesAndCombinations?.Count > 0) + { + if (requestedResultTypes == null || requestedResultTypes.Count == 0) + { + throw new SpeckleException( + "Adjust publish settings - no result type input for the requested load cases and combinations" + ); + } + + if (!_csiApplicationService.SapModel.GetModelIsLocked()) + { + throw new SpeckleException("Model unlocked, no access to analysis results"); + } + + try + { + var analysisResults = _analysisResultsExtractor.ExtractAnalysisResults( + selectedCasesAndCombinations, + requestedResultTypes, + objectSelectionSummary + ); + rootObjectCollection[RootKeys.ANALYSIS_RESULTS] = analysisResults; + } + catch (Exception ex) + { + throw new SpeckleException("Analysis result extraction failed", ex); + } + } + + // Process root collection and wait for all uploads + await sendPipeline.Process(rootObjectCollection); + await sendPipeline.WaitForUpload(); + + return new RootObjectBuilderResult(rootObjectCollection, results); + } + + private async Task ConvertCsiObject( + ICsiWrapper csiObject, + Collection typeCollection, + SendPipeline sendPipeline + ) + { + string sourceType = csiObject.ObjectName; + string applicationId = csiObject switch + { + CsiJointWrapper jointWrapper => jointWrapper.GetSpeckleApplicationId(_csiApplicationService.SapModel), + CsiFrameWrapper frameWrapper => frameWrapper.GetSpeckleApplicationId(_csiApplicationService.SapModel), + CsiCableWrapper cableWrapper => cableWrapper.GetSpeckleApplicationId(_csiApplicationService.SapModel), + CsiTendonWrapper tendonWrapper => tendonWrapper.ObjectName, + CsiShellWrapper shellWrapper => shellWrapper.GetSpeckleApplicationId(_csiApplicationService.SapModel), + CsiSolidWrapper solidWrapper => solidWrapper.GetSpeckleApplicationId(_csiApplicationService.SapModel), + CsiLinkWrapper linkWrapper => linkWrapper.GetSpeckleApplicationId(_csiApplicationService.SapModel), + _ => throw new ArgumentException($"Unsupported wrapper type: {csiObject.GetType()}", nameof(csiObject)) + }; + + try + { + Base converted = _rootToSpeckleConverter.Convert(csiObject); + + var collection = _sendCollectionManager.AddObjectCollectionToRoot(converted, typeCollection); + + // NOTE: this is the main part that differentiate from the main root object builder + var reference = await sendPipeline.Process(converted).ConfigureAwait(false); + collection.elements.Add(reference); + + return new(Status.SUCCESS, applicationId, sourceType, reference); + } + catch (NotImplementedException ex) + { + _logger.LogError(ex, "Failed to convert object {sourceType}", sourceType); + return new(Status.WARNING, applicationId, sourceType, null, ex); + } + catch (Exception ex) when (!ex.IsFatal()) + { + _logger.LogError(ex, "Failed to convert object {sourceType}", sourceType); + return new(Status.ERROR, applicationId, sourceType, null, ex); + } + } + + private Dictionary> GetObjectSummary(IReadOnlyList csiObjects) => + csiObjects + .GroupBy(csiObject => csiObject.ObjectType) + .ToDictionary(group => group.Key, group => group.Select(obj => obj.Name).ToList()); + + private (string, string) GetForceAndTemperatureUnits() + { + var forceUnit = eForce.NotApplicable; + var lengthUnit = eLength.NotApplicable; + var temperatureUnit = eTemperature.NotApplicable; + + _converterSettings.Current.SapModel.GetDatabaseUnits_2(ref forceUnit, ref lengthUnit, ref temperatureUnit); + + return (forceUnit.ToString(), temperatureUnit.ToString()); + } +} diff --git a/Connectors/CSi/Speckle.Connectors.CSiShared/Operations/Send/CsiRootObjectBuilder.cs b/Connectors/CSi/Speckle.Connectors.CSiShared/Operations/Send/CsiRootObjectBuilder.cs index 0df394a30..00c110ba3 100644 --- a/Connectors/CSi/Speckle.Connectors.CSiShared/Operations/Send/CsiRootObjectBuilder.cs +++ b/Connectors/CSi/Speckle.Connectors.CSiShared/Operations/Send/CsiRootObjectBuilder.cs @@ -13,6 +13,7 @@ using Speckle.Sdk; using Speckle.Sdk.Logging; using Speckle.Sdk.Models; using Speckle.Sdk.Models.Collections; +using Speckle.Sdk.Pipelines.Progress; namespace Speckle.Connectors.CSiShared.Builders; diff --git a/Connectors/CSi/Speckle.Connectors.CSiShared/ServiceRegistration.cs b/Connectors/CSi/Speckle.Connectors.CSiShared/ServiceRegistration.cs index 0a3942caf..c0b224cd6 100644 --- a/Connectors/CSi/Speckle.Connectors.CSiShared/ServiceRegistration.cs +++ b/Connectors/CSi/Speckle.Connectors.CSiShared/ServiceRegistration.cs @@ -43,6 +43,7 @@ public static class ServiceRegistration services.AddScoped(); services.AddScoped(); services.AddScoped, CsiRootObjectBuilder>(); + services.AddScoped, CsiContinuousTraversalBuilder>(); services.AddScoped>(); services.AddScoped(); diff --git a/Connectors/CSi/Speckle.Connectors.CSiShared/Speckle.Connectors.CSiShared.projitems b/Connectors/CSi/Speckle.Connectors.CSiShared/Speckle.Connectors.CSiShared.projitems index 7dd388f54..9e9dca126 100644 --- a/Connectors/CSi/Speckle.Connectors.CSiShared/Speckle.Connectors.CSiShared.projitems +++ b/Connectors/CSi/Speckle.Connectors.CSiShared/Speckle.Connectors.CSiShared.projitems @@ -29,6 +29,7 @@ + diff --git a/Connectors/Navisworks/Speckle.Connectors.NavisworksShared/Bindings/NavisworksSendBinding.cs b/Connectors/Navisworks/Speckle.Connectors.NavisworksShared/Bindings/NavisworksSendBinding.cs index e5afddd40..a76394d72 100644 --- a/Connectors/Navisworks/Speckle.Connectors.NavisworksShared/Bindings/NavisworksSendBinding.cs +++ b/Connectors/Navisworks/Speckle.Connectors.NavisworksShared/Bindings/NavisworksSendBinding.cs @@ -5,7 +5,6 @@ using Speckle.Connector.Navisworks.Operations.Send.Filters; using Speckle.Connector.Navisworks.Operations.Send.Settings; using Speckle.Connector.Navisworks.Services; using Speckle.Connectors.Common.Cancellation; -using Speckle.Connectors.Common.Operations; using Speckle.Connectors.Common.Threading; using Speckle.Connectors.DUI.Bindings; using Speckle.Connectors.DUI.Bridge; @@ -18,6 +17,7 @@ using Speckle.Converter.Navisworks.Services; using Speckle.Converter.Navisworks.Settings; using Speckle.Converters.Common; using Speckle.Sdk.Common; +using Speckle.Sdk.Pipelines.Progress; namespace Speckle.Connector.Navisworks.Bindings; diff --git a/Connectors/Navisworks/Speckle.Connectors.NavisworksShared/DependencyInjection/NavisworksConnectorServiceRegistration.cs b/Connectors/Navisworks/Speckle.Connectors.NavisworksShared/DependencyInjection/NavisworksConnectorServiceRegistration.cs index 845b6ed62..cb9ab254e 100644 --- a/Connectors/Navisworks/Speckle.Connectors.NavisworksShared/DependencyInjection/NavisworksConnectorServiceRegistration.cs +++ b/Connectors/Navisworks/Speckle.Connectors.NavisworksShared/DependencyInjection/NavisworksConnectorServiceRegistration.cs @@ -57,6 +57,7 @@ public static class NavisworksConnectorServiceRegistration // Sending operations serviceCollection.AddScoped, NavisworksRootObjectBuilder>(); + serviceCollection.AddScoped, NavisworksContinuousTraversalBuilder>(); serviceCollection.AddScoped>(); serviceCollection.AddSingleton(DefaultTraversal.CreateTraversalFunc()); serviceCollection.AddSingleton(); diff --git a/Connectors/Navisworks/Speckle.Connectors.NavisworksShared/Operations/Send/NavisworksContinuousTraversalBuilder.cs b/Connectors/Navisworks/Speckle.Connectors.NavisworksShared/Operations/Send/NavisworksContinuousTraversalBuilder.cs new file mode 100644 index 000000000..ab1563a46 --- /dev/null +++ b/Connectors/Navisworks/Speckle.Connectors.NavisworksShared/Operations/Send/NavisworksContinuousTraversalBuilder.cs @@ -0,0 +1,400 @@ +using Microsoft.Extensions.Logging; +using Speckle.Connector.Navisworks.HostApp; +using Speckle.Connectors.Common.Builders; +using Speckle.Connectors.Common.Conversion; +using Speckle.Converter.Navisworks.Helpers; +using Speckle.Converter.Navisworks.Services; +using Speckle.Converter.Navisworks.Settings; +using Speckle.Converters.Common; +using Speckle.Objects.Data; +using Speckle.Sdk; +using Speckle.Sdk.Logging; +using Speckle.Sdk.Models; +using Speckle.Sdk.Models.Collections; +using Speckle.Sdk.Models.Instances; +using Speckle.Sdk.Pipelines.Progress; +using Speckle.Sdk.Pipelines.Send; +using static Speckle.Connector.Navisworks.Operations.Send.GeometryNodeMerger; +using static Speckle.Connectors.Common.Operations.ProxyKeys; +using static Speckle.Converter.Navisworks.Constants.InstanceConstants; + +namespace Speckle.Connector.Navisworks.Operations.Send; + +/// +/// Continuous traversal builder for Navisworks that streams objects through a +/// for packfile-based uploads. Same conversion/grouping logic as , +/// but processes final elements through the pipeline after all post-processing is complete. +/// +public class NavisworksContinuousTraversalBuilder( + IRootToSpeckleConverter rootToSpeckleConverter, + IConverterSettingsStore converterSettings, + ILogger logger, + ISdkActivityFactory activityFactory, + NavisworksMaterialUnpacker materialUnpacker, + NavisworksColorUnpacker colorUnpacker, + Speckle.Converter.Navisworks.Constants.Registers.IInstanceFragmentRegistry instanceRegistry, + IElementSelectionService elementSelectionService, + IUiUnitsCache uiUnitsCache +) : IRootContinuousTraversalBuilder +{ + private bool SkipNodeMerging { get; set; } + private bool DisableGroupingForInstanceTesting { get; set; } + + public async Task Build( + IReadOnlyList navisworksModelItems, + string projectId, + SendPipeline sendPipeline, + IProgress onOperationProgressed, + CancellationToken cancellationToken + ) + { +#if DEBUG + SkipNodeMerging = false; + DisableGroupingForInstanceTesting = false; +#endif + using var activity = activityFactory.Start("Build"); + + ValidateInputs(navisworksModelItems, projectId, onOperationProgressed); + + var rootCollection = InitializeRootCollection(); + (Dictionary convertedElements, List conversionResults) = + await ConvertModelItemsAsync(navisworksModelItems, onOperationProgressed, cancellationToken); + + ValidateConversionResults(conversionResults); + + var groupedNodes = SkipNodeMerging ? [] : GroupSiblingGeometryNodes(navisworksModelItems); + var finalElements = BuildFinalElements(convertedElements, groupedNodes); + + await AddProxiesToCollection(rootCollection, navisworksModelItems, groupedNodes); + + AddInstanceDefinitionsToCollection(rootCollection, ref finalElements); + + // Process each final element through the send pipeline + var processedElements = new List(finalElements.Count); + foreach (var element in finalElements) + { + cancellationToken.ThrowIfCancellationRequested(); + // NOTE: this is the main part that differentiate from the main root object builder + var reference = await sendPipeline.Process(element).ConfigureAwait(false); + processedElements.Add(reference); + } + + rootCollection.elements = processedElements; + + // Process the root collection and wait for all uploads to complete + await sendPipeline.Process(rootCollection); + await sendPipeline.WaitForUpload(); + + return new RootObjectBuilderResult(rootCollection, conversionResults); + } + + private static void ValidateInputs( + IReadOnlyList navisworksModelItems, + string projectId, + IProgress onOperationProgressed + ) + { + if (!navisworksModelItems.Any()) + { + throw new SpeckleException("No objects to convert"); + } + + if (navisworksModelItems == null) + { + throw new ArgumentNullException(nameof(navisworksModelItems)); + } + + if (onOperationProgressed == null || projectId == null) + { + throw new ArgumentNullException( + onOperationProgressed == null ? nameof(onOperationProgressed) : nameof(projectId) + ); + } + } + + private Collection InitializeRootCollection() => + new() + { + name = NavisworksApp.ActiveDocument.Title ?? "Unnamed model", + ["units"] = converterSettings.Current.Derived.SpeckleUnits + }; + + private Task<(Dictionary converted, List results)> ConvertModelItemsAsync( + IReadOnlyList navisworksModelItems, + IProgress onOperationProgressed, + CancellationToken cancellationToken + ) + { + var results = new List(navisworksModelItems.Count); + var convertedBases = new Dictionary(); + int processedCount = 0; + int totalCount = navisworksModelItems.Count; + + foreach (var item in navisworksModelItems) + { + cancellationToken.ThrowIfCancellationRequested(); + var converted = ConvertNavisworksItem(item, convertedBases); + results.Add(converted); + + processedCount++; + onOperationProgressed.Report(new CardProgress("Converting", (double)processedCount / totalCount)); + } + + return Task.FromResult((convertedBases, results)); + } + + private static void ValidateConversionResults(List results) + { + if (results.All(x => x.Status == Status.ERROR)) + { + throw new SpeckleException("Failed to convert all objects."); + } + } + + private List BuildFinalElements( + Dictionary convertedBases, + Dictionary> groupedNodes + ) + { + var finalElements = new List(); + var processedPaths = new HashSet(); + + if (!DisableGroupingForInstanceTesting) + { + AddGroupedElements(finalElements, convertedBases, groupedNodes, processedPaths); + logger.LogInformation( + "After grouping: {grouped} paths processed, {elements} elements in collection", + processedPaths.Count, + finalElements.Count + ); + } + else + { + logger.LogInformation("Grouping disabled for instance testing"); + } + + if (converterSettings.Current.User.PreserveModelHierarchy) + { + logger.LogInformation("Building hierarchy (PreserveModelHierarchy=true)"); + var hierarchyBuilder = new NavisworksHierarchyBuilder( + convertedBases, + rootToSpeckleConverter, + elementSelectionService + ); + + return hierarchyBuilder.BuildHierarchy(); + } + + logger.LogInformation("Adding remaining elements (flat mode)"); + AddRemainingElements(finalElements, convertedBases, processedPaths); + + logger.LogInformation("Final elements count: {count}", finalElements.Count); + return finalElements; + } + + private void AddGroupedElements( + List finalElements, + Dictionary convertedBases, + Dictionary> groupedNodes, + HashSet processedPaths + ) + { + foreach (var group in groupedNodes) + { + var siblingBases = new List(group.Value.Count); + foreach (var itemPath in group.Value.Select(elementSelectionService.GetModelItemPath)) + { + processedPaths.Add(itemPath); + if (convertedBases.TryGetValue(itemPath, out var convertedBase) && convertedBase != null) + { + siblingBases.Add(convertedBase); + } + } + + if (siblingBases.Count > 0) + { + finalElements.Add(CreateNavisworksObject(group.Key, siblingBases)); + } + } + } + + private void AddRemainingElements( + List finalElements, + Dictionary convertedBases, + HashSet processedPaths + ) + { + foreach (var kvp in convertedBases.Where(kvp => !processedPaths.Contains(kvp.Key))) + { + switch (kvp.Value) + { + case null: + continue; + case Collection collection: + finalElements.Add(collection); + break; + default: + if (CreateNavisworksObject(kvp.Value) is { } navisworksObject) + { + finalElements.Add(navisworksObject); + } + + break; + } + } + } + + private (string name, string path) GetElementNameAndPath(string applicationId) + { + var modelItem = elementSelectionService.GetModelItemFromPath(applicationId); + var context = HierarchyHelper.ExtractContext(modelItem); + return (context.Name, context.Path); + } + + private NavisworksObject CreateNavisworksObject(string groupKey, List siblingBases) + { + string cleanParentPath = ElementSelectionHelper.GetCleanPath(groupKey); + (string name, string path) = GetElementNameAndPath(cleanParentPath); + + int estimatedCapacity = siblingBases.Sum(b => (b["displayValue"] as List)?.Count ?? 0); + var displayValues = new List(estimatedCapacity); + displayValues.AddRange( + siblingBases + .Where(sibling => sibling["displayValue"] is List) + .SelectMany(sibling => (List)sibling["displayValue"]!) + ); + + return new NavisworksObject + { + name = name, + displayValue = displayValues, + properties = siblingBases.First()["properties"] as Dictionary ?? [], + units = converterSettings.Current.Derived.SpeckleUnits, + applicationId = groupKey, + ["path"] = path + }; + } + + private NavisworksObject? CreateNavisworksObject(Base convertedBase) + { + if (convertedBase.applicationId == null) + { + return null; + } + + (string name, string path) = GetElementNameAndPath(convertedBase.applicationId); + + var units = uiUnitsCache.Ensure(); + + return new NavisworksObject + { + name = name, + displayValue = convertedBase["displayValue"] as List ?? [], + properties = convertedBase["properties"] as Dictionary ?? [], + units = units.ToString(), + applicationId = convertedBase.applicationId, + ["path"] = path + }; + } + + private Task AddProxiesToCollection( + Collection rootCollection, + IReadOnlyList navisworksModelItems, + Dictionary> groupedNodes + ) + { + using var _ = activityFactory.Start("UnpackProxies"); + + var renderMaterials = materialUnpacker.UnpackRenderMaterial(navisworksModelItems, groupedNodes); + if (renderMaterials.Count > 0) + { + rootCollection[RENDER_MATERIAL] = renderMaterials; + } + + var colors = colorUnpacker.UnpackColor(navisworksModelItems, groupedNodes); + if (colors.Count > 0) + { + rootCollection[COLOR] = colors; + } + + return Task.CompletedTask; + } + + private void AddInstanceDefinitionsToCollection(Collection rootCollection, ref List finalElements) + { + using var _ = activityFactory.Start("BuildInstanceDefinitions"); + + var allDefinitions = instanceRegistry.GetAllDefinitionGeometries(); + + if (allDefinitions.Count == 0) + { + logger.LogInformation("No instance definitions found - instancing may be disabled"); + return; + } + + logger.LogInformation("Building instance structure for {count} definition groups", allDefinitions.Count); + + var instanceDefinitionProxies = new List(allDefinitions.Count); + + int estimatedGeometryCount = allDefinitions.Sum(kvp => kvp.Value.Count); + var allDefinitionGeometries = new List(estimatedGeometryCount); + + foreach (var kvp in allDefinitions) + { + var groupKey = kvp.Key; + var geometries = kvp.Value; + var groupKeyPath = groupKey.ToPathString(); + + var defProxy = new InstanceDefinitionProxy + { + name = $"Shared Geometry {groupKeyPath}", + objects = geometries.Select(g => g.applicationId ?? "").Where(id => !string.IsNullOrEmpty(id)).ToList(), + applicationId = $"{DEFINITION_ID_PREFIX}{groupKeyPath}", + maxDepth = 0 + }; + + instanceDefinitionProxies.Add(defProxy); + allDefinitionGeometries.AddRange(geometries); + } + + rootCollection[INSTANCE_DEFINITION] = instanceDefinitionProxies; + var geometryDefinitionsCollection = new Collection + { + name = "Geometry Definitions", + elements = allDefinitionGeometries + }; + + var objectCollection = new Collection { name = "", elements = finalElements }; + + finalElements = [geometryDefinitionsCollection, objectCollection]; + + logger.LogInformation( + "Added {proxyCount} instance definition proxies and {geomCount} definition geometries", + instanceDefinitionProxies.Count, + allDefinitionGeometries.Count + ); + } + + private SendConversionResult ConvertNavisworksItem( + NAV.ModelItem navisworksItem, + Dictionary convertedBases + ) + { + string applicationId = elementSelectionService.GetModelItemPath(navisworksItem); + string sourceType = navisworksItem.GetType().Name; + + try + { + Base converted = rootToSpeckleConverter.Convert(navisworksItem); + + convertedBases[applicationId] = converted; + + return new SendConversionResult(Status.SUCCESS, applicationId, sourceType, converted); + } + catch (Exception ex) when (!ex.IsFatal()) + { + logger.LogError(ex, "Failed to convert model item {id}", applicationId); + return new SendConversionResult(Status.ERROR, applicationId, "ModelItem", null, ex); + } + } +} diff --git a/Connectors/Navisworks/Speckle.Connectors.NavisworksShared/Operations/Send/NavisworksRootObjectBuilder.cs b/Connectors/Navisworks/Speckle.Connectors.NavisworksShared/Operations/Send/NavisworksRootObjectBuilder.cs index 9297b38af..45185cf43 100644 --- a/Connectors/Navisworks/Speckle.Connectors.NavisworksShared/Operations/Send/NavisworksRootObjectBuilder.cs +++ b/Connectors/Navisworks/Speckle.Connectors.NavisworksShared/Operations/Send/NavisworksRootObjectBuilder.cs @@ -3,7 +3,6 @@ using Speckle.Connector.Navisworks.HostApp; using Speckle.Connectors.Common.Builders; using Speckle.Connectors.Common.Caching; using Speckle.Connectors.Common.Conversion; -using Speckle.Connectors.Common.Operations; using Speckle.Converter.Navisworks.Helpers; using Speckle.Converter.Navisworks.Services; using Speckle.Converter.Navisworks.Settings; @@ -14,6 +13,7 @@ using Speckle.Sdk.Logging; using Speckle.Sdk.Models; using Speckle.Sdk.Models.Collections; using Speckle.Sdk.Models.Instances; +using Speckle.Sdk.Pipelines.Progress; using static Speckle.Connector.Navisworks.Operations.Send.GeometryNodeMerger; using static Speckle.Connectors.Common.Operations.ProxyKeys; using static Speckle.Converter.Navisworks.Constants.InstanceConstants; diff --git a/Connectors/Navisworks/Speckle.Connectors.NavisworksShared/Speckle.Connectors.NavisworksShared.projitems b/Connectors/Navisworks/Speckle.Connectors.NavisworksShared/Speckle.Connectors.NavisworksShared.projitems index 9e2fd10e7..7581669d9 100644 --- a/Connectors/Navisworks/Speckle.Connectors.NavisworksShared/Speckle.Connectors.NavisworksShared.projitems +++ b/Connectors/Navisworks/Speckle.Connectors.NavisworksShared/Speckle.Connectors.NavisworksShared.projitems @@ -24,6 +24,7 @@ + diff --git a/Connectors/Revit/Speckle.Connectors.RevitShared/DependencyInjection/RevitConnectorModule.cs b/Connectors/Revit/Speckle.Connectors.RevitShared/DependencyInjection/RevitConnectorModule.cs index 7971cbb99..b496f4e1c 100644 --- a/Connectors/Revit/Speckle.Connectors.RevitShared/DependencyInjection/RevitConnectorModule.cs +++ b/Connectors/Revit/Speckle.Connectors.RevitShared/DependencyInjection/RevitConnectorModule.cs @@ -65,6 +65,7 @@ public static class ServiceRegistration serviceCollection.AddScoped(); serviceCollection.AddScoped(); serviceCollection.AddScoped, RevitRootObjectBuilder>(); + serviceCollection.AddScoped, RevitContinuousTraversalBuilder>(); serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); diff --git a/Connectors/Revit/Speckle.Connectors.RevitShared/HostApp/RevitFamilyBaker.cs b/Connectors/Revit/Speckle.Connectors.RevitShared/HostApp/RevitFamilyBaker.cs index b8c3d71da..85d6704fd 100644 --- a/Connectors/Revit/Speckle.Connectors.RevitShared/HostApp/RevitFamilyBaker.cs +++ b/Connectors/Revit/Speckle.Connectors.RevitShared/HostApp/RevitFamilyBaker.cs @@ -4,7 +4,6 @@ using Autodesk.Revit.DB; using Autodesk.Revit.DB.Structure; using Microsoft.Extensions.Logging; using Speckle.Connectors.Common.Conversion; -using Speckle.Connectors.Common.Operations; using Speckle.Converters.Common; using Speckle.Converters.Common.Objects; using Speckle.Converters.RevitShared.Helpers; @@ -18,6 +17,7 @@ using Speckle.Sdk.Models; using Speckle.Sdk.Models.Collections; using Speckle.Sdk.Models.GraphTraversal; using Speckle.Sdk.Models.Instances; +using Speckle.Sdk.Pipelines.Progress; using DB = Autodesk.Revit.DB; using Document = Autodesk.Revit.DB.Document; diff --git a/Connectors/Revit/Speckle.Connectors.RevitShared/Operations/Receive/RevitHostObjectBuilder.cs b/Connectors/Revit/Speckle.Connectors.RevitShared/Operations/Receive/RevitHostObjectBuilder.cs index dc598a92e..054a31959 100644 --- a/Connectors/Revit/Speckle.Connectors.RevitShared/Operations/Receive/RevitHostObjectBuilder.cs +++ b/Connectors/Revit/Speckle.Connectors.RevitShared/Operations/Receive/RevitHostObjectBuilder.cs @@ -24,6 +24,7 @@ using Speckle.Sdk.Models; using Speckle.Sdk.Models.Collections; using Speckle.Sdk.Models.GraphTraversal; using Speckle.Sdk.Models.Instances; +using Speckle.Sdk.Pipelines.Progress; namespace Speckle.Connectors.Revit.Operations.Receive; diff --git a/Connectors/Revit/Speckle.Connectors.RevitShared/Operations/Send/RevitContinuousTraversalBuilder.cs b/Connectors/Revit/Speckle.Connectors.RevitShared/Operations/Send/RevitContinuousTraversalBuilder.cs new file mode 100644 index 000000000..9cdf309f7 --- /dev/null +++ b/Connectors/Revit/Speckle.Connectors.RevitShared/Operations/Send/RevitContinuousTraversalBuilder.cs @@ -0,0 +1,318 @@ +using System.Diagnostics.CodeAnalysis; +using Autodesk.Revit.DB; +using Microsoft.Extensions.Logging; +using Speckle.Connectors.Common.Builders; +using Speckle.Connectors.Common.Caching; +using Speckle.Connectors.Common.Conversion; +using Speckle.Connectors.Common.Extensions; +using Speckle.Connectors.Common.Operations; +using Speckle.Connectors.Common.Threading; +using Speckle.Connectors.DUI.Exceptions; +using Speckle.Connectors.DUI.Settings; +using Speckle.Connectors.Revit.HostApp; +using Speckle.Converters.Common; +using Speckle.Converters.RevitShared.Helpers; +using Speckle.Converters.RevitShared.Settings; +using Speckle.Sdk; +using Speckle.Sdk.Common; +using Speckle.Sdk.Models; +using Speckle.Sdk.Models.Collections; +using Speckle.Sdk.Pipelines.Progress; +using Speckle.Sdk.Pipelines.Send; + +namespace Speckle.Connectors.Revit.Operations.Send; + +public class RevitContinuousTraversalBuilder( + IRootToSpeckleConverter converter, + IConverterSettingsStore converterSettings, + ISendConversionCache sendConversionCache, + ElementUnpacker elementUnpacker, + LevelUnpacker levelUnpacker, + ViewUnpacker viewUnpacker, + IThreadContext threadContext, + SendCollectionManager sendCollectionManager, + ILogger logger, + RevitToSpeckleCacheSingleton revitToSpeckleCacheSingleton, + LinkedModelHandler linkedModelHandler, + IConfigStore configStore +) : IRootContinuousTraversalBuilder +{ + public async Task Build( + IReadOnlyList documentElementContexts, + string projectId, + SendPipeline sendPipeline, + IProgress onOperationProgressed, + CancellationToken cancellationToken + ) + { + return await threadContext.RunOnMainAsync( + async () => + await BuildMainThread( + documentElementContexts, + projectId, + sendPipeline, + onOperationProgressed, + cancellationToken + ) + ); + } + + [SuppressMessage("Maintainability", "CA1502:Avoid excessive class coupling")] + [SuppressMessage("Maintainability", "CA1506:Avoid excessive class coupling")] + private async Task BuildMainThread( + IReadOnlyList documentElementContexts, + string projectId, + SendPipeline sendPipeline, + IProgress onOperationProgressed, + CancellationToken cancellationToken + ) + { + var doc = converterSettings.Current.Document; + + if (doc.IsFamilyDocument) + { + throw new SpeckleException("Family Environment documents are not supported."); + } + + // init the root + Collection rootObject = + new() { name = converterSettings.Current.Document.PathName.Split('\\').Last().Split('.').First() }; + rootObject["units"] = converterSettings.Current.SpeckleUnits; + + var filteredDocumentsToConvert = new List(); + bool sendWithLinkedModels = converterSettings.Current.SendLinkedModels; + List results = new(); + + // Prepare linked model display names if needed + if (sendWithLinkedModels) + { + linkedModelHandler.PrepareLinkedModelNames(documentElementContexts); + } + + foreach (var documentElementContext in documentElementContexts) + { + // add appropriate warnings for linked documents + if (documentElementContext.Doc.IsLinked && !sendWithLinkedModels) + { + results.Add( + new( + Status.WARNING, + documentElementContext.Doc.PathName, + typeof(RevitLinkInstance).ToString(), + null, + new SpeckleException("Enable linked model support from the settings to send this object") + ) + ); + continue; + } + + // filter for valid elements + // if send linked models setting is disabled List will be empty, and we won't enter foreach loop + var elementsInTransform = new List(); + foreach (var el in documentElementContext.Elements) + { + if (el == null || el.Category == null) + { + continue; + } + elementsInTransform.Add(el); + } + + // only add contexts with elements + if (elementsInTransform.Count > 0) + { + filteredDocumentsToConvert.Add(documentElementContext with { Elements = elementsInTransform }); + } + } + + // TODO: check the exception!!!! + if (filteredDocumentsToConvert.Count == 0) + { + throw new SpeckleSendFilterException("No objects were found. Please update your publish filter!"); + } + + // Unpack groups (& other complex data structures) + var atomicObjectsByDocumentAndTransform = new List(); + var atomicObjectCount = 0; + foreach (var filteredDocumentToConvert in filteredDocumentsToConvert) + { + using ( + converterSettings.Push(currentSettings => currentSettings with { Document = filteredDocumentToConvert.Doc }) + ) + { + var atomicObjects = elementUnpacker + .UnpackSelectionForConversion(filteredDocumentToConvert.Elements, filteredDocumentToConvert.Doc) + .ToList(); + atomicObjectsByDocumentAndTransform.Add(filteredDocumentToConvert with { Elements = atomicObjects }); + atomicObjectCount += atomicObjects.Count; + } + } + + var count = 0; + var cacheHitCount = 0; + var skippedObjectCount = 0; + + var config = configStore.GetConnectorConfig(); + + foreach (var atomicObjectByDocumentAndTransform in atomicObjectsByDocumentAndTransform) + { + string? modelDisplayName = null; + if (atomicObjectByDocumentAndTransform.Doc.IsLinked) + { + string id = linkedModelHandler.GetIdFromDocumentToConvert(atomicObjectByDocumentAndTransform); + linkedModelHandler.LinkedModelDisplayNames.TryGetValue(id, out modelDisplayName); + } + + // here we do magic for changing the transform and the related document according to model. first one is always the main model. + using ( + converterSettings.Push(currentSettings => + currentSettings with + { + ReferencePointTransform = atomicObjectByDocumentAndTransform.Transform, + Document = atomicObjectByDocumentAndTransform.Doc, + } + ) + ) + { + var atomicObjects = atomicObjectByDocumentAndTransform.Elements; + foreach (Element revitElement in atomicObjects) + { + cancellationToken.ThrowIfCancellationRequested(); + string applicationId = revitElement.UniqueId; + string sourceType = revitElement.GetType().Name; + try + { + if (!SupportedCategoriesUtils.IsSupportedCategory(revitElement.Category)) + { + var cat = revitElement.Category != null ? revitElement.Category.Name : "No category"; + results.Add( + new( + Status.WARNING, + revitElement.UniqueId, + cat, + null, + new SpeckleException($"Category {cat} is not supported.") + ) + ); + skippedObjectCount++; + continue; + } + + Base converted; + bool hasTransform = atomicObjectByDocumentAndTransform.Transform != null; + + // non-transformed elements can safely rely on cache + // TODO: Potential here to transform cached objects and NOT reconvert, + // TODO: we wont do !hasTransform here, and re-set application id before this + + if ( + !hasTransform + && !config.DocumentChangeListeningDisabled //This is experimental + && sendConversionCache.TryGetValue(projectId, applicationId, out ObjectReference? value) + ) + { + converted = value; + cacheHitCount++; + } + // not in cache means we convert + else + { + // if it has a transform we append transform hash to the applicationId to distinguish the elements from other instances + if (hasTransform) + { + string transformHash = linkedModelHandler.GetTransformHash( + atomicObjectByDocumentAndTransform.Transform.NotNull() + ); + applicationId = $"{applicationId}_t{transformHash}"; + } + // normal conversions + converted = converter.Convert(revitElement); + converted.applicationId = applicationId; + } + + // NOTE: this is the main part that differentiate from the main root object builder + var reference = await sendPipeline.Process(converted).ConfigureAwait(true); + + var collection = sendCollectionManager.GetAndCreateObjectHostCollection( + revitElement, + rootObject, + sendWithLinkedModels, + modelDisplayName + ); + + collection.elements.Add(reference); + results.Add(new(Status.SUCCESS, applicationId, sourceType, reference)); + } + catch (Exception ex) when (!ex.IsFatal()) + { + logger.LogSendConversionError(ex, sourceType); + results.Add(new(Status.ERROR, applicationId, sourceType, null, ex)); + } + + count++; + onOperationProgressed.Report( + new($"Converting objects... ({count:N0} / {atomicObjectCount:N0})", (double)count / atomicObjectCount) + ); + } + } + } + + // if we ended up skipping everything, there is a reason for this, that users can diagnose themselves + // this can occur if a published view contains only unsupported objects or if user trying to ONLY send linked model + // docs but the setting is disabled + if (skippedObjectCount == atomicObjectCount) + { + throw new SpeckleException("No supported objects visible. Update publish filter or check publish settings."); + } + + // this is, I suppose, fully on us? + if (results.All(x => x.Status == Status.ERROR)) + { + throw new SpeckleException("Failed to convert all objects."); + } + + // STEP 5: Unpack proxies to attach to root collection + var flatElements = atomicObjectsByDocumentAndTransform.SelectMany(t => t.Elements).ToList(); + var idsAndSubElementIds = elementUnpacker.GetElementsAndSubelementIdsFromAtomicObjects(flatElements); + + var renderMaterialProxies = revitToSpeckleCacheSingleton.GetRenderMaterialProxyListForObjects(idsAndSubElementIds); + rootObject[ProxyKeys.RENDER_MATERIAL] = renderMaterialProxies; + + var levelProxies = levelUnpacker.Unpack(flatElements); + rootObject[ProxyKeys.LEVEL] = levelProxies; + + rootObject[ProxyKeys.INSTANCE_DEFINITION] = revitToSpeckleCacheSingleton.GetInstanceDefinitionProxiesForObjects( + idsAndSubElementIds + ); + rootObject.elements.Add( + new Collection() + { + elements = revitToSpeckleCacheSingleton.GetBaseObjectsForObjects(idsAndSubElementIds), + name = "definitionGeometry" + } + ); + + // STEP 6: Unpack all other objects to attach to root collection + List views = viewUnpacker.Unpack(converterSettings.Current.Document); + + if (views.Count > 0) + { + rootObject[RootKeys.VIEW] = views; + } + + // NOTE: these are currently not used anywhere, we'll skip them until someone calls for it back + // rootObject[ProxyKeys.PARAMETER_DEFINITIONS] = _parameterDefinitionHandler.Definitions; + + // we want to store transform data for chosen reference point setting + if (converterSettings.Current.ReferencePointTransform is Transform transform) + { + var transformMatrix = ReferencePointHelper.CreateTransformDataForRootObject(transform); + rootObject[RootKeys.REFERENCE_POINT_TRANSFORM] = transformMatrix; + } + + await sendPipeline.Process(rootObject); + await sendPipeline.WaitForUpload(); + + return new RootObjectBuilderResult(rootObject, results); + } +} diff --git a/Connectors/Revit/Speckle.Connectors.RevitShared/Operations/Send/RevitRootObjectBuilder.cs b/Connectors/Revit/Speckle.Connectors.RevitShared/Operations/Send/RevitRootObjectBuilder.cs index d10c99190..7b3e58a9d 100644 --- a/Connectors/Revit/Speckle.Connectors.RevitShared/Operations/Send/RevitRootObjectBuilder.cs +++ b/Connectors/Revit/Speckle.Connectors.RevitShared/Operations/Send/RevitRootObjectBuilder.cs @@ -17,6 +17,7 @@ using Speckle.Sdk; using Speckle.Sdk.Common; using Speckle.Sdk.Models; using Speckle.Sdk.Models.Collections; +using Speckle.Sdk.Pipelines.Progress; namespace Speckle.Connectors.Revit.Operations.Send; @@ -39,7 +40,7 @@ public class RevitRootObjectBuilder( IReadOnlyList documentElementContexts, string projectId, IProgress onOperationProgressed, - CancellationToken ct = default + CancellationToken ct ) => threadContext.RunOnMainAsync( () => Task.FromResult(BuildSync(documentElementContexts, projectId, onOperationProgressed, ct)) diff --git a/Connectors/Revit/Speckle.Connectors.RevitShared/Speckle.Connectors.RevitShared.projitems b/Connectors/Revit/Speckle.Connectors.RevitShared/Speckle.Connectors.RevitShared.projitems index 0730a2580..f8431209c 100644 --- a/Connectors/Revit/Speckle.Connectors.RevitShared/Speckle.Connectors.RevitShared.projitems +++ b/Connectors/Revit/Speckle.Connectors.RevitShared/Speckle.Connectors.RevitShared.projitems @@ -60,6 +60,7 @@ + diff --git a/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Components/Operations/Receive/ReceiveAsyncComponent.cs b/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Components/Operations/Receive/ReceiveAsyncComponent.cs index 2aa2573f9..75d9a9e88 100644 --- a/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Components/Operations/Receive/ReceiveAsyncComponent.cs +++ b/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Components/Operations/Receive/ReceiveAsyncComponent.cs @@ -8,7 +8,6 @@ using GrasshopperAsyncComponent; using Rhino; using Speckle.Connectors.Common; using Speckle.Connectors.Common.Analytics; -using Speckle.Connectors.Common.Operations; using Speckle.Connectors.Common.Operations.Receive; using Speckle.Connectors.GrasshopperShared.HostApp; using Speckle.Connectors.GrasshopperShared.Operations.Receive; @@ -22,6 +21,7 @@ using Speckle.Sdk.Credentials; using Speckle.Sdk.Models; using Speckle.Sdk.Models.Collections; using Speckle.Sdk.Models.Extensions; +using Speckle.Sdk.Pipelines.Progress; namespace Speckle.Connectors.GrasshopperShared.Components.Operations.Receive; diff --git a/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Components/Operations/Receive/ReceiveComponent.cs b/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Components/Operations/Receive/ReceiveComponent.cs index 015913adc..1f27dfa66 100644 --- a/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Components/Operations/Receive/ReceiveComponent.cs +++ b/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Components/Operations/Receive/ReceiveComponent.cs @@ -3,7 +3,6 @@ using Microsoft.Extensions.DependencyInjection; using Rhino; using Speckle.Connectors.Common; using Speckle.Connectors.Common.Analytics; -using Speckle.Connectors.Common.Operations; using Speckle.Connectors.Common.Operations.Receive; using Speckle.Connectors.GrasshopperShared.Components.BaseComponents; using Speckle.Connectors.GrasshopperShared.HostApp; @@ -16,6 +15,7 @@ using Speckle.Sdk.Api; using Speckle.Sdk.Api.GraphQL.Models; using Speckle.Sdk.Credentials; using Speckle.Sdk.Models.Collections; +using Speckle.Sdk.Pipelines.Progress; namespace Speckle.Connectors.GrasshopperShared.Components.Operations.Receive; diff --git a/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Components/Operations/Send/IngestionTracker.cs b/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Components/Operations/Send/IngestionTracker.cs new file mode 100644 index 000000000..6238a5662 --- /dev/null +++ b/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Components/Operations/Send/IngestionTracker.cs @@ -0,0 +1,56 @@ +using Speckle.Sdk; +using Speckle.Sdk.Api; +using Speckle.Sdk.Api.GraphQL.Enums; +using Speckle.Sdk.Common; + +namespace Speckle.Connectors.GrasshopperShared.Components.Operations.Send; + +/// +/// Polls ingestion status via the SDK's GraphQL query API +/// and blocks until the ingestion reaches a terminal state (success/failed/cancelled). +/// +/// +/// We use polling instead of subscriptions because GH components call WaitForIngestionCompletion +/// after SendViaPackfile returns — by that point the server may have already completed +/// the ingestion. Setting up a new WebSocket subscription is too slow to catch fast completions. +/// Polling with Ingestion.Get() is reliable regardless of timing. +/// +public class IngestionTracker +{ + private static readonly TimeSpan s_pollInterval = TimeSpan.FromSeconds(1); + + public async Task WaitForIngestionCompletion( + IClient client, + string projectId, + string ingestionId, + Action? reportProgress, + string? reportProgressId, + CancellationToken cancellationToken + ) + { + // NOTE: before start hating from this - read the class description + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + var ingestion = await client.Ingestion.Get(ingestionId, projectId, cancellationToken).ConfigureAwait(false); + var status = ingestion.statusData.status; + + switch (status) + { + case ModelIngestionStatus.success: + return ingestion.statusData.versionId.NotNull(); + case ModelIngestionStatus.failed: + throw new SpeckleException($"Server processing failed: {ingestion.statusData.progressMessage}"); + case ModelIngestionStatus.cancelled: + throw new OperationCanceledException("Ingestion was cancelled by the server"); + case ModelIngestionStatus.processing: + case ModelIngestionStatus.queued: + reportProgress?.Invoke(reportProgressId ?? "Server", 0); + break; + } + + await Task.Delay(s_pollInterval, cancellationToken).ConfigureAwait(false); + } + } +} diff --git a/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Components/Operations/Send/SendAsyncComponent.cs b/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Components/Operations/Send/SendAsyncComponent.cs index 9f25e9676..bd45a382c 100644 --- a/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Components/Operations/Send/SendAsyncComponent.cs +++ b/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Components/Operations/Send/SendAsyncComponent.cs @@ -20,6 +20,7 @@ using Speckle.Sdk.Api; using Speckle.Sdk.Common; using Speckle.Sdk.Credentials; using Speckle.Sdk.Models.Extensions; +using Speckle.Sdk.Pipelines.Progress; namespace Speckle.Connectors.GrasshopperShared.Components.Operations.Send; @@ -503,10 +504,26 @@ public class SendComponentWorker : WorkerInstance }); using var scope = PriorityLoader.CreateScopeForActiveDocument(); var sendOperation = scope.ServiceProvider.GetRequiredService>(); - (SendOperationResult result, string versionId) = await sendOperation + (SendOperationResult result, string versionId, string? ingestionId) = await sendOperation .Send([rootCollectionWrapper], sendInfo, fileName, fileBytes, Parent.VersionMessage, progress, CancellationToken) .ConfigureAwait(false); + if (ingestionId != null) + { + Parent.Message = "Remote processing"; + var ingestionTracker = scope.ServiceProvider.GetRequiredService(); + versionId = await ingestionTracker + .WaitForIngestionCompletion( + Parent.ApiClient, + sendInfo.ProjectId, + ingestionId, + reportProgress, + Id, + CancellationToken + ) + .ConfigureAwait(false); + } + // TODO: If we have NodeRun events later, better to have `ComponentTracker` to use across components var customProperties = new Dictionary() { { "isAsync", true }, { "auto", Parent.AutoSend } }; if (sendInfo.WorkspaceId != null) diff --git a/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Components/Operations/Send/SendComponent.cs b/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Components/Operations/Send/SendComponent.cs index 985df6ef8..e4ebd5077 100644 --- a/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Components/Operations/Send/SendComponent.cs +++ b/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Components/Operations/Send/SendComponent.cs @@ -15,6 +15,7 @@ using Speckle.Sdk.Api; using Speckle.Sdk.Common; using Speckle.Sdk.Credentials; using Speckle.Sdk.Models.Collections; +using Speckle.Sdk.Pipelines.Progress; namespace Speckle.Connectors.GrasshopperShared.Components.Operations.Send; @@ -281,10 +282,26 @@ public class SendComponent : SpeckleTaskCapableComponent(); + versionId = await ingestionTracker + .WaitForIngestionCompletion( + client, + sendInfo.ProjectId, + ingestionId, + reportProgress: null, + reportProgressId: null, + cancellationToken + ) + .ConfigureAwait(false); + } + // TODO: If we have NodeRun events later, better to have `ComponentTracker` to use across components var customProperties = new Dictionary { { "isAsync", false } }; if (sendInfo.WorkspaceId != null) @@ -295,12 +312,13 @@ public class SendComponent : SpeckleTaskCapableComponent(); await mixpanel.TrackEvent(MixPanelEvents.Send, account, customProperties); - SpeckleUrlLatestModelVersionResource createdVersionResource = + SpeckleUrlModelVersionResource createdVersionResource = new( new(sendInfo.Account.id, null, sendInfo.Account.serverInfo.url), sendInfo.WorkspaceId, sendInfo.ProjectId, - sendInfo.ModelId + sendInfo.ModelId, + versionId ); Url = $"{sendInfo.Account.serverInfo.url}/projects/{sendInfo.ProjectId}/models/{sendInfo.ModelId}"; return new SendComponentOutput(createdVersionResource, versionId); diff --git a/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Operations/Receive/GrasshopperReceiveOperation.cs b/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Operations/Receive/GrasshopperReceiveOperation.cs index b8f9d0af8..6ce8d6e7a 100644 --- a/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Operations/Receive/GrasshopperReceiveOperation.cs +++ b/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Operations/Receive/GrasshopperReceiveOperation.cs @@ -3,6 +3,7 @@ using Speckle.Connectors.Common.Operations; using Speckle.Sdk.Api; using Speckle.Sdk.Logging; using Speckle.Sdk.Models; +using Speckle.Sdk.Pipelines.Progress; using Speckle.Sdk.Transports; namespace Speckle.Connectors.GrasshopperShared.Operations.Receive; diff --git a/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Operations/Send/GrasshopperContinuousTraversalBuilder.cs b/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Operations/Send/GrasshopperContinuousTraversalBuilder.cs new file mode 100644 index 000000000..75c6811a1 --- /dev/null +++ b/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Operations/Send/GrasshopperContinuousTraversalBuilder.cs @@ -0,0 +1,221 @@ +using Speckle.Connectors.Common.Builders; +using Speckle.Connectors.Common.Instances; +using Speckle.Connectors.Common.Operations; +using Speckle.Connectors.GrasshopperShared.HostApp; +using Speckle.Connectors.GrasshopperShared.Parameters; +using Speckle.Sdk.Models; +using Speckle.Sdk.Models.Collections; +using Speckle.Sdk.Pipelines.Progress; +using Speckle.Sdk.Pipelines.Send; +using DataObject = Speckle.Objects.Data.DataObject; + +namespace Speckle.Connectors.GrasshopperShared.Operations.Send; + +/// +/// Continuous traversal builder for Grasshopper that processes each object through the +/// as it unwraps. This enables the packfile send path (streaming objects to S3 during build). +/// +public class GrasshopperContinuousTraversalBuilder( + IInstanceObjectsManager> instanceObjectsManager +) : IRootContinuousTraversalBuilder +{ + public async Task Build( + IReadOnlyList objects, + string projectId, + SendPipeline sendPipeline, + IProgress onOperationProgressed, + CancellationToken cancellationToken + ) + { + // create root collection + var rootCollectionGoo = (SpeckleRootCollectionWrapperGoo)objects[0].Duplicate(); + rootCollectionGoo.Value.Name = "Grasshopper Model"; + RootCollection rootCollection = + new(rootCollectionGoo.Value.Name) + { + applicationId = rootCollectionGoo.Value.ApplicationId, + properties = rootCollectionGoo.Value.Properties ?? new() + }; + + // create packers for colors and render materials + GrasshopperColorPacker colorPacker = new(); + GrasshopperMaterialPacker materialPacker = new(); + GrasshopperBlockPacker blockPacker = new(instanceObjectsManager); + + // unwrap the input collection, processing each object through the send pipeline + await Unwrap( + rootCollectionGoo.Value, + rootCollection, + colorPacker, + materialPacker, + blockPacker, + sendPipeline, + cancellationToken + ) + .ConfigureAwait(false); + + // add proxies + rootCollection[ProxyKeys.COLOR] = colorPacker.ColorProxies.Values.ToList(); + rootCollection[ProxyKeys.RENDER_MATERIAL] = materialPacker.RenderMaterialProxies.Values.ToList(); + rootCollection[ProxyKeys.INSTANCE_DEFINITION] = blockPacker.InstanceDefinitionProxies.Values.ToList(); + + // process the root collection through the pipeline and wait for all uploads + await sendPipeline.Process(rootCollection).ConfigureAwait(false); + await sendPipeline.WaitForUpload().ConfigureAwait(false); + + // TODO: Not getting any conversion results yet + return new RootObjectBuilderResult(rootCollection, []); + } + + private async Task Unwrap( + SpeckleCollectionWrapper wrapper, + Collection targetCollection, + GrasshopperColorPacker colorPacker, + GrasshopperMaterialPacker materialPacker, + GrasshopperBlockPacker blockPacker, + SendPipeline sendPipeline, + CancellationToken cancellationToken + ) + { + colorPacker.ProcessColor(wrapper.ApplicationId, wrapper.Color); + materialPacker.ProcessMaterial(wrapper.ApplicationId, wrapper.Material); + + int skippedNulls = 0; + + foreach (ISpeckleCollectionObject? element in wrapper.Elements) + { + cancellationToken.ThrowIfCancellationRequested(); + + switch (element) + { + case null: + skippedNulls++; + continue; + + case SpeckleCollectionWrapper collWrapper: + collWrapper.ApplicationId ??= collWrapper.GetSpeckleApplicationId(); + targetCollection.elements.Add(collWrapper.Collection); + await Unwrap( + collWrapper, + collWrapper.Collection, + colorPacker, + materialPacker, + blockPacker, + sendPipeline, + cancellationToken + ) + .ConfigureAwait(false); + break; + + case SpeckleGeometryWrapper so: + Base objectBase = UnwrapGeometry(so); + string applicationId = objectBase.applicationId!; + + // NOTE: This is how it differentiate from 'GrasshopperSendOperation' + // It process through send pipeline before adding to collection + var reference = await sendPipeline.Process(objectBase).ConfigureAwait(false); + targetCollection.elements.Add(reference); + + if (so is SpeckleBlockInstanceWrapper blockInstance) + { + await ProcessBlockInstanceDefinition( + blockInstance, + colorPacker, + materialPacker, + blockPacker, + targetCollection, + sendPipeline, + cancellationToken + ) + .ConfigureAwait(false); + } + + colorPacker.ProcessColor(applicationId, so.Color); + materialPacker.ProcessMaterial(applicationId, so.Material); + break; + + case SpeckleDataObjectWrapper dataObjectWrapper: + DataObject dataObject = UnwrapDataObject(dataObjectWrapper, colorPacker, materialPacker); + + // process data object through send pipeline + var dataRef = await sendPipeline.Process(dataObject).ConfigureAwait(false); + targetCollection.elements.Add(dataRef); + break; + } + } + + // clear topology when nulls are present (CNX-2855) + if (skippedNulls > 0) + { + targetCollection[Constants.TOPOLOGY_PROP] = null; + } + } + + private Base UnwrapGeometry(SpeckleGeometryWrapper wrapper) + { + Dictionary props = []; + Base baseObject = wrapper.Base; + if (wrapper.Properties.CastTo(ref props)) + { + baseObject["properties"] = props; + } + + return baseObject; + } + + private async Task ProcessBlockInstanceDefinition( + SpeckleBlockInstanceWrapper blockInstance, + GrasshopperColorPacker colorPacker, + GrasshopperMaterialPacker materialPacker, + GrasshopperBlockPacker blockPacker, + Collection currentColl, + SendPipeline sendPipeline, + CancellationToken cancellationToken + ) + { + cancellationToken.ThrowIfCancellationRequested(); + + var definitionObjects = blockPacker.ProcessInstance(blockInstance); + + if (definitionObjects != null) + { + foreach (var definitionObject in definitionObjects) + { + Base defObjectBase = UnwrapGeometry(definitionObject); + string applicationId = defObjectBase.applicationId!; + + var reference = await sendPipeline.Process(defObjectBase).ConfigureAwait(false); + currentColl.elements.Add(reference); + + colorPacker.ProcessColor(applicationId, definitionObject.Color); + materialPacker.ProcessMaterial(applicationId, definitionObject.Material); + } + } + } + + private DataObject UnwrapDataObject( + SpeckleDataObjectWrapper wrapper, + GrasshopperColorPacker colorPacker, + GrasshopperMaterialPacker materialPacker + ) + { + DataObject dataObject = wrapper.DataObject; + + var displayValue = new List(); + foreach (var geometryWrapper in wrapper.Geometries) + { + Base geometryBase = UnwrapGeometry(geometryWrapper); + displayValue.Add(geometryBase); + + if (geometryWrapper.ApplicationId != null) + { + colorPacker.ProcessColor(geometryWrapper.ApplicationId, geometryWrapper.Color); + materialPacker.ProcessMaterial(geometryWrapper.ApplicationId, geometryWrapper.Material); + } + } + + dataObject.displayValue = displayValue; + + return dataObject; + } +} diff --git a/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Operations/Send/GrasshopperSendOperation.cs b/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Operations/Send/GrasshopperSendOperation.cs index 8450fc871..37692db3a 100644 --- a/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Operations/Send/GrasshopperSendOperation.cs +++ b/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Operations/Send/GrasshopperSendOperation.cs @@ -5,6 +5,7 @@ using Speckle.Connectors.GrasshopperShared.HostApp; using Speckle.Connectors.GrasshopperShared.Parameters; using Speckle.Sdk.Models; using Speckle.Sdk.Models.Collections; +using Speckle.Sdk.Pipelines.Progress; using DataObject = Speckle.Objects.Data.DataObject; namespace Speckle.Connectors.GrasshopperShared.Operations.Send; diff --git a/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Registration/PriorityLoader.cs b/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Registration/PriorityLoader.cs index bd1fec8b7..0d7c9532b 100644 --- a/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Registration/PriorityLoader.cs +++ b/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Registration/PriorityLoader.cs @@ -9,6 +9,7 @@ using Speckle.Connectors.Common.Operations; using Speckle.Connectors.Common.Operations.Receive; using Speckle.Connectors.Common.Threading; using Speckle.Connectors.GrasshopperShared.Components; +using Speckle.Connectors.GrasshopperShared.Components.Operations.Send; using Speckle.Connectors.GrasshopperShared.HostApp; using Speckle.Connectors.GrasshopperShared.Operations.Receive; using Speckle.Connectors.GrasshopperShared.Operations.Send; @@ -61,9 +62,14 @@ public class PriorityLoader : GH_AssemblyPriority services.AddTransient(); services.AddScoped(); services.AddTransient(); + services.AddTransient(); // send services.AddTransient, GrasshopperRootObjectBuilder>(); + services.AddTransient< + IRootContinuousTraversalBuilder, + GrasshopperContinuousTraversalBuilder + >(); services.AddTransient>(); services.AddSingleton(new DefaultThreadContext()); services.AddScoped< diff --git a/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Speckle.Connectors.GrasshopperShared.projitems b/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Speckle.Connectors.GrasshopperShared.projitems index 11c8a8743..169d58a67 100644 --- a/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Speckle.Connectors.GrasshopperShared.projitems +++ b/Connectors/Rhino/Speckle.Connectors.GrasshopperShared/Speckle.Connectors.GrasshopperShared.projitems @@ -58,7 +58,9 @@ + + diff --git a/Connectors/Rhino/Speckle.Connectors.RhinoImporter/packages.lock.json b/Connectors/Rhino/Speckle.Connectors.RhinoImporter/packages.lock.json index 2d412e93b..2db534724 100644 --- a/Connectors/Rhino/Speckle.Connectors.RhinoImporter/packages.lock.json +++ b/Connectors/Rhino/Speckle.Connectors.RhinoImporter/packages.lock.json @@ -29,20 +29,20 @@ }, "RhinoCommon": { "type": "Direct", - "requested": "[8.25.25328.11001, )", - "resolved": "8.25.25328.11001", - "contentHash": "PDKR9GwqyUXUkTulV4J0dzDIf/aWqSJkL7nkS8ReAx8xhnt/+RQpE8gTjOSCmkSU2tjG6WzclowbTxwMTU7VAA==", + "requested": "[8.28.26041.11001, )", + "resolved": "8.28.26041.11001", + "contentHash": "5mByZF+IdHvRLbYvr6Ek95Pwam6otewAyVHIGSC0sUE1+OscSpd9gbrFWnzo1arfCYmUJcUdsGhf7VayW09fQA==", "dependencies": { "System.Drawing.Common": "7.0.0" } }, "RhinoWindows": { "type": "Direct", - "requested": "[8.25.25328.11001, )", - "resolved": "8.25.25328.11001", - "contentHash": "I/+++piwtYTue+iAAQqcMF5QlontqwNnC7Leyhiv2FiF8JpAl6K44ZsJqB7ZEUC6ns0LDfa3mbFzQwUfHwYumQ==", + "requested": "[8.28.26041.11001, )", + "resolved": "8.28.26041.11001", + "contentHash": "7MBG231k5c0/V/USTzbXpaTCiZCECQOuB80DnpgEvvEA3r//IQQDTbrqawDYmN9BEw/FHiFn82bsf6tV+SS5Iw==", "dependencies": { - "RhinoCommon": "[8.25.25328.11001]" + "RhinoCommon": "[8.28.26041.11001]" } }, "Speckle.InterfaceGenerator": { diff --git a/Connectors/Rhino/Speckle.Connectors.RhinoShared/HostApp/RhinoInstanceBaker.cs b/Connectors/Rhino/Speckle.Connectors.RhinoShared/HostApp/RhinoInstanceBaker.cs index 0e6109ac8..77920b99b 100644 --- a/Connectors/Rhino/Speckle.Connectors.RhinoShared/HostApp/RhinoInstanceBaker.cs +++ b/Connectors/Rhino/Speckle.Connectors.RhinoShared/HostApp/RhinoInstanceBaker.cs @@ -5,7 +5,6 @@ using Rhino.Geometry; using Rhino.Render; using Speckle.Connectors.Common.Conversion; using Speckle.Connectors.Common.Instances; -using Speckle.Connectors.Common.Operations; using Speckle.Connectors.Rhino.Extensions; using Speckle.Converters.Common.ToHost; using Speckle.DoubleNumerics; @@ -15,6 +14,7 @@ using Speckle.Sdk.Common.Exceptions; using Speckle.Sdk.Models; using Speckle.Sdk.Models.Collections; using Speckle.Sdk.Models.Instances; +using Speckle.Sdk.Pipelines.Progress; namespace Speckle.Connectors.Rhino.HostApp; diff --git a/Connectors/Rhino/Speckle.Connectors.RhinoShared/Operations/Receive/RhinoHostObjectBuilder.cs b/Connectors/Rhino/Speckle.Connectors.RhinoShared/Operations/Receive/RhinoHostObjectBuilder.cs index c0cc8edf3..2874f7aed 100644 --- a/Connectors/Rhino/Speckle.Connectors.RhinoShared/Operations/Receive/RhinoHostObjectBuilder.cs +++ b/Connectors/Rhino/Speckle.Connectors.RhinoShared/Operations/Receive/RhinoHostObjectBuilder.cs @@ -19,6 +19,7 @@ using Speckle.Sdk.Logging; using Speckle.Sdk.Models; using Speckle.Sdk.Models.Collections; using Speckle.Sdk.Models.Instances; +using Speckle.Sdk.Pipelines.Progress; namespace Speckle.Connectors.Rhino.Operations.Receive; diff --git a/Connectors/Rhino/Speckle.Connectors.RhinoShared/Operations/Send/RhinoContinousTraversalBuilder.cs b/Connectors/Rhino/Speckle.Connectors.RhinoShared/Operations/Send/RhinoContinousTraversalBuilder.cs new file mode 100644 index 000000000..5aa87a06c --- /dev/null +++ b/Connectors/Rhino/Speckle.Connectors.RhinoShared/Operations/Send/RhinoContinousTraversalBuilder.cs @@ -0,0 +1,220 @@ +using Microsoft.Extensions.Logging; +using Rhino.DocObjects; +using Speckle.Connectors.Common.Builders; +using Speckle.Connectors.Common.Caching; +using Speckle.Connectors.Common.Conversion; +using Speckle.Connectors.Common.Extensions; +using Speckle.Connectors.Common.Instances; +using Speckle.Connectors.Common.Operations; +using Speckle.Connectors.DUI.Models.Card.SendFilter; +using Speckle.Connectors.Rhino.HostApp; +using Speckle.Connectors.Rhino.HostApp.Properties; +using Speckle.Converters.Common; +using Speckle.Converters.Rhino; +using Speckle.Sdk; +using Speckle.Sdk.Logging; +using Speckle.Sdk.Models; +using Speckle.Sdk.Models.Collections; +using Speckle.Sdk.Models.Instances; +using Speckle.Sdk.Pipelines.Progress; +using Speckle.Sdk.Pipelines.Send; +using Layer = Rhino.DocObjects.Layer; + +namespace Speckle.Connectors.Rhino.Operations.Send; + +/// +/// NOTE: I am not happy this is a mostly copy paste of the Root object builder, but i'm also not too worried. The main (hot) path +/// should be this one going forward, so we should not touch the og root object builder besides to delete it. +/// Stateless builder object to turn an into a object +/// +public class RhinoContinuousTraversalBuilder : IRootContinuousTraversalBuilder +{ + private readonly IRootToSpeckleConverter _rootToSpeckleConverter; + private readonly ISendConversionCache _sendConversionCache; + private readonly IConverterSettingsStore _converterSettings; + private readonly RhinoLayerUnpacker _layerUnpacker; + private readonly RhinoInstanceUnpacker _instanceUnpacker; + private readonly RhinoGroupUnpacker _groupUnpacker; + private readonly RhinoMaterialUnpacker _materialUnpacker; + private readonly RhinoColorUnpacker _colorUnpacker; + private readonly RhinoViewUnpacker _viewUnpacker; + private readonly PropertiesExtractor _propertiesExtractor; + private readonly ILogger _logger; + private readonly ISdkActivityFactory _activityFactory; + + public RhinoContinuousTraversalBuilder( + IRootToSpeckleConverter rootToSpeckleConverter, + ISendConversionCache sendConversionCache, + IConverterSettingsStore converterSettings, + RhinoLayerUnpacker layerUnpacker, + RhinoInstanceUnpacker instanceUnpacker, + RhinoGroupUnpacker groupUnpacker, + RhinoMaterialUnpacker materialUnpacker, + RhinoColorUnpacker colorUnpacker, + RhinoViewUnpacker viewUnpacker, + PropertiesExtractor propertiesExtractor, + ILogger logger, + ISdkActivityFactory activityFactory + ) + { + _sendConversionCache = sendConversionCache; + _converterSettings = converterSettings; + _layerUnpacker = layerUnpacker; + _instanceUnpacker = instanceUnpacker; + _groupUnpacker = groupUnpacker; + _rootToSpeckleConverter = rootToSpeckleConverter; + _materialUnpacker = materialUnpacker; + _colorUnpacker = colorUnpacker; + _viewUnpacker = viewUnpacker; + _propertiesExtractor = propertiesExtractor; + _logger = logger; + _activityFactory = activityFactory; + } + + public async Task Build( + IReadOnlyList rhinoObjects, + string projectId, + SendPipeline sendPipeline, + IProgress onOperationProgressed, + CancellationToken cancellationToken + ) + { + using var activity = _activityFactory.Start("Build"); + // 0 - Init the root + Collection rootObjectCollection = new() { name = _converterSettings.Current.Document.Name ?? "Unnamed document" }; + rootObjectCollection["units"] = _converterSettings.Current.SpeckleUnits; + + // 1 - Unpack the instances + UnpackResult unpackResults; + using (var _ = _activityFactory.Start("UnpackSelection")) + { + unpackResults = _instanceUnpacker.UnpackSelection(rhinoObjects); + } + + var (atomicObjects, _, instanceProxies, instanceDefinitionProxies) = unpackResults; + // POC: we should formalise this, sooner or later - or somehow fix it a bit more + rootObjectCollection[ProxyKeys.INSTANCE_DEFINITION] = instanceDefinitionProxies; // this won't work re traversal on receive + + // 2 - Unpack the groups + using (var _ = _activityFactory.Start("Unpack Groups")) + { + _groupUnpacker.UnpackGroups(rhinoObjects); + } + rootObjectCollection[ProxyKeys.GROUP] = _groupUnpacker.GroupProxies.Values; + + // 3 - Convert atomic objects + List results = new(atomicObjects.Count); + int count = 0; + using (var _ = _activityFactory.Start("Convert all")) + { + foreach (RhinoObject rhinoObject in atomicObjects) + { + cancellationToken.ThrowIfCancellationRequested(); + + // handle layer and store object layer *and all layer parents* to the version layers + // this is important because we need to unpack colors and materials on intermediate layers that do not have objects as well. + Layer layer = _converterSettings.Current.Document.Layers[rhinoObject.Attributes.LayerIndex]; + Collection collectionHost = _layerUnpacker.GetHostObjectCollection(layer, rootObjectCollection); + + var result = await ConvertRhinoObject(rhinoObject, collectionHost, instanceProxies, projectId, sendPipeline); + results.Add(result); + + count++; + onOperationProgressed.Report( + new($"Converting objects... ({count:N0} / {atomicObjects.Count:N0})", (double)count / atomicObjects.Count) + ); + } + } + + if (results.All(x => x.Status == Status.ERROR)) + { + throw new SpeckleException("Failed to convert all objects."); // fail fast instead creating empty commit! It will appear as model card error with red color. + } + + // 4 - Unpack all proxies for the root + // Get all layers from the created collections on the root object commit for proxy processing + List layers = _layerUnpacker.GetUsedLayers().ToList(); + + using (var _ = _activityFactory.Start("UnpackRenderMaterials")) + { + rootObjectCollection[ProxyKeys.RENDER_MATERIAL] = _materialUnpacker.UnpackRenderMaterials(atomicObjects, layers); + } + + using (var _ = _activityFactory.Start("UnpackColors")) + { + rootObjectCollection[ProxyKeys.COLOR] = _colorUnpacker.UnpackColors(atomicObjects, layers); + } + + // 5 - Unpack all other objects for the root + using (var _ = _activityFactory.Start("UnpackViews")) + { + List views = _viewUnpacker.UnpackViews(_converterSettings.Current.Document.NamedViews); + if (views.Count > 0) + { + rootObjectCollection[RootKeys.VIEW] = views; + } + } + + await sendPipeline.Process(rootObjectCollection); + await sendPipeline.WaitForUpload(); + return new RootObjectBuilderResult(rootObjectCollection, results); + } + + private async Task ConvertRhinoObject( + RhinoObject rhinoObject, + Collection collectionHost, + IReadOnlyDictionary instanceProxies, + string projectId, + SendPipeline sendPipeline + ) + { + string applicationId = rhinoObject.Id.ToString(); + string sourceType = rhinoObject.ObjectType.ToString(); + try + { + // get from cache or convert: + // What we actually do here is check if the object has been previously converted AND has not changed. + // If that's the case, we insert in the host collection just its object reference which has been saved from the prior conversion. + Base converted; + if (rhinoObject is InstanceObject) + { + converted = instanceProxies[applicationId]; + } + else if (_sendConversionCache.TryGetValue(projectId, applicationId, out ObjectReference? value)) + { + converted = value; + } + else + { + converted = _rootToSpeckleConverter.Convert(rhinoObject); + converted.applicationId = applicationId; + } + + // add name and properties + // POC: this is NOT done in the converter because we don't have a RootToSpeckle converter that captures all top level converters + if (!string.IsNullOrEmpty(rhinoObject.Attributes.Name)) + { + converted["name"] = rhinoObject.Attributes.Name; + } + + var properties = _propertiesExtractor.GetProperties(rhinoObject); + if (properties.Count > 0) + { + converted["properties"] = properties; + } + + // NOTE: this is the main part that differentiate from the main root object builder + var reference = await sendPipeline.Process(converted).ConfigureAwait(false); + + // add to host + collectionHost.elements.Add(reference); + + return new(Status.SUCCESS, applicationId, sourceType, reference); + } + catch (Exception ex) when (!ex.IsFatal()) + { + _logger.LogSendConversionError(ex, sourceType); + return new(Status.ERROR, applicationId, sourceType, null, ex); + } + } +} diff --git a/Connectors/Rhino/Speckle.Connectors.RhinoShared/Operations/Send/RhinoRootObjectBuilder.cs b/Connectors/Rhino/Speckle.Connectors.RhinoShared/Operations/Send/RhinoRootObjectBuilder.cs index 8394ec6c5..938b5c91c 100644 --- a/Connectors/Rhino/Speckle.Connectors.RhinoShared/Operations/Send/RhinoRootObjectBuilder.cs +++ b/Connectors/Rhino/Speckle.Connectors.RhinoShared/Operations/Send/RhinoRootObjectBuilder.cs @@ -16,6 +16,7 @@ using Speckle.Sdk.Logging; using Speckle.Sdk.Models; using Speckle.Sdk.Models.Collections; using Speckle.Sdk.Models.Instances; +using Speckle.Sdk.Pipelines.Progress; using Layer = Rhino.DocObjects.Layer; namespace Speckle.Connectors.Rhino.Operations.Send; diff --git a/Connectors/Rhino/Speckle.Connectors.RhinoShared/Registration/ServiceRegistration.cs b/Connectors/Rhino/Speckle.Connectors.RhinoShared/Registration/ServiceRegistration.cs index 6369b4b29..e202defd5 100644 --- a/Connectors/Rhino/Speckle.Connectors.RhinoShared/Registration/ServiceRegistration.cs +++ b/Connectors/Rhino/Speckle.Connectors.RhinoShared/Registration/ServiceRegistration.cs @@ -71,6 +71,7 @@ public static class ServiceRegistration serviceCollection.AddSingleton(DefaultTraversal.CreateTraversalFunc()); serviceCollection.AddScoped, RhinoRootObjectBuilder>(); + serviceCollection.AddScoped, RhinoContinuousTraversalBuilder>(); serviceCollection.AddScoped< IInstanceObjectsManager>, InstanceObjectsManager> diff --git a/Connectors/Rhino/Speckle.Connectors.RhinoShared/Speckle.Connectors.RhinoShared.projitems b/Connectors/Rhino/Speckle.Connectors.RhinoShared/Speckle.Connectors.RhinoShared.projitems index 8da708e54..ed833bdb9 100644 --- a/Connectors/Rhino/Speckle.Connectors.RhinoShared/Speckle.Connectors.RhinoShared.projitems +++ b/Connectors/Rhino/Speckle.Connectors.RhinoShared/Speckle.Connectors.RhinoShared.projitems @@ -38,6 +38,7 @@ + diff --git a/Connectors/Tekla/Speckle.Connector.TeklaShared/Operations/Send/TeklaContinuousTraversalBuilder.cs b/Connectors/Tekla/Speckle.Connector.TeklaShared/Operations/Send/TeklaContinuousTraversalBuilder.cs new file mode 100644 index 000000000..180e5d99e --- /dev/null +++ b/Connectors/Tekla/Speckle.Connector.TeklaShared/Operations/Send/TeklaContinuousTraversalBuilder.cs @@ -0,0 +1,141 @@ +using Microsoft.Extensions.Logging; +using Speckle.Connectors.Common.Builders; +using Speckle.Connectors.Common.Caching; +using Speckle.Connectors.Common.Conversion; +using Speckle.Connectors.Common.Operations; +using Speckle.Connectors.TeklaShared.Extensions; +using Speckle.Connectors.TeklaShared.HostApp; +using Speckle.Converters.Common; +using Speckle.Converters.TeklaShared; +using Speckle.Sdk; +using Speckle.Sdk.Logging; +using Speckle.Sdk.Models; +using Speckle.Sdk.Models.Collections; +using Speckle.Sdk.Pipelines.Progress; +using Speckle.Sdk.Pipelines.Send; + +namespace Speckle.Connectors.TeklaShared.Operations.Send; + +/// +/// Continuous traversal builder for Tekla that streams objects through a +/// for packfile-based uploads. Same conversion logic as . +/// +public class TeklaContinuousTraversalBuilder : IRootContinuousTraversalBuilder +{ + private readonly IRootToSpeckleConverter _rootToSpeckleConverter; + private readonly ISendConversionCache _sendConversionCache; + private readonly IConverterSettingsStore _converterSettings; + private readonly SendCollectionManager _sendCollectionManager; + private readonly ILogger _logger; + private readonly ISdkActivityFactory _activityFactory; + private readonly TeklaMaterialUnpacker _materialUnpacker; + + public TeklaContinuousTraversalBuilder( + IRootToSpeckleConverter rootToSpeckleConverter, + ISendConversionCache sendConversionCache, + IConverterSettingsStore converterSettings, + SendCollectionManager sendCollectionManager, + ILogger logger, + ISdkActivityFactory activityFactory, + TeklaMaterialUnpacker materialUnpacker + ) + { + _sendConversionCache = sendConversionCache; + _converterSettings = converterSettings; + _sendCollectionManager = sendCollectionManager; + _rootToSpeckleConverter = rootToSpeckleConverter; + _logger = logger; + _activityFactory = activityFactory; + _materialUnpacker = materialUnpacker; + } + + public async Task Build( + IReadOnlyList teklaObjects, + string projectId, + SendPipeline sendPipeline, + IProgress onOperationProgressed, + CancellationToken cancellationToken + ) + { + using var activity = _activityFactory.Start("Build"); + + var model = new TSM.Model(); + string modelName = model.GetInfo().ModelName ?? "Unnamed model"; + + Collection rootObjectCollection = new() { name = modelName }; + rootObjectCollection["units"] = _converterSettings.Current.SpeckleUnits; + + List results = new(teklaObjects.Count); + int count = 0; + + using (var _ = _activityFactory.Start("Convert all")) + { + foreach (TSM.ModelObject teklaObject in teklaObjects) + { + cancellationToken.ThrowIfCancellationRequested(); + var result = await ConvertTeklaObject(teklaObject, rootObjectCollection, projectId, sendPipeline); + results.Add(result); + + ++count; + onOperationProgressed.Report( + new($"Converting objects... ({count:N0} / {teklaObjects.Count:N0})", (double)count / teklaObjects.Count) + ); + await Task.Yield(); + } + } + + if (results.All(x => x.Status == Status.ERROR)) + { + throw new SpeckleException("Failed to convert all objects."); + } + + var renderMaterialProxies = _materialUnpacker.UnpackRenderMaterial(teklaObjects.ToList()); + if (renderMaterialProxies.Count > 0) + { + rootObjectCollection[ProxyKeys.RENDER_MATERIAL] = renderMaterialProxies; + } + + // Process root collection and wait for all uploads + await sendPipeline.Process(rootObjectCollection); + await sendPipeline.WaitForUpload(); + + return new RootObjectBuilderResult(rootObjectCollection, results); + } + + private async Task ConvertTeklaObject( + TSM.ModelObject teklaObject, + Collection collectionHost, + string projectId, + SendPipeline sendPipeline + ) + { + string applicationId = teklaObject.GetSpeckleApplicationId(); + string sourceType = teklaObject.GetType().Name; + + try + { + Base converted; + if (_sendConversionCache.TryGetValue(projectId, applicationId, out ObjectReference? value)) + { + converted = value; + } + else + { + converted = _rootToSpeckleConverter.Convert(teklaObject); + } + + var collection = _sendCollectionManager.GetAndCreateObjectHostCollection(teklaObject, collectionHost); + + // NOTE: this is the main part that differentiate from the main root object builder + var reference = await sendPipeline.Process(converted).ConfigureAwait(false); + collection.elements.Add(reference); + + return new(Status.SUCCESS, applicationId, sourceType, reference); + } + catch (Exception ex) when (!ex.IsFatal()) + { + _logger.LogError(ex, "Failed to convert object {SourceType}", sourceType); + return new(Status.ERROR, applicationId, sourceType, null, ex); + } + } +} diff --git a/Connectors/Tekla/Speckle.Connector.TeklaShared/Operations/Send/TeklaRootObjectBuilder.cs b/Connectors/Tekla/Speckle.Connector.TeklaShared/Operations/Send/TeklaRootObjectBuilder.cs index e41216c3b..d1aeb2550 100644 --- a/Connectors/Tekla/Speckle.Connector.TeklaShared/Operations/Send/TeklaRootObjectBuilder.cs +++ b/Connectors/Tekla/Speckle.Connector.TeklaShared/Operations/Send/TeklaRootObjectBuilder.cs @@ -11,6 +11,7 @@ using Speckle.Sdk; using Speckle.Sdk.Logging; using Speckle.Sdk.Models; using Speckle.Sdk.Models.Collections; +using Speckle.Sdk.Pipelines.Progress; namespace Speckle.Connectors.TeklaShared.Operations.Send; diff --git a/Connectors/Tekla/Speckle.Connector.TeklaShared/ServiceRegistration.cs b/Connectors/Tekla/Speckle.Connector.TeklaShared/ServiceRegistration.cs index a4a4a389f..2398982f5 100644 --- a/Connectors/Tekla/Speckle.Connector.TeklaShared/ServiceRegistration.cs +++ b/Connectors/Tekla/Speckle.Connector.TeklaShared/ServiceRegistration.cs @@ -54,6 +54,7 @@ public static class ServiceRegistration services.AddSingleton(DefaultTraversal.CreateTraversalFunc()); services.AddScoped(); services.AddScoped, TeklaRootObjectBuilder>(); + services.AddScoped, TeklaContinuousTraversalBuilder>(); services.AddScoped>(); services.AddSingleton(); diff --git a/Connectors/Tekla/Speckle.Connector.TeklaShared/Speckle.Connectors.TeklaShared.projitems b/Connectors/Tekla/Speckle.Connector.TeklaShared/Speckle.Connectors.TeklaShared.projitems index 86a8ddc2d..bdf99d4f4 100644 --- a/Connectors/Tekla/Speckle.Connector.TeklaShared/Speckle.Connectors.TeklaShared.projitems +++ b/Connectors/Tekla/Speckle.Connector.TeklaShared/Speckle.Connectors.TeklaShared.projitems @@ -37,6 +37,7 @@ + diff --git a/Converters/Rhino/Speckle.Converters.Rhino8/packages.lock.json b/Converters/Rhino/Speckle.Converters.Rhino8/packages.lock.json index a0fb0d0c5..bedaabff4 100644 --- a/Converters/Rhino/Speckle.Converters.Rhino8/packages.lock.json +++ b/Converters/Rhino/Speckle.Converters.Rhino8/packages.lock.json @@ -346,9 +346,9 @@ }, "RhinoCommon": { "type": "Direct", - "requested": "[8.25.25328.11001, )", - "resolved": "8.25.25328.11001", - "contentHash": "PDKR9GwqyUXUkTulV4J0dzDIf/aWqSJkL7nkS8ReAx8xhnt/+RQpE8gTjOSCmkSU2tjG6WzclowbTxwMTU7VAA==", + "requested": "[8.28.26041.11001, )", + "resolved": "8.28.26041.11001", + "contentHash": "5mByZF+IdHvRLbYvr6Ek95Pwam6otewAyVHIGSC0sUE1+OscSpd9gbrFWnzo1arfCYmUJcUdsGhf7VayW09fQA==", "dependencies": { "System.Drawing.Common": "7.0.0" } diff --git a/DUI3/Speckle.Connectors.DUI.Tests/ReceiveOperationManagerTests.cs b/DUI3/Speckle.Connectors.DUI.Tests/ReceiveOperationManagerTests.cs index 3112474f2..311fbe12f 100644 --- a/DUI3/Speckle.Connectors.DUI.Tests/ReceiveOperationManagerTests.cs +++ b/DUI3/Speckle.Connectors.DUI.Tests/ReceiveOperationManagerTests.cs @@ -12,6 +12,7 @@ using Speckle.Connectors.DUI.Models; using Speckle.Connectors.DUI.Models.Card; using Speckle.Sdk; using Speckle.Sdk.Credentials; +using Speckle.Sdk.Pipelines.Progress; using Speckle.Testing; namespace Speckle.Connectors.DUI.Tests; diff --git a/DUI3/Speckle.Connectors.DUI/Bindings/ISendBindingUICommands.cs b/DUI3/Speckle.Connectors.DUI/Bindings/ISendBindingUICommands.cs index 0fb6ddc10..f732aa369 100644 --- a/DUI3/Speckle.Connectors.DUI/Bindings/ISendBindingUICommands.cs +++ b/DUI3/Speckle.Connectors.DUI/Bindings/ISendBindingUICommands.cs @@ -13,7 +13,8 @@ public interface ISendBindingUICommands Task SetModelSendResult( string modelCardId, string versionId, - IEnumerable sendConversionResults + IEnumerable sendConversionResults, + string? ingestionId = null ); IBrowserBridge Bridge { get; } diff --git a/DUI3/Speckle.Connectors.DUI/Bindings/OperationProgressManager.cs b/DUI3/Speckle.Connectors.DUI/Bindings/OperationProgressManager.cs index c8dd08700..ccc047dac 100644 --- a/DUI3/Speckle.Connectors.DUI/Bindings/OperationProgressManager.cs +++ b/DUI3/Speckle.Connectors.DUI/Bindings/OperationProgressManager.cs @@ -1,8 +1,8 @@ using System.Collections.Concurrent; -using Speckle.Connectors.Common.Operations; using Speckle.Connectors.DUI.Bridge; using Speckle.Connectors.DUI.Models.Card; using Speckle.InterfaceGenerator; +using Speckle.Sdk.Pipelines.Progress; namespace Speckle.Connectors.DUI.Bindings; @@ -21,7 +21,7 @@ public class OperationProgressManager : IOperationProgressManager private const string SET_MODEL_PROGRESS_UI_COMMAND_NAME = "setModelProgress"; private static readonly ConcurrentDictionary s_lastProgressValues = new(); - private const int THROTTLE_INTERVAL_MS = 200; + private const int THROTTLE_INTERVAL_MS = 400; public IProgress CreateOperationProgressEventHandler( IBrowserBridge bridge, @@ -65,11 +65,10 @@ public class OperationProgressManager : IOperationProgressManager var currentTime = DateTime.Now; var elapsedMs = (currentTime - t.Item1).Milliseconds; - if (elapsedMs < THROTTLE_INTERVAL_MS && t.Item2 == progress.Status) + if (elapsedMs < THROTTLE_INTERVAL_MS) { return; } - Console.WriteLine($"Progress: {progress.Status} - {progress.Progress}"); s_lastProgressValues[modelCardId] = (currentTime, progress.Status); SendProgress(bridge, modelCardId, progress); } diff --git a/DUI3/Speckle.Connectors.DUI/Bindings/SendBindingUICommands.cs b/DUI3/Speckle.Connectors.DUI/Bindings/SendBindingUICommands.cs index eeafee7b3..8c0bc944c 100644 --- a/DUI3/Speckle.Connectors.DUI/Bindings/SendBindingUICommands.cs +++ b/DUI3/Speckle.Connectors.DUI/Bindings/SendBindingUICommands.cs @@ -37,7 +37,8 @@ public class SendBindingUICommands(IBrowserBridge bridge) public async Task SetModelSendResult( string modelCardId, string versionId, - IEnumerable sendConversionResults + IEnumerable sendConversionResults, + string? ingestionId = null ) => await Bridge.Send( SET_MODEL_SEND_RESULT_UI_COMMAND_NAME, @@ -45,7 +46,8 @@ public class SendBindingUICommands(IBrowserBridge bridge) { modelCardId, versionId, - sendConversionResults + sendConversionResults, + ingestionId } ); } diff --git a/DUI3/Speckle.Connectors.DUI/Bindings/SendOperationManager.cs b/DUI3/Speckle.Connectors.DUI/Bindings/SendOperationManager.cs index 069b661a3..c7216dd7d 100644 --- a/DUI3/Speckle.Connectors.DUI/Bindings/SendOperationManager.cs +++ b/DUI3/Speckle.Connectors.DUI/Bindings/SendOperationManager.cs @@ -13,6 +13,7 @@ using Speckle.Sdk.Api; using Speckle.Sdk.Common; using Speckle.Sdk.Credentials; using Speckle.Sdk.Logging; +using Speckle.Sdk.Pipelines.Progress; namespace Speckle.Connectors.DUI.Bindings; @@ -98,7 +99,7 @@ public sealed class SendOperationManager( cancellationItem.Token ); - var objects = await gatherObjects(modelCard, progress); + var objects = await gatherObjects.Invoke(modelCard, progress); if (objects.Count == 0) { @@ -108,7 +109,7 @@ public sealed class SendOperationManager( var sendOperation = serviceScope.ServiceProvider.GetRequiredService>(); - var (result, versionId) = await sendOperation.Send( + var (result, versionId, ingestionId) = await sendOperation.Send( objects, sendInfo, fileName, @@ -118,7 +119,7 @@ public sealed class SendOperationManager( cancellationItem.Token ); - await commands.SetModelSendResult(modelCardId, versionId, result.ConversionResults); + await commands.SetModelSendResult(modelCardId, versionId, result.ConversionResults, ingestionId); } catch (OperationCanceledException) { diff --git a/DUI3/Speckle.Connectors.DUI/Models/Card/ModelCardProgress.cs b/DUI3/Speckle.Connectors.DUI/Models/Card/ModelCardProgress.cs index aff002bd2..4827bdfa0 100644 --- a/DUI3/Speckle.Connectors.DUI/Models/Card/ModelCardProgress.cs +++ b/DUI3/Speckle.Connectors.DUI/Models/Card/ModelCardProgress.cs @@ -4,7 +4,7 @@ /// Progress value between 0 and 1 to calculate UI progress bar width. /// If it is null it will swooshing on UI. /// -public record ModelCardProgress(string ModelCardId, string Status, double? Progress) +public readonly record struct ModelCardProgress(string ModelCardId, string Status, double? Progress) { public override string ToString() => $"{ModelCardId} - {Status} - {Progress}"; } diff --git a/Directory.Packages.props b/Directory.Packages.props index 45fb141af..b31fd4e18 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -18,7 +18,7 @@ - + @@ -28,10 +28,10 @@ - + - - + + @@ -43,8 +43,8 @@ - - + + diff --git a/Importers/Rhino/Speckle.Importers.JobProcessor/Blobs/ImportJobFileFactory.cs b/Importers/Rhino/Speckle.Importers.JobProcessor/Blobs/ImportJobFileFactory.cs index ea6f9d077..cd583a9b0 100644 --- a/Importers/Rhino/Speckle.Importers.JobProcessor/Blobs/ImportJobFileFactory.cs +++ b/Importers/Rhino/Speckle.Importers.JobProcessor/Blobs/ImportJobFileFactory.cs @@ -1,22 +1,35 @@ using Microsoft.Extensions.Logging; using Speckle.Importers.JobProcessor.Domain; using Speckle.Sdk.Api; +using Speckle.Sdk.Logging; namespace Speckle.Importers.JobProcessor.Blobs; -internal sealed class ImportJobFileDownloader(ILogger logger) +internal sealed class ImportJobFileDownloader(ILogger logger, ISdkActivityFactory activityFactory) { public async Task DownloadFile(FileimportJob job, IClient client, CancellationToken cancellationToken) { - var directory = Directory.CreateTempSubdirectory("speckle-file-import"); - string targetFilePath = $"{directory.FullName}/{job.Payload.BlobId}.{job.Payload.FileType}"; - await client.FileImport.DownloadFile( - job.Payload.ProjectId, - job.Payload.BlobId, - targetFilePath, - null, - cancellationToken - ); - return new ImportJobFile(logger, new FileInfo(targetFilePath)); + using var activity = activityFactory.Start(); + try + { + var directory = Directory.CreateTempSubdirectory("speckle-file-import"); + string targetFilePath = $"{directory.FullName}/{job.Payload.BlobId}.{job.Payload.FileType}"; + await client.FileImport.DownloadFile( + job.Payload.ProjectId, + job.Payload.BlobId, + targetFilePath, + null, + cancellationToken + ); + var ret = new ImportJobFile(logger, new FileInfo(targetFilePath)); + activity?.SetStatus(SdkActivityStatusCode.Ok); + return ret; + } + catch (Exception ex) + { + activity?.RecordException(ex); + activity?.SetStatus(SdkActivityStatusCode.Error); + throw; + } } } diff --git a/Importers/Rhino/Speckle.Importers.JobProcessor/Domain/FileimportPayload.cs b/Importers/Rhino/Speckle.Importers.JobProcessor/Domain/FileimportPayload.cs index b347b1d4a..58b41eaf5 100644 --- a/Importers/Rhino/Speckle.Importers.JobProcessor/Domain/FileimportPayload.cs +++ b/Importers/Rhino/Speckle.Importers.JobProcessor/Domain/FileimportPayload.cs @@ -1,4 +1,6 @@ -namespace Speckle.Importers.JobProcessor.Domain; +using System.Text.Json.Serialization; + +namespace Speckle.Importers.JobProcessor.Domain; /// /// Payload for the fileimport job @@ -17,4 +19,13 @@ internal sealed class FileimportPayload public required Uri ServerUrl { get; init; } public required int PayloadVersion { get; init; } public required int TimeOutSeconds { get; init; } + + [JsonPropertyName("_traceContext")] + public TraceContext? TraceContext { get; init; } +} + +public sealed class TraceContext +{ + [JsonPropertyName("traceparent")] + public string? TraceParent { get; init; } } diff --git a/Importers/Rhino/Speckle.Importers.JobProcessor/JobHandlers/IPCModels.cs b/Importers/Rhino/Speckle.Importers.JobProcessor/JobHandlers/IPCModels.cs index 0aa777426..2f93f65f6 100644 --- a/Importers/Rhino/Speckle.Importers.JobProcessor/JobHandlers/IPCModels.cs +++ b/Importers/Rhino/Speckle.Importers.JobProcessor/JobHandlers/IPCModels.cs @@ -12,6 +12,7 @@ internal readonly struct ImporterArgs public required string BlobId { get; init; } public required int Attempt { get; init; } public required string ResultsPath { get; init; } + public required string? TraceContext { get; init; } public required Project Project { get; init; } public required ModelIngestion Ingestion { get; init; } public required Account Account { get; init; } diff --git a/Importers/Rhino/Speckle.Importers.JobProcessor/JobHandlers/RhinoJobHandler.cs b/Importers/Rhino/Speckle.Importers.JobProcessor/JobHandlers/RhinoJobHandler.cs index 995e1634b..30eaf02da 100644 --- a/Importers/Rhino/Speckle.Importers.JobProcessor/JobHandlers/RhinoJobHandler.cs +++ b/Importers/Rhino/Speckle.Importers.JobProcessor/JobHandlers/RhinoJobHandler.cs @@ -10,6 +10,7 @@ using Speckle.Sdk.Api; using Speckle.Sdk.Api.GraphQL.Inputs; using Speckle.Sdk.Api.GraphQL.Models; using Speckle.Sdk.Common; +using Speckle.Sdk.Logging; using JsonSerializer = System.Text.Json.JsonSerializer; namespace Speckle.Importers.JobProcessor.JobHandlers; @@ -17,7 +18,8 @@ namespace Speckle.Importers.JobProcessor.JobHandlers; internal sealed class RhinoJobHandler( ILogger logger, ImportJobFileDownloader fileDownloader, - ISpeckleApplication application + ISpeckleApplication application, + ISdkActivityFactory activityFactory ) : IJobHandler { private readonly JsonSerializerSettings _settings = @@ -50,21 +52,27 @@ internal sealed class RhinoJobHandler( ), cancellationToken ); + string resultsPath = $"{file.FileInfo.DirectoryName}/results.json"; - var importerArgs = new ImporterArgs + using (var activity = activityFactory.Start("Await sub-process")) { - FilePath = file.FileInfo.FullName, - ResultsPath = $"{file.FileInfo.DirectoryName}/results.json", - Account = client.Account, - Project = project, - Ingestion = ingestion, - JobId = job.Id, - BlobId = job.Payload.BlobId, - Attempt = job.Attempt, - HostApplication = handlerApplication, - }; - await RunSubProcess(importerArgs, cancellationToken); - var response = await DeserializeResponse(importerArgs.ResultsPath, cancellationToken); + var importerArgs = new ImporterArgs + { + FilePath = file.FileInfo.FullName, + ResultsPath = resultsPath, + TraceContext = $"00-{activity?.TraceId}-{activity?.SpanId}-01", + Account = client.Account, + Project = project, + Ingestion = ingestion, + JobId = job.Id, + BlobId = job.Payload.BlobId, + Attempt = job.Attempt, + HostApplication = handlerApplication, + }; + await RunSubProcess(importerArgs, cancellationToken); + } + + var response = await DeserializeResponse(resultsPath, cancellationToken); if (response.RootObjectId is null) { @@ -94,7 +102,12 @@ internal sealed class RhinoJobHandler( var processStart = new ProcessStartInfo() { FileName = $"{path}/Speckle.Importers.Rhino.exe", - Environment = { }, + Environment = + { + ["DOTNET_ENVIRONMENT"] = Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT"), + ["SEQ_API_KEY"] = Environment.GetEnvironmentVariable("SEQ_API_KEY"), + ["SPECKLE_COLLECTOR_API_TOKEN"] = Environment.GetEnvironmentVariable("SPECKLE_COLLECTOR_API_TOKEN"), + }, RedirectStandardError = true, RedirectStandardOutput = true, UseShellExecute = false, diff --git a/Importers/Rhino/Speckle.Importers.JobProcessor/JobProcessor.cs b/Importers/Rhino/Speckle.Importers.JobProcessor/JobProcessor.cs index d849f2346..453ec4d4a 100644 --- a/Importers/Rhino/Speckle.Importers.JobProcessor/JobProcessor.cs +++ b/Importers/Rhino/Speckle.Importers.JobProcessor/JobProcessor.cs @@ -71,7 +71,10 @@ internal sealed class JobProcessorInstance( job.RemainingComputeBudgetSeconds ); - using var activity = activityFactory.Start(); + using var activity = job.Payload.TraceContext?.TraceParent is not null + ? activityFactory.StartRemote(job.Payload.TraceContext.TraceParent, SdkActivityKind.Consumer, "Picked up a job") + : activityFactory.Start("Picked up a job", SdkActivityKind.Consumer); + using var scopeJobId = ActivityScope.SetTag("jobId", job.Id); using var scopeJobType = ActivityScope.SetTag("jobType", job.Payload.JobType); using var scopeAttempt = ActivityScope.SetTag("job.attempt", job.Attempt.ToString()); @@ -96,27 +99,6 @@ internal sealed class JobProcessorInstance( } } - private async Task ReportSuccess( - FileimportJob job, - string rootObjectId, - IClient client, - double elapsedSeconds, - CancellationToken cancellationToken - ) - { - string versionId = await client.Ingestion.Complete( - new(job.Payload.ModelIngestionId, job.Payload.ProjectId, rootObjectId, null), - cancellationToken - ); - logger.LogInformation( - "Attempt {Attempt} of {JobId} has succeeded creating {VersionId} after {ElapsedSeconds}", - job.Attempt, - job.Id, - versionId, - elapsedSeconds - ); - } - private async Task ReportCancelled(FileimportJob job, IClient client, Exception ex, double elapsedSeconds) { await client.Ingestion.FailWithCancel( @@ -186,11 +168,9 @@ internal sealed class JobProcessorInstance( throw new MaxAttemptsExceededException("Unhandled error silently failed the job multiple times"); } - string rootObjectId = await ExecuteJobWithTimeout(job, speckleClient, serviceCancellationToken); + await ExecuteJobWithTimeout(job, speckleClient, serviceCancellationToken); totalElapsedSeconds = stopwatch.Elapsed.TotalSeconds; - await ReportSuccess(job, rootObjectId, speckleClient, totalElapsedSeconds, serviceCancellationToken); - activity?.SetStatus(SdkActivityStatusCode.Ok); } catch (Exception ex) diff --git a/Importers/Rhino/Speckle.Importers.JobProcessor/Program.cs b/Importers/Rhino/Speckle.Importers.JobProcessor/Program.cs index 49cb6ffc5..a5ca899f7 100644 --- a/Importers/Rhino/Speckle.Importers.JobProcessor/Program.cs +++ b/Importers/Rhino/Speckle.Importers.JobProcessor/Program.cs @@ -15,17 +15,17 @@ public static class Program // Dapper doesn't understand how to handle JSON deserialization, so we need to tell it what types can be deserialzied SqlMapper.AddTypeHandler(new JsonHandler()); - var host = ConfigureAppHost(args); + using var loggerDisposable = ConfigureAppHost(args, out IHost host); await host.RunAsync(); } - private static IHost ConfigureAppHost(string[] args) + private static IDisposable ConfigureAppHost(string[] args, out IHost host) { // DI setup var builder = Host.CreateApplicationBuilder(args); - builder.Services.AddJobProcessor(); + var loggingDisposable = builder.Services.AddJobProcessor(); builder.Services.AddWindowsService(); builder.Services.AddTransient(); @@ -35,6 +35,7 @@ public static class Program settings.Filter = (_, level) => level >= LogLevel.Information; }); - return builder.Build(); + host = builder.Build(); + return loggingDisposable; } } diff --git a/Importers/Rhino/Speckle.Importers.JobProcessor/ServiceRegistration.cs b/Importers/Rhino/Speckle.Importers.JobProcessor/ServiceRegistration.cs index c4e4cadd0..2dd154e4b 100644 --- a/Importers/Rhino/Speckle.Importers.JobProcessor/ServiceRegistration.cs +++ b/Importers/Rhino/Speckle.Importers.JobProcessor/ServiceRegistration.cs @@ -7,6 +7,9 @@ using Speckle.Importers.JobProcessor.Blobs; using Speckle.Importers.JobProcessor.JobQueue; using Speckle.Objects.Geometry; using Speckle.Sdk; +#if !DEBUG && !LOCAL +using Speckle.Sdk.Common; +#endif namespace Speckle.Importers.JobProcessor; @@ -15,7 +18,7 @@ internal static class ServiceRegistration private static readonly Application s_application = new(".NET File Import Job Processor", "jobprocessor"); private const HostAppVersion HOST_APP_VERSION = HostAppVersion.v3; - public static IServiceCollection AddJobProcessor(this IServiceCollection serviceCollection) + public static IDisposable AddJobProcessor(this IServiceCollection serviceCollection) { var assemblyVersion = Assembly.GetExecutingAssembly().GetVersion(); @@ -26,17 +29,18 @@ internal static class ServiceRegistration typeof(Point).Assembly ); - serviceCollection.AddLoggingConfig(); + var loggingDisposable = serviceCollection.AddLoggingConfig(); serviceCollection.AddTransient(); serviceCollection.AddTransient(); serviceCollection.AddHostedService(); - return serviceCollection; + return loggingDisposable; } - private static void AddLoggingConfig(this IServiceCollection serviceCollection) + private static IDisposable AddLoggingConfig(this IServiceCollection serviceCollection) { - serviceCollection.AddSeqLogging( + return serviceCollection.AddOpenTelemetry( + "Speckle.Importers.JobProcessor", s_application, HOST_APP_VERSION, #if DEBUG || LOCAL @@ -51,7 +55,23 @@ internal static class ServiceRegistration [ new( Endpoint: new Uri("https://seq.speckle.systems/ingest/otlp/v1/logs"), - Headers: new() { { "X-Seq-ApiKey", "zG4cU1MbOhMD699iGlAq" } } + Headers: new() + { + // We're using a different token than connectors for seq because we want to beable to + // trust the client's timestamps (rather than use the server's timestamps) for better tracing + // This setting has more opportunity for abuse, so we're keeping it secret, unlike the connectors token. + { "X-Seq-ApiKey", Environment.GetEnvironmentVariable("SEQ_API_KEY").NotNullOrWhiteSpace() } + } + ), + new( + Endpoint: new Uri("https://collector.speckle.dev/v1/logs"), + Headers: new() + { + { + "authorization", + Environment.GetEnvironmentVariable("SPECKLE_COLLECTOR_API_TOKEN").NotNullOrWhiteSpace() + } + } ) ], MinimumLevel: SpeckleLogLevel.Information @@ -62,7 +82,20 @@ internal static class ServiceRegistration [ new( Endpoint: new Uri("https://seq.speckle.systems/ingest/otlp/v1/traces"), - Headers: new() { { "X-Seq-ApiKey", "zG4cU1MbOhMD699iGlAq" } } + Headers: new() + { + { "X-Seq-ApiKey", Environment.GetEnvironmentVariable("SEQ_API_KEY").NotNullOrWhiteSpace() } + } + ), + new( + Endpoint: new Uri("https://collector.speckle.dev/v1/traces"), + Headers: new() + { + { + "authorization", + Environment.GetEnvironmentVariable("SPECKLE_COLLECTOR_API_TOKEN").NotNullOrWhiteSpace() + } + } ) ] ), diff --git a/Importers/Rhino/Speckle.Importers.JobProcessor/packages.lock.json b/Importers/Rhino/Speckle.Importers.JobProcessor/packages.lock.json index badaf7ee8..a5cc59d02 100644 --- a/Importers/Rhino/Speckle.Importers.JobProcessor/packages.lock.json +++ b/Importers/Rhino/Speckle.Importers.JobProcessor/packages.lock.json @@ -40,9 +40,9 @@ }, "Npgsql": { "type": "Direct", - "requested": "[9.0.3, )", - "resolved": "9.0.3", - "contentHash": "tPvY61CxOAWxNsKLEBg+oR646X4Bc8UmyQ/tJszL/7mEmIXQnnBhVJZrZEEUv0Bstu0mEsHZD5At3EO8zQRAYw==", + "requested": "[9.0.4, )", + "resolved": "9.0.4", + "contentHash": "68BASXH0FAEuL/J4J0eRfYC8/3vzqQTmoW8zDzNf0JgaVxc7LZeEkS6jaG0ib3voFLxY5ZiCwJG+uQM+mzuu0Q==", "dependencies": { "Microsoft.Extensions.Logging.Abstractions": "8.0.2" } diff --git a/Importers/Rhino/Speckle.Importers.Rhino/Internal/IPCModels.cs b/Importers/Rhino/Speckle.Importers.Rhino/Internal/IPCModels.cs index 91e3bce1e..82c792521 100644 --- a/Importers/Rhino/Speckle.Importers.Rhino/Internal/IPCModels.cs +++ b/Importers/Rhino/Speckle.Importers.Rhino/Internal/IPCModels.cs @@ -12,6 +12,7 @@ internal readonly struct ImporterArgs public required string BlobId { get; init; } public required int Attempt { get; init; } public required string ResultsPath { get; init; } + public required string? TraceContext { get; init; } public required Project Project { get; init; } public required ModelIngestion Ingestion { get; init; } public required Account Account { get; init; } diff --git a/Importers/Rhino/Speckle.Importers.Rhino/Internal/ImporterInstance.cs b/Importers/Rhino/Speckle.Importers.Rhino/Internal/ImporterInstance.cs index 1bd58f9e5..ef826299e 100644 --- a/Importers/Rhino/Speckle.Importers.Rhino/Internal/ImporterInstance.cs +++ b/Importers/Rhino/Speckle.Importers.Rhino/Internal/ImporterInstance.cs @@ -2,60 +2,99 @@ using System.Diagnostics.Contracts; using Microsoft.Extensions.Logging; using Rhino; using Rhino.Runtime.InProcess; +using Speckle.Connectors.Common.Builders; using Speckle.Connectors.Common.Extensions; using Speckle.Connectors.Logging; using Speckle.Importers.Rhino.Internal.FileTypeConfig; using Speckle.Sdk.Api; -using Speckle.Sdk.Serialisation.V2.Send; +using Speckle.Sdk.Logging; namespace Speckle.Importers.Rhino.Internal; -internal sealed class ImporterInstance( - ImporterArgs args, - Sender sender, - IClient speckleClient, - ILogger logger -) : IDisposable +internal sealed class ImporterInstance : IDisposable { private readonly RhinoCore _rhinoInstance = new(["/netcore-8"], WindowStyle.NoWindow); - private readonly RhinoDoc _rhinoDoc = OpenDocument(args, logger); + private readonly RhinoDoc _rhinoDoc; - private readonly IReadOnlyList _scopes = - [ - ActivityScope.SetTag("jobId", args.JobId), - ActivityScope.SetTag("job.attempt", args.Attempt.ToString()), - // ActivityScope.SetTag("jobType", args.JobType), - ActivityScope.SetTag("serverUrl", args.Account.serverInfo.url), - ActivityScope.SetTag("projectId", args.Project.id), - ActivityScope.SetTag("modelIngestion.Id", args.Ingestion.id), - ActivityScope.SetTag("modelId", args.Ingestion.modelId), - ActivityScope.SetTag("blobId", args.BlobId), - ActivityScope.SetTag("fileType", Path.GetExtension(args.FilePath).TrimStart('.')), - UserActivityScope.AddUserScope(args.Account), - ]; + private readonly IReadOnlyList _scopes; - public async Task RunRhinoImport(CancellationToken cancellationToken) + private readonly ImporterArgs _args; + private readonly Sender _sender; + private readonly IClient _speckleClient; + private readonly ILogger _logger; + private readonly ISdkActivityFactory _activityFactory; + + public ImporterInstance( + ImporterArgs args, + Sender sender, + IClient speckleClient, + ILogger logger, + ISdkActivityFactory activityFactory + ) { + _args = args; + _sender = sender; + _speckleClient = speckleClient; + _logger = logger; + _activityFactory = activityFactory; + _rhinoDoc = OpenDocument(); + _scopes = + [ + ActivityScope.SetTag("jobId", args.JobId), + ActivityScope.SetTag("job.attempt", args.Attempt.ToString()), + // ActivityScope.SetTag("jobType", args.JobType), + ActivityScope.SetTag("serverUrl", new Uri(args.Account.serverInfo.url).ToString()), + ActivityScope.SetTag("projectId", args.Project.id), + ActivityScope.SetTag("modelIngestion.id", args.Ingestion.id), + ActivityScope.SetTag("modelId", args.Ingestion.modelId), + ActivityScope.SetTag("blobId", args.BlobId), + ActivityScope.SetTag("fileType", Path.GetExtension(args.FilePath).TrimStart('.')), + UserActivityScope.AddUserScope(args.Account), + ]; + } + + public async Task RunRhinoImport(CancellationToken cancellationToken) + { + using var activity = _activityFactory.Start(); try { RhinoDoc.ActiveDoc = _rhinoDoc; - var results = await sender - .Send(args.Project, args.Ingestion, speckleClient, cancellationToken) + var results = await _sender + .Send(_args.Project, _args.Ingestion, _speckleClient, cancellationToken) .ConfigureAwait(false); + activity?.SetStatus(SdkActivityStatusCode.Ok); return results; } + catch (Exception ex) + { + activity?.RecordException(ex); + activity?.SetStatus(SdkActivityStatusCode.Error); + throw; + } finally { RhinoDoc.ActiveDoc = null; } } - private static RhinoDoc OpenDocument(ImporterArgs args, ILogger logger) + private RhinoDoc OpenDocument() { - using var config = GetConfig(Path.GetExtension(args.FilePath)); - logger.LogInformation("Opening file {FilePath}", args.FilePath); - return config.OpenInHeadlessDocument(args.FilePath); + using var activity = _activityFactory.Start(); + try + { + using var config = GetConfig(Path.GetExtension(_args.FilePath)); + RhinoDoc openedDoc = config.OpenInHeadlessDocument(_args.FilePath); + + activity?.SetStatus(SdkActivityStatusCode.Ok); + return openedDoc; + } + catch (Exception ex) + { + activity?.RecordException(ex); + activity?.SetStatus(SdkActivityStatusCode.Error); + throw; + } } [Pure] @@ -82,6 +121,6 @@ internal sealed class ImporterInstance( { scope.Dispose(); } - speckleClient.Dispose(); + _speckleClient.Dispose(); } } diff --git a/Importers/Rhino/Speckle.Importers.Rhino/Internal/ImporterInstanceFactory.cs b/Importers/Rhino/Speckle.Importers.Rhino/Internal/ImporterInstanceFactory.cs index e5d2aa600..59ffd2619 100644 --- a/Importers/Rhino/Speckle.Importers.Rhino/Internal/ImporterInstanceFactory.cs +++ b/Importers/Rhino/Speckle.Importers.Rhino/Internal/ImporterInstanceFactory.cs @@ -1,17 +1,19 @@ using Microsoft.Extensions.Logging; using Speckle.Sdk.Api; +using Speckle.Sdk.Logging; namespace Speckle.Importers.Rhino.Internal; internal sealed class ImporterInstanceFactory( Sender sender, IClientFactory clientFactory, - ILogger logger + ILogger logger, + ISdkActivityFactory activityFactory ) { public ImporterInstance Create(ImporterArgs args) { var speckleClient = clientFactory.Create(args.Account); - return new(args, sender, speckleClient, logger); + return new(args, sender, speckleClient, logger, activityFactory); } } diff --git a/Importers/Rhino/Speckle.Importers.Rhino/Internal/Sender.cs b/Importers/Rhino/Speckle.Importers.Rhino/Internal/Sender.cs index 136c711d3..638ff513f 100644 --- a/Importers/Rhino/Speckle.Importers.Rhino/Internal/Sender.cs +++ b/Importers/Rhino/Speckle.Importers.Rhino/Internal/Sender.cs @@ -3,8 +3,7 @@ using Microsoft.Extensions.Logging; using Rhino; using Rhino.DocObjects; using Speckle.Connectors.Common.Analytics; -using Speckle.Connectors.Common.Operations; -using Speckle.Connectors.Common.Operations.Send; +using Speckle.Connectors.Common.Builders; using Speckle.Converters.Common; using Speckle.Converters.Rhino; using Speckle.Sdk; @@ -12,7 +11,8 @@ using Speckle.Sdk.Api; using Speckle.Sdk.Api.GraphQL.Models; using Speckle.Sdk.Credentials; using Speckle.Sdk.Logging; -using Speckle.Sdk.Serialisation.V2.Send; +using Speckle.Sdk.Pipelines.Progress; +using Speckle.Sdk.Pipelines.Send; namespace Speckle.Importers.Rhino.Internal; @@ -22,10 +22,11 @@ internal sealed class Sender( IRhinoConversionSettingsFactory rhinoConversionSettingsFactory, IMixPanelManager mixpanel, IIngestionProgressManagerFactory progressManagerFactory, + ISendPipelineFactory sendPipelineFactory, ILogger logger ) { - public async Task Send( + public async Task Send( Project project, ModelIngestion ingestion, IClient speckleClient, @@ -35,7 +36,6 @@ internal sealed class Sender( var progressManager = progressManagerFactory.CreateInstance( speckleClient, ingestion, - project.id, TimeSpan.FromSeconds(1.5), cancellationToken ); @@ -56,20 +56,30 @@ internal sealed class Sender( throw new SpeckleException("There are no objects found in the file"); } - var operation = scope.ServiceProvider.GetRequiredService>(); - var buildResults = await operation.Build(rhinoObjects, project.id, progressManager, cancellationToken); - var results = await operation.SendObjects( - buildResults.RootObject, + var rootContinuousTraversalBuilder = scope.ServiceProvider.GetRequiredService< + IRootContinuousTraversalBuilder + >(); + + var sendPipeline = sendPipelineFactory.CreateInstance( project.id, + ingestion.id, speckleClient.Account, + new RenderedStreamProgress(progressManager), + cancellationToken + ); + var buildResult = await rootContinuousTraversalBuilder.Build( + rhinoObjects, + project.id, + sendPipeline, progressManager, cancellationToken ); + buildResult.RootObject["version"] = 3; await TrackSendMetrics(project, speckleClient.Account); - logger.LogInformation("Root: {RootId}", results.RootId); - return results; + logger.LogInformation("Root: {RootId}", buildResult.RootObject.id); + return buildResult; } private async Task TrackSendMetrics(Project project, Account account) diff --git a/Importers/Rhino/Speckle.Importers.Rhino/Internal/ServiceRegistration.cs b/Importers/Rhino/Speckle.Importers.Rhino/Internal/ServiceRegistration.cs index 7f99d6aa6..00b47ce2c 100644 --- a/Importers/Rhino/Speckle.Importers.Rhino/Internal/ServiceRegistration.cs +++ b/Importers/Rhino/Speckle.Importers.Rhino/Internal/ServiceRegistration.cs @@ -1,35 +1,120 @@ -using Microsoft.Extensions.DependencyInjection; +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; using Speckle.Connectors.Common; -using Speckle.Connectors.Common.Operations.Send; +using Speckle.Connectors.Common.Common; using Speckle.Connectors.Common.Threading; +using Speckle.Connectors.Logging; using Speckle.Connectors.Rhino.DependencyInjection; using Speckle.Converters.Rhino; +using Speckle.Objects.Geometry; using Speckle.Sdk; using Speckle.Sdk.SQLite; +#if !DEBUG && !LOCAL +using Speckle.Sdk.Common; +#endif namespace Speckle.Importers.Rhino.Internal; internal static class ServiceRegistration { - public static IServiceCollection AddRhinoImporter(this IServiceCollection services, Application applicationInfo) - { - services.Initialize(applicationInfo, HostAppVersion.v8); - services.AddSingleton(applicationInfo); + private const HostAppVersion HOST_APP_VERSION = HostAppVersion.v3; - services.AddRhino(false); - services.AddRhinoConverters(); - services.AddTransient(); - services.AddTransient(); - services.AddTransient(); - services.AddTransient(); - services.AddTransient(); + public static IServiceCollection AddRhinoImporter( + this IServiceCollection serviceCollection, + Application applicationInfo + ) + { + var assemblyVersion = Assembly.GetExecutingAssembly().GetVersion(); + + serviceCollection.AddSpeckleSdk( + applicationInfo, + HostApplications.GetVersion(HOST_APP_VERSION), + assemblyVersion, + typeof(Point).Assembly + ); + + serviceCollection.AddSingleton(applicationInfo); + + serviceCollection.AddRhino(false); + serviceCollection.AddRhinoConverters(); + serviceCollection.AddTransient(); + serviceCollection.AddTransient(); + serviceCollection.AddTransient(); // override default thread context - services.AddSingleton(new ImporterThreadContext()); + serviceCollection.AddSingleton(new ImporterThreadContext()); // override sqlite cache, since we don't want to persist to disk any object data - services.AddTransient(); + serviceCollection.AddTransient(); - return services; + return serviceCollection; + } + + // Important to respect disposal, because disposal ensures pending messages are flushed + public static IDisposable AddLoggingConfig(this IServiceCollection serviceCollection, Application applicationInfo) + { + return serviceCollection.AddOpenTelemetry( + "Speckle.Importers.Rhino", + applicationInfo, + HOST_APP_VERSION, +#if DEBUG || LOCAL + new SpeckleLogging(Console: true, File: new(), MinimumLevel: SpeckleLogLevel.Debug), + new SpeckleTracing(Console: false), + new SpeckleMetrics(Console: false) +#else + new SpeckleLogging( + Console: true, + File: new(), + Otel: + [ + new( + Endpoint: new Uri("https://seq.speckle.systems/ingest/otlp/v1/logs"), + Headers: new() + { + // We're using a different token than connectors for seq because we want to beable to + // trust the client's timestamps (rather than use the server's timestamps) for better tracing + // This setting has more opportunity for abuse, so we're keeping it secret, unlike the connectors token. + { "X-Seq-ApiKey", Environment.GetEnvironmentVariable("SEQ_API_KEY").NotNullOrWhiteSpace() } + } + ), + new( + Endpoint: new Uri("https://collector.speckle.dev/v1/logs"), + Headers: new() + { + { + "authorization", + Environment.GetEnvironmentVariable("SPECKLE_COLLECTOR_API_TOKEN").NotNullOrWhiteSpace() + } + } + ) + ], + MinimumLevel: SpeckleLogLevel.Information + ), + new SpeckleTracing( + Console: false, + Otel: + [ + new( + Endpoint: new Uri("https://seq.speckle.systems/ingest/otlp/v1/traces"), + Headers: new() + { + { "X-Seq-ApiKey", Environment.GetEnvironmentVariable("SEQ_API_KEY").NotNullOrWhiteSpace() } + } + ), + new( + Endpoint: new Uri("https://collector.speckle.dev/v1/traces"), + Headers: new() + { + { + "authorization", + Environment.GetEnvironmentVariable("SPECKLE_COLLECTOR_API_TOKEN").NotNullOrWhiteSpace() + } + } + ) + ] + ), + null +#endif + ); } } diff --git a/Importers/Rhino/Speckle.Importers.Rhino/Program.cs b/Importers/Rhino/Speckle.Importers.Rhino/Program.cs index 60c47b93c..0e279712b 100644 --- a/Importers/Rhino/Speckle.Importers.Rhino/Program.cs +++ b/Importers/Rhino/Speckle.Importers.Rhino/Program.cs @@ -4,7 +4,9 @@ using System.Text.Json.Serialization; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using RhinoInside; +using Speckle.Connectors.Logging; using Speckle.Importers.Rhino.Internal; +using Speckle.Sdk.Logging; namespace Speckle.Importers.Rhino; @@ -22,6 +24,8 @@ public static class Program [SuppressMessage("Design", "CA1031:Do not catch general exception types", Justification = "IPC")] public static async Task Main(string[] args) { + //Thread.Sleep(10000); //For Debugging purposes, gives you enough time to attach your IDE to the running process + ILogger? logger = null; ImporterInstance? importer = null; @@ -31,11 +35,17 @@ public static class Program var serviceCollection = new ServiceCollection(); serviceCollection.AddRhinoImporter(importerArgs.HostApplication); + using var otel = serviceCollection.AddLoggingConfig(importerArgs.HostApplication); using var serviceProvider = serviceCollection.BuildServiceProvider(); logger = serviceProvider.GetRequiredService>(); TaskScheduler.UnobservedTaskException += (_, eventArgs) => logger.LogCritical(eventArgs.Exception, "Unobserved Task Exception"); + ISdkActivityFactory activityFactory = serviceProvider.GetRequiredService(); + using var activity = importerArgs.TraceContext is not null + ? activityFactory.StartRemote(importerArgs.TraceContext, SdkActivityKind.Consumer) + : activityFactory.Start(); + var factory = serviceProvider.GetRequiredService(); // Error handling flow below here looks a bit of a mess, but we're having to navigate threading issues with rhino inside @@ -58,7 +68,7 @@ public static class Program try { var results = await importer.RunRhinoImport(CancellationToken.None).ConfigureAwait(false); - WriteResult(new() { RootObjectId = results.RootId }, importerArgs.ResultsPath); + WriteResult(new() { RootObjectId = results.RootObject.id }, importerArgs.ResultsPath); } catch (Exception ex) { diff --git a/Importers/Rhino/Speckle.Importers.Rhino/Speckle.Importers.Rhino.csproj b/Importers/Rhino/Speckle.Importers.Rhino/Speckle.Importers.Rhino.csproj index 64a83f87d..b19933521 100644 --- a/Importers/Rhino/Speckle.Importers.Rhino/Speckle.Importers.Rhino.csproj +++ b/Importers/Rhino/Speckle.Importers.Rhino/Speckle.Importers.Rhino.csproj @@ -8,9 +8,9 @@ - - - + + + diff --git a/Importers/Rhino/Speckle.Importers.Rhino/packages.lock.json b/Importers/Rhino/Speckle.Importers.Rhino/packages.lock.json index 80ed6ae16..309d86b45 100644 --- a/Importers/Rhino/Speckle.Importers.Rhino/packages.lock.json +++ b/Importers/Rhino/Speckle.Importers.Rhino/packages.lock.json @@ -4,11 +4,11 @@ "net8.0-windows7.0": { "Grasshopper": { "type": "Direct", - "requested": "[8.25.25328.11001, )", - "resolved": "8.25.25328.11001", - "contentHash": "1uFL9pmgCEbYFd1b3JFGHaLEjQuRgFsiRHmP5u73mdEMdO6+5F/pNV1dQZr3YGLDovUmgrc+jmpsme7wr/J01A==", + "requested": "[8.28.26041.11001, )", + "resolved": "8.28.26041.11001", + "contentHash": "xlLD67F4/c5quwUrLxtAXT3M4Xam7GnhXyepC8/AblIBFcWbKEBzfOv85FlzO0ZPmDJwNjM5kRYML/sAStktZg==", "dependencies": { - "RhinoCommon": "[8.25.25328.11001]" + "RhinoCommon": "[8.28.26041.11001]" } }, "Microsoft.NETFramework.ReferenceAssemblies": { @@ -48,20 +48,20 @@ }, "RhinoCommon": { "type": "Direct", - "requested": "[8.25.25328.11001, )", - "resolved": "8.25.25328.11001", - "contentHash": "PDKR9GwqyUXUkTulV4J0dzDIf/aWqSJkL7nkS8ReAx8xhnt/+RQpE8gTjOSCmkSU2tjG6WzclowbTxwMTU7VAA==", + "requested": "[8.28.26041.11001, )", + "resolved": "8.28.26041.11001", + "contentHash": "5mByZF+IdHvRLbYvr6Ek95Pwam6otewAyVHIGSC0sUE1+OscSpd9gbrFWnzo1arfCYmUJcUdsGhf7VayW09fQA==", "dependencies": { "System.Drawing.Common": "7.0.0" } }, "RhinoWindows": { "type": "Direct", - "requested": "[8.25.25328.11001, )", - "resolved": "8.25.25328.11001", - "contentHash": "I/+++piwtYTue+iAAQqcMF5QlontqwNnC7Leyhiv2FiF8JpAl6K44ZsJqB7ZEUC6ns0LDfa3mbFzQwUfHwYumQ==", + "requested": "[8.28.26041.11001, )", + "resolved": "8.28.26041.11001", + "contentHash": "7MBG231k5c0/V/USTzbXpaTCiZCECQOuB80DnpgEvvEA3r//IQQDTbrqawDYmN9BEw/FHiFn82bsf6tV+SS5Iw==", "dependencies": { - "RhinoCommon": "[8.25.25328.11001]" + "RhinoCommon": "[8.28.26041.11001]" } }, "Speckle.InterfaceGenerator": { diff --git a/Sdk/Speckle.Connectors.Common.Tests/Operations/ReceiveOperationTests.cs b/Sdk/Speckle.Connectors.Common.Tests/Operations/ReceiveOperationTests.cs index 224bdae55..4ac7def16 100644 --- a/Sdk/Speckle.Connectors.Common.Tests/Operations/ReceiveOperationTests.cs +++ b/Sdk/Speckle.Connectors.Common.Tests/Operations/ReceiveOperationTests.cs @@ -12,6 +12,7 @@ using Speckle.Sdk.Api.GraphQL.Models; using Speckle.Sdk.Credentials; using Speckle.Sdk.Logging; using Speckle.Sdk.Models; +using Speckle.Sdk.Pipelines.Progress; using Speckle.Testing; namespace Speckle.Connectors.Common.Tests.Operations; diff --git a/Sdk/Speckle.Connectors.Common.Tests/Operations/ReceiveProgressTests.cs b/Sdk/Speckle.Connectors.Common.Tests/Operations/ReceiveProgressTests.cs index 414e5b235..3b98b54eb 100644 --- a/Sdk/Speckle.Connectors.Common.Tests/Operations/ReceiveProgressTests.cs +++ b/Sdk/Speckle.Connectors.Common.Tests/Operations/ReceiveProgressTests.cs @@ -1,6 +1,7 @@ using Moq; using NUnit.Framework; using Speckle.Connectors.Common.Operations; +using Speckle.Sdk.Pipelines.Progress; using Speckle.Sdk.Transports; using Speckle.Testing; diff --git a/Sdk/Speckle.Connectors.Common.Tests/Operations/SendProgressTests.cs b/Sdk/Speckle.Connectors.Common.Tests/Operations/SendProgressTests.cs index 5c10f566e..53117b6ac 100644 --- a/Sdk/Speckle.Connectors.Common.Tests/Operations/SendProgressTests.cs +++ b/Sdk/Speckle.Connectors.Common.Tests/Operations/SendProgressTests.cs @@ -1,6 +1,7 @@ using Moq; using NUnit.Framework; using Speckle.Connectors.Common.Operations; +using Speckle.Sdk.Pipelines.Progress; using Speckle.Sdk.Transports; using Speckle.Testing; diff --git a/Sdk/Speckle.Connectors.Common/Builders/IHostObjectBuilder.cs b/Sdk/Speckle.Connectors.Common/Builders/IHostObjectBuilder.cs index aa19cc90f..b558410f2 100644 --- a/Sdk/Speckle.Connectors.Common/Builders/IHostObjectBuilder.cs +++ b/Sdk/Speckle.Connectors.Common/Builders/IHostObjectBuilder.cs @@ -1,6 +1,6 @@ using Speckle.Connectors.Common.Conversion; -using Speckle.Connectors.Common.Operations; using Speckle.Sdk.Models; +using Speckle.Sdk.Pipelines.Progress; namespace Speckle.Connectors.Common.Builders; diff --git a/Sdk/Speckle.Connectors.Common/Builders/IRootObjectBuilder.cs b/Sdk/Speckle.Connectors.Common/Builders/IRootObjectBuilder.cs index 1699bc758..e5bc0a3b7 100644 --- a/Sdk/Speckle.Connectors.Common/Builders/IRootObjectBuilder.cs +++ b/Sdk/Speckle.Connectors.Common/Builders/IRootObjectBuilder.cs @@ -1,6 +1,7 @@ using Speckle.Connectors.Common.Conversion; -using Speckle.Connectors.Common.Operations; using Speckle.Sdk.Models; +using Speckle.Sdk.Pipelines.Progress; +using Speckle.Sdk.Pipelines.Send; namespace Speckle.Connectors.Common.Builders; @@ -14,4 +15,15 @@ public interface IRootObjectBuilder ); } +public interface IRootContinuousTraversalBuilder +{ + public Task Build( + IReadOnlyList objects, + string projectId, + SendPipeline sendPipeline, + IProgress onOperationProgressed, + CancellationToken cancellationToken + ); +} + public record RootObjectBuilderResult(Base RootObject, IReadOnlyList ConversionResults); diff --git a/Sdk/Speckle.Connectors.Common/Caching/ISendConversionCache.cs b/Sdk/Speckle.Connectors.Common/Caching/ISendConversionCache.cs index 4d396b403..afd1210b0 100644 --- a/Sdk/Speckle.Connectors.Common/Caching/ISendConversionCache.cs +++ b/Sdk/Speckle.Connectors.Common/Caching/ISendConversionCache.cs @@ -15,6 +15,8 @@ public interface ISendConversionCache { void StoreSendResult(string projectId, IReadOnlyDictionary convertedReferences); + void AppendSendResult(string projectId, string applicationId, ObjectReference convertedReference); + /// /// Call this method whenever you need to invalidate a set of objects that have changed in the host app. /// Failure to do so correctly will result in cache poisoning and incorrect version creation (stale objects). diff --git a/Sdk/Speckle.Connectors.Common/Caching/NullSendConversionCache.cs b/Sdk/Speckle.Connectors.Common/Caching/NullSendConversionCache.cs index 1ae66b67b..845055d5e 100644 --- a/Sdk/Speckle.Connectors.Common/Caching/NullSendConversionCache.cs +++ b/Sdk/Speckle.Connectors.Common/Caching/NullSendConversionCache.cs @@ -13,6 +13,8 @@ public class NullSendConversionCache : ISendConversionCache public void EvictObjects(IEnumerable objectIds) { } + public void AppendSendResult(string projectId, string applicationId, ObjectReference convertedReference) { } + public void ClearCache() { } public bool TryGetValue( diff --git a/Sdk/Speckle.Connectors.Common/Caching/SendConversionCache.cs b/Sdk/Speckle.Connectors.Common/Caching/SendConversionCache.cs index 521fe2bf1..d2a74e5f4 100644 --- a/Sdk/Speckle.Connectors.Common/Caching/SendConversionCache.cs +++ b/Sdk/Speckle.Connectors.Common/Caching/SendConversionCache.cs @@ -17,6 +17,11 @@ public class SendConversionCache : ISendConversionCache } } + public void AppendSendResult(string projectId, string applicationId, ObjectReference convertedReference) + { + Cache[(applicationId, projectId)] = convertedReference; + } + /// public void EvictObjects(IEnumerable objectIds) => Cache = Cache diff --git a/Sdk/Speckle.Connectors.Common/Connector.cs b/Sdk/Speckle.Connectors.Common/Connector.cs index 9d5b878e9..e45908635 100644 --- a/Sdk/Speckle.Connectors.Common/Connector.cs +++ b/Sdk/Speckle.Connectors.Common/Connector.cs @@ -37,7 +37,8 @@ public static class Connector typeof(Point).Assembly ); - return serviceCollection.AddSeqLogging( + return serviceCollection.AddOpenTelemetry( + "Connector", application, version, #if DEBUG || LOCAL @@ -72,8 +73,9 @@ public static class Connector ); } - public static IDisposable AddSeqLogging( + public static IDisposable AddOpenTelemetry( this IServiceCollection serviceCollection, + string serviceName, Application application, HostAppVersion version, SpeckleLogging loggingConfig, @@ -83,6 +85,7 @@ public static class Connector { var assemblyVersion = Assembly.GetExecutingAssembly().GetVersion(); var (logging, tracing, metrics) = Observability.Initialize( + serviceName, application.Name + " " + HostApplications.GetVersion(version), application.Slug, assemblyVersion, diff --git a/Sdk/Speckle.Connectors.Common/Conversion/ReportResult.cs b/Sdk/Speckle.Connectors.Common/Conversion/ReportResult.cs index c27e3bba8..835982cb0 100644 --- a/Sdk/Speckle.Connectors.Common/Conversion/ReportResult.cs +++ b/Sdk/Speckle.Connectors.Common/Conversion/ReportResult.cs @@ -1,3 +1,4 @@ +using Speckle.Newtonsoft.Json; using Speckle.Sdk.Common; using Speckle.Sdk.Models; @@ -34,8 +35,12 @@ public sealed class SendConversionResult : ConversionResult SourceType = sourceType; ResultId = result?.id; ResultType = result?.speckle_type; + Result = result; Error = FormatError(exception); } + + [JsonIgnore] + public Base? Result { get; } } // HACK: I've unsealed this for Grasshopper, non-ideal. Should be discussed and a better pattern may be implemented. @@ -63,7 +68,6 @@ public class ReceiveConversionResult : ConversionResult /// send conversion result or a receive conversion result - but i do not believe this requires fully separate classes, especially /// for what this is meant to be at its core: a list of green or red checkmarks in the UI. To make DX easier, the two classes above embody /// this one and provided clean constructors for each case. -/// POC: Inherits from Base so we can attach the conversion report to the root commit object. Can be revisited later (it's not a problem to not inherit from base). /// public abstract class ConversionResult { diff --git a/Sdk/Speckle.Connectors.Common/Extensions/RootObjectBuilderExtensions.cs b/Sdk/Speckle.Connectors.Common/Extensions/RootObjectBuilderExtensions.cs index fa630c4c8..bb3b6f51f 100644 --- a/Sdk/Speckle.Connectors.Common/Extensions/RootObjectBuilderExtensions.cs +++ b/Sdk/Speckle.Connectors.Common/Extensions/RootObjectBuilderExtensions.cs @@ -6,6 +6,21 @@ namespace Speckle.Connectors.Common.Extensions; public static class RootObjectBuilderExtensions { + public static void LogSendConversionError( + this ILogger> logger, + Exception ex, + string objectType + ) + { + LogLevel logLevel = ex switch + { + SpeckleException => LogLevel.Information, + _ => LogLevel.Error + }; + + logger.Log(logLevel, ex, "Conversion of object {ObjectType} was not successful", objectType); + } + public static void LogSendConversionError( this ILogger> logger, Exception ex, diff --git a/Sdk/Speckle.Connectors.Common/Extensions/StopwatchPollyfills.cs b/Sdk/Speckle.Connectors.Common/Extensions/StopwatchPollyfills.cs deleted file mode 100644 index 6bb7932f7..000000000 --- a/Sdk/Speckle.Connectors.Common/Extensions/StopwatchPollyfills.cs +++ /dev/null @@ -1,21 +0,0 @@ -using System.Diagnostics; - -namespace Speckle.Connectors.Common.Extensions; - -public static class StopwatchPollyfills -{ -#if !NET7_0_OR_GREATER - private static readonly double s_tickFrequency = (double)TimeSpan.TicksPerSecond / Stopwatch.Frequency; -#endif - - public static TimeSpan GetElapsedTime(long startingTimestamp) - { -#if NET7_0_OR_GREATER - return Stopwatch.GetElapsedTime(startingTimestamp); -#else - - long elapsedTicks = Stopwatch.GetTimestamp() - startingTimestamp; - return new TimeSpan((long)(elapsedTicks * s_tickFrequency)); -#endif - } -} diff --git a/Sdk/Speckle.Connectors.Common/Instances/IInstanceBaker.cs b/Sdk/Speckle.Connectors.Common/Instances/IInstanceBaker.cs index a2a8e559f..316e3ca67 100644 --- a/Sdk/Speckle.Connectors.Common/Instances/IInstanceBaker.cs +++ b/Sdk/Speckle.Connectors.Common/Instances/IInstanceBaker.cs @@ -1,6 +1,6 @@ -using Speckle.Connectors.Common.Operations; using Speckle.Sdk.Models.Collections; using Speckle.Sdk.Models.Instances; +using Speckle.Sdk.Pipelines.Progress; namespace Speckle.Connectors.Common.Instances; diff --git a/Sdk/Speckle.Connectors.Common/Operations/CardProgress.cs b/Sdk/Speckle.Connectors.Common/Operations/CardProgress.cs deleted file mode 100644 index d1fbd4a8f..000000000 --- a/Sdk/Speckle.Connectors.Common/Operations/CardProgress.cs +++ /dev/null @@ -1,3 +0,0 @@ -namespace Speckle.Connectors.Common.Operations; - -public readonly record struct CardProgress(string Status, double? Progress); diff --git a/Sdk/Speckle.Connectors.Common/Operations/ReceiveOperation.cs b/Sdk/Speckle.Connectors.Common/Operations/ReceiveOperation.cs index 242694a5a..8f2b795d5 100644 --- a/Sdk/Speckle.Connectors.Common/Operations/ReceiveOperation.cs +++ b/Sdk/Speckle.Connectors.Common/Operations/ReceiveOperation.cs @@ -8,6 +8,7 @@ using Speckle.Sdk.Credentials; using Speckle.Sdk.Logging; using Speckle.Sdk.Models; using Speckle.Sdk.Models.Extensions; +using Speckle.Sdk.Pipelines.Progress; namespace Speckle.Connectors.Common.Operations; diff --git a/Sdk/Speckle.Connectors.Common/Operations/ReceiveProgress.cs b/Sdk/Speckle.Connectors.Common/Operations/ReceiveProgress.cs index 3e0cf800d..e46fe67a1 100644 --- a/Sdk/Speckle.Connectors.Common/Operations/ReceiveProgress.cs +++ b/Sdk/Speckle.Connectors.Common/Operations/ReceiveProgress.cs @@ -1,4 +1,5 @@ using Speckle.InterfaceGenerator; +using Speckle.Sdk.Pipelines.Progress; using Speckle.Sdk.Transports; namespace Speckle.Connectors.Common.Operations; diff --git a/Sdk/Speckle.Connectors.Common/Operations/Send/AggregateProgress.cs b/Sdk/Speckle.Connectors.Common/Operations/Send/AggregateProgress.cs deleted file mode 100644 index 8bcf2f5d6..000000000 --- a/Sdk/Speckle.Connectors.Common/Operations/Send/AggregateProgress.cs +++ /dev/null @@ -1,19 +0,0 @@ -namespace Speckle.Connectors.Common.Operations.Send; - -public sealed class AggregateProgress : IProgress -{ - private readonly IProgress[] _progresses; - - public AggregateProgress(params IProgress[] progresses) - { - _progresses = progresses; - } - - public void Report(T value) - { - foreach (var progress in _progresses) - { - progress.Report(value); - } - } -} diff --git a/Sdk/Speckle.Connectors.Common/Operations/Send/IngestionProgressManager.cs b/Sdk/Speckle.Connectors.Common/Operations/Send/IngestionProgressManager.cs deleted file mode 100644 index 4b4b4e6cd..000000000 --- a/Sdk/Speckle.Connectors.Common/Operations/Send/IngestionProgressManager.cs +++ /dev/null @@ -1,92 +0,0 @@ -using System.Diagnostics; -using Microsoft.Extensions.Logging; -using Speckle.Connectors.Common.Extensions; -using Speckle.InterfaceGenerator; -using Speckle.Sdk.Api; -using Speckle.Sdk.Api.GraphQL.Inputs; -using Speckle.Sdk.Api.GraphQL.Models; - -namespace Speckle.Connectors.Common.Operations.Send; - -public partial interface IIngestionProgressManager : IProgress; - -/// -/// An implementation for the entire client side Ingestion progress update reporting -/// Will throttles ingestion progress messages and reports their progress -/// -/// -/// The same class exists also in the RVT ODA codebase -/// -[GenerateAutoInterface] -public sealed class IngestionProgressManager( - ILogger logger, - IClient speckleClient, - ModelIngestion ingestion, - string projectId, - TimeSpan updateInterval, - CancellationToken cancellationToken -) : IIngestionProgressManager -{ - /// - /// We've picked quite a coarse throttle window to try and avoid over pressure - /// - private Task? _lastUpdate; - private long _lastUpdatedAt; - private readonly object _lock = new(); - - [AutoInterfaceIgnore] - public void Report(CardProgress value) - { - cancellationToken.ThrowIfCancellationRequested(); - - string trimmedMessage; - lock (_lock) - { - if (ShouldIgnoreProgressUpdate()) - { - return; - } - - _lastUpdatedAt = Stopwatch.GetTimestamp(); - - trimmedMessage = value.Status.TrimEnd('.'); - - _lastUpdate = speckleClient - .Ingestion.UpdateProgress( - new ModelIngestionUpdateInput(ingestion.id, projectId, trimmedMessage, value.Progress), - cancellationToken - ) - .ContinueWith( - HandleFaultedContinuation, - CancellationToken.None, - TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously, - TaskScheduler.Default - ); - } - - logger.LogInformation("Progress update {Message} {Progress}", trimmedMessage, value.Progress); - } - - /// if the update should be ignored, otherwise - private bool ShouldIgnoreProgressUpdate() - { - if (_lastUpdate is not null && !_lastUpdate.IsCompleted) - { - return true; - } - - TimeSpan msSinceLastUpdate = StopwatchPollyfills.GetElapsedTime(_lastUpdatedAt); - return msSinceLastUpdate < updateInterval; - } - - private void HandleFaultedContinuation(Task updateTask) - { - // The progress report failed... could be many reasons. - // For now, we're not letting this fail the Ingestion in any way - // we'll log but otherwise let it slide while leaving no unobserved task exceptions - if (updateTask.IsFaulted) - { - logger.LogWarning(updateTask.Exception, "A progress update failed unexpectedly"); - } - } -} diff --git a/Sdk/Speckle.Connectors.Common/Operations/Send/IngestionProgressManagerFactory.cs b/Sdk/Speckle.Connectors.Common/Operations/Send/IngestionProgressManagerFactory.cs deleted file mode 100644 index c969ee25f..000000000 --- a/Sdk/Speckle.Connectors.Common/Operations/Send/IngestionProgressManagerFactory.cs +++ /dev/null @@ -1,22 +0,0 @@ -using Microsoft.Extensions.Logging; -using Speckle.InterfaceGenerator; -using Speckle.Sdk.Api; -using Speckle.Sdk.Api.GraphQL.Models; - -namespace Speckle.Connectors.Common.Operations.Send; - -[GenerateAutoInterface] -public sealed class IngestionProgressManagerFactory(ILogger logger) - : IIngestionProgressManagerFactory -{ - public IIngestionProgressManager CreateInstance( - IClient speckleClient, - ModelIngestion ingestion, - string projectId, - TimeSpan updateInterval, - CancellationToken cancellationToken - ) - { - return new IngestionProgressManager(logger, speckleClient, ingestion, projectId, updateInterval, cancellationToken); - } -} diff --git a/Sdk/Speckle.Connectors.Common/Operations/SendOperation.cs b/Sdk/Speckle.Connectors.Common/Operations/SendOperation.cs index cb1167fb0..cbd415dd3 100644 --- a/Sdk/Speckle.Connectors.Common/Operations/SendOperation.cs +++ b/Sdk/Speckle.Connectors.Common/Operations/SendOperation.cs @@ -1,7 +1,6 @@ using Speckle.Connectors.Common.Builders; using Speckle.Connectors.Common.Caching; using Speckle.Connectors.Common.Conversion; -using Speckle.Connectors.Common.Operations.Send; using Speckle.Connectors.Common.Threading; using Speckle.Connectors.Logging; using Speckle.InterfaceGenerator; @@ -10,11 +9,17 @@ using Speckle.Sdk.Api; using Speckle.Sdk.Api.GraphQL.Inputs; using Speckle.Sdk.Api.GraphQL.Models; using Speckle.Sdk.Credentials; +using Speckle.Sdk.Helpers; using Speckle.Sdk.Logging; using Speckle.Sdk.Models; +using Speckle.Sdk.Pipelines.Progress; +using Speckle.Sdk.Pipelines.Send; using Speckle.Sdk.Serialisation; using Speckle.Sdk.Serialisation.V2.Send; using Version = Speckle.Sdk.Api.GraphQL.Models.Version; +#if !NET8_0_OR_GREATER +using System.Net.Http; +#endif namespace Speckle.Connectors.Common.Operations; @@ -24,13 +29,16 @@ public sealed class SendOperation( ISendConversionCache sendConversionCache, ISendProgress sendProgress, ISendOperationExecutor sendOperationExecutor, - ISdkActivityFactory activityFactory, IThreadContext threadContext, + ISdkActivityFactory activityFactory, ISpeckleApplication speckleApplication, - IIngestionProgressManagerFactory ingestionProgressManagerFactory + IIngestionProgressManagerFactory ingestionProgressManagerFactory, + ISpeckleHttp speckleHttp, + ISendPipelineFactory sendPipelineFactory, + IRootContinuousTraversalBuilder? rootContinuousTraversalBuilder = null ) : ISendOperation { - public async Task<(SendOperationResult sendResult, string versionId)> Send( + public async Task<(SendOperationResult sendResult, string versionId, string? ingestionId)> Send( IReadOnlyList objects, SendInfo sendInfo, string? fileName, @@ -43,6 +51,20 @@ public sealed class SendOperation( bool useModelIngestionSend = await CheckUseModelIngestionSend(sendInfo); if (useModelIngestionSend) { + bool usePackfileSend = + rootContinuousTraversalBuilder != null && await CheckPackfileSendEndpoints(sendInfo, cancellationToken); + if (usePackfileSend) + { + return await SendViaPackfile( + objects, + sendInfo, + fileName, + fileSizeBytes, + versionMessage, + uiProgress, + cancellationToken + ); + } return await SendViaIngestion( objects, sendInfo, @@ -59,22 +81,30 @@ public sealed class SendOperation( } } - private async Task<(SendOperationResult sendResult, string versionId)> SendViaIngestion( + private async Task<(SendOperationResult sendResult, string versionId, string? ingestionId)> SendViaPackfile( IReadOnlyList objects, SendInfo sendInfo, string? fileName, long? fileSizeBytes, +#pragma warning disable IDE0060 string? versionMessage, +#pragma warning restore IDE0060 IProgress uiProgress, CancellationToken cancellationToken ) { + if (rootContinuousTraversalBuilder == null) + { + throw new InvalidOperationException("rootContinuousTraversalBuilder cannot be null"); + } + ModelIngestion ingestion = await sendInfo.Client.Ingestion.Create( new( sendInfo.ModelId, sendInfo.ProjectId, $"Sending from {speckleApplication.ApplicationAndVersion}", - new(speckleApplication.Slug, speckleApplication.HostApplicationVersion, fileName, fileSizeBytes) + new(speckleApplication.Slug, speckleApplication.HostApplicationVersion, fileName, fileSizeBytes), + 600 ), cancellationToken ); @@ -83,21 +113,42 @@ public sealed class SendOperation( var ingestionProgress = ingestionProgressManagerFactory.CreateInstance( sendInfo.Client, ingestion, - sendInfo.ProjectId, - TimeSpan.FromSeconds(5), + TimeSpan.FromSeconds(10), cancellationToken ); + AggregateProgress progress = new(ingestionProgress, uiProgress); try { - SendOperationResult result = await ConvertAndSend(objects, sendInfo, progress, cancellationToken); - - string createdVersionId = await sendInfo.Client.Ingestion.Complete( - new(ingestion.id, sendInfo.ProjectId, result.RootObjId, versionMessage), - CancellationToken.None + var sendPipeline = sendPipelineFactory.CreateInstance( + sendInfo.ProjectId, + ingestion.id, + sendInfo.Account, + new RenderedStreamProgress(progress), + cancellationToken + ); + var buildResult = await rootContinuousTraversalBuilder.Build( + objects, + sendInfo.ProjectId, + sendPipeline, + progress, + cancellationToken ); - return (result, createdVersionId); + buildResult.RootObject["version"] = 3; + + WriteReferencesToCache(buildResult.ConversionResults, sendInfo.ProjectId); + + SendOperationResult result = + new(buildResult.RootObject.id!, new Dictionary(), buildResult.ConversionResults); + + // NOTE: clients do not need to complete the ingestion - that's going to the be the server's job + // string createdVersionId = await sendInfo.Client.Ingestion.Complete( + // new(ingestion.id, sendInfo.ProjectId, result.RootObjId, versionMessage), + // CancellationToken.None + // ); + + return (result, "latest", ingestion.id); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { @@ -117,7 +168,68 @@ public sealed class SendOperation( } } - private async Task<(SendOperationResult sendResult, string versionId)> SendViaVersionCreate( + private async Task<(SendOperationResult sendResult, string versionId, string? ingestionId)> SendViaIngestion( + IReadOnlyList objects, + SendInfo sendInfo, + string? fileName, + long? fileSizeBytes, + string? versionMessage, + IProgress uiProgress, + CancellationToken cancellationToken + ) + { + ModelIngestion ingestion = await sendInfo.Client.Ingestion.Create( + new( + sendInfo.ModelId, + sendInfo.ProjectId, + $"Sending from {speckleApplication.ApplicationAndVersion}", + new(speckleApplication.Slug, speckleApplication.HostApplicationVersion, fileName, fileSizeBytes), + 600 + ), + cancellationToken + ); + using var ingestionScope = ActivityScope.SetTag("modelIngestionId", ingestion.id); + + var ingestionProgress = ingestionProgressManagerFactory.CreateInstance( + sendInfo.Client, + ingestion, + TimeSpan.FromSeconds(5), + cancellationToken + ); + AggregateProgress progress = new(ingestionProgress, uiProgress); + try + { + SendOperationResult result = await ConvertAndSend(objects, sendInfo, progress, cancellationToken); + + string createdVersionId = await sendInfo.Client.Ingestion.Complete( + new(ingestion.id, sendInfo.ProjectId, result.RootObjId, versionMessage), + CancellationToken.None + ); + + // NOTE: it might seem weird to pass null for ingestion.id 'null' here but there is a reason. + // Because we complete ingestion here in .NET which is safe to pass null ingestion id that we don't want DUI explicitly subscribe to ingestion changes. + // I am hoping we will get rid of from logical branching once we have model ingestion on public server + return (result, createdVersionId, null); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + _ = await sendInfo.Client.Ingestion.FailWithCancel( + new(ingestion.id, sendInfo.ProjectId, "User requested cancellation"), + CancellationToken.None + ); + throw; + } + catch (Exception ex) + { + _ = await sendInfo.Client.Ingestion.FailWithError( + ModelIngestionFailedInput.FromException(ingestion.id, sendInfo.ProjectId, ex), + CancellationToken.None + ); + throw; + } + } + + private async Task<(SendOperationResult sendResult, string versionId, string? ingestionId)> SendViaVersionCreate( IReadOnlyList objects, SendInfo sendInfo, string? versionMessage, @@ -137,7 +249,7 @@ public sealed class SendOperation( ), cancellationToken ); - return (result, version.id); + return (result, version.id, null); } public async Task ConvertAndSend( @@ -239,6 +351,66 @@ public sealed class SendOperation( return useModelIngestionSend; } + + /// + /// + /// server returned a response, but it was neither true nor false (case insensitive) + /// Request failed, or the server returned a non-successful status code that wasn't 404 + /// + /// Returns if the server supports the new packfile data uploads, + /// if the server doesn't explicitly, or implicitly via a 404 response. + /// Will throw for unexpected cases. + /// + private async Task CheckPackfileSendEndpoints(SendInfo sendInfo, CancellationToken cancellationToken) + { + Uri url = new Uri(new Uri(sendInfo.Account.serverInfo.url), "/api/v1/data-module-enabled"); + using HttpClient client = speckleHttp.CreateHttpClient(); + using var response = await client.GetAsync(url, cancellationToken); + if (response.StatusCode != System.Net.HttpStatusCode.NotFound) + { + response.EnsureSuccessStatusCode(); +#if NET8_0_OR_GREATER + string responseBody = await response.Content.ReadAsStringAsync(cancellationToken); +#else + string responseBody = await response.Content.ReadAsStringAsync(); +#endif + return bool.Parse(responseBody); + } + + return false; + } + + /// + /// Reads the conversion results and any will be written to cache. + /// All other values will be ignored. + /// + /// + /// For the connectors that support send caching, we are reporting all results as either or + /// For Navisworks, which we no longer support send caching, it reports other subtypes, and those will not be cached. + /// + /// + /// + private void WriteReferencesToCache(IReadOnlyList conversionResults, string projectId) + { + // We write the objects to the cache after they've been uploaded to the server + // There is still an inbuilt "bad" assumption here that, successfully uploading NDJson means the server is able to re-materialize ids - + // but this is only true once the `Version` object is created + // Since for many reasons, the server could fail to process the json... + // This would leave this send cache out-of-sync with the server, and lead to failed processing of subsequent NDJson uploads + // For now, we've taken the decision that it's unlikely to happen... + + var references = new Dictionary(); + foreach (var x in conversionResults) + { + if (x.Result is ObjectReference r) + { + // NOTE: why not ToDictionary -> we might end up reoccurring object references for any reason. instancing, linked models etc. + // ToDictionary throws 'item already exists' errors. but safe to override items in references dictionary since they are unique + references[new Id(x.SourceId)] = r; + } + } + sendConversionCache.StoreSendResult(projectId, references); + } } public record SendOperationResult( diff --git a/Sdk/Speckle.Connectors.Common/Operations/SendProgress.cs b/Sdk/Speckle.Connectors.Common/Operations/SendProgress.cs index e77d17cbd..e1ea5b361 100644 --- a/Sdk/Speckle.Connectors.Common/Operations/SendProgress.cs +++ b/Sdk/Speckle.Connectors.Common/Operations/SendProgress.cs @@ -1,4 +1,5 @@ using Speckle.InterfaceGenerator; +using Speckle.Sdk.Pipelines.Progress; using Speckle.Sdk.Transports; namespace Speckle.Connectors.Common.Operations; diff --git a/Sdk/Speckle.Connectors.Common/Speckle.Connectors.Common.csproj b/Sdk/Speckle.Connectors.Common/Speckle.Connectors.Common.csproj index 8ae753d87..40d067edf 100644 --- a/Sdk/Speckle.Connectors.Common/Speckle.Connectors.Common.csproj +++ b/Sdk/Speckle.Connectors.Common/Speckle.Connectors.Common.csproj @@ -13,10 +13,11 @@ - + - + - + + diff --git a/Sdk/Speckle.Connectors.Logging/Consts.cs b/Sdk/Speckle.Connectors.Logging/Consts.cs index d5e463447..fa8595073 100644 --- a/Sdk/Speckle.Connectors.Logging/Consts.cs +++ b/Sdk/Speckle.Connectors.Logging/Consts.cs @@ -4,16 +4,18 @@ namespace Speckle.Connectors.Logging; public static class Consts { + public const string DEPLOYMENT_ENVIRONMENT = "deployment.environment.name"; public const string SERVICE_NAME = "connector.name"; public const string SERVICE_SLUG = "connector.slug"; public const string OS_NAME = "os.name"; public const string OS_TYPE = "os.type"; public const string OS_SLUG = "os.slug"; - public const string RUNTIME_NAME = "runtime.name"; + public const string RUNTIME_NAME = "process.runtime.name"; + public const string RUNTIME_VERSION = "process.runtime.version"; public const string USER_ID = "user.id"; public const string USER_DISTINCT_ID = "user.distinctId"; public const string USER_SERVER_URL = "user.server_url"; - public const string TRACING_SOURCE = "speckle"; + public const string TRACING_SOURCE = "connector"; /// /// A random GUID for adding to the logging context to correlate the service.instance.id diff --git a/Sdk/Speckle.Connectors.Logging/Internal/ResourceCreator.cs b/Sdk/Speckle.Connectors.Logging/Internal/ResourceCreator.cs index 497544b23..fa0a3cc95 100644 --- a/Sdk/Speckle.Connectors.Logging/Internal/ResourceCreator.cs +++ b/Sdk/Speckle.Connectors.Logging/Internal/ResourceCreator.cs @@ -5,24 +5,33 @@ namespace Speckle.Connectors.Logging.Internal; internal static class ResourceCreator { - internal static ResourceBuilder Create(string applicationAndVersion, string slug, string connectorVersion) => - ResourceBuilder + internal static ResourceBuilder Create( + string serviceName, + string applicationAndVersion, + string slug, + string connectorVersion + ) + { + string deploymentEnvironment = + Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT") + ?? Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT") + ?? "Development"; + return ResourceBuilder .CreateEmpty() - .AddService( - serviceName: Consts.TRACING_SOURCE, - serviceVersion: connectorVersion, - serviceInstanceId: Consts.StaticSessionId - ) + .AddService(serviceName: serviceName, serviceVersion: connectorVersion, serviceInstanceId: Consts.StaticSessionId) .AddAttributes( [ + new(Consts.DEPLOYMENT_ENVIRONMENT, deploymentEnvironment.ToLowerInvariant()), new(Consts.SERVICE_NAME, applicationAndVersion), new(Consts.SERVICE_SLUG, slug), new(Consts.OS_NAME, Environment.OSVersion.ToString()), new(Consts.OS_TYPE, RuntimeInformation.ProcessArchitecture.ToString()), new(Consts.OS_SLUG, DetermineHostOsSlug()), - new(Consts.RUNTIME_NAME, RuntimeInformation.FrameworkDescription), + new(Consts.RUNTIME_NAME, ".NET"), + new(Consts.RUNTIME_VERSION, RuntimeInformation.FrameworkDescription), ] ); + } private static string DetermineHostOsSlug() { diff --git a/Sdk/Speckle.Connectors.Logging/Observability.cs b/Sdk/Speckle.Connectors.Logging/Observability.cs index 59405aed4..c4b226c91 100644 --- a/Sdk/Speckle.Connectors.Logging/Observability.cs +++ b/Sdk/Speckle.Connectors.Logging/Observability.cs @@ -5,13 +5,14 @@ namespace Speckle.Connectors.Logging; public static class Observability { public static (LoggerProvider, IDisposable, IDisposable) Initialize( + string serviceName, string applicationAndVersion, string slug, string connectorVersion, SpeckleObservability observability ) { - var resourceBuilder = ResourceCreator.Create(applicationAndVersion, slug, connectorVersion); + var resourceBuilder = ResourceCreator.Create(serviceName, applicationAndVersion, slug, connectorVersion); var logging = LogBuilder.Initialize( applicationAndVersion, connectorVersion, diff --git a/Sdk/Speckle.Connectors.Logging/packages.lock.json b/Sdk/Speckle.Connectors.Logging/packages.lock.json index e56cd1682..296dcbfcf 100644 --- a/Sdk/Speckle.Connectors.Logging/packages.lock.json +++ b/Sdk/Speckle.Connectors.Logging/packages.lock.json @@ -41,23 +41,22 @@ }, "OpenTelemetry.Exporter.OpenTelemetryProtocol": { "type": "Direct", - "requested": "[1.11.1, )", - "resolved": "1.11.1", - "contentHash": "UiZBa+2b396Oxx9RX7h4ch+yZvX8nezxVkihPLU6zdEUfJbbVY2mNypJKEoW2Vh4xCaCp0fB6na3Kti+KfTVaw==", + "requested": "[1.13.1, )", + "resolved": "1.13.1", + "contentHash": "WFqOpqMkjd8lM/asQRgfLP72UCqThQIGoylDgoYR8x0Bh9UCrdmBJCDU4pVZgI9CtSq1sWXeQuUsRXAl5U4yKg==", "dependencies": { - "Grpc.Core": "[2.44.0, 3.0.0)", - "OpenTelemetry": "1.11.1" + "OpenTelemetry": "1.13.1" } }, "OpenTelemetry.Instrumentation.Http": { "type": "Direct", - "requested": "[1.11.0, )", - "resolved": "1.11.0", - "contentHash": "1ncYPNmMaYNPX664uo3FlmSVGETBKQbBvarbGgB5ZynERTFmCsZ7UqefvVe3vnPYOIAGOjbMAbprYF2BfMielg==", + "requested": "[1.13.0, )", + "resolved": "1.13.0", + "contentHash": "B+nmCn3/orrLhUuCC6WwHh9JUkIV/wubpZ+vYOf2CedjOZupgcQcx96Kwy6UVjdNDWGxsEw0jXWXZUQlYnmRqA==", "dependencies": { "Microsoft.Extensions.Configuration": "9.0.0", "Microsoft.Extensions.Options": "9.0.0", - "OpenTelemetry.Api.ProviderBuilderExtensions": "[1.11.1, 2.0.0)" + "OpenTelemetry.Api.ProviderBuilderExtensions": "[1.13.1, 2.0.0)" } }, "PolySharp": { @@ -120,23 +119,6 @@ "resolved": "0.9.6", "contentHash": "HKH7tYrYYlCK1ct483hgxERAdVdMtl7gUKW9ijWXxA1UsYR4Z+TrRHYmzZ9qmpu1NnTycSrp005NYM78GDKV1w==" }, - "Grpc.Core": { - "type": "Transitive", - "resolved": "2.44.0", - "contentHash": "H2rTNePSYeEqUgBBqiE5KZ/yOX7jEtETjWjv4gGtnYxrlpZ79VAHe+4KtZAnt2KJMiGvqTm7eo1SPk+QpiPfaw==", - "dependencies": { - "Grpc.Core.Api": "2.44.0", - "System.Memory": "4.5.3" - } - }, - "Grpc.Core.Api": { - "type": "Transitive", - "resolved": "2.44.0", - "contentHash": "FBfPMvKwT8q98T8lWa5z6nBMLdH/Mmo5g4yyYYMvbXLWDzo4beqa7CUU5QH3PKvo2X6/b+UAZ2IymXlrYG3IXg==", - "dependencies": { - "System.Memory": "4.5.3" - } - }, "ILRepack": { "type": "Transitive", "resolved": "2.0.33", @@ -264,29 +246,29 @@ }, "OpenTelemetry": { "type": "Transitive", - "resolved": "1.11.1", - "contentHash": "F+HBI2bE7RKmb8Bj0kBtZIVzCfpTe1ZyY6kYP/jny1+9oq7IdBnNsVXZlPev9OqQzRp3iXpJ1UsnN1YOEwdtkQ==", + "resolved": "1.13.1", + "contentHash": "rrM2NlZ0Xla2Ar8zzU09n+HoLFq8b+Kjx7vrmR0tdYfWLYWGNcXHOITXUiyaK8RrFEceMEoF45VBsaf4QPmKcg==", "dependencies": { "Microsoft.Extensions.Diagnostics.Abstractions": "9.0.0", "Microsoft.Extensions.Logging.Configuration": "9.0.0", - "OpenTelemetry.Api.ProviderBuilderExtensions": "1.11.1" + "OpenTelemetry.Api.ProviderBuilderExtensions": "1.13.1" } }, "OpenTelemetry.Api": { "type": "Transitive", - "resolved": "1.11.1", - "contentHash": "KaBjGMqrqQv41mIkvPUvmAG7yxDlI6qchKhjXlOF3ZwsdcRRLrdrkiDLIJ90iZgUoKVdP8fE1fCri9nc+ug0Cg==", + "resolved": "1.13.1", + "contentHash": "tieglRERo7Rgu8oE8aamnuXCMPEW5fXIqO5ngTMCNk9pOEXanc0SdQ86ZAD1goNiGcjWHn+P3WMZp0FZSJgCoQ==", "dependencies": { "System.Diagnostics.DiagnosticSource": "9.0.0" } }, "OpenTelemetry.Api.ProviderBuilderExtensions": { "type": "Transitive", - "resolved": "1.11.1", - "contentHash": "vMdNMQeW55jXIa/Kybec/br6jC+rWybniTi6DCW5lz1kGghKso+J+FC3uBgiq0/pTqusfeDbO5PEHGM/r5z8Ow==", + "resolved": "1.13.1", + "contentHash": "x8QXMsrIyp+XzDUFQAM+C4upAPNbwaBIPjTWEoonziWAav6weS8OxsMKrE4wz7Zly8ATlsoxk0mWZ+PHO3Wg0w==", "dependencies": { "Microsoft.Extensions.DependencyInjection.Abstractions": "9.0.0", - "OpenTelemetry.Api": "1.11.1" + "OpenTelemetry.Api": "1.13.1" } }, "System.Buffers": { @@ -417,22 +399,22 @@ }, "OpenTelemetry.Exporter.OpenTelemetryProtocol": { "type": "Direct", - "requested": "[1.11.1, )", - "resolved": "1.11.1", - "contentHash": "UiZBa+2b396Oxx9RX7h4ch+yZvX8nezxVkihPLU6zdEUfJbbVY2mNypJKEoW2Vh4xCaCp0fB6na3Kti+KfTVaw==", + "requested": "[1.13.1, )", + "resolved": "1.13.1", + "contentHash": "WFqOpqMkjd8lM/asQRgfLP72UCqThQIGoylDgoYR8x0Bh9UCrdmBJCDU4pVZgI9CtSq1sWXeQuUsRXAl5U4yKg==", "dependencies": { - "OpenTelemetry": "1.11.1" + "OpenTelemetry": "1.13.1" } }, "OpenTelemetry.Instrumentation.Http": { "type": "Direct", - "requested": "[1.11.0, )", - "resolved": "1.11.0", - "contentHash": "1ncYPNmMaYNPX664uo3FlmSVGETBKQbBvarbGgB5ZynERTFmCsZ7UqefvVe3vnPYOIAGOjbMAbprYF2BfMielg==", + "requested": "[1.13.0, )", + "resolved": "1.13.0", + "contentHash": "B+nmCn3/orrLhUuCC6WwHh9JUkIV/wubpZ+vYOf2CedjOZupgcQcx96Kwy6UVjdNDWGxsEw0jXWXZUQlYnmRqA==", "dependencies": { - "Microsoft.Extensions.Configuration": "9.0.0", - "Microsoft.Extensions.Options": "9.0.0", - "OpenTelemetry.Api.ProviderBuilderExtensions": "[1.11.1, 2.0.0)" + "Microsoft.Extensions.Configuration": "8.0.0", + "Microsoft.Extensions.Options": "8.0.0", + "OpenTelemetry.Api.ProviderBuilderExtensions": "[1.13.1, 2.0.0)" } }, "PolySharp": { @@ -594,29 +576,29 @@ }, "OpenTelemetry": { "type": "Transitive", - "resolved": "1.11.1", - "contentHash": "F+HBI2bE7RKmb8Bj0kBtZIVzCfpTe1ZyY6kYP/jny1+9oq7IdBnNsVXZlPev9OqQzRp3iXpJ1UsnN1YOEwdtkQ==", + "resolved": "1.13.1", + "contentHash": "rrM2NlZ0Xla2Ar8zzU09n+HoLFq8b+Kjx7vrmR0tdYfWLYWGNcXHOITXUiyaK8RrFEceMEoF45VBsaf4QPmKcg==", "dependencies": { "Microsoft.Extensions.Diagnostics.Abstractions": "9.0.0", "Microsoft.Extensions.Logging.Configuration": "9.0.0", - "OpenTelemetry.Api.ProviderBuilderExtensions": "1.11.1" + "OpenTelemetry.Api.ProviderBuilderExtensions": "1.13.1" } }, "OpenTelemetry.Api": { "type": "Transitive", - "resolved": "1.11.1", - "contentHash": "KaBjGMqrqQv41mIkvPUvmAG7yxDlI6qchKhjXlOF3ZwsdcRRLrdrkiDLIJ90iZgUoKVdP8fE1fCri9nc+ug0Cg==", + "resolved": "1.13.1", + "contentHash": "tieglRERo7Rgu8oE8aamnuXCMPEW5fXIqO5ngTMCNk9pOEXanc0SdQ86ZAD1goNiGcjWHn+P3WMZp0FZSJgCoQ==", "dependencies": { "System.Diagnostics.DiagnosticSource": "9.0.0" } }, "OpenTelemetry.Api.ProviderBuilderExtensions": { "type": "Transitive", - "resolved": "1.11.1", - "contentHash": "vMdNMQeW55jXIa/Kybec/br6jC+rWybniTi6DCW5lz1kGghKso+J+FC3uBgiq0/pTqusfeDbO5PEHGM/r5z8Ow==", + "resolved": "1.13.1", + "contentHash": "x8QXMsrIyp+XzDUFQAM+C4upAPNbwaBIPjTWEoonziWAav6weS8OxsMKrE4wz7Zly8ATlsoxk0mWZ+PHO3Wg0w==", "dependencies": { "Microsoft.Extensions.DependencyInjection.Abstractions": "9.0.0", - "OpenTelemetry.Api": "1.11.1" + "OpenTelemetry.Api": "1.13.1" } }, "System.Diagnostics.DiagnosticSource": { diff --git a/Speckle.Rhino-importer.slnx b/Speckle.Rhino-importer.slnx index 5adab161d..a49671c2a 100644 --- a/Speckle.Rhino-importer.slnx +++ b/Speckle.Rhino-importer.slnx @@ -22,21 +22,16 @@ - + - + - - - - - - + diff --git a/todos.md b/todos.md new file mode 100644 index 000000000..b7945f4da --- /dev/null +++ b/todos.md @@ -0,0 +1,11 @@ +# TODOs + +## Packfile send route for importers + +Enable the importer flow to use the packfile send path (`SendViaPackfile` via `SendPipeline` + `IRootContinuousTraversalBuilder`). In this route, the server creates the version at the end of its processing job, so the job processor must skip calling `Ingestion.Complete()`. + +The `null` root object ID signals "server handles completion": + +1. **`Importers/Rhino/Speckle.Importers.Rhino/Internal/Sender.cs`**: When packfile route is used, return `null` for `RootId` (e.g. `SerializeProcessResults` won't have one — write `RootObjectId = null` in `Program.cs:61`). +2. **`Importers/Rhino/Speckle.Importers.JobProcessor/JobHandlers/RhinoJobHandler.cs` (line 69-73)**: Stop treating `null` `RootObjectId` as an error. Null means "server handles it", not failure. +3. **`Importers/Rhino/Speckle.Importers.JobProcessor/JobProcessor.cs` (line 189-192)**: Make `rootObjectId` nullable, skip `ReportSuccess()` when null.