feat(api)!: Implement new packfile based sends via SendPipline (aka DuckDB changes) (#1277)

* Dim/quack lets go (#1275)

* Add model ingestion to sharp connectors

* correct ingestion message

* Progress

* grasshopper

* GH exception messages

* fix GH

* file names

* revit file name

* grasshopper file names

* etabs file names

* delete tests

* tekla maybe

* ingestion  scope

* bad boolean logic

* Longer TimeSpan

* wip upload pipe

* 10s

* passthrough ingestion id

* happy hack time: prevent ingestion completion

this is handled server-side in the processing logic.

* add packfile send endpoint detection and routing

Route to SendViaPackfile when the server supports the upload-signing
endpoint (POST probe, 404 = unsupported) and a continuous traversal
builder is registered.

* Adds Continuous Traversal Builder

Introduces a Continuous Traversal Builder to manage the conversion and processing of Revit elements within a Send Pipeline.

---------

Co-authored-by: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com>

* feat(api): DI Refactor for Duck DB + Gergo's API endpoint changes (#1282)

* Di

* undo accidental change

* Feat (duck): dui ingestion updates post upload (#1295)

* Pass optional ingestion id to DUI

* Make ingestion id null for the SendViaIngestion, see the note :)

* feat!: Duckdev progress reporitng (#1296)

* Di

* throwaway from laptop

* Progress reporting

* Use matching logger

* Revit and revert rhino unpacker progress

* more revertion

* make pr even cleaner

* and this one

* fix build issues with other connectors

* SDK nuget (#1299)

* Bump to 3.14.0-alpha.2

* Feat(duck): grasshopper (#1297)

* Duck x Grasshopper - who would win?

* Fix registration for new builder

* missing imports

* return version id grasshopper

* Align sync resource to sync

---------

Co-authored-by: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com>

* Bump SDK

* feat(importer): rhino file importer changes for packfile (#1301)

* rhino importer changes

* correct deps

* Bump SDK

* Fix build issues

* ditto

* Fix build issue

* Lower standards

* Fix build

* feat: duck for acad, civil, navis, tekla, etabs (#1300)

* duck: acad, civil, etabs, tekla, navis and bump channels to 10.0.0

* notes

* fix conflicts

* more conflicts

* Ready for testing

* fix(duck): Fix send caching (#1302)

* potential fix

* undo-rhino chnages

* fix xml comment

* amended comment

* revit

* Fix build

* Aligned converting message

* fix: reoccurring object references

* Bump sdk and resolve merge conflict issues

* Merge pull request #1317 from specklesystems/jrm/importer-tracing

feat(otel): Tracing and OTEL changes for Rhino importer

* Fix revit linked model progress (#1312)

* Revert otel packages

* bump SDK

* Trace unpacking groups

* Align trace context nullability with app

* Disable send caching in Navisworks

* comments

* Update FileimportPayload.cs

* fix using directive

---------

Co-authored-by: Jedd Morgan <45512892+JR-Morgan@users.noreply.github.com>

* Fix merge conflicts

---------

Co-authored-by: Dimitrie Stefanescu <didimitrie@gmail.com>
Co-authored-by: Oğuzhan Koral <45078678+oguzhankoral@users.noreply.github.com>
Co-authored-by: Björn Steinhagen <88777268+bjoernsteinhagen@users.noreply.github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Sebastian Witt <sebastian.witt@rwth-aachen.de>
This commit is contained in:
Jedd Morgan
2026-04-08 10:07:56 +01:00
committed by GitHub
parent 38b9415e81
commit 7860c44f4e
102 changed files with 2644 additions and 435 deletions
+1 -1
View File
@@ -22,4 +22,4 @@ coverage.xml
output/
Images/Thumbs.db
.claude
.claude/
+1 -1
View File
@@ -1,4 +1,4 @@
CA1502: 25
CA1501: 5
CA1506(Method): 50
CA1506(Method): 60
CA1506(Type): 95
@@ -18,6 +18,10 @@ public static class AutocadConnectorModule
// Send
serviceCollection.LoadSend();
serviceCollection.AddScoped<IRootObjectBuilder<AutocadRootObject>, AutocadRootObjectBuilder>();
serviceCollection.AddScoped<
IRootContinuousTraversalBuilder<AutocadRootObject>,
AutocadContinuousTraversalBuilder
>();
// Receive
serviceCollection.LoadReceive();
@@ -1,9 +1,9 @@
using Autodesk.AutoCAD.Colors;
using Microsoft.Extensions.Logging;
using Speckle.Connectors.Common.Operations;
using Speckle.InterfaceGenerator;
using Speckle.Sdk;
using Speckle.Sdk.Models.Proxies;
using Speckle.Sdk.Pipelines.Progress;
using AutocadColor = Autodesk.AutoCAD.Colors.Color;
namespace Speckle.Connectors.Autocad.HostApp;
@@ -5,7 +5,6 @@ using Microsoft.Extensions.Logging;
using Speckle.Connectors.Autocad.HostApp.Extensions;
using Speckle.Connectors.Common.Conversion;
using Speckle.Connectors.Common.Instances;
using Speckle.Connectors.Common.Operations;
using Speckle.Converters.Autocad;
using Speckle.Converters.Common;
using Speckle.DoubleNumerics;
@@ -16,6 +15,7 @@ using Speckle.Sdk.Dependencies;
using Speckle.Sdk.Models;
using Speckle.Sdk.Models.Collections;
using Speckle.Sdk.Models.Instances;
using Speckle.Sdk.Pipelines.Progress;
using AutocadColor = Autodesk.AutoCAD.Colors.Color;
namespace Speckle.Connectors.Autocad.HostApp;
@@ -3,12 +3,12 @@ using Autodesk.AutoCAD.DatabaseServices;
using Autodesk.AutoCAD.GraphicsInterface;
using Microsoft.Extensions.Logging;
using Speckle.Connectors.Common.Conversion;
using Speckle.Connectors.Common.Operations;
using Speckle.InterfaceGenerator;
using Speckle.Objects.Other;
using Speckle.Sdk;
using Speckle.Sdk.Common;
using Speckle.Sdk.Models;
using Speckle.Sdk.Pipelines.Progress;
using Material = Autodesk.AutoCAD.DatabaseServices.Material;
using RenderMaterial = Speckle.Objects.Other.RenderMaterial;
@@ -12,6 +12,7 @@ using Speckle.Sdk.Dependencies;
using Speckle.Sdk.Models;
using Speckle.Sdk.Models.Collections;
using Speckle.Sdk.Models.Instances;
using Speckle.Sdk.Pipelines.Progress;
using AutocadColor = Autodesk.AutoCAD.Colors.Color;
namespace Speckle.Connectors.Autocad.Operations.Receive;
@@ -0,0 +1,197 @@
using System.Diagnostics.CodeAnalysis;
using Autodesk.AutoCAD.DatabaseServices;
using Microsoft.Extensions.Logging;
using Speckle.Connectors.Autocad.HostApp;
using Speckle.Connectors.Common.Builders;
using Speckle.Connectors.Common.Caching;
using Speckle.Connectors.Common.Conversion;
using Speckle.Connectors.Common.Extensions;
using Speckle.Connectors.Common.Operations;
using Speckle.Converters.Common;
using Speckle.Sdk;
using Speckle.Sdk.Logging;
using Speckle.Sdk.Models;
using Speckle.Sdk.Models.Collections;
using Speckle.Sdk.Models.Instances;
using Speckle.Sdk.Pipelines.Progress;
using Speckle.Sdk.Pipelines.Send;
namespace Speckle.Connectors.Autocad.Operations.Send;
/// <summary>
/// Abstract base class for AutoCAD continuous traversal builders that stream objects through a
/// <see cref="SendPipeline"/> for packfile-based uploads. Same conversion logic as
/// <see cref="AutocadRootObjectBaseBuilder"/>, but processes elements through the pipeline.
/// </summary>
public abstract class AutocadContinuousTraversalBaseBuilder : IRootContinuousTraversalBuilder<AutocadRootObject>
{
private readonly IRootToSpeckleConverter _converter;
private readonly string[] _documentPathSeparator = ["\\"];
private readonly ISendConversionCache _sendConversionCache;
private readonly AutocadInstanceUnpacker _instanceUnpacker;
private readonly AutocadMaterialUnpacker _materialUnpacker;
private readonly AutocadColorUnpacker _colorUnpacker;
private readonly AutocadGroupUnpacker _groupUnpacker;
private readonly ILogger<AutocadRootObjectBuilder> _logger;
private readonly ISdkActivityFactory _activityFactory;
protected AutocadContinuousTraversalBaseBuilder(
IRootToSpeckleConverter converter,
ISendConversionCache sendConversionCache,
AutocadInstanceUnpacker instanceObjectManager,
AutocadMaterialUnpacker materialUnpacker,
AutocadColorUnpacker colorUnpacker,
AutocadGroupUnpacker groupUnpacker,
ILogger<AutocadRootObjectBuilder> logger,
ISdkActivityFactory activityFactory
)
{
_converter = converter;
_sendConversionCache = sendConversionCache;
_instanceUnpacker = instanceObjectManager;
_materialUnpacker = materialUnpacker;
_colorUnpacker = colorUnpacker;
_groupUnpacker = groupUnpacker;
_logger = logger;
_activityFactory = activityFactory;
}
[SuppressMessage(
"Maintainability",
"CA1506:Avoid excessive class coupling",
Justification = """
It is already simplified but has many different references since it is a builder. Do not know can we simplify it now.
Later we might consider to refactor proxies from one proxy manager? but we do not know the shape of it all potential
proxy classes yet. So I'm supressing this one now!!!
"""
)]
public async Task<RootObjectBuilderResult> Build(
IReadOnlyList<AutocadRootObject> objects,
string projectId,
SendPipeline sendPipeline,
IProgress<CardProgress> onOperationProgressed,
CancellationToken cancellationToken
)
{
// 0 - Init the root
Collection root =
new()
{
name = Application
.DocumentManager.CurrentDocument.Name.Split(_documentPathSeparator, StringSplitOptions.None)
.Reverse()
.First()
};
Document doc = Application.DocumentManager.CurrentDocument;
using Transaction tr = doc.Database.TransactionManager.StartTransaction();
// 1 - Unpack the instances
var (atomicObjects, _, instanceProxies, instanceDefinitionProxies) = _instanceUnpacker.UnpackSelection(objects);
root[ProxyKeys.INSTANCE_DEFINITION] = instanceDefinitionProxies;
// 2 - Unpack the groups
root[ProxyKeys.GROUP] = _groupUnpacker.UnpackGroups(atomicObjects);
using (var _ = _activityFactory.Start("Converting objects"))
{
// 3 - Convert atomic objects and process through pipeline
List<LayerTableRecord> usedAcadLayers = new();
List<SendConversionResult> results = new();
int count = 0;
foreach (var (entity, applicationId) in atomicObjects)
{
cancellationToken.ThrowIfCancellationRequested();
(Collection objectCollection, LayerTableRecord? autocadLayer) = CreateObjectCollection(entity, tr);
if (autocadLayer is not null)
{
usedAcadLayers.Add(autocadLayer);
root.elements.Add(objectCollection);
}
var result = await ConvertAutocadEntity(
entity,
applicationId,
objectCollection,
instanceProxies,
projectId,
sendPipeline
);
results.Add(result);
onOperationProgressed.Report(
new($"Converting objects... ({count:N0} / {atomicObjects.Count:N0})", (double)++count / atomicObjects.Count)
);
}
if (results.All(x => x.Status == Status.ERROR))
{
throw new SpeckleException("Failed to convert all objects.");
}
// 4 - Unpack the render material proxies
root[ProxyKeys.RENDER_MATERIAL] = _materialUnpacker.UnpackMaterials(atomicObjects, usedAcadLayers);
// 5 - Unpack the color proxies
root[ProxyKeys.COLOR] = _colorUnpacker.UnpackColors(atomicObjects, usedAcadLayers);
// add any additional properties (most likely from verticals)
AddAdditionalProxiesToRoot(root);
// Process root collection and wait for all uploads
await sendPipeline.Process(root);
await sendPipeline.WaitForUpload();
return new RootObjectBuilderResult(root, results);
}
}
public virtual (Collection, LayerTableRecord?) CreateObjectCollection(Entity entity, Transaction tr)
{
return (new(), null);
}
public virtual void AddAdditionalProxiesToRoot(Collection rootCollection)
{
return;
}
private async Task<SendConversionResult> ConvertAutocadEntity(
Entity entity,
string applicationId,
Collection collectionHost,
IReadOnlyDictionary<string, InstanceProxy> instanceProxies,
string projectId,
SendPipeline sendPipeline
)
{
string sourceType = entity.GetType().ToString();
try
{
Base converted;
if (entity is BlockReference && instanceProxies.TryGetValue(applicationId, out InstanceProxy? instanceProxy))
{
converted = instanceProxy;
}
else if (_sendConversionCache.TryGetValue(projectId, applicationId, out ObjectReference? value))
{
converted = value;
}
else
{
converted = _converter.Convert(entity);
converted.applicationId = applicationId;
}
// NOTE: this is the main part that differentiate from the main root object builder
var reference = await sendPipeline.Process(converted).ConfigureAwait(false);
collectionHost.elements.Add(reference);
return new(Status.SUCCESS, applicationId, sourceType, reference);
}
catch (Exception ex) when (!ex.IsFatal())
{
_logger.LogSendConversionError(ex, sourceType);
return new(Status.ERROR, applicationId, sourceType, null, ex);
}
}
}
@@ -0,0 +1,46 @@
using Autodesk.AutoCAD.DatabaseServices;
using Microsoft.Extensions.Logging;
using Speckle.Connectors.Autocad.HostApp;
using Speckle.Connectors.Common.Caching;
using Speckle.Converters.Common;
using Speckle.Sdk.Logging;
using Speckle.Sdk.Models.Collections;
namespace Speckle.Connectors.Autocad.Operations.Send;
public sealed class AutocadContinuousTraversalBuilder : AutocadContinuousTraversalBaseBuilder
{
private readonly AutocadLayerUnpacker _layerUnpacker;
public AutocadContinuousTraversalBuilder(
AutocadLayerUnpacker layerUnpacker,
IRootToSpeckleConverter converter,
ISendConversionCache sendConversionCache,
AutocadInstanceUnpacker instanceObjectManager,
AutocadMaterialUnpacker materialUnpacker,
AutocadColorUnpacker colorUnpacker,
AutocadGroupUnpacker groupUnpacker,
ILogger<AutocadRootObjectBuilder> logger,
ISdkActivityFactory activityFactory
)
: base(
converter,
sendConversionCache,
instanceObjectManager,
materialUnpacker,
colorUnpacker,
groupUnpacker,
logger,
activityFactory
)
{
_layerUnpacker = layerUnpacker;
}
public override (Collection, LayerTableRecord?) CreateObjectCollection(Entity entity, Transaction tr)
{
Layer layer = _layerUnpacker.GetOrCreateSpeckleLayer(entity, tr, out LayerTableRecord? autocadLayer);
return (layer, autocadLayer);
}
}
@@ -16,6 +16,7 @@ using Speckle.Sdk.Logging;
using Speckle.Sdk.Models;
using Speckle.Sdk.Models.Collections;
using Speckle.Sdk.Models.Instances;
using Speckle.Sdk.Pipelines.Progress;
namespace Speckle.Connectors.Autocad.Operations.Send;
@@ -41,6 +41,8 @@
<Compile Include="$(MSBuildThisFileDirectory)Operations\Receive\AutocadHostObjectBaseBuilder.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Receive\AutocadHostObjectBuilder.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\AutocadRootObject.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\AutocadContinuousTraversalBaseBuilder.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\AutocadContinuousTraversalBuilder.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\AutocadRootObjectBaseBuilder.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\AutocadRootObjectBuilder.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Plugin\AutocadRibbon.cs" />
@@ -22,6 +22,10 @@ public static class Civil3dConnectorModule
// add send
serviceCollection.LoadSend();
serviceCollection.AddScoped<IRootObjectBuilder<AutocadRootObject>, Civil3dRootObjectBuilder>();
serviceCollection.AddScoped<
IRootContinuousTraversalBuilder<AutocadRootObject>,
Civil3dContinuousTraversalBuilder
>();
serviceCollection.AddSingleton<IBinding, Civil3dSendBinding>();
// add receive
@@ -0,0 +1,57 @@
using Autodesk.AutoCAD.DatabaseServices;
using Microsoft.Extensions.Logging;
using Speckle.Connectors.Autocad.HostApp;
using Speckle.Connectors.Autocad.Operations.Send;
using Speckle.Connectors.Common.Caching;
using Speckle.Connectors.Common.Operations;
using Speckle.Converters.Civil3dShared.ToSpeckle;
using Speckle.Converters.Common;
using Speckle.Sdk.Logging;
using Speckle.Sdk.Models.Collections;
namespace Speckle.Connectors.Civil3dShared.Operations.Send;
public sealed class Civil3dContinuousTraversalBuilder : AutocadContinuousTraversalBaseBuilder
{
private readonly AutocadLayerUnpacker _layerUnpacker;
private readonly PropertySetDefinitionHandler _propertySetDefinitionHandler;
public Civil3dContinuousTraversalBuilder(
AutocadLayerUnpacker layerUnpacker,
PropertySetDefinitionHandler propertySetDefinitionHandler,
IRootToSpeckleConverter converter,
ISendConversionCache sendConversionCache,
AutocadInstanceUnpacker instanceObjectManager,
AutocadMaterialUnpacker materialUnpacker,
AutocadColorUnpacker colorUnpacker,
AutocadGroupUnpacker groupUnpacker,
ILogger<AutocadRootObjectBuilder> logger,
ISdkActivityFactory activityFactory
)
: base(
converter,
sendConversionCache,
instanceObjectManager,
materialUnpacker,
colorUnpacker,
groupUnpacker,
logger,
activityFactory
)
{
_layerUnpacker = layerUnpacker;
_propertySetDefinitionHandler = propertySetDefinitionHandler;
}
public override (Collection, LayerTableRecord?) CreateObjectCollection(Entity entity, Transaction tr)
{
Layer layer = _layerUnpacker.GetOrCreateSpeckleLayer(entity, tr, out LayerTableRecord? autocadLayer);
return (layer, autocadLayer);
}
public override void AddAdditionalProxiesToRoot(Collection rootObject)
{
rootObject[ProxyKeys.PROPERTYSET_DEFINITIONS] = _propertySetDefinitionHandler.Definitions;
}
}
@@ -13,6 +13,7 @@
<Compile Include="$(MSBuildThisFileDirectory)DependencyInjection\Civil3dConnectorModule.cs" />
<Compile Include="$(MSBuildThisFileDirectory)HostApp\PropertySetBaker.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Receive\Civil3dHostObjectBuilder.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\Civil3dContinuousTraversalBuilder.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\Civil3dRootObjectBuilder.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Bindings\Civil3dSendBinding.cs" />
</ItemGroup>
@@ -0,0 +1,211 @@
using Microsoft.Extensions.Logging;
using Speckle.Connectors.Common.Builders;
using Speckle.Connectors.Common.Conversion;
using Speckle.Connectors.Common.Operations;
using Speckle.Connectors.CSiShared.HostApp;
using Speckle.Connectors.CSiShared.HostApp.Helpers;
using Speckle.Connectors.CSiShared.Utils;
using Speckle.Converters.Common;
using Speckle.Converters.CSiShared;
using Speckle.Converters.CSiShared.Extensions;
using Speckle.Converters.CSiShared.Utils;
using Speckle.Sdk;
using Speckle.Sdk.Logging;
using Speckle.Sdk.Models;
using Speckle.Sdk.Models.Collections;
using Speckle.Sdk.Pipelines.Progress;
using Speckle.Sdk.Pipelines.Send;
namespace Speckle.Connectors.CSiShared.Builders;
/// <summary>
/// Continuous traversal builder for CSi that streams objects through a <see cref="SendPipeline"/>
/// for packfile-based uploads. Same conversion logic as <see cref="CsiRootObjectBuilder"/>.
/// </summary>
public class CsiContinuousTraversalBuilder : IRootContinuousTraversalBuilder<ICsiWrapper>
{
private readonly IRootToSpeckleConverter _rootToSpeckleConverter;
private readonly IConverterSettingsStore<CsiConversionSettings> _converterSettings;
private readonly CsiSendCollectionManager _sendCollectionManager;
private readonly IMaterialUnpacker _materialUnpacker;
private readonly ISectionUnpacker _sectionUnpacker;
private readonly ILogger<CsiRootObjectBuilder> _logger;
private readonly ISdkActivityFactory _activityFactory;
private readonly ICsiApplicationService _csiApplicationService;
private readonly AnalysisResultsExtractor _analysisResultsExtractor;
public CsiContinuousTraversalBuilder(
IRootToSpeckleConverter rootToSpeckleConverter,
IConverterSettingsStore<CsiConversionSettings> converterSettings,
CsiSendCollectionManager sendCollectionManager,
IMaterialUnpacker materialUnpacker,
ISectionUnpacker sectionUnpacker,
ILogger<CsiRootObjectBuilder> logger,
ISdkActivityFactory activityFactory,
ICsiApplicationService csiApplicationService,
AnalysisResultsExtractor analysisResultsExtractor
)
{
_converterSettings = converterSettings;
_sendCollectionManager = sendCollectionManager;
_materialUnpacker = materialUnpacker;
_sectionUnpacker = sectionUnpacker;
_rootToSpeckleConverter = rootToSpeckleConverter;
_logger = logger;
_activityFactory = activityFactory;
_csiApplicationService = csiApplicationService;
_analysisResultsExtractor = analysisResultsExtractor;
}
public async Task<RootObjectBuilderResult> Build(
IReadOnlyList<ICsiWrapper> csiObjects,
string projectId,
SendPipeline sendPipeline,
IProgress<CardProgress> onOperationProgressed,
CancellationToken cancellationToken
)
{
using var activity = _activityFactory.Start("Build");
string modelFileName = _csiApplicationService.SapModel.GetModelFilename(false) ?? "Unnamed model";
(string forceUnit, string tempUnit) = GetForceAndTemperatureUnits();
Collection rootObjectCollection =
new()
{
name = modelFileName,
["units"] = _converterSettings.Current.SpeckleUnits,
["forceUnits"] = forceUnit,
["temperatureUnits"] = tempUnit
};
List<SendConversionResult> results = new(csiObjects.Count);
int count = 0;
using (var _ = _activityFactory.Start("Convert all"))
{
foreach (ICsiWrapper csiObject in csiObjects)
{
cancellationToken.ThrowIfCancellationRequested();
var result = await ConvertCsiObject(csiObject, rootObjectCollection, sendPipeline);
results.Add(result);
count++;
onOperationProgressed.Report(
new($"Converting objects... ({count:N0} / {csiObjects.Count:N0})", (double)count / csiObjects.Count)
);
await Task.Yield();
}
}
if (results.All(x => x.Status == Status.ERROR))
{
throw new SpeckleException("Failed to convert all objects");
}
using (var _ = _activityFactory.Start("Process Proxies"))
{
rootObjectCollection[ProxyKeys.MATERIAL] = _materialUnpacker.UnpackMaterials().ToList();
rootObjectCollection[ProxyKeys.SECTION] = _sectionUnpacker.UnpackSections().ToList();
}
// Extract analysis results (if applicable)
var objectSelectionSummary = GetObjectSummary(csiObjects);
var selectedCasesAndCombinations = _converterSettings.Current.SelectedLoadCasesAndCombinations;
var requestedResultTypes = _converterSettings.Current.SelectedResultTypes;
if (selectedCasesAndCombinations?.Count > 0)
{
if (requestedResultTypes == null || requestedResultTypes.Count == 0)
{
throw new SpeckleException(
"Adjust publish settings - no result type input for the requested load cases and combinations"
);
}
if (!_csiApplicationService.SapModel.GetModelIsLocked())
{
throw new SpeckleException("Model unlocked, no access to analysis results");
}
try
{
var analysisResults = _analysisResultsExtractor.ExtractAnalysisResults(
selectedCasesAndCombinations,
requestedResultTypes,
objectSelectionSummary
);
rootObjectCollection[RootKeys.ANALYSIS_RESULTS] = analysisResults;
}
catch (Exception ex)
{
throw new SpeckleException("Analysis result extraction failed", ex);
}
}
// Process root collection and wait for all uploads
await sendPipeline.Process(rootObjectCollection);
await sendPipeline.WaitForUpload();
return new RootObjectBuilderResult(rootObjectCollection, results);
}
private async Task<SendConversionResult> ConvertCsiObject(
ICsiWrapper csiObject,
Collection typeCollection,
SendPipeline sendPipeline
)
{
string sourceType = csiObject.ObjectName;
string applicationId = csiObject switch
{
CsiJointWrapper jointWrapper => jointWrapper.GetSpeckleApplicationId(_csiApplicationService.SapModel),
CsiFrameWrapper frameWrapper => frameWrapper.GetSpeckleApplicationId(_csiApplicationService.SapModel),
CsiCableWrapper cableWrapper => cableWrapper.GetSpeckleApplicationId(_csiApplicationService.SapModel),
CsiTendonWrapper tendonWrapper => tendonWrapper.ObjectName,
CsiShellWrapper shellWrapper => shellWrapper.GetSpeckleApplicationId(_csiApplicationService.SapModel),
CsiSolidWrapper solidWrapper => solidWrapper.GetSpeckleApplicationId(_csiApplicationService.SapModel),
CsiLinkWrapper linkWrapper => linkWrapper.GetSpeckleApplicationId(_csiApplicationService.SapModel),
_ => throw new ArgumentException($"Unsupported wrapper type: {csiObject.GetType()}", nameof(csiObject))
};
try
{
Base converted = _rootToSpeckleConverter.Convert(csiObject);
var collection = _sendCollectionManager.AddObjectCollectionToRoot(converted, typeCollection);
// NOTE: this is the main part that differentiate from the main root object builder
var reference = await sendPipeline.Process(converted).ConfigureAwait(false);
collection.elements.Add(reference);
return new(Status.SUCCESS, applicationId, sourceType, reference);
}
catch (NotImplementedException ex)
{
_logger.LogError(ex, "Failed to convert object {sourceType}", sourceType);
return new(Status.WARNING, applicationId, sourceType, null, ex);
}
catch (Exception ex) when (!ex.IsFatal())
{
_logger.LogError(ex, "Failed to convert object {sourceType}", sourceType);
return new(Status.ERROR, applicationId, sourceType, null, ex);
}
}
private Dictionary<ModelObjectType, List<string>> GetObjectSummary(IReadOnlyList<ICsiWrapper> csiObjects) =>
csiObjects
.GroupBy(csiObject => csiObject.ObjectType)
.ToDictionary(group => group.Key, group => group.Select(obj => obj.Name).ToList());
private (string, string) GetForceAndTemperatureUnits()
{
var forceUnit = eForce.NotApplicable;
var lengthUnit = eLength.NotApplicable;
var temperatureUnit = eTemperature.NotApplicable;
_converterSettings.Current.SapModel.GetDatabaseUnits_2(ref forceUnit, ref lengthUnit, ref temperatureUnit);
return (forceUnit.ToString(), temperatureUnit.ToString());
}
}
@@ -13,6 +13,7 @@ using Speckle.Sdk;
using Speckle.Sdk.Logging;
using Speckle.Sdk.Models;
using Speckle.Sdk.Models.Collections;
using Speckle.Sdk.Pipelines.Progress;
namespace Speckle.Connectors.CSiShared.Builders;
@@ -43,6 +43,7 @@ public static class ServiceRegistration
services.AddScoped<ISendFilter, CsiSharedSelectionFilter>();
services.AddScoped<CsiSendCollectionManager>();
services.AddScoped<IRootObjectBuilder<ICsiWrapper>, CsiRootObjectBuilder>();
services.AddScoped<IRootContinuousTraversalBuilder<ICsiWrapper>, CsiContinuousTraversalBuilder>();
services.AddScoped<SendOperation<ICsiWrapper>>();
services.AddScoped<CsiMaterialPropertyExtractor>();
@@ -29,6 +29,7 @@
<Compile Include="$(MSBuildThisFileDirectory)HostApp\Helpers\IApplicationSectionPropertyExtractor.cs" />
<Compile Include="$(MSBuildThisFileDirectory)HostApp\Helpers\ISectionPropertyExtractor.cs" />
<Compile Include="$(MSBuildThisFileDirectory)HostApp\Helpers\ISectionUnpacker.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\CsiContinuousTraversalBuilder.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\CsiRootObjectBuilder.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\Settings\ToSpeckleSettingsManager.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Plugin\CsiPluginBase.cs" />
@@ -5,7 +5,6 @@ using Speckle.Connector.Navisworks.Operations.Send.Filters;
using Speckle.Connector.Navisworks.Operations.Send.Settings;
using Speckle.Connector.Navisworks.Services;
using Speckle.Connectors.Common.Cancellation;
using Speckle.Connectors.Common.Operations;
using Speckle.Connectors.Common.Threading;
using Speckle.Connectors.DUI.Bindings;
using Speckle.Connectors.DUI.Bridge;
@@ -18,6 +17,7 @@ using Speckle.Converter.Navisworks.Services;
using Speckle.Converter.Navisworks.Settings;
using Speckle.Converters.Common;
using Speckle.Sdk.Common;
using Speckle.Sdk.Pipelines.Progress;
namespace Speckle.Connector.Navisworks.Bindings;
@@ -57,6 +57,7 @@ public static class NavisworksConnectorServiceRegistration
// Sending operations
serviceCollection.AddScoped<IRootObjectBuilder<NAV.ModelItem>, NavisworksRootObjectBuilder>();
serviceCollection.AddScoped<IRootContinuousTraversalBuilder<NAV.ModelItem>, NavisworksContinuousTraversalBuilder>();
serviceCollection.AddScoped<SendOperation<NAV.ModelItem>>();
serviceCollection.AddSingleton(DefaultTraversal.CreateTraversalFunc());
serviceCollection.AddSingleton<IOperationProgressManager, OperationProgressManager>();
@@ -0,0 +1,400 @@
using Microsoft.Extensions.Logging;
using Speckle.Connector.Navisworks.HostApp;
using Speckle.Connectors.Common.Builders;
using Speckle.Connectors.Common.Conversion;
using Speckle.Converter.Navisworks.Helpers;
using Speckle.Converter.Navisworks.Services;
using Speckle.Converter.Navisworks.Settings;
using Speckle.Converters.Common;
using Speckle.Objects.Data;
using Speckle.Sdk;
using Speckle.Sdk.Logging;
using Speckle.Sdk.Models;
using Speckle.Sdk.Models.Collections;
using Speckle.Sdk.Models.Instances;
using Speckle.Sdk.Pipelines.Progress;
using Speckle.Sdk.Pipelines.Send;
using static Speckle.Connector.Navisworks.Operations.Send.GeometryNodeMerger;
using static Speckle.Connectors.Common.Operations.ProxyKeys;
using static Speckle.Converter.Navisworks.Constants.InstanceConstants;
namespace Speckle.Connector.Navisworks.Operations.Send;
/// <summary>
/// Continuous traversal builder for Navisworks that streams objects through a <see cref="SendPipeline"/>
/// for packfile-based uploads. Same conversion/grouping logic as <see cref="NavisworksRootObjectBuilder"/>,
/// but processes final elements through the pipeline after all post-processing is complete.
/// </summary>
public class NavisworksContinuousTraversalBuilder(
IRootToSpeckleConverter rootToSpeckleConverter,
IConverterSettingsStore<NavisworksConversionSettings> converterSettings,
ILogger<NavisworksContinuousTraversalBuilder> logger,
ISdkActivityFactory activityFactory,
NavisworksMaterialUnpacker materialUnpacker,
NavisworksColorUnpacker colorUnpacker,
Speckle.Converter.Navisworks.Constants.Registers.IInstanceFragmentRegistry instanceRegistry,
IElementSelectionService elementSelectionService,
IUiUnitsCache uiUnitsCache
) : IRootContinuousTraversalBuilder<NAV.ModelItem>
{
private bool SkipNodeMerging { get; set; }
private bool DisableGroupingForInstanceTesting { get; set; }
public async Task<RootObjectBuilderResult> Build(
IReadOnlyList<NAV.ModelItem> navisworksModelItems,
string projectId,
SendPipeline sendPipeline,
IProgress<CardProgress> onOperationProgressed,
CancellationToken cancellationToken
)
{
#if DEBUG
SkipNodeMerging = false;
DisableGroupingForInstanceTesting = false;
#endif
using var activity = activityFactory.Start("Build");
ValidateInputs(navisworksModelItems, projectId, onOperationProgressed);
var rootCollection = InitializeRootCollection();
(Dictionary<string, Base?> convertedElements, List<SendConversionResult> conversionResults) =
await ConvertModelItemsAsync(navisworksModelItems, onOperationProgressed, cancellationToken);
ValidateConversionResults(conversionResults);
var groupedNodes = SkipNodeMerging ? [] : GroupSiblingGeometryNodes(navisworksModelItems);
var finalElements = BuildFinalElements(convertedElements, groupedNodes);
await AddProxiesToCollection(rootCollection, navisworksModelItems, groupedNodes);
AddInstanceDefinitionsToCollection(rootCollection, ref finalElements);
// Process each final element through the send pipeline
var processedElements = new List<Base>(finalElements.Count);
foreach (var element in finalElements)
{
cancellationToken.ThrowIfCancellationRequested();
// NOTE: this is the main part that differentiate from the main root object builder
var reference = await sendPipeline.Process(element).ConfigureAwait(false);
processedElements.Add(reference);
}
rootCollection.elements = processedElements;
// Process the root collection and wait for all uploads to complete
await sendPipeline.Process(rootCollection);
await sendPipeline.WaitForUpload();
return new RootObjectBuilderResult(rootCollection, conversionResults);
}
private static void ValidateInputs(
IReadOnlyList<NAV.ModelItem> navisworksModelItems,
string projectId,
IProgress<CardProgress> onOperationProgressed
)
{
if (!navisworksModelItems.Any())
{
throw new SpeckleException("No objects to convert");
}
if (navisworksModelItems == null)
{
throw new ArgumentNullException(nameof(navisworksModelItems));
}
if (onOperationProgressed == null || projectId == null)
{
throw new ArgumentNullException(
onOperationProgressed == null ? nameof(onOperationProgressed) : nameof(projectId)
);
}
}
private Collection InitializeRootCollection() =>
new()
{
name = NavisworksApp.ActiveDocument.Title ?? "Unnamed model",
["units"] = converterSettings.Current.Derived.SpeckleUnits
};
private Task<(Dictionary<string, Base?> converted, List<SendConversionResult> results)> ConvertModelItemsAsync(
IReadOnlyList<NAV.ModelItem> navisworksModelItems,
IProgress<CardProgress> onOperationProgressed,
CancellationToken cancellationToken
)
{
var results = new List<SendConversionResult>(navisworksModelItems.Count);
var convertedBases = new Dictionary<string, Base?>();
int processedCount = 0;
int totalCount = navisworksModelItems.Count;
foreach (var item in navisworksModelItems)
{
cancellationToken.ThrowIfCancellationRequested();
var converted = ConvertNavisworksItem(item, convertedBases);
results.Add(converted);
processedCount++;
onOperationProgressed.Report(new CardProgress("Converting", (double)processedCount / totalCount));
}
return Task.FromResult((convertedBases, results));
}
private static void ValidateConversionResults(List<SendConversionResult> results)
{
if (results.All(x => x.Status == Status.ERROR))
{
throw new SpeckleException("Failed to convert all objects.");
}
}
private List<Base> BuildFinalElements(
Dictionary<string, Base?> convertedBases,
Dictionary<string, List<NAV.ModelItem>> groupedNodes
)
{
var finalElements = new List<Base>();
var processedPaths = new HashSet<string>();
if (!DisableGroupingForInstanceTesting)
{
AddGroupedElements(finalElements, convertedBases, groupedNodes, processedPaths);
logger.LogInformation(
"After grouping: {grouped} paths processed, {elements} elements in collection",
processedPaths.Count,
finalElements.Count
);
}
else
{
logger.LogInformation("Grouping disabled for instance testing");
}
if (converterSettings.Current.User.PreserveModelHierarchy)
{
logger.LogInformation("Building hierarchy (PreserveModelHierarchy=true)");
var hierarchyBuilder = new NavisworksHierarchyBuilder(
convertedBases,
rootToSpeckleConverter,
elementSelectionService
);
return hierarchyBuilder.BuildHierarchy();
}
logger.LogInformation("Adding remaining elements (flat mode)");
AddRemainingElements(finalElements, convertedBases, processedPaths);
logger.LogInformation("Final elements count: {count}", finalElements.Count);
return finalElements;
}
private void AddGroupedElements(
List<Base> finalElements,
Dictionary<string, Base?> convertedBases,
Dictionary<string, List<NAV.ModelItem>> groupedNodes,
HashSet<string> processedPaths
)
{
foreach (var group in groupedNodes)
{
var siblingBases = new List<Base>(group.Value.Count);
foreach (var itemPath in group.Value.Select(elementSelectionService.GetModelItemPath))
{
processedPaths.Add(itemPath);
if (convertedBases.TryGetValue(itemPath, out var convertedBase) && convertedBase != null)
{
siblingBases.Add(convertedBase);
}
}
if (siblingBases.Count > 0)
{
finalElements.Add(CreateNavisworksObject(group.Key, siblingBases));
}
}
}
private void AddRemainingElements(
List<Base> finalElements,
Dictionary<string, Base?> convertedBases,
HashSet<string> processedPaths
)
{
foreach (var kvp in convertedBases.Where(kvp => !processedPaths.Contains(kvp.Key)))
{
switch (kvp.Value)
{
case null:
continue;
case Collection collection:
finalElements.Add(collection);
break;
default:
if (CreateNavisworksObject(kvp.Value) is { } navisworksObject)
{
finalElements.Add(navisworksObject);
}
break;
}
}
}
private (string name, string path) GetElementNameAndPath(string applicationId)
{
var modelItem = elementSelectionService.GetModelItemFromPath(applicationId);
var context = HierarchyHelper.ExtractContext(modelItem);
return (context.Name, context.Path);
}
private NavisworksObject CreateNavisworksObject(string groupKey, List<Base> siblingBases)
{
string cleanParentPath = ElementSelectionHelper.GetCleanPath(groupKey);
(string name, string path) = GetElementNameAndPath(cleanParentPath);
int estimatedCapacity = siblingBases.Sum(b => (b["displayValue"] as List<Base>)?.Count ?? 0);
var displayValues = new List<Base>(estimatedCapacity);
displayValues.AddRange(
siblingBases
.Where(sibling => sibling["displayValue"] is List<Base>)
.SelectMany(sibling => (List<Base>)sibling["displayValue"]!)
);
return new NavisworksObject
{
name = name,
displayValue = displayValues,
properties = siblingBases.First()["properties"] as Dictionary<string, object?> ?? [],
units = converterSettings.Current.Derived.SpeckleUnits,
applicationId = groupKey,
["path"] = path
};
}
private NavisworksObject? CreateNavisworksObject(Base convertedBase)
{
if (convertedBase.applicationId == null)
{
return null;
}
(string name, string path) = GetElementNameAndPath(convertedBase.applicationId);
var units = uiUnitsCache.Ensure();
return new NavisworksObject
{
name = name,
displayValue = convertedBase["displayValue"] as List<Base> ?? [],
properties = convertedBase["properties"] as Dictionary<string, object?> ?? [],
units = units.ToString(),
applicationId = convertedBase.applicationId,
["path"] = path
};
}
private Task AddProxiesToCollection(
Collection rootCollection,
IReadOnlyList<NAV.ModelItem> navisworksModelItems,
Dictionary<string, List<NAV.ModelItem>> groupedNodes
)
{
using var _ = activityFactory.Start("UnpackProxies");
var renderMaterials = materialUnpacker.UnpackRenderMaterial(navisworksModelItems, groupedNodes);
if (renderMaterials.Count > 0)
{
rootCollection[RENDER_MATERIAL] = renderMaterials;
}
var colors = colorUnpacker.UnpackColor(navisworksModelItems, groupedNodes);
if (colors.Count > 0)
{
rootCollection[COLOR] = colors;
}
return Task.CompletedTask;
}
private void AddInstanceDefinitionsToCollection(Collection rootCollection, ref List<Base> finalElements)
{
using var _ = activityFactory.Start("BuildInstanceDefinitions");
var allDefinitions = instanceRegistry.GetAllDefinitionGeometries();
if (allDefinitions.Count == 0)
{
logger.LogInformation("No instance definitions found - instancing may be disabled");
return;
}
logger.LogInformation("Building instance structure for {count} definition groups", allDefinitions.Count);
var instanceDefinitionProxies = new List<InstanceDefinitionProxy>(allDefinitions.Count);
int estimatedGeometryCount = allDefinitions.Sum(kvp => kvp.Value.Count);
var allDefinitionGeometries = new List<Base>(estimatedGeometryCount);
foreach (var kvp in allDefinitions)
{
var groupKey = kvp.Key;
var geometries = kvp.Value;
var groupKeyPath = groupKey.ToPathString();
var defProxy = new InstanceDefinitionProxy
{
name = $"Shared Geometry {groupKeyPath}",
objects = geometries.Select(g => g.applicationId ?? "").Where(id => !string.IsNullOrEmpty(id)).ToList(),
applicationId = $"{DEFINITION_ID_PREFIX}{groupKeyPath}",
maxDepth = 0
};
instanceDefinitionProxies.Add(defProxy);
allDefinitionGeometries.AddRange(geometries);
}
rootCollection[INSTANCE_DEFINITION] = instanceDefinitionProxies;
var geometryDefinitionsCollection = new Collection
{
name = "Geometry Definitions",
elements = allDefinitionGeometries
};
var objectCollection = new Collection { name = "", elements = finalElements };
finalElements = [geometryDefinitionsCollection, objectCollection];
logger.LogInformation(
"Added {proxyCount} instance definition proxies and {geomCount} definition geometries",
instanceDefinitionProxies.Count,
allDefinitionGeometries.Count
);
}
private SendConversionResult ConvertNavisworksItem(
NAV.ModelItem navisworksItem,
Dictionary<string, Base?> convertedBases
)
{
string applicationId = elementSelectionService.GetModelItemPath(navisworksItem);
string sourceType = navisworksItem.GetType().Name;
try
{
Base converted = rootToSpeckleConverter.Convert(navisworksItem);
convertedBases[applicationId] = converted;
return new SendConversionResult(Status.SUCCESS, applicationId, sourceType, converted);
}
catch (Exception ex) when (!ex.IsFatal())
{
logger.LogError(ex, "Failed to convert model item {id}", applicationId);
return new SendConversionResult(Status.ERROR, applicationId, "ModelItem", null, ex);
}
}
}
@@ -3,7 +3,6 @@ using Speckle.Connector.Navisworks.HostApp;
using Speckle.Connectors.Common.Builders;
using Speckle.Connectors.Common.Caching;
using Speckle.Connectors.Common.Conversion;
using Speckle.Connectors.Common.Operations;
using Speckle.Converter.Navisworks.Helpers;
using Speckle.Converter.Navisworks.Services;
using Speckle.Converter.Navisworks.Settings;
@@ -14,6 +13,7 @@ using Speckle.Sdk.Logging;
using Speckle.Sdk.Models;
using Speckle.Sdk.Models.Collections;
using Speckle.Sdk.Models.Instances;
using Speckle.Sdk.Pipelines.Progress;
using static Speckle.Connector.Navisworks.Operations.Send.GeometryNodeMerger;
using static Speckle.Connectors.Common.Operations.ProxyKeys;
using static Speckle.Converter.Navisworks.Constants.InstanceConstants;
@@ -24,6 +24,7 @@
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\Filters\SavedItemHelpers.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\GeometryNodeMerger.cs"/>
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\NavisworksHierarchyBuilder.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\NavisworksContinuousTraversalBuilder.cs"/>
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\NavisworksRootObjectBuilder.cs"/>
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\Settings\ConvertHiddenElementsSetting.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\Settings\IncludeInternalPropertiesSetting.cs"/>
@@ -65,6 +65,7 @@ public static class ServiceRegistration
serviceCollection.AddScoped<ViewUnpacker>();
serviceCollection.AddScoped<SendCollectionManager>();
serviceCollection.AddScoped<IRootObjectBuilder<DocumentToConvert>, RevitRootObjectBuilder>();
serviceCollection.AddScoped<IRootContinuousTraversalBuilder<DocumentToConvert>, RevitContinuousTraversalBuilder>();
serviceCollection.AddSingleton<ISendConversionCache, SendConversionCache>();
serviceCollection.AddSingleton<ToSpeckleSettingsManager>();
serviceCollection.AddSingleton<ToHostSettingsManager>();
@@ -4,7 +4,6 @@ using Autodesk.Revit.DB;
using Autodesk.Revit.DB.Structure;
using Microsoft.Extensions.Logging;
using Speckle.Connectors.Common.Conversion;
using Speckle.Connectors.Common.Operations;
using Speckle.Converters.Common;
using Speckle.Converters.Common.Objects;
using Speckle.Converters.RevitShared.Helpers;
@@ -18,6 +17,7 @@ using Speckle.Sdk.Models;
using Speckle.Sdk.Models.Collections;
using Speckle.Sdk.Models.GraphTraversal;
using Speckle.Sdk.Models.Instances;
using Speckle.Sdk.Pipelines.Progress;
using DB = Autodesk.Revit.DB;
using Document = Autodesk.Revit.DB.Document;
@@ -24,6 +24,7 @@ using Speckle.Sdk.Models;
using Speckle.Sdk.Models.Collections;
using Speckle.Sdk.Models.GraphTraversal;
using Speckle.Sdk.Models.Instances;
using Speckle.Sdk.Pipelines.Progress;
namespace Speckle.Connectors.Revit.Operations.Receive;
@@ -0,0 +1,318 @@
using System.Diagnostics.CodeAnalysis;
using Autodesk.Revit.DB;
using Microsoft.Extensions.Logging;
using Speckle.Connectors.Common.Builders;
using Speckle.Connectors.Common.Caching;
using Speckle.Connectors.Common.Conversion;
using Speckle.Connectors.Common.Extensions;
using Speckle.Connectors.Common.Operations;
using Speckle.Connectors.Common.Threading;
using Speckle.Connectors.DUI.Exceptions;
using Speckle.Connectors.DUI.Settings;
using Speckle.Connectors.Revit.HostApp;
using Speckle.Converters.Common;
using Speckle.Converters.RevitShared.Helpers;
using Speckle.Converters.RevitShared.Settings;
using Speckle.Sdk;
using Speckle.Sdk.Common;
using Speckle.Sdk.Models;
using Speckle.Sdk.Models.Collections;
using Speckle.Sdk.Pipelines.Progress;
using Speckle.Sdk.Pipelines.Send;
namespace Speckle.Connectors.Revit.Operations.Send;
public class RevitContinuousTraversalBuilder(
IRootToSpeckleConverter converter,
IConverterSettingsStore<RevitConversionSettings> converterSettings,
ISendConversionCache sendConversionCache,
ElementUnpacker elementUnpacker,
LevelUnpacker levelUnpacker,
ViewUnpacker viewUnpacker,
IThreadContext threadContext,
SendCollectionManager sendCollectionManager,
ILogger<RevitRootObjectBuilder> logger,
RevitToSpeckleCacheSingleton revitToSpeckleCacheSingleton,
LinkedModelHandler linkedModelHandler,
IConfigStore configStore
) : IRootContinuousTraversalBuilder<DocumentToConvert>
{
public async Task<RootObjectBuilderResult> Build(
IReadOnlyList<DocumentToConvert> documentElementContexts,
string projectId,
SendPipeline sendPipeline,
IProgress<CardProgress> onOperationProgressed,
CancellationToken cancellationToken
)
{
return await threadContext.RunOnMainAsync(
async () =>
await BuildMainThread(
documentElementContexts,
projectId,
sendPipeline,
onOperationProgressed,
cancellationToken
)
);
}
[SuppressMessage("Maintainability", "CA1502:Avoid excessive class coupling")]
[SuppressMessage("Maintainability", "CA1506:Avoid excessive class coupling")]
private async Task<RootObjectBuilderResult> BuildMainThread(
IReadOnlyList<DocumentToConvert> documentElementContexts,
string projectId,
SendPipeline sendPipeline,
IProgress<CardProgress> onOperationProgressed,
CancellationToken cancellationToken
)
{
var doc = converterSettings.Current.Document;
if (doc.IsFamilyDocument)
{
throw new SpeckleException("Family Environment documents are not supported.");
}
// init the root
Collection rootObject =
new() { name = converterSettings.Current.Document.PathName.Split('\\').Last().Split('.').First() };
rootObject["units"] = converterSettings.Current.SpeckleUnits;
var filteredDocumentsToConvert = new List<DocumentToConvert>();
bool sendWithLinkedModels = converterSettings.Current.SendLinkedModels;
List<SendConversionResult> results = new();
// Prepare linked model display names if needed
if (sendWithLinkedModels)
{
linkedModelHandler.PrepareLinkedModelNames(documentElementContexts);
}
foreach (var documentElementContext in documentElementContexts)
{
// add appropriate warnings for linked documents
if (documentElementContext.Doc.IsLinked && !sendWithLinkedModels)
{
results.Add(
new(
Status.WARNING,
documentElementContext.Doc.PathName,
typeof(RevitLinkInstance).ToString(),
null,
new SpeckleException("Enable linked model support from the settings to send this object")
)
);
continue;
}
// filter for valid elements
// if send linked models setting is disabled List<Elements> will be empty, and we won't enter foreach loop
var elementsInTransform = new List<Element>();
foreach (var el in documentElementContext.Elements)
{
if (el == null || el.Category == null)
{
continue;
}
elementsInTransform.Add(el);
}
// only add contexts with elements
if (elementsInTransform.Count > 0)
{
filteredDocumentsToConvert.Add(documentElementContext with { Elements = elementsInTransform });
}
}
// TODO: check the exception!!!!
if (filteredDocumentsToConvert.Count == 0)
{
throw new SpeckleSendFilterException("No objects were found. Please update your publish filter!");
}
// Unpack groups (& other complex data structures)
var atomicObjectsByDocumentAndTransform = new List<DocumentToConvert>();
var atomicObjectCount = 0;
foreach (var filteredDocumentToConvert in filteredDocumentsToConvert)
{
using (
converterSettings.Push(currentSettings => currentSettings with { Document = filteredDocumentToConvert.Doc })
)
{
var atomicObjects = elementUnpacker
.UnpackSelectionForConversion(filteredDocumentToConvert.Elements, filteredDocumentToConvert.Doc)
.ToList();
atomicObjectsByDocumentAndTransform.Add(filteredDocumentToConvert with { Elements = atomicObjects });
atomicObjectCount += atomicObjects.Count;
}
}
var count = 0;
var cacheHitCount = 0;
var skippedObjectCount = 0;
var config = configStore.GetConnectorConfig();
foreach (var atomicObjectByDocumentAndTransform in atomicObjectsByDocumentAndTransform)
{
string? modelDisplayName = null;
if (atomicObjectByDocumentAndTransform.Doc.IsLinked)
{
string id = linkedModelHandler.GetIdFromDocumentToConvert(atomicObjectByDocumentAndTransform);
linkedModelHandler.LinkedModelDisplayNames.TryGetValue(id, out modelDisplayName);
}
// here we do magic for changing the transform and the related document according to model. first one is always the main model.
using (
converterSettings.Push(currentSettings =>
currentSettings with
{
ReferencePointTransform = atomicObjectByDocumentAndTransform.Transform,
Document = atomicObjectByDocumentAndTransform.Doc,
}
)
)
{
var atomicObjects = atomicObjectByDocumentAndTransform.Elements;
foreach (Element revitElement in atomicObjects)
{
cancellationToken.ThrowIfCancellationRequested();
string applicationId = revitElement.UniqueId;
string sourceType = revitElement.GetType().Name;
try
{
if (!SupportedCategoriesUtils.IsSupportedCategory(revitElement.Category))
{
var cat = revitElement.Category != null ? revitElement.Category.Name : "No category";
results.Add(
new(
Status.WARNING,
revitElement.UniqueId,
cat,
null,
new SpeckleException($"Category {cat} is not supported.")
)
);
skippedObjectCount++;
continue;
}
Base converted;
bool hasTransform = atomicObjectByDocumentAndTransform.Transform != null;
// non-transformed elements can safely rely on cache
// TODO: Potential here to transform cached objects and NOT reconvert,
// TODO: we wont do !hasTransform here, and re-set application id before this
if (
!hasTransform
&& !config.DocumentChangeListeningDisabled //This is experimental
&& sendConversionCache.TryGetValue(projectId, applicationId, out ObjectReference? value)
)
{
converted = value;
cacheHitCount++;
}
// not in cache means we convert
else
{
// if it has a transform we append transform hash to the applicationId to distinguish the elements from other instances
if (hasTransform)
{
string transformHash = linkedModelHandler.GetTransformHash(
atomicObjectByDocumentAndTransform.Transform.NotNull()
);
applicationId = $"{applicationId}_t{transformHash}";
}
// normal conversions
converted = converter.Convert(revitElement);
converted.applicationId = applicationId;
}
// NOTE: this is the main part that differentiate from the main root object builder
var reference = await sendPipeline.Process(converted).ConfigureAwait(true);
var collection = sendCollectionManager.GetAndCreateObjectHostCollection(
revitElement,
rootObject,
sendWithLinkedModels,
modelDisplayName
);
collection.elements.Add(reference);
results.Add(new(Status.SUCCESS, applicationId, sourceType, reference));
}
catch (Exception ex) when (!ex.IsFatal())
{
logger.LogSendConversionError(ex, sourceType);
results.Add(new(Status.ERROR, applicationId, sourceType, null, ex));
}
count++;
onOperationProgressed.Report(
new($"Converting objects... ({count:N0} / {atomicObjectCount:N0})", (double)count / atomicObjectCount)
);
}
}
}
// if we ended up skipping everything, there is a reason for this, that users can diagnose themselves
// this can occur if a published view contains only unsupported objects or if user trying to ONLY send linked model
// docs but the setting is disabled
if (skippedObjectCount == atomicObjectCount)
{
throw new SpeckleException("No supported objects visible. Update publish filter or check publish settings.");
}
// this is, I suppose, fully on us?
if (results.All(x => x.Status == Status.ERROR))
{
throw new SpeckleException("Failed to convert all objects.");
}
// STEP 5: Unpack proxies to attach to root collection
var flatElements = atomicObjectsByDocumentAndTransform.SelectMany(t => t.Elements).ToList();
var idsAndSubElementIds = elementUnpacker.GetElementsAndSubelementIdsFromAtomicObjects(flatElements);
var renderMaterialProxies = revitToSpeckleCacheSingleton.GetRenderMaterialProxyListForObjects(idsAndSubElementIds);
rootObject[ProxyKeys.RENDER_MATERIAL] = renderMaterialProxies;
var levelProxies = levelUnpacker.Unpack(flatElements);
rootObject[ProxyKeys.LEVEL] = levelProxies;
rootObject[ProxyKeys.INSTANCE_DEFINITION] = revitToSpeckleCacheSingleton.GetInstanceDefinitionProxiesForObjects(
idsAndSubElementIds
);
rootObject.elements.Add(
new Collection()
{
elements = revitToSpeckleCacheSingleton.GetBaseObjectsForObjects(idsAndSubElementIds),
name = "definitionGeometry"
}
);
// STEP 6: Unpack all other objects to attach to root collection
List<Objects.Other.Camera> views = viewUnpacker.Unpack(converterSettings.Current.Document);
if (views.Count > 0)
{
rootObject[RootKeys.VIEW] = views;
}
// NOTE: these are currently not used anywhere, we'll skip them until someone calls for it back
// rootObject[ProxyKeys.PARAMETER_DEFINITIONS] = _parameterDefinitionHandler.Definitions;
// we want to store transform data for chosen reference point setting
if (converterSettings.Current.ReferencePointTransform is Transform transform)
{
var transformMatrix = ReferencePointHelper.CreateTransformDataForRootObject(transform);
rootObject[RootKeys.REFERENCE_POINT_TRANSFORM] = transformMatrix;
}
await sendPipeline.Process(rootObject);
await sendPipeline.WaitForUpload();
return new RootObjectBuilderResult(rootObject, results);
}
}
@@ -17,6 +17,7 @@ using Speckle.Sdk;
using Speckle.Sdk.Common;
using Speckle.Sdk.Models;
using Speckle.Sdk.Models.Collections;
using Speckle.Sdk.Pipelines.Progress;
namespace Speckle.Connectors.Revit.Operations.Send;
@@ -39,7 +40,7 @@ public class RevitRootObjectBuilder(
IReadOnlyList<DocumentToConvert> documentElementContexts,
string projectId,
IProgress<CardProgress> onOperationProgressed,
CancellationToken ct = default
CancellationToken ct
) =>
threadContext.RunOnMainAsync(
() => Task.FromResult(BuildSync(documentElementContexts, projectId, onOperationProgressed, ct))
@@ -60,6 +60,7 @@
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\Filters\RevitCategoriesFilter.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\Filters\RevitSelectionFilter.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\Filters\RevitViewsFilter.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\RevitContinuousTraversalBuilder.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\RevitRootObjectBuilder.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\Settings\LinkedModelsSetting.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\Settings\SendParameterNullOrEmptyStringsSetting.cs" />
@@ -8,7 +8,6 @@ using GrasshopperAsyncComponent;
using Rhino;
using Speckle.Connectors.Common;
using Speckle.Connectors.Common.Analytics;
using Speckle.Connectors.Common.Operations;
using Speckle.Connectors.Common.Operations.Receive;
using Speckle.Connectors.GrasshopperShared.HostApp;
using Speckle.Connectors.GrasshopperShared.Operations.Receive;
@@ -22,6 +21,7 @@ using Speckle.Sdk.Credentials;
using Speckle.Sdk.Models;
using Speckle.Sdk.Models.Collections;
using Speckle.Sdk.Models.Extensions;
using Speckle.Sdk.Pipelines.Progress;
namespace Speckle.Connectors.GrasshopperShared.Components.Operations.Receive;
@@ -3,7 +3,6 @@ using Microsoft.Extensions.DependencyInjection;
using Rhino;
using Speckle.Connectors.Common;
using Speckle.Connectors.Common.Analytics;
using Speckle.Connectors.Common.Operations;
using Speckle.Connectors.Common.Operations.Receive;
using Speckle.Connectors.GrasshopperShared.Components.BaseComponents;
using Speckle.Connectors.GrasshopperShared.HostApp;
@@ -16,6 +15,7 @@ using Speckle.Sdk.Api;
using Speckle.Sdk.Api.GraphQL.Models;
using Speckle.Sdk.Credentials;
using Speckle.Sdk.Models.Collections;
using Speckle.Sdk.Pipelines.Progress;
namespace Speckle.Connectors.GrasshopperShared.Components.Operations.Receive;
@@ -0,0 +1,56 @@
using Speckle.Sdk;
using Speckle.Sdk.Api;
using Speckle.Sdk.Api.GraphQL.Enums;
using Speckle.Sdk.Common;
namespace Speckle.Connectors.GrasshopperShared.Components.Operations.Send;
/// <summary>
/// Polls ingestion status via the SDK's GraphQL query API
/// and blocks until the ingestion reaches a terminal state (success/failed/cancelled).
/// </summary>
/// <remarks>
/// We use polling instead of subscriptions because GH components call WaitForIngestionCompletion
/// after SendViaPackfile returns — by that point the server may have already completed
/// the ingestion. Setting up a new WebSocket subscription is too slow to catch fast completions.
/// Polling with Ingestion.Get() is reliable regardless of timing.
/// </remarks>
public class IngestionTracker
{
private static readonly TimeSpan s_pollInterval = TimeSpan.FromSeconds(1);
public async Task<string> WaitForIngestionCompletion(
IClient client,
string projectId,
string ingestionId,
Action<string, double>? reportProgress,
string? reportProgressId,
CancellationToken cancellationToken
)
{
// NOTE: before start hating from this - read the class description
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
var ingestion = await client.Ingestion.Get(ingestionId, projectId, cancellationToken).ConfigureAwait(false);
var status = ingestion.statusData.status;
switch (status)
{
case ModelIngestionStatus.success:
return ingestion.statusData.versionId.NotNull();
case ModelIngestionStatus.failed:
throw new SpeckleException($"Server processing failed: {ingestion.statusData.progressMessage}");
case ModelIngestionStatus.cancelled:
throw new OperationCanceledException("Ingestion was cancelled by the server");
case ModelIngestionStatus.processing:
case ModelIngestionStatus.queued:
reportProgress?.Invoke(reportProgressId ?? "Server", 0);
break;
}
await Task.Delay(s_pollInterval, cancellationToken).ConfigureAwait(false);
}
}
}
@@ -20,6 +20,7 @@ using Speckle.Sdk.Api;
using Speckle.Sdk.Common;
using Speckle.Sdk.Credentials;
using Speckle.Sdk.Models.Extensions;
using Speckle.Sdk.Pipelines.Progress;
namespace Speckle.Connectors.GrasshopperShared.Components.Operations.Send;
@@ -503,10 +504,26 @@ public class SendComponentWorker : WorkerInstance<SendAsyncComponent>
});
using var scope = PriorityLoader.CreateScopeForActiveDocument();
var sendOperation = scope.ServiceProvider.GetRequiredService<SendOperation<SpeckleCollectionWrapperGoo>>();
(SendOperationResult result, string versionId) = await sendOperation
(SendOperationResult result, string versionId, string? ingestionId) = await sendOperation
.Send([rootCollectionWrapper], sendInfo, fileName, fileBytes, Parent.VersionMessage, progress, CancellationToken)
.ConfigureAwait(false);
if (ingestionId != null)
{
Parent.Message = "Remote processing";
var ingestionTracker = scope.ServiceProvider.GetRequiredService<IngestionTracker>();
versionId = await ingestionTracker
.WaitForIngestionCompletion(
Parent.ApiClient,
sendInfo.ProjectId,
ingestionId,
reportProgress,
Id,
CancellationToken
)
.ConfigureAwait(false);
}
// TODO: If we have NodeRun events later, better to have `ComponentTracker` to use across components
var customProperties = new Dictionary<string, object>() { { "isAsync", true }, { "auto", Parent.AutoSend } };
if (sendInfo.WorkspaceId != null)
@@ -15,6 +15,7 @@ using Speckle.Sdk.Api;
using Speckle.Sdk.Common;
using Speckle.Sdk.Credentials;
using Speckle.Sdk.Models.Collections;
using Speckle.Sdk.Pipelines.Progress;
namespace Speckle.Connectors.GrasshopperShared.Components.Operations.Send;
@@ -281,10 +282,26 @@ public class SendComponent : SpeckleTaskCapableComponent<SendComponentInput, Sen
using var client = clientFactory.Create(account);
var sendInfo = await input.Resource.GetSendInfo(client, cancellationToken).ConfigureAwait(false);
var (result, versionId) = await sendOperation
var (result, versionId, ingestionId) = await sendOperation
.Send([collectionToSend], sendInfo, fileName, fileBytes, VersionMessage, progress, cancellationToken)
.ConfigureAwait(false);
if (ingestionId != null)
{
Message = "Remote processing";
var ingestionTracker = scope.ServiceProvider.GetRequiredService<IngestionTracker>();
versionId = await ingestionTracker
.WaitForIngestionCompletion(
client,
sendInfo.ProjectId,
ingestionId,
reportProgress: null,
reportProgressId: null,
cancellationToken
)
.ConfigureAwait(false);
}
// TODO: If we have NodeRun events later, better to have `ComponentTracker` to use across components
var customProperties = new Dictionary<string, object> { { "isAsync", false } };
if (sendInfo.WorkspaceId != null)
@@ -295,12 +312,13 @@ public class SendComponent : SpeckleTaskCapableComponent<SendComponentInput, Sen
var mixpanel = PriorityLoader.Container.GetRequiredService<IMixPanelManager>();
await mixpanel.TrackEvent(MixPanelEvents.Send, account, customProperties);
SpeckleUrlLatestModelVersionResource createdVersionResource =
SpeckleUrlModelVersionResource createdVersionResource =
new(
new(sendInfo.Account.id, null, sendInfo.Account.serverInfo.url),
sendInfo.WorkspaceId,
sendInfo.ProjectId,
sendInfo.ModelId
sendInfo.ModelId,
versionId
);
Url = $"{sendInfo.Account.serverInfo.url}/projects/{sendInfo.ProjectId}/models/{sendInfo.ModelId}";
return new SendComponentOutput(createdVersionResource, versionId);
@@ -3,6 +3,7 @@ using Speckle.Connectors.Common.Operations;
using Speckle.Sdk.Api;
using Speckle.Sdk.Logging;
using Speckle.Sdk.Models;
using Speckle.Sdk.Pipelines.Progress;
using Speckle.Sdk.Transports;
namespace Speckle.Connectors.GrasshopperShared.Operations.Receive;
@@ -0,0 +1,221 @@
using Speckle.Connectors.Common.Builders;
using Speckle.Connectors.Common.Instances;
using Speckle.Connectors.Common.Operations;
using Speckle.Connectors.GrasshopperShared.HostApp;
using Speckle.Connectors.GrasshopperShared.Parameters;
using Speckle.Sdk.Models;
using Speckle.Sdk.Models.Collections;
using Speckle.Sdk.Pipelines.Progress;
using Speckle.Sdk.Pipelines.Send;
using DataObject = Speckle.Objects.Data.DataObject;
namespace Speckle.Connectors.GrasshopperShared.Operations.Send;
/// <summary>
/// Continuous traversal builder for Grasshopper that processes each object through the <see cref="SendPipeline"/>
/// as it unwraps. This enables the packfile send path (streaming objects to S3 during build).
/// </summary>
public class GrasshopperContinuousTraversalBuilder(
IInstanceObjectsManager<SpeckleGeometryWrapper, List<string>> instanceObjectsManager
) : IRootContinuousTraversalBuilder<SpeckleCollectionWrapperGoo>
{
public async Task<RootObjectBuilderResult> Build(
IReadOnlyList<SpeckleCollectionWrapperGoo> objects,
string projectId,
SendPipeline sendPipeline,
IProgress<CardProgress> onOperationProgressed,
CancellationToken cancellationToken
)
{
// create root collection
var rootCollectionGoo = (SpeckleRootCollectionWrapperGoo)objects[0].Duplicate();
rootCollectionGoo.Value.Name = "Grasshopper Model";
RootCollection rootCollection =
new(rootCollectionGoo.Value.Name)
{
applicationId = rootCollectionGoo.Value.ApplicationId,
properties = rootCollectionGoo.Value.Properties ?? new()
};
// create packers for colors and render materials
GrasshopperColorPacker colorPacker = new();
GrasshopperMaterialPacker materialPacker = new();
GrasshopperBlockPacker blockPacker = new(instanceObjectsManager);
// unwrap the input collection, processing each object through the send pipeline
await Unwrap(
rootCollectionGoo.Value,
rootCollection,
colorPacker,
materialPacker,
blockPacker,
sendPipeline,
cancellationToken
)
.ConfigureAwait(false);
// add proxies
rootCollection[ProxyKeys.COLOR] = colorPacker.ColorProxies.Values.ToList();
rootCollection[ProxyKeys.RENDER_MATERIAL] = materialPacker.RenderMaterialProxies.Values.ToList();
rootCollection[ProxyKeys.INSTANCE_DEFINITION] = blockPacker.InstanceDefinitionProxies.Values.ToList();
// process the root collection through the pipeline and wait for all uploads
await sendPipeline.Process(rootCollection).ConfigureAwait(false);
await sendPipeline.WaitForUpload().ConfigureAwait(false);
// TODO: Not getting any conversion results yet
return new RootObjectBuilderResult(rootCollection, []);
}
private async Task Unwrap(
SpeckleCollectionWrapper wrapper,
Collection targetCollection,
GrasshopperColorPacker colorPacker,
GrasshopperMaterialPacker materialPacker,
GrasshopperBlockPacker blockPacker,
SendPipeline sendPipeline,
CancellationToken cancellationToken
)
{
colorPacker.ProcessColor(wrapper.ApplicationId, wrapper.Color);
materialPacker.ProcessMaterial(wrapper.ApplicationId, wrapper.Material);
int skippedNulls = 0;
foreach (ISpeckleCollectionObject? element in wrapper.Elements)
{
cancellationToken.ThrowIfCancellationRequested();
switch (element)
{
case null:
skippedNulls++;
continue;
case SpeckleCollectionWrapper collWrapper:
collWrapper.ApplicationId ??= collWrapper.GetSpeckleApplicationId();
targetCollection.elements.Add(collWrapper.Collection);
await Unwrap(
collWrapper,
collWrapper.Collection,
colorPacker,
materialPacker,
blockPacker,
sendPipeline,
cancellationToken
)
.ConfigureAwait(false);
break;
case SpeckleGeometryWrapper so:
Base objectBase = UnwrapGeometry(so);
string applicationId = objectBase.applicationId!;
// NOTE: This is how it differentiate from 'GrasshopperSendOperation'
// It process through send pipeline before adding to collection
var reference = await sendPipeline.Process(objectBase).ConfigureAwait(false);
targetCollection.elements.Add(reference);
if (so is SpeckleBlockInstanceWrapper blockInstance)
{
await ProcessBlockInstanceDefinition(
blockInstance,
colorPacker,
materialPacker,
blockPacker,
targetCollection,
sendPipeline,
cancellationToken
)
.ConfigureAwait(false);
}
colorPacker.ProcessColor(applicationId, so.Color);
materialPacker.ProcessMaterial(applicationId, so.Material);
break;
case SpeckleDataObjectWrapper dataObjectWrapper:
DataObject dataObject = UnwrapDataObject(dataObjectWrapper, colorPacker, materialPacker);
// process data object through send pipeline
var dataRef = await sendPipeline.Process(dataObject).ConfigureAwait(false);
targetCollection.elements.Add(dataRef);
break;
}
}
// clear topology when nulls are present (CNX-2855)
if (skippedNulls > 0)
{
targetCollection[Constants.TOPOLOGY_PROP] = null;
}
}
private Base UnwrapGeometry(SpeckleGeometryWrapper wrapper)
{
Dictionary<string, object?> props = [];
Base baseObject = wrapper.Base;
if (wrapper.Properties.CastTo(ref props))
{
baseObject["properties"] = props;
}
return baseObject;
}
private async Task ProcessBlockInstanceDefinition(
SpeckleBlockInstanceWrapper blockInstance,
GrasshopperColorPacker colorPacker,
GrasshopperMaterialPacker materialPacker,
GrasshopperBlockPacker blockPacker,
Collection currentColl,
SendPipeline sendPipeline,
CancellationToken cancellationToken
)
{
cancellationToken.ThrowIfCancellationRequested();
var definitionObjects = blockPacker.ProcessInstance(blockInstance);
if (definitionObjects != null)
{
foreach (var definitionObject in definitionObjects)
{
Base defObjectBase = UnwrapGeometry(definitionObject);
string applicationId = defObjectBase.applicationId!;
var reference = await sendPipeline.Process(defObjectBase).ConfigureAwait(false);
currentColl.elements.Add(reference);
colorPacker.ProcessColor(applicationId, definitionObject.Color);
materialPacker.ProcessMaterial(applicationId, definitionObject.Material);
}
}
}
private DataObject UnwrapDataObject(
SpeckleDataObjectWrapper wrapper,
GrasshopperColorPacker colorPacker,
GrasshopperMaterialPacker materialPacker
)
{
DataObject dataObject = wrapper.DataObject;
var displayValue = new List<Base>();
foreach (var geometryWrapper in wrapper.Geometries)
{
Base geometryBase = UnwrapGeometry(geometryWrapper);
displayValue.Add(geometryBase);
if (geometryWrapper.ApplicationId != null)
{
colorPacker.ProcessColor(geometryWrapper.ApplicationId, geometryWrapper.Color);
materialPacker.ProcessMaterial(geometryWrapper.ApplicationId, geometryWrapper.Material);
}
}
dataObject.displayValue = displayValue;
return dataObject;
}
}
@@ -5,6 +5,7 @@ using Speckle.Connectors.GrasshopperShared.HostApp;
using Speckle.Connectors.GrasshopperShared.Parameters;
using Speckle.Sdk.Models;
using Speckle.Sdk.Models.Collections;
using Speckle.Sdk.Pipelines.Progress;
using DataObject = Speckle.Objects.Data.DataObject;
namespace Speckle.Connectors.GrasshopperShared.Operations.Send;
@@ -9,6 +9,7 @@ using Speckle.Connectors.Common.Operations;
using Speckle.Connectors.Common.Operations.Receive;
using Speckle.Connectors.Common.Threading;
using Speckle.Connectors.GrasshopperShared.Components;
using Speckle.Connectors.GrasshopperShared.Components.Operations.Send;
using Speckle.Connectors.GrasshopperShared.HostApp;
using Speckle.Connectors.GrasshopperShared.Operations.Receive;
using Speckle.Connectors.GrasshopperShared.Operations.Send;
@@ -61,9 +62,14 @@ public class PriorityLoader : GH_AssemblyPriority
services.AddTransient<TraversalContextUnpacker>();
services.AddScoped<IDataObjectInstanceRegistry, DataObjectInstanceRegistry>();
services.AddTransient<LocalToGlobalMapHandler>();
services.AddTransient<IngestionTracker>();
// send
services.AddTransient<IRootObjectBuilder<SpeckleCollectionWrapperGoo>, GrasshopperRootObjectBuilder>();
services.AddTransient<
IRootContinuousTraversalBuilder<SpeckleCollectionWrapperGoo>,
GrasshopperContinuousTraversalBuilder
>();
services.AddTransient<SendOperation<SpeckleCollectionWrapperGoo>>();
services.AddSingleton<IThreadContext>(new DefaultThreadContext());
services.AddScoped<
@@ -58,7 +58,9 @@
<Compile Include="$(MSBuildThisFileDirectory)Components\Operations\Receive\ReceiveComponent.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\GrasshopperBlockPacker.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\GrasshopperMaterialPacker.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\GrasshopperContinuousTraversalBuilder.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\GrasshopperColorPacker.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Components\Operations\Send\IngestionTracker.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Components\Operations\Send\SendAsyncComponent.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Components\Operations\Send\SendComponent.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Components\Operations\SpeckleSelectModelComponent.cs" />
@@ -29,20 +29,20 @@
},
"RhinoCommon": {
"type": "Direct",
"requested": "[8.25.25328.11001, )",
"resolved": "8.25.25328.11001",
"contentHash": "PDKR9GwqyUXUkTulV4J0dzDIf/aWqSJkL7nkS8ReAx8xhnt/+RQpE8gTjOSCmkSU2tjG6WzclowbTxwMTU7VAA==",
"requested": "[8.28.26041.11001, )",
"resolved": "8.28.26041.11001",
"contentHash": "5mByZF+IdHvRLbYvr6Ek95Pwam6otewAyVHIGSC0sUE1+OscSpd9gbrFWnzo1arfCYmUJcUdsGhf7VayW09fQA==",
"dependencies": {
"System.Drawing.Common": "7.0.0"
}
},
"RhinoWindows": {
"type": "Direct",
"requested": "[8.25.25328.11001, )",
"resolved": "8.25.25328.11001",
"contentHash": "I/+++piwtYTue+iAAQqcMF5QlontqwNnC7Leyhiv2FiF8JpAl6K44ZsJqB7ZEUC6ns0LDfa3mbFzQwUfHwYumQ==",
"requested": "[8.28.26041.11001, )",
"resolved": "8.28.26041.11001",
"contentHash": "7MBG231k5c0/V/USTzbXpaTCiZCECQOuB80DnpgEvvEA3r//IQQDTbrqawDYmN9BEw/FHiFn82bsf6tV+SS5Iw==",
"dependencies": {
"RhinoCommon": "[8.25.25328.11001]"
"RhinoCommon": "[8.28.26041.11001]"
}
},
"Speckle.InterfaceGenerator": {
@@ -5,7 +5,6 @@ using Rhino.Geometry;
using Rhino.Render;
using Speckle.Connectors.Common.Conversion;
using Speckle.Connectors.Common.Instances;
using Speckle.Connectors.Common.Operations;
using Speckle.Connectors.Rhino.Extensions;
using Speckle.Converters.Common.ToHost;
using Speckle.DoubleNumerics;
@@ -15,6 +14,7 @@ using Speckle.Sdk.Common.Exceptions;
using Speckle.Sdk.Models;
using Speckle.Sdk.Models.Collections;
using Speckle.Sdk.Models.Instances;
using Speckle.Sdk.Pipelines.Progress;
namespace Speckle.Connectors.Rhino.HostApp;
@@ -19,6 +19,7 @@ using Speckle.Sdk.Logging;
using Speckle.Sdk.Models;
using Speckle.Sdk.Models.Collections;
using Speckle.Sdk.Models.Instances;
using Speckle.Sdk.Pipelines.Progress;
namespace Speckle.Connectors.Rhino.Operations.Receive;
@@ -0,0 +1,220 @@
using Microsoft.Extensions.Logging;
using Rhino.DocObjects;
using Speckle.Connectors.Common.Builders;
using Speckle.Connectors.Common.Caching;
using Speckle.Connectors.Common.Conversion;
using Speckle.Connectors.Common.Extensions;
using Speckle.Connectors.Common.Instances;
using Speckle.Connectors.Common.Operations;
using Speckle.Connectors.DUI.Models.Card.SendFilter;
using Speckle.Connectors.Rhino.HostApp;
using Speckle.Connectors.Rhino.HostApp.Properties;
using Speckle.Converters.Common;
using Speckle.Converters.Rhino;
using Speckle.Sdk;
using Speckle.Sdk.Logging;
using Speckle.Sdk.Models;
using Speckle.Sdk.Models.Collections;
using Speckle.Sdk.Models.Instances;
using Speckle.Sdk.Pipelines.Progress;
using Speckle.Sdk.Pipelines.Send;
using Layer = Rhino.DocObjects.Layer;
namespace Speckle.Connectors.Rhino.Operations.Send;
/// <summary>
/// NOTE: I am not happy this is a mostly copy paste of the Root object builder, but i'm also not too worried. The main (hot) path
/// should be this one going forward, so we should not touch the og root object builder besides to delete it.
/// Stateless builder object to turn an <see cref="ISendFilter"/> into a <see cref="Base"/> object
/// </summary>
public class RhinoContinuousTraversalBuilder : IRootContinuousTraversalBuilder<RhinoObject>
{
private readonly IRootToSpeckleConverter _rootToSpeckleConverter;
private readonly ISendConversionCache _sendConversionCache;
private readonly IConverterSettingsStore<RhinoConversionSettings> _converterSettings;
private readonly RhinoLayerUnpacker _layerUnpacker;
private readonly RhinoInstanceUnpacker _instanceUnpacker;
private readonly RhinoGroupUnpacker _groupUnpacker;
private readonly RhinoMaterialUnpacker _materialUnpacker;
private readonly RhinoColorUnpacker _colorUnpacker;
private readonly RhinoViewUnpacker _viewUnpacker;
private readonly PropertiesExtractor _propertiesExtractor;
private readonly ILogger<RhinoContinuousTraversalBuilder> _logger;
private readonly ISdkActivityFactory _activityFactory;
public RhinoContinuousTraversalBuilder(
IRootToSpeckleConverter rootToSpeckleConverter,
ISendConversionCache sendConversionCache,
IConverterSettingsStore<RhinoConversionSettings> converterSettings,
RhinoLayerUnpacker layerUnpacker,
RhinoInstanceUnpacker instanceUnpacker,
RhinoGroupUnpacker groupUnpacker,
RhinoMaterialUnpacker materialUnpacker,
RhinoColorUnpacker colorUnpacker,
RhinoViewUnpacker viewUnpacker,
PropertiesExtractor propertiesExtractor,
ILogger<RhinoContinuousTraversalBuilder> logger,
ISdkActivityFactory activityFactory
)
{
_sendConversionCache = sendConversionCache;
_converterSettings = converterSettings;
_layerUnpacker = layerUnpacker;
_instanceUnpacker = instanceUnpacker;
_groupUnpacker = groupUnpacker;
_rootToSpeckleConverter = rootToSpeckleConverter;
_materialUnpacker = materialUnpacker;
_colorUnpacker = colorUnpacker;
_viewUnpacker = viewUnpacker;
_propertiesExtractor = propertiesExtractor;
_logger = logger;
_activityFactory = activityFactory;
}
public async Task<RootObjectBuilderResult> Build(
IReadOnlyList<RhinoObject> rhinoObjects,
string projectId,
SendPipeline sendPipeline,
IProgress<CardProgress> onOperationProgressed,
CancellationToken cancellationToken
)
{
using var activity = _activityFactory.Start("Build");
// 0 - Init the root
Collection rootObjectCollection = new() { name = _converterSettings.Current.Document.Name ?? "Unnamed document" };
rootObjectCollection["units"] = _converterSettings.Current.SpeckleUnits;
// 1 - Unpack the instances
UnpackResult<RhinoObject> unpackResults;
using (var _ = _activityFactory.Start("UnpackSelection"))
{
unpackResults = _instanceUnpacker.UnpackSelection(rhinoObjects);
}
var (atomicObjects, _, instanceProxies, instanceDefinitionProxies) = unpackResults;
// POC: we should formalise this, sooner or later - or somehow fix it a bit more
rootObjectCollection[ProxyKeys.INSTANCE_DEFINITION] = instanceDefinitionProxies; // this won't work re traversal on receive
// 2 - Unpack the groups
using (var _ = _activityFactory.Start("Unpack Groups"))
{
_groupUnpacker.UnpackGroups(rhinoObjects);
}
rootObjectCollection[ProxyKeys.GROUP] = _groupUnpacker.GroupProxies.Values;
// 3 - Convert atomic objects
List<SendConversionResult> results = new(atomicObjects.Count);
int count = 0;
using (var _ = _activityFactory.Start("Convert all"))
{
foreach (RhinoObject rhinoObject in atomicObjects)
{
cancellationToken.ThrowIfCancellationRequested();
// handle layer and store object layer *and all layer parents* to the version layers
// this is important because we need to unpack colors and materials on intermediate layers that do not have objects as well.
Layer layer = _converterSettings.Current.Document.Layers[rhinoObject.Attributes.LayerIndex];
Collection collectionHost = _layerUnpacker.GetHostObjectCollection(layer, rootObjectCollection);
var result = await ConvertRhinoObject(rhinoObject, collectionHost, instanceProxies, projectId, sendPipeline);
results.Add(result);
count++;
onOperationProgressed.Report(
new($"Converting objects... ({count:N0} / {atomicObjects.Count:N0})", (double)count / atomicObjects.Count)
);
}
}
if (results.All(x => x.Status == Status.ERROR))
{
throw new SpeckleException("Failed to convert all objects."); // fail fast instead creating empty commit! It will appear as model card error with red color.
}
// 4 - Unpack all proxies for the root
// Get all layers from the created collections on the root object commit for proxy processing
List<Layer> layers = _layerUnpacker.GetUsedLayers().ToList();
using (var _ = _activityFactory.Start("UnpackRenderMaterials"))
{
rootObjectCollection[ProxyKeys.RENDER_MATERIAL] = _materialUnpacker.UnpackRenderMaterials(atomicObjects, layers);
}
using (var _ = _activityFactory.Start("UnpackColors"))
{
rootObjectCollection[ProxyKeys.COLOR] = _colorUnpacker.UnpackColors(atomicObjects, layers);
}
// 5 - Unpack all other objects for the root
using (var _ = _activityFactory.Start("UnpackViews"))
{
List<Objects.Other.Camera> views = _viewUnpacker.UnpackViews(_converterSettings.Current.Document.NamedViews);
if (views.Count > 0)
{
rootObjectCollection[RootKeys.VIEW] = views;
}
}
await sendPipeline.Process(rootObjectCollection);
await sendPipeline.WaitForUpload();
return new RootObjectBuilderResult(rootObjectCollection, results);
}
private async Task<SendConversionResult> ConvertRhinoObject(
RhinoObject rhinoObject,
Collection collectionHost,
IReadOnlyDictionary<string, InstanceProxy> instanceProxies,
string projectId,
SendPipeline sendPipeline
)
{
string applicationId = rhinoObject.Id.ToString();
string sourceType = rhinoObject.ObjectType.ToString();
try
{
// get from cache or convert:
// What we actually do here is check if the object has been previously converted AND has not changed.
// If that's the case, we insert in the host collection just its object reference which has been saved from the prior conversion.
Base converted;
if (rhinoObject is InstanceObject)
{
converted = instanceProxies[applicationId];
}
else if (_sendConversionCache.TryGetValue(projectId, applicationId, out ObjectReference? value))
{
converted = value;
}
else
{
converted = _rootToSpeckleConverter.Convert(rhinoObject);
converted.applicationId = applicationId;
}
// add name and properties
// POC: this is NOT done in the converter because we don't have a RootToSpeckle converter that captures all top level converters
if (!string.IsNullOrEmpty(rhinoObject.Attributes.Name))
{
converted["name"] = rhinoObject.Attributes.Name;
}
var properties = _propertiesExtractor.GetProperties(rhinoObject);
if (properties.Count > 0)
{
converted["properties"] = properties;
}
// NOTE: this is the main part that differentiate from the main root object builder
var reference = await sendPipeline.Process(converted).ConfigureAwait(false);
// add to host
collectionHost.elements.Add(reference);
return new(Status.SUCCESS, applicationId, sourceType, reference);
}
catch (Exception ex) when (!ex.IsFatal())
{
_logger.LogSendConversionError(ex, sourceType);
return new(Status.ERROR, applicationId, sourceType, null, ex);
}
}
}
@@ -16,6 +16,7 @@ using Speckle.Sdk.Logging;
using Speckle.Sdk.Models;
using Speckle.Sdk.Models.Collections;
using Speckle.Sdk.Models.Instances;
using Speckle.Sdk.Pipelines.Progress;
using Layer = Rhino.DocObjects.Layer;
namespace Speckle.Connectors.Rhino.Operations.Send;
@@ -71,6 +71,7 @@ public static class ServiceRegistration
serviceCollection.AddSingleton(DefaultTraversal.CreateTraversalFunc());
serviceCollection.AddScoped<IRootObjectBuilder<RhinoObject>, RhinoRootObjectBuilder>();
serviceCollection.AddScoped<IRootContinuousTraversalBuilder<RhinoObject>, RhinoContinuousTraversalBuilder>();
serviceCollection.AddScoped<
IInstanceObjectsManager<RhinoObject, List<string>>,
InstanceObjectsManager<RhinoObject, List<string>>
@@ -38,6 +38,7 @@
<Compile Include="$(MSBuildThisFileDirectory)Mapper\Revit\RevitMappingResolver.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\Filters\RhinoSelectionFilter.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\Filters\RhinoLayersFilter.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\RhinoContinousTraversalBuilder.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\Settings\AddVisualizationProperties.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\Settings\ToSpeckleSettingsManager.cs" />
<Compile Include="$(MSBuildThisFileDirectory)RhinoEvents.cs" />
@@ -0,0 +1,141 @@
using Microsoft.Extensions.Logging;
using Speckle.Connectors.Common.Builders;
using Speckle.Connectors.Common.Caching;
using Speckle.Connectors.Common.Conversion;
using Speckle.Connectors.Common.Operations;
using Speckle.Connectors.TeklaShared.Extensions;
using Speckle.Connectors.TeklaShared.HostApp;
using Speckle.Converters.Common;
using Speckle.Converters.TeklaShared;
using Speckle.Sdk;
using Speckle.Sdk.Logging;
using Speckle.Sdk.Models;
using Speckle.Sdk.Models.Collections;
using Speckle.Sdk.Pipelines.Progress;
using Speckle.Sdk.Pipelines.Send;
namespace Speckle.Connectors.TeklaShared.Operations.Send;
/// <summary>
/// Continuous traversal builder for Tekla that streams objects through a <see cref="SendPipeline"/>
/// for packfile-based uploads. Same conversion logic as <see cref="TeklaRootObjectBuilder"/>.
/// </summary>
public class TeklaContinuousTraversalBuilder : IRootContinuousTraversalBuilder<TSM.ModelObject>
{
private readonly IRootToSpeckleConverter _rootToSpeckleConverter;
private readonly ISendConversionCache _sendConversionCache;
private readonly IConverterSettingsStore<TeklaConversionSettings> _converterSettings;
private readonly SendCollectionManager _sendCollectionManager;
private readonly ILogger<TeklaRootObjectBuilder> _logger;
private readonly ISdkActivityFactory _activityFactory;
private readonly TeklaMaterialUnpacker _materialUnpacker;
public TeklaContinuousTraversalBuilder(
IRootToSpeckleConverter rootToSpeckleConverter,
ISendConversionCache sendConversionCache,
IConverterSettingsStore<TeklaConversionSettings> converterSettings,
SendCollectionManager sendCollectionManager,
ILogger<TeklaRootObjectBuilder> logger,
ISdkActivityFactory activityFactory,
TeklaMaterialUnpacker materialUnpacker
)
{
_sendConversionCache = sendConversionCache;
_converterSettings = converterSettings;
_sendCollectionManager = sendCollectionManager;
_rootToSpeckleConverter = rootToSpeckleConverter;
_logger = logger;
_activityFactory = activityFactory;
_materialUnpacker = materialUnpacker;
}
public async Task<RootObjectBuilderResult> Build(
IReadOnlyList<TSM.ModelObject> teklaObjects,
string projectId,
SendPipeline sendPipeline,
IProgress<CardProgress> onOperationProgressed,
CancellationToken cancellationToken
)
{
using var activity = _activityFactory.Start("Build");
var model = new TSM.Model();
string modelName = model.GetInfo().ModelName ?? "Unnamed model";
Collection rootObjectCollection = new() { name = modelName };
rootObjectCollection["units"] = _converterSettings.Current.SpeckleUnits;
List<SendConversionResult> results = new(teklaObjects.Count);
int count = 0;
using (var _ = _activityFactory.Start("Convert all"))
{
foreach (TSM.ModelObject teklaObject in teklaObjects)
{
cancellationToken.ThrowIfCancellationRequested();
var result = await ConvertTeklaObject(teklaObject, rootObjectCollection, projectId, sendPipeline);
results.Add(result);
++count;
onOperationProgressed.Report(
new($"Converting objects... ({count:N0} / {teklaObjects.Count:N0})", (double)count / teklaObjects.Count)
);
await Task.Yield();
}
}
if (results.All(x => x.Status == Status.ERROR))
{
throw new SpeckleException("Failed to convert all objects.");
}
var renderMaterialProxies = _materialUnpacker.UnpackRenderMaterial(teklaObjects.ToList());
if (renderMaterialProxies.Count > 0)
{
rootObjectCollection[ProxyKeys.RENDER_MATERIAL] = renderMaterialProxies;
}
// Process root collection and wait for all uploads
await sendPipeline.Process(rootObjectCollection);
await sendPipeline.WaitForUpload();
return new RootObjectBuilderResult(rootObjectCollection, results);
}
private async Task<SendConversionResult> ConvertTeklaObject(
TSM.ModelObject teklaObject,
Collection collectionHost,
string projectId,
SendPipeline sendPipeline
)
{
string applicationId = teklaObject.GetSpeckleApplicationId();
string sourceType = teklaObject.GetType().Name;
try
{
Base converted;
if (_sendConversionCache.TryGetValue(projectId, applicationId, out ObjectReference? value))
{
converted = value;
}
else
{
converted = _rootToSpeckleConverter.Convert(teklaObject);
}
var collection = _sendCollectionManager.GetAndCreateObjectHostCollection(teklaObject, collectionHost);
// NOTE: this is the main part that differentiate from the main root object builder
var reference = await sendPipeline.Process(converted).ConfigureAwait(false);
collection.elements.Add(reference);
return new(Status.SUCCESS, applicationId, sourceType, reference);
}
catch (Exception ex) when (!ex.IsFatal())
{
_logger.LogError(ex, "Failed to convert object {SourceType}", sourceType);
return new(Status.ERROR, applicationId, sourceType, null, ex);
}
}
}
@@ -11,6 +11,7 @@ using Speckle.Sdk;
using Speckle.Sdk.Logging;
using Speckle.Sdk.Models;
using Speckle.Sdk.Models.Collections;
using Speckle.Sdk.Pipelines.Progress;
namespace Speckle.Connectors.TeklaShared.Operations.Send;
@@ -54,6 +54,7 @@ public static class ServiceRegistration
services.AddSingleton(DefaultTraversal.CreateTraversalFunc());
services.AddScoped<SendCollectionManager>();
services.AddScoped<IRootObjectBuilder<ModelObject>, TeklaRootObjectBuilder>();
services.AddScoped<IRootContinuousTraversalBuilder<ModelObject>, TeklaContinuousTraversalBuilder>();
services.AddScoped<SendOperation<ModelObject>>();
services.AddSingleton<ToSpeckleSettingsManager>();
@@ -37,6 +37,7 @@
<Compile Include="$(MSBuildThisFileDirectory)HostApp\TeklaMaterialUnpacker.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\Settings\SendRebarsAsSolidSetting.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\Settings\ToSpeckleSettingsManager.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\TeklaContinuousTraversalBuilder.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Operations\Send\TeklaRootObjectBuilder.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Plugin\TeklaPlugin.cs" />
<Compile Include="$(MSBuildThisFileDirectory)ServiceRegistration.cs" />
@@ -346,9 +346,9 @@
},
"RhinoCommon": {
"type": "Direct",
"requested": "[8.25.25328.11001, )",
"resolved": "8.25.25328.11001",
"contentHash": "PDKR9GwqyUXUkTulV4J0dzDIf/aWqSJkL7nkS8ReAx8xhnt/+RQpE8gTjOSCmkSU2tjG6WzclowbTxwMTU7VAA==",
"requested": "[8.28.26041.11001, )",
"resolved": "8.28.26041.11001",
"contentHash": "5mByZF+IdHvRLbYvr6Ek95Pwam6otewAyVHIGSC0sUE1+OscSpd9gbrFWnzo1arfCYmUJcUdsGhf7VayW09fQA==",
"dependencies": {
"System.Drawing.Common": "7.0.0"
}
@@ -12,6 +12,7 @@ using Speckle.Connectors.DUI.Models;
using Speckle.Connectors.DUI.Models.Card;
using Speckle.Sdk;
using Speckle.Sdk.Credentials;
using Speckle.Sdk.Pipelines.Progress;
using Speckle.Testing;
namespace Speckle.Connectors.DUI.Tests;
@@ -13,7 +13,8 @@ public interface ISendBindingUICommands
Task SetModelSendResult(
string modelCardId,
string versionId,
IEnumerable<SendConversionResult> sendConversionResults
IEnumerable<SendConversionResult> sendConversionResults,
string? ingestionId = null
);
IBrowserBridge Bridge { get; }
@@ -1,8 +1,8 @@
using System.Collections.Concurrent;
using Speckle.Connectors.Common.Operations;
using Speckle.Connectors.DUI.Bridge;
using Speckle.Connectors.DUI.Models.Card;
using Speckle.InterfaceGenerator;
using Speckle.Sdk.Pipelines.Progress;
namespace Speckle.Connectors.DUI.Bindings;
@@ -21,7 +21,7 @@ public class OperationProgressManager : IOperationProgressManager
private const string SET_MODEL_PROGRESS_UI_COMMAND_NAME = "setModelProgress";
private static readonly ConcurrentDictionary<string, (DateTime lastCallTime, string status)> s_lastProgressValues =
new();
private const int THROTTLE_INTERVAL_MS = 200;
private const int THROTTLE_INTERVAL_MS = 400;
public IProgress<CardProgress> CreateOperationProgressEventHandler(
IBrowserBridge bridge,
@@ -65,11 +65,10 @@ public class OperationProgressManager : IOperationProgressManager
var currentTime = DateTime.Now;
var elapsedMs = (currentTime - t.Item1).Milliseconds;
if (elapsedMs < THROTTLE_INTERVAL_MS && t.Item2 == progress.Status)
if (elapsedMs < THROTTLE_INTERVAL_MS)
{
return;
}
Console.WriteLine($"Progress: {progress.Status} - {progress.Progress}");
s_lastProgressValues[modelCardId] = (currentTime, progress.Status);
SendProgress(bridge, modelCardId, progress);
}
@@ -37,7 +37,8 @@ public class SendBindingUICommands(IBrowserBridge bridge)
public async Task SetModelSendResult(
string modelCardId,
string versionId,
IEnumerable<SendConversionResult> sendConversionResults
IEnumerable<SendConversionResult> sendConversionResults,
string? ingestionId = null
) =>
await Bridge.Send(
SET_MODEL_SEND_RESULT_UI_COMMAND_NAME,
@@ -45,7 +46,8 @@ public class SendBindingUICommands(IBrowserBridge bridge)
{
modelCardId,
versionId,
sendConversionResults
sendConversionResults,
ingestionId
}
);
}
@@ -13,6 +13,7 @@ using Speckle.Sdk.Api;
using Speckle.Sdk.Common;
using Speckle.Sdk.Credentials;
using Speckle.Sdk.Logging;
using Speckle.Sdk.Pipelines.Progress;
namespace Speckle.Connectors.DUI.Bindings;
@@ -98,7 +99,7 @@ public sealed class SendOperationManager(
cancellationItem.Token
);
var objects = await gatherObjects(modelCard, progress);
var objects = await gatherObjects.Invoke(modelCard, progress);
if (objects.Count == 0)
{
@@ -108,7 +109,7 @@ public sealed class SendOperationManager(
var sendOperation = serviceScope.ServiceProvider.GetRequiredService<ISendOperation<T>>();
var (result, versionId) = await sendOperation.Send(
var (result, versionId, ingestionId) = await sendOperation.Send(
objects,
sendInfo,
fileName,
@@ -118,7 +119,7 @@ public sealed class SendOperationManager(
cancellationItem.Token
);
await commands.SetModelSendResult(modelCardId, versionId, result.ConversionResults);
await commands.SetModelSendResult(modelCardId, versionId, result.ConversionResults, ingestionId);
}
catch (OperationCanceledException)
{
@@ -4,7 +4,7 @@
/// Progress value between 0 and 1 to calculate UI progress bar width.
/// If it is null it will swooshing on UI.
/// </summary>
public record ModelCardProgress(string ModelCardId, string Status, double? Progress)
public readonly record struct ModelCardProgress(string ModelCardId, string Status, double? Progress)
{
public override string ToString() => $"{ModelCardId} - {Status} - {Progress}";
}
+6 -6
View File
@@ -18,7 +18,7 @@
<PackageVersion Include="Moq" Version="4.20.70" />
<PackageVersion Include="Microsoft.Build" Version="17.11.48" />
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
<PackageVersion Include="Npgsql" Version="9.0.3" />
<PackageVersion Include="Npgsql" Version="9.0.4" />
<PackageVersion Include="Microsoft.Extensions.Hosting.WindowsServices" Version="9.0.9" />
<PackageVersion Include="Microsoft.Extensions.Logging" Version="2.2.0" />
<PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="2.2.0" />
@@ -28,10 +28,10 @@
<PackageVersion Include="NUnit.Analyzers" Version="4.2.0" />
<PackageVersion Include="NUnit3TestAdapter" version="4.6.0" />
<PackageVersion Include="Revit.Async" Version="2.1.1" />
<PackageVersion Include="RhinoCommon" Version="8.25.25328.11001" />
<PackageVersion Include="RhinoCommon" Version="8.28.26041.11001" />
<PackageVersion Include="Rhino.Inside" Version="8.0.7-beta" />
<PackageVersion Include="Grasshopper" Version="8.9.24194.18121" />
<PackageVersion Include="RhinoWindows" Version="8.25.25328.11001" />
<PackageVersion Include="Grasshopper" Version="8.28.26041.11001" />
<PackageVersion Include="RhinoWindows" Version="8.28.26041.11001" />
<PackageVersion Include="Semver" Version="3.0.0" />
<PackageVersion Include="Serilog.Formatting.Compact" Version="3.0.0" />
<PackageVersion Include="Speckle.CSI.API" Version="2.4.0" />
@@ -43,8 +43,8 @@
<PackageVersion Include="Tekla.Structures.Drawing" Version="2024.0.4" />
<PackageVersion Include="Tekla.Structures.Model" Version="2024.0.4" />
<PackageVersion Include="Tekla.Structures.Plugins" Version="2024.0.4" PrivateAssets="all" IncludeAssets="compile; build" />
<PackageVersion Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.11.1" />
<PackageVersion Include="OpenTelemetry.Instrumentation.Http" version="1.11.0" />
<PackageVersion Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.13.1" />
<PackageVersion Include="OpenTelemetry.Instrumentation.Http" version="1.13.0" />
<PackageVersion Include="Serilog" Version="4.0.1" />
<PackageVersion Include="Serilog.Exceptions" Version="8.4.0" />
<PackageVersion Include="Serilog.Extensions.Logging" Version="8.0.0" />
@@ -1,12 +1,16 @@
using Microsoft.Extensions.Logging;
using Speckle.Importers.JobProcessor.Domain;
using Speckle.Sdk.Api;
using Speckle.Sdk.Logging;
namespace Speckle.Importers.JobProcessor.Blobs;
internal sealed class ImportJobFileDownloader(ILogger<ImportJobFile> logger)
internal sealed class ImportJobFileDownloader(ILogger<ImportJobFile> logger, ISdkActivityFactory activityFactory)
{
public async Task<ImportJobFile> DownloadFile(FileimportJob job, IClient client, CancellationToken cancellationToken)
{
using var activity = activityFactory.Start();
try
{
var directory = Directory.CreateTempSubdirectory("speckle-file-import");
string targetFilePath = $"{directory.FullName}/{job.Payload.BlobId}.{job.Payload.FileType}";
@@ -17,6 +21,15 @@ internal sealed class ImportJobFileDownloader(ILogger<ImportJobFile> logger)
null,
cancellationToken
);
return new ImportJobFile(logger, new FileInfo(targetFilePath));
var ret = new ImportJobFile(logger, new FileInfo(targetFilePath));
activity?.SetStatus(SdkActivityStatusCode.Ok);
return ret;
}
catch (Exception ex)
{
activity?.RecordException(ex);
activity?.SetStatus(SdkActivityStatusCode.Error);
throw;
}
}
}
@@ -1,4 +1,6 @@
namespace Speckle.Importers.JobProcessor.Domain;
using System.Text.Json.Serialization;
namespace Speckle.Importers.JobProcessor.Domain;
/// <summary>
/// Payload for the fileimport job
@@ -17,4 +19,13 @@ internal sealed class FileimportPayload
public required Uri ServerUrl { get; init; }
public required int PayloadVersion { get; init; }
public required int TimeOutSeconds { get; init; }
[JsonPropertyName("_traceContext")]
public TraceContext? TraceContext { get; init; }
}
public sealed class TraceContext
{
[JsonPropertyName("traceparent")]
public string? TraceParent { get; init; }
}
@@ -12,6 +12,7 @@ internal readonly struct ImporterArgs
public required string BlobId { get; init; }
public required int Attempt { get; init; }
public required string ResultsPath { get; init; }
public required string? TraceContext { get; init; }
public required Project Project { get; init; }
public required ModelIngestion Ingestion { get; init; }
public required Account Account { get; init; }
@@ -10,6 +10,7 @@ using Speckle.Sdk.Api;
using Speckle.Sdk.Api.GraphQL.Inputs;
using Speckle.Sdk.Api.GraphQL.Models;
using Speckle.Sdk.Common;
using Speckle.Sdk.Logging;
using JsonSerializer = System.Text.Json.JsonSerializer;
namespace Speckle.Importers.JobProcessor.JobHandlers;
@@ -17,7 +18,8 @@ namespace Speckle.Importers.JobProcessor.JobHandlers;
internal sealed class RhinoJobHandler(
ILogger<RhinoJobHandler> logger,
ImportJobFileDownloader fileDownloader,
ISpeckleApplication application
ISpeckleApplication application,
ISdkActivityFactory activityFactory
) : IJobHandler
{
private readonly JsonSerializerSettings _settings =
@@ -50,11 +52,15 @@ internal sealed class RhinoJobHandler(
),
cancellationToken
);
string resultsPath = $"{file.FileInfo.DirectoryName}/results.json";
using (var activity = activityFactory.Start("Await sub-process"))
{
var importerArgs = new ImporterArgs
{
FilePath = file.FileInfo.FullName,
ResultsPath = $"{file.FileInfo.DirectoryName}/results.json",
ResultsPath = resultsPath,
TraceContext = $"00-{activity?.TraceId}-{activity?.SpanId}-01",
Account = client.Account,
Project = project,
Ingestion = ingestion,
@@ -64,7 +70,9 @@ internal sealed class RhinoJobHandler(
HostApplication = handlerApplication,
};
await RunSubProcess(importerArgs, cancellationToken);
var response = await DeserializeResponse(importerArgs.ResultsPath, cancellationToken);
}
var response = await DeserializeResponse(resultsPath, cancellationToken);
if (response.RootObjectId is null)
{
@@ -94,7 +102,12 @@ internal sealed class RhinoJobHandler(
var processStart = new ProcessStartInfo()
{
FileName = $"{path}/Speckle.Importers.Rhino.exe",
Environment = { },
Environment =
{
["DOTNET_ENVIRONMENT"] = Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT"),
["SEQ_API_KEY"] = Environment.GetEnvironmentVariable("SEQ_API_KEY"),
["SPECKLE_COLLECTOR_API_TOKEN"] = Environment.GetEnvironmentVariable("SPECKLE_COLLECTOR_API_TOKEN"),
},
RedirectStandardError = true,
RedirectStandardOutput = true,
UseShellExecute = false,
@@ -71,7 +71,10 @@ internal sealed class JobProcessorInstance(
job.RemainingComputeBudgetSeconds
);
using var activity = activityFactory.Start();
using var activity = job.Payload.TraceContext?.TraceParent is not null
? activityFactory.StartRemote(job.Payload.TraceContext.TraceParent, SdkActivityKind.Consumer, "Picked up a job")
: activityFactory.Start("Picked up a job", SdkActivityKind.Consumer);
using var scopeJobId = ActivityScope.SetTag("jobId", job.Id);
using var scopeJobType = ActivityScope.SetTag("jobType", job.Payload.JobType);
using var scopeAttempt = ActivityScope.SetTag("job.attempt", job.Attempt.ToString());
@@ -96,27 +99,6 @@ internal sealed class JobProcessorInstance(
}
}
private async Task ReportSuccess(
FileimportJob job,
string rootObjectId,
IClient client,
double elapsedSeconds,
CancellationToken cancellationToken
)
{
string versionId = await client.Ingestion.Complete(
new(job.Payload.ModelIngestionId, job.Payload.ProjectId, rootObjectId, null),
cancellationToken
);
logger.LogInformation(
"Attempt {Attempt} of {JobId} has succeeded creating {VersionId} after {ElapsedSeconds}",
job.Attempt,
job.Id,
versionId,
elapsedSeconds
);
}
private async Task ReportCancelled(FileimportJob job, IClient client, Exception ex, double elapsedSeconds)
{
await client.Ingestion.FailWithCancel(
@@ -186,11 +168,9 @@ internal sealed class JobProcessorInstance(
throw new MaxAttemptsExceededException("Unhandled error silently failed the job multiple times");
}
string rootObjectId = await ExecuteJobWithTimeout(job, speckleClient, serviceCancellationToken);
await ExecuteJobWithTimeout(job, speckleClient, serviceCancellationToken);
totalElapsedSeconds = stopwatch.Elapsed.TotalSeconds;
await ReportSuccess(job, rootObjectId, speckleClient, totalElapsedSeconds, serviceCancellationToken);
activity?.SetStatus(SdkActivityStatusCode.Ok);
}
catch (Exception ex)
@@ -15,17 +15,17 @@ public static class Program
// Dapper doesn't understand how to handle JSON deserialization, so we need to tell it what types can be deserialzied
SqlMapper.AddTypeHandler(new JsonHandler<FileimportPayload>());
var host = ConfigureAppHost(args);
using var loggerDisposable = ConfigureAppHost(args, out IHost host);
await host.RunAsync();
}
private static IHost ConfigureAppHost(string[] args)
private static IDisposable ConfigureAppHost(string[] args, out IHost host)
{
// DI setup
var builder = Host.CreateApplicationBuilder(args);
builder.Services.AddJobProcessor();
var loggingDisposable = builder.Services.AddJobProcessor();
builder.Services.AddWindowsService();
builder.Services.AddTransient<IJobHandler, RhinoJobHandler>();
@@ -35,6 +35,7 @@ public static class Program
settings.Filter = (_, level) => level >= LogLevel.Information;
});
return builder.Build();
host = builder.Build();
return loggingDisposable;
}
}
@@ -7,6 +7,9 @@ using Speckle.Importers.JobProcessor.Blobs;
using Speckle.Importers.JobProcessor.JobQueue;
using Speckle.Objects.Geometry;
using Speckle.Sdk;
#if !DEBUG && !LOCAL
using Speckle.Sdk.Common;
#endif
namespace Speckle.Importers.JobProcessor;
@@ -15,7 +18,7 @@ internal static class ServiceRegistration
private static readonly Application s_application = new(".NET File Import Job Processor", "jobprocessor");
private const HostAppVersion HOST_APP_VERSION = HostAppVersion.v3;
public static IServiceCollection AddJobProcessor(this IServiceCollection serviceCollection)
public static IDisposable AddJobProcessor(this IServiceCollection serviceCollection)
{
var assemblyVersion = Assembly.GetExecutingAssembly().GetVersion();
@@ -26,17 +29,18 @@ internal static class ServiceRegistration
typeof(Point).Assembly
);
serviceCollection.AddLoggingConfig();
var loggingDisposable = serviceCollection.AddLoggingConfig();
serviceCollection.AddTransient<Repository>();
serviceCollection.AddTransient<ImportJobFileDownloader>();
serviceCollection.AddHostedService<JobProcessorInstance>();
return serviceCollection;
return loggingDisposable;
}
private static void AddLoggingConfig(this IServiceCollection serviceCollection)
private static IDisposable AddLoggingConfig(this IServiceCollection serviceCollection)
{
serviceCollection.AddSeqLogging(
return serviceCollection.AddOpenTelemetry(
"Speckle.Importers.JobProcessor",
s_application,
HOST_APP_VERSION,
#if DEBUG || LOCAL
@@ -51,7 +55,23 @@ internal static class ServiceRegistration
[
new(
Endpoint: new Uri("https://seq.speckle.systems/ingest/otlp/v1/logs"),
Headers: new() { { "X-Seq-ApiKey", "zG4cU1MbOhMD699iGlAq" } }
Headers: new()
{
// We're using a different token than connectors for seq because we want to beable to
// trust the client's timestamps (rather than use the server's timestamps) for better tracing
// This setting has more opportunity for abuse, so we're keeping it secret, unlike the connectors token.
{ "X-Seq-ApiKey", Environment.GetEnvironmentVariable("SEQ_API_KEY").NotNullOrWhiteSpace() }
}
),
new(
Endpoint: new Uri("https://collector.speckle.dev/v1/logs"),
Headers: new()
{
{
"authorization",
Environment.GetEnvironmentVariable("SPECKLE_COLLECTOR_API_TOKEN").NotNullOrWhiteSpace()
}
}
)
],
MinimumLevel: SpeckleLogLevel.Information
@@ -62,7 +82,20 @@ internal static class ServiceRegistration
[
new(
Endpoint: new Uri("https://seq.speckle.systems/ingest/otlp/v1/traces"),
Headers: new() { { "X-Seq-ApiKey", "zG4cU1MbOhMD699iGlAq" } }
Headers: new()
{
{ "X-Seq-ApiKey", Environment.GetEnvironmentVariable("SEQ_API_KEY").NotNullOrWhiteSpace() }
}
),
new(
Endpoint: new Uri("https://collector.speckle.dev/v1/traces"),
Headers: new()
{
{
"authorization",
Environment.GetEnvironmentVariable("SPECKLE_COLLECTOR_API_TOKEN").NotNullOrWhiteSpace()
}
}
)
]
),
@@ -40,9 +40,9 @@
},
"Npgsql": {
"type": "Direct",
"requested": "[9.0.3, )",
"resolved": "9.0.3",
"contentHash": "tPvY61CxOAWxNsKLEBg+oR646X4Bc8UmyQ/tJszL/7mEmIXQnnBhVJZrZEEUv0Bstu0mEsHZD5At3EO8zQRAYw==",
"requested": "[9.0.4, )",
"resolved": "9.0.4",
"contentHash": "68BASXH0FAEuL/J4J0eRfYC8/3vzqQTmoW8zDzNf0JgaVxc7LZeEkS6jaG0ib3voFLxY5ZiCwJG+uQM+mzuu0Q==",
"dependencies": {
"Microsoft.Extensions.Logging.Abstractions": "8.0.2"
}
@@ -12,6 +12,7 @@ internal readonly struct ImporterArgs
public required string BlobId { get; init; }
public required int Attempt { get; init; }
public required string ResultsPath { get; init; }
public required string? TraceContext { get; init; }
public required Project Project { get; init; }
public required ModelIngestion Ingestion { get; init; }
public required Account Account { get; init; }
@@ -2,60 +2,99 @@ using System.Diagnostics.Contracts;
using Microsoft.Extensions.Logging;
using Rhino;
using Rhino.Runtime.InProcess;
using Speckle.Connectors.Common.Builders;
using Speckle.Connectors.Common.Extensions;
using Speckle.Connectors.Logging;
using Speckle.Importers.Rhino.Internal.FileTypeConfig;
using Speckle.Sdk.Api;
using Speckle.Sdk.Serialisation.V2.Send;
using Speckle.Sdk.Logging;
namespace Speckle.Importers.Rhino.Internal;
internal sealed class ImporterInstance(
ImporterArgs args,
Sender sender,
IClient speckleClient,
ILogger<ImporterInstance> logger
) : IDisposable
internal sealed class ImporterInstance : IDisposable
{
private readonly RhinoCore _rhinoInstance = new(["/netcore-8"], WindowStyle.NoWindow);
private readonly RhinoDoc _rhinoDoc = OpenDocument(args, logger);
private readonly RhinoDoc _rhinoDoc;
private readonly IReadOnlyList<IDisposable> _scopes =
private readonly IReadOnlyList<IDisposable> _scopes;
private readonly ImporterArgs _args;
private readonly Sender _sender;
private readonly IClient _speckleClient;
private readonly ILogger<ImporterInstance> _logger;
private readonly ISdkActivityFactory _activityFactory;
public ImporterInstance(
ImporterArgs args,
Sender sender,
IClient speckleClient,
ILogger<ImporterInstance> logger,
ISdkActivityFactory activityFactory
)
{
_args = args;
_sender = sender;
_speckleClient = speckleClient;
_logger = logger;
_activityFactory = activityFactory;
_rhinoDoc = OpenDocument();
_scopes =
[
ActivityScope.SetTag("jobId", args.JobId),
ActivityScope.SetTag("job.attempt", args.Attempt.ToString()),
// ActivityScope.SetTag("jobType", args.JobType),
ActivityScope.SetTag("serverUrl", args.Account.serverInfo.url),
ActivityScope.SetTag("serverUrl", new Uri(args.Account.serverInfo.url).ToString()),
ActivityScope.SetTag("projectId", args.Project.id),
ActivityScope.SetTag("modelIngestion.Id", args.Ingestion.id),
ActivityScope.SetTag("modelIngestion.id", args.Ingestion.id),
ActivityScope.SetTag("modelId", args.Ingestion.modelId),
ActivityScope.SetTag("blobId", args.BlobId),
ActivityScope.SetTag("fileType", Path.GetExtension(args.FilePath).TrimStart('.')),
UserActivityScope.AddUserScope(args.Account),
];
}
public async Task<SerializeProcessResults> RunRhinoImport(CancellationToken cancellationToken)
public async Task<RootObjectBuilderResult> RunRhinoImport(CancellationToken cancellationToken)
{
using var activity = _activityFactory.Start();
try
{
RhinoDoc.ActiveDoc = _rhinoDoc;
var results = await sender
.Send(args.Project, args.Ingestion, speckleClient, cancellationToken)
var results = await _sender
.Send(_args.Project, _args.Ingestion, _speckleClient, cancellationToken)
.ConfigureAwait(false);
activity?.SetStatus(SdkActivityStatusCode.Ok);
return results;
}
catch (Exception ex)
{
activity?.RecordException(ex);
activity?.SetStatus(SdkActivityStatusCode.Error);
throw;
}
finally
{
RhinoDoc.ActiveDoc = null;
}
}
private static RhinoDoc OpenDocument(ImporterArgs args, ILogger logger)
private RhinoDoc OpenDocument()
{
using var config = GetConfig(Path.GetExtension(args.FilePath));
logger.LogInformation("Opening file {FilePath}", args.FilePath);
return config.OpenInHeadlessDocument(args.FilePath);
using var activity = _activityFactory.Start();
try
{
using var config = GetConfig(Path.GetExtension(_args.FilePath));
RhinoDoc openedDoc = config.OpenInHeadlessDocument(_args.FilePath);
activity?.SetStatus(SdkActivityStatusCode.Ok);
return openedDoc;
}
catch (Exception ex)
{
activity?.RecordException(ex);
activity?.SetStatus(SdkActivityStatusCode.Error);
throw;
}
}
[Pure]
@@ -82,6 +121,6 @@ internal sealed class ImporterInstance(
{
scope.Dispose();
}
speckleClient.Dispose();
_speckleClient.Dispose();
}
}
@@ -1,17 +1,19 @@
using Microsoft.Extensions.Logging;
using Speckle.Sdk.Api;
using Speckle.Sdk.Logging;
namespace Speckle.Importers.Rhino.Internal;
internal sealed class ImporterInstanceFactory(
Sender sender,
IClientFactory clientFactory,
ILogger<ImporterInstance> logger
ILogger<ImporterInstance> logger,
ISdkActivityFactory activityFactory
)
{
public ImporterInstance Create(ImporterArgs args)
{
var speckleClient = clientFactory.Create(args.Account);
return new(args, sender, speckleClient, logger);
return new(args, sender, speckleClient, logger, activityFactory);
}
}
@@ -3,8 +3,7 @@ using Microsoft.Extensions.Logging;
using Rhino;
using Rhino.DocObjects;
using Speckle.Connectors.Common.Analytics;
using Speckle.Connectors.Common.Operations;
using Speckle.Connectors.Common.Operations.Send;
using Speckle.Connectors.Common.Builders;
using Speckle.Converters.Common;
using Speckle.Converters.Rhino;
using Speckle.Sdk;
@@ -12,7 +11,8 @@ using Speckle.Sdk.Api;
using Speckle.Sdk.Api.GraphQL.Models;
using Speckle.Sdk.Credentials;
using Speckle.Sdk.Logging;
using Speckle.Sdk.Serialisation.V2.Send;
using Speckle.Sdk.Pipelines.Progress;
using Speckle.Sdk.Pipelines.Send;
namespace Speckle.Importers.Rhino.Internal;
@@ -22,10 +22,11 @@ internal sealed class Sender(
IRhinoConversionSettingsFactory rhinoConversionSettingsFactory,
IMixPanelManager mixpanel,
IIngestionProgressManagerFactory progressManagerFactory,
ISendPipelineFactory sendPipelineFactory,
ILogger<Sender> logger
)
{
public async Task<SerializeProcessResults> Send(
public async Task<RootObjectBuilderResult> Send(
Project project,
ModelIngestion ingestion,
IClient speckleClient,
@@ -35,7 +36,6 @@ internal sealed class Sender(
var progressManager = progressManagerFactory.CreateInstance(
speckleClient,
ingestion,
project.id,
TimeSpan.FromSeconds(1.5),
cancellationToken
);
@@ -56,20 +56,30 @@ internal sealed class Sender(
throw new SpeckleException("There are no objects found in the file");
}
var operation = scope.ServiceProvider.GetRequiredService<SendOperation<RhinoObject>>();
var buildResults = await operation.Build(rhinoObjects, project.id, progressManager, cancellationToken);
var results = await operation.SendObjects(
buildResults.RootObject,
var rootContinuousTraversalBuilder = scope.ServiceProvider.GetRequiredService<
IRootContinuousTraversalBuilder<RhinoObject>
>();
var sendPipeline = sendPipelineFactory.CreateInstance(
project.id,
ingestion.id,
speckleClient.Account,
new RenderedStreamProgress(progressManager),
cancellationToken
);
var buildResult = await rootContinuousTraversalBuilder.Build(
rhinoObjects,
project.id,
sendPipeline,
progressManager,
cancellationToken
);
buildResult.RootObject["version"] = 3;
await TrackSendMetrics(project, speckleClient.Account);
logger.LogInformation("Root: {RootId}", results.RootId);
return results;
logger.LogInformation("Root: {RootId}", buildResult.RootObject.id);
return buildResult;
}
private async Task TrackSendMetrics(Project project, Account account)
@@ -1,35 +1,120 @@
using Microsoft.Extensions.DependencyInjection;
using System.Reflection;
using Microsoft.Extensions.DependencyInjection;
using Speckle.Connectors.Common;
using Speckle.Connectors.Common.Operations.Send;
using Speckle.Connectors.Common.Common;
using Speckle.Connectors.Common.Threading;
using Speckle.Connectors.Logging;
using Speckle.Connectors.Rhino.DependencyInjection;
using Speckle.Converters.Rhino;
using Speckle.Objects.Geometry;
using Speckle.Sdk;
using Speckle.Sdk.SQLite;
#if !DEBUG && !LOCAL
using Speckle.Sdk.Common;
#endif
namespace Speckle.Importers.Rhino.Internal;
internal static class ServiceRegistration
{
public static IServiceCollection AddRhinoImporter(this IServiceCollection services, Application applicationInfo)
{
services.Initialize(applicationInfo, HostAppVersion.v8);
services.AddSingleton(applicationInfo);
private const HostAppVersion HOST_APP_VERSION = HostAppVersion.v3;
services.AddRhino(false);
services.AddRhinoConverters();
services.AddTransient<Sender>();
services.AddTransient<ImporterInstance>();
services.AddTransient<ImporterInstanceFactory>();
services.AddTransient<IIngestionProgressManager, IngestionProgressManager>();
services.AddTransient<IIngestionProgressManagerFactory, IngestionProgressManagerFactory>();
public static IServiceCollection AddRhinoImporter(
this IServiceCollection serviceCollection,
Application applicationInfo
)
{
var assemblyVersion = Assembly.GetExecutingAssembly().GetVersion();
serviceCollection.AddSpeckleSdk(
applicationInfo,
HostApplications.GetVersion(HOST_APP_VERSION),
assemblyVersion,
typeof(Point).Assembly
);
serviceCollection.AddSingleton(applicationInfo);
serviceCollection.AddRhino(false);
serviceCollection.AddRhinoConverters();
serviceCollection.AddTransient<Sender>();
serviceCollection.AddTransient<ImporterInstance>();
serviceCollection.AddTransient<ImporterInstanceFactory>();
// override default thread context
services.AddSingleton<IThreadContext>(new ImporterThreadContext());
serviceCollection.AddSingleton<IThreadContext>(new ImporterThreadContext());
// override sqlite cache, since we don't want to persist to disk any object data
services.AddTransient<ISqLiteJsonCacheManagerFactory, DummySqliteJsonCacheManagerFactory>();
serviceCollection.AddTransient<ISqLiteJsonCacheManagerFactory, DummySqliteJsonCacheManagerFactory>();
return services;
return serviceCollection;
}
// Important to respect disposal, because disposal ensures pending messages are flushed
public static IDisposable AddLoggingConfig(this IServiceCollection serviceCollection, Application applicationInfo)
{
return serviceCollection.AddOpenTelemetry(
"Speckle.Importers.Rhino",
applicationInfo,
HOST_APP_VERSION,
#if DEBUG || LOCAL
new SpeckleLogging(Console: true, File: new(), MinimumLevel: SpeckleLogLevel.Debug),
new SpeckleTracing(Console: false),
new SpeckleMetrics(Console: false)
#else
new SpeckleLogging(
Console: true,
File: new(),
Otel:
[
new(
Endpoint: new Uri("https://seq.speckle.systems/ingest/otlp/v1/logs"),
Headers: new()
{
// We're using a different token than connectors for seq because we want to beable to
// trust the client's timestamps (rather than use the server's timestamps) for better tracing
// This setting has more opportunity for abuse, so we're keeping it secret, unlike the connectors token.
{ "X-Seq-ApiKey", Environment.GetEnvironmentVariable("SEQ_API_KEY").NotNullOrWhiteSpace() }
}
),
new(
Endpoint: new Uri("https://collector.speckle.dev/v1/logs"),
Headers: new()
{
{
"authorization",
Environment.GetEnvironmentVariable("SPECKLE_COLLECTOR_API_TOKEN").NotNullOrWhiteSpace()
}
}
)
],
MinimumLevel: SpeckleLogLevel.Information
),
new SpeckleTracing(
Console: false,
Otel:
[
new(
Endpoint: new Uri("https://seq.speckle.systems/ingest/otlp/v1/traces"),
Headers: new()
{
{ "X-Seq-ApiKey", Environment.GetEnvironmentVariable("SEQ_API_KEY").NotNullOrWhiteSpace() }
}
),
new(
Endpoint: new Uri("https://collector.speckle.dev/v1/traces"),
Headers: new()
{
{
"authorization",
Environment.GetEnvironmentVariable("SPECKLE_COLLECTOR_API_TOKEN").NotNullOrWhiteSpace()
}
}
)
]
),
null
#endif
);
}
}
@@ -4,7 +4,9 @@ using System.Text.Json.Serialization;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using RhinoInside;
using Speckle.Connectors.Logging;
using Speckle.Importers.Rhino.Internal;
using Speckle.Sdk.Logging;
namespace Speckle.Importers.Rhino;
@@ -22,6 +24,8 @@ public static class Program
[SuppressMessage("Design", "CA1031:Do not catch general exception types", Justification = "IPC")]
public static async Task Main(string[] args)
{
//Thread.Sleep(10000); //For Debugging purposes, gives you enough time to attach your IDE to the running process
ILogger? logger = null;
ImporterInstance? importer = null;
@@ -31,11 +35,17 @@ public static class Program
var serviceCollection = new ServiceCollection();
serviceCollection.AddRhinoImporter(importerArgs.HostApplication);
using var otel = serviceCollection.AddLoggingConfig(importerArgs.HostApplication);
using var serviceProvider = serviceCollection.BuildServiceProvider();
logger = serviceProvider.GetRequiredService<ILogger<object>>();
TaskScheduler.UnobservedTaskException += (_, eventArgs) =>
logger.LogCritical(eventArgs.Exception, "Unobserved Task Exception");
ISdkActivityFactory activityFactory = serviceProvider.GetRequiredService<ISdkActivityFactory>();
using var activity = importerArgs.TraceContext is not null
? activityFactory.StartRemote(importerArgs.TraceContext, SdkActivityKind.Consumer)
: activityFactory.Start();
var factory = serviceProvider.GetRequiredService<ImporterInstanceFactory>();
// Error handling flow below here looks a bit of a mess, but we're having to navigate threading issues with rhino inside
@@ -58,7 +68,7 @@ public static class Program
try
{
var results = await importer.RunRhinoImport(CancellationToken.None).ConfigureAwait(false);
WriteResult(new() { RootObjectId = results.RootId }, importerArgs.ResultsPath);
WriteResult(new() { RootObjectId = results.RootObject.id }, importerArgs.ResultsPath);
}
catch (Exception ex)
{
@@ -8,9 +8,9 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="RhinoCommon" IncludeAssets="compile; build" PrivateAssets="all" VersionOverride="8.25.25328.11001"/>
<PackageReference Include="Grasshopper" IncludeAssets="compile; build" PrivateAssets="all" VersionOverride="8.25.25328.11001"/>
<PackageReference Include="RhinoWindows" IncludeAssets="compile; build" PrivateAssets="all" VersionOverride="8.25.25328.11001"/>
<PackageReference Include="RhinoCommon" IncludeAssets="compile; build" PrivateAssets="all" />
<PackageReference Include="Grasshopper" IncludeAssets="compile; build" PrivateAssets="all" />
<PackageReference Include="RhinoWindows" IncludeAssets="compile; build" PrivateAssets="all" />
<PackageReference Include="Rhino.Inside" />
</ItemGroup>
@@ -4,11 +4,11 @@
"net8.0-windows7.0": {
"Grasshopper": {
"type": "Direct",
"requested": "[8.25.25328.11001, )",
"resolved": "8.25.25328.11001",
"contentHash": "1uFL9pmgCEbYFd1b3JFGHaLEjQuRgFsiRHmP5u73mdEMdO6+5F/pNV1dQZr3YGLDovUmgrc+jmpsme7wr/J01A==",
"requested": "[8.28.26041.11001, )",
"resolved": "8.28.26041.11001",
"contentHash": "xlLD67F4/c5quwUrLxtAXT3M4Xam7GnhXyepC8/AblIBFcWbKEBzfOv85FlzO0ZPmDJwNjM5kRYML/sAStktZg==",
"dependencies": {
"RhinoCommon": "[8.25.25328.11001]"
"RhinoCommon": "[8.28.26041.11001]"
}
},
"Microsoft.NETFramework.ReferenceAssemblies": {
@@ -48,20 +48,20 @@
},
"RhinoCommon": {
"type": "Direct",
"requested": "[8.25.25328.11001, )",
"resolved": "8.25.25328.11001",
"contentHash": "PDKR9GwqyUXUkTulV4J0dzDIf/aWqSJkL7nkS8ReAx8xhnt/+RQpE8gTjOSCmkSU2tjG6WzclowbTxwMTU7VAA==",
"requested": "[8.28.26041.11001, )",
"resolved": "8.28.26041.11001",
"contentHash": "5mByZF+IdHvRLbYvr6Ek95Pwam6otewAyVHIGSC0sUE1+OscSpd9gbrFWnzo1arfCYmUJcUdsGhf7VayW09fQA==",
"dependencies": {
"System.Drawing.Common": "7.0.0"
}
},
"RhinoWindows": {
"type": "Direct",
"requested": "[8.25.25328.11001, )",
"resolved": "8.25.25328.11001",
"contentHash": "I/+++piwtYTue+iAAQqcMF5QlontqwNnC7Leyhiv2FiF8JpAl6K44ZsJqB7ZEUC6ns0LDfa3mbFzQwUfHwYumQ==",
"requested": "[8.28.26041.11001, )",
"resolved": "8.28.26041.11001",
"contentHash": "7MBG231k5c0/V/USTzbXpaTCiZCECQOuB80DnpgEvvEA3r//IQQDTbrqawDYmN9BEw/FHiFn82bsf6tV+SS5Iw==",
"dependencies": {
"RhinoCommon": "[8.25.25328.11001]"
"RhinoCommon": "[8.28.26041.11001]"
}
},
"Speckle.InterfaceGenerator": {
@@ -12,6 +12,7 @@ using Speckle.Sdk.Api.GraphQL.Models;
using Speckle.Sdk.Credentials;
using Speckle.Sdk.Logging;
using Speckle.Sdk.Models;
using Speckle.Sdk.Pipelines.Progress;
using Speckle.Testing;
namespace Speckle.Connectors.Common.Tests.Operations;
@@ -1,6 +1,7 @@
using Moq;
using NUnit.Framework;
using Speckle.Connectors.Common.Operations;
using Speckle.Sdk.Pipelines.Progress;
using Speckle.Sdk.Transports;
using Speckle.Testing;
@@ -1,6 +1,7 @@
using Moq;
using NUnit.Framework;
using Speckle.Connectors.Common.Operations;
using Speckle.Sdk.Pipelines.Progress;
using Speckle.Sdk.Transports;
using Speckle.Testing;
@@ -1,6 +1,6 @@
using Speckle.Connectors.Common.Conversion;
using Speckle.Connectors.Common.Operations;
using Speckle.Sdk.Models;
using Speckle.Sdk.Pipelines.Progress;
namespace Speckle.Connectors.Common.Builders;
@@ -1,6 +1,7 @@
using Speckle.Connectors.Common.Conversion;
using Speckle.Connectors.Common.Operations;
using Speckle.Sdk.Models;
using Speckle.Sdk.Pipelines.Progress;
using Speckle.Sdk.Pipelines.Send;
namespace Speckle.Connectors.Common.Builders;
@@ -14,4 +15,15 @@ public interface IRootObjectBuilder<in T>
);
}
public interface IRootContinuousTraversalBuilder<in T>
{
public Task<RootObjectBuilderResult> Build(
IReadOnlyList<T> objects,
string projectId,
SendPipeline sendPipeline,
IProgress<CardProgress> onOperationProgressed,
CancellationToken cancellationToken
);
}
public record RootObjectBuilderResult(Base RootObject, IReadOnlyList<SendConversionResult> ConversionResults);
@@ -15,6 +15,8 @@ public interface ISendConversionCache
{
void StoreSendResult(string projectId, IReadOnlyDictionary<Id, ObjectReference> convertedReferences);
void AppendSendResult(string projectId, string applicationId, ObjectReference convertedReference);
/// <summary>
/// <para>Call this method whenever you need to invalidate a set of objects that have changed in the host app.</para>
/// <para><b>Failure to do so correctly will result in cache poisoning and incorrect version creation (stale objects).</b></para>
@@ -13,6 +13,8 @@ public class NullSendConversionCache : ISendConversionCache
public void EvictObjects(IEnumerable<string> objectIds) { }
public void AppendSendResult(string projectId, string applicationId, ObjectReference convertedReference) { }
public void ClearCache() { }
public bool TryGetValue(
@@ -17,6 +17,11 @@ public class SendConversionCache : ISendConversionCache
}
}
public void AppendSendResult(string projectId, string applicationId, ObjectReference convertedReference)
{
Cache[(applicationId, projectId)] = convertedReference;
}
/// <inheritdoc/>
public void EvictObjects(IEnumerable<string> objectIds) =>
Cache = Cache
+5 -2
View File
@@ -37,7 +37,8 @@ public static class Connector
typeof(Point).Assembly
);
return serviceCollection.AddSeqLogging(
return serviceCollection.AddOpenTelemetry(
"Connector",
application,
version,
#if DEBUG || LOCAL
@@ -72,8 +73,9 @@ public static class Connector
);
}
public static IDisposable AddSeqLogging(
public static IDisposable AddOpenTelemetry(
this IServiceCollection serviceCollection,
string serviceName,
Application application,
HostAppVersion version,
SpeckleLogging loggingConfig,
@@ -83,6 +85,7 @@ public static class Connector
{
var assemblyVersion = Assembly.GetExecutingAssembly().GetVersion();
var (logging, tracing, metrics) = Observability.Initialize(
serviceName,
application.Name + " " + HostApplications.GetVersion(version),
application.Slug,
assemblyVersion,
@@ -1,3 +1,4 @@
using Speckle.Newtonsoft.Json;
using Speckle.Sdk.Common;
using Speckle.Sdk.Models;
@@ -34,8 +35,12 @@ public sealed class SendConversionResult : ConversionResult
SourceType = sourceType;
ResultId = result?.id;
ResultType = result?.speckle_type;
Result = result;
Error = FormatError(exception);
}
[JsonIgnore]
public Base? Result { get; }
}
// HACK: I've unsealed this for Grasshopper, non-ideal. Should be discussed and a better pattern may be implemented.
@@ -63,7 +68,6 @@ public class ReceiveConversionResult : ConversionResult
/// send conversion result or a receive conversion result - but i do not believe this requires fully separate classes, especially
/// for what this is meant to be at its core: a list of green or red checkmarks in the UI. To make DX easier, the two classes above embody
/// this one and provided clean constructors for each case.
/// POC: Inherits from Base so we can attach the conversion report to the root commit object. Can be revisited later (it's not a problem to not inherit from base).
/// </summary>
public abstract class ConversionResult
{
@@ -6,6 +6,21 @@ namespace Speckle.Connectors.Common.Extensions;
public static class RootObjectBuilderExtensions
{
public static void LogSendConversionError<T>(
this ILogger<IRootContinuousTraversalBuilder<T>> logger,
Exception ex,
string objectType
)
{
LogLevel logLevel = ex switch
{
SpeckleException => LogLevel.Information,
_ => LogLevel.Error
};
logger.Log(logLevel, ex, "Conversion of object {ObjectType} was not successful", objectType);
}
public static void LogSendConversionError<T>(
this ILogger<IRootObjectBuilder<T>> logger,
Exception ex,
@@ -1,21 +0,0 @@
using System.Diagnostics;
namespace Speckle.Connectors.Common.Extensions;
public static class StopwatchPollyfills
{
#if !NET7_0_OR_GREATER
private static readonly double s_tickFrequency = (double)TimeSpan.TicksPerSecond / Stopwatch.Frequency;
#endif
public static TimeSpan GetElapsedTime(long startingTimestamp)
{
#if NET7_0_OR_GREATER
return Stopwatch.GetElapsedTime(startingTimestamp);
#else
long elapsedTicks = Stopwatch.GetTimestamp() - startingTimestamp;
return new TimeSpan((long)(elapsedTicks * s_tickFrequency));
#endif
}
}
@@ -1,6 +1,6 @@
using Speckle.Connectors.Common.Operations;
using Speckle.Sdk.Models.Collections;
using Speckle.Sdk.Models.Instances;
using Speckle.Sdk.Pipelines.Progress;
namespace Speckle.Connectors.Common.Instances;
@@ -1,3 +0,0 @@
namespace Speckle.Connectors.Common.Operations;
public readonly record struct CardProgress(string Status, double? Progress);
@@ -8,6 +8,7 @@ using Speckle.Sdk.Credentials;
using Speckle.Sdk.Logging;
using Speckle.Sdk.Models;
using Speckle.Sdk.Models.Extensions;
using Speckle.Sdk.Pipelines.Progress;
namespace Speckle.Connectors.Common.Operations;
@@ -1,4 +1,5 @@
using Speckle.InterfaceGenerator;
using Speckle.Sdk.Pipelines.Progress;
using Speckle.Sdk.Transports;
namespace Speckle.Connectors.Common.Operations;
@@ -1,19 +0,0 @@
namespace Speckle.Connectors.Common.Operations.Send;
public sealed class AggregateProgress<T> : IProgress<T>
{
private readonly IProgress<T>[] _progresses;
public AggregateProgress(params IProgress<T>[] progresses)
{
_progresses = progresses;
}
public void Report(T value)
{
foreach (var progress in _progresses)
{
progress.Report(value);
}
}
}
@@ -1,92 +0,0 @@
using System.Diagnostics;
using Microsoft.Extensions.Logging;
using Speckle.Connectors.Common.Extensions;
using Speckle.InterfaceGenerator;
using Speckle.Sdk.Api;
using Speckle.Sdk.Api.GraphQL.Inputs;
using Speckle.Sdk.Api.GraphQL.Models;
namespace Speckle.Connectors.Common.Operations.Send;
public partial interface IIngestionProgressManager : IProgress<CardProgress>;
/// <summary>
/// An <see langword="IProgress{IngestionProgressEventArgs}"/> implementation for the entire client side Ingestion progress update reporting
/// Will throttles ingestion progress messages and reports their progress
/// </summary>
/// <remarks>
/// The same class exists also in the RVT ODA codebase
/// </remarks>
[GenerateAutoInterface]
public sealed class IngestionProgressManager(
ILogger<IngestionProgressManager> logger,
IClient speckleClient,
ModelIngestion ingestion,
string projectId,
TimeSpan updateInterval,
CancellationToken cancellationToken
) : IIngestionProgressManager
{
/// <remarks>
/// We've picked quite a coarse throttle window to try and avoid over pressure
/// </remarks>
private Task? _lastUpdate;
private long _lastUpdatedAt;
private readonly object _lock = new();
[AutoInterfaceIgnore]
public void Report(CardProgress value)
{
cancellationToken.ThrowIfCancellationRequested();
string trimmedMessage;
lock (_lock)
{
if (ShouldIgnoreProgressUpdate())
{
return;
}
_lastUpdatedAt = Stopwatch.GetTimestamp();
trimmedMessage = value.Status.TrimEnd('.');
_lastUpdate = speckleClient
.Ingestion.UpdateProgress(
new ModelIngestionUpdateInput(ingestion.id, projectId, trimmedMessage, value.Progress),
cancellationToken
)
.ContinueWith(
HandleFaultedContinuation,
CancellationToken.None,
TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously,
TaskScheduler.Default
);
}
logger.LogInformation("Progress update {Message} {Progress}", trimmedMessage, value.Progress);
}
/// <returns><see langword="true"/> if the update should be ignored, otherwise <see langword="false"/></returns>
private bool ShouldIgnoreProgressUpdate()
{
if (_lastUpdate is not null && !_lastUpdate.IsCompleted)
{
return true;
}
TimeSpan msSinceLastUpdate = StopwatchPollyfills.GetElapsedTime(_lastUpdatedAt);
return msSinceLastUpdate < updateInterval;
}
private void HandleFaultedContinuation(Task updateTask)
{
// The progress report failed... could be many reasons.
// For now, we're not letting this fail the Ingestion in any way
// we'll log but otherwise let it slide while leaving no unobserved task exceptions
if (updateTask.IsFaulted)
{
logger.LogWarning(updateTask.Exception, "A progress update failed unexpectedly");
}
}
}
@@ -1,22 +0,0 @@
using Microsoft.Extensions.Logging;
using Speckle.InterfaceGenerator;
using Speckle.Sdk.Api;
using Speckle.Sdk.Api.GraphQL.Models;
namespace Speckle.Connectors.Common.Operations.Send;
[GenerateAutoInterface]
public sealed class IngestionProgressManagerFactory(ILogger<IngestionProgressManager> logger)
: IIngestionProgressManagerFactory
{
public IIngestionProgressManager CreateInstance(
IClient speckleClient,
ModelIngestion ingestion,
string projectId,
TimeSpan updateInterval,
CancellationToken cancellationToken
)
{
return new IngestionProgressManager(logger, speckleClient, ingestion, projectId, updateInterval, cancellationToken);
}
}
@@ -1,7 +1,6 @@
using Speckle.Connectors.Common.Builders;
using Speckle.Connectors.Common.Caching;
using Speckle.Connectors.Common.Conversion;
using Speckle.Connectors.Common.Operations.Send;
using Speckle.Connectors.Common.Threading;
using Speckle.Connectors.Logging;
using Speckle.InterfaceGenerator;
@@ -10,11 +9,17 @@ using Speckle.Sdk.Api;
using Speckle.Sdk.Api.GraphQL.Inputs;
using Speckle.Sdk.Api.GraphQL.Models;
using Speckle.Sdk.Credentials;
using Speckle.Sdk.Helpers;
using Speckle.Sdk.Logging;
using Speckle.Sdk.Models;
using Speckle.Sdk.Pipelines.Progress;
using Speckle.Sdk.Pipelines.Send;
using Speckle.Sdk.Serialisation;
using Speckle.Sdk.Serialisation.V2.Send;
using Version = Speckle.Sdk.Api.GraphQL.Models.Version;
#if !NET8_0_OR_GREATER
using System.Net.Http;
#endif
namespace Speckle.Connectors.Common.Operations;
@@ -24,13 +29,16 @@ public sealed class SendOperation<T>(
ISendConversionCache sendConversionCache,
ISendProgress sendProgress,
ISendOperationExecutor sendOperationExecutor,
ISdkActivityFactory activityFactory,
IThreadContext threadContext,
ISdkActivityFactory activityFactory,
ISpeckleApplication speckleApplication,
IIngestionProgressManagerFactory ingestionProgressManagerFactory
IIngestionProgressManagerFactory ingestionProgressManagerFactory,
ISpeckleHttp speckleHttp,
ISendPipelineFactory sendPipelineFactory,
IRootContinuousTraversalBuilder<T>? rootContinuousTraversalBuilder = null
) : ISendOperation<T>
{
public async Task<(SendOperationResult sendResult, string versionId)> Send(
public async Task<(SendOperationResult sendResult, string versionId, string? ingestionId)> Send(
IReadOnlyList<T> objects,
SendInfo sendInfo,
string? fileName,
@@ -43,6 +51,20 @@ public sealed class SendOperation<T>(
bool useModelIngestionSend = await CheckUseModelIngestionSend(sendInfo);
if (useModelIngestionSend)
{
bool usePackfileSend =
rootContinuousTraversalBuilder != null && await CheckPackfileSendEndpoints(sendInfo, cancellationToken);
if (usePackfileSend)
{
return await SendViaPackfile(
objects,
sendInfo,
fileName,
fileSizeBytes,
versionMessage,
uiProgress,
cancellationToken
);
}
return await SendViaIngestion(
objects,
sendInfo,
@@ -59,22 +81,30 @@ public sealed class SendOperation<T>(
}
}
private async Task<(SendOperationResult sendResult, string versionId)> SendViaIngestion(
private async Task<(SendOperationResult sendResult, string versionId, string? ingestionId)> SendViaPackfile(
IReadOnlyList<T> objects,
SendInfo sendInfo,
string? fileName,
long? fileSizeBytes,
#pragma warning disable IDE0060
string? versionMessage,
#pragma warning restore IDE0060
IProgress<CardProgress> uiProgress,
CancellationToken cancellationToken
)
{
if (rootContinuousTraversalBuilder == null)
{
throw new InvalidOperationException("rootContinuousTraversalBuilder cannot be null");
}
ModelIngestion ingestion = await sendInfo.Client.Ingestion.Create(
new(
sendInfo.ModelId,
sendInfo.ProjectId,
$"Sending from {speckleApplication.ApplicationAndVersion}",
new(speckleApplication.Slug, speckleApplication.HostApplicationVersion, fileName, fileSizeBytes)
new(speckleApplication.Slug, speckleApplication.HostApplicationVersion, fileName, fileSizeBytes),
600
),
cancellationToken
);
@@ -83,21 +113,42 @@ public sealed class SendOperation<T>(
var ingestionProgress = ingestionProgressManagerFactory.CreateInstance(
sendInfo.Client,
ingestion,
sendInfo.ProjectId,
TimeSpan.FromSeconds(5),
TimeSpan.FromSeconds(10),
cancellationToken
);
AggregateProgress<CardProgress> progress = new(ingestionProgress, uiProgress);
try
{
SendOperationResult result = await ConvertAndSend(objects, sendInfo, progress, cancellationToken);
string createdVersionId = await sendInfo.Client.Ingestion.Complete(
new(ingestion.id, sendInfo.ProjectId, result.RootObjId, versionMessage),
CancellationToken.None
var sendPipeline = sendPipelineFactory.CreateInstance(
sendInfo.ProjectId,
ingestion.id,
sendInfo.Account,
new RenderedStreamProgress(progress),
cancellationToken
);
var buildResult = await rootContinuousTraversalBuilder.Build(
objects,
sendInfo.ProjectId,
sendPipeline,
progress,
cancellationToken
);
return (result, createdVersionId);
buildResult.RootObject["version"] = 3;
WriteReferencesToCache(buildResult.ConversionResults, sendInfo.ProjectId);
SendOperationResult result =
new(buildResult.RootObject.id!, new Dictionary<Id, ObjectReference>(), buildResult.ConversionResults);
// NOTE: clients do not need to complete the ingestion - that's going to the be the server's job
// string createdVersionId = await sendInfo.Client.Ingestion.Complete(
// new(ingestion.id, sendInfo.ProjectId, result.RootObjId, versionMessage),
// CancellationToken.None
// );
return (result, "latest", ingestion.id);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
@@ -117,7 +168,68 @@ public sealed class SendOperation<T>(
}
}
private async Task<(SendOperationResult sendResult, string versionId)> SendViaVersionCreate(
private async Task<(SendOperationResult sendResult, string versionId, string? ingestionId)> SendViaIngestion(
IReadOnlyList<T> objects,
SendInfo sendInfo,
string? fileName,
long? fileSizeBytes,
string? versionMessage,
IProgress<CardProgress> uiProgress,
CancellationToken cancellationToken
)
{
ModelIngestion ingestion = await sendInfo.Client.Ingestion.Create(
new(
sendInfo.ModelId,
sendInfo.ProjectId,
$"Sending from {speckleApplication.ApplicationAndVersion}",
new(speckleApplication.Slug, speckleApplication.HostApplicationVersion, fileName, fileSizeBytes),
600
),
cancellationToken
);
using var ingestionScope = ActivityScope.SetTag("modelIngestionId", ingestion.id);
var ingestionProgress = ingestionProgressManagerFactory.CreateInstance(
sendInfo.Client,
ingestion,
TimeSpan.FromSeconds(5),
cancellationToken
);
AggregateProgress<CardProgress> progress = new(ingestionProgress, uiProgress);
try
{
SendOperationResult result = await ConvertAndSend(objects, sendInfo, progress, cancellationToken);
string createdVersionId = await sendInfo.Client.Ingestion.Complete(
new(ingestion.id, sendInfo.ProjectId, result.RootObjId, versionMessage),
CancellationToken.None
);
// NOTE: it might seem weird to pass null for ingestion.id 'null' here but there is a reason.
// Because we complete ingestion here in .NET which is safe to pass null ingestion id that we don't want DUI explicitly subscribe to ingestion changes.
// I am hoping we will get rid of from logical branching once we have model ingestion on public server
return (result, createdVersionId, null);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
_ = await sendInfo.Client.Ingestion.FailWithCancel(
new(ingestion.id, sendInfo.ProjectId, "User requested cancellation"),
CancellationToken.None
);
throw;
}
catch (Exception ex)
{
_ = await sendInfo.Client.Ingestion.FailWithError(
ModelIngestionFailedInput.FromException(ingestion.id, sendInfo.ProjectId, ex),
CancellationToken.None
);
throw;
}
}
private async Task<(SendOperationResult sendResult, string versionId, string? ingestionId)> SendViaVersionCreate(
IReadOnlyList<T> objects,
SendInfo sendInfo,
string? versionMessage,
@@ -137,7 +249,7 @@ public sealed class SendOperation<T>(
),
cancellationToken
);
return (result, version.id);
return (result, version.id, null);
}
public async Task<SendOperationResult> ConvertAndSend(
@@ -239,6 +351,66 @@ public sealed class SendOperation<T>(
return useModelIngestionSend;
}
/// <param name="sendInfo"></param>
/// <param name="cancellationToken"></param>
/// <exception cref="FormatException">server returned a response, but it was neither <c>true</c> nor <c>false</c> (case insensitive)</exception>
/// <exception cref="HttpRequestException ">Request failed, or the server returned a non-successful status code that wasn't <c>404</c></exception>
/// <returns>
/// Returns <see langword="true"/> if the server supports the new packfile data uploads,
/// <see langword="false"/> if the server doesn't explicitly, or implicitly via a <c>404</c> response.
/// Will throw for unexpected cases.
/// </returns>
private async Task<bool> CheckPackfileSendEndpoints(SendInfo sendInfo, CancellationToken cancellationToken)
{
Uri url = new Uri(new Uri(sendInfo.Account.serverInfo.url), "/api/v1/data-module-enabled");
using HttpClient client = speckleHttp.CreateHttpClient();
using var response = await client.GetAsync(url, cancellationToken);
if (response.StatusCode != System.Net.HttpStatusCode.NotFound)
{
response.EnsureSuccessStatusCode();
#if NET8_0_OR_GREATER
string responseBody = await response.Content.ReadAsStringAsync(cancellationToken);
#else
string responseBody = await response.Content.ReadAsStringAsync();
#endif
return bool.Parse(responseBody);
}
return false;
}
/// <summary>
/// Reads the conversion results and any <see cref="ObjectReference"/> will be written to cache.
/// All other values will be ignored.
/// </summary>
/// <remarks>
/// For the connectors that support send caching, we are reporting all results as either <see cref="ObjectReference"/> or <see langword="null"/>
/// For Navisworks, which we no longer support send caching, it reports other <see cref="Base"/> subtypes, and those will not be cached.
/// </remarks>
/// <param name="conversionResults"></param>
/// <param name="projectId"></param>
private void WriteReferencesToCache(IReadOnlyList<SendConversionResult> conversionResults, string projectId)
{
// We write the objects to the cache after they've been uploaded to the server
// There is still an inbuilt "bad" assumption here that, successfully uploading NDJson means the server is able to re-materialize ids -
// but this is only true once the `Version` object is created
// Since for many reasons, the server could fail to process the json...
// This would leave this send cache out-of-sync with the server, and lead to failed processing of subsequent NDJson uploads
// For now, we've taken the decision that it's unlikely to happen...
var references = new Dictionary<Id, ObjectReference>();
foreach (var x in conversionResults)
{
if (x.Result is ObjectReference r)
{
// NOTE: why not ToDictionary -> we might end up reoccurring object references for any reason. instancing, linked models etc.
// ToDictionary throws 'item already exists' errors. but safe to override items in references dictionary since they are unique
references[new Id(x.SourceId)] = r;
}
}
sendConversionCache.StoreSendResult(projectId, references);
}
}
public record SendOperationResult(
@@ -1,4 +1,5 @@
using Speckle.InterfaceGenerator;
using Speckle.Sdk.Pipelines.Progress;
using Speckle.Sdk.Transports;
namespace Speckle.Connectors.Common.Operations;
@@ -19,4 +19,5 @@
<ItemGroup Condition="'$(Configuration)' != 'Local'">
<PackageReference Include="Speckle.Objects" />
</ItemGroup>
</Project>
+4 -2
View File
@@ -4,16 +4,18 @@ namespace Speckle.Connectors.Logging;
public static class Consts
{
public const string DEPLOYMENT_ENVIRONMENT = "deployment.environment.name";
public const string SERVICE_NAME = "connector.name";
public const string SERVICE_SLUG = "connector.slug";
public const string OS_NAME = "os.name";
public const string OS_TYPE = "os.type";
public const string OS_SLUG = "os.slug";
public const string RUNTIME_NAME = "runtime.name";
public const string RUNTIME_NAME = "process.runtime.name";
public const string RUNTIME_VERSION = "process.runtime.version";
public const string USER_ID = "user.id";
public const string USER_DISTINCT_ID = "user.distinctId";
public const string USER_SERVER_URL = "user.server_url";
public const string TRACING_SOURCE = "speckle";
public const string TRACING_SOURCE = "connector";
/// <summary>
/// A random GUID for adding to the logging context to correlate the <c>service.instance.id</c>
@@ -5,24 +5,33 @@ namespace Speckle.Connectors.Logging.Internal;
internal static class ResourceCreator
{
internal static ResourceBuilder Create(string applicationAndVersion, string slug, string connectorVersion) =>
ResourceBuilder
.CreateEmpty()
.AddService(
serviceName: Consts.TRACING_SOURCE,
serviceVersion: connectorVersion,
serviceInstanceId: Consts.StaticSessionId
internal static ResourceBuilder Create(
string serviceName,
string applicationAndVersion,
string slug,
string connectorVersion
)
{
string deploymentEnvironment =
Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT")
?? Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT")
?? "Development";
return ResourceBuilder
.CreateEmpty()
.AddService(serviceName: serviceName, serviceVersion: connectorVersion, serviceInstanceId: Consts.StaticSessionId)
.AddAttributes(
[
new(Consts.DEPLOYMENT_ENVIRONMENT, deploymentEnvironment.ToLowerInvariant()),
new(Consts.SERVICE_NAME, applicationAndVersion),
new(Consts.SERVICE_SLUG, slug),
new(Consts.OS_NAME, Environment.OSVersion.ToString()),
new(Consts.OS_TYPE, RuntimeInformation.ProcessArchitecture.ToString()),
new(Consts.OS_SLUG, DetermineHostOsSlug()),
new(Consts.RUNTIME_NAME, RuntimeInformation.FrameworkDescription),
new(Consts.RUNTIME_NAME, ".NET"),
new(Consts.RUNTIME_VERSION, RuntimeInformation.FrameworkDescription),
]
);
}
private static string DetermineHostOsSlug()
{
@@ -5,13 +5,14 @@ namespace Speckle.Connectors.Logging;
public static class Observability
{
public static (LoggerProvider, IDisposable, IDisposable) Initialize(
string serviceName,
string applicationAndVersion,
string slug,
string connectorVersion,
SpeckleObservability observability
)
{
var resourceBuilder = ResourceCreator.Create(applicationAndVersion, slug, connectorVersion);
var resourceBuilder = ResourceCreator.Create(serviceName, applicationAndVersion, slug, connectorVersion);
var logging = LogBuilder.Initialize(
applicationAndVersion,
connectorVersion,
@@ -41,23 +41,22 @@
},
"OpenTelemetry.Exporter.OpenTelemetryProtocol": {
"type": "Direct",
"requested": "[1.11.1, )",
"resolved": "1.11.1",
"contentHash": "UiZBa+2b396Oxx9RX7h4ch+yZvX8nezxVkihPLU6zdEUfJbbVY2mNypJKEoW2Vh4xCaCp0fB6na3Kti+KfTVaw==",
"requested": "[1.13.1, )",
"resolved": "1.13.1",
"contentHash": "WFqOpqMkjd8lM/asQRgfLP72UCqThQIGoylDgoYR8x0Bh9UCrdmBJCDU4pVZgI9CtSq1sWXeQuUsRXAl5U4yKg==",
"dependencies": {
"Grpc.Core": "[2.44.0, 3.0.0)",
"OpenTelemetry": "1.11.1"
"OpenTelemetry": "1.13.1"
}
},
"OpenTelemetry.Instrumentation.Http": {
"type": "Direct",
"requested": "[1.11.0, )",
"resolved": "1.11.0",
"contentHash": "1ncYPNmMaYNPX664uo3FlmSVGETBKQbBvarbGgB5ZynERTFmCsZ7UqefvVe3vnPYOIAGOjbMAbprYF2BfMielg==",
"requested": "[1.13.0, )",
"resolved": "1.13.0",
"contentHash": "B+nmCn3/orrLhUuCC6WwHh9JUkIV/wubpZ+vYOf2CedjOZupgcQcx96Kwy6UVjdNDWGxsEw0jXWXZUQlYnmRqA==",
"dependencies": {
"Microsoft.Extensions.Configuration": "9.0.0",
"Microsoft.Extensions.Options": "9.0.0",
"OpenTelemetry.Api.ProviderBuilderExtensions": "[1.11.1, 2.0.0)"
"OpenTelemetry.Api.ProviderBuilderExtensions": "[1.13.1, 2.0.0)"
}
},
"PolySharp": {
@@ -120,23 +119,6 @@
"resolved": "0.9.6",
"contentHash": "HKH7tYrYYlCK1ct483hgxERAdVdMtl7gUKW9ijWXxA1UsYR4Z+TrRHYmzZ9qmpu1NnTycSrp005NYM78GDKV1w=="
},
"Grpc.Core": {
"type": "Transitive",
"resolved": "2.44.0",
"contentHash": "H2rTNePSYeEqUgBBqiE5KZ/yOX7jEtETjWjv4gGtnYxrlpZ79VAHe+4KtZAnt2KJMiGvqTm7eo1SPk+QpiPfaw==",
"dependencies": {
"Grpc.Core.Api": "2.44.0",
"System.Memory": "4.5.3"
}
},
"Grpc.Core.Api": {
"type": "Transitive",
"resolved": "2.44.0",
"contentHash": "FBfPMvKwT8q98T8lWa5z6nBMLdH/Mmo5g4yyYYMvbXLWDzo4beqa7CUU5QH3PKvo2X6/b+UAZ2IymXlrYG3IXg==",
"dependencies": {
"System.Memory": "4.5.3"
}
},
"ILRepack": {
"type": "Transitive",
"resolved": "2.0.33",
@@ -264,29 +246,29 @@
},
"OpenTelemetry": {
"type": "Transitive",
"resolved": "1.11.1",
"contentHash": "F+HBI2bE7RKmb8Bj0kBtZIVzCfpTe1ZyY6kYP/jny1+9oq7IdBnNsVXZlPev9OqQzRp3iXpJ1UsnN1YOEwdtkQ==",
"resolved": "1.13.1",
"contentHash": "rrM2NlZ0Xla2Ar8zzU09n+HoLFq8b+Kjx7vrmR0tdYfWLYWGNcXHOITXUiyaK8RrFEceMEoF45VBsaf4QPmKcg==",
"dependencies": {
"Microsoft.Extensions.Diagnostics.Abstractions": "9.0.0",
"Microsoft.Extensions.Logging.Configuration": "9.0.0",
"OpenTelemetry.Api.ProviderBuilderExtensions": "1.11.1"
"OpenTelemetry.Api.ProviderBuilderExtensions": "1.13.1"
}
},
"OpenTelemetry.Api": {
"type": "Transitive",
"resolved": "1.11.1",
"contentHash": "KaBjGMqrqQv41mIkvPUvmAG7yxDlI6qchKhjXlOF3ZwsdcRRLrdrkiDLIJ90iZgUoKVdP8fE1fCri9nc+ug0Cg==",
"resolved": "1.13.1",
"contentHash": "tieglRERo7Rgu8oE8aamnuXCMPEW5fXIqO5ngTMCNk9pOEXanc0SdQ86ZAD1goNiGcjWHn+P3WMZp0FZSJgCoQ==",
"dependencies": {
"System.Diagnostics.DiagnosticSource": "9.0.0"
}
},
"OpenTelemetry.Api.ProviderBuilderExtensions": {
"type": "Transitive",
"resolved": "1.11.1",
"contentHash": "vMdNMQeW55jXIa/Kybec/br6jC+rWybniTi6DCW5lz1kGghKso+J+FC3uBgiq0/pTqusfeDbO5PEHGM/r5z8Ow==",
"resolved": "1.13.1",
"contentHash": "x8QXMsrIyp+XzDUFQAM+C4upAPNbwaBIPjTWEoonziWAav6weS8OxsMKrE4wz7Zly8ATlsoxk0mWZ+PHO3Wg0w==",
"dependencies": {
"Microsoft.Extensions.DependencyInjection.Abstractions": "9.0.0",
"OpenTelemetry.Api": "1.11.1"
"OpenTelemetry.Api": "1.13.1"
}
},
"System.Buffers": {
@@ -417,22 +399,22 @@
},
"OpenTelemetry.Exporter.OpenTelemetryProtocol": {
"type": "Direct",
"requested": "[1.11.1, )",
"resolved": "1.11.1",
"contentHash": "UiZBa+2b396Oxx9RX7h4ch+yZvX8nezxVkihPLU6zdEUfJbbVY2mNypJKEoW2Vh4xCaCp0fB6na3Kti+KfTVaw==",
"requested": "[1.13.1, )",
"resolved": "1.13.1",
"contentHash": "WFqOpqMkjd8lM/asQRgfLP72UCqThQIGoylDgoYR8x0Bh9UCrdmBJCDU4pVZgI9CtSq1sWXeQuUsRXAl5U4yKg==",
"dependencies": {
"OpenTelemetry": "1.11.1"
"OpenTelemetry": "1.13.1"
}
},
"OpenTelemetry.Instrumentation.Http": {
"type": "Direct",
"requested": "[1.11.0, )",
"resolved": "1.11.0",
"contentHash": "1ncYPNmMaYNPX664uo3FlmSVGETBKQbBvarbGgB5ZynERTFmCsZ7UqefvVe3vnPYOIAGOjbMAbprYF2BfMielg==",
"requested": "[1.13.0, )",
"resolved": "1.13.0",
"contentHash": "B+nmCn3/orrLhUuCC6WwHh9JUkIV/wubpZ+vYOf2CedjOZupgcQcx96Kwy6UVjdNDWGxsEw0jXWXZUQlYnmRqA==",
"dependencies": {
"Microsoft.Extensions.Configuration": "9.0.0",
"Microsoft.Extensions.Options": "9.0.0",
"OpenTelemetry.Api.ProviderBuilderExtensions": "[1.11.1, 2.0.0)"
"Microsoft.Extensions.Configuration": "8.0.0",
"Microsoft.Extensions.Options": "8.0.0",
"OpenTelemetry.Api.ProviderBuilderExtensions": "[1.13.1, 2.0.0)"
}
},
"PolySharp": {
@@ -594,29 +576,29 @@
},
"OpenTelemetry": {
"type": "Transitive",
"resolved": "1.11.1",
"contentHash": "F+HBI2bE7RKmb8Bj0kBtZIVzCfpTe1ZyY6kYP/jny1+9oq7IdBnNsVXZlPev9OqQzRp3iXpJ1UsnN1YOEwdtkQ==",
"resolved": "1.13.1",
"contentHash": "rrM2NlZ0Xla2Ar8zzU09n+HoLFq8b+Kjx7vrmR0tdYfWLYWGNcXHOITXUiyaK8RrFEceMEoF45VBsaf4QPmKcg==",
"dependencies": {
"Microsoft.Extensions.Diagnostics.Abstractions": "9.0.0",
"Microsoft.Extensions.Logging.Configuration": "9.0.0",
"OpenTelemetry.Api.ProviderBuilderExtensions": "1.11.1"
"OpenTelemetry.Api.ProviderBuilderExtensions": "1.13.1"
}
},
"OpenTelemetry.Api": {
"type": "Transitive",
"resolved": "1.11.1",
"contentHash": "KaBjGMqrqQv41mIkvPUvmAG7yxDlI6qchKhjXlOF3ZwsdcRRLrdrkiDLIJ90iZgUoKVdP8fE1fCri9nc+ug0Cg==",
"resolved": "1.13.1",
"contentHash": "tieglRERo7Rgu8oE8aamnuXCMPEW5fXIqO5ngTMCNk9pOEXanc0SdQ86ZAD1goNiGcjWHn+P3WMZp0FZSJgCoQ==",
"dependencies": {
"System.Diagnostics.DiagnosticSource": "9.0.0"
}
},
"OpenTelemetry.Api.ProviderBuilderExtensions": {
"type": "Transitive",
"resolved": "1.11.1",
"contentHash": "vMdNMQeW55jXIa/Kybec/br6jC+rWybniTi6DCW5lz1kGghKso+J+FC3uBgiq0/pTqusfeDbO5PEHGM/r5z8Ow==",
"resolved": "1.13.1",
"contentHash": "x8QXMsrIyp+XzDUFQAM+C4upAPNbwaBIPjTWEoonziWAav6weS8OxsMKrE4wz7Zly8ATlsoxk0mWZ+PHO3Wg0w==",
"dependencies": {
"Microsoft.Extensions.DependencyInjection.Abstractions": "9.0.0",
"OpenTelemetry.Api": "1.11.1"
"OpenTelemetry.Api": "1.13.1"
}
},
"System.Diagnostics.DiagnosticSource": {

Some files were not shown because too many files have changed in this diff Show More