wip pipe implementation

Connectors/Revit/Speckle.Connectors.RevitShared/Operations/Send/RevitRootObjectBuilder.cs

@@ -42,7 +42,11 @@ public class RevitRootObjectBuilder(
       () => Task.FromResult(BuildSync(documentElementContexts, projectId, onOperationProgressed, ct))
     );
 
+#pragma warning disable CA1506
+#pragma warning disable CA1502
   private RootObjectBuilderResult BuildSync(
+#pragma warning restore CA1506
+#pragma warning restore CA1502
     IReadOnlyList<DocumentToConvert> documentElementContexts,
     string projectId,
     IProgress<CardProgress> onOperationProgressed,
@@ -56,6 +60,9 @@ public class RevitRootObjectBuilder(
       throw new SpeckleException("Family Environment documents are not supported.");
     }
 
+    // create a new send pipeline
+    using var sendPipeline = new Speckle.Sdk.Pipeline.Send();
+
     // init the root
     Collection rootObject =
       new() { name = converterSettings.Current.Document.PathName.Split('\\').Last().Split('.').First() };
@@ -184,10 +191,12 @@ public class RevitRootObjectBuilder(
             // non-transformed elements can safely rely on cache
             // TODO: Potential here to transform cached objects and NOT reconvert,
             // TODO: we wont do !hasTransform here, and re-set application id before this
-
+            bool wasCached = false;
             if (!hasTransform && sendConversionCache.TryGetValue(projectId, applicationId, out ObjectReference? value))
             {
+              // TODO: cahce hit
               converted = value;
+              wasCached = true;
               cacheHitCount++;
             }
             // not in cache means we convert
@@ -206,6 +215,12 @@ public class RevitRootObjectBuilder(
               converted.applicationId = applicationId;
             }
 
+            var reference = sendPipeline.Process(converted).Result; // .Wait(cancellationToken);//.ConfigureAwait(false);
+            if (!wasCached)
+            {
+              sendConversionCache.AppendSendResult(projectId, applicationId, reference);
+            }
+
             var collection = sendCollectionManager.GetAndCreateObjectHostCollection(
               revitElement,
               rootObject,
@@ -213,7 +228,7 @@ public class RevitRootObjectBuilder(
               modelDisplayName
             );
 
-            collection.elements.Add(converted);
+            collection.elements.Add(reference);
             results.Add(new(Status.SUCCESS, applicationId, sourceType, converted));
           }
           catch (Exception ex) when (!ex.IsFatal())
@@ -254,13 +269,20 @@ public class RevitRootObjectBuilder(
     rootObject[ProxyKeys.INSTANCE_DEFINITION] = revitToSpeckleCacheSingleton.GetInstanceDefinitionProxiesForObjects(
       idsAndSubElementIds
     );
-    rootObject.elements.Add(
+    // NOTE: i might be overdoing things in here, but tldr:
-      new Collection()
+    // - all instance objects (meshes) are processed individually
-      {
+    // - process their collection individually, and then attach it to the root collection
-        elements = revitToSpeckleCacheSingleton.GetBaseObjectsForObjects(idsAndSubElementIds),
+    // we could, theoretically, just process the collection as a whole (but it can be big?)
-        name = "revitInstancedObjects"
+    // note/ask: do these need to go in the conversion cache? or not?
-      }
+    var instanceObjects = revitToSpeckleCacheSingleton.GetBaseObjectsForObjects(idsAndSubElementIds);
-    );
+    var instanceReferences = new Collection("revitInstancedObjects");
+    foreach (var instanceObject in instanceObjects)
+    {
+      var referenceInstanceObject = sendPipeline.Process(instanceObject).Result;
+      instanceReferences.elements.Add(referenceInstanceObject);
+    }
+    var instanceReferenceCollection = sendPipeline.Process(instanceReferences).Result;
+    rootObject.elements.Add(instanceReferenceCollection);
 
     // STEP 6: Unpack all other objects to attach to root collection
     List<Objects.Other.Camera> views = viewUnpacker.Unpack(converterSettings.Current.Document);
@@ -279,6 +301,10 @@ public class RevitRootObjectBuilder(
       rootObject[RootKeys.REFERENCE_POINT_TRANSFORM] = transformMatrix;
     }
 
-    return new RootObjectBuilderResult(rootObject, results);
+    // NOTE: could be
+    sendPipeline.Process(rootObject).Wait(cancellationToken);
+    sendPipeline.WaitForUpload().Wait(cancellationToken);
+
+    return new RootObjectBuilderResult(new Collection() { name = "ignore" }, results);
   }
 }

Connectors/Rhino/Speckle.Connectors.RhinoShared/Operations/Send/RhinoRootObjectBuilder.cs

@@ -75,6 +75,8 @@ public class RhinoRootObjectBuilder : IRootObjectBuilder<RhinoObject>
   )
   {
     using var activity = _activityFactory.Start("Build");
+    using var sendPipeline = new Speckle.Sdk.Pipeline.Send();
+
     // 0 - Init the root
     Collection rootObjectCollection = new() { name = _converterSettings.Current.Document.Name ?? "Unnamed document" };
     rootObjectCollection["units"] = _converterSettings.Current.SpeckleUnits;
@@ -97,6 +99,7 @@ public class RhinoRootObjectBuilder : IRootObjectBuilder<RhinoObject>
     // 3 - Convert atomic objects
     List<SendConversionResult> results = new(atomicObjects.Count);
     int count = 0;
+
     using (var _ = _activityFactory.Start("Convert all"))
     {
       foreach (RhinoObject rhinoObject in atomicObjects)
@@ -108,9 +111,8 @@ public class RhinoRootObjectBuilder : IRootObjectBuilder<RhinoObject>
         Layer layer = _converterSettings.Current.Document.Layers[rhinoObject.Attributes.LayerIndex];
         Collection collectionHost = _layerUnpacker.GetHostObjectCollection(layer, rootObjectCollection);
 
-        var result = ConvertRhinoObject(rhinoObject, collectionHost, instanceProxies, projectId);
+        var result = await ConvertRhinoObject(rhinoObject, collectionHost, instanceProxies, projectId, sendPipeline);
         results.Add(result);
-
         ++count;
         onOperationProgressed.Report(new("Converting", (double)count / atomicObjects.Count));
         await Task.Yield();
@@ -149,18 +151,23 @@ public class RhinoRootObjectBuilder : IRootObjectBuilder<RhinoObject>
       }
     }
 
-    return new RootObjectBuilderResult(rootObjectCollection, results);
+    await sendPipeline.Process(rootObjectCollection);
+    await sendPipeline.WaitForUpload();
+
+    return new RootObjectBuilderResult(new Collection() { name = "ignore" }, results);
   }
 
-  private SendConversionResult ConvertRhinoObject(
+  private async Task<SendConversionResult> ConvertRhinoObject(
     RhinoObject rhinoObject,
     Collection collectionHost,
     IReadOnlyDictionary<string, InstanceProxy> instanceProxies,
-    string projectId
+    string projectId,
+    Sdk.Pipeline.Send sendPipeline
   )
   {
     string applicationId = rhinoObject.Id.ToString();
     string sourceType = rhinoObject.ObjectType.ToString();
+    bool wasCached = false;
     try
     {
       // get from cache or convert:
@@ -174,6 +181,7 @@ public class RhinoRootObjectBuilder : IRootObjectBuilder<RhinoObject>
       else if (_sendConversionCache.TryGetValue(projectId, applicationId, out ObjectReference? value))
       {
         converted = value;
+        wasCached = true;
       }
       else
       {
@@ -194,10 +202,17 @@ public class RhinoRootObjectBuilder : IRootObjectBuilder<RhinoObject>
         converted["properties"] = properties;
       }
 
-      // add to host
+      // process in pipeline
-      collectionHost.elements.Add(converted);
+      var reference = await sendPipeline.Process(converted).ConfigureAwait(false);
+      if (!wasCached)
+      {
+        _sendConversionCache.AppendSendResult(projectId, applicationId, reference);
+      }
 
-      return new(Status.SUCCESS, applicationId, sourceType, converted);
+      // add to host
+      collectionHost.elements.Add(reference);
+
+      return new(Status.SUCCESS, applicationId, sourceType, reference);
     }
     catch (Exception ex) when (!ex.IsFatal())
     {

Sdk/Speckle.Connectors.Common/Caching/ISendConversionCache.cs

@@ -14,6 +14,7 @@ namespace Speckle.Connectors.Common.Caching;
 public interface ISendConversionCache
 {
   void StoreSendResult(string projectId, IReadOnlyDictionary<Id, ObjectReference> convertedReferences);
+  void AppendSendResult(string projectId, string applicationId, ObjectReference convertedReference);
 
   /// <summary>
   /// <para>Call this method whenever you need to invalidate a set of objects that have changed in the host app.</para>

Sdk/Speckle.Connectors.Common/Caching/NullSendConversionCache.cs

@@ -11,6 +11,8 @@ public class NullSendConversionCache : ISendConversionCache
 {
   public void StoreSendResult(string projectId, IReadOnlyDictionary<Id, ObjectReference> convertedReferences) { }
 
+  public void AppendSendResult(string projectId, string applicationId, ObjectReference convertedReference) { }
+
   public void EvictObjects(IEnumerable<string> objectIds) { }
 
   public void ClearCache() { }

Sdk/Speckle.Connectors.Common/Caching/SendConversionCache.cs

@@ -17,6 +17,11 @@ public class SendConversionCache : ISendConversionCache
     }
   }
 
+  public void AppendSendResult(string projectId, string applicationId, ObjectReference convertedReference)
+  {
+    Cache[(applicationId, projectId)] = convertedReference;
+  }
+
   /// <inheritdoc/>
   public void EvictObjects(IEnumerable<string> objectIds) =>
     Cache = Cache