Merge pull request #58 from specklesystems/2.13.0-new-ui

2.13.0 new UI

.circleci/config.yml

@@ -55,6 +55,21 @@ jobs:
                 root: ./
                 paths:
                   - speckle-sharp-ci-tools/Installers
+                  - speckle_arcgis_installer
+  publish-github-release:
+    docker:
+      - image: cimg/go:1.20.0
+    steps:
+      - attach_workspace:
+          at: ./
+      - run:
+          name: "Publish Release on GitHub"
+          command: |
+            set -x
+            go install github.com/tcnksm/ghr@v0.16.0
+            VERSION="${CIRCLE_TAG:-0.0.0}"
+            VERSION_SHORT=$(echo "${VERSION}" | cut -d- -f1)
+            ghr -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete "${VERSION}" "./speckle_arcgis_installer/speckle_toolbox-${VERSION_SHORT}-py3-none-any.whl"
 
   get-ci-tools: # Clones our ci tools and persists them to the workspace
     docker:
@@ -142,6 +157,15 @@ workflows: #happens with every PR to main
             branches:
               ignore: /.*/
           context: innosetup
+      - publish-github-release:
+          requires:
+            - build-deploy-connector-win
+          filters:
+            tags:
+              only: /([0-9]+)\.([0-9]+)\.([0-9]+)(?:-\w+)?$/
+            branches:
+              ignore: /.*/
+          context: arcgis-github-release
       - deploy-manager2:
           slug: arcgis
           os: Win

patch_version.py

@@ -6,7 +6,7 @@ def patch_installer(tag):
     iss_file = "speckle-sharp-ci-tools/arcgis.iss"
     setup_whl_file = "setup.py"
     conda_file = "speckle_arcgis_installer/conda_clone_activate.py"
-    toolbox_install_file = "speckle_arcgis_installer/toolbox_install.py"
+    #toolbox_install_file = "speckle_arcgis_installer/toolbox_install.py"
     toolbox_manual_install_file = "speckle_arcgis_installer/toolbox_install_manual.py"
 
     #py_tag = get_specklepy_version()
@@ -48,7 +48,7 @@ def patch_installer(tag):
         file.close()
 
     whlFileRename(conda_file)
-    whlFileRename(toolbox_install_file)
+    #whlFileRename(toolbox_install_file)
     whlFileRename(toolbox_manual_install_file)
 
 

scripts/debugging_panel.py

@@ -13,7 +13,7 @@ except:
 
 from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
 from arcpy.management import (CreateFeatureclass, MakeFeatureLayer,
-                              AddFields, AlterField, DefineProjection )
+                              AddFields, AlterField, DefineProjection, SelectLayerByAttribute, GetCount )
 
 from specklepy.objects import Base
 
@@ -40,6 +40,13 @@ for layer in active_map.listLayers():
             if geomType == "Point" and layerPoint is None: layerPoint = layer 
             if geomType == "Multipoint" and layerMultiPoint is None: layerMultiPoint = layer 
 
+################################ select/ clear selection ###########################
+for layer in project.activeMap.listLayers(): 
+    if (layer.isFeatureLayer) or layer.isRasterLayer: 
+        arcpy.SelectLayerByAttribute_management(layer.longName,"ADD_TO_SELECTION",'"OBJECTID" = 1')
+        print(arcpy.GetCount_management(layer.longName).getOutput(0))
+        arcpy.SelectLayerByAttribute_management(layer.longName, "CLEAR_SELECTION")
+
 ################## reset symbology if needed:
 sym = layerPolygon.symbology
 print(sym.renderer.type)
@@ -66,7 +73,7 @@ for k, grp in enumerate(sym.renderer.groups):
 
 
 
-from speckle.converter.layers.symbologyTemplates import get_polygon_simpleRenderer 
+from speckle.converter.layers.symbology import get_polygon_simpleRenderer 
 from arcpy._mp import ArcGISProject
 
 aprx = ArcGISProject('CURRENT')

speckle_arcgis_installer/Readme.md

@@ -1,10 +1,10 @@
 ### Manual installation
 
-1. Download present "speckle_arcgis_installer" folder
-2. Clone the default ArcGIS Pro conda environment and restart ArcGIS Pro
+1. From the [latest release](https://github.com/specklesystems/speckle-arcgis/releases) download the whl file and the source code zip, unzip and locate the subfolder "speckle_arcgis_installer" on your machine and place whl file in it. 
+2. Clone the default ArcGIS Pro conda environment (or set the one you use, except the default one) and restart ArcGIS Pro
        - for 2.9.0: Project-> Python-> Manage Environments-> Clone Default
        - for 3.0.0: Project-> Package Manager-> Active Environment (Environment Manager)-> Clone arcgispro-py3
-3. Change the path to your new environemnt Python.exe if necessary (variable "pythonPath" in "toolbox_install_manual.py") 
+3. Adjust the path to your new environment python executable (variable "pythonPath" in "speckle_arcgis_installer/toolbox_install_manual.py") 
 4. Enter the location of 'toolbox_install_manual.py' in the following command and run this command in ArcGIS Python console (View -> Python Window)
 
 ```python

speckle_arcgis_installer/speckle_arcgis_install.bat

@@ -1,3 +0,0 @@
-@echo off
-"%PROGRAMFILES%\ArcGIS\Pro\bin\Python\envs\arcgispro-py3\python.exe" "conda_clone_activate.py"
-"%LOCALAPPDATA%\ESRI\conda\envs\arcgispro-py3-speckle\python.exe" "toolbox_install.py"

speckle_arcgis_installer/toolbox_install.py

@@ -1,61 +0,0 @@
-
-from subprocess_call import subprocess_call
-import os 
-
-pythonPath = os.getenv('APPDATA').replace("\\Roaming","") + r"\Local\ESRI\conda\envs\arcgispro-py3-speckle\python.exe"
-
-def installToolbox(newExec: str):
-    print("Installing Speckle Toolbox")
-    whl_file = os.path.join(os.path.dirname(__file__), "speckle_toolbox-2.9.4-py3-none-any.whl" ) 
-    print(whl_file)
-    subprocess_call([newExec, '-m','pip','install','--upgrade', '--force-reinstall', whl_file])
-    # to uninstall: cmd.exe "C:\\Users\\username\\AppData\\Local\\ESRI\\conda\\envs\\arcgispro-2.9.4-py3-none-any.whl
-    return
-
-def installDependencies(pythonExec: str, pkgName: str, pkgVersion: str):
-    # install pip
-    print(pythonExec)
-    try:
-        import pip
-    except:
-        getPipFilePath = os.path.join(os.path.dirname(__file__), "get_pip.py") #TODO: give actual folder path 
-        exec(open(getPipFilePath).read())
-        # just in case the included version is old
-        subprocess_call([pythonExec, "-m", "pip", "install", "--upgrade", "pip"])
-        
-    # install package
-    try:
-        import importlib
-        importlib.import_module(pkgName)
-    except Exception as e:
-        print(f"{pkgName} not installed")
-        subprocess_call( [pythonExec, "-m", "pip", "install", f"{pkgName}=={pkgVersion}"])
-
-    # Check if package needs updating
-    try:
-        print(f"Attempting to update {pkgName} to {pkgVersion}")
-        result = subprocess_call(
-            [
-                pythonExec,
-                "-m",
-                "pip",
-                "install",
-                "--upgrade",
-                f"{pkgName}=={pkgVersion}",
-            ]
-        )
-        if result == True:
-            print(f"{pkgName} upgraded")
-            return True
-        else: 
-            return False
-    except Exception as e:
-        print(e)
-        print(e.with_traceback)
-    return True
-
-
-installToolbox(pythonPath)
-installDependencies(pythonPath, "specklepy", "2.9.0"  )
-installDependencies(pythonPath, "panda3d", "1.10.11" )
-

speckle_arcgis_installer/toolbox_install_manual.py

@@ -17,10 +17,10 @@ pythonPath = os.getenv('APPDATA').replace("\\Roaming","") + r"\Local\ESRI\conda\
 def installToolbox(newExec: str):
     print("Installing Speckle Toolbox")
     mypath = os.path.dirname(__file__)
-    onlyfiles = [f for f in listdir(mypath) if (isfile(join(mypath, f)) and "-2.11.0-py3-none-any.whl" in str(f))]
+    onlyfiles = [f for f in listdir(mypath) if (isfile(join(mypath, f)) and "py3-none-any.whl" in str(f))]
     onlyfiles.sort(key = lambda x: int(x.replace("speckle_toolbox-","").replace("-py3-none-any.whl","").split(".")[1]) ) 
-    whl_file = onlyfiles[len(onlyfiles)-1]
-    #whl_file = os.path.join(os.path.dirname(__file__), "speckle_toolbox-2.11.0-py3-none-any.whl" ) 
+    whl_file = mypath + "\\" + onlyfiles[len(onlyfiles)-1]
+    #whl_file = os.path.join(os.path.dirname(__file__), "speckle_toolbox-2.11.3-py3-none-any.whl" ) 
     subprocess_call([newExec, '-m','pip','install','--upgrade', '--force-reinstall', whl_file])
     return
 

speckle_toolbox/esri/toolboxes/Speckle.pyt

@@ -1,2 +1,83 @@
-try: from speckle.speckle_arcgis import *
-except: from speckle_toolbox.esri.toolboxes.speckle.speckle_arcgis import *
+     
+import os
+import sys
+import threading
+from typing import List
+
+from PyQt5.QtWidgets import (QMainWindow, QLabel, QApplication,
+    QDockWidget, QVBoxLayout, QWidget)
+from PyQt5.QtCore import Qt 
+from PyQt5 import QtGui, uic
+import arcpy
+
+try: 
+    from speckle.ui.speckle_qgis_dialog import SpeckleGISDialog
+    from speckle.speckle_arcgis import SpeckleGIS 
+except: 
+    from speckle_toolbox.esri.toolboxes.speckle.ui.speckle_qgis_dialog import SpeckleGISDialog
+    from speckle_toolbox.esri.toolboxes.speckle.speckle_arcgis import SpeckleGIS 
+
+def startThread(sp_class): 
+    print("START THREAD")
+    t = threading.Thread(target=qtApp, args=(sp_class,))
+    t.start()
+    threads = threading.enumerate()
+    print("__Total threads: " + str(len(threads)))
+
+def qtApp(text: str):
+    print("MAIN function")
+    
+    threads = threading.enumerate()
+    print("__Total threads: " + str(len(threads)))
+    app = QApplication(sys.argv)
+    ex = SpeckleGIS()
+    #ex.show()
+    sys.exit(app.exec_())
+
+class Toolbox:
+    def __init__(self):
+        """Define the toolbox (the name of the toolbox is the name of the .pyt file)."""
+        print("___start_Toolbox")
+        self.label = "Speckle Tools"
+        self.alias = "speckle_toolbox_"  
+        # List of tool classes associated with this toolbox
+        self.tools = [Speckle]  
+
+class Speckle:
+    #instances = []
+    def __init__(self):  
+        
+        print("___start speckle tool_________")
+
+        self.label       = "Speckle"
+        self.description = "Allows you to send and receive your layers " + \
+                           "to/from other software using Speckle server." 
+
+    def getParameterInfo(self):
+        cat1 = "category 1"
+
+        param0 = arcpy.Parameter(
+            displayName="""▷ Run to launch Speckle Connector
+""", #▶ 
+            name="param0",
+            datatype="GPString",
+            parameterType="Optional",
+            direction="Input",
+            enabled="True",
+            )
+        param0.value = """This is an experimental version of plugin.
+
+Save your work before using!
+
+Report issues at https://speckle.community/"""
+        return [param0]
+
+    def isLicensed(self): #optional
+        return True
+
+    def updateParameters(self, parameters: List, toRefresh = False): #optional
+        return 
+
+    def execute(self, parameters: List, messages): 
+        qtApp("")
+        #startThread("")

speckle_toolbox/esri/toolboxes/Speckle.pyt.xml

@@ -1,5 +1,5 @@
 <?xml version="1.0"?>
-<metadata xml:lang="en"><Esri><CreaDate>20220718</CreaDate><CreaTime>13500100</CreaTime><ArcGISFormat>1.0</ArcGISFormat><SyncOnce>TRUE</SyncOnce><ModDate>20221207</ModDate><ModTime>175959</ModTime><scaleRange><minScale>150000000</minScale><maxScale>5000</maxScale></scaleRange><ArcGISProfile>ItemDescription</ArcGISProfile></Esri><toolbox name="Speckle" alias="speckle_toolbox_"><arcToolboxHelpPath>c:\program files\arcgis\pro\Resources\Help\gp</arcToolboxHelpPath><toolsets/></toolbox><dataIdInfo><idCitation><resTitle>Speckle</resTitle></idCitation><idPurp>Speckle connector for ArcGIS</idPurp><searchKeys><keyword>speckle3d</keyword></searchKeys></dataIdInfo><distInfo><distributor><distorFormat><formatName>ArcToolbox Toolbox</formatName></distorFormat></distributor></distInfo><mdHrLv><ScopeCd value="005"></ScopeCd></mdHrLv><Binary><Thumbnail><Data EsriPropertyType="PictureX">/9j/4AAQSkZJRgABAQEAYABgAAD/4gxYSUNDX1BST0ZJTEUAAQEAAAxITGlubwIQAABtbnRyUkdC
+<metadata xml:lang="en"><Esri><CreaDate>20220718</CreaDate><CreaTime>13500100</CreaTime><ArcGISFormat>1.0</ArcGISFormat><SyncOnce>TRUE</SyncOnce><ModDate>20230303</ModDate><ModTime>104937</ModTime><scaleRange><minScale>150000000</minScale><maxScale>5000</maxScale></scaleRange><ArcGISProfile>ItemDescription</ArcGISProfile></Esri><toolbox name="Speckle" alias="speckle_toolbox_"><arcToolboxHelpPath>c:\program files\arcgis\pro\Resources\Help\gp</arcToolboxHelpPath><toolsets/></toolbox><dataIdInfo><idCitation><resTitle>Speckle</resTitle></idCitation><idPurp>Speckle connector for ArcGIS</idPurp><searchKeys><keyword>speckle3d</keyword></searchKeys></dataIdInfo><distInfo><distributor><distorFormat><formatName>ArcToolbox Toolbox</formatName></distorFormat></distributor></distInfo><mdHrLv><ScopeCd value="005"></ScopeCd></mdHrLv><Binary><Thumbnail><Data EsriPropertyType="PictureX">/9j/4AAQSkZJRgABAQEAYABgAAD/4gxYSUNDX1BST0ZJTEUAAQEAAAxITGlubwIQAABtbnRyUkdC
 IFhZWiAHzgACAAkABgAxAABhY3NwTVNGVAAAAABJRUMgc1JHQgAAAAAAAAAAAAAAAAAA9tYAAQAA
 AADTLUhQICAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABFj
 cHJ0AAABUAAAADNkZXNjAAABhAAAAGx3dHB0AAAB8AAAABRia3B0AAACBAAAABRyWFlaAAACGAAA

speckle_toolbox/esri/toolboxes/speckle/converter/geometry/__init__.py

@@ -0,0 +1,211 @@
+
+from regex import F
+from specklepy.objects import Base
+from specklepy.objects.geometry import Line, Mesh, Point, Polyline, Curve, Arc, Circle, Polycurve, Ellipse 
+
+import arcpy 
+from typing import Any, List, Union, Sequence
+try:
+    from speckle.converter.geometry.polygon import polygonToNative, polygonToSpeckle, multiPolygonToSpeckle, polygonToSpeckleMesh
+    from speckle.converter.geometry.polyline import arcToNative, ellipseToNative, circleToNative, curveToNative, lineToNative, polycurveToNative, polylineFromVerticesToSpeckle, polylineToNative, polylineToSpeckle
+    from speckle.converter.geometry.point import pointToCoord, pointToNative, pointToSpeckle, multiPointToSpeckle
+    from speckle.converter.geometry.polyline import speckleArcCircleToPoints, specklePolycurveToPoints, multiPolylineToSpeckle
+    from speckle.plugin_utils.logger import logToUser
+    from speckle.converter.geometry.mesh import meshToNative
+except:
+    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.polygon import polygonToNative, polygonToSpeckle, multiPolygonToSpeckle, polygonToSpeckleMesh
+    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.polyline import arcToNative, ellipseToNative, circleToNative, curveToNative, lineToNative, polycurveToNative, polylineFromVerticesToSpeckle, polylineToNative, polylineToSpeckle
+    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.point import pointToCoord, pointToNative, pointToSpeckle, multiPointToSpeckle
+    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.polyline import speckleArcCircleToPoints, specklePolycurveToPoints, multiPolylineToSpeckle
+    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.mesh import meshToNative
+    from speckle_toolbox.esri.toolboxes.speckle.plugin_utils.logger import logToUser
+
+import numpy as np
+
+def convertToSpeckle(feature, index: str, layer, geomType, featureType) -> Union[Base, Sequence[Base], None]:
+    """Converts the provided layer feature to Speckle objects"""
+    print("___convertToSpeckle____________")
+    try:
+        geom = feature
+        print(geom.isMultipart) # e.g. False 
+        print(geom.partCount)
+        geomMultiType = geom.isMultipart
+        hasCurves = feature.hasCurves 
+        
+        # feature is <geoprocessing describe geometry object object at 0x000002A75D6A4BD0>
+        
+        #print(featureType) # e.g. Simple 
+        #print(geomType) # e.g. Polygon 
+        #geomSingleType = (featureType=="Simple") # Simple,SimpleJunction,SimpleJunction,ComplexEdge,Annotation,CoverageAnnotation,Dimension,RasterCatalogItem 
+
+        if geomType == "Point": #Polygon, Point, Polyline, Multipoint, MultiPatch
+            for pt in geom:
+                return pointToSpeckle(pt, feature, layer)
+        elif geomType == "Polyline":
+            #if geom.hasCurves: 
+            #    geom, feature = curvesToSegments(geom, feature, layer, geomMultiType)
+            #    geomMultiType = geom.isMultipart
+            #    return polylineToSpeckle(geom, feature, layer, geomMultiType)
+            #else:
+            if geom.partCount > 1: return multiPolylineToSpeckle(geom, feature, layer, geomMultiType)
+            else: return polylineToSpeckle(geom, feature, layer, geomMultiType)
+        elif geomType == "Polygon":
+            if geom.partCount > 1: return multiPolygonToSpeckle(geom, index, layer, geomMultiType)
+            else: return polygonToSpeckle(geom, index, layer, geomMultiType)
+        elif geomType == "Multipoint":
+            return multiPointToSpeckle(geom, feature, layer, geomMultiType)
+        elif geomType == "MultiPatch":
+            return polygonToSpeckleMesh(geom, index, layer, False)
+        else:
+            logToUser("Unsupported or invalid geometry in layer " + layer.name, 1)
+        return None
+    except Exception as e:
+        logToUser(e)
+        return None
+
+
+def convertToNative(base: Base, sr: arcpy.SpatialReference) -> Union[Any, None]:
+    """Converts any given base object to QgsGeometry."""
+    print("___Convert to Native SingleType___")
+    converted = None
+    try:
+        #print(base)
+        conversions = [
+            (Point, pointToNative),
+            (Line, lineToNative),
+            (Polyline, polylineToNative),
+            (Curve, curveToNative),
+            (Arc, arcToNative),
+            (Circle, circleToNative),
+            (Ellipse, ellipseToNative),
+            #(Mesh, meshToNative),
+            (Polycurve, polycurveToNative),
+            (Base, polygonToNative), # temporary solution for polygons (Speckle has no type Polygon yet)
+        ]
+
+        for conversion in conversions:
+            if isinstance(base, conversion[0]):
+                #print(conversion[0])
+                converted = conversion[1](base, sr)
+                break
+        #print(converted)
+    except Exception as e:
+        logToUser(e)
+    return converted
+
+def multiPointToNative(items: List[Point], sr: arcpy.SpatialReference):
+    print("___Create MultiPoint")
+    features = None
+    try:
+        all_pts = []
+        # example https://pro.arcgis.com/en/pro-app/2.8/arcpy/classes/multipoint.htm
+        for item in items:
+            pt = pointToCoord(item) # [x, y, z]
+            all_pts.append( arcpy.Point(pt[0], pt[1], pt[2]) )
+        #print(all_pts)
+        features = arcpy.Multipoint( arcpy.Array(all_pts) )
+        #if len(features)==0: features = None
+    except Exception as e:
+        logToUser(e)
+    return features
+
+def multiPolylineToNative(items: List[Polyline], sr: arcpy.SpatialReference):
+    print("_______Drawing Multipolylines____")
+    poly = None
+    try:
+        #print(items)
+        poly = None
+        full_array_list = []
+        for item in items: # will be 1 item
+            pointsSpeckle = []
+            try: pointsSpeckle = item.as_points()
+            except: continue 
+            pts = [pointToCoord(pt) for pt in pointsSpeckle]
+
+            if item.closed is True: 
+                pts.append( pointToCoord(item.as_points()[0]) )
+            
+            arr = [arcpy.Point(*coords) for coords in pts]
+            full_array_list.append(arr)
+
+        poly = arcpy.Polyline( arcpy.Array(full_array_list), sr, has_z=True )
+    except Exception as e:
+        logToUser(e)
+    return poly
+
+def multiPolygonToNative(items: List[Base], sr: arcpy.SpatialReference): #TODO fix multi features
+    
+    print("_______Drawing Multipolygons____")
+    polygon = None 
+    try: 
+        #print(items)
+        full_array_list = []
+
+        for item in items: # will be 1 item
+            #print(item)
+            #pts = [pointToCoord(pt) for pt in item["boundary"].as_points()]
+            pointsSpeckle = []
+            if isinstance(item["boundary"], Circle) or isinstance(item["boundary"], Arc): 
+                pointsSpeckle = speckleArcCircleToPoints(item["boundary"]) 
+            elif isinstance(item["boundary"], Polycurve): 
+                pointsSpeckle = specklePolycurveToPoints(item["boundary"]) 
+            elif isinstance(item["boundary"], Line): pass
+            else: 
+                try: pointsSpeckle = item["boundary"].as_points()
+                except: pass # if Line
+
+            pts = [pointToCoord(pt) for pt in pointsSpeckle]
+            #print(pts)
+
+            outer_arr = [arcpy.Point(*coords) for coords in pts]
+            outer_arr.append(outer_arr[0])
+            geomPart = []
+            try:
+                for void in item["voids"]: 
+                    #print(void)
+                    #pts = [pointToCoord(pt) for pt in void.as_points()]
+                    pointsSpeckle = []
+                    if isinstance(void, Circle) or isinstance(void, Arc): 
+                        pointsSpeckle = speckleArcCircleToPoints(void) 
+                    elif isinstance(void, Polycurve): 
+                        pointsSpeckle = specklePolycurveToPoints(void) 
+                    elif isinstance(void, Line): pass
+                    else: 
+                        try: pointsSpeckle = void.as_points()
+                        except: pass # if Line
+                    pts = [pointToCoord(pt) for pt in pointsSpeckle]
+
+                    inner_arr = [arcpy.Point(*coords) for coords in pts]
+                    inner_arr.append(inner_arr[0])
+                    geomPart.append(arcpy.Array(inner_arr))
+            except:pass
+        
+            geomPart.insert(0, arcpy.Array(outer_arr))
+            full_array_list.extend(geomPart) # outlines are written one by one, with no separation to "parts"
+
+        geomPartArray = arcpy.Array(full_array_list)
+        polygon = arcpy.Polygon(geomPartArray, sr, has_z=True)
+        
+        print(polygon)
+    except Exception as e:
+        logToUser(e)
+    
+    return polygon
+
+def convertToNativeMulti(items: List[Base], sr: arcpy.SpatialReference): 
+    print("___Convert to Native MultiType___")
+    try:
+        first = items[0]
+        if isinstance(first, Point):
+            return multiPointToNative(items, sr)
+        elif isinstance(first, Line) or isinstance(first, Polyline):
+            return multiPolylineToNative(items, sr)
+        elif isinstance(first, Base): 
+            try:
+                if first["boundary"] is not None and first["voids"] is not None:
+                    return multiPolygonToNative(items, sr)
+            except: return None 
+    except Exception as e:
+        logToUser(e)
+        return None 
+    

speckle_toolbox/esri/toolboxes/speckle/converter/geometry/_init_.py

@@ -1,183 +0,0 @@
-
-from regex import F
-from specklepy.objects import Base
-from specklepy.objects.geometry import Line, Mesh, Point, Polyline, Curve, Arc, Circle, Polycurve, Ellipse 
-
-import arcpy 
-from typing import Any, List, Union, Sequence
-try:
-    from speckle.converter.geometry.polygon import polygonToNative, polygonToSpeckle, multiPolygonToSpeckle
-    from speckle.converter.geometry.polyline import arcToNative, ellipseToNative, circleToNative, curveToNative, lineToNative, polycurveToNative, polylineFromVerticesToSpeckle, polylineToNative, polylineToSpeckle
-    from speckle.converter.geometry.point import pointToCoord, pointToNative, pointToSpeckle, multiPointToSpeckle
-    from speckle.converter.geometry.polyline import speckleArcCircleToPoints, specklePolycurveToPoints, multiPolylineToSpeckle
-    from speckle.converter.geometry.mesh import meshToNative
-except:
-    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.polygon import polygonToNative, polygonToSpeckle, multiPolygonToSpeckle
-    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.polyline import arcToNative, ellipseToNative, circleToNative, curveToNative, lineToNative, polycurveToNative, polylineFromVerticesToSpeckle, polylineToNative, polylineToSpeckle
-    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.point import pointToCoord, pointToNative, pointToSpeckle, multiPointToSpeckle
-    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.polyline import speckleArcCircleToPoints, specklePolycurveToPoints, multiPolylineToSpeckle
-    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.mesh import meshToNative
-
-import numpy as np
-
-def convertToSpeckle(feature, index: str, layer, geomType, featureType) -> Union[Base, Sequence[Base], None]:
-    """Converts the provided layer feature to Speckle objects"""
-    print("___convertToSpeckle____________")
-    geom = feature
-    print(geom.isMultipart) # e.g. False 
-    print(geom.partCount)
-    geomMultiType = geom.isMultipart
-    hasCurves = feature.hasCurves 
-    
-    # feature is <geoprocessing describe geometry object object at 0x000002A75D6A4BD0>
-    
-    #print(featureType) # e.g. Simple 
-    #print(geomType) # e.g. Polygon 
-    #geomSingleType = (featureType=="Simple") # Simple,SimpleJunction,SimpleJunction,ComplexEdge,Annotation,CoverageAnnotation,Dimension,RasterCatalogItem 
-
-    if geomType == "Point": #Polygon, Point, Polyline, Multipoint, MultiPatch
-        for pt in geom:
-            return pointToSpeckle(pt, feature, layer)
-    elif geomType == "Polyline":
-        #if geom.hasCurves: 
-        #    geom, feature = curvesToSegments(geom, feature, layer, geomMultiType)
-        #    geomMultiType = geom.isMultipart
-        #    return polylineToSpeckle(geom, feature, layer, geomMultiType)
-        #else:
-        if geom.partCount > 1: return multiPolylineToSpeckle(geom, feature, layer, geomMultiType)
-        else: return polylineToSpeckle(geom, feature, layer, geomMultiType)
-    elif geomType == "Polygon":
-        if geom.partCount > 1: return multiPolygonToSpeckle(geom, feature, index, layer, geomMultiType)
-        else: return polygonToSpeckle(geom, feature, index, layer, geomMultiType)
-    elif geomType == "Multipoint":
-        return multiPointToSpeckle(geom, feature, layer, geomMultiType)
-    else:
-        arcpy.AddWarning("Unsupported or invalid geometry in layer " + layer.name)
-    return None
-
-
-def convertToNative(base: Base, sr: arcpy.SpatialReference) -> Union[Any, None]:
-    """Converts any given base object to QgsGeometry."""
-    print("___Convert to Native SingleType___")
-    #print(base)
-    converted = None
-    conversions = [
-        (Point, pointToNative),
-        (Line, lineToNative),
-        (Polyline, polylineToNative),
-        (Curve, curveToNative),
-        (Arc, arcToNative),
-        (Circle, circleToNative),
-        (Ellipse, ellipseToNative),
-        #(Mesh, meshToNative),
-        (Polycurve, polycurveToNative),
-        (Base, polygonToNative), # temporary solution for polygons (Speckle has no type Polygon yet)
-    ]
-
-    for conversion in conversions:
-        if isinstance(base, conversion[0]):
-            #print(conversion[0])
-            converted = conversion[1](base, sr)
-            break
-    #print(converted)
-    return converted
-
-def multiPointToNative(items: List[Point], sr: arcpy.SpatialReference):
-    print("___Create MultiPoint")
-    all_pts = []
-    # example https://pro.arcgis.com/en/pro-app/2.8/arcpy/classes/multipoint.htm
-    for item in items:
-        pt = pointToCoord(item) # [x, y, z]
-        all_pts.append( arcpy.Point(pt[0], pt[1], pt[2]) )
-    #print(all_pts)
-    features = arcpy.Multipoint( arcpy.Array(all_pts) )
-    #if len(features)==0: features = None
-    return features
-
-def multiPolylineToNative(items: List[Polyline], sr: arcpy.SpatialReference):
-    print("_______Drawing Multipolylines____")
-    #print(items)
-    poly = None
-    full_array_list = []
-    for item in items: # will be 1 item
-        pointsSpeckle = []
-        try: pointsSpeckle = item.as_points()
-        except: continue 
-        pts = [pointToCoord(pt) for pt in pointsSpeckle]
-
-        if item.closed is True: 
-            pts.append( pointToCoord(item.as_points()[0]) )
-        
-        arr = [arcpy.Point(*coords) for coords in pts]
-        full_array_list.append(arr)
-
-    poly = arcpy.Polyline( arcpy.Array(full_array_list), sr, has_z=True )
-    return poly
-
-def multiPolygonToNative(items: List[Base], sr: arcpy.SpatialReference): #TODO fix multi features
-    
-    print("_______Drawing Multipolygons____")
-    #print(items)
-    full_array_list = []
-
-    for item in items: # will be 1 item
-        #print(item)
-        #pts = [pointToCoord(pt) for pt in item["boundary"].as_points()]
-        pointsSpeckle = []
-        if isinstance(item["boundary"], Circle) or isinstance(item["boundary"], Arc): 
-            pointsSpeckle = speckleArcCircleToPoints(item["boundary"]) 
-        elif isinstance(item["boundary"], Polycurve): 
-            pointsSpeckle = specklePolycurveToPoints(item["boundary"]) 
-        elif isinstance(item["boundary"], Line): pass
-        else: 
-            try: pointsSpeckle = item["boundary"].as_points()
-            except: pass # if Line
-
-        pts = [pointToCoord(pt) for pt in pointsSpeckle]
-        #print(pts)
-
-        outer_arr = [arcpy.Point(*coords) for coords in pts]
-        outer_arr.append(outer_arr[0])
-        geomPart = []
-        try:
-            for void in item["voids"]: 
-                #print(void)
-                #pts = [pointToCoord(pt) for pt in void.as_points()]
-                pointsSpeckle = []
-                if isinstance(void, Circle) or isinstance(void, Arc): 
-                    pointsSpeckle = speckleArcCircleToPoints(void) 
-                elif isinstance(void, Polycurve): 
-                    pointsSpeckle = specklePolycurveToPoints(void) 
-                elif isinstance(void, Line): pass
-                else: 
-                    try: pointsSpeckle = void.as_points()
-                    except: pass # if Line
-                pts = [pointToCoord(pt) for pt in pointsSpeckle]
-
-                inner_arr = [arcpy.Point(*coords) for coords in pts]
-                inner_arr.append(inner_arr[0])
-                geomPart.append(arcpy.Array(inner_arr))
-        except:pass
-    
-        geomPart.insert(0, arcpy.Array(outer_arr))
-        full_array_list.extend(geomPart) # outlines are written one by one, with no separation to "parts"
-
-    geomPartArray = arcpy.Array(full_array_list)
-    polygon = arcpy.Polygon(geomPartArray, sr, has_z=True)
-    
-    print(polygon)
-    
-    return polygon
-
-def convertToNativeMulti(items: List[Base], sr: arcpy.SpatialReference): 
-    print("___Convert to Native MultiType___")
-    first = items[0]
-    if isinstance(first, Point):
-        return multiPointToNative(items, sr)
-    elif isinstance(first, Line) or isinstance(first, Polyline):
-        return multiPolylineToNative(items, sr)
-    elif isinstance(first, Base): 
-        try:
-            if first["boundary"] is not None and first["voids"] is not None:
-                return multiPolygonToNative(items, sr)
-        except: return None 

speckle_toolbox/esri/toolboxes/speckle/converter/geometry/mesh.py

@@ -1,47 +1,275 @@
 from typing import List
 import arcpy
+import math
 
-from specklepy.objects.geometry import Mesh
+from specklepy.objects.geometry import Mesh, Point
+from specklepy.objects.other import RenderMaterial
 
 import shapefile
-from shapefile import TRIANGLE_STRIP, TRIANGLE_FAN
-try: from speckle.converter.layers.utils import get_scale_factor
-except: from speckle_toolbox.esri.toolboxes.speckle.converter.layers.utils import get_scale_factor
+from shapefile import TRIANGLE_STRIP, TRIANGLE_FAN, OUTER_RING
+try: 
+    from speckle.converter.layers.utils import get_scale_factor
+    from speckle.converter.geometry.point import pointToNative
+    from speckle.converter.layers.symbology import featureColorfromNativeRenderer
+    from speckle.converter.layers.utils import get_scale_factor
+    from speckle.plugin_utils.logger import logToUser
+except: 
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.utils import get_scale_factor
+    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.point import pointToNative
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.symbology import featureColorfromNativeRenderer
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.utils import get_scale_factor
+    from speckle_toolbox.esri.toolboxes.speckle.plugin_utils.logger import logToUser
+
+from panda3d.core import Triangulator
 
 def meshToNative(meshes: List[Mesh], path: str):
     """Converts a Speckle Mesh to MultiPatch"""
     print("06___________________Mesh to Native")
-    #print(meshes)
-    #print(mesh.units)
-    w = shapefile.Writer(path) 
-    w.field('speckleTyp', 'C')
+    try:
+        #print(meshes)
+        #print(mesh.units)
+        w = shapefile.Writer(path) 
+        w.field('speckleTyp', 'C')
 
-    shapes = []
-    for geom in meshes:
-        
-        try: 
-            if geom.displayValue and isinstance(geom.displayValue, Mesh): 
-                mesh = geom.displayValue
-                w = fill_mesh_parts(w, mesh)
-            elif geom.displayValue and isinstance(geom.displayValue, List): 
-                for part in geom.displayValue:
-                    if isinstance(part, Mesh): 
-                        mesh = part
-                        w = fill_mesh_parts(w, mesh)
-        except: 
+        shapes = []
+        for geom in meshes:
+            
             try: 
-                if geom.displayMesh and isinstance(geom.displayMesh, Mesh): 
-                    mesh = geom.displayMesh
+                if geom.displayValue and isinstance(geom.displayValue, Mesh): 
+                    mesh = geom.displayValue
                     w = fill_mesh_parts(w, mesh)
-                elif geom.displayMesh and isinstance(geom.displayMesh, List): 
-                    for part in geom.displayMesh:
+                elif geom.displayValue and isinstance(geom.displayValue, List): 
+                    for part in geom.displayValue:
                         if isinstance(part, Mesh): 
                             mesh = part
                             w = fill_mesh_parts(w, mesh)
-            except: pass
-    w.close()
+            except: 
+                try: 
+                    if geom.displayMesh and isinstance(geom.displayMesh, Mesh): 
+                        mesh = geom.displayMesh
+                        w = fill_mesh_parts(w, mesh)
+                    elif geom.displayMesh and isinstance(geom.displayMesh, List): 
+                        for part in geom.displayMesh:
+                            if isinstance(part, Mesh): 
+                                mesh = part
+                                w = fill_mesh_parts(w, mesh)
+                except: pass
+        w.close()
+    except Exception as e:
+        logToUser(e)
     return path
 
+
+def writeMeshToShp(meshes: List[Mesh], path: str):
+    """Converts a Speckle Mesh to native geometry"""
+    print("06___________________Mesh to Native")
+    try:
+        #print(meshes)
+        #print(mesh.units)
+        w = shapefile.Writer(path) 
+        w.field('speckle_id', 'C')
+
+        shapes = []
+        for geom in meshes:
+
+            if geom.speckle_type =='Objects.Geometry.Mesh' and isinstance(geom, Mesh):
+                mesh = geom
+                w = fill_mesh_parts(w, mesh, geom.id)
+            else:
+                try: 
+                    if geom.displayValue and isinstance(geom.displayValue, Mesh): 
+                        mesh = geom.displayValue
+                        w = fill_mesh_parts(w, mesh, geom.id)
+                    elif geom.displayValue and isinstance(geom.displayValue, List): 
+                        w = fill_multi_mesh_parts(w, geom.displayValue, geom.id)
+                except: 
+                    try: 
+                        if geom["@displayValue"] and isinstance(geom["@displayValue"], Mesh): 
+                            mesh = geom["@displayValue"]
+                            w = fill_mesh_parts(w, mesh, geom.id)
+                        elif geom["@displayValue"] and isinstance(geom["@displayValue"], List): 
+                            w = fill_multi_mesh_parts(w, geom["@displayValue"], geom.id)
+                    except:
+                        try: 
+                            if geom.displayMesh and isinstance(geom.displayMesh, Mesh): 
+                                mesh = geom.displayMesh
+                                w = fill_mesh_parts(w, mesh, geom.id)
+                            elif geom.displayMesh and isinstance(geom.displayMesh, List): 
+                                w = fill_multi_mesh_parts(w, geom.displayMesh, geom.id)
+                        except: pass
+        w.close()
+    except Exception as e:
+        logToUser(e)
+    return path
+
+
+def fill_multi_mesh_parts(w: shapefile.Writer, meshes: List[Mesh], geom_id: str):
+    try:
+        parts_list = []
+        types_list = []
+        for mesh in meshes:
+            if not isinstance(mesh, Mesh): continue
+            try:
+                #print(f"Fill multi-mesh parts # {geom_id}")
+                parts_list_x, types_list_x = deconstructSpeckleMesh(mesh) 
+                parts_list.extend(parts_list_x)
+                types_list.extend(types_list_x)
+            except Exception as e: pass 
+        
+        w.multipatch(parts_list, partTypes=types_list ) # one type for each part
+        w.record(geom_id)
+    except Exception as e:
+        logToUser(e)
+    return w
+
+def fill_mesh_parts(w: shapefile.Writer, mesh: Mesh, geom_id: str):
+    
+    try:
+        #print(f"Fill mesh parts # {geom_id}")
+        parts_list, types_list = deconstructSpeckleMesh(mesh) 
+        w.multipatch(parts_list, partTypes=types_list ) # one type for each part
+        w.record(geom_id)
+
+    except Exception as e:
+        logToUser(e)
+    return w
+
+def deconstructSpeckleMesh(mesh: Mesh):
+    parts_list = []
+    types_list = []
+    try:
+        scale = get_scale_factor(mesh.units)
+
+        count = 0 # sequence of vertex (not of flat coord list) 
+        for f in mesh.faces: # real number of loops will be at least 3 times less 
+            try:
+                vertices = mesh.faces[count]
+                if mesh.faces[count] == 0: vertices = 3
+                if mesh.faces[count] == 1: vertices = 4
+                
+                face = []
+                for i in range(vertices):
+                    index_faces = count + 1 + i 
+                    index_vertices = mesh.faces[index_faces]*3
+                    face.append([ scale * mesh.vertices[index_vertices], scale * mesh.vertices[index_vertices+1], scale * mesh.vertices[index_vertices+2] ]) 
+
+                parts_list.append(face)
+                types_list.append(OUTER_RING)
+                count += vertices + 1
+            except: break # when out of range 
+
+    except Exception as e:
+        logToUser(e)
+    return parts_list, types_list
+
+def constructMesh(vertices, faces, colors):
+    mesh = None
+    try:
+        mesh = Mesh.create(vertices, faces, colors)
+        mesh.units = "m"
+        material = RenderMaterial()
+        material.diffuse = colors[0]
+        mesh.renderMaterial = material 
+    except Exception as e:
+        logToUser(e)
+    return mesh
+
+def meshPartsFromPolygon(polyBorder: List[Point], voidsAsPts: List[List[Point]], existing_vert: int, index: int, layer):
+    
+    try:
+        #print("__meshPartsFromPolygon__")
+        vertices = []
+        total_vertices = 0
+        #print(layer)
+        try: sr = arcpy.Describe(layer.dataSource).spatialReference
+        except Exception as e: 
+            print(e)
+            sr = None
+        #print(sr)
+        coef = 1
+        maxPoints = 5000
+        if len(polyBorder) >= maxPoints: coef = int(len(polyBorder)/maxPoints)
+        
+        if len(voidsAsPts) == 0: # only if there is a mesh with no voids and large amount of points
+            #print("mesh with no voids")
+            for k, ptt in enumerate(polyBorder): #pointList:
+                pt = polyBorder[k*coef]
+                if k < maxPoints:
+                    #if isinstance(pt, QgsPointXY):
+                    #    pt = QgsPoint(pt)
+                    #print(pt)
+                    if isinstance(pt,Point):
+                        pt = pointToNative(pt, sr).getPart()
+                    x = pt.X
+                    y = pt.Y
+                    z = 0 if math.isnan(pt.Z) else pt.Z
+                    vertices.extend([x, y, z])
+                    total_vertices += 1
+                else: break
+
+            ran = range(0, total_vertices)
+            faces = [total_vertices]
+            faces.extend([i + existing_vert for i in ran])
+            # else: https://docs.panda3d.org/1.10/python/reference/panda3d.core.Triangulator
+        else: # if there are voids 
+            print("mesh with voids")
+            # if its a large polygon with voids to be triangualted, lower the coef even more:
+            maxPoints = 100
+            if len(polyBorder) >= maxPoints: coef = int(len(polyBorder)/maxPoints)
+
+            trianglator = Triangulator()
+            faces = []
+
+            pt_count = 0
+            # add extra middle point for border
+            for k, ptt in enumerate(polyBorder): #pointList:
+                pt = polyBorder[k*coef]
+                if k < maxPoints:
+                    if pt_count < len(polyBorder)-1 and k < (maxPoints-1): 
+                        pt2 = polyBorder[(k+1)*coef]
+                    else: pt2 = polyBorder[0]
+                            
+                    trianglator.addPolygonVertex(trianglator.addVertex(pt.x, pt.y))
+                    vertices.extend([pt.x, pt.y, pt.z])
+                    trianglator.addPolygonVertex(trianglator.addVertex((pt.x+pt2.x)/2, (pt.y+pt2.y)/2))
+                    vertices.extend([(pt.x+pt2.x)/2, (pt.y+pt2.y)/2, (pt.z+pt2.z)/2])
+                    total_vertices += 2
+                    pt_count += 1
+                else: break
+
+            #add void points
+            for pts in voidsAsPts:
+                trianglator.beginHole()
+
+                coefVoid = 1
+                if len(pts) >= maxPoints: coefVoid = int(len(pts)/maxPoints)
+                for k, ptt in enumerate(pts):
+                    pt = pts[k*coefVoid]
+                    if k < maxPoints:
+                        trianglator.addHoleVertex(trianglator.addVertex(pt.x, pt.y))
+                        vertices.extend([pt.x, pt.y, pt.z])
+                        total_vertices += 1
+                    else: break
+            
+            trianglator.triangulate()
+            i = 0
+            #print(trianglator.getNumTriangles())
+            while i < trianglator.getNumTriangles():
+                tr = [trianglator.getTriangleV0(i),trianglator.getTriangleV1(i),trianglator.getTriangleV2(i)]
+                faces.extend([3, tr[0] + existing_vert, tr[1] + existing_vert, tr[2] + existing_vert])
+                i+=1
+            ran = range(0, total_vertices)
+
+        #print("color")
+        col = featureColorfromNativeRenderer(index, layer) #(100<<16) + (100<<8) + 100 
+        colors = [col for i in ran] # apply same color for all vertices
+
+        return total_vertices, vertices, faces, colors
+    except Exception as e:
+        logToUser(e)
+        return None, None, None, None 
+
+r'''
 def fill_mesh_parts(w: shapefile.Writer, mesh: Mesh):
     scale = get_scale_factor(mesh.units)
 
@@ -69,8 +297,13 @@ def fill_mesh_parts(w: shapefile.Writer, mesh: Mesh):
 
     except Exception as e: pass #; print(e)
     return w
-    
+'''
+
 def rasterToMesh(vertices, faces, colors):
-    mesh = Mesh.create(vertices, faces, colors)
-    mesh.units = "m"
-    return mesh
+    try:
+        mesh = Mesh.create(vertices, faces, colors)
+        mesh.units = "m"
+        return mesh
+    except Exception as e:
+        logToUser(e)
+        return None

speckle_toolbox/esri/toolboxes/speckle/converter/geometry/point.py

@@ -3,24 +3,27 @@ from typing import List
 from specklepy.objects.geometry import Point
 import arcpy
 
-try: from speckle.converter.layers.utils import get_scale_factor
-except: from speckle_toolbox.esri.toolboxes.speckle.converter.layers.utils import get_scale_factor
+try: 
+    from speckle.converter.layers.utils import get_scale_factor
+    from speckle.plugin_utils.logger import logToUser
+except: 
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.utils import get_scale_factor
+    from speckle_toolbox.esri.toolboxes.speckle.plugin_utils.logger import logToUser
 
 
 def multiPointToSpeckle(geom, feature, layer, multiType: bool):
     """Converts a Point to Speckle"""
-    #try: 
-    #print("___Point to Speckle____")
-    #point = Point(units = "m")
+    
     pointList = []
     #print(geom) # <geoprocessing describe geometry object object at 0x0000020F1D94AB10>
-    #print(multiType)
-
-    if multiType is False: 
-        for pt in geom:
-            #print(pt) # 284394.58100903 5710688.11602606 NaN NaN <class 'arcpy.arcobjects.arcobjects.Point'> 
-            #print(type(pt))
-            if pt != None: pointList.append(pointToSpeckle(pt, feature, layer)) 
+    try:
+        if multiType is False: 
+            for pt in geom:
+                #print(pt) # 284394.58100903 5710688.11602606 NaN NaN <class 'arcpy.arcobjects.arcobjects.Point'> 
+                #print(type(pt))
+                if pt != None: pointList.append(pointToSpeckle(pt, feature, layer)) 
+    except Exception as e:
+        logToUser(e)
     return pointList
 
 def pointToSpeckle(pt, feature, layer):
@@ -30,48 +33,67 @@ def pointToSpeckle(pt, feature, layer):
     # when unset, z() returns "nan"
     #print(pt) # 4.9046319 52.3592043 NaN NaN
     #print("____Point to Speckle___")
-    x = pt.X
-    y = pt.Y
-    if pt.Z: z = pt.Z 
-    else: z = 0
-    specklePoint = Point(units = "m")
-    specklePoint.x = x
-    specklePoint.y = y
-    specklePoint.z = z
-    '''
-    if feature is not None and layer is not None: # can be if it's a point from raster layer 
-        col = featureColorfromNativeRenderer(feature, layer)
-        specklePoint['displayStyle'] = {}
-        specklePoint['displayStyle']['color'] = col
-    '''
-    #print(specklePoint)
-    return specklePoint
+    try:
+        x = pt.X
+        y = pt.Y
+        if pt.Z: z = pt.Z 
+        else: z = 0
+        specklePoint = Point(units = "m")
+        specklePoint.x = x
+        specklePoint.y = y
+        specklePoint.z = z
+        '''
+        if feature is not None and layer is not None: # can be if it's a point from raster layer 
+            col = featureColorfromNativeRenderer(feature, layer)
+            specklePoint['displayStyle'] = {}
+            specklePoint['displayStyle']['color'] = col
+        '''
+        #print(specklePoint)
+        return specklePoint
+    except Exception as e:
+        logToUser(e)
+        return None
 
 def pointToNative(pt: Point, sr: arcpy.SpatialReference) -> arcpy.PointGeometry:
     """Converts a Speckle Point to QgsPoint"""
-    #print("___pointToNative__")
-    #print(pt)
-    pt = scalePointToNative(pt, pt.units)
-    geom = arcpy.PointGeometry(arcpy.Point(pt.x, pt.y, pt.z), sr, has_z = True)
-    #print(geom)
-    return geom
+    try:
+        pt = scalePointToNative(pt, pt.units)
+        geom = arcpy.PointGeometry(arcpy.Point(pt.x, pt.y, pt.z), sr, has_z = True)
+        #print(geom)
+        return geom
+    
+    except Exception as e:
+        logToUser(e)
+        return None
 
 def pointToCoord(point: Point) -> List[float]:
     """Converts a Speckle Point to QgsPoint"""
-    pt = scalePointToNative(point, point.units)
-    coords = [pt.x, pt.y, pt.z]
-    #print(coords)
-    return coords
+    try:
+        pt = scalePointToNative(point, point.units)
+        coords = [pt.x, pt.y, pt.z]
+        #print(coords)
+        return coords
+    except Exception as e:
+        logToUser(e)
+    return [None, None, None]
 
 def scalePointToNative(point: Point, units: str) -> Point:
     """Scale point coordinates to meters"""
-    scaleFactor = get_scale_factor(units)
-    pt = Point(units = "m")
-    pt.x = point.x * scaleFactor
-    pt.y = point.y * scaleFactor
-    pt.z = 0 if math.isnan(point.z) else point.z * scaleFactor
-    return pt
+    try:
+        scaleFactor = get_scale_factor(units)
+        pt = Point(units = "m")
+        pt.x = point.x * scaleFactor
+        pt.y = point.y * scaleFactor
+        pt.z = 0 if math.isnan(point.z) else point.z * scaleFactor
+        return pt
+    except Exception as e:
+        logToUser(e)
+        return None
 
 def addZtoPoint(coords: List): 
-    if len(coords) == 2: coords.append(0)
-    return coords
+    try:
+        if len(coords) == 2: coords.append(0)
+        return coords
+    except Exception as e:
+        logToUser(e)
+        return None 

speckle_toolbox/esri/toolboxes/speckle/converter/geometry/polygon.py

@@ -1,4 +1,4 @@
-from typing import Sequence
+from typing import List, Sequence, Union
 import arcpy 
 import json
 from arcpy.arcobjects.arcobjects import SpatialReference 
@@ -7,221 +7,286 @@ from specklepy.objects import Base
 from specklepy.objects.geometry import Point, Arc, Circle, Polycurve, Polyline, Line
 
 try:
-    from speckle.converter.geometry.mesh import rasterToMesh
+    from speckle.converter.geometry.mesh import rasterToMesh, constructMesh, meshPartsFromPolygon
     from speckle.converter.geometry.point import pointToCoord, pointToNative
-    from speckle.converter.layers.symbologyTemplates import featureColorfromNativeRenderer
+    from speckle.converter.layers.symbology import featureColorfromNativeRenderer
     from speckle.converter.geometry.polyline import (polylineFromVerticesToSpeckle, 
                                                     circleToSpeckle, 
                                                     speckleArcCircleToPoints, 
                                                     curveToSpeckle, 
                                                     specklePolycurveToPoints
                                                     )
+    from speckle.converter.geometry.utils import speckleBoundaryToSpecklePts
+    from speckle.plugin_utils.logger import logToUser
+
 except: 
-    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.mesh import rasterToMesh
+    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.mesh import rasterToMesh, constructMesh, meshPartsFromPolygon
     from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.point import pointToCoord, pointToNative
-    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.symbologyTemplates import featureColorfromNativeRenderer
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.symbology import featureColorfromNativeRenderer
     from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.polyline import (polylineFromVerticesToSpeckle, 
                                                     circleToSpeckle, 
                                                     speckleArcCircleToPoints, 
                                                     curveToSpeckle, 
                                                     specklePolycurveToPoints
                                                     )
+    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.utils import speckleBoundaryToSpecklePts
+    from speckle_toolbox.esri.toolboxes.speckle.plugin_utils.logger import logToUser
+
 
 import math
 from panda3d.core import Triangulator
 
 
+def polygonToSpeckleMesh(feature, index: int, layer, multitype: bool):
+    print("________polygonToSpeckleMesh_____")
+    print(feature)
+    polygon = Base(units = "m")
+    try:
 
-def multiPolygonToSpeckle(geom, feature, index: str, layer, multiType: bool):
+        vertices = []
+        faces = [] 
+        colors = []
+        existing_vert = 0
+        
+        for i, p in enumerate(feature): 
+            #print("____start enumerate feature")
+            #print(p) #<geoprocessing array object object at 0x0000026796C77110>
+            
+            boundary, voids = getPolyBoundaryVoids(p, layer, multitype)
+            #print(boundary)
+            #print(voids)
+            polyBorder = speckleBoundaryToSpecklePts(boundary)
+            #print(polyBorder)
+            voidsAsPts = []
+            for v in voids:
+                pts = speckleBoundaryToSpecklePts(v)
+                voidsAsPts.append(pts)
+            #print(voidsAsPts)
+            #print("__to start meshPartsFromPolygon")
+            total_vert, vertices_x, faces_x, colors_x = meshPartsFromPolygon(polyBorder, voidsAsPts, existing_vert, index, layer)
+
+            existing_vert += total_vert
+            vertices.extend(vertices_x)
+            faces.extend(faces_x)
+            colors.extend(colors_x)
+        
+        #print("Colors: ")
+        #print(colors)
+        mesh = constructMesh(vertices, faces, colors)
+        polygon.displayValue = [ mesh ] 
+
+    except Exception as e:
+        logToUser(e)
+    return polygon 
+
+def getPolyBoundaryVoids(geom, layer, multiType: bool):
+    #print("__getPolyBoundaryVoids__")
+    voids: List[Union[None, Polyline, Arc, Line, Polycurve]] = []
+    #print(voids)
+    boundary = None
+    pointList = []
+    try:
+        partsBoundaries = []
+        partsVoids = []
+        if multiType is False: # Multipolygon
+            try: # might be no property "has curves"
+                if geom.hasCurves: 
+                    print("has curves")
+                    # geometry SHAPE@ tokens: https://pro.arcgis.com/en/pro-app/latest/arcpy/get-started/reading-geometries.htm
+                    print(geom.JSON) 
+                    boundary = curveToSpeckle(geom, "Polygon", geom, layer)
+                else: 
+                    print("no curves")
+                    for p in geom:
+                        for pt in p: 
+                            #print(pt)
+                            if pt != None: pointList.append(pt) 
+                    boundary = polylineFromVerticesToSpeckle(pointList, True, geom, layer) 
+                    print(boundary)
+            except: # for multipatches, no property "has curves" 
+                #print(geom)
+                for pt in geom: 
+                    #print(pt)
+                    if pt != None: pointList.append(pt) 
+                boundary = polylineFromVerticesToSpeckle(pointList, True, geom, layer) 
+
+            partsBoundaries.append(boundary)
+            partsVoids.append([])
+            
+        else: 
+            print("multi type")
+            for i, p in enumerate(geom):
+                #print(p)
+                for pt in p:  
+                    #print(pt) # 284394.58100903 5710688.11602606 NaN NaN
+                    if pt == None and boundary == None:  # first break 
+                        boundary = polylineFromVerticesToSpeckle(pointList, True, geom, layer) 
+                        pointList = []
+                    elif pt == None and boundary != None: # breaks btw voids
+                        void = polylineFromVerticesToSpeckle(pointList, True, geom, layer)
+                        voids.append(void)
+                        pointList = []
+                    elif pt != None: # add points to whatever list (boundary or void) 
+                        pointList.append(pt)
+
+                if boundary != None and len(pointList)>0: # remaining polyline
+                    void = polylineFromVerticesToSpeckle(pointList, True, geom, layer)
+                    voids.append(void)
+    
+    except Exception as e:
+        logToUser(e)
+    return boundary, voids
+
+def multiPolygonToSpeckle(geom, index: str, layer, multiType: bool):
 
     print("___MultiPolygon to Speckle____")
     polygon = []
-    #print(enumerate(geom.getPart())) # this method ignores curvature and voids
-    #print(json.loads(geom.JSON))
-    #js = json.loads(geom.JSON)['rings']
-    #https://desktop.arcgis.com/en/arcmap/latest/analyze/python/reading-geometries.htm
-    for i,x in enumerate(feature): # [[x,x,x]
-        print("Part # " + str(i+1))
-        print(x)
-        boundaryFinished = 0
-        arrBoundary = []
-        arrInnerRings = []
-        for ptn in x: # arcpy.Point
-            if ptn is None: 
-                boundaryFinished += 1
-                arrInnerRings.append([]) # start of new Inner Ring
-            elif boundaryFinished == 0 and ptn is not None: 
-                arrBoundary.append(ptn)
-            elif boundaryFinished == 1 and ptn is not None: 
-                arrInnerRings[len(arrInnerRings)-1].append(ptn)
+    try: 
+        #print(enumerate(geom.getPart())) # this method ignores curvature and voids
+        #print(json.loads(geom.JSON))
+        #js = json.loads(geom.JSON)['rings']
+        #https://desktop.arcgis.com/en/arcmap/latest/analyze/python/reading-geometries.htm
+        for i,x in enumerate(geom): # [[x,x,x]
+            print("Part # " + str(i+1))
+            print(x)
+            boundaryFinished = 0
+            arrBoundary = []
+            arrInnerRings = []
+            for ptn in x: # arcpy.Point
+                if ptn is None: 
+                    boundaryFinished += 1
+                    arrInnerRings.append([]) # start of new Inner Ring
+                elif boundaryFinished == 0 and ptn is not None: 
+                    arrBoundary.append(ptn)
+                elif boundaryFinished == 1 and ptn is not None: 
+                    arrInnerRings[len(arrInnerRings)-1].append(ptn)
 
-        full_arr = [arrBoundary] + arrInnerRings
-        #print(full_arr)
-        poly = arcpy.Polygon(arcpy.Array(full_arr), arcpy.Describe(layer.dataSource).SpatialReference, has_z = True)
-        #print(poly) #<geoprocessing describe geometry object object at 0x000002B2D3E338D0> 
-        polygon.append(polygonToSpeckle(poly, feature, index, layer, poly.isMultipart))
+            full_arr = [arrBoundary] + arrInnerRings
+            #print(full_arr)
+            poly = arcpy.Polygon(arcpy.Array(full_arr), arcpy.Describe(layer.dataSource).SpatialReference, has_z = True)
+            #print(poly) #<geoprocessing describe geometry object object at 0x000002B2D3E338D0> 
+            polygon.append(polygonToSpeckle(poly, index, layer, poly.isMultipart))
 
+    except Exception as e:
+        logToUser(e)
     return polygon
 
 
-def polygonToSpeckle(geom, feature, index: int, layer, multiType: bool):
+def polygonToSpeckle(geom, index: int, layer, multitype: bool):
     """Converts a Polygon to Speckle"""
-    #try: 
-    print("___Polygon to Speckle____")
-    print(geom)
     polygon = Base(units = "m")
-    pointList = []
-    voidPointList = []
-    voids = []
-    boundary = None
-    data = arcpy.Describe(layer.dataSource)
-    sr = data.spatialReference
+    try: 
+        print("___Polygon to Speckle____")
+        print(geom)
 
-    print(multiType)
-    partsBoundaries = []
-    partsVoids = []
-    
+        boundary, voids = getPolyBoundaryVoids(geom, layer, multitype)
+
+        data = arcpy.Describe(layer.dataSource)
+        sr = data.spatialReference
+
+        if boundary is None: return None 
+        polygon.boundary = boundary
+        polygon.voids = voids
+        polygon.displayValue = [ boundary ] + voids
+        #print(boundary)
+
+        ############# mesh 
+        vertices = []
+        polyBorder = []
+        total_vertices = 0
+        if isinstance(boundary, Circle) or isinstance(boundary, Arc): 
+            polyBorder = speckleArcCircleToPoints(boundary) 
+        elif isinstance(boundary, Polycurve): 
+            polyBorder = specklePolycurveToPoints(boundary) 
+            #polygon.boundary.displayValue.closed = True 
+        elif isinstance(boundary, Line): pass
+        elif isinstance(boundary, Polyline): 
+            try: polyBorder = boundary.as_points()
+            except: pass # if Line
+        #print(polyBorder)
         
-    #else: # no curves
-    if multiType is False: # Multipolygon
-        if geom.hasCurves: 
-            print("has curves")
-            # geometry SHAPE@ tokens: https://pro.arcgis.com/en/pro-app/latest/arcpy/get-started/reading-geometries.htm
-            print(geom.JSON) 
-            boundary = curveToSpeckle(geom, "Polygon", feature, layer)
-        else: 
-            print("no curves")
-            for p in geom:
-                for pt in p: 
-                    if pt != None: pointList.append(pt) 
-            boundary = polylineFromVerticesToSpeckle(pointList, True, feature, layer) 
-        partsBoundaries.append(boundary)
-        partsVoids.append([])
 
-    else: 
-        print("multi type")
-        for i, p in enumerate(geom):
-            print(p)
-            for pt in p:  
-                #print(pt) # 284394.58100903 5710688.11602606 NaN NaN
-                if pt == None and boundary == None:  # first break 
-                    boundary = polylineFromVerticesToSpeckle(pointList, True, feature, layer) 
-                    pointList = []
-                elif pt == None and boundary != None: # breaks btw voids
-                    void = polylineFromVerticesToSpeckle(pointList, True, feature, layer)
-                    voids.append(void)
-                    pointList = []
-                elif pt != None: # add points to whatever list (boundary or void) 
-                    pointList.append(pt)
+        if len(polyBorder)>2: # at least 3 points 
+            print("make meshes from polygons")
+            if len(voids) == 0: # if there is a mesh with no voids
+                for pt in polyBorder:
+                    if isinstance(pt, Point): pt = pointToNative(pt, sr).getPart() # SR unknown
+                    x = pt.X
+                    y = pt.Y
+                    z = 0 if math.isnan(pt.Z) else pt.Z
+                    vertices.extend([x, y, z])
+                    total_vertices += 1
+                #print(vertices)
+                ran = range(0, total_vertices)
+                faces = [total_vertices]
+                faces.extend([i for i in ran])
+                #print(faces)
+                # else: https://docs.panda3d.org/1.10/python/reference/panda3d.core.Triangulator
+            else:
+                trianglator = Triangulator()
+                faces = []
 
-            if boundary != None and len(pointList)>0: # remaining polyline
-                void = polylineFromVerticesToSpeckle(pointList, True, feature, layer)
-                voids.append(void)
+                # add boundary points
+                #polyBorder = boundary.as_points()
+                pt_count = 0
+                # add extra middle point for border
+                for pt in polyBorder:
+                    if pt_count < len(polyBorder)-1: 
+                        pt2 = polyBorder[pt_count+1]
+                    else: pt2 = polyBorder[0]
+                    
+                    trianglator.addPolygonVertex(trianglator.addVertex(pt.x, pt.y))
+                    vertices.extend([pt.x, pt.y, pt.z])
 
-            #partsBoundaries.append(boundary)
-            #partsVoids.append(local_voids)
+                    #trianglator.addPolygonVertex(trianglator.addVertex((pt.x+pt2.x)/4*3, (pt.y+pt2.y)/4*3))
+                    #vertices.extend([(pt.x+pt2.x)/4*3, (pt.y+pt2.y)/4*3, (pt.z+pt2.z)/4*3])
 
-    if boundary is None: return None 
-    polygon.boundary = boundary
-    polygon.voids = voids
-    polygon.displayValue = [ boundary ] + voids
-    #print(boundary)
+                    trianglator.addPolygonVertex(trianglator.addVertex((pt.x+pt2.x)/2, (pt.y+pt2.y)/2))
+                    vertices.extend([(pt.x+pt2.x)/2, (pt.y+pt2.y)/2, (pt.z+pt2.z)/2])
+                    
+                    #trianglator.addPolygonVertex(trianglator.addVertex((pt.x+pt2.x)/4, (pt.y+pt2.y)/4))
+                    #vertices.extend([(pt.x+pt2.x)/4, (pt.y+pt2.y)/4, (pt.z+pt2.z)/4])
 
-    ############# mesh 
-    vertices = []
-    polyBorder = []
-    total_vertices = 0
-    if isinstance(boundary, Circle) or isinstance(boundary, Arc): 
-        polyBorder = speckleArcCircleToPoints(boundary) 
-    elif isinstance(boundary, Polycurve): 
-        polyBorder = specklePolycurveToPoints(boundary) 
-        #polygon.boundary.displayValue.closed = True 
-    elif isinstance(boundary, Line): pass
-    elif isinstance(boundary, Polyline): 
-        try: polyBorder = boundary.as_points()
-        except: pass # if Line
-    #print(polyBorder)
-    
+                    total_vertices += 2
+                    pt_count += 1
 
-    if len(polyBorder)>2: # at least 3 points 
-        print("make meshes from polygons")
-        if len(voids) == 0: # if there is a mesh with no voids
-            for pt in polyBorder:
-                if isinstance(pt, Point): pt = pointToNative(pt, sr).getPart() # SR unknown
-                x = pt.X
-                y = pt.Y
-                z = 0 if math.isnan(pt.Z) else pt.Z
-                vertices.extend([x, y, z])
-                total_vertices += 1
-            #print(vertices)
-            ran = range(0, total_vertices)
-            faces = [total_vertices]
-            faces.extend([i for i in ran])
-            #print(faces)
-            # else: https://docs.panda3d.org/1.10/python/reference/panda3d.core.Triangulator
-        else:
-            trianglator = Triangulator()
-            faces = []
-
-            # add boundary points
-            #polyBorder = boundary.as_points()
-            pt_count = 0
-            # add extra middle point for border
-            for pt in polyBorder:
-              if pt_count < len(polyBorder)-1: 
-                  pt2 = polyBorder[pt_count+1]
-              else: pt2 = polyBorder[0]
-              
-              trianglator.addPolygonVertex(trianglator.addVertex(pt.x, pt.y))
-              vertices.extend([pt.x, pt.y, pt.z])
-
-              #trianglator.addPolygonVertex(trianglator.addVertex((pt.x+pt2.x)/4*3, (pt.y+pt2.y)/4*3))
-              #vertices.extend([(pt.x+pt2.x)/4*3, (pt.y+pt2.y)/4*3, (pt.z+pt2.z)/4*3])
-
-              trianglator.addPolygonVertex(trianglator.addVertex((pt.x+pt2.x)/2, (pt.y+pt2.y)/2))
-              vertices.extend([(pt.x+pt2.x)/2, (pt.y+pt2.y)/2, (pt.z+pt2.z)/2])
-              
-              #trianglator.addPolygonVertex(trianglator.addVertex((pt.x+pt2.x)/4, (pt.y+pt2.y)/4))
-              #vertices.extend([(pt.x+pt2.x)/4, (pt.y+pt2.y)/4, (pt.z+pt2.z)/4])
-
-              total_vertices += 2
-              pt_count += 1
-
-            #add void points
-            for i in range(len(voids)):
-              trianglator.beginHole()
+                #add void points
+                for i in range(len(voids)):
+                    trianglator.beginHole()
 
 
-              pts = []
-              if isinstance(voids[i], Circle) or isinstance(voids[i], Arc): 
-                  pts = speckleArcCircleToPoints(voids[i]) 
-              elif isinstance(voids[i], Polycurve): 
-                  pts = specklePolycurveToPoints(voids[i]) 
-              elif isinstance(voids[i], Line): pass
-              else: 
-                  try: pts = voids[i].as_points()
-                  except: pass # if Line
-              #pts = voids[i].as_points()
-              for pt in pts:
-                trianglator.addHoleVertex(trianglator.addVertex(pt.x, pt.y))
-                vertices.extend([pt.x, pt.y, pt.z])
-                total_vertices += 1
+                    pts = []
+                    if isinstance(voids[i], Circle) or isinstance(voids[i], Arc): 
+                        pts = speckleArcCircleToPoints(voids[i]) 
+                    elif isinstance(voids[i], Polycurve): 
+                        pts = specklePolycurveToPoints(voids[i]) 
+                    elif isinstance(voids[i], Line): pass
+                    else: 
+                        try: pts = voids[i].as_points()
+                        except: pass # if Line
+                    #pts = voids[i].as_points()
+                    for pt in pts:
+                        trianglator.addHoleVertex(trianglator.addVertex(pt.x, pt.y))
+                        vertices.extend([pt.x, pt.y, pt.z])
+                        total_vertices += 1
 
-            trianglator.triangulate()
-            i = 0
-            while i < trianglator.getNumTriangles():
-              tr = [trianglator.getTriangleV0(i),trianglator.getTriangleV1(i),trianglator.getTriangleV2(i)]
-              faces.extend([3, tr[0], tr[1], tr[2]])
-              i+=1
-            ran = range(0, total_vertices)
-        
+                trianglator.triangulate()
+                i = 0
+                while i < trianglator.getNumTriangles():
+                    tr = [trianglator.getTriangleV0(i),trianglator.getTriangleV1(i),trianglator.getTriangleV2(i)]
+                    faces.extend([3, tr[0], tr[1], tr[2]])
+                    i+=1
+                ran = range(0, total_vertices)
+            
+            #print(polygon)
+            col = featureColorfromNativeRenderer(index, layer)
+            colors = [col for i in ran] # apply same color for all vertices
+            mesh = rasterToMesh(vertices, faces, colors)
+            polygon.displayValue = mesh 
+        #print("print resulted polygon")
         #print(polygon)
-        col = featureColorfromNativeRenderer(index, layer)
-        colors = [col for i in ran] # apply same color for all vertices
-        mesh = rasterToMesh(vertices, faces, colors)
-        polygon.displayValue = mesh 
-    #print("print resulted polygon")
-    #print(polygon)
+    except Exception as e:
+        logToUser(e)
     return polygon
 
 def polygonToNative(poly: Base, sr: arcpy.SpatialReference) -> arcpy.Polygon:
@@ -230,44 +295,48 @@ def polygonToNative(poly: Base, sr: arcpy.SpatialReference) -> arcpy.Polygon:
     Each being a Speckle Polyline and List of polylines respectively."""
 
     print("_______Drawing polygons____")
-    #pts = [pointToCoord(pt) for pt in poly["boundary"].as_points()]
-    pointsSpeckle = []
-    if isinstance(poly["boundary"], Circle) or isinstance(poly["boundary"], Arc): 
-        pointsSpeckle = speckleArcCircleToPoints(poly["boundary"]) 
-    elif isinstance(poly["boundary"], Polycurve): 
-        pointsSpeckle = specklePolycurveToPoints(poly["boundary"]) 
-    elif isinstance(poly["boundary"], Line): pass
-    else: 
-        try: pointsSpeckle = poly["boundary"].as_points()
-        except: pass # if Line
+    polygon = None
+    try: 
+        #pts = [pointToCoord(pt) for pt in poly["boundary"].as_points()]
+        pointsSpeckle = []
+        if isinstance(poly["boundary"], Circle) or isinstance(poly["boundary"], Arc): 
+            pointsSpeckle = speckleArcCircleToPoints(poly["boundary"]) 
+        elif isinstance(poly["boundary"], Polycurve): 
+            pointsSpeckle = specklePolycurveToPoints(poly["boundary"]) 
+        elif isinstance(poly["boundary"], Line): pass
+        else: 
+            try: pointsSpeckle = poly["boundary"].as_points()
+            except: pass # if Line
 
-    pts = [pointToCoord(pt) for pt in pointsSpeckle]
-    #print(pts)
+        pts = [pointToCoord(pt) for pt in pointsSpeckle]
+        #print(pts)
 
-    outer_arr = [arcpy.Point(*coords) for coords in pts]
-    outer_arr.append(outer_arr[0])
-    geomPart = []
-    try:
-        for void in poly["voids"]: 
-            #print(void)
-            #pts = [pointToCoord(pt) for pt in void.as_points()]
-            pointsSpeckle = []
-            if isinstance(void, Circle) or isinstance(void, Arc): 
-                pointsSpeckle = speckleArcCircleToPoints(void) 
-            elif isinstance(void, Polycurve): 
-                pointsSpeckle = specklePolycurveToPoints(void) 
-            elif isinstance(void, Line): pass
-            else: 
-                try: pointsSpeckle = void.as_points()
-                except: pass # if Line
-            pts = [pointToCoord(pt) for pt in pointsSpeckle]
+        outer_arr = [arcpy.Point(*coords) for coords in pts]
+        outer_arr.append(outer_arr[0])
+        geomPart = []
+        try:
+            for void in poly["voids"]: 
+                #print(void)
+                #pts = [pointToCoord(pt) for pt in void.as_points()]
+                pointsSpeckle = []
+                if isinstance(void, Circle) or isinstance(void, Arc): 
+                    pointsSpeckle = speckleArcCircleToPoints(void) 
+                elif isinstance(void, Polycurve): 
+                    pointsSpeckle = specklePolycurveToPoints(void) 
+                elif isinstance(void, Line): pass
+                else: 
+                    try: pointsSpeckle = void.as_points()
+                    except: pass # if Line
+                pts = [pointToCoord(pt) for pt in pointsSpeckle]
 
-            inner_arr = [arcpy.Point(*coords) for coords in pts]
-            inner_arr.append(inner_arr[0])
-            geomPart.append(arcpy.Array(inner_arr))
-    except:pass
-    geomPart.insert(0, outer_arr)
-    geomPartArray = arcpy.Array(geomPart)
-    polygon = arcpy.Polygon(geomPartArray, sr, has_z=True)
+                inner_arr = [arcpy.Point(*coords) for coords in pts]
+                inner_arr.append(inner_arr[0])
+                geomPart.append(arcpy.Array(inner_arr))
+        except:pass
+        geomPart.insert(0, outer_arr)
+        geomPartArray = arcpy.Array(geomPart)
+        polygon = arcpy.Polygon(geomPartArray, sr, has_z=True)
 
+    except Exception as e:
+        logToUser(e)
     return polygon

speckle_toolbox/esri/toolboxes/speckle/converter/geometry/utils.py

@@ -0,0 +1,29 @@
+
+from specklepy.objects.geometry import Point, Line, Polyline, Circle, Arc, Polycurve
+from specklepy.objects import Base
+from typing import List, Union
+
+try:
+    from speckle.converter.geometry.polyline import speckleArcCircleToPoints, specklePolycurveToPoints
+    from speckle.plugin_utils.logger import logToUser
+except:
+    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.polyline import speckleArcCircleToPoints, specklePolycurveToPoints
+    from speckle_toolbox.esri.toolboxes.speckle.plugin_utils.logger import logToUser
+
+
+def speckleBoundaryToSpecklePts(boundary: Union[None, Polyline, Arc, Line, Polycurve]) -> List[Point]:
+    #print("__speckleBoundaryToSpecklePts__")
+    # add boundary points
+    polyBorder = []
+    try:
+        if isinstance(boundary, Circle) or isinstance(boundary, Arc): 
+            polyBorder = speckleArcCircleToPoints(boundary) 
+        elif isinstance(boundary, Polycurve): 
+            polyBorder = specklePolycurveToPoints(boundary) 
+        elif isinstance(boundary, Line): pass
+        else: 
+            try: polyBorder = boundary.as_points()
+            except: pass # if Line or None
+    except Exception as e:
+        logToUser(e)
+    return polyBorder

speckle_toolbox/esri/toolboxes/speckle/converter/layers/_init_.py

@@ -1,833 +0,0 @@
-"""
-Contains all Layer related classes and methods.
-"""
-import os
-from typing import Any, List, Tuple, Union
-
-#from regex import D
-
-try:
-    from speckle.converter.layers.CRS import CRS
-    from speckle.converter.layers.Layer import Layer, VectorLayer, RasterLayer
-    from speckle.converter.layers.symbologyTemplates import vectorRendererToNative, rasterRendererToNative, rendererToSpeckle 
-    from speckle.converter.layers.feature import featureToNative, featureToSpeckle, cadFeatureToNative, bimFeatureToNative, rasterFeatureToSpeckle
-
-    from speckle.converter.geometry.mesh import rasterToMesh, meshToNative
-    from speckle.converter.layers.utils import findTransformation
-    from speckle.converter.layers.utils import getLayerAttributes, newLayerGroupAndName, validate_path
-except: 
-    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.CRS import CRS
-    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.Layer import Layer, VectorLayer, RasterLayer
-    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.symbologyTemplates import vectorRendererToNative, rasterRendererToNative, rendererToSpeckle 
-    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.feature import featureToNative, featureToSpeckle, cadFeatureToNative, bimFeatureToNative, rasterFeatureToSpeckle
-
-    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.mesh import rasterToMesh, meshToNative
-    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.utils import findTransformation
-    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.utils import getLayerAttributes, newLayerGroupAndName, validate_path
-
-from specklepy.objects import Base
-from specklepy.objects.geometry import Mesh
-
-import arcgisscripting
-import pandas as pd
-import arcpy
-from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
-from arcpy.management import (CreateFeatureclass, MakeFeatureLayer,
-                              AddFields, AlterField, DefineProjection )
-
-import numpy as np
-
-def convertSelectedLayers(all_layers: List[arcLayer], selected_layers: List[str], project: ArcGISProject) -> List[Union[VectorLayer,Layer]]:
-    """Converts the current selected layers to Speckle"""
-    print("________Convert Layers_________")
-    result = []
-    for layer in selected_layers:
-        layerToSend = None
-        for c in range(len(all_layers)):
-            if int(layer.split("-",1)[0]) == c:
-                layerToSend = all_layers[c]
-                break
-        if layerToSend is not None: 
-            ds = layerToSend.dataSource #file path
-            #if layerToSend.isFeatureLayer: 
-            newBaseLayer = layerToSpeckle(layerToSend, project)
-            if newBaseLayer is not None: 
-                result.append(newBaseLayer)
-            #elif layerToSend.isRasterLayer: pass
-            print(result)
-
-    return result
-
-def layerToSpeckle(layer: arcLayer, project: ArcGISProject) -> Union[VectorLayer, RasterLayer]: #now the input is QgsVectorLayer instead of qgis._core.QgsLayerTreeLayer
-    """Converts a given QGIS Layer to Speckle"""
-    print("________Convert Feature Layer_________")
-
-    speckleLayer = None
-
-    projectCRS = project.activeMap.spatialReference
-    try: data = arcpy.Describe(layer.dataSource)
-    except OSError as e: arcpy.AddWarning(str(e.args[0])); return
-
-    layerName = layer.name
-    crs = data.SpatialReference
-    units = "m"
-    layerObjs = []
-
-    # Convert CRS to speckle, use the projectCRS
-    speckleReprojectedCrs = CRS(name = projectCRS.name, wkt = projectCRS.exportToString(), units = units)
-    layerCRS = CRS(name=crs.name, wkt=crs.exportToString(), units = units) 
-    
-    #renderer = selectedLayer.renderer()
-    #layerRenderer = rendererToSpeckle(renderer) 
-    
-    if layer.isFeatureLayer: 
-        print("VECTOR LAYER HERE")
-        
-        speckleLayer = VectorLayer(units = "m")
-        speckleLayer.type="VectorLayer"
-        speckleLayer.name = layerName
-        speckleLayer.crs = speckleReprojectedCrs
-        speckleLayer.renderer = rendererToSpeckle(project, project.activeMap, layer, None)
-        #speckleLayer.datum = datum
-
-
-        try: # https://pro.arcgis.com/en/pro-app/2.8/arcpy/get-started/the-spatial-reference-object.htm
-            
-            #print(data.datasetType) # FeatureClass
-            if data.datasetType == "FeatureClass": #FeatureClass, ?Table Properties, ?Datasets
-                
-                # write feature attributes
-                fieldnames = [field.name for field in data.fields]
-                rows_shapes = arcpy.da.SearchCursor(layer.longName, "Shape@") # arcpy.da.SearchCursor(in_table, field_names, {where_clause}, {spatial_reference}, {explode_to_points}, {sql_clause})
-                print("__ start iterating features")
-                row_shapes_list = [x for k, x in enumerate(rows_shapes)]
-                for i, features in enumerate(row_shapes_list):
-
-                    print("____error Feature # " + str(i+1)) # + " / " + str(sum(1 for _ in enumerate(rows_shapes))))
-                    if features[0] is None: continue 
-                    feat = features[0]
-                    #print(feat) # <geoprocessing describe geometry object object at 0x000002A75D6A4BD0>
-                    #print(feat.hasCurves)
-                    #print(feat.partCount)
-
-                    if feat is not None: 
-                        print(feat)
-                        rows_attributes = arcpy.da.SearchCursor(layer.longName, fieldnames)
-                        row_attr = []
-                        for k, attrs in enumerate(rows_attributes):
-                            if i == k: row_attr = attrs; break
-
-                        # if curves detected, createa new feature class, turn to segments and get the same feature but in straigt lines
-                        #print(feat.hasCurves)
-                        if feat.hasCurves:
-                            #f_class_modified = curvedFeatureClassToSegments(layer)
-                            #rows_shapes_modified = arcpy.da.SearchCursor(f_class_modified, "Shape@") 
-                            #row_shapes_list_modified = [x for k, x in enumerate(rows_shapes_modified)]
-
-                            feat = feat.densify("ANGLE", 1000, 0.12)
-                            #print(feat)
-
-
-                        b = featureToSpeckle(fieldnames, row_attr, i, feat, projectCRS, project, layer)
-                        if b is not None: layerObjs.append(b)
-                        
-                    print("____End of Feature # " + str(i+1))
-                    
-                print("__ finish iterating features")
-                speckleLayer.features=layerObjs
-                speckleLayer.geomType = data.shapeType
-
-                if len(speckleLayer.features) == 0: return None
-
-                #layerBase.renderer = layerRenderer
-                #layerBase.applicationId = selectedLayer.id()
-
-        except OSError as e: 
-            arcpy.AddWarning(str(e))
-            return
-
-    elif layer.isRasterLayer:
-        print("RASTER IN DA HOUSE")
-        print(layer.name) # London_square.tif
-        print(arcpy.Describe(layer.dataSource)) # <geoprocessing describe data object object at 0x000002507C7F3BB0>
-        print(arcpy.Describe(layer.dataSource).datasetType) # RasterDataset
-        b = rasterFeatureToSpeckle(layer, projectCRS, project)
-        if b is not None: layerObjs.append(b)
-
-        speckleLayer = RasterLayer(units = "m", type="RasterLayer")
-        speckleLayer.name = layerName
-        speckleLayer.crs = speckleReprojectedCrs
-        speckleLayer.rasterCrs = layerCRS
-        speckleLayer.type="RasterLayer"
-        #speckleLayer.geomType="Raster"
-        speckleLayer.features = layerObjs
-        
-        speckleLayer.renderer = rendererToSpeckle(project, project.activeMap, layer, b)
-        
-        #speckleLayer.renderer = layerRenderer
-        #speckleLayer.applicationId = selectedLayer.id()
-
-    return speckleLayer
-
-def layerToNative(layer: Union[Layer, VectorLayer, RasterLayer], streamBranch: str, project: ArcGISProject):
-
-    if layer.type is None:
-        # Handle this case
-        return
-    elif layer.type.endswith("VectorLayer"):
-        return vectorLayerToNative(layer, streamBranch, project)
-    elif layer.type.endswith("RasterLayer"):
-        return rasterLayerToNative(layer, streamBranch, project)
-    return None
-
-def bimLayerToNative(layerContentList: List[Base], layerName: str, streamBranch: str, project: ArcGISProject) :
-    print("01______BIM layer to native")
-    print(layerName)
-    geom_meshes = []
-    layer_meshes = None
-    #filter speckle objects by type within each layer, create sub-layer for each type (points, lines, polygons, mesh?)
-    for geom in layerContentList:
-        try: 
-            if geom.displayMesh: geom_meshes.append(geom)
-        except:
-            try: 
-                if geom.displayValue: geom_meshes.append(geom)
-            except: pass
-
-    if len(geom_meshes)>0: layer_meshes = bimVectorLayerToNative(geom_meshes, layerName, "Mesh", streamBranch, project)
-
-    return True
-
-
-def bimVectorLayerToNative(geomList, layerName: str, geomType: str, streamBranch: str, project: ArcGISProject): 
-    # no support for mltipatches, maybe in 3.1: https://community.esri.com/t5/arcgis-pro-ideas/better-support-for-multipatches-in-arcpy/idi-p/953614/page/2#comments
-    print("02_________BIM vector layer to native_____")
-    #get Project CRS, use it by default for the new received layer
-    
-    vl = None
-    layerName = layerName + "_" + geomType
-    layerName = layerName.replace("[","_").replace("]","_").replace(" ","_").replace("-","_").replace("(","_").replace(")","_").replace(":","_").replace("\\","_").replace("/","_").replace("\"","_").replace("&","_").replace("@","_").replace("$","_").replace("%","_").replace("^","_")
-    #if not "__Structural_Foundations_Mesh" in layerName: return None
-    
-    sr = arcpy.SpatialReference(text = project.activeMap.spatialReference.exportToString())
-    active_map = project.activeMap
-    
-    path = project.filePath.replace("aprx","gdb") #
-    path_bim = "\\".join(project.filePath.split("\\")[:-1]) + "\\Layers_Speckle\\BIM_layers_speckle\\" + streamBranch+ "\\" + layerName + "\\" #arcpy.env.workspace + "\\" #
-    print(path_bim)
-    
-    if not os.path.exists(path_bim): os.makedirs(path_bim)
-    print(path)
-    
-    if sr.type == "Geographic": 
-        arcpy.AddMessage(f"Project CRS is set to Geographic type, and objects in linear units might not be received correctly")
-
-    #CREATE A GROUP "received blabla" with sublayers
-    layerGroup = None
-    newGroupName = f'{streamBranch}'
-    #print(newGroupName)
-    for l in active_map.listLayers():
-        if l.longName == newGroupName: layerGroup = l; break 
-    
-    #find ID of the layer with a matching name in the "latest" group 
-    newName = f'{streamBranch.split("_")[len(streamBranch.split("_"))-1]}_{layerName}'
-    print(newName)
-
-    all_layer_names = []
-    layerExists = 0
-    for l in project.activeMap.listLayers(): 
-        if l.longName.startswith(newGroupName + "\\"):
-            all_layer_names.append(l.longName)
-    #print(all_layer_names)
-
-    longName = streamBranch + "\\" + newName 
-    if longName in all_layer_names: 
-        for index, letter in enumerate('234567890abcdefghijklmnopqrstuvwxyz'):
-            if (longName + "_" + letter) not in all_layer_names: newName += "_"+letter; layerExists +=1; break 
-
-    # particularly if the layer comes from ArcGIS
-    if "mesh" in geomType.lower(): geomType = "Multipatch"
-
-    #print("Create feature class (cad): ")
-    # should be created inside the workspace to be a proper Feature class (not .shp) with Nullable Fields
-    class_name = ("f_class_" + newName)
-    #f_class = CreateFeatureclass(path, class_name, geomType, has_z="ENABLED", spatial_reference = sr)
-
-
-    shp = meshToNative(geomList, path_bim + newName)
-    print("____ meshes saved___")
-    print(shp)
-    #print(path)
-    #print(class_name)
-    validated_class_path = validate_path(class_name)
-    print(validated_class_path)
-    validated_class_name = validated_class_path.split("\\")[len(validated_class_path.split("\\"))-1]
-    print(validated_class_name)
-    f_class = arcpy.conversion.FeatureClassToFeatureClass(shp, path, validated_class_name)
-    # , spatial_reference = sr
-    #arcpy.management.Project(in_dataset, f_class, sr, in_coor_system=sr)
-
-    print(f_class)
-    #print(geomList)
-
-    # get and set Layer attribute fields
-    # example: https://resource.esriuk.com/blog/an-introductory-slice-of-arcpy-in-arcgis-pro/
-    newFields = getLayerAttributes(geomList)
-    
-    fields_to_ignore = ["arcgisgeomfromspeckle", "shape", "objectid", "displayMesh"]
-    matrix = []
-    all_keys = []
-    all_key_types = []
-    max_len = 52
-
-    print("___ after layer attributes: ___________")
-    print(newFields.items())
-    #try:
-    for key, value in newFields.items(): 
-        existingFields = [fl.name for fl in arcpy.ListFields(validated_class_name)]
-        #print(existingFields)
-        if key not in existingFields  and key.lower() not in fields_to_ignore: # exclude geometry and default existing fields
-            #print(key)
-            # signs that should not be used as field names and table names: https://support.esri.com/en/technical-article/000005588
-            key = key.replace(" ","_").replace("-","_").replace("(","_").replace(")","_").replace(":","_").replace("\\","_").replace("/","_").replace("\"","_").replace("&","_").replace("@","_").replace("$","_").replace("%","_").replace("^","_") 
-            if key[0] in ['0','1','2','3','4','5','6','7','8','9']: key = "_"+key
-            if len(key)>max_len: key = key[:max_len]
-            #print(all_keys)
-            if key in all_keys:
-                for index, letter in enumerate('1234567890abcdefghijklmnopqrstuvwxyz'):
-                    if len(key)<max_len and (key+letter) not in all_keys: key+=letter; break 
-                    if len(key) == max_len and (key[:9] + letter) not in all_keys: key=key[:9] + letter; break 
-            if key not in all_keys: 
-                all_keys.append(key)
-                all_key_types.append(value)
-                #print(all_keys)
-                matrix.append([key, value, key, 255])
-    print(all_keys)
-    print(len(all_keys))
-    if len(matrix)>0: AddFields(str(f_class), matrix)
-    print(matrix)
-    
-    fets = []
-    print("_________BIM FeatureS To Native___________")
-    for f in geomList[:]: 
-        new_feat = bimFeatureToNative(f, newFields, sr, path_bim)
-        if new_feat != "" and new_feat != None: 
-            fets.append(new_feat)
-    print(len(fets))
-        
-    
-    if len(fets) == 0: return None
-    count = 0
-    rowValues = []
-    for i, feat in enumerate(fets):
-
-        row = []
-        heads = []
-        for key in all_keys:
-            try:
-                row.append(feat[key])
-                heads.append(key)
-            except Exception as e: 
-                row.append(None)
-                heads.append(key)
-
-        rowValues.append(row)
-        count += 1
-    print(heads)
-    
-    with arcpy.da.UpdateCursor(f_class, heads) as cur:
-        # For each row, evaluate the WELL_YIELD value (index position 
-        # of 0), and update WELL_CLASS (index position of 1)
-        shp_num = 0
-        #print(heads)
-        try:
-            for rowShape in cur: 
-                #print(rowShape)
-                for i,r in enumerate(rowShape):
-                    #print(heads[i])
-                    #print(matrix[i])
-                    rowShape[i] = rowValues[shp_num][i]
-                    #print(type(rowShape[i]))
-                    if matrix[i][1] == 'TEXT' and rowShape[i] is not None: rowShape[i] = str(rowValues[shp_num][i]) 
-                    #print(type(rowShape[i]))
-                    if isinstance(rowValues[shp_num][i], str): # cut if string is too long
-                        rowShape[i] = rowValues[shp_num][i][:255]
-                    #print(rowShape[i])
-                #print(rowShape)
-                cur.updateRow(rowShape)
-                shp_num += 1
-                #print(shp_num)
-        except Exception as e: 
-            print("Layer attribute error: " + str(e))
-            #print(i)
-            print(shp_num)
-            print(len(rowValues))
-            #print(rowValues[i])
-            #print(len(rowValues[i]))
-            arcpy.AddWarning("Layer attribute error: " + e)
-    del cur 
-    
-    print("create layer:")
-    vl = MakeFeatureLayer(str(f_class), newName).getOutput(0)
-
-    active_map.addLayerToGroup(layerGroup, vl)
-    print("created2")
-    #os.remove(path_bim)
-
-    return True #last one
-
-def cadLayerToNative(layerContentList: List[Base], layerName: str, streamBranch: str, project: ArcGISProject) :
-    print("01______Cad vector layer to native")
-    print(layerName)
-    geom_points = []
-    geom_polylines = []
-    geom_polygones = []
-    geom_meshes = []
-    #filter speckle objects by type within each layer, create sub-layer for each type (points, lines, polygons, mesh?)
-    print(layerContentList)
-    for geom in layerContentList:
-        #print(geom)
-        if geom.speckle_type == "Objects.Geometry.Point": 
-            geom_points.append(geom)
-        if geom.speckle_type == "Objects.Geometry.Line" or geom.speckle_type == "Objects.Geometry.Polyline" or geom.speckle_type == "Objects.Geometry.Curve" or geom.speckle_type == "Objects.Geometry.Arc" or geom.speckle_type == "Objects.Geometry.Circle" or geom.speckle_type == "Objects.Geometry.Ellipse" or geom.speckle_type == "Objects.Geometry.Polycurve":
-            geom_polylines.append(geom)
-    
-    if len(geom_points)>0: layer_points = cadVectorLayerToNative(geom_points, layerName, "Points", streamBranch, project)
-    if len(geom_polylines)>0: layer_polylines = cadVectorLayerToNative(geom_polylines, layerName, "Polylines", streamBranch, project)
-
-    return [layer_points, layer_polylines]
-
-def cadVectorLayerToNative(geomList, layerName: str, geomType: str, streamBranch: str, project: ArcGISProject): 
-    print("02_________CAD vector layer to native_____")
-    #get Project CRS, use it by default for the new received layer
-    vl = None
-    layerName = layerName.replace("[","_").replace("]","_").replace(" ","_").replace("-","_").replace("(","_").replace(")","_").replace(":","_").replace("\\","_").replace("/","_").replace("\"","_").replace("&","_").replace("@","_").replace("$","_").replace("%","_").replace("^","_")
-    layerName = layerName + "_" + geomType
-    print(layerName)
-    
-    sr = arcpy.SpatialReference(text = project.activeMap.spatialReference.exportToString())
-    active_map = project.activeMap
-    path = project.filePath.replace("aprx","gdb") #"\\".join(project.filePath.split("\\")[:-1]) + "\\speckle_layers\\" #arcpy.env.workspace + "\\" #
-    
-    print(path)
-    print(streamBranch)
-    if sr.type == "Geographic": 
-        arcpy.AddMessage(f"Project CRS is set to Geographic type, and objects in linear units might not be received correctly")
-
-    #CREATE A GROUP "received blabla" with sublayers
-    layerGroup = None
-    newGroupName = f'{streamBranch}'
-    print(newGroupName)
-    #print(newGroupName)
-    for l in active_map.listLayers():
-        if l.longName == newGroupName: layerGroup = l; break 
-    
-    #find ID of the layer with a matching name in the "latest" group 
-    newName = f'{streamBranch.split("_")[len(streamBranch.split("_"))-1]}_{layerName}'
-
-    all_layer_names = []
-    layerExists = 0
-    for l in project.activeMap.listLayers(): 
-        if l.longName.startswith(newGroupName + "\\"):
-            all_layer_names.append(l.longName)
-    #print(all_layer_names)
-
-    longName = streamBranch + "\\" + newName 
-    if longName in all_layer_names: 
-        for index, letter in enumerate('234567890abcdefghijklmnopqrstuvwxyz'):
-            if (longName + "_" + letter) not in all_layer_names: newName += "_"+letter; layerExists +=1; break 
-
-    # particularly if the layer comes from ArcGIS
-    if "polygon" in geomType.lower(): geomType = "Polygon"
-    if "line" in geomType.lower(): geomType = "Polyline"
-    if "multipoint" in geomType.lower(): geomType = "Multipoint"
-    elif "point" in geomType.lower(): geomType = "Point"
-    #print(geomType)
-    
-    #print(newName)
-    #path = r"C:\Users\username\Documents\ArcGIS\Projects\MyProject-test\MyProject-test.gdb\\"
-    #https://community.esri.com/t5/arcgis-pro-questions/is-it-possible-to-create-a-new-group-layer-with/td-p/1068607
-
-    #print("Create feature class (cad): ")
-    # should be created inside the workspace to be a proper Feature class (not .shp) with Nullable Fields
-    class_name = ("f_class_" + newName)
-    f_class = CreateFeatureclass(path, class_name, geomType, has_z="ENABLED", spatial_reference = sr)
-    print(f_class)
-    #print(geomList)
-
-    # get and set Layer attribute fields
-    # example: https://resource.esriuk.com/blog/an-introductory-slice-of-arcpy-in-arcgis-pro/
-    newFields = getLayerAttributes(geomList)
-    
-    fields_to_ignore = ["arcgisgeomfromspeckle", "shape", "objectid"]
-    matrix = []
-    all_keys = []
-    all_key_types = []
-    max_len = 52
-    for key, value in newFields.items(): 
-        existingFields = [fl.name for fl in arcpy.ListFields(class_name)]
-        if key not in existingFields  and key.lower() not in fields_to_ignore: # exclude geometry and default existing fields
-            # signs that should not be used as field names and table names: https://support.esri.com/en/technical-article/000005588
-            key = key.replace(" ","_").replace("-","_").replace("(","_").replace(")","_").replace(":","_").replace("\\","_").replace("/","_").replace("\"","_").replace("&","_").replace("@","_").replace("$","_").replace("%","_").replace("^","_") 
-            if key[0] in ['0','1','2','3','4','5','6','7','8','9']: key = "_"+key
-            if len(key)>max_len: key = key[:max_len]
-            #print(all_keys)
-            if key in all_keys:
-                for index, letter in enumerate('1234567890abcdefghijklmnopqrstuvwxyz'):
-                    if len(key)<max_len and (key+letter) not in all_keys: key+=letter; break 
-                    if len(key) == max_len and (key[:9] + letter) not in all_keys: key=key[:9] + letter; break 
-            if key not in all_keys: 
-                all_keys.append(key)
-                all_key_types.append(value)
-                #print(all_keys)
-                matrix.append([key, value, key, 255])
-                #print(matrix)
-    if len(matrix)>0: AddFields(str(f_class), matrix)
-
-    fets = []
-    for f in geomList[:]: 
-        new_feat = cadFeatureToNative(f, newFields, sr)
-        if new_feat != "" and new_feat != None: 
-            fets.append(new_feat)
-    print("features created")
-    print(len(fets))
-    print(all_keys)
-    
-    if len(fets) == 0: return None
-    count = 0
-    rowValues = []
-    for feat in fets:
-        try: feat['applicationId'] 
-        except: feat.update({'applicationId': count})
-
-        row = [feat['arcGisGeomFromSpeckle'], feat['applicationId']]
-        heads = [ 'Shape@', 'OBJECTID']
-
-        for key,value in feat.items(): 
-            #print(key, str(value))
-            if key in all_keys and key.lower() not in fields_to_ignore: 
-                heads.append(key)
-                row.append(value)
-        rowValues.append(row)
-        count += 1
-    cur = arcpy.da.InsertCursor(str(f_class), tuple(heads) )
-    #print(heads)
-    for row in rowValues: 
-        try:
-            #print(row)
-            cur.insertRow(tuple(row))
-        except Exception as e:
-            print(e)
-    del cur 
-    vl = MakeFeatureLayer(str(f_class), newName).getOutput(0)
-
-    #adding layers from code solved: https://gis.stackexchange.com/questions/344343/arcpy-makefeaturelayer-management-function-not-creating-feature-layer-in-arcgis
-    #active_map.addLayer(new_layer)
-    active_map.addLayerToGroup(layerGroup, vl)
-    print("Layer created")
-
-    return vl
-
-def vectorLayerToNative(layer: Union[Layer, VectorLayer], streamBranch: str, project: ArcGISProject):
-    print("_________Vector Layer to Native_________")
-    vl = None
-    layerName = layer.name.replace(" ","_").replace("-","_").replace("(","_").replace(")","_").replace(":","_").replace("\\","_").replace("/","_").replace("\"","_").replace("&","_").replace("@","_").replace("$","_").replace("%","_").replace("^","_")
-    
-    print(layerName)
-    sr = arcpy.SpatialReference(text=layer.crs.wkt) 
-    active_map = project.activeMap
-    path = project.filePath.replace("aprx","gdb") #"\\".join(project.filePath.split("\\")[:-1]) + "\\speckle_layers\\" #arcpy.env.workspace + "\\" #
-    #if not os.path.exists(path): os.makedirs(path)
-    #print(path) 
-
-    newName, layerGroup = newLayerGroupAndName(layerName, streamBranch, project)
-
-    # particularly if the layer comes from ArcGIS
-    geomType = layer.geomType # for ArcGIS: Polygon, Point, Polyline, Multipoint, MultiPatch
-    print(geomType)
-    if "polygon" in geomType.lower(): geomType = "Polygon"
-    if "line" in geomType.lower(): geomType = "Polyline"
-    if "multipoint" in geomType.lower(): geomType = "Multipoint"
-    elif "point" in geomType.lower(): geomType = "Point"
-    #print(geomType)
-    
-    #print(newName)
-    #path = r"C:\Users\username\Documents\ArcGIS\Projects\MyProject-test\MyProject-test.gdb\\"
-    #https://community.esri.com/t5/arcgis-pro-questions/is-it-possible-to-create-a-new-group-layer-with/td-p/1068607
-    #print(project.filePath.replace("aprx","gdb"))
-    #print("_________create feature class___________________________________")
-    # should be created inside the workspace to be a proper Feature class (not .shp) with Nullable Fields
-    class_name = "f_class_" + newName
-    #print(class_name)
-    try: f_class = CreateFeatureclass(path, class_name, geomType, has_z="ENABLED", spatial_reference = sr)
-    except arcgisscripting.ExecuteError: class_name+="_"; f_class = CreateFeatureclass(path, class_name, geomType, has_z="ENABLED", spatial_reference = sr)
-
-    # get and set Layer attribute fields
-    # example: https://resource.esriuk.com/blog/an-introductory-slice-of-arcpy-in-arcgis-pro/
-    newFields = getLayerAttributes(layer.features)
-    fields_to_ignore = ["arcgisgeomfromspeckle", "shape", "objectid"]
-    matrix = []
-    all_keys = []
-    all_key_types = []
-    max_len = 52
-    for key, value in newFields.items(): 
-        existingFields = [fl.name for fl in arcpy.ListFields(class_name)]
-        if key not in existingFields  and key.lower() not in fields_to_ignore: # exclude geometry and default existing fields
-            # signs that should not be used as field names and table names: https://support.esri.com/en/technical-article/000005588
-            key = key.replace(" ","_").replace("-","_").replace("(","_").replace(")","_").replace(":","_").replace("\\","_").replace("/","_").replace("\"","_").replace("&","_").replace("@","_").replace("$","_").replace("%","_").replace("^","_") 
-            if key[0] in ['0','1','2','3','4','5','6','7','8','9']: key = "_"+key
-            if len(key)>max_len: key = key[:max_len]
-            #print(all_keys)
-            if key in all_keys:
-                for index, letter in enumerate('1234567890abcdefghijklmnopqrstuvwxyz'):
-                    if len(key)<max_len and (key+letter) not in all_keys: key+=letter; break 
-                    if len(key) == max_len and (key[:9] + letter) not in all_keys: key=key[:9] + letter; break 
-            if key not in all_keys: 
-                all_keys.append(key)
-                all_key_types.append(value)
-                #print(all_keys)
-                matrix.append([key, value, key, 255])
-                #print(matrix)
-    if len(matrix)>0: AddFields(str(f_class), matrix)
-
-    fets = []
-    for f in layer.features: 
-        new_feat = featureToNative(f, newFields, geomType, sr)
-        if new_feat != "" and new_feat!= None: fets.append(new_feat)
-    
-    #print(fets)
-    if len(fets) == 0: return None
-    count = 0
-    rowValues = []
-    heads = None
-    for feat in fets:
-        #print(feat)
-        try: feat['applicationId'] 
-        except: feat.update({'applicationId': count})
-
-        row = [feat['arcGisGeomFromSpeckle'], feat['applicationId']]
-        heads = [ 'Shape@', 'OBJECTID']
-
-        for key,value in feat.items(): 
-            if key in all_keys and key.lower() not in fields_to_ignore: 
-                heads.append(key)
-                row.append(value)
-        rowValues.append(row)
-        count += 1
-    cur = arcpy.da.InsertCursor(str(f_class), tuple(heads) )
-    for row in rowValues: 
-        #print(tuple(heads))
-        #print(tuple(row))
-        cur.insertRow(tuple(row))
-    del cur 
-
-    vl = MakeFeatureLayer(str(f_class), newName).getOutput(0)
-
-    #adding layers from code solved: https://gis.stackexchange.com/questions/344343/arcpy-makefeaturelayer-management-function-not-creating-feature-layer-in-arcgis
-    
-    active_map.addLayerToGroup(layerGroup, vl)
-    vl2 = None
-    print(newName)
-    for l in project.activeMap.listLayers(): 
-        #print(l.longName)
-        if l.longName == layerGroup.longName + "\\" + newName:
-            vl2 = l 
-            break
-    path_lyr = vectorRendererToNative(project, active_map, layerGroup, layer, vl2, f_class, heads)
-    #if path_lyr is not None: 
-    #    active_map.removeLayer(path_lyr)    
-
-    r'''
-    # rename back the layer if was renamed due to existing duplicate
-    if layerExists:  
-        vl.name = newName[:len(newName)-2]
-        for lyr in project.activeMap.listLayers():
-            print(lyr.longName)
-            if (streamBranch + "\\" + newName) == lyr.longName: 
-                lyr.name = lyr.name.replace( lyr.name, lyr.name[:len(lyr.name)-2] )
-                lyr.longName = lyr.longName.replace( lyr.longName, lyr.longName[:len(newName)-2] )
-                break
-    '''
-
-    r'''
-    pr.addFeatures(fets)
-    vl.updateExtents()
-    vl.commitChanges()
-    #layerGroup.addLayer(vl)
-
-    rendererNew = vectorRendererToNative(layer)
-    if rendererNew is None:
-        symbol = QgsSymbol.defaultSymbol(QgsWkbTypes.geometryType(QgsWkbTypes.parseType(geomType)))
-        rendererNew = QgsSingleSymbolRenderer(symbol)
-
-    try: vl.setRenderer(rendererNew)
-    except: pass
-    '''
-    return vl
-
-def rasterLayerToNative(layer: RasterLayer, streamBranch: str, project: ArcGISProject):
-
-    rasterLayer = None
-
-    layerName = layer.name.replace(" ","_").replace("-","_").replace("(","_").replace(")","_").replace(":","_").replace("\\","_").replace("/","_").replace("\"","_").replace("&","_").replace("@","_").replace("$","_").replace("%","_").replace("^","_")
-    
-    print(layerName)
-    sr = arcpy.SpatialReference(text=layer.crs.wkt) 
-    print(layer.crs.wkt)
-    active_map = project.activeMap
-    path = project.filePath.replace("aprx","gdb")
-    #path = '.'.join(path.split("\\")[:-1])
-    rasterHasSr = False
-    print(path)
-
-    path_bands = "\\".join(path.split("\\")[:-1]) + "\\Layers_Speckle\\rasters_Speckle\\" + streamBranch 
-    if not os.path.exists(path_bands): os.makedirs(path_bands)
-
-    try: 
-        srRasterWkt = str(layer.rasterCrs.wkt)
-        print(layer.rasterCrs.wkt)
-        srRaster = arcpy.SpatialReference(text=srRasterWkt) # by native raster SR
-        rasterHasSr = True
-    except: 
-        srRasterWkt = str(layer.crs.wkt)
-        srRaster: arcpy.SpatialReference = sr # by layer
-    #print(layer.rasterCrs.wkt)
-    print(srRaster)
-
-    newName, layerGroup = newLayerGroupAndName(layerName, streamBranch, project)
-    print(newName)
-    if "." in newName: newName = '.'.join(newName.split(".")[:-1])
-    print(newName)
-    
-    feat = layer.features[0]
-    bandNames = feat["Band names"]
-    bandValues = [feat["@(10000)" + name + "_values"] for name in bandNames]
-
-    xsize= int(feat["X pixels"])
-    ysize= int(feat["Y pixels"])
-    xres = float(feat["X resolution"])
-    yres = float(feat["Y resolution"])
-    bandsCount=int(feat["Band count"])
-    originPt = arcpy.Point(feat['displayValue'][0].x, feat['displayValue'][0].y, feat['displayValue'][0].z)
-    print(originPt)
-    #if source projection is different from layer display projection, convert display OriginPt to raster source projection 
-    if rasterHasSr is True and srRaster.exportToString() != sr.exportToString():
-        originPt = findTransformation(arcpy.PointGeometry(originPt, sr, has_z = True), "Point", sr, srRaster, None).getPart()
-    print(originPt)
-
-    bandDatasets = ""
-    rastersToMerge = []
-    rasterPathsToMerge = []
-
-
-    arcpy.env.workspace = path 
-    arcpy.env.overwriteOutput = True
-    # https://pro.arcgis.com/en/pro-app/latest/tool-reference/data-management/composite-bands.htm
-    
-    
-    for i in range(bandsCount):
-        print(i)
-        print(bandNames[i])
-        rasterbandPath = path_bands + "\\" + newName + "_Band_" + str(i+1) + ".tif"
-        bandDatasets += rasterbandPath + ";"
-        rasterband = np.array(bandValues[i])
-        rasterband = np.reshape(rasterband,(ysize, xsize))
-        print(rasterband)
-        print(np.shape(rasterband))
-        print(xsize)
-        print(xres)
-        print(ysize)
-        print(yres)
-        leftLowerCorner = arcpy.Point(originPt.X, originPt.Y + (ysize*yres), originPt.Z)
-        #upperRightCorner = arcpy.Point(originPt.X + (xsize*xres), originPt.Y, originPt.Z)
-        print(leftLowerCorner)
-        #print(upperRightCorner)
-
-        # # Convert array to a geodatabase raster, add to layers 
-        try: myRaster = arcpy.NumPyArrayToRaster(rasterband, leftLowerCorner, abs(xres), abs(yres), float(feat["NoDataVal"][i]) ) 
-        except: myRaster = arcpy.NumPyArrayToRaster(rasterband, leftLowerCorner, abs(xres), abs(yres)) 
-
-        rasterbandPath = validate_path(rasterbandPath) #solved file saving issue 
-        print(rasterbandPath)
-        #mergedRaster = arcpy.ia.Merge(rastersToMerge) # glues all bands together
-        myRaster.save(rasterbandPath)
-
-        print(myRaster.width)
-        print(myRaster.height)
-
-        rastersToMerge.append(myRaster)
-        rasterPathsToMerge.append(rasterbandPath)
-        print(rasterbandPath)
-
-    #mergedRaster.setProperty("spatialReference", crsRaster)
-
-    full_path = validate_path(path + "\\" + newName) #solved file saving issue 
-    print("RASTER FULL PATH")
-    print(full_path)
-    if os.path.exists(full_path):
-        #print(full_path)
-        for index, letter in enumerate('1234567890abcdefghijklmnopqrstuvwxyz'):
-            print(full_path + letter)
-            if os.path.exists(full_path + letter): pass
-            else: full_path += letter; break 
-    print("RASTER new PATH")
-    print(full_path)
-    #mergedRaster = arcpy.ia.Merge(rastersToMerge) # glues all bands together
-    #mergedRaster.save(full_path) # similar errors: https://community.esri.com/t5/python-questions/error-010240-could-not-save-raster-dataset/td-p/321690
-    
-    try: 
-        arcpy.management.CompositeBands(rasterPathsToMerge, full_path)
-    except: # if already exists
-        full_path += "_"
-        arcpy.management.CompositeBands(rasterPathsToMerge, full_path)
-    print(path + "\\" + newName)
-    arcpy.management.DefineProjection(full_path, srRaster)
-
-    rasterLayer = arcpy.management.MakeRasterLayer(full_path, newName).getOutput(0)
-    print(layerGroup)
-    active_map.addLayerToGroup(layerGroup, rasterLayer)
-
-    rl2 = None
-    for l in active_map.listLayers(): 
-        if l.longName == layerGroup.longName + "\\" + newName:
-            print(l.longName)
-            rl2 = l 
-            break
-    rasterLayer = rasterRendererToNative(project, active_map, layerGroup, layer, rl2, rasterPathsToMerge, newName)
-
-    try: os.remove(path_bands)
-    except: pass
-
-    r'''
-    if arcpy.Exists(fileout):
-        arcpy.management.Delete(fileout)
-    arcpy.management.Rename(filelist[0], fileout)
-
-    # Remove temporary files
-    for fileitem in filelist:
-        if arcpy.Exists(fileitem):
-            arcpy.management.Delete(fileitem)
-
-    # Release raster objects from memory
-    del myRasterBlock
-    del myRaster
-    '''
-
-    r'''
-    rasterComposite = arcpy.management.CompositeBands(bandDatasets, path + "\\" + newName) # "band1.tif;band2.tif;band3.tif", "compbands.tif"
-    
-    # https://pro.arcgis.com/en/pro-app/latest/tool-reference/data-management/make-raster-layer.htm
-    rasterLayer = arcpy.MakeRasterLayer_management(rasterComposite, newName)
-    '''
-
-    
-    r'''
-    # WORKS: 
-    arcpy.CreateRasterDataset_management(r"C:\Users\Kateryna\Documents\ArcGIS\Projects\MyProject-test",
-    "EmptyTIFF.tif",
-    "2",
-    "8_BIT_UNSIGNED",
-    "PROJCS['DHDN_3_Degree_Gauss_Zone_3',GEOGCS['GCS_Deutsches_Hauptdreiecksnetz',DATUM['D_Deutsches_Hauptdreiecksnetz',SPHEROID['Bessel_1841',6377397.155,299.1528128]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]],PROJECTION['Gauss_Kruger'],PARAMETER['False_Easting',3500000.0],PARAMETER['False_Northing',0.0],PARAMETER['Central_Meridian',9.0],PARAMETER['Scale_Factor',1.0],PARAMETER['Latitude_Of_Origin',0.0],UNIT['Meter',1.0]]", "3", "", "PYRAMIDS -1 NEAREST JPEG", "128 128", "NONE", "")
-    '''
-
-    return rasterLayer

speckle_toolbox/esri/toolboxes/speckle/converter/layers/symbology.py

@@ -0,0 +1,702 @@
+import json
+from typing import Any, List, Tuple, Union
+import copy
+import os
+
+from typing import Dict
+
+import arcpy 
+from arcpy._mp import ArcGISProject, Layer as arcLayer
+from arcpy.management import (CreateFeatureclass, MakeFeatureLayer,
+                              AddFields, AlterField, DefineProjection )
+
+from specklepy.objects import Base
+from specklepy.objects.other import RenderMaterial
+
+try:
+    from speckle.converter.layers.Layer import Layer, VectorLayer, RasterLayer 
+    from speckle.plugin_utils.logger import logToUser
+except:
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.Layer import Layer, VectorLayer, RasterLayer 
+    from speckle_toolbox.esri.toolboxes.speckle.plugin_utils.logger import logToUser
+
+def jsonFromLayerStyle(layerArcgis, path_style):
+    # write updated renderer to file and get layerStyle variable 
+    try:
+        arcpy.management.SaveToLayerFile(layerArcgis, path_style, False)
+        f = open(path_style, "r")
+        layerStyle = json.loads(f.read())
+        f.close()
+        os.remove(path_style)
+        return layerStyle
+    except Exception as e:
+        logToUser(e)
+        return None 
+
+def symbol_color_to_speckle(color: dict):
+    newColor = (0<<16) + (0<<8) + 0
+    try: 
+        r = int(color['RGB'][0])
+        g = int(color['RGB'][1])
+        b = int(color['RGB'][2])
+        newColor = (r<<16) + (g<<8) + b
+    except Exception as e:
+        logToUser(e, 1)
+    return newColor
+
+
+def colorFromRenderMaterial(material):
+
+    color = {'RGB': [245, 245, 245, 100]} #Objects.Other.RenderMaterial
+    if material is not None:
+        try: 
+            rgb = material.diffuse
+            r = (rgb & 0xFF0000) >> 16
+            g = (rgb & 0xFF00) >> 8
+            b = rgb & 0xFF 
+            color = {'RGB': [r, g, b, 100]}
+            #print(color)
+        except Exception as e:
+            logToUser(e, 1)
+    return color
+
+def cadBimRendererToNative(project: ArcGISProject, active_map, layerGroup, fetColors: List[RenderMaterial], layerArcgis, f_class, existingAttrs: List) -> Union[None, Dict[str, Any]] :
+    print("___________APPLY VECTOR RENDERER______________")
+    print(layerArcgis)
+    print(f_class)
+    print(fetColors)
+
+    attribute = "Speckle_ID"
+    try:
+        root_path = "\\".join(project.filePath.split("\\")[:-1])
+        #path_style = root_path + '\\' + str(f_class).split('\\')[-1] + '_old.lyrx'
+
+        data = arcpy.Describe(layerArcgis.dataSource)
+        if layerArcgis.isFeatureLayer:
+            geomType = data.shapeType
+            sym = layerArcgis.symbology
+
+            cursor = arcpy.da.SearchCursor(f_class, attribute)
+            class_shapes = [shp_id[0] for n, shp_id in enumerate(cursor)]
+            del cursor 
+
+            sym.updateRenderer('UniqueValueRenderer')
+            print(sym.renderer.type)
+            print(existingAttrs)
+            print(attribute)
+
+            sym.renderer.fields = [attribute]
+            for k, grp in enumerate(sym.renderer.groups):
+                for itm in grp.items:
+                    transVal = itm.values[0][0] #Grab the first "percent" value in the list of potential values 
+                    #print(transVal)
+                    for i in range(len(class_shapes)):
+                        label = class_shapes[i]
+                        #print(label)
+                        if label is None or label=="" or str(label)=="": label = "<Null>"
+
+                        if str(transVal) == label:
+                            #print("found label")
+                            material = fetColors[i]
+                            #print(material)
+                            itm.symbol.color = colorFromRenderMaterial(material) 
+                            itm.label = label
+                            break
+            layerArcgis.symbology = sym
+            #print(layerArcgis)
+            return layerArcgis 
+    except Exception as e:
+        logToUser(e)
+        return None
+
+
+def vectorRendererToNative(project: ArcGISProject, active_map, layerGroup, layerSpeckle: Union[Layer, VectorLayer], layerArcgis, f_class, existingAttrs: List) -> Union[None, Dict[str, Any]] :
+    print("___________APPLY VECTOR RENDERER______________")
+    print(layerArcgis)
+    print(f_class)
+    try:
+        renderer = layerSpeckle.renderer 
+
+        if renderer and renderer['type']:
+            print(renderer['type'])
+
+            root_path = "\\".join(project.filePath.split("\\")[:-1])
+            #path_style = root_path + '\\' + str(f_class).split('\\')[-1] + '_old.lyrx'
+
+            data = arcpy.Describe(layerArcgis.dataSource)
+            if layerArcgis.isFeatureLayer:
+                geomType = data.shapeType
+                sym = layerArcgis.symbology
+
+                if renderer['type'] == 'singleSymbol':
+                    print("RENDERER SINGLE")
+                    print(renderer)
+
+                    r,g,b = get_rgb_from_speckle(renderer['properties']['symbol']['symbColor']) 
+                    #print(r,g,b)
+                    #print(sym.renderer.symbol.color)
+                    sym.renderer.symbol.color = {'RGB': [r, g, b, 100]}
+                    #print(sym.renderer.symbol.color)
+                    layerArcgis.symbology = sym # SimpleRenderer
+                    #print(layerArcgis)
+                    return layerArcgis 
+
+                elif renderer['type']  == 'categorizedSymbol':
+                    print("RENDERER CATEGORIZED")
+                    print(renderer)
+
+                    cats = renderer['properties']['categories']
+                    attribute = renderer['properties']['attribute']
+                    if attribute not in existingAttrs: return layerArcgis 
+
+                    #vl2 = active_map.addLayer(layerArcgis)[0]
+                    #sym = layerArcgis.symbology
+                    sym.updateRenderer('UniqueValueRenderer')
+                    print(sym.renderer.type)
+                    print(existingAttrs)
+                    print(attribute)
+
+                    sym.renderer.fields = [attribute]
+                    for k, grp in enumerate(sym.renderer.groups):
+                        for itm in grp.items:
+                            transVal = itm.values[0][0] #Grab the first "percent" value in the list of potential values
+                            for i in range(len(cats)):
+                                label = cats[i]['value'] 
+                                if label is None or label=="" or str(label)=="": label = "<Null>"
+                                r,g,b = get_rgb_from_speckle(cats[i]['symbColor']) 
+
+                                if str(transVal) == label:
+                                    itm.symbol.color = {'RGB': [r, g, b, 100]}
+                                    itm.label = label
+                                    break
+                    layerArcgis.symbology = sym
+                    return layerArcgis 
+
+                elif renderer['type'] == 'graduatedSymbol': 
+                    print("RENDERER GRADUATED")
+                    print(renderer)
+
+                    attribute = renderer['properties']['attribute']
+                    gradMetod = renderer['properties']['gradMethod'] # by color or by size
+                    if gradMetod  != 0: 
+                        r,g,b = get_rgb_from_speckle(renderer['properties']['sourceSymbColor'] ) 
+                        sym.renderer.symbol.color = {'RGB': [r, g, b, 100]}
+                        layerArcgis.symbology = sym # SimpleRenderer
+                        return layerArcgis 
+                    if attribute not in existingAttrs or gradMetod != 0: return layerArcgis # by color, not line width 
+
+                    sym.updateRenderer('GraduatedColorsRenderer')
+                    print(sym.renderer.type)
+
+                    r,g,b = get_rgb_from_speckle(renderer['properties']['sourceSymbColor']) 
+                    ramp = renderer['properties']['ramp'] # {discrete, rampType, stops}
+                    ranges = renderer['properties']['ranges'] # []
+
+                    # get all existing values 
+                    all_values = []
+                    with arcpy.da.UpdateCursor(f_class, attribute) as cur:
+                        for rowShape in cur: 
+                            all_values.append(rowShape[0])
+                        print(all_values)
+                    del cur 
+
+                    print(len(ranges))
+                    sym.renderer.classificationField = attribute
+                    print(sym.renderer.breakCount)
+                    sym.renderer.breakCount = len(ranges)
+                    print(sym.renderer.breakCount)
+                    
+                    if len(sym.renderer.classBreaks) > 0:
+                        totalClasses = 0
+                        for k, br in enumerate(ranges):
+                            
+                            print(totalClasses)
+                            if sym.renderer.breakCount < len(ranges):
+                                valFits = 0
+                                # check if any existing value fits in this range:
+                                for val in all_values: 
+                                    if val <= ranges[k]["upper"] and (totalClasses==0 or (totalClasses>0 and sym.renderer.classBreaks[totalClasses-1].upperBound<val)):
+                                        valFits+=1
+                                        break
+                                if valFits == 0: continue
+                                
+                            r,g,b = get_rgb_from_speckle(ranges[k]['symbColor']) 
+
+                            #classBreak.upperBound = ranges[k]["upper"]
+                            sym.renderer.classBreaks[totalClasses].upperBound = ranges[k]["upper"]
+                            sym.renderer.classBreaks[totalClasses].label = ranges[k]["label"]
+                            sym.renderer.classBreaks[totalClasses].symbol.color = {'RGB': [r, g, b, 100]}
+                            totalClasses += 1 
+
+                            #layerArcgis.symbology = sym
+                            print(ranges[k]["label"])
+                            print(ranges[k]["upper"])
+
+                        sym.renderer.classBreaks[0].upperBound = ranges[0]["upper"] # otherwise its assigned maximum value
+
+                    layerArcgis.symbology = sym
+                    return layerArcgis 
+
+                else: return None 
+    except Exception as e:
+        logToUser(e)
+        return None
+
+def get_rgb_from_speckle(rgb: int) -> Tuple[int, int, int]:
+    r = g = b = 0
+    try: 
+        r = (rgb & 0xFF0000) >> 16
+        g = (rgb & 0xFF00) >> 8
+        b = rgb & 0xFF 
+    except: r = g = b = 0
+    
+    r,g,b = check_rgb(r,g,b)
+    return r,g,b 
+
+def check_rgb(r:int, g:int, b:int) -> Tuple[int, int, int]:
+    try:
+        if not isinstance(r, int) or r<0 or r>255: r=g=b=0
+        if not isinstance(g, int) or g<0 or g>255: r=g=b=0
+        if not isinstance(b, int) or b<0 or b>255: r=g=b=0
+        return r,g,b
+    
+    except Exception as e:
+        logToUser(e) 
+        return 0, 0, 0
+
+
+
+def rasterRendererToNative(project: ArcGISProject, active_map, layerGroup,  layer: RasterLayer, arcLayer, rasterPathsToMerge, newName):
+    print("_____rasterRenderer ToNative______")
+    try:
+        renderer = layer.renderer
+        rendererNew = None
+        print(renderer)
+
+        feat = layer.features[0]
+        print(feat)
+
+        bandNames = feat["Band names"]
+        print(bandNames)
+
+        sym = arcLayer.symbology
+        symJson = None
+        path_style = ""
+        path_style2 = ""
+
+        print(sym)
+
+        if renderer and renderer['type']:
+            
+            if not hasattr(arcLayer.symbology, 'colorizer'): 
+                # multiband raster, CIMRasterRGBColorizer 
+                # arcpy doesnt support multiband raster symbology: https://community.esri.com/t5/arcgis-api-for-python-questions/why-does-arcpy-mp-arcgis-pro-2-6-mosaic-dataset/td-p/1016312 
+                root_path = "\\".join(project.filePath.split("\\")[:-1])
+                if not os.path.exists(root_path + '\\Layers_Speckle\\raster_bands'): os.makedirs(root_path + '\\Layers_Speckle\\raster_bands')
+                path_style = root_path + '\\Layers_Speckle\\raster_bands\\' + newName + '_old.lyrx'
+                path_style2 = root_path + '\\Layers_Speckle\\raster_bands\\' + newName + '_new.lyrx'
+                symJson = jsonFromLayerStyle(arcLayer, path_style)
+
+            if renderer['type']  == 'singlebandgray':
+                print("Singleband grey")
+                band_index = renderer['properties']['band']-1
+                if symJson is None:
+                    sym.updateColorizer('RasterStretchColorizer')
+                    sym.colorizer.band = band_index
+                    arcLayer.symbology = sym
+                else: 
+                    temp = arcpy.management.MakeRasterLayer(rasterPathsToMerge[band_index], newName + "_temp").getOutput(0)
+                    active_map.addLayerToGroup(layerGroup, temp)
+                    temp_layer = None
+                    for l in active_map.listLayers(): 
+                        if l.longName == layerGroup.longName + "\\" + newName + "_temp":
+                            print(l.longName)
+                            temp_layer = l 
+                            break
+
+                    sym = temp_layer.symbology
+                    sym.updateColorizer('RasterStretchColorizer')
+                    sym.colorizer.band = band_index
+                    arcLayer.symbology = sym
+
+                    active_map.removeLayer(temp_layer) 
+                    
+
+            elif renderer['type']  == 'multibandcolor':
+                print("Multiband")
+                if symJson is None: 
+                    sym.updateColorizer('RasterStretchColorizer')
+                    arcLayer.symbology = sym 
+                else:
+                    
+                    redSt =  copy.deepcopy(symJson["layerDefinitions"][0]["colorizer"]["stretchStatsRed"])
+                    greenSt =  copy.deepcopy(symJson["layerDefinitions"][0]["colorizer"]["stretchStatsGreen"])
+                    blueSt =  copy.deepcopy(symJson["layerDefinitions"][0]["colorizer"]["stretchStatsBlue"])
+
+                    redBand = renderer['properties']['redBand']
+                    greenBand = renderer['properties']['greenBand']
+                    blueBand = renderer['properties']['blueBand']
+                    try: symJson["layerDefinitions"][0]["colorizer"]["greenBandIndex"] = greenBand-1
+                    except: symJson["layerDefinitions"][0]["colorizer"]["greenBandIndex"] = 0
+
+                    try: symJson["layerDefinitions"][0]["colorizer"]["redBandIndex"] = redBand-1
+                    except: symJson["layerDefinitions"][0]["colorizer"]["redBandIndex"] = 0
+                    
+                    try: symJson["layerDefinitions"][0]["colorizer"]["blueBandIndex"] = blueBand-1
+                    except: symJson["layerDefinitions"][0]["colorizer"]["blueBandIndex"] = 0
+                    
+                    print(symJson)
+                    f = open(path_style2, "w")
+                    f.write(json.dumps(symJson, indent=2))
+                    f.close()
+
+                    active_map.removeLayer(arcLayer) 
+                    lyrFile = arcpy.mp.LayerFile(path_style2)
+                    active_map.addLayerToGroup(layerGroup, lyrFile )
+                    
+                    os.remove(path_style2) 
+
+            elif renderer['type']  == 'paletted':
+                print("Paletted")
+                band_index = renderer['properties']['band']-1
+
+                if symJson is None:
+                    for br in sym.colorizer.groups:
+                        print(br.heading) #"Value"
+                        # go through all values classified 
+                        for k, itm in enumerate(br.items):
+                            if k< len(renderer['properties']['classes']):
+                                #go through saved renderer classes
+                                for n, cl in enumerate(renderer['properties']['classes']):
+                                    if k == n:
+                                        r,g,b = get_rgb_from_speckle(cl['color']) 
+                                        itm.color = {'RGB': [r,g,b, 100]}
+                                        itm.label = cl['label']
+                                        itm.values = cl['value']
+                            else: pass
+                    arcLayer.symbology = sym 
+            else: 
+                sym.updateColorizer('RasterStretchColorizer')
+                arcLayer.symbology = sym
+    except Exception as e:
+        logToUser(e)
+    
+    return arcLayer
+
+def rendererToSpeckle(project: ArcGISProject, active_map, arcLayer, rasterFeat: Base):
+    print("_____renderer To Speckle______")
+    try:
+        if arcLayer.isRasterLayer: 
+            try: 
+                rType = arcLayer.symbology.colorizer.type # 'singleSymbol','categorizedSymbol','graduatedSymbol',
+                if rType =='RasterStretchColorizer':  rType = 'singlebandgray'
+                elif rType =='RasterUniqueValueColorizer':  rType = 'paletted' # only for 1-band raster 
+                else: rType = 'singlebandgray'
+            except: 
+                rType = "multibandcolor"
+                root_path = "\\".join(project.filePath.split("\\")[:-1])
+                if not os.path.exists(root_path + '\\Layers_Speckle\\raster_bands'): os.makedirs(root_path + '\\Layers_Speckle\\raster_bands')
+                path_style = root_path + '\\Layers_Speckle\\raster_bands\\' + arcLayer.name + '_temp.lyrx'
+                #path_style2 = root_path + '\\' + newName + '_new.lyrx'
+                symJson = jsonFromLayerStyle(arcLayer, path_style)
+
+            layerRenderer: Dict[str, Any] = {}
+            layerRenderer['type'] = rType
+            print(rType)
+            my_raster = arcpy.Raster(arcLayer.dataSource)  
+            rasterBandNames = my_raster.bandNames
+            
+            #bandNames = rasterFeat["Band names"]
+            bandValues = [rasterFeat["@(10000)" + name + "_values"] for name in rasterBandNames]
+
+            if rType == "singlebandgray":  
+                try: band = arcLayer.symbology.colorizer.band 
+                except: band = 0 
+                try: 
+                    bVals = bandValues[band]
+                    bvalMin = min(bVals)
+                    bvalMax = max(bVals)
+                except: 
+                    bvalMin = 0
+                    bvalMax = 255
+                layerRenderer.update({'properties': {'max':bvalMax,'min':bvalMin,'band':band+1,'contrast':1}}) 
+
+            elif rType == "multibandcolor":
+
+                try: greenBand = symJson["layerDefinitions"][0]["colorizer"]["greenBandIndex"] +1
+                except: greenBand = None
+                try: blueBand = symJson["layerDefinitions"][0]["colorizer"]["blueBandIndex"] +1
+                except: blueBand = None
+                try: redBand = symJson["layerDefinitions"][0]["colorizer"]["redBandIndex"] +1
+                except: 
+                    print(greenBand)
+                    print(blueBand)
+                    if blueBand!=1 and greenBand!=1: redBand= 1
+                    else: redBand = None
+                print(redBand)
+
+                try: 
+                    rbVals = bandValues[redBand-1]
+                    rbvalMin = min(rbVals)
+                    rbvalMax = max(rbVals)
+                    print(rbvalMin)
+                    print(rbvalMax)
+                except: 
+                    rbvalMin = 0
+                    rbvalMax = 255
+                try: 
+                    gbVals = bandValues[greenBand-1]
+                    gbvalMin = min(gbVals)
+                    gbvalMax = max(gbVals)
+                except: 
+                    gbvalMin = 0
+                    gbvalMax = 255
+                try: 
+                    bbVals = bandValues[blueBand-1]
+                    bbvalMin = min(bbVals)
+                    bbvalMax = max(bbVals) 
+                except: 
+                    bbvalMin = 0
+                    bbvalMax = 255
+
+                layerRenderer.update({'properties': {'greenBand':greenBand,'blueBand':blueBand,'redBand':redBand}})
+                layerRenderer['properties'].update({'redContrast':1,'redMin':rbvalMin,'redMax':rbvalMax})
+                layerRenderer['properties'].update({'greenContrast':1,'greenMin':gbvalMin,'greenMax':gbvalMax})
+                layerRenderer['properties'].update({'blueContrast':1,'blueMin':bbvalMin,'blueMax':bbvalMax})
+            elif rType == "paletted":  
+                band = 0 
+                rendererClasses = arcLayer.symbology.colorizer.groups
+                classes = []
+                sourceRamp = {}
+
+                for i, cl in enumerate(rendererClasses):
+                    if cl.heading == 'Value':
+                        for k, itm in enumerate(cl.items):
+                            value = itm.values[0]
+                            label = itm.label 
+                            try: 
+                                r,g,b = itm.color['RGB'][0], itm.color['RGB'][1], itm.color['RGB'][2]
+                                sColor = (r<<16) + (g<<8) + b
+                                classes.append({'color':sColor,'value':value,'label':label})
+                            except: pass 
+                layerRenderer.update({'properties': {'classes':classes,'ramp':sourceRamp,'band':band+1}})
+
+            return layerRenderer 
+        elif arcLayer.isFeatureLayer: 
+            layerRenderer: Dict[str, Any] = {}
+
+            sym = arcLayer.symbology
+            print(sym.renderer.type)
+
+            if sym.renderer.type == 'SimpleRenderer':
+                layerRenderer['type'] = 'singleSymbol'
+                layerRenderer['properties'] = {'symbol':{}, 'symbType':""}
+                symbolColor = symbol_color_to_speckle(sym.renderer.symbol.color)
+                layerRenderer['properties'].update({'symbol':{'symbColor': symbolColor}, 'symbType':''})
+
+            elif sym.renderer.type == 'UniqueValueRenderer':
+                layerRenderer['type'] = 'categorizedSymbol'
+                layerRenderer['properties'] = {'attribute': '', 'symbType': ''} #{'symbol':{}, 'ramp':{}, 'ranges':{}, 'gradMethod':"", 'symbType':"", 'legendClassificationAttribute': ""}
+                
+                attribute = sym.renderer.fields[0]
+                layerRenderer['properties']['attribute'] = attribute
+                sourceSymbColor = symbol_color_to_speckle(sym.renderer.defaultSymbol.color)
+                layerRenderer['properties'].update( {'sourceSymbColor': sourceSymbColor} )
+    
+                categories = sym.renderer.groups
+                layerRenderer['properties']['categories'] = []
+
+                for i, grp in enumerate(categories):
+                    for itm in grp.items:
+                        value = itm.values[0][0]
+                        symbColor = symbol_color_to_speckle(itm.symbol.color)
+                        label = itm.label
+                        layerRenderer['properties']['categories'].append({'value':value,'symbColor':symbColor,'symbOpacity':1, 'sourceSymbColor': sourceSymbColor,'label':label})
+
+            elif sym.renderer.type == 'GraduatedColorsRenderer' or sym.renderer.type == 'GraduatedSymbolsRenderer':
+                layerRenderer['type'] = 'graduatedSymbol'
+                layerRenderer['properties'] = {'symbol':{}, 'ramp':{}, 'ranges':{}, 'gradMethod':"", 'symbType':""}
+            
+                attribute = sym.renderer.classificationField 
+                sourceSymbColor = (0<<16) + (0<<8) + 0  
+                layerRenderer['properties'].update( {'attribute': attribute, 'symbType': '', 'gradMethod': 0, 'sourceSymbColor': sourceSymbColor} )
+
+                rRamp = sym.renderer.colorRamp # QgsGradientColorRamp
+                layerRenderer['properties']['ramp'] = {} # gradientColorRampToSpeckle(rRamp)
+
+                rRanges = sym.renderer.classBreaks
+                layerRenderer['properties']['ranges'] = []
+                for itm in rRanges:
+                    try: lower = float(itm.label.split(" - ")[0]) if (" - " in rRanges.label) else float(rRanges.label[0])
+                    except: lower = 0
+                    upper = itm.upperBound
+                    symbColor = symbol_color_to_speckle(itm.symbol.color) 
+                    label = itm.label
+                    width = 0.26
+                    # {'label': '1 - 1.4', 'lower': 1.0, 'symbColor': <PyQt5.QtGui.QColor ...BD9B9D4A0>, 'symbOpacity': 1.0, 'upper': 1.4}
+                    layerRenderer['properties']['ranges'].append({'lower':lower,'upper':upper,'symbColor':symbColor,'symbOpacity':1,'label':label,'width':width})
+        
+            elif sym.renderer.type == 'UnclassedColorsRenderer':
+                layerRenderer['type'] = 'graduatedSymbol'
+                layerRenderer['properties'] = {'symbol':{}, 'ramp':{}, 'ranges':{}, 'gradMethod':"", 'symbType':""}
+            
+                attribute = sym.renderer.field 
+                sourceSymbColor = (0<<16) + (0<<8) + 0  
+                layerRenderer['properties'].update( {'attribute': attribute, 'symbType': '', 'gradMethod': 0, 'sourceSymbColor': sourceSymbColor} )
+                layerRenderer['properties']['ramp'] = {} # gradientColorRampToSpeckle(rRamp)
+
+                lowest = sym.renderer.lowerLabel
+                highest = sym.renderer.upperLabel
+
+                # trick to get colors 
+                rRamp = sym.renderer.colorRamp # QgsGradientColorRamp
+                arcRamp = project.listColorRamps('White to Black')[0]
+                sym.updateRenderer('GraduatedColorsRenderer')
+                sym.renderer.colorRamp = arcRamp
+                sym.renderer.classificationField = attribute
+                rows_attributes = arcpy.da.SearchCursor(arcLayer.dataSource, attribute)
+                row_attrs = []
+                row_max = -1000000000
+                row_min = 1000000000
+                for k, attrs in enumerate(rows_attributes): 
+                    row_attrs.append(attrs[0])
+                    if attrs[0] < row_min: row_min = attrs[0]
+                    if attrs[0] > row_max: row_max = attrs[0]
+                row_range = row_max - row_min
+                breakCount = len(list(set(row_attrs))) # only unique values 
+                sym.renderer.breakCount = breakCount
+
+                # run as gradient colors 
+                rRanges = sym.renderer.classBreaks
+                layerRenderer['properties']['ranges'] = []
+                for itm in rRanges:
+                    try: lower = float(itm.label.split(" - ")[0]) if (" - " in rRanges.label) else float(rRanges.label[0])
+                    except: lower = 0
+                    upper = itm.upperBound
+                    if row_range==0: rgb = 0
+                    else: rgb = 255 - int((itm.upperBound - row_min) / row_range * 255 ) 
+                    symbColor = (rgb<<16) + (rgb<<8) + rgb 
+                    label = itm.label
+                    width = 0.26
+                    # {'label': '1 - 1.4', 'lower': 1.0, 'symbColor': <PyQt5.QtGui.QColor ...BD9B9D4A0>, 'symbOpacity': 1.0, 'upper': 1.4}
+                    layerRenderer['properties']['ranges'].append({'lower':lower,'upper':upper,'symbColor':symbColor,'symbOpacity':1,'label':label,'width':width})
+
+            else: return None
+                
+            return layerRenderer
+
+        else: return None
+    
+    except Exception as e:
+        logToUser(e)
+        return None
+
+
+def featureColorfromNativeRenderer(index: int, arcLayer: arcLayer) -> int:
+    # case with one color for the entire layer
+    #try:
+    color = {'RGB': [100,100,100,100]}
+    try:
+        sym = arcLayer.symbology
+
+        if sym.renderer.type == 'SimpleRenderer':
+            print('SimpleRenderer')
+            color = sym.renderer.symbol.color
+
+        elif sym.renderer.type == 'UniqueValueRenderer':
+            #print('Unique Value Renderer')
+            #print(index)
+            #print(arcLayer)
+
+            attribute = sym.renderer.fields[0]
+            color = sym.renderer.defaultSymbol.color
+            categories = sym.renderer.groups
+
+            rows_attributes = arcpy.da.SearchCursor(arcLayer.dataSource, attribute)
+            row_shapes_list = [x for k, x in enumerate(rows_attributes)]
+
+            color_found = 0
+            for i, grp in enumerate(categories):
+                if color_found == 1: break
+                for n, itm in enumerate(grp.items):
+                    for k, attrs in enumerate(row_shapes_list): 
+                        if str(itm.values[0][0]) == "<Null>": itm.values[0][0] = None 
+                        if k == index and ( str(attrs[0]) == str(itm.values[0][0]) or (attrs[0] is None and str(itm.values[0][0]) == "<Null>") ): 
+                            color = itm.symbol.color
+                            #print("symbol color: ")
+                            #print(color)
+                            color_found = 1
+                            break 
+
+        elif sym.renderer.type == 'GraduatedColorsRenderer' or sym.renderer.type == 'GraduatedSymbolsRenderer':
+            print('Graduated Colors / Sybmols Renderer')
+            attribute = sym.renderer.classificationField 
+            rows_attributes = arcpy.da.SearchCursor(arcLayer.dataSource, attribute)
+            row_shapes_list = [x for k, x in enumerate(rows_attributes)]
+
+            rRanges = sym.renderer.classBreaks
+            upperBounds = [-10000000000000000000]
+            color_found = 0
+            for itm in rRanges:
+                print(itm)
+                if color_found == 1: break
+                for k, attrs in enumerate(row_shapes_list):  
+                    try:
+                        if k == index and float(attrs[0]) <= float(itm.upperBound) and (k==0 or float(attrs[0]) > float(upperBounds[-1]) ):
+                            color = itm.symbol.color
+                            color_found = 1
+                            break 
+                    except: pass
+                upperBounds.append(itm.upperBound)
+
+        elif sym.renderer.type == 'UnclassedColorsRenderer': 
+            print('UnclassedColorsRenderer')
+            attribute = sym.renderer.field  
+
+            sym.updateRenderer('GraduatedColorsRenderer')
+            sym.renderer.classificationField = attribute
+
+            rows_attributes = arcpy.da.SearchCursor(arcLayer.dataSource, attribute)
+            row_shapes_list = [x for k, x in enumerate(rows_attributes)]
+            row_attrs = []
+            row_max = -10000000000000000000
+            row_min = 10000000000000000000
+            feat_value = None
+            for k, attrs in enumerate(row_shapes_list): 
+                row_attrs.append(attrs[0])
+                if attrs[0] < row_min: row_min = attrs[0]
+                if attrs[0] > row_max: row_max = attrs[0]
+                if k == index: feat_value = attrs[0]
+            row_range = row_max - row_min
+            breakCount = len(list(set(row_attrs))) # only unique values 
+            sym.renderer.breakCount = breakCount
+
+            # run as gradient colors 
+            
+            upperBounds = [-10000000000000000000]
+            rRanges = sym.renderer.classBreaks
+
+            for itm in rRanges:
+                print(itm)
+                try:
+                    if row_range!=0 and float(feat_value) <= float(itm.upperBound) and (len(upperBounds)==0 or float(feat_value) > float(upperBounds[-1])):
+                        rgb = 255 - int((itm.upperBound - row_min) / row_range * 255 ) 
+                        color = {'RGB':[rgb,rgb,rgb,100]}
+                        print(color) 
+                        break
+                except: pass 
+                upperBounds.append(itm.upperBound)
+
+        else: 
+            print('Else')
+            return (100<<16) + (100<<8) + 100
+    except Exception as e:
+        logToUser(e)
+    
+    #print("final color: ")
+    #print(color) 
+    # construct RGB color
+    col = symbol_color_to_speckle(color)
+    #print(col)
+    return col
+

speckle_toolbox/esri/toolboxes/speckle/converter/layers/symbologyTemplates.py

@@ -1,606 +0,0 @@
-import json
-from typing import Any, List, Union
-import copy
-import os
-
-from typing import Dict
-
-import arcpy 
-from arcpy._mp import ArcGISProject
-from arcpy.management import (CreateFeatureclass, MakeFeatureLayer,
-                              AddFields, AlterField, DefineProjection )
-
-from specklepy.objects import Base
-try:
-    from speckle.converter.layers.Layer import Layer, VectorLayer, RasterLayer 
-except:
-    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.Layer import Layer, VectorLayer, RasterLayer 
-
-def jsonFromLayerStyle(layerArcgis, path_style):
-    # write updated renderer to file and get layerStyle variable 
-    arcpy.management.SaveToLayerFile(layerArcgis, path_style, False)
-    f = open(path_style, "r")
-    layerStyle = json.loads(f.read())
-    f.close()
-    os.remove(path_style)
-    return layerStyle
-
-def symbol_color_to_speckle(color: dict):
-    newColor = (0<<16) + (0<<8) + 0
-    try: 
-        r = int(color['RGB'][0])
-        g = int(color['RGB'][1])
-        b = int(color['RGB'][2])
-        newColor = (r<<16) + (g<<8) + b
-    except: pass 
-    return newColor
-
-def vectorRendererToNative(project: ArcGISProject, active_map, layerGroup, layerSpeckle: Union[Layer, VectorLayer], layerArcgis, f_class, existingAttrs: List) -> Union[None, Dict[str, Any]] :
-    print("___________APPLY VECTOR RENDERER______________")
-    print(layerArcgis)
-    print(f_class)
-    renderer = layerSpeckle.renderer 
-
-    if renderer and renderer['type']:
-        print(renderer['type'])
-
-        root_path = "\\".join(project.filePath.split("\\")[:-1])
-        #path_style = root_path + '\\' + str(f_class).split('\\')[-1] + '_old.lyrx'
-
-        data = arcpy.Describe(layerArcgis.dataSource)
-        if layerArcgis.isFeatureLayer:
-            geomType = data.shapeType
-            sym = layerArcgis.symbology
-
-            if renderer['type'] == 'singleSymbol':
-                print("RENDERER SINGLE")
-                print(renderer)
-
-                r,g,b = get_rgb_from_speckle(renderer['properties']['symbol']['symbColor']) 
-                #print(r,g,b)
-                #print(sym.renderer.symbol.color)
-                sym.renderer.symbol.color = {'RGB': [r, g, b, 100]}
-                #print(sym.renderer.symbol.color)
-                layerArcgis.symbology = sym # SimpleRenderer
-                #print(layerArcgis)
-                return layerArcgis 
-
-            elif renderer['type']  == 'categorizedSymbol':
-                print("RENDERER CATEGORIZED")
-                print(renderer)
-
-                cats = renderer['properties']['categories']
-                attribute = renderer['properties']['attribute']
-                if attribute not in existingAttrs: return layerArcgis 
-
-                #vl2 = active_map.addLayer(layerArcgis)[0]
-                #sym = layerArcgis.symbology
-                sym.updateRenderer('UniqueValueRenderer')
-                print(sym.renderer.type)
-                print(existingAttrs)
-                print(attribute)
-
-                sym.renderer.fields = [attribute]
-                for k, grp in enumerate(sym.renderer.groups):
-                    for itm in grp.items:
-                        transVal = itm.values[0][0] #Grab the first "percent" value in the list of potential values
-                        for i in range(len(cats)):
-                            label = cats[i]['value'] 
-                            if label is None or label=="" or str(label)=="": label = "<Null>"
-                            r,g,b = get_rgb_from_speckle(cats[i]['symbColor']) 
-
-                            if str(transVal) == label:
-                                itm.symbol.color = {'RGB': [r, g, b, 100]}
-                                itm.label = label
-                                break
-                layerArcgis.symbology = sym
-                return layerArcgis 
-
-            elif renderer['type'] == 'graduatedSymbol': 
-                print("RENDERER GRADUATED")
-                print(renderer)
-
-                attribute = renderer['properties']['attribute']
-                gradMetod = renderer['properties']['gradMethod'] # by color or by size
-                if gradMetod  != 0: 
-                    r,g,b = get_rgb_from_speckle(renderer['properties']['sourceSymbColor'] ) 
-                    sym.renderer.symbol.color = {'RGB': [r, g, b, 100]}
-                    layerArcgis.symbology = sym # SimpleRenderer
-                    return layerArcgis 
-                if attribute not in existingAttrs or gradMetod != 0: return layerArcgis # by color, not line width 
-
-                sym.updateRenderer('GraduatedColorsRenderer')
-                print(sym.renderer.type)
-
-                r,g,b = get_rgb_from_speckle(renderer['properties']['sourceSymbColor']) 
-                ramp = renderer['properties']['ramp'] # {discrete, rampType, stops}
-                ranges = renderer['properties']['ranges'] # []
-
-                # get all existing values 
-                all_values = []
-                with arcpy.da.UpdateCursor(f_class, attribute) as cur:
-                    for rowShape in cur: 
-                        all_values.append(rowShape[0])
-                    print(all_values)
-                del cur 
-
-                print(len(ranges))
-                sym.renderer.classificationField = attribute
-                print(sym.renderer.breakCount)
-                sym.renderer.breakCount = len(ranges)
-                print(sym.renderer.breakCount)
-                
-                if len(sym.renderer.classBreaks) > 0:
-                    totalClasses = 0
-                    for k, br in enumerate(ranges):
-                        
-                        print(totalClasses)
-                        if sym.renderer.breakCount < len(ranges):
-                            valFits = 0
-                            # check if any existing value fits in this range:
-                            for val in all_values: 
-                                if val <= ranges[k]["upper"] and (totalClasses==0 or (totalClasses>0 and sym.renderer.classBreaks[totalClasses-1].upperBound<val)):
-                                    valFits+=1
-                                    break
-                            if valFits == 0: continue
-                            
-                        r,g,b = get_rgb_from_speckle(ranges[k]['symbColor']) 
-
-                        #classBreak.upperBound = ranges[k]["upper"]
-                        sym.renderer.classBreaks[totalClasses].upperBound = ranges[k]["upper"]
-                        sym.renderer.classBreaks[totalClasses].label = ranges[k]["label"]
-                        sym.renderer.classBreaks[totalClasses].symbol.color = {'RGB': [r, g, b, 100]}
-                        totalClasses += 1 
-
-                        #layerArcgis.symbology = sym
-                        print(ranges[k]["label"])
-                        print(ranges[k]["upper"])
-
-                    sym.renderer.classBreaks[0].upperBound = ranges[0]["upper"] # otherwise its assigned maximum value
-
-                layerArcgis.symbology = sym
-                return layerArcgis 
-
-            else: return None 
-
-def get_rgb_from_speckle(rgb: int) -> tuple[int, int, int]:
-    r = g = b = 0
-    try: 
-        r = (rgb & 0xFF0000) >> 16
-        g = (rgb & 0xFF00) >> 8
-        b = rgb & 0xFF 
-    except: r = g = b = 0
-    
-    r,g,b = check_rgb(r,g,b)
-    return r,g,b 
-
-def check_rgb(r:int, g:int, b:int) -> tuple[int, int, int]:
-
-    if not isinstance(r, int) or r<0 or r>255: r=g=b=0
-    if not isinstance(g, int) or g<0 or g>255: r=g=b=0
-    if not isinstance(b, int) or b<0 or b>255: r=g=b=0
-    return r,g,b 
-
-
-
-def rasterRendererToNative(project: ArcGISProject, active_map, layerGroup,  layer: RasterLayer, arcLayer, rasterPathsToMerge, newName):
-    print("_____rasterRenderer ToNative______")
-    renderer = layer.renderer
-    rendererNew = None
-    print(renderer)
-
-    feat = layer.features[0]
-    print(feat)
-
-    bandNames = feat["Band names"]
-    print(bandNames)
-
-    sym = arcLayer.symbology
-    symJson = None
-    path_style = ""
-    path_style2 = ""
-
-    print(sym)
-
-    if renderer and renderer['type']:
-        
-        if not hasattr(arcLayer.symbology, 'colorizer'): 
-            # multiband raster, CIMRasterRGBColorizer 
-            # arcpy doesnt support multiband raster symbology: https://community.esri.com/t5/arcgis-api-for-python-questions/why-does-arcpy-mp-arcgis-pro-2-6-mosaic-dataset/td-p/1016312 
-            root_path = "\\".join(project.filePath.split("\\")[:-1])
-            if not os.path.exists(root_path + '\\Layers_Speckle\\rasters_Speckle'): os.makedirs(root_path + '\\Layers_Speckle\\rasters_Speckle')
-            path_style = root_path + '\\Layers_Speckle\\rasters_Speckle\\' + newName + '_old.lyrx'
-            path_style2 = root_path + '\\Layers_Speckle\\rasters_Speckle\\' + newName + '_new.lyrx'
-            symJson = jsonFromLayerStyle(arcLayer, path_style)
-
-        if renderer['type']  == 'singlebandgray':
-            print("Singleband grey")
-            band_index = renderer['properties']['band']-1
-            if symJson is None:
-                sym.updateColorizer('RasterStretchColorizer')
-                sym.colorizer.band = band_index
-                arcLayer.symbology = sym
-            else: 
-                temp = arcpy.management.MakeRasterLayer(rasterPathsToMerge[band_index], newName + "_temp").getOutput(0)
-                active_map.addLayerToGroup(layerGroup, temp)
-                temp_layer = None
-                for l in active_map.listLayers(): 
-                    if l.longName == layerGroup.longName + "\\" + newName + "_temp":
-                        print(l.longName)
-                        temp_layer = l 
-                        break
-
-                sym = temp_layer.symbology
-                sym.updateColorizer('RasterStretchColorizer')
-                sym.colorizer.band = band_index
-                arcLayer.symbology = sym
-
-                active_map.removeLayer(temp_layer) 
-                
-
-        elif renderer['type']  == 'multibandcolor':
-            print("Multiband")
-            if symJson is None: 
-                sym.updateColorizer('RasterStretchColorizer')
-                arcLayer.symbology = sym 
-            else:
-                
-                redSt =  copy.deepcopy(symJson["layerDefinitions"][0]["colorizer"]["stretchStatsRed"])
-                greenSt =  copy.deepcopy(symJson["layerDefinitions"][0]["colorizer"]["stretchStatsGreen"])
-                blueSt =  copy.deepcopy(symJson["layerDefinitions"][0]["colorizer"]["stretchStatsBlue"])
-
-                redBand = renderer['properties']['redBand']
-                greenBand = renderer['properties']['greenBand']
-                blueBand = renderer['properties']['blueBand']
-                try: symJson["layerDefinitions"][0]["colorizer"]["greenBandIndex"] = greenBand-1
-                except: symJson["layerDefinitions"][0]["colorizer"]["greenBandIndex"] = 0
-
-                try: symJson["layerDefinitions"][0]["colorizer"]["redBandIndex"] = redBand-1
-                except: symJson["layerDefinitions"][0]["colorizer"]["redBandIndex"] = 0
-                
-                try: symJson["layerDefinitions"][0]["colorizer"]["blueBandIndex"] = blueBand-1
-                except: symJson["layerDefinitions"][0]["colorizer"]["blueBandIndex"] = 0
-                
-                print(symJson)
-                f = open(path_style2, "w")
-                f.write(json.dumps(symJson, indent=2))
-                f.close()
-
-                active_map.removeLayer(arcLayer) 
-                lyrFile = arcpy.mp.LayerFile(path_style2)
-                active_map.addLayerToGroup(layerGroup, lyrFile )
-                
-                os.remove(path_style2) 
-
-        elif renderer['type']  == 'paletted':
-            print("Paletted")
-            band_index = renderer['properties']['band']-1
-
-            if symJson is None:
-                for br in sym.colorizer.groups:
-                    print(br.heading) #"Value"
-                    # go through all values classified 
-                    for k, itm in enumerate(br.items):
-                        if k< len(renderer['properties']['classes']):
-                            #go through saved renderer classes
-                            for n, cl in enumerate(renderer['properties']['classes']):
-                                if k == n:
-                                    r,g,b = get_rgb_from_speckle(cl['color']) 
-                                    itm.color = {'RGB': [r,g,b, 100]}
-                                    itm.label = cl['label']
-                                    itm.values = cl['value']
-                        else: pass
-                arcLayer.symbology = sym 
-        else: 
-            sym.updateColorizer('RasterStretchColorizer')
-            arcLayer.symbology = sym
-    return arcLayer
-    
-
-def rendererToSpeckle(project: ArcGISProject, active_map, arcLayer, rasterFeat: Base):
-    print("_____rasterRenderer To Speckle______")
-    if arcLayer.isRasterLayer: 
-        try: 
-            rType = arcLayer.symbology.colorizer.type # 'singleSymbol','categorizedSymbol','graduatedSymbol',
-            if rType =='RasterStretchColorizer':  rType = 'singlebandgray'
-            elif rType =='RasterUniqueValueColorizer':  rType = 'paletted' # only for 1-band raster 
-            else: rType = 'singlebandgray'
-        except: 
-            rType = "multibandcolor"
-            root_path = "\\".join(project.filePath.split("\\")[:-1])
-            if not os.path.exists(root_path + '\\Layers_Speckle\\rasters_Speckle'): os.makedirs(root_path + '\\Layers_Speckle\\rasters_Speckle')
-            path_style = root_path + '\\Layers_Speckle\\rasters_Speckle\\' + arcLayer.name + '_temp.lyrx'
-            #path_style2 = root_path + '\\' + newName + '_new.lyrx'
-            symJson = jsonFromLayerStyle(arcLayer, path_style)
-
-        layerRenderer: Dict[str, Any] = {}
-        layerRenderer['type'] = rType
-        print(rType)
-        my_raster = arcpy.Raster(arcLayer.dataSource)  
-        rasterBandNames = my_raster.bandNames
-        
-        #bandNames = rasterFeat["Band names"]
-        bandValues = [rasterFeat["@(10000)" + name + "_values"] for name in rasterBandNames]
-
-        if rType == "singlebandgray":  
-            try: band = arcLayer.symbology.colorizer.band 
-            except: band = 0 
-            try: 
-                bVals = bandValues[band]
-                bvalMin = min(bVals)
-                bvalMax = max(bVals)
-            except: 
-                bvalMin = 0
-                bvalMax = 255
-            layerRenderer.update({'properties': {'max':bvalMax,'min':bvalMin,'band':band+1,'contrast':1}}) 
-
-        elif rType == "multibandcolor":
-
-            try: greenBand = symJson["layerDefinitions"][0]["colorizer"]["greenBandIndex"] +1
-            except: greenBand = None
-            try: blueBand = symJson["layerDefinitions"][0]["colorizer"]["blueBandIndex"] +1
-            except: blueBand = None
-            try: redBand = symJson["layerDefinitions"][0]["colorizer"]["redBandIndex"] +1
-            except: 
-                print(greenBand)
-                print(blueBand)
-                if blueBand!=1 and greenBand!=1: redBand= 1
-                else: redBand = None
-            print(redBand)
-
-            try: 
-                rbVals = bandValues[redBand-1]
-                rbvalMin = min(rbVals)
-                rbvalMax = max(rbVals)
-                print(rbvalMin)
-                print(rbvalMax)
-            except: 
-                rbvalMin = 0
-                rbvalMax = 255
-            try: 
-                gbVals = bandValues[greenBand-1]
-                gbvalMin = min(gbVals)
-                gbvalMax = max(gbVals)
-            except: 
-                gbvalMin = 0
-                gbvalMax = 255
-            try: 
-                bbVals = bandValues[blueBand-1]
-                bbvalMin = min(bbVals)
-                bbvalMax = max(bbVals) 
-            except: 
-                bbvalMin = 0
-                bbvalMax = 255
-
-            layerRenderer.update({'properties': {'greenBand':greenBand,'blueBand':blueBand,'redBand':redBand}})
-            layerRenderer['properties'].update({'redContrast':1,'redMin':rbvalMin,'redMax':rbvalMax})
-            layerRenderer['properties'].update({'greenContrast':1,'greenMin':gbvalMin,'greenMax':gbvalMax})
-            layerRenderer['properties'].update({'blueContrast':1,'blueMin':bbvalMin,'blueMax':bbvalMax})
-        elif rType == "paletted":  
-            band = 0 
-            rendererClasses = arcLayer.symbology.colorizer.groups
-            classes = []
-            sourceRamp = {}
-
-            for i, cl in enumerate(rendererClasses):
-                if cl.heading == 'Value':
-                    for k, itm in enumerate(cl.items):
-                        value = itm.values[0]
-                        label = itm.label 
-                        try: 
-                            r,g,b = itm.color['RGB'][0], itm.color['RGB'][1], itm.color['RGB'][2]
-                            sColor = (r<<16) + (g<<8) + b
-                            classes.append({'color':sColor,'value':value,'label':label})
-                        except: pass 
-            layerRenderer.update({'properties': {'classes':classes,'ramp':sourceRamp,'band':band+1}})
-
-        return layerRenderer 
-    elif arcLayer.isFeatureLayer: 
-        layerRenderer: Dict[str, Any] = {}
-
-        sym = arcLayer.symbology
-        print(sym.renderer.type)
-
-        if sym.renderer.type == 'SimpleRenderer':
-            layerRenderer['type'] = 'singleSymbol'
-            layerRenderer['properties'] = {'symbol':{}, 'symbType':""}
-            symbolColor = symbol_color_to_speckle(sym.renderer.symbol.color)
-            layerRenderer['properties'].update({'symbol':{'symbColor': symbolColor}, 'symbType':''})
-
-        elif sym.renderer.type == 'UniqueValueRenderer':
-            layerRenderer['type'] = 'categorizedSymbol'
-            layerRenderer['properties'] = {'attribute': '', 'symbType': ''} #{'symbol':{}, 'ramp':{}, 'ranges':{}, 'gradMethod':"", 'symbType':"", 'legendClassificationAttribute': ""}
-            
-            attribute = sym.renderer.fields[0]
-            layerRenderer['properties']['attribute'] = attribute
-            sourceSymbColor = symbol_color_to_speckle(sym.renderer.defaultSymbol.color)
-            layerRenderer['properties'].update( {'sourceSymbColor': sourceSymbColor} )
- 
-            categories = sym.renderer.groups
-            layerRenderer['properties']['categories'] = []
-
-            for i, grp in enumerate(categories):
-                for itm in grp.items:
-                    value = itm.values[0][0]
-                    symbColor = symbol_color_to_speckle(itm.symbol.color)
-                    label = itm.label
-                    layerRenderer['properties']['categories'].append({'value':value,'symbColor':symbColor,'symbOpacity':1, 'sourceSymbColor': sourceSymbColor,'label':label})
-
-        elif sym.renderer.type == 'GraduatedColorsRenderer' or sym.renderer.type == 'GraduatedSymbolsRenderer':
-            layerRenderer['type'] = 'graduatedSymbol'
-            layerRenderer['properties'] = {'symbol':{}, 'ramp':{}, 'ranges':{}, 'gradMethod':"", 'symbType':""}
-        
-            attribute = sym.renderer.classificationField 
-            sourceSymbColor = (0<<16) + (0<<8) + 0  
-            layerRenderer['properties'].update( {'attribute': attribute, 'symbType': '', 'gradMethod': 0, 'sourceSymbColor': sourceSymbColor} )
-
-            rRamp = sym.renderer.colorRamp # QgsGradientColorRamp
-            layerRenderer['properties']['ramp'] = {} # gradientColorRampToSpeckle(rRamp)
-
-            rRanges = sym.renderer.classBreaks
-            layerRenderer['properties']['ranges'] = []
-            for itm in rRanges:
-                try: lower = float(itm.label.split(" - ")[0]) if (" - " in rRanges.label) else float(rRanges.label[0])
-                except: lower = 0
-                upper = itm.upperBound
-                symbColor = symbol_color_to_speckle(itm.symbol.color) 
-                label = itm.label
-                width = 0.26
-                # {'label': '1 - 1.4', 'lower': 1.0, 'symbColor': <PyQt5.QtGui.QColor ...BD9B9D4A0>, 'symbOpacity': 1.0, 'upper': 1.4}
-                layerRenderer['properties']['ranges'].append({'lower':lower,'upper':upper,'symbColor':symbColor,'symbOpacity':1,'label':label,'width':width})
-     
-        elif sym.renderer.type == 'UnclassedColorsRenderer':
-            layerRenderer['type'] = 'graduatedSymbol'
-            layerRenderer['properties'] = {'symbol':{}, 'ramp':{}, 'ranges':{}, 'gradMethod':"", 'symbType':""}
-        
-            attribute = sym.renderer.field 
-            sourceSymbColor = (0<<16) + (0<<8) + 0  
-            layerRenderer['properties'].update( {'attribute': attribute, 'symbType': '', 'gradMethod': 0, 'sourceSymbColor': sourceSymbColor} )
-            layerRenderer['properties']['ramp'] = {} # gradientColorRampToSpeckle(rRamp)
-
-            lowest = sym.renderer.lowerLabel
-            highest = sym.renderer.upperLabel
-
-            # trick to get colors 
-            rRamp = sym.renderer.colorRamp # QgsGradientColorRamp
-            arcRamp = project.listColorRamps('White to Black')[0]
-            sym.updateRenderer('GraduatedColorsRenderer')
-            sym.renderer.colorRamp = arcRamp
-            sym.renderer.classificationField = attribute
-            rows_attributes = arcpy.da.SearchCursor(arcLayer.longName, attribute)
-            row_attrs = []
-            row_max = -1000000000
-            row_min = 1000000000
-            for k, attrs in enumerate(rows_attributes): 
-                row_attrs.append(attrs[0])
-                if attrs[0] < row_min: row_min = attrs[0]
-                if attrs[0] > row_max: row_max = attrs[0]
-            row_range = row_max - row_min
-            breakCount = len(list(set(row_attrs))) # only unique values 
-            sym.renderer.breakCount = breakCount
-
-            # run as gradient colors 
-            rRanges = sym.renderer.classBreaks
-            layerRenderer['properties']['ranges'] = []
-            for itm in rRanges:
-                try: lower = float(itm.label.split(" - ")[0]) if (" - " in rRanges.label) else float(rRanges.label[0])
-                except: lower = 0
-                upper = itm.upperBound
-                if row_range==0: rgb = 0
-                else: rgb = 255 - int((itm.upperBound - row_min) / row_range * 255 ) 
-                symbColor = (rgb<<16) + (rgb<<8) + rgb 
-                label = itm.label
-                width = 0.26
-                # {'label': '1 - 1.4', 'lower': 1.0, 'symbColor': <PyQt5.QtGui.QColor ...BD9B9D4A0>, 'symbOpacity': 1.0, 'upper': 1.4}
-                layerRenderer['properties']['ranges'].append({'lower':lower,'upper':upper,'symbColor':symbColor,'symbOpacity':1,'label':label,'width':width})
-
-        else: return None
-            
-        return layerRenderer
-
-    else: return None
-
-
-def featureColorfromNativeRenderer(index: int, arcLayer) -> int:
-    # case with one color for the entire layer
-    #try:
-    sym = arcLayer.symbology
-    color = {'RGB': [100,100,100,100]}
-
-    if sym.renderer.type == 'SimpleRenderer':
-        print('SimpleRenderer')
-        color = sym.renderer.symbol.color
-
-    elif sym.renderer.type == 'UniqueValueRenderer':
-        print('Unique Value Renderer')
-
-        attribute = sym.renderer.fields[0]
-        color = sym.renderer.defaultSymbol.color
-        categories = sym.renderer.groups
-
-        rows_attributes = arcpy.da.SearchCursor(arcLayer.longName, attribute)
-        row_shapes_list = [x for k, x in enumerate(rows_attributes)]
-
-        color_found = 0
-        for i, grp in enumerate(categories):
-            if color_found == 1: break
-            for n, itm in enumerate(grp.items):
-                for k, attrs in enumerate(row_shapes_list): 
-                    if str(itm.values[0][0]) == "<Null>": itm.values[0][0] = None 
-                    if k == index and ( str(attrs[0]) == str(itm.values[0][0]) or (attrs[0] is None and str(itm.values[0][0]) == "<Null>") ): 
-                        color = itm.symbol.color
-                        print("symbol color: ")
-                        print(color)
-                        color_found = 1
-                        break 
-
-    elif sym.renderer.type == 'GraduatedColorsRenderer' or sym.renderer.type == 'GraduatedSymbolsRenderer':
-        print('Graduated Colors / Sybmols Renderer')
-        attribute = sym.renderer.classificationField 
-        rows_attributes = arcpy.da.SearchCursor(arcLayer.longName, attribute)
-        row_shapes_list = [x for k, x in enumerate(rows_attributes)]
-
-        rRanges = sym.renderer.classBreaks
-        upperBounds = [-10000000000000000000]
-        color_found = 0
-        for itm in rRanges:
-            print(itm)
-            if color_found == 1: break
-            for k, attrs in enumerate(row_shapes_list):  
-                try:
-                    if k == index and float(attrs[0]) <= float(itm.upperBound) and (k==0 or float(attrs[0]) > float(upperBounds[-1]) ):
-                        color = itm.symbol.color
-                        color_found = 1
-                        break 
-                except: pass
-            upperBounds.append(itm.upperBound)
-
-    elif sym.renderer.type == 'UnclassedColorsRenderer': 
-        print('UnclassedColorsRenderer')
-        attribute = sym.renderer.field  
-
-        sym.updateRenderer('GraduatedColorsRenderer')
-        sym.renderer.classificationField = attribute
-
-        rows_attributes = arcpy.da.SearchCursor(arcLayer.longName, attribute)
-        row_shapes_list = [x for k, x in enumerate(rows_attributes)]
-        row_attrs = []
-        row_max = -10000000000000000000
-        row_min = 10000000000000000000
-        feat_value = None
-        for k, attrs in enumerate(row_shapes_list): 
-            row_attrs.append(attrs[0])
-            if attrs[0] < row_min: row_min = attrs[0]
-            if attrs[0] > row_max: row_max = attrs[0]
-            if k == index: feat_value = attrs[0]
-        row_range = row_max - row_min
-        breakCount = len(list(set(row_attrs))) # only unique values 
-        sym.renderer.breakCount = breakCount
-
-        # run as gradient colors 
-        
-        upperBounds = [-10000000000000000000]
-        rRanges = sym.renderer.classBreaks
-
-        for itm in rRanges:
-            print(itm)
-            try:
-                if row_range!=0 and float(feat_value) <= float(itm.upperBound) and (len(upperBounds)==0 or float(feat_value) > float(upperBounds[-1])):
-                    rgb = 255 - int((itm.upperBound - row_min) / row_range * 255 ) 
-                    color = {'RGB':[rgb,rgb,rgb,100]}
-                    print(color) 
-                    break
-            except: pass 
-            upperBounds.append(itm.upperBound)
-
-    else: 
-        print('Else')
-        return (100<<16) + (100<<8) + 100
-    
-    print("final color: ")
-    print(color) 
-    # construct RGB color
-    col = symbol_color_to_speckle(color)
-    print(col)
-    return col
-

speckle_toolbox/esri/toolboxes/speckle/converter/layers/utils.py

@@ -2,130 +2,190 @@ from typing import Dict, Any, List, Union
 import json 
 from specklepy.objects import Base
 import arcpy 
-
 from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
 import os
 
-ATTRS_REMOVE = ['geometry','applicationId','bbox','displayStyle', 'id', 'renderMaterial', 'displayMesh', 'displayValue'] 
+
+try: 
+    from speckle.converter.layers.emptyLayerTemplates import createGroupLayer
+    from speckle.plugin_utils.helpers import findOrCreatePath
+    from speckle.plugin_utils.logger import logToUser
+except:
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.emptyLayerTemplates import createGroupLayer
+    from speckle_toolbox.esri.toolboxes.speckle.plugin_utils.helpers import findOrCreatePath
+    from speckle_toolbox.esri.toolboxes.speckle.plugin_utils.logger import logToUser
+
+#ATTRS_REMOVE = ['geometry','applicationId','bbox','displayStyle', 'id', 'renderMaterial', 'displayMesh', 'displayValue'] 
+ATTRS_REMOVE = ['speckleTyp','speckle_id','geometry','applicationId','bbox','displayStyle', 'id', 'renderMaterial', 'displayMesh', 'displayValue'] 
+
+    
+def findAndClearLayerGroup(gis_project: ArcGISProject, newGroupName: str = ""):
+    print("find And Clear LayerGroup")
+    try:
+        groupExists = 0
+        print(newGroupName)
+        for l in gis_project.activeMap.listLayers(): 
+            #print(l.longName)
+            if l.longName.startswith(newGroupName + "\\"):
+                #print(l.longName)
+                gis_project.activeMap.removeLayer(l)
+                groupExists+=1
+            elif l.longName == newGroupName: 
+                groupExists+=1
+        print(newGroupName)
+        if groupExists == 0:
+            # create empty group layer file "\\Layers_Speckle\\
+            path = "\\".join(gis_project.filePath.split("\\")[:-1]) + "\\Layers_Speckle\\"
+            findOrCreatePath(path)
+            lyr_path = path + newGroupName + ".lyrx"
+            print(lyr_path)
+            try:
+                f = open(lyr_path, "w")
+                content = createGroupLayer().replace("TestGroupLayer", newGroupName)
+                f.write(content)
+                f.close()
+                newGroupLayer = arcpy.mp.LayerFile(lyr_path)
+                layerGroup = gis_project.activeMap.addLayer(newGroupLayer)[0]
+                print(layerGroup)
+            except: # for 3.0.0
+                if gis_project.active_map is not None:
+                    print("try creating the group")
+                    layerGroup = gis_project.activeMap.createGroupLayer(newGroupName)
+                    print(layerGroup)
+                else:
+                    logToUser("The map didn't fully load, try selecting the project Map or/and refreshing the plugin.", 1)
+                    return
+    except Exception as e:
+        logToUser(e)
+
 
 def getVariantFromValue(value: Any) -> Union[str, None]:
     #print("_________get variant from value_______")
     # TODO add Base object
-    pairs = [
-        (str, "TEXT"), # 10
-        (float, "FLOAT"),
-        (int, "LONG"),
-        (bool, "SHORT") 
-    ]
     res = None
-    for p in pairs:
-        if isinstance(value, p[0]): 
-            res = p[1]
-            try:
-                if res == "LONG" and (value>= 2147483647 or value<= -2147483647):
-                    #https://pro.arcgis.com/en/pro-app/latest/help/data/geodatabases/overview/arcgis-field-data-types.htm
-                    res = "FLOAT"
-            except Exception as e: print(e)
-            break
+    try:
+        pairs = [
+            (str, "TEXT"), # 10
+            (float, "FLOAT"),
+            (int, "LONG"),
+            (bool, "SHORT") 
+        ]
+        for p in pairs:
+            if isinstance(value, p[0]): 
+                res = p[1]
+                try:
+                    if res == "LONG" and (value>= 2147483647 or value<= -2147483647):
+                        #https://pro.arcgis.com/en/pro-app/latest/help/data/geodatabases/overview/arcgis-field-data-types.htm
+                        res = "FLOAT"
+                except Exception as e: print(e)
+                break
+
+    except Exception as e:
+        logToUser(e)
 
     return res
 
 def getLayerAttributes(featuresList: List[Base], attrsToRemove: List[str] = ATTRS_REMOVE ) -> Dict[str, str]:
     print("03________ get layer attributes")
-
-    if not isinstance(featuresList, list): features = [featuresList]
-    else: features = featuresList[:]
-    
     fields = {}
-    all_props = []
-    for feature in features: 
-        #get object properties to add as attributes
-        dynamicProps = feature.get_dynamic_member_names()
-        for att in attrsToRemove:
-            try: dynamicProps.remove(att)
-            except: pass
-        dynamicProps.sort()
+    try: 
+        if not isinstance(featuresList, list): features = [featuresList]
+        else: features = featuresList[:]
+        
+        all_props = []
+        for feature in features: 
+            #get object properties to add as attributes
+            dynamicProps = feature.get_dynamic_member_names()
+            for att in ATTRS_REMOVE:
+                try: dynamicProps.remove(att)
+                except: pass
+                
+            dynamicProps.sort()
 
-        # add field names and variands 
-        for name in dynamicProps:
-            #if name not in all_props: all_props.append(name)
+            # add field names and variands 
+            for name in dynamicProps:
+                #if name not in all_props: all_props.append(name)
 
-            value = feature[name]
-            variant = getVariantFromValue(value)
-            #if name == 'area': print(value); print(variant)
-            if not variant: variant = None #LongLong #4 
+                value = feature[name]
+                variant = getVariantFromValue(value)
+                #if name == 'area': print(value); print(variant)
+                if not variant: variant = None #LongLong #4 
 
-            # go thought the dictionary object
-            if value and isinstance(value, list):
-                #all_props.remove(name) # remove generic dict name
-                for i, val_item in enumerate(value):
-                    newF, newVals = traverseDict( {}, {}, name+"_"+str(i), val_item)
+                # go thought the dictionary object
+                if value and isinstance(value, list):
+                    #all_props.remove(name) # remove generic dict name
+                    for i, val_item in enumerate(value):
+                        newF, newVals = traverseDict( {}, {}, name+"_"+str(i), val_item)
 
+                        for i, (k,v) in enumerate(newF.items()):
+                            if k not in all_props: all_props.append(k)
+                            if k not in fields.keys(): fields.update({k: v}) 
+                            else: #check if the field was empty previously: 
+                                oldVariant = fields[k]
+                                # replace if new one is NOT Float (too large integers)
+                                #if oldVariant != "FLOAT" and v == "FLOAT": 
+                                #    fields.update({k: v}) 
+                                # replace if new one is NOT LongLong or IS String
+                                if oldVariant != "TEXT" and v == "TEXT": 
+                                    fields.update({k: v}) 
+                
+                # add a field if not existing yet 
+                else: # if str, Base, etc
+                    newF, newVals = traverseDict( {}, {}, name, value)
+                    
                     for i, (k,v) in enumerate(newF.items()):
                         if k not in all_props: all_props.append(k)
-                        if k not in fields.keys(): fields.update({k: v}) 
+                        if k not in fields.keys(): fields.update({k: v}) #if variant is known
                         else: #check if the field was empty previously: 
                             oldVariant = fields[k]
                             # replace if new one is NOT Float (too large integers)
-                            if oldVariant != "FLOAT" and v == "FLOAT": 
-                                fields.update({k: v}) 
+                            #print(oldVariant, v)
+                            #if oldVariant == "LONG" and v == "FLOAT": 
+                            #    fields.update({k: v}) 
                             # replace if new one is NOT LongLong or IS String
                             if oldVariant != "TEXT" and v == "TEXT": 
                                 fields.update({k: v}) 
-            
-            # add a field if not existing yet 
-            else: # if str, Base, etc
-                newF, newVals = traverseDict( {}, {}, name, value)
-                
-                for i, (k,v) in enumerate(newF.items()):
-                    if k not in all_props: all_props.append(k)
-                    if k not in fields.keys(): fields.update({k: v}) #if variant is known
-                    else: #check if the field was empty previously: 
-                        oldVariant = fields[k]
-                        # replace if new one is NOT Float (too large integers)
-                        #print(oldVariant, v)
-                        if oldVariant == "LONG" and v == "FLOAT": 
-                            fields.update({k: v}) 
-                        # replace if new one is NOT LongLong or IS String
-                        if oldVariant != "TEXT" and v == "TEXT": 
-                            fields.update({k: v}) 
-                        #print(fields)
-    # replace all empty ones wit String
-    all_props.append("Speckle_ID") 
-    for name in all_props:
-        if name not in fields.keys(): 
-            fields.update({name: 'TEXT'}) 
+                            #print(fields)
+        # replace all empty ones wit String
+        all_props.append("Speckle_ID") 
+        for name in all_props:
+            if name not in fields.keys(): 
+                fields.update({name: 'TEXT'}) 
 
-    fields_sorted = {k: v for k, v in sorted(fields.items(), key=lambda item: item[0])}
-    return fields_sorted
+        #fields_sorted = {k: v for k, v in sorted(fields.items(), key=lambda item: item[0])}
+    except Exception as e:
+        logToUser(e)
+    return fields
 
 def traverseDict(newF: dict, newVals: dict, nam: str, val: Any):
-    
-    if isinstance(val, dict):
-        for i, (k,v) in enumerate(val.items()):
-            newF, newVals = traverseDict( newF, newVals, nam+"_"+k, v)
-    elif isinstance(val, Base):
-        dynamicProps = val.get_dynamic_member_names()
-        for att in ATTRS_REMOVE:
-            try: dynamicProps.remove(att)
-            except: pass
-        dynamicProps.sort()
+    try:
+        if isinstance(val, dict):
+            for i, (k,v) in enumerate(val.items()):
+                newF, newVals = traverseDict( newF, newVals, nam+"_"+k, v)
+        elif isinstance(val, Base):
+            dynamicProps = val.get_dynamic_member_names()
+            for att in ATTRS_REMOVE:
+                try: dynamicProps.remove(att)
+                except: pass
+            dynamicProps.sort()
 
-        item_dict = {} 
-        for prop in dynamicProps:
-            item_dict.update({prop: val[prop]})
+            item_dict = {} 
+            for prop in dynamicProps:
+                item_dict.update({prop: val[prop]})
 
-        for i, (k,v) in enumerate(item_dict.items()):
-            newF, newVals = traverseDict( newF, newVals, nam+"_"+k, v)
-    else: 
-        var = getVariantFromValue(val)
-        if var is None: 
-            var = 'TEXT'
-            val = str(val)
-        #print(var)
-        newF.update({nam: var})
-        newVals.update({nam: val})  
+            for i, (k,v) in enumerate(item_dict.items()):
+                newF, newVals = traverseDict( newF, newVals, nam+"_"+k, v)
+        else: 
+            var = getVariantFromValue(val)
+            if var is None: 
+                var = 'TEXT'
+                val = str(val)
+            #print(var)
+            newF.update({nam: var})
+            newVals.update({nam: val})  
 
+    except Exception as e:
+        logToUser(e)
     return newF, newVals
 
 def get_scale_factor(units: str) -> float:
@@ -147,162 +207,190 @@ def get_scale_factor(units: str) -> float:
     }
     if units is not None and units.lower() in unit_scale.keys():
         return unit_scale[units]
-    arcpy.AddWarning(f"Units {units} are not supported. Meters will be applied by default.")
+    logToUser(f"Units {units} are not supported. Meters will be applied by default.",0)
     return 1.0
 
 def findTransformation(f_shape, geomType, layer_sr: arcpy.SpatialReference, projectCRS: arcpy.SpatialReference, selectedLayer: arcLayer):
     #apply transformation if needed
-    if layer_sr.name != projectCRS.name:
-        tr0 = tr1 = tr2 = tr_custom = None
-        print(layer_sr)
-        try:
-            transformations = arcpy.ListTransformations(layer_sr, projectCRS)
-            customTransformName = "layer_sr.name"+"_To_"+ projectCRS.name
-            if len(transformations) == 0:
-                midSr = arcpy.SpatialReference("WGS 1984") # GCS_WGS_1984
-                try:
-                    tr1 = arcpy.ListTransformations(layer_sr, midSr)[0]
-                    tr2 = arcpy.ListTransformations(midSr, projectCRS)[0]
-                except: 
-                    #customGeoTransfm = "GEOGTRAN[METHOD['Geocentric_Translation'],PARAMETER['X_Axis_Translation',''],PARAMETER['Y_Axis_Translation',''],PARAMETER['Z_Axis_Translation','']]"
-                    #CreateCustomGeoTransformation(customTransformName, layer_sr, projectCRS)
-                    tr_custom = customTransformName
-            else: 
-                #print("else")
-                # choose equation based instead of file-based/grid-based method, 
-                # to be consistent with QGIS: https://desktop.arcgis.com/en/arcmap/latest/map/projections/choosing-an-appropriate-transformation.htm
-                selecterTr = {}
-                for tr in transformations:
-                    if "NTv2" not in tr and "NADCON" not in tr: 
-                        set1 = set( layer_sr.name.split("_") + projectCRS.name.split("_") )
-                        set2 = set( tr.split("_") )
-                        diff = len( set(set1).symmetric_difference(set2) )
-                        selecterTr.update({tr: diff})
-                selecterTr = dict(sorted(selecterTr.items(), key=lambda item: item[1]))
-                tr0 = list(selecterTr.keys())[0]
+    try:
+        if layer_sr.name != projectCRS.name:
+            tr0 = tr1 = tr2 = tr_custom = None
+            print(layer_sr)
+            try:
+                transformations = arcpy.ListTransformations(layer_sr, projectCRS)
+                customTransformName = "layer_sr.name"+"_To_"+ projectCRS.name
+                if len(transformations) == 0:
+                    midSr = arcpy.SpatialReference("WGS 1984") # GCS_WGS_1984
+                    try:
+                        tr1 = arcpy.ListTransformations(layer_sr, midSr)[0]
+                        tr2 = arcpy.ListTransformations(midSr, projectCRS)[0]
+                    except: 
+                        #customGeoTransfm = "GEOGTRAN[METHOD['Geocentric_Translation'],PARAMETER['X_Axis_Translation',''],PARAMETER['Y_Axis_Translation',''],PARAMETER['Z_Axis_Translation','']]"
+                        #CreateCustomGeoTransformation(customTransformName, layer_sr, projectCRS)
+                        tr_custom = customTransformName
+                else: 
+                    #print("else")
+                    # choose equation based instead of file-based/grid-based method, 
+                    # to be consistent with QGIS: https://desktop.arcgis.com/en/arcmap/latest/map/projections/choosing-an-appropriate-transformation.htm
+                    selecterTr = {}
+                    for tr in transformations:
+                        if "NTv2" not in tr and "NADCON" not in tr: 
+                            set1 = set( layer_sr.name.split("_") + projectCRS.name.split("_") )
+                            set2 = set( tr.split("_") )
+                            diff = len( set(set1).symmetric_difference(set2) )
+                            selecterTr.update({tr: diff})
+                    selecterTr = dict(sorted(selecterTr.items(), key=lambda item: item[1]))
+                    tr0 = list(selecterTr.keys())[0]
 
-            if geomType != "Point" and geomType != "Polyline" and geomType != "Polygon" and geomType != "Multipoint":
-                try: arcpy.AddWarning("Unsupported or invalid geometry in layer " + selectedLayer.name)
-                except: arcpy.AddWarning("Unsupported or invalid geometry")
+                if geomType != "Point" and geomType != "Polyline" and geomType != "Polygon" and geomType != "Multipoint":
+                    try: logToUser("Unsupported or invalid geometry in layer " + selectedLayer.name)
+                    except: logToUser("Unsupported or invalid geometry")
 
-            # reproject geometry using chosen transformstion(s)
-            if tr0 is not None:
-                ptgeo1 = f_shape.projectAs(projectCRS, tr0)
-                f_shape = ptgeo1
-            elif tr1 is not None and tr2 is not None:
-                ptgeo1 = f_shape.projectAs(midSr, tr1)
-                ptgeo2 = ptgeo1.projectAs(projectCRS, tr2)
-                f_shape = ptgeo2
-            else:
-                ptgeo1 = f_shape.projectAs(projectCRS)
-                f_shape = ptgeo1
-        
-        except:
-            arcpy.AddWarning(f"Spatial Transformation not found for layer {selectedLayer.name}")
-            return None
+                # reproject geometry using chosen transformstion(s)
+                if tr0 is not None:
+                    ptgeo1 = f_shape.projectAs(projectCRS, tr0)
+                    f_shape = ptgeo1
+                elif tr1 is not None and tr2 is not None:
+                    ptgeo1 = f_shape.projectAs(midSr, tr1)
+                    ptgeo2 = ptgeo1.projectAs(projectCRS, tr2)
+                    f_shape = ptgeo2
+                else:
+                    ptgeo1 = f_shape.projectAs(projectCRS)
+                    f_shape = ptgeo1
+            
+            except:
+                logToUser(f"Spatial Transformation not found for layer {selectedLayer.name}")
+                return None
 
+    except Exception as e:
+        logToUser(e)
     return f_shape    
 
 def traverseDictByKey(d: Dict, key:str ="", result = None) -> Dict:
     print("__traverse")
-
-    result = None
-    #print(d)
-    for k, v in d.items():
-        
-        try: v = json.loads(v)
-        except: pass 
-        if isinstance(v, dict):
-            #print("__dict__")
-            if k == key: print("__break loop"); result = v; return result
-            else: 
-                result = traverseDictByKey(v, key, result)
-                if result is not None: return result
-        if isinstance(v, list):
-            for item in v: 
-                #print(item) 
-                if isinstance(item, dict): 
-                    result = traverseDictByKey(item, key, result)
+    try:
+        result = None
+        #print(d)
+        for k, v in d.items():
+            
+            try: v = json.loads(v)
+            except: pass 
+            if isinstance(v, dict):
+                #print("__dict__")
+                if k == key: print("__break loop"); result = v; return result
+                else: 
+                    result = traverseDictByKey(v, key, result)
                     if result is not None: return result
+            if isinstance(v, list):
+                for item in v: 
+                    #print(item) 
+                    if isinstance(item, dict): 
+                        result = traverseDictByKey(item, key, result)
+                        if result is not None: return result
+    except Exception as e:
+        logToUser(e)
+        return None
     #print("__result is: ____________")
     #return result 
 
 def hsv_to_rgb(listHSV):
-    h, s, v = listHSV[0], listHSV[1], listHSV[2]
-    if s == 0.0: v*=255; return (v, v, v)
-    i = int(h*6.) # XXX assume int() truncates!
-    f = (h*6.)-i; p,q,t = int(255*(v*(1.-s))), int(255*(v*(1.-s*f))), int(255*(v*(1.-s*(1.-f)))); v*=255; i%=6
-    if i == 0: return (v, t, p)
-    if i == 1: return (q, v, p)
-    if i == 2: return (p, v, t)
-    if i == 3: return (p, q, v)
-    if i == 4: return (t, p, v)
-    if i == 5: return (v, p, q)
+    try:
+        h, s, v = listHSV[0], listHSV[1], listHSV[2]
+        if s == 0.0: v*=255; return (v, v, v)
+        i = int(h*6.) # XXX assume int() truncates!
+        f = (h*6.)-i; p,q,t = int(255*(v*(1.-s))), int(255*(v*(1.-s*f))), int(255*(v*(1.-s*(1.-f)))); v*=255; i%=6
+        if i == 0: return (v, t, p)
+        if i == 1: return (q, v, p)
+        if i == 2: return (p, v, t)
+        if i == 3: return (p, q, v)
+        if i == 4: return (t, p, v)
+        if i == 5: return (v, p, q)
+        
+    except Exception as e:
+        logToUser(e)
+        return (0,0,0)
 
 def cmyk_to_rgb(c, m, y, k, cmyk_scale, rgb_scale=255):
-    r = rgb_scale * (1.0 - c / float(cmyk_scale)) * (1.0 - k / float(cmyk_scale))
-    g = rgb_scale * (1.0 - m / float(cmyk_scale)) * (1.0 - k / float(cmyk_scale))
-    b = rgb_scale * (1.0 - y / float(cmyk_scale)) * (1.0 - k / float(cmyk_scale))
-    return r, g, b
+    try:
+        r = rgb_scale * (1.0 - c / float(cmyk_scale)) * (1.0 - k / float(cmyk_scale))
+        g = rgb_scale * (1.0 - m / float(cmyk_scale)) * (1.0 - k / float(cmyk_scale))
+        b = rgb_scale * (1.0 - y / float(cmyk_scale)) * (1.0 - k / float(cmyk_scale))
+        return r, g, b
+    except Exception as e:
+        logToUser(e)
+        return 0,0,0
 
 def newLayerGroupAndName(layerName: str, streamBranch: str, project: ArcGISProject) -> str:
     print("___new Layer Group and Name")
-    #CREATE A GROUP "received blabla" with sublayers
     layerGroup = None
     newGroupName = f'{streamBranch}'
-    print(newGroupName)
-    for l in project.activeMap.listLayers():
-        if l.longName == newGroupName: layerGroup = l; break 
-    
-    #find a layer with a matching name in the "latest" group 
-    newName = f'{streamBranch.split("_")[len(streamBranch.split("_"))-1]}_{layerName}'
+    try:
+    #CREATE A GROUP "received blabla" with sublayers
+        print(newGroupName)
+        for l in project.activeMap.listLayers():
+            if l.longName == newGroupName: layerGroup = l; break 
+        
+        #find a layer with a matching name in the "latest" group 
+        newName = f'{streamBranch.split("_")[len(streamBranch.split("_"))-1]}_{layerName}'
 
-    all_layer_names = []
-    layerExists = 0
-    for l in project.activeMap.listLayers(): 
-        if l.longName.startswith(newGroupName + "\\"):
-            all_layer_names.append(l.longName)
-    #print(all_layer_names)
-    print(newName)
+        all_layer_names = []
+        layerExists = 0
+        for l in project.activeMap.listLayers(): 
+            if l.longName.startswith(newGroupName + "\\"):
+                all_layer_names.append(l.longName)
+        #print(all_layer_names)
+        print(newName)
 
-    longName = streamBranch + "\\" + newName 
-    if longName in all_layer_names: 
-        for index, letter in enumerate('234567890abcdefghijklmnopqrstuvwxyz'):
-            if (longName + "_" + letter) not in all_layer_names: 
-                newName += "_"+letter 
-                layerExists +=1 
-                break 
-    print(newName)              
-    return newName, layerGroup 
+        longName = streamBranch + "\\" + newName 
+        if longName in all_layer_names: 
+            for index, letter in enumerate('234567890abcdefghijklmnopqrstuvwxyz'):
+                if (longName + "_" + letter) not in all_layer_names: 
+                    newName += "_"+letter 
+                    layerExists +=1 
+                    break 
+        print(newName)       
+        return newName, layerGroup        
+    except Exception as e:
+        logToUser(e)
+        return None, None
 
 
 def curvedFeatureClassToSegments(layer) -> str:
     print("___densify___")
-    data = arcpy.Describe(layer.dataSource)
-    dataPath = data.catalogPath
-    print(dataPath)
-    newPath = dataPath+"_backup"
+    try:
+        data = arcpy.Describe(layer.dataSource)
+        dataPath = data.catalogPath
+        print(dataPath)
+        newPath = dataPath+"_backup"
 
-    arcpy.management.CopyFeatures(dataPath, newPath) # features copied like this do not preserve curved segments
+        arcpy.management.CopyFeatures(dataPath, newPath) # features copied like this do not preserve curved segments
 
-    arcpy.edit.Densify(in_features = newPath, densification_method = "ANGLE", max_angle = 0.01, max_vertex_per_segment = 100) # https://pro.arcgis.com/en/pro-app/latest/tool-reference/editing/densify.htm
-    print(newPath)
-    return newPath
+        arcpy.edit.Densify(in_features = newPath, densification_method = "ANGLE", max_angle = 0.01, max_vertex_per_segment = 100) # https://pro.arcgis.com/en/pro-app/latest/tool-reference/editing/densify.htm
+        print(newPath)
+        return newPath
+     
+    except Exception as e:
+        logToUser(e)
+        return None
 
 def validate_path(path: str):
-    # https://github.com/EsriOceans/btm/commit/a9c0529485c9b0baa78c1f094372c0f9d83c0aaf
     """If our path contains a DB name, make sure we have a valid DB name and not a standard file name."""
-    dirname, file_name = os.path.split(path)
-    #print(dirname)
-    #print(file_name)
-    file_base = os.path.splitext(file_name)[0]
-    if dirname == '':
-        # a relative path only, relying on the workspace
-        dirname = arcpy.env.workspace
-    path_ext = os.path.splitext(dirname)[1].lower()
-    if path_ext in ['.mdb', '.gdb', '.sde']:
-        # we're working in a database
-        file_name = arcpy.ValidateTableName(file_base) # e.g. add a letter in front of the name 
-    validated_path = os.path.join(dirname, file_name)
-    #msg("validated path: %s; (from %s)" % (validated_path, path))
-    return validated_path
+    try:
+        # https://github.com/EsriOceans/btm/commit/a9c0529485c9b0baa78c1f094372c0f9d83c0aaf
+        dirname, file_name = os.path.split(path)
+        #print(dirname)
+        #print(file_name)
+        file_base = os.path.splitext(file_name)[0]
+        if dirname == '':
+            # a relative path only, relying on the workspace
+            dirname = arcpy.env.workspace
+        path_ext = os.path.splitext(dirname)[1].lower()
+        if path_ext in ['.mdb', '.gdb', '.sde']:
+            # we're working in a database
+            file_name = arcpy.ValidateTableName(file_base) # e.g. add a letter in front of the name 
+        validated_path = os.path.join(dirname, file_name)
+        #msg("validated path: %s; (from %s)" % (validated_path, path))
+        return validated_path
+    except Exception as e:
+        logToUser(e)
+        return None 

speckle_toolbox/esri/toolboxes/speckle/plugin_utils/helpers.py

@@ -0,0 +1,39 @@
+import os
+from typing import List
+try:
+    from speckle.plugin_utils.logger import logToUser
+except:
+    from speckle_toolbox.esri.toolboxes.speckle.plugin_utils.logger import logToUser
+
+
+def findOrCreatePath(path: str):
+    if not os.path.exists(path): 
+        os.makedirs(path)
+
+def validateNewFclassName(newName: str, prefix: str, all_layer_names: List[str]) -> str:
+    
+    fixed_name = newName
+
+    if (prefix + fixed_name) in all_layer_names: 
+    
+        layerNameCreated = 0
+        for index, letter in enumerate('234567890abcdefghijklmnopqrstuvwxyz'):
+            if ((prefix + fixed_name) + "_" + letter) not in all_layer_names: 
+                fixed_name += "_"+letter
+                layerNameCreated +=1
+                break 
+        if layerNameCreated == 0:
+            for index, letter in enumerate('234567890abcdefghijklmnopqrstuvwxyz'):
+                test_fixed_name = validateNewFclassName((fixed_name + "_" + letter), prefix, all_layer_names)
+                if (prefix + test_fixed_name) not in all_layer_names: 
+                    fixed_name = test_fixed_name 
+                    layerNameCreated +=1
+                    break 
+                #else: layerNameCreated +=1
+
+        if layerNameCreated == 0:
+            logToUser('Feature class name already exists')
+            #return fixed_name
+
+    return fixed_name
+    

speckle_toolbox/esri/toolboxes/speckle/plugin_utils/logger.py

@@ -0,0 +1,22 @@
+
+from PyQt5.QtWidgets import QMessageBox
+from PyQt5 import QtCore
+
+def logToUser(msg: str, level: int = 2):
+        # https://www.techwithtim.net/tutorials/pyqt5-tutorial/messageboxes/
+        window = QMessageBox()
+        #msg.setWindowFlag(QtCore.Qt.WindowStaysOnTopHint)
+        if level==0: 
+              window.setWindowTitle("Info")
+              window.setIcon(QMessageBox.Icon.Information)
+        if level==1: 
+              window.setWindowTitle("Warning")
+              window.setIcon(QMessageBox.Icon.Warning)
+        elif level==2: 
+              window.setWindowTitle("Error")
+              window.setIcon(QMessageBox.Icon.Critical)
+        window.setFixedWidth(200)
+        window.setText(str(msg))
+        window.exec_() 
+        
+        return

speckle_toolbox/esri/toolboxes/speckle/plugin_utils/object_utils.py

@@ -0,0 +1,135 @@
+
+from typing import Any, Callable, List, Optional 
+
+
+from specklepy.objects import Base
+
+try:
+    from speckle.plugin_utils.logger import logToUser
+    from speckle.converter.layers.Layer import VectorLayer, RasterLayer, Layer
+    from speckle.converter.layers import bimLayerToNative, cadLayerToNative, layerToNative
+except:
+    from speckle_toolbox.esri.toolboxes.speckle.plugin_utils.logger import logToUser
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.Layer import VectorLayer, RasterLayer, Layer
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers import bimLayerToNative, cadLayerToNative, layerToNative
+
+import arcpy
+
+SPECKLE_TYPES_TO_READ = ["Objects.Geometry.", "Objects.BuiltElements.", "IFC"] # will properly traverse and check for displayValue
+
+def traverseObject(
+    base: Base,
+    callback: Optional[Callable[[Base, str], bool]],
+    check: Optional[Callable[[Base], bool]],
+    streamBranch: str,
+):
+    try:
+        #print("traverse Object")
+        #print(base)
+        if check and check(base):
+            res = callback(base, streamBranch) if callback else False
+            #print(res)
+            if res:
+                return
+        memberNames = base.get_member_names()
+        #print(base)
+        #print(memberNames)
+        for name in memberNames:
+            try:
+                if ["id", "applicationId", "units", "speckle_type"].index(name):
+                    continue
+            except:
+                pass
+            #print(name)
+            traverseValue(base[name], callback, check, streamBranch)
+    except Exception as e:
+        logToUser(e)
+
+def traverseValue(
+    value: Any,
+    callback: Optional[Callable[[Base, str], bool]],
+    check: Optional[Callable[[Base], bool]],
+    streamBranch: str,
+):
+    try:
+        #print("traverse Value")
+        #print(value)
+        if isinstance(value, Base):
+            traverseObject(value, callback, check, streamBranch)
+        if isinstance(value, List):
+            for item in value:
+                traverseValue(item, callback, check, streamBranch)
+    except Exception as e:
+        logToUser(e)
+
+def callback(base: Base, streamBranch: str) -> bool:
+    try:
+        #print("callback")
+        if isinstance(base, VectorLayer) or isinstance(base, Layer) or isinstance(base, RasterLayer):
+            if isinstance(base, Layer):
+                logToUser(f"Speckle class \"Layer\" will be deprecated in future updates in favour of \"VectorLayer\" or \"RasterLayer\"", 0) 
+            layer = layerToNative(base, streamBranch)
+            #print(layer)
+            if layer is not None:
+                arcpy.AddMessage("Layer created: " + layer.name)
+        else:
+            loopObj(base, "", streamBranch)
+        return True
+    except Exception as e:
+        logToUser(e)
+        return False
+
+def loopObj(base: Base, baseName: str, streamBranch: str):
+    try:
+        memberNames = base.get_member_names()
+        for name in memberNames:
+            if name in ["id", "applicationId", "units", "speckle_type"]: continue
+            # skip if traversal goes to displayValue of an object, that will be readable anyway:
+            if (name == "displayValue" or name == "@displayValue") and base.speckle_type.startswith(tuple(SPECKLE_TYPES_TO_READ)): continue 
+
+            try: loopVal(base[name], baseName + "/" + name, streamBranch)
+            except: pass
+    
+    except Exception as e:
+        logToUser(e)
+
+def loopVal(value: Any, name: str, streamBranch: str): # "name" is the parent object/property/layer name
+    try:
+        if isinstance(value, Base): 
+            try: # loop through objects with Speckletype prop, but don't go through parts of Speckle Geometry object
+                if not value.speckle_type.startswith("Objects.Geometry."): 
+                    loopObj(value, name, streamBranch)
+            except: 
+                loopObj(value, name, streamBranch)
+
+        elif isinstance(value, List):
+            streamBranch = streamBranch.replace("[","_").replace("]","_").replace(" ","_").replace("-","_").replace("(","_").replace(")","_").replace(":","_").replace("\\","_").replace("/","_").replace("\"","_").replace("&","_").replace("@","_").replace("$","_").replace("%","_").replace("^","_")
+
+            objectListConverted = 0
+            #print("loop val - List")
+            for i, item in enumerate(value):
+                loopVal(item, name, streamBranch)
+                if item.speckle_type and item.speckle_type.startswith("IFC"): 
+                    # keep traversing infinitely, just don't run repeated conversion for the same list of objects
+                    try: 
+                        if item["displayValue"] is not None and objectListConverted == 0: 
+                            bimLayerToNative(value, name, streamBranch)
+                            objectListConverted += 1
+                    except: 
+                        try: 
+                            if item["@displayValue"] is not None and objectListConverted == 0: 
+                                bimLayerToNative(value, name, streamBranch)
+                                objectListConverted += 1
+                        except: pass 
+                
+                elif item.speckle_type and (item.speckle_type == "Objects.Geometry.Mesh" or item.speckle_type == "Objects.Geometry.Brep" or item.speckle_type.startswith("Objects.BuiltElements.")):
+                    bimLayerToNative(value, name, streamBranch)
+                    break
+                elif item.speckle_type and item.speckle_type != "Objects.Geometry.Mesh" and item.speckle_type != "Objects.Geometry.Brep" and item.speckle_type.startswith("Objects.Geometry."): # or item.speckle_type == 'Objects.BuiltElements.Alignment'): 
+                    pt, pl = cadLayerToNative(value, name, streamBranch)
+                    if pt is not None: arcpy.AddMessage("Layer group created: " + str(pt.name))
+                    if pl is not None: arcpy.AddMessage("Layer group created: " + str(pl.name))
+                    break
+    except Exception as e:
+        logToUser(e)
+

speckle_toolbox/esri/toolboxes/speckle/speckle_arcgis_old.py

@@ -0,0 +1,787 @@
+
+# -*- coding: utf-8 -*-
+
+from typing import Any, Callable, List, Optional, Tuple
+#r'''
+from collections import defaultdict
+
+import arcpy
+#from arcpy import toolbox
+from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
+from arcpy import metadata as md
+
+try:
+    from speckle.converter.layers.Layer import Layer, RasterLayer
+    from speckle.converter.layers._init_ import convertSelectedLayers, layerToNative, cadLayerToNative, bimLayerToNative
+    from speckle.ui.project_vars import toolboxInputsClass, speckleInputsClass
+    from speckle.converter.layers.emptyLayerTemplates import createGroupLayer
+    from speckle.converter.layers.Layer import VectorLayer
+except: 
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.Layer import Layer, RasterLayer
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers import convertSelectedLayers, layerToNative, cadLayerToNative, bimLayerToNative
+    from speckle_toolbox.esri.toolboxes.speckle.ui.project_vars import toolboxInputsClass, speckleInputsClass
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.emptyLayerTemplates import createGroupLayer
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.Layer import VectorLayer
+    
+from arcgis.features import FeatureLayer
+import os
+import os.path
+import sys 
+
+import specklepy
+from specklepy.api.models import Branch, Stream, Streams
+from specklepy.transports.server.server import ServerTransport
+from specklepy.api.credentials import get_local_accounts
+from specklepy.api.client import SpeckleClient
+from specklepy.api import operations
+from specklepy.logging.exceptions import (
+    GraphQLException,
+    SpeckleException,
+    SpeckleWarning,
+)
+#from specklepy.api.credentials import StreamWrapper
+from specklepy.api.wrapper import StreamWrapper
+from specklepy.objects import Base
+from specklepy.logging import metrics
+
+#'''
+
+def traverseObject(
+        base: Base,
+        callback: Optional[Callable[[Base], bool]],
+        check: Optional[Callable[[Base], bool]],
+    ):
+        if check and check(base):
+            res = callback(base) if callback else False
+            if res:
+                return
+        memberNames = base.get_member_names()
+        for name in memberNames:
+            try:
+                if ["id", "applicationId", "units", "speckle_type"].index(name):
+                    continue
+            except:
+                pass
+            traverseValue(base[name], callback, check)
+            
+def traverseValue(
+    value: Any,
+    callback: Optional[Callable[[Base], bool]],
+    check: Optional[Callable[[Base], bool]],
+):
+    if isinstance(value, Base):
+        traverseObject(value, callback, check)
+    if isinstance(value, List):
+        for item in value:
+            traverseValue(item, callback, check)
+
+
+class Toolbox:
+    def __init__(self):
+        """Define the toolbox (the name of the toolbox is the name of the
+        .pyt file)."""
+        print("___ping_Toolbox")
+        self.label = "Speckle Tools"
+        self.alias = "speckle_toolbox_"  
+        # List of tool classes associated with this toolbox
+        self.tools = [Speckle]  
+
+        try: 
+            version = arcpy.GetInstallInfo()['Version']
+            python_version = f"python {'.'.join(map(str, sys.version_info[:2]))}"
+            metrics.set_host_app("ArcGIS", ', '.join([f"ArcGIS {version}", python_version])) 
+        except: 
+            metrics.set_host_app("ArcGIS")
+        # https://pro.arcgis.com/en/pro-app/2.8/arcpy/mapping/alphabeticallistofclasses.htm#except: print("something happened")
+
+class Speckle:
+    def __init__(self):  
+        print("__________________INIT SPECKLE TOOL_________")
+        self.label       = "Speckle"
+        self.description = "Allows you to send and receive your layers " + \
+                           "to/from other software using Speckle server." 
+
+        self.toRefresh = False
+        self.speckleInputs = None
+        self.toolboxInputs = None
+
+        total = len(speckleInputsClass.instances)
+        print(total)
+        for i in range(total):
+            if speckleInputsClass.instances[total-i-1] is not None: 
+                try: 
+                    y = speckleInputsClass.instances[total-i-1].streams_default 
+                    #if not isinstance(speckleInputsClass.instances[total-i-1].streams_default, SpeckleException): # also will throw exception in case not initialized properly 
+                    self.speckleInputs = speckleInputsClass.instances[total-i-1] # take latest (first in reverted list)
+                    break
+                except: pass
+        if self.speckleInputs is None or isinstance(self.speckleInputs.streams_default, SpeckleException): self.speckleInputs = speckleInputsClass()
+        #print(self.speckleInputs.streams_default)
+        print(len(speckleInputsClass.instances))
+
+        total = len(toolboxInputsClass.instances)
+
+        for i in range(total):
+            if toolboxInputsClass.instances[total-i-1] is not None: 
+                self.toolboxInputs = toolboxInputsClass.instances[total-i-1] # take latest (first in reverted list)
+                break
+        if self.toolboxInputs is None: self.toolboxInputs = toolboxInputsClass()
+
+        #print(self.speckleInputs.accounts)
+        if len(self.speckleInputs.accounts) == 0:
+            arcpy.AddError("Speckle accounts not found")
+
+    def getParameterInfo(self):
+        #data types: https://pro.arcgis.com/en/pro-app/2.8/arcpy/geoprocessing_and_python/defining-parameter-data-types-in-a-python-toolbox.htm
+        # parameter details: https://pro.arcgis.com/en/pro-app/latest/arcpy/geoprocessing_and_python/customizing-tool-behavior-in-a-python-toolbox.htm
+        print("Get parameter values")
+        cat1 = "Add Streams"
+        cat2 = "Send/Receive"
+        cat3 = "Create custom Spatial Reference"
+
+        streamsDefalut = arcpy.Parameter(
+            displayName="Add stream from default account",
+            name="streamsDefalut",
+            datatype="GPString",
+            parameterType="Optional",
+            direction="Input",
+            category=cat1
+            )
+        streamsDefalut.filter.type = 'ValueList'
+        if isinstance(self.speckleInputs.streams_default, SpeckleException):
+            arcpy.AddError("Speckle account not accessible")
+            streamsDefalut.filter.list = []
+        elif self.speckleInputs.streams_default is not None:
+            streamsDefalut.filter.list = [ (str(st.name) + " - " + str(st.id)) for st in self.speckleInputs.streams_default ]
+        else:
+            streamsDefalut.filter.list = []
+            arcpy.AddError("Error connecting to default Speckle account")
+        
+        addDefStreams = arcpy.Parameter(
+            displayName="Add",
+            name="addDefStreams",
+            datatype="GPBoolean",
+            parameterType="Optional",
+            direction="Input",
+            category=cat1
+            )
+        addDefStreams.value = False 
+
+        streamUrl = arcpy.Parameter(
+            displayName="Add stream by URL",
+            name="streamUrl",
+            datatype="GPString",
+            parameterType="Optional",
+            direction="Input",
+            category=cat1
+            )
+        streamUrl.value = ""
+
+        addUrlStreams = arcpy.Parameter(
+            displayName="Add",
+            name="addUrlStreams",
+            datatype="GPBoolean",
+            parameterType="Optional",
+            direction="Input",
+            category=cat1
+            )
+        addUrlStreams.value = False 
+
+        ############################################################################
+        
+        lat = arcpy.Parameter(
+            displayName="Origin point LAT",
+            name="lat",
+            datatype="GPString",
+            parameterType="Optional",
+            direction="Input",
+            category=cat3
+            )
+        lat.value = str(self.toolboxInputs.lat)
+
+        lon = arcpy.Parameter(
+            displayName="Origin point LON",
+            name="lon",
+            datatype="GPString",
+            parameterType="Optional",
+            direction="Input",
+            category=cat3
+            )
+        lon.value = str(self.toolboxInputs.lon)
+        
+        setLatLon = arcpy.Parameter(
+            displayName="Create and apply",
+            name="setLatLon",
+            datatype="GPBoolean",
+            parameterType="Optional",
+            direction="Input",
+            category=cat3
+            )
+        setLatLon.value = False
+        
+        ####################################################################################################
+
+        savedStreams = arcpy.Parameter(
+            displayName="Select Stream",
+            name="savedStreams",
+            datatype="GPString",
+            parameterType="Required",
+            direction="Input",
+            multiValue=False,
+            )
+        savedStreams.filter.list = [f"Stream not accessible - {stream[0].stream_id}" if stream[1] is None or isinstance(stream[1], SpeckleException) else f"{stream[1].name} - {stream[1].id}" for i,stream in enumerate(self.speckleInputs.saved_streams)] 
+
+        removeStream = arcpy.Parameter(
+            displayName="Remove",
+            name="removeStream",
+            datatype="GPBoolean",
+            parameterType="Optional",
+            direction="Input"
+            )
+        removeStream.value = False 
+
+        branch = arcpy.Parameter(
+            displayName="Branch",
+            name="branch",
+            datatype="GPString",
+            parameterType="Required",
+            direction="Input",
+            )
+        branch.value = ""
+        branch.filter.type = 'ValueList'
+
+        commit = arcpy.Parameter(
+            displayName="Commit",
+            name="commit",
+            datatype="GPString",
+            parameterType="Optional",
+            direction="Input",
+            )
+        commit.value = ""
+        commit.filter.type = 'ValueList'
+
+        msg = arcpy.Parameter(
+            displayName="Message",
+            name="msg",
+            datatype="GPString",
+            parameterType="Optional",
+            direction="Input",
+            multiValue=False,
+            )
+        msg.value = ""
+
+        selectedLayers = arcpy.Parameter(
+            displayName="Selected Layers",
+            name="selectedLayers",
+            datatype="GPString",
+            parameterType="Optional",
+            direction="Input",
+            multiValue=True,
+            )
+        selectedLayers.filter.list = [str(i) + "-" + l.longName for i,l in enumerate(self.speckleInputs.all_layers)] #"Polyline"
+
+        
+        action = arcpy.Parameter(
+            displayName="",
+            name="action",
+            datatype="GPString",
+            parameterType="Required",
+            direction="Input",
+            multiValue=False,
+            )
+        action.value = "Send" 
+        action.filter.list = ["Send", "Receive"]  
+
+        refresh = arcpy.Parameter(
+            displayName="Refresh",
+            name="refresh",
+            datatype="GPBoolean",
+            parameterType="Optional",
+            direction="Input"
+            ) 
+        refresh.value = False 
+
+        parameters = [streamsDefalut, addDefStreams, streamUrl, addUrlStreams, lat, lon, setLatLon, savedStreams, removeStream, branch, commit, selectedLayers, msg, action, refresh]
+        return parameters
+
+    def isLicensed(self): #optional
+        return True
+
+    def updateParameters(self, parameters: List, toRefresh = False): #optional
+        print("UPDATING PARAMETERS")
+        
+        for i, par in enumerate(parameters): 
+            
+
+            if par.name == "addDefStreams" and par.altered and par.value == True: 
+                for p in parameters:
+                    if p.name == "streamsDefalut" and p.valueAsText is not None:
+
+                        # add value from streamsDefault to saved streams
+                        selected_stream_name = p.valueAsText[:]
+
+                        for stream in self.speckleInputs.streams_default:
+                            if stream.name == selected_stream_name.split(" - ")[0]:
+                                print("_____Add from list___")
+                                wr = StreamWrapper(f"{self.speckleInputs.account.serverInfo.url}/streams/{stream.id}?u={self.speckleInputs.account.userInfo.id}")
+                                self.toolboxInputs.setProjectStreams(wr)
+                                
+                                for p_saved in parameters:
+                                    if p_saved.name == "savedStreams": 
+                                        saved_streams = self.speckleInputs.getProjectStreams()
+                                        self.speckleInputs.saved_streams = saved_streams
+                                        p_saved.filter.list = [f"Stream not accessible - {stream[0].stream_id}" if stream[1] is None or isinstance(stream[1], SpeckleException) else f"{stream[1].name} - {stream[1].id}" for i,stream in enumerate(saved_streams)] 
+                                        if len(p_saved.filter.list)>0: print(p_saved.filter.list); p_saved.value = p_saved.filter.list[0]
+                                break
+                        p.value = None
+                par.value = False
+            
+            if par.name == "addUrlStreams" and par.altered and par.value == True: 
+                for p in parameters:
+                    if p.name == "streamUrl" and p.valueAsText is not None:
+
+                        # add value from streamsDefault to saved streams
+                        query = p.valueAsText[:]
+                        if "http" in query and len(query.split("/")) >= 3: # URL
+                            steamId = query
+                            try: steamId = query.split("/streams/")[1].split("/")[0] 
+                            except: pass
+
+                            # query stream, add to saved
+                            stream = self.speckleInputs.speckle_client.stream.get(id = steamId, branch_limit = 100, commit_limit = 100)
+                            if isinstance(stream, Stream): 
+                                print("_____Add by URL___")
+                                wr = StreamWrapper(f"{self.speckleInputs.account.serverInfo.url}/streams/{stream.id}?u={self.speckleInputs.account.userInfo.id}")
+                                self.toolboxInputs.setProjectStreams(wr)
+                                
+                                for p_saved in parameters:
+                                    if p_saved.name == "savedStreams": 
+                                        saved_streams = self.speckleInputs.getProjectStreams()
+                                        self.speckleInputs.saved_streams = saved_streams
+                                        p_saved.filter.list = [f"Stream not accessible - {st[0].stream_id}" if st[1] is None or isinstance(st[1], SpeckleException) else f"{st[1].name} - {st[1].id}" for i,st in enumerate(saved_streams)] 
+                                        if len(p_saved.filter.list)>0: print(p_saved.filter.list); p_saved.value = p_saved.filter.list[0]
+                            else: pass
+
+                        p.value = None
+                        break
+                par.value = False
+
+
+            if par.name == "removeStream" and par.altered and par.value == True: 
+                for p in parameters:
+                    if p.name == "savedStreams" and p.valueAsText is not None:
+
+                        # get value from savedStreams 
+                        selected_stream_name = p.valueAsText[:]
+                        for streamTup in self.speckleInputs.saved_streams:
+                            stream = streamTup[1]
+                            if stream.name == selected_stream_name.split(" - ")[0]:
+                                print("_____Remove stream___")
+                                wr = StreamWrapper(f"{self.speckleInputs.account.serverInfo.url}/streams/{stream.id}?u={self.speckleInputs.account.userInfo.id}")
+                                self.toolboxInputs.setProjectStreams(wr, False)
+                                
+                                for p_saved in parameters:
+                                    if p_saved.name == "savedStreams": 
+                                        saved_streams = self.speckleInputs.getProjectStreams()
+                                        self.speckleInputs.saved_streams = saved_streams
+                                        p_saved.filter.list = [f"Stream not accessible - {st[0].stream_id}" if st[1] is None or isinstance(st[1], SpeckleException) else f"{st[1].name} - {st[1].id}" for i,st in enumerate(saved_streams)] 
+                                        p_saved.value = None
+                                break
+                        p.value = None
+                par.value = False
+
+            ####################################################################### 
+            if par.name == "setLatLon" and par.altered and par.value == True: 
+                lat = lon = 0
+                for p in parameters:
+                    if p.name == "lat" and p.valueAsText is not None:
+                        # add value from the UI to saved lat
+                        lat = p.valueAsText[:].replace(",","").replace(" ","").replace(";","").replace("_","")
+                        try: lat = float(lat)
+                        except: lat = 0; p.value = "0.0"
+
+                    if p.name == "lon" and p.valueAsText is not None:
+                        # add value from the UI to saved lat
+                        lon = p.valueAsText[:].replace(",","").replace(" ","").replace(";","").replace("_","")
+                        try: lon = float(lon)
+                        except: lon = 0; p.value = "0.0"
+                    coords = [lat, lon]
+                    self.toolboxInputs.set_survey_point(coords)
+                par.value = False
+            
+            ####################################################################### 
+
+            if par.name == "savedStreams" and par.altered:
+                # Search for the stream by name
+                if par.value is not None and "Stream not accessible" not in par.valueAsText[:]:
+                    selected_stream_name = par.valueAsText[:]
+                    self.toolboxInputs.active_stream = None
+                    self.toolboxInputs.active_stream_wrapper = None
+                    for st in self.speckleInputs.saved_streams:
+                        if st[1].name == selected_stream_name.split(" - ")[0]: 
+                            self.toolboxInputs.active_stream = st[1]
+                            self.toolboxInputs.active_stream_wrapper = st[0]
+                            break
+
+                    # edit branches: globals and UI 
+                    branch_list = [branch.name for branch in self.toolboxInputs.active_stream.branches.items]
+                    for p in parameters:
+                        if p.name == "branch":
+                            p.filter.list = branch_list
+
+                            if p.valueAsText not in branch_list: 
+                                p.value = "main"
+                            for b in self.toolboxInputs.active_stream.branches.items:
+                                if b.name == p.value: 
+                                    self.toolboxInputs.active_branch = b
+                                    break 
+                    
+                    # setting commit value and list 
+                    for p in parameters:
+                        if p.name == "commit":
+                            try: 
+                                p.filter.list = [f"{commit.id}"+ " - " + f"{commit.message}" for commit in self.toolboxInputs.active_branch.commits.items]
+                                if p.valueAsText not in p.filter.list:
+                                    p.value = self.toolboxInputs.active_branch.commits.items[0].id + " - " + self.toolboxInputs.active_branch.commits.items[0].message 
+                                    self.toolboxInputs.active_commit = self.toolboxInputs.active_branch.commits.items[0] 
+                            except: 
+                                p.filter.list = []
+                                p.value = None
+                                self.toolboxInputs.active_commit = None 
+                else: par.value = None
+
+            if par.name == "branch" and par.altered: # branches
+                if par.value is not None:
+                    selected_branch_name = par.valueAsText[:]
+                    self.toolboxInputs.active_branch = None
+                    if self.toolboxInputs.active_stream is not None:
+                        for br in self.toolboxInputs.active_stream.branches.items:
+                            if br.name == selected_branch_name:
+                                self.toolboxInputs.active_branch = br
+                                break
+                # edit commit values 
+                if self.toolboxInputs.active_branch is not None: 
+                    for p in parameters:
+                        if p.name == "commit":
+                            try: 
+                                p.filter.list = [f"{commit.id}"+ " - " + f"{commit.message}" for commit in self.toolboxInputs.active_branch.commits.items]
+                                if p.valueAsText not in p.filter.list:
+                                    p.value = self.toolboxInputs.active_branch.commits.items[0].id + " - " + self.toolboxInputs.active_branch.commits.items[0].message 
+                                    self.toolboxInputs.active_commit = self.toolboxInputs.active_branch.commits.items[0]
+                            except: 
+                                p.filter.list = []
+                                p.value = None
+                                self.toolboxInputs.active_commit = None 
+
+            if par.name == "commit" and par.altered: # commits
+                if par.value is not None:
+                    selected_commit_id = par.valueAsText[:].split(" - ")[0]
+                    self.toolboxInputs.active_commit = None
+                    if self.toolboxInputs.active_branch is not None:
+                        for c in self.toolboxInputs.active_branch.commits.items:
+                            if c.id == selected_commit_id: 
+                                self.toolboxInputs.active_commit = c 
+                                break
+            
+            if par.name == "selectedLayers" and par.altered: # selected layers
+                if par.value is not None:
+                    self.toolboxInputs.selected_layers = par.values
+
+            if par.name == "msg" and par.altered and par.valueAsText is not None:
+                self.toolboxInputs.messageSpeckle = par.valueAsText
+                print(self.toolboxInputs.messageSpeckle)
+
+            if par.name == "action" and par.altered:
+                if par.valueAsText == "Send": self.toolboxInputs.action = 1
+                else: self.toolboxInputs.action = 0
+
+            if par.name == "refresh" and par.altered: # refresh btn
+                if par.value == True: 
+                    self.refresh(parameters) 
+            if self.toRefresh == True:
+                self.refresh(parameters) 
+                self.toRefresh = False
+        
+        print("____________________________parameters___________________________")
+        return 
+    
+    def refresh(self, parameters: List[Any]): 
+        print("Refresh______")
+        self.speckleInputs: speckleInputsClass = speckleInputsClass()
+        self.toolboxInputs: toolboxInputsClass = toolboxInputsClass()
+        
+        for par in parameters: 
+            if par.name == "streamUrl": par.value = None
+            if par.name == "streamsDefalut": par.value = None
+            if par.name == "savedStreams": par.value = None
+            if par.name == "branch": par.value = ""; par.filter.list = []
+            if par.name == "commit": par.value = None; par.filter.list = []
+            if par.name == "selectedLayers": par.value = None
+            if par.name == "msg": par.value = ""
+            if par.name == "action": par.value = "Send"
+            if par.name == "refresh": par.value = False
+            
+            if par.name == "lat": par.value = str(self.toolboxInputs.get_survey_point()[0])
+            if par.name == "lon": par.value = str(self.toolboxInputs.get_survey_point()[1])
+            if par.name == "streamsDefalut": 
+                if isinstance(self.speckleInputs.streams_default, SpeckleException):
+                    arcpy.AddError("Speckle account not accessible")
+                    par.filter.list = []
+                else:
+                    par.filter.list = [ (st.name + " - " + st.id) for st in self.speckleInputs.streams_default ]
+            if par.name == "savedStreams": 
+                saved_streams = self.speckleInputs.getProjectStreams()
+                par.filter.list = [f"Stream not accessible - {stream[0].stream_id}" if stream[1] is None or isinstance(stream[1], SpeckleException) else f"{stream[1].name} - {stream[1].id}" for i,stream in enumerate(saved_streams)] 
+            if par.name == "selectedLayers": par.filter.list = [str(i) + "-" + l.longName for i,l in enumerate(self.speckleInputs.all_layers)]
+        
+        return parameters
+
+    def updateMessages(self, parameters): #optional
+        return
+
+    def execute(self, parameters: List, messages): 
+        # https://pro.arcgis.com/en/pro-app/latest/arcpy/get-started/what-is-arcpy-.htm
+        #Warning if any of the fields is invalid/empty 
+        print("___________________________Run___________________________")
+        check = self.validateStreamBranch(parameters) # apparently pdate needed to assign proper self.values
+        
+        print(self.toolboxInputs.selected_layers)
+        print(self.toolboxInputs.action)
+        
+        if self.toolboxInputs.action == 1 and check is True: self.onSend(parameters)
+        elif self.toolboxInputs.action == 0 and check is True: self.onReceive(parameters)
+        print("__________________________Run_end___________________________")
+
+    def validateStreamBranch(self, parameters: List):
+        	
+        self.updateParameters(parameters)
+        if self.toolboxInputs.active_stream is None:
+            arcpy.AddError("Choose a valid stream")
+            return False
+        if self.toolboxInputs.active_branch is None: 
+            arcpy.AddError("Choose a valid branch")
+            return False
+        return True 
+
+    def onSend(self, parameters: List):
+
+        print("______________SEND_______________")
+
+        if len(self.toolboxInputs.selected_layers) == 0: 
+            arcpy.AddError("No layers selected for sending")
+            return
+
+        streamId = self.toolboxInputs.active_stream.id #stream_id
+        client = self.toolboxInputs.active_stream_wrapper.get_client()
+        #client = self.speckleInputs.speckle_client # ?
+        
+        # Get the stream wrapper
+        #streamWrapper = StreamWrapper(None)
+        #client = streamWrapper.get_client()
+        # Ensure the stream actually exists
+        #try:
+        #    client.stream.get(streamId)
+        #except SpeckleException as error:
+        #    print(str(error))
+        #    return
+
+        # next create a server transport - this is the vehicle through which you will send and receive
+        transport = ServerTransport(client=client, stream_id=streamId)
+
+        ##################################### conversions ################################################
+        base_obj = Base(units = "m")
+        base_obj.layers = convertSelectedLayers(self.speckleInputs.all_layers, self.toolboxInputs.selected_layers, self.speckleInputs.project)
+        
+        if len(base_obj.layers) == 0: 
+            arcpy.AddMessage("No data sent to stream " + streamId)
+            return 
+        try:
+            # this serialises the block and sends it to the transport
+            objId = operations.send(base=base_obj, transports=[transport])
+        except SpeckleException as error:
+            arcpy.AddError("Error sending data")
+            #print("Error sending data")
+            return
+        except SpeckleWarning as warning:
+            arcpy.AddMessage("SpeckleWarning: " + str(warning.args[0]))
+            
+
+        message = self.toolboxInputs.messageSpeckle
+        print(message)
+        if message is None or ( isinstance(message, str) and len(message) == 0):  message = "Sent from ArcGIS"
+        print(message)
+        try:
+            # you can now create a commit on your stream with this object
+            client.commit.create(
+                stream_id=streamId,
+                object_id=objId,
+                branch_name=self.toolboxInputs.active_branch.name,
+                message=message,
+                source_application="ArcGIS",
+            )
+            arcpy.AddMessage("Successfully sent data to stream: " + streamId)
+        except:
+            arcpy.AddError("Error creating commit")
+
+    def onReceive(self, parameters: List[Any]): 
+        
+        print("______________RECEIVE_______________")
+
+        #if self.validateStreamBranch(parameters) == False: return
+
+        try: 
+            streamId = self.toolboxInputs.active_stream.id #stream_id
+            client = self.toolboxInputs.active_stream_wrapper.get_client()
+            #client = self.speckleInputs.speckle_client # 
+        except SpeckleWarning as warning: 
+            arcpy.AddWarning(str(warning.args[0]))
+
+        # get commit 
+        commit = None 
+        try: 
+            #commit = self.toolboxInputs.active_branch.commits.items[0]
+            commit = self.toolboxInputs.active_commit
+            commitId = commit.id # text to make sure commit exists 
+        except: 
+            try: 
+                commit = self.toolboxInputs.active_branch.commits.items[0]
+                commitId = commit.id 
+                arcpy.AddWarning("Failed to find a commit. Getting the last commit of the branch")
+            except:
+                arcpy.AddError("Failed to find a commit")
+                return
+
+        # next create a server transport - this is the vehicle through which you will send and receive
+        try: 
+            transport = ServerTransport(client=client, stream_id=streamId)
+            
+            client.commit.received(
+            streamId,
+            commit.id,
+            source_application="ArcGIS",
+            message="Received commit in ArcGIS",
+            )
+        except: 
+            arcpy.AddError("Make sure your account has access to the chosen stream")
+            return
+        try:
+            #print(commit)
+            objId = commit.referencedObject
+            commitDetailed = client.commit.get(streamId, commit.id)
+            if isinstance(commitDetailed, GraphQLException): 
+                arcpy.AddError("Access error")
+                return
+            app = commitDetailed.sourceApplication
+            if objId is None:
+                return
+            commitObj = operations.receive(objId, transport, None)
+            
+            if app != "QGIS" and app != "ArcGIS": 
+                if self.speckleInputs.project.activeMap.spatialReference.type == "Geographic" or self.speckleInputs.project.activeMap.spatialReference is None: #TODO test with invalid CRS
+                    arcpy.AddMessage("It is advisable to set the project Spatial reference to Projected type before receiving CAD geometry (e.g. EPSG:32631), or create a custom one from geographic coordinates")
+                    print("It is advisable to set the project Spatial reference to Projected type before receiving CAD geometry (e.g. EPSG:32631), or create a custom one from geographic coordinates")
+            print(f"Successfully received {objId}")
+
+            # Clear 'latest' group
+            streamBranch = streamId + "_" + self.toolboxInputs.active_branch.name + "_" + str(commit.id)
+            streamBranch = streamBranch.replace("[","_").replace("]","_").replace(" ","_").replace("-","_").replace("(","_").replace(")","_").replace(":","_").replace("\\","_").replace("/","_").replace("\"","_").replace("&","_").replace("@","_").replace("$","_").replace("%","_").replace("^","_")
+    
+            newGroupName = f'{streamBranch}'
+            
+            groupExists = 0
+            print(newGroupName)
+            for l in self.speckleInputs.project.activeMap.listLayers(): 
+                #print(l.longName)
+                if l.longName.startswith(newGroupName + "\\"):
+                    #print(l.longName)
+                    self.speckleInputs.project.activeMap.removeLayer(l)
+                    groupExists+=1
+                elif l.longName == newGroupName: 
+                    groupExists+=1
+            print(newGroupName)
+            if groupExists == 0:
+                # create empty group layer file 
+                path = self.speckleInputs.project.filePath.replace("aprx","gdb") #"\\".join(self.toolboxInputs.project.filePath.split("\\")[:-1]) + "\\speckle_layers\\"
+                print(path)
+                try:
+                    f = open(path + "\\" + newGroupName + ".lyrx", "w")
+                    content = createGroupLayer().replace("TestGroupLayer", newGroupName)
+                    f.write(content)
+                    f.close()
+                    newGroupLayer = arcpy.mp.LayerFile(path + "\\" + newGroupName + ".lyrx")
+                    layerGroup = self.speckleInputs.project.activeMap.addLayer(newGroupLayer)[0]
+                except: # for 3.0.0
+                    if self.speckleInputs.active_map is not None:
+                        layerGroup = self.speckleInputs.active_map.createGroupLayer(newGroupName)
+                    else:
+                        arcpy.AddWarning("The map didn't fully load, try refreshing the plugin.")
+                        return
+
+                print(layerGroup)
+                print("layer added")
+                layerGroup.name = newGroupName
+                print(newGroupName)
+
+            if app == "QGIS" or app == "ArcGIS": check: Callable[[Base], bool] = lambda base: isinstance(base, Layer) or isinstance(base, VectorLayer) or isinstance(base, RasterLayer)
+            else: check: Callable[[Base], bool] = lambda base: isinstance(base, Base)
+
+            def callback(base: Base) -> bool:
+                print("callback")
+                #print(base)
+                if isinstance(base, Layer) or isinstance(base, VectorLayer) or isinstance(base, RasterLayer):
+                    layer = layerToNative(base, streamBranch, self.speckleInputs.project)
+                    if layer is not None:
+                        print("Layer created: " + layer.name)
+                else:
+                    loopObj(base, "")
+                return True
+            
+            def loopObj(base: Base, baseName: str):
+                memberNames = base.get_member_names()
+                for name in memberNames:
+                    if name in ["id", "applicationId", "units", "speckle_type"]: continue
+                    try: loopVal(base[name], baseName + "/" + name) # loop properties not included above
+                    except: pass
+
+            def loopVal(value: Any, name: str): # "name" is the parent object/property/layer name
+                if name.endswith('/displayValue'): return 
+                
+                if isinstance(value, Base): 
+                    try: # dont go through parts of Speckle Geometry object
+                        print("objects to loop through: " + value.speckle_type)
+                        if value.speckle_type.startswith("Objects.Geometry."): pass #.Brep") or value.speckle_type.startswith("Objects.Geometry.Mesh") or value.speckle_type.startswith("Objects.Geometry.Surface") or value.speckle_type.startswith("Objects.Geometry.Extrusion"): pass
+                        else: loopObj(value, name)
+                    except: loopObj(value, name)
+
+                if isinstance(value, List):
+                    for item in value:
+                        loopVal(item, name)
+                        
+                        print(item)
+                        pt = None
+                        if item.speckle_type and item.speckle_type.startswith("Objects.Geometry."): 
+
+                            pt, pl = cadLayerToNative(value, name, streamBranch, self.speckleInputs.project)
+                            if pt is not None: print("Layer group created: " + pt.name())
+                            if pl is not None: print("Layer group created: " + pl.name())
+                            break
+                        
+                        if item.speckle_type and (item.speckle_type.startswith("Objects.BuiltElements.") or item.speckle_type.startswith("Objects.Structural.Geometry")): #  and "Revit" in item.speckle_type
+                            print("__receiving structures__")
+                            msh_bool = bimLayerToNative(value, name, streamBranch, self.speckleInputs.project)
+                            #if msh is not None: print("Layer group created: " + msh.name())
+                            break
+                
+            traverseObject(commitObj, callback, check)
+      
+        except (SpeckleException, GraphQLException) as e:
+            print("Receive failed: " + str(e))
+            arcpy.AddError("Receive failed: " + str(e))
+            return
+        
+        print("received")
+        #self.updateParameters(parameters, True)
+        #self.refresh(parameters)
+    
+    
+#__all__ = ["Toolbox", "Speckle"]

speckle_toolbox/esri/toolboxes/speckle/ui/add_stream_modal.py

@@ -0,0 +1,161 @@
+import os
+from typing import List, Union
+#import ui.speckle_qgis_dialog
+
+from PyQt5 import QtWidgets, uic, QtCore
+from PyQt5.QtCore import pyqtSignal
+from specklepy.api.models import Stream
+from specklepy.api.client import SpeckleClient
+from specklepy.logging.exceptions import SpeckleException
+
+from specklepy.api.credentials import get_local_accounts #, StreamWrapper
+from specklepy.api.wrapper import StreamWrapper
+from gql import gql
+
+try:
+    from speckle.plugin_utils.logger import logToUser
+except:
+    from speckle_toolbox.esri.toolboxes.speckle.plugin_utils.logger import logToUser
+
+import arcpy
+
+# This loads your .ui file so that PyQt can populate your plugin with the elements from Qt Designer
+ui_class = os.path.dirname(os.path.abspath(__file__)) + "/add_stream_modal.ui"
+
+class AddStreamModalDialog(QtWidgets.QWidget):
+
+    search_button: QtWidgets.QPushButton = None
+    search_text_field: QtWidgets.QLineEdit = None
+    search_results_list: QtWidgets.QListWidget = None
+    dialog_button_box: QtWidgets.QDialogButtonBox = None
+    accounts_dropdown: QtWidgets.QComboBox
+
+    stream_results: List[Stream] = []
+    speckle_client: Union[SpeckleClient, None] = None
+
+    #Events
+    handleStreamAdd = pyqtSignal(StreamWrapper)
+
+    def __init__(self, parent=None, speckle_client: SpeckleClient = None):
+        super(AddStreamModalDialog,self).__init__(parent,QtCore.Qt.WindowStaysOnTopHint)
+        uic.loadUi(ui_class, self) # Load the .ui file
+        self.show()
+        try:
+        
+            self.speckle_client = speckle_client
+            
+            self.setWindowTitle("Add Speckle stream")
+
+            self.dialog_button_box.button(QtWidgets.QDialogButtonBox.Ok).setEnabled(False) 
+
+            self.search_button.clicked.connect(self.onSearchClicked)
+            self.search_results_list.currentItemChanged.connect( self.searchResultChanged )
+            self.dialog_button_box.button(QtWidgets.QDialogButtonBox.Ok).clicked.connect(self.onOkClicked)
+            self.dialog_button_box.button(QtWidgets.QDialogButtonBox.Cancel).clicked.connect(self.onCancelClicked)
+            self.accounts_dropdown.currentIndexChanged.connect(self.onAccountSelected)
+            self.populate_accounts_dropdown()
+        except Exception as e:
+            logToUser(e)
+
+    def searchResultChanged(self):
+        try:
+            index = self.search_results_list.currentIndex().row()
+            if index == -1: self.dialog_button_box.button(QtWidgets.QDialogButtonBox.Ok).setEnabled(False) 
+            else: self.dialog_button_box.button(QtWidgets.QDialogButtonBox.Ok).setEnabled(True) 
+        except Exception as e: 
+            logToUser(str(e)) 
+
+    def onSearchClicked(self):
+        try:
+            query = self.search_text_field.text()
+            sw = None 
+            results = []
+            if "http" in query and len(query.split("/")) >= 3: # URL
+                sw = StreamWrapper(query)
+                stream = sw.get_client().stream.get(sw.stream_id)
+                if isinstance(stream, Stream): results = [stream]
+                else: results = []
+            
+            elif self.speckle_client is not None: 
+                results = self.speckle_client.stream.search(query)
+            elif self.speckle_client is None: 
+                logToUser(f"Account cannot be authenticated: {self.accounts_dropdown.currentText()}", 2) 
+            
+            self.stream_results = results
+            self.populateResultsList(sw)
+            
+        except Exception as e: 
+            logToUser(str(e)) 
+    
+    def populateResultsList(self, sw):
+        try:
+            self.search_results_list.clear()
+            if isinstance(self.stream_results, SpeckleException): 
+                logToUser("Some streams cannot be accessed", 1)
+                return 
+            for stream in self.stream_results:
+                host = ""
+                if sw is not None:
+                    host = sw.get_account().serverInfo.url
+                else: 
+                    host = self.speckle_client.account.serverInfo.url
+                
+                if isinstance(stream, SpeckleException): 
+                    logToUser("Some streams cannot be accessed", 1)
+                else: 
+                    self.search_results_list.addItems([
+                        f"{stream.name}, {stream.id} | {host}" #for stream in self.stream_results 
+                    ])
+        
+        except Exception as e: 
+            logToUser(str(e)) 
+
+    def onOkClicked(self):
+        try:
+            if isinstance(self.stream_results, SpeckleException):
+                logToUser("Selected stream cannot be accessed", 1)
+                return
+            #elif index == -1 or len(self.stream_results) == 0:
+            #    logger.logToUser("Select stream from \"Search Results\". No stream selected", Qgis.Warning)
+            #    return 
+            else:
+                try:
+                    index = self.search_results_list.currentIndex().row()
+                    stream = self.stream_results[index]
+                    item = self.search_results_list.item(index)
+                    url = item.text().split(" | ")[1] + "/streams/" + item.text().split(", ")[1].split(" | ")[0]
+                    sw = StreamWrapper(url) 
+                    #acc = sw.get_account() #get_local_accounts()[self.accounts_dropdown.currentIndex()]
+                    self.handleStreamAdd.emit(sw) #StreamWrapper(f"{acc.serverInfo.url}/streams/{stream.id}?u={acc.userInfo.id}"))
+                    self.close()
+                except Exception as e:
+                    logToUser("Some streams cannot be accessed: " + str(e), 1)
+                    return 
+        except Exception as e: 
+            logToUser(str(e)) 
+
+    def onCancelClicked(self):
+        self.close()
+
+    def onAccountSelected(self, index):
+        try:
+            account = self.speckle_accounts[index]
+            self.speckle_client = SpeckleClient(account.serverInfo.url, account.serverInfo.url.startswith("https"))
+            self.speckle_client.authenticate_with_token(token=account.token)
+        except Exception as e: 
+            logToUser(str(e)) 
+
+    def populate_accounts_dropdown(self):
+        # Populate the accounts comboBox
+        try:
+            self.speckle_accounts = get_local_accounts()
+            self.accounts_dropdown.clear()
+            self.accounts_dropdown.addItems(
+                [
+                    f"{acc.userInfo.name}, {acc.userInfo.email} | {acc.serverInfo.url}"
+                    for acc in self.speckle_accounts
+                ]
+            )
+        except Exception as e: 
+            logToUser(str(e)) 
+

speckle_toolbox/esri/toolboxes/speckle/ui/add_stream_modal.ui

@@ -0,0 +1,87 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>AddStreamDialog</class>
+ <widget class="QWidget" name="AddStreamDialog">
+  <property name="windowModality">
+   <enum>Qt::NonModal</enum>
+  </property>
+  <property name="sizePolicy">
+   <sizepolicy hsizetype="MinimumExpanding" vsizetype="MinimumExpanding">
+    <horstretch>0</horstretch>
+    <verstretch>0</verstretch>
+   </sizepolicy>
+  </property>
+  <property name="windowTitle">
+   <string>Form</string>
+  </property>
+  <layout class="QGridLayout" name="gridLayout_2">
+   <item row="0" column="0">
+    <layout class="QVBoxLayout" name="verticalLayout">
+     <property name="sizeConstraint">
+      <enum>QLayout::SetNoConstraint</enum>
+     </property>
+     <item>
+      <layout class="QFormLayout" name="search_form">
+       <item row="1" column="0">
+        <widget class="QLabel" name="search_label">
+         <property name="text">
+          <string>Search Stream by name or URL</string>
+         </property>
+        </widget>
+       </item>
+       <item row="1" column="1">
+        <widget class="QLineEdit" name="search_text_field"/>
+       </item>
+       <item row="2" column="1">
+        <widget class="QPushButton" name="search_button">
+         <property name="text">
+          <string>Search</string>
+         </property>
+        </widget>
+       </item>
+       <item row="0" column="1">
+        <widget class="QComboBox" name="accounts_dropdown"/>
+       </item>
+       <item row="0" column="0">
+        <widget class="QLabel" name="accounts_label">
+         <property name="text">
+          <string>Account</string>
+         </property>
+        </widget>
+       </item>
+      </layout>
+     </item>
+     <item>
+      <widget class="QLabel" name="search_results_label">
+       <property name="text">
+        <string>Search Results</string>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <widget class="QListWidget" name="search_results_list">
+       <property name="minimumSize">
+        <size>
+         <width>0</width>
+         <height>100</height>
+        </size>
+       </property>
+       <property name="sizeAdjustPolicy">
+        <enum>QAbstractScrollArea::AdjustToContents</enum>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <widget class="QDialogButtonBox" name="dialog_button_box">
+       <property name="standardButtons">
+        <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
+       </property>
+      </widget>
+     </item>
+    </layout>
+   </item>
+  </layout>
+ </widget>
+ <resources/>
+ <connections/>
+</ui>

speckle_toolbox/esri/toolboxes/speckle/ui/create_branch.py

@@ -0,0 +1,81 @@
+import os
+from typing import List, Tuple, Union
+#import ui.speckle_qgis_dialog
+
+from PyQt5 import QtWidgets, uic, QtCore
+from PyQt5.QtCore import pyqtSignal
+from specklepy.api.models import Stream
+from specklepy.api.client import SpeckleClient
+from specklepy.logging.exceptions import SpeckleException
+
+from specklepy.api.credentials import Account, get_local_accounts #, StreamWrapper
+from specklepy.api.wrapper import StreamWrapper
+from gql import gql
+
+import arcpy 
+try:
+    from speckle.plugin_utils.logger import logToUser
+except:
+    from speckle_toolbox.esri.toolboxes.speckle.plugin_utils.logger import logToUser
+
+# This loads your .ui file so that PyQt can populate your plugin with the elements from Qt Designer
+
+ui_class = os.path.dirname(os.path.abspath(__file__)) + "/create_branch.ui"
+
+class CreateBranchModalDialog(QtWidgets.QWidget):
+
+    name_field: QtWidgets.QLineEdit = None
+    description_field: QtWidgets.QLineEdit = None
+    dialog_button_box: QtWidgets.QDialogButtonBox = None
+    speckle_client: Union[SpeckleClient, None] = None
+
+    #Events
+    handleBranchCreate = pyqtSignal(str,str)
+
+    def __init__(self, parent=None, speckle_client: SpeckleClient = None):
+        super(CreateBranchModalDialog,self).__init__(parent,QtCore.Qt.WindowStaysOnTopHint)
+        uic.loadUi(ui_class, self) # Load the .ui file
+        self.show() 
+        try:
+        
+            self.speckle_client = speckle_client
+            
+            self.setWindowTitle("Create New Branch")
+
+            self.name_field.textChanged.connect(self.nameCheck)
+            self.dialog_button_box.button(QtWidgets.QDialogButtonBox.Ok).setEnabled(False) 
+            self.dialog_button_box.button(QtWidgets.QDialogButtonBox.Ok).clicked.connect(self.onOkClicked)
+            self.dialog_button_box.button(QtWidgets.QDialogButtonBox.Cancel).clicked.connect(self.onCancelClicked)
+        except Exception as e:
+            logToUser(e)
+
+    def nameCheck(self):
+        try:
+            if len(self.name_field.text()) >= 3:
+                self.dialog_button_box.button(QtWidgets.QDialogButtonBox.Ok).setEnabled(True) 
+            else: 
+                self.dialog_button_box.button(QtWidgets.QDialogButtonBox.Ok).setEnabled(False) 
+            return
+        except Exception as e: 
+            logToUser(str(e)) 
+
+    def onOkClicked(self):
+        try:
+            name = self.name_field.text()
+            description = self.description_field.text()
+            self.handleBranchCreate.emit(name, description)
+            self.close()
+        except Exception as e:
+            logToUser(str(e))
+            return 
+
+    def onCancelClicked(self):
+        self.close()
+
+    def onAccountSelected(self, index):
+        try:
+            account = self.speckle_accounts[index]
+            self.speckle_client = SpeckleClient(account.serverInfo.url, account.serverInfo.url.startswith("https"))
+            self.speckle_client.authenticate_with_token(token=account.token)
+        except Exception as e: 
+            logToUser(str(e)) 

speckle_toolbox/esri/toolboxes/speckle/ui/create_branch.ui

@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>CreateStreamDialog</class>
+ <widget class="QWidget" name="AddBranchDialog">
+  <property name="windowModality">
+   <enum>Qt::NonModal</enum>
+  </property>
+  <property name="sizePolicy">
+   <sizepolicy hsizetype="MinimumExpanding" vsizetype="MinimumExpanding">
+    <horstretch>0</horstretch>
+    <verstretch>0</verstretch>
+   </sizepolicy>
+  </property>
+  <property name="windowTitle">
+   <string>Form</string>
+  </property>
+  <layout class="QGridLayout" name="gridLayout_2">
+   <item row="0" column="0">
+    <layout class="QVBoxLayout" name="verticalLayout">
+     <property name="sizeConstraint">
+      <enum>QLayout::SetNoConstraint</enum>
+     </property>
+     <item>
+      <layout class="QFormLayout" name="search_form">
+
+       <item row="0" column="0">
+        <widget class="QLabel" name="name_label">
+         <property name="text">
+          <string>Branch Name</string>
+         </property>
+        </widget>
+       </item>
+       <item row="1" column="0">
+        <widget class="QLabel" name="name_label">
+         <property name="text">
+          <string>Description</string>
+         </property>
+        </widget>
+       </item>
+
+       <item row="0" column="1">
+        <widget class="QLineEdit" name="name_field"/>
+       </item>
+       <item row="1" column="1">
+        <widget class="QLineEdit" name="description_field"/>
+       </item>
+
+      </layout>
+     </item>
+     <item>
+      <widget class="QDialogButtonBox" name="dialog_button_box">
+       <property name="standardButtons">
+        <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
+       </property>
+      </widget>
+     </item>
+
+    </layout>
+   </item>
+  </layout>
+ </widget>
+ <resources/>
+ <connections/>
+</ui>

speckle_toolbox/esri/toolboxes/speckle/ui/create_stream.py

@@ -0,0 +1,106 @@
+import os
+from typing import List, Tuple, Union
+#import ui.speckle_qgis_dialog
+
+
+from PyQt5 import QtWidgets, uic, QtCore
+from PyQt5.QtCore import pyqtSignal
+
+import arcpy
+
+from specklepy.api.models import Stream
+from specklepy.api.client import SpeckleClient
+from specklepy.logging.exceptions import SpeckleException
+
+from specklepy.api.credentials import Account, get_local_accounts #, StreamWrapper
+from specklepy.api.wrapper import StreamWrapper
+from gql import gql
+try:
+    from speckle.plugin_utils.logger import logToUser
+except:
+    from speckle_toolbox.esri.toolboxes.speckle.plugin_utils.logger import logToUser
+
+# This loads your .ui file so that PyQt can populate your plugin with the elements from Qt Designer
+
+ui_class = os.path.dirname(os.path.abspath(__file__)) + "/create_stream.ui"
+
+class CreateStreamModalDialog(QtWidgets.QWidget):
+
+    name_field: QtWidgets.QLineEdit = None
+    description_field: QtWidgets.QLineEdit = None
+    dialog_button_box: QtWidgets.QDialogButtonBox = None
+    accounts_dropdown: QtWidgets.QComboBox
+    public_toggle: QtWidgets.QCheckBox
+
+    speckle_client: Union[SpeckleClient, None] = None
+
+    #Events
+    handleStreamCreate = pyqtSignal(Account, str, str, bool)
+
+    def __init__(self, parent=None, speckle_client: SpeckleClient = None):
+        super(CreateStreamModalDialog,self).__init__(parent,QtCore.Qt.WindowStaysOnTopHint)
+        uic.loadUi(ui_class, self) # Load the .ui file
+        self.show() 
+        try:
+        
+            self.speckle_client = speckle_client
+            
+            self.setWindowTitle("Create New Stream")
+
+            self.name_field.textChanged.connect(self.nameCheck)
+            self.dialog_button_box.button(QtWidgets.QDialogButtonBox.Ok).setEnabled(True) 
+            self.dialog_button_box.button(QtWidgets.QDialogButtonBox.Ok).clicked.connect(self.onOkClicked)
+            self.dialog_button_box.button(QtWidgets.QDialogButtonBox.Cancel).clicked.connect(self.onCancelClicked)
+            self.accounts_dropdown.currentIndexChanged.connect(self.onAccountSelected)
+            self.populate_accounts_dropdown()
+        except Exception as e:
+            logToUser(e)
+
+    def nameCheck(self):
+        try:
+            if len(self.name_field.text()) == 0 or len(self.name_field.text()) >= 3:
+                self.dialog_button_box.button(QtWidgets.QDialogButtonBox.Ok).setEnabled(True) 
+            else: 
+                self.dialog_button_box.button(QtWidgets.QDialogButtonBox.Ok).setEnabled(False) 
+            return
+        except Exception as e: 
+            logToUser(str(e)) 
+
+    def onOkClicked(self):
+        try:
+            acc = get_local_accounts()[self.accounts_dropdown.currentIndex()]
+            name = self.name_field.text()
+            description = self.description_field.text()
+            public = self.public_toggle.isChecked()
+            self.handleStreamCreate.emit(acc,name,description,public)
+            self.close()
+        except Exception as e:
+            logToUser(str(e))
+            return 
+
+    def onCancelClicked(self):
+        #self.handleCancelStreamCreate.emit()
+        self.close()
+
+    def onAccountSelected(self, index):
+        try:
+            account = self.speckle_accounts[index]
+            self.speckle_client = SpeckleClient(account.serverInfo.url, account.serverInfo.url.startswith("https"))
+            self.speckle_client.authenticate_with_token(token=account.token)
+        except Exception as e: 
+            logToUser(str(e)) 
+
+    def populate_accounts_dropdown(self):
+        try:
+            # Populate the accounts comboBox
+            self.speckle_accounts = get_local_accounts()
+            self.accounts_dropdown.clear()
+            self.accounts_dropdown.addItems(
+                [
+                    f"{acc.userInfo.name}, {acc.userInfo.email} | {acc.serverInfo.url}"
+                    for acc in self.speckle_accounts
+                ]
+            )
+        except Exception as e: 
+            logToUser(str(e)) 
+

speckle_toolbox/esri/toolboxes/speckle/ui/create_stream.ui

@@ -0,0 +1,85 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>CreateStreamDialog</class>
+ <widget class="QWidget" name="AddStreamDialog">
+  <property name="windowModality">
+   <enum>Qt::NonModal</enum>
+  </property>
+  <property name="sizePolicy">
+   <sizepolicy hsizetype="MinimumExpanding" vsizetype="MinimumExpanding">
+    <horstretch>0</horstretch>
+    <verstretch>0</verstretch>
+   </sizepolicy>
+  </property>
+  <property name="windowTitle">
+   <string>Form</string>
+  </property>
+  <layout class="QGridLayout" name="gridLayout_2">
+   <item row="0" column="0">
+    <layout class="QVBoxLayout" name="verticalLayout">
+     <property name="sizeConstraint">
+      <enum>QLayout::SetNoConstraint</enum>
+     </property>
+     <item>
+      <layout class="QFormLayout" name="search_form">
+       
+       <item row="0" column="0">
+        <widget class="QLabel" name="accounts_label">
+         <property name="text">
+          <string>Account</string>
+         </property>
+        </widget>
+       </item>
+
+       <item row="1" column="0">
+        <widget class="QLabel" name="name_label">
+         <property name="text">
+          <string>Stream Name</string>
+         </property>
+        </widget>
+       </item>
+       <item row="2" column="0">
+        <widget class="QLabel" name="name_label">
+         <property name="text">
+          <string>Description</string>
+         </property>
+        </widget>
+       </item>
+       <item row="3" column="0">
+        <widget class="QLabel" name="name_label">
+         <property name="text">
+          <string>Public</string>
+         </property>
+        </widget>
+       </item>
+
+       <item row="0" column="1">
+        <widget class="QComboBox" name="accounts_dropdown"/>
+       </item>
+       <item row="1" column="1">
+        <widget class="QLineEdit" name="name_field"/>
+       </item>
+       <item row="2" column="1">
+        <widget class="QLineEdit" name="description_field"/>
+       </item>
+       <item row="3" column="1">
+        <widget class="QCheckBox" name="public_toggle"/>
+       </item>
+
+      </layout>
+     </item>
+     <item>
+      <widget class="QDialogButtonBox" name="dialog_button_box">
+       <property name="standardButtons">
+        <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
+       </property>
+      </widget>
+     </item>
+
+    </layout>
+   </item>
+  </layout>
+ </widget>
+ <resources/>
+ <connections/>
+</ui>

speckle_toolbox/esri/toolboxes/speckle/ui/linkWidget.py

@@ -0,0 +1,74 @@
+
+from PyQt5.QtCore import QCoreApplication, QSettings, Qt, QTranslator, QRect, QObject
+from PyQt5.QtWidgets import QAction, QDockWidget, QVBoxLayout, QWidget
+from PyQt5 import QtWidgets
+import webbrowser
+
+try:
+    from speckle.plugin_utils.logger import logToUser
+except:
+    from speckle_toolbox.esri.toolboxes.speckle.plugin_utils.logger import logToUser
+
+SPECKLE_COLOR = (59,130,246)
+SPECKLE_COLOR_LIGHT = (69,140,255)
+
+class LinkWidget(QWidget):
+    
+    # constructor
+    def __init__(self, parent=None):
+        super(LinkWidget, self).__init__(parent)
+        print("start LinkWidget")
+        self.parentWidget = parent
+        print(self.parentWidget)
+        # create a temporary floating button 
+        width = 0 #parent.frameSize().width()
+        height = 0# parent.frameSize().height()
+        backgr_color = f"background-color: rgb{str(SPECKLE_COLOR)};"
+        backgr_color_light = f"background-color: rgb{str(SPECKLE_COLOR_LIGHT)};"
+        
+        self.setAccessibleName("commit_link")
+        connect_box = QVBoxLayout(self)
+
+        
+        commit_link_btn = QtWidgets.QPushButton(f"👌 Data sent \n View it online") # to '{streamName}' Sent , v
+        commit_link_btn.setStyleSheet("QPushButton {color: white;border: 0px;border-radius: 17px;padding: 20px;height: 40px;text-align: left;"+ f"{backgr_color}" + "} QPushButton:hover { "+ f"{backgr_color_light}" + " }")
+
+        connect_box.addWidget(commit_link_btn) #, alignment=Qt.AlignCenter) 
+        connect_box.setContentsMargins(0, 0, 0, 30)
+        connect_box.setAlignment(Qt.AlignBottom)  
+        self.setGeometry(0, 0, width, height)
+        #self.mouseReleaseEvent = lambda event: self.closeLinkWidget(parent)
+        commit_link_btn.clicked.connect(lambda: self.openLink())
+        
+
+    # overriding the mouseReleaseEvent method
+    def mouseReleaseEvent(self, event):
+        print("Mouse Release Event")
+        self.parentWidget.hideLink()
+
+    def openLink(self, url = ""):
+        try:
+            if url == "": 
+                url = self.parentWidget.link_url
+            webbrowser.open(url, new=0, autoraise=True)
+            self.parentWidget.hideLink()
+        except Exception as e: 
+            logToUser(str(e)) 
+
+    def closeLinkWidget(self):
+        return
+        #self.parentWidget.hideLink()
+        r'''
+        try: 
+            # https://stackoverflow.com/questions/5899826/pyqt-how-to-remove-a-widget
+            print(self.parentWidget.layout())
+            self.parentWidget.layout().removeWidget(self.parentWidget.link)
+            print(self.parentWidget.layout())
+            self.parentWidget.link_url = "" 
+            self.parentWidget.link = None
+            return True
+        except Exception as e: 
+            logToUser(str(e)) 
+            return True 
+        '''
+

speckle_toolbox/esri/toolboxes/speckle/ui/project_vars.py

@@ -2,25 +2,347 @@
 from typing import Any, List, Optional, Tuple, Union
 import arcpy
 from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
-from specklepy.api.models import Branch, Stream, Streams
+from arcpy.management import CreateTable
+
 import os.path
 
-from specklepy.api.credentials import get_local_accounts
+from specklepy.api.credentials import Account, get_local_accounts
 from specklepy.api.client import SpeckleClient
 from specklepy.logging.exceptions import (
     GraphQLException,
     SpeckleException,
 )
-from specklepy.api.wrapper import StreamWrapper
+from specklepy.api.wrapper import StreamWrapper 
+from specklepy.api.models import Branch, Stream, Streams
 from osgeo import osr
 
+try: 
+    from speckle.ui.validation import tryGetStream
+    from speckle.speckle_arcgis import SpeckleGIS
+    from speckle.converter.layers import getAllProjLayers
+    from speckle.plugin_utils.logger import logToUser
+except: 
+    from speckle_toolbox.esri.toolboxes.speckle.ui.validation import tryGetStream
+    from speckle_toolbox.esri.toolboxes.speckle.speckle_arcgis import SpeckleGIS
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers import getAllProjLayers
+    from speckle_toolbox.esri.toolboxes.speckle.plugin_utils.logger import logToUser
+
+FIELDS = ["project_streams","project_layer_selection", "lat_lon"]
+
+def get_project_streams(self: SpeckleGIS, content: str = None):
+    try:
+        print("get proj streams")
+        
+        print("GET proj streams")
+        project = self.gis_project
+        table = findOrCreateSpeckleTable(project)
+        if table is None: return 
+
+        rows = arcpy.da.SearchCursor(table, "project_streams") 
+        saved_streams = []
+        for x in rows:
+            saved_streams.append(x[0])
+
+        temp = []
+        ######### need to check whether saved streams are available (account reachable)
+        if len(saved_streams) > 0:
+            for url in saved_streams:
+                try:
+                    sw = StreamWrapper(url)
+                    try: 
+                        stream = tryGetStream(sw)
+                    except SpeckleException as e:
+                        logToUser(e.message, 2)
+                        stream = None
+                    #strId = stream.id # will cause exception if invalid
+                    temp.append((sw, stream))
+                except SpeckleException as e:
+                    logToUser(e.message, 2)
+                #except GraphQLException as e:
+                #    logger.logToUser(e.message, Qgis.Warning)
+        self.current_streams = temp
+    except Exception as e: 
+        logToUser(str(e)) 
+    
+def set_project_streams(self: SpeckleGIS):
+    try:
+        print("SET proj streams")
+        project = self.gis_project
+        table = findOrCreateSpeckleTable(project)
+        print("SET proj streams 2")
+        
+        value = [stream[0].stream_url for stream in self.current_streams] #",".join()
+        print(value)
+
+        if table is not None:
+            proj_layers = []
+            lan_lot = ""
+            with arcpy.da.UpdateCursor(table, FIELDS) as cursor: 
+                for row in cursor: # just one row
+                    if row[1] is not None and row[1] != "": proj_layers.append(row[1]) 
+                    if row[2] is not None and row[2] != "": lan_lot = row[2] 
+                    cursor.deleteRow()
+            del cursor 
+            if len(proj_layers) == 0: proj_layers.append("")
+            if len(value) == 0: value.append("")
+            
+            cursor = arcpy.da.InsertCursor(table, FIELDS ) 
+            length = max(len(proj_layers), len(value))
+
+            for i in range(length): 
+                if i==0: 
+                    cursor.insertRow([value[i], proj_layers[i] , lan_lot]) 
+                else: 
+                    try:
+                        cursor.insertRow([value[i], proj_layers[i] , ""]) 
+                    except: 
+                        if len(value) <= i: cursor.insertRow(["", proj_layers[i] , ""]) 
+                        if len(proj_layers) <= i: cursor.insertRow([value[i], "" , ""])
+            del cursor 
+    except Exception as e: 
+        logToUser(str(e)) 
+  
+def get_project_layer_selection(self: SpeckleGIS):
+    try:
+        print("GET project layer selection from the table")
+        project = self.gis_project
+        table = findOrCreateSpeckleTable(project)
+        if table is None: return 
+        
+        rows = arcpy.da.SearchCursor(table, "project_layer_selection") 
+        saved_layers = []
+        for x in rows:
+            saved_layers.append(x[0]) 
+
+        
+        temp = []
+        proj_layers = getAllProjLayers(project)
+        ######### need to check whether saved streams are available (account reachable)
+        if len(saved_layers) > 0:
+            for layerPath in saved_layers:
+                if layerPath == "": continue
+                found = 0
+                for layer in proj_layers:
+                    print(layer.dataSource)
+                    if layer.dataSource == layerPath:
+                        temp.append((layer.name, layer))
+                        found += 1
+                        break
+                if found == 0: 
+                    logToUser(f'Saved layer not found: "{layerPath}"', 1)
+        self.current_layers = temp
+    except Exception as e: 
+        logToUser(str(e)) 
+
+def set_project_layer_selection(self: SpeckleGIS):
+    try:
+        print("SET project layer selection function")
+        project = self.gis_project
+        value: List[str] = [layer[1].dataSource for layer in self.current_layers] #",".join([layer[1].dataSource for layer in self.current_layers]) 
+        print(value)
+
+        table = findOrCreateSpeckleTable(project)
+        #print(table)
+        if table is not None:
+            lan_lot = ""
+            proj_streams = []
+            with arcpy.da.UpdateCursor(table, FIELDS) as cursor:
+                for row in cursor: # just one row
+                    if row[0] is not None and row[0] != "": proj_streams.append(row[0]) 
+                    if row[2] is not None and row[2] != "": lan_lot = row[2]
+                    cursor.deleteRow()
+            del cursor 
+            if len(proj_streams) == 0: proj_streams.append("")
+            if len(value) == 0: value.append("")
+            #print(proj_streams)
+            
+            cursor = arcpy.da.InsertCursor(table, FIELDS )
+            length = max(len(proj_streams), len(value))
+            #print(length)
+            for i in range(length): 
+                #print(i)
+                if i==0: 
+                    cursor.insertRow([proj_streams[i], value[i] , lan_lot]) 
+                    print(i)
+                else: 
+                    try:
+                        cursor.insertRow([proj_streams[i], value[i] , ""]) 
+                    except: 
+                        if len(proj_streams) <= i: cursor.insertRow(["", value[i] , ""]) 
+                        if len(value) <= i: cursor.insertRow([proj_streams[i], "" , ""])
+                #print(i)
+            del cursor 
+            #print(table)
+    except Exception as e: 
+        logToUser(str(e)) 
+
+    print("SET project layer selection 2")
+
+def get_survey_point(self: SpeckleGIS, content = None):
+    try:
+        print("get survey point")
+        project = self.gis_project
+        table = findOrCreateSpeckleTable(project)
+        if table is None: return 
+
+        rows = arcpy.da.SearchCursor(table, "lat_lon") 
+        points = ""
+        for x in rows:
+            points = x[0]
+            break
+
+        if points != "": 
+            vals: List[str] = points.replace(" ","").split(";")[:2]
+            self.lat, self.lon = [float(i) for i in vals]
+
+    except Exception as e: 
+        logToUser(str(e)) 
+    
+def set_survey_point(self: SpeckleGIS):
+
+    try:
+        # from widget (2 strings) to local vars + update SR of the map
+        print("SET survey point")
+        
+        project = self.gis_project
+        vals =[ str(self.dockwidget.surveyPointLat.text()), str(self.dockwidget.surveyPointLon.text()) ]
+
+        self.lat, self.lon = [float(i.replace(" ","")) for i in vals]
+        pt = str(self.lat) + ";" + str(self.lon) 
+
+        table = findOrCreateSpeckleTable(project)
+        if table is not None:
+            with arcpy.da.UpdateCursor(table, ["lat_lon"]) as cursor:
+                for row in cursor: # just one row
+                    cursor.updateRow([pt])
+                    break
+            del cursor   
+        
+        setProjectReferenceSystem(self)
+        return True
+
+    except Exception as e:
+        self.dockwidget.surveyPointLat.setText(str(self.lat))
+        self.dockwidget.surveyPointLon.setText(str(self.lon))
+        logToUser("Lat, Lon values invalid: " + str(e))
+        return False 
+
+def setProjectReferenceSystem(self: SpeckleGIS):
+    try: 
+        # save to project; create SR
+        newCrsString = "+proj=tmerc +ellps=WGS84 +datum=WGS84 +units=m +no_defs +lon_0=" + str(self.lon) + " lat_0=" + str(self.lat) + " +x_0=0 +y_0=0 +k_0=1"
+        newCrs = osr.SpatialReference()
+        newCrs.ImportFromProj4(newCrsString)
+        newCrs.MorphToESRI() # converts the WKT to an ESRI-compatible format
+        
+        validate = True if len(newCrs.ExportToWkt())>10 else False
+
+        if validate: 
+            newProjSR = arcpy.SpatialReference()
+            newProjSR.loadFromString(newCrs.ExportToWkt())
+
+            #source = osr.SpatialReference() 
+            #source.ImportFromWkt(self.project.activeMap.spatialReference.exportToString())
+            #transform = osr.CoordinateTransformation(source, newCrs)
+
+            self.gis_project.activeMap.spatialReference =  newProjSR
+            logToUser("Custom project Spatial Reference successfully applied", 0)
+        else:
+            logToUser("Custom Spatial Reference could not be created", 1)
+
+        return True
+    except Exception as e: 
+        logToUser(str(e)) 
+        return False
+
+def findOrCreateSpeckleTable(project: ArcGISProject) -> Union[str, None]:
+    try:
+        path = project.filePath.replace("aprx","gdb") #"\\".join(project.filePath.split("\\")[:-1]) + "\\speckle_layers\\" #arcpy.env.workspace + "\\" #
+    
+        if 'speckle_gis' not in arcpy.ListTables():
+            try: 
+                table = CreateTable(path, "speckle_gis")
+                arcpy.management.AddField(table, "project_streams", "TEXT")
+                arcpy.management.AddField(table, "project_layer_selection", "TEXT")
+                arcpy.management.AddField(table, "lat_lon", "TEXT")
+
+                cursor = arcpy.da.InsertCursor(table, FIELDS )
+                cursor.insertRow(["",""])
+                del cursor
+            
+            except Exception as e:
+                logToUser("Error creating a table: " + str(e), 1)
+                return None
+        else: 
+            #print("table already exists")
+            # make sure fileds exist 
+            table = path + "\\speckle_gis" 
+            findOrCreateTableField(table, FIELDS[0])
+            findOrCreateTableField(table, FIELDS[1])
+            findOrCreateTableField(table, FIELDS[2])
+            
+            findOrCreateRow(table, FIELDS)
+
+        return table
+    
+    except Exception as e: 
+        logToUser(str(e)) 
+        return None 
+
+def findOrCreateTableField(table: str, field: str):
+    try: 
+        with arcpy.da.UpdateCursor(table, [field]) as cursor:
+            value = None
+            for row in cursor:
+                value = row # tuple(val,)
+                if value[0] is None: cursor.updateRow("")
+                break # look at the 1st row only 
+        del cursor
+
+        #if value is None: # if there are no rows 
+        #    cursor = arcpy.da.InsertCursor(table, [field])
+        #    cursor.insertRow([""]) 
+        #    del cursor
+    
+    except: # if field doesn't exist
+        arcpy.management.AddField(table, field, "TEXT")
+        #cursor = arcpy.da.InsertCursor(table, [field] )
+        #cursor.insertRow([""])
+        del cursor
+
+def findOrCreateRow(table:str, fields: List[str]):
+    try:
+        # check if the row exists 
+        cursor = arcpy.da.SearchCursor(table, fields)
+        k=-1
+        for k, row in enumerate(cursor): 
+            #print(row)
+            break
+        del cursor
+        
+        # if no rows
+        if k == -1:
+            cursor = arcpy.da.InsertCursor(table, fields)
+            cursor.insertRow(["", "", ""]) 
+            del cursor
+        else: 
+            with arcpy.da.UpdateCursor(table, fields) as cursor:
+                for row in cursor:
+                    if None in row: cursor.updateRow(["","",""])
+                    break # look at the 1st row only 
+            del cursor
+    
+    except Exception as e: 
+        logToUser(str(e)) 
+
+r'''
 class speckleInputsClass:
     #def __init__(self):
     print("CREATING speckle inputs first time________")
     instances = []
-    accounts = get_local_accounts()
+    accounts: List[Account] = get_local_accounts()
     account = None
-    streams_default: None or List[Stream] = None 
+    streams_default: Optional[List[Stream]] = None
 
     project = None
     active_map = None
@@ -123,7 +445,7 @@ class speckleInputsClass:
             except: pass
 
             client = sw.get_client()
-            stream = client.stream.get(steamId)
+            stream = client.stream.get(id = steamId, branch_limit = 100, commit_limit = 100)
             if isinstance(stream, GraphQLException):
                 raise SpeckleException(stream.errors[0]['message'])
             return stream
@@ -273,5 +595,5 @@ class toolboxInputsClass:
             arcpy.AddWarning("Custom CRS could not be created: not enough coordinates provided")
 
         return True
-
+'''
     

speckle_toolbox/esri/toolboxes/speckle/ui/speckle_qgis_dialog.py

@@ -0,0 +1,622 @@
+
+import os
+import sys
+from typing import List
+#from speckle.converter.layers import getLayers
+
+#import ui.speckle_qgis_dialog
+
+from specklepy.logging.exceptions import (SpeckleException, GraphQLException)
+from PyQt5 import QtWidgets, uic
+from PyQt5 import QtGui
+from PyQt5.QtGui import QIcon, QPixmap
+from PyQt5.QtWidgets import (QMainWindow, QApplication, QWidget, 
+    QListWidgetItem, QAction, QDockWidget, QVBoxLayout, 
+    QHBoxLayout, QWidget, QLabel)
+from PyQt5 import QtCore
+from PyQt5.QtCore import pyqtSignal, Qt, QSize, QEvent 
+from PyQt5 import QtGui, uic 
+
+from specklepy.api.credentials import get_local_accounts
+
+import importlib
+
+from specklepy.api.wrapper import StreamWrapper
+from specklepy.api.client import SpeckleClient
+
+import arcpy 
+
+try:
+    #from speckle.speckle_arcgis_new import Speckle
+    from speckle.converter.layers import getLayers
+    from speckle.converter.layers import getAllProjLayers
+    from speckle.plugin_utils.logger import logToUser
+    from speckle.ui.linkWidget import LinkWidget
+except:
+    #from speckle_toolbox.esri.toolboxes.speckle.speckle_arcgis_new import Speckle
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers import getLayers
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers import getAllProjLayers
+    from speckle_toolbox.esri.toolboxes.speckle.plugin_utils.logger import logToUser
+    from speckle_toolbox.esri.toolboxes.speckle.ui.linkWidget import LinkWidget
+
+#from ui.validation import tryGetStream
+
+# Create module-like object
+#pytPath = os.path.dirname(os.path.abspath(__file__)).replace("/speckle/ui","/Speckle.pyt")
+#print(pytPath)
+#pytModule = importlib.machinery.SourceFileLoader("specklePyt", pytPath )
+#specklePyt = pytModule.load_module("specklePyt")
+
+
+# This loads your .ui file so that PyQt can populate your plugin with the elements from Qt Designer
+
+COLOR_HIGHLIGHT = (210,210,210)
+
+SPECKLE_COLOR = (59,130,246)
+SPECKLE_COLOR_LIGHT = (69,140,255)
+ICON_LOGO = os.path.dirname(os.path.abspath(__file__)) + "/logo-slab-white@0.5x.png"
+
+ICON_SEARCH = os.path.dirname(os.path.abspath(__file__)) + "/magnify.png"
+
+ICON_DELETE = os.path.dirname(os.path.abspath(__file__)) + "/delete.png"
+ICON_DELETE_BLUE = os.path.dirname(os.path.abspath(__file__)) + "/delete-blue.png"
+
+ICON_SEND = os.path.dirname(os.path.abspath(__file__)) + "/cube-send.png"
+ICON_RECEIVE = os.path.dirname(os.path.abspath(__file__)) + "/cube-receive.png"
+
+ICON_SEND_BLACK = os.path.dirname(os.path.abspath(__file__)) + "/cube-send-black.png"
+ICON_RECEIVE_BLACK = os.path.dirname(os.path.abspath(__file__)) + "/cube-receive-black.png"
+
+ICON_SEND_BLUE = os.path.dirname(os.path.abspath(__file__)) + "/cube-send-blue.png"
+ICON_RECEIVE_BLUE = os.path.dirname(os.path.abspath(__file__)) + "/cube-receive-blue.png"
+
+ui_class = os.path.dirname(os.path.abspath(__file__)) + "/speckle_qgis_dialog_base.ui"
+print(os.path.dirname(__file__))
+
+class SpeckleGISDialog(QMainWindow):
+
+    closingPlugin = pyqtSignal()
+    streamList: QtWidgets.QComboBox
+    sendModeButton: QtWidgets.QPushButton
+    receiveModeButton: QtWidgets.QPushButton
+    streamBranchDropdown: QtWidgets.QComboBox
+    layerSendModeDropdown: QtWidgets.QComboBox
+    commitDropdown: QtWidgets.QComboBox
+    layersWidget: QtWidgets.QListWidget
+    saveLayerSelection: QtWidgets.QPushButton
+    runButton: QtWidgets.QPushButton
+    link = None
+    link_url: str = ""
+
+    gridLayoutTitleBar = QtWidgets.QGridLayout
+    
+    def __init__(self):
+        """Constructor."""
+        print("START MAIN WINDOW")
+        super(SpeckleGISDialog, self).__init__(None)#, QtCore.Qt.WindowStaysOnTopHint)
+        uic.loadUi(ui_class, self) # Load the .ui file
+        #self.installEventFilter(self)
+        self.show() 
+        #self.instances.append(1)
+        try:
+            self.streamBranchDropdown.setMaxCount(100)
+            self.commitDropdown.setMaxCount(100)
+
+            self.streams_add_button.setFlat(True)
+            self.streams_remove_button.setFlat(True)
+            self.saveSurveyPoint.setFlat(True)
+            self.saveLayerSelection.setFlat(True)
+            self.reloadButton.setFlat(True)
+            self.closeButton.setFlat(True)
+
+            color = f"color: rgb{str(SPECKLE_COLOR)};"
+            backgr_color = f"background-color: rgb{str(SPECKLE_COLOR)};"
+            backgr_color_light = f"background-color: rgb{str(SPECKLE_COLOR_LIGHT)};"
+            backgr_image_del = f"border-image: url({ICON_DELETE_BLUE});"
+            self.streams_add_button.setIcon(QIcon(ICON_SEARCH))
+            self.streams_add_button.setMaximumWidth(25)
+            self.streams_add_button.setStyleSheet("QPushButton {padding:3px;padding-left:5px;border: none; text-align: left;} QPushButton:hover { " + f"background-color: rgb{str(COLOR_HIGHLIGHT)};" + f"{color}" + " }")
+            self.streams_remove_button.setIcon(QIcon(ICON_DELETE))
+            self.streams_remove_button.setMaximumWidth(25)
+            self.streams_remove_button.setStyleSheet("QPushButton {padding:3px;padding-left:5px;border: none; text-align: left; image-position:right} QPushButton:hover { " + f"background-color: rgb{str(COLOR_HIGHLIGHT)};" + f"{color}" + " }") #+ f"{backgr_image_del}" 
+
+            self.saveLayerSelection.setStyleSheet("QPushButton {text-align: right;} QPushButton:hover { " + f"{color}" + " }")
+            self.saveSurveyPoint.setStyleSheet("QPushButton {text-align: right;} QPushButton:hover { " + f"{color}" + " }")
+            self.reloadButton.setStyleSheet("QPushButton {text-align: left;} QPushButton:hover { " + f"{color}" + " }")
+            self.closeButton.setStyleSheet("QPushButton {text-align: right;} QPushButton:hover { " + f"{color}" + " }")
+
+            exitIcon = QPixmap(ICON_LOGO)
+            exitActIcon = QIcon(exitIcon)
+
+            backgr_color = f"background-color: rgb{str(SPECKLE_COLOR)};"
+            backgr_color_light = f"background-color: rgb{str(SPECKLE_COLOR_LIGHT)};"
+
+            # create a label 
+            text_label = QtWidgets.QPushButton(" for ArcGIS")
+            text_label.setStyleSheet("border: 0px;"
+                                "color: white;"
+                                f"{backgr_color}"
+                                "top-margin: 40 px;"
+                                "padding: 10px;"
+                                "padding-left: 20px;"
+                                "font-size: 15px;"
+                                "height: 30px;"
+                                "text-align: left;"
+                                )
+            text_label.setIcon(exitActIcon)
+            text_label.setIconSize(QSize(300, 93))
+            text_label.setMinimumSize(QSize(100, 40))
+            text_label.setMaximumWidth(220)
+
+            version = ""
+            try:
+                metadata_file = os.path.dirname(__file__)[:-2] + "metadata.txt"
+                with open(metadata_file, "r") as file:
+                    lines = file.readlines()
+                    for i, line in enumerate(lines):
+                        if "version=" in line: 
+                            version = "v " + line.replace("version=", "")
+                            break
+            except: pass 
+
+            version_label = QtWidgets.QPushButton(f"{version}")
+            version_label.setStyleSheet("border: 0px;"
+                                "color: white;"
+                                f"{backgr_color}"
+                                "padding-top: 15px;"
+                                "padding-left: 0px;"
+                                "margin-left: 0px;"
+                                "font-size: 10px;"
+                                "height: 30px;"
+                                "text-align: left;"
+                                )
+
+            widget = QWidget()
+            widget.setStyleSheet(f"{backgr_color}")
+            connect_box = QHBoxLayout(widget)
+            connect_box.addWidget(text_label) #, alignment=Qt.AlignCenter) 
+            connect_box.addWidget(version_label) 
+            connect_box.setContentsMargins(0, 0, 0, 0)
+            self.gridLayoutTitleBar.addWidget(widget) # fro QMainWindow
+            #self.setTitleBarWidget(widget) # for QDockWidget 
+
+            self.sendModeButton.setStyleSheet("QPushButton {padding: 10px; border: 0px; " + f"color: rgb{str(SPECKLE_COLOR)};"+ "} QPushButton:hover { "  + "}" ) 
+            self.sendModeButton.setIcon(QIcon(ICON_SEND_BLUE))
+            
+            self.receiveModeButton.setFlat(True)
+            self.receiveModeButton.setStyleSheet("QPushButton {padding: 10px; border: 0px;}"+ "QPushButton:hover { "  + f"background-color: rgb{str(COLOR_HIGHLIGHT)};" + "}" ) 
+            self.receiveModeButton.setIcon(QIcon(ICON_RECEIVE_BLACK))
+
+            self.runButton.setStyleSheet("QPushButton {color: white;border: 0px;border-radius: 17px;padding: 10px;"+ f"{backgr_color}" + "} QPushButton:hover { "+ f"{backgr_color_light}" + " }")
+            self.runButton.setMaximumWidth(200)
+            self.runButton.setIcon(QIcon(ICON_SEND))
+
+            widgetLink = LinkWidget(parent=self)
+            self.layout().addWidget(widgetLink)
+            self.link = widgetLink 
+
+        except Exception as e: 
+            logToUser(str(e)) 
+
+    def showLink(self):
+        print("showLink")
+        try: 
+            self.link.setGeometry(0, 0, self.frameSize().width(), self.frameSize().height())
+        except Exception as e: 
+            logToUser(str(e)) 
+
+    def hideLink(self):
+        if self.link is None: return 
+        try: 
+            self.link.setGeometry(0, 0, 0, 0)
+        except Exception as e: 
+            logToUser(str(e)) 
+
+
+    def closeEvent(self, event):
+        try:
+            #import threading
+            print("Close event")
+            #threads = threading.enumerate()
+            #print(f"Threads total: {str(len(threads))}: {str(threads)}")
+
+            #print(self.instances)
+            
+            self.closingPlugin.emit()
+            event.accept()
+        
+        except Exception as e: 
+            logToUser(str(e)) 
+        
+    def clearDropdown(self):
+        try:
+            #self.streamIdField.clear()
+            self.streamBranchDropdown.clear()
+            self.commitDropdown.clear()
+            #self.layerSendModeDropdown.clear()
+        except Exception as e: 
+            logToUser(str(e)) 
+
+    def reloadDialogUI(self, plugin):
+        try:
+            self.clearDropdown()
+            self.populateUI(plugin) 
+            self.enableElements(plugin)
+        except Exception as e: 
+            logToUser(str(e)) 
+
+        
+
+    def run(self, plugin): 
+        try:
+            print("dockwidget run")
+            # Setup events on first load only!
+            self.setupOnFirstLoad(plugin)
+            # Connect streams section events
+            self.completeStreamSection(plugin)
+            # Populate the UI dropdowns
+            self.populateUI(plugin) 
+            print("dockwidget run end")
+        except Exception as e: 
+            logToUser(str(e)) 
+
+
+    def setupOnFirstLoad(self, plugin):
+        try:
+            self.runButton.clicked.connect(plugin.onRunButtonClicked)
+            
+            self.streams_add_button.clicked.connect( plugin.onStreamAddButtonClicked )
+            self.reloadButton.clicked.connect(plugin.reloadUI)
+            self.closeButton.clicked.connect(plugin.onClosePlugin)
+            self.saveSurveyPoint.clicked.connect(plugin.set_survey_point)
+            self.saveLayerSelection.clicked.connect(lambda: self.populateLayerDropdown(plugin))
+            self.sendModeButton.clicked.connect(lambda: self.setSendMode(plugin))
+            self.layerSendModeDropdown.currentIndexChanged.connect( lambda: self.layerSendModeChange(plugin) )
+            self.receiveModeButton.clicked.connect(lambda: self.setReceiveMode(plugin))
+
+            self.streamBranchDropdown.currentIndexChanged.connect( lambda: self.runBtnStatusChanged(plugin) )
+            self.commitDropdown.currentIndexChanged.connect( lambda: self.runBtnStatusChanged(plugin) )
+
+            self.closingPlugin.connect(plugin.onClosePlugin)
+            return 
+        
+        except Exception as e: 
+            logToUser(str(e)) 
+
+    def setSendMode(self, plugin):
+        try:
+            plugin.btnAction = 0 # send 
+            color = f"color: rgb{str(SPECKLE_COLOR)};"
+            self.sendModeButton.setStyleSheet("border: 0px;"
+                                        f"color: rgb{str(SPECKLE_COLOR)};"
+                                        "padding: 10px;")
+            self.sendModeButton.setIcon(QIcon(ICON_SEND_BLUE))
+            self.sendModeButton.setFlat(False)
+            self.receiveModeButton.setFlat(True)
+            self.receiveModeButton.setStyleSheet("QPushButton {border: 0px; color: black; padding: 10px; } QPushButton:hover { " + f"background-color: rgb{str(COLOR_HIGHLIGHT)};" +  " };")
+            self.receiveModeButton.setIcon(QIcon(ICON_RECEIVE_BLACK))
+            #self.receiveModeButton.setFlat(True)
+            self.runButton.setProperty("text", " SEND")
+            self.runButton.setIcon(QIcon(ICON_SEND))
+
+            # enable sections only if in "saved streams" mode 
+            if self.layerSendModeDropdown.currentIndex() == 1: self.layersWidget.setEnabled(True)
+            if self.layerSendModeDropdown.currentIndex() == 1: self.saveLayerSelection.setEnabled(True)
+            self.commitDropdown.setEnabled(False)
+            self.messageInput.setEnabled(True)
+            self.layerSendModeDropdown.setEnabled(True)
+
+            self.runBtnStatusChanged(plugin)
+            return
+        except Exception as e: 
+            logToUser(str(e)) 
+    
+    def setReceiveMode(self, plugin):
+        try:
+            plugin.btnAction = 1 # receive 
+            color = f"color: rgb{str(SPECKLE_COLOR)};"
+            self.receiveModeButton.setStyleSheet("border: 0px;"
+                                        f"color: rgb{str(SPECKLE_COLOR)};"
+                                        "padding: 10px;")
+            self.sendModeButton.setIcon(QIcon(ICON_SEND_BLACK))
+            self.sendModeButton.setStyleSheet("QPushButton {border: 0px; color: black; padding: 10px;} QPushButton:hover { " + f"background-color: rgb{str(COLOR_HIGHLIGHT)};"  + " };")
+            self.receiveModeButton.setIcon(QIcon(ICON_RECEIVE_BLUE))
+            self.sendModeButton.setFlat(True)
+            self.receiveModeButton.setFlat(False)
+            #self.sendModeButton.setFlat(True)
+            self.runButton.setProperty("text", " RECEIVE")
+            self.runButton.setIcon(QIcon(ICON_RECEIVE))
+            #self.layerSendModeChange(plugin, 1)
+            self.commitDropdown.setEnabled(True)
+            self.layersWidget.setEnabled(False)
+            self.messageInput.setEnabled(False)
+            self.saveLayerSelection.setEnabled(False)
+            self.layerSendModeDropdown.setEnabled(False)
+
+            self.runBtnStatusChanged(plugin)
+            return
+        except Exception as e: 
+            logToUser(str(e)) 
+
+    def completeStreamSection(self, plugin):
+        self.streams_remove_button.clicked.connect( lambda: self.onStreamRemoveButtonClicked(plugin) )
+        self.streamList.currentIndexChanged.connect( lambda: self.onActiveStreamChanged(plugin) )
+        self.streamBranchDropdown.currentIndexChanged.connect( lambda: self.populateActiveCommitDropdown(plugin) )
+        return
+
+    def populateUI(self, plugin):
+        try:
+            self.populateLayerSendModeDropdown()
+            self.populateLayerDropdown(plugin, False)
+            #items = [self.layersWidget.item(x).text() for x in range(self.layersWidget.count())]
+            self.populateProjectStreams(plugin)
+            self.populateSurveyPoint(plugin)
+
+            self.runBtnStatusChanged(plugin)
+            self.runButton.setEnabled(False) 
+        except Exception as e: 
+            logToUser(str(e)) 
+  
+    def runBtnStatusChanged(self, plugin):
+        try:
+            commitStr = str(self.commitDropdown.currentText())
+            branchStr = str(self.streamBranchDropdown.currentText())
+
+            if plugin.btnAction == 1: # on receive
+                if commitStr == "": 
+                    self.runButton.setEnabled(False) 
+                else: 
+                    self.runButton.setEnabled(True) 
+            
+            if plugin.btnAction == 0: # on send 
+                if branchStr == "": 
+                    self.runButton.setEnabled(False) 
+                elif branchStr != "" and self.layerSendModeDropdown.currentIndex() == 1 and len(plugin.current_layers) == 0: # saved layers; but the list is empty 
+                    self.runButton.setEnabled(False)
+                else:
+                    self.runButton.setEnabled(True)
+        except Exception as e: 
+            logToUser(str(e)) 
+            
+
+    def layerSendModeChange(self, plugin, runMode = None):
+        try:
+            print("Send mode changed")
+
+            if self.layerSendModeDropdown.currentIndex() == 0 or runMode == 1: # by manual selection OR receive mode
+                self.current_layers = []
+                self.layersWidget.setEnabled(False)
+                self.saveLayerSelection.setEnabled(False)
+                
+            elif self.layerSendModeDropdown.currentIndex() == 1 and (runMode == 0 or runMode is None): # by saved AND when Send mode
+                self.layersWidget.setEnabled(True)
+                self.saveLayerSelection.setEnabled(True)
+            
+            branchStr = str(self.streamBranchDropdown.currentText())
+            if self.layerSendModeDropdown.currentIndex() == 0:
+                if branchStr == "": self.runButton.setEnabled(False) # by manual selection
+                else: self.runButton.setEnabled(True) # by manual selection
+            elif self.layerSendModeDropdown.currentIndex() == 1: self.runBtnStatusChanged(plugin) # by saved
+        except Exception as e: 
+            logToUser(str(e)) 
+
+
+    def populateLayerDropdown(self, plugin, bySelection: bool = True):
+        print("populate layer dropdown / clicked save selection")
+        if not self: return
+        try: 
+            from speckle.ui.project_vars import set_project_layer_selection
+        except: 
+            from speckle_toolbox.esri.toolboxes.speckle.ui.project_vars import set_project_layer_selection
+        
+        try:
+            self.layersWidget.clear()
+            nameDisplay = [] 
+            project = plugin.gis_project 
+
+            if bySelection is False: # read from project data 
+                print("populate layers from saved data")
+                #print(project)
+                #print(project.activeMap)
+
+                all_layers_ids = [l.dataSource for l in getAllProjLayers(project)]
+                for layer_tuple in plugin.current_layers:
+                    if layer_tuple[1].dataSource in all_layers_ids: 
+                        listItem = self.fillLayerList(layer_tuple[1]) 
+                        self.layersWidget.addItem(listItem)
+
+            else: # read selected layers 
+                # Fetch selected layers
+                print("populate layers from selection")
+                
+                plugin.current_layers = []
+                layers = getLayers(plugin, bySelection) # List[QgsLayerTreeNode]
+                print(layers)
+                for i, layer in enumerate(layers):
+                    plugin.current_layers.append((layer.name, layer)) 
+                    listItem = self.fillLayerList(layer)
+                    self.layersWidget.addItem(listItem)
+                print("populate layers from selection 2")
+                set_project_layer_selection(plugin)
+                print("populate layers from selection 3")
+
+            self.layersWidget.setIconSize(QSize(20, 20))
+            self.runBtnStatusChanged(plugin)
+            
+            return
+        except Exception as e: 
+            logToUser(str(e)) 
+
+    def fillLayerList(self, layer):
+        print("Fill layer list")
+        
+        try:
+            ICON_XXL = os.path.dirname(os.path.abspath(__file__)) + "/size-xxl.png"
+            ICON_RASTER = os.path.dirname(os.path.abspath(__file__)) + "/legend_raster.png"
+            ICON_POLYGON = os.path.dirname(os.path.abspath(__file__)) + "/legend_polygon.png"
+            ICON_LINE = os.path.dirname(os.path.abspath(__file__)) + "/legend_line.png"
+            ICON_POINT = os.path.dirname(os.path.abspath(__file__)) + "/legend_point.png"
+
+            listItem = QListWidgetItem(layer.name) 
+            #print(listItem)
+
+            if layer.isRasterLayer: # and layer.width()*layer.height() > 1000000:
+                listItem.setIcon(QIcon(ICON_RASTER))
+            
+            elif layer.isFeatureLayer: # and layer.featureCount() > 20000:
+                geomType = arcpy.Describe(layer.dataSource).shapeType
+                if geomType == "Polygon": listItem.setIcon(QIcon(ICON_POLYGON))
+                elif geomType == "Polyline": listItem.setIcon(QIcon(ICON_LINE))
+                elif geomType == "Point" or geomType == "Multipoint": listItem.setIcon(QIcon(ICON_POINT))
+                else: 
+                    listItem.setIcon(QIcon(ICON_XXL))
+            #else: 
+            #    icon = QgsIconUtils().iconForLayer(layer)
+            #    listItem.setIcon(icon)
+            
+            return listItem
+        except Exception as e: 
+            logToUser(str(e)) 
+        
+    def populateSurveyPoint(self, plugin):
+        if not self:
+            return
+        try:
+            self.surveyPointLat.clear()
+            self.surveyPointLat.setText(str(plugin.lat))
+            self.surveyPointLon.clear()
+            self.surveyPointLon.setText(str(plugin.lon))
+        
+        except Exception as e: 
+            logToUser(str(e)) 
+
+    def enableElements(self, plugin):
+        try:
+            self.sendModeButton.setEnabled(plugin.is_setup)
+            self.receiveModeButton.setEnabled(plugin.is_setup)
+            self.runButton.setEnabled(plugin.is_setup)
+            self.streams_add_button.setEnabled(plugin.is_setup)
+            if plugin.is_setup is False: self.streams_remove_button.setEnabled(plugin.is_setup) 
+            self.streamBranchDropdown.setEnabled(plugin.is_setup)
+            self.layerSendModeDropdown.setEnabled(plugin.is_setup)
+            self.commitDropdown.setEnabled(False)
+            self.show()
+        except Exception as e: 
+            logToUser(str(e)) 
+
+    def populateProjectStreams(self, plugin):
+
+        try:   
+            from speckle.ui.project_vars import set_project_streams
+        except: 
+            from speckle_toolbox.esri.toolboxes.speckle.ui.project_vars import set_project_streams
+        
+        try:
+            if not self: return
+            self.streamList.clear()
+            for stream in plugin.current_streams: 
+                self.streamList.addItems(
+                [f"Stream not accessible - {stream[0].stream_id}" if stream[1] is None or isinstance(stream[1], SpeckleException) else f"{stream[1].name}, {stream[1].id} | {stream[0].stream_url.split('/streams')[0]}"] 
+            )
+            if len(plugin.current_streams)==0: self.streamList.addItems([""])
+            self.streamList.addItems(["Create New Stream"])
+            set_project_streams(plugin)
+            index = self.streamList.currentIndex()
+            if index == -1: self.streams_remove_button.setEnabled(False)
+            else: self.streams_remove_button.setEnabled(True)
+
+            if len(plugin.current_streams)>0: plugin.active_stream = plugin.current_streams[0]
+        except Exception as e: 
+            logToUser(str(e)) 
+        
+
+    def onActiveStreamChanged(self, plugin):
+
+        if not self: return
+        try:
+            index = self.streamList.currentIndex()
+            if (len(plugin.current_streams) == 0 and index ==1) or (len(plugin.current_streams)>0 and index == len(plugin.current_streams)): 
+                self.populateProjectStreams(plugin)
+                plugin.onStreamCreateClicked()
+                return
+            if len(plugin.current_streams) == 0: return
+            if index == -1: return
+
+            try: plugin.active_stream = plugin.current_streams[index]
+            except: plugin.active_stream = None
+
+            self.populateActiveStreamBranchDropdown(plugin)
+            self.populateActiveCommitDropdown(plugin)
+        except Exception as e: 
+            logToUser(str(e)) 
+        
+    def populateLayerSendModeDropdown(self):
+        if not self: return
+        try:
+            self.layerSendModeDropdown.clear()
+            self.layerSendModeDropdown.addItems(
+                ["Send visible layers", "Send saved layers"]
+            )
+        except Exception as e: 
+            logToUser(str(e)) 
+
+    def populateActiveStreamBranchDropdown(self, plugin):
+        if not self: return
+        if plugin.active_stream is None: return
+        try:
+            self.streamBranchDropdown.clear()
+            if isinstance(plugin.active_stream[1], SpeckleException): 
+                #logger.logToUser("Some streams cannot be accessed", Qgis.Warning)
+                return
+            elif plugin.active_stream is None or plugin.active_stream[1] is None or plugin.active_stream[1].branches is None:
+                return
+            self.streamBranchDropdown.addItems(
+                [f"{branch.name}" for branch in plugin.active_stream[1].branches.items]
+            )
+            self.streamBranchDropdown.addItems(["Create New Branch"])
+        except Exception as e: 
+            logToUser(str(e)) 
+
+    def populateActiveCommitDropdown(self, plugin):
+        if not self: return
+        try:
+            self.commitDropdown.clear()
+            if plugin.active_stream is None: return
+            branchName = self.streamBranchDropdown.currentText()
+            if branchName == "Create New Branch": 
+                self.streamBranchDropdown.setCurrentText("main")
+                plugin.onBranchCreateClicked()
+                return
+            branch = None
+            if isinstance(plugin.active_stream[1], SpeckleException): 
+                #logger.logToUser("Some streams cannot be accessed", Qgis.Warning)
+                return
+            elif plugin.active_stream[1]:
+                for b in plugin.active_stream[1].branches.items:
+                    if b.name == branchName:
+                        branch = b
+                        break
+            try:
+                self.commitDropdown.addItems(
+                    [f"{commit.id}"+ " | " + f"{commit.message}" for commit in branch.commits.items]
+                )
+            except: pass
+        except Exception as e: 
+            logToUser(str(e)) 
+
+    def onStreamRemoveButtonClicked(self, plugin):
+        try:
+            #from ui.project_vars import set_project_streams
+            if not self: return
+            index = self.streamList.currentIndex()
+            if len(plugin.current_streams) > 0: plugin.current_streams.pop(index)
+            plugin.active_stream = None
+            self.streamBranchDropdown.clear()
+            self.commitDropdown.clear()
+            #self.streamIdField.setText("")
+
+            #set_project_streams(plugin)
+            self.populateProjectStreams(plugin)
+        except Exception as e: 
+            logToUser(str(e)) 
+

speckle_toolbox/esri/toolboxes/speckle/ui/speckle_qgis_dialog_base.ui

@@ -0,0 +1,316 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>SpeckleQArcGISDialog</class>
+ <widget class="QMainWindow" name="SpeckleQArcGISDialog">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>400</width>
+    <height>600</height>
+   </rect>
+  </property>
+  <widget class="QWidget" name="dockWidgetContents">
+   <layout class="QGridLayout" name="gridLayout">
+    <item row="0" column="0">
+     <layout class="QVBoxLayout" name="verticalLayout">
+      <item>
+        <layout class="QGridLayout" name="gridLayoutTitleBar">
+        </layout>
+      </item>
+      
+      <item>
+       <layout class="QFormLayout" name="formLayout">
+        <property name="leftMargin">
+         <number>20</number>
+        </property>
+        <property name="topMargin">
+         <number>10</number>
+        </property>
+        <property name="rightMargin">
+         <number>30</number>
+        </property>
+        <property name="bottomMargin">
+         <number>10</number>
+        </property>
+
+        
+        <item row="0" column="1">
+          <layout class="QHBoxLayout" name="streamListButtons">
+          </layout>
+        </item>
+
+        <item row="1" column="0">
+         <widget class="QLabel" name="streamListLabel">
+          <property name="text">
+           <string>Stream</string>
+          </property>
+         </widget>
+        </item>
+
+        <item row="1" column="1">
+          <layout class="QHBoxLayout" name="streamListButtons" stretch="20,1">
+            <item>
+              <widget class="QComboBox" name="streamList"/>
+            </item>
+            <item>
+            <widget class="QPushButton" name="streams_add_button">
+              <property name="text">
+              <string> </string>
+              </property>
+            </widget>
+            </item>
+            <item>
+              <widget class="QPushButton" name="streams_remove_button">
+                <property name="text">
+                <string> </string>
+                </property>
+                  <property name="geometry">
+                    <rect>
+                      <x>0</x>
+                      <y>0</y>
+                      <width>10</width>
+                      <height>10</height>
+                    </rect>
+                  </property>
+              </widget>
+            </item>
+          </layout>
+        </item>
+
+
+
+        <item row="2" column="0">
+         <widget class="QLabel" name="streamBranchLabel">
+          <property name="text">
+           <string>Branch</string>
+          </property>
+         </widget>
+        </item>
+        <item row="2" column="1">
+         <widget class="QComboBox" name="streamBranchDropdown"/>
+        </item>
+        <item row="3" column="0">
+         <widget class="QLabel" name="commitLabel">
+          <property name="text">
+           <string>Commit</string>
+          </property>
+         </widget>
+        </item>
+        <item row="3" column="1">
+         <widget class="QComboBox" name="commitDropdown"/>
+        </item>
+        
+        <item row="4" column="1">
+          <layout class="QHBoxLayout" name="horizontalLayout">
+         </layout>
+        </item>
+
+        
+        <item row="5" column="1">
+         <layout class="QHBoxLayout" name="horizontalLayout">
+          <item>
+           <widget class="QPushButton" name="sendModeButton">
+            <property name="enabled">
+             <bool>true</bool>
+            </property>
+            <property name="text">
+             <string>Send</string>
+            </property>
+           </widget>
+          </item>
+          <item>
+           <widget class="QPushButton" name="receiveModeButton">
+            <property name="text">
+             <string>Receive</string>
+            </property>
+           </widget>
+          </item>
+         </layout>
+        </item>
+
+        <item row="6" column="1">
+         <widget class="QComboBox" name="layerSendModeDropdown"/>
+        </item>
+
+
+
+        <item row="7" column="1">
+         <widget class="QListWidget" name="layersWidget">
+          <property name="selectionMode">
+           <enum>QAbstractItemView::NoSelection</enum>
+          </property>
+          <property name="sizePolicy">
+           <sizepolicy hsizetype="Expanding" vsizetype="MinimumExpanding">
+            <horstretch>0</horstretch>
+            <verstretch>0</verstretch>
+           </sizepolicy>
+          </property>
+          <property name="sizeAdjustPolicy">
+           <enum>QAbstractScrollArea::AdjustToContents</enum>
+          </property>
+          <property name="resizeMode">
+           <enum>QListView::Fixed</enum>
+          </property>
+          <property name="viewMode">
+           <enum>QListView::ListMode</enum>
+          </property>
+         </widget>
+        </item>
+
+
+        <item row="8" column="1">
+          <layout class="QHBoxLayout" name="horizontalLayout"  stretch="1,1">
+            <item>
+            <spacer name="horizontalSpacer">
+              <property name="orientation">
+              <enum>Qt::Horizontal</enum>
+              </property>
+              <property name="sizeType">
+              <enum>QSizePolicy::Expanding</enum>
+              </property>
+            </spacer>
+            </item>
+
+            <item>
+              <widget class="QPushButton" name="saveLayerSelection">
+                <property name="enabled">
+                  <bool>true</bool>
+                </property>
+                <property name="text">
+                  <string>Set visible layers as selection</string>
+                </property>
+              </widget>
+            </item>
+          </layout>
+        </item>
+
+        <item row="9" column="0">
+         <widget class="QLabel" name="messageLabel">
+          <property name="text">
+           <string>Message</string>
+          </property>
+         </widget>
+        </item>
+        <item row="9" column="1">
+         <widget class="QLineEdit" name="messageInput">
+          <property name="placeholderText">
+           <string>Sent XXX objects from ArcGIS</string>
+          </property>
+         </widget>
+        </item>
+
+        
+
+        <item row="10" column="1">
+         <layout class="QHBoxLayout" name="horizontalLayout">
+          <item>
+          <spacer name="horizontalSpacer">
+            <property name="orientation">
+             <enum>Qt::Horizontal</enum>
+            </property>
+            <property name="sizeType">
+             <enum>QSizePolicy::Expanding</enum>
+            </property>
+          </spacer>
+          </item>
+
+          <item>
+           <widget class="QPushButton" name="runButton">
+            <property name="text">
+             <string> SEND</string>
+            </property>
+           </widget>
+          </item>
+
+          <item>
+          <spacer name="horizontalSpacer">
+            <property name="orientation">
+             <enum>Qt::Horizontal</enum>
+            </property>
+            <property name="sizeType">
+             <enum>QSizePolicy::Expanding</enum>
+            </property>
+          </spacer>
+          </item>
+
+         </layout>
+        </item>
+
+        <item row="11" column="0">
+         <widget class="QLabel" name="surveyPointLabel">
+          <property name="text">
+           <string>Lat, Lon</string>
+          </property>
+         </widget>
+        </item>
+        <item row="11" column="1">
+          <layout class="QHBoxLayout" name="horizontalLayout">
+            <item>
+              <widget class="QLineEdit" name="surveyPointLat">
+                <property name="placeholderText">
+                <string>0.0</string>
+                </property>
+              </widget>
+            </item>
+            <item>
+              <widget class="QLineEdit" name="surveyPointLon">
+                <property name="placeholderText">
+                <string>0.0</string>
+                </property>
+              </widget>
+            </item>
+
+            <item>
+              <widget class="QPushButton" name="saveSurveyPoint">
+                <property name="enabled">
+                  <bool>true</bool>
+                </property>
+                <property name="text">
+                  <string>Set as a project center</string>
+                </property>
+              </widget>
+            </item>
+
+          </layout>
+        </item>
+        
+        <item row="12" column="1">
+          <layout class="QHBoxLayout" name="horizontalLayout">
+            
+            <item>
+              <widget class="QPushButton" name="reloadButton">
+                <property name="enabled">
+                <bool>true</bool>
+                </property>
+                <property name="text">
+                <string>Refresh</string>
+                </property>
+              </widget>
+            </item>
+            <item>
+              <widget class="QPushButton" name="closeButton">
+                <property name="enabled">
+                  <bool>true</bool>
+                </property>
+                <property name="text">
+                  <string>Close</string>
+                </property>
+              </widget>
+            </item>
+
+          </layout>
+        </item>
+
+        
+
+       </layout>
+      </item>
+     </layout>
+    </item>
+   </layout>
+  </widget>
+ </widget>
+ <resources/>
+ <connections/>
+</ui>

speckle_toolbox/esri/toolboxes/speckle/ui/streamlist_dialog.py

@@ -0,0 +1,19 @@
+import os
+from PyQt5 import QtWidgets, uic
+from PyQt5.QtCore import pyqtSignal
+
+# This loads your .ui file so that PyQt can populate your plugin with the elements from Qt Designer
+
+ui_class = os.path.dirname(os.path.abspath(__file__)) + "/streamlist_dialog.ui"
+
+class StreamListDialog(QtWidgets.QWidget):
+    streams_add_button: QtWidgets.QPushButton
+    streams_reload_button: QtWidgets.QPushButton
+    streams_remove_button: QtWidgets.QPushButton
+
+
+    def __init__(self, parent=None):
+        super(StreamListDialog, self).__init__(parent)
+        uic.loadUi(ui_class, self) # Load the .ui file
+        self.show()
+

speckle_toolbox/esri/toolboxes/speckle/ui/streamlist_dialog.ui

@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>StreamListDialog</class>
+ <widget class="QWidget" name="StreamListDialog">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>400</width>
+    <height>70</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string>Form</string>
+  </property>
+  <layout class="QHBoxLayout" name="horizontalLayout">
+   <item>
+    <widget class="QPushButton" name="pushButton_3">
+     <property name="text">
+      <string>PushButton</string>
+     </property>
+    </widget>
+   </item>
+   <item>
+    <widget class="QPushButton" name="pushButton">
+     <property name="text">
+      <string>PushButton</string>
+     </property>
+    </widget>
+   </item>
+   <item>
+    <widget class="QPushButton" name="pushButton_2">
+     <property name="text">
+      <string>PushButton</string>
+     </property>
+    </widget>
+   </item>
+  </layout>
+ </widget>
+ <resources/>
+ <connections/>
+</ui>

speckle_toolbox/esri/toolboxes/speckle/ui/validation.py

@@ -0,0 +1,92 @@
+
+from typing import Union
+from specklepy.api.wrapper import StreamWrapper
+from specklepy.api.models import Stream, Branch, Commit 
+from specklepy.transports.server import ServerTransport
+from specklepy.api.client import SpeckleClient
+from specklepy.logging.exceptions import SpeckleException, GraphQLException
+
+import arcpy 
+try:
+    from speckle.plugin_utils.logger import logToUser
+except:
+    from speckle_toolbox.esri.toolboxes.speckle.plugin_utils.logger import logToUser
+  
+def tryGetStream (sw: StreamWrapper) -> Union[Stream, None]:
+    try:
+        client = sw.get_client()
+        stream = client.stream.get(id = sw.stream_id, branch_limit = 100, commit_limit = 100)
+        if isinstance(stream, GraphQLException):
+            raise SpeckleException(stream.errors[0]['message'])
+        return stream
+    except Exception as e: 
+        logToUser(str(e)) 
+        return None 
+
+def validateStream(streamWrapper: StreamWrapper) -> Union[Stream, None]:
+    try: 
+        stream = tryGetStream(streamWrapper)
+        if isinstance(stream, SpeckleException): return None
+
+        if stream.branches is None:
+            logToUser("Stream has no branches")
+            return None
+        return stream
+    except Exception as e:
+        logToUser(e)
+        return None
+
+
+def validateBranch(stream: Stream, branchName: str, checkCommits: bool) ->  Union[Branch, None]:
+    try:
+        branch = None
+        if not stream.branches or not stream.branches.items: 
+            return None
+        for b in stream.branches.items:
+            if b.name == branchName:
+                branch = b
+                break
+        if branch is None: 
+            logToUser("Failed to find a branch")
+            return None
+        if checkCommits == True:
+            if branch.commits is None:
+                logToUser("Failed to find a branch")
+                return None
+            if len(branch.commits.items)==0:
+                logToUser("Branch contains no commits")
+                return None
+        return branch
+    
+    except Exception as e: 
+        logToUser(str(e)) 
+        return None
+                
+def validateCommit(branch: Branch, commitId: str) -> Union[Commit, None]:
+    try:
+        commit = None
+        try: commitId = commitId.split(" | ")[0]
+        except: logToUser("Commit ID is not valid")
+
+        for i in branch.commits.items:
+            if i.id == commitId:
+                commit = i
+                break
+        if commit is None:
+            try: 
+                commit = branch.commits.items[0]
+                logToUser("Failed to find a commit. Receiving Latest")
+            except: 
+                logToUser("Failed to find a commit")
+                return None
+        return commit
+    except Exception as e:
+        logToUser(e)
+
+def validateTransport(client: SpeckleClient, streamId: str) -> Union[ServerTransport, None]:
+    try: 
+        transport = ServerTransport(client=client, stream_id=streamId)
+        return transport
+    except Exception as e: 
+        logToUser("Make sure you have sufficient permissions: " + str(e))
+        return None

tests/tests_unittest.py

@@ -10,6 +10,13 @@ from specklepy.logging.exceptions import GraphQLException, SpeckleException
 from specklepy.api.credentials import Account
 
 import unittest   # The test framework
+# remove SetUp
+# add different scenqrios for Streqm Wrapper including wrng ones
+# tree of all options for input or class outcome
+# use dict for types chech
+# issue with untestable class init 
+# "mocking objects" for tests or "faking"
+# get functions ut of INIT 
 
 class Test_InitializingClasses(unittest.TestCase):
     def setUp(self) -> None:
@@ -17,7 +24,7 @@ class Test_InitializingClasses(unittest.TestCase):
         self.speckle_input = speckleInputsClass()
         self.toolbox = Toolbox()
         self.speckleTool = Speckle()
-        self.test_stream = "https://speckle.xyz/streams/17b0b76d13"
+        self.test_stream = "https://speckle.xyz/streams////17b0b76d13"
     
     def text_all_toolbox(self):
         self.assertTrue(isinstance(self.toolbox.tools[0], Speckle))
@@ -34,6 +41,17 @@ class Test_InitializingClasses(unittest.TestCase):
         self.assertIsNone(self.toolbox_input.project)
         self.assertEqual(self.toolbox_input.stream_file_path, "")
 
+    def test_something(self):
+        # Arrange
+        toolbox_input: toolboxInputsClass = toolboxInputsClass()
+        
+        # Act
+        toolbox_input.setProjectStreams(StreamWrapper(self.test_stream))
+
+        # Assert
+        os.path.exists(self.toolbox_input.stream_file_path)
+
+
     def test_toolbox_inputs_functions(self):
         self.toolbox_input.setProjectStreams(StreamWrapper(self.test_stream))
         if os.path.exists(self.toolbox_input.stream_file_path):
@@ -48,6 +66,7 @@ class Test_InitializingClasses(unittest.TestCase):
             existing_content = f.read()
             f.close()
             self.assertTrue(isinstance(existing_content, str))
+        self.assertIsInstance()
 
         self.assertTrue( isinstance(self.toolbox_input.get_survey_point(), tuple))
         self.assertTrue( isinstance(self.toolbox_input.get_survey_point()[0], float) or isinstance(self.toolbox_input.get_survey_point()[0], int))