Merge pull request #51 from specklesystems/kate/2.12

Kate/2.12

.circleci/config.yml

@@ -23,6 +23,12 @@ jobs:
       - checkout
       - attach_workspace:
           at: ./
+      - run:
+          name: Create Innosetup signing cert
+          shell: powershell.exe
+          command: |
+            echo $env:PFX_B64 > "speckle-sharp-ci-tools\SignTool\AEC Systems Ltd.txt"
+            certutil -decode "speckle-sharp-ci-tools\SignTool\AEC Systems Ltd.txt" "speckle-sharp-ci-tools\SignTool\AEC Systems Ltd.pfx"
       - run:
           name: Patch
           shell: powershell.exe
@@ -34,7 +40,14 @@ jobs:
             $version = "$($ver).$($env:CIRCLE_BUILD_NUM)"
             echo $semver
             python patch_version.py $semver
-            speckle-sharp-ci-tools\InnoSetup\ISCC.exe speckle-sharp-ci-tools\arcgis.iss
+            python setup.py sdist bdist_wheel
+            Copy-Item -Path "dist\speckle_toolbox-$($ver)-py3-none-any.whl" -Destination "speckle_arcgis_installer"
+      - run: 
+          name: Build Installer
+          shell: cmd.exe
+          command:
+            | # If no tag, use 0.0.0.1 and don't make any YML (for testing only!)
+            speckle-sharp-ci-tools\InnoSetup\ISCC.exe speckle-sharp-ci-tools\arcgis.iss /Sbyparam=$p
       - when:
           condition: << parameters.installer >>
           steps:
@@ -42,14 +55,38 @@ jobs:
                 root: ./
                 paths:
                   - speckle-sharp-ci-tools/Installers
+  publish-github-release:
+    docker:
+      - image: cimg/base:2021.01
+    steps:
+      - attach_workspace:
+          at: ./speckle_arcgis_installer
+      - run:
+          name: "Publish Release on GitHub"
+          command: |
+            go get github.com/tcnksm/ghr
+            VERSION=$(my-binary --version)
+            $tag = if([string]::IsNullOrEmpty($env:CIRCLE_TAG)) { "0.0.0" } else { $env:CIRCLE_TAG }
+            $semver = if($tag.Contains('/')) {$tag.Split("/")[1] } else { $tag }
+            $ver = if($semver.Contains('-')) {$semver.Split("-")[0] } else { $semver }
+            ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete ${VERSION} ./speckle_arcgis_installer/speckle_toolbox-$($ver)-py3-none-any.whl
 
   get-ci-tools: # Clones our ci tools and persists them to the workspace
     docker:
       - image: cimg/base:2021.01
     steps:
-      - run: # Could not get ssh to work, so using a personal token
+      - add_ssh_keys:
+          fingerprints:
+            - "77:64:03:93:c5:f3:1d:a6:fd:bd:fb:d1:05:56:ca:e9"
+      - run:
+          name: I know Github as a host
+          command: |
+            mkdir ~/.ssh
+            touch ~/.ssh/known_hosts
+            ssh-keyscan github.com >> ~/.ssh/known_hosts
+      - run:
           name: Clone
-          command: git clone https://$GITHUB_TOKEN@github.com/specklesystems/speckle-sharp-ci-tools.git speckle-sharp-ci-tools
+          command: git clone git@github.com:specklesystems/speckle-sharp-ci-tools.git speckle-sharp-ci-tools
       - persist_to_workspace:
           root: ./
           paths:
@@ -82,6 +119,7 @@ workflows: #happens with every PR to main
   build: # build the installers, but don't persist to workspace for deployment
     jobs:
       - get-ci-tools:
+          context: github-dev-bot
           filters:
             branches:
               only:
@@ -95,10 +133,21 @@ workflows: #happens with every PR to main
               only:
                 - main
                 - /ci\/.*/
+          context: innosetup
+      - publish-github-release:
+          requires:
+            - build-connector-win
+          filters:
+            tags:
+              only: /([0-9]+)\.([0-9]+)\.([0-9]+)(?:-\w+)?$/
+            branches:
+              ignore: /.*/
+          context: innosetup
 
   deploy: # build installers and deploy
     jobs:
       - get-ci-tools:
+          context: github-dev-bot
           filters:
             tags:
               only: /.*/
@@ -116,16 +165,25 @@ workflows: #happens with every PR to main
               only: /([0-9]+)\.([0-9]+)\.([0-9]+)(?:-\w+)?$/
             branches:
               ignore: /.*/
-
-      - deploy-manager2:
-          slug: arcgis
-          os: Win
-          extension: exe
+          context: innosetup
+      - publish-github-release:
           requires:
-            - get-ci-tools
             - build-deploy-connector-win
           filters:
             tags:
               only: /([0-9]+)\.([0-9]+)\.([0-9]+)(?:-\w+)?$/
             branches:
-              ignore: /.*/ # For testing only! /ci\/.*/
+              ignore: /.*/
+          context: innosetup
+      - deploy-manager2:
+          slug: arcgis
+          os: Win
+          extension: exe
+          requires:
+            - build-deploy-connector-win
+          filters:
+            tags:
+              only: /([0-9]+)\.([0-9]+)\.([0-9]+)(?:-\w+)?$/
+            branches:
+              ignore: /.*/ # For testing only! /ci\/.*/
+          context: do-spaces-speckle-releases

.github/workflows/close-issue.yml

@@ -0,0 +1,12 @@
+name: Update issue Status
+
+on:
+  issues:
+    types: [closed]
+
+jobs:
+  update_issue:
+    uses: specklesystems/github-actions/.github/workflows/project-add-issue.yml@main
+    secrets: inherit
+    with:
+      issue-id: ${{ github.event.issue.node_id }}

.github/workflows/open-issue.yml

@@ -0,0 +1,12 @@
+name: Move new issues into Project
+
+on:
+  issues:
+    types: [opened]
+
+jobs:
+  track_issue:
+    uses: specklesystems/github-actions/.github/workflows/project-add-issue.yml@main
+    secrets: inherit
+    with:
+      issue-id: ${{ github.event.issue.node_id }}

.gitignore

@@ -118,4 +118,6 @@ scratch.py
 settings.json
 **/.DS_Store
 zip_build
-.qt_for_python
+.qt_for_python
+*.pyt.xml
+

Readme.md

@@ -74,7 +74,7 @@ Setup is adapted from [this tutorial](https://pro.arcgis.com/en/pro-app/2.8/arcp
 
 #### Dev Environment
 
-For a better development experience in your editor, we recommend creating a [virtual environment in ArcGIS](https://pro.arcgis.com/en/pro-app/2.8/arcpy/get-started/work-with-python-environments.htm). In the venv, you'll just need to install `specklepy`. 
+For a better development experience in your editor, we recommend creating a [virtual conda environment in ArcGIS](https://pro.arcgis.com/en/pro-app/2.8/arcpy/get-started/work-with-python-environments.htm). In the new conda environment, you'll just need to install `specklepy` and `panda3d`. 
 
 ### Debugging
 

patch_version.py

@@ -4,16 +4,52 @@ import sys
 def patch_installer(tag):
     """Patches the installer with the correct connector version and specklepy version"""
     iss_file = "speckle-sharp-ci-tools/arcgis.iss"
+    setup_whl_file = "setup.py"
+    conda_file = "speckle_arcgis_installer/conda_clone_activate.py"
+    toolbox_install_file = "speckle_arcgis_installer/toolbox_install.py"
+    toolbox_manual_install_file = "speckle_arcgis_installer/toolbox_install_manual.py"
 
     #py_tag = get_specklepy_version()
     with open(iss_file, "r") as file:
         lines = file.readlines()
-        lines.insert(12, f'#define AppVersion "{tag.split("-")[0]}"\n')
-        lines.insert(13, f'#define AppInfoVersion "{tag}"\n')
-
+        for i, line in enumerate(lines):
+            if "#define AppVersion " in line: 
+                lines[i] = f'#define AppVersion "{tag.split("-")[0]}"\n'
+            if "#define AppInfoVersion " in line: 
+                lines[i] = f'#define AppInfoVersion "{tag}"\n'
         with open(iss_file, "w") as file:
             file.writelines(lines)
             print(f"Patched installer with connector v{tag} and specklepy ")
+    file.close()
+
+    with open(setup_whl_file, "r") as file:
+        lines = file.readlines()
+        for i, line in enumerate(lines):
+            if "version=" in line: 
+                lines[i] = f'\t\t\tversion="{tag.split("-")[0]}",\n'
+                break
+        with open(setup_whl_file, "w") as file:
+            file.writelines(lines)
+            print(f"Patched whl setup with connector v{tag} and specklepy ")
+    file.close()
+
+    def whlFileRename(fileName: str): 
+        with open(fileName, "r") as file:
+            lines = file.readlines()
+            for i, line in enumerate(lines):
+                if "-py3-none-any.whl" in line and '.sort' not in line: 
+                    p1 = line.split("-py3-none-any.whl")[0].split("-")[0]
+                    p2 = f'{tag.split("-")[0]}'
+                    p3 = line.split("-py3-none-any.whl")[1]
+                    lines[i] = p1+"-"+p2+"-py3-none-any.whl"+p3
+            with open(fileName, "w") as file:
+                file.writelines(lines)
+                print(f"Patched toolbox_installer with connector v{tag} and specklepy ")
+        file.close()
+
+    whlFileRename(conda_file)
+    whlFileRename(toolbox_install_file)
+    whlFileRename(toolbox_manual_install_file)
 
 
 def main():

requirements.txt

@@ -0,0 +1,3 @@
+specklepy==2.9.1
+panda3d==1.10.11
+

scripts/debugging.py

@@ -0,0 +1,2 @@
+print("Hello")
+

scripts/debugging_panel.py

@@ -0,0 +1,324 @@
+
+from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
+import arcpy
+import json
+import os
+
+try: 
+    from speckle.converter.layers.CRS import CRS
+    from speckle.converter.layers.Layer import Layer, VectorLayer, RasterLayer
+except: 
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.CRS import CRS
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.Layer import Layer, VectorLayer, RasterLayer
+
+from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
+from arcpy.management import (CreateFeatureclass, MakeFeatureLayer,
+                              AddFields, AlterField, DefineProjection )
+
+from specklepy.objects import Base
+
+##################################################### get example layers from the project #######
+project = ArcGISProject('CURRENT')
+active_map = project.activeMap
+all_layers = []
+
+layerPolygon = None
+layerPolyline = None
+layerPoint = None
+layerMultiPoint = None
+layerRaster = None
+#get layer of interest
+for layer in active_map.listLayers(): 
+   if layer.isFeatureLayer or layer.isRasterLayer: 
+        all_layers.append(layer)
+        data = arcpy.Describe(layer.dataSource)
+        if layer.isRasterLayer and layerRaster is None: layerRaster = layer 
+        if layer.isFeatureLayer:
+            geomType = data.shapeType
+            if geomType == "Polygon" and layerPolygon is None: layerPolygon = layer 
+            if geomType == "Polyline" and layerPolyline is None: layerPolyline = layer 
+            if geomType == "Point" and layerPoint is None: layerPoint = layer 
+            if geomType == "Multipoint" and layerMultiPoint is None: layerMultiPoint = layer 
+
+################## reset symbology if needed:
+sym = layerPolygon.symbology
+print(sym.renderer.type)
+sym.updateRenderer('UniqueValueRenderer')
+layerPolygon.symbology = sym
+print(sym.updateRenderer('UniqueValueRenderer'))
+print(layerPolygon.symbology.renderer.type)
+# SimpleRenderer, GraduatedColorsRenderer, GraduatedSymbolsRenderer, UnclassedColorsRenderer, UniqueValueRenderer 
+
+######################################### change symbology ################################# 
+
+for k, grp in enumerate(sym.renderer.groups):
+    for itm in grp.items:
+        print(itm)
+        print(itm.values)
+        print(itm.symbol.color)
+        transVal = itm.values[0][0] #Grab the first "percent" value in the list of potential values
+        print(transVal)
+        for i in range(len(cats)):
+            label = cats[i]['value'] 
+            print(label)
+            if label is None or label=="": label = "<Null>"
+            print(label)
+
+
+
+from speckle.converter.layers.symbologyTemplates import get_polygon_simpleRenderer 
+from arcpy._mp import ArcGISProject
+
+aprx = ArcGISProject('CURRENT')
+root_path = "\\".join(aprx.filePath.split("\\")[:-1])
+
+path_style = root_path + '\\layer_speckle_symbology.lyrx'
+path_style2 = root_path + '\\layer_speckle_symbology2.lyrx'
+#arcpy.management.SaveToLayerFile(layerPolygon, path_style, False)
+print(layerPolygon.dataSource)
+arcpy.management.ApplySymbologyFromLayer(
+                        in_layer=layerPolygon.dataSource, 
+                        in_symbology_layer=path_style2, 
+                        update_symbology='UPDATE')
+
+
+
+f = open(path_style, "r")
+renderer = json.loads(f.read())
+
+renderer["layerDefinitions"][0]["renderer"] = get_polygon_simpleRenderer(1,2,150)
+f = open(path_style2, "w")
+f.write(json.dumps(renderer, indent=4))
+f.close()
+arcpy.management.ApplySymbologyFromLayer(str(layerPolygon), path_style2)
+os.remove(path_style)
+os.remove(path_style2)
+
+###########################################################################
+layer = all_layers[0] 
+if isinstance(layer, arcLayer): 
+
+    projectCRS = project.activeMap.spatialReference
+    try: data = arcpy.Describe(layer.dataSource)
+    except OSError as e: print(e)
+
+    layerName = layer.name
+    crs = data.SpatialReference
+    units = "m"
+    layerObjs = []
+
+    # Convert CRS to speckle, use the projectCRS
+    speckleReprojectedCrs = CRS(name = projectCRS.name, wkt = projectCRS.exportToString(), units = units)
+    layerCRS = CRS(name=crs.name, wkt=crs.exportToString(), units = units) 
+
+    if layer.isFeatureLayer: 
+        print("VECTOR LAYER HERE")
+        
+        speckleLayer = VectorLayer(units = "m")
+        speckleLayer.type="VectorLayer"
+        speckleLayer.name = layerName
+        speckleLayer.crs = speckleReprojectedCrs
+
+        if data.datasetType == "FeatureClass": #FeatureClass, ?Table Properties, ?Datasets
+            # write feature attributes
+            fieldnames = [field.name for field in data.fields]
+            rows_shapes = arcpy.da.SearchCursor(layer.longName, "Shape@") # arcpy.da.SearchCursor(in_table, field_names, {where_clause}, {spatial_reference}, {explode_to_points}, {sql_clause})
+            print("__ start iterating features")
+            row_shapes_list = [x for k, x in enumerate(rows_shapes)]
+            for i, features in enumerate(row_shapes_list):
+
+                print("____error Feature # " + str(i+1)) # + " / " + str(sum(1 for _ in enumerate(rows_shapes))))
+                if features[0] is None: continue 
+                feat = features[0]
+
+                if feat is not None: 
+                    print(feat)
+                    rows_attributes = arcpy.da.SearchCursor(layer.longName, fieldnames)
+                    row_attr = []
+                    for k, attrs in enumerate(rows_attributes):
+                        if i == k: row_attr = attrs; break
+                    if feat.hasCurves: feat = feat.densify("ANGLE", 1000, 0.12)
+
+                    print("___________Feature to Speckle____________")
+
+                    b = Base(units = "m")
+                    data = arcpy.Describe(layer.dataSource)
+                    layer_sr = data.spatialReference # if sr.type == "Projected":
+                    geomType = data.shapeType #Polygon, Point, Polyline, Multipoint, MultiPatch
+                    featureType = data.featureType # Simple,SimpleJunction,SimpleJunction,ComplexEdge,Annotation,CoverageAnnotation,Dimension,RasterCatalogItem 
+                    print(geomType)
+                    print(hasattr(data, "isRevit")) 
+                    print(hasattr(data, "isIFC")) 
+                    print(hasattr(data, "bimLevels")) 
+                    print(hasattr(data, "hasSpatialIndex")) 
+                    if geomType == "MultiPatch" or hasattr(data, "isRevit") or hasattr(data, "isIFC") or hasattr(data, "bimLevels"): 
+                        print(f"Layer {layer.name} has unsupported data type")
+
+                        print("___convertToSpeckle____________")
+                        geom = feat
+                        print(geom.isMultipart) # e.g. False 
+                        print(geom.hasCurves)
+                        print(geom.partCount)
+                        geomMultiType = geom.isMultipart
+                        hasCurves = feat.hasCurves 
+
+                        geomPart = []
+                        for i,x in enumerate(feat): # [[x,x,x]
+                            
+                            if i==0:
+                                print("Part # " + str(i+1))
+                                print(x)
+
+                                inner_arr = []
+                                for k,ptn in enumerate(x):  
+                                    if k<10: print(ptn) # e.g. 6.25128173828125 -9.42138671875 22.2768999999971 NaN
+
+                                    inner_arr.append(ptn)
+                                    #inner_arr.append(inner_arr[0]) #add first in the end
+                                geomPart.append(arcpy.Array(inner_arr))
+
+                                geomPartArray = arcpy.Array(inner_arr)
+                                sr = project.activeMap.spatialReference
+
+                                multipatch = arcpy.Multipatch(arcpy.Array(x), sr, has_z=True) # error 
+                                print(multipatch)
+                        
+                    else:
+                        print("___convertToSpeckle____________")
+                        geom = feat
+                        print(geom.isMultipart) # e.g. False 
+                        print(geom.hasCurves)
+                        print(geom.partCount)
+                        geomMultiType = geom.isMultipart
+                        hasCurves = feat.hasCurves 
+
+                        for i,x in enumerate(feat): # [[x,x,x]
+                            print("Part # " + str(i+1))
+                            print(x)
+                            for k,ptn in enumerate(x): 
+                                if k<10: print(ptn) # e.g. 6.25128173828125 -9.42138671875 22.2768999999971 NaN
+
+
+path: str = project.filePath.replace("aprx","gdb") 
+sr = project.activeMap.spatialReference
+print(sr)
+f_class = CreateFeatureclass(path, "NewTestLayer", "Multipatch", has_z="ENABLED", spatial_reference = sr)
+fets = []
+print("04_____Feature To Native____________") 
+new_feat = {}
+new_feat.update({"arcGisGeomFromSpeckle": multipatch})
+fets.append(new_feat)
+
+vl = MakeFeatureLayer(str(f_class), "NewTestLayer").getOutput(0)
+
+
+
+############################# write shapefile ##################################
+
+import shapefile
+from shapefile import TRIANGLE_STRIP, TRIANGLE_FAN
+
+from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
+from arcpy.management import (CreateFeatureclass, MakeFeatureLayer,
+                              AddFields, AlterField, DefineProjection )
+
+from specklepy.objects import Base
+
+project = ArcGISProject('CURRENT')
+path: str = project.filePath.replace("aprx","gdb") 
+
+#with shapefile.Writer(path + "\contextwriter") as w:
+#    w.field('field1', 'C')
+#    pass
+
+w = shapefile.Writer(path + '\\dtype')
+w.field('TEXT', 'C')
+w.field('SHORT_TEXT', 'C', size=5)
+w.field('LONG_TEXT', 'C', size=250)
+w.null()
+w.record('Hello', 'World', 'World'*50)
+w.close()
+
+r = shapefile.Reader(path + '\\dtype')
+assert r.record(0) == ['Hello', 'World', 'World'*50]
+################################################################### WORKS #################################
+
+w = shapefile.Writer(path + '\\dtype')
+w.field('INT', 'N')
+w.field('LOWPREC', 'N', decimal=2)
+w.field('MEDPREC', 'N', decimal=10)
+w.field('HIGHPREC', 'N', decimal=30)
+w.field('FTYPE', 'F', decimal=10)
+w.field('LARGENR', 'N', 101)
+w.field('FIRST_FLD','C','40')
+w.field('SECOND_FLD','C','40')
+nr = 1.3217328
+w.null()
+w.null()
+w.record(INT=nr, LOWPREC=nr, MEDPREC=nr, HIGHPREC=-3.2302e-25, FTYPE=nr, LARGENR=int(nr)*10**100, FIRST_FLD='First', SECOND_FLD='Line')
+w.record(None, None, None, None, None, None, '', '')
+w.close()
+
+r = shapefile.Reader(path + '\\dtype')
+assert r.record(0) == [1, 1.32, 1.3217328, -3.2302e-25, 1.3217328, 10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000, 'First', 'Line']
+assert r.record(1) == [None, None, None, None, None, None, '', '']
+
+################################################################# Add point ####################
+
+w = shapefile.Writer(path + '\\dtypeShapes')
+w.field('name', 'C')
+
+w.point(122, 37) 
+w.record('point1')
+
+w.close()
+
+################################################################# Add Multipatch ####################
+
+w = shapefile.Writer(path + '\\MultipatchTest2')
+w.field('name', 'C')
+
+w.multipatch([
+			 [[0,0,0],[0,0,3],[5,0,0],[5,0,3],[5,5,0],[5,5,3],[0,5,0],[0,5,3],[0,0,0],[0,0,3]], # TRIANGLE_STRIP for house walls
+			 [[2.5,2.5,5],[0,0,3],[5,0,3],[5,5,3],[0,5,3],[0,0,3]], # TRIANGLE_FAN for pointed house roof
+			 ],
+			 partTypes=[TRIANGLE_STRIP, TRIANGLE_FAN]) # one type for each part
+
+w.record('house1')
+w.close()
+
+r = shapefile.Reader(path + '\\MultipatchTest2')
+assert r.record(0) == ['house1']
+
+
+active_map.addDataFromPath(path + '\\MultipatchTest2.shp')
+
+########################################################################## reader 
+sf = shapefile.Reader(path + '\\MultipatchTest2.shp')
+sf.shapeType # e.g. 31 - multipatch
+sf.bbox # e.g. [0.0, 0.0, 5.0, 5.0]
+shapefile.Shape
+
+
+##################################################### cerate multipatch layer #################################
+result = arcpy.management.CreateFeatureclass(arcpy.env.scratchGDB, "test_multipatch", "MULTIPATCH", has_z="ENABLED", spatial_reference=4326)
+feature_class = result[0]
+
+
+################################# reading shapefile - works ####################
+
+fc = r'C:\Users\katri\Documents\ArcGIS\Projects\MyProject\Layers_Speckle\BIM_layers_speckle\00f70159b9104180f622cca87f5dd2cb.shp'
+rows = arcpy.da.SearchCursor(fc, 'Shape@')
+for r in rows:
+        if r is not None: shape = r
+print(shape)
+cl = arcpy.conversion.FeatureClassToFeatureClass(r'C:\Users\katri\Documents\ArcGIS\Projects\MyProject\Layers_Speckle\BIM_layers_speckle\16d73b756a_main_2f8cfa8644\__Floors_Mesh\00c7696966e4cfda2bd8c03860a414a6', r'C:\Users\katri\Documents\ArcGIS\tests', 'copyclass')
+
+##################################### update rows in feature class - working #############
+with arcpy.da.UpdateCursor('f_class_2f8cfa8644___Structural_Framing_Mesh', 'name') as cursor:
+    # For each row, evaluate the WELL_YIELD value (index position 
+    # of 0), and update WELL_CLASS (index position of 1)
+    for row in cursor: 
+        row[0] = "newName"
+        cursor.updateRow(row)
+

setup.py

@@ -1,7 +1,7 @@
 # to build an installer: run cmd from this folder or use terminal: "%PROGRAMFILES%\\ArcGIS\\Pro\\bin\\Python\\envs\\arcgispro-py3\\python.exe" 
 # 
 # 1) python patch_version.py 2.x.x 
-# 2) (if needed: pip install wheel) python setup.py sdist bdist_wheel   #C:\\Users\\username\\Documents\\00_Speckle\\GitHub\\speckle-arcgis\\setup.py sdist bdist_wheel 
+# 2) python setup.py sdist bdist_wheel   #C:\\Users\\username\\Documents\\00_Speckle\\GitHub\\speckle-arcgis\\setup.py sdist bdist_wheel 
 # copy .whl from "dist" to "speckle_arcgis_installer" 
 
 # ref: https://pro.arcgis.com/en/pro-app/2.8/arcpy/geoprocessing_and_python/distributing-python-modules.htm
@@ -14,20 +14,23 @@ def read(fname):
     return open(os.path.join(os.path.dirname(__file__), fname)).read() 
  
 setup(name='speckle_toolbox', 
-      version='0.1',
       author='SpeckleSystems',
+			version="2.9.4",
+      author_email="connectors@speckle.systems",
+      url="https://speckle.systems/",
       description=("Example for extending geoprocessing through Python modules"),
       long_description=read('Readme.md'),
       python_requires='~=3.3',
+      setup_requires=['wheel'],
       packages=['speckle_toolbox'], 
-      package_data={'speckle_toolbox':['esri/toolboxes/*',  
+      package_data={'speckle_toolbox':[
                   'esri/arcpy/*', 
                   'esri/help/gp/*', 'esri/help/gp/toolboxes/*', 'esri/help/gp/messages/*',
                   'esri/toolboxes/*','esri/toolboxes/speckle/*',
                   'esri/toolboxes/speckle/converter/*', 'esri/toolboxes/speckle/converter/geometry/*', 'esri/toolboxes/speckle/converter/layers/*',
-                  'esri/toolboxes/speckle/plugin_utils/*'] 
+                  'esri/toolboxes/speckle/plugin_utils/*',
+                  'esri/toolboxes/speckle/ui/*'] 
                   }, 
-      setup_requires=['wheel'],
       )
 
 # then to install in ArcGIS:

speckle_arcgis_installer/Readme.md

@@ -0,0 +1,13 @@
+### Manual installation
+
+1. Download present "speckle_arcgis_installer" folder
+2. Clone the default ArcGIS Pro conda environment and restart ArcGIS Pro
+       - for 2.9.0: Project-> Python-> Manage Environments-> Clone Default
+       - for 3.0.0: Project-> Package Manager-> Active Environment (Environment Manager)-> Clone arcgispro-py3
+3. Change the path to your new environemnt Python.exe if necessary (variable "pythonPath" in "toolbox_install_manual.py") 
+4. Enter the location of 'toolbox_install_manual.py' in the following command and run this command in ArcGIS Python console (View -> Python Window)
+
+```python
+import sysconfig; import subprocess; x = sysconfig.get_paths()['data'] + r"\python.exe"; subprocess.run((x, 'C:\\Users\\pathToFolder\\speckle_arcgis_installer\\toolbox_install_manual.py'), capture_output=True, text=True, shell=True, timeout=1000 )
+```
+

speckle_arcgis_installer/Speckle.pyt

@@ -0,0 +1,2 @@
+try: from speckle.speckle_arcgis import *
+except: from speckle_toolbox.esri.toolboxes.speckle.speckle_arcgis import *

speckle_arcgis_installer/conda_clone_activate.py

@@ -8,50 +8,128 @@ import subprocess
 from subprocess import CalledProcessError
 from subprocess_call import subprocess_call
 
-from msilib.schema import Error
 import sys
 
-import arcpy 
-
 def setup():
-    #print(plugin_dir) 
     pythonExec = get_python_path() # import numpy; import os; print(os.path.abspath(numpy.__file__)) 
-    #print(pythonExec)
-    
-    if pythonExec is None: # env is default, need to restart ArcGIS
-        return False
+    return pythonExec # None if not successful 
     
 def get_python_path(): # create a full copy of default env 
+    #print("Get Python path")
     # or: import site; site.getsitepackages()[0]
     # import specklepy; import os; print(os.path.abspath(specklepy.__file__)) ##currentPythonExec = sysconfig.get_paths()['data'] + r"\python.exe"
 
     pythonExec = os.environ["ProgramFiles"] + r'\ArcGIS\Pro\bin\Python\envs\arcgispro-py3\python.exe' #(r"%PROGRAMFILES%\ArcGIS\Pro\bin\Python\envs\arcgispro-py3\python.exe") #os.path.dirname(sys.executable) + "\\python.exe" # default python.exe
     #print(pythonExec)
+    if not os.path.exists(pythonExec):
+        pythonExec = os.getenv('APPDATA').replace("Roaming", "Local") + r'\Programs\ArcGIS\Pro\bin\Python\envs\arcgispro-py3\python.exe'
+        if not os.path.exists(pythonExec): return None
+    #print(os.getenv('APPDATA') + r'\Programs\ArcGIS\Pro\bin\Python\envs\arcgispro-py3\python.exe')
+
     if sys.platform == "win32":
         env_new_name = "arcgispro-py3-speckle"
-        #clone_env(pythonExec, env_new_name) # only if doesn't exist yet 
+        newExec = clone_env(pythonExec, env_new_name) # only if doesn't exist yet 
+        if not os.path.exists(newExec): return None
+        
         activate_env(env_new_name)
-        return pythonExec
-    else: pass
+        return newExec
+    else: return None
 
 def clone_env(pythonExec_old: str, env_new_name: str): 
     install_folder = os.getenv('APPDATA').replace("\\Roaming","") + r"\Local\ESRI\conda\envs"   #r"%LOCALAPPDATA%\ESRI\conda\envs" 
-    #print("Clone default ArcGIS Pro conda env")
-    #print(install_folder)
-    #if not os.path.exists(install_folder): os.makedirs(install_folder)
+    if not os.path.exists(install_folder): os.makedirs(install_folder)
     
     default_env = pythonExec_old.replace("Pro\\bin\\Python\\envs\\arcgispro-py3\\python.exe","Pro\\bin\\Python\\envs\\arcgispro-py3") # + "\\" + 'arcgispro-py3'
     conda_exe = pythonExec_old.replace("Pro\\bin\\Python\\envs\\arcgispro-py3\\python.exe","Pro\\bin\\Python\\Scripts\\conda.exe") #%PROGRAMFILES%\ArcGIS\Pro\bin\Python\Scripts\conda.exe #base: %PROGRAMDATA%\Anaconda3\condabin\conda.bat
     new_env = install_folder + "\\" + env_new_name # %LOCALAPPDATA%\ESRI\conda\envs\...
 
-    subprocess_call( [ conda_exe, 'create', '--clone', default_env, '-p', new_env] ) # will not execute if already exists
+    if os.path.exists(conda_exe) and os.path.exists(default_env) and os.path.exists(new_env) and not os.path.exists(new_env + "\\python.exe"): 
+        # conda environment invalid: delete it's folder 
+        print(f"Removing invalid environment {new_env}")
+        os.remove(new_env)
 
+    if os.path.exists(conda_exe) and os.path.exists(default_env) and not os.path.exists(new_env): 
+        print("Wait for the default ArcGIS Pro conda environment to be cloned")
+        subprocess_call( [ conda_exe, 'config', '--set', 'ssl_verify', 'False'] )
+        subprocess_call( [ conda_exe, 'create', '--clone', default_env, '-p', new_env] ) # will not execute if already exists
+        subprocess_call( [ conda_exe, 'config', '--set', 'ssl_verify', 'True'] )
+
+    elif os.path.exists(new_env) and os.path.exists(new_env + "\\python.exe"):
+        print(f"Environment {new_env} already exists, preparing to install packages..")
+
+    print(new_env + "\\python.exe")
     return new_env + "\\python.exe"
 
 def activate_env(env_new_name: str):
     # using Popen, because process does not return result; subprocess.run will hang indefinitely 
     variable = subprocess.Popen((f'proswap {env_new_name}'),stdout = subprocess.PIPE,stderr = subprocess.PIPE,text = True,shell = True)
-    #print(variable) 
     # activate new env : https://support.esri.com/en/technical-article/000024206
 
-setup()
+
+def installToolbox(newExec: str):
+    print("Installing Speckle Toolbox")
+    whl_file = os.path.join(os.path.dirname(__file__), "speckle_toolbox-2.9.4-py3-none-any.whl" ) 
+    print(whl_file)
+    subprocess_call([newExec, '-m','pip','install','--upgrade', '--force-reinstall', whl_file])
+    # to uninstall: cmd.exe "C:\\Users\\username\\AppData\\Local\\ESRI\\conda\\envs\\arcgispro-2.9.4-py3-none-any.whl
+    return
+
+def installDependencies(pythonExec: str, pkgName: str, pkgVersion: str):
+    # install pip
+    print(pythonExec)
+    try:
+        import pip
+    except:
+        getPipFilePath = os.path.join(os.path.dirname(__file__), "get_pip.py") #TODO: give actual folder path 
+        exec(open(getPipFilePath).read())
+        # just in case the included version is old
+        subprocess_call([pythonExec, "-m", "pip", "install", "--upgrade", "pip"])
+        
+    # install package
+    try:
+        #import importlib #importlib.import_module(pkgName)
+        if pkgName == "specklepy":
+            import specklepy 
+            if pythonExec.replace("\\python.exe","") not in (os.path.abspath(specklepy.__file__)): 
+                print(f"Installing {pkgName} to {pythonExec}")
+                #subprocess_call( [pythonExec, "-m", "pip", "uninstall", f"{pkgName}"])
+                subprocess_call( [pythonExec, "-m", "pip", "install", f"{pkgName}=={pkgVersion}"])
+        elif pkgName == "panda3d":
+            import panda3d 
+            if pythonExec.replace("\\python.exe","") not in (os.path.abspath(panda3d.__file__)): 
+                print(f"Installing {pkgName} to {pythonExec}")
+                subprocess_call( [pythonExec, "-m", "pip", "install", f"{pkgName}=={pkgVersion}"])
+    except Exception as e:
+        print(f"{pkgName} not installed")
+        subprocess_call( [pythonExec, "-m", "pip", "install", f"{pkgName}=={pkgVersion}"])
+
+    # Check if package needs updating
+    r'''
+    try:
+        print(f"Attempting to update {pkgName} to {pkgVersion}")
+        result = subprocess_call(
+            [
+                pythonExec,
+                "-m",
+                "pip",
+                "install",
+                "--upgrade",
+                f"{pkgName}=={pkgVersion}",
+            ]
+        )
+        if result == True:
+            print(f"{pkgName} upgraded")
+            return True
+        else: 
+            return False
+    except Exception as e:
+        print(e)
+        print(e.with_traceback)
+    '''
+    return True
+    
+pythonPath = setup()
+if pythonPath is not None:
+    installToolbox(pythonPath)
+    installDependencies(pythonPath, "specklepy", "2.9.0"  )
+    installDependencies(pythonPath, "panda3d", "1.10.11" )

speckle_arcgis_installer/manual_toolbox_install.py

@@ -1,73 +0,0 @@
-# MANUAL INSTALLATION: 
-#   1. enter correct path to Python exe of your new environemnt in line 10 
-#   2. enter the location of 'manual_toolbox_install.py' in the following command and run this command in ArcGIS Python console (View -> Python Window)
-#   import sysconfig; import subprocess; x = sysconfig.get_paths()['data'] + r"\python.exe"; subprocess.run((x, 'C:\\...\\manual_toolbox_install.py'), capture_output=True, text=True, shell=True, timeout=1000 )
-# then restart
-
-from subprocess_call import subprocess_call
-import os 
-
-pythonPath = "C:\\ ...\\custom_environment_name\\python.exe"
-
-def installToolbox(newExec: str):
-    print("Installing Speckle Toolbox")
-    whl_file = os.path.join(os.path.dirname(__file__), "speckle_toolbox-0.1-py3-none-any.whl" ) 
-    subprocess_call([newExec, '-m','pip','install','--upgrade', '--force-reinstall', whl_file])
-    return
-
-def installDependencies(pythonExec: str):
-    print("Installing dependencies")
-    print(pythonExec)
-    try:
-        import pip
-    except:
-        getPipFilePath = os.path.join(os.path.dirname(__file__), "get_pip.py") 
-        exec(open(getPipFilePath).read())
-
-        # just in case the included version is old
-        subprocess_call([pythonExec, "-m", "pip", "install", "--upgrade", "pip"])
-    
-    pkgVersion = "2.7.4" 
-    pkgName = "specklepy"
-    try:
-        import specklepy 
-    except Exception as e:
-        subprocess_call([ pythonExec, "-m", "pip", "install", f"{pkgName}=={pkgVersion}"])
-
-    
-    pkgVersion = "1.10.11" 
-    pkgName = "panda3d"
-    try:
-        import panda3d
-    except Exception as e:
-        print("panda3d not installed")
-        subprocess_call( [pythonExec, "-m", "pip", "install", f"{pkgName}=={pkgVersion}"])
-
-    # Check if specklpy needs updating
-    try:
-        print(f"Attempting to update specklepy to {pkgVersion}")
-        # pip.main(['install', "specklepy==2.7.4"])
-        result = subprocess_call(
-            [
-                pythonExec,
-                "-m",
-                "pip",
-                "install",
-                "--upgrade",
-                f"{pkgName}=={pkgVersion}",
-            ]
-        )
-        if result == True:
-            print("specklepy upgraded")
-            return True
-        else: 
-            return False
-
-    except Exception as e:
-        print(e)
-        print(e.with_traceback)
-    return True
-
-installToolbox(pythonPath)
-installDependencies(pythonPath)
-

speckle_arcgis_installer/subprocess_call.py

@@ -14,12 +14,12 @@ def subprocess_call(*args, **kwargs):
         startupinfo.dwFlags = subprocess.CREATE_NEW_CONSOLE | subprocess.STARTF_USESHOWWINDOW
         startupinfo.wShowWindow = subprocess.SW_HIDE
         kwargs['startupinfo'] = startupinfo
-        print("start")
+        #print("start")
         #print(*args)
     try: 
         # if manually: cmd.exe -> conda activate [env folder] -> pip install specklepy 
         result = subprocess.run(*args, capture_output=True, text=True, shell=True, timeout=1000)
-        #print(result)
+        print(result)
         #result = subprocess.Popen( arg, shell=True, stdout=subprocess.PIPE) #, stderr=subprocess.STDOUT)
         #retcode = subprocess.check_call(*args, **kwargs) # Creates infinite loop, known issue: https://github.com/python/cpython/issues/87512
     except CalledProcessError as e: 
@@ -32,7 +32,7 @@ def subprocess_call(*args, **kwargs):
         #print(str(e))
         return False
     except: print("unknown error") 
-    print("end")
+    #print("end")
     return True
 
     

speckle_arcgis_installer/toolbox_install.py

@@ -6,43 +6,34 @@ pythonPath = os.getenv('APPDATA').replace("\\Roaming","") + r"\Local\ESRI\conda\
 
 def installToolbox(newExec: str):
     print("Installing Speckle Toolbox")
-    whl_file = os.path.join(os.path.dirname(__file__), "speckle_toolbox-0.1-py3-none-any.whl" ) 
+    whl_file = os.path.join(os.path.dirname(__file__), "speckle_toolbox-2.9.4-py3-none-any.whl" ) 
+    print(whl_file)
     subprocess_call([newExec, '-m','pip','install','--upgrade', '--force-reinstall', whl_file])
-    # to uninstall: cmd.exe "C:\\Users\\username\\AppData\\Local\\ESRI\\conda\\envs\\arcgispro-py3-speckle\\python.exe" -m pip uninstall C:\\Users\\username\\Downloads\\speckle-arcgis\\foo-0.1-py3-none-any.whl
+    # to uninstall: cmd.exe "C:\\Users\\username\\AppData\\Local\\ESRI\\conda\\envs\\arcgispro-2.9.4-py3-none-any.whl
     return
 
-def installDependencies(pythonExec: str):
-    #print("Installing dependencies")
+def installDependencies(pythonExec: str, pkgName: str, pkgVersion: str):
+    # install pip
     print(pythonExec)
     try:
         import pip
     except:
         getPipFilePath = os.path.join(os.path.dirname(__file__), "get_pip.py") #TODO: give actual folder path 
         exec(open(getPipFilePath).read())
-
         # just in case the included version is old
         subprocess_call([pythonExec, "-m", "pip", "install", "--upgrade", "pip"])
-    
-    pkgVersion = "2.9.0" 
-    pkgName = "specklepy"
+        
+    # install package
     try:
-        import specklepy # C:\Users\username\AppData\Roaming\Python\Python37\site-packages\specklepy\__init__.py 
+        import importlib
+        importlib.import_module(pkgName)
     except Exception as e:
-        subprocess_call([ pythonExec, "-m", "pip", "install", f"{pkgName}=={pkgVersion}"])
-
-    
-    pkgVersion = "1.10.11" 
-    pkgName = "panda3d"
-    try:
-        import panda3d
-    except Exception as e:
-        print("panda3d not installed")
+        print(f"{pkgName} not installed")
         subprocess_call( [pythonExec, "-m", "pip", "install", f"{pkgName}=={pkgVersion}"])
 
-    # Check if specklpy needs updating
+    # Check if package needs updating
     try:
-        print(f"Attempting to update specklepy to {pkgVersion}")
-        # pip.main(['install', "specklepy==2.7.4"])
+        print(f"Attempting to update {pkgName} to {pkgVersion}")
         result = subprocess_call(
             [
                 pythonExec,
@@ -54,16 +45,17 @@ def installDependencies(pythonExec: str):
             ]
         )
         if result == True:
-            print("specklepy upgraded")
+            print(f"{pkgName} upgraded")
             return True
         else: 
             return False
-
     except Exception as e:
         print(e)
         print(e.with_traceback)
     return True
 
-installToolbox(pythonPath)
-installDependencies(pythonPath)
+
+installToolbox(pythonPath)
+installDependencies(pythonPath, "specklepy", "2.9.0"  )
+installDependencies(pythonPath, "panda3d", "1.10.11" )
 

speckle_arcgis_installer/toolbox_install_manual.py

@@ -0,0 +1,63 @@
+# MANUAL INSTALLATION: 
+#   1. Clone the default ArcGIS Pro conda environment 
+#       for 2.9.0: Project-> Python-> Manage Environments-> Clone Default
+#       for 3.0.0: Project-> Package Manager-> Active Environment (Environment Manager)-> Clone arcgispro-py3
+#   2. Change the path to your new environemnt Python.exe if necessary (in variable "pythonPath" below, line 13) 
+#   3. Enter the location of 'toolbox_install_manual.py' in the following command and run this command in ArcGIS Python console (View -> Python Window)
+#   import sysconfig; import subprocess; x = sysconfig.get_paths()['data'] + r"\python.exe"; subprocess.run((x, 'C:\\Users\\myusername\\Documents\\toolbox_install_manual.py'), capture_output=True, text=True, shell=True, timeout=1000 )
+#   4. Restart ArcGIS Pro 
+
+from subprocess_call import subprocess_call
+import os 
+from os import listdir
+from os.path import isfile, join
+
+pythonPath = os.getenv('APPDATA').replace("\\Roaming","") + r"\Local\ESRI\conda\envs\arcgispro-py3-speckle\python.exe"
+
+def installToolbox(newExec: str):
+    print("Installing Speckle Toolbox")
+    mypath = os.path.dirname(__file__)
+    onlyfiles = [f for f in listdir(mypath) if (isfile(join(mypath, f)) and "py3-none-any.whl" in str(f))]
+    onlyfiles.sort(key = lambda x: int(x.replace("speckle_toolbox-","").replace("-py3-none-any.whl","").split(".")[1]) ) 
+    whl_file = onlyfiles[len(onlyfiles)-1]
+    #whl_file = os.path.join(os.path.dirname(__file__), "speckle_toolbox-2.11.3-py3-none-any.whl" ) 
+    subprocess_call([newExec, '-m','pip','install','--upgrade', '--force-reinstall', whl_file])
+    return
+
+def installDependencies(pythonExec: str, pkgName: str, pkgVersion: str):
+    # install package
+    try:
+        #import importlib #importlib.import_module(pkgName)
+        if pkgName == "specklepy": import specklepy 
+        elif pkgName == "panda3d": import panda3d 
+    except Exception as e:
+        print(f"{pkgName} not installed")
+        subprocess_call( [pythonExec, "-m", "pip", "install", f"{pkgName}=={pkgVersion}"])
+
+    # Check if package needs updating
+    try:
+        print(f"Attempting to update {pkgName} to {pkgVersion}")
+        result = subprocess_call(
+            [
+                pythonExec,
+                "-m",
+                "pip",
+                "install",
+                "--upgrade",
+                f"{pkgName}=={pkgVersion}",
+            ]
+        )
+        if result == True:
+            print(f"{pkgName} upgraded")
+            return True
+        else: 
+            return False
+    except Exception as e:
+        print(e)
+        print(e.with_traceback)
+    return True
+
+installToolbox(pythonPath)
+installDependencies(pythonPath, "specklepy", "2.9.0"  )
+installDependencies(pythonPath, "panda3d", "1.10.11" )
+

speckle_toolbox/esri/toolboxes/Speckle.pyt

@@ -1 +1,2 @@
-from speckle.speckle_arcgis import *
+try: from speckle.speckle_arcgis import *
+except: from speckle_toolbox.esri.toolboxes.speckle.speckle_arcgis import *

speckle_toolbox/esri/toolboxes/Speckle.pyt.xml

@@ -1,5 +1,5 @@
 <?xml version="1.0"?>
-<metadata xml:lang="en"><Esri><CreaDate>20220718</CreaDate><CreaTime>13500100</CreaTime><ArcGISFormat>1.0</ArcGISFormat><SyncOnce>TRUE</SyncOnce><ModDate>20221024</ModDate><ModTime>110535</ModTime><scaleRange><minScale>150000000</minScale><maxScale>5000</maxScale></scaleRange><ArcGISProfile>ItemDescription</ArcGISProfile></Esri><toolbox name="Speckle" alias="speckle_toolbox_"><arcToolboxHelpPath>c:\program files\arcgis\pro\Resources\Help\gp</arcToolboxHelpPath><toolsets/></toolbox><dataIdInfo><idCitation><resTitle>Speckle</resTitle></idCitation><idPurp>Speckle connector for ArcGIS</idPurp><searchKeys><keyword>speckle3d</keyword></searchKeys></dataIdInfo><distInfo><distributor><distorFormat><formatName>ArcToolbox Toolbox</formatName></distorFormat></distributor></distInfo><mdHrLv><ScopeCd value="005"></ScopeCd></mdHrLv><Binary><Thumbnail><Data EsriPropertyType="PictureX">/9j/4AAQSkZJRgABAQEAYABgAAD/4gxYSUNDX1BST0ZJTEUAAQEAAAxITGlubwIQAABtbnRyUkdC
+<metadata xml:lang="en"><Esri><CreaDate>20220718</CreaDate><CreaTime>13500100</CreaTime><ArcGISFormat>1.0</ArcGISFormat><SyncOnce>TRUE</SyncOnce><ModDate>20221207</ModDate><ModTime>175959</ModTime><scaleRange><minScale>150000000</minScale><maxScale>5000</maxScale></scaleRange><ArcGISProfile>ItemDescription</ArcGISProfile></Esri><toolbox name="Speckle" alias="speckle_toolbox_"><arcToolboxHelpPath>c:\program files\arcgis\pro\Resources\Help\gp</arcToolboxHelpPath><toolsets/></toolbox><dataIdInfo><idCitation><resTitle>Speckle</resTitle></idCitation><idPurp>Speckle connector for ArcGIS</idPurp><searchKeys><keyword>speckle3d</keyword></searchKeys></dataIdInfo><distInfo><distributor><distorFormat><formatName>ArcToolbox Toolbox</formatName></distorFormat></distributor></distInfo><mdHrLv><ScopeCd value="005"></ScopeCd></mdHrLv><Binary><Thumbnail><Data EsriPropertyType="PictureX">/9j/4AAQSkZJRgABAQEAYABgAAD/4gxYSUNDX1BST0ZJTEUAAQEAAAxITGlubwIQAABtbnRyUkdC
 IFhZWiAHzgACAAkABgAxAABhY3NwTVNGVAAAAABJRUMgc1JHQgAAAAAAAAAAAAAAAAAA9tYAAQAA
 AADTLUhQICAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABFj
 cHJ0AAABUAAAADNkZXNjAAABhAAAAGx3dHB0AAAB8AAAABRia3B0AAACBAAAABRyWFlaAAACGAAA

speckle_toolbox/esri/toolboxes/speckle/converter/geometry/_init_.py

@@ -5,13 +5,22 @@ from specklepy.objects.geometry import Line, Mesh, Point, Polyline, Curve, Arc,
 
 import arcpy 
 from typing import Any, List, Union, Sequence
-from speckle.converter.geometry.polygon import polygonToNative, polygonToSpeckle, multiPolygonToSpeckle
-from speckle.converter.geometry.polyline import arcToNative, ellipseToNative, circleToNative, curveToNative, lineToNative, polycurveToNative, polylineFromVerticesToSpeckle, polylineToNative, polylineToSpeckle
-from speckle.converter.geometry.point import pointToCoord, pointToNative, pointToSpeckle, multiPointToSpeckle
-from speckle.converter.geometry.polyline import speckleArcCircleToPoints, specklePolycurveToPoints, multiPolylineToSpeckle
+try:
+    from speckle.converter.geometry.polygon import polygonToNative, polygonToSpeckle, multiPolygonToSpeckle
+    from speckle.converter.geometry.polyline import arcToNative, ellipseToNative, circleToNative, curveToNative, lineToNative, polycurveToNative, polylineFromVerticesToSpeckle, polylineToNative, polylineToSpeckle
+    from speckle.converter.geometry.point import pointToCoord, pointToNative, pointToSpeckle, multiPointToSpeckle
+    from speckle.converter.geometry.polyline import speckleArcCircleToPoints, specklePolycurveToPoints, multiPolylineToSpeckle
+    from speckle.converter.geometry.mesh import meshToNative
+except:
+    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.polygon import polygonToNative, polygonToSpeckle, multiPolygonToSpeckle
+    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.polyline import arcToNative, ellipseToNative, circleToNative, curveToNative, lineToNative, polycurveToNative, polylineFromVerticesToSpeckle, polylineToNative, polylineToSpeckle
+    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.point import pointToCoord, pointToNative, pointToSpeckle, multiPointToSpeckle
+    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.polyline import speckleArcCircleToPoints, specklePolycurveToPoints, multiPolylineToSpeckle
+    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.mesh import meshToNative
+
 import numpy as np
 
-def convertToSpeckle(feature, layer, geomType, featureType) -> Union[Base, Sequence[Base], None]:
+def convertToSpeckle(feature, index: str, layer, geomType, featureType) -> Union[Base, Sequence[Base], None]:
     """Converts the provided layer feature to Speckle objects"""
     print("___convertToSpeckle____________")
     geom = feature
@@ -38,8 +47,8 @@ def convertToSpeckle(feature, layer, geomType, featureType) -> Union[Base, Seque
         if geom.partCount > 1: return multiPolylineToSpeckle(geom, feature, layer, geomMultiType)
         else: return polylineToSpeckle(geom, feature, layer, geomMultiType)
     elif geomType == "Polygon":
-        if geom.partCount > 1: return multiPolygonToSpeckle(geom, feature, layer, geomMultiType)
-        else: return polygonToSpeckle(geom, feature, layer, geomMultiType)
+        if geom.partCount > 1: return multiPolygonToSpeckle(geom, feature, index, layer, geomMultiType)
+        else: return polygonToSpeckle(geom, feature, index, layer, geomMultiType)
     elif geomType == "Multipoint":
         return multiPointToSpeckle(geom, feature, layer, geomMultiType)
     else:
@@ -125,7 +134,7 @@ def multiPolygonToNative(items: List[Base], sr: arcpy.SpatialReference): #TODO f
             except: pass # if Line
 
         pts = [pointToCoord(pt) for pt in pointsSpeckle]
-        print(pts)
+        #print(pts)
 
         outer_arr = [arcpy.Point(*coords) for coords in pts]
         outer_arr.append(outer_arr[0])

speckle_toolbox/esri/toolboxes/speckle/converter/geometry/mesh.py

@@ -1,10 +1,75 @@
+from typing import List
+import arcpy
+
 from specklepy.objects.geometry import Mesh
 
+import shapefile
+from shapefile import TRIANGLE_STRIP, TRIANGLE_FAN
+try: from speckle.converter.layers.utils import get_scale_factor
+except: from speckle_toolbox.esri.toolboxes.speckle.converter.layers.utils import get_scale_factor
 
-def meshToNative(mesh: Mesh):
-    """Converts a Speckle Mesh to QgsGeometry. Currently UNSUPPORTED"""
-    return None
+def meshToNative(meshes: List[Mesh], path: str):
+    """Converts a Speckle Mesh to MultiPatch"""
+    print("06___________________Mesh to Native")
+    #print(meshes)
+    #print(mesh.units)
+    w = shapefile.Writer(path) 
+    w.field('speckleTyp', 'C')
 
+    shapes = []
+    for geom in meshes:
+        
+        try: 
+            if geom.displayValue and isinstance(geom.displayValue, Mesh): 
+                mesh = geom.displayValue
+                w = fill_mesh_parts(w, mesh)
+            elif geom.displayValue and isinstance(geom.displayValue, List): 
+                for part in geom.displayValue:
+                    if isinstance(part, Mesh): 
+                        mesh = part
+                        w = fill_mesh_parts(w, mesh)
+        except: 
+            try: 
+                if geom.displayMesh and isinstance(geom.displayMesh, Mesh): 
+                    mesh = geom.displayMesh
+                    w = fill_mesh_parts(w, mesh)
+                elif geom.displayMesh and isinstance(geom.displayMesh, List): 
+                    for part in geom.displayMesh:
+                        if isinstance(part, Mesh): 
+                            mesh = part
+                            w = fill_mesh_parts(w, mesh)
+            except: pass
+    w.close()
+    return path
+
+def fill_mesh_parts(w: shapefile.Writer, mesh: Mesh):
+    scale = get_scale_factor(mesh.units)
+
+    parts_list = []
+    types_list = []
+    count = 0 # sequence of vertex (not of flat coord list) 
+    try:
+        #print(len(mesh.faces))
+        if len(mesh.faces) % 4 == 0 and (mesh.faces[0] == 0 or mesh.faces[0] == 3):
+            for f in mesh.faces:
+                try:
+                    if mesh.faces[count] == 0 or mesh.faces[count] == 3: # only handle triangles
+                        f1 = [ scale*mesh.vertices[mesh.faces[count+1]*3], scale*mesh.vertices[mesh.faces[count+1]*3+1], scale*mesh.vertices[mesh.faces[count+1]*3+2] ]
+                        f2 = [ scale*mesh.vertices[mesh.faces[(count+2)]*3], scale*mesh.vertices[mesh.faces[(count+2)]*3+1], scale*mesh.vertices[mesh.faces[(count+2)]*3+2] ]
+                        f3 = [ scale*mesh.vertices[mesh.faces[(count+3)]*3], scale*mesh.vertices[mesh.faces[(count+3)]*3+1], scale*mesh.vertices[mesh.faces[(count+3)]*3+2] ]
+                        parts_list.append([ f1, f2, f3 ])
+                        types_list.append(TRIANGLE_FAN)
+                        count += 4
+                    else: 
+                        count += mesh.faces[count+1]
+                except: break
+            w.multipatch(parts_list, partTypes=types_list ) # one type for each part
+            w.record('displayMesh')
+        else: print("not triangulated mesh")
+
+    except Exception as e: pass #; print(e)
+    return w
+    
 def rasterToMesh(vertices, faces, colors):
     mesh = Mesh.create(vertices, faces, colors)
     mesh.units = "m"

speckle_toolbox/esri/toolboxes/speckle/converter/geometry/point.py

@@ -3,7 +3,8 @@ from typing import List
 from specklepy.objects.geometry import Point
 import arcpy
 
-from speckle.converter.layers.utils import get_scale_factor
+try: from speckle.converter.layers.utils import get_scale_factor
+except: from speckle_toolbox.esri.toolboxes.speckle.converter.layers.utils import get_scale_factor
 
 
 def multiPointToSpeckle(geom, feature, layer, multiType: bool):
@@ -55,19 +56,20 @@ def pointToNative(pt: Point, sr: arcpy.SpatialReference) -> arcpy.PointGeometry:
     #print(geom)
     return geom
 
-def pointToCoord(pt: Point) -> List[float]:
+def pointToCoord(point: Point) -> List[float]:
     """Converts a Speckle Point to QgsPoint"""
-    pt = scalePointToNative(pt, pt.units)
+    pt = scalePointToNative(point, point.units)
     coords = [pt.x, pt.y, pt.z]
     #print(coords)
     return coords
 
-def scalePointToNative(pt: Point, units: str) -> Point:
+def scalePointToNative(point: Point, units: str) -> Point:
     """Scale point coordinates to meters"""
     scaleFactor = get_scale_factor(units)
-    pt.x = pt.x * scaleFactor
-    pt.y = pt.y * scaleFactor
-    pt.z = 0 if math.isnan(pt.z) else pt.z * scaleFactor
+    pt = Point(units = "m")
+    pt.x = point.x * scaleFactor
+    pt.y = point.y * scaleFactor
+    pt.z = 0 if math.isnan(point.z) else point.z * scaleFactor
     return pt
 
 def addZtoPoint(coords: List): 

speckle_toolbox/esri/toolboxes/speckle/converter/geometry/polygon.py

@@ -5,20 +5,34 @@ from arcpy.arcobjects.arcobjects import SpatialReference
 
 from specklepy.objects import Base
 from specklepy.objects.geometry import Point, Arc, Circle, Polycurve, Polyline, Line
-from speckle.converter.geometry.mesh import rasterToMesh
-from speckle.converter.geometry.point import pointToCoord, pointToNative
-from speckle.converter.geometry.polyline import (polylineFromVerticesToSpeckle, 
-                                                circleToSpeckle, 
-                                                speckleArcCircleToPoints, 
-                                                curveToSpeckle, 
-                                                specklePolycurveToPoints
-                                                )
+
+try:
+    from speckle.converter.geometry.mesh import rasterToMesh
+    from speckle.converter.geometry.point import pointToCoord, pointToNative
+    from speckle.converter.layers.symbologyTemplates import featureColorfromNativeRenderer
+    from speckle.converter.geometry.polyline import (polylineFromVerticesToSpeckle, 
+                                                    circleToSpeckle, 
+                                                    speckleArcCircleToPoints, 
+                                                    curveToSpeckle, 
+                                                    specklePolycurveToPoints
+                                                    )
+except: 
+    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.mesh import rasterToMesh
+    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.point import pointToCoord, pointToNative
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.symbologyTemplates import featureColorfromNativeRenderer
+    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.polyline import (polylineFromVerticesToSpeckle, 
+                                                    circleToSpeckle, 
+                                                    speckleArcCircleToPoints, 
+                                                    curveToSpeckle, 
+                                                    specklePolycurveToPoints
+                                                    )
 
 import math
 from panda3d.core import Triangulator
 
 
-def multiPolygonToSpeckle(geom, feature, layer, multiType: bool):
+
+def multiPolygonToSpeckle(geom, feature, index: str, layer, multiType: bool):
 
     print("___MultiPolygon to Speckle____")
     polygon = []
@@ -42,15 +56,15 @@ def multiPolygonToSpeckle(geom, feature, layer, multiType: bool):
                 arrInnerRings[len(arrInnerRings)-1].append(ptn)
 
         full_arr = [arrBoundary] + arrInnerRings
-        print(full_arr)
+        #print(full_arr)
         poly = arcpy.Polygon(arcpy.Array(full_arr), arcpy.Describe(layer.dataSource).SpatialReference, has_z = True)
-        print(poly) #<geoprocessing describe geometry object object at 0x000002B2D3E338D0> 
-        polygon.append(polygonToSpeckle(poly, feature, layer, poly.isMultipart))
+        #print(poly) #<geoprocessing describe geometry object object at 0x000002B2D3E338D0> 
+        polygon.append(polygonToSpeckle(poly, feature, index, layer, poly.isMultipart))
 
     return polygon
 
 
-def polygonToSpeckle(geom, feature, layer, multiType: bool):
+def polygonToSpeckle(geom, feature, index: int, layer, multiType: bool):
     """Converts a Polygon to Speckle"""
     #try: 
     print("___Polygon to Speckle____")
@@ -202,7 +216,7 @@ def polygonToSpeckle(geom, feature, layer, multiType: bool):
             ran = range(0, total_vertices)
         
         #print(polygon)
-        col = (100<<16) + (100<<8) + 100 #featureColorfromNativeRenderer(feature, layer)
+        col = featureColorfromNativeRenderer(index, layer)
         colors = [col for i in ran] # apply same color for all vertices
         mesh = rasterToMesh(vertices, faces, colors)
         polygon.displayValue = mesh 
@@ -228,7 +242,7 @@ def polygonToNative(poly: Base, sr: arcpy.SpatialReference) -> arcpy.Polygon:
         except: pass # if Line
 
     pts = [pointToCoord(pt) for pt in pointsSpeckle]
-    print(pts)
+    #print(pts)
 
     outer_arr = [arcpy.Point(*coords) for coords in pts]
     outer_arr.append(outer_arr[0])

speckle_toolbox/esri/toolboxes/speckle/converter/geometry/polyline.py

@@ -8,8 +8,13 @@ from specklepy.objects.geometry import Box, Vector, Point, Line, Polyline, Curve
 import arcpy 
 import numpy as np
 
-from speckle.converter.geometry.point import pointToCoord, pointToSpeckle, addZtoPoint
-from speckle.converter.layers.utils import get_scale_factor
+try:
+    from speckle.converter.geometry.point import pointToCoord, pointToSpeckle, addZtoPoint
+    from speckle.converter.layers.utils import get_scale_factor
+except:
+    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.point import pointToCoord, pointToSpeckle, addZtoPoint
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.utils import get_scale_factor
+
 
 def circleToSpeckle(center, point):
     print("___Circle to Speckle____")
@@ -26,6 +31,7 @@ def circleToSpeckle(center, point):
     #print(args) 
     c = Circle.from_list(args)
     c.plane.origin.units = "m"
+    c.units = "m"
     #print(c)
     return c 
 
@@ -107,6 +113,7 @@ def arc3ptToSpeckle(p0: List, p1: List, p2: List, feature, layer) -> Arc:
     arc.plane = Plane() #.from_list(Point(), Vector(Point(0, 0, 1)), Vector(Point(0,1,0)), Vector(Point(-1,0,0)))
     arc.plane.origin = Point.from_list(center)
     arc.plane.origin.units = "m" 
+    arc.units = "m"
     arc.angleRadians, startAngle, endAngle = getArcRadianAngle(arc)
 
     arc.radius = radius
@@ -173,7 +180,7 @@ def curveToSpeckle(geom, geomType, feature, layer) -> Union[Circle, Arc, Polylin
     # a - elliptical arc (endPt, centralPt) e.g. for circle: [[[631307.05960000027,5803698.4477999993,0],{"a":[[631307.05960000027,5803698.4477999993,0],[631307.05960000027,5803414.92656173],0,1]}]]
     # c - circular arc (endPt, throughPt) e.g. [[[633242.45179999992,5803058.0354999993,0],{"c":[[633718.26040000003,5803496.4210000001,0],[633337.75764975848,5803431.9997026781]]},[633242.45179999992,5803058.0354999993,0]]]
     
-    boundary = Polycurve()
+    boundary = Polycurve(units = "m")
     if geomType == "Polyline": boundary.closed = False
     else: boundary.closed = True 
     segments = [] 
@@ -326,7 +333,7 @@ def lineFrom2pt(pt1: List[float], pt2: List[float]):
     dist = math.sqrt( math.pow((pt2[0] - pt1[0]), 2) + math.pow((pt2[1] - pt1[1]), 2) + math.pow((pt2[2] - pt1[2]), 2) ) 
     print(dist) 
     domain = [0, dist, 0, 0] 
-    line = Line()#.from_list([*pt1, *pt2, *domain]) 
+    line = Line(units = "m" )#.from_list([*pt1, *pt2, *domain]) 
     line.start = Point.from_list(pt1)
     line.end = Point.from_list(pt2)
     line.start.units = line.end.units = "m" 
@@ -334,7 +341,7 @@ def lineFrom2pt(pt1: List[float], pt2: List[float]):
 
 def polylineToNative(poly: Polyline, sr: arcpy.SpatialReference) -> arcpy.Polyline:
     """Converts a Speckle Polyline to QgsLineString"""
-    print("__ convert poly to native __")
+    print("__ convert polyline to native __")
 
     if isinstance(poly, Polycurve): 
         poly = specklePolycurveToPoints(poly)
@@ -377,7 +384,7 @@ def ellipseToNative():
 
 def circleToNative(poly: Circle, sr: arcpy.SpatialReference) -> arcpy.Polyline:
     """Converts a Speckle Circle to QgsLineString"""
-    print("___Convert Circle from Native___")
+    print("___Convert Circle to Native___")
     points = []
     angle1 = math.pi/2
     
@@ -393,7 +400,8 @@ def circleToNative(poly: Circle, sr: arcpy.SpatialReference) -> arcpy.Polyline:
         if poly.plane.normal.z == 0: normal = 1
         else: normal = poly.plane.normal.z
         angle = angle1 + k * math.pi*2 * normal
-        pt = Point( x = poly.plane.origin.x + radScaled * cos(angle), y = poly.plane.origin.y + radScaled * sin(angle), z = 0) 
+        pt = Point( x = poly.plane.origin.x * get_scale_factor(poly.units) + radScaled * cos(angle), y = poly.plane.origin.y * get_scale_factor(poly.units) + radScaled * sin(angle), z = 0) 
+        print(pt)
         pt.units = "m"
         points.append(pointToCoord(pt))
     points.append(points[0])
@@ -404,10 +412,10 @@ def polycurveToNative(poly: Polycurve, sr: arcpy.SpatialReference) -> arcpy.Poly
     points = []
     curve = None
     print("___Polycurve to native___")
-    
-    try:
-        for segm in poly.segments: # Line, Polyline, Curve, Arc, Circle
-            #print(segm)
+
+    try: 
+        for i, segm in enumerate(poly.segments): # Line, Polyline, Curve, Arc, Circle
+            print("___start segment")            
             if isinstance(segm,Line):  converted = lineToNative(segm, sr) # QgsLineString
             elif isinstance(segm,Polyline):  converted = polylineToNative(segm, sr) # QgsLineString
             elif isinstance(segm,Curve):  converted = curveToNative(segm, sr) # QgsLineString
@@ -420,6 +428,7 @@ def polycurveToNative(poly: Polycurve, sr: arcpy.SpatialReference) -> arcpy.Poly
             if converted is not None: 
                 #print(converted) # <geoprocessing describe geometry object object at 0x000002B2D3E338D0>
                 for part in converted:
+                    #print("Part: ")
                     #print(part) # <geoprocessing array object object at 0x000002B2D2E09530>
                     for pt in part: 
                         #print(pt) # 64.4584221540162 5.5 NaN NaN
@@ -428,8 +437,7 @@ def polycurveToNative(poly: Polycurve, sr: arcpy.SpatialReference) -> arcpy.Poly
                         #print(pt_z)
                         #print(len(points)) 
                         if len(points)>0 and pt.X == points[len(points)-1][0] and pt.Y == points[len(points)-1][1] and pt_z == points[len(points)-1][2]: pass
-                        else: points.append(pointToCoord(Point(x=pt.X, y = pt.Y, z = pt_z)))
-                        #print(points)
+                        else: points.append(pointToCoord(Point(x=pt.X, y = pt.Y, z = pt_z, units = "m"))) # e.g. [[64.4584221540162, 5.499999999999999, 0.0], [64.45461685210796, 5.587155742747657, 0.0]]
             else:
                 arcpy.AddWarning(f"Part of the polycurve cannot be converted")
                 curve = arcpy.Polyline( arcpy.Array([arcpy.Point(*coords) for coords in points]), sr, has_z=True )
@@ -441,7 +449,7 @@ def polycurveToNative(poly: Polycurve, sr: arcpy.SpatialReference) -> arcpy.Poly
     return curve
 
 def arcToNativePolyline(poly: Union[Arc, Circle], sr: arcpy.SpatialReference):
-    print("__Arc/Circle to native__")
+    print("__Arc/Circle to native polyline__")
     pointsSpeckle = speckleArcCircleToPoints(poly)
     points = [pointToCoord(p) for p in pointsSpeckle]
     curve = arcpy.Polyline( arcpy.Array([arcpy.Point(*coords) for coords in points]), sr, has_z=True )
@@ -453,7 +461,8 @@ def specklePolycurveToPoints(poly: Polycurve) -> List[Point]:
     print("_____Speckle Polycurve to points____")
     points = []
     for segm in poly.segments:
-        print(segm)
+        #print(segm)
+        pts = []
         if isinstance(segm, Arc) or isinstance(segm, Circle): # or isinstance(segm, Curve):
             print("Arc or Curve")
             pts: List[Point] = speckleArcCircleToPoints(segm) 
@@ -470,8 +479,8 @@ def specklePolycurveToPoints(poly: Polycurve) -> List[Point]:
 def speckleArcCircleToPoints(poly: Union[Arc, Circle]) -> List[Point]: 
     print("__Arc or Circle to Points___")
     points = []
-    print(poly.plane) 
-    print(poly.plane.normal) 
+    #print(poly.plane) 
+    #print(poly.plane.normal) 
     if poly.plane is None or poly.plane.normal.z == 0: normal = 1 
     else: normal = poly.plane.normal.z 
     #print(poly.plane.origin)
@@ -511,7 +520,6 @@ def speckleArcCircleToPoints(poly: Union[Arc, Circle]) -> List[Point]:
         
         pt.units = poly.plane.origin.units 
         points.append(pt)
-
     if isinstance(poly, Arc): points.append(poly.endPoint)
     return points
 

speckle_toolbox/esri/toolboxes/speckle/converter/layers/Layer.py

@@ -1,7 +1,10 @@
 from typing import List, Optional
 from specklepy.objects.base import Base
 
-from speckle.converter.layers.CRS import CRS
+try: 
+    from speckle.converter.layers.CRS import CRS
+except:
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.CRS import CRS
 
 
 

speckle_toolbox/esri/toolboxes/speckle/converter/layers/_init_.py

@@ -4,11 +4,29 @@ Contains all Layer related classes and methods.
 import os
 from typing import Any, List, Tuple, Union
 
-from regex import D
-from speckle.converter.layers.CRS import CRS
-from speckle.converter.layers.Layer import Layer, VectorLayer, RasterLayer
-from speckle.converter.layers.feature import featureToNative, featureToSpeckle, cadFeatureToNative, rasterFeatureToSpeckle
+#from regex import D
+
+try:
+    from speckle.converter.layers.CRS import CRS
+    from speckle.converter.layers.Layer import Layer, VectorLayer, RasterLayer
+    from speckle.converter.layers.symbologyTemplates import vectorRendererToNative, rasterRendererToNative, rendererToSpeckle 
+    from speckle.converter.layers.feature import featureToNative, featureToSpeckle, cadFeatureToNative, bimFeatureToNative, rasterFeatureToSpeckle
+
+    from speckle.converter.geometry.mesh import rasterToMesh, meshToNative
+    from speckle.converter.layers.utils import findTransformation
+    from speckle.converter.layers.utils import getLayerAttributes, newLayerGroupAndName, validate_path
+except: 
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.CRS import CRS
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.Layer import Layer, VectorLayer, RasterLayer
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.symbologyTemplates import vectorRendererToNative, rasterRendererToNative, rendererToSpeckle 
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.feature import featureToNative, featureToSpeckle, cadFeatureToNative, bimFeatureToNative, rasterFeatureToSpeckle
+
+    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.mesh import rasterToMesh, meshToNative
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.utils import findTransformation
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.utils import getLayerAttributes, newLayerGroupAndName, validate_path
+
 from specklepy.objects import Base
+from specklepy.objects.geometry import Mesh
 
 import arcgisscripting
 import pandas as pd
@@ -17,12 +35,8 @@ from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
 from arcpy.management import (CreateFeatureclass, MakeFeatureLayer,
                               AddFields, AlterField, DefineProjection )
 
-from speckle.converter.layers.utils import getLayerAttributes, newLayerGroupAndName, validate_path
 import numpy as np
 
-from speckle.converter.layers.utils import findTransformation
-
-
 def convertSelectedLayers(all_layers: List[arcLayer], selected_layers: List[str], project: ArcGISProject) -> List[Union[VectorLayer,Layer]]:
     """Converts the current selected layers to Speckle"""
     print("________Convert Layers_________")
@@ -37,9 +51,10 @@ def convertSelectedLayers(all_layers: List[arcLayer], selected_layers: List[str]
             ds = layerToSend.dataSource #file path
             #if layerToSend.isFeatureLayer: 
             newBaseLayer = layerToSpeckle(layerToSend, project)
-            if newBaseLayer is not None: result.append(newBaseLayer)
-
-            elif layerToSend.isRasterLayer: pass
+            if newBaseLayer is not None: 
+                result.append(newBaseLayer)
+            #elif layerToSend.isRasterLayer: pass
+            print(result)
 
     return result
 
@@ -72,6 +87,7 @@ def layerToSpeckle(layer: arcLayer, project: ArcGISProject) -> Union[VectorLayer
         speckleLayer.type="VectorLayer"
         speckleLayer.name = layerName
         speckleLayer.crs = speckleReprojectedCrs
+        speckleLayer.renderer = rendererToSpeckle(project, project.activeMap, layer, None)
         #speckleLayer.datum = datum
 
 
@@ -112,7 +128,7 @@ def layerToSpeckle(layer: arcLayer, project: ArcGISProject) -> Union[VectorLayer
                             #print(feat)
 
 
-                        b = featureToSpeckle(fieldnames, row_attr, feat, projectCRS, project, layer)
+                        b = featureToSpeckle(fieldnames, row_attr, i, feat, projectCRS, project, layer)
                         if b is not None: layerObjs.append(b)
                         
                     print("____End of Feature # " + str(i+1))
@@ -121,6 +137,8 @@ def layerToSpeckle(layer: arcLayer, project: ArcGISProject) -> Union[VectorLayer
                 speckleLayer.features=layerObjs
                 speckleLayer.geomType = data.shapeType
 
+                if len(speckleLayer.features) == 0: return None
+
                 #layerBase.renderer = layerRenderer
                 #layerBase.applicationId = selectedLayer.id()
 
@@ -144,6 +162,8 @@ def layerToSpeckle(layer: arcLayer, project: ArcGISProject) -> Union[VectorLayer
         #speckleLayer.geomType="Raster"
         speckleLayer.features = layerObjs
         
+        speckleLayer.renderer = rendererToSpeckle(project, project.activeMap, layer, b)
+        
         #speckleLayer.renderer = layerRenderer
         #speckleLayer.applicationId = selectedLayer.id()
 
@@ -160,7 +180,203 @@ def layerToNative(layer: Union[Layer, VectorLayer, RasterLayer], streamBranch: s
         return rasterLayerToNative(layer, streamBranch, project)
     return None
 
-def cadLayerToNative(layerContentList:Base, layerName: str, streamBranch: str, project: ArcGISProject) :
+def bimLayerToNative(layerContentList: List[Base], layerName: str, streamBranch: str, project: ArcGISProject) :
+    print("01______BIM layer to native")
+    print(layerName)
+    geom_meshes = []
+    layer_meshes = None
+    #filter speckle objects by type within each layer, create sub-layer for each type (points, lines, polygons, mesh?)
+    for geom in layerContentList:
+        try: 
+            if geom.displayMesh: geom_meshes.append(geom)
+        except:
+            try: 
+                if geom.displayValue: geom_meshes.append(geom)
+            except: pass
+
+    if len(geom_meshes)>0: layer_meshes = bimVectorLayerToNative(geom_meshes, layerName, "Mesh", streamBranch, project)
+
+    return True
+
+
+def bimVectorLayerToNative(geomList, layerName: str, geomType: str, streamBranch: str, project: ArcGISProject): 
+    # no support for mltipatches, maybe in 3.1: https://community.esri.com/t5/arcgis-pro-ideas/better-support-for-multipatches-in-arcpy/idi-p/953614/page/2#comments
+    print("02_________BIM vector layer to native_____")
+    #get Project CRS, use it by default for the new received layer
+    
+    vl = None
+    layerName = layerName + "_" + geomType
+    layerName = layerName.replace("[","_").replace("]","_").replace(" ","_").replace("-","_").replace("(","_").replace(")","_").replace(":","_").replace("\\","_").replace("/","_").replace("\"","_").replace("&","_").replace("@","_").replace("$","_").replace("%","_").replace("^","_")
+    #if not "__Structural_Foundations_Mesh" in layerName: return None
+    
+    sr = arcpy.SpatialReference(text = project.activeMap.spatialReference.exportToString())
+    active_map = project.activeMap
+    
+    path = project.filePath.replace("aprx","gdb") #
+    path_bim = "\\".join(project.filePath.split("\\")[:-1]) + "\\Layers_Speckle\\BIM_layers\\" + streamBranch+ "\\" + layerName + "\\" #arcpy.env.workspace + "\\" #
+    print(path_bim)
+    
+    if not os.path.exists(path_bim): os.makedirs(path_bim)
+    print(path)
+    
+    if sr.type == "Geographic": 
+        arcpy.AddMessage(f"Project CRS is set to Geographic type, and objects in linear units might not be received correctly")
+
+    #CREATE A GROUP "received blabla" with sublayers
+    layerGroup = None
+    newGroupName = f'{streamBranch}'
+    #print(newGroupName)
+    for l in active_map.listLayers():
+        if l.longName == newGroupName: layerGroup = l; break 
+    
+    #find ID of the layer with a matching name in the "latest" group 
+    newName = f'{streamBranch.split("_")[len(streamBranch.split("_"))-1]}_{layerName}'
+    print(newName)
+
+    all_layer_names = []
+    layerExists = 0
+    for l in project.activeMap.listLayers(): 
+        if l.longName.startswith(newGroupName + "\\"):
+            all_layer_names.append(l.longName)
+    #print(all_layer_names)
+
+    longName = streamBranch + "\\" + newName 
+    if longName in all_layer_names: 
+        for index, letter in enumerate('234567890abcdefghijklmnopqrstuvwxyz'):
+            if (longName + "_" + letter) not in all_layer_names: newName += "_"+letter; layerExists +=1; break 
+
+    # particularly if the layer comes from ArcGIS
+    if "mesh" in geomType.lower(): geomType = "Multipatch"
+
+    #print("Create feature class (cad): ")
+    # should be created inside the workspace to be a proper Feature class (not .shp) with Nullable Fields
+    class_name = ("f_class_" + newName)
+    #f_class = CreateFeatureclass(path, class_name, geomType, has_z="ENABLED", spatial_reference = sr)
+
+
+    shp = meshToNative(geomList, path_bim + newName)
+    print("____ meshes saved___")
+    print(shp)
+    #print(path)
+    #print(class_name)
+    validated_class_path = validate_path(class_name)
+    print(validated_class_path)
+    validated_class_name = validated_class_path.split("\\")[len(validated_class_path.split("\\"))-1]
+    print(validated_class_name)
+    f_class = arcpy.conversion.FeatureClassToFeatureClass(shp, path, validated_class_name)
+    # , spatial_reference = sr
+    #arcpy.management.Project(in_dataset, f_class, sr, in_coor_system=sr)
+
+    print(f_class)
+    #print(geomList)
+
+    # get and set Layer attribute fields
+    # example: https://resource.esriuk.com/blog/an-introductory-slice-of-arcpy-in-arcgis-pro/
+    newFields = getLayerAttributes(geomList)
+    
+    fields_to_ignore = ["arcgisgeomfromspeckle", "shape", "objectid", "displayMesh"]
+    matrix = []
+    all_keys = []
+    all_key_types = []
+    max_len = 52
+
+    print("___ after layer attributes: ___________")
+    print(newFields.items())
+    #try:
+    for key, value in newFields.items(): 
+        existingFields = [fl.name for fl in arcpy.ListFields(validated_class_name)]
+        #print(existingFields)
+        if key not in existingFields  and key.lower() not in fields_to_ignore: # exclude geometry and default existing fields
+            #print(key)
+            # signs that should not be used as field names and table names: https://support.esri.com/en/technical-article/000005588
+            key = key.replace(" ","_").replace("-","_").replace("(","_").replace(")","_").replace(":","_").replace("\\","_").replace("/","_").replace("\"","_").replace("&","_").replace("@","_").replace("$","_").replace("%","_").replace("^","_") 
+            if key[0] in ['0','1','2','3','4','5','6','7','8','9']: key = "_"+key
+            if len(key)>max_len: key = key[:max_len]
+            #print(all_keys)
+            if key in all_keys:
+                for index, letter in enumerate('1234567890abcdefghijklmnopqrstuvwxyz'):
+                    if len(key)<max_len and (key+letter) not in all_keys: key+=letter; break 
+                    if len(key) == max_len and (key[:9] + letter) not in all_keys: key=key[:9] + letter; break 
+            if key not in all_keys: 
+                all_keys.append(key)
+                all_key_types.append(value)
+                #print(all_keys)
+                matrix.append([key, value, key, 255])
+    print(all_keys)
+    print(len(all_keys))
+    if len(matrix)>0: AddFields(str(f_class), matrix)
+    print(matrix)
+    
+    fets = []
+    print("_________BIM FeatureS To Native___________")
+    for f in geomList[:]: 
+        new_feat = bimFeatureToNative(f, newFields, sr, path_bim)
+        if new_feat != "" and new_feat != None: 
+            fets.append(new_feat)
+    print(len(fets))
+        
+    
+    if len(fets) == 0: return None
+    count = 0
+    rowValues = []
+    for i, feat in enumerate(fets):
+
+        row = []
+        heads = []
+        for key in all_keys:
+            try:
+                row.append(feat[key])
+                heads.append(key)
+            except Exception as e: 
+                row.append(None)
+                heads.append(key)
+
+        rowValues.append(row)
+        count += 1
+    print(heads)
+    
+    with arcpy.da.UpdateCursor(f_class, heads) as cur:
+        # For each row, evaluate the WELL_YIELD value (index position 
+        # of 0), and update WELL_CLASS (index position of 1)
+        shp_num = 0
+        #print(heads)
+        try:
+            for rowShape in cur: 
+                #print(rowShape)
+                for i,r in enumerate(rowShape):
+                    #print(heads[i])
+                    #print(matrix[i])
+                    rowShape[i] = rowValues[shp_num][i]
+                    #print(type(rowShape[i]))
+                    if matrix[i][1] == 'TEXT' and rowShape[i] is not None: rowShape[i] = str(rowValues[shp_num][i]) 
+                    #print(type(rowShape[i]))
+                    if isinstance(rowValues[shp_num][i], str): # cut if string is too long
+                        rowShape[i] = rowValues[shp_num][i][:255]
+                    #print(rowShape[i])
+                #print(rowShape)
+                cur.updateRow(rowShape)
+                shp_num += 1
+                #print(shp_num)
+        except Exception as e: 
+            print("Layer attribute error: " + str(e))
+            #print(i)
+            print(shp_num)
+            print(len(rowValues))
+            #print(rowValues[i])
+            #print(len(rowValues[i]))
+            arcpy.AddWarning("Layer attribute error: " + e)
+    del cur 
+    
+    print("create layer:")
+    vl = MakeFeatureLayer(str(f_class), newName).getOutput(0)
+
+    active_map.addLayerToGroup(layerGroup, vl)
+    print("created2")
+    #os.remove(path_bim)
+
+    return True #last one
+
+def cadLayerToNative(layerContentList: List[Base], layerName: str, streamBranch: str, project: ArcGISProject) :
     print("01______Cad vector layer to native")
     print(layerName)
     geom_points = []
@@ -168,6 +384,7 @@ def cadLayerToNative(layerContentList:Base, layerName: str, streamBranch: str, p
     geom_polygones = []
     geom_meshes = []
     #filter speckle objects by type within each layer, create sub-layer for each type (points, lines, polygons, mesh?)
+    print(layerContentList)
     for geom in layerContentList:
         #print(geom)
         if geom.speckle_type == "Objects.Geometry.Point": 
@@ -186,17 +403,21 @@ def cadVectorLayerToNative(geomList, layerName: str, geomType: str, streamBranch
     vl = None
     layerName = layerName.replace("[","_").replace("]","_").replace(" ","_").replace("-","_").replace("(","_").replace(")","_").replace(":","_").replace("\\","_").replace("/","_").replace("\"","_").replace("&","_").replace("@","_").replace("$","_").replace("%","_").replace("^","_")
     layerName = layerName + "_" + geomType
+    print(layerName)
     
-    sr = arcpy.SpatialReference(project.activeMap.spatialReference.name)
+    sr = arcpy.SpatialReference(text = project.activeMap.spatialReference.exportToString())
     active_map = project.activeMap
     path = project.filePath.replace("aprx","gdb") #"\\".join(project.filePath.split("\\")[:-1]) + "\\speckle_layers\\" #arcpy.env.workspace + "\\" #
     
+    print(path)
+    print(streamBranch)
     if sr.type == "Geographic": 
         arcpy.AddMessage(f"Project CRS is set to Geographic type, and objects in linear units might not be received correctly")
 
     #CREATE A GROUP "received blabla" with sublayers
     layerGroup = None
     newGroupName = f'{streamBranch}'
+    print(newGroupName)
     #print(newGroupName)
     for l in active_map.listLayers():
         if l.longName == newGroupName: layerGroup = l; break 
@@ -231,7 +452,7 @@ def cadVectorLayerToNative(geomList, layerName: str, geomType: str, streamBranch
     # should be created inside the workspace to be a proper Feature class (not .shp) with Nullable Fields
     class_name = ("f_class_" + newName)
     f_class = CreateFeatureclass(path, class_name, geomType, has_z="ENABLED", spatial_reference = sr)
-    #print(f_class)
+    print(f_class)
     #print(geomList)
 
     # get and set Layer attribute fields
@@ -268,8 +489,9 @@ def cadVectorLayerToNative(geomList, layerName: str, geomType: str, streamBranch
         new_feat = cadFeatureToNative(f, newFields, sr)
         if new_feat != "" and new_feat != None: 
             fets.append(new_feat)
-    #print("features created")
-    #print(fets)
+    print("features created")
+    print(len(fets))
+    print(all_keys)
     
     if len(fets) == 0: return None
     count = 0
@@ -282,24 +504,27 @@ def cadVectorLayerToNative(geomList, layerName: str, geomType: str, streamBranch
         heads = [ 'Shape@', 'OBJECTID']
 
         for key,value in feat.items(): 
+            #print(key, str(value))
             if key in all_keys and key.lower() not in fields_to_ignore: 
                 heads.append(key)
                 row.append(value)
         rowValues.append(row)
         count += 1
-    #print(heads)
     cur = arcpy.da.InsertCursor(str(f_class), tuple(heads) )
+    #print(heads)
     for row in rowValues: 
-        #print(tuple(heads))
-        #print(tuple(row))
-        cur.insertRow(tuple(row))
+        try:
+            #print(row)
+            cur.insertRow(tuple(row))
+        except Exception as e:
+            print(e)
     del cur 
-    #print(f_class)
     vl = MakeFeatureLayer(str(f_class), newName).getOutput(0)
 
     #adding layers from code solved: https://gis.stackexchange.com/questions/344343/arcpy-makefeaturelayer-management-function-not-creating-feature-layer-in-arcgis
     #active_map.addLayer(new_layer)
     active_map.addLayerToGroup(layerGroup, vl)
+    print("Layer created")
 
     return vl
 
@@ -370,7 +595,7 @@ def vectorLayerToNative(layer: Union[Layer, VectorLayer], streamBranch: str, pro
         new_feat = featureToNative(f, newFields, geomType, sr)
         if new_feat != "" and new_feat!= None: fets.append(new_feat)
     
-    print(fets)
+    #print(fets)
     if len(fets) == 0: return None
     count = 0
     rowValues = []
@@ -391,16 +616,27 @@ def vectorLayerToNative(layer: Union[Layer, VectorLayer], streamBranch: str, pro
         count += 1
     cur = arcpy.da.InsertCursor(str(f_class), tuple(heads) )
     for row in rowValues: 
-        print(tuple(heads))
-        print(tuple(row))
+        #print(tuple(heads))
+        #print(tuple(row))
         cur.insertRow(tuple(row))
     del cur 
 
     vl = MakeFeatureLayer(str(f_class), newName).getOutput(0)
 
     #adding layers from code solved: https://gis.stackexchange.com/questions/344343/arcpy-makefeaturelayer-management-function-not-creating-feature-layer-in-arcgis
-    #active_map.addLayer(new_layer)
+    
     active_map.addLayerToGroup(layerGroup, vl)
+    vl2 = None
+    print(newName)
+    for l in project.activeMap.listLayers(): 
+        #print(l.longName)
+        if l.longName == layerGroup.longName + "\\" + newName:
+            vl2 = l 
+            break
+    path_lyr = vectorRendererToNative(project, active_map, layerGroup, layer, vl2, f_class, heads)
+    #if path_lyr is not None: 
+    #    active_map.removeLayer(path_lyr)    
+
     r'''
     # rename back the layer if was renamed due to existing duplicate
     if layerExists:  
@@ -444,6 +680,9 @@ def rasterLayerToNative(layer: RasterLayer, streamBranch: str, project: ArcGISPr
     rasterHasSr = False
     print(path)
 
+    path_bands = "\\".join(path.split("\\")[:-1]) + "\\Layers_Speckle\\raster_bands\\" + streamBranch 
+    if not os.path.exists(path_bands): os.makedirs(path_bands)
+
     try: 
         srRasterWkt = str(layer.rasterCrs.wkt)
         print(layer.rasterCrs.wkt)
@@ -489,7 +728,7 @@ def rasterLayerToNative(layer: RasterLayer, streamBranch: str, project: ArcGISPr
     for i in range(bandsCount):
         print(i)
         print(bandNames[i])
-        rasterbandPath = path + "\\" + newName + "_Band_" + str(i+1) #+ ".tif"
+        rasterbandPath = path_bands + "\\" + newName + "_Band_" + str(i+1) + ".tif"
         bandDatasets += rasterbandPath + ";"
         rasterband = np.array(bandValues[i])
         rasterband = np.reshape(rasterband,(ysize, xsize))
@@ -523,23 +762,42 @@ def rasterLayerToNative(layer: RasterLayer, streamBranch: str, project: ArcGISPr
     #mergedRaster.setProperty("spatialReference", crsRaster)
 
     full_path = validate_path(path + "\\" + newName) #solved file saving issue 
+    print("RASTER FULL PATH")
+    print(full_path)
     if os.path.exists(full_path):
+        #print(full_path)
         for index, letter in enumerate('1234567890abcdefghijklmnopqrstuvwxyz'):
+            print(full_path + letter)
             if os.path.exists(full_path + letter): pass
             else: full_path += letter; break 
-
+    print("RASTER new PATH")
     print(full_path)
     #mergedRaster = arcpy.ia.Merge(rastersToMerge) # glues all bands together
     #mergedRaster.save(full_path) # similar errors: https://community.esri.com/t5/python-questions/error-010240-could-not-save-raster-dataset/td-p/321690
     
-    arcpy.management.CompositeBands(rasterPathsToMerge, full_path)
+    try: 
+        arcpy.management.CompositeBands(rasterPathsToMerge, full_path)
+    except: # if already exists
+        full_path += "_"
+        arcpy.management.CompositeBands(rasterPathsToMerge, full_path)
     print(path + "\\" + newName)
     arcpy.management.DefineProjection(full_path, srRaster)
 
-    rasterLayer = arcpy.management.MakeRasterLayer(full_path, newName + "_").getOutput(0)
+    rasterLayer = arcpy.management.MakeRasterLayer(full_path, newName).getOutput(0)
     print(layerGroup)
     active_map.addLayerToGroup(layerGroup, rasterLayer)
 
+    rl2 = None
+    for l in active_map.listLayers(): 
+        if l.longName == layerGroup.longName + "\\" + newName:
+            print(l.longName)
+            rl2 = l 
+            break
+    rasterLayer = rasterRendererToNative(project, active_map, layerGroup, layer, rl2, rasterPathsToMerge, newName)
+
+    try: os.remove(path_bands)
+    except: pass
+
     r'''
     if arcpy.Exists(fileout):
         arcpy.management.Delete(fileout)

speckle_toolbox/esri/toolboxes/speckle/converter/layers/feature.py

@@ -9,32 +9,49 @@ import arcpy
 from arcpy.management import CreateCustomGeoTransformation
 from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
 
-from speckle.converter.geometry._init_ import convertToSpeckle, convertToNative, convertToNativeMulti
-from speckle.converter.layers.utils import (findTransformation, getVariantFromValue, traverseDict, 
-                                            traverseDictByKey, hsv_to_rgb)
-
-from speckle.converter.geometry.point import pointToSpeckle
-from speckle.converter.geometry.mesh import rasterToMesh
+try: 
+    from speckle.converter.geometry._init_ import convertToSpeckle, convertToNative, convertToNativeMulti
+    from speckle.converter.layers.utils import (findTransformation, getVariantFromValue, traverseDict, 
+                                                traverseDictByKey, hsv_to_rgb)
+    from speckle.converter.geometry.point import pointToSpeckle
+    from speckle.converter.geometry.mesh import rasterToMesh, meshToNative
+    from speckle.converter.layers.symbologyTemplates import jsonFromLayerStyle
+except: 
+    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry._init_ import convertToSpeckle, convertToNative, convertToNativeMulti
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.utils import (findTransformation, getVariantFromValue, traverseDict, 
+                                                traverseDictByKey, hsv_to_rgb)
+    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.point import pointToSpeckle
+    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.mesh import rasterToMesh, meshToNative
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.symbologyTemplates import jsonFromLayerStyle
 
 import numpy as np
 import colorsys
 
 
-def featureToSpeckle(fieldnames, attr_list, f_shape, projectCRS: arcpy.SpatialReference, project: ArcGISProject, selectedLayer: arcLayer):
+def featureToSpeckle(fieldnames, attr_list, index: int, f_shape, projectCRS: arcpy.SpatialReference, project: ArcGISProject, selectedLayer: arcLayer):
     print("___________Feature to Speckle____________")
     b = Base(units = "m")
     data = arcpy.Describe(selectedLayer.dataSource)
     layer_sr = data.spatialReference # if sr.type == "Projected":
     geomType = data.shapeType #Polygon, Point, Polyline, Multipoint, MultiPatch
     featureType = data.featureType # Simple,SimpleJunction,SimpleJunction,ComplexEdge,Annotation,CoverageAnnotation,Dimension,RasterCatalogItem 
-
+    print(geomType)
+    print(hasattr(data, "isRevit")) 
+    print(hasattr(data, "isIFC")) 
+    print(hasattr(data, "bimLevels")) 
+    print(hasattr(data, "hasSpatialIndex")) 
+    if geomType == "MultiPatch" or hasattr(data, "isRevit") or hasattr(data, "isIFC") or hasattr(data, "bimLevels"): 
+        print(f"Layer {selectedLayer.name} has unsupported data type")
+        arcpy.AddWarning(f"Layer {selectedLayer.name} has unsupported data type")
+        return None 
     #print(layer_sr.name)
     #print(projectCRS.name)
     f_shape = findTransformation(f_shape, geomType, layer_sr, projectCRS, selectedLayer)
+    if f_shape is None: return None
 
     ######################################### Convert geometry ##########################################
     try:
-        geom = convertToSpeckle(f_shape, selectedLayer, geomType, featureType) 
+        geom = convertToSpeckle(f_shape, index, selectedLayer, geomType, featureType) 
         if geom is not None: print(geom); b["geometry"] = geom 
     except Exception as error:
         print("Error converting geometry: " + str(error))
@@ -58,7 +75,7 @@ def featureToNative(feature: Base, fields: dict, geomType: str, sr: arcpy.Spatia
     feat = {}
     try: speckle_geom = feature["geometry"] # for created in QGIS / ArcGIS Layer type
     except:  speckle_geom = feature # for created in other software
-    print(speckle_geom)
+    #print(speckle_geom)
     if isinstance(speckle_geom, list):
         if len(speckle_geom)>1 or geomType == "Multipoint": arcGisGeom = convertToNativeMulti(speckle_geom, sr)
         else: arcGisGeom = convertToNative(speckle_geom[0], sr)
@@ -71,9 +88,14 @@ def featureToNative(feature: Base, fields: dict, geomType: str, sr: arcpy.Spatia
         return None
 
     for key, variant in fields.items(): 
+        try: value = feature[key]
+        except: 
+            if key == 'Speckle_ID': value = feature['id'] 
+            else: 
+                arcpy.AddWarning(f'Field {key} not found')
+                return None 
 
-        value = feature[key]
-        if variant == "TEXT": value = str(feature[key]) 
+        if variant == "TEXT": value = str(value) 
         if variant == getVariantFromValue(value) and value != "NULL" and value != "None": 
             feat.update({key: value})
         else: 
@@ -82,6 +104,23 @@ def featureToNative(feature: Base, fields: dict, geomType: str, sr: arcpy.Spatia
             if variant == "LONG": feat.update({key: None})
             if variant == "SHORT": feat.update({key: None})
     return feat
+             
+def bimFeatureToNative(feature: Base, fields: dict, sr: arcpy.SpatialReference, path: str):
+    print("04_________BIM Feature To Native____________")
+
+    feat = {}
+    try: speckle_geom = feature["geometry"] # for created in QGIS Layer type
+    except:  speckle_geom = feature # for created in other software
+
+    feat.update({"arcGisGeomFromSpeckle": ""})
+
+    try: 
+        if "Speckle_ID" not in fields.keys() and feature["id"]: feat.update("Speckle_ID", "TEXT")
+    except: pass
+    feat_updated = updateFeat(feat, fields, feature)
+
+    return feat_updated
+                
 
 def cadFeatureToNative(feature: Base, fields: dict, sr: arcpy.SpatialReference):
     print("04_________CAD Feature To Native____________")
@@ -94,61 +133,85 @@ def cadFeatureToNative(feature: Base, fields: dict, sr: arcpy.SpatialReference):
         else: arcGisGeom = convertToNative(speckle_geom[0], sr) 
     else:
         arcGisGeom = convertToNative(speckle_geom, sr)
-    print(feat)
+        
     if arcGisGeom is not None:
         feat.update({"arcGisGeomFromSpeckle": arcGisGeom})
     else: return None
-    print(feat)
+    
     try: 
         if "Speckle_ID" not in fields.keys() and feature["id"]: feat.update("Speckle_ID", "TEXT")
     except: pass
-    print(feat)
-    #### setting attributes to feature
-    for key, variant in fields.items(): 
-        #value = feature[key]
-        #print()
-        if key == "Speckle_ID": 
-            value = str(feature["id"])
-            feat[key] = value 
-        else:
-            try: value = feature[key]
-            except:
-                rootName = key.split("_")[0]
-                newF, newVals = traverseDict({}, {}, rootName, feature[rootName][0])
-                for i, (k,v) in enumerate(newVals.items()):
-                    if k == key: value = v; break
-        # for all values: 
-        if variant == "TEXT": value = str(value) 
 
-        if variant == getVariantFromValue(value) and value != "NULL" and value != "None": 
-            feat.update({key: value})   
-        else: 
-            if variant == "TEXT": feat.update({key: None})
-            if variant == "FLOAT": feat.update({key: None})
-            if variant == "LONG": feat.update({key: None})
-            if variant == "SHORT": feat.update({key: None})
-            
-    print(feat) 
-    return feat
+    #### setting attributes to feature
+    feat_updated = updateFeat(feat, fields, feature)
+    print(feat)
+    print(fields)
+    return feat_updated
+
+def addFeatVariant(key, variant, value, f):
     
+    feat = f
+    if variant == "TEXT": value = str(value) 
+    if value != "NULL" and value != "None":
+        #if key == 'area': print(value); print(type(value)); print(getVariantFromValue(value))
+        if variant == getVariantFromValue(value) or (variant=="FLOAT" and isinstance(value, int)): 
+            feat.update({key: value}) 
+        elif variant == "LONG" and isinstance(value, float): # if object has been modified 
+            feat.update({key: int(value)}) 
+        elif variant == "FLOAT" and isinstance(value, int): # if object has been modified 
+            feat.update({key: float(value)}) 
+        else: feat.update({key: None}) 
+    elif variant == "TEXT" or variant == "FLOAT" or variant == "LONG" or variant == "SHORT": feat.update({key: None})
+    return feat 
+
+def updateFeat(feat:dict, fields: dict, feature: Base) -> Dict[str, Any]:
+    
+    for key, variant in fields.items(): 
+        try:
+            if key == "Speckle_ID": 
+                value = str(feature["id"])
+                
+                feat[key] = value 
+
+                feat = addFeatVariant(key, variant, value, feat)
+            else:
+                try: 
+                    value = feature[key] 
+                    #if key == "area": print(feature[key]); print(type(feature[key]))
+                    feat = addFeatVariant(key, variant, value, feat)
+                except:
+                    value = None
+                    rootName = key.split("_")[0]
+                    #try: # if the root category exists
+                    # if its'a list 
+                    if isinstance(feature[rootName], list):
+                        for i in range(len(feature[rootName])):
+                            try:
+                                newF, newVals = traverseDict({}, {}, rootName + "_" + str(i), feature[rootName][i])
+                                for i, (key,value) in enumerate(newVals.items()):
+                                    for k, (x,y) in enumerate(newF.items()):
+                                        if key == x: variant = y; break
+                                    feat = addFeatVariant(key, variant, value, feat)
+                            except Exception as e: print(e)
+                    #except: # if not a list
+                    else:
+                        try:
+                            newF, newVals = traverseDict({}, {}, rootName, feature[rootName])
+                            for i, (key,value) in enumerate(newVals.items()):
+                                for k, (x,y) in enumerate(newF.items()):
+                                    if key == x: variant = y; break
+                                feat = addFeatVariant(key, variant, value, feat)
+                        except Exception as e: feat.update({key: None})
+        except Exception as e: 
+            feat.update({key: None})
+    feat_sorted = {k: v for k, v in sorted(feat.items(), key=lambda item: item[0])}
+    #print("_________________end of updating a feature_________________________")
+    return feat_sorted
+
 def rasterFeatureToSpeckle(selectedLayer: arcLayer, projectCRS: arcpy.SpatialReference, project: ArcGISProject) -> Base:
     print("_________ Raster feature to speckle______") 
     # https://pro.arcgis.com/en/pro-app/latest/arcpy/classes/raster-object.htm 
 
-    r'''
-    # Save layer file to read symbology 
-    # https://pro.arcgis.com/en/pro-app/latest/tool-reference/data-management/save-to-layer-file.htm
-    layerFile = project.homeFolder + "\\" + selectedLayer.name.split(".")[0] 
-    arcpy.management.SaveToLayerFile(selectedLayer.name, layerFile, "ABSOLUTE")
-
-    # read the file and then delete
-    f = open(layerFile + ".lyrx", "r")
-    layerFileContent = json.loads(f.read())
-    print(layerFileContent)
-    f.close()
-    os.remove(layerFile + ".lyrx")
-    '''
-
     # get Raster object of entire raster dataset 
     my_raster = arcpy.Raster(selectedLayer.dataSource)
     print(my_raster.mdinfo) # None
@@ -161,7 +224,10 @@ def rasterFeatureToSpeckle(selectedLayer: arcLayer, projectCRS: arcpy.SpatialRef
 
     #ds = gdal.Open(selectedLayer.source(), gdal.GA_ReadOnly)
     extent = my_raster.extent
-    rasterOriginPoint = arcpy.PointGeometry(arcpy.Point(extent.XMin, extent.YMin, extent.ZMin), my_raster.spatialReference, has_z = True)
+    print(extent.XMin)
+    print(extent.YMin)
+    rasterOriginPoint = arcpy.PointGeometry(arcpy.Point(extent.XMin, extent.YMax, extent.ZMin), my_raster.spatialReference, has_z = True)
+    #if extent.YMin>0: rasterOriginPoint = arcpy.PointGeometry(arcpy.Point(extent.XMin, extent.YMax, extent.ZMin), my_raster.spatialReference, has_z = True)
     print(rasterOriginPoint)
     rasterResXY = [my_raster.meanCellWidth, my_raster.meanCellHeight] #[float(ds.GetGeoTransform()[1]), float(ds.GetGeoTransform()[5])]
     rasterBandNoDataVal = [] #list(my_raster.noDataValues)
@@ -178,6 +244,8 @@ def rasterFeatureToSpeckle(selectedLayer: arcLayer, projectCRS: arcpy.SpatialRef
         reprojectedPt = rasterOriginPoint
         if my_raster.spatialReference.name != projectCRS.name: 
             reprojectedPt = findTransformation(reprojectedPt, "Point", my_raster.spatialReference, projectCRS, selectedLayer)
+            if reprojectedPt is None: 
+                reprojectedPt = rasterOriginPoint
         geom = pointToSpeckle(reprojectedPt.getPart(), None, None)
         if (geom != None):
             b['displayValue'] = [geom]
@@ -192,7 +260,7 @@ def rasterFeatureToSpeckle(selectedLayer: arcLayer, projectCRS: arcpy.SpatialRef
         print(np.shape(rb.read()))
         valMin = rb.minimum
         valMax = rb.maximum
-        bandVals = np.flip(np.swapaxes(rb.read(), 1, 2), 0).flatten() #.tolist()
+        bandVals = np.swapaxes(rb.read(), 1, 2).flatten() #.tolist() np.flip( , 0)
 
         bandValsFlat = []
         bandValsFlat.extend(bandVals.tolist())
@@ -229,6 +297,8 @@ def rasterFeatureToSpeckle(selectedLayer: arcLayer, projectCRS: arcpy.SpatialRef
             else: rasterBandNoDataVal.append(rb.noDataValue)
 
         except: rasterBandNoDataVal.append(rb.noDataValue)
+        if rasterBandNoDataVal[len(rasterBandNoDataVal)-1] is None: 
+            rasterBandNoDataVal[len(rasterBandNoDataVal)-1] = 'None'
 
         
         rasterBandVals.append(bandValsFlat)
@@ -240,7 +310,7 @@ def rasterFeatureToSpeckle(selectedLayer: arcLayer, projectCRS: arcpy.SpatialRef
         b["@(10000)" + item + "_values"] = bandValsFlat #[0:int(max_values/rasterBandCount)]
     
     b["X resolution"] = rasterResXY[0]
-    b["Y resolution"] = rasterResXY[1]
+    b["Y resolution"] = -1* rasterResXY[1]
     b["X pixels"] = rasterDimensions[0]
     b["Y pixels"] = rasterDimensions[1]
     b["Band count"] = rasterBandCount
@@ -252,7 +322,7 @@ def rasterFeatureToSpeckle(selectedLayer: arcLayer, projectCRS: arcpy.SpatialRef
     colors = []
     count = 0
     
-    print(my_raster.variables)
+    #print(my_raster.variables)
     print(selectedLayer.symbology) #None
     colorizer = None
     #renderer = selectedLayer.symbology.renderer
@@ -261,8 +331,58 @@ def rasterFeatureToSpeckle(selectedLayer: arcLayer, projectCRS: arcpy.SpatialRef
 
         print(colorizer) # <arcpy._colorizer.RasterStretchColorizer object at 0x000001780497FBC8>
         print(colorizer.type) # RasterStretchColorizer 
+    else: 
+        #RGB colorizer 
+        root_path = "\\".join(project.filePath.split("\\")[:-1])
+        if not os.path.exists(root_path + '\\Layers_Speckle\\raster_bands'): os.makedirs(root_path + '\\Layers_Speckle\\raster_bands')
+        path_style = root_path + '\\Layers_Speckle\\raster_bands\\' + selectedLayer.name + '_temp.lyrx'
+        symJson = jsonFromLayerStyle(selectedLayer, path_style)
+
+        # read from Json
+        print(symJson["layerDefinitions"][0]["colorizer"])
+        try: greenBand = symJson["layerDefinitions"][0]["colorizer"]["greenBandIndex"]
+        except: greenBand = None
+        try: blueBand = symJson["layerDefinitions"][0]["colorizer"]["blueBandIndex"] 
+        except: blueBand = None
+        
+        try: redBand = symJson["layerDefinitions"][0]["colorizer"]["redBandIndex"]
+        except: 
+            if blueBand!=0 and greenBand!=0: redBand= 0
+            else: redBand = None
+        print("bands")
+        print(redBand)
+        print(greenBand)
+        print(blueBand)
+        try: 
+            rbVals = rasterBandVals[redBand] #my_raster.getRasterBands(rasterBandNames[redBand])
+            rbvalMin = min(rbVals)
+            rbvalMax = max(rbVals)
+            rvalRange = float(rbvalMax) - float(rbvalMin)
+            print(rbvalMin)
+            print(rbvalMax)
+            print(rvalRange)
+        except Exception as e: print(e); rvalRange = None
+        try: 
+            gbVals = rasterBandVals[greenBand]
+            gbvalMin = min(gbVals)
+            gbvalMax = max(gbVals)
+            gvalRange = float(gbvalMax) - float(gbvalMin)
+            print(gbvalMin)
+            print(gbvalMax)
+            print(gvalRange)
+        except: gvalRange = None
+        try: 
+            bbVals = rasterBandVals[blueBand]
+            bbvalMin = min(bbVals)
+            bbvalMax = max(bbVals)
+            bvalRange = float(bbvalMax) - float(bbvalMin)
+            print(bbvalMin)
+            print(bbvalMax)
+            print(bvalRange)
+        except: bvalRange = None
+
     rendererType = ""
-    if hasattr(selectedLayer.symbology, 'renderer'): rendererType = selectedLayer.symbology.renderer.type #e.g. SimpleRenderer
+    #if hasattr(selectedLayer.symbology, 'renderer'): rendererType = selectedLayer.symbology.renderer.type #e.g. SimpleRenderer
     # custom color ramp {"type": "algorithmic", "fromColor": [115, 76, 0, 255],"toColor": [255, 25, 86, 255], "algorithm": "esriHSVAlgorithm"}.
     # custom color map {'values': [0, 1, 2, 3, 4, 5], 'colors': ['#000000', '#DCFFDF', '#B8FFBE', '#85FF90', '#50FF60','#00AB10']}
 
@@ -316,12 +436,12 @@ def rasterFeatureToSpeckle(selectedLayer: arcLayer, projectCRS: arcpy.SpatialRef
     # identify symbology type and if Multiband, which band is which color
     for v in range(rasterDimensions[1] ): #each row, Y
         for h in range(rasterDimensions[0] ): #item in a row, X
-            pt1 = arcpy.PointGeometry(arcpy.Point(extent.XMin+h*rasterResXY[0],extent.YMin+v*rasterResXY[1]), my_raster.spatialReference, has_z = True)
-            pt2 = arcpy.PointGeometry(arcpy.Point(extent.XMin+h*rasterResXY[0], extent.YMin+(v+1)*rasterResXY[1]), my_raster.spatialReference, has_z = True)
-            pt3 = arcpy.PointGeometry(arcpy.Point(extent.XMin+(h+1)*rasterResXY[0], extent.YMin+(v+1)*rasterResXY[1]), my_raster.spatialReference, has_z = True)
-            pt4 = arcpy.PointGeometry(arcpy.Point(extent.XMin+(h+1)*rasterResXY[0], extent.YMin+v*rasterResXY[1]), my_raster.spatialReference, has_z = True)
+            pt1 = arcpy.PointGeometry(arcpy.Point(extent.XMin+h*rasterResXY[0],extent.YMax-v*rasterResXY[1]), my_raster.spatialReference, has_z = True)
+            pt2 = arcpy.PointGeometry(arcpy.Point(extent.XMin+h*rasterResXY[0], extent.YMax-(v+1)*rasterResXY[1]), my_raster.spatialReference, has_z = True)
+            pt3 = arcpy.PointGeometry(arcpy.Point(extent.XMin+(h+1)*rasterResXY[0], extent.YMax-(v+1)*rasterResXY[1]), my_raster.spatialReference, has_z = True)
+            pt4 = arcpy.PointGeometry(arcpy.Point(extent.XMin+(h+1)*rasterResXY[0], extent.YMax-v*rasterResXY[1]), my_raster.spatialReference, has_z = True)
             # first, get point coordinates with correct position and resolution, then reproject each:
-            if my_raster.spatialReference.name != projectCRS.name:
+            if my_raster.spatialReference.exportToString() != projectCRS.exportToString():
                 pt1 = findTransformation(pt1, "Point", my_raster.spatialReference, projectCRS, selectedLayer)
                 pt2 = findTransformation(pt2, "Point", my_raster.spatialReference, projectCRS, selectedLayer)
                 pt3 = findTransformation(pt3, "Point", my_raster.spatialReference, projectCRS, selectedLayer)
@@ -380,28 +500,71 @@ def rasterFeatureToSpeckle(selectedLayer: arcLayer, projectCRS: arcpy.SpatialRef
                 print(colorizer.groups)
             '''
             #else:  
-            if colorizer:   
-                try: bandIndex = int(colorizer.band)
-                except: pass            
-                if rasterBandVals[bandIndex][int(count/4)] >= float(colorizer.minLabel) and rasterBandVals[bandIndex][int(count/4)] <= float(colorizer.maxLabel) : #rasterBandMinVal[bandIndex]: 
-                    # REMAP band values to (0,255) range
-                    valRange = float(colorizer.maxLabel) - float(colorizer.minLabel) #(rasterBandMaxVal[bandIndex] - rasterBandMinVal[bandIndex])
-                    colorVal = int( (rasterBandVals[bandIndex][int(count/4)] - float(colorizer.minLabel)) / valRange * 255 )
-                    if colorizer.invertColorRamp is True: colorVal = int( (-rasterBandVals[bandIndex][int(count/4)] + float(colorizer.maxLabel)) / valRange * 255 )
-                    color =  (colorVal<<16) + (colorVal<<8) + colorVal
-            else:
-                # REMAP band values to (0,255) range
-                rbVals = my_raster.getRasterBands(rasterBandNames[0])
-                try:
-                    rbvalMin = rbVals.minimum
-                    rbvalMax = rbVals.maximum
-                except:
-                    rbvalMin = min(rbVals)
-                    rbvalMax = max(rbVals)
+            if hasattr(selectedLayer.symbology, 'colorizer'): # only 1 band 
+                try: bandIndex = int(colorizer.band) # if stretched 
+                except: pass     
+                if colorizer.type =='RasterUniqueValueColorizer': 
+                    # REDO !!!!!!!!!!!!
+                    colorRVal = colorGVal = colorBVal = 0 
+                    try:
+                        for br in colorizer.groups:
+                            print(br.heading) #"Value"
+                            # go through all values classified 
+                            if br.heading != 'Value': print(int('x')) #call exception
+                            for k, itm in enumerate(br.items):
+                                print(itm.values)
+                                if itm.values[0] == rasterBandVals[bandIndex][int(count/4)]:
+                                    print(itm.values[0])
+                                    colorRVal, colorGVal, colorBVal = itm.color['RGB'][0], itm.color['RGB'][1], itm.color['RGB'][2]
+                                    break
+                                # if string covering float
+                                try: 
+                                    if float(itm.values[0]) == float(rasterBandVals[bandIndex][int(count/4)]):
+                                        print(itm.values[0])
+                                        colorRVal, colorGVal, colorBVal = itm.color['RGB'][0], itm.color['RGB'][1], itm.color['RGB'][2]
+                                        break
+                                except Exception as e: print(e); pass
 
-                valRange = float(rbvalMax) - float(rbvalMin) #(rasterBandMaxVal[bandIndex] - rasterBandMinVal[bandIndex])
-                colorVal = int( (rasterBandVals[bandIndex][int(count/4)] - float(rbvalMin)) / valRange * 255 )
-                color =  (colorVal<<16) + (colorVal<<8) + colorVal
+
+                    except Exception as e: # if no Min Max labels:
+                        # REMAP band values to (0,255) range
+                        print(e)
+                        valRange = max(rasterBandVals[bandIndex]) - min(rasterBandVals[bandIndex]) #(rasterBandMaxVal[bandIndex] - rasterBandMinVal[bandIndex])
+                        colorRVal = colorGVal = colorBVal = int( (rasterBandVals[bandIndex][int(count/4)] - min(rasterBandVals[bandIndex])) / valRange * 255 )
+                    
+                    print("__pixel color_")
+                    print(colorRVal)
+                    print(colorGVal)
+                    print(colorBVal)
+                    color =  (colorRVal<<16) + (colorGVal<<8) + colorBVal
+
+                else:
+                    try:
+                        if rasterBandVals[bandIndex][int(count/4)] >= float(colorizer.minLabel) and rasterBandVals[bandIndex][int(count/4)] <= float(colorizer.maxLabel) : #rasterBandMinVal[bandIndex]: 
+                            # REMAP band values to (0,255) range
+                            valRange = float(colorizer.maxLabel) - float(colorizer.minLabel) #(rasterBandMaxVal[bandIndex] - rasterBandMinVal[bandIndex])
+                            colorVal = int( (rasterBandVals[bandIndex][int(count/4)] - float(colorizer.minLabel)) / valRange * 255 )
+                            if colorizer.invertColorRamp is True: colorVal = int( (-rasterBandVals[bandIndex][int(count/4)] + float(colorizer.maxLabel)) / valRange * 255 )
+                            color =  (colorVal<<16) + (colorVal<<8) + colorVal
+                    except: # if no Min Max labels:
+                        # REMAP band values to (0,255) range
+                        valRange = max(rasterBandVals[bandIndex]) - min(rasterBandVals[bandIndex]) #(rasterBandMaxVal[bandIndex] - rasterBandMinVal[bandIndex])
+                        colorVal = int( (rasterBandVals[bandIndex][int(count/4)] - min(rasterBandVals[bandIndex])) / valRange * 255 )
+                        color =  (colorVal<<16) + (colorVal<<8) + colorVal
+            else: # rgb
+                # REMAP band values to (0,255) range
+                if rvalRange is not None and redBand is not None: colorRVal = int( (rasterBandVals[redBand][int(count/4)] - float(rbvalMin)) / rvalRange * 255 )
+                else: colorRVal = 0
+                if gvalRange is not None and greenBand is not None: colorGVal = int( (rasterBandVals[greenBand][int(count/4)] - float(gbvalMin)) / gvalRange * 255 )
+                else: colorGVal = 0
+                if bvalRange is not None and blueBand is not None: colorBVal = int( (rasterBandVals[blueBand][int(count/4)] - float(bbvalMin)) / bvalRange * 255 )
+                else: colorBVal = 0
+                print("__pixel color_")
+                print(colorRVal)
+                print(colorGVal)
+                print(colorBVal)
+
+                color =  (colorRVal<<16) + (colorGVal<<8) + colorBVal
 
             colors.extend([color,color,color,color])
             count += 4

speckle_toolbox/esri/toolboxes/speckle/converter/layers/symbologyTemplates.py

@@ -0,0 +1,606 @@
+import json
+from typing import Any, List, Tuple, Union
+import copy
+import os
+
+from typing import Dict
+
+import arcpy 
+from arcpy._mp import ArcGISProject
+from arcpy.management import (CreateFeatureclass, MakeFeatureLayer,
+                              AddFields, AlterField, DefineProjection )
+
+from specklepy.objects import Base
+try:
+    from speckle.converter.layers.Layer import Layer, VectorLayer, RasterLayer 
+except:
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.Layer import Layer, VectorLayer, RasterLayer 
+
+def jsonFromLayerStyle(layerArcgis, path_style):
+    # write updated renderer to file and get layerStyle variable 
+    arcpy.management.SaveToLayerFile(layerArcgis, path_style, False)
+    f = open(path_style, "r")
+    layerStyle = json.loads(f.read())
+    f.close()
+    os.remove(path_style)
+    return layerStyle
+
+def symbol_color_to_speckle(color: dict):
+    newColor = (0<<16) + (0<<8) + 0
+    try: 
+        r = int(color['RGB'][0])
+        g = int(color['RGB'][1])
+        b = int(color['RGB'][2])
+        newColor = (r<<16) + (g<<8) + b
+    except: pass 
+    return newColor
+
+def vectorRendererToNative(project: ArcGISProject, active_map, layerGroup, layerSpeckle: Union[Layer, VectorLayer], layerArcgis, f_class, existingAttrs: List) -> Union[None, Dict[str, Any]] :
+    print("___________APPLY VECTOR RENDERER______________")
+    print(layerArcgis)
+    print(f_class)
+    renderer = layerSpeckle.renderer 
+
+    if renderer and renderer['type']:
+        print(renderer['type'])
+
+        root_path = "\\".join(project.filePath.split("\\")[:-1])
+        #path_style = root_path + '\\' + str(f_class).split('\\')[-1] + '_old.lyrx'
+
+        data = arcpy.Describe(layerArcgis.dataSource)
+        if layerArcgis.isFeatureLayer:
+            geomType = data.shapeType
+            sym = layerArcgis.symbology
+
+            if renderer['type'] == 'singleSymbol':
+                print("RENDERER SINGLE")
+                print(renderer)
+
+                r,g,b = get_rgb_from_speckle(renderer['properties']['symbol']['symbColor']) 
+                #print(r,g,b)
+                #print(sym.renderer.symbol.color)
+                sym.renderer.symbol.color = {'RGB': [r, g, b, 100]}
+                #print(sym.renderer.symbol.color)
+                layerArcgis.symbology = sym # SimpleRenderer
+                #print(layerArcgis)
+                return layerArcgis 
+
+            elif renderer['type']  == 'categorizedSymbol':
+                print("RENDERER CATEGORIZED")
+                print(renderer)
+
+                cats = renderer['properties']['categories']
+                attribute = renderer['properties']['attribute']
+                if attribute not in existingAttrs: return layerArcgis 
+
+                #vl2 = active_map.addLayer(layerArcgis)[0]
+                #sym = layerArcgis.symbology
+                sym.updateRenderer('UniqueValueRenderer')
+                print(sym.renderer.type)
+                print(existingAttrs)
+                print(attribute)
+
+                sym.renderer.fields = [attribute]
+                for k, grp in enumerate(sym.renderer.groups):
+                    for itm in grp.items:
+                        transVal = itm.values[0][0] #Grab the first "percent" value in the list of potential values
+                        for i in range(len(cats)):
+                            label = cats[i]['value'] 
+                            if label is None or label=="" or str(label)=="": label = "<Null>"
+                            r,g,b = get_rgb_from_speckle(cats[i]['symbColor']) 
+
+                            if str(transVal) == label:
+                                itm.symbol.color = {'RGB': [r, g, b, 100]}
+                                itm.label = label
+                                break
+                layerArcgis.symbology = sym
+                return layerArcgis 
+
+            elif renderer['type'] == 'graduatedSymbol': 
+                print("RENDERER GRADUATED")
+                print(renderer)
+
+                attribute = renderer['properties']['attribute']
+                gradMetod = renderer['properties']['gradMethod'] # by color or by size
+                if gradMetod  != 0: 
+                    r,g,b = get_rgb_from_speckle(renderer['properties']['sourceSymbColor'] ) 
+                    sym.renderer.symbol.color = {'RGB': [r, g, b, 100]}
+                    layerArcgis.symbology = sym # SimpleRenderer
+                    return layerArcgis 
+                if attribute not in existingAttrs or gradMetod != 0: return layerArcgis # by color, not line width 
+
+                sym.updateRenderer('GraduatedColorsRenderer')
+                print(sym.renderer.type)
+
+                r,g,b = get_rgb_from_speckle(renderer['properties']['sourceSymbColor']) 
+                ramp = renderer['properties']['ramp'] # {discrete, rampType, stops}
+                ranges = renderer['properties']['ranges'] # []
+
+                # get all existing values 
+                all_values = []
+                with arcpy.da.UpdateCursor(f_class, attribute) as cur:
+                    for rowShape in cur: 
+                        all_values.append(rowShape[0])
+                    print(all_values)
+                del cur 
+
+                print(len(ranges))
+                sym.renderer.classificationField = attribute
+                print(sym.renderer.breakCount)
+                sym.renderer.breakCount = len(ranges)
+                print(sym.renderer.breakCount)
+                
+                if len(sym.renderer.classBreaks) > 0:
+                    totalClasses = 0
+                    for k, br in enumerate(ranges):
+                        
+                        print(totalClasses)
+                        if sym.renderer.breakCount < len(ranges):
+                            valFits = 0
+                            # check if any existing value fits in this range:
+                            for val in all_values: 
+                                if val <= ranges[k]["upper"] and (totalClasses==0 or (totalClasses>0 and sym.renderer.classBreaks[totalClasses-1].upperBound<val)):
+                                    valFits+=1
+                                    break
+                            if valFits == 0: continue
+                            
+                        r,g,b = get_rgb_from_speckle(ranges[k]['symbColor']) 
+
+                        #classBreak.upperBound = ranges[k]["upper"]
+                        sym.renderer.classBreaks[totalClasses].upperBound = ranges[k]["upper"]
+                        sym.renderer.classBreaks[totalClasses].label = ranges[k]["label"]
+                        sym.renderer.classBreaks[totalClasses].symbol.color = {'RGB': [r, g, b, 100]}
+                        totalClasses += 1 
+
+                        #layerArcgis.symbology = sym
+                        print(ranges[k]["label"])
+                        print(ranges[k]["upper"])
+
+                    sym.renderer.classBreaks[0].upperBound = ranges[0]["upper"] # otherwise its assigned maximum value
+
+                layerArcgis.symbology = sym
+                return layerArcgis 
+
+            else: return None 
+
+def get_rgb_from_speckle(rgb: int) -> Tuple[int, int, int]:
+    r = g = b = 0
+    try: 
+        r = (rgb & 0xFF0000) >> 16
+        g = (rgb & 0xFF00) >> 8
+        b = rgb & 0xFF 
+    except: r = g = b = 0
+    
+    r,g,b = check_rgb(r,g,b)
+    return r,g,b 
+
+def check_rgb(r:int, g:int, b:int) -> Tuple[int, int, int]:
+
+    if not isinstance(r, int) or r<0 or r>255: r=g=b=0
+    if not isinstance(g, int) or g<0 or g>255: r=g=b=0
+    if not isinstance(b, int) or b<0 or b>255: r=g=b=0
+    return r,g,b 
+
+
+
+def rasterRendererToNative(project: ArcGISProject, active_map, layerGroup,  layer: RasterLayer, arcLayer, rasterPathsToMerge, newName):
+    print("_____rasterRenderer ToNative______")
+    renderer = layer.renderer
+    rendererNew = None
+    print(renderer)
+
+    feat = layer.features[0]
+    print(feat)
+
+    bandNames = feat["Band names"]
+    print(bandNames)
+
+    sym = arcLayer.symbology
+    symJson = None
+    path_style = ""
+    path_style2 = ""
+
+    print(sym)
+
+    if renderer and renderer['type']:
+        
+        if not hasattr(arcLayer.symbology, 'colorizer'): 
+            # multiband raster, CIMRasterRGBColorizer 
+            # arcpy doesnt support multiband raster symbology: https://community.esri.com/t5/arcgis-api-for-python-questions/why-does-arcpy-mp-arcgis-pro-2-6-mosaic-dataset/td-p/1016312 
+            root_path = "\\".join(project.filePath.split("\\")[:-1])
+            if not os.path.exists(root_path + '\\Layers_Speckle\\raster_bands'): os.makedirs(root_path + '\\Layers_Speckle\\raster_bands')
+            path_style = root_path + '\\Layers_Speckle\\raster_bands\\' + newName + '_old.lyrx'
+            path_style2 = root_path + '\\Layers_Speckle\\raster_bands\\' + newName + '_new.lyrx'
+            symJson = jsonFromLayerStyle(arcLayer, path_style)
+
+        if renderer['type']  == 'singlebandgray':
+            print("Singleband grey")
+            band_index = renderer['properties']['band']-1
+            if symJson is None:
+                sym.updateColorizer('RasterStretchColorizer')
+                sym.colorizer.band = band_index
+                arcLayer.symbology = sym
+            else: 
+                temp = arcpy.management.MakeRasterLayer(rasterPathsToMerge[band_index], newName + "_temp").getOutput(0)
+                active_map.addLayerToGroup(layerGroup, temp)
+                temp_layer = None
+                for l in active_map.listLayers(): 
+                    if l.longName == layerGroup.longName + "\\" + newName + "_temp":
+                        print(l.longName)
+                        temp_layer = l 
+                        break
+
+                sym = temp_layer.symbology
+                sym.updateColorizer('RasterStretchColorizer')
+                sym.colorizer.band = band_index
+                arcLayer.symbology = sym
+
+                active_map.removeLayer(temp_layer) 
+                
+
+        elif renderer['type']  == 'multibandcolor':
+            print("Multiband")
+            if symJson is None: 
+                sym.updateColorizer('RasterStretchColorizer')
+                arcLayer.symbology = sym 
+            else:
+                
+                redSt =  copy.deepcopy(symJson["layerDefinitions"][0]["colorizer"]["stretchStatsRed"])
+                greenSt =  copy.deepcopy(symJson["layerDefinitions"][0]["colorizer"]["stretchStatsGreen"])
+                blueSt =  copy.deepcopy(symJson["layerDefinitions"][0]["colorizer"]["stretchStatsBlue"])
+
+                redBand = renderer['properties']['redBand']
+                greenBand = renderer['properties']['greenBand']
+                blueBand = renderer['properties']['blueBand']
+                try: symJson["layerDefinitions"][0]["colorizer"]["greenBandIndex"] = greenBand-1
+                except: symJson["layerDefinitions"][0]["colorizer"]["greenBandIndex"] = 0
+
+                try: symJson["layerDefinitions"][0]["colorizer"]["redBandIndex"] = redBand-1
+                except: symJson["layerDefinitions"][0]["colorizer"]["redBandIndex"] = 0
+                
+                try: symJson["layerDefinitions"][0]["colorizer"]["blueBandIndex"] = blueBand-1
+                except: symJson["layerDefinitions"][0]["colorizer"]["blueBandIndex"] = 0
+                
+                print(symJson)
+                f = open(path_style2, "w")
+                f.write(json.dumps(symJson, indent=2))
+                f.close()
+
+                active_map.removeLayer(arcLayer) 
+                lyrFile = arcpy.mp.LayerFile(path_style2)
+                active_map.addLayerToGroup(layerGroup, lyrFile )
+                
+                os.remove(path_style2) 
+
+        elif renderer['type']  == 'paletted':
+            print("Paletted")
+            band_index = renderer['properties']['band']-1
+
+            if symJson is None:
+                for br in sym.colorizer.groups:
+                    print(br.heading) #"Value"
+                    # go through all values classified 
+                    for k, itm in enumerate(br.items):
+                        if k< len(renderer['properties']['classes']):
+                            #go through saved renderer classes
+                            for n, cl in enumerate(renderer['properties']['classes']):
+                                if k == n:
+                                    r,g,b = get_rgb_from_speckle(cl['color']) 
+                                    itm.color = {'RGB': [r,g,b, 100]}
+                                    itm.label = cl['label']
+                                    itm.values = cl['value']
+                        else: pass
+                arcLayer.symbology = sym 
+        else: 
+            sym.updateColorizer('RasterStretchColorizer')
+            arcLayer.symbology = sym
+    return arcLayer
+    
+
+def rendererToSpeckle(project: ArcGISProject, active_map, arcLayer, rasterFeat: Base):
+    print("_____rasterRenderer To Speckle______")
+    if arcLayer.isRasterLayer: 
+        try: 
+            rType = arcLayer.symbology.colorizer.type # 'singleSymbol','categorizedSymbol','graduatedSymbol',
+            if rType =='RasterStretchColorizer':  rType = 'singlebandgray'
+            elif rType =='RasterUniqueValueColorizer':  rType = 'paletted' # only for 1-band raster 
+            else: rType = 'singlebandgray'
+        except: 
+            rType = "multibandcolor"
+            root_path = "\\".join(project.filePath.split("\\")[:-1])
+            if not os.path.exists(root_path + '\\Layers_Speckle\\raster_bands'): os.makedirs(root_path + '\\Layers_Speckle\\raster_bands')
+            path_style = root_path + '\\Layers_Speckle\\raster_bands\\' + arcLayer.name + '_temp.lyrx'
+            #path_style2 = root_path + '\\' + newName + '_new.lyrx'
+            symJson = jsonFromLayerStyle(arcLayer, path_style)
+
+        layerRenderer: Dict[str, Any] = {}
+        layerRenderer['type'] = rType
+        print(rType)
+        my_raster = arcpy.Raster(arcLayer.dataSource)  
+        rasterBandNames = my_raster.bandNames
+        
+        #bandNames = rasterFeat["Band names"]
+        bandValues = [rasterFeat["@(10000)" + name + "_values"] for name in rasterBandNames]
+
+        if rType == "singlebandgray":  
+            try: band = arcLayer.symbology.colorizer.band 
+            except: band = 0 
+            try: 
+                bVals = bandValues[band]
+                bvalMin = min(bVals)
+                bvalMax = max(bVals)
+            except: 
+                bvalMin = 0
+                bvalMax = 255
+            layerRenderer.update({'properties': {'max':bvalMax,'min':bvalMin,'band':band+1,'contrast':1}}) 
+
+        elif rType == "multibandcolor":
+
+            try: greenBand = symJson["layerDefinitions"][0]["colorizer"]["greenBandIndex"] +1
+            except: greenBand = None
+            try: blueBand = symJson["layerDefinitions"][0]["colorizer"]["blueBandIndex"] +1
+            except: blueBand = None
+            try: redBand = symJson["layerDefinitions"][0]["colorizer"]["redBandIndex"] +1
+            except: 
+                print(greenBand)
+                print(blueBand)
+                if blueBand!=1 and greenBand!=1: redBand= 1
+                else: redBand = None
+            print(redBand)
+
+            try: 
+                rbVals = bandValues[redBand-1]
+                rbvalMin = min(rbVals)
+                rbvalMax = max(rbVals)
+                print(rbvalMin)
+                print(rbvalMax)
+            except: 
+                rbvalMin = 0
+                rbvalMax = 255
+            try: 
+                gbVals = bandValues[greenBand-1]
+                gbvalMin = min(gbVals)
+                gbvalMax = max(gbVals)
+            except: 
+                gbvalMin = 0
+                gbvalMax = 255
+            try: 
+                bbVals = bandValues[blueBand-1]
+                bbvalMin = min(bbVals)
+                bbvalMax = max(bbVals) 
+            except: 
+                bbvalMin = 0
+                bbvalMax = 255
+
+            layerRenderer.update({'properties': {'greenBand':greenBand,'blueBand':blueBand,'redBand':redBand}})
+            layerRenderer['properties'].update({'redContrast':1,'redMin':rbvalMin,'redMax':rbvalMax})
+            layerRenderer['properties'].update({'greenContrast':1,'greenMin':gbvalMin,'greenMax':gbvalMax})
+            layerRenderer['properties'].update({'blueContrast':1,'blueMin':bbvalMin,'blueMax':bbvalMax})
+        elif rType == "paletted":  
+            band = 0 
+            rendererClasses = arcLayer.symbology.colorizer.groups
+            classes = []
+            sourceRamp = {}
+
+            for i, cl in enumerate(rendererClasses):
+                if cl.heading == 'Value':
+                    for k, itm in enumerate(cl.items):
+                        value = itm.values[0]
+                        label = itm.label 
+                        try: 
+                            r,g,b = itm.color['RGB'][0], itm.color['RGB'][1], itm.color['RGB'][2]
+                            sColor = (r<<16) + (g<<8) + b
+                            classes.append({'color':sColor,'value':value,'label':label})
+                        except: pass 
+            layerRenderer.update({'properties': {'classes':classes,'ramp':sourceRamp,'band':band+1}})
+
+        return layerRenderer 
+    elif arcLayer.isFeatureLayer: 
+        layerRenderer: Dict[str, Any] = {}
+
+        sym = arcLayer.symbology
+        print(sym.renderer.type)
+
+        if sym.renderer.type == 'SimpleRenderer':
+            layerRenderer['type'] = 'singleSymbol'
+            layerRenderer['properties'] = {'symbol':{}, 'symbType':""}
+            symbolColor = symbol_color_to_speckle(sym.renderer.symbol.color)
+            layerRenderer['properties'].update({'symbol':{'symbColor': symbolColor}, 'symbType':''})
+
+        elif sym.renderer.type == 'UniqueValueRenderer':
+            layerRenderer['type'] = 'categorizedSymbol'
+            layerRenderer['properties'] = {'attribute': '', 'symbType': ''} #{'symbol':{}, 'ramp':{}, 'ranges':{}, 'gradMethod':"", 'symbType':"", 'legendClassificationAttribute': ""}
+            
+            attribute = sym.renderer.fields[0]
+            layerRenderer['properties']['attribute'] = attribute
+            sourceSymbColor = symbol_color_to_speckle(sym.renderer.defaultSymbol.color)
+            layerRenderer['properties'].update( {'sourceSymbColor': sourceSymbColor} )
+ 
+            categories = sym.renderer.groups
+            layerRenderer['properties']['categories'] = []
+
+            for i, grp in enumerate(categories):
+                for itm in grp.items:
+                    value = itm.values[0][0]
+                    symbColor = symbol_color_to_speckle(itm.symbol.color)
+                    label = itm.label
+                    layerRenderer['properties']['categories'].append({'value':value,'symbColor':symbColor,'symbOpacity':1, 'sourceSymbColor': sourceSymbColor,'label':label})
+
+        elif sym.renderer.type == 'GraduatedColorsRenderer' or sym.renderer.type == 'GraduatedSymbolsRenderer':
+            layerRenderer['type'] = 'graduatedSymbol'
+            layerRenderer['properties'] = {'symbol':{}, 'ramp':{}, 'ranges':{}, 'gradMethod':"", 'symbType':""}
+        
+            attribute = sym.renderer.classificationField 
+            sourceSymbColor = (0<<16) + (0<<8) + 0  
+            layerRenderer['properties'].update( {'attribute': attribute, 'symbType': '', 'gradMethod': 0, 'sourceSymbColor': sourceSymbColor} )
+
+            rRamp = sym.renderer.colorRamp # QgsGradientColorRamp
+            layerRenderer['properties']['ramp'] = {} # gradientColorRampToSpeckle(rRamp)
+
+            rRanges = sym.renderer.classBreaks
+            layerRenderer['properties']['ranges'] = []
+            for itm in rRanges:
+                try: lower = float(itm.label.split(" - ")[0]) if (" - " in rRanges.label) else float(rRanges.label[0])
+                except: lower = 0
+                upper = itm.upperBound
+                symbColor = symbol_color_to_speckle(itm.symbol.color) 
+                label = itm.label
+                width = 0.26
+                # {'label': '1 - 1.4', 'lower': 1.0, 'symbColor': <PyQt5.QtGui.QColor ...BD9B9D4A0>, 'symbOpacity': 1.0, 'upper': 1.4}
+                layerRenderer['properties']['ranges'].append({'lower':lower,'upper':upper,'symbColor':symbColor,'symbOpacity':1,'label':label,'width':width})
+     
+        elif sym.renderer.type == 'UnclassedColorsRenderer':
+            layerRenderer['type'] = 'graduatedSymbol'
+            layerRenderer['properties'] = {'symbol':{}, 'ramp':{}, 'ranges':{}, 'gradMethod':"", 'symbType':""}
+        
+            attribute = sym.renderer.field 
+            sourceSymbColor = (0<<16) + (0<<8) + 0  
+            layerRenderer['properties'].update( {'attribute': attribute, 'symbType': '', 'gradMethod': 0, 'sourceSymbColor': sourceSymbColor} )
+            layerRenderer['properties']['ramp'] = {} # gradientColorRampToSpeckle(rRamp)
+
+            lowest = sym.renderer.lowerLabel
+            highest = sym.renderer.upperLabel
+
+            # trick to get colors 
+            rRamp = sym.renderer.colorRamp # QgsGradientColorRamp
+            arcRamp = project.listColorRamps('White to Black')[0]
+            sym.updateRenderer('GraduatedColorsRenderer')
+            sym.renderer.colorRamp = arcRamp
+            sym.renderer.classificationField = attribute
+            rows_attributes = arcpy.da.SearchCursor(arcLayer.longName, attribute)
+            row_attrs = []
+            row_max = -1000000000
+            row_min = 1000000000
+            for k, attrs in enumerate(rows_attributes): 
+                row_attrs.append(attrs[0])
+                if attrs[0] < row_min: row_min = attrs[0]
+                if attrs[0] > row_max: row_max = attrs[0]
+            row_range = row_max - row_min
+            breakCount = len(list(set(row_attrs))) # only unique values 
+            sym.renderer.breakCount = breakCount
+
+            # run as gradient colors 
+            rRanges = sym.renderer.classBreaks
+            layerRenderer['properties']['ranges'] = []
+            for itm in rRanges:
+                try: lower = float(itm.label.split(" - ")[0]) if (" - " in rRanges.label) else float(rRanges.label[0])
+                except: lower = 0
+                upper = itm.upperBound
+                if row_range==0: rgb = 0
+                else: rgb = 255 - int((itm.upperBound - row_min) / row_range * 255 ) 
+                symbColor = (rgb<<16) + (rgb<<8) + rgb 
+                label = itm.label
+                width = 0.26
+                # {'label': '1 - 1.4', 'lower': 1.0, 'symbColor': <PyQt5.QtGui.QColor ...BD9B9D4A0>, 'symbOpacity': 1.0, 'upper': 1.4}
+                layerRenderer['properties']['ranges'].append({'lower':lower,'upper':upper,'symbColor':symbColor,'symbOpacity':1,'label':label,'width':width})
+
+        else: return None
+            
+        return layerRenderer
+
+    else: return None
+
+
+def featureColorfromNativeRenderer(index: int, arcLayer) -> int:
+    # case with one color for the entire layer
+    #try:
+    sym = arcLayer.symbology
+    color = {'RGB': [100,100,100,100]}
+
+    if sym.renderer.type == 'SimpleRenderer':
+        print('SimpleRenderer')
+        color = sym.renderer.symbol.color
+
+    elif sym.renderer.type == 'UniqueValueRenderer':
+        print('Unique Value Renderer')
+
+        attribute = sym.renderer.fields[0]
+        color = sym.renderer.defaultSymbol.color
+        categories = sym.renderer.groups
+
+        rows_attributes = arcpy.da.SearchCursor(arcLayer.longName, attribute)
+        row_shapes_list = [x for k, x in enumerate(rows_attributes)]
+
+        color_found = 0
+        for i, grp in enumerate(categories):
+            if color_found == 1: break
+            for n, itm in enumerate(grp.items):
+                for k, attrs in enumerate(row_shapes_list): 
+                    if str(itm.values[0][0]) == "<Null>": itm.values[0][0] = None 
+                    if k == index and ( str(attrs[0]) == str(itm.values[0][0]) or (attrs[0] is None and str(itm.values[0][0]) == "<Null>") ): 
+                        color = itm.symbol.color
+                        print("symbol color: ")
+                        print(color)
+                        color_found = 1
+                        break 
+
+    elif sym.renderer.type == 'GraduatedColorsRenderer' or sym.renderer.type == 'GraduatedSymbolsRenderer':
+        print('Graduated Colors / Sybmols Renderer')
+        attribute = sym.renderer.classificationField 
+        rows_attributes = arcpy.da.SearchCursor(arcLayer.longName, attribute)
+        row_shapes_list = [x for k, x in enumerate(rows_attributes)]
+
+        rRanges = sym.renderer.classBreaks
+        upperBounds = [-10000000000000000000]
+        color_found = 0
+        for itm in rRanges:
+            print(itm)
+            if color_found == 1: break
+            for k, attrs in enumerate(row_shapes_list):  
+                try:
+                    if k == index and float(attrs[0]) <= float(itm.upperBound) and (k==0 or float(attrs[0]) > float(upperBounds[-1]) ):
+                        color = itm.symbol.color
+                        color_found = 1
+                        break 
+                except: pass
+            upperBounds.append(itm.upperBound)
+
+    elif sym.renderer.type == 'UnclassedColorsRenderer': 
+        print('UnclassedColorsRenderer')
+        attribute = sym.renderer.field  
+
+        sym.updateRenderer('GraduatedColorsRenderer')
+        sym.renderer.classificationField = attribute
+
+        rows_attributes = arcpy.da.SearchCursor(arcLayer.longName, attribute)
+        row_shapes_list = [x for k, x in enumerate(rows_attributes)]
+        row_attrs = []
+        row_max = -10000000000000000000
+        row_min = 10000000000000000000
+        feat_value = None
+        for k, attrs in enumerate(row_shapes_list): 
+            row_attrs.append(attrs[0])
+            if attrs[0] < row_min: row_min = attrs[0]
+            if attrs[0] > row_max: row_max = attrs[0]
+            if k == index: feat_value = attrs[0]
+        row_range = row_max - row_min
+        breakCount = len(list(set(row_attrs))) # only unique values 
+        sym.renderer.breakCount = breakCount
+
+        # run as gradient colors 
+        
+        upperBounds = [-10000000000000000000]
+        rRanges = sym.renderer.classBreaks
+
+        for itm in rRanges:
+            print(itm)
+            try:
+                if row_range!=0 and float(feat_value) <= float(itm.upperBound) and (len(upperBounds)==0 or float(feat_value) > float(upperBounds[-1])):
+                    rgb = 255 - int((itm.upperBound - row_min) / row_range * 255 ) 
+                    color = {'RGB':[rgb,rgb,rgb,100]}
+                    print(color) 
+                    break
+            except: pass 
+            upperBounds.append(itm.upperBound)
+
+    else: 
+        print('Else')
+        return (100<<16) + (100<<8) + 100
+    
+    print("final color: ")
+    print(color) 
+    # construct RGB color
+    col = symbol_color_to_speckle(color)
+    print(col)
+    return col
+

speckle_toolbox/esri/toolboxes/speckle/converter/layers/utils.py

@@ -6,95 +6,126 @@ import arcpy
 from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
 import os
 
+ATTRS_REMOVE = ['geometry','applicationId','bbox','displayStyle', 'id', 'renderMaterial', 'displayMesh', 'displayValue'] 
 
 def getVariantFromValue(value: Any) -> Union[str, None]:
+    #print("_________get variant from value_______")
     # TODO add Base object
     pairs = [
         (str, "TEXT"), # 10
         (float, "FLOAT"),
         (int, "LONG"),
-        (bool, "SHORT")
-        #date: "SHORT"
+        (bool, "SHORT") 
     ]
     res = None
     for p in pairs:
-        if isinstance(value, p[0]): res = p[1]; break
-    #t = type(value)
-    
-    #try: res = pairs[t]
-    #except: pass
-    #if isinstance(value, str) and "PyQt5.QtCore.QDate(" in value: res = QVariant.Date #14
-    #elif isinstance(value, str) and "PyQt5.QtCore.QDateTime(" in value: res = QVariant.DateTime #16
+        if isinstance(value, p[0]): 
+            res = p[1]
+            try:
+                if res == "LONG" and (value>= 2147483647 or value<= -2147483647):
+                    #https://pro.arcgis.com/en/pro-app/latest/help/data/geodatabases/overview/arcgis-field-data-types.htm
+                    res = "FLOAT"
+            except Exception as e: print(e)
+            break
 
     return res
 
-def getLayerAttributes(features: List[Base], attrsToRemove: List[str] =['geometry','applicationId','bbox','displayStyle', 'id', 'renderMaterial', 'geometry'] ) -> dict:
+def getLayerAttributes(featuresList: List[Base], attrsToRemove: List[str] = ATTRS_REMOVE ) -> Dict[str, str]:
     print("03________ get layer attributes")
+
+    if not isinstance(featuresList, list): features = [featuresList]
+    else: features = featuresList[:]
+    
     fields = {}
     all_props = []
     for feature in features: 
         #get object properties to add as attributes
         dynamicProps = feature.get_dynamic_member_names()
-        #attrsToRemove = ['geometry','applicationId','bbox','displayStyle', 'id', 'renderMaterial', 'geometry']
         for att in attrsToRemove:
             try: dynamicProps.remove(att)
             except: pass
-
         dynamicProps.sort()
-        print(dynamicProps)
 
         # add field names and variands 
         for name in dynamicProps:
-            if name not in all_props: all_props.append(name)
+            #if name not in all_props: all_props.append(name)
 
             value = feature[name]
             variant = getVariantFromValue(value)
+            #if name == 'area': print(value); print(variant)
             if not variant: variant = None #LongLong #4 
 
             # go thought the dictionary object
-            print("go thought the dictionary object")
-            if value and isinstance(value, list) and isinstance(value[0], dict) :
-                all_props.remove(name) # remove generic dict name
-                newF, newVals = traverseDict( {}, {}, name, value[0])
-                #print(newF)
-                #print(newF.items())
-                for i, (k,v) in enumerate(newF.items()):
-                    fields.update({k: v}) 
-                    if k not in all_props: all_props.append(k)
-                #print(fields)
-            
-            # add a field if not existing yet AND if variant is known
-            elif variant and (name not in fields.keys()): 
-                fields.update({name: variant})
-            
-            elif name in fields.keys(): #check if the field was empty previously: 
-                oldVariant = fields[name]
-                # replace if new one is NOT LongLong or IS String
-                if oldVariant != "TEXT" and variant == "TEXT": 
-                    fields.update({name: variant}) 
-        #print(all_props)
+            if value and isinstance(value, list):
+                #all_props.remove(name) # remove generic dict name
+                for i, val_item in enumerate(value):
+                    newF, newVals = traverseDict( {}, {}, name+"_"+str(i), val_item)
 
+                    for i, (k,v) in enumerate(newF.items()):
+                        if k not in all_props: all_props.append(k)
+                        if k not in fields.keys(): fields.update({k: v}) 
+                        else: #check if the field was empty previously: 
+                            oldVariant = fields[k]
+                            # replace if new one is NOT Float (too large integers)
+                            if oldVariant != "FLOAT" and v == "FLOAT": 
+                                fields.update({k: v}) 
+                            # replace if new one is NOT LongLong or IS String
+                            if oldVariant != "TEXT" and v == "TEXT": 
+                                fields.update({k: v}) 
+            
+            # add a field if not existing yet 
+            else: # if str, Base, etc
+                newF, newVals = traverseDict( {}, {}, name, value)
+                
+                for i, (k,v) in enumerate(newF.items()):
+                    if k not in all_props: all_props.append(k)
+                    if k not in fields.keys(): fields.update({k: v}) #if variant is known
+                    else: #check if the field was empty previously: 
+                        oldVariant = fields[k]
+                        # replace if new one is NOT Float (too large integers)
+                        #print(oldVariant, v)
+                        if oldVariant == "LONG" and v == "FLOAT": 
+                            fields.update({k: v}) 
+                        # replace if new one is NOT LongLong or IS String
+                        if oldVariant != "TEXT" and v == "TEXT": 
+                            fields.update({k: v}) 
+                        #print(fields)
     # replace all empty ones wit String
+    all_props.append("Speckle_ID") 
     for name in all_props:
         if name not in fields.keys(): 
-            fields.update({name: "TEXT"}) 
-    print(fields)
-    return fields
+            fields.update({name: 'TEXT'}) 
+
+    fields_sorted = {k: v for k, v in sorted(fields.items(), key=lambda item: item[0])}
+    return fields_sorted
 
 def traverseDict(newF: dict, newVals: dict, nam: str, val: Any):
-    #print("Traverse Dict")
+    
     if isinstance(val, dict):
         for i, (k,v) in enumerate(val.items()):
-            traverseDict( newF, newVals, nam+"_"+k, v)
+            newF, newVals = traverseDict( newF, newVals, nam+"_"+k, v)
+    elif isinstance(val, Base):
+        dynamicProps = val.get_dynamic_member_names()
+        for att in ATTRS_REMOVE:
+            try: dynamicProps.remove(att)
+            except: pass
+        dynamicProps.sort()
+
+        item_dict = {} 
+        for prop in dynamicProps:
+            item_dict.update({prop: val[prop]})
+
+        for i, (k,v) in enumerate(item_dict.items()):
+            newF, newVals = traverseDict( newF, newVals, nam+"_"+k, v)
     else: 
         var = getVariantFromValue(val)
-        if not var: var = None #LongLong #4 
-        else: 
-            newF.update({nam: var})
-            newVals.update({nam: val})  
-    #print(newF)
-    #print(newVals)
-    #print("traverse end")
+        if var is None: 
+            var = 'TEXT'
+            val = str(val)
+        #print(var)
+        newF.update({nam: var})
+        newVals.update({nam: val})  
+
     return newF, newVals
 
 def get_scale_factor(units: str) -> float:
@@ -123,46 +154,52 @@ def findTransformation(f_shape, geomType, layer_sr: arcpy.SpatialReference, proj
     #apply transformation if needed
     if layer_sr.name != projectCRS.name:
         tr0 = tr1 = tr2 = tr_custom = None
-        transformations = arcpy.ListTransformations(layer_sr, projectCRS)
-        customTransformName = "layer_sr.name"+"_To_"+ projectCRS.name
-        if len(transformations) == 0:
-            midSr = arcpy.SpatialReference("WGS 1984") # GCS_WGS_1984
-            try:
-                tr1 = arcpy.ListTransformations(layer_sr, midSr)[0]
-                tr2 = arcpy.ListTransformations(midSr, projectCRS)[0]
-            except: 
-                #customGeoTransfm = "GEOGTRAN[METHOD['Geocentric_Translation'],PARAMETER['X_Axis_Translation',''],PARAMETER['Y_Axis_Translation',''],PARAMETER['Z_Axis_Translation','']]"
-                #CreateCustomGeoTransformation(customTransformName, layer_sr, projectCRS)
-                tr_custom = customTransformName
-        else: 
-            #print("else")
-            # choose equation based instead of file-based/grid-based method, 
-            # to be consistent with QGIS: https://desktop.arcgis.com/en/arcmap/latest/map/projections/choosing-an-appropriate-transformation.htm
-            selecterTr = {}
-            for tr in transformations:
-                if "NTv2" not in tr and "NADCON" not in tr: 
-                    set1 = set( layer_sr.name.split("_") + projectCRS.name.split("_") )
-                    set2 = set( tr.split("_") )
-                    diff = len( set(set1).symmetric_difference(set2) )
-                    selecterTr.update({tr: diff})
-            selecterTr = dict(sorted(selecterTr.items(), key=lambda item: item[1]))
-            tr0 = list(selecterTr.keys())[0]
+        print(layer_sr)
+        try:
+            transformations = arcpy.ListTransformations(layer_sr, projectCRS)
+            customTransformName = "layer_sr.name"+"_To_"+ projectCRS.name
+            if len(transformations) == 0:
+                midSr = arcpy.SpatialReference("WGS 1984") # GCS_WGS_1984
+                try:
+                    tr1 = arcpy.ListTransformations(layer_sr, midSr)[0]
+                    tr2 = arcpy.ListTransformations(midSr, projectCRS)[0]
+                except: 
+                    #customGeoTransfm = "GEOGTRAN[METHOD['Geocentric_Translation'],PARAMETER['X_Axis_Translation',''],PARAMETER['Y_Axis_Translation',''],PARAMETER['Z_Axis_Translation','']]"
+                    #CreateCustomGeoTransformation(customTransformName, layer_sr, projectCRS)
+                    tr_custom = customTransformName
+            else: 
+                #print("else")
+                # choose equation based instead of file-based/grid-based method, 
+                # to be consistent with QGIS: https://desktop.arcgis.com/en/arcmap/latest/map/projections/choosing-an-appropriate-transformation.htm
+                selecterTr = {}
+                for tr in transformations:
+                    if "NTv2" not in tr and "NADCON" not in tr: 
+                        set1 = set( layer_sr.name.split("_") + projectCRS.name.split("_") )
+                        set2 = set( tr.split("_") )
+                        diff = len( set(set1).symmetric_difference(set2) )
+                        selecterTr.update({tr: diff})
+                selecterTr = dict(sorted(selecterTr.items(), key=lambda item: item[1]))
+                tr0 = list(selecterTr.keys())[0]
 
-        if geomType != "Point" and geomType != "Polyline" and geomType != "Polygon" and geomType != "Multipoint":
-            try: arcpy.AddWarning("Unsupported or invalid geometry in layer " + selectedLayer.name)
-            except: arcpy.AddWarning("Unsupported or invalid geometry")
+            if geomType != "Point" and geomType != "Polyline" and geomType != "Polygon" and geomType != "Multipoint":
+                try: arcpy.AddWarning("Unsupported or invalid geometry in layer " + selectedLayer.name)
+                except: arcpy.AddWarning("Unsupported or invalid geometry")
 
-        # reproject geometry using chosen transformstion(s)
-        if tr0 is not None:
-            ptgeo1 = f_shape.projectAs(projectCRS, tr0)
-            f_shape = ptgeo1
-        elif tr1 is not None and tr2 is not None:
-            ptgeo1 = f_shape.projectAs(midSr, tr1)
-            ptgeo2 = ptgeo1.projectAs(projectCRS, tr2)
-            f_shape = ptgeo2
-        else:
-            ptgeo1 = f_shape.projectAs(projectCRS)
-            f_shape = ptgeo1
+            # reproject geometry using chosen transformstion(s)
+            if tr0 is not None:
+                ptgeo1 = f_shape.projectAs(projectCRS, tr0)
+                f_shape = ptgeo1
+            elif tr1 is not None and tr2 is not None:
+                ptgeo1 = f_shape.projectAs(midSr, tr1)
+                ptgeo2 = ptgeo1.projectAs(projectCRS, tr2)
+                f_shape = ptgeo2
+            else:
+                ptgeo1 = f_shape.projectAs(projectCRS)
+                f_shape = ptgeo1
+        
+        except:
+            arcpy.AddWarning(f"Spatial Transformation not found for layer {selectedLayer.name}")
+            return None
 
     return f_shape    
 
@@ -252,12 +289,12 @@ def curvedFeatureClassToSegments(layer) -> str:
     print(newPath)
     return newPath
 
-def validate_path(path):
+def validate_path(path: str):
     # https://github.com/EsriOceans/btm/commit/a9c0529485c9b0baa78c1f094372c0f9d83c0aaf
     """If our path contains a DB name, make sure we have a valid DB name and not a standard file name."""
     dirname, file_name = os.path.split(path)
-    print(dirname)
-    print(file_name)
+    #print(dirname)
+    #print(file_name)
     file_base = os.path.splitext(file_name)[0]
     if dirname == '':
         # a relative path only, relying on the workspace

speckle_toolbox/esri/toolboxes/speckle/plugin_utils/install_message.py

@@ -1,13 +1,13 @@
 # -*- coding: utf-8 -*-
-
+r'''
 import arcpy
 
 class Toolbox(object):
     def __init__(self):
         """Define the toolbox (the name of the toolbox is the name of the
         .pyt file)."""
-        self.label = "Speckle Toolbox"
-        self.alias = "speckle_toolbox_"  
+        self.label = "Speckle something"
+        self.alias = "speckle_toolbox"  
         self.tools = [Speckle]    
 
 class Speckle(object):
@@ -45,4 +45,4 @@ class Speckle(object):
     def execute(self, parameters): 
         return
     
-    
+'''    

speckle_toolbox/esri/toolboxes/speckle/plugin_utils/testing_from_file.py

@@ -0,0 +1,73 @@
+#python speckle_toolbox\esri\toolboxes\speckle\plugin_utils\testing_from_file.py
+import arcpy
+from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
+
+import json
+import os
+
+from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
+from arcpy.management import (CreateFeatureclass, MakeFeatureLayer,
+                              AddFields, AlterField, DefineProjection )
+
+##################################################### get example layers from the project #######
+path = r'C:\Users\katri\Documents\ArcGIS\Projects\MyProject\MyProject.gdb'
+arcpy.env.workspace = path
+project = ArcGISProject(path.replace("gdb","aprx"))
+active_map = project.listMaps()[0] #.activeMap
+all_layers = []
+
+layerPolygon = None
+layerPolyline = None
+layerPoint = None
+layerMultiPoint = None
+#get layer of interest
+for layer in active_map.listLayers(): 
+   if layer.isFeatureLayer or layer.isRasterLayer: 
+        all_layers.append(layer)
+        data = arcpy.Describe(layer.dataSource)
+        if layer.isFeatureLayer:
+            geomType = data.shapeType
+            if geomType == "Polygon" and layerPolygon is None: layerPolygon = layer 
+            if geomType == "Polyline" and layerPolyline is None: layerPolyline = layer 
+            if geomType == "Point" and layerPoint is None: layerPoint = layer 
+            if geomType == "Multipoint" and layerMultiPoint is None: layerMultiPoint = layer 
+
+root_path = "\\".join(project.filePath.split("\\")[:-1])
+#path_style = root_path + '\\layer_speckle_symbology.lyrx'
+path_style2 = root_path + '\\layer_speckle_symbology2.lyrx'
+
+for layer in active_map.listLayers(): 
+    if layer.longName == layerPolygon.longName:
+        layerPolygon = layer
+        break
+
+print(layerPolygon.dataSource)
+r'''
+source = str(layerPolygon.dataSource).split('\\')
+
+layerPolygon = arcpy.ApplySymbologyFromLayer_management(
+                        in_layer= str(layerPolygon.dataSource), 
+                        in_symbology_layer=path_style2, 
+                        update_symbology='UPDATE')[0]
+'''
+#vl2 = MakeFeatureLayer(layerPolygon.dataSource, 'someName').getOutput(0)
+#active_map.addLayer(arcpy.mp.LayerFile(path_style2))
+
+################## reset symbology if needed:
+
+sym = layerPolygon.symbology
+print(sym.renderer.type)
+sym.updateRenderer('UniqueValueRenderer')
+print(sym.renderer.type)
+layerPolygon.symbology = sym
+print(sym.renderer.type)
+r'''
+sym = layerPolygon.symbology
+print(sym.renderer.type)
+sym.updateRenderer('UniqueValueRenderer')
+layerPolygon.symbology = sym
+print(sym.updateRenderer('UniqueValueRenderer'))
+print(layerPolygon.symbology.renderer.type)
+# SimpleRenderer, GraduatedColorsRenderer, GraduatedSymbolsRenderer, UnclassedColorsRenderer, UniqueValueRenderer 
+'''
+project.save() 

speckle_toolbox/esri/toolboxes/speckle/plugin_utils/unit_tests.py

@@ -0,0 +1,13 @@
+
+import arcpy
+from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
+
+import json
+import os
+
+from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
+from arcpy.management import (CreateFeatureclass, MakeFeatureLayer,
+                              AddFields, AlterField, DefineProjection )
+
+import unittest 
+

speckle_toolbox/esri/toolboxes/speckle/speckle_arcgis.py

@@ -1,3 +1,4 @@
+
 # -*- coding: utf-8 -*-
 
 from typing import Any, Callable, List, Optional, Tuple
@@ -9,15 +10,26 @@ import arcpy
 from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
 from arcpy import metadata as md
 
-from specklepy.api.models import Branch, Stream, Streams
-from speckle.converter.layers.Layer import Layer, RasterLayer
-
-from speckle.converter.layers._init_ import convertSelectedLayers, layerToNative, cadLayerToNative
+try:
+    from speckle.converter.layers.Layer import Layer, RasterLayer
+    from speckle.converter.layers._init_ import convertSelectedLayers, layerToNative, cadLayerToNative, bimLayerToNative
+    from speckle.ui.project_vars import toolboxInputsClass, speckleInputsClass
+    from speckle.converter.layers.emptyLayerTemplates import createGroupLayer
+    from speckle.converter.layers.Layer import VectorLayer
+except: 
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.Layer import Layer, RasterLayer
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers._init_ import convertSelectedLayers, layerToNative, cadLayerToNative, bimLayerToNative
+    from speckle_toolbox.esri.toolboxes.speckle.ui.project_vars import toolboxInputsClass, speckleInputsClass
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.emptyLayerTemplates import createGroupLayer
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.Layer import VectorLayer
+    
 from arcgis.features import FeatureLayer
 import os
 import os.path
+import sys 
 
 import specklepy
+from specklepy.api.models import Branch, Stream, Streams
 from specklepy.transports.server.server import ServerTransport
 from specklepy.api.credentials import get_local_accounts
 from specklepy.api.client import SpeckleClient
@@ -32,10 +44,6 @@ from specklepy.api.wrapper import StreamWrapper
 from specklepy.objects import Base
 from specklepy.logging import metrics
 
-from speckle.ui.project_vars import toolboxInputsClass, speckleInputsClass
-from speckle.converter.layers.emptyLayerTemplates import createGroupLayer
-from speckle.converter.layers.Layer import VectorLayer
-
 #'''
 
 def traverseObject(
@@ -77,13 +85,18 @@ class Toolbox:
         self.alias = "speckle_toolbox_"  
         # List of tool classes associated with this toolbox
         self.tools = [Speckle]  
-        metrics.set_host_app("ArcGIS")  
 
+        try: 
+            version = arcpy.GetInstallInfo()['Version']
+            python_version = f"python {'.'.join(map(str, sys.version_info[:2]))}"
+            metrics.set_host_app("ArcGIS", ', '.join([f"ArcGIS {version}", python_version])) 
+        except: 
+            metrics.set_host_app("ArcGIS")
         # https://pro.arcgis.com/en/pro-app/2.8/arcpy/mapping/alphabeticallistofclasses.htm#except: print("something happened")
 
 class Speckle:
-    def __init__(self):
-        #print("________________reset_______________")   
+    def __init__(self):  
+        print("__________________INIT SPECKLE TOOL_________")
         self.label       = "Speckle"
         self.description = "Allows you to send and receive your layers " + \
                            "to/from other software using Speckle server." 
@@ -91,39 +104,32 @@ class Speckle:
         self.toRefresh = False
         self.speckleInputs = None
         self.toolboxInputs = None
-        #print("ping_Speckle1")
-        
-        #print(speckleInputsClass.instances)
+
         total = len(speckleInputsClass.instances)
-        #print(total)
+        print(total)
         for i in range(total):
-            #print(i)
-            #print(speckleInputsClass.instances[total-i-1])
             if speckleInputsClass.instances[total-i-1] is not None: 
                 try: 
-                    #print(speckleInputsClass.instances[total-i-1].streams_default)
-                    y = speckleInputsClass.instances[total-i-1].streams_default # in case not initialized properly 
+                    y = speckleInputsClass.instances[total-i-1].streams_default 
+                    #if not isinstance(speckleInputsClass.instances[total-i-1].streams_default, SpeckleException): # also will throw exception in case not initialized properly 
                     self.speckleInputs = speckleInputsClass.instances[total-i-1] # take latest (first in reverted list)
-                    #print("FOUND INSTANCE")
                     break
                 except: pass
-        #print(self.speckleInputs)
-        if self.speckleInputs is None: self.speckleInputs = speckleInputsClass()
+        if self.speckleInputs is None or isinstance(self.speckleInputs.streams_default, SpeckleException): self.speckleInputs = speckleInputsClass()
+        #print(self.speckleInputs.streams_default)
+        print(len(speckleInputsClass.instances))
 
-
-        #print(toolboxInputsClass.instances)
-        #print("TOTAL = ...................") 
         total = len(toolboxInputsClass.instances)
-        #print(total)
+
         for i in range(total):
             if toolboxInputsClass.instances[total-i-1] is not None: 
                 self.toolboxInputs = toolboxInputsClass.instances[total-i-1] # take latest (first in reverted list)
-                #print("FOUND INSTANCE")
                 break
-        #print(self.toolboxInputs)
         if self.toolboxInputs is None: self.toolboxInputs = toolboxInputsClass()
-        
-        #print("ping_Speckle2")
+
+        #print(self.speckleInputs.accounts)
+        if len(self.speckleInputs.accounts) == 0:
+            arcpy.AddError("Speckle accounts not found")
 
     def getParameterInfo(self):
         #data types: https://pro.arcgis.com/en/pro-app/2.8/arcpy/geoprocessing_and_python/defining-parameter-data-types-in-a-python-toolbox.htm
@@ -138,19 +144,24 @@ class Speckle:
             name="streamsDefalut",
             datatype="GPString",
             parameterType="Optional",
-            #category="Sending data",
             direction="Input",
             category=cat1
             )
         streamsDefalut.filter.type = 'ValueList'
-        streamsDefalut.filter.list = [ (st.name + " - " + st.id) for st in self.speckleInputs.streams_default ]
-
+        if isinstance(self.speckleInputs.streams_default, SpeckleException):
+            arcpy.AddError("Speckle account not accessible")
+            streamsDefalut.filter.list = []
+        elif self.speckleInputs.streams_default is not None:
+            streamsDefalut.filter.list = [ (str(st.name) + " - " + str(st.id)) for st in self.speckleInputs.streams_default ]
+        else:
+            streamsDefalut.filter.list = []
+            arcpy.AddError("Error connecting to default Speckle account")
+        
         addDefStreams = arcpy.Parameter(
             displayName="Add",
             name="addDefStreams",
             datatype="GPBoolean",
             parameterType="Optional",
-            #category="Sending data",
             direction="Input",
             category=cat1
             )
@@ -217,7 +228,6 @@ class Speckle:
             parameterType="Required",
             direction="Input",
             multiValue=False,
-            #category=cat2
             )
         savedStreams.filter.list = [f"Stream not accessible - {stream[0].stream_id}" if stream[1] is None or isinstance(stream[1], SpeckleException) else f"{stream[1].name} - {stream[1].id}" for i,stream in enumerate(self.speckleInputs.saved_streams)] 
 
@@ -235,9 +245,7 @@ class Speckle:
             name="branch",
             datatype="GPString",
             parameterType="Required",
-            #category="Sending data",
             direction="Input",
-            #category=cat2
             )
         branch.value = ""
         branch.filter.type = 'ValueList'
@@ -247,21 +255,18 @@ class Speckle:
             name="commit",
             datatype="GPString",
             parameterType="Optional",
-            #category="Sending data",
             direction="Input",
-            #category=cat2
             )
         commit.value = ""
         commit.filter.type = 'ValueList'
 
         msg = arcpy.Parameter(
             displayName="Message",
-            name="message",
+            name="msg",
             datatype="GPString",
             parameterType="Optional",
             direction="Input",
             multiValue=False,
-            #category=cat2
             )
         msg.value = ""
 
@@ -272,7 +277,6 @@ class Speckle:
             parameterType="Optional",
             direction="Input",
             multiValue=True,
-            #category=cat2
             )
         selectedLayers.filter.list = [str(i) + "-" + l.longName for i,l in enumerate(self.speckleInputs.all_layers)] #"Polyline"
 
@@ -282,13 +286,10 @@ class Speckle:
             name="action",
             datatype="GPString",
             parameterType="Required",
-            #category="Sending data",
             direction="Input",
             multiValue=False,
-            #category=cat2
             )
         action.value = "Send" 
-        #action.filter.type = 'ValueList'
         action.filter.list = ["Send", "Receive"]  
 
         refresh = arcpy.Parameter(
@@ -297,8 +298,7 @@ class Speckle:
             datatype="GPBoolean",
             parameterType="Optional",
             direction="Input"
-            )
-        #refresh.filter.type = "ValueList"   
+            ) 
         refresh.value = False 
 
         parameters = [streamsDefalut, addDefStreams, streamUrl, addUrlStreams, lat, lon, setLatLon, savedStreams, removeStream, branch, commit, selectedLayers, msg, action, refresh]
@@ -316,11 +316,11 @@ class Speckle:
             if par.name == "addDefStreams" and par.altered and par.value == True: 
                 for p in parameters:
                     if p.name == "streamsDefalut" and p.valueAsText is not None:
+
                         # add value from streamsDefault to saved streams
                         selected_stream_name = p.valueAsText[:]
-                        #print(selected_stream_name)
+
                         for stream in self.speckleInputs.streams_default:
-                            #print(stream)
                             if stream.name == selected_stream_name.split(" - ")[0]:
                                 print("_____Add from list___")
                                 wr = StreamWrapper(f"{self.speckleInputs.account.serverInfo.url}/streams/{stream.id}?u={self.speckleInputs.account.userInfo.id}")
@@ -331,7 +331,7 @@ class Speckle:
                                         saved_streams = self.speckleInputs.getProjectStreams()
                                         self.speckleInputs.saved_streams = saved_streams
                                         p_saved.filter.list = [f"Stream not accessible - {stream[0].stream_id}" if stream[1] is None or isinstance(stream[1], SpeckleException) else f"{stream[1].name} - {stream[1].id}" for i,stream in enumerate(saved_streams)] 
-                                        p_saved.value = p_saved.filter.list[0]
+                                        if len(p_saved.filter.list)>0: print(p_saved.filter.list); p_saved.value = p_saved.filter.list[0]
                                 break
                         p.value = None
                 par.value = False
@@ -346,8 +346,9 @@ class Speckle:
                             steamId = query
                             try: steamId = query.split("/streams/")[1].split("/")[0] 
                             except: pass
-                            # quesry stream, add to saved
-                            stream = self.speckleInputs.speckle_client.stream.get(steamId)
+
+                            # query stream, add to saved
+                            stream = self.speckleInputs.speckle_client.stream.get(id = steamId, branch_limit = 100, commit_limit = 100)
                             if isinstance(stream, Stream): 
                                 print("_____Add by URL___")
                                 wr = StreamWrapper(f"{self.speckleInputs.account.serverInfo.url}/streams/{stream.id}?u={self.speckleInputs.account.userInfo.id}")
@@ -358,7 +359,7 @@ class Speckle:
                                         saved_streams = self.speckleInputs.getProjectStreams()
                                         self.speckleInputs.saved_streams = saved_streams
                                         p_saved.filter.list = [f"Stream not accessible - {st[0].stream_id}" if st[1] is None or isinstance(st[1], SpeckleException) else f"{st[1].name} - {st[1].id}" for i,st in enumerate(saved_streams)] 
-                                        p_saved.value = p_saved.filter.list[0]
+                                        if len(p_saved.filter.list)>0: print(p_saved.filter.list); p_saved.value = p_saved.filter.list[0]
                             else: pass
 
                         p.value = None
@@ -370,11 +371,9 @@ class Speckle:
                 for p in parameters:
                     if p.name == "savedStreams" and p.valueAsText is not None:
 
-                         # get value from savedStreams 
+                        # get value from savedStreams 
                         selected_stream_name = p.valueAsText[:]
-                        #print(selected_stream_name)
                         for streamTup in self.speckleInputs.saved_streams:
-                            #print(stream)
                             stream = streamTup[1]
                             if stream.name == selected_stream_name.split(" - ")[0]:
                                 print("_____Remove stream___")
@@ -397,12 +396,13 @@ class Speckle:
                 for p in parameters:
                     if p.name == "lat" and p.valueAsText is not None:
                         # add value from the UI to saved lat
-                        lat = p.valueAsText[:].replace(",","").replace(" ","").replace(";","").replace("-","").replace("_","")
+                        lat = p.valueAsText[:].replace(",","").replace(" ","").replace(";","").replace("_","")
                         try: lat = float(lat)
                         except: lat = 0; p.value = "0.0"
+
                     if p.name == "lon" and p.valueAsText is not None:
                         # add value from the UI to saved lat
-                        lon = p.valueAsText[:].replace(",","").replace(" ","").replace(";","").replace("-","").replace("_","")
+                        lon = p.valueAsText[:].replace(",","").replace(" ","").replace(";","").replace("_","")
                         try: lon = float(lon)
                         except: lon = 0; p.value = "0.0"
                     coords = [lat, lon]
@@ -414,12 +414,13 @@ class Speckle:
             if par.name == "savedStreams" and par.altered:
                 # Search for the stream by name
                 if par.value is not None and "Stream not accessible" not in par.valueAsText[:]:
-                    #print("SAVED STREAMS - selection")
                     selected_stream_name = par.valueAsText[:]
                     self.toolboxInputs.active_stream = None
+                    self.toolboxInputs.active_stream_wrapper = None
                     for st in self.speckleInputs.saved_streams:
                         if st[1].name == selected_stream_name.split(" - ")[0]: 
                             self.toolboxInputs.active_stream = st[1]
+                            self.toolboxInputs.active_stream_wrapper = st[0]
                             break
 
                     # edit branches: globals and UI 
@@ -448,7 +449,6 @@ class Speckle:
                                 p.value = None
                                 self.toolboxInputs.active_commit = None 
                 else: par.value = None
-                #print(self.toolboxInputs.action)
 
             if par.name == "branch" and par.altered: # branches
                 if par.value is not None:
@@ -487,22 +487,14 @@ class Speckle:
                 if par.value is not None:
                     self.toolboxInputs.selected_layers = par.values
 
-                    #print("selected layers changed")
-                    #print(self.toolboxInputs.action)
-                    #print(self.toolboxInputs.selected_layers)
-
             if par.name == "msg" and par.altered and par.valueAsText is not None:
                 self.toolboxInputs.messageSpeckle = par.valueAsText
+                print(self.toolboxInputs.messageSpeckle)
 
             if par.name == "action" and par.altered:
-                #print("action changed")
-                #print(par.valueAsText)
                 if par.valueAsText == "Send": self.toolboxInputs.action = 1
                 else: self.toolboxInputs.action = 0
 
-                #print(self.toolboxInputs.action)
-                #print(self.toolboxInputs.selected_layers)
-
             if par.name == "refresh" and par.altered: # refresh btn
                 if par.value == True: 
                     self.refresh(parameters) 
@@ -511,9 +503,6 @@ class Speckle:
                 self.toRefresh = False
         
         print("____________________________parameters___________________________")
-        #[print(str(x.name) + " - " + str(x.valueAsText)) for x in parameters]
-        #[x.clearMessage() for x in parameters] # https://pro.arcgis.com/en/pro-app/latest/arcpy/geoprocessing_and_python/programming-a-toolvalidator-class.htm
-        #[print(x.valueAsText) for x in parameters]
         return 
     
     def refresh(self, parameters: List[Any]): 
@@ -525,8 +514,8 @@ class Speckle:
             if par.name == "streamUrl": par.value = None
             if par.name == "streamsDefalut": par.value = None
             if par.name == "savedStreams": par.value = None
-            if par.name == "branch": par.value = ""
-            if par.name == "commit": par.value = None
+            if par.name == "branch": par.value = ""; par.filter.list = []
+            if par.name == "commit": par.value = None; par.filter.list = []
             if par.name == "selectedLayers": par.value = None
             if par.name == "msg": par.value = ""
             if par.name == "action": par.value = "Send"
@@ -534,11 +523,14 @@ class Speckle:
             
             if par.name == "lat": par.value = str(self.toolboxInputs.get_survey_point()[0])
             if par.name == "lon": par.value = str(self.toolboxInputs.get_survey_point()[1])
-            if par.name == "streamsDefalut": par.filter.list = [ (st.name + " - " + st.id) for st in self.speckleInputs.streams_default ]
+            if par.name == "streamsDefalut": 
+                if isinstance(self.speckleInputs.streams_default, SpeckleException):
+                    arcpy.AddError("Speckle account not accessible")
+                    par.filter.list = []
+                else:
+                    par.filter.list = [ (st.name + " - " + st.id) for st in self.speckleInputs.streams_default ]
             if par.name == "savedStreams": 
-                #print("par.name")
                 saved_streams = self.speckleInputs.getProjectStreams()
-                #print(saved_streams)
                 par.filter.list = [f"Stream not accessible - {stream[0].stream_id}" if stream[1] is None or isinstance(stream[1], SpeckleException) else f"{stream[1].name} - {stream[1].id}" for i,stream in enumerate(saved_streams)] 
             if par.name == "selectedLayers": par.filter.list = [str(i) + "-" + l.longName for i,l in enumerate(self.speckleInputs.all_layers)]
         
@@ -575,14 +567,13 @@ class Speckle:
 
         print("______________SEND_______________")
 
-        #if self.validateStreamBranch(parameters) == False: return
-
         if len(self.toolboxInputs.selected_layers) == 0: 
             arcpy.AddError("No layers selected for sending")
             return
 
         streamId = self.toolboxInputs.active_stream.id #stream_id
-        client = self.speckleInputs.speckle_client # ?
+        client = self.toolboxInputs.active_stream_wrapper.get_client()
+        #client = self.speckleInputs.speckle_client # ?
         
         # Get the stream wrapper
         #streamWrapper = StreamWrapper(None)
@@ -601,6 +592,9 @@ class Speckle:
         base_obj = Base(units = "m")
         base_obj.layers = convertSelectedLayers(self.speckleInputs.all_layers, self.toolboxInputs.selected_layers, self.speckleInputs.project)
         
+        if len(base_obj.layers) == 0: 
+            arcpy.AddMessage("No data sent to stream " + streamId)
+            return 
         try:
             # this serialises the block and sends it to the transport
             objId = operations.send(base=base_obj, transports=[transport])
@@ -613,7 +607,9 @@ class Speckle:
             
 
         message = self.toolboxInputs.messageSpeckle
+        print(message)
         if message is None or ( isinstance(message, str) and len(message) == 0):  message = "Sent from ArcGIS"
+        print(message)
         try:
             # you can now create a commit on your stream with this object
             client.commit.create(
@@ -635,7 +631,8 @@ class Speckle:
 
         try: 
             streamId = self.toolboxInputs.active_stream.id #stream_id
-            client = self.speckleInputs.speckle_client # 
+            client = self.toolboxInputs.active_stream_wrapper.get_client()
+            #client = self.speckleInputs.speckle_client # 
         except SpeckleWarning as warning: 
             arcpy.AddWarning(str(warning.args[0]))
 
@@ -667,11 +664,13 @@ class Speckle:
         except: 
             arcpy.AddError("Make sure your account has access to the chosen stream")
             return
-
         try:
             #print(commit)
             objId = commit.referencedObject
             commitDetailed = client.commit.get(streamId, commit.id)
+            if isinstance(commitDetailed, GraphQLException): 
+                arcpy.AddError("Access error")
+                return
             app = commitDetailed.sourceApplication
             if objId is None:
                 return
@@ -681,14 +680,16 @@ class Speckle:
                 if self.speckleInputs.project.activeMap.spatialReference.type == "Geographic" or self.speckleInputs.project.activeMap.spatialReference is None: #TODO test with invalid CRS
                     arcpy.AddMessage("It is advisable to set the project Spatial reference to Projected type before receiving CAD geometry (e.g. EPSG:32631), or create a custom one from geographic coordinates")
                     print("It is advisable to set the project Spatial reference to Projected type before receiving CAD geometry (e.g. EPSG:32631), or create a custom one from geographic coordinates")
-            print(f"Succesfully received {objId}")
+            print(f"Successfully received {objId}")
 
             # Clear 'latest' group
             streamBranch = streamId + "_" + self.toolboxInputs.active_branch.name + "_" + str(commit.id)
+            streamBranch = streamBranch.replace("[","_").replace("]","_").replace(" ","_").replace("-","_").replace("(","_").replace(")","_").replace(":","_").replace("\\","_").replace("/","_").replace("\"","_").replace("&","_").replace("@","_").replace("$","_").replace("%","_").replace("^","_")
+    
             newGroupName = f'{streamBranch}'
             
             groupExists = 0
-            #print(newGroupName)
+            print(newGroupName)
             for l in self.speckleInputs.project.activeMap.listLayers(): 
                 #print(l.longName)
                 if l.longName.startswith(newGroupName + "\\"):
@@ -697,18 +698,29 @@ class Speckle:
                     groupExists+=1
                 elif l.longName == newGroupName: 
                     groupExists+=1
+            print(newGroupName)
             if groupExists == 0:
                 # create empty group layer file 
                 path = self.speckleInputs.project.filePath.replace("aprx","gdb") #"\\".join(self.toolboxInputs.project.filePath.split("\\")[:-1]) + "\\speckle_layers\\"
-                #print(path)
-                f = open(path + "\\" + newGroupName + ".lyrx", "w")
-                content = createGroupLayer().replace("TestGroupLayer", newGroupName)
-                f.write(content)
-                f.close()
-                smth = arcpy.mp.LayerFile(path + "\\" + newGroupName + ".lyrx")
-                #print(smth)
-                layerGroup = self.speckleInputs.project.activeMap.addLayer(smth)[0]
+                print(path)
+                try:
+                    f = open(path + "\\" + newGroupName + ".lyrx", "w")
+                    content = createGroupLayer().replace("TestGroupLayer", newGroupName)
+                    f.write(content)
+                    f.close()
+                    newGroupLayer = arcpy.mp.LayerFile(path + "\\" + newGroupName + ".lyrx")
+                    layerGroup = self.speckleInputs.project.activeMap.addLayer(newGroupLayer)[0]
+                except: # for 3.0.0
+                    if self.speckleInputs.active_map is not None:
+                        layerGroup = self.speckleInputs.active_map.createGroupLayer(newGroupName)
+                    else:
+                        arcpy.AddWarning("The map didn't fully load, try refreshing the plugin.")
+                        return
+
+                print(layerGroup)
+                print("layer added")
                 layerGroup.name = newGroupName
+                print(newGroupName)
 
             if app == "QGIS" or app == "ArcGIS": check: Callable[[Base], bool] = lambda base: isinstance(base, Layer) or isinstance(base, VectorLayer) or isinstance(base, RasterLayer)
             else: check: Callable[[Base], bool] = lambda base: isinstance(base, Base)
@@ -732,6 +744,8 @@ class Speckle:
                     except: pass
 
             def loopVal(value: Any, name: str): # "name" is the parent object/property/layer name
+                if name.endswith('/displayValue'): return 
+                
                 if isinstance(value, Base): 
                     try: # dont go through parts of Speckle Geometry object
                         print("objects to loop through: " + value.speckle_type)
@@ -742,18 +756,29 @@ class Speckle:
                 if isinstance(value, List):
                     for item in value:
                         loopVal(item, name)
-                        #print(item)
+                        
+                        print(item)
+                        pt = None
                         if item.speckle_type and item.speckle_type.startswith("Objects.Geometry."): 
+
                             pt, pl = cadLayerToNative(value, name, streamBranch, self.speckleInputs.project)
                             if pt is not None: print("Layer group created: " + pt.name())
                             if pl is not None: print("Layer group created: " + pl.name())
                             break
+                        
+                        if item.speckle_type and (item.speckle_type.startswith("Objects.BuiltElements.") or item.speckle_type.startswith("Objects.Structural.Geometry")): #  and "Revit" in item.speckle_type
+                            print("__receiving structures__")
+                            msh_bool = bimLayerToNative(value, name, streamBranch, self.speckleInputs.project)
+                            #if msh is not None: print("Layer group created: " + msh.name())
+                            break
                 
             traverseObject(commitObj, callback, check)
       
-        except SpeckleException as e:
-            print("Receive failed")
+        except (SpeckleException, GraphQLException) as e:
+            print("Receive failed: " + str(e))
+            arcpy.AddError("Receive failed: " + str(e))
             return
+        
         print("received")
         #self.updateParameters(parameters, True)
         #self.refresh(parameters)

speckle_toolbox/esri/toolboxes/speckle/ui/project_vars.py

@@ -5,13 +5,12 @@ from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
 from specklepy.api.models import Branch, Stream, Streams
 import os.path
 
-from specklepy.api.credentials import get_local_accounts
+from specklepy.api.credentials import Account, get_local_accounts
 from specklepy.api.client import SpeckleClient
 from specklepy.logging.exceptions import (
     GraphQLException,
     SpeckleException,
 )
-#from specklepy.api.credentials import StreamWrapper
 from specklepy.api.wrapper import StreamWrapper
 from osgeo import osr
 
@@ -19,20 +18,26 @@ class speckleInputsClass:
     #def __init__(self):
     print("CREATING speckle inputs first time________")
     instances = []
-    accounts = get_local_accounts()
+    accounts: List[Account] = get_local_accounts()
     account = None
-    streams_default: None or Streams = None 
+    streams_default: Optional[List[Stream]] = None
 
     project = None
     active_map = None
-    saved_streams: List[None or Tuple[StreamWrapper, Stream]] = []
+    saved_streams: List[Optional[Tuple[StreamWrapper, Stream]]] = []
     stream_file_path: str = ""
     all_layers: List[arcLayer] = []
+    clients: List[SpeckleClient] = []
 
     for acc in accounts:
-        if acc.isDefault:
-            account = acc
-            break
+        if acc.isDefault: account = acc
+        new_client = SpeckleClient(
+            acc.serverInfo.url,
+            acc.serverInfo.url.startswith("https")
+        )
+        new_client.authenticate_with_token(token=acc.token)
+        clients.append(new_client)
+
     speckle_client = None
     if account:
         speckle_client = SpeckleClient(
@@ -44,15 +49,18 @@ class speckleInputsClass:
 
     def __init__(self) -> None:
         print("___start speckle inputs________")
+        self.all_layers = []
         try:
             aprx = ArcGISProject('CURRENT')
             self.project = aprx
+            # following will fail if no project found 
             self.active_map = aprx.activeMap
             
             if self.active_map is not None and isinstance(self.active_map, Map): # if project loaded
                 for layer in self.active_map.listLayers(): 
-                    #print(layer)
-                    if layer.isFeatureLayer or layer.isRasterLayer: self.all_layers.append(layer) #type: 'arcpy._mp.Layer'
+                    try: geomType = arcpy.Describe(layer.dataSource).shapeType.lower()
+                    except: geomType = '' #print(arcpy.Describe(layer.dataSource)) #and arcpy.Describe(layer.dataSource).shapeType.lower() != "multipatch")
+                    if (layer.isFeatureLayer and geomType != "multipatch") or layer.isRasterLayer: self.all_layers.append(layer) #type: 'arcpy._mp.Layer'
             self.stream_file_path: str = aprx.filePath.replace("aprx","gdb") + "\\speckle_streams.txt"
 
             if os.path.exists(self.stream_file_path): 
@@ -63,7 +71,9 @@ class speckleInputsClass:
                     f.close()
                 except: pass
                 
-            else: 
+            elif len(self.stream_file_path) >10: 
+                f = open(self.stream_file_path, "x")
+                f.close()
                 f = open(self.stream_file_path, "w")
                 content = ""
                 f.write(content)
@@ -89,6 +99,7 @@ class speckleInputsClass:
                 streamExists = 0
                 index = 0
                 try:
+                    #print(url)
                     sw = StreamWrapper(url)
                     stream = self.tryGetStream(sw)
 
@@ -101,33 +112,33 @@ class speckleInputsClass:
                     streamsTuples.insert(0,(sw, stream))
 
                 except SpeckleException as e:
-                    arcpy.AddMessage(str(e.args[0]))
+                    arcpy.AddMessage(str(e.args))
             return streamsTuples
         else: return []
 
     def tryGetStream (self,sw: StreamWrapper) -> Stream:
-        #print("Try get streams")
-
-        steamId = sw.stream_id
-        try: steamId = sw.stream_id.split("/streams/")[1].split("/")[0] 
-        except: pass
-
-        client = sw.get_client()
-        stream = client.stream.get(steamId)
-        if isinstance(stream, GraphQLException):
-            raise SpeckleException(stream.errors[0]['message'])
-        return stream
+        if isinstance(sw, StreamWrapper):
+            steamId = sw.stream_id
+            try: steamId = sw.stream_id.split("/streams/")[1].split("/")[0] 
+            except: pass
 
+            client = sw.get_client()
+            stream = client.stream.get(id = steamId, branch_limit = 100, commit_limit = 100)
+            if isinstance(stream, GraphQLException):
+                raise SpeckleException(stream.errors[0]['message'])
+            return stream
+        else: 
+            raise SpeckleException('Invalid StreamWrapper provided')
 
 class toolboxInputsClass:
-    #def __init__(self):
+
     print("CREATING UI inputs first time________")
-    # self.instances.append(self)
     instances = []
     lat: float = 0.0
     lon: float = 0.0
-    active_stream: None or Stream = None
-    active_branch: None or Branch = None
+    active_stream: Optional[Stream] = None
+    active_stream_wrapper: Optional[StreamWrapper] = None
+    active_branch: Optional[Branch] = None
     active_commit = None
     selected_layers: List[Any] = []
     messageSpeckle: str = ""
@@ -153,15 +164,14 @@ class toolboxInputsClass:
         try:
             aprx = ArcGISProject('CURRENT')
             self.project = aprx
-        except: self.project = None; print("Project not found")
+        except: self.project = None; print("Project not found"); arcpy.AddWarning("Project not found")
         self.instances.append(self)
 
     def setProjectStreams(self, wr: StreamWrapper, add = True): 
         # ERROR 032659 Error queueing metrics request: 
-        # Cannot parse  into a stream wrapper class - invalid URL provided.
-        print("SET proj streamz")
+        print("SET proj streams")
 
-        if os.path.exists(self.stream_file_path): 
+        if os.path.exists(self.stream_file_path) and ".gdb\\speckle_streams.txt" in self.stream_file_path: 
 
             new_content = ""
 
@@ -180,7 +190,9 @@ class toolboxInputsClass:
 
             f.write(new_content)
             f.close()
-        else: 
+        elif ".gdb\\speckle_streams.txt" in self.stream_file_path: 
+            f = open(self.stream_file_path, "x")
+            f.close()
             f = open(self.stream_file_path, "w")
             f.write(str(wr.stream_url) + ",")
             f.close()
@@ -190,63 +202,75 @@ class toolboxInputsClass:
         print("get survey point")
         x = y = 0
         if not content: 
-            content = self.stream_file_path
-            try: 
-                f = open(self.stream_file_path, "r")
-                content = f.read()
-                f.close()
-            except: pass
+            content = None 
+            if os.path.exists(self.stream_file_path) and ".gdb\\speckle_streams.txt" in self.stream_file_path: 
+                try: 
+                    f = open(self.stream_file_path, "r")
+                    content = f.read()
+                    f.close()
+                except: pass
         if content:
-            temp = []
             for i, coords in enumerate(content.split(",")):
                 if "speckle_sr_origin_" in coords: 
                     try:
                         x, y = [float(c) for c in coords.replace("speckle_sr_origin_","").split(";")]
                     except: pass
-        return (x, y)
+        return (x, y) 
 
     def set_survey_point(self, coords: List[float]):
         # from widget (2 strings) to local vars + update SR of the map
         print("SET survey point")
 
-        pt = "speckle_sr_origin_" + str(coords[0]) + ";" + str(coords[1]) 
-        if os.path.exists(self.stream_file_path): 
+        if len(coords) == 2: 
+            pt = "speckle_sr_origin_" + str(coords[0]) + ";" + str(coords[1]) 
+            if os.path.exists(self.stream_file_path) and ".gdb\\speckle_streams.txt" in self.stream_file_path: 
 
-            new_content = ""
-            f = open(self.stream_file_path, "r")
-            existing_content = f.read()
-            f.close()
+                new_content = ""
+                f = open(self.stream_file_path, "r")
+                existing_content = f.read()
+                f.close()
 
-            f = open(self.stream_file_path, "w")
-            if pt in existing_content: 
-                new_content = existing_content.replace( pt , "")
-            else: 
-                new_content = existing_content 
+                f = open(self.stream_file_path, "w")
+                if pt in existing_content: 
+                    new_content = existing_content.replace( pt , "")
+                else: 
+                    new_content = existing_content 
+                
+                new_content += pt + "," # add point
+                f.write(new_content)
+                f.close()
+            elif ".gdb\\speckle_streams.txt" in self.stream_file_path: 
+                f = open(self.stream_file_path, "x")
+                f.close()
+                f = open(self.stream_file_path, "w")
+                f.write(pt + ",")
+                f.close()
             
-            new_content += pt + "," # add point
-            f.write(new_content)
-            f.close()
-        else: 
-            f = open(self.stream_file_path, "w")
-            f.write(pt + ",")
-            f.close()
+            # save to project; crearte SR
+            self.lat, self.lon = coords[0], coords[1]
+            newCrsString = "+proj=tmerc +ellps=WGS84 +datum=WGS84 +units=m +no_defs +lon_0=" + str(self.lon) + " lat_0=" + str(self.lat) + " +x_0=0 +y_0=0 +k_0=1"
+            newCrs = osr.SpatialReference()
+            newCrs.ImportFromProj4(newCrsString)
+            newCrs.MorphToESRI() # converts the WKT to an ESRI-compatible format
+            
+
+            validate = True if len(newCrs.ExportToWkt())>10 else False
+
+            if validate: 
+                newProjSR = arcpy.SpatialReference()
+                newProjSR.loadFromString(newCrs.ExportToWkt())
+
+                #source = osr.SpatialReference() 
+                #source.ImportFromWkt(self.project.activeMap.spatialReference.exportToString())
+                #transform = osr.CoordinateTransformation(source, newCrs)
+
+                self.project.activeMap.spatialReference =  newProjSR
+                arcpy.AddMessage("Custom project CRS successfully applied")
+            else:
+                arcpy.AddWarning("Custom CRS could not be created")
         
-        # save to project; crearte SR
-        self.lat, self.lon = coords[0], coords[1]
-        newCrsString = "+proj=tmerc +ellps=WGS84 +datum=WGS84 +units=m +no_defs +lon_0=" + str(self.lon) + " lat_0=" + str(self.lat) + " +x_0=0 +y_0=0 +k_0=1"
-        newCrs = osr.SpatialReference()
-        newCrs.ImportFromProj4(newCrsString)
-        print(newCrs.ExportToWkt())
-
-        validate = True if len(newCrs.ExportToWkt())>10 else False
-
-        if validate: 
-            newProjSR = arcpy.SpatialReference()
-            newProjSR.loadFromString(newCrs.ExportToWkt())
-            self.project.activeMap.spatialReference =  newProjSR
-            arcpy.AddWarning("Custom project CRS successfully applied")
         else:
-            arcpy.AddWarning("Custom CRS could not be created")
+            arcpy.AddWarning("Custom CRS could not be created: not enough coordinates provided")
 
         return True
 

tests/tests_unittest.py

@@ -0,0 +1,104 @@
+from speckle_toolbox.esri.toolboxes.speckle.speckle_arcgis import Toolbox, Speckle   # The code to test
+from speckle_toolbox.esri.toolboxes.speckle.ui.project_vars import speckleInputsClass, toolboxInputsClass
+
+import arcpy 
+import os
+
+from specklepy.api.wrapper import StreamWrapper
+from specklepy.api.models import Branch, Stream, Streams
+from specklepy.logging.exceptions import GraphQLException, SpeckleException
+from specklepy.api.credentials import Account
+
+import unittest   # The test framework
+# remove SetUp
+# add different scenqrios for Streqm Wrapper including wrng ones
+# tree of all options for input or class outcome
+# use dict for types chech
+# issue with untestable class init 
+# "mocking objects" for tests or "faking"
+# get functions ut of INIT 
+
+class Test_InitializingClasses(unittest.TestCase):
+    def setUp(self) -> None:
+        self.toolbox_input = toolboxInputsClass()
+        self.speckle_input = speckleInputsClass()
+        self.toolbox = Toolbox()
+        self.speckleTool = Speckle()
+        self.test_stream = "https://speckle.xyz/streams////17b0b76d13"
+    
+    def text_all_toolbox(self):
+        self.assertTrue(isinstance(self.toolbox.tools[0], Speckle))
+
+    def test_toolbox_inputs(self):
+        self.assertEqual(self.toolbox_input.lat, 0)
+        self.assertEqual(self.toolbox_input.lon, 0)
+        self.assertIsNone(self.toolbox_input.active_stream)
+        self.assertIsNone(self.toolbox_input.active_branch)
+        self.assertIsNone(self.toolbox_input.active_commit)
+        self.assertEqual(len(self.toolbox_input.selected_layers), 0)
+        self.assertEqual(self.toolbox_input.messageSpeckle, "")
+        self.assertEqual(self.toolbox_input.action, 1)
+        self.assertIsNone(self.toolbox_input.project)
+        self.assertEqual(self.toolbox_input.stream_file_path, "")
+
+    def test_something(self):
+        # Arrange
+        toolbox_input: toolboxInputsClass = toolboxInputsClass()
+        
+        # Act
+        toolbox_input.setProjectStreams(StreamWrapper(self.test_stream))
+
+        # Assert
+        os.path.exists(self.toolbox_input.stream_file_path)
+
+
+    def test_toolbox_inputs_functions(self):
+        self.toolbox_input.setProjectStreams(StreamWrapper(self.test_stream))
+        if os.path.exists(self.toolbox_input.stream_file_path):
+            f = open(self.toolbox_input.stream_file_path, "r")
+            existing_content = f.read()
+            f.close()
+            self.assertTrue(isinstance(existing_content, str))
+        
+        self.toolbox_input.setProjectStreams(None)
+        if os.path.exists(self.toolbox_input.stream_file_path):
+            f = open(self.toolbox_input.stream_file_path, "r")
+            existing_content = f.read()
+            f.close()
+            self.assertTrue(isinstance(existing_content, str))
+        self.assertIsInstance()
+
+        self.assertTrue( isinstance(self.toolbox_input.get_survey_point(), tuple))
+        self.assertTrue( isinstance(self.toolbox_input.get_survey_point()[0], float) or isinstance(self.toolbox_input.get_survey_point()[0], int))
+        self.assertTrue( isinstance(self.toolbox_input.get_survey_point()[1], float) or isinstance(self.toolbox_input.get_survey_point()[1], int))
+
+        self.assertTrue( self.toolbox_input.set_survey_point )
+
+    def test_speckle_inputs(self):
+        self.assertTrue(isinstance(self.speckle_input.accounts, list))
+        self.assertTrue(self.speckle_input.account is None or isinstance(self.speckle_input.account, Account))
+        self.assertTrue(self.speckle_input.streams_default is None or isinstance(self.speckle_input.streams_default, list))
+        self.assertIsNone(self.speckle_input.project)
+        self.assertIsNone(self.speckle_input.active_map)
+        self.assertEqual(self.speckle_input.stream_file_path, "")
+        self.assertTrue(isinstance(self.speckle_input.saved_streams, list))
+        self.assertTrue(isinstance(self.speckle_input.all_layers, list))
+        self.assertTrue(isinstance(self.speckle_input.clients, list))
+    
+    def test_speckle_inputs_functions(self):    
+        
+        self.assertTrue(isinstance(self.speckle_input.getProjectStreams(), list))
+        getStreams = self.speckle_input.getProjectStreams(self.test_stream)
+        self.assertTrue(isinstance(getStreams[0][0], StreamWrapper) and isinstance(getStreams[0][1], Stream))
+
+        self.assertTrue(isinstance(self.speckle_input.tryGetStream(StreamWrapper(self.test_stream)), Stream))
+        self.assertRaises(SpeckleException, lambda: self.speckle_input.tryGetStream(None))
+
+    def test_parameters(self):
+        actual = len(self.speckleTool.getParameterInfo())
+        expected = 15
+        self.assertEqual(actual, expected)
+
+if __name__ == '__main__':
+    unittest.main()
+