Merge pull request #78 from specklesystems/JR-Morgan-patch-1

Update config.yml

.circleci/config.yml

@@ -8,68 +8,6 @@ orbs:
   aws-s3: circleci/aws-s3@2.0.0
 
 jobs:
-  build-connector-win: # Reusable job for basic connectors
-    executor:
-      name: win/default # comes with python 3.7.3
-      shell: cmd.exe
-    parameters:
-      slug:
-        type: string
-        default: "arcgis"
-      installer:
-        type: boolean
-        default: false
-    steps:
-      - checkout
-      - attach_workspace:
-          at: ./
-      - run:
-          name: Create Innosetup signing cert
-          shell: powershell.exe
-          command: |
-            echo $env:PFX_B64 > "speckle-sharp-ci-tools\SignTool\AEC Systems Ltd.txt"
-            certutil -decode "speckle-sharp-ci-tools\SignTool\AEC Systems Ltd.txt" "speckle-sharp-ci-tools\SignTool\AEC Systems Ltd.pfx"
-      - run:
-          name: Patch
-          shell: powershell.exe
-          command:
-            | # If no tag, use 0.0.0.1 and don't make any YML (for testing only!)
-            $tag = if([string]::IsNullOrEmpty($env:CIRCLE_TAG)) { "0.0.0" } else { $env:CIRCLE_TAG }
-            $semver = if($tag.Contains('/')) {$tag.Split("/")[1] } else { $tag }
-            $ver = if($semver.Contains('-')) {$semver.Split("-")[0] } else { $semver }
-            $version = "$($ver).$($env:CIRCLE_BUILD_NUM)"
-            echo $semver
-            python patch_version.py $semver
-            python setup.py sdist bdist_wheel
-            Copy-Item -Path "dist\speckle_toolbox-$($ver)-py3-none-any.whl" -Destination "speckle_arcgis_installer"
-      - run: 
-          name: Build Installer
-          shell: cmd.exe
-          command:
-            | # If no tag, use 0.0.0.1 and don't make any YML (for testing only!)
-            speckle-sharp-ci-tools\InnoSetup\ISCC.exe speckle-sharp-ci-tools\arcgis.iss /Sbyparam=$p
-      - when:
-          condition: << parameters.installer >>
-          steps:
-            - persist_to_workspace:
-                root: ./
-                paths:
-                  - speckle-sharp-ci-tools/Installers
-  publish-github-release:
-    docker:
-      - image: cimg/go:1.20.0
-    steps:
-      - attach_workspace:
-          at: ./speckle_arcgis_installer
-      - run:
-          name: "Publish Release on GitHub"
-          command: |
-            set -x
-            go install github.com/tcnksm/ghr@v0.16.0
-            VERSION="${CIRCLE_TAG:-0.0.0}"
-            VERSION_SHORT=$(echo "${VERSION}" | cut -d- -f1)
-            ghr -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete "${VERSION}" "./speckle_arcgis_installer/speckle_toolbox-${VERSION_SHORT}-py3-none-any.whl"
-
   get-ci-tools: # Clones our ci tools and persists them to the workspace
     docker:
       - image: cimg/base:2021.01
@@ -91,6 +29,129 @@ jobs:
           paths:
             - speckle-sharp-ci-tools
 
+  get-ui: # Clones our ci tools and persists them to the workspace
+    docker:
+      - image: cimg/base:2021.01
+    steps:
+      - checkout
+      - attach_workspace:
+          at: ./
+      - add_ssh_keys:
+          fingerprints:
+            - "d1:d5:96:4d:ed:58:6e:7f:58:cc:21:5f:94:20:76:49"
+      - run:
+          name: I know Github as a host
+          command: |
+            touch ~/.ssh/known_hosts
+            ssh-keyscan github.com >> ~/.ssh/known_hosts
+      - run:
+          name: Clone
+          command:  |
+            git clone git@github.com:specklesystems/specklepy_qt_ui.git speckle_toolbox/esri/toolboxes/speckle/specklepy_qt_ui
+      - run:
+          name: Remove Git Artifacts
+          command: |
+            rm -rf ./speckle_toolbox/esri/toolboxes/speckle/specklepy_qt_ui/.git/
+            rm ./speckle_toolbox/esri/toolboxes/speckle/specklepy_qt_ui/.gitignore
+      - persist_to_workspace:
+          root: ./
+          paths:
+            - speckle_toolbox/esri/toolboxes/speckle/specklepy_qt_ui
+
+  build-connector-win: # Reusable job for basic connectors
+    executor:
+      name: win/default # comes with python 3.7.3
+    parameters:
+      slug:
+        type: string
+        default: "arcgis"
+      installer:
+        type: boolean
+        default: false
+    steps:
+      - checkout
+      - attach_workspace:
+          at: ./
+      - run:
+          name: Patch
+          shell: powershell.exe
+          command:
+            | # If no tag, use 0.0.0.1 and don't make any YML (for testing only!)
+            $tag = if([string]::IsNullOrEmpty($env:CIRCLE_TAG)) { "0.0.0" } else { $env:CIRCLE_TAG }
+            $semver = if($tag.Contains('/')) {$tag.Split("/")[1] } else { $tag }
+            $ver = if($semver.Contains('-')) {$semver.Split("-")[0] } else { $semver }
+            $version = "$($ver).$($env:CIRCLE_BUILD_NUM)"
+            echo $semver
+            python patch_version.py $semver
+            pip install setuptools
+            python setup.py sdist bdist_wheel
+            Copy-Item -Path "dist\speckle_toolbox-$($ver)-py3-none-any.whl" -Destination "speckle_arcgis_installer"
+
+      - run:
+          name: Exit if External PR
+          shell: bash.exe
+          command: if [ "$CIRCLE_PR_REPONAME" ]; then circleci-agent step halt; fi
+      
+      - unless: # Build installers unsigned on non-tagged builds
+          condition: << pipeline.git.tag >>
+          steps:
+            - run:
+                name: Build Installer
+                shell: cmd.exe #does not work in powershell
+                command: 
+                  speckle-sharp-ci-tools\InnoSetup\ISCC.exe speckle-sharp-ci-tools\arcgis.iss /Sbyparam=$p
+
+      - when: # Setup certificates and build installers signed for tagged builds
+          condition: << pipeline.git.tag >>
+          steps:
+            - run: # Installs digicert signing tools for windows
+                name: "Digicert Signing Manager Setup"
+                command: |
+                  cd C:\
+                  curl.exe -X GET  https://one.digicert.com/signingmanager/api-ui/v1/releases/smtools-windows-x64.msi/download -H "x-api-key:$env:SM_API_KEY" -o smtools-windows-x64.msi
+                  msiexec.exe /i smtools-windows-x64.msi /quiet /qn | Wait-Process
+            - run: # Creates the Auth cert and the signing public PEM cert
+                name: Create Auth & OV Signing Cert
+                command: |
+                  cd C:\
+                  echo $env:SM_CLIENT_CERT_FILE_B64 > certificate.txt
+                  certutil -decode certificate.txt certificate.p12
+            - run: # Syncs certificates from Digicert into local user store
+                name: Sync Certs
+                command: |
+                  & $env:SSM\smksp_cert_sync.exe
+            - run: 
+                name: Build Installer
+                shell: cmd.exe
+                command:
+                  | # If no tag, use 0.0.0.1 and don't make any YML (for testing only!)
+                  speckle-sharp-ci-tools\InnoSetup\ISCC.exe speckle-sharp-ci-tools\arcgis.iss /Sbyparam=$p /DSIGN_INSTALLER /DCODE_SIGNING_CERT_FINGERPRINT=%SM_CODE_SIGNING_CERT_SHA1_HASH%
+      - when:
+          condition: << parameters.installer >>
+          steps:
+            - persist_to_workspace:
+                root: ./
+                paths:
+                  - speckle-sharp-ci-tools/Installers
+                  - speckle_arcgis_installer
+    environment:
+      SSM: 'C:\Program Files\DigiCert\DigiCert One Signing Manager Tools'
+
+  publish-github-release:
+    docker:
+      - image: cimg/go:1.20.0
+    steps:
+      - attach_workspace:
+          at: ./
+      - run:
+          name: "Publish Release on GitHub"
+          command: |
+            set -x
+            go install github.com/tcnksm/ghr@v0.16.0
+            VERSION="${CIRCLE_TAG:-0.0.0}"
+            VERSION_SHORT=$(echo "${VERSION}" | cut -d- -f1)
+            ghr -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete "${VERSION}" "./speckle_arcgis_installer/speckle_toolbox-${VERSION_SHORT}-py3-none-any.whl"
+
   deploy-manager2:
     docker:
       - image: mcr.microsoft.com/dotnet/sdk:6.0
@@ -114,56 +175,47 @@ jobs:
             TAG=$(if [ "${CIRCLE_TAG}" ]; then echo $CIRCLE_TAG; else echo "0.0.0"; fi;)
             SEMVER=$(echo "$TAG" | sed -e 's/\/[a-zA-Z-]*//')
             /root/.dotnet/tools/Speckle.Manager.Feed deploy -s << parameters.slug >> -v ${SEMVER} -u https://releases.speckle.dev/installers/<< parameters.slug >>/<< parameters.slug >>-${SEMVER}.<< parameters.extension >> -o << parameters.os >> -f speckle-sharp-ci-tools/Installers/<< parameters.slug >>/<< parameters.slug >>-${SEMVER}.<< parameters.extension >>
+
 workflows: #happens with every PR to main
   build: # build the installers, but don't persist to workspace for deployment
     jobs:
+      - get-ui:
+          context: github-dev-bot
       - get-ci-tools:
           context: github-dev-bot
-          filters:
-            branches:
-              only:
-                - main
-                - /ci\/.*/
       - build-connector-win:
           requires:
+            - get-ui
             - get-ci-tools
-          filters:
-            branches:
-              only:
-                - main
-                - /ci\/.*/
-          context: innosetup
+          context: digicert-keylocker
 
   deploy: # build installers and deploy
     jobs:
-      - get-ci-tools:
+      - get-ui:
           context: github-dev-bot
-          filters:
+          filters: &deploy_filters
             tags:
-              only: /.*/
+              only: /([0-9]+)\.([0-9]+)\.([0-9]+)(?:-\w+)?$/
             branches:
               ignore: /.*/
 
+      - get-ci-tools:
+          context: github-dev-bot
+          filters: *deploy_filters
+
       - build-connector-win:
           name: build-deploy-connector-win
           slug: arcgis
           installer: true
           requires:
+            - get-ui
             - get-ci-tools
-          filters:
-            tags:
-              only: /([0-9]+)\.([0-9]+)\.([0-9]+)(?:-\w+)?$/
-            branches:
-              ignore: /.*/
-          context: innosetup
+          filters: *deploy_filters
+          context: digicert-keylocker
       - publish-github-release:
           requires:
             - build-deploy-connector-win
-          filters:
-            tags:
-              only: /([0-9]+)\.([0-9]+)\.([0-9]+)(?:-\w+)?$/
-            branches:
-              ignore: /.*/
+          filters: *deploy_filters
           context: arcgis-github-release
       - deploy-manager2:
           slug: arcgis
@@ -171,9 +223,5 @@ workflows: #happens with every PR to main
           extension: exe
           requires:
             - build-deploy-connector-win
-          filters:
-            tags:
-              only: /([0-9]+)\.([0-9]+)\.([0-9]+)(?:-\w+)?$/
-            branches:
-              ignore: /.*/ # For testing only! /ci\/.*/
-          context: do-spaces-speckle-releases
+          filters: *deploy_filters
+          context: do-spaces-speckle-releases

.gitignore

@@ -120,4 +120,5 @@ settings.json
 zip_build
 .qt_for_python
 *.pyt.xml
-
+*specklepy_qt_ui
+*.whl 

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

patch_version.py

@@ -6,8 +6,9 @@ def patch_installer(tag):
     iss_file = "speckle-sharp-ci-tools/arcgis.iss"
     setup_whl_file = "setup.py"
     conda_file = "speckle_arcgis_installer/conda_clone_activate.py"
-    toolbox_install_file = "speckle_arcgis_installer/toolbox_install.py"
+    #toolbox_install_file = "speckle_arcgis_installer/toolbox_install.py"
     toolbox_manual_install_file = "speckle_arcgis_installer/toolbox_install_manual.py"
+    plugin_start_file = "speckle_toolbox/esri/toolboxes/speckle/speckle/speckle_arcgis.py"
 
     #py_tag = get_specklepy_version()
     with open(iss_file, "r") as file:
@@ -33,6 +34,18 @@ def patch_installer(tag):
             print(f"Patched whl setup with connector v{tag} and specklepy ")
     file.close()
 
+    with open(plugin_start_file, "r") as file:
+        lines = file.readlines()
+        for i, line in enumerate(lines):
+            if 'self.version = ' in line: 
+                lines[i] = lines[i].split("\"")[0] + "\"" + tag.split('-')[0] + "\"" + lines[i].split("\"")[2]
+                break
+        with open(plugin_start_file, "w") as file:
+            file.writelines(lines)
+            print(f"Patched GIS start file with connector v{tag} and specklepy ")
+    file.close()
+
+
     def whlFileRename(fileName: str): 
         with open(fileName, "r") as file:
             lines = file.readlines()
@@ -48,7 +61,7 @@ def patch_installer(tag):
         file.close()
 
     whlFileRename(conda_file)
-    whlFileRename(toolbox_install_file)
+    #whlFileRename(toolbox_install_file)
     whlFileRename(toolbox_manual_install_file)
 
 

requirements.txt

@@ -1,3 +1,3 @@
-specklepy==2.9.1
+specklepy==2.17.17
 panda3d==1.10.11
 

scripts/debugging_panel.py

@@ -1,3 +1,11 @@
+import os
+import subprocess
+pythonExec = os.environ["ProgramFiles"]+ r"\ArcGIS\Pro\bin\Python\envs\arcgispro-py3\python.exe"
+result = subprocess.run([pythonExec, "-m", "pip", "install", "--upgrade", "--ignore-installed", "specklepy==2.17.17"], capture_output=True, text=True, shell=True, timeout=1000)
+result = subprocess.run([pythonExec, "-m", "pip", "install", "--upgrade", "--ignore-installed", "panda3d==1.10.11"], capture_output=True, text=True, shell=True, timeout=1000)
+result = subprocess.run([pythonExec, "-m", "pip", "install", "--upgrade", "--ignore-installed", "PyQt5==5.15.9"], capture_output=True, text=True, shell=True, timeout=1000)
+
+
 
 from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
 import arcpy
@@ -5,15 +13,15 @@ import json
 import os
 
 try: 
-    from speckle.converter.layers.CRS import CRS
-    from speckle.converter.layers.Layer import Layer, VectorLayer, RasterLayer
+    from speckle.speckle.converter.layers.CRS import CRS
+    from specklepy.objects.GIS.layers import Layer, VectorLayer, RasterLayer
 except: 
     from speckle_toolbox.esri.toolboxes.speckle.converter.layers.CRS import CRS
     from speckle_toolbox.esri.toolboxes.speckle.converter.layers.Layer import Layer, VectorLayer, RasterLayer
 
 from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
 from arcpy.management import (CreateFeatureclass, MakeFeatureLayer,
-                              AddFields, AlterField, DefineProjection )
+                              AddFields, AlterField, DefineProjection, SelectLayerByAttribute, GetCount )
 
 from specklepy.objects import Base
 
@@ -40,6 +48,13 @@ for layer in active_map.listLayers():
             if geomType == "Point" and layerPoint is None: layerPoint = layer 
             if geomType == "Multipoint" and layerMultiPoint is None: layerMultiPoint = layer 
 
+################################ select/ clear selection ###########################
+for layer in project.activeMap.listLayers(): 
+    if (layer.isFeatureLayer) or layer.isRasterLayer: 
+        arcpy.SelectLayerByAttribute_management(layer.longName,"ADD_TO_SELECTION",'"OBJECTID" = 1')
+        print(arcpy.GetCount_management(layer.longName).getOutput(0))
+        arcpy.SelectLayerByAttribute_management(layer.longName, "CLEAR_SELECTION")
+
 ################## reset symbology if needed:
 sym = layerPolygon.symbology
 print(sym.renderer.type)
@@ -66,7 +81,7 @@ for k, grp in enumerate(sym.renderer.groups):
 
 
 
-from speckle.converter.layers.symbologyTemplates import get_polygon_simpleRenderer 
+from speckle.speckle.converter.layers.symbology import get_polygon_simpleRenderer 
 from arcpy._mp import ArcGISProject
 
 aprx = ArcGISProject('CURRENT')

setup.py

@@ -1,39 +1,56 @@
-# to build an installer: run cmd from this folder or use terminal: "%PROGRAMFILES%\\ArcGIS\\Pro\\bin\\Python\\envs\\arcgispro-py3\\python.exe" 
-# 
-# 1) python patch_version.py 2.x.x 
-# 2) python setup.py sdist bdist_wheel   #C:\\Users\\username\\Documents\\00_Speckle\\GitHub\\speckle-arcgis\\setup.py sdist bdist_wheel 
-# copy .whl from "dist" to "speckle_arcgis_installer" 
+# to build an installer: run cmd from this folder or use terminal: "%PROGRAMFILES%\\ArcGIS\\Pro\\bin\\Python\\envs\\arcgispro-py3\\python.exe"
+#
+# 1) python patch_version.py 2.x.x
+# 2) python setup.py sdist bdist_wheel   #C:\\Users\\username\\Documents\\00_Speckle\\GitHub\\speckle-arcgis\\setup.py sdist bdist_wheel
+# copy .whl from "dist" to "speckle_arcgis_installer"
 
 # ref: https://pro.arcgis.com/en/pro-app/2.8/arcpy/geoprocessing_and_python/distributing-python-modules.htm
 
-import os 
-from setuptools import setup 
+import os
+from setuptools import setup
 
 
-def read(fname): 
-    return open(os.path.join(os.path.dirname(__file__), fname)).read() 
- 
-setup(name='speckle_toolbox', 
-      author='SpeckleSystems',
-			version="2.9.4",
-      author_email="connectors@speckle.systems",
-      url="https://speckle.systems/",
-      description=("Example for extending geoprocessing through Python modules"),
-      long_description=read('Readme.md'),
-      python_requires='~=3.3',
-      setup_requires=['wheel'],
-      packages=['speckle_toolbox'], 
-      package_data={'speckle_toolbox':[
-                  'esri/arcpy/*', 
-                  'esri/help/gp/*', 'esri/help/gp/toolboxes/*', 'esri/help/gp/messages/*',
-                  'esri/toolboxes/*','esri/toolboxes/speckle/*',
-                  'esri/toolboxes/speckle/converter/*', 'esri/toolboxes/speckle/converter/geometry/*', 'esri/toolboxes/speckle/converter/layers/*',
-                  'esri/toolboxes/speckle/plugin_utils/*',
-                  'esri/toolboxes/speckle/ui/*'] 
-                  }, 
-      )
+def read(fname):
+    return open(os.path.join(os.path.dirname(__file__), fname)).read()
+
+
+setup(
+    name="speckle_toolbox",
+    author="SpeckleSystems",
+    version="0.0.99",
+    author_email="connectors@speckle.systems",
+    url="https://speckle.systems/",
+    description=("Example for extending geoprocessing through Python modules"),
+    long_description=read("Readme.md"),
+    python_requires="~=3.3",
+    setup_requires=["wheel"],
+    packages=["speckle_toolbox"],
+    package_data={
+        "speckle_toolbox": [
+            "esri/arcpy/*",
+            "esri/help/gp/*",
+            "esri/help/gp/toolboxes/*",
+            "esri/help/gp/messages/*",
+            "esri/toolboxes/*",
+            "esri/toolboxes/speckle/*",
+            "esri/toolboxes/speckle/speckle/*",
+            "esri/toolboxes/speckle/speckle/converter/*",
+            "esri/toolboxes/speckle/speckle/converter/features/*",
+            "esri/toolboxes/speckle/speckle/converter/geometry/*",
+            "esri/toolboxes/speckle/speckle/converter/layers/*",
+            "esri/toolboxes/speckle/speckle/plugin_utils/*",
+            "esri/toolboxes/speckle/speckle/utils/*",
+            "esri/toolboxes/speckle/specklepy_qt_ui/*",
+            "esri/toolboxes/speckle/specklepy_qt_ui/qt_ui/*",
+            "esri/toolboxes/speckle/specklepy_qt_ui/qt_ui/ui/*",
+            "esri/toolboxes/speckle/specklepy_qt_ui/qt_ui/utils/*",
+            "esri/toolboxes/speckle/specklepy_qt_ui/qt_ui/utils/assets/*",
+            "esri/toolboxes/speckle/ui_widgets/*",
+        ]
+    },
+)
 
 # then to install in ArcGIS:
 # import sysconfig; import subprocess; x = sysconfig.get_paths()['data'] + r"\python.exe"; subprocess.run((x, '-m','pip', 'install', 'C:\\Users\\username\\Documents\\00_Speckle\\GitHub\\speckle-arcgis\\dist\\foo-0.1-py3-none-any.whl'), capture_output=True, text=True, shell=True, timeout=1000 )
-# to uninstall: 
+# to uninstall:
 # "C:\\Users\\username\\AppData\\Local\\ESRI\\conda\\envs\\arcgispro-py3-speckle\\python.exe" -m pip uninstall C:\\Users\\username\\Documents\\00_Speckle\\GitHub\\speckle-arcgis\\dist\\foo-0.1-py3-none-any.whl

speckle_arcgis_installer/Readme.md

@@ -1,10 +1,9 @@
 ### Manual installation
 
-1. Download present "speckle_arcgis_installer" folder
-2. Clone the default ArcGIS Pro conda environment and restart ArcGIS Pro
-       - for 2.9.0: Project-> Python-> Manage Environments-> Clone Default
+1. From the [latest release](https://github.com/specklesystems/speckle-arcgis/releases) download the whl file and the source code zip, unzip and locate the subfolder "speckle_arcgis_installer" on your machine and place whl file in it. 
+2. Clone the default ArcGIS Pro conda environment (or set the one you use, except the default one) and restart ArcGIS Pro
        - for 3.0.0: Project-> Package Manager-> Active Environment (Environment Manager)-> Clone arcgispro-py3
-3. Change the path to your new environemnt Python.exe if necessary (variable "pythonPath" in "toolbox_install_manual.py") 
+3. Adjust the path to your new environment python executable (variable "pythonPath" in "speckle_arcgis_installer/toolbox_install_manual.py") 
 4. Enter the location of 'toolbox_install_manual.py' in the following command and run this command in ArcGIS Python console (View -> Python Window)
 
 ```python

speckle_arcgis_installer/Speckle.pyt

@@ -1,2 +1,4 @@
-try: from speckle.speckle_arcgis import *
-except: from speckle_toolbox.esri.toolboxes.speckle.speckle_arcgis import *
+try:
+    from speckle.speckle.speckle_arcgis import *
+except:
+    from speckle_toolbox.esri.toolboxes.speckle.speckle_arcgis import *

speckle_arcgis_installer/conda_clone_activate.py

@@ -1,4 +1,4 @@
-# clone env, install toolbox & dependencies into the cloned env; and into current one (if not default) 
+# clone env, install toolbox & dependencies into the cloned env; and into current one (if not default)
 import sys
 import sysconfig
 import os.path
@@ -10,126 +10,240 @@ from subprocess_call import subprocess_call
 
 import sys
 
+ENV_NEW_NAME = "arcgispro-py3-speckle"
+PROSWAP = "proswap"
+
+
 def setup():
-    pythonExec = get_python_path() # import numpy; import os; print(os.path.abspath(numpy.__file__)) 
-    return pythonExec # None if not successful 
-    
-def get_python_path(): # create a full copy of default env 
-    #print("Get Python path")
+    pythonExec = (
+        get_python_path()
+    )  # import numpy; import os; print(os.path.abspath(numpy.__file__))
+    return pythonExec  # None if not successful
+
+
+def get_default_python():
+    pythonExec = (
+        os.environ["ProgramFiles"]
+        + r"\ArcGIS\Pro\bin\Python\envs\arcgispro-py3\python.exe"
+    )  # (r"%PROGRAMFILES%\ArcGIS\Pro\bin\Python\envs\arcgispro-py3\python.exe") #os.path.dirname(sys.executable) + "\\python.exe" # default python.exe
+    # print(pythonExec)
+    if not os.path.exists(pythonExec):
+        pythonExec = (
+            os.getenv("APPDATA").replace("Roaming", "Local")
+            + r"\Programs\ArcGIS\Pro\bin\Python\envs\arcgispro-py3\python.exe"
+        )
+        if not os.path.exists(pythonExec):
+            return None
+    return pythonExec
+
+
+def get_python_path():  # create a full copy of default env
+    # print("Get Python path")
     # or: import site; site.getsitepackages()[0]
     # import specklepy; import os; print(os.path.abspath(specklepy.__file__)) ##currentPythonExec = sysconfig.get_paths()['data'] + r"\python.exe"
-
-    pythonExec = os.environ["ProgramFiles"] + r'\ArcGIS\Pro\bin\Python\envs\arcgispro-py3\python.exe' #(r"%PROGRAMFILES%\ArcGIS\Pro\bin\Python\envs\arcgispro-py3\python.exe") #os.path.dirname(sys.executable) + "\\python.exe" # default python.exe
-    #print(pythonExec)
-    if not os.path.exists(pythonExec):
-        pythonExec = os.getenv('APPDATA').replace("Roaming", "Local") + r'\Programs\ArcGIS\Pro\bin\Python\envs\arcgispro-py3\python.exe'
-        if not os.path.exists(pythonExec): return None
-    #print(os.getenv('APPDATA') + r'\Programs\ArcGIS\Pro\bin\Python\envs\arcgispro-py3\python.exe')
+    def_exec = get_default_python()
+    # print(os.getenv('APPDATA') + r'\Programs\ArcGIS\Pro\bin\Python\envs\arcgispro-py3\python.exe')
 
     if sys.platform == "win32":
-        env_new_name = "arcgispro-py3-speckle"
-        newExec = clone_env(pythonExec, env_new_name) # only if doesn't exist yet 
-        if not os.path.exists(newExec): return None
-        
-        activate_env(env_new_name)
+        newExec = clone_env(def_exec)  # only if doesn't exist yet
+        if not os.path.exists(newExec):
+            return None
+
+        activate_env()
         return newExec
-    else: return None
+    else:
+        return None
 
-def clone_env(pythonExec_old: str, env_new_name: str): 
-    install_folder = os.getenv('APPDATA').replace("\\Roaming","") + r"\Local\ESRI\conda\envs"   #r"%LOCALAPPDATA%\ESRI\conda\envs" 
-    if not os.path.exists(install_folder): os.makedirs(install_folder)
-    
-    default_env = pythonExec_old.replace("Pro\\bin\\Python\\envs\\arcgispro-py3\\python.exe","Pro\\bin\\Python\\envs\\arcgispro-py3") # + "\\" + 'arcgispro-py3'
-    conda_exe = pythonExec_old.replace("Pro\\bin\\Python\\envs\\arcgispro-py3\\python.exe","Pro\\bin\\Python\\Scripts\\conda.exe") #%PROGRAMFILES%\ArcGIS\Pro\bin\Python\Scripts\conda.exe #base: %PROGRAMDATA%\Anaconda3\condabin\conda.bat
-    new_env = install_folder + "\\" + env_new_name # %LOCALAPPDATA%\ESRI\conda\envs\...
 
-    if os.path.exists(conda_exe) and os.path.exists(default_env) and os.path.exists(new_env) and not os.path.exists(new_env + "\\python.exe"): 
-        # conda environment invalid: delete it's folder 
+def clone_env(pythonExec_old: str):
+    install_folder = (
+        os.getenv("APPDATA").replace("\\Roaming", "") + r"\Local\ESRI\conda\envs"
+    )  # r"%LOCALAPPDATA%\ESRI\conda\envs"
+    if not os.path.exists(install_folder):
+        os.makedirs(install_folder)
+
+    default_env = pythonExec_old.replace(
+        "Pro\\bin\\Python\\envs\\arcgispro-py3\\python.exe",
+        "Pro\\bin\\Python\\envs\\arcgispro-py3",
+    )  # + "\\" + 'arcgispro-py3'
+    conda_exe = pythonExec_old.replace(
+        "Pro\\bin\\Python\\envs\\arcgispro-py3\\python.exe",
+        "Pro\\bin\\Python\\Scripts\\conda.exe",
+    )  # %PROGRAMFILES%\ArcGIS\Pro\bin\Python\Scripts\conda.exe #base: %PROGRAMDATA%\Anaconda3\condabin\conda.bat
+    global PROSWAP
+    PROSWAP = conda_exe.replace(
+        "conda.exe",
+        "proswap.bat",
+    )
+    print(PROSWAP)
+    new_env = install_folder + "\\" + ENV_NEW_NAME  # %LOCALAPPDATA%\ESRI\conda\envs\...
+
+    # first check if venv invalid:
+    if (
+        os.path.exists(conda_exe)
+        and os.path.exists(default_env)
+        and os.path.exists(new_env)
+    ):
+        # delete existing venv
         print(f"Removing invalid environment {new_env}")
-        os.remove(new_env)
-
-    if os.path.exists(conda_exe) and os.path.exists(default_env) and not os.path.exists(new_env): 
+        try:
+            os.remove(new_env)
+        except PermissionError as e:
+            print(e)
+    elif (
+        os.path.exists(conda_exe)
+        and os.path.exists(default_env)
+        and not os.path.exists(new_env)
+    ):
         print("Wait for the default ArcGIS Pro conda environment to be cloned")
-        subprocess_call( [ conda_exe, 'config', '--set', 'ssl_verify', 'False'] )
-        subprocess_call( [ conda_exe, 'create', '--clone', default_env, '-p', new_env] ) # will not execute if already exists
-        subprocess_call( [ conda_exe, 'config', '--set', 'ssl_verify', 'True'] )
+        subprocess_call([conda_exe, "config", "--set", "ssl_verify", "False"])
+        subprocess_call(
+            [conda_exe, "create", "--clone", default_env, "-p", new_env]
+        )  # will not execute if already exists
+        subprocess_call([conda_exe, "config", "--set", "ssl_verify", "True"])
 
-    elif os.path.exists(new_env) and os.path.exists(new_env + "\\python.exe"):
-        print(f"Environment {new_env} already exists, preparing to install packages..")
+    # final check
+    if os.path.exists(new_env) and os.path.exists(new_env + "\\python.exe"):
+        print("Preparing to install packages..")
 
     print(new_env + "\\python.exe")
     return new_env + "\\python.exe"
 
-def activate_env(env_new_name: str):
-    # using Popen, because process does not return result; subprocess.run will hang indefinitely 
-    variable = subprocess.Popen((f'proswap {env_new_name}'),stdout = subprocess.PIPE,stderr = subprocess.PIPE,text = True,shell = True)
+
+def activate_env():
+    # using Popen, because process does not return result; subprocess.run will hang indefinitely
+    # print(os.environ["TMPDIR"])
+    my_env = os.environ.copy()
+    print(PROSWAP)
+    # my_env["TMPDIR"] = r"C:\Users\katri\AppData\Roaming\Speckle\connector_installations"
+    pop = subprocess.Popen(
+        (f"{PROSWAP} {ENV_NEW_NAME}"),
+        text=True,
+        env=my_env,
+    )
     # activate new env : https://support.esri.com/en/technical-article/000024206
 
 
 def installToolbox(newExec: str):
     print("Installing Speckle Toolbox")
-    whl_file = os.path.join(os.path.dirname(__file__), "speckle_toolbox-2.9.4-py3-none-any.whl" ) 
+    whl_file = os.path.join(
+        os.path.dirname(__file__), "speckle_toolbox-2.9.99-py3-none-any.whl"
+    )
     print(whl_file)
-    subprocess_call([newExec, '-m','pip','install','--upgrade', '--force-reinstall', whl_file])
-    # to uninstall: cmd.exe "C:\\Users\\username\\AppData\\Local\\ESRI\\conda\\envs\\arcgispro-2.9.4-py3-none-any.whl
+    subprocess_call(
+        [newExec, "-m", "pip", "install", "--upgrade", "--force-reinstall", whl_file]
+    )
+    # to uninstall: cmd.exe "X:\\xxx.whl
     return
 
+
+def clearToolbox(pythonExec: str):
+    # install pip
+    print("CLEAR toolbox")
+    print(pythonExec)
+    try:
+
+        speckle_path = pythonExec.replace("python.exe", "Lib\\site-packages\\")
+
+        print(speckle_path)
+        paths = os.listdir(speckle_path)
+        for p in paths:
+            if "speckle_toolbox" in p:
+                print("remove: " + str(p))
+                os.remove(p)
+    except Exception as e:
+        print(e)
+        pass
+
+
 def installDependencies(pythonExec: str, pkgName: str, pkgVersion: str):
     # install pip
     print(pythonExec)
     try:
         import pip
     except:
-        getPipFilePath = os.path.join(os.path.dirname(__file__), "get_pip.py") #TODO: give actual folder path 
+        getPipFilePath = os.path.join(
+            os.path.dirname(__file__), "get_pip.py"
+        )  # TODO: give actual folder path
         exec(open(getPipFilePath).read())
         # just in case the included version is old
         subprocess_call([pythonExec, "-m", "pip", "install", "--upgrade", "pip"])
-        
+
     # install package
     try:
-        #import importlib #importlib.import_module(pkgName)
+        # import importlib #importlib.import_module(pkgName)
         if pkgName == "specklepy":
-            import specklepy 
-            if pythonExec.replace("\\python.exe","") not in (os.path.abspath(specklepy.__file__)): 
+            import specklepy
+
+            if pythonExec.replace("\\python.exe", "") not in (
+                os.path.abspath(specklepy.__file__)
+            ):
                 print(f"Installing {pkgName} to {pythonExec}")
-                #subprocess_call( [pythonExec, "-m", "pip", "uninstall", f"{pkgName}"])
-                subprocess_call( [pythonExec, "-m", "pip", "install", f"{pkgName}=={pkgVersion}"])
+                # subprocess_call( [pythonExec, "-m", "pip", "uninstall", f"{pkgName}"])
+                subprocess_call(
+                    [
+                        pythonExec,
+                        "-m",
+                        "pip",
+                        "install",
+                        "--ignore-installed",
+                        f"{pkgName}=={pkgVersion}",
+                    ]
+                )
         elif pkgName == "panda3d":
-            import panda3d 
-            if pythonExec.replace("\\python.exe","") not in (os.path.abspath(panda3d.__file__)): 
+            import panda3d
+
+            if pythonExec.replace("\\python.exe", "") not in (
+                os.path.abspath(panda3d.__file__)
+            ):
                 print(f"Installing {pkgName} to {pythonExec}")
-                subprocess_call( [pythonExec, "-m", "pip", "install", f"{pkgName}=={pkgVersion}"])
+                subprocess_call(
+                    [
+                        pythonExec,
+                        "-m",
+                        "pip",
+                        "install",
+                        "--ignore-installed",
+                        f"{pkgName}=={pkgVersion}",
+                    ]
+                )
+        elif pkgName == "PyQt5":
+            import PyQt5
+
+            if pythonExec.replace("\\python.exe", "") not in (
+                os.path.abspath(PyQt5.__file__)
+            ):
+                print(f"Installing {pkgName} to {pythonExec}")
+                subprocess_call(
+                    [
+                        pythonExec,
+                        "-m",
+                        "pip",
+                        "install",
+                        "--ignore-installed",
+                        f"{pkgName}=={pkgVersion}",
+                    ]
+                )
     except Exception as e:
         print(f"{pkgName} not installed")
-        subprocess_call( [pythonExec, "-m", "pip", "install", f"{pkgName}=={pkgVersion}"])
-
-    # Check if package needs updating
-    r'''
-    try:
-        print(f"Attempting to update {pkgName} to {pkgVersion}")
-        result = subprocess_call(
-            [
-                pythonExec,
-                "-m",
-                "pip",
-                "install",
-                "--upgrade",
-                f"{pkgName}=={pkgVersion}",
-            ]
+        subprocess_call(
+            [pythonExec, "-m", "pip", "install", f"{pkgName}=={pkgVersion}"]
         )
-        if result == True:
-            print(f"{pkgName} upgraded")
-            return True
-        else: 
-            return False
-    except Exception as e:
-        print(e)
-        print(e.with_traceback)
-    '''
     return True
-    
+
+
 pythonPath = setup()
+print(pythonPath)
 if pythonPath is not None:
+
+    # def_exec = get_default_python()
+    # conda_exe = def_exec.replace("Pro\\bin\\Python\\envs\\arcgispro-py3\\python.exe","Pro\\bin\\Python\\Scripts\\conda.exe") #%PROGRAMFILES%\ArcGIS\Pro\bin\Python\Scripts\conda.exe #base: %PROGRAMDATA%\Anaconda3\condabin\conda.bat
+    # subprocess_call([conda_exe, 'proup','-n', ENV_NEW_NAME])
+
+    clearToolbox(pythonPath)
     installToolbox(pythonPath)
-    installDependencies(pythonPath, "specklepy", "2.9.0"  )
-    installDependencies(pythonPath, "panda3d", "1.10.11" )
+    installDependencies(pythonPath, "specklepy", "2.17.17")
+    installDependencies(pythonPath, "panda3d", "1.10.11")
+    installDependencies(pythonPath, "PyQt5", "5.15.9")
+
+# manual: import sysconfig; import subprocess; x = sysconfig.get_paths()['data'] + r"\python.exe"; subprocess.run((x, "-m", "pip", "install", "PyQt5==5.15.9"), capture_output=True, text=True, shell=True, timeout=1000 )

speckle_arcgis_installer/speckle_arcgis_install.bat

@@ -1,3 +0,0 @@
-@echo off
-"%PROGRAMFILES%\ArcGIS\Pro\bin\Python\envs\arcgispro-py3\python.exe" "conda_clone_activate.py"
-"%LOCALAPPDATA%\ESRI\conda\envs\arcgispro-py3-speckle\python.exe" "toolbox_install.py"

speckle_arcgis_installer/toolbox_install.py

@@ -1,61 +0,0 @@
-
-from subprocess_call import subprocess_call
-import os 
-
-pythonPath = os.getenv('APPDATA').replace("\\Roaming","") + r"\Local\ESRI\conda\envs\arcgispro-py3-speckle\python.exe"
-
-def installToolbox(newExec: str):
-    print("Installing Speckle Toolbox")
-    whl_file = os.path.join(os.path.dirname(__file__), "speckle_toolbox-2.9.4-py3-none-any.whl" ) 
-    print(whl_file)
-    subprocess_call([newExec, '-m','pip','install','--upgrade', '--force-reinstall', whl_file])
-    # to uninstall: cmd.exe "C:\\Users\\username\\AppData\\Local\\ESRI\\conda\\envs\\arcgispro-2.9.4-py3-none-any.whl
-    return
-
-def installDependencies(pythonExec: str, pkgName: str, pkgVersion: str):
-    # install pip
-    print(pythonExec)
-    try:
-        import pip
-    except:
-        getPipFilePath = os.path.join(os.path.dirname(__file__), "get_pip.py") #TODO: give actual folder path 
-        exec(open(getPipFilePath).read())
-        # just in case the included version is old
-        subprocess_call([pythonExec, "-m", "pip", "install", "--upgrade", "pip"])
-        
-    # install package
-    try:
-        import importlib
-        importlib.import_module(pkgName)
-    except Exception as e:
-        print(f"{pkgName} not installed")
-        subprocess_call( [pythonExec, "-m", "pip", "install", f"{pkgName}=={pkgVersion}"])
-
-    # Check if package needs updating
-    try:
-        print(f"Attempting to update {pkgName} to {pkgVersion}")
-        result = subprocess_call(
-            [
-                pythonExec,
-                "-m",
-                "pip",
-                "install",
-                "--upgrade",
-                f"{pkgName}=={pkgVersion}",
-            ]
-        )
-        if result == True:
-            print(f"{pkgName} upgraded")
-            return True
-        else: 
-            return False
-    except Exception as e:
-        print(e)
-        print(e.with_traceback)
-    return True
-
-
-installToolbox(pythonPath)
-installDependencies(pythonPath, "specklepy", "2.9.0"  )
-installDependencies(pythonPath, "panda3d", "1.10.11" )
-

speckle_arcgis_installer/toolbox_install_manual.py

@@ -19,8 +19,8 @@ def installToolbox(newExec: str):
     mypath = os.path.dirname(__file__)
     onlyfiles = [f for f in listdir(mypath) if (isfile(join(mypath, f)) and "py3-none-any.whl" in str(f))]
     onlyfiles.sort(key = lambda x: int(x.replace("speckle_toolbox-","").replace("-py3-none-any.whl","").split(".")[1]) ) 
-    whl_file = onlyfiles[len(onlyfiles)-1]
-    #whl_file = os.path.join(os.path.dirname(__file__), "speckle_toolbox-2.11.3-py3-none-any.whl" ) 
+    whl_file = mypath + "\\" + onlyfiles[len(onlyfiles)-1]
+    #whl_file = os.path.join(os.path.dirname(__file__), "speckle_toolbox-2.9.9-py3-none-any.whl" ) 
     subprocess_call([newExec, '-m','pip','install','--upgrade', '--force-reinstall', whl_file])
     return
 

speckle_toolbox/esri/toolboxes/Speckle.pyt

@@ -1,2 +1,4 @@
-try: from speckle.speckle_arcgis import *
-except: from speckle_toolbox.esri.toolboxes.speckle.speckle_arcgis import *
+try:
+    from speckle.speckle.speckle_arcgis import *
+except:
+    from speckle_toolbox.esri.toolboxes.speckle.speckle.speckle_arcgis import *

speckle_toolbox/esri/toolboxes/Speckle.pyt.xml

@@ -1,77 +1,2 @@
-<?xml version="1.0"?>
-<metadata xml:lang="en"><Esri><CreaDate>20220718</CreaDate><CreaTime>13500100</CreaTime><ArcGISFormat>1.0</ArcGISFormat><SyncOnce>TRUE</SyncOnce><ModDate>20221207</ModDate><ModTime>175959</ModTime><scaleRange><minScale>150000000</minScale><maxScale>5000</maxScale></scaleRange><ArcGISProfile>ItemDescription</ArcGISProfile></Esri><toolbox name="Speckle" alias="speckle_toolbox_"><arcToolboxHelpPath>c:\program files\arcgis\pro\Resources\Help\gp</arcToolboxHelpPath><toolsets/></toolbox><dataIdInfo><idCitation><resTitle>Speckle</resTitle></idCitation><idPurp>Speckle connector for ArcGIS</idPurp><searchKeys><keyword>speckle3d</keyword></searchKeys></dataIdInfo><distInfo><distributor><distorFormat><formatName>ArcToolbox Toolbox</formatName></distorFormat></distributor></distInfo><mdHrLv><ScopeCd value="005"></ScopeCd></mdHrLv><Binary><Thumbnail><Data EsriPropertyType="PictureX">/9j/4AAQSkZJRgABAQEAYABgAAD/4gxYSUNDX1BST0ZJTEUAAQEAAAxITGlubwIQAABtbnRyUkdC
-IFhZWiAHzgACAAkABgAxAABhY3NwTVNGVAAAAABJRUMgc1JHQgAAAAAAAAAAAAAAAAAA9tYAAQAA
-AADTLUhQICAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABFj
-cHJ0AAABUAAAADNkZXNjAAABhAAAAGx3dHB0AAAB8AAAABRia3B0AAACBAAAABRyWFlaAAACGAAA
-ABRnWFlaAAACLAAAABRiWFlaAAACQAAAABRkbW5kAAACVAAAAHBkbWRkAAACxAAAAIh2dWVkAAAD
-TAAAAIZ2aWV3AAAD1AAAACRsdW1pAAAD+AAAABRtZWFzAAAEDAAAACR0ZWNoAAAEMAAAAAxyVFJD
-AAAEPAAACAxnVFJDAAAEPAAACAxiVFJDAAAEPAAACAx0ZXh0AAAAAENvcHlyaWdodCAoYykgMTk5
-OCBIZXdsZXR0LVBhY2thcmQgQ29tcGFueQAAZGVzYwAAAAAAAAASc1JHQiBJRUM2MTk2Ni0yLjEA
-AAAAAAAAAAAAABJzUkdCIElFQzYxOTY2LTIuMQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAWFlaIAAAAAAAAPNRAAEAAAABFsxYWVogAAAAAAAAAAAAAAAA
-AAAAAFhZWiAAAAAAAABvogAAOPUAAAOQWFlaIAAAAAAAAGKZAAC3hQAAGNpYWVogAAAAAAAAJKAA
-AA+EAAC2z2Rlc2MAAAAAAAAAFklFQyBodHRwOi8vd3d3LmllYy5jaAAAAAAAAAAAAAAAFklFQyBo
-dHRwOi8vd3d3LmllYy5jaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAABkZXNjAAAAAAAAAC5JRUMgNjE5NjYtMi4xIERlZmF1bHQgUkdCIGNvbG91ciBzcGFjZSAt
-IHNSR0IAAAAAAAAAAAAAAC5JRUMgNjE5NjYtMi4xIERlZmF1bHQgUkdCIGNvbG91ciBzcGFjZSAt
-IHNSR0IAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZGVzYwAAAAAAAAAsUmVmZXJlbmNlIFZpZXdpbmcg
-Q29uZGl0aW9uIGluIElFQzYxOTY2LTIuMQAAAAAAAAAAAAAALFJlZmVyZW5jZSBWaWV3aW5nIENv
-bmRpdGlvbiBpbiBJRUM2MTk2Ni0yLjEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHZpZXcAAAAA
-ABOk/gAUXy4AEM8UAAPtzAAEEwsAA1yeAAAAAVhZWiAAAAAAAEwJVgBQAAAAVx/nbWVhcwAAAAAA
-AAABAAAAAAAAAAAAAAAAAAAAAAAAAo8AAAACc2lnIAAAAABDUlQgY3VydgAAAAAAAAQAAAAABQAK
-AA8AFAAZAB4AIwAoAC0AMgA3ADsAQABFAEoATwBUAFkAXgBjAGgAbQByAHcAfACBAIYAiwCQAJUA
-mgCfAKQAqQCuALIAtwC8AMEAxgDLANAA1QDbAOAA5QDrAPAA9gD7AQEBBwENARMBGQEfASUBKwEy
-ATgBPgFFAUwBUgFZAWABZwFuAXUBfAGDAYsBkgGaAaEBqQGxAbkBwQHJAdEB2QHhAekB8gH6AgMC
-DAIUAh0CJgIvAjgCQQJLAlQCXQJnAnECegKEAo4CmAKiAqwCtgLBAssC1QLgAusC9QMAAwsDFgMh
-Ay0DOANDA08DWgNmA3IDfgOKA5YDogOuA7oDxwPTA+AD7AP5BAYEEwQgBC0EOwRIBFUEYwRxBH4E
-jASaBKgEtgTEBNME4QTwBP4FDQUcBSsFOgVJBVgFZwV3BYYFlgWmBbUFxQXVBeUF9gYGBhYGJwY3
-BkgGWQZqBnsGjAadBq8GwAbRBuMG9QcHBxkHKwc9B08HYQd0B4YHmQesB78H0gflB/gICwgfCDII
-RghaCG4IggiWCKoIvgjSCOcI+wkQCSUJOglPCWQJeQmPCaQJugnPCeUJ+woRCicKPQpUCmoKgQqY
-Cq4KxQrcCvMLCwsiCzkLUQtpC4ALmAuwC8gL4Qv5DBIMKgxDDFwMdQyODKcMwAzZDPMNDQ0mDUAN
-Wg10DY4NqQ3DDd4N+A4TDi4OSQ5kDn8Omw62DtIO7g8JDyUPQQ9eD3oPlg+zD88P7BAJECYQQxBh
-EH4QmxC5ENcQ9RETETERTxFtEYwRqhHJEegSBxImEkUSZBKEEqMSwxLjEwMTIxNDE2MTgxOkE8UT
-5RQGFCcUSRRqFIsUrRTOFPAVEhU0FVYVeBWbFb0V4BYDFiYWSRZsFo8WshbWFvoXHRdBF2UXiReu
-F9IX9xgbGEAYZRiKGK8Y1Rj6GSAZRRlrGZEZtxndGgQaKhpRGncanhrFGuwbFBs7G2MbihuyG9oc
-AhwqHFIcexyjHMwc9R0eHUcdcB2ZHcMd7B4WHkAeah6UHr4e6R8THz4faR+UH78f6iAVIEEgbCCY
-IMQg8CEcIUghdSGhIc4h+yInIlUigiKvIt0jCiM4I2YjlCPCI/AkHyRNJHwkqyTaJQklOCVoJZcl
-xyX3JicmVyaHJrcm6CcYJ0kneierJ9woDSg/KHEooijUKQYpOClrKZ0p0CoCKjUqaCqbKs8rAis2
-K2krnSvRLAUsOSxuLKIs1y0MLUEtdi2rLeEuFi5MLoIuty7uLyQvWi+RL8cv/jA1MGwwpDDbMRIx
-SjGCMbox8jIqMmMymzLUMw0zRjN/M7gz8TQrNGU0njTYNRM1TTWHNcI1/TY3NnI2rjbpNyQ3YDec
-N9c4FDhQOIw4yDkFOUI5fzm8Ofk6Njp0OrI67zstO2s7qjvoPCc8ZTykPOM9Ij1hPaE94D4gPmA+
-oD7gPyE/YT+iP+JAI0BkQKZA50EpQWpBrEHuQjBCckK1QvdDOkN9Q8BEA0RHRIpEzkUSRVVFmkXe
-RiJGZ0arRvBHNUd7R8BIBUhLSJFI10kdSWNJqUnwSjdKfUrESwxLU0uaS+JMKkxyTLpNAk1KTZNN
-3E4lTm5Ot08AT0lPk0/dUCdQcVC7UQZRUFGbUeZSMVJ8UsdTE1NfU6pT9lRCVI9U21UoVXVVwlYP
-VlxWqVb3V0RXklfgWC9YfVjLWRpZaVm4WgdaVlqmWvVbRVuVW+VcNVyGXNZdJ114XcleGl5sXr1f
-D19hX7NgBWBXYKpg/GFPYaJh9WJJYpxi8GNDY5dj62RAZJRk6WU9ZZJl52Y9ZpJm6Gc9Z5Nn6Wg/
-aJZo7GlDaZpp8WpIap9q92tPa6dr/2xXbK9tCG1gbbluEm5rbsRvHm94b9FwK3CGcOBxOnGVcfBy
-S3KmcwFzXXO4dBR0cHTMdSh1hXXhdj52m3b4d1Z3s3gReG54zHkqeYl553pGeqV7BHtje8J8IXyB
-fOF9QX2hfgF+Yn7CfyN/hH/lgEeAqIEKgWuBzYIwgpKC9INXg7qEHYSAhOOFR4Wrhg6GcobXhzuH
-n4gEiGmIzokziZmJ/opkisqLMIuWi/yMY4zKjTGNmI3/jmaOzo82j56QBpBukNaRP5GokhGSepLj
-k02TtpQglIqU9JVflcmWNJaflwqXdZfgmEyYuJkkmZCZ/JpomtWbQpuvnByciZz3nWSd0p5Anq6f
-HZ+Ln/qgaaDYoUehtqImopajBqN2o+akVqTHpTilqaYapoum/adup+CoUqjEqTepqaocqo+rAqt1
-q+msXKzQrUStuK4trqGvFq+LsACwdbDqsWCx1rJLssKzOLOutCW0nLUTtYq2AbZ5tvC3aLfguFm4
-0blKucK6O7q1uy67p7whvJu9Fb2Pvgq+hL7/v3q/9cBwwOzBZ8Hjwl/C28NYw9TEUcTOxUvFyMZG
-xsPHQce/yD3IvMk6ybnKOMq3yzbLtsw1zLXNNc21zjbOts83z7jQOdC60TzRvtI/0sHTRNPG1EnU
-y9VO1dHWVdbY11zX4Nhk2OjZbNnx2nba+9uA3AXcit0Q3ZbeHN6i3ynfr+A24L3hROHM4lPi2+Nj
-4+vkc+T85YTmDeaW5x/nqegy6LzpRunQ6lvq5etw6/vshu0R7ZzuKO6070DvzPBY8OXxcvH/8ozz
-GfOn9DT0wvVQ9d72bfb794r4Gfio+Tj5x/pX+uf7d/wH/Jj9Kf26/kv+3P9t////2wBDAAMCAgMC
-AgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIU
-FRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQU
-FBQUFBQUFBQUFBQUFBT/wAARCAAgACADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAEC
-AwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0Kx
-wRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1
-dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ
-2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QA
-tREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYk
-NOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaH
-iImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq
-8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD9Kfif8SNJ+EngjUPFeui4OlWLwrObWMPIolmSINtJGQDI
-Ccc4BwCcA8rbftG+F7y3iuLe31CeCVBJHLGkTK6kZDAiTBBHeuZ/bq/5NW8b/wDbj/6XW9fmzpP/
-ACC7P/rin/oIr9T4V4XwueYKVetJqSk18rRfl3Z8NxLnlfJfZypK/MfpTq37XHhmxvpbeFLVhGdj
-C71KKGRWHDKUG7GD716V8OfihoHxP0l7vRr+2uZ4Nq3drDOsj27HOA209Dg4bvg9CCB+S1fZf/BO
-v/moH/cP/wDbmvX4j4OwOV5XUxlBvmhy/O8ktfvvofO5DxVjMxzKGFrJcs7/ACsm/wBD1H9ur/k1
-bxv/ANuP/pdb1+bOk/8AILs/+uKf+giv2Y1DT7XVrC5sb62hvLK5iaGe2uIxJHLGwIZGU8MpBIIP
-BBrw/Rf2JPhRpLXQk0e81GGWTdDDdahMFtU7RxmNlJUDAy5ZuOWNeTwlxVg8jwlShioybcuZctne
-6Stq1ta+/wDwfoOKMjxOcqlHDNK173dvyTPzhr7L/wCCdf8AzUD/ALh//tzXsv8Awx58If8AoUf/
-ACpXn/x6uz+HXwd8IfCf+0P+EV0j+yv7Q8v7T/pM03mbN2z/AFjtjG9umOtepxDxpl+bZZVwVCE1
-KXLa6jbSSfST7djwMi4Sx2WZjSxdacHGN72bvrFrrFd+5//Z</Data></Thumbnail></Binary><mdDateSt Sync="TRUE">20220725</mdDateSt></metadata>
+<?xml version="1.0" encoding="UTF-8"?>
+<metadata xml:lang="en"><Esri><CreaDate>20231206</CreaDate><CreaTime>23391500</CreaTime><ArcGISFormat>1.0</ArcGISFormat><SyncOnce>TRUE</SyncOnce><ModDate>20231206</ModDate><ModTime>233915</ModTime></Esri><toolbox name="Speckle" alias="speckle_toolbox_"><arcToolboxHelpPath>c:\program files\arcgis\pro\Resources\Help\gp</arcToolboxHelpPath><toolsets/></toolbox><dataIdInfo><idCitation><resTitle>Speckle</resTitle></idCitation></dataIdInfo><distInfo><distributor><distorFormat><formatName>ArcToolbox Toolbox</formatName></distorFormat></distributor></distInfo></metadata>

speckle_toolbox/esri/toolboxes/speckle/converter/geometry/_init_.py

@@ -1,183 +0,0 @@
-
-from regex import F
-from specklepy.objects import Base
-from specklepy.objects.geometry import Line, Mesh, Point, Polyline, Curve, Arc, Circle, Polycurve, Ellipse 
-
-import arcpy 
-from typing import Any, List, Union, Sequence
-try:
-    from speckle.converter.geometry.polygon import polygonToNative, polygonToSpeckle, multiPolygonToSpeckle
-    from speckle.converter.geometry.polyline import arcToNative, ellipseToNative, circleToNative, curveToNative, lineToNative, polycurveToNative, polylineFromVerticesToSpeckle, polylineToNative, polylineToSpeckle
-    from speckle.converter.geometry.point import pointToCoord, pointToNative, pointToSpeckle, multiPointToSpeckle
-    from speckle.converter.geometry.polyline import speckleArcCircleToPoints, specklePolycurveToPoints, multiPolylineToSpeckle
-    from speckle.converter.geometry.mesh import meshToNative
-except:
-    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.polygon import polygonToNative, polygonToSpeckle, multiPolygonToSpeckle
-    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.polyline import arcToNative, ellipseToNative, circleToNative, curveToNative, lineToNative, polycurveToNative, polylineFromVerticesToSpeckle, polylineToNative, polylineToSpeckle
-    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.point import pointToCoord, pointToNative, pointToSpeckle, multiPointToSpeckle
-    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.polyline import speckleArcCircleToPoints, specklePolycurveToPoints, multiPolylineToSpeckle
-    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.mesh import meshToNative
-
-import numpy as np
-
-def convertToSpeckle(feature, index: str, layer, geomType, featureType) -> Union[Base, Sequence[Base], None]:
-    """Converts the provided layer feature to Speckle objects"""
-    print("___convertToSpeckle____________")
-    geom = feature
-    print(geom.isMultipart) # e.g. False 
-    print(geom.partCount)
-    geomMultiType = geom.isMultipart
-    hasCurves = feature.hasCurves 
-    
-    # feature is <geoprocessing describe geometry object object at 0x000002A75D6A4BD0>
-    
-    #print(featureType) # e.g. Simple 
-    #print(geomType) # e.g. Polygon 
-    #geomSingleType = (featureType=="Simple") # Simple,SimpleJunction,SimpleJunction,ComplexEdge,Annotation,CoverageAnnotation,Dimension,RasterCatalogItem 
-
-    if geomType == "Point": #Polygon, Point, Polyline, Multipoint, MultiPatch
-        for pt in geom:
-            return pointToSpeckle(pt, feature, layer)
-    elif geomType == "Polyline":
-        #if geom.hasCurves: 
-        #    geom, feature = curvesToSegments(geom, feature, layer, geomMultiType)
-        #    geomMultiType = geom.isMultipart
-        #    return polylineToSpeckle(geom, feature, layer, geomMultiType)
-        #else:
-        if geom.partCount > 1: return multiPolylineToSpeckle(geom, feature, layer, geomMultiType)
-        else: return polylineToSpeckle(geom, feature, layer, geomMultiType)
-    elif geomType == "Polygon":
-        if geom.partCount > 1: return multiPolygonToSpeckle(geom, feature, index, layer, geomMultiType)
-        else: return polygonToSpeckle(geom, feature, index, layer, geomMultiType)
-    elif geomType == "Multipoint":
-        return multiPointToSpeckle(geom, feature, layer, geomMultiType)
-    else:
-        arcpy.AddWarning("Unsupported or invalid geometry in layer " + layer.name)
-    return None
-
-
-def convertToNative(base: Base, sr: arcpy.SpatialReference) -> Union[Any, None]:
-    """Converts any given base object to QgsGeometry."""
-    print("___Convert to Native SingleType___")
-    #print(base)
-    converted = None
-    conversions = [
-        (Point, pointToNative),
-        (Line, lineToNative),
-        (Polyline, polylineToNative),
-        (Curve, curveToNative),
-        (Arc, arcToNative),
-        (Circle, circleToNative),
-        (Ellipse, ellipseToNative),
-        #(Mesh, meshToNative),
-        (Polycurve, polycurveToNative),
-        (Base, polygonToNative), # temporary solution for polygons (Speckle has no type Polygon yet)
-    ]
-
-    for conversion in conversions:
-        if isinstance(base, conversion[0]):
-            #print(conversion[0])
-            converted = conversion[1](base, sr)
-            break
-    #print(converted)
-    return converted
-
-def multiPointToNative(items: List[Point], sr: arcpy.SpatialReference):
-    print("___Create MultiPoint")
-    all_pts = []
-    # example https://pro.arcgis.com/en/pro-app/2.8/arcpy/classes/multipoint.htm
-    for item in items:
-        pt = pointToCoord(item) # [x, y, z]
-        all_pts.append( arcpy.Point(pt[0], pt[1], pt[2]) )
-    #print(all_pts)
-    features = arcpy.Multipoint( arcpy.Array(all_pts) )
-    #if len(features)==0: features = None
-    return features
-
-def multiPolylineToNative(items: List[Polyline], sr: arcpy.SpatialReference):
-    print("_______Drawing Multipolylines____")
-    #print(items)
-    poly = None
-    full_array_list = []
-    for item in items: # will be 1 item
-        pointsSpeckle = []
-        try: pointsSpeckle = item.as_points()
-        except: continue 
-        pts = [pointToCoord(pt) for pt in pointsSpeckle]
-
-        if item.closed is True: 
-            pts.append( pointToCoord(item.as_points()[0]) )
-        
-        arr = [arcpy.Point(*coords) for coords in pts]
-        full_array_list.append(arr)
-
-    poly = arcpy.Polyline( arcpy.Array(full_array_list), sr, has_z=True )
-    return poly
-
-def multiPolygonToNative(items: List[Base], sr: arcpy.SpatialReference): #TODO fix multi features
-    
-    print("_______Drawing Multipolygons____")
-    #print(items)
-    full_array_list = []
-
-    for item in items: # will be 1 item
-        #print(item)
-        #pts = [pointToCoord(pt) for pt in item["boundary"].as_points()]
-        pointsSpeckle = []
-        if isinstance(item["boundary"], Circle) or isinstance(item["boundary"], Arc): 
-            pointsSpeckle = speckleArcCircleToPoints(item["boundary"]) 
-        elif isinstance(item["boundary"], Polycurve): 
-            pointsSpeckle = specklePolycurveToPoints(item["boundary"]) 
-        elif isinstance(item["boundary"], Line): pass
-        else: 
-            try: pointsSpeckle = item["boundary"].as_points()
-            except: pass # if Line
-
-        pts = [pointToCoord(pt) for pt in pointsSpeckle]
-        #print(pts)
-
-        outer_arr = [arcpy.Point(*coords) for coords in pts]
-        outer_arr.append(outer_arr[0])
-        geomPart = []
-        try:
-            for void in item["voids"]: 
-                #print(void)
-                #pts = [pointToCoord(pt) for pt in void.as_points()]
-                pointsSpeckle = []
-                if isinstance(void, Circle) or isinstance(void, Arc): 
-                    pointsSpeckle = speckleArcCircleToPoints(void) 
-                elif isinstance(void, Polycurve): 
-                    pointsSpeckle = specklePolycurveToPoints(void) 
-                elif isinstance(void, Line): pass
-                else: 
-                    try: pointsSpeckle = void.as_points()
-                    except: pass # if Line
-                pts = [pointToCoord(pt) for pt in pointsSpeckle]
-
-                inner_arr = [arcpy.Point(*coords) for coords in pts]
-                inner_arr.append(inner_arr[0])
-                geomPart.append(arcpy.Array(inner_arr))
-        except:pass
-    
-        geomPart.insert(0, arcpy.Array(outer_arr))
-        full_array_list.extend(geomPart) # outlines are written one by one, with no separation to "parts"
-
-    geomPartArray = arcpy.Array(full_array_list)
-    polygon = arcpy.Polygon(geomPartArray, sr, has_z=True)
-    
-    print(polygon)
-    
-    return polygon
-
-def convertToNativeMulti(items: List[Base], sr: arcpy.SpatialReference): 
-    print("___Convert to Native MultiType___")
-    first = items[0]
-    if isinstance(first, Point):
-        return multiPointToNative(items, sr)
-    elif isinstance(first, Line) or isinstance(first, Polyline):
-        return multiPolylineToNative(items, sr)
-    elif isinstance(first, Base): 
-        try:
-            if first["boundary"] is not None and first["voids"] is not None:
-                return multiPolygonToNative(items, sr)
-        except: return None 

speckle_toolbox/esri/toolboxes/speckle/converter/geometry/mesh.py

@@ -1,76 +0,0 @@
-from typing import List
-import arcpy
-
-from specklepy.objects.geometry import Mesh
-
-import shapefile
-from shapefile import TRIANGLE_STRIP, TRIANGLE_FAN
-try: from speckle.converter.layers.utils import get_scale_factor
-except: from speckle_toolbox.esri.toolboxes.speckle.converter.layers.utils import get_scale_factor
-
-def meshToNative(meshes: List[Mesh], path: str):
-    """Converts a Speckle Mesh to MultiPatch"""
-    print("06___________________Mesh to Native")
-    #print(meshes)
-    #print(mesh.units)
-    w = shapefile.Writer(path) 
-    w.field('speckleTyp', 'C')
-
-    shapes = []
-    for geom in meshes:
-        
-        try: 
-            if geom.displayValue and isinstance(geom.displayValue, Mesh): 
-                mesh = geom.displayValue
-                w = fill_mesh_parts(w, mesh)
-            elif geom.displayValue and isinstance(geom.displayValue, List): 
-                for part in geom.displayValue:
-                    if isinstance(part, Mesh): 
-                        mesh = part
-                        w = fill_mesh_parts(w, mesh)
-        except: 
-            try: 
-                if geom.displayMesh and isinstance(geom.displayMesh, Mesh): 
-                    mesh = geom.displayMesh
-                    w = fill_mesh_parts(w, mesh)
-                elif geom.displayMesh and isinstance(geom.displayMesh, List): 
-                    for part in geom.displayMesh:
-                        if isinstance(part, Mesh): 
-                            mesh = part
-                            w = fill_mesh_parts(w, mesh)
-            except: pass
-    w.close()
-    return path
-
-def fill_mesh_parts(w: shapefile.Writer, mesh: Mesh):
-    scale = get_scale_factor(mesh.units)
-
-    parts_list = []
-    types_list = []
-    count = 0 # sequence of vertex (not of flat coord list) 
-    try:
-        #print(len(mesh.faces))
-        if len(mesh.faces) % 4 == 0 and (mesh.faces[0] == 0 or mesh.faces[0] == 3):
-            for f in mesh.faces:
-                try:
-                    if mesh.faces[count] == 0 or mesh.faces[count] == 3: # only handle triangles
-                        f1 = [ scale*mesh.vertices[mesh.faces[count+1]*3], scale*mesh.vertices[mesh.faces[count+1]*3+1], scale*mesh.vertices[mesh.faces[count+1]*3+2] ]
-                        f2 = [ scale*mesh.vertices[mesh.faces[(count+2)]*3], scale*mesh.vertices[mesh.faces[(count+2)]*3+1], scale*mesh.vertices[mesh.faces[(count+2)]*3+2] ]
-                        f3 = [ scale*mesh.vertices[mesh.faces[(count+3)]*3], scale*mesh.vertices[mesh.faces[(count+3)]*3+1], scale*mesh.vertices[mesh.faces[(count+3)]*3+2] ]
-                        parts_list.append([ f1, f2, f3 ])
-                        types_list.append(TRIANGLE_FAN)
-                        count += 4
-                    else: 
-                        count += mesh.faces[count+1]
-                except: break
-            w.multipatch(parts_list, partTypes=types_list ) # one type for each part
-            w.record('displayMesh')
-        else: print("not triangulated mesh")
-
-    except Exception as e: pass #; print(e)
-    return w
-    
-def rasterToMesh(vertices, faces, colors):
-    mesh = Mesh.create(vertices, faces, colors)
-    mesh.units = "m"
-    return mesh

speckle_toolbox/esri/toolboxes/speckle/converter/geometry/point.py

@@ -1,77 +0,0 @@
-import math
-from typing import List
-from specklepy.objects.geometry import Point
-import arcpy
-
-try: from speckle.converter.layers.utils import get_scale_factor
-except: from speckle_toolbox.esri.toolboxes.speckle.converter.layers.utils import get_scale_factor
-
-
-def multiPointToSpeckle(geom, feature, layer, multiType: bool):
-    """Converts a Point to Speckle"""
-    #try: 
-    #print("___Point to Speckle____")
-    #point = Point(units = "m")
-    pointList = []
-    #print(geom) # <geoprocessing describe geometry object object at 0x0000020F1D94AB10>
-    #print(multiType)
-
-    if multiType is False: 
-        for pt in geom:
-            #print(pt) # 284394.58100903 5710688.11602606 NaN NaN <class 'arcpy.arcobjects.arcobjects.Point'> 
-            #print(type(pt))
-            if pt != None: pointList.append(pointToSpeckle(pt, feature, layer)) 
-    return pointList
-
-def pointToSpeckle(pt, feature, layer):
-  
-    """Converts a Point to Speckle"""
-    #print("___Point to Speckle____")
-    # when unset, z() returns "nan"
-    #print(pt) # 4.9046319 52.3592043 NaN NaN
-    #print("____Point to Speckle___")
-    x = pt.X
-    y = pt.Y
-    if pt.Z: z = pt.Z 
-    else: z = 0
-    specklePoint = Point(units = "m")
-    specklePoint.x = x
-    specklePoint.y = y
-    specklePoint.z = z
-    '''
-    if feature is not None and layer is not None: # can be if it's a point from raster layer 
-        col = featureColorfromNativeRenderer(feature, layer)
-        specklePoint['displayStyle'] = {}
-        specklePoint['displayStyle']['color'] = col
-    '''
-    #print(specklePoint)
-    return specklePoint
-
-def pointToNative(pt: Point, sr: arcpy.SpatialReference) -> arcpy.PointGeometry:
-    """Converts a Speckle Point to QgsPoint"""
-    #print("___pointToNative__")
-    #print(pt)
-    pt = scalePointToNative(pt, pt.units)
-    geom = arcpy.PointGeometry(arcpy.Point(pt.x, pt.y, pt.z), sr, has_z = True)
-    #print(geom)
-    return geom
-
-def pointToCoord(point: Point) -> List[float]:
-    """Converts a Speckle Point to QgsPoint"""
-    pt = scalePointToNative(point, point.units)
-    coords = [pt.x, pt.y, pt.z]
-    #print(coords)
-    return coords
-
-def scalePointToNative(point: Point, units: str) -> Point:
-    """Scale point coordinates to meters"""
-    scaleFactor = get_scale_factor(units)
-    pt = Point(units = "m")
-    pt.x = point.x * scaleFactor
-    pt.y = point.y * scaleFactor
-    pt.z = 0 if math.isnan(point.z) else point.z * scaleFactor
-    return pt
-
-def addZtoPoint(coords: List): 
-    if len(coords) == 2: coords.append(0)
-    return coords

speckle_toolbox/esri/toolboxes/speckle/converter/geometry/polygon.py

@@ -1,273 +0,0 @@
-from typing import Sequence
-import arcpy 
-import json
-from arcpy.arcobjects.arcobjects import SpatialReference 
-
-from specklepy.objects import Base
-from specklepy.objects.geometry import Point, Arc, Circle, Polycurve, Polyline, Line
-
-try:
-    from speckle.converter.geometry.mesh import rasterToMesh
-    from speckle.converter.geometry.point import pointToCoord, pointToNative
-    from speckle.converter.layers.symbologyTemplates import featureColorfromNativeRenderer
-    from speckle.converter.geometry.polyline import (polylineFromVerticesToSpeckle, 
-                                                    circleToSpeckle, 
-                                                    speckleArcCircleToPoints, 
-                                                    curveToSpeckle, 
-                                                    specklePolycurveToPoints
-                                                    )
-except: 
-    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.mesh import rasterToMesh
-    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.point import pointToCoord, pointToNative
-    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.symbologyTemplates import featureColorfromNativeRenderer
-    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.polyline import (polylineFromVerticesToSpeckle, 
-                                                    circleToSpeckle, 
-                                                    speckleArcCircleToPoints, 
-                                                    curveToSpeckle, 
-                                                    specklePolycurveToPoints
-                                                    )
-
-import math
-from panda3d.core import Triangulator
-
-
-
-def multiPolygonToSpeckle(geom, feature, index: str, layer, multiType: bool):
-
-    print("___MultiPolygon to Speckle____")
-    polygon = []
-    #print(enumerate(geom.getPart())) # this method ignores curvature and voids
-    #print(json.loads(geom.JSON))
-    #js = json.loads(geom.JSON)['rings']
-    #https://desktop.arcgis.com/en/arcmap/latest/analyze/python/reading-geometries.htm
-    for i,x in enumerate(feature): # [[x,x,x]
-        print("Part # " + str(i+1))
-        print(x)
-        boundaryFinished = 0
-        arrBoundary = []
-        arrInnerRings = []
-        for ptn in x: # arcpy.Point
-            if ptn is None: 
-                boundaryFinished += 1
-                arrInnerRings.append([]) # start of new Inner Ring
-            elif boundaryFinished == 0 and ptn is not None: 
-                arrBoundary.append(ptn)
-            elif boundaryFinished == 1 and ptn is not None: 
-                arrInnerRings[len(arrInnerRings)-1].append(ptn)
-
-        full_arr = [arrBoundary] + arrInnerRings
-        #print(full_arr)
-        poly = arcpy.Polygon(arcpy.Array(full_arr), arcpy.Describe(layer.dataSource).SpatialReference, has_z = True)
-        #print(poly) #<geoprocessing describe geometry object object at 0x000002B2D3E338D0> 
-        polygon.append(polygonToSpeckle(poly, feature, index, layer, poly.isMultipart))
-
-    return polygon
-
-
-def polygonToSpeckle(geom, feature, index: int, layer, multiType: bool):
-    """Converts a Polygon to Speckle"""
-    #try: 
-    print("___Polygon to Speckle____")
-    print(geom)
-    polygon = Base(units = "m")
-    pointList = []
-    voidPointList = []
-    voids = []
-    boundary = None
-    data = arcpy.Describe(layer.dataSource)
-    sr = data.spatialReference
-
-    print(multiType)
-    partsBoundaries = []
-    partsVoids = []
-    
-        
-    #else: # no curves
-    if multiType is False: # Multipolygon
-        if geom.hasCurves: 
-            print("has curves")
-            # geometry SHAPE@ tokens: https://pro.arcgis.com/en/pro-app/latest/arcpy/get-started/reading-geometries.htm
-            print(geom.JSON) 
-            boundary = curveToSpeckle(geom, "Polygon", feature, layer)
-        else: 
-            print("no curves")
-            for p in geom:
-                for pt in p: 
-                    if pt != None: pointList.append(pt) 
-            boundary = polylineFromVerticesToSpeckle(pointList, True, feature, layer) 
-        partsBoundaries.append(boundary)
-        partsVoids.append([])
-
-    else: 
-        print("multi type")
-        for i, p in enumerate(geom):
-            print(p)
-            for pt in p:  
-                #print(pt) # 284394.58100903 5710688.11602606 NaN NaN
-                if pt == None and boundary == None:  # first break 
-                    boundary = polylineFromVerticesToSpeckle(pointList, True, feature, layer) 
-                    pointList = []
-                elif pt == None and boundary != None: # breaks btw voids
-                    void = polylineFromVerticesToSpeckle(pointList, True, feature, layer)
-                    voids.append(void)
-                    pointList = []
-                elif pt != None: # add points to whatever list (boundary or void) 
-                    pointList.append(pt)
-
-            if boundary != None and len(pointList)>0: # remaining polyline
-                void = polylineFromVerticesToSpeckle(pointList, True, feature, layer)
-                voids.append(void)
-
-            #partsBoundaries.append(boundary)
-            #partsVoids.append(local_voids)
-
-    if boundary is None: return None 
-    polygon.boundary = boundary
-    polygon.voids = voids
-    polygon.displayValue = [ boundary ] + voids
-    #print(boundary)
-
-    ############# mesh 
-    vertices = []
-    polyBorder = []
-    total_vertices = 0
-    if isinstance(boundary, Circle) or isinstance(boundary, Arc): 
-        polyBorder = speckleArcCircleToPoints(boundary) 
-    elif isinstance(boundary, Polycurve): 
-        polyBorder = specklePolycurveToPoints(boundary) 
-        #polygon.boundary.displayValue.closed = True 
-    elif isinstance(boundary, Line): pass
-    elif isinstance(boundary, Polyline): 
-        try: polyBorder = boundary.as_points()
-        except: pass # if Line
-    #print(polyBorder)
-    
-
-    if len(polyBorder)>2: # at least 3 points 
-        print("make meshes from polygons")
-        if len(voids) == 0: # if there is a mesh with no voids
-            for pt in polyBorder:
-                if isinstance(pt, Point): pt = pointToNative(pt, sr).getPart() # SR unknown
-                x = pt.X
-                y = pt.Y
-                z = 0 if math.isnan(pt.Z) else pt.Z
-                vertices.extend([x, y, z])
-                total_vertices += 1
-            #print(vertices)
-            ran = range(0, total_vertices)
-            faces = [total_vertices]
-            faces.extend([i for i in ran])
-            #print(faces)
-            # else: https://docs.panda3d.org/1.10/python/reference/panda3d.core.Triangulator
-        else:
-            trianglator = Triangulator()
-            faces = []
-
-            # add boundary points
-            #polyBorder = boundary.as_points()
-            pt_count = 0
-            # add extra middle point for border
-            for pt in polyBorder:
-              if pt_count < len(polyBorder)-1: 
-                  pt2 = polyBorder[pt_count+1]
-              else: pt2 = polyBorder[0]
-              
-              trianglator.addPolygonVertex(trianglator.addVertex(pt.x, pt.y))
-              vertices.extend([pt.x, pt.y, pt.z])
-
-              #trianglator.addPolygonVertex(trianglator.addVertex((pt.x+pt2.x)/4*3, (pt.y+pt2.y)/4*3))
-              #vertices.extend([(pt.x+pt2.x)/4*3, (pt.y+pt2.y)/4*3, (pt.z+pt2.z)/4*3])
-
-              trianglator.addPolygonVertex(trianglator.addVertex((pt.x+pt2.x)/2, (pt.y+pt2.y)/2))
-              vertices.extend([(pt.x+pt2.x)/2, (pt.y+pt2.y)/2, (pt.z+pt2.z)/2])
-              
-              #trianglator.addPolygonVertex(trianglator.addVertex((pt.x+pt2.x)/4, (pt.y+pt2.y)/4))
-              #vertices.extend([(pt.x+pt2.x)/4, (pt.y+pt2.y)/4, (pt.z+pt2.z)/4])
-
-              total_vertices += 2
-              pt_count += 1
-
-            #add void points
-            for i in range(len(voids)):
-              trianglator.beginHole()
-
-
-              pts = []
-              if isinstance(voids[i], Circle) or isinstance(voids[i], Arc): 
-                  pts = speckleArcCircleToPoints(voids[i]) 
-              elif isinstance(voids[i], Polycurve): 
-                  pts = specklePolycurveToPoints(voids[i]) 
-              elif isinstance(voids[i], Line): pass
-              else: 
-                  try: pts = voids[i].as_points()
-                  except: pass # if Line
-              #pts = voids[i].as_points()
-              for pt in pts:
-                trianglator.addHoleVertex(trianglator.addVertex(pt.x, pt.y))
-                vertices.extend([pt.x, pt.y, pt.z])
-                total_vertices += 1
-
-            trianglator.triangulate()
-            i = 0
-            while i < trianglator.getNumTriangles():
-              tr = [trianglator.getTriangleV0(i),trianglator.getTriangleV1(i),trianglator.getTriangleV2(i)]
-              faces.extend([3, tr[0], tr[1], tr[2]])
-              i+=1
-            ran = range(0, total_vertices)
-        
-        #print(polygon)
-        col = featureColorfromNativeRenderer(index, layer)
-        colors = [col for i in ran] # apply same color for all vertices
-        mesh = rasterToMesh(vertices, faces, colors)
-        polygon.displayValue = mesh 
-    #print("print resulted polygon")
-    #print(polygon)
-    return polygon
-
-def polygonToNative(poly: Base, sr: arcpy.SpatialReference) -> arcpy.Polygon:
-    """Converts a Speckle Polygon base object to QgsPolygon.
-    This object must have a 'boundary' and 'voids' properties.
-    Each being a Speckle Polyline and List of polylines respectively."""
-
-    print("_______Drawing polygons____")
-    #pts = [pointToCoord(pt) for pt in poly["boundary"].as_points()]
-    pointsSpeckle = []
-    if isinstance(poly["boundary"], Circle) or isinstance(poly["boundary"], Arc): 
-        pointsSpeckle = speckleArcCircleToPoints(poly["boundary"]) 
-    elif isinstance(poly["boundary"], Polycurve): 
-        pointsSpeckle = specklePolycurveToPoints(poly["boundary"]) 
-    elif isinstance(poly["boundary"], Line): pass
-    else: 
-        try: pointsSpeckle = poly["boundary"].as_points()
-        except: pass # if Line
-
-    pts = [pointToCoord(pt) for pt in pointsSpeckle]
-    #print(pts)
-
-    outer_arr = [arcpy.Point(*coords) for coords in pts]
-    outer_arr.append(outer_arr[0])
-    geomPart = []
-    try:
-        for void in poly["voids"]: 
-            #print(void)
-            #pts = [pointToCoord(pt) for pt in void.as_points()]
-            pointsSpeckle = []
-            if isinstance(void, Circle) or isinstance(void, Arc): 
-                pointsSpeckle = speckleArcCircleToPoints(void) 
-            elif isinstance(void, Polycurve): 
-                pointsSpeckle = specklePolycurveToPoints(void) 
-            elif isinstance(void, Line): pass
-            else: 
-                try: pointsSpeckle = void.as_points()
-                except: pass # if Line
-            pts = [pointToCoord(pt) for pt in pointsSpeckle]
-
-            inner_arr = [arcpy.Point(*coords) for coords in pts]
-            inner_arr.append(inner_arr[0])
-            geomPart.append(arcpy.Array(inner_arr))
-    except:pass
-    geomPart.insert(0, outer_arr)
-    geomPartArray = arcpy.Array(geomPart)
-    polygon = arcpy.Polygon(geomPartArray, sr, has_z=True)
-
-    return polygon

speckle_toolbox/esri/toolboxes/speckle/converter/geometry/polyline.py

@@ -1,605 +0,0 @@
-
-from math import atan, cos, sin
-import math
-import json 
-from typing import List, Union, Tuple
-from specklepy.objects import Base 
-from specklepy.objects.geometry import Box, Vector, Point, Line, Polyline, Curve, Ellipse, Arc, Circle, Polycurve, Plane, Interval  
-import arcpy 
-import numpy as np
-
-try:
-    from speckle.converter.geometry.point import pointToCoord, pointToSpeckle, addZtoPoint
-    from speckle.converter.layers.utils import get_scale_factor
-except:
-    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.point import pointToCoord, pointToSpeckle, addZtoPoint
-    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.utils import get_scale_factor
-
-
-def circleToSpeckle(center, point):
-    print("___Circle to Speckle____")
-    rad = math.sqrt(math.pow((center[0] - point[0]),2) + math.pow((center[1] - point[1]),2) )
-    #print(rad)
-    if len(center)>2: center_z = center[2]
-    else: center_z = 0
-    length = rad*2*math.pi
-    domain = [0, length] 
-    plane = [center[0], center[1], center_z, 0,0,1, 1,0,0, 0,1,0] 
-    units = 3 #"m" 
-
-    args = [0] + [rad] + domain + plane + [units] 
-    #print(args) 
-    c = Circle.from_list(args)
-    c.plane.origin.units = "m"
-    c.units = "m"
-    #print(c)
-    return c 
-
-def multiPolylineToSpeckle(geom, feature, layer, multiType: bool):
-
-    print("___MultiPolyline to Speckle____")
-    polyline = []
-    print(enumerate(geom.getPart()))
-    for i,x in enumerate(geom.getPart()):
-        poly = arcpy.Polyline(x, arcpy.Describe(layer.dataSource).SpatialReference, has_z = True)
-        print(poly)
-        polyline.append(polylineToSpeckle(poly, feature, layer, poly.isMultipart))
-
-    return polyline
-
-def polylineToSpeckle(geom, feature, layer, multiType: bool):
-    print("___Polyline to Speckle____")
-    polyline = None
-    pointList = []
-    print(geom.hasCurves) 
-
-    if multiType is False: 
-        if geom.hasCurves: 
-            print("has curves")
-            # geometry SHAPE@ tokens: https://pro.arcgis.com/en/pro-app/latest/arcpy/get-started/reading-geometries.htm
-            print(geom.JSON) 
-            polyline = curveToSpeckle(geom, "Polyline", feature, layer)
-        else:
-            for p in geom: 
-                for pt in p: 
-                    if pt != None: pointList.append(pt)#; print(pt.Z)
-            closed = False
-            if pointList[0] == pointList[len(pointList)-1]: 
-                closed = True
-                pointList = pointList[:-1]
-            polyline = polylineFromVerticesToSpeckle(pointList, closed, feature, layer) 
-    return polyline
-
-def polylineFromVerticesToSpeckle(vertices: List[Point], closed: bool, feature, layer) -> Polyline:
-    """Converts a Polyline to Speckle"""
-    
-    print("___Polyline from vertices to Speckle____")
-    
-    if isinstance(vertices, list): 
-        if len(vertices) > 0 and isinstance(vertices[0], Point):
-            specklePts = vertices
-        else: specklePts = [pointToSpeckle(pt, feature, layer) for pt in vertices] #breaks unexplainably
-    #elif isinstance(vertices, QgsVertexIterator):
-    #    specklePts = [pointToSpeckle(pt, feature, layer) for pt in vertices]
-    else: return None
-
-    specklePts = []
-    for pt in vertices:
-        newPt = pointToSpeckle(pt, feature, layer) 
-        specklePts.append(newPt)
-
-    # TODO: Replace with `from_points` function when fix is pushed.
-    polyline = Polyline(units = "m")
-    polyline.value = []
-    polyline.closed = closed
-    polyline.units = specklePts[0].units
-    for i, point in enumerate(specklePts):
-        if closed and i == len(specklePts) - 1 and specklePts[0] == point: 
-            continue # if we consider the last pt, do not add is coincides with the first (and type is Closed) 
-        polyline.value.extend([point.x, point.y, point.z])
-
-    return polyline
-
-def arc3ptToSpeckle(p0: List, p1: List, p2: List, feature, layer) -> Arc: 
-    print("____arc 3pt to Speckle___")
-    p0 = addZtoPoint(p0)
-    p1 = addZtoPoint(p1)
-    p2 = addZtoPoint(p2)
-    arc = Arc()
-    arc.startPoint = pointToSpeckle(arcpy.Point(*p0), feature, layer)
-    arc.midPoint = pointToSpeckle(arcpy.Point(*p1), feature, layer)
-    arc.endPoint = pointToSpeckle(arcpy.Point(*p2), feature, layer)
-    center, radius = getArcCenter(Point.from_list(p0), Point.from_list(p1), Point.from_list(p2))
-    arc.plane = Plane() #.from_list(Point(), Vector(Point(0, 0, 1)), Vector(Point(0,1,0)), Vector(Point(-1,0,0)))
-    arc.plane.origin = Point.from_list(center)
-    arc.plane.origin.units = "m" 
-    arc.units = "m"
-    arc.angleRadians, startAngle, endAngle = getArcRadianAngle(arc)
-
-    arc.radius = radius
-    
-    arc.plane.normal = getArcNormal(arc, arc.midPoint)
-
-    #arc.angleRadians = abs(angle1 + angle2)
-    #print(arc.angleRadians)
-
-    #col = featureColorfromNativeRenderer(feature, layer)
-    #arc['displayStyle'] = {}
-    #arc['displayStyle']['color'] = col
-
-    return arc
-
-def curveBezierToSpeckle(segmStartCoord, segmEndCoord, knots, feature, layer): 
-    print("____bezier curve to Speckle____")
-    degree = 3
-    points = [
-        tuple(knots[0]), tuple(segmStartCoord), tuple(knots[1]), tuple(segmEndCoord)
-    ] #[segmStartCoord, *coords]
-    print(points)
-    num_points = len(points) #2
-
-    knot_count = num_points + degree - 1 #4
-    knots = [0] * knot_count
-    print(knots)
-    for i in range(1, len(knots)):
-        knots[i] = i // 3
-        print(knots[i])
-
-    length = 1 #spline.calc_length()
-    domain = Interval(start=0, end=length, totalChildrenCount=0)
-    points = [tuple(pt) for pt in points]
-    curve = Curve(
-        degree = degree,
-        closed = False, 
-        periodic= True if (segmStartCoord == segmEndCoord) else False,
-        points= list(sum(points, ())),  # magic (flatten list of tuples)
-        weights=[1] * num_points,
-        knots=knots,
-        rational=False,
-        area=0,
-        volume=0,
-        length=length,
-        domain=domain,
-        units="m",
-        bbox=Box(area=0.0, volume=0.0),
-    )
-    print(curve) 
-    return curve
-
-
-def curveToSpeckle(geom, geomType, feature, layer) -> Union[Circle, Arc, Polyline, Polycurve]:
-    print("____curve to Speckle____")
-    print(geomType)
-    # look for "curvePaths" or "curveRings"[[ (startPt, {arcs, beziers etc}, optional(endPt))],[],...], "rings" 
-    # examples: https://developers.arcgis.com/documentation/common-data-types/geometry-objects.htm
-    # e.g. {"hasZ":true,
-    # "curveRings":[[[631307.05960000027,5803698.4477999993,0],{"a":[[631307.05960000027,5803698.4477999993,0],[631307.05960000027,5803414.92656173],0,1]}]],
-    # "spatialReference":{"wkid":32631,"latestWkid":32631}}
-    
-    # b - bezier curve (endPt, controlPts) 
-    # a - elliptical arc (endPt, centralPt) e.g. for circle: [[[631307.05960000027,5803698.4477999993,0],{"a":[[631307.05960000027,5803698.4477999993,0],[631307.05960000027,5803414.92656173],0,1]}]]
-    # c - circular arc (endPt, throughPt) e.g. [[[633242.45179999992,5803058.0354999993,0],{"c":[[633718.26040000003,5803496.4210000001,0],[633337.75764975848,5803431.9997026781]]},[633242.45179999992,5803058.0354999993,0]]]
-    
-    boundary = Polycurve(units = "m")
-    if geomType == "Polyline": boundary.closed = False
-    else: boundary.closed = True 
-    segments = [] 
-
-    for key, val in json.loads(geom.JSON).items(): 
-        print(key)
-        if key == "curveRings" or key == "curvePaths": 
-            
-            #boundary.closed = True
-            includesLines = 0
-
-            for segm in val: # segm: List
-                print(segm) #e.g. [[631307.05960000027,5803698.4477999993,0], {"a":[[631307.05960000027,5803698.4477999993,0],[631307.05960000027,5803414.92656173],0,1]}] 
-                segmStartCoord: List = addZtoPoint(segm[0])
-                
-                # go through all elements (points, a, c, ...)
-                for k in range(1, len(segm)):
-                    # e.g. one from the list: "curveRings":[[[631750.87200000044,5803159.6126000006,0],
-                    # {"c":[[632429.8348000003,5803507.1132999994,0],[631988.22772700491,5803532.9008129537]]},
-                    # {"c":[[632590.21970000025,5803127.5355999991,0],[633018.51899157302,5803532.1801161235]]},
-                    # [631750.87200000044,5803159.6126000006,0]]]
-
-                    # if previous segments exist 
-                    if len(segments) > 0:  
-                            segmOldData = segm[k-1] 
-                            if isinstance(segmOldData, dict):  # get "end point" of previous segment 
-                                for key3, val3 in segmOldData.items(): 
-                                    segmStartCoord: List = addZtoPoint(val2[0]) 
-                            elif isinstance(segmOldData, list) and isinstance(segmOldData[0], float): 
-                                  segmStartCoord: List = segmOldData 
-                    segmStartCoord = addZtoPoint(segmStartCoord)
-
-                    if isinstance(segm[k], dict): 
-                        for key2, val2 in segm[k].items():
-                            if key2 == "a": # elliptical arc (endPt, centralPt)
-                                # e.g. {'a': [[633883.1035000002, 5802972.5812, 0], [634028.3379278888, 5802908.342895357], 0, 1, 1.1543577096027686, 473.59966687227444, 0.33531864204900685]}
-                                segmEndCoord = addZtoPoint(val2[0]) # [631307.05960000027,5803698.4477999993,0]
-                                segmCenter = addZtoPoint(val2[1]) # [631307.05960000027,5803414.92656173]
-                                
-                                if segmStartCoord == segmEndCoord: 
-                                    if len(val2) == 4: 
-                                        print("full circle") 
-                                        #boundary.closed = True 
-                                        segmentLocal = circleToSpeckle(segmCenter, segmEndCoord)
-                                        segments.append(segmentLocal)
-                                        lastPt = segmEndCoord 
-                                        print("segmentLocal:")
-                                        print(segmentLocal)
-                                        print(segmStartCoord)
-                                        print(segmEndCoord)
-                                    else: # ellipse
-                                        arcpy.AddMessage("SpeckleWarning: ellipse geometry not supported yet") 
-                                        segments = []
-                                        break
-                                else: # elliptical curve
-                                    arcpy.AddMessage("SpeckleWarning: ellipse geometry not supported yet") 
-                                    segments = []
-                                    break
-
-                            if key2 == "c": # circular arc (endPt, throughPt) 
-                                
-                                segmEndCoord: List = addZtoPoint(val2[0]) # [633718.26040000003,5803496.4210000001,0] 
-                                segmThrough: List = addZtoPoint(val2[1]) # [633337.7576497585, 5803431.999702678] 
-
-                                segmentLocal = arc3ptToSpeckle(segmStartCoord, segmThrough, segmEndCoord, feature, layer) 
-                                segments.append(segmentLocal)
-                                print("segmentLocal:")
-                                print(segmentLocal)
-                                print(segmStartCoord)
-                                print(segmEndCoord)
-                                lastPt = segmEndCoord 
-
-                            if key2 == "b": # bezier curve (endPt, controlPts) 
-                                arcpy.AddMessage("SpeckleWarning: bezier curve geometry not supported yet") 
-                                segments = []
-                                break
-                                r'''
-                                segmEndCoord: List = addZtoPoint(val2[0])  # [633718.26040000003,5803496.4210000001,0] 
-                                #segmThrough: List = val2[1] # [633337.7576497585, 5803431.999702678] 
-                                coords = val2[1:]
-                                segmentLocal = curveBezierToSpeckle(segmStartCoord, segmEndCoord, coords, feature, layer) 
-                                segments.append(segmentLocal)
-                                print("segmentLocal:")
-                                print(segmentLocal)
-                                print(segmStartCoord)
-                                print(segmEndCoord)
-                                
-                                lastPt = segmEndCoord 
-                                '''
-                    
-                    elif isinstance(segm[k], list) and isinstance(segm[k][0], float): # add line to point 
-                        print("add line")
-                        segm[k] = addZtoPoint(segm[k]) 
-                        segmentLocal = lineFrom2pt(segmStartCoord, segm[k])
-                        includesLines = 1
-                        segments.append(segmentLocal) 
-                        lastPt = segm[k]
-
-                        print("segmentLocal:")
-                        print(segmentLocal)
-                        print(segmentLocal.start)
-                        print(segmentLocal.end)
-
-                    # for the last point
-                    if k == len(segm)-1 and isinstance(segm[k], list): 
-                        print("last element is a point (adding line)")
-                        lastPt = addZtoPoint(lastPt)
-                        if lastPt != segm[0]: 
-                            #segmentLocal = lineFrom2pt(lastPt, segm[0])
-                            #segments.append(segmentLocal) 
-                            #includesLines = 1
-                            #print("segmentLocal:")
-                            #print(segmentLocal)
-                            #print(segmentLocal.start)
-                            #print(segmentLocal.end)
-                            boundary.closed = True 
-                    #pts = speckleArcCircleToPoints(segmentLocal)
-                    #pts.append(segm[k])
-                    #arcgisPts = [arcpy.Point(pt[0], pt[1], pt[2]) for pt in pts]
-                    #segmentLocal = polylineFromVerticesToSpeckle(arcgisPts, True, feature, layer)
-    
-    boundary.segments = segments
-    print(segments)
-
-    if len(segments) == 1:
-        boundary = segments[0]
-        #if isinstance(boundary, Arc) or isinstance(boundary, Circle): 
-        #    boundary.displayValue = Polyline.from_points(speckleArcCircleToPoints(boundary)) 
-        
-    elif len(segments) > 1: # and includesLines == 0:
-        #boundary.displayValue = Polyline.from_points(specklePolycurveToPoints(boundary)) 
-        pass
-        #boundary.closed = True 
-    #elif len(segments) > 1 and includesLines == 1: 
-    #    print("includes lines!")
-    #    points = specklePolycurveToPoints(boundary) 
-    #    boundary = Polyline.from_points(points)
-    else: return None
-
-    #boundary.displayValue = Polyline.from_points(specklePolycurveToPoints(boundary)) 
-    
-    print(boundary)  
-    return boundary 
-
-
-
-def lineFrom2pt(pt1: List[float], pt2: List[float]): 
-    pt1 = addZtoPoint(pt1)
-    pt2 = addZtoPoint(pt2)
-    dist = math.sqrt( math.pow((pt2[0] - pt1[0]), 2) + math.pow((pt2[1] - pt1[1]), 2) + math.pow((pt2[2] - pt1[2]), 2) ) 
-    print(dist) 
-    domain = [0, dist, 0, 0] 
-    line = Line(units = "m" )#.from_list([*pt1, *pt2, *domain]) 
-    line.start = Point.from_list(pt1)
-    line.end = Point.from_list(pt2)
-    line.start.units = line.end.units = "m" 
-    return line 
-
-def polylineToNative(poly: Polyline, sr: arcpy.SpatialReference) -> arcpy.Polyline:
-    """Converts a Speckle Polyline to QgsLineString"""
-    print("__ convert polyline to native __")
-
-    if isinstance(poly, Polycurve): 
-        poly = specklePolycurveToPoints(poly)
-    if isinstance(poly, Arc) or isinstance(poly, Circle): 
-        try: poly = poly["displayValue"]
-        except: poly = speckleArcCircleToPoints(poly)
-        
-    if isinstance(poly, list): pts = [pointToCoord(pt) for pt in poly]
-    else: pts = [pointToCoord(pt) for pt in poly.as_points()]
-
-    if poly.closed is True: 
-        pts.append( pointToCoord(poly.as_points()[0]) )
-
-    pts_coord_list = [arcpy.Point(*coords) for coords in pts]
-    polyline = arcpy.Polyline( arcpy.Array(pts_coord_list), sr, has_z=True )
-    #print(polyline.JSON)
-    return polyline
-
-
-def lineToNative(line: Line, sr: arcpy.SpatialReference) -> arcpy.Polyline:
-    """Converts a Speckle Line to Native"""
-    print("___Line to Native___")
-    pts = [pointToCoord(pt) for pt in [line.start, line.end]]
-    line = arcpy.Polyline( arcpy.Array([arcpy.Point(*coords) for coords in pts]), sr , has_z=True)
-    return line
-
-def curveToNative(poly: Curve, sr: arcpy.SpatialReference) -> arcpy.Polyline:
-    """Converts a Speckle Curve to Native"""
-    display = poly.displayValue
-    curve = polylineToNative(display, sr) 
-    return curve
-
-def arcToNative(poly: Arc, sr: arcpy.SpatialReference) -> arcpy.Polyline:
-    """Converts a Speckle Arc to Native"""
-    arc = arcToNativePolyline(poly, sr) #QgsCircularString(pointToNative(poly.startPoint), pointToNative(poly.midPoint), pointToNative(poly.endPoint))
-    return arc
-
-def ellipseToNative():
-    return
-
-def circleToNative(poly: Circle, sr: arcpy.SpatialReference) -> arcpy.Polyline:
-    """Converts a Speckle Circle to QgsLineString"""
-    print("___Convert Circle to Native___")
-    points = []
-    angle1 = math.pi/2
-    
-    pointsNum = math.floor(math.pi*2) * 12 
-    if pointsNum <4: pointsNum = 4
-    points.append(pointToCoord(poly.plane.origin))
-
-    radScaled = poly.radius * get_scale_factor(poly.units)
-    points[0][1] += radScaled
-
-    for i in range(1, pointsNum + 1): 
-        k = i/pointsNum # to reset values from 1/10 to 1
-        if poly.plane.normal.z == 0: normal = 1
-        else: normal = poly.plane.normal.z
-        angle = angle1 + k * math.pi*2 * normal
-        pt = Point( x = poly.plane.origin.x * get_scale_factor(poly.units) + radScaled * cos(angle), y = poly.plane.origin.y * get_scale_factor(poly.units) + radScaled * sin(angle), z = 0) 
-        print(pt)
-        pt.units = "m"
-        points.append(pointToCoord(pt))
-    points.append(points[0])
-    curve = arcpy.Polyline( arcpy.Array([arcpy.Point(*coords) for coords in points]), sr , has_z=True)
-    return curve
-
-def polycurveToNative(poly: Polycurve, sr: arcpy.SpatialReference) -> arcpy.Polyline:
-    points = []
-    curve = None
-    print("___Polycurve to native___")
-
-    try: 
-        for i, segm in enumerate(poly.segments): # Line, Polyline, Curve, Arc, Circle
-            print("___start segment")            
-            if isinstance(segm,Line):  converted = lineToNative(segm, sr) # QgsLineString
-            elif isinstance(segm,Polyline):  converted = polylineToNative(segm, sr) # QgsLineString
-            elif isinstance(segm,Curve):  converted = curveToNative(segm, sr) # QgsLineString
-            elif isinstance(segm,Circle):  converted = circleToNative(segm, sr) # QgsLineString
-            elif isinstance(segm,Arc):  converted = arcToNativePolyline(segm, sr) # QgsLineString
-            else: # either return a part of the curve, of skip this segment and try next
-                arcpy.AddWarning(f"Part of the polycurve cannot be converted")
-                curve = arcpy.Polyline( arcpy.Array([arcpy.Point(*coords) for coords in points]), sr , has_z=True)
-                return curve
-            if converted is not None: 
-                #print(converted) # <geoprocessing describe geometry object object at 0x000002B2D3E338D0>
-                for part in converted:
-                    #print("Part: ")
-                    #print(part) # <geoprocessing array object object at 0x000002B2D2E09530>
-                    for pt in part: 
-                        #print(pt) # 64.4584221540162 5.5 NaN NaN
-                        if pt.Z != None: pt_z = pt.Z
-                        else: pt_z = 0
-                        #print(pt_z)
-                        #print(len(points)) 
-                        if len(points)>0 and pt.X == points[len(points)-1][0] and pt.Y == points[len(points)-1][1] and pt_z == points[len(points)-1][2]: pass
-                        else: points.append(pointToCoord(Point(x=pt.X, y = pt.Y, z = pt_z, units = "m"))) # e.g. [[64.4584221540162, 5.499999999999999, 0.0], [64.45461685210796, 5.587155742747657, 0.0]]
-            else:
-                arcpy.AddWarning(f"Part of the polycurve cannot be converted")
-                curve = arcpy.Polyline( arcpy.Array([arcpy.Point(*coords) for coords in points]), sr, has_z=True )
-                return curve
-    except: curve = None
-    #print(curve)
-    
-    curve = arcpy.Polyline( arcpy.Array([arcpy.Point(*coords) for coords in points]), sr, has_z=True )
-    return curve
-
-def arcToNativePolyline(poly: Union[Arc, Circle], sr: arcpy.SpatialReference):
-    print("__Arc/Circle to native polyline__")
-    pointsSpeckle = speckleArcCircleToPoints(poly)
-    points = [pointToCoord(p) for p in pointsSpeckle]
-    curve = arcpy.Polyline( arcpy.Array([arcpy.Point(*coords) for coords in points]), sr, has_z=True )
-    return curve
-
-
-
-def specklePolycurveToPoints(poly: Polycurve) -> List[Point]:
-    print("_____Speckle Polycurve to points____")
-    points = []
-    for segm in poly.segments:
-        #print(segm)
-        pts = []
-        if isinstance(segm, Arc) or isinstance(segm, Circle): # or isinstance(segm, Curve):
-            print("Arc or Curve")
-            pts: List[Point] = speckleArcCircleToPoints(segm) 
-        elif isinstance(segm, Line): 
-            print("Line")
-            pts: List[Point] = [segm.start, segm.end]
-        elif isinstance(segm, Polyline): 
-            print("Polyline")
-            pts: List[Point] = segm.as_points()
-
-        points.extend(pts)
-    return points
-
-def speckleArcCircleToPoints(poly: Union[Arc, Circle]) -> List[Point]: 
-    print("__Arc or Circle to Points___")
-    points = []
-    #print(poly.plane) 
-    #print(poly.plane.normal) 
-    if poly.plane is None or poly.plane.normal.z == 0: normal = 1 
-    else: normal = poly.plane.normal.z 
-    #print(poly.plane.origin)
-    if isinstance(poly, Circle):
-        interval = 2*math.pi
-        range_start = 0
-        angle1 = 0
-
-    else: # if Arc
-        points.append(poly.startPoint)
-        range_start = 0 
-
-        #angle1, angle2 = getArcAngles(poly)
-        
-        interval, angle1, angle2 = getArcRadianAngle(poly)
-        interval = abs(angle2 - angle1)
-        
-        #print(angle1)
-        #print(angle2)
-        
-        if (angle1 > angle2 and normal == -1) or (angle2 > angle1 and normal == 1): pass
-        if angle1 > angle2 and normal == 1: interval = abs( (2*math.pi-angle1) + angle2)
-        if angle2 > angle1 and normal == -1: interval = abs( (2*math.pi-angle2) + angle1)
-
-        #print(interval)
-        #print(normal)
-    
-    pointsNum = math.floor( abs(interval)) * 12
-    if pointsNum <4: pointsNum = 4
-
-    for i in range(range_start, pointsNum + 1): 
-        k = i/pointsNum # to reset values from 1/10 to 1
-        angle = angle1 + k * interval * normal
-        #print(k)
-        #print(angle)
-        pt = Point( x = poly.plane.origin.x + poly.radius * cos(angle), y = poly.plane.origin.y + poly.radius * sin(angle), z = 0) 
-        
-        pt.units = poly.plane.origin.units 
-        points.append(pt)
-    if isinstance(poly, Arc): points.append(poly.endPoint)
-    return points
-
-
-def getArcRadianAngle(arc: Arc) -> List[float]:
-
-    interval = None
-    normal = arc.plane.normal.z 
-    angle1, angle2 = getArcAngles(arc)
-    if angle1 is None or angle2 is  None: return None
-    interval = abs(angle2 - angle1)
-
-    if (angle1 > angle2 and normal == -1) or (angle2 > angle1 and normal == 1): pass
-    if angle1 > angle2 and normal == 1: interval = abs( (2*math.pi-angle1) + angle2)
-    if angle2 > angle1 and normal == -1: interval = abs( (2*math.pi-angle2) + angle1)
-    return interval, angle1, angle2
-
-def getArcAngles(poly: Arc) -> Tuple[float]: 
-    
-    if poly.startPoint.x == poly.plane.origin.x: angle1 = math.pi/2
-    else: angle1 = atan( abs ((poly.startPoint.y - poly.plane.origin.y) / (poly.startPoint.x - poly.plane.origin.x) )) # between 0 and pi/2
-    
-    if poly.plane.origin.x < poly.startPoint.x and poly.plane.origin.y > poly.startPoint.y: angle1 = 2*math.pi - angle1
-    if poly.plane.origin.x > poly.startPoint.x and poly.plane.origin.y > poly.startPoint.y: angle1 = math.pi + angle1
-    if poly.plane.origin.x > poly.startPoint.x and poly.plane.origin.y < poly.startPoint.y: angle1 = math.pi - angle1
-
-    if poly.endPoint.x == poly.plane.origin.x: angle2 = math.pi/2
-    else: angle2 = atan( abs ((poly.endPoint.y - poly.plane.origin.y) / (poly.endPoint.x - poly.plane.origin.x) )) # between 0 and pi/2
-
-    if poly.plane.origin.x < poly.endPoint.x and poly.plane.origin.y > poly.endPoint.y: angle2 = 2*math.pi - angle2
-    if poly.plane.origin.x > poly.endPoint.x and poly.plane.origin.y > poly.endPoint.y: angle2 = math.pi + angle2
-    if poly.plane.origin.x > poly.endPoint.x and poly.plane.origin.y < poly.endPoint.y: angle2 = math.pi - angle2
-
-    return angle1, angle2 
-
-def getArcNormal(poly: Arc, midPt: Point): 
-    print("____getArcNormal___")
-    angle1, angle2 = getArcAngles(poly)
-
-    if midPt.x == poly.plane.origin.x: angle = math.pi/2
-    else: angle = atan( abs ((midPt.y - poly.plane.origin.y) / (midPt.x - poly.plane.origin.x) )) # between 0 and pi/2
-    
-    if poly.plane.origin.x < midPt.x and poly.plane.origin.y > midPt.y: angle = 2*math.pi - angle
-    if poly.plane.origin.x > midPt.x and poly.plane.origin.y > midPt.y: angle = math.pi + angle
-    if poly.plane.origin.x > midPt.x and poly.plane.origin.y < midPt.y: angle = math.pi - angle
-
-    normal = Vector()
-    normal.x = normal.y = 0
-
-    if angle1 > angle > angle2: normal.z = -1  
-    if angle1 > angle2 > angle: normal.z = 1  
-
-    if angle2 > angle1 > angle: normal.z = -1  
-    if angle > angle1 > angle2: normal.z = 1  
-
-    if angle2 > angle > angle1: normal.z = 1  
-    if angle > angle2 > angle1: normal.z = -1  
-    
-    print(angle1)
-    print(angle)
-    print(angle2)
-    print(normal)
-
-    return normal
-
-
-def getArcCenter(p1: Point, p2: Point, p3: Point) -> Tuple[List, float]:
-    #print(p1)
-    p1 = np.array(p1.to_list())
-    p2 = np.array(p2.to_list())
-    p3 = np.array(p3.to_list())
-    a = np.linalg.norm(p3 - p2)
-    b = np.linalg.norm(p3 - p1)
-    c = np.linalg.norm(p2 - p1)
-    s = (a + b + c) / 2
-    radius = a*b*c / 4 / np.sqrt(s * (s - a) * (s - b) * (s - c))
-    b1 = a*a * (b*b + c*c - a*a)
-    b2 = b*b * (a*a + c*c - b*b)
-    b3 = c*c * (a*a + b*b - c*c)
-    center = np.column_stack((p1, p2, p3)).dot(np.hstack((b1, b2, b3)))
-    center /= b1 + b2 + b3
-    center = center.tolist()
-    return center, radius

speckle_toolbox/esri/toolboxes/speckle/converter/layers/_init_.py

@@ -1,833 +0,0 @@
-"""
-Contains all Layer related classes and methods.
-"""
-import os
-from typing import Any, List, Tuple, Union
-
-#from regex import D
-
-try:
-    from speckle.converter.layers.CRS import CRS
-    from speckle.converter.layers.Layer import Layer, VectorLayer, RasterLayer
-    from speckle.converter.layers.symbologyTemplates import vectorRendererToNative, rasterRendererToNative, rendererToSpeckle 
-    from speckle.converter.layers.feature import featureToNative, featureToSpeckle, cadFeatureToNative, bimFeatureToNative, rasterFeatureToSpeckle
-
-    from speckle.converter.geometry.mesh import rasterToMesh, meshToNative
-    from speckle.converter.layers.utils import findTransformation
-    from speckle.converter.layers.utils import getLayerAttributes, newLayerGroupAndName, validate_path
-except: 
-    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.CRS import CRS
-    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.Layer import Layer, VectorLayer, RasterLayer
-    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.symbologyTemplates import vectorRendererToNative, rasterRendererToNative, rendererToSpeckle 
-    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.feature import featureToNative, featureToSpeckle, cadFeatureToNative, bimFeatureToNative, rasterFeatureToSpeckle
-
-    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.mesh import rasterToMesh, meshToNative
-    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.utils import findTransformation
-    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.utils import getLayerAttributes, newLayerGroupAndName, validate_path
-
-from specklepy.objects import Base
-from specklepy.objects.geometry import Mesh
-
-import arcgisscripting
-import pandas as pd
-import arcpy
-from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
-from arcpy.management import (CreateFeatureclass, MakeFeatureLayer,
-                              AddFields, AlterField, DefineProjection )
-
-import numpy as np
-
-def convertSelectedLayers(all_layers: List[arcLayer], selected_layers: List[str], project: ArcGISProject) -> List[Union[VectorLayer,Layer]]:
-    """Converts the current selected layers to Speckle"""
-    print("________Convert Layers_________")
-    result = []
-    for layer in selected_layers:
-        layerToSend = None
-        for c in range(len(all_layers)):
-            if int(layer.split("-",1)[0]) == c:
-                layerToSend = all_layers[c]
-                break
-        if layerToSend is not None: 
-            ds = layerToSend.dataSource #file path
-            #if layerToSend.isFeatureLayer: 
-            newBaseLayer = layerToSpeckle(layerToSend, project)
-            if newBaseLayer is not None: 
-                result.append(newBaseLayer)
-            #elif layerToSend.isRasterLayer: pass
-            print(result)
-
-    return result
-
-def layerToSpeckle(layer: arcLayer, project: ArcGISProject) -> Union[VectorLayer, RasterLayer]: #now the input is QgsVectorLayer instead of qgis._core.QgsLayerTreeLayer
-    """Converts a given QGIS Layer to Speckle"""
-    print("________Convert Feature Layer_________")
-
-    speckleLayer = None
-
-    projectCRS = project.activeMap.spatialReference
-    try: data = arcpy.Describe(layer.dataSource)
-    except OSError as e: arcpy.AddWarning(str(e.args[0])); return
-
-    layerName = layer.name
-    crs = data.SpatialReference
-    units = "m"
-    layerObjs = []
-
-    # Convert CRS to speckle, use the projectCRS
-    speckleReprojectedCrs = CRS(name = projectCRS.name, wkt = projectCRS.exportToString(), units = units)
-    layerCRS = CRS(name=crs.name, wkt=crs.exportToString(), units = units) 
-    
-    #renderer = selectedLayer.renderer()
-    #layerRenderer = rendererToSpeckle(renderer) 
-    
-    if layer.isFeatureLayer: 
-        print("VECTOR LAYER HERE")
-        
-        speckleLayer = VectorLayer(units = "m")
-        speckleLayer.type="VectorLayer"
-        speckleLayer.name = layerName
-        speckleLayer.crs = speckleReprojectedCrs
-        speckleLayer.renderer = rendererToSpeckle(project, project.activeMap, layer, None)
-        #speckleLayer.datum = datum
-
-
-        try: # https://pro.arcgis.com/en/pro-app/2.8/arcpy/get-started/the-spatial-reference-object.htm
-            
-            #print(data.datasetType) # FeatureClass
-            if data.datasetType == "FeatureClass": #FeatureClass, ?Table Properties, ?Datasets
-                
-                # write feature attributes
-                fieldnames = [field.name for field in data.fields]
-                rows_shapes = arcpy.da.SearchCursor(layer.longName, "Shape@") # arcpy.da.SearchCursor(in_table, field_names, {where_clause}, {spatial_reference}, {explode_to_points}, {sql_clause})
-                print("__ start iterating features")
-                row_shapes_list = [x for k, x in enumerate(rows_shapes)]
-                for i, features in enumerate(row_shapes_list):
-
-                    print("____error Feature # " + str(i+1)) # + " / " + str(sum(1 for _ in enumerate(rows_shapes))))
-                    if features[0] is None: continue 
-                    feat = features[0]
-                    #print(feat) # <geoprocessing describe geometry object object at 0x000002A75D6A4BD0>
-                    #print(feat.hasCurves)
-                    #print(feat.partCount)
-
-                    if feat is not None: 
-                        print(feat)
-                        rows_attributes = arcpy.da.SearchCursor(layer.longName, fieldnames)
-                        row_attr = []
-                        for k, attrs in enumerate(rows_attributes):
-                            if i == k: row_attr = attrs; break
-
-                        # if curves detected, createa new feature class, turn to segments and get the same feature but in straigt lines
-                        #print(feat.hasCurves)
-                        if feat.hasCurves:
-                            #f_class_modified = curvedFeatureClassToSegments(layer)
-                            #rows_shapes_modified = arcpy.da.SearchCursor(f_class_modified, "Shape@") 
-                            #row_shapes_list_modified = [x for k, x in enumerate(rows_shapes_modified)]
-
-                            feat = feat.densify("ANGLE", 1000, 0.12)
-                            #print(feat)
-
-
-                        b = featureToSpeckle(fieldnames, row_attr, i, feat, projectCRS, project, layer)
-                        if b is not None: layerObjs.append(b)
-                        
-                    print("____End of Feature # " + str(i+1))
-                    
-                print("__ finish iterating features")
-                speckleLayer.features=layerObjs
-                speckleLayer.geomType = data.shapeType
-
-                if len(speckleLayer.features) == 0: return None
-
-                #layerBase.renderer = layerRenderer
-                #layerBase.applicationId = selectedLayer.id()
-
-        except OSError as e: 
-            arcpy.AddWarning(str(e))
-            return
-
-    elif layer.isRasterLayer:
-        print("RASTER IN DA HOUSE")
-        print(layer.name) # London_square.tif
-        print(arcpy.Describe(layer.dataSource)) # <geoprocessing describe data object object at 0x000002507C7F3BB0>
-        print(arcpy.Describe(layer.dataSource).datasetType) # RasterDataset
-        b = rasterFeatureToSpeckle(layer, projectCRS, project)
-        if b is not None: layerObjs.append(b)
-
-        speckleLayer = RasterLayer(units = "m", type="RasterLayer")
-        speckleLayer.name = layerName
-        speckleLayer.crs = speckleReprojectedCrs
-        speckleLayer.rasterCrs = layerCRS
-        speckleLayer.type="RasterLayer"
-        #speckleLayer.geomType="Raster"
-        speckleLayer.features = layerObjs
-        
-        speckleLayer.renderer = rendererToSpeckle(project, project.activeMap, layer, b)
-        
-        #speckleLayer.renderer = layerRenderer
-        #speckleLayer.applicationId = selectedLayer.id()
-
-    return speckleLayer
-
-def layerToNative(layer: Union[Layer, VectorLayer, RasterLayer], streamBranch: str, project: ArcGISProject):
-
-    if layer.type is None:
-        # Handle this case
-        return
-    elif layer.type.endswith("VectorLayer"):
-        return vectorLayerToNative(layer, streamBranch, project)
-    elif layer.type.endswith("RasterLayer"):
-        return rasterLayerToNative(layer, streamBranch, project)
-    return None
-
-def bimLayerToNative(layerContentList: List[Base], layerName: str, streamBranch: str, project: ArcGISProject) :
-    print("01______BIM layer to native")
-    print(layerName)
-    geom_meshes = []
-    layer_meshes = None
-    #filter speckle objects by type within each layer, create sub-layer for each type (points, lines, polygons, mesh?)
-    for geom in layerContentList:
-        try: 
-            if geom.displayMesh: geom_meshes.append(geom)
-        except:
-            try: 
-                if geom.displayValue: geom_meshes.append(geom)
-            except: pass
-
-    if len(geom_meshes)>0: layer_meshes = bimVectorLayerToNative(geom_meshes, layerName, "Mesh", streamBranch, project)
-
-    return True
-
-
-def bimVectorLayerToNative(geomList, layerName: str, geomType: str, streamBranch: str, project: ArcGISProject): 
-    # no support for mltipatches, maybe in 3.1: https://community.esri.com/t5/arcgis-pro-ideas/better-support-for-multipatches-in-arcpy/idi-p/953614/page/2#comments
-    print("02_________BIM vector layer to native_____")
-    #get Project CRS, use it by default for the new received layer
-    
-    vl = None
-    layerName = layerName + "_" + geomType
-    layerName = layerName.replace("[","_").replace("]","_").replace(" ","_").replace("-","_").replace("(","_").replace(")","_").replace(":","_").replace("\\","_").replace("/","_").replace("\"","_").replace("&","_").replace("@","_").replace("$","_").replace("%","_").replace("^","_")
-    #if not "__Structural_Foundations_Mesh" in layerName: return None
-    
-    sr = arcpy.SpatialReference(text = project.activeMap.spatialReference.exportToString())
-    active_map = project.activeMap
-    
-    path = project.filePath.replace("aprx","gdb") #
-    path_bim = "\\".join(project.filePath.split("\\")[:-1]) + "\\Layers_Speckle\\BIM_layers\\" + streamBranch+ "\\" + layerName + "\\" #arcpy.env.workspace + "\\" #
-    print(path_bim)
-    
-    if not os.path.exists(path_bim): os.makedirs(path_bim)
-    print(path)
-    
-    if sr.type == "Geographic": 
-        arcpy.AddMessage(f"Project CRS is set to Geographic type, and objects in linear units might not be received correctly")
-
-    #CREATE A GROUP "received blabla" with sublayers
-    layerGroup = None
-    newGroupName = f'{streamBranch}'
-    #print(newGroupName)
-    for l in active_map.listLayers():
-        if l.longName == newGroupName: layerGroup = l; break 
-    
-    #find ID of the layer with a matching name in the "latest" group 
-    newName = f'{streamBranch.split("_")[len(streamBranch.split("_"))-1]}_{layerName}'
-    print(newName)
-
-    all_layer_names = []
-    layerExists = 0
-    for l in project.activeMap.listLayers(): 
-        if l.longName.startswith(newGroupName + "\\"):
-            all_layer_names.append(l.longName)
-    #print(all_layer_names)
-
-    longName = streamBranch + "\\" + newName 
-    if longName in all_layer_names: 
-        for index, letter in enumerate('234567890abcdefghijklmnopqrstuvwxyz'):
-            if (longName + "_" + letter) not in all_layer_names: newName += "_"+letter; layerExists +=1; break 
-
-    # particularly if the layer comes from ArcGIS
-    if "mesh" in geomType.lower(): geomType = "Multipatch"
-
-    #print("Create feature class (cad): ")
-    # should be created inside the workspace to be a proper Feature class (not .shp) with Nullable Fields
-    class_name = ("f_class_" + newName)
-    #f_class = CreateFeatureclass(path, class_name, geomType, has_z="ENABLED", spatial_reference = sr)
-
-
-    shp = meshToNative(geomList, path_bim + newName)
-    print("____ meshes saved___")
-    print(shp)
-    #print(path)
-    #print(class_name)
-    validated_class_path = validate_path(class_name)
-    print(validated_class_path)
-    validated_class_name = validated_class_path.split("\\")[len(validated_class_path.split("\\"))-1]
-    print(validated_class_name)
-    f_class = arcpy.conversion.FeatureClassToFeatureClass(shp, path, validated_class_name)
-    # , spatial_reference = sr
-    #arcpy.management.Project(in_dataset, f_class, sr, in_coor_system=sr)
-
-    print(f_class)
-    #print(geomList)
-
-    # get and set Layer attribute fields
-    # example: https://resource.esriuk.com/blog/an-introductory-slice-of-arcpy-in-arcgis-pro/
-    newFields = getLayerAttributes(geomList)
-    
-    fields_to_ignore = ["arcgisgeomfromspeckle", "shape", "objectid", "displayMesh"]
-    matrix = []
-    all_keys = []
-    all_key_types = []
-    max_len = 52
-
-    print("___ after layer attributes: ___________")
-    print(newFields.items())
-    #try:
-    for key, value in newFields.items(): 
-        existingFields = [fl.name for fl in arcpy.ListFields(validated_class_name)]
-        #print(existingFields)
-        if key not in existingFields  and key.lower() not in fields_to_ignore: # exclude geometry and default existing fields
-            #print(key)
-            # signs that should not be used as field names and table names: https://support.esri.com/en/technical-article/000005588
-            key = key.replace(" ","_").replace("-","_").replace("(","_").replace(")","_").replace(":","_").replace("\\","_").replace("/","_").replace("\"","_").replace("&","_").replace("@","_").replace("$","_").replace("%","_").replace("^","_") 
-            if key[0] in ['0','1','2','3','4','5','6','7','8','9']: key = "_"+key
-            if len(key)>max_len: key = key[:max_len]
-            #print(all_keys)
-            if key in all_keys:
-                for index, letter in enumerate('1234567890abcdefghijklmnopqrstuvwxyz'):
-                    if len(key)<max_len and (key+letter) not in all_keys: key+=letter; break 
-                    if len(key) == max_len and (key[:9] + letter) not in all_keys: key=key[:9] + letter; break 
-            if key not in all_keys: 
-                all_keys.append(key)
-                all_key_types.append(value)
-                #print(all_keys)
-                matrix.append([key, value, key, 255])
-    print(all_keys)
-    print(len(all_keys))
-    if len(matrix)>0: AddFields(str(f_class), matrix)
-    print(matrix)
-    
-    fets = []
-    print("_________BIM FeatureS To Native___________")
-    for f in geomList[:]: 
-        new_feat = bimFeatureToNative(f, newFields, sr, path_bim)
-        if new_feat != "" and new_feat != None: 
-            fets.append(new_feat)
-    print(len(fets))
-        
-    
-    if len(fets) == 0: return None
-    count = 0
-    rowValues = []
-    for i, feat in enumerate(fets):
-
-        row = []
-        heads = []
-        for key in all_keys:
-            try:
-                row.append(feat[key])
-                heads.append(key)
-            except Exception as e: 
-                row.append(None)
-                heads.append(key)
-
-        rowValues.append(row)
-        count += 1
-    print(heads)
-    
-    with arcpy.da.UpdateCursor(f_class, heads) as cur:
-        # For each row, evaluate the WELL_YIELD value (index position 
-        # of 0), and update WELL_CLASS (index position of 1)
-        shp_num = 0
-        #print(heads)
-        try:
-            for rowShape in cur: 
-                #print(rowShape)
-                for i,r in enumerate(rowShape):
-                    #print(heads[i])
-                    #print(matrix[i])
-                    rowShape[i] = rowValues[shp_num][i]
-                    #print(type(rowShape[i]))
-                    if matrix[i][1] == 'TEXT' and rowShape[i] is not None: rowShape[i] = str(rowValues[shp_num][i]) 
-                    #print(type(rowShape[i]))
-                    if isinstance(rowValues[shp_num][i], str): # cut if string is too long
-                        rowShape[i] = rowValues[shp_num][i][:255]
-                    #print(rowShape[i])
-                #print(rowShape)
-                cur.updateRow(rowShape)
-                shp_num += 1
-                #print(shp_num)
-        except Exception as e: 
-            print("Layer attribute error: " + str(e))
-            #print(i)
-            print(shp_num)
-            print(len(rowValues))
-            #print(rowValues[i])
-            #print(len(rowValues[i]))
-            arcpy.AddWarning("Layer attribute error: " + e)
-    del cur 
-    
-    print("create layer:")
-    vl = MakeFeatureLayer(str(f_class), newName).getOutput(0)
-
-    active_map.addLayerToGroup(layerGroup, vl)
-    print("created2")
-    #os.remove(path_bim)
-
-    return True #last one
-
-def cadLayerToNative(layerContentList: List[Base], layerName: str, streamBranch: str, project: ArcGISProject) :
-    print("01______Cad vector layer to native")
-    print(layerName)
-    geom_points = []
-    geom_polylines = []
-    geom_polygones = []
-    geom_meshes = []
-    #filter speckle objects by type within each layer, create sub-layer for each type (points, lines, polygons, mesh?)
-    print(layerContentList)
-    for geom in layerContentList:
-        #print(geom)
-        if geom.speckle_type == "Objects.Geometry.Point": 
-            geom_points.append(geom)
-        if geom.speckle_type == "Objects.Geometry.Line" or geom.speckle_type == "Objects.Geometry.Polyline" or geom.speckle_type == "Objects.Geometry.Curve" or geom.speckle_type == "Objects.Geometry.Arc" or geom.speckle_type == "Objects.Geometry.Circle" or geom.speckle_type == "Objects.Geometry.Ellipse" or geom.speckle_type == "Objects.Geometry.Polycurve":
-            geom_polylines.append(geom)
-    
-    if len(geom_points)>0: layer_points = cadVectorLayerToNative(geom_points, layerName, "Points", streamBranch, project)
-    if len(geom_polylines)>0: layer_polylines = cadVectorLayerToNative(geom_polylines, layerName, "Polylines", streamBranch, project)
-
-    return [layer_points, layer_polylines]
-
-def cadVectorLayerToNative(geomList, layerName: str, geomType: str, streamBranch: str, project: ArcGISProject): 
-    print("02_________CAD vector layer to native_____")
-    #get Project CRS, use it by default for the new received layer
-    vl = None
-    layerName = layerName.replace("[","_").replace("]","_").replace(" ","_").replace("-","_").replace("(","_").replace(")","_").replace(":","_").replace("\\","_").replace("/","_").replace("\"","_").replace("&","_").replace("@","_").replace("$","_").replace("%","_").replace("^","_")
-    layerName = layerName + "_" + geomType
-    print(layerName)
-    
-    sr = arcpy.SpatialReference(text = project.activeMap.spatialReference.exportToString())
-    active_map = project.activeMap
-    path = project.filePath.replace("aprx","gdb") #"\\".join(project.filePath.split("\\")[:-1]) + "\\speckle_layers\\" #arcpy.env.workspace + "\\" #
-    
-    print(path)
-    print(streamBranch)
-    if sr.type == "Geographic": 
-        arcpy.AddMessage(f"Project CRS is set to Geographic type, and objects in linear units might not be received correctly")
-
-    #CREATE A GROUP "received blabla" with sublayers
-    layerGroup = None
-    newGroupName = f'{streamBranch}'
-    print(newGroupName)
-    #print(newGroupName)
-    for l in active_map.listLayers():
-        if l.longName == newGroupName: layerGroup = l; break 
-    
-    #find ID of the layer with a matching name in the "latest" group 
-    newName = f'{streamBranch.split("_")[len(streamBranch.split("_"))-1]}_{layerName}'
-
-    all_layer_names = []
-    layerExists = 0
-    for l in project.activeMap.listLayers(): 
-        if l.longName.startswith(newGroupName + "\\"):
-            all_layer_names.append(l.longName)
-    #print(all_layer_names)
-
-    longName = streamBranch + "\\" + newName 
-    if longName in all_layer_names: 
-        for index, letter in enumerate('234567890abcdefghijklmnopqrstuvwxyz'):
-            if (longName + "_" + letter) not in all_layer_names: newName += "_"+letter; layerExists +=1; break 
-
-    # particularly if the layer comes from ArcGIS
-    if "polygon" in geomType.lower(): geomType = "Polygon"
-    if "line" in geomType.lower(): geomType = "Polyline"
-    if "multipoint" in geomType.lower(): geomType = "Multipoint"
-    elif "point" in geomType.lower(): geomType = "Point"
-    #print(geomType)
-    
-    #print(newName)
-    #path = r"C:\Users\username\Documents\ArcGIS\Projects\MyProject-test\MyProject-test.gdb\\"
-    #https://community.esri.com/t5/arcgis-pro-questions/is-it-possible-to-create-a-new-group-layer-with/td-p/1068607
-
-    #print("Create feature class (cad): ")
-    # should be created inside the workspace to be a proper Feature class (not .shp) with Nullable Fields
-    class_name = ("f_class_" + newName)
-    f_class = CreateFeatureclass(path, class_name, geomType, has_z="ENABLED", spatial_reference = sr)
-    print(f_class)
-    #print(geomList)
-
-    # get and set Layer attribute fields
-    # example: https://resource.esriuk.com/blog/an-introductory-slice-of-arcpy-in-arcgis-pro/
-    newFields = getLayerAttributes(geomList)
-    
-    fields_to_ignore = ["arcgisgeomfromspeckle", "shape", "objectid"]
-    matrix = []
-    all_keys = []
-    all_key_types = []
-    max_len = 52
-    for key, value in newFields.items(): 
-        existingFields = [fl.name for fl in arcpy.ListFields(class_name)]
-        if key not in existingFields  and key.lower() not in fields_to_ignore: # exclude geometry and default existing fields
-            # signs that should not be used as field names and table names: https://support.esri.com/en/technical-article/000005588
-            key = key.replace(" ","_").replace("-","_").replace("(","_").replace(")","_").replace(":","_").replace("\\","_").replace("/","_").replace("\"","_").replace("&","_").replace("@","_").replace("$","_").replace("%","_").replace("^","_") 
-            if key[0] in ['0','1','2','3','4','5','6','7','8','9']: key = "_"+key
-            if len(key)>max_len: key = key[:max_len]
-            #print(all_keys)
-            if key in all_keys:
-                for index, letter in enumerate('1234567890abcdefghijklmnopqrstuvwxyz'):
-                    if len(key)<max_len and (key+letter) not in all_keys: key+=letter; break 
-                    if len(key) == max_len and (key[:9] + letter) not in all_keys: key=key[:9] + letter; break 
-            if key not in all_keys: 
-                all_keys.append(key)
-                all_key_types.append(value)
-                #print(all_keys)
-                matrix.append([key, value, key, 255])
-                #print(matrix)
-    if len(matrix)>0: AddFields(str(f_class), matrix)
-
-    fets = []
-    for f in geomList[:]: 
-        new_feat = cadFeatureToNative(f, newFields, sr)
-        if new_feat != "" and new_feat != None: 
-            fets.append(new_feat)
-    print("features created")
-    print(len(fets))
-    print(all_keys)
-    
-    if len(fets) == 0: return None
-    count = 0
-    rowValues = []
-    for feat in fets:
-        try: feat['applicationId'] 
-        except: feat.update({'applicationId': count})
-
-        row = [feat['arcGisGeomFromSpeckle'], feat['applicationId']]
-        heads = [ 'Shape@', 'OBJECTID']
-
-        for key,value in feat.items(): 
-            #print(key, str(value))
-            if key in all_keys and key.lower() not in fields_to_ignore: 
-                heads.append(key)
-                row.append(value)
-        rowValues.append(row)
-        count += 1
-    cur = arcpy.da.InsertCursor(str(f_class), tuple(heads) )
-    #print(heads)
-    for row in rowValues: 
-        try:
-            #print(row)
-            cur.insertRow(tuple(row))
-        except Exception as e:
-            print(e)
-    del cur 
-    vl = MakeFeatureLayer(str(f_class), newName).getOutput(0)
-
-    #adding layers from code solved: https://gis.stackexchange.com/questions/344343/arcpy-makefeaturelayer-management-function-not-creating-feature-layer-in-arcgis
-    #active_map.addLayer(new_layer)
-    active_map.addLayerToGroup(layerGroup, vl)
-    print("Layer created")
-
-    return vl
-
-def vectorLayerToNative(layer: Union[Layer, VectorLayer], streamBranch: str, project: ArcGISProject):
-    print("_________Vector Layer to Native_________")
-    vl = None
-    layerName = layer.name.replace(" ","_").replace("-","_").replace("(","_").replace(")","_").replace(":","_").replace("\\","_").replace("/","_").replace("\"","_").replace("&","_").replace("@","_").replace("$","_").replace("%","_").replace("^","_")
-    
-    print(layerName)
-    sr = arcpy.SpatialReference(text=layer.crs.wkt) 
-    active_map = project.activeMap
-    path = project.filePath.replace("aprx","gdb") #"\\".join(project.filePath.split("\\")[:-1]) + "\\speckle_layers\\" #arcpy.env.workspace + "\\" #
-    #if not os.path.exists(path): os.makedirs(path)
-    #print(path) 
-
-    newName, layerGroup = newLayerGroupAndName(layerName, streamBranch, project)
-
-    # particularly if the layer comes from ArcGIS
-    geomType = layer.geomType # for ArcGIS: Polygon, Point, Polyline, Multipoint, MultiPatch
-    print(geomType)
-    if "polygon" in geomType.lower(): geomType = "Polygon"
-    if "line" in geomType.lower(): geomType = "Polyline"
-    if "multipoint" in geomType.lower(): geomType = "Multipoint"
-    elif "point" in geomType.lower(): geomType = "Point"
-    #print(geomType)
-    
-    #print(newName)
-    #path = r"C:\Users\username\Documents\ArcGIS\Projects\MyProject-test\MyProject-test.gdb\\"
-    #https://community.esri.com/t5/arcgis-pro-questions/is-it-possible-to-create-a-new-group-layer-with/td-p/1068607
-    #print(project.filePath.replace("aprx","gdb"))
-    #print("_________create feature class___________________________________")
-    # should be created inside the workspace to be a proper Feature class (not .shp) with Nullable Fields
-    class_name = "f_class_" + newName
-    #print(class_name)
-    try: f_class = CreateFeatureclass(path, class_name, geomType, has_z="ENABLED", spatial_reference = sr)
-    except arcgisscripting.ExecuteError: class_name+="_"; f_class = CreateFeatureclass(path, class_name, geomType, has_z="ENABLED", spatial_reference = sr)
-
-    # get and set Layer attribute fields
-    # example: https://resource.esriuk.com/blog/an-introductory-slice-of-arcpy-in-arcgis-pro/
-    newFields = getLayerAttributes(layer.features)
-    fields_to_ignore = ["arcgisgeomfromspeckle", "shape", "objectid"]
-    matrix = []
-    all_keys = []
-    all_key_types = []
-    max_len = 52
-    for key, value in newFields.items(): 
-        existingFields = [fl.name for fl in arcpy.ListFields(class_name)]
-        if key not in existingFields  and key.lower() not in fields_to_ignore: # exclude geometry and default existing fields
-            # signs that should not be used as field names and table names: https://support.esri.com/en/technical-article/000005588
-            key = key.replace(" ","_").replace("-","_").replace("(","_").replace(")","_").replace(":","_").replace("\\","_").replace("/","_").replace("\"","_").replace("&","_").replace("@","_").replace("$","_").replace("%","_").replace("^","_") 
-            if key[0] in ['0','1','2','3','4','5','6','7','8','9']: key = "_"+key
-            if len(key)>max_len: key = key[:max_len]
-            #print(all_keys)
-            if key in all_keys:
-                for index, letter in enumerate('1234567890abcdefghijklmnopqrstuvwxyz'):
-                    if len(key)<max_len and (key+letter) not in all_keys: key+=letter; break 
-                    if len(key) == max_len and (key[:9] + letter) not in all_keys: key=key[:9] + letter; break 
-            if key not in all_keys: 
-                all_keys.append(key)
-                all_key_types.append(value)
-                #print(all_keys)
-                matrix.append([key, value, key, 255])
-                #print(matrix)
-    if len(matrix)>0: AddFields(str(f_class), matrix)
-
-    fets = []
-    for f in layer.features: 
-        new_feat = featureToNative(f, newFields, geomType, sr)
-        if new_feat != "" and new_feat!= None: fets.append(new_feat)
-    
-    #print(fets)
-    if len(fets) == 0: return None
-    count = 0
-    rowValues = []
-    heads = None
-    for feat in fets:
-        #print(feat)
-        try: feat['applicationId'] 
-        except: feat.update({'applicationId': count})
-
-        row = [feat['arcGisGeomFromSpeckle'], feat['applicationId']]
-        heads = [ 'Shape@', 'OBJECTID']
-
-        for key,value in feat.items(): 
-            if key in all_keys and key.lower() not in fields_to_ignore: 
-                heads.append(key)
-                row.append(value)
-        rowValues.append(row)
-        count += 1
-    cur = arcpy.da.InsertCursor(str(f_class), tuple(heads) )
-    for row in rowValues: 
-        #print(tuple(heads))
-        #print(tuple(row))
-        cur.insertRow(tuple(row))
-    del cur 
-
-    vl = MakeFeatureLayer(str(f_class), newName).getOutput(0)
-
-    #adding layers from code solved: https://gis.stackexchange.com/questions/344343/arcpy-makefeaturelayer-management-function-not-creating-feature-layer-in-arcgis
-    
-    active_map.addLayerToGroup(layerGroup, vl)
-    vl2 = None
-    print(newName)
-    for l in project.activeMap.listLayers(): 
-        #print(l.longName)
-        if l.longName == layerGroup.longName + "\\" + newName:
-            vl2 = l 
-            break
-    path_lyr = vectorRendererToNative(project, active_map, layerGroup, layer, vl2, f_class, heads)
-    #if path_lyr is not None: 
-    #    active_map.removeLayer(path_lyr)    
-
-    r'''
-    # rename back the layer if was renamed due to existing duplicate
-    if layerExists:  
-        vl.name = newName[:len(newName)-2]
-        for lyr in project.activeMap.listLayers():
-            print(lyr.longName)
-            if (streamBranch + "\\" + newName) == lyr.longName: 
-                lyr.name = lyr.name.replace( lyr.name, lyr.name[:len(lyr.name)-2] )
-                lyr.longName = lyr.longName.replace( lyr.longName, lyr.longName[:len(newName)-2] )
-                break
-    '''
-
-    r'''
-    pr.addFeatures(fets)
-    vl.updateExtents()
-    vl.commitChanges()
-    #layerGroup.addLayer(vl)
-
-    rendererNew = vectorRendererToNative(layer)
-    if rendererNew is None:
-        symbol = QgsSymbol.defaultSymbol(QgsWkbTypes.geometryType(QgsWkbTypes.parseType(geomType)))
-        rendererNew = QgsSingleSymbolRenderer(symbol)
-
-    try: vl.setRenderer(rendererNew)
-    except: pass
-    '''
-    return vl
-
-def rasterLayerToNative(layer: RasterLayer, streamBranch: str, project: ArcGISProject):
-
-    rasterLayer = None
-
-    layerName = layer.name.replace(" ","_").replace("-","_").replace("(","_").replace(")","_").replace(":","_").replace("\\","_").replace("/","_").replace("\"","_").replace("&","_").replace("@","_").replace("$","_").replace("%","_").replace("^","_")
-    
-    print(layerName)
-    sr = arcpy.SpatialReference(text=layer.crs.wkt) 
-    print(layer.crs.wkt)
-    active_map = project.activeMap
-    path = project.filePath.replace("aprx","gdb")
-    #path = '.'.join(path.split("\\")[:-1])
-    rasterHasSr = False
-    print(path)
-
-    path_bands = "\\".join(path.split("\\")[:-1]) + "\\Layers_Speckle\\raster_bands\\" + streamBranch 
-    if not os.path.exists(path_bands): os.makedirs(path_bands)
-
-    try: 
-        srRasterWkt = str(layer.rasterCrs.wkt)
-        print(layer.rasterCrs.wkt)
-        srRaster = arcpy.SpatialReference(text=srRasterWkt) # by native raster SR
-        rasterHasSr = True
-    except: 
-        srRasterWkt = str(layer.crs.wkt)
-        srRaster: arcpy.SpatialReference = sr # by layer
-    #print(layer.rasterCrs.wkt)
-    print(srRaster)
-
-    newName, layerGroup = newLayerGroupAndName(layerName, streamBranch, project)
-    print(newName)
-    if "." in newName: newName = '.'.join(newName.split(".")[:-1])
-    print(newName)
-    
-    feat = layer.features[0]
-    bandNames = feat["Band names"]
-    bandValues = [feat["@(10000)" + name + "_values"] for name in bandNames]
-
-    xsize= int(feat["X pixels"])
-    ysize= int(feat["Y pixels"])
-    xres = float(feat["X resolution"])
-    yres = float(feat["Y resolution"])
-    bandsCount=int(feat["Band count"])
-    originPt = arcpy.Point(feat['displayValue'][0].x, feat['displayValue'][0].y, feat['displayValue'][0].z)
-    print(originPt)
-    #if source projection is different from layer display projection, convert display OriginPt to raster source projection 
-    if rasterHasSr is True and srRaster.exportToString() != sr.exportToString():
-        originPt = findTransformation(arcpy.PointGeometry(originPt, sr, has_z = True), "Point", sr, srRaster, None).getPart()
-    print(originPt)
-
-    bandDatasets = ""
-    rastersToMerge = []
-    rasterPathsToMerge = []
-
-
-    arcpy.env.workspace = path 
-    arcpy.env.overwriteOutput = True
-    # https://pro.arcgis.com/en/pro-app/latest/tool-reference/data-management/composite-bands.htm
-    
-    
-    for i in range(bandsCount):
-        print(i)
-        print(bandNames[i])
-        rasterbandPath = path_bands + "\\" + newName + "_Band_" + str(i+1) + ".tif"
-        bandDatasets += rasterbandPath + ";"
-        rasterband = np.array(bandValues[i])
-        rasterband = np.reshape(rasterband,(ysize, xsize))
-        print(rasterband)
-        print(np.shape(rasterband))
-        print(xsize)
-        print(xres)
-        print(ysize)
-        print(yres)
-        leftLowerCorner = arcpy.Point(originPt.X, originPt.Y + (ysize*yres), originPt.Z)
-        #upperRightCorner = arcpy.Point(originPt.X + (xsize*xres), originPt.Y, originPt.Z)
-        print(leftLowerCorner)
-        #print(upperRightCorner)
-
-        # # Convert array to a geodatabase raster, add to layers 
-        try: myRaster = arcpy.NumPyArrayToRaster(rasterband, leftLowerCorner, abs(xres), abs(yres), float(feat["NoDataVal"][i]) ) 
-        except: myRaster = arcpy.NumPyArrayToRaster(rasterband, leftLowerCorner, abs(xres), abs(yres)) 
-
-        rasterbandPath = validate_path(rasterbandPath) #solved file saving issue 
-        print(rasterbandPath)
-        #mergedRaster = arcpy.ia.Merge(rastersToMerge) # glues all bands together
-        myRaster.save(rasterbandPath)
-
-        print(myRaster.width)
-        print(myRaster.height)
-
-        rastersToMerge.append(myRaster)
-        rasterPathsToMerge.append(rasterbandPath)
-        print(rasterbandPath)
-
-    #mergedRaster.setProperty("spatialReference", crsRaster)
-
-    full_path = validate_path(path + "\\" + newName) #solved file saving issue 
-    print("RASTER FULL PATH")
-    print(full_path)
-    if os.path.exists(full_path):
-        #print(full_path)
-        for index, letter in enumerate('1234567890abcdefghijklmnopqrstuvwxyz'):
-            print(full_path + letter)
-            if os.path.exists(full_path + letter): pass
-            else: full_path += letter; break 
-    print("RASTER new PATH")
-    print(full_path)
-    #mergedRaster = arcpy.ia.Merge(rastersToMerge) # glues all bands together
-    #mergedRaster.save(full_path) # similar errors: https://community.esri.com/t5/python-questions/error-010240-could-not-save-raster-dataset/td-p/321690
-    
-    try: 
-        arcpy.management.CompositeBands(rasterPathsToMerge, full_path)
-    except: # if already exists
-        full_path += "_"
-        arcpy.management.CompositeBands(rasterPathsToMerge, full_path)
-    print(path + "\\" + newName)
-    arcpy.management.DefineProjection(full_path, srRaster)
-
-    rasterLayer = arcpy.management.MakeRasterLayer(full_path, newName).getOutput(0)
-    print(layerGroup)
-    active_map.addLayerToGroup(layerGroup, rasterLayer)
-
-    rl2 = None
-    for l in active_map.listLayers(): 
-        if l.longName == layerGroup.longName + "\\" + newName:
-            print(l.longName)
-            rl2 = l 
-            break
-    rasterLayer = rasterRendererToNative(project, active_map, layerGroup, layer, rl2, rasterPathsToMerge, newName)
-
-    try: os.remove(path_bands)
-    except: pass
-
-    r'''
-    if arcpy.Exists(fileout):
-        arcpy.management.Delete(fileout)
-    arcpy.management.Rename(filelist[0], fileout)
-
-    # Remove temporary files
-    for fileitem in filelist:
-        if arcpy.Exists(fileitem):
-            arcpy.management.Delete(fileitem)
-
-    # Release raster objects from memory
-    del myRasterBlock
-    del myRaster
-    '''
-
-    r'''
-    rasterComposite = arcpy.management.CompositeBands(bandDatasets, path + "\\" + newName) # "band1.tif;band2.tif;band3.tif", "compbands.tif"
-    
-    # https://pro.arcgis.com/en/pro-app/latest/tool-reference/data-management/make-raster-layer.htm
-    rasterLayer = arcpy.MakeRasterLayer_management(rasterComposite, newName)
-    '''
-
-    
-    r'''
-    # WORKS: 
-    arcpy.CreateRasterDataset_management(r"C:\Users\Kateryna\Documents\ArcGIS\Projects\MyProject-test",
-    "EmptyTIFF.tif",
-    "2",
-    "8_BIT_UNSIGNED",
-    "PROJCS['DHDN_3_Degree_Gauss_Zone_3',GEOGCS['GCS_Deutsches_Hauptdreiecksnetz',DATUM['D_Deutsches_Hauptdreiecksnetz',SPHEROID['Bessel_1841',6377397.155,299.1528128]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]],PROJECTION['Gauss_Kruger'],PARAMETER['False_Easting',3500000.0],PARAMETER['False_Northing',0.0],PARAMETER['Central_Meridian',9.0],PARAMETER['Scale_Factor',1.0],PARAMETER['Latitude_Of_Origin',0.0],UNIT['Meter',1.0]]", "3", "", "PYRAMIDS -1 NEAREST JPEG", "128 128", "NONE", "")
-    '''
-
-    return rasterLayer

speckle_toolbox/esri/toolboxes/speckle/converter/layers/feature.py

@@ -1,614 +0,0 @@
-import json
-import math
-import os
-
-from typing import Dict, Any, Callable, List, Optional, Tuple
-
-from specklepy.objects import Base
-import arcpy 
-from arcpy.management import CreateCustomGeoTransformation
-from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
-
-try: 
-    from speckle.converter.geometry._init_ import convertToSpeckle, convertToNative, convertToNativeMulti
-    from speckle.converter.layers.utils import (findTransformation, getVariantFromValue, traverseDict, 
-                                                traverseDictByKey, hsv_to_rgb)
-    from speckle.converter.geometry.point import pointToSpeckle
-    from speckle.converter.geometry.mesh import rasterToMesh, meshToNative
-    from speckle.converter.layers.symbologyTemplates import jsonFromLayerStyle
-except: 
-    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry._init_ import convertToSpeckle, convertToNative, convertToNativeMulti
-    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.utils import (findTransformation, getVariantFromValue, traverseDict, 
-                                                traverseDictByKey, hsv_to_rgb)
-    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.point import pointToSpeckle
-    from speckle_toolbox.esri.toolboxes.speckle.converter.geometry.mesh import rasterToMesh, meshToNative
-    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.symbologyTemplates import jsonFromLayerStyle
-
-import numpy as np
-import colorsys
-
-
-def featureToSpeckle(fieldnames, attr_list, index: int, f_shape, projectCRS: arcpy.SpatialReference, project: ArcGISProject, selectedLayer: arcLayer):
-    print("___________Feature to Speckle____________")
-    b = Base(units = "m")
-    data = arcpy.Describe(selectedLayer.dataSource)
-    layer_sr = data.spatialReference # if sr.type == "Projected":
-    geomType = data.shapeType #Polygon, Point, Polyline, Multipoint, MultiPatch
-    featureType = data.featureType # Simple,SimpleJunction,SimpleJunction,ComplexEdge,Annotation,CoverageAnnotation,Dimension,RasterCatalogItem 
-    print(geomType)
-    print(hasattr(data, "isRevit")) 
-    print(hasattr(data, "isIFC")) 
-    print(hasattr(data, "bimLevels")) 
-    print(hasattr(data, "hasSpatialIndex")) 
-    if geomType == "MultiPatch" or hasattr(data, "isRevit") or hasattr(data, "isIFC") or hasattr(data, "bimLevels"): 
-        print(f"Layer {selectedLayer.name} has unsupported data type")
-        arcpy.AddWarning(f"Layer {selectedLayer.name} has unsupported data type")
-        return None 
-    #print(layer_sr.name)
-    #print(projectCRS.name)
-    f_shape = findTransformation(f_shape, geomType, layer_sr, projectCRS, selectedLayer)
-    if f_shape is None: return None
-
-    ######################################### Convert geometry ##########################################
-    try:
-        geom = convertToSpeckle(f_shape, index, selectedLayer, geomType, featureType) 
-        if geom is not None: print(geom); b["geometry"] = geom 
-    except Exception as error:
-        print("Error converting geometry: " + str(error))
-        print(selectedLayer)
-        arcpy.AddError("Error converting geometry: " + str(error))
-    #print(geomType) 
-    #print(featureType) 
-    for i, name in enumerate(fieldnames):
-        corrected = name.replace("/", "_").replace(".", "-")
-        if corrected != "Shape" and corrected != "Shape@": 
-            # different ID behaviors: https://support.esri.com/en/technical-article/000010834 
-            # save all attribute, duplicate one into applicationId 
-            b[corrected] = attr_list[i]
-            if corrected == "FID" or corrected == "OID" or corrected == "OBJECTID": b["applicationId"] = str(attr_list[i])
-    #print(b)
-    print("______end of __Feature to Speckle____________________")
-    return b
-
-def featureToNative(feature: Base, fields: dict, geomType: str, sr: arcpy.SpatialReference):
-    print("04_____Feature To Native____________") 
-    feat = {}
-    try: speckle_geom = feature["geometry"] # for created in QGIS / ArcGIS Layer type
-    except:  speckle_geom = feature # for created in other software
-    #print(speckle_geom)
-    if isinstance(speckle_geom, list):
-        if len(speckle_geom)>1 or geomType == "Multipoint": arcGisGeom = convertToNativeMulti(speckle_geom, sr)
-        else: arcGisGeom = convertToNative(speckle_geom[0], sr)
-    else:
-        arcGisGeom = convertToNative(speckle_geom, sr)
-
-    if arcGisGeom is not None:
-        feat.update({"arcGisGeomFromSpeckle": arcGisGeom})
-    else:
-        return None
-
-    for key, variant in fields.items(): 
-        try: value = feature[key]
-        except: 
-            if key == 'Speckle_ID': value = feature['id'] 
-            else: 
-                arcpy.AddWarning(f'Field {key} not found')
-                return None 
-
-        if variant == "TEXT": value = str(value) 
-        if variant == getVariantFromValue(value) and value != "NULL" and value != "None": 
-            feat.update({key: value})
-        else: 
-            if variant == "TEXT": feat.update({key: None})
-            if variant == "FLOAT": feat.update({key: None})
-            if variant == "LONG": feat.update({key: None})
-            if variant == "SHORT": feat.update({key: None})
-    return feat
-             
-def bimFeatureToNative(feature: Base, fields: dict, sr: arcpy.SpatialReference, path: str):
-    print("04_________BIM Feature To Native____________")
-
-    feat = {}
-    try: speckle_geom = feature["geometry"] # for created in QGIS Layer type
-    except:  speckle_geom = feature # for created in other software
-
-    feat.update({"arcGisGeomFromSpeckle": ""})
-
-    try: 
-        if "Speckle_ID" not in fields.keys() and feature["id"]: feat.update("Speckle_ID", "TEXT")
-    except: pass
-    feat_updated = updateFeat(feat, fields, feature)
-
-    return feat_updated
-                
-
-def cadFeatureToNative(feature: Base, fields: dict, sr: arcpy.SpatialReference):
-    print("04_________CAD Feature To Native____________")
-    feat = {}
-    try: speckle_geom = feature["geometry"] # for created in QGIS Layer type
-    except:  speckle_geom = feature # for created in other software
-
-    if isinstance(speckle_geom, list):
-        if len(speckle_geom)>1: arcGisGeom = convertToNativeMulti(speckle_geom, sr)
-        else: arcGisGeom = convertToNative(speckle_geom[0], sr) 
-    else:
-        arcGisGeom = convertToNative(speckle_geom, sr)
-        
-    if arcGisGeom is not None:
-        feat.update({"arcGisGeomFromSpeckle": arcGisGeom})
-    else: return None
-    
-    try: 
-        if "Speckle_ID" not in fields.keys() and feature["id"]: feat.update("Speckle_ID", "TEXT")
-    except: pass
-
-    #### setting attributes to feature
-    feat_updated = updateFeat(feat, fields, feature)
-    print(feat)
-    print(fields)
-    return feat_updated
-
-def addFeatVariant(key, variant, value, f):
-    
-    feat = f
-    if variant == "TEXT": value = str(value) 
-    if value != "NULL" and value != "None":
-        #if key == 'area': print(value); print(type(value)); print(getVariantFromValue(value))
-        if variant == getVariantFromValue(value) or (variant=="FLOAT" and isinstance(value, int)): 
-            feat.update({key: value}) 
-        elif variant == "LONG" and isinstance(value, float): # if object has been modified 
-            feat.update({key: int(value)}) 
-        elif variant == "FLOAT" and isinstance(value, int): # if object has been modified 
-            feat.update({key: float(value)}) 
-        else: feat.update({key: None}) 
-    elif variant == "TEXT" or variant == "FLOAT" or variant == "LONG" or variant == "SHORT": feat.update({key: None})
-    return feat 
-
-def updateFeat(feat:dict, fields: dict, feature: Base) -> Dict[str, Any]:
-    
-    for key, variant in fields.items(): 
-        try:
-            if key == "Speckle_ID": 
-                value = str(feature["id"])
-                
-                feat[key] = value 
-
-                feat = addFeatVariant(key, variant, value, feat)
-            else:
-                try: 
-                    value = feature[key] 
-                    #if key == "area": print(feature[key]); print(type(feature[key]))
-                    feat = addFeatVariant(key, variant, value, feat)
-                except:
-                    value = None
-                    rootName = key.split("_")[0]
-                    #try: # if the root category exists
-                    # if its'a list 
-                    if isinstance(feature[rootName], list):
-                        for i in range(len(feature[rootName])):
-                            try:
-                                newF, newVals = traverseDict({}, {}, rootName + "_" + str(i), feature[rootName][i])
-                                for i, (key,value) in enumerate(newVals.items()):
-                                    for k, (x,y) in enumerate(newF.items()):
-                                        if key == x: variant = y; break
-                                    feat = addFeatVariant(key, variant, value, feat)
-                            except Exception as e: print(e)
-                    #except: # if not a list
-                    else:
-                        try:
-                            newF, newVals = traverseDict({}, {}, rootName, feature[rootName])
-                            for i, (key,value) in enumerate(newVals.items()):
-                                for k, (x,y) in enumerate(newF.items()):
-                                    if key == x: variant = y; break
-                                feat = addFeatVariant(key, variant, value, feat)
-                        except Exception as e: feat.update({key: None})
-        except Exception as e: 
-            feat.update({key: None})
-    feat_sorted = {k: v for k, v in sorted(feat.items(), key=lambda item: item[0])}
-    #print("_________________end of updating a feature_________________________")
-    return feat_sorted
-
-def rasterFeatureToSpeckle(selectedLayer: arcLayer, projectCRS: arcpy.SpatialReference, project: ArcGISProject) -> Base:
-    print("_________ Raster feature to speckle______") 
-    # https://pro.arcgis.com/en/pro-app/latest/arcpy/classes/raster-object.htm 
-
-    # get Raster object of entire raster dataset 
-    my_raster = arcpy.Raster(selectedLayer.dataSource)
-    print(my_raster.mdinfo) # None
-
-    rasterBandCount = my_raster.bandCount
-    rasterBandNames = my_raster.bandNames
-    rasterDimensions = [my_raster.width, my_raster.height]
-    if rasterDimensions[0]*rasterDimensions[1] > 1000000 :
-        arcpy.AddWarning("Large layer: ")
-
-    #ds = gdal.Open(selectedLayer.source(), gdal.GA_ReadOnly)
-    extent = my_raster.extent
-    print(extent.XMin)
-    print(extent.YMin)
-    rasterOriginPoint = arcpy.PointGeometry(arcpy.Point(extent.XMin, extent.YMax, extent.ZMin), my_raster.spatialReference, has_z = True)
-    #if extent.YMin>0: rasterOriginPoint = arcpy.PointGeometry(arcpy.Point(extent.XMin, extent.YMax, extent.ZMin), my_raster.spatialReference, has_z = True)
-    print(rasterOriginPoint)
-    rasterResXY = [my_raster.meanCellWidth, my_raster.meanCellHeight] #[float(ds.GetGeoTransform()[1]), float(ds.GetGeoTransform()[5])]
-    rasterBandNoDataVal = [] #list(my_raster.noDataValues)
-    rasterBandMinVal = []
-    rasterBandMaxVal = []
-    rasterBandVals = []
-
-    
-    b = Base(units = "m")
-    # Try to extract geometry
-    reprojectedPt = None
-
-    try:
-        reprojectedPt = rasterOriginPoint
-        if my_raster.spatialReference.name != projectCRS.name: 
-            reprojectedPt = findTransformation(reprojectedPt, "Point", my_raster.spatialReference, projectCRS, selectedLayer)
-            if reprojectedPt is None: 
-                reprojectedPt = rasterOriginPoint
-        geom = pointToSpeckle(reprojectedPt.getPart(), None, None)
-        if (geom != None):
-            b['displayValue'] = [geom]
-        print(geom)
-    except Exception as error:
-        arcpy.AddError("Error converting point geometry: " + str(error)) 
-
-    for i, item in enumerate(rasterBandNames):
-        print(item)
-        rb = my_raster.getRasterBands(item)
-        print(rb)
-        print(np.shape(rb.read()))
-        valMin = rb.minimum
-        valMax = rb.maximum
-        bandVals = np.swapaxes(rb.read(), 1, 2).flatten() #.tolist() np.flip( , 0)
-
-        bandValsFlat = []
-        bandValsFlat.extend(bandVals.tolist())
-        #print(bandValsFlat)
-
-        const = float(-1* math.pow(10,30))
-        defaultNoData = rb.noDataValue
-
-        # check whether NA value is too small or raster has too small values
-        # assign min value of an actual list; re-assign NA val; replace list items to new NA val
-        try:
-            # create "safe" fake NA value; replace extreme values with it
-            fakeNA = max(bandValsFlat) + 1 
-            bandValsFlatFake = [fakeNA if val<=const else val for val in bandValsFlat] # replace all values corresponding to NoData value 
-            
-            #if default NA value is too small
-            if (isinstance(defaultNoData, float) or isinstance(defaultNoData, int)) and defaultNoData < const:
-                # find and rewrite min of actual band values; create new NA value
-                valMin = min(bandValsFlatFake)
-                noDataValNew = valMin - 1000 # use new adequate value
-                rasterBandNoDataVal.append(noDataValNew)
-                # replace fake NA with new NA
-                bandValsFlat = [noDataValNew if val == fakeNA else val for val in bandValsFlatFake] # replace all values corresponding to NoData value 
-            
-            # if default val unaccessible and minimum val is too small 
-            elif (isinstance(defaultNoData, str) or defaultNoData is None) and valMin < const: # if there are extremely small values but default NA unaccessible 
-                noDataValNew = valMin 
-                rasterBandNoDataVal.append(noDataValNew)
-                # replace fake NA with new NA
-                bandValsFlat = [noDataValNew if val == fakeNA else val for val in bandValsFlatFake] # replace all values corresponding to NoData value 
-                # last, change minValto actual one
-                valMin = min(bandValsFlatFake)
-
-            else: rasterBandNoDataVal.append(rb.noDataValue)
-
-        except: rasterBandNoDataVal.append(rb.noDataValue)
-        if rasterBandNoDataVal[len(rasterBandNoDataVal)-1] is None: 
-            rasterBandNoDataVal[len(rasterBandNoDataVal)-1] = 'None'
-
-        
-        rasterBandVals.append(bandValsFlat)
-        rasterBandMinVal.append(valMin)
-        rasterBandMaxVal.append(valMax)
-
-        #print(rb.getColormap()) #None
-
-        b["@(10000)" + item + "_values"] = bandValsFlat #[0:int(max_values/rasterBandCount)]
-    
-    b["X resolution"] = rasterResXY[0]
-    b["Y resolution"] = -1* rasterResXY[1]
-    b["X pixels"] = rasterDimensions[0]
-    b["Y pixels"] = rasterDimensions[1]
-    b["Band count"] = rasterBandCount
-    b["Band names"] = rasterBandNames
-    b["NoDataVal"] = rasterBandNoDataVal
-    # creating a mesh
-    vertices = []
-    faces = []
-    colors = []
-    count = 0
-    
-    #print(my_raster.variables)
-    print(selectedLayer.symbology) #None
-    colorizer = None
-    #renderer = selectedLayer.symbology.renderer
-    if hasattr(selectedLayer.symbology, 'colorizer'): 
-        colorizer = selectedLayer.symbology.colorizer
-
-        print(colorizer) # <arcpy._colorizer.RasterStretchColorizer object at 0x000001780497FBC8>
-        print(colorizer.type) # RasterStretchColorizer 
-    else: 
-        #RGB colorizer 
-        root_path = "\\".join(project.filePath.split("\\")[:-1])
-        if not os.path.exists(root_path + '\\Layers_Speckle\\raster_bands'): os.makedirs(root_path + '\\Layers_Speckle\\raster_bands')
-        path_style = root_path + '\\Layers_Speckle\\raster_bands\\' + selectedLayer.name + '_temp.lyrx'
-        symJson = jsonFromLayerStyle(selectedLayer, path_style)
-
-        # read from Json
-        print(symJson["layerDefinitions"][0]["colorizer"])
-        try: greenBand = symJson["layerDefinitions"][0]["colorizer"]["greenBandIndex"]
-        except: greenBand = None
-        try: blueBand = symJson["layerDefinitions"][0]["colorizer"]["blueBandIndex"] 
-        except: blueBand = None
-        
-        try: redBand = symJson["layerDefinitions"][0]["colorizer"]["redBandIndex"]
-        except: 
-            if blueBand!=0 and greenBand!=0: redBand= 0
-            else: redBand = None
-        print("bands")
-        print(redBand)
-        print(greenBand)
-        print(blueBand)
-        try: 
-            rbVals = rasterBandVals[redBand] #my_raster.getRasterBands(rasterBandNames[redBand])
-            rbvalMin = min(rbVals)
-            rbvalMax = max(rbVals)
-            rvalRange = float(rbvalMax) - float(rbvalMin)
-            print(rbvalMin)
-            print(rbvalMax)
-            print(rvalRange)
-        except Exception as e: print(e); rvalRange = None
-        try: 
-            gbVals = rasterBandVals[greenBand]
-            gbvalMin = min(gbVals)
-            gbvalMax = max(gbVals)
-            gvalRange = float(gbvalMax) - float(gbvalMin)
-            print(gbvalMin)
-            print(gbvalMax)
-            print(gvalRange)
-        except: gvalRange = None
-        try: 
-            bbVals = rasterBandVals[blueBand]
-            bbvalMin = min(bbVals)
-            bbvalMax = max(bbVals)
-            bvalRange = float(bbvalMax) - float(bbvalMin)
-            print(bbvalMin)
-            print(bbvalMax)
-            print(bvalRange)
-        except: bvalRange = None
-
-    rendererType = ""
-    #if hasattr(selectedLayer.symbology, 'renderer'): rendererType = selectedLayer.symbology.renderer.type #e.g. SimpleRenderer
-    # custom color ramp {"type": "algorithmic", "fromColor": [115, 76, 0, 255],"toColor": [255, 25, 86, 255], "algorithm": "esriHSVAlgorithm"}.
-    # custom color map {'values': [0, 1, 2, 3, 4, 5], 'colors': ['#000000', '#DCFFDF', '#B8FFBE', '#85FF90', '#50FF60','#00AB10']}
-
-    bandIndex = 0
-    r'''
-    if colorizer.type == "RasterStretchColorizer":
-        print("___Color cell: RasterStretchColorizer___")
-        print(colorizer.band)
-        #colorRamps = project.listColorRamps()
-        bandIndex = colorizer.band
-
-        colorizerData = None
-        colorRamp = None
-
-        colorizerData = traverseDictByKey(layerFileContent, "colorizer", None)
-        print(colorizerData) # {'type': 'CIMRasterStretchColorizer', 'resamplingType': 
-
-        #noDataColor: List[float] = traverseDictByKey(colorizerData, "noDataColor")['values']
-        colorRamp = traverseDictByKey(colorizerData, "colorRamp", None)
-
-        colorsFromRamp: List[List[float]] = []
-        colorsFromRampType = []
-        try:
-            for i, item in enumerate(colorRamp['colorRamps']): 
-                colorsFromRamp.append(item['fromColor']['values'])
-                colorsFromRampType.append(item['fromColor']['type'])
-                if i == len(colorRamp['colorRamps'])-1 : 
-                    colorsFromRamp.append(item['toColor']['values'])
-                    colorsFromRampType.append(item['toColor']['type'])
-        except: pass
-        print(colorsFromRamp) # [[220, 100, 45, 100], [214.12, 100, 100, 100], [201, 25, 100, 100]]
-        rangesNumber = len(colorsFromRamp) - 1 # 2 (if 3 colors)
-
-        colorsFromRampRGB = []
-        for i, item in enumerate(colorsFromRamp): 
-            if ("CIMHSVColor" in colorsFromRampType[i]): 
-                # https://pro.arcgis.com/en/pro-app/latest/arcpy/mapping/rasterclassbreak-class.htm
-                newR, newG, newB = colorsys.hsv_to_rgb(item[0],item[1],item[2])
-                colorsFromRampRGB.append( ( int(newR*255), int( newG*255), int(newB*255) )  )
-            elif ("HSL" in colorsFromRampType[i]): 
-                # https://pro.arcgis.com/en/pro-app/latest/arcpy/mapping/rasterclassbreak-class.htm
-                newR, newG, newB = colorsys.hsl_to_rgb(item[0],item[1],item[2])
-                colorsFromRampRGB.append( ( int(newR*255), int( newG*255), int(newB*255) )  )
-            else:
-                colorsFromRampRGB.append(item)
-
-        rs = [float(x[0]) for x in colorsFromRampRGB]
-        gs = [float(x[1]) for x in colorsFromRampRGB]
-        bs = [float(x[2]) for x in colorsFromRampRGB]
-    '''    
-    # identify symbology type and if Multiband, which band is which color
-    for v in range(rasterDimensions[1] ): #each row, Y
-        for h in range(rasterDimensions[0] ): #item in a row, X
-            pt1 = arcpy.PointGeometry(arcpy.Point(extent.XMin+h*rasterResXY[0],extent.YMax-v*rasterResXY[1]), my_raster.spatialReference, has_z = True)
-            pt2 = arcpy.PointGeometry(arcpy.Point(extent.XMin+h*rasterResXY[0], extent.YMax-(v+1)*rasterResXY[1]), my_raster.spatialReference, has_z = True)
-            pt3 = arcpy.PointGeometry(arcpy.Point(extent.XMin+(h+1)*rasterResXY[0], extent.YMax-(v+1)*rasterResXY[1]), my_raster.spatialReference, has_z = True)
-            pt4 = arcpy.PointGeometry(arcpy.Point(extent.XMin+(h+1)*rasterResXY[0], extent.YMax-v*rasterResXY[1]), my_raster.spatialReference, has_z = True)
-            # first, get point coordinates with correct position and resolution, then reproject each:
-            if my_raster.spatialReference.exportToString() != projectCRS.exportToString():
-                pt1 = findTransformation(pt1, "Point", my_raster.spatialReference, projectCRS, selectedLayer)
-                pt2 = findTransformation(pt2, "Point", my_raster.spatialReference, projectCRS, selectedLayer)
-                pt3 = findTransformation(pt3, "Point", my_raster.spatialReference, projectCRS, selectedLayer)
-                pt4 = findTransformation(pt4, "Point", my_raster.spatialReference, projectCRS, selectedLayer)
-            vertices.extend([pt1.getPart().X, pt1.getPart().Y, pt1.getPart().Z, pt2.getPart().X, pt2.getPart().Y, pt2.getPart().Z, pt3.getPart().X, pt3.getPart().Y, pt3.getPart().Z, pt4.getPart().X, pt4.getPart().Y, pt4.getPart().Z]) ## add 4 points
-            faces.extend([4, count, count+1, count+2, count+3])
-
-            # color vertices according to QGIS renderer
-            color = (0<<16) + (0<<8) + 0
-            noValColor = [0,0,0] #selectedLayer.renderer().nodataColor().getRgb()
-
-            r'''
-            if colorizer.type == "RasterStretchColorizer":   
-
-                # find position of the alue on the range
-                if rasterBandVals[bandIndex][int(count/4)] >= float(colorizer.minLabel) and rasterBandVals[bandIndex][int(count/4)] <= float(colorizer.maxLabel) : #rasterBandMinVal[bandIndex]: 
-                    # REMAP band values to (0,255) range
-                    valRange = float(colorizer.maxLabel) - float(colorizer.minLabel) #(rasterBandMaxVal[bandIndex] - rasterBandMinVal[bandIndex])
-                    position = (rasterBandVals[bandIndex][int(count/4)] - float(colorizer.minLabel)) / valRange 
-                    
-                    print(position) # 0.8461538461538461
-                    print("calc range")
-                    localPosition = 0
-                    for n in range(rangesNumber): # 0, 1
-                        print(n)
-                        start = n/rangesNumber
-                        end = (n+1)/rangesNumber
-                        if position <= end and position >= start: 
-                            localRange = end-start
-                            localPosition = position - start
-                            break # n - is the range we need, number bentween n and (n+1)
-                    print(localPosition) # 0.34615384615384615
-                    print(n)
-
-                    localColor = []
-                    for c in [rs,gs,bs]: 
-                        print(c)
-                        # go through each color: 
-                        localColor.append( int( (c[n+1] - c[n]) * localPosition + c[n] ) )
-                        print(localColor)
-                    color =  (localColor[0]<<16) + (localColor[1]<<8) + localColor[2]
-                    print(color)
-
-
-            elif colorizer.type == "RasterClassifyColorizer":
-                print(colorizer.noDataColor)
-                print(colorizer.breakCount) # number of classes 
-                print(colorizer.classBreaks)
-                print(colorizer.classificationField)
-                print(colorizer.classificationMethod)
-                print(colorizer.colorRamp)
-            elif colorizer.type == "RasterUniqueValueColorizer":
-                print(colorizer.noDataColor)
-                print(colorizer.colorRamp)
-                print(colorizer.field)
-                print(colorizer.groups)
-            '''
-            #else:  
-            if hasattr(selectedLayer.symbology, 'colorizer'): # only 1 band 
-                try: bandIndex = int(colorizer.band) # if stretched 
-                except: pass     
-                if colorizer.type =='RasterUniqueValueColorizer': 
-                    # REDO !!!!!!!!!!!!
-                    colorRVal = colorGVal = colorBVal = 0 
-                    try:
-                        for br in colorizer.groups:
-                            print(br.heading) #"Value"
-                            # go through all values classified 
-                            if br.heading != 'Value': print(int('x')) #call exception
-                            for k, itm in enumerate(br.items):
-                                print(itm.values)
-                                if itm.values[0] == rasterBandVals[bandIndex][int(count/4)]:
-                                    print(itm.values[0])
-                                    colorRVal, colorGVal, colorBVal = itm.color['RGB'][0], itm.color['RGB'][1], itm.color['RGB'][2]
-                                    break
-                                # if string covering float
-                                try: 
-                                    if float(itm.values[0]) == float(rasterBandVals[bandIndex][int(count/4)]):
-                                        print(itm.values[0])
-                                        colorRVal, colorGVal, colorBVal = itm.color['RGB'][0], itm.color['RGB'][1], itm.color['RGB'][2]
-                                        break
-                                except Exception as e: print(e); pass
-
-
-                    except Exception as e: # if no Min Max labels:
-                        # REMAP band values to (0,255) range
-                        print(e)
-                        valRange = max(rasterBandVals[bandIndex]) - min(rasterBandVals[bandIndex]) #(rasterBandMaxVal[bandIndex] - rasterBandMinVal[bandIndex])
-                        colorRVal = colorGVal = colorBVal = int( (rasterBandVals[bandIndex][int(count/4)] - min(rasterBandVals[bandIndex])) / valRange * 255 )
-                    
-                    print("__pixel color_")
-                    print(colorRVal)
-                    print(colorGVal)
-                    print(colorBVal)
-                    color =  (colorRVal<<16) + (colorGVal<<8) + colorBVal
-
-                else:
-                    try:
-                        if rasterBandVals[bandIndex][int(count/4)] >= float(colorizer.minLabel) and rasterBandVals[bandIndex][int(count/4)] <= float(colorizer.maxLabel) : #rasterBandMinVal[bandIndex]: 
-                            # REMAP band values to (0,255) range
-                            valRange = float(colorizer.maxLabel) - float(colorizer.minLabel) #(rasterBandMaxVal[bandIndex] - rasterBandMinVal[bandIndex])
-                            colorVal = int( (rasterBandVals[bandIndex][int(count/4)] - float(colorizer.minLabel)) / valRange * 255 )
-                            if colorizer.invertColorRamp is True: colorVal = int( (-rasterBandVals[bandIndex][int(count/4)] + float(colorizer.maxLabel)) / valRange * 255 )
-                            color =  (colorVal<<16) + (colorVal<<8) + colorVal
-                    except: # if no Min Max labels:
-                        # REMAP band values to (0,255) range
-                        valRange = max(rasterBandVals[bandIndex]) - min(rasterBandVals[bandIndex]) #(rasterBandMaxVal[bandIndex] - rasterBandMinVal[bandIndex])
-                        colorVal = int( (rasterBandVals[bandIndex][int(count/4)] - min(rasterBandVals[bandIndex])) / valRange * 255 )
-                        color =  (colorVal<<16) + (colorVal<<8) + colorVal
-            else: # rgb
-                # REMAP band values to (0,255) range
-                if rvalRange is not None and redBand is not None: colorRVal = int( (rasterBandVals[redBand][int(count/4)] - float(rbvalMin)) / rvalRange * 255 )
-                else: colorRVal = 0
-                if gvalRange is not None and greenBand is not None: colorGVal = int( (rasterBandVals[greenBand][int(count/4)] - float(gbvalMin)) / gvalRange * 255 )
-                else: colorGVal = 0
-                if bvalRange is not None and blueBand is not None: colorBVal = int( (rasterBandVals[blueBand][int(count/4)] - float(bbvalMin)) / bvalRange * 255 )
-                else: colorBVal = 0
-                print("__pixel color_")
-                print(colorRVal)
-                print(colorGVal)
-                print(colorBVal)
-
-                color =  (colorRVal<<16) + (colorGVal<<8) + colorBVal
-
-            colors.extend([color,color,color,color])
-            count += 4
-            
-    mesh = rasterToMesh(vertices, faces, colors)
-    if(b['displayValue'] is None):
-        b['displayValue'] = []
-    b['displayValue'].append(mesh)
-
-    return b
-
-r'''
-    # example raster stretch colorizer
-
-    {'type': 'CIMRasterStretchColorizer', 'resamplingType': 'NearestNeighbor', 
-    'noDataColor': {'type': 'CIMRGBColor', 'values': [255, 255, 255, 0]}, 
-    'backgroundColor': {'type': 'CIMRGBColor', 'values': [255, 255, 255, 0]}, 
-    'colorRamp': 
-    {
-      'type': 'CIMMultipartColorRamp',
-      'colorRamps': 
-        [{
-          'type': 'CIMPolarContinuousColorRamp', 
-          'colorSpace': {'type': 'CIMICCColorSpace', 'url': 'Default RGB'}, 
-          'fromColor': {'type': 'CIMHSVColor', 'values': [220, 100, 45, 100]}, 
-          'toColor': {'type': 'CIMHSVColor', 'values': [214, 100, 100, 100]}, 
-          'interpolationSpace': 'HSV', 'polarDirection': 'Counterclockwise'
-          }, 
-          {
-            'type': 'CIMPolarContinuousColorRamp', 
-            'colorSpace': {'type': 'CIMICCColorSpace', 'url': 'Default RGB'}, 
-            'fromColor': {'type': 'CIMHSVColor', 'values': [214.12, 100, 100, 100]}, 
-            'toColor': {'type': 'CIMHSVColor', 'values': [201, 25, 100, 100]}, 
-            'interpolationSpace': 'HSV', 'polarDirection': 'Counterclockwise'
-          }], 
-          'weights': [1, 1]}, 
-    'colorScheme': 'Bathymetry #3', 'customStretchMax': 1, 'gammaValue': 1, 'hillshadeZFactor': 1, 
-    'maxPercent': 2, 'minPercent': 2, 'standardDeviationParam': 2, 'statsType': 'Dataset', 
-    'stretchClasses': [
-      {'type': 'CIMRasterStretchClass', 'label': '3', 'value': 3}, 
-      {'type': 'CIMRasterStretchClass', 'value': 22.5}, 
-      {'type': 'CIMRasterStretchClass', 'label': '42', 'value': 42}
-      ], 
-      'stretchStats': {'type': 'StatsHistogram', 'min': 3, 'max': 42, 'mean': 21.761538461538, 'stddev': 11.387670241563, 'resolution': 0.15294117647058825}, 
-      'stretchType': 'StandardDeviations'}
-    '''
-

speckle_toolbox/esri/toolboxes/speckle/converter/layers/symbologyTemplates.py

@@ -1,606 +0,0 @@
-import json
-from typing import Any, List, Tuple, Union
-import copy
-import os
-
-from typing import Dict
-
-import arcpy 
-from arcpy._mp import ArcGISProject
-from arcpy.management import (CreateFeatureclass, MakeFeatureLayer,
-                              AddFields, AlterField, DefineProjection )
-
-from specklepy.objects import Base
-try:
-    from speckle.converter.layers.Layer import Layer, VectorLayer, RasterLayer 
-except:
-    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.Layer import Layer, VectorLayer, RasterLayer 
-
-def jsonFromLayerStyle(layerArcgis, path_style):
-    # write updated renderer to file and get layerStyle variable 
-    arcpy.management.SaveToLayerFile(layerArcgis, path_style, False)
-    f = open(path_style, "r")
-    layerStyle = json.loads(f.read())
-    f.close()
-    os.remove(path_style)
-    return layerStyle
-
-def symbol_color_to_speckle(color: dict):
-    newColor = (0<<16) + (0<<8) + 0
-    try: 
-        r = int(color['RGB'][0])
-        g = int(color['RGB'][1])
-        b = int(color['RGB'][2])
-        newColor = (r<<16) + (g<<8) + b
-    except: pass 
-    return newColor
-
-def vectorRendererToNative(project: ArcGISProject, active_map, layerGroup, layerSpeckle: Union[Layer, VectorLayer], layerArcgis, f_class, existingAttrs: List) -> Union[None, Dict[str, Any]] :
-    print("___________APPLY VECTOR RENDERER______________")
-    print(layerArcgis)
-    print(f_class)
-    renderer = layerSpeckle.renderer 
-
-    if renderer and renderer['type']:
-        print(renderer['type'])
-
-        root_path = "\\".join(project.filePath.split("\\")[:-1])
-        #path_style = root_path + '\\' + str(f_class).split('\\')[-1] + '_old.lyrx'
-
-        data = arcpy.Describe(layerArcgis.dataSource)
-        if layerArcgis.isFeatureLayer:
-            geomType = data.shapeType
-            sym = layerArcgis.symbology
-
-            if renderer['type'] == 'singleSymbol':
-                print("RENDERER SINGLE")
-                print(renderer)
-
-                r,g,b = get_rgb_from_speckle(renderer['properties']['symbol']['symbColor']) 
-                #print(r,g,b)
-                #print(sym.renderer.symbol.color)
-                sym.renderer.symbol.color = {'RGB': [r, g, b, 100]}
-                #print(sym.renderer.symbol.color)
-                layerArcgis.symbology = sym # SimpleRenderer
-                #print(layerArcgis)
-                return layerArcgis 
-
-            elif renderer['type']  == 'categorizedSymbol':
-                print("RENDERER CATEGORIZED")
-                print(renderer)
-
-                cats = renderer['properties']['categories']
-                attribute = renderer['properties']['attribute']
-                if attribute not in existingAttrs: return layerArcgis 
-
-                #vl2 = active_map.addLayer(layerArcgis)[0]
-                #sym = layerArcgis.symbology
-                sym.updateRenderer('UniqueValueRenderer')
-                print(sym.renderer.type)
-                print(existingAttrs)
-                print(attribute)
-
-                sym.renderer.fields = [attribute]
-                for k, grp in enumerate(sym.renderer.groups):
-                    for itm in grp.items:
-                        transVal = itm.values[0][0] #Grab the first "percent" value in the list of potential values
-                        for i in range(len(cats)):
-                            label = cats[i]['value'] 
-                            if label is None or label=="" or str(label)=="": label = "<Null>"
-                            r,g,b = get_rgb_from_speckle(cats[i]['symbColor']) 
-
-                            if str(transVal) == label:
-                                itm.symbol.color = {'RGB': [r, g, b, 100]}
-                                itm.label = label
-                                break
-                layerArcgis.symbology = sym
-                return layerArcgis 
-
-            elif renderer['type'] == 'graduatedSymbol': 
-                print("RENDERER GRADUATED")
-                print(renderer)
-
-                attribute = renderer['properties']['attribute']
-                gradMetod = renderer['properties']['gradMethod'] # by color or by size
-                if gradMetod  != 0: 
-                    r,g,b = get_rgb_from_speckle(renderer['properties']['sourceSymbColor'] ) 
-                    sym.renderer.symbol.color = {'RGB': [r, g, b, 100]}
-                    layerArcgis.symbology = sym # SimpleRenderer
-                    return layerArcgis 
-                if attribute not in existingAttrs or gradMetod != 0: return layerArcgis # by color, not line width 
-
-                sym.updateRenderer('GraduatedColorsRenderer')
-                print(sym.renderer.type)
-
-                r,g,b = get_rgb_from_speckle(renderer['properties']['sourceSymbColor']) 
-                ramp = renderer['properties']['ramp'] # {discrete, rampType, stops}
-                ranges = renderer['properties']['ranges'] # []
-
-                # get all existing values 
-                all_values = []
-                with arcpy.da.UpdateCursor(f_class, attribute) as cur:
-                    for rowShape in cur: 
-                        all_values.append(rowShape[0])
-                    print(all_values)
-                del cur 
-
-                print(len(ranges))
-                sym.renderer.classificationField = attribute
-                print(sym.renderer.breakCount)
-                sym.renderer.breakCount = len(ranges)
-                print(sym.renderer.breakCount)
-                
-                if len(sym.renderer.classBreaks) > 0:
-                    totalClasses = 0
-                    for k, br in enumerate(ranges):
-                        
-                        print(totalClasses)
-                        if sym.renderer.breakCount < len(ranges):
-                            valFits = 0
-                            # check if any existing value fits in this range:
-                            for val in all_values: 
-                                if val <= ranges[k]["upper"] and (totalClasses==0 or (totalClasses>0 and sym.renderer.classBreaks[totalClasses-1].upperBound<val)):
-                                    valFits+=1
-                                    break
-                            if valFits == 0: continue
-                            
-                        r,g,b = get_rgb_from_speckle(ranges[k]['symbColor']) 
-
-                        #classBreak.upperBound = ranges[k]["upper"]
-                        sym.renderer.classBreaks[totalClasses].upperBound = ranges[k]["upper"]
-                        sym.renderer.classBreaks[totalClasses].label = ranges[k]["label"]
-                        sym.renderer.classBreaks[totalClasses].symbol.color = {'RGB': [r, g, b, 100]}
-                        totalClasses += 1 
-
-                        #layerArcgis.symbology = sym
-                        print(ranges[k]["label"])
-                        print(ranges[k]["upper"])
-
-                    sym.renderer.classBreaks[0].upperBound = ranges[0]["upper"] # otherwise its assigned maximum value
-
-                layerArcgis.symbology = sym
-                return layerArcgis 
-
-            else: return None 
-
-def get_rgb_from_speckle(rgb: int) -> Tuple[int, int, int]:
-    r = g = b = 0
-    try: 
-        r = (rgb & 0xFF0000) >> 16
-        g = (rgb & 0xFF00) >> 8
-        b = rgb & 0xFF 
-    except: r = g = b = 0
-    
-    r,g,b = check_rgb(r,g,b)
-    return r,g,b 
-
-def check_rgb(r:int, g:int, b:int) -> Tuple[int, int, int]:
-
-    if not isinstance(r, int) or r<0 or r>255: r=g=b=0
-    if not isinstance(g, int) or g<0 or g>255: r=g=b=0
-    if not isinstance(b, int) or b<0 or b>255: r=g=b=0
-    return r,g,b 
-
-
-
-def rasterRendererToNative(project: ArcGISProject, active_map, layerGroup,  layer: RasterLayer, arcLayer, rasterPathsToMerge, newName):
-    print("_____rasterRenderer ToNative______")
-    renderer = layer.renderer
-    rendererNew = None
-    print(renderer)
-
-    feat = layer.features[0]
-    print(feat)
-
-    bandNames = feat["Band names"]
-    print(bandNames)
-
-    sym = arcLayer.symbology
-    symJson = None
-    path_style = ""
-    path_style2 = ""
-
-    print(sym)
-
-    if renderer and renderer['type']:
-        
-        if not hasattr(arcLayer.symbology, 'colorizer'): 
-            # multiband raster, CIMRasterRGBColorizer 
-            # arcpy doesnt support multiband raster symbology: https://community.esri.com/t5/arcgis-api-for-python-questions/why-does-arcpy-mp-arcgis-pro-2-6-mosaic-dataset/td-p/1016312 
-            root_path = "\\".join(project.filePath.split("\\")[:-1])
-            if not os.path.exists(root_path + '\\Layers_Speckle\\raster_bands'): os.makedirs(root_path + '\\Layers_Speckle\\raster_bands')
-            path_style = root_path + '\\Layers_Speckle\\raster_bands\\' + newName + '_old.lyrx'
-            path_style2 = root_path + '\\Layers_Speckle\\raster_bands\\' + newName + '_new.lyrx'
-            symJson = jsonFromLayerStyle(arcLayer, path_style)
-
-        if renderer['type']  == 'singlebandgray':
-            print("Singleband grey")
-            band_index = renderer['properties']['band']-1
-            if symJson is None:
-                sym.updateColorizer('RasterStretchColorizer')
-                sym.colorizer.band = band_index
-                arcLayer.symbology = sym
-            else: 
-                temp = arcpy.management.MakeRasterLayer(rasterPathsToMerge[band_index], newName + "_temp").getOutput(0)
-                active_map.addLayerToGroup(layerGroup, temp)
-                temp_layer = None
-                for l in active_map.listLayers(): 
-                    if l.longName == layerGroup.longName + "\\" + newName + "_temp":
-                        print(l.longName)
-                        temp_layer = l 
-                        break
-
-                sym = temp_layer.symbology
-                sym.updateColorizer('RasterStretchColorizer')
-                sym.colorizer.band = band_index
-                arcLayer.symbology = sym
-
-                active_map.removeLayer(temp_layer) 
-                
-
-        elif renderer['type']  == 'multibandcolor':
-            print("Multiband")
-            if symJson is None: 
-                sym.updateColorizer('RasterStretchColorizer')
-                arcLayer.symbology = sym 
-            else:
-                
-                redSt =  copy.deepcopy(symJson["layerDefinitions"][0]["colorizer"]["stretchStatsRed"])
-                greenSt =  copy.deepcopy(symJson["layerDefinitions"][0]["colorizer"]["stretchStatsGreen"])
-                blueSt =  copy.deepcopy(symJson["layerDefinitions"][0]["colorizer"]["stretchStatsBlue"])
-
-                redBand = renderer['properties']['redBand']
-                greenBand = renderer['properties']['greenBand']
-                blueBand = renderer['properties']['blueBand']
-                try: symJson["layerDefinitions"][0]["colorizer"]["greenBandIndex"] = greenBand-1
-                except: symJson["layerDefinitions"][0]["colorizer"]["greenBandIndex"] = 0
-
-                try: symJson["layerDefinitions"][0]["colorizer"]["redBandIndex"] = redBand-1
-                except: symJson["layerDefinitions"][0]["colorizer"]["redBandIndex"] = 0
-                
-                try: symJson["layerDefinitions"][0]["colorizer"]["blueBandIndex"] = blueBand-1
-                except: symJson["layerDefinitions"][0]["colorizer"]["blueBandIndex"] = 0
-                
-                print(symJson)
-                f = open(path_style2, "w")
-                f.write(json.dumps(symJson, indent=2))
-                f.close()
-
-                active_map.removeLayer(arcLayer) 
-                lyrFile = arcpy.mp.LayerFile(path_style2)
-                active_map.addLayerToGroup(layerGroup, lyrFile )
-                
-                os.remove(path_style2) 
-
-        elif renderer['type']  == 'paletted':
-            print("Paletted")
-            band_index = renderer['properties']['band']-1
-
-            if symJson is None:
-                for br in sym.colorizer.groups:
-                    print(br.heading) #"Value"
-                    # go through all values classified 
-                    for k, itm in enumerate(br.items):
-                        if k< len(renderer['properties']['classes']):
-                            #go through saved renderer classes
-                            for n, cl in enumerate(renderer['properties']['classes']):
-                                if k == n:
-                                    r,g,b = get_rgb_from_speckle(cl['color']) 
-                                    itm.color = {'RGB': [r,g,b, 100]}
-                                    itm.label = cl['label']
-                                    itm.values = cl['value']
-                        else: pass
-                arcLayer.symbology = sym 
-        else: 
-            sym.updateColorizer('RasterStretchColorizer')
-            arcLayer.symbology = sym
-    return arcLayer
-    
-
-def rendererToSpeckle(project: ArcGISProject, active_map, arcLayer, rasterFeat: Base):
-    print("_____rasterRenderer To Speckle______")
-    if arcLayer.isRasterLayer: 
-        try: 
-            rType = arcLayer.symbology.colorizer.type # 'singleSymbol','categorizedSymbol','graduatedSymbol',
-            if rType =='RasterStretchColorizer':  rType = 'singlebandgray'
-            elif rType =='RasterUniqueValueColorizer':  rType = 'paletted' # only for 1-band raster 
-            else: rType = 'singlebandgray'
-        except: 
-            rType = "multibandcolor"
-            root_path = "\\".join(project.filePath.split("\\")[:-1])
-            if not os.path.exists(root_path + '\\Layers_Speckle\\raster_bands'): os.makedirs(root_path + '\\Layers_Speckle\\raster_bands')
-            path_style = root_path + '\\Layers_Speckle\\raster_bands\\' + arcLayer.name + '_temp.lyrx'
-            #path_style2 = root_path + '\\' + newName + '_new.lyrx'
-            symJson = jsonFromLayerStyle(arcLayer, path_style)
-
-        layerRenderer: Dict[str, Any] = {}
-        layerRenderer['type'] = rType
-        print(rType)
-        my_raster = arcpy.Raster(arcLayer.dataSource)  
-        rasterBandNames = my_raster.bandNames
-        
-        #bandNames = rasterFeat["Band names"]
-        bandValues = [rasterFeat["@(10000)" + name + "_values"] for name in rasterBandNames]
-
-        if rType == "singlebandgray":  
-            try: band = arcLayer.symbology.colorizer.band 
-            except: band = 0 
-            try: 
-                bVals = bandValues[band]
-                bvalMin = min(bVals)
-                bvalMax = max(bVals)
-            except: 
-                bvalMin = 0
-                bvalMax = 255
-            layerRenderer.update({'properties': {'max':bvalMax,'min':bvalMin,'band':band+1,'contrast':1}}) 
-
-        elif rType == "multibandcolor":
-
-            try: greenBand = symJson["layerDefinitions"][0]["colorizer"]["greenBandIndex"] +1
-            except: greenBand = None
-            try: blueBand = symJson["layerDefinitions"][0]["colorizer"]["blueBandIndex"] +1
-            except: blueBand = None
-            try: redBand = symJson["layerDefinitions"][0]["colorizer"]["redBandIndex"] +1
-            except: 
-                print(greenBand)
-                print(blueBand)
-                if blueBand!=1 and greenBand!=1: redBand= 1
-                else: redBand = None
-            print(redBand)
-
-            try: 
-                rbVals = bandValues[redBand-1]
-                rbvalMin = min(rbVals)
-                rbvalMax = max(rbVals)
-                print(rbvalMin)
-                print(rbvalMax)
-            except: 
-                rbvalMin = 0
-                rbvalMax = 255
-            try: 
-                gbVals = bandValues[greenBand-1]
-                gbvalMin = min(gbVals)
-                gbvalMax = max(gbVals)
-            except: 
-                gbvalMin = 0
-                gbvalMax = 255
-            try: 
-                bbVals = bandValues[blueBand-1]
-                bbvalMin = min(bbVals)
-                bbvalMax = max(bbVals) 
-            except: 
-                bbvalMin = 0
-                bbvalMax = 255
-
-            layerRenderer.update({'properties': {'greenBand':greenBand,'blueBand':blueBand,'redBand':redBand}})
-            layerRenderer['properties'].update({'redContrast':1,'redMin':rbvalMin,'redMax':rbvalMax})
-            layerRenderer['properties'].update({'greenContrast':1,'greenMin':gbvalMin,'greenMax':gbvalMax})
-            layerRenderer['properties'].update({'blueContrast':1,'blueMin':bbvalMin,'blueMax':bbvalMax})
-        elif rType == "paletted":  
-            band = 0 
-            rendererClasses = arcLayer.symbology.colorizer.groups
-            classes = []
-            sourceRamp = {}
-
-            for i, cl in enumerate(rendererClasses):
-                if cl.heading == 'Value':
-                    for k, itm in enumerate(cl.items):
-                        value = itm.values[0]
-                        label = itm.label 
-                        try: 
-                            r,g,b = itm.color['RGB'][0], itm.color['RGB'][1], itm.color['RGB'][2]
-                            sColor = (r<<16) + (g<<8) + b
-                            classes.append({'color':sColor,'value':value,'label':label})
-                        except: pass 
-            layerRenderer.update({'properties': {'classes':classes,'ramp':sourceRamp,'band':band+1}})
-
-        return layerRenderer 
-    elif arcLayer.isFeatureLayer: 
-        layerRenderer: Dict[str, Any] = {}
-
-        sym = arcLayer.symbology
-        print(sym.renderer.type)
-
-        if sym.renderer.type == 'SimpleRenderer':
-            layerRenderer['type'] = 'singleSymbol'
-            layerRenderer['properties'] = {'symbol':{}, 'symbType':""}
-            symbolColor = symbol_color_to_speckle(sym.renderer.symbol.color)
-            layerRenderer['properties'].update({'symbol':{'symbColor': symbolColor}, 'symbType':''})
-
-        elif sym.renderer.type == 'UniqueValueRenderer':
-            layerRenderer['type'] = 'categorizedSymbol'
-            layerRenderer['properties'] = {'attribute': '', 'symbType': ''} #{'symbol':{}, 'ramp':{}, 'ranges':{}, 'gradMethod':"", 'symbType':"", 'legendClassificationAttribute': ""}
-            
-            attribute = sym.renderer.fields[0]
-            layerRenderer['properties']['attribute'] = attribute
-            sourceSymbColor = symbol_color_to_speckle(sym.renderer.defaultSymbol.color)
-            layerRenderer['properties'].update( {'sourceSymbColor': sourceSymbColor} )
- 
-            categories = sym.renderer.groups
-            layerRenderer['properties']['categories'] = []
-
-            for i, grp in enumerate(categories):
-                for itm in grp.items:
-                    value = itm.values[0][0]
-                    symbColor = symbol_color_to_speckle(itm.symbol.color)
-                    label = itm.label
-                    layerRenderer['properties']['categories'].append({'value':value,'symbColor':symbColor,'symbOpacity':1, 'sourceSymbColor': sourceSymbColor,'label':label})
-
-        elif sym.renderer.type == 'GraduatedColorsRenderer' or sym.renderer.type == 'GraduatedSymbolsRenderer':
-            layerRenderer['type'] = 'graduatedSymbol'
-            layerRenderer['properties'] = {'symbol':{}, 'ramp':{}, 'ranges':{}, 'gradMethod':"", 'symbType':""}
-        
-            attribute = sym.renderer.classificationField 
-            sourceSymbColor = (0<<16) + (0<<8) + 0  
-            layerRenderer['properties'].update( {'attribute': attribute, 'symbType': '', 'gradMethod': 0, 'sourceSymbColor': sourceSymbColor} )
-
-            rRamp = sym.renderer.colorRamp # QgsGradientColorRamp
-            layerRenderer['properties']['ramp'] = {} # gradientColorRampToSpeckle(rRamp)
-
-            rRanges = sym.renderer.classBreaks
-            layerRenderer['properties']['ranges'] = []
-            for itm in rRanges:
-                try: lower = float(itm.label.split(" - ")[0]) if (" - " in rRanges.label) else float(rRanges.label[0])
-                except: lower = 0
-                upper = itm.upperBound
-                symbColor = symbol_color_to_speckle(itm.symbol.color) 
-                label = itm.label
-                width = 0.26
-                # {'label': '1 - 1.4', 'lower': 1.0, 'symbColor': <PyQt5.QtGui.QColor ...BD9B9D4A0>, 'symbOpacity': 1.0, 'upper': 1.4}
-                layerRenderer['properties']['ranges'].append({'lower':lower,'upper':upper,'symbColor':symbColor,'symbOpacity':1,'label':label,'width':width})
-     
-        elif sym.renderer.type == 'UnclassedColorsRenderer':
-            layerRenderer['type'] = 'graduatedSymbol'
-            layerRenderer['properties'] = {'symbol':{}, 'ramp':{}, 'ranges':{}, 'gradMethod':"", 'symbType':""}
-        
-            attribute = sym.renderer.field 
-            sourceSymbColor = (0<<16) + (0<<8) + 0  
-            layerRenderer['properties'].update( {'attribute': attribute, 'symbType': '', 'gradMethod': 0, 'sourceSymbColor': sourceSymbColor} )
-            layerRenderer['properties']['ramp'] = {} # gradientColorRampToSpeckle(rRamp)
-
-            lowest = sym.renderer.lowerLabel
-            highest = sym.renderer.upperLabel
-
-            # trick to get colors 
-            rRamp = sym.renderer.colorRamp # QgsGradientColorRamp
-            arcRamp = project.listColorRamps('White to Black')[0]
-            sym.updateRenderer('GraduatedColorsRenderer')
-            sym.renderer.colorRamp = arcRamp
-            sym.renderer.classificationField = attribute
-            rows_attributes = arcpy.da.SearchCursor(arcLayer.longName, attribute)
-            row_attrs = []
-            row_max = -1000000000
-            row_min = 1000000000
-            for k, attrs in enumerate(rows_attributes): 
-                row_attrs.append(attrs[0])
-                if attrs[0] < row_min: row_min = attrs[0]
-                if attrs[0] > row_max: row_max = attrs[0]
-            row_range = row_max - row_min
-            breakCount = len(list(set(row_attrs))) # only unique values 
-            sym.renderer.breakCount = breakCount
-
-            # run as gradient colors 
-            rRanges = sym.renderer.classBreaks
-            layerRenderer['properties']['ranges'] = []
-            for itm in rRanges:
-                try: lower = float(itm.label.split(" - ")[0]) if (" - " in rRanges.label) else float(rRanges.label[0])
-                except: lower = 0
-                upper = itm.upperBound
-                if row_range==0: rgb = 0
-                else: rgb = 255 - int((itm.upperBound - row_min) / row_range * 255 ) 
-                symbColor = (rgb<<16) + (rgb<<8) + rgb 
-                label = itm.label
-                width = 0.26
-                # {'label': '1 - 1.4', 'lower': 1.0, 'symbColor': <PyQt5.QtGui.QColor ...BD9B9D4A0>, 'symbOpacity': 1.0, 'upper': 1.4}
-                layerRenderer['properties']['ranges'].append({'lower':lower,'upper':upper,'symbColor':symbColor,'symbOpacity':1,'label':label,'width':width})
-
-        else: return None
-            
-        return layerRenderer
-
-    else: return None
-
-
-def featureColorfromNativeRenderer(index: int, arcLayer) -> int:
-    # case with one color for the entire layer
-    #try:
-    sym = arcLayer.symbology
-    color = {'RGB': [100,100,100,100]}
-
-    if sym.renderer.type == 'SimpleRenderer':
-        print('SimpleRenderer')
-        color = sym.renderer.symbol.color
-
-    elif sym.renderer.type == 'UniqueValueRenderer':
-        print('Unique Value Renderer')
-
-        attribute = sym.renderer.fields[0]
-        color = sym.renderer.defaultSymbol.color
-        categories = sym.renderer.groups
-
-        rows_attributes = arcpy.da.SearchCursor(arcLayer.longName, attribute)
-        row_shapes_list = [x for k, x in enumerate(rows_attributes)]
-
-        color_found = 0
-        for i, grp in enumerate(categories):
-            if color_found == 1: break
-            for n, itm in enumerate(grp.items):
-                for k, attrs in enumerate(row_shapes_list): 
-                    if str(itm.values[0][0]) == "<Null>": itm.values[0][0] = None 
-                    if k == index and ( str(attrs[0]) == str(itm.values[0][0]) or (attrs[0] is None and str(itm.values[0][0]) == "<Null>") ): 
-                        color = itm.symbol.color
-                        print("symbol color: ")
-                        print(color)
-                        color_found = 1
-                        break 
-
-    elif sym.renderer.type == 'GraduatedColorsRenderer' or sym.renderer.type == 'GraduatedSymbolsRenderer':
-        print('Graduated Colors / Sybmols Renderer')
-        attribute = sym.renderer.classificationField 
-        rows_attributes = arcpy.da.SearchCursor(arcLayer.longName, attribute)
-        row_shapes_list = [x for k, x in enumerate(rows_attributes)]
-
-        rRanges = sym.renderer.classBreaks
-        upperBounds = [-10000000000000000000]
-        color_found = 0
-        for itm in rRanges:
-            print(itm)
-            if color_found == 1: break
-            for k, attrs in enumerate(row_shapes_list):  
-                try:
-                    if k == index and float(attrs[0]) <= float(itm.upperBound) and (k==0 or float(attrs[0]) > float(upperBounds[-1]) ):
-                        color = itm.symbol.color
-                        color_found = 1
-                        break 
-                except: pass
-            upperBounds.append(itm.upperBound)
-
-    elif sym.renderer.type == 'UnclassedColorsRenderer': 
-        print('UnclassedColorsRenderer')
-        attribute = sym.renderer.field  
-
-        sym.updateRenderer('GraduatedColorsRenderer')
-        sym.renderer.classificationField = attribute
-
-        rows_attributes = arcpy.da.SearchCursor(arcLayer.longName, attribute)
-        row_shapes_list = [x for k, x in enumerate(rows_attributes)]
-        row_attrs = []
-        row_max = -10000000000000000000
-        row_min = 10000000000000000000
-        feat_value = None
-        for k, attrs in enumerate(row_shapes_list): 
-            row_attrs.append(attrs[0])
-            if attrs[0] < row_min: row_min = attrs[0]
-            if attrs[0] > row_max: row_max = attrs[0]
-            if k == index: feat_value = attrs[0]
-        row_range = row_max - row_min
-        breakCount = len(list(set(row_attrs))) # only unique values 
-        sym.renderer.breakCount = breakCount
-
-        # run as gradient colors 
-        
-        upperBounds = [-10000000000000000000]
-        rRanges = sym.renderer.classBreaks
-
-        for itm in rRanges:
-            print(itm)
-            try:
-                if row_range!=0 and float(feat_value) <= float(itm.upperBound) and (len(upperBounds)==0 or float(feat_value) > float(upperBounds[-1])):
-                    rgb = 255 - int((itm.upperBound - row_min) / row_range * 255 ) 
-                    color = {'RGB':[rgb,rgb,rgb,100]}
-                    print(color) 
-                    break
-            except: pass 
-            upperBounds.append(itm.upperBound)
-
-    else: 
-        print('Else')
-        return (100<<16) + (100<<8) + 100
-    
-    print("final color: ")
-    print(color) 
-    # construct RGB color
-    col = symbol_color_to_speckle(color)
-    print(col)
-    return col
-

speckle_toolbox/esri/toolboxes/speckle/converter/layers/utils.py

@@ -1,308 +0,0 @@
-from typing import Dict, Any, List, Union
-import json 
-from specklepy.objects import Base
-import arcpy 
-
-from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
-import os
-
-ATTRS_REMOVE = ['geometry','applicationId','bbox','displayStyle', 'id', 'renderMaterial', 'displayMesh', 'displayValue'] 
-
-def getVariantFromValue(value: Any) -> Union[str, None]:
-    #print("_________get variant from value_______")
-    # TODO add Base object
-    pairs = [
-        (str, "TEXT"), # 10
-        (float, "FLOAT"),
-        (int, "LONG"),
-        (bool, "SHORT") 
-    ]
-    res = None
-    for p in pairs:
-        if isinstance(value, p[0]): 
-            res = p[1]
-            try:
-                if res == "LONG" and (value>= 2147483647 or value<= -2147483647):
-                    #https://pro.arcgis.com/en/pro-app/latest/help/data/geodatabases/overview/arcgis-field-data-types.htm
-                    res = "FLOAT"
-            except Exception as e: print(e)
-            break
-
-    return res
-
-def getLayerAttributes(featuresList: List[Base], attrsToRemove: List[str] = ATTRS_REMOVE ) -> Dict[str, str]:
-    print("03________ get layer attributes")
-
-    if not isinstance(featuresList, list): features = [featuresList]
-    else: features = featuresList[:]
-    
-    fields = {}
-    all_props = []
-    for feature in features: 
-        #get object properties to add as attributes
-        dynamicProps = feature.get_dynamic_member_names()
-        for att in attrsToRemove:
-            try: dynamicProps.remove(att)
-            except: pass
-        dynamicProps.sort()
-
-        # add field names and variands 
-        for name in dynamicProps:
-            #if name not in all_props: all_props.append(name)
-
-            value = feature[name]
-            variant = getVariantFromValue(value)
-            #if name == 'area': print(value); print(variant)
-            if not variant: variant = None #LongLong #4 
-
-            # go thought the dictionary object
-            if value and isinstance(value, list):
-                #all_props.remove(name) # remove generic dict name
-                for i, val_item in enumerate(value):
-                    newF, newVals = traverseDict( {}, {}, name+"_"+str(i), val_item)
-
-                    for i, (k,v) in enumerate(newF.items()):
-                        if k not in all_props: all_props.append(k)
-                        if k not in fields.keys(): fields.update({k: v}) 
-                        else: #check if the field was empty previously: 
-                            oldVariant = fields[k]
-                            # replace if new one is NOT Float (too large integers)
-                            if oldVariant != "FLOAT" and v == "FLOAT": 
-                                fields.update({k: v}) 
-                            # replace if new one is NOT LongLong or IS String
-                            if oldVariant != "TEXT" and v == "TEXT": 
-                                fields.update({k: v}) 
-            
-            # add a field if not existing yet 
-            else: # if str, Base, etc
-                newF, newVals = traverseDict( {}, {}, name, value)
-                
-                for i, (k,v) in enumerate(newF.items()):
-                    if k not in all_props: all_props.append(k)
-                    if k not in fields.keys(): fields.update({k: v}) #if variant is known
-                    else: #check if the field was empty previously: 
-                        oldVariant = fields[k]
-                        # replace if new one is NOT Float (too large integers)
-                        #print(oldVariant, v)
-                        if oldVariant == "LONG" and v == "FLOAT": 
-                            fields.update({k: v}) 
-                        # replace if new one is NOT LongLong or IS String
-                        if oldVariant != "TEXT" and v == "TEXT": 
-                            fields.update({k: v}) 
-                        #print(fields)
-    # replace all empty ones wit String
-    all_props.append("Speckle_ID") 
-    for name in all_props:
-        if name not in fields.keys(): 
-            fields.update({name: 'TEXT'}) 
-
-    fields_sorted = {k: v for k, v in sorted(fields.items(), key=lambda item: item[0])}
-    return fields_sorted
-
-def traverseDict(newF: dict, newVals: dict, nam: str, val: Any):
-    
-    if isinstance(val, dict):
-        for i, (k,v) in enumerate(val.items()):
-            newF, newVals = traverseDict( newF, newVals, nam+"_"+k, v)
-    elif isinstance(val, Base):
-        dynamicProps = val.get_dynamic_member_names()
-        for att in ATTRS_REMOVE:
-            try: dynamicProps.remove(att)
-            except: pass
-        dynamicProps.sort()
-
-        item_dict = {} 
-        for prop in dynamicProps:
-            item_dict.update({prop: val[prop]})
-
-        for i, (k,v) in enumerate(item_dict.items()):
-            newF, newVals = traverseDict( newF, newVals, nam+"_"+k, v)
-    else: 
-        var = getVariantFromValue(val)
-        if var is None: 
-            var = 'TEXT'
-            val = str(val)
-        #print(var)
-        newF.update({nam: var})
-        newVals.update({nam: val})  
-
-    return newF, newVals
-
-def get_scale_factor(units: str) -> float:
-    unit_scale = {
-    "meters": 1.0,
-    "centimeters": 0.01,
-    "millimeters": 0.001,
-    "inches": 0.0254,
-    "feet": 0.3048,
-    "kilometers": 1000.0,
-    "mm": 0.001,
-    "cm": 0.01,
-    "m": 1.0,
-    "km": 1000.0,
-    "in": 0.0254,
-    "ft": 0.3048,
-    "yd": 0.9144,
-    "mi": 1609.340,
-    }
-    if units is not None and units.lower() in unit_scale.keys():
-        return unit_scale[units]
-    arcpy.AddWarning(f"Units {units} are not supported. Meters will be applied by default.")
-    return 1.0
-
-def findTransformation(f_shape, geomType, layer_sr: arcpy.SpatialReference, projectCRS: arcpy.SpatialReference, selectedLayer: arcLayer):
-    #apply transformation if needed
-    if layer_sr.name != projectCRS.name:
-        tr0 = tr1 = tr2 = tr_custom = None
-        print(layer_sr)
-        try:
-            transformations = arcpy.ListTransformations(layer_sr, projectCRS)
-            customTransformName = "layer_sr.name"+"_To_"+ projectCRS.name
-            if len(transformations) == 0:
-                midSr = arcpy.SpatialReference("WGS 1984") # GCS_WGS_1984
-                try:
-                    tr1 = arcpy.ListTransformations(layer_sr, midSr)[0]
-                    tr2 = arcpy.ListTransformations(midSr, projectCRS)[0]
-                except: 
-                    #customGeoTransfm = "GEOGTRAN[METHOD['Geocentric_Translation'],PARAMETER['X_Axis_Translation',''],PARAMETER['Y_Axis_Translation',''],PARAMETER['Z_Axis_Translation','']]"
-                    #CreateCustomGeoTransformation(customTransformName, layer_sr, projectCRS)
-                    tr_custom = customTransformName
-            else: 
-                #print("else")
-                # choose equation based instead of file-based/grid-based method, 
-                # to be consistent with QGIS: https://desktop.arcgis.com/en/arcmap/latest/map/projections/choosing-an-appropriate-transformation.htm
-                selecterTr = {}
-                for tr in transformations:
-                    if "NTv2" not in tr and "NADCON" not in tr: 
-                        set1 = set( layer_sr.name.split("_") + projectCRS.name.split("_") )
-                        set2 = set( tr.split("_") )
-                        diff = len( set(set1).symmetric_difference(set2) )
-                        selecterTr.update({tr: diff})
-                selecterTr = dict(sorted(selecterTr.items(), key=lambda item: item[1]))
-                tr0 = list(selecterTr.keys())[0]
-
-            if geomType != "Point" and geomType != "Polyline" and geomType != "Polygon" and geomType != "Multipoint":
-                try: arcpy.AddWarning("Unsupported or invalid geometry in layer " + selectedLayer.name)
-                except: arcpy.AddWarning("Unsupported or invalid geometry")
-
-            # reproject geometry using chosen transformstion(s)
-            if tr0 is not None:
-                ptgeo1 = f_shape.projectAs(projectCRS, tr0)
-                f_shape = ptgeo1
-            elif tr1 is not None and tr2 is not None:
-                ptgeo1 = f_shape.projectAs(midSr, tr1)
-                ptgeo2 = ptgeo1.projectAs(projectCRS, tr2)
-                f_shape = ptgeo2
-            else:
-                ptgeo1 = f_shape.projectAs(projectCRS)
-                f_shape = ptgeo1
-        
-        except:
-            arcpy.AddWarning(f"Spatial Transformation not found for layer {selectedLayer.name}")
-            return None
-
-    return f_shape    
-
-def traverseDictByKey(d: Dict, key:str ="", result = None) -> Dict:
-    print("__traverse")
-
-    result = None
-    #print(d)
-    for k, v in d.items():
-        
-        try: v = json.loads(v)
-        except: pass 
-        if isinstance(v, dict):
-            #print("__dict__")
-            if k == key: print("__break loop"); result = v; return result
-            else: 
-                result = traverseDictByKey(v, key, result)
-                if result is not None: return result
-        if isinstance(v, list):
-            for item in v: 
-                #print(item) 
-                if isinstance(item, dict): 
-                    result = traverseDictByKey(item, key, result)
-                    if result is not None: return result
-    #print("__result is: ____________")
-    #return result 
-
-def hsv_to_rgb(listHSV):
-    h, s, v = listHSV[0], listHSV[1], listHSV[2]
-    if s == 0.0: v*=255; return (v, v, v)
-    i = int(h*6.) # XXX assume int() truncates!
-    f = (h*6.)-i; p,q,t = int(255*(v*(1.-s))), int(255*(v*(1.-s*f))), int(255*(v*(1.-s*(1.-f)))); v*=255; i%=6
-    if i == 0: return (v, t, p)
-    if i == 1: return (q, v, p)
-    if i == 2: return (p, v, t)
-    if i == 3: return (p, q, v)
-    if i == 4: return (t, p, v)
-    if i == 5: return (v, p, q)
-
-def cmyk_to_rgb(c, m, y, k, cmyk_scale, rgb_scale=255):
-    r = rgb_scale * (1.0 - c / float(cmyk_scale)) * (1.0 - k / float(cmyk_scale))
-    g = rgb_scale * (1.0 - m / float(cmyk_scale)) * (1.0 - k / float(cmyk_scale))
-    b = rgb_scale * (1.0 - y / float(cmyk_scale)) * (1.0 - k / float(cmyk_scale))
-    return r, g, b
-
-def newLayerGroupAndName(layerName: str, streamBranch: str, project: ArcGISProject) -> str:
-    print("___new Layer Group and Name")
-    #CREATE A GROUP "received blabla" with sublayers
-    layerGroup = None
-    newGroupName = f'{streamBranch}'
-    print(newGroupName)
-    for l in project.activeMap.listLayers():
-        if l.longName == newGroupName: layerGroup = l; break 
-    
-    #find a layer with a matching name in the "latest" group 
-    newName = f'{streamBranch.split("_")[len(streamBranch.split("_"))-1]}_{layerName}'
-
-    all_layer_names = []
-    layerExists = 0
-    for l in project.activeMap.listLayers(): 
-        if l.longName.startswith(newGroupName + "\\"):
-            all_layer_names.append(l.longName)
-    #print(all_layer_names)
-    print(newName)
-
-    longName = streamBranch + "\\" + newName 
-    if longName in all_layer_names: 
-        for index, letter in enumerate('234567890abcdefghijklmnopqrstuvwxyz'):
-            if (longName + "_" + letter) not in all_layer_names: 
-                newName += "_"+letter 
-                layerExists +=1 
-                break 
-    print(newName)              
-    return newName, layerGroup 
-
-
-def curvedFeatureClassToSegments(layer) -> str:
-    print("___densify___")
-    data = arcpy.Describe(layer.dataSource)
-    dataPath = data.catalogPath
-    print(dataPath)
-    newPath = dataPath+"_backup"
-
-    arcpy.management.CopyFeatures(dataPath, newPath) # features copied like this do not preserve curved segments
-
-    arcpy.edit.Densify(in_features = newPath, densification_method = "ANGLE", max_angle = 0.01, max_vertex_per_segment = 100) # https://pro.arcgis.com/en/pro-app/latest/tool-reference/editing/densify.htm
-    print(newPath)
-    return newPath
-
-def validate_path(path: str):
-    # https://github.com/EsriOceans/btm/commit/a9c0529485c9b0baa78c1f094372c0f9d83c0aaf
-    """If our path contains a DB name, make sure we have a valid DB name and not a standard file name."""
-    dirname, file_name = os.path.split(path)
-    #print(dirname)
-    #print(file_name)
-    file_base = os.path.splitext(file_name)[0]
-    if dirname == '':
-        # a relative path only, relying on the workspace
-        dirname = arcpy.env.workspace
-    path_ext = os.path.splitext(dirname)[1].lower()
-    if path_ext in ['.mdb', '.gdb', '.sde']:
-        # we're working in a database
-        file_name = arcpy.ValidateTableName(file_base) # e.g. add a letter in front of the name 
-    validated_path = os.path.join(dirname, file_name)
-    #msg("validated path: %s; (from %s)" % (validated_path, path))
-    return validated_path

speckle_toolbox/esri/toolboxes/speckle/speckle/converter/features/feature_conversions.py

@@ -0,0 +1,997 @@
+from datetime import datetime
+import inspect
+import math
+import os
+from typing import List, Union
+import arcpy
+from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
+
+import numpy as np
+
+import scipy as sp
+from speckle.speckle.plugin_utils.helpers import (
+    findOrCreatePath,
+    get_scale_factor_to_meter,
+)
+from speckle.speckle.converter.layers.utils import (
+    apply_reproject,
+    findTransformation,
+    getVariantFromValue,
+    traverseDict,
+    validateAttributeName,
+)
+
+# from speckle.speckle.converter.geometry.conversions import transform
+from speckle.speckle.converter.geometry.conversions import (
+    convertToNative,
+    convertToNativeMulti,
+    convertToSpeckle,
+)
+from speckle.speckle.converter.geometry.mesh import constructMeshFromRaster
+from speckle.speckle.converter.geometry.utils import apply_pt_offsets_rotation_on_send
+from speckle.speckle.utils.panel_logging import logToUser
+from speckle.speckle.converter.features.utils import updateFeat
+from specklepy.objects.GIS.geometry import (
+    GisRasterElement,
+    GisPolygonGeometry,
+    GisNonGeometryElement,
+    GisTopography,
+)
+
+from specklepy.objects import Base
+
+from speckle.speckle.converter.geometry.point import pointToSpeckle
+from speckle.speckle.converter.layers.symbology import jsonFromLayerStyle
+
+
+def featureToSpeckle(
+    fieldnames,
+    attr_list,
+    index: int,
+    f_shape,
+    projectCRS: arcpy.SpatialReference,
+    selectedLayer: arcLayer,
+    plugin,
+):
+    dataStorage = plugin.dataStorage
+    if dataStorage is None:
+        return
+    units = dataStorage.currentUnits
+    new_report = {"obj_type": "", "errors": ""}
+    iterations = 0
+    try:
+        geom = None
+        data = arcpy.Describe(selectedLayer.dataSource)
+        geomType = data.shapeType
+        if (
+            hasattr(data, "isRevit")
+            or hasattr(data, "isIFC")
+            or hasattr(data, "bimLevels")
+        ):
+            # print(f"Layer {selectedLayer.name} has unsupported data type")
+            logToUser(
+                f"Layer {selectedLayer.name} has unsupported data type",
+                level=1,
+                func=inspect.stack()[0][3],
+            )
+            return None
+
+        skipped_msg = f"'{geomType}' feature skipped due to invalid geometry"
+        try:
+            geom, iterations = convertToSpeckle(
+                f_shape, index, selectedLayer, data, dataStorage
+            )
+            print(geom)
+            if geom is not None and geom != "None":
+                if not isinstance(geom.geometry, List):
+                    logToUser(
+                        "Geometry not in list format",
+                        level=2,
+                        func=inspect.stack()[0][3],
+                    )
+                    return None
+
+                all_errors = ""
+                for g in geom.geometry:
+                    if g is None or g == "None":
+                        all_errors += skipped_msg + ", "
+                        logToUser(skipped_msg, level=2, func=inspect.stack()[0][3])
+                    elif isinstance(g, GisPolygonGeometry):
+                        if len(g.displayValue) == 0:
+                            all_errors += (
+                                "Polygon converted, but display mesh not generated"
+                                + ", "
+                            )
+                            logToUser(
+                                "Polygon converted, but display mesh not generated",
+                                level=1,
+                                func=inspect.stack()[0][3],
+                            )
+                        elif iterations is not None and iterations > 0:
+                            all_errors += "Polygon display mesh is simplified" + ", "
+                            logToUser(
+                                "Polygon display mesh is simplified",
+                                level=1,
+                                func=inspect.stack()[0][3],
+                            )
+
+                if len(geom.geometry) == 0:
+                    all_errors = "No geometry converted"
+                new_report.update({"obj_type": geom.speckle_type, "errors": all_errors})
+
+            else:  # geom is None
+                new_report = {"obj_type": "", "errors": skipped_msg}
+                logToUser(skipped_msg, level=2, func=inspect.stack()[0][3])
+
+                dataStorage.latestActionFeaturesReport[
+                    len(dataStorage.latestActionFeaturesReport) - 1
+                ].update(new_report)
+                return
+                # geom = GisNonGeometryElement()
+        except Exception as error:
+            new_report = {
+                "obj_type": "",
+                "errors": "Error converting geometry: " + str(error),
+            }
+            logToUser(
+                "Error converting geometry: " + str(error),
+                level=2,
+                func=inspect.stack()[0][3],
+            )
+
+        # print(fieldnames)
+        # print(attr_list)
+        attributes = Base()
+        for i, name in enumerate(fieldnames):
+            corrected = validateAttributeName(name, fieldnames)
+            # print(corrected)
+            f_val = attr_list[i]
+            if f_val == "NULL" or f_val is None or str(f_val) == "NULL":
+                f_val = None
+            if isinstance(attr_list[i], list):
+                x = ""
+                for i, attr in enumerate(attr_list[i]):
+                    if i == 0:
+                        x += str(attr)
+                    else:
+                        x += ", " + str(attr)
+                f_val = x
+            attributes[corrected] = f_val
+
+        if geom is not None and geom != "None":
+            geom.attributes = attributes
+        # print(geom.attributes)
+
+        dataStorage.latestActionFeaturesReport[
+            len(dataStorage.latestActionFeaturesReport) - 1
+        ].update(new_report)
+        return geom
+
+    except Exception as e:
+        new_report.update({"errors": e})
+        dataStorage.latestActionFeaturesReport[
+            len(dataStorage.latestActionFeaturesReport) - 1
+        ].update(new_report)
+        logToUser(e, level=2, func=inspect.stack()[0][3])
+        return geom
+
+
+def rasterFeatureToSpeckle(
+    selectedLayer: arcLayer,
+    projectCRS: arcpy.SpatialReference,
+    plugin,
+) -> Base:
+    dataStorage = plugin.dataStorage
+    if dataStorage is None:
+        return
+
+    b = GisRasterElement(units=dataStorage.currentUnits)
+    try:
+        # get Raster object of entire raster dataset
+        my_raster = arcpy.Raster(selectedLayer.dataSource)
+        print(my_raster.mdinfo)  # None
+
+        rasterBandCount = my_raster.bandCount
+        rasterBandNames = my_raster.bandNames
+        #rasterDimensions = [my_raster.width, my_raster.height] #wrong result
+        
+        rb = my_raster.getRasterBands(rasterBandNames[0])
+        size = np.shape(rb.read())
+        rasterDimensions = [size[1], size[0]]
+        print(rasterDimensions)
+
+        # ds = gdal.Open(selectedLayer.source(), gdal.GA_ReadOnly)
+        extent = my_raster.extent
+        print(extent.XMin)
+        print(extent.YMin)
+        # if extent.YMin>0: rasterOriginPoint = arcpy.PointGeometry(arcpy.Point(extent.XMin, extent.YMax, extent.ZMin), my_raster.spatialReference, has_z = True)
+        rasterResXY = [
+            my_raster.meanCellWidth,
+            my_raster.meanCellHeight,
+        ]  # [float(ds.GetGeoTransform()[1]), float(ds.GetGeoTransform()[5])]
+        
+        b.x_resolution = rasterResXY[0]
+        b.y_resolution = -1 * rasterResXY[1]
+        b.x_size = rasterDimensions[0]
+        b.y_size = rasterDimensions[1]
+        b.band_count = rasterBandCount
+        b.band_names = rasterBandNames
+
+        rasterBandNoDataVal = []  # list(my_raster.noDataValues)
+        rasterBandMinVal = []
+        rasterBandMaxVal = []
+        rasterBandVals = []
+
+        rasterOriginPoint = arcpy.PointGeometry(
+            arcpy.Point(extent.XMin, extent.YMax),
+            my_raster.spatialReference,
+        )
+        rasterOriginPoint_max = arcpy.PointGeometry(
+            arcpy.Point(
+                extent.XMin + rasterDimensions[0] * b.x_resolution,
+                extent.YMax + rasterDimensions[1] * b.y_resolution
+            ),
+            my_raster.spatialReference,
+        )
+
+        # Try to extract geometry
+        x_form: tuple = findTransformation(
+            "Point",
+            my_raster.spatialReference,
+            projectCRS,
+            selectedLayer,
+        )
+
+        reprojectedPt = apply_reproject(rasterOriginPoint, x_form, dataStorage)
+        reprojectedPt_max = apply_reproject(rasterOriginPoint_max, x_form, dataStorage)
+        if reprojectedPt is None:
+            reprojectedPt = rasterOriginPoint
+            reprojectedPt_max = rasterOriginPoint_max
+
+        new_x_min, new_y_min = apply_pt_offsets_rotation_on_send(reprojectedPt.getPart().X, reprojectedPt.getPart().Y, dataStorage)
+        new_x_max, new_y_max = apply_pt_offsets_rotation_on_send(reprojectedPt_max.getPart().X, reprojectedPt_max.getPart().Y, dataStorage)
+        res_x = (new_x_max - new_x_min) / rasterDimensions[0]
+        res_y = (new_y_max - new_y_min) / rasterDimensions[1]
+
+        for index in range(rasterBandCount):
+            item = rasterBandNames[index]
+
+            rb = my_raster.getRasterBands(item)
+            print(rb)
+            size = np.shape(rb.read())
+            print(size)
+            rasterDimensions = [size[1], size[0]]
+
+            valMin = rb.minimum
+            valMax = rb.maximum
+            bandVals = np.swapaxes(rb.read(), 1, 2).flatten()  # .tolist() np.flip( , 0)
+
+            bandValsFlat = bandVals.tolist()
+            # print(bandValsFlat)
+
+            const = float(-1 * math.pow(10, 30))
+            defaultNoData = rb.noDataValue
+            print(defaultNoData)
+
+            # check whether NA value is too small or raster has too small values
+            # assign min value of an actual list; re-assign NA val; replace list items to new NA val
+            try:
+                # create "safe" fake NA value; replace extreme values with it
+                fakeNA = max(bandValsFlat) + 1
+                bandValsFlatFake = [
+                    fakeNA if val <= const else val for val in bandValsFlat
+                ]  # replace all values corresponding to NoData value
+
+                # if default NA value is too small
+                if (
+                    isinstance(defaultNoData, float) or isinstance(defaultNoData, int)
+                ) and defaultNoData < const:
+                    # find and rewrite min of actual band values; create new NA value
+                    valMin = min(bandValsFlatFake)
+                    noDataValNew = valMin - 1000  # use new adequate value
+                    rasterBandNoDataVal.append(noDataValNew)
+                    # replace fake NA with new NA
+                    bandValsFlat = [
+                        noDataValNew if val == fakeNA else val
+                        for val in bandValsFlatFake
+                    ]  # replace all values corresponding to NoData value
+
+                # if default val unaccessible and minimum val is too small
+                elif (
+                    isinstance(defaultNoData, str) or defaultNoData is None
+                ) and valMin < const:  # if there are extremely small values but default NA unaccessible
+                    noDataValNew = valMin
+                    rasterBandNoDataVal.append(noDataValNew)
+                    # replace fake NA with new NA
+                    bandValsFlat = [
+                        noDataValNew if val == fakeNA else val
+                        for val in bandValsFlatFake
+                    ]  # replace all values corresponding to NoData value
+                    # last, change minValto actual one
+                    valMin = min(bandValsFlatFake)
+
+                else:
+                    rasterBandNoDataVal.append(rb.noDataValue)
+
+            except:
+                rasterBandNoDataVal.append(rb.noDataValue)
+
+            # if rasterBandNoDataVal[len(rasterBandNoDataVal) - 1] is None:
+            #   rasterBandNoDataVal[len(rasterBandNoDataVal) - 1] = np.nan
+            print(len(bandValsFlat))
+            rasterBandVals.append(bandValsFlat)
+            rasterBandMinVal.append(valMin)
+            rasterBandMaxVal.append(valMax)
+
+            b["@(10000)" + item + "_values"] = (
+                bandValsFlat  # [0:int(max_values/rasterBandCount)]
+            )
+
+        b.x_origin, b.y_origin = apply_pt_offsets_rotation_on_send(
+            reprojectedPt.getPart().X, reprojectedPt.getPart().Y, dataStorage
+        )
+        try:
+            b.noDataValue = [float(val) for val in rasterBandNoDataVal]
+        except:
+            pass
+        # creating a mesh
+        vertices = []
+        faces = []
+        colors = []
+        count = 0
+
+        # print(selectedLayer.symbology)  # None
+        colorizer = None
+
+        # print(rendererType)
+        # identify symbology type and if Multiband, which band is which color
+
+        #############################################################
+
+        largeTransform = False
+        if rasterDimensions[0] * rasterDimensions[1] > 10000:
+            logToUser(
+                f"Transformation of the layer '{selectedLayer.name}' might take a while 🕒",
+                level=0,
+                plugin=plugin.dockwidget,
+            )
+            largeTransform = True
+
+        ############################################################
+
+        if hasattr(selectedLayer.symbology, "colorizer"):
+            colorizer = selectedLayer.symbology.colorizer
+            # print(
+            #    colorizer
+            # )  # <arcpy._colorizer.RasterStretchColorizer object at 0x000001780497FBC8>
+            # print(colorizer.type)  # RasterStretchColorizer
+        else:
+            redBand = greenBand = blueBand = None
+            # RGB colorizer
+            root_path: str = (
+                os.path.expandvars(r"%LOCALAPPDATA%")
+                + "\\Temp\\Speckle_ArcGIS_temp\\"
+                + datetime.now().strftime("%Y-%m-%d_%H-%M")
+            )
+            root_path += "\\Layers_Speckle\\raster_bands\\"
+            findOrCreatePath(root_path)
+
+            # if not os.path.exists(root_path + '\\Layers_Speckle\\raster_bands'): os.makedirs(root_path + '\\Layers_Speckle\\raster_bands')
+            path_style = root_path + selectedLayer.name + "_temp.lyrx"
+            symJson = jsonFromLayerStyle(selectedLayer, path_style)
+
+            # read from Json
+            try:
+                greenBand = symJson["layerDefinitions"][0]["colorizer"][
+                    "greenBandIndex"
+                ]
+            except:
+                if len(rasterBandVals) > 1:
+                    greenBand = 1
+            try:
+                blueBand = symJson["layerDefinitions"][0]["colorizer"]["blueBandIndex"]
+            except:
+                if len(rasterBandVals) > 2:
+                    blueBand = 2
+            try:
+                redBand = symJson["layerDefinitions"][0]["colorizer"]["redBandIndex"]
+            except:
+                if blueBand != 0 and greenBand != 0:
+                    redBand = 0
+                else:
+                    redBand = None
+            print("bands")
+            print(redBand)
+            print(greenBand)
+            print(blueBand)
+            try:
+                rbVals = rasterBandVals[
+                    redBand
+                ]  # my_raster.getRasterBands(rasterBandNames[redBand])
+                rbvalMin = rasterBandMinVal[redBand]
+                rbvalMax = rasterBandMaxVal[redBand]
+                rvalRange = float(rbvalMax) - float(rbvalMin)
+                print(rbvalMin)
+                print(rbvalMax)
+                print(rvalRange)
+            except Exception as e:
+                print(e)
+                rvalRange = None
+            try:
+                gbVals = rasterBandVals[greenBand]
+                gbvalMin = rasterBandMinVal[greenBand]
+                gbvalMax = rasterBandMaxVal[greenBand]
+                gvalRange = float(gbvalMax) - float(gbvalMin)
+                print(gbvalMin)
+                print(gbvalMax)
+                print(gvalRange)
+            except:
+                gvalRange = None
+            try:
+                bbVals = rasterBandVals[blueBand]
+                bbvalMin = rasterBandMinVal[blueBand]
+                bbvalMax = rasterBandMaxVal[blueBand]
+                bvalRange = float(bbvalMax) - float(bbvalMin)
+                print(bbvalMin)
+                print(bbvalMax)
+                print(bvalRange)
+            except:
+                bvalRange = None
+
+        rendererType = ""
+        bandIndex = 0
+
+        array_z = []  # size is large by 1 than the raster size, in both dimensions
+        time0 = datetime.now()
+
+        print(projectCRS)
+        print(projectCRS.factoryCode)
+        # if isinstance(projectCRS.factoryCode, int) and projectCRS.factoryCode > 1:
+        #    reprojected_raster = arcpy.ia.Reproject(
+        #        my_raster, {"wkid": projectCRS.factoryCode}
+        #    )
+        print(my_raster.spatialReference)
+        print(my_raster.spatialReference.factoryCode)
+        for v in range(rasterDimensions[1]):  # each row, Y
+            print(v)
+            if largeTransform is True:
+                if v == int(rasterDimensions[1] / 20):
+                    logToUser(
+                        f"Converting layer '{selectedLayer.name}': 5%...",
+                        level=0,
+                        plugin=plugin.dockwidget,
+                    )
+                elif v == int(rasterDimensions[1] / 10):
+                    logToUser(
+                        f"Converting layer '{selectedLayer.name}': 10%...",
+                        level=0,
+                        plugin=plugin.dockwidget,
+                    )
+                elif v == int(rasterDimensions[1] / 5):
+                    logToUser(
+                        f"Converting layer '{selectedLayer.name}': 20%...",
+                        level=0,
+                        plugin=plugin.dockwidget,
+                    )
+                elif v == int(rasterDimensions[1] * 2 / 5):
+                    logToUser(
+                        f"Converting layer '{selectedLayer.name}': 40%...",
+                        level=0,
+                        plugin=plugin.dockwidget,
+                    )
+                elif v == int(rasterDimensions[1] * 3 / 5):
+                    logToUser(
+                        f"Converting layer '{selectedLayer.name}': 60%...",
+                        level=0,
+                        plugin=plugin.dockwidget,
+                    )
+                elif v == int(rasterDimensions[1] * 4 / 5):
+                    logToUser(
+                        f"Converting layer '{selectedLayer.name}': 80%...",
+                        level=0,
+                        plugin=plugin.dockwidget,
+                    )
+                elif v == int(rasterDimensions[1] * 9 / 10):
+                    logToUser(
+                        f"Converting layer '{selectedLayer.name}': 90%...",
+                        level=0,
+                        plugin=plugin.dockwidget,
+                    )
+
+            for h in range(rasterDimensions[0]):  # item in a row, X
+                vertices.extend(
+                    [
+                        new_x_min + h * res_x,
+                        new_y_min + v * res_y,
+                        0,
+                        new_x_min + h * res_x,
+                        new_y_min + (v + 1) * res_y,
+                        0,
+                        new_x_min + (h + 1) * res_x,
+                        new_y_min + (v + 1) * res_y,
+                        0,
+                        new_x_min + (h + 1) * res_x,
+                        new_y_min + v * res_y,
+                        0,
+                    ]
+                )  ## add 4 points
+                faces.extend([4, count, count + 1, count + 2, count + 3])
+
+                # color vertices according to QGIS renderer
+                color = (0 << 16) + (0 << 8) + 0
+                noValColor = (
+                    0,
+                    0,
+                    0,
+                )
+
+                if hasattr(selectedLayer.symbology, "colorizer"):  # only 1 band
+                    try:
+                        bandIndex = int(colorizer.band)  # if stretched
+                    except:
+                        pass
+                    if colorizer.type == "RasterUniqueValueColorizer":
+                        # REDO !!!!!!!!!!!!
+                        colorRVal = colorGVal = colorBVal = 0
+                        try:
+                            for br in colorizer.groups:
+                                print(br.heading)  # "Value"
+                                # go through all values classified
+                                if br.heading != "Value":
+                                    raise Exception('br.heading != "Value"')
+                                for k, itm in enumerate(br.items):
+                                    # print(itm.values)
+                                    if (
+                                        itm.values[0]
+                                        == rasterBandVals[bandIndex][int(count / 4)]
+                                    ):
+                                        print(itm.values[0])
+                                        colorRVal, colorGVal, colorBVal = (
+                                            itm.color["RGB"][0],
+                                            itm.color["RGB"][1],
+                                            itm.color["RGB"][2],
+                                        )
+                                        break
+                                    # if string covering float
+                                    try:
+                                        if float(itm.values[0]) == float(
+                                            rasterBandVals[bandIndex][int(count / 4)]
+                                        ):
+                                            print(itm.values[0])
+                                            colorRVal, colorGVal, colorBVal = (
+                                                itm.color["RGB"][0],
+                                                itm.color["RGB"][1],
+                                                itm.color["RGB"][2],
+                                            )
+                                            break
+                                    except Exception as e:
+                                        print(e)
+                                        pass
+
+                        except Exception as e:  # if no Min Max labels:
+                            # REMAP band values to (0,255) range
+                            print(e)
+                            valRange = max(rasterBandVals[bandIndex]) - min(
+                                rasterBandVals[bandIndex]
+                            )  # (rasterBandMaxVal[bandIndex] - rasterBandMinVal[bandIndex])
+                            if valRange == 0:
+                                if min(rasterBandVals[bandIndex]) == 0:
+                                    colorVal = 0
+                                else:
+                                    colorVal = 255
+                            else:
+                                colorRVal = colorGVal = colorBVal = int(
+                                    (
+                                        rasterBandVals[bandIndex][int(count / 4)]
+                                        - min(rasterBandVals[bandIndex])
+                                    )
+                                    / valRange
+                                    * 255
+                                )
+
+                        color = (colorRVal << 16) + (colorGVal << 8) + colorBVal
+
+                    else:
+                        try:
+                            if rasterBandVals[bandIndex][int(count / 4)] >= float(
+                                colorizer.minLabel
+                            ) and rasterBandVals[bandIndex][int(count / 4)] <= float(
+                                colorizer.maxLabel
+                            ):  # rasterBandMinVal[bandIndex]:
+                                # REMAP band values to (0,255) range
+                                valRange = float(colorizer.maxLabel) - float(
+                                    colorizer.minLabel
+                                )  # (rasterBandMaxVal[bandIndex] - rasterBandMinVal[bandIndex])
+                                colorVal = int(
+                                    (
+                                        rasterBandVals[bandIndex][int(count / 4)]
+                                        - float(colorizer.minLabel)
+                                    )
+                                    / valRange
+                                    * 255
+                                )
+                                if colorizer.invertColorRamp is True:
+                                    if valRange == 0:
+                                        if float(colorizer.maxLabel) == 0:
+                                            colorVal = 0
+                                        else:
+                                            colorVal = 255
+                                    else:
+                                        colorVal = int(
+                                            (
+                                                -rasterBandVals[bandIndex][
+                                                    int(count / 4)
+                                                ]
+                                                + float(colorizer.maxLabel)
+                                            )
+                                            / valRange
+                                            * 255
+                                        )
+                                color = (colorVal << 16) + (colorVal << 8) + colorVal
+                        except:  # if no Min Max labels:
+                            # REMAP band values to (0,255) range
+                            valRange = max(rasterBandVals[bandIndex]) - min(
+                                rasterBandVals[bandIndex]
+                            )  # (rasterBandMaxVal[bandIndex] - rasterBandMinVal[bandIndex])
+
+                            if valRange == 0:
+                                if min(rasterBandVals[bandIndex]) == 0:
+                                    colorVal = 0
+                                else:
+                                    colorVal = 255
+                            else:
+                                colorVal = int(
+                                    (
+                                        rasterBandVals[bandIndex][int(count / 4)]
+                                        - min(rasterBandVals[bandIndex])
+                                    )
+                                    / valRange
+                                    * 255
+                                )
+                            color = (colorVal << 16) + (colorVal << 8) + colorVal
+                else:  # rgb
+                    # REMAP band values to (0,255) range
+                    if rvalRange is not None and redBand is not None:
+                        if rvalRange == 0:
+                            if float(rbvalMin) == 0:
+                                colorVal = 0
+                            else:
+                                colorVal = 255
+                        else:
+                            colorRVal = int(
+                                (
+                                    rasterBandVals[redBand][int(count / 4)]
+                                    - float(rbvalMin)
+                                )
+                                / rvalRange
+                                * 255
+                            )
+                    else:
+                        colorRVal = 0
+                    if gvalRange is not None and greenBand is not None:
+                        if gvalRange == 0:
+                            if float(gbvalMin) == 0:
+                                colorVal = 0
+                            else:
+                                colorVal = 255
+                        else:
+                            colorGVal = int(
+                                (
+                                    rasterBandVals[greenBand][int(count / 4)]
+                                    - float(gbvalMin)
+                                )
+                                / gvalRange
+                                * 255
+                            )
+                    else:
+                        colorGVal = 0
+                    if bvalRange is not None and blueBand is not None:
+                        if bvalRange == 0:
+                            if float(bbvalMin) == 0:
+                                colorVal = 0
+                            else:
+                                colorVal = 255
+                        else:
+                            colorBVal = int(
+                                (
+                                    rasterBandVals[blueBand][int(count / 4)]
+                                    - float(bbvalMin)
+                                )
+                                / bvalRange
+                                * 255
+                            )
+                    else:
+                        colorBVal = 0
+                    # print("__pixel color_")
+                    # print(colorRVal)
+                    # print(colorGVal)
+                    # print(colorBVal)
+
+                    color = (colorRVal << 16) + (colorGVal << 8) + colorBVal
+
+                colors.extend([color, color, color, color])
+                count += 4
+
+        mesh = constructMeshFromRaster(vertices, faces, colors)
+
+        time1 = datetime.now()
+        # print(f"Time to get Raster: {(time1-time0).total_seconds()} sec")
+        # after the entire loop
+
+        if mesh is not None:
+            mesh.units = dataStorage.currentUnits
+            b.displayValue = [mesh]
+        else:
+            logToUser(
+                "Something went wrong. Mesh cannot be created, only raster data will be sent. ",
+                level=2,
+                plugin=plugin.dockwidget,
+            )
+
+        return b
+
+    except Exception as e:
+        logToUser(e, level=2, func=inspect.stack()[0][3])
+        return None
+
+
+def featureToNative(
+    feature: Base, fields: dict, geomType: str, sr: arcpy.SpatialReference, dataStorage
+):
+    print("04_____Feature To Native correct____________")
+    feat = {}
+    try:
+        try:
+            speckle_geom = feature[
+                "geometry"
+            ]  # for created in QGIS / ArcGIS Layer type
+        except:
+            speckle_geom = feature  # for created in other software
+
+        print(speckle_geom)
+        arcGisGeom = None
+        if isinstance(speckle_geom, list):
+            if len(speckle_geom) > 1 or geomType == "Multipoint":
+                arcGisGeom = convertToNativeMulti(speckle_geom, sr, dataStorage)
+            else:
+                if len(speckle_geom) > 0:
+                    arcGisGeom = convertToNative(speckle_geom[0], sr, dataStorage)
+        else:
+            arcGisGeom = convertToNative(speckle_geom, sr, dataStorage)
+
+        if arcGisGeom is not None:
+            feat.update({"arcGisGeomFromSpeckle": arcGisGeom})
+        else:
+            return None
+        try:
+            [print(p for p in arcGisGeom.getPart())]
+        except:
+            print(arcGisGeom.getPart())
+        # print(feat)
+        for key, variant in fields.items():
+            value = None
+            try:
+                value = feature[key]
+            except:
+                if key == "Speckle_ID":
+                    try:
+                        value = str(
+                            feature["speckle_id"]
+                        )  # if GIS already generated this field
+                    except Exception as e:
+                        # print(e)
+                        value = str(feature["id"])
+                else:
+                    # print(key)
+                    # arcpy.AddWarning(f'Field {key} not found')
+                    try:
+                        value = feature.attributes[key]
+                    except:
+                        try:
+                            value = feature.attributes[key.replace("attributes_", "")]
+                        except:
+                            pass
+
+            if variant == "TEXT":
+                value = str(value)
+                if len(value) > 255:
+                    # print(len(value))
+                    value = value[:255]
+                    logToUser(
+                        f'Field "{key}" values are trimmed at 255 characters',
+                        level=2,
+                        func=inspect.stack()[0][3],
+                    )
+                    # arcpy.AddWarning(
+                    #    f'Field "{key}" values are trimmed at 255 characters'
+                    # )
+            if (
+                variant == getVariantFromValue(value)
+                and value != "NULL"
+                and value != "None"
+            ):
+                feat.update({key: value})
+            else:
+                if variant == "TEXT":
+                    feat.update({key: None})
+                if variant == "FLOAT":
+                    feat.update({key: None})
+                if variant == "LONG":
+                    feat.update({key: None})
+                if variant == "SHORT":
+                    feat.update({key: None})
+        # print(feat)
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return feat
+
+
+r"""
+def featureToNative(feature: Base, fields: "QgsFields", dataStorage):
+    feat = QgsFeature()
+    # print("___featureToNative")
+    try:
+        qgsGeom = None
+
+        if isinstance(feature, GisNonGeometryElement):
+            pass
+        else:
+            try:
+                speckle_geom = (
+                    feature.geometry
+                )  # for QGIS / ArcGIS Layer type from 2.14
+            except:
+                try:
+                    speckle_geom = feature[
+                        "geometry"
+                    ]  # for QGIS / ArcGIS Layer type before 2.14
+                except:
+                    speckle_geom = feature  # for created in other software
+
+            if not isinstance(speckle_geom, list):
+                qgsGeom = convertToNative(speckle_geom, dataStorage)
+
+            elif isinstance(speckle_geom, list):
+                if len(speckle_geom) == 1:
+                    qgsGeom = convertToNative(speckle_geom[0], dataStorage)
+                elif len(speckle_geom) > 1:
+                    qgsGeom = convertToNativeMulti(speckle_geom, dataStorage)
+                else:
+                    logToUser(
+                        f"Feature '{feature.id}' does not contain geometry",
+                        level=2,
+                        func=inspect.stack()[0][3],
+                    )
+
+            if qgsGeom is not None:
+                feat.setGeometry(qgsGeom)
+            else:
+                return None
+
+        feat.setFields(fields)
+        for field in fields:
+            name = str(field.name())
+            variant = field.type()
+            # if name == "id": feat[name] = str(feature["applicationId"])
+
+            try:
+                value = feature.attributes[name]  # fro 2.14 onwards
+            except:
+                try:
+                    value = feature[name]
+                except:
+                    if name == "Speckle_ID":
+                        try:
+                            value = str(
+                                feature["Speckle_ID"]
+                            )  # if GIS already generated this field
+                        except:
+                            try:
+                                value = str(feature["speckle_id"])
+                            except:
+                                value = str(feature["id"])
+                    else:
+                        value = None
+                        # logger.logToUser(f"Field {name} not found", Qgis.Warning)
+                        # return None
+
+            if variant == QVariant.String:
+                value = str(value)
+
+            if isinstance(value, str) and variant == QVariant.Date:  # 14
+                y, m, d = value.split("(")[1].split(")")[0].split(",")[:3]
+                value = QDate(int(y), int(m), int(d))
+            elif isinstance(value, str) and variant == QVariant.DateTime:
+                y, m, d, t1, t2 = value.split("(")[1].split(")")[0].split(",")[:5]
+                value = QDateTime(int(y), int(m), int(d), int(t1), int(t2))
+
+            if (
+                variant == getVariantFromValue(value)
+                and value != "NULL"
+                and value != "None"
+            ):
+                feat[name] = value
+
+        return feat
+    except Exception as e:
+        logToUser(e, level=2, func=inspect.stack()[0][3])
+        return feat
+"""
+
+
+def bimFeatureToNative(
+    feature: Base,
+    fields: dict,
+    sr: arcpy.SpatialReference,
+    path: str,
+    dataStorage,
+):
+    # print("04_________BIM Feature To Native____________")
+    feat_updated = {}
+    try:
+        feat = {}
+        feat.update({"arcGisGeomFromSpeckle": ""})
+        # feat_updated = updateFeat(exist_feat, fields, feature)
+
+        try:
+            if "Speckle_ID" not in fields.keys() and feature["id"]:
+                feat.update("Speckle_ID", "TEXT")
+        except:
+            pass
+        feat_updated = updateFeat(feat, fields, feature)
+
+        return feat_updated
+    except Exception as e:
+        logToUser(e, level=2, func=inspect.stack()[0][3])
+    return feat_updated
+
+
+def nonGeomFeatureToNative(feature: Base, fields: "QgsFields", dataStorage):
+    try:
+        exist_feat = QgsFeature()
+        exist_feat.setFields(fields)
+        feat_updated = updateFeat(exist_feat, fields, feature)
+        return feat_updated
+
+    except Exception as e:
+        logToUser(e, level=2, func=inspect.stack()[0][3])
+        return
+
+
+def cadFeatureToNative(
+    feature: Base, fields: dict, sr: arcpy.SpatialReference, dataStorage
+):
+
+    print("04_________CAD Feature To Native____________")
+    feat = {}
+    try:
+        try:
+            speckle_geom = feature["geometry"]  # for created in QGIS Layer type
+        except:
+            speckle_geom = feature  # for created in other software
+
+        if isinstance(speckle_geom, list):
+            arcGisGeom = convertToNativeMulti(speckle_geom, sr, dataStorage)
+        else:
+            arcGisGeom = convertToNative(speckle_geom, sr, dataStorage)
+
+        if arcGisGeom is not None:
+            feat.update({"arcGisGeomFromSpeckle": arcGisGeom})
+        else:
+            return
+
+        try:
+            if "Speckle_ID" not in fields.keys() and feature["id"]:
+                feat.update("Speckle_ID", "TEXT")
+        except:
+            pass
+
+        #### setting attributes to feature
+        feat_updated = updateFeat(feat, fields, feature)
+        # print(feat)
+        # print(fields)
+        return feat_updated
+
+    except Exception as e:
+        logToUser(e, level=2, func=inspect.stack()[0][3])
+        return

speckle_toolbox/esri/toolboxes/speckle/speckle/converter/features/utils.py

@@ -0,0 +1,218 @@
+import inspect
+import random
+from typing import Any, Dict, Union
+from speckle.speckle.converter.layers.utils import getVariantFromValue, traverseDict
+
+from speckle.speckle.utils.panel_logging import logToUser
+
+from specklepy.objects import Base
+
+
+def addFeatVariant(key, variant, value, f):
+    # print("Add feat variant")
+    feat = f
+    try:
+        if variant == "TEXT":
+            value = str(value)[:255]
+            if len(value) > 255:
+                print(len(value))
+                value = value[:255]
+                logToUser(
+                    f'Field "{key}" values are trimmed at 255 characters',
+                    level=2,
+                    func=inspect.stack()[0][3],
+                )
+
+        if value != "NULL" and value != "None":
+            # if key == 'area': print(value); print(type(value)); print(getVariantFromValue(value))
+            if variant == getVariantFromValue(
+                value
+            ):  # or (variant=="FLOAT" and isinstance(value, int)):
+                feat.update({key: value})
+            elif variant == "LONG" and isinstance(
+                value, float
+            ):  # if object has been modified
+                feat.update({key: int(value)})
+            elif variant == "FLOAT" and isinstance(
+                value, int
+            ):  # if object has been modified
+                feat.update({key: float(value)})
+            else:
+                feat.update({key: None})
+        elif (
+            variant == "TEXT"
+            or variant == "FLOAT"
+            or variant == "LONG"
+            or variant == "SHORT"
+        ):
+            feat.update({key: None})
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return feat
+
+
+def addFeatVariant_qgis(key, variant, value, f):
+    try:
+        feat = f
+        if variant == 10:
+            value = str(value)  # string
+
+        if value != "NULL" and value != "None":
+            if variant == getVariantFromValue(value):
+                feat[key] = value
+            elif (
+                isinstance(value, float) and variant == 4
+            ):  # float, but expecting Long (integer)
+                feat[key] = int(value)
+            elif (
+                isinstance(value, int) and variant == 6
+            ):  # int (longlong), but expecting float
+                feat[key] = float(value)
+            else:
+                feat[key] = None
+                # print(key); print(value); print(type(value)); print(variant); print(getVariantFromValue(value))
+        elif isinstance(variant, int):
+            feat[key] = None
+        return feat
+    except Exception as e:
+        logToUser(e, level=2, func=inspect.stack()[0][3])
+        return feat
+
+
+def updateFeat(feat: dict, fields: dict, feature: Base) -> Union[Dict[str, Any], None]:
+    feat_sorted = {}
+    try:
+        for key, variant in fields.items():
+            try:
+                if key == "Speckle_ID":
+                    value = str(feature["id"])
+
+                    feat[key] = value
+
+                    feat = addFeatVariant(key, variant, value, feat)
+                else:
+                    try:
+                        value = feature[key]
+                        # if key == "area": print(feature[key]); print(type(feature[key]))
+                        feat = addFeatVariant(key, variant, value, feat)
+                    except:
+                        value = None
+                        rootName = key.split("_")[0]
+                        # try: # if the root category exists
+                        # if its'a list
+                        if isinstance(feature[rootName], list):
+                            for i in range(len(feature[rootName])):
+                                try:
+                                    newF, newVals = traverseDict(
+                                        {},
+                                        {},
+                                        rootName + "_" + str(i),
+                                        feature[rootName][i],
+                                    )
+                                    for i, (key, value) in enumerate(newVals.items()):
+                                        for k, (x, y) in enumerate(newF.items()):
+                                            if key == x:
+                                                variant = y
+                                                break
+                                        feat = addFeatVariant(key, variant, value, feat)
+                                except Exception as e:
+                                    pass  # print(e)
+                        # except: # if not a list
+                        else:
+                            try:
+                                newF, newVals = traverseDict(
+                                    {}, {}, rootName, feature[rootName]
+                                )
+                                for i, (key, value) in enumerate(newVals.items()):
+                                    for k, (x, y) in enumerate(newF.items()):
+                                        if key == x:
+                                            variant = y
+                                            break
+                                    feat = addFeatVariant(key, variant, value, feat)
+                            except Exception as e:
+                                feat.update({key: None})
+            except Exception as e:
+                feat.update({key: None})
+        feat_sorted = {k: v for k, v in sorted(feat.items(), key=lambda item: item[0])}
+        # print("_________________end of updating a feature_________________________")
+        return feat_sorted
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+
+
+def getPolygonFeatureHeight(
+    feature: "QgsFeature", layer: "QgsVectorLayer", dataStorage: "DataStorage"
+) -> Union[int, float, None]:
+    height = None
+    ignore = False
+    if dataStorage.savedTransforms is not None:
+        for item in dataStorage.savedTransforms:
+            layer_name = item.split("  ->  ")[0].split(" ('")[0]
+            transform_name = item.split("  ->  ")[1].lower()
+            if "ignore" in transform_name:
+                ignore = True
+
+            if layer_name == layer.name():
+                attribute = None
+                if " ('" in item:
+                    attribute = item.split(" ('")[1].split("') ")[0]
+
+                if attribute is None and ignore is False:
+                    logToUser(
+                        "Attribute for extrusion not selected",
+                        level=1,
+                        func=inspect.stack()[0][3],
+                    )
+                    return None
+
+                # print("Apply transform: " + transform_name)
+                if "extrude" in transform_name and "polygon" in transform_name:
+                    # additional check:
+                    try:
+                        if dataStorage.project.crs().isGeographic():
+                            return None
+                    except:
+                        return None
+
+                    try:
+                        existing_height = float(feature[attribute])
+                        if (
+                            existing_height is None or str(feature[attribute]) == "NULL"
+                        ):  # if attribute value invalid
+                            if ignore is True:
+                                return None
+                            else:  # find approximate value
+                                all_existing_vals = [
+                                    f[attribute]
+                                    for f in layer.getFeatures()
+                                    if (
+                                        f[attribute] is not None
+                                        and (
+                                            isinstance(f[attribute], float)
+                                            or isinstance(f[attribute], int)
+                                        )
+                                    )
+                                ]
+                                try:
+                                    if len(all_existing_vals) > 5:
+                                        height_average = all_existing_vals[
+                                            int(len(all_existing_vals) / 2)
+                                        ]
+                                        height = random.randint(
+                                            height_average - 5, height_average + 5
+                                        )
+                                    else:
+                                        height = random.randint(10, 20)
+                                except:
+                                    height = random.randint(10, 20)
+                        else:  # if acceptable value: reading from existing attribute
+                            height = existing_height
+
+                    except:  # if no Height attribute
+                        if ignore is True:
+                            height = None
+                        else:
+                            height = random.randint(10, 20)
+
+    return height

speckle_toolbox/esri/toolboxes/speckle/speckle/converter/geometry/conversions.py

@@ -0,0 +1,357 @@
+from regex import F
+from specklepy.objects import Base
+from specklepy.objects.geometry import (
+    Line,
+    Mesh,
+    Point,
+    Polyline,
+    Curve,
+    Arc,
+    Circle,
+    Polycurve,
+    Ellipse,
+)
+
+import arcpy
+from typing import Any, List, Tuple, Union, Sequence
+
+import inspect
+from specklepy.objects.GIS.geometry import (
+    GisLineElement,
+    GisPointElement,
+    GisPolygonElement,
+)
+from specklepy.objects.GIS.geometry import GisPolygonGeometry
+
+from speckle.speckle.converter.geometry.mesh import meshToNative
+from speckle.speckle.converter.geometry.polygon import (
+    polygonToNative,
+    multiPolygonToNative,
+    polygonToSpeckle,
+    multiPolygonToSpeckle,
+    polygonToSpeckleMesh,
+)
+from speckle.speckle.converter.geometry.utils import (
+    specklePolycurveToPoints,
+    addCorrectUnits,
+)
+from speckle.speckle.converter.geometry.polyline import (
+    anyLineToSpeckle,
+    arcToNative,
+    ellipseToNative,
+    circleToNative,
+    curveToNative,
+    lineToNative,
+    polycurveToNative,
+    polylineToNative,
+    polylineToSpeckle,
+    speckleArcCircleToPoints,
+    multiPolylineToSpeckle,
+)
+from speckle.speckle.converter.geometry.point import (
+    pointToCoord,
+    pointToNative,
+    pointToSpeckle,
+    multiPointToSpeckle,
+)
+
+from speckle.speckle.converter.layers.utils import apply_reproject, findTransformation
+from speckle.speckle.utils.panel_logging import logToUser
+
+import numpy as np
+
+
+def convertToSpeckle(
+    feature, index, layer, data, dataStorage
+) -> Tuple[Union[Base, Sequence[Base], None], int]:
+    """Converts the provided layer feature to Speckle objects"""
+    print("___convertToSpeckle____________")
+    try:
+        iterations = 0
+        layer_sr = data.spatialReference  # if sr.type == "Projected":
+        geomType = data.shapeType  # Polygon, Point, Polyline, Multipoint, MultiPatch
+        featureType = data.featureType
+        projectCRS = dataStorage.project.activeMap.spatialReference
+        units = dataStorage.currentUnits
+        x_form: tuple = findTransformation(geomType, layer_sr, projectCRS, layer)
+        try:
+            [print(p for p in feature.getPart())]
+        except:
+            print(feature.getPart())
+        # geomMultiType = feature.isMultipart
+        hasCurves = feature.hasCurves
+
+        # feature is <geoprocessing describe geometry object object at 0x000002A75D6A4BD0>
+
+        # print(featureType) # e.g. Simple
+        # print(geomType) # e.g. Polygon
+        # geomSingleType = (featureType=="Simple") # Simple,SimpleJunction,SimpleJunction,ComplexEdge,Annotation,CoverageAnnotation,Dimension,RasterCatalogItem
+
+        if geomType == "Point":  # Polygon, Point, Polyline, Multipoint, MultiPatch
+            print("__Point conversion")
+
+            f_shape = apply_reproject(feature, x_form, dataStorage).getPart()
+            if f_shape is None:
+                return None
+            result = [pointToSpeckle(f_shape, feature, layer, dataStorage)]
+            for r in result:
+                r.units = units
+
+            element = GisPointElement(units=units, geometry=result)
+
+        elif geomType == "Multipoint":
+            print("__Multipoint conversion")
+
+            f_shape = apply_reproject(feature, x_form, dataStorage).getPart()
+            if f_shape is None:
+                return None
+            result = [pointToSpeckle(pt, feature, layer, dataStorage) for pt in f_shape]
+            for r in result:
+                r.units = units
+
+            element = GisPointElement(units=units, geometry=result)
+
+        elif geomType == "Polyline":
+            print("__Polyline conversion")
+            # if feature.partCount == 1:
+            #    result = anyLineToSpeckle(
+            #        feature, feature, layer, dataStorage, xform_vars
+            #    )
+            #    result = addCorrectUnits(result, dataStorage)
+            #    result = [result]
+            # else:
+            all_parts = []
+            for part in feature.getPart():
+                all_parts.append(
+                    arcpy.Polyline(
+                        part,
+                        arcpy.Describe(layer.dataSource).SpatialReference,
+                        has_z=True,
+                    )
+                )
+            result = [
+                anyLineToSpeckle(poly, feature, layer, dataStorage, x_form)
+                for poly in all_parts
+            ]
+            for r in result:
+                r = addCorrectUnits(r, dataStorage)
+
+            element = GisLineElement(units=units, geometry=result)
+
+        elif geomType == "Polygon":
+            print("__Polygon conversion")
+            # if feature.partCount > 1:
+            r"""
+            if feature.partCount == 1:
+                result = polygonToSpeckle(
+                    feature, feature, index, layer, dataStorage, xform_vars
+                )
+                result = [result]
+
+            else:
+            """
+            f_shape = apply_reproject(feature, x_form, dataStorage).getPart()
+            result = [
+                polygonToSpeckle(geom, feature, index, layer, dataStorage, x_form)
+                for geom in f_shape
+            ]
+
+            for r in result:
+                if r is None:
+                    continue
+                r.units = units
+                if r.boundary is not None:
+                    r.boundary.units = units
+                if r.voids is not None:
+                    for v in r.voids:
+                        if v is not None:
+                            v.units = units
+                    for v in r.displayValue:
+                        if v is not None:
+                            v.units = units
+            element = GisPolygonElement(units=units, geometry=result)
+
+        elif geomType == "MultiPatch":
+            f_shape = apply_reproject(feature, x_form, dataStorage).getPart()
+            if f_shape is None:
+                return None
+            result = [polygonToSpeckleMesh(f_shape, index, layer, False, dataStorage)]
+            for r in result:
+                if r is None:
+                    continue
+                r.units = units
+                if r.boundary is not None:
+                    r.boundary.units = units
+                if r.voids is not None:
+                    for v in r.voids:
+                        if v is not None:
+                            v.units = units
+                    for v in r.displayValue:
+                        if v is not None:
+                            v.units = units
+            element = GisPolygonElement(units=units, geometry=result)
+            print(element)
+        else:
+            logToUser(
+                "Unsupported or invalid geometry in layer " + layer.name,
+                level=1,
+                func=inspect.stack()[0][3],
+            )
+        try:
+            print(result)
+        except:
+            pass
+        return element, iterations
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None, None
+
+
+def convertToNative(
+    base: Base, sr: arcpy.SpatialReference, dataStorage
+) -> Union[Any, None]:
+    """Converts any given base object to QgsGeometry."""
+    print("___Convert to Native SingleType___")
+    converted = None
+    try:
+        # print(base)
+        conversions = [
+            (Point, pointToNative),
+            (Line, lineToNative),
+            (Polyline, polylineToNative),
+            (Curve, curveToNative),
+            (Arc, arcToNative),
+            (Circle, circleToNative),
+            (Ellipse, ellipseToNative),
+            # (Mesh, meshToNative),
+            (Polycurve, polycurveToNative),
+            # temporary solution for polygons (Speckle has no type Polygon yet)
+        ]
+
+        for conversion in conversions:
+            if isinstance(base, conversion[0]):
+                # print(conversion[0])
+                converted = conversion[1](base, sr, dataStorage)
+                break
+
+        if converted is None:
+            # distinguish normal QGIS polygons and the ones sent as Mesh only
+            try:
+                if isinstance(base, GisPolygonGeometry):
+                    if base.boundary is None:
+                        try:
+                            converted = meshToNative(base.displayValue, sr, dataStorage)
+                        except KeyError as e:
+                            # print(e)
+                            converted = meshToNative(
+                                base["@displayValue"], sr, dataStorage
+                            )
+                    else:
+                        converted = multiPolygonToNative(base, sr, dataStorage)
+                else:
+                    # for older commits
+                    boundary = base.boundary  # will throw exception if not polygon
+                    if boundary is None:
+                        try:
+                            converted = meshToNative(base.displayValue, sr, dataStorage)
+                        except:
+                            converted = meshToNative(
+                                base["@displayValue"], sr, dataStorage
+                            )
+                    elif boundary is not None and isinstance(base, conversion[0]):
+                        converted = multiPolygonToNative(base, sr, dataStorage)
+
+            except (
+                Exception
+            ) as e:  # if no "boundary" found (either old Mesh from QGIS or other object)
+                print(e)
+                try:  # check for a QGIS Mesh
+                    try:
+                        # if sent as Mesh
+                        colors = base.displayValue[0].colors  # will throw exception
+                        if isinstance(base.displayValue[0], Mesh):
+                            converted = meshToNative(
+                                base.displayValue, sr, dataStorage
+                            )  # only called for Meshes created in QGIS before
+                    except:
+                        # if sent as Mesh
+                        colors = base["@displayValue"][0].colors  # will throw exception
+                        if isinstance(base["@displayValue"][0], Mesh):
+                            converted = meshToNative(
+                                base["@displayValue"], sr, dataStorage
+                            )  # only called for Meshes created in QGIS before
+
+                except:  # any other object(
+                    pass
+        if converted is None:
+            converted = multiPolygonToNative(base, sr, dataStorage)
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return converted
+
+
+def multiPointToNative(items: List[Point], sr: arcpy.SpatialReference, dataStorage):
+    print("___Create MultiPoint")
+    features = None
+    try:
+        all_pts = []
+        # example https://pro.arcgis.com/en/pro-app/2.8/arcpy/classes/multipoint.htm
+        for item in items:
+            pt = pointToCoord(item)  # [x, y, z]
+            all_pts.append(arcpy.Point(pt[0], pt[1], pt[2]))
+        # print(all_pts)
+        features = arcpy.Multipoint(arcpy.Array(all_pts))
+        # if len(features)==0: features = None
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return features
+
+
+def multiPolylineToNative(
+    items: List[Polyline], sr: arcpy.SpatialReference, dataStorage
+):
+    print("_______Drawing Multipolylines____")
+    poly = None
+    try:
+        # print(items)
+        poly = None
+        full_array_list = []
+        for item in items:  # will be 1 item
+            pointsSpeckle = []
+            try:
+                pointsSpeckle = item.as_points()
+            except:
+                continue
+            pts = [pointToCoord(pt) for pt in pointsSpeckle]
+
+            if item.closed is True:
+                pts.append(pointToCoord(item.as_points()[0]))
+
+            arr = [arcpy.Point(*coords) for coords in pts]
+            full_array_list.append(arr)
+
+        poly = arcpy.Polyline(arcpy.Array(full_array_list), sr, has_z=True)
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return poly
+
+
+def convertToNativeMulti(items: List[Base], sr: arcpy.SpatialReference, dataStorage):
+    print("___Convert to Native MultiType___")
+    try:
+        first = items[0]
+        if isinstance(first, Point):
+            return multiPointToNative(items, sr, dataStorage)
+        elif isinstance(first, Line) or isinstance(first, Polyline):
+            return multiPolylineToNative(items, sr, dataStorage)
+        elif isinstance(first, Base):
+            try:
+                if first["boundary"] is not None and first["voids"] is not None:
+                    print(first["boundary"])
+                    print(first["voids"])
+                    return multiPolygonToNative(items, sr, dataStorage)
+            except:
+                return None
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None

speckle_toolbox/esri/toolboxes/speckle/speckle/converter/geometry/mesh.py

@@ -0,0 +1,325 @@
+from datetime import datetime
+import os
+import time
+from typing import List
+import arcpy
+import math
+
+from specklepy.objects.geometry import Mesh, Point, Polyline
+from specklepy.objects.other import RenderMaterial
+from specklepy.objects.GIS.geometry import GisPolygonGeometry
+
+
+import inspect
+
+import shapefile
+from shapefile import TRIANGLE_STRIP, TRIANGLE_FAN, OUTER_RING
+
+
+from speckle.speckle.converter.layers.utils import get_scale_factor, getDisplayValueList
+from speckle.speckle.converter.geometry.point import pointToNative
+from speckle.speckle.converter.layers.symbology import featureColorfromNativeRenderer
+from speckle.speckle.converter.layers.utils import get_scale_factor
+from speckle.speckle.utils.panel_logging import logToUser
+from speckle.speckle.plugin_utils.helpers import findOrCreatePath, validateNewFclassName
+
+from panda3d.core import Triangulator
+
+from speckle.speckle.converter.geometry.utils import apply_pt_transform_matrix
+from arcpy.management import CreateFeatureclass
+
+
+def meshToNative(meshes: Mesh, sr, dataStorage=None):
+    """Converts a Speckle Mesh to MultiPatch"""
+
+    result = []
+
+    print("06___________________Mesh to Native")
+    new_path = writeMeshToShp(meshes, "", dataStorage)
+
+    # time.sleep(0.1)
+    try:
+        cursor = arcpy.da.SearchCursor(new_path, "Speckle_ID")
+    except:
+        class_name = new_path.split("\\")[-1]
+        all_classes = arcpy.ListFeatureClasses()
+        validated_class_name = validateNewFclassName(class_name, all_classes)
+        path = dataStorage.workspace  # project.filePath.replace("aprx","gdb") #
+
+        f_class = arcpy.conversion.FeatureClassToFeatureClass(
+            new_path, path, validated_class_name
+        )
+        arcpy.management.DefineProjection(f_class, sr.exportToString())
+        cursor = arcpy.da.SearchCursor(new_path, "Speckle_ID")
+
+    class_shapes = [shp_id[0] for n, shp_id in enumerate(cursor)]
+    del cursor
+    return class_shapes[0]
+
+
+def writeMeshToShp(meshes: List[Mesh], path: str, dataStorage):
+    """Converts a Speckle Mesh to native geometry"""
+    print("06___________________Mesh to Native SHP")
+    try:
+        try:
+            if path == "":
+                path_folder = (
+                    os.path.expandvars(r"%LOCALAPPDATA%")
+                    + "\\Temp\\Speckle_ArcGIS_temp\\"
+                    + datetime.now().strftime("%Y_%m_%d__%H_%M")
+                    + "\\Layers_Speckle\\BIM_layers\\"
+                )
+                findOrCreatePath(path_folder)
+                path = (
+                    path_folder
+                    + "mesh_"
+                    + datetime.now().strftime("%Y_%m_%d__%H_%M_%S")
+                    + "_shp"
+                )
+            w = shapefile.Writer(path)  # + "\\" + str(meshes[0].id))
+        except Exception as e:
+            logToUser(e)
+            return
+
+        w.field("speckle_id", "C")
+
+        for _, geom in enumerate(meshes):
+            meshList: List = getDisplayValueList(geom)
+            print(meshList)
+            w = fill_multi_mesh_parts(w, meshList, geom.id, dataStorage)
+        w.close()
+        return path
+
+    except Exception as e:
+        logToUser(e, level=2, func=inspect.stack()[0][3])
+        return None
+
+
+def fill_multi_mesh_parts(
+    w: shapefile.Writer, meshes: List[Mesh], geom_id: str, dataStorage
+):
+    try:
+        parts_list = []
+        types_list = []
+        for mesh in meshes:
+            if not isinstance(mesh, Mesh):
+                continue
+            try:
+                # print(f"Fill multi-mesh parts # {geom_id}")
+                parts_list_x, types_list_x = deconstructSpeckleMesh(mesh, dataStorage)
+                parts_list.extend(parts_list_x)
+                types_list.extend(types_list_x)
+            except Exception as e:
+                pass
+
+        w.multipatch(parts_list, partTypes=types_list)  # one type for each part
+        w.record(geom_id)
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return w
+
+
+def fill_mesh_parts(w: shapefile.Writer, mesh: Mesh, geom_id: str, dataStorage):
+
+    try:
+        # print(f"Fill mesh parts # {geom_id}")
+        parts_list, types_list = deconstructSpeckleMesh(mesh, dataStorage)
+        w.multipatch(parts_list, partTypes=types_list)  # one type for each part
+        w.record(geom_id)
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return w
+
+
+def deconstructSpeckleMesh(mesh: Mesh, dataStorage):
+    parts_list = []
+    types_list = []
+    try:
+        scale = get_scale_factor(mesh.units)
+
+        count = 0  # sequence of vertex (not of flat coord list)
+        for f in mesh.faces:  # real number of loops will be at least 3 times less
+            try:
+                vertices = mesh.faces[count]
+                if mesh.faces[count] == 0:
+                    vertices = 3
+                if mesh.faces[count] == 1:
+                    vertices = 4
+
+                face = []
+                for i in range(vertices):
+                    index_faces = count + 1 + i
+                    index_vertices = mesh.faces[index_faces] * 3
+                    pt_coords = [
+                        mesh.vertices[index_vertices],
+                        mesh.vertices[index_vertices + 1],
+                        mesh.vertices[index_vertices + 2],
+                    ]
+                    pt_coords_new = apply_pt_transform_matrix(pt_coords, dataStorage)
+                    face.append([scale * coord for coord in pt_coords_new])
+
+                parts_list.append(face)
+                types_list.append(OUTER_RING)
+                count += vertices + 1
+            except:
+                break  # when out of range
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return parts_list, types_list
+
+
+def constructMeshFromRaster(vertices, faces, colors):
+    mesh = None
+    try:
+        mesh = Mesh.create(vertices, faces, colors)
+        mesh.units = "m"
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return mesh
+
+
+def constructMesh(vertices, faces, colors):
+    mesh = None
+    try:
+        mesh = Mesh.create(vertices, faces, colors)
+        mesh.units = "m"
+        material = RenderMaterial()
+        material.diffuse = colors[0]
+        mesh.renderMaterial = material
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return mesh
+
+
+def meshPartsFromPolygon(
+    polyBorder: List[Point],
+    voidsAsPts: List[List[Point]],
+    existing_vert: int,
+    index: int,
+    layer,
+    dataStorage,
+):
+
+    try:
+        # print("__meshPartsFromPolygon__")
+        vertices = []
+        total_vertices = 0
+        # print(layer)
+        try:
+            sr = arcpy.Describe(layer.dataSource).spatialReference
+        except Exception as e:
+            print(e)
+            sr = None
+        # print(sr)
+        coef = 1
+        maxPoints = 5000
+        if len(polyBorder) >= maxPoints:
+            coef = int(len(polyBorder) / maxPoints)
+
+        if (
+            len(voidsAsPts) == 0
+        ):  # only if there is a mesh with no voids and large amount of points
+            # print("mesh with no voids")
+            for k, ptt in enumerate(polyBorder):  # pointList:
+                pt = polyBorder[k * coef]
+                if k < maxPoints:
+                    # if isinstance(pt, QgsPointXY):
+                    #    pt = QgsPoint(pt)
+                    # print(pt)
+                    if isinstance(pt, Point):
+                        x = pt.x
+                        y = pt.y
+                        z = pt.z
+                        # pt = pointToNative(pt, sr, dataStorage).getPart()
+                    else:
+                        x = pt.X
+                        y = pt.Y
+                        z = 0 if math.isnan(pt.Z) else pt.Z
+                    vertices.extend([x, y, z])
+                    total_vertices += 1
+                else:
+                    break
+
+            ran = range(0, total_vertices)
+            faces = [total_vertices]
+            faces.extend([i + existing_vert for i in ran])
+            # else: https://docs.panda3d.org/1.10/python/reference/panda3d.core.Triangulator
+        else:  # if there are voids
+            print("mesh with voids")
+            # if its a large polygon with voids to be triangualted, lower the coef even more:
+            maxPoints = 100
+            if len(polyBorder) >= maxPoints:
+                coef = int(len(polyBorder) / maxPoints)
+
+            trianglator = Triangulator()
+            faces = []
+
+            pt_count = 0
+            # add extra middle point for border
+            for k, ptt in enumerate(polyBorder):  # pointList:
+                pt = polyBorder[k * coef]
+                if k < maxPoints:
+                    if pt_count < len(polyBorder) - 1 and k < (maxPoints - 1):
+                        pt2 = polyBorder[(k + 1) * coef]
+                    else:
+                        pt2 = polyBorder[0]
+
+                    trianglator.addPolygonVertex(trianglator.addVertex(pt.x, pt.y))
+                    vertices.extend([pt.x, pt.y, pt.z])
+                    trianglator.addPolygonVertex(
+                        trianglator.addVertex((pt.x + pt2.x) / 2, (pt.y + pt2.y) / 2)
+                    )
+                    vertices.extend(
+                        [(pt.x + pt2.x) / 2, (pt.y + pt2.y) / 2, (pt.z + pt2.z) / 2]
+                    )
+                    total_vertices += 2
+                    pt_count += 1
+                else:
+                    break
+
+            # add void points
+            for pts in voidsAsPts:
+                trianglator.beginHole()
+
+                coefVoid = 1
+                if len(pts) >= maxPoints:
+                    coefVoid = int(len(pts) / maxPoints)
+                for k, ptt in enumerate(pts):
+                    pt = pts[k * coefVoid]
+                    if k < maxPoints:
+                        trianglator.addHoleVertex(trianglator.addVertex(pt.x, pt.y))
+                        vertices.extend([pt.x, pt.y, pt.z])
+                        total_vertices += 1
+                    else:
+                        break
+
+            trianglator.triangulate()
+            i = 0
+            # print(trianglator.getNumTriangles())
+            while i < trianglator.getNumTriangles():
+                tr = [
+                    trianglator.getTriangleV0(i),
+                    trianglator.getTriangleV1(i),
+                    trianglator.getTriangleV2(i),
+                ]
+                faces.extend(
+                    [
+                        3,
+                        tr[0] + existing_vert,
+                        tr[1] + existing_vert,
+                        tr[2] + existing_vert,
+                    ]
+                )
+                i += 1
+            ran = range(0, total_vertices)
+
+        # print("color")
+        col = featureColorfromNativeRenderer(index, layer)  # (100<<16) + (100<<8) + 100
+        colors = [col for i in ran]  # apply same color for all vertices
+
+        return total_vertices, vertices, faces, colors
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None, None, None, None

speckle_toolbox/esri/toolboxes/speckle/speckle/converter/geometry/point.py

@@ -0,0 +1,144 @@
+import math
+from typing import List
+from specklepy.objects.geometry import Point
+import arcpy
+
+import inspect
+from speckle.speckle.converter.geometry.utils import (
+    apply_pt_offsets_rotation_on_send,
+    transform_speckle_pt_on_receive,
+    apply_pt_transform_matrix,
+)
+
+from speckle.speckle.converter.layers.utils import get_scale_factor
+from speckle.speckle.utils.panel_logging import logToUser
+
+
+def multiPointToSpeckle(geom, feature, layer, multiType: bool, dataStorage):
+    """Converts a Point to Speckle"""
+
+    pointList = []
+    # print(geom) # <geoprocessing describe geometry object object at 0x0000020F1D94AB10>
+    try:
+        if multiType is False:
+            for pt in geom:
+                # print(pt) # 284394.58100903 5710688.11602606 NaN NaN <class 'arcpy.arcobjects.arcobjects.Point'>
+                # print(type(pt))
+                if pt != None:
+                    pointList.append(pointToSpeckle(pt, feature, layer, dataStorage))
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return pointList
+
+
+def pointToSpeckle(pt, feature, layer, dataStorage):
+    """Converts a Point to Speckle"""
+    # print("___Point to Speckle____")
+    # when unset, z() returns "nan"
+    # print(pt) # 4.9046319 52.3592043 NaN NaN
+    # print("____Point to Speckle___")
+    try:
+        r"""
+        if isinstance(pt, arcpy.PointGeometry):
+            x = pt[0]
+            y = pt[1]
+            if len(pt) > 2:
+                z = pt[2]
+            else:
+                z = 0
+        """
+        x = pt.X
+        y = pt.Y
+        if pt.Z:
+            z = pt.Z
+        else:
+            z = 0
+
+        specklePoint = Point(units="m")
+        specklePoint.x = x
+        specklePoint.y = y
+        specklePoint.z = z
+        specklePoint.units = "m"
+        """
+        if feature is not None and layer is not None: # can be if it's a point from raster layer 
+            col = featureColorfromNativeRenderer(feature, layer)
+            specklePoint['displayStyle'] = {}
+            specklePoint['displayStyle']['color'] = col
+        """
+        specklePoint.x, specklePoint.y = apply_pt_offsets_rotation_on_send(
+            x, y, dataStorage
+        )
+        return specklePoint
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None
+
+
+def pointToNative(
+    pt: Point, sr: arcpy.SpatialReference, dataStorage
+) -> arcpy.PointGeometry:
+    """Converts a Speckle Point to QgsPoint"""
+    try:
+        new_pt = scalePointToNative(pt, pt.units, dataStorage)
+        new_pt = apply_pt_transform_matrix(new_pt, dataStorage)
+        newPt = transform_speckle_pt_on_receive(new_pt, dataStorage)
+
+        geom = arcpy.PointGeometry(arcpy.Point(pt.x, pt.y, pt.z), sr, has_z=True)
+        # print(geom)
+        return geom
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None
+
+
+def pointToNativeWithoutTransforms(pt: Point, sr: arcpy.SpatialReference, dataStorage):
+    """Converts a Speckle Point to QgsPoint"""
+    try:
+        new_pt = scalePointToNative(pt, pt.units, dataStorage)
+        new_pt = apply_pt_transform_matrix(new_pt, dataStorage)
+
+        geom = arcpy.PointGeometry(arcpy.Point(pt.x, pt.y, pt.z), sr, has_z=True)
+        # print(geom)
+        return geom
+
+    except Exception as e:
+        logToUser(e, level=2, func=inspect.stack()[0][3])
+        return None
+
+
+def pointToCoord(point: Point) -> List[float]:
+    """Converts a Speckle Point to QgsPoint"""
+    try:
+        # pt = scalePointToNative(point, point.units)
+        pt = point
+        coords = [pt.x, pt.y, pt.z]
+        # print(coords)
+        return coords
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return [None, None, None]
+
+
+def scalePointToNative(point: Point, units: str, dataStorage=None) -> Point:
+    """Scale point coordinates to meters"""
+    try:
+        scaleFactor = get_scale_factor(units)
+        pt = Point(units="m")
+        pt.x = point.x * scaleFactor
+        pt.y = point.y * scaleFactor
+        pt.z = 0 if math.isnan(point.z) else point.z * scaleFactor
+        return pt
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None
+
+
+def addZtoPoint(coords: List):
+    try:
+        if len(coords) == 2:
+            coords.append(0)
+        return coords
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None

speckle_toolbox/esri/toolboxes/speckle/speckle/converter/geometry/polygon.py

@@ -0,0 +1,484 @@
+from typing import List, Sequence, Union
+import arcpy
+import json
+from arcpy.arcobjects.arcobjects import SpatialReference
+
+from specklepy.objects import Base
+from specklepy.objects.GIS.geometry import GisPolygonGeometry
+from specklepy.objects.geometry import Point, Arc, Circle, Polycurve, Polyline, Line
+
+import inspect
+
+from speckle.speckle.converter.geometry.mesh import (
+    constructMesh,
+    constructMeshFromRaster,
+    meshPartsFromPolygon,
+)
+from speckle.speckle.converter.geometry.point import pointToCoord, pointToNative
+from speckle.speckle.converter.layers.symbology import featureColorfromNativeRenderer
+from speckle.speckle.converter.geometry.polyline import (
+    anyLineToSpeckle,
+    polylineFromVerticesToSpeckle,
+    speckleArcCircleToPoints,
+    curveToSpeckle,
+)
+from speckle.speckle.converter.geometry.utils import (
+    speckleBoundaryToSpecklePts,
+    specklePolycurveToPoints,
+)
+from speckle.speckle.utils.panel_logging import logToUser
+
+import math
+from panda3d.core import Triangulator
+
+
+def polygonToSpeckleMesh(geom, index: int, layer, multitype: bool, dataStorage):
+    print("________polygonToSpeckleMesh_____")
+    # print(geom)
+    polygon = GisPolygonGeometry(units="m")
+    try:
+
+        vertices = []
+        faces = []
+        colors = []
+        existing_vert = 0
+
+        for i, p in enumerate(geom):
+            # print("____start enumerate feature")
+            # print(p) #<geoprocessing array object object at 0x0000026796C77110>
+            boundaries, voids = getPolyBoundaryVoids(p, layer, dataStorage)
+            # boundary = boundaries[0]
+            for boundary in boundaries:
+                # print(boundary)
+                # print(voids)
+                polyBorder = speckleBoundaryToSpecklePts(boundary)
+                print(boundary.as_points())
+                # print(polyBorder)
+                if polyBorder is None or (
+                    isinstance(polyBorder, list) and len(polyBorder) < 3
+                ):
+                    continue
+                # print(existing_vert)
+                voidsAsPts = []
+                for v in voids:
+                    pts = speckleBoundaryToSpecklePts(v)
+                    voidsAsPts.append(pts)
+                # print(voidsAsPts)
+                total_vert, vertices_x, faces_x, colors_x = meshPartsFromPolygon(
+                    polyBorder, voidsAsPts, existing_vert, index, layer, dataStorage
+                )
+
+                existing_vert += total_vert
+                vertices.extend(vertices_x)
+                faces.extend(faces_x)
+                colors.extend(colors_x)
+
+        # print("Colors: ")
+        # print(colors)
+        # print(faces)
+        # print(vertices)
+        mesh = constructMesh(vertices, faces, colors)
+        # print(mesh)
+        if mesh is not None:
+            polygon.displayValue = [mesh]
+        else:
+            logToUser(
+                "Mesh creation from Polygon failed. Boundaries will be used as displayValue",
+                level=1,
+                func=inspect.stack()[0][3],
+            )
+        return polygon
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None
+
+
+def getPolyBoundaryVoids(feature, layer, dataStorage, x_form=None):
+    print("__getPolyBoundaryVoids__")
+    voids: List[Union[None, Polyline, Arc, Line, Polycurve]] = []
+    boundary = None
+    pointList = []
+    all_boundaries = []
+    try:
+        for i, pt in enumerate(feature):
+            # print(pt)  # 284394.58100903 5710688.11602606 NaN NaN
+            if pt == None and boundary == None:  # first break
+                boundary = polylineFromVerticesToSpeckle(
+                    pointList, True, feature, layer, dataStorage
+                )
+                pointList = []
+                all_boundaries.append(boundary)
+            elif pt == None and boundary != None:  # breaks btw voids
+                void = polylineFromVerticesToSpeckle(
+                    pointList, True, feature, layer, dataStorage
+                )
+                voids.append(void)
+                pointList = []
+            elif (
+                len(pointList) > 0
+                and pt.X == pointList[0].X
+                and pt.Y == pointList[0].Y
+                and pt.Z == pointList[0].Z
+            ):  # new condition
+                # if multiple boundaries
+                new_boundary = polylineFromVerticesToSpeckle(
+                    pointList, True, feature, layer, dataStorage
+                )
+                pointList = []
+                # print(new_boundary)
+                all_boundaries.append(new_boundary)
+            elif pt != None:  # add points to whatever list (boundary or void)
+                pointList.append(pt)
+
+        if boundary != None and len(pointList) > 0:  # remaining polyline
+            void = polylineFromVerticesToSpeckle(
+                pointList, True, feature, layer, dataStorage
+            )
+            voids.append(void)
+
+        elif boundary is None:  # no voids
+            boundary = polylineFromVerticesToSpeckle(
+                pointList, True, feature, layer, dataStorage
+            )
+            all_boundaries.append(boundary)
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return all_boundaries, voids
+
+
+def multiPolygonToSpeckle(geom, index: str, layer, multiType: bool, dataStorage):
+
+    print("___MultiPolygon to Speckle____")
+    polygon = []
+    try:
+        # print(enumerate(geom.getPart())) # this method ignores curvature and voids
+        # print(json.loads(geom.JSON))
+        # js = json.loads(geom.JSON)['rings']
+        # https://desktop.arcgis.com/en/arcmap/latest/analyze/python/reading-geometries.htm
+        for i, x in enumerate(geom):  # [[x,x,x]
+            # print("Part # " + str(i + 1))
+            # print(x)
+            boundaryFinished = 0
+            arrBoundary = []
+            arrInnerRings = []
+            for ptn in x:  # arcpy.Point
+                if ptn is None:
+                    boundaryFinished += 1
+                    arrInnerRings.append([])  # start of new Inner Ring
+                elif boundaryFinished == 0 and ptn is not None:
+                    arrBoundary.append(ptn)
+                elif boundaryFinished == 1 and ptn is not None:
+                    arrInnerRings[len(arrInnerRings) - 1].append(ptn)
+
+            full_arr = [arrBoundary] + arrInnerRings
+            # print(full_arr)
+            poly = arcpy.Polygon(
+                arcpy.Array(full_arr),
+                arcpy.Describe(layer.dataSource).SpatialReference,
+                has_z=True,
+            )
+            # print(poly) #<geoprocessing describe geometry object object at 0x000002B2D3E338D0>
+            polygon.append(
+                polygonToSpeckle(poly, index, layer, poly.isMultipart, dataStorage)
+            )
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return polygon
+
+
+def polygonToSpeckle(geom, feature, index: int, layer, dataStorage, x_form):
+    """Converts a Polygon to Speckle"""
+    # polygon = Base(units="m")
+    polygon = GisPolygonGeometry(units="m")
+    try:
+        print("___Polygon to Speckle____")
+        # print(geom)  # array
+
+        boundaries, voids = getPolyBoundaryVoids(geom, layer, dataStorage, x_form)
+        boundary = boundaries[0]
+        # print(boundary)
+        # print(voids)
+
+        data = arcpy.Describe(layer.dataSource)
+        sr = data.spatialReference
+
+        if boundary is None:
+            return None
+        polygon.boundary = boundary
+        polygon.voids = voids
+        # polygon.displayValue = [boundary] + voids
+        # print(boundary)
+
+        ############# mesh
+        vertices = []
+        polyBorder = boundary.as_points()
+        total_vertices = 0
+
+        if len(polyBorder) > 2:  # at least 3 points
+            # print("make meshes from polygons")
+            if len(voids) == 0:  # if there is a mesh with no voids
+                # print("no voids")
+                # print(polyBorder)
+                for pt in polyBorder:
+                    if isinstance(pt, Point):
+                        pt = pointToNative(pt, sr, dataStorage).getPart()  # SR unknown
+                    x = pt.X
+                    y = pt.Y
+                    z = 0 if math.isnan(pt.Z) else pt.Z
+                    vertices.extend([x, y, z])
+                    total_vertices += 1
+                # print(vertices)
+                ran = range(0, total_vertices)
+                faces = [total_vertices]
+                faces.extend([i for i in ran])
+                # print(faces)
+                # else: https://docs.panda3d.org/1.10/python/reference/panda3d.core.Triangulator
+            else:
+                # print("voids")
+                # print(polyBorder)
+                trianglator = Triangulator()
+                faces = []
+
+                pt_count = 0
+                # add extra middle point for border
+                for pt in polyBorder:
+                    if pt_count < len(polyBorder) - 1:
+                        pt2 = polyBorder[pt_count + 1]
+                    else:
+                        pt2 = polyBorder[0]
+
+                    trianglator.addPolygonVertex(trianglator.addVertex(pt.x, pt.y))
+                    vertices.extend([pt.x, pt.y, pt.z])
+
+                    # trianglator.addPolygonVertex(trianglator.addVertex((pt.x+pt2.x)/4*3, (pt.y+pt2.y)/4*3))
+                    # vertices.extend([(pt.x+pt2.x)/4*3, (pt.y+pt2.y)/4*3, (pt.z+pt2.z)/4*3])
+
+                    trianglator.addPolygonVertex(
+                        trianglator.addVertex((pt.x + pt2.x) / 2, (pt.y + pt2.y) / 2)
+                    )
+                    vertices.extend(
+                        [(pt.x + pt2.x) / 2, (pt.y + pt2.y) / 2, (pt.z + pt2.z) / 2]
+                    )
+
+                    # trianglator.addPolygonVertex(trianglator.addVertex((pt.x+pt2.x)/4, (pt.y+pt2.y)/4))
+                    # vertices.extend([(pt.x+pt2.x)/4, (pt.y+pt2.y)/4, (pt.z+pt2.z)/4])
+
+                    total_vertices += 2
+                    pt_count += 1
+
+                # add void points
+                for i in range(len(voids)):
+                    trianglator.beginHole()
+
+                    pts = []
+                    if isinstance(voids[i], Circle) or isinstance(voids[i], Arc):
+                        pts = speckleArcCircleToPoints(voids[i])
+                    elif isinstance(voids[i], Polycurve):
+                        pts = specklePolycurveToPoints(voids[i])
+                    elif isinstance(voids[i], Line):
+                        pass
+                    else:
+                        try:
+                            pts = voids[i].as_points()
+                        except:
+                            pass  # if Line
+                    # pts = voids[i].as_points()
+                    for pt in pts:
+                        trianglator.addHoleVertex(trianglator.addVertex(pt.x, pt.y))
+                        vertices.extend([pt.x, pt.y, pt.z])
+                        total_vertices += 1
+
+                trianglator.triangulate()
+                i = 0
+                while i < trianglator.getNumTriangles():
+                    tr = [
+                        trianglator.getTriangleV0(i),
+                        trianglator.getTriangleV1(i),
+                        trianglator.getTriangleV2(i),
+                    ]
+                    faces.extend([3, tr[0], tr[1], tr[2]])
+                    i += 1
+                ran = range(0, total_vertices)
+
+            # print(polygon)
+            col = featureColorfromNativeRenderer(index, layer)
+            colors = [col for i in ran]  # apply same color for all vertices
+            mesh = constructMesh(vertices, faces, colors)
+
+            # print(mesh)
+            if mesh is not None:
+                polygon.displayValue = [mesh]
+            else:
+                logToUser(
+                    "Mesh creation from Polygon failed. Boundaries will be used as displayValue",
+                    level=1,
+                    func=inspect.stack()[0][3],
+                )
+            return polygon
+        else:
+            logToUser(
+                "Not enough points for Polygon boundary",
+                level=1,
+                func=inspect.stack()[0][3],
+            )
+            return None
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None
+
+
+def polygonToNative(
+    poly: Base, sr: arcpy.SpatialReference, dataStorage
+) -> arcpy.Polygon:
+    """Converts a Speckle Polygon base object to QgsPolygon.
+    This object must have a 'boundary' and 'voids' properties.
+    Each being a Speckle Polyline and List of polylines respectively."""
+
+    print("_______Drawing polygons____")
+    polygon = None
+    try:
+        try:
+            poly = poly["geometry"]
+        except:
+            pass
+        # pts = [pointToCoord(pt) for pt in poly["boundary"].as_points()]
+        pointsSpeckle = []
+        if isinstance(poly["boundary"], Circle) or isinstance(poly["boundary"], Arc):
+            pointsSpeckle = speckleArcCircleToPoints(poly["boundary"])
+        elif isinstance(poly["boundary"], Polycurve):
+            pointsSpeckle = specklePolycurveToPoints(poly["boundary"])
+        elif isinstance(poly["boundary"], Line):
+            pass
+        else:
+            try:
+                pointsSpeckle = poly["boundary"].as_points()
+            except:
+                pass  # if Line
+
+        pts = [pointToCoord(pt) for pt in pointsSpeckle]
+        # print(pts)
+
+        outer_arr = [arcpy.Point(*coords) for coords in pts]
+        outer_arr.append(outer_arr[0])
+        geomPart = []
+        try:
+            for void in poly["voids"]:
+                # print(void)
+                # pts = [pointToCoord(pt) for pt in void.as_points()]
+                pointsSpeckle = []
+                if isinstance(void, Circle) or isinstance(void, Arc):
+                    pointsSpeckle = speckleArcCircleToPoints(void)
+                elif isinstance(void, Polycurve):
+                    pointsSpeckle = specklePolycurveToPoints(void)
+                elif isinstance(void, Line):
+                    pass
+                else:
+                    try:
+                        pointsSpeckle = void.as_points()
+                    except:
+                        pass  # if Line
+                pts = [pointToCoord(pt) for pt in pointsSpeckle]
+
+                inner_arr = [arcpy.Point(*coords) for coords in pts]
+                inner_arr.append(inner_arr[0])
+                geomPart.append(arcpy.Array(inner_arr))
+        except:
+            pass
+        geomPart.insert(0, outer_arr)
+        geomPartArray = arcpy.Array(geomPart)
+        polygon = arcpy.Polygon(geomPartArray, sr, has_z=True)
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return polygon
+
+
+def multiPolygonToNative(
+    items: List[Base], sr: arcpy.SpatialReference, dataStorage
+):  # TODO fix multi features
+
+    print("_______Drawing Multipolygons____")
+    polygon = None
+    if not isinstance(items, List):
+        items = [items]
+    try:
+        # print(items)
+        full_array_list = []
+        for item_geom in items:  # will be 1 item
+            try:
+                item_geom = item_geom["geometry"]
+            except:
+                item_geom = [item_geom]
+            for item in item_geom:
+                # print(item)
+                # pts = [pointToCoord(pt) for pt in item["boundary"].as_points()]
+                pointsSpeckle = []
+                if isinstance(item["boundary"], Circle) or isinstance(
+                    item["boundary"], Arc
+                ):
+                    pointsSpeckle = speckleArcCircleToPoints(item["boundary"])
+                elif isinstance(item["boundary"], Polycurve):
+                    pointsSpeckle = specklePolycurveToPoints(item["boundary"])
+                elif isinstance(item["boundary"], Line):
+                    pass
+                else:
+                    try:
+                        pointsSpeckle = item["boundary"].as_points()
+                    except Exception as e:
+                        print(e)  # if Line
+                # print(pointsSpeckle)
+                pts = [pointToCoord(pt) for pt in pointsSpeckle]
+                # print(pts)
+
+                outer_arr = [arcpy.Point(*coords) for coords in pts]
+                if pts[0] != pts[-1]:
+                    outer_arr.append(outer_arr[0])
+                # print("outer border")
+                # print(outer_arr)
+                geomPart = []
+                try:
+                    for void in item["voids"]:
+                        # print("void")
+                        # print(void)
+                        # pts = [pointToCoord(pt) for pt in void.as_points()]
+                        pointsSpeckle = []
+                        if isinstance(void, Circle) or isinstance(void, Arc):
+                            pointsSpeckle = speckleArcCircleToPoints(void)
+                        elif isinstance(void, Polycurve):
+                            pointsSpeckle = specklePolycurveToPoints(void)
+                        elif isinstance(void, Line):
+                            pass
+                        else:
+                            try:
+                                pointsSpeckle = void.as_points()
+                            except:
+                                pass  # if Line
+                        pts = [pointToCoord(pt) for pt in pointsSpeckle]
+
+                        inner_arr = [arcpy.Point(*coords) for coords in pts]
+                        if pts[0] != pts[-1]:
+                            inner_arr.append(inner_arr[0])
+                        geomPart.append(arcpy.Array(inner_arr))
+                except Exception as e:
+                    print(e)
+
+                geomPart.insert(0, arcpy.Array(outer_arr))
+                full_array_list.extend(
+                    geomPart
+                )  # outlines are written one by one, with no separation to "parts"
+                # print(full_array_list)  # array of points
+
+            # print("end of loop1")
+        # print("end of loop2")
+        geomPartArray = arcpy.Array(full_array_list)
+        polygon = arcpy.Polygon(geomPartArray, sr, has_z=True)
+
+        print(polygon)
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+
+    return polygon

speckle_toolbox/esri/toolboxes/speckle/speckle/converter/geometry/polyline.py

@@ -0,0 +1,734 @@
+from math import atan, cos, sin
+import math
+import json
+from typing import List, Union, Tuple
+from specklepy.objects import Base
+from specklepy.objects.geometry import (
+    Box,
+    Vector,
+    Point,
+    Line,
+    Polyline,
+    Curve,
+    Ellipse,
+    Arc,
+    Circle,
+    Polycurve,
+    Plane,
+    Interval,
+)
+import arcpy
+import numpy as np
+
+import inspect
+
+from speckle.speckle.converter.geometry.point import (
+    pointToCoord,
+    pointToSpeckle,
+    addZtoPoint,
+)
+from speckle.speckle.converter.geometry.utils import (
+    speckleArcCircleToPoints,
+    specklePolycurveToPoints,
+)
+from speckle.speckle.converter.layers.utils import apply_reproject, get_scale_factor
+from speckle.speckle.utils.panel_logging import logToUser
+
+
+def circleToSpeckle(center, point):
+    print("___Circle to Speckle____")
+    try:
+        rad = math.sqrt(
+            math.pow((center[0] - point[0]), 2) + math.pow((center[1] - point[1]), 2)
+        )
+        # print(rad)
+        if len(center) > 2:
+            center_z = center[2]
+        else:
+            center_z = 0
+        length = rad * 2 * math.pi
+        domain = [0, length]
+        plane = [center[0], center[1], center_z, 0, 0, 1, 1, 0, 0, 0, 1, 0]
+        units = 3  # "m"
+
+        args = [0] + [rad] + domain + plane + [units]
+        # print(args)
+        c = Circle.from_list(args)
+        c.plane.origin.units = "m"
+        c.units = "m"
+        # print(c)
+        return c
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None
+
+
+def multiPolylineToSpeckle(
+    geom, feature, layer, multiType: bool, dataStorage, xform_vars=None
+):
+
+    print("___MultiPolyline to Speckle____")
+    polyline = []
+    try:
+        print(enumerate(geom.getPart()))
+        for i, x in enumerate(geom.getPart()):
+            poly = arcpy.Polyline(
+                x, arcpy.Describe(layer.dataSource).SpatialReference, has_z=True
+            )
+            print(poly)
+            polyline.append(
+                polylineToSpeckle(poly, feature, layer, poly.isMultipart, dataStorage)
+            )
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return polyline
+
+
+def anyLineToSpeckle(geom, feature, layer, dataStorage, x_form=None):
+    print("___Any line to Speckle____")
+    polyline = None
+    try:
+        pointList = []
+        print(geom.hasCurves)
+        # multiType = feature.isMultipart
+
+        # if multiType is False:
+        try:
+            if geom.hasCurves:
+                print("has curves")
+                # geometry SHAPE@ tokens: https://pro.arcgis.com/en/pro-app/latest/arcpy/get-started/reading-geometries.htm
+                # print(geom.JSON)
+                new_geom = geom.densify("GEODESIC", 0.1)
+            else:
+                new_geom = geom
+        except:  # no curves
+            new_geom = geom
+
+        if x_form is not None:
+            new_geom = apply_reproject(new_geom, x_form, dataStorage).getPart()
+            if new_geom is None:
+                return None
+
+        # print(new_geom)  # describe geometry object
+        for p in new_geom:
+            # print(p)  # array
+            for pt in p:
+                # print(pt)
+                if pt != None:
+                    pointList.append(pt)  # ; print(pt.Z)
+        closed = False
+        if pointList[0] == pointList[len(pointList) - 1]:
+            closed = True
+            pointList = pointList[:-1]
+        polyline = polylineFromVerticesToSpeckle(
+            pointList, closed, feature, layer, dataStorage
+        )
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return polyline
+
+
+def polylineToSpeckle(
+    geom, feature, layer, multiType: bool, dataStorage, xform_vars=None
+):
+    print("___Polyline to Speckle____")
+    polyline = None
+    try:
+        pointList = []
+        print(geom.hasCurves)
+
+        if multiType is False:
+            if geom.hasCurves:
+                print("has curves")
+                # geometry SHAPE@ tokens: https://pro.arcgis.com/en/pro-app/latest/arcpy/get-started/reading-geometries.htm
+                print(geom.JSON)
+                polyline = curveToSpeckle(geom, "Polyline", feature, layer, dataStorage)
+            else:
+                for p in geom:
+                    for pt in p:
+                        if pt != None:
+                            pointList.append(pt)  # ; print(pt.Z)
+                closed = False
+                if pointList[0] == pointList[len(pointList) - 1]:
+                    closed = True
+                    pointList = pointList[:-1]
+                polyline = polylineFromVerticesToSpeckle(
+                    pointList, closed, feature, layer, dataStorage
+                )
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return polyline
+
+
+def polylineFromVerticesToSpeckle(
+    vertices: List[Point], closed: bool, feature, layer, dataStorage
+) -> Polyline:
+    """Converts a Polyline to Speckle"""
+    polyline = Polyline(units="m")
+    try:
+        # print("__polylineFromVerticesToSpeckle")
+        # print(vertices)
+        if isinstance(vertices, list):
+            if len(vertices) > 0 and isinstance(vertices[0], Point):
+                specklePts = vertices
+            else:
+                specklePts = [
+                    pointToSpeckle(pt, feature, layer, dataStorage) for pt in vertices
+                ]  # breaks unexplainably
+        # elif isinstance(vertices, QgsVertexIterator):
+        #    specklePts = [pointToSpeckle(pt, feature, layer) for pt in vertices]
+        else:
+            return None
+
+        specklePts = []
+        for pt in vertices:
+            newPt = pointToSpeckle(pt, feature, layer, dataStorage)
+            specklePts.append(newPt)
+
+        # TODO: Replace with `from_points` function when fix is pushed.
+        polyline.value = []
+        polyline.closed = closed
+        polyline.units = specklePts[0].units
+        for i, point in enumerate(specklePts):
+            if closed and i == len(specklePts) - 1 and specklePts[0] == point:
+                continue  # if we consider the last pt, do not add is coincides with the first (and type is Closed)
+            polyline.value.extend([point.x, point.y, point.z])
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return polyline
+
+
+def arc3ptToSpeckle(p0: List, p1: List, p2: List, feature, layer) -> Arc:
+    print("____arc 3pt to Speckle___")
+    arc = Arc()
+    try:
+        p0 = addZtoPoint(p0)
+        p1 = addZtoPoint(p1)
+        p2 = addZtoPoint(p2)
+        arc.startPoint = pointToSpeckle(arcpy.Point(*p0), feature, layer, dataStorage)
+        arc.midPoint = pointToSpeckle(arcpy.Point(*p1), feature, layer, dataStorage)
+        arc.endPoint = pointToSpeckle(arcpy.Point(*p2), feature, layer, dataStorage)
+        center, radius = getArcCenter(
+            Point.from_list(p0), Point.from_list(p1), Point.from_list(p2)
+        )
+        arc.plane = (
+            Plane()
+        )  # .from_list(Point(), Vector(Point(0, 0, 1)), Vector(Point(0,1,0)), Vector(Point(-1,0,0)))
+        arc.plane.origin = Point.from_list(center)
+        arc.plane.origin.units = "m"
+        arc.units = "m"
+        arc.angleRadians, startAngle, endAngle = getArcRadianAngle(arc)
+
+        arc.radius = radius
+
+        arc.plane.normal = getArcNormal(arc, arc.midPoint)
+
+        # arc.angleRadians = abs(angle1 + angle2)
+        # print(arc.angleRadians)
+
+        # col = featureColorfromNativeRenderer(feature, layer)
+        # arc['displayStyle'] = {}
+        # arc['displayStyle']['color'] = col
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return arc
+
+
+def curveBezierToSpeckle(segmStartCoord, segmEndCoord, knots, feature, layer):
+    print("____bezier curve to Speckle____")
+    try:
+        degree = 3
+        points = [
+            tuple(knots[0]),
+            tuple(segmStartCoord),
+            tuple(knots[1]),
+            tuple(segmEndCoord),
+        ]  # [segmStartCoord, *coords]
+        print(points)
+        num_points = len(points)  # 2
+
+        knot_count = num_points + degree - 1  # 4
+        knots = [0] * knot_count
+        print(knots)
+        for i in range(1, len(knots)):
+            knots[i] = i // 3
+            print(knots[i])
+
+        length = 1  # spline.calc_length()
+        domain = Interval(start=0, end=length, totalChildrenCount=0)
+        points = [tuple(pt) for pt in points]
+        curve = Curve(
+            degree=degree,
+            closed=False,
+            periodic=True if (segmStartCoord == segmEndCoord) else False,
+            points=list(sum(points, ())),  # magic (flatten list of tuples)
+            weights=[1] * num_points,
+            knots=knots,
+            rational=False,
+            area=0,
+            volume=0,
+            length=length,
+            domain=domain,
+            units="m",
+            bbox=Box(area=0.0, volume=0.0),
+        )
+        print(curve)
+        return curve
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None
+
+
+def curveToSpeckle(
+    geom, geomType, feature, layer, dataStorage
+) -> Union[Circle, Arc, Polyline, Polycurve]:
+    print("____curve to Speckle____")
+    boundary = Polycurve(units="m")
+    print(geomType)
+    try:
+        # look for "curvePaths" or "curveRings"[[ (startPt, {arcs, beziers etc}, optional(endPt))],[],...], "rings"
+        # examples: https://developers.arcgis.com/documentation/common-data-types/geometry-objects.htm
+        # e.g. {"hasZ":true,
+        # "curveRings":[[[631307.05960000027,5803698.4477999993,0],{"a":[[631307.05960000027,5803698.4477999993,0],[631307.05960000027,5803414.92656173],0,1]}]],
+        # "spatialReference":{"wkid":32631,"latestWkid":32631}}
+
+        # b - bezier curve (endPt, controlPts)
+        # a - elliptical arc (endPt, centralPt) e.g. for circle: [[[631307.05960000027,5803698.4477999993,0],{"a":[[631307.05960000027,5803698.4477999993,0],[631307.05960000027,5803414.92656173],0,1]}]]
+        # c - circular arc (endPt, throughPt) e.g. [[[633242.45179999992,5803058.0354999993,0],{"c":[[633718.26040000003,5803496.4210000001,0],[633337.75764975848,5803431.9997026781]]},[633242.45179999992,5803058.0354999993,0]]]
+
+        if geomType == "Polyline":
+            boundary.closed = False
+        else:
+            boundary.closed = True
+        segments = []
+
+        for key, val in json.loads(geom.JSON).items():
+            print(key)
+            if key == "curveRings" or key == "curvePaths":
+
+                # boundary.closed = True
+                includesLines = 0
+
+                for segm in val:  # segm: List
+                    print(
+                        segm
+                    )  # e.g. [[631307.05960000027,5803698.4477999993,0], {"a":[[631307.05960000027,5803698.4477999993,0],[631307.05960000027,5803414.92656173],0,1]}]
+                    segmStartCoord: List = addZtoPoint(segm[0])
+
+                    # go through all elements (points, a, c, ...)
+                    for k in range(1, len(segm)):
+                        # e.g. one from the list: "curveRings":[[[631750.87200000044,5803159.6126000006,0],
+                        # {"c":[[632429.8348000003,5803507.1132999994,0],[631988.22772700491,5803532.9008129537]]},
+                        # {"c":[[632590.21970000025,5803127.5355999991,0],[633018.51899157302,5803532.1801161235]]},
+                        # [631750.87200000044,5803159.6126000006,0]]]
+
+                        # if previous segments exist
+                        if len(segments) > 0:
+                            segmOldData = segm[k - 1]
+                            if isinstance(
+                                segmOldData, dict
+                            ):  # get "end point" of previous segment
+                                for key3, val3 in segmOldData.items():
+                                    segmStartCoord: List = addZtoPoint(val2[0])
+                            elif isinstance(segmOldData, list) and isinstance(
+                                segmOldData[0], float
+                            ):
+                                segmStartCoord: List = segmOldData
+                        segmStartCoord = addZtoPoint(segmStartCoord)
+
+                        if isinstance(segm[k], dict):
+                            for key2, val2 in segm[k].items():
+                                if key2 == "a":  # elliptical arc (endPt, centralPt)
+                                    # e.g. {'a': [[633883.1035000002, 5802972.5812, 0], [634028.3379278888, 5802908.342895357], 0, 1, 1.1543577096027686, 473.59966687227444, 0.33531864204900685]}
+                                    segmEndCoord = addZtoPoint(
+                                        val2[0]
+                                    )  # [631307.05960000027,5803698.4477999993,0]
+                                    segmCenter = addZtoPoint(
+                                        val2[1]
+                                    )  # [631307.05960000027,5803414.92656173]
+
+                                    if segmStartCoord == segmEndCoord:
+                                        if len(val2) == 4:
+                                            print("full circle")
+                                            # boundary.closed = True
+                                            segmentLocal = circleToSpeckle(
+                                                segmCenter, segmEndCoord
+                                            )
+                                            segments.append(segmentLocal)
+                                            lastPt = segmEndCoord
+                                            print("segmentLocal:")
+                                            print(segmentLocal)
+                                            print(segmStartCoord)
+                                            print(segmEndCoord)
+                                        else:  # ellipse
+                                            logToUser(
+                                                "SpeckleWarning: ellipse geometry not supported yet",
+                                                level=1,
+                                                func=inspect.stack()[0][3],
+                                            )
+                                            segments = []
+                                            break
+                                    else:  # elliptical curve
+                                        logToUser(
+                                            "SpeckleWarning: ellipse geometry not supported yet",
+                                            level=1,
+                                            func=inspect.stack()[0][3],
+                                        )
+                                        segments = []
+                                        break
+
+                                if key2 == "c":  # circular arc (endPt, throughPt)
+
+                                    segmEndCoord: List = addZtoPoint(
+                                        val2[0]
+                                    )  # [633718.26040000003,5803496.4210000001,0]
+                                    segmThrough: List = addZtoPoint(
+                                        val2[1]
+                                    )  # [633337.7576497585, 5803431.999702678]
+
+                                    segmentLocal = arc3ptToSpeckle(
+                                        segmStartCoord,
+                                        segmThrough,
+                                        segmEndCoord,
+                                        geom,
+                                        layer,
+                                    )
+                                    segments.append(segmentLocal)
+                                    print("segmentLocal:")
+                                    print(segmentLocal)
+                                    print(segmStartCoord)
+                                    print(segmEndCoord)
+                                    lastPt = segmEndCoord
+
+                                if key2 == "b":  # bezier curve (endPt, controlPts)
+                                    logToUser(
+                                        "SpeckleWarning: bezier curve geometry not supported yet",
+                                        level=1,
+                                        func=inspect.stack()[0][3],
+                                    )
+                                    segments = []
+                                    break
+                                    r"""
+                                    segmEndCoord: List = addZtoPoint(val2[0])  # [633718.26040000003,5803496.4210000001,0] 
+                                    #segmThrough: List = val2[1] # [633337.7576497585, 5803431.999702678] 
+                                    coords = val2[1:]
+                                    segmentLocal = curveBezierToSpeckle(segmStartCoord, segmEndCoord, coords, feature, layer) 
+                                    segments.append(segmentLocal)
+                                    print("segmentLocal:")
+                                    print(segmentLocal)
+                                    print(segmStartCoord)
+                                    print(segmEndCoord)
+                                    
+                                    lastPt = segmEndCoord 
+                                    """
+
+                        elif isinstance(segm[k], list) and isinstance(
+                            segm[k][0], float
+                        ):  # add line to point
+                            print("add line")
+                            segm[k] = addZtoPoint(segm[k])
+                            segmentLocal = lineFrom2pt(segmStartCoord, segm[k])
+                            includesLines = 1
+                            segments.append(segmentLocal)
+                            lastPt = segm[k]
+
+                            print("segmentLocal:")
+                            print(segmentLocal)
+                            print(segmentLocal.start)
+                            print(segmentLocal.end)
+
+                        # for the last point
+                        if k == len(segm) - 1 and isinstance(segm[k], list):
+                            print("last element is a point (adding line)")
+                            lastPt = addZtoPoint(lastPt)
+                            if lastPt != segm[0]:
+                                # segmentLocal = lineFrom2pt(lastPt, segm[0])
+                                # segments.append(segmentLocal)
+                                # includesLines = 1
+                                # print("segmentLocal:")
+                                # print(segmentLocal)
+                                # print(segmentLocal.start)
+                                # print(segmentLocal.end)
+                                boundary.closed = True
+                        # pts = speckleArcCircleToPoints(segmentLocal)
+                        # pts.append(segm[k])
+                        # arcgisPts = [arcpy.Point(pt[0], pt[1], pt[2]) for pt in pts]
+                        # segmentLocal = polylineFromVerticesToSpeckle(arcgisPts, True, feature, layer)
+
+        boundary.segments = segments
+        print(segments)
+
+        if len(segments) == 1:
+            boundary = segments[0]
+            # if isinstance(boundary, Arc) or isinstance(boundary, Circle):
+            #    boundary.displayValue = Polyline.from_points(speckleArcCircleToPoints(boundary))
+
+        elif len(segments) > 1:  # and includesLines == 0:
+            # boundary.displayValue = Polyline.from_points(specklePolycurveToPoints(boundary))
+            pass
+            # boundary.closed = True
+        # elif len(segments) > 1 and includesLines == 1:
+        #    print("includes lines!")
+        #    points = specklePolycurveToPoints(boundary)
+        #    boundary = Polyline.from_points(points)
+        else:
+            return None
+
+        # boundary.displayValue = Polyline.from_points(specklePolycurveToPoints(boundary))
+
+        print(boundary)
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return boundary
+
+
+def lineFrom2pt(pt1: List[float], pt2: List[float]):
+    line = Line(units="m")  # .from_list([*pt1, *pt2, *domain])
+    try:
+        pt1 = addZtoPoint(pt1)
+        pt2 = addZtoPoint(pt2)
+        dist = math.sqrt(
+            math.pow((pt2[0] - pt1[0]), 2)
+            + math.pow((pt2[1] - pt1[1]), 2)
+            + math.pow((pt2[2] - pt1[2]), 2)
+        )
+        print(dist)
+        domain = [0, dist, 0, 0]
+        line.start = Point.from_list(pt1)
+        line.end = Point.from_list(pt2)
+        line.start.units = line.end.units = "m"
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return line
+
+
+def polylineToNative(
+    poly: Polyline, sr: arcpy.SpatialReference, dataStorage
+) -> arcpy.Polyline:
+    """Converts a Speckle Polyline to QgsLineString"""
+    print("__ convert polyline to native __")
+    polyline = None
+    try:
+
+        if isinstance(poly, Polycurve):
+            poly = specklePolycurveToPoints(poly)
+        if isinstance(poly, Arc) or isinstance(poly, Circle):
+            try:
+                poly = poly["displayValue"]
+            except:
+                poly = speckleArcCircleToPoints(poly)
+
+        if isinstance(poly, list):
+            pts = [pointToCoord(pt) for pt in poly]
+        else:
+            pts = [pointToCoord(pt) for pt in poly.as_points()]
+
+        if poly.closed is True:
+            pts.append(pointToCoord(poly.as_points()[0]))
+
+        scale = get_scale_factor(poly.units)
+        pts = [[pt[0] * scale, pt[1] * scale, pt[2] * scale] for pt in pts]
+
+        pts_coord_list = [arcpy.Point(*coords) for coords in pts]
+        polyline = arcpy.Polyline(arcpy.Array(pts_coord_list), sr, has_z=True)
+        # print(polyline.JSON)
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return polyline
+
+
+def lineToNative(line: Line, sr: arcpy.SpatialReference, dataStorage) -> arcpy.Polyline:
+    """Converts a Speckle Line to Native"""
+    print("___Line to Native___")
+    try:
+        pts = [pointToCoord(pt) for pt in [line.start, line.end]]
+        scale = get_scale_factor(line.units)
+        pts = [[pt[0] * scale, pt[1] * scale, pt[2] * scale] for pt in pts]
+
+        line = arcpy.Polyline(
+            arcpy.Array([arcpy.Point(*coords) for coords in pts]), sr, has_z=True
+        )
+        return line
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None
+
+
+def curveToNative(
+    poly: Curve, sr: arcpy.SpatialReference, dataStorage
+) -> arcpy.Polyline:
+    """Converts a Speckle Curve to Native"""
+    try:
+        display = poly.displayValue
+        curve = polylineToNative(display, sr, dataStorage)
+        return curve
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None
+
+
+def arcToNative(poly: Arc, sr: arcpy.SpatialReference, dataStorage) -> arcpy.Polyline:
+    """Converts a Speckle Arc to Native"""
+    try:
+        arc = arcToNativePolyline(poly, sr, dataStorage)
+        return arc
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None
+
+
+def ellipseToNative(poly: Ellipse, sr: arcpy.SpatialReference, dataStorage):
+    logToUser("Ellipse geometry is not supported yet", level=1)
+    return
+
+
+def circleToNative(
+    poly: Circle, sr: arcpy.SpatialReference, dataStorage
+) -> arcpy.Polyline:
+    """Converts a Speckle Circle to QgsLineString"""
+    print("___Convert Circle to Native___")
+    curve = None
+    try:
+        points = []
+        angle1 = math.pi / 2
+
+        pointsNum = math.floor(math.pi * 2) * 12
+        if pointsNum < 4:
+            pointsNum = 4
+        points.append(pointToCoord(poly.plane.origin))
+
+        radScaled = poly.radius * get_scale_factor(poly.units)
+        points[0][1] += radScaled
+
+        for i in range(1, pointsNum + 1):
+            k = i / pointsNum  # to reset values from 1/10 to 1
+            if poly.plane.normal.z == 0:
+                normal = 1
+            else:
+                normal = poly.plane.normal.z
+            angle = angle1 + k * math.pi * 2 * normal
+            pt = Point(
+                x=poly.plane.origin.x * get_scale_factor(poly.units)
+                + radScaled * cos(angle),
+                y=poly.plane.origin.y * get_scale_factor(poly.units)
+                + radScaled * sin(angle),
+                z=0,
+            )
+            print(pt)
+            pt.units = "m"
+            points.append(pointToCoord(pt))
+        points.append(points[0])
+
+        scale = get_scale_factor(poly.units)
+        points = [[pt[0] * scale, pt[1] * scale, pt[2] * scale] for pt in points]
+
+        curve = arcpy.Polyline(
+            arcpy.Array([arcpy.Point(*coords) for coords in points]), sr, has_z=True
+        )
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return curve
+
+
+def polycurveToNative(
+    poly: Polycurve, sr: arcpy.SpatialReference, dataStorage
+) -> arcpy.Polyline:
+    points = []
+    curve = None
+    print("___Polycurve to native___")
+
+    try:
+        for i, segm in enumerate(poly.segments):  # Line, Polyline, Curve, Arc, Circle
+            print("___start segment")
+            if isinstance(segm, Line):
+                converted = lineToNative(segm, sr, dataStorage)  # QgsLineString
+            elif isinstance(segm, Polyline):
+                converted = polylineToNative(segm, sr, dataStorage)  # QgsLineString
+            elif isinstance(segm, Curve):
+                converted = curveToNative(segm, sr, dataStorage)  # QgsLineString
+            elif isinstance(segm, Circle):
+                converted = circleToNative(segm, sr, dataStorage)  # QgsLineString
+            elif isinstance(segm, Arc):
+                converted = arcToNativePolyline(segm, sr, dataStorage)  # QgsLineString
+            else:  # either return a part of the curve, of skip this segment and try next
+                logToUser(
+                    f"Part of the polycurve cannot be converted",
+                    level=1,
+                    func=inspect.stack()[0][3],
+                )
+                curve = arcpy.Polyline(
+                    arcpy.Array([arcpy.Point(*coords) for coords in points]),
+                    sr,
+                    has_z=True,
+                )
+                return curve
+            if converted is not None:
+                # print(converted) # <geoprocessing describe geometry object object at 0x000002B2D3E338D0>
+                for part in converted:
+                    # print("Part: ")
+                    # print(part) # <geoprocessing array object object at 0x000002B2D2E09530>
+                    for pt in part:
+                        # print(pt) # 64.4584221540162 5.5 NaN NaN
+                        if pt.Z != None:
+                            pt_z = pt.Z
+                        else:
+                            pt_z = 0
+                        # print(pt_z)
+                        # print(len(points))
+                        if (
+                            len(points) > 0
+                            and pt.X == points[len(points) - 1][0]
+                            and pt.Y == points[len(points) - 1][1]
+                            and pt_z == points[len(points) - 1][2]
+                        ):
+                            pass
+                        else:
+                            points.append(
+                                pointToCoord(Point(x=pt.X, y=pt.Y, z=pt_z, units="m"))
+                            )  # e.g. [[64.4584221540162, 5.499999999999999, 0.0], [64.45461685210796, 5.587155742747657, 0.0]]
+            else:
+                logToUser(
+                    f"Part of the polycurve cannot be converted",
+                    level=1,
+                    func=inspect.stack()[0][3],
+                )
+                curve = arcpy.Polyline(
+                    arcpy.Array([arcpy.Point(*coords) for coords in points]),
+                    sr,
+                    has_z=True,
+                )
+                return curve
+        # print(curve)
+
+        curve = arcpy.Polyline(
+            arcpy.Array([arcpy.Point(*coords) for coords in points]), sr, has_z=True
+        )
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return curve
+
+
+def arcToNativePolyline(
+    poly: Union[Arc, Circle], sr: arcpy.SpatialReference, dataStorage
+):
+    print("__Arc/Circle to native polyline__")
+    curve = None
+    try:
+        pointsSpeckle = speckleArcCircleToPoints(poly)
+        points = [pointToCoord(p) for p in pointsSpeckle]
+
+        scale = get_scale_factor(poly.units)
+        points = [[pt[0] * scale, pt[1] * scale, pt[2] * scale] for pt in points]
+
+        curve = arcpy.Polyline(
+            arcpy.Array([arcpy.Point(*coords) for coords in points]), sr, has_z=True
+        )
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return curve

speckle_toolbox/esri/toolboxes/speckle/speckle/converter/geometry/utils.py

@@ -0,0 +1,426 @@
+import math
+from math import cos, sin, atan
+import numpy as np
+from specklepy.objects.geometry import (
+    Point,
+    Line,
+    Polyline,
+    Circle,
+    Arc,
+    Polycurve,
+    Vector,
+)
+from specklepy.objects import Base
+from typing import List, Tuple, Union
+
+import inspect
+
+from speckle.speckle.utils.panel_logging import logToUser
+
+
+def addCorrectUnits(geom: Base, dataStorage) -> Base:
+    if not isinstance(geom, Base):
+        return None
+    units = dataStorage.currentUnits
+
+    geom.units = units
+    if isinstance(geom, Arc):
+        geom.plane.origin.units = units
+        geom.startPoint.units = units
+        geom.midPoint.units = units
+        geom.endPoint.units = units
+
+    elif isinstance(geom, Polycurve):
+        for s in geom.segments:
+            s.units = units
+            if isinstance(s, Arc):
+                s.plane.origin.units = units
+                s.startPoint.units = units
+                s.midPoint.units = units
+                s.endPoint.units = units
+
+    return geom
+
+
+def apply_pt_offsets_rotation_on_send(
+    x: float, y: float, dataStorage
+) -> Tuple[float, float]:  # on Send
+    try:
+        offset_x = dataStorage.crs_offset_x
+        offset_y = dataStorage.crs_offset_y
+        rotation = dataStorage.crs_rotation
+        if offset_x is not None and isinstance(offset_x, float):
+            x -= offset_x
+        if offset_y is not None and isinstance(offset_y, float):
+            y -= offset_y
+        if (
+            rotation is not None
+            and (isinstance(rotation, float) or isinstance(rotation, int))
+            and -360 < rotation < 360
+        ):
+            a = rotation * math.pi / 180
+            x2 = x * math.cos(a) + y * math.sin(a)
+            y2 = -x * math.sin(a) + y * math.cos(a)
+            x = x2
+            y = y2
+        return x, y
+    except Exception as e:
+        logToUser(e, level=2, func=inspect.stack()[0][3])
+        return x, y
+
+
+def transform_speckle_pt_on_receive(pt_original: Point, dataStorage) -> Point:
+    offset_x = dataStorage.crs_offset_x
+    offset_y = dataStorage.crs_offset_y
+    rotation = dataStorage.crs_rotation
+
+    pt = Point(
+        x=pt_original.x, y=pt_original.y, z=pt_original.z, units=pt_original.units
+    )
+
+    gisLayer = None
+    try:
+        gisLayer = dataStorage.latestHostApp.lower().endswith("gis")
+        applyTransforms = False if (gisLayer and gisLayer is True) else True
+    except Exception as e:
+        print(e)
+        applyTransforms = True
+
+    # for non-GIS layers
+    if applyTransforms is True:
+        if (
+            rotation is not None
+            and (isinstance(rotation, float) or isinstance(rotation, int))
+            and -360 < rotation < 360
+        ):
+            a = rotation * math.pi / 180
+            x2 = pt.x
+            y2 = pt.y
+
+            # if a > 0: # turn counterclockwise on receive
+            x2 = pt.x * math.cos(a) - pt.y * math.sin(a)
+            y2 = pt.x * math.sin(a) + pt.y * math.cos(a)
+
+            pt.x = x2
+            pt.y = y2
+        if (
+            offset_x is not None
+            and isinstance(offset_x, float)
+            and offset_y is not None
+            and isinstance(offset_y, float)
+        ):
+            pt.x += offset_x
+            pt.y += offset_y
+
+    # for GIS layers
+    if gisLayer is True:
+        try:
+            offset_x = dataStorage.current_layer_crs_offset_x
+            offset_y = dataStorage.current_layer_crs_offset_y
+            rotation = dataStorage.current_layer_crs_rotation
+
+            if (
+                rotation is not None
+                and isinstance(rotation, float)
+                and -360 < rotation < 360
+            ):
+                a = rotation * math.pi / 180
+                x2 = pt.x
+                y2 = pt.y
+
+                # if a > 0: # turn counterclockwise on receive
+                x2 = pt.x * math.cos(a) - pt.y * math.sin(a)
+                y2 = pt.x * math.sin(a) + pt.y * math.cos(a)
+
+                pt.x = x2
+                pt.y = y2
+            if (
+                offset_x is not None
+                and isinstance(offset_x, float)
+                and offset_y is not None
+                and isinstance(offset_y, float)
+            ):
+                pt.x += offset_x
+                pt.y += offset_y
+        except Exception as e:
+            print(e)
+
+    return pt
+
+
+def apply_pt_transform_matrix(pt_coords: List, dataStorage) -> List:
+    try:
+        if dataStorage.matrix is not None:
+            b = np.matrix(pt_coords + [1])
+            # print(b)
+            # print(dataStorage.matrix)
+            res = b * dataStorage.matrix
+            x, y, z = res.item(0), res.item(1), res.item(2)
+            return [x, y, z]
+    except Exception as e:
+        pass
+        # print(e)
+    return pt_coords
+
+
+def speckleBoundaryToSpecklePts(
+    boundary: Union[None, Polyline, Arc, Line, Polycurve]
+) -> List[Point]:
+    # print("__speckleBoundaryToSpecklePts__")
+    # add boundary points
+    polyBorder = []
+    try:
+        if isinstance(boundary, Circle) or isinstance(boundary, Arc):
+            polyBorder = speckleArcCircleToPoints(boundary)
+        elif isinstance(boundary, Polycurve):
+            polyBorder = specklePolycurveToPoints(boundary)
+        elif isinstance(boundary, Line):
+            pass
+        else:
+            try:
+                polyBorder = boundary.as_points()
+            except:
+                pass  # if Line or None
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return polyBorder
+
+
+def speckleArcCircleToPoints(poly: Union[Arc, Circle]) -> List[Point]:
+    print("__Arc or Circle to Points___")
+    points = []
+    try:
+        # print(poly.plane)
+        # print(poly.plane.normal)
+        if poly.plane is None or poly.plane.normal.z == 0:
+            normal = 1
+        else:
+            normal = poly.plane.normal.z
+        # print(poly.plane.origin)
+        if isinstance(poly, Circle):
+            interval = 2 * math.pi
+            range_start = 0
+            angle1 = 0
+
+        else:  # if Arc
+            points.append(poly.startPoint)
+            range_start = 0
+
+            # angle1, angle2 = getArcAngles(poly)
+
+            interval, angle1, angle2 = getArcRadianAngle(poly)
+            interval = abs(angle2 - angle1)
+
+            # print(angle1)
+            # print(angle2)
+
+            if (angle1 > angle2 and normal == -1) or (angle2 > angle1 and normal == 1):
+                pass
+            if angle1 > angle2 and normal == 1:
+                interval = abs((2 * math.pi - angle1) + angle2)
+            if angle2 > angle1 and normal == -1:
+                interval = abs((2 * math.pi - angle2) + angle1)
+
+            # print(interval)
+            # print(normal)
+
+        pointsNum = math.floor(abs(interval)) * 12
+        if pointsNum < 4:
+            pointsNum = 4
+
+        for i in range(range_start, pointsNum + 1):
+            k = i / pointsNum  # to reset values from 1/10 to 1
+            angle = angle1 + k * interval * normal
+            # print(k)
+            # print(angle)
+            pt = Point(
+                x=poly.plane.origin.x + poly.radius * cos(angle),
+                y=poly.plane.origin.y + poly.radius * sin(angle),
+                z=0,
+            )
+
+            pt.units = poly.plane.origin.units
+            points.append(pt)
+        if isinstance(poly, Arc):
+            points.append(poly.endPoint)
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return points
+
+
+def getArcRadianAngle(arc: Arc) -> List[float]:
+    try:
+        interval = None
+        normal = arc.plane.normal.z
+        angle1, angle2 = getArcAngles(arc)
+        if angle1 is None or angle2 is None:
+            return None
+        interval = abs(angle2 - angle1)
+
+        if (angle1 > angle2 and normal == -1) or (angle2 > angle1 and normal == 1):
+            pass
+        if angle1 > angle2 and normal == 1:
+            interval = abs((2 * math.pi - angle1) + angle2)
+        if angle2 > angle1 and normal == -1:
+            interval = abs((2 * math.pi - angle2) + angle1)
+        return interval, angle1, angle2
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None, None, None
+
+
+def getArcAngles(poly: Arc) -> Tuple[float]:
+    try:
+        if poly.startPoint.x == poly.plane.origin.x:
+            angle1 = math.pi / 2
+        else:
+            angle1 = atan(
+                abs(
+                    (poly.startPoint.y - poly.plane.origin.y)
+                    / (poly.startPoint.x - poly.plane.origin.x)
+                )
+            )  # between 0 and pi/2
+
+        if (
+            poly.plane.origin.x < poly.startPoint.x
+            and poly.plane.origin.y > poly.startPoint.y
+        ):
+            angle1 = 2 * math.pi - angle1
+        if (
+            poly.plane.origin.x > poly.startPoint.x
+            and poly.plane.origin.y > poly.startPoint.y
+        ):
+            angle1 = math.pi + angle1
+        if (
+            poly.plane.origin.x > poly.startPoint.x
+            and poly.plane.origin.y < poly.startPoint.y
+        ):
+            angle1 = math.pi - angle1
+
+        if poly.endPoint.x == poly.plane.origin.x:
+            angle2 = math.pi / 2
+        else:
+            angle2 = atan(
+                abs(
+                    (poly.endPoint.y - poly.plane.origin.y)
+                    / (poly.endPoint.x - poly.plane.origin.x)
+                )
+            )  # between 0 and pi/2
+
+        if (
+            poly.plane.origin.x < poly.endPoint.x
+            and poly.plane.origin.y > poly.endPoint.y
+        ):
+            angle2 = 2 * math.pi - angle2
+        if (
+            poly.plane.origin.x > poly.endPoint.x
+            and poly.plane.origin.y > poly.endPoint.y
+        ):
+            angle2 = math.pi + angle2
+        if (
+            poly.plane.origin.x > poly.endPoint.x
+            and poly.plane.origin.y < poly.endPoint.y
+        ):
+            angle2 = math.pi - angle2
+
+        return angle1, angle2
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None, None
+
+
+def specklePolycurveToPoints(poly: Polycurve) -> List[Point]:
+    print("_____Speckle Polycurve to points____")
+    points = []
+    try:
+        for segm in poly.segments:
+            # print(segm)
+            pts = []
+            if isinstance(segm, Arc) or isinstance(
+                segm, Circle
+            ):  # or isinstance(segm, Curve):
+                print("Arc or Curve")
+                pts: List[Point] = speckleArcCircleToPoints(segm)
+            elif isinstance(segm, Line):
+                print("Line")
+                pts: List[Point] = [segm.start, segm.end]
+            elif isinstance(segm, Polyline):
+                print("Polyline")
+                pts: List[Point] = segm.as_points()
+
+            points.extend(pts)
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return points
+
+
+def getArcNormal(poly: Arc, midPt: Point):
+    print("____getArcNormal___")
+    try:
+        angle1, angle2 = getArcAngles(poly)
+
+        if midPt.x == poly.plane.origin.x:
+            angle = math.pi / 2
+        else:
+            angle = atan(
+                abs((midPt.y - poly.plane.origin.y) / (midPt.x - poly.plane.origin.x))
+            )  # between 0 and pi/2
+
+        if poly.plane.origin.x < midPt.x and poly.plane.origin.y > midPt.y:
+            angle = 2 * math.pi - angle
+        if poly.plane.origin.x > midPt.x and poly.plane.origin.y > midPt.y:
+            angle = math.pi + angle
+        if poly.plane.origin.x > midPt.x and poly.plane.origin.y < midPt.y:
+            angle = math.pi - angle
+
+        normal = Vector()
+        normal.x = normal.y = 0
+
+        if angle1 > angle > angle2:
+            normal.z = -1
+        if angle1 > angle2 > angle:
+            normal.z = 1
+
+        if angle2 > angle1 > angle:
+            normal.z = -1
+        if angle > angle1 > angle2:
+            normal.z = 1
+
+        if angle2 > angle > angle1:
+            normal.z = 1
+        if angle > angle2 > angle1:
+            normal.z = -1
+
+        print(angle1)
+        print(angle)
+        print(angle2)
+        print(normal)
+        return normal
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None
+
+
+def getArcCenter(p1: Point, p2: Point, p3: Point) -> Tuple[List, float]:
+    # print(p1)
+    try:
+        p1 = np.array(p1.to_list())
+        p2 = np.array(p2.to_list())
+        p3 = np.array(p3.to_list())
+        a = np.linalg.norm(p3 - p2)
+        b = np.linalg.norm(p3 - p1)
+        c = np.linalg.norm(p2 - p1)
+        s = (a + b + c) / 2
+        radius = a * b * c / 4 / np.sqrt(s * (s - a) * (s - b) * (s - c))
+        b1 = a * a * (b * b + c * c - a * a)
+        b2 = b * b * (a * a + c * c - b * b)
+        b3 = c * c * (a * a + b * b - c * c)
+        center = np.column_stack((p1, p2, p3)).dot(np.hstack((b1, b2, b3)))
+        center /= b1 + b2 + b3
+        center = center.tolist()
+        return center, radius
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None, None

speckle_toolbox/esri/toolboxes/speckle/speckle/converter/layers/Layer.py

@@ -2,13 +2,13 @@ from typing import List, Optional
 from specklepy.objects.base import Base
 
 try: 
-    from speckle.converter.layers.CRS import CRS
+    from speckle.speckle.converter.layers.CRS import CRS
 except:
-    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.CRS import CRS
+    from speckle_toolbox.esri.toolboxes.speckle.speckle.converter.layers.CRS import CRS
 
 
 
-class Layer(Base, chunkable={"features": 100}):
+class Layer(Base, chunkable={"elements": 100}):
     """A GIS Layer"""
 
     def __init__(
@@ -16,7 +16,7 @@ class Layer(Base, chunkable={"features": 100}):
         name: Optional[str] = None,
         crs: Optional[CRS] = None,
         datum: Optional[CRS] = None,
-        features: List[Base] = [],
+        elements: List[Base] = [],
         layerType: str = "None",
         geomType: str = "None",
         renderer: dict = {},
@@ -26,12 +26,12 @@ class Layer(Base, chunkable={"features": 100}):
         self.name = name
         self.crs = crs
         self.datum = datum
-        self.type = layerType
-        self.features = features
+        self.collectionType = layerType
+        self.elements = elements
         self.geomType = geomType
         self.renderer = renderer 
 
-class VectorLayer(Base, chunkable={"features": 100}):
+class VectorLayer(Base, chunkable={"elements": 100}):
     """A GIS Vector Layer"""
 
     def __init__(
@@ -39,7 +39,7 @@ class VectorLayer(Base, chunkable={"features": 100}):
         name: Optional[str] = None,
         crs: Optional[CRS] = None,
         datum: Optional[CRS] = None,
-        features: List[Base] = [],
+        elements: List[Base] = [],
         layerType: str = "None",
         geomType: str = "None",
         renderer: dict = {},
@@ -49,12 +49,12 @@ class VectorLayer(Base, chunkable={"features": 100}):
         self.name = name
         self.crs = crs
         self.datum = datum
-        self.type = layerType
-        self.features = features
+        self.collectionType = layerType
+        self.elements = elements
         self.geomType = geomType
         self.renderer = renderer 
 
-class RasterLayer(Base, chunkable={"features": 100}):
+class RasterLayer(Base, chunkable={"elements": 100}):
     """A GIS Raster Layer"""
 
     def __init__(
@@ -62,7 +62,7 @@ class RasterLayer(Base, chunkable={"features": 100}):
         name: Optional[str] = None,
         crs: Optional[CRS] =None,
         rasterCrs: Optional[CRS] = None,
-        features: List[Base] = [],
+        elements: List[Base] = [],
         layerType: str = "None",
         geomType: str = "None",
         renderer: dict = {},
@@ -72,8 +72,8 @@ class RasterLayer(Base, chunkable={"features": 100}):
         self.name = name
         self.crs = crs
         self.rasterCrs = rasterCrs
-        self.type = layerType
-        self.features = features
+        self.collectionType = layerType
+        self.elements = elements
         self.geomType = geomType
         self.renderer = renderer 
     

speckle_toolbox/esri/toolboxes/speckle/speckle/converter/layers/__init__.py

@@ -0,0 +1,137 @@
+"""
+Contains all Layer related classes and methods.
+"""
+
+from typing import Any, List, Tuple, Union
+
+import inspect
+
+from speckle.speckle.plugin_utils.helpers import (
+    SYMBOL,
+)
+from speckle.speckle.utils.panel_logging import logToUser
+
+from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
+
+
+def getAllProjLayers(plugin) -> List[arcLayer]:
+    # print("get all project layers")
+    layers = []
+    try:
+        project: ArcGISProject = plugin.project
+        if project.activeMap is not None and isinstance(
+            project.activeMap, Map
+        ):  # if project loaded
+            # print(type(project.activeMap))
+            for layer in project.activeMap.listLayers():
+                if (layer.isFeatureLayer) or layer.isRasterLayer:
+                    layers.append(layer)  # type: 'arcpy._mp.Layer'
+        else:
+            # print(type(project.activeMap))
+            logToUser(
+                "Cannot get Project layers, Project Active Map not loaded or not selected",
+                level=1,
+                func=inspect.stack()[0][3],
+                plugin=plugin.dockwidget,
+            )
+            return None
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return layers
+
+
+def getLayersWithStructure(
+    plugin, bySelection=False
+) -> Tuple[List[arcLayer], List[str]]:
+    """Gets a list of all layers in the map"""
+    layers = []
+    structure = []
+    try:
+        print("___ get layers list ___")
+
+        # issue with getting selected layers: https://community.esri.com/t5/python-questions/determining-selected-layers-in-the-table-of/td-p/252098
+
+        self = plugin.dockwidget
+        project = plugin.project
+        map = project.activeMap
+        if map is None:
+            logToUser(
+                "Project Active Map not loaded or not selected",
+                level=2,
+                func=inspect.stack()[0][3],
+            )
+            return None, None
+
+        if bySelection is True:  # by selection
+            # print("get selected layers")
+            for layer in project.activeMap.listLayers():
+                if layer.visible and (layer.isFeatureLayer or layer.isRasterLayer):
+
+                    # find possibly hidden parent groups
+                    layerGroupsHidden = 0
+                    for group in project.activeMap.listLayers():
+                        if (
+                            layer.longName.startswith(group.longName + "\\")
+                            and group.visible is False
+                        ):
+                            for sub_layer in group.listLayers():
+                                if sub_layer.longName == layer.longName:
+                                    layerGroupsHidden += 1
+                                    break
+                            # .isGroupLayer method is broken: https://community.esri.com/t5/python-questions/arcpy-property-layer-isgrouplayer-not-working-as/td-p/709250
+                            r""" 
+                            if group.isGroupLayer:
+                                print("__groups")
+                                print(group.longName + "_" + str(group.visible))
+                                if layer.longName.startswith(group.longName + "\\"):
+                                    print(group.visible)
+                                    if group.visible is False:
+                                        layerGroupsHidden += 1
+                                        break
+                            """
+                    if layerGroupsHidden == 0:
+                        layers.append(layer)
+                        structure.append(
+                            ("\\".join(layer.longName.split("\\")[:-1]) + "\\").replace(
+                                "\\", SYMBOL
+                            )
+                        )
+            # print("layers selected and saved")
+        else:  # from project data
+            # all_layers_ids = [l.id() for l in project.mapLayers().values()]
+            for item in plugin.dataStorage.current_layers:
+                try:
+                    layerPath = item[1].dataSource
+
+                    found = 0
+
+                    all_layers = getAllProjLayers(plugin)
+                    if all_layers is None:
+                        return None
+                    for l in all_layers:
+                        if l.dataSource == layerPath:
+                            layers.append(l)
+                            structure.append(
+                                "\\".join(l.longName.split("\\")[:-1] + "\\").replace(
+                                    "\\", SYMBOL
+                                )
+                            )
+                            found += 1
+                            break
+                    if found == 0:
+                        logToUser(
+                            f'Saved layer not found: "{item[0]}"',
+                            level=1,
+                            func=inspect.stack()[0][3],
+                        )
+
+                except:
+                    logToUser(
+                        f'Saved layer not found: "{item[0]}"',
+                        level=1,
+                        func=inspect.stack()[0][3],
+                    )
+                    continue
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return layers, structure

speckle_toolbox/esri/toolboxes/speckle/speckle/converter/layers/utils.py

@@ -0,0 +1,817 @@
+import copy
+from datetime import datetime
+import random
+from typing import Dict, Any, List, Tuple, Union
+import json
+import hashlib
+from specklepy.objects import Base
+from specklepy.objects.geometry import Mesh
+from specklepy.objects.other import Collection
+
+import arcpy
+from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
+import os
+
+import inspect
+
+from PyQt5.QtGui import QColor
+
+from speckle.speckle.converter.layers.emptyLayerTemplates import createGroupLayer
+from speckle.speckle.plugin_utils.helpers import findOrCreatePath, SYMBOL
+from speckle.speckle.utils.panel_logging import logToUser
+from speckle.speckle.plugin_utils.helpers import validateNewFclassName
+
+
+# ATTRS_REMOVE = ['geometry','applicationId','bbox','displayStyle', 'id', 'renderMaterial', 'displayMesh', 'displayValue']
+ATTRS_REMOVE = [
+    "speckleTyp",
+    "speckle_id",
+    "geometry",
+    "applicationId",
+    "bbox",
+    "displayStyle",
+    "id",
+    "renderMaterial",
+    "displayMesh",
+    "displayValue",
+]
+
+
+def generate_qgis_app_id(
+    base: Base,
+    layer,
+    f,
+):
+    """Generate unique ID for Vector feature."""
+    return ""
+    try:
+        fieldnames = [str(field.name()) for field in layer.fields()]
+        props = [str(f[prop]) for prop in fieldnames]
+        try:
+            geoms = f.geometry()
+        except Exception as e:
+            geoms = ""
+
+        id_data: str = (
+            layer.id()
+            + str(layer.wkbType())
+            + str(fieldnames)
+            + str(props)
+            + str(geoms)
+        )
+        return hashlib.md5(id_data.encode("utf-8")).hexdigest()
+
+    except Exception as e:
+        logToUser(
+            f"Application ID not generated for feature in layer {layer.name()}: {e}",
+            level=1,
+        )
+        return ""
+
+
+def generate_qgis_raster_app_id(rasterLayer):
+    """Generate unique ID for Raster layer."""
+    return ""
+    try:
+        id_data = str(get_raster_stats(rasterLayer))
+        file_ds = gdal.Open(rasterLayer.source(), gdal.GA_ReadOnly)
+        for i in range(rasterLayer.bandCount()):
+            band = file_ds.GetRasterBand(i + 1)
+            id_data += str(band.ReadAsArray())
+        return hashlib.md5(id_data.encode("utf-8")).hexdigest()
+
+    except Exception as e:
+        logToUser(
+            f"Application ID not generated for layer {rasterLayer.name()}: {e}",
+            level=1,
+        )
+        return ""
+
+
+def findUpdateJsonItemPath(tree: Dict, full_path_str: str):
+    try:
+        new_tree = copy.deepcopy(tree)
+
+        path_list_original = full_path_str.split(SYMBOL)
+        path_list = []
+        for x in path_list_original:
+            if len(x) > 0:
+                path_list.append(x)
+        attr_found = False
+
+        for i, item in enumerate(new_tree.items()):
+            attr, val_dict = item
+
+            if attr == path_list[0]:
+                attr_found = True
+                path_list.pop(0)
+                if len(path_list) > 0:  # if the path is not finished:
+                    all_names = val_dict.keys()
+                    if (
+                        len(path_list) == 1 and path_list[0] in all_names
+                    ):  # already in a tree
+                        return new_tree
+                    else:
+                        branch = findUpdateJsonItemPath(
+                            val_dict, SYMBOL.join(path_list)
+                        )
+                        new_tree.update({attr: branch})
+
+        if (
+            attr_found is False and len(path_list) > 0
+        ):  # create a new branch at the top level
+            if len(path_list) == 1:
+                new_tree.update({path_list[0]: {}})
+                return new_tree
+            else:
+                branch = findUpdateJsonItemPath(
+                    {path_list[0]: {}}, SYMBOL.join(path_list)
+                )
+                new_tree.update(branch)
+        return new_tree
+    except Exception as e:
+        # logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return tree
+
+
+def collectionsFromJson(
+    jsonObj: dict, levels: list, layerConverted, baseCollection: Collection
+):
+    if jsonObj == {} or len(levels) == 0:
+        # print("RETURN")
+        baseCollection.elements.append(layerConverted)
+        return baseCollection
+
+    lastLevel = baseCollection
+    for i, l in enumerate(levels):
+        sub_collection_found = 0
+        for item in lastLevel.elements:
+            # print("___ITEM")
+            # print(l)
+            if item.name == l:
+                # print("___ITEM FOUND")
+                # print(l)
+                lastLevel = item
+                sub_collection_found = 1
+                break
+        if sub_collection_found == 0:
+            # print("___ SUB COLLECTION NOT FOUND")
+            subCollection = Collection(
+                units="m", collectionType="ArcGIS Layer Group", name=l, elements=[]
+            )
+            lastLevel.elements.append(subCollection)
+            lastLevel = lastLevel.elements[
+                len(lastLevel.elements) - 1
+            ]  # reassign last element
+
+        if i == len(levels) - 1:  # if last level
+            lastLevel.elements.append(layerConverted)
+
+    return baseCollection
+
+
+def findAndClearLayerGroup(project: ArcGISProject, newGroupName: str = "", plugin=None):
+    print("find And Clear LayerGroup")
+    try:
+        groupExists = 0
+        for l in project.activeMap.listLayers():
+            if l.longName.startswith(newGroupName + "\\"):
+                if l.isFeatureLayer:
+                    # condition for feature layers:
+                    fields = [f.name for f in arcpy.ListFields(l.dataSource)]
+
+                    if "Speckle_ID" in fields or "speckle_id" in fields:
+                        project.activeMap.removeLayer(l)
+                        groupExists += 1
+                elif l.isRasterLayer:
+                    # condition for raster layers:
+                    if "_Speckle" in l.name:
+                        project.activeMap.removeLayer(l)
+                        groupExists += 1
+
+            elif l.longName == newGroupName:
+                groupExists += 1
+
+        if groupExists == 0:
+            layerGroup = create_layer_group(project, newGroupName, plugin)
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3], plugin=plugin.dockwidget)
+
+
+def create_layer_group(project: ArcGISProject, newGroupName: str, plugin):
+    try:
+        if project.activeMap is not None:
+            # print("try creating the group")
+            # check for full match
+            for l in project.activeMap.listLayers():
+                # print(newGroupName + "  __  " + l.longName)
+                if l.longName == newGroupName and l.isGroupLayer:
+                    layerGroup = l
+                    return layerGroup
+            # check for parent layer
+            for l in project.activeMap.listLayers():
+                # print(newGroupName + "  __  " + l.longName)
+                if (
+                    l.longName == "\\".join(newGroupName.split("\\")[:-1])
+                    and l.isGroupLayer
+                ):
+                    short_name = newGroupName.split("\\")[-1]
+                    new_group = project.activeMap.createGroupLayer(short_name, l)
+                    return new_group
+
+            layerGroup = project.activeMap.createGroupLayer(newGroupName)
+            # print(layerGroup)
+            return layerGroup
+        else:
+            logToUser(
+                "The map didn't fully load, try selecting the project Map or/and refreshing the plugin.",
+                level=1,
+                func=inspect.stack()[0][3],
+            )
+            return
+    except Exception as e:
+        print(e)
+
+
+def getVariantFromValue(value: Any) -> Union[str, None]:
+    # print("_________get variant from value_______")
+    # TODO add Base object
+    res = None
+    try:
+        pairs = [(str, "TEXT"), (float, "FLOAT"), (int, "LONG"), (bool, "SHORT")]  # 10
+        for p in pairs:
+            if isinstance(value, p[0]):
+                res = p[1]
+                try:
+                    if res == "LONG" and (value >= 2147483647 or value <= -2147483647):
+                        # https://pro.arcgis.com/en/pro-app/latest/help/data/geodatabases/overview/arcgis-field-data-types.htm
+                        res = "FLOAT"
+                except Exception as e:
+                    print(e)
+                break
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+
+    return res
+
+
+def colorFromSpeckle(rgb):
+    try:
+        color = QColor.fromRgb(245, 245, 245)
+        if isinstance(rgb, int):
+            r = (rgb & 0xFF0000) >> 16
+            g = (rgb & 0xFF00) >> 8
+            b = rgb & 0xFF
+            color = QColor.fromRgb(r, g, b)
+        return color
+    except Exception as e:
+        logToUser(e, level=2, func=inspect.stack()[0][3])
+        return QColor.fromRgb(245, 245, 245)
+
+
+def getDisplayValueList(geom: Any) -> List:
+    try:
+        # print("___getDisplayValueList")
+        val = []
+        # get list of display values for Meshes
+        if isinstance(geom, Mesh):
+            val = [geom]
+        elif isinstance(geom, List) and len(geom) > 0:
+            if isinstance(geom[0], Mesh):
+                val = geom
+            else:
+                print("not an individual geometry")
+        else:
+            try:
+                val = geom.displayValue  # list
+            except Exception as e:
+                # print(e)
+                try:
+                    val = geom["@displayValue"]  # list
+                except Exception as e:
+                    # print(e)
+                    try:
+                        val = geom.displayMesh
+                    except:
+                        pass
+        return val
+    except Exception as e:
+        # print(e)
+        return []
+
+
+def getLayerGeomType(layer) -> str:
+    return
+
+
+def tryCreateGroupTree(project: ArcGISProject, fullGroupName, plugin=None):
+
+    # CREATE A GROUP "received blabla" with sublayers
+    # print("_________CREATE GROUP TREE: " + fullGroupName)
+
+    # receive_layer_tree: dict = plugin.receive_layer_tree
+    receive_layer_list = fullGroupName.split(SYMBOL)
+    path_list = []
+    for x in receive_layer_list:
+        if len(x) > 0:
+            path_list.append(x)
+    group_to_create_name = path_list[0]
+    layerGroup = create_layer_group(project, group_to_create_name, plugin)
+    path_list.pop(0)
+
+    if len(path_list) > 0:
+        path_list[0] = f"{group_to_create_name}\\{path_list[0]}"
+        layerGroup = tryCreateGroupTree(project, SYMBOL.join(path_list), plugin)
+
+    return layerGroup
+
+
+def validateAttributeName(name: str, fieldnames: List[str]) -> str:
+    try:
+        new_list = [x for x in fieldnames if x != name]
+
+        corrected = name.replace("/", "_").replace(".", "_")
+        if corrected == "id":
+            corrected = "applicationId"
+
+        for i, x in enumerate(corrected):
+            if corrected[0] != "_" and corrected not in new_list:
+                break
+            else:
+                corrected = corrected[1:]
+
+        if len(corrected) <= 1 and len(name) > 1:
+            corrected = "0" + name  # if the loop removed the property name completely
+
+        return corrected
+    except Exception as e:
+        logToUser(e, level=2, func=inspect.stack()[0][3])
+        return
+
+
+def trySaveCRS(crs, streamBranch: str = ""):
+    return
+    try:
+        authid = crs.authid()
+        wkt = crs.toWkt()
+        if authid == "":
+            crs_id = crs.saveAsUserCrs("SpeckleCRS_" + streamBranch)
+            return crs_id
+        else:
+            return crs.srsid()
+    except Exception as e:
+        logToUser(e, level=2, func=inspect.stack()[0][3])
+        return
+
+
+def getLayerAttributes(
+    featuresList: List[Base], attrsToRemove: List[str] = ATTRS_REMOVE
+) -> Dict[str, str]:
+    # print("03________ get layer attributes")
+    fields = {}
+    try:
+        if not isinstance(featuresList, list):
+            features = [featuresList]
+        else:
+            features = featuresList[:]
+        # print(features)
+        all_props = []
+        for feature in features:
+            # get object properties to add as attributes
+            try:
+                dynamicProps = (
+                    feature.attributes.get_dynamic_member_names()
+                )  # for 2.14 onwards
+            except:
+                dynamicProps = feature.get_dynamic_member_names()
+
+            for att in ATTRS_REMOVE:
+                try:
+                    dynamicProps.remove(att)
+                except ValueError:
+                    pass
+
+            dynamicProps.sort()
+
+            # add field names and variands
+            for name in dynamicProps:
+                # if name not in all_props: all_props.append(name)
+
+                try:
+                    value = feature.attributes[name]
+                except:
+                    value = feature[name]
+                variant = getVariantFromValue(value)
+                # if name == 'area': print(value); print(variant)
+                if not variant:
+                    variant = None  # LongLong #4
+
+                # go thought the dictionary object
+                if value and isinstance(value, list):
+                    # all_props.remove(name) # remove generic dict name
+                    for i, val_item in enumerate(value):
+                        newF, newVals = traverseDict(
+                            {}, {}, name + "_" + str(i), val_item
+                        )
+
+                        for i, (k, v) in enumerate(newF.items()):
+                            if k not in all_props:
+                                all_props.append(k)
+                            if k not in fields.keys():
+                                fields.update({k: v})
+                            else:  # check if the field was empty previously:
+                                oldVariant = fields[k]
+                                # replace if new one is NOT Float (too large integers)
+                                # if oldVariant != "FLOAT" and v == "FLOAT":
+                                #    fields.update({k: v})
+                                # replace if new one is NOT LongLong or IS String
+                                if oldVariant != "TEXT" and v == "TEXT":
+                                    fields.update({k: v})
+
+                # add a field if not existing yet
+                else:  # if str, Base, etc
+                    newF, newVals = traverseDict({}, {}, name, value)
+
+                    for i, (k, v) in enumerate(newF.items()):
+                        if k not in all_props:
+                            all_props.append(k)
+                        if k not in fields.keys():
+                            fields.update({k: v})  # if variant is known
+                        else:  # check if the field was empty previously:
+                            oldVariant = fields[k]
+                            # replace if new one is NOT Float (too large integers)
+                            # print(oldVariant, v)
+                            # if oldVariant == "LONG" and v == "FLOAT":
+                            #    fields.update({k: v})
+                            # replace if new one is NOT LongLong or IS String
+                            if oldVariant != "TEXT" and v == "TEXT":
+                                fields.update({k: v})
+                            # print(fields)
+        # replace all empty ones wit String
+        all_props.append("Speckle_ID")
+        for name in all_props:
+            if name not in fields.keys():
+                fields.update({name: "TEXT"})
+        # print(fields)
+        # fields_sorted = {k: v for k, v in sorted(fields.items(), key=lambda item: item[0])}
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return fields
+
+
+def traverseDict(newF: dict, newVals: dict, nam: str, val: Any):
+    try:
+        if isinstance(val, dict):
+            for i, (k, v) in enumerate(val.items()):
+                newF, newVals = traverseDict(newF, newVals, nam + "_" + k, v)
+        elif isinstance(val, Base):
+            dynamicProps = val.get_dynamic_member_names()
+            for att in ATTRS_REMOVE:
+                try:
+                    dynamicProps.remove(att)
+                except:
+                    pass
+            dynamicProps.sort()
+
+            item_dict = {}
+            for prop in dynamicProps:
+                item_dict.update({prop: val[prop]})
+
+            for i, (k, v) in enumerate(item_dict.items()):
+                newF, newVals = traverseDict(newF, newVals, nam + "_" + k, v)
+        else:
+            var = getVariantFromValue(val)
+            if var is None:
+                var = "TEXT"
+                val = str(val)
+            # print(var)
+            newF.update({nam: var})
+            newVals.update({nam: val})
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return newF, newVals
+
+
+def get_scale_factor(units: str) -> float:
+    unit_scale = {
+        "meters": 1.0,
+        "centimeters": 0.01,
+        "millimeters": 0.001,
+        "inches": 0.0254,
+        "feet": 0.3048,
+        "kilometers": 1000.0,
+        "mm": 0.001,
+        "cm": 0.01,
+        "m": 1.0,
+        "km": 1000.0,
+        "in": 0.0254,
+        "ft": 0.3048,
+        "yd": 0.9144,
+        "mi": 1609.340,
+    }
+    if units is not None and units.lower() in unit_scale.keys():
+        return unit_scale[units]
+    logToUser(
+        f"Units {units} are not supported. Meters will be applied by default.",
+        level=0,
+        func=inspect.stack()[0][3],
+    )
+    return 1.0
+
+
+def findTransformation(
+    geomType,
+    layer_sr: arcpy.SpatialReference,
+    projectCRS: arcpy.SpatialReference,
+    selectedLayer: arcLayer,
+) -> Tuple:
+    # apply transformation if needed
+    try:
+        print("___findTransformation___")
+        print(layer_sr.name)
+        print(projectCRS.name)
+        tr0 = tr1 = tr2 = tr_custom = None
+        if (
+            geomType != "Point"
+            and geomType != "Polyline"
+            and geomType != "Polygon"
+            and geomType != "Multipoint"
+            and geomType != "MultiPatch"
+        ):
+            try:
+                logToUser(
+                    "Unsupported or invalid geometry in layer " + selectedLayer.name,
+                    level=2,
+                    func=inspect.stack()[0][3],
+                )
+            except:
+                logToUser(
+                    "Unsupported or invalid geometry",
+                    level=2,
+                    func=inspect.stack()[0][3],
+                )
+            return layer_sr, None, None, None, None
+
+        if layer_sr.name == projectCRS.name:
+            return layer_sr, None, None, None, None
+
+        # get list of transforms and find similar one
+        transformations = arcpy.ListTransformations(layer_sr, projectCRS)
+
+        customTransformName = (
+            hashlib.md5(layer_sr.exportToString().encode("utf-8")).hexdigest()
+            + "_To_"
+            + hashlib.md5(projectCRS.exportToString().encode("utf-8")).hexdigest()
+        )
+        print(customTransformName)
+        for tr in transformations:
+            print(tr)
+            if tr == customTransformName:
+                tr0 = tr
+                print("found")
+                return layer_sr, tr0, tr1, tr2, tr_custom
+
+        if tr0 is None:  # if no valid transforms found
+            try:  # try find intermediate transforms
+                midSr = arcpy.SpatialReference("WGS 1984")  # GCS_WGS_1984
+                tr1 = arcpy.ListTransformations(layer_sr, midSr)[0]
+                tr2 = arcpy.ListTransformations(midSr, projectCRS)[0]
+            except Exception as e:  # create custom transform in nothing else works
+                print(e)
+                layer_sr = arcpy.SpatialReference(text=layer_sr.exportToString())
+                projectCRS = arcpy.SpatialReference(text=projectCRS.exportToString())
+                customGeoTransfm = "GEOGTRAN[METHOD['Geocentric_Translation'],PARAMETER['X_Axis_Translation',''],PARAMETER['Y_Axis_Translation',''],PARAMETER['Z_Axis_Translation','']]"
+                try:
+                    arcpy.management.CreateCustomGeoTransformation(
+                        customTransformName, layer_sr, projectCRS, customGeoTransfm
+                    )
+                except Exception as e:  # if already exists
+                    logToUser(
+                        f"Spatial Transformation cannot be created: {e}",
+                        level=2,
+                        func=inspect.stack()[0][3],
+                    )
+                tr_custom = customTransformName
+                print(tr_custom)
+            return layer_sr, tr0, tr1, tr2, tr_custom
+        r"""
+        else:
+            print("else")
+            # choose equation based instead of file-based/grid-based method,
+            # to be consistent with QGIS: https://desktop.arcgis.com/en/arcmap/latest/map/projections/choosing-an-appropriate-transformation.htm
+            selecterTr = {}
+            for tr in transformations:
+                if "NTv2" not in tr and "NADCON" not in tr:
+                    set1 = set(
+                        layer_sr.name.split("_") + projectCRS.name.split("_")
+                    )
+                    set2 = set(tr.split("_"))
+                    diff = len(set(set1).symmetric_difference(set2))
+                    selecterTr.update({tr: diff})
+            selecterTr = dict(
+                sorted(selecterTr.items(), key=lambda item: item[1])
+            )
+            tr0 = list(selecterTr.keys())[0]
+            # print(tr0)
+        """
+
+    except Exception as e:
+        logToUser(
+            f"Spatial Transformation not found for layer {selectedLayer.name}: {e}",
+            level=2,
+            func=inspect.stack()[0][3],
+        )
+        return layer_sr, None, None, None, None
+
+
+def apply_reproject(f_shape, transforms: Tuple, dataStorage):
+    try:
+        layer_sr, tr0, tr1, tr2, tr_custom = transforms
+        projectCRS = dataStorage.project.activeMap.spatialReference
+        # reproject geometry using chosen transformstion(s)
+        if tr0 is not None:
+            ptgeo1 = f_shape.projectAs(projectCRS, tr0)
+            f_shape = ptgeo1
+        elif tr1 is not None and tr2 is not None:
+            midSr = arcpy.SpatialReference("WGS 1984")
+            ptgeo1 = f_shape.projectAs(midSr, tr1)
+            ptgeo2 = ptgeo1.projectAs(projectCRS, tr2)
+            f_shape = ptgeo2
+            r"""
+            elif tr_custom is not None:
+                print(tr_custom)
+                ptgeo1 = f_shape.projectAs(projectCRS, tr_custom)
+                f_shape = ptgeo1
+            """
+        else:
+            projectCRS = arcpy.SpatialReference(text=projectCRS.exportToString())
+            ptgeo1 = f_shape.projectAs(projectCRS)
+            f_shape = ptgeo1
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return f_shape
+
+
+def traverseDictByKey(d: Dict, key: str = "", result=None) -> Dict:
+    print("__traverse")
+    try:
+        result = None
+        # print(d)
+        for k, v in d.items():
+
+            try:
+                v = json.loads(v)
+            except:
+                pass
+            if isinstance(v, dict):
+                # print("__dict__")
+                if k == key:
+                    print("__break loop")
+                    result = v
+                    return result
+                else:
+                    result = traverseDictByKey(v, key, result)
+                    if result is not None:
+                        return result
+            if isinstance(v, list):
+                for item in v:
+                    # print(item)
+                    if isinstance(item, dict):
+                        result = traverseDictByKey(item, key, result)
+                        if result is not None:
+                            return result
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None
+    # print("__result is: ____________")
+    # return result
+
+
+def hsv_to_rgb(listHSV):
+    try:
+        h, s, v = listHSV[0], listHSV[1], listHSV[2]
+        if s == 0.0:
+            v *= 255
+            return (v, v, v)
+        i = int(h * 6.0)  # XXX assume int() truncates!
+        f = (h * 6.0) - i
+        p, q, t = (
+            int(255 * (v * (1.0 - s))),
+            int(255 * (v * (1.0 - s * f))),
+            int(255 * (v * (1.0 - s * (1.0 - f)))),
+        )
+        v *= 255
+        i %= 6
+        if i == 0:
+            return (v, t, p)
+        if i == 1:
+            return (q, v, p)
+        if i == 2:
+            return (p, v, t)
+        if i == 3:
+            return (p, q, v)
+        if i == 4:
+            return (t, p, v)
+        if i == 5:
+            return (v, p, q)
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return (0, 0, 0)
+
+
+def cmyk_to_rgb(c, m, y, k, cmyk_scale, rgb_scale=255):
+    try:
+        r = rgb_scale * (1.0 - c / float(cmyk_scale)) * (1.0 - k / float(cmyk_scale))
+        g = rgb_scale * (1.0 - m / float(cmyk_scale)) * (1.0 - k / float(cmyk_scale))
+        b = rgb_scale * (1.0 - y / float(cmyk_scale)) * (1.0 - k / float(cmyk_scale))
+        return r, g, b
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return 0, 0, 0
+
+
+def newLayerGroupAndName(
+    layerName: str, streamBranch: str, project: ArcGISProject
+) -> str:
+    print("___new Layer Group and Name")
+    layerGroup = None
+    newGroupName = f"{streamBranch}"
+    try:
+        # CREATE A GROUP "received blabla" with sublayers
+        print(newGroupName)
+        for l in project.activeMap.listLayers():
+            if l.longName == newGroupName:
+                layerGroup = l
+                break
+
+        # find a layer with a matching name in the "latest" group
+        newName = (
+            f'{streamBranch.split("_")[len(streamBranch.split("_"))-1]}_{layerName}'
+        )
+
+        all_layer_names = []
+        layerExists = 0
+        for l in project.activeMap.listLayers():
+            if l.longName.startswith(newGroupName + "\\"):
+                all_layer_names.append(l.longName)
+        # print(all_layer_names)
+        print(newName)
+
+        newName = validateNewFclassName(newName, all_layer_names, streamBranch + "\\")
+
+        print(newName)
+        return newName, layerGroup
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None, None
+
+
+def curvedFeatureClassToSegments(layer) -> str:
+    print("___densify___")
+    try:
+        data = arcpy.Describe(layer.dataSource)
+        dataPath = data.catalogPath
+        print(dataPath)
+        newPath = dataPath + "_backup"
+
+        arcpy.management.CopyFeatures(
+            dataPath, newPath
+        )  # features copied like this do not preserve curved segments
+
+        arcpy.edit.Densify(
+            in_features=newPath,
+            densification_method="ANGLE",
+            max_angle=0.01,
+            max_vertex_per_segment=100,
+        )  # https://pro.arcgis.com/en/pro-app/latest/tool-reference/editing/densify.htm
+        print(newPath)
+        return newPath
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None
+
+
+def validate_path(path: str, plugin) -> str:
+    """If our path contains a DB name, make sure we have a valid DB name and not a standard file name."""
+    # https://github.com/EsriOceans/btm/commit/a9c0529485c9b0baa78c1f094372c0f9d83c0aaf
+    dirname, file_name = os.path.split(path)
+    # print(dirname)
+    # print(file_name)
+    file_base = os.path.splitext(file_name)[0]
+    if dirname == "":
+        # a relative path only, relying on the workspace
+        dirname = plugin.workspace
+    path_ext = os.path.splitext(dirname)[1].lower()
+    if path_ext in [".mdb", ".gdb", ".sde"]:
+        # we're working in a database
+        file_name = arcpy.ValidateTableName(
+            file_base
+        )  # e.g. add a letter in front of the name
+    validated_path = os.path.join(dirname, file_name)
+    # msg("validated path: %s; (from %s)" % (validated_path, path))
+    return validated_path

speckle_toolbox/esri/toolboxes/speckle/speckle/plugin_utils/helpers.py

@@ -0,0 +1,218 @@
+import os
+from typing import List
+import inspect
+
+SYMBOL = "_x_x_"
+UNSUPPORTED_PROVIDERS = ["WFS", "wms", "wcs", "vectortile"]
+
+
+def get_scale_factor(units: str, dataStorage) -> float:
+    scale_to_meter = get_scale_factor_to_meter(units)
+    if dataStorage is not None:
+        scale_back = scale_to_meter / get_scale_factor_to_meter(
+            dataStorage.currentUnits
+        )
+        return scale_back
+    else:
+        return scale_to_meter
+
+
+def get_scale_factor_to_meter(units: str) -> float:
+    try:
+        unit_scale = {
+            "meters": 1.0,
+            "centimeters": 0.01,
+            "millimeters": 0.001,
+            "inches": 0.0254,
+            "feet": 0.3048,
+            "kilometers": 1000.0,
+            "mm": 0.001,
+            "cm": 0.01,
+            "m": 1.0,
+            "km": 1000.0,
+            "in": 0.0254,
+            "ft": 0.3048,
+            "yd": 0.9144,
+            "mi": 1609.340,
+        }
+        if (
+            units is not None
+            and isinstance(units, str)
+            and units.lower() in unit_scale.keys()
+        ):
+            return unit_scale[units]
+        try:
+            from speckle.speckle.utils.panel_logging import logToUser
+
+            logToUser(
+                f"Units {units} are not supported. Meters will be applied by default.",
+                level=1,
+                func=inspect.stack()[0][3],
+            )
+            return 1.0
+        except:
+            print(
+                f"Units {units} are not supported. Meters will be applied by default."
+            )
+            return 1.0
+    except Exception as e:
+        try:
+            from speckle.speckle.utils.panel_logging import logToUser
+
+            logToUser(
+                f"{e}. Meters will be applied by default.",
+                level=2,
+                func=inspect.stack()[0][3],
+            )
+            return 1.0
+        except:
+            print(f"{e}. Meters will be applied by default.")
+            return 1.0
+
+
+def getAppName(name: str) -> str:
+    new_name = ""
+    for i, x in enumerate(str(name)):
+        if x.lower() in [a for k, a in enumerate("abcdefghijklmnopqrstuvwxyz")]:
+            new_name += x
+        else:
+            break
+    return new_name
+
+
+def findOrCreatePath(path: str):
+    if not os.path.exists(path):
+        os.makedirs(path)
+
+
+def removeSpecialCharacters(text: str) -> str:
+    new_text = (
+        text.replace("[", "_")
+        .replace("]", "_")
+        .replace(".", "_")
+        .replace(" ", "_")
+        .replace("-", "_")
+        .replace("—", "_")
+        .replace("(", "_")
+        .replace(")", "_")
+        .replace(":", "_")
+        .replace("\\", "_")
+        .replace("/", "_")
+        .replace('"', "_")
+        .replace("&", "_")
+        .replace("@", "_")
+        .replace("$", "_")
+        .replace("%", "_")
+        .replace("^", "_")
+    )
+    # new_text = text.encode('iso-8859-1', errors='ignore').decode('utf-8')
+    return new_text
+
+
+def splitTextIntoLines(text: str = "", number: int = 40) -> str:
+    print("__splitTextIntoLines")
+    print(text)
+    msg = ""
+    try:
+        if len(text) > number:
+            try:
+                for i, x in enumerate(text):
+                    msg += x
+                    if i != 0 and i % number == 0:
+                        msg += "\n"
+            except Exception as e:
+                print(e)
+        else:
+            msg = text
+    except Exception as e:
+        print(e)
+        print(text)
+
+    return msg
+
+
+def jsonFromList(jsonObj: dict, levels: list):
+    # print("jsonFromList")
+    if len(levels) == 0:
+        return jsonObj
+    lastLevel = jsonObj
+    for l in levels:
+        # print(lastLevel)
+        try:
+            lastLevel = lastLevel[l]
+        except:
+            lastLevel.update({l: {}})
+    # print(jsonObj)
+    return jsonObj
+
+
+def validateNewFclassName(
+    newName: str, all_layer_names: List[str], prefix: str = ""
+) -> str:
+
+    fixed_name = newName
+
+    if (prefix + fixed_name) in all_layer_names:
+
+        layerNameCreated = 0
+        for index, letter in enumerate("234567890abcdefghijklmnopqrstuvwxyz"):
+            if ((prefix + fixed_name) + "_" + letter) not in all_layer_names:
+                fixed_name += "_" + letter
+                layerNameCreated += 1
+                break
+        if layerNameCreated == 0:
+            for index, letter in enumerate("234567890abcdefghijklmnopqrstuvwxyz"):
+                test_fixed_name = validateNewFclassName(
+                    (fixed_name + "_" + letter), all_layer_names, prefix
+                )
+                if (prefix + test_fixed_name) not in all_layer_names:
+                    fixed_name = test_fixed_name
+                    layerNameCreated += 1
+                    break
+                # else: layerNameCreated +=1
+
+        if layerNameCreated == 0:
+            pass  # logToUser('Feature class name already exists', level=2, func = inspect.stack()[0][3])
+            # return fixed_name
+
+    return fixed_name
+
+
+def findFeatColors(fetColors, f):
+
+    colorFound = 0
+    try:  # get render material from any part of the mesh (list of items in displayValue)
+        for k, item in enumerate(f.displayValue):
+            try:
+                fetColors.append(item.renderMaterial.diffuse)
+                colorFound += 1
+                break
+            except:
+                pass
+        if colorFound == 0:
+            fetColors.append(f.renderMaterial.diffuse)
+    except:
+        try:
+            for k, item in enumerate(f["@displayValue"]):
+                try:
+                    fetColors.append(item.renderMaterial.diffuse)
+                    colorFound += 1
+                    break
+                except:
+                    pass
+            if colorFound == 0:
+                fetColors.append(f.renderMaterial.diffuse)
+        except:
+            # the Mesh itself has a renderer
+            try:  # get render material from any part of the mesh (list of items in displayValue)
+                fetColors.append(f.renderMaterial.diffuse)
+                colorFound += 1
+            except:
+                try:
+                    fetColors.append(f.displayStyle.color)
+                    colorFound += 1
+                except:
+                    pass
+    if colorFound == 0:
+        fetColors.append(None)
+    return fetColors

speckle_toolbox/esri/toolboxes/speckle/speckle/plugin_utils/object_utils.py

@@ -0,0 +1,365 @@
+import time
+from typing import Any, Callable, List, Optional
+
+import inspect
+import numpy as np
+
+from specklepy.objects import Base
+from specklepy.objects.GIS.layers import VectorLayer, RasterLayer, Layer
+
+
+from speckle.speckle.converter.layers.utils import findUpdateJsonItemPath
+from speckle.speckle.plugin_utils.helpers import SYMBOL, removeSpecialCharacters
+from speckle.speckle.utils.panel_logging import logToUser
+from speckle.speckle.converter.layers.layer_conversions import (
+    geometryLayerToNative,
+    layerToNative,
+    nonGeometryLayerToNative,
+)
+
+import arcpy
+
+SPECKLE_TYPES_TO_READ = [
+    "Objects.Geometry.",
+    "Objects.BuiltElements.",
+    "IFC",
+]  # will properly traverse and check for displayValue
+
+
+def traverseObject(
+    plugin,
+    base: Base,
+    callback: Optional[Callable[[Base, str, Any], bool]],
+    check: Optional[Callable[[Base], bool]],
+    streamBranch: str,
+    nameBase: str = "",
+):
+    try:
+        if check and check(base):
+            res = callback(base, streamBranch, nameBase, plugin) if callback else False
+            if res:
+                return
+        memberNames = base.get_member_names()
+        for name in memberNames:
+            if name in ["id", "applicationId", "units", "speckle_type"]:
+                continue
+            try:  # some of the static members cannot be called via [""]
+                base[name]
+            except KeyError:
+                continue
+
+            if (
+                nameBase == SYMBOL + "ArcGIS commit"
+                or nameBase == SYMBOL + "QGIS commit"
+            ):
+                name_pass = getBaseValidName(base, name)
+            else:
+                name_pass = nameBase + SYMBOL + getBaseValidName(base, name)
+            # check again
+            if (
+                name_pass == SYMBOL + "ArcGIS commit"
+                or name_pass == SYMBOL + "QGIS commit"
+            ):
+                name_pass = ""
+
+            traverseValue(plugin, base[name], callback, check, streamBranch, name_pass)
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+
+
+def traverseValue(
+    plugin,
+    value: Any,
+    callback: Optional[Callable[[Base, str, Any], bool]],
+    check: Optional[Callable[[Base], bool]],
+    streamBranch: str,
+    name: str,
+):
+    if isinstance(value, Base):
+        traverseObject(plugin, value, callback, check, streamBranch, name)
+    if isinstance(value, List):
+        for item in value:
+            traverseValue(plugin, item, callback, check, streamBranch, name)
+
+
+def callback(base: Base, streamBranch: str, nameBase: str, plugin) -> bool:
+
+    try:
+        if (
+            isinstance(base, VectorLayer)
+            or isinstance(base, Layer)
+            or isinstance(base, RasterLayer)
+            or base.speckle_type.endswith("VectorLayer")
+            or base.speckle_type.endswith("RasterLayer")
+        ):
+            layerToNative(base, streamBranch, nameBase, plugin)
+        else:
+            loopObj(base, "", streamBranch, plugin, [])
+        return True
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+
+
+def getBaseValidName(base: Base, name: str) -> str:
+    name_pass = name
+    search = 0
+    try:
+        if name == "elements" and isinstance(base[name], list):
+            search = 1
+    except:
+        pass
+    try:
+        if name == "displayValue" or name == "@displayValue":
+            search = 1
+    except:
+        pass
+    try:
+        if name == "definition" and isinstance(base[name], Base):
+            search = 1
+    except:
+        pass
+
+    try:
+        if search == 1:
+            try:
+                if (
+                    (base["name"], str)
+                    and len(base["name"]) > 1
+                    and base["name"] != "null"
+                ):
+                    name_pass = base["name"]
+                else:
+                    raise Exception
+            except:
+                try:
+                    if (
+                        (base["Name"], str)
+                        and len(base["Name"]) > 1
+                        and base["Name"] != "null"
+                    ):
+                        name_pass = base["Name"]
+                    else:
+                        raise Exception
+                except:
+                    try:
+                        if (
+                            (base["type"], str)
+                            and len(base["type"]) > 1
+                            and base["type"] != "null"
+                        ):
+                            name_pass = base["type"]
+                        else:
+                            raise Exception
+                    except:
+                        try:
+                            if (
+                                (base["category"], str)
+                                and len(base["category"]) > 1
+                                and base["category"] != "null"
+                            ):
+                                name_pass = base["category"]
+                            else:
+                                raise Exception
+                        except:
+                            name_pass = name
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+
+    return name_pass
+
+
+def loopObj(
+    base: Base, baseName: str, streamBranch: str, plugin, used_ids, matrix=None
+):
+    try:
+        # dont loop primitives
+        if not isinstance(base, Base):
+            return
+        # print(base)
+        memberNames = base.get_member_names()
+
+        baseName_pass = removeSpecialCharacters(baseName)
+
+        plugin.receive_layer_tree = findUpdateJsonItemPath(
+            plugin.receive_layer_tree, streamBranch + SYMBOL + baseName_pass
+        )
+
+        for name in memberNames:
+            if name in ["id", "applicationId", "units", "speckle_type"]:
+                continue
+            # skip if traversal goes to displayValue of an object, that will be readable anyway:
+
+            if (
+                name == "displayValue" or name == "@displayValue"
+            ) and base.speckle_type.startswith(tuple(SPECKLE_TYPES_TO_READ)):
+                continue
+
+            try:
+                if (
+                    "View" in base[name].speckle_type
+                    or "RevitMaterial" in base[name].speckle_type
+                ):
+                    continue
+            except:
+                pass
+
+            name_pass = baseName_pass + SYMBOL + getBaseValidName(base, name)
+
+            if base[name] is not None:
+                if name.endswith("definition"):
+                    # print("___Definition object: " + name)
+                    try:
+                        matrixList = base["transform"].matrix
+                        try:
+                            matrix2 = np.matrix(matrixList).reshape(4, 4)
+                            matrix2 = matrix2.transpose()
+                            if matrix2 is None:
+                                geometryLayerToNative(
+                                    [base[name]],
+                                    name,
+                                    base.id,
+                                    streamBranch,
+                                    plugin,
+                                    None,
+                                )
+
+                            else:  # both not None
+                                if matrix is not None:
+                                    matrix = matrix2 * matrix
+                                else:  # matrix is None
+                                    matrix = matrix2
+                                # print(matrix)
+                                geometryLayerToNative(
+                                    [base[name]],
+                                    name_pass,
+                                    base.id,
+                                    streamBranch,
+                                    plugin,
+                                    matrix,
+                                )
+
+                        except:
+                            matrix = None
+                        time.sleep(0.3)
+                    except Exception as e:
+                        print(f"ERROR: {e}")
+                loopVal(
+                    base[name],
+                    name_pass,
+                    base.id,
+                    streamBranch,
+                    plugin,
+                    used_ids,
+                    matrix,
+                )
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+
+
+def loopVal(
+    value: Any, name: str, val_id: str, streamBranch: str, plugin, used_ids, matrix=None
+):  # "name" is the parent object/property/layer name
+    try:
+        name = removeSpecialCharacters(name)
+        if isinstance(value, Base):
+            try:  # loop through objects with Speckletype prop, but don't go through parts of Speckle Geometry object
+                if (
+                    "View" in value.speckle_type
+                    or "RevitMaterial" in value.speckle_type
+                ):
+                    return
+
+                if not value.speckle_type.startswith("Objects.Geometry."):
+                    loopObj(value, name, streamBranch, plugin, used_ids, matrix)
+                elif value.id not in used_ids:  # if geometry
+                    used_ids.append(value.id)
+                    loopVal(
+                        [value], name, value.id, streamBranch, plugin, used_ids, matrix
+                    )
+            except:
+                loopObj(value, name, streamBranch, plugin, used_ids, matrix)
+
+        elif isinstance(value, List):
+            streamBranch = removeSpecialCharacters(streamBranch)
+
+            objectListConverted = 0
+            for i, item in enumerate(value):
+                if not isinstance(item, Base):
+                    continue
+
+                used_ids.append(item.id)
+                loopVal(item, name, item.id, streamBranch, plugin, used_ids, matrix)
+
+                if not isinstance(item, Base):
+                    continue
+                if "View" in item.speckle_type or "RevitMaterial" in item.speckle_type:
+                    continue
+
+                if item.speckle_type and item.speckle_type.startswith("IFC"):
+                    # keep traversing infinitely, just don't run repeated conversion for the same list of objects
+                    try:
+                        if (
+                            item["displayValue"] is not None
+                            and objectListConverted == 0
+                        ):
+                            geometryLayerToNative(
+                                value, name, val_id, streamBranch, plugin
+                            )
+                            time.sleep(0.3)
+                            objectListConverted += 1
+                    except:
+                        try:
+                            if (
+                                item["@displayValue"] is not None
+                                and objectListConverted == 0
+                            ):
+                                geometryLayerToNative(
+                                    value, name, val_id, streamBranch, plugin
+                                )
+                                time.sleep(0.3)
+                                objectListConverted += 1
+                        except:
+                            pass
+                elif item.speckle_type and item.speckle_type.endswith(".ModelCurve"):
+                    if item["baseCurve"] is not None:
+                        geometryLayerToNative(value, name, val_id, streamBranch, plugin)
+                        time.sleep(0.3)
+                        break
+                elif (
+                    plugin.dataStorage.latestHostApp.lower().endswith("excel")
+                    or item.speckle_type == "Objects.Organization.DataTable"
+                ):
+                    # should be before the check for "BuiltElements"
+                    nonGeometryLayerToNative(value, name, val_id, streamBranch, plugin)
+                    time.sleep(0.3)
+                    break
+                elif item.speckle_type and (
+                    item.speckle_type == "Objects.Geometry.Mesh"
+                    or item.speckle_type == "Objects.Geometry.Brep"
+                    or item.speckle_type.startswith("Objects.BuiltElements.")
+                ):
+                    geometryLayerToNative(value, name, val_id, streamBranch, plugin)
+                    time.sleep(0.3)
+                    break
+                elif (
+                    item.speckle_type
+                    and item.speckle_type != "Objects.Geometry.Mesh"
+                    and item.speckle_type != "Objects.Geometry.Brep"
+                    and item.speckle_type.startswith("Objects.Geometry.")
+                ):  # or item.speckle_type == 'Objects.BuiltElements.Alignment'):
+                    geometryLayerToNative(value, name, val_id, streamBranch, plugin)
+                    time.sleep(0.3)
+                    break
+                elif item.speckle_type:
+                    try:
+                        if item["baseLine"] is not None:
+                            geometryLayerToNative(
+                                value, name, val_id, streamBranch, plugin
+                            )
+                            time.sleep(0.3)
+                            break
+                    except Exception as e:
+                        pass
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])

speckle_toolbox/esri/toolboxes/speckle/speckle/plugin_utils/threads.py

@@ -0,0 +1,63 @@
+
+import sys
+import trace
+import threading
+
+class KThread(threading.Thread):
+    """A subclass of threading.Thread, with a kill()
+    method."""
+    # https://web.archive.org/web/20130503082442/http://mail.python.org/pipermail/python-list/2004-May/281943.html 
+    def __init__(self, *args, **keywords):
+        threading.Thread.__init__(self, *args, **keywords)
+        self.killed = False
+
+    def start(self):
+        """Start the thread."""
+        self.__run_backup = self.run
+        self.run = self.__run      # Force the Thread to install our trace.
+        threading.Thread.start(self)
+
+    def __run(self):
+        """Hacked run function, which installs the trace."""
+        sys.settrace(self.globaltrace)
+        self.__run_backup()
+        self.run = self.__run_backup
+
+    def globaltrace(self, frame, why, arg):
+        if why == 'call':
+            return self.localtrace
+        else:
+            return None
+
+    def localtrace(self, frame, why, arg):
+        if self.killed:
+            if why == 'line':
+                raise SystemExit()
+        return self.localtrace
+
+    def kill(self):
+        self.killed = True
+
+class KillableThread(threading.Thread):
+    # is NOT running in the background 
+    # https://stackoverflow.com/questions/323972/is-there-any-way-to-kill-a-thread 
+    def __init__(self, sleep_interval=1):
+        super().__init__()
+        self._kill = threading.Event()
+        self._interval = sleep_interval
+
+    def run(self):
+        while True:
+            print("Do Something")
+
+            # If no kill signal is set, sleep for the interval,
+            # If kill signal comes in while sleeping, immediately
+            #  wake up and handle
+            is_killed = self._kill.wait(self._interval)
+            if is_killed:
+                break
+
+        print("Killing Thread")
+
+    def kill(self):
+        self._kill.set()

speckle_toolbox/esri/toolboxes/speckle/speckle/utils/logger.py

@@ -0,0 +1,74 @@
+from PyQt5.QtWidgets import QMessageBox
+from PyQt5 import QtCore
+import arcpy
+
+try:
+    from speckle.speckle.plugin_utils.helpers import splitTextIntoLines
+except:
+    from speckle_toolbox.esri.toolboxes.speckle.speckle.plugin_utils.helpers import (
+        splitTextIntoLines,
+    )
+
+import inspect
+
+
+def logToUser(msg: str, func=None, level: int = 2, plugin=None, url="", blue=False):
+    # print("Log to user")
+    # print(msg)
+
+    msg = str(msg)
+    dockwidget = plugin
+
+    try:
+        if url == "" and blue is False:  # only for info messages
+            msg = addLevelSymbol(msg, level)
+            if func is not None:
+                msg += "::" + str(func)
+        writeToLog(msg, level)
+
+        if dockwidget is None:
+            return
+
+        new_msg = splitTextIntoLines(msg, 70)
+
+        dockwidget.msgLog.addButton(new_msg, level=level, url=url, blue=blue)
+
+    except Exception as e:
+        print(e)
+        return
+
+
+def logToUserWithAction(msg: str, level: int = 0, plugin=None, url=""):
+    print("Log to user with action")
+    return
+    msg = str(msg)
+    dockwidget = plugin
+    if dockwidget is None:
+        return
+    try:
+        new_msg = splitTextIntoLines(msg, 70)
+        dockwidget.msgLog.addButton(new_msg, level=level, url=url)
+        writeToLog(new_msg, level)
+    except Exception as e:
+        print(e)
+        return
+
+
+def addLevelSymbol(msg: str, level: int):
+    if level == 0:
+        msg = "🛈 " + msg
+    if level == 1:
+        msg = "⚠️ " + msg
+    if level == 2:
+        msg = "❗ " + msg
+    return msg
+
+
+def writeToLog(msg: str = "", level: int = 2):
+    print(msg)
+    if level == 0:
+        arcpy.AddMessage(msg)
+    if level == 1:
+        arcpy.AddWarning(msg)
+    if level == 2:
+        arcpy.AddError(msg)

speckle_toolbox/esri/toolboxes/speckle/speckle/utils/panel_logging.py

@@ -0,0 +1,128 @@
+"""Logging Utility Module for Speckle QGIS"""
+
+import inspect
+from typing import Union
+import webbrowser
+
+import arcpy
+
+
+def logToUser(
+    msg: Union[str, Exception],
+    func=None,
+    level: int = 2,
+    plugin=None,
+    url="",
+    blue=False,
+    report=False,
+):
+    from speckle.specklepy_qt_ui.qt_ui.utils.logger import logToUser as logToUser_UI
+
+    msg = str(msg)
+    print(msg + "::" + str(func))
+    logToUser_UI(msg, func, level, plugin, url, blue, report)
+    logger.writeToLog(msg.replace("\n", ". ") + " " + url, level, func)
+
+
+class Logging:
+    """Holds utility methods for logging messages to QGIS"""
+
+    qgisInterface = None
+
+    def __init__(self, iface) -> None:
+        self.qgisInterface = iface
+
+    def log(self, message: str, level: int = 0):
+        """Logs a specific message to the Speckle messages panel."""
+        try:
+            if level == 0:
+                arcpy.AddMessage(message)
+            elif level == 1:
+                arcpy.AddWarning(message)
+            # elif level == 2: 3 error will quit pluging
+            #    arcpy.AddError(message)
+        except Exception as e:
+            try:
+                logToUser(e, level=2, func=inspect.stack()[0][3])
+            except:
+                pass
+
+    def btnClicked(url):
+        try:
+            if url == "":
+                return
+            webbrowser.open(url, new=0, autoraise=True)
+        except Exception as e:
+            pass
+
+    def logToUserWithAction(
+        self,
+        message: str,
+        action_text: str,
+        url: str = "",
+        level: int = 0,
+        duration: int = 120,
+    ):
+        self.log(message, level)
+        return
+        if not self.qgisInterface:
+            return
+        try:
+            from qgis.core import Qgis
+            from qgis.PyQt.QtWidgets import QPushButton
+
+            if level == 0:
+                level = Qgis.Info
+            elif level == 1:
+                level = Qgis.Warning
+            elif level == 2:
+                level = Qgis.Critical
+
+            widget = self.qgisInterface.messageBar().createMessage("Speckle", message)
+            button = QPushButton(widget)
+            button.setText(action_text)
+            button.pressed.connect(lambda: self.btnClicked(url))
+            widget.layout().addWidget(button)
+            self.qgisInterface.messageBar().pushWidget(widget, level, duration)
+        except ImportError:
+            pass
+
+    def logToUserPanel(
+        self,
+        message: str,
+        level: int = 0,
+        duration: int = 20,
+        func=None,
+        plugin=None,
+    ):
+        """Logs a specific message to the user in QGIS"""
+        return
+        self.log(message, level)
+
+        if not self.qgisInterface:
+            return
+        try:
+            from qgis.core import Qgis
+
+            if level == 0:
+                level = Qgis.Info
+            if level == 1:
+                level = Qgis.Warning
+            if level == 2:
+                level = Qgis.Critical
+
+            if self.qgisInterface:
+                self.qgisInterface.messageBar().pushMessage(
+                    "Speckle", message, level=level, duration=duration
+                )
+        except ImportError:
+            pass
+
+    def writeToLog(self, msg: str = "", level: int = 2, func=None, plugin=None):
+        msg = str(msg)
+        if func is not None and func != "None":
+            msg += "::" + str(func)
+        self.log(msg, level)
+
+
+logger = Logging(None)

speckle_toolbox/esri/toolboxes/speckle/speckle/utils/project_vars.py

@@ -0,0 +1,616 @@
+from typing import Any, List, Optional, Tuple, Union
+import arcpy
+from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
+from arcpy.management import CreateTable
+
+import os.path
+
+from specklepy.api.credentials import Account, get_local_accounts
+from specklepy.api.client import SpeckleClient
+from specklepy.logging.exceptions import (
+    GraphQLException,
+    SpeckleException,
+)
+from specklepy.api.wrapper import StreamWrapper
+from specklepy.api.models import Branch, Stream, Streams
+from specklepy.logging import metrics
+
+from osgeo import osr
+
+import inspect
+
+from speckle.speckle.utils.validation import tryGetStream
+
+# from speckle.speckle.speckle_arcgis import SpeckleGIS
+from speckle.speckle.converter.layers import getAllProjLayers
+from speckle.speckle.utils.panel_logging import logToUser
+
+FIELDS = [
+    "project_streams",
+    "project_layer_selection",
+    "lat_lon",
+    "crs_rotation",
+    "crs_offsets",
+]
+
+
+def get_project_streams(plugin: "SpeckleGIS", content: str = None):
+    try:
+        print("GET proj streams")
+        project = plugin.project
+        table = findOrCreateSpeckleTable(project, plugin)
+        #logToUser(table, level=0, func=inspect.stack()[0][3])
+
+        rows = arcpy.da.SearchCursor(table, "project_streams")
+        saved_streams = []
+        for x in rows:
+            # logToUser(x[0], level=0, func=inspect.stack()[0][3])
+            saved_streams.append(x[0])
+
+        temp = []
+        ######### need to check whether saved streams are available (account reachable)
+        if len(saved_streams) > 0:
+            for url in saved_streams:
+                if url == "":
+                    continue
+                try:
+                    sw = StreamWrapper(url)
+                    try:
+                        stream = tryGetStream(sw, plugin.dataStorage)
+                    except SpeckleException as e:
+                        logToUser(e.message, level=2, func=inspect.stack()[0][3])
+                        stream = None
+                    # strId = stream.id # will cause exception if invalid
+                    temp.append((sw, stream))
+                except SpeckleException as e:
+                    logToUser(e.message, level=2, func=inspect.stack()[0][3])
+                # except GraphQLException as e:
+                #    logger.logToUser(e.message, Qgis.Warning)
+        plugin.current_streams = temp
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+
+
+def set_project_streams(plugin: "SpeckleGIS"):
+    try:
+        print("SET proj streams")
+        project = plugin.project
+        table = findOrCreateSpeckleTable(project, plugin)
+        print("SET proj streams 2")
+
+        current_streams = [
+            stream[0].stream_url for stream in plugin.current_streams
+        ]  # ",".join()
+        print(current_streams)
+        logToUser(current_streams, level=0, func=inspect.stack()[0][3])
+
+        if table is not None:
+            proj_layers = []
+            lan_lot = ""
+            with arcpy.da.UpdateCursor(table, FIELDS) as cursor:
+                for row in cursor:  # just one row
+                    if row[1] is not None and row[1] != "":
+                        proj_layers.append(row[1])
+                    if row[2] is not None and row[2] != "":
+                        lan_lot = row[2]
+                    cursor.deleteRow()
+            del cursor
+            if len(proj_layers) == 0:
+                proj_layers.append("")
+            if len(current_streams) == 0:
+                current_streams.append("")
+
+            cursor = arcpy.da.InsertCursor(table, FIELDS[:3])
+            length = max(len(proj_layers), len(current_streams))
+
+            for i in range(length):
+                if i == 0:
+                    cursor.insertRow([current_streams[i], proj_layers[i], lan_lot])
+                else:
+                    try:
+                        cursor.insertRow([current_streams[i], proj_layers[i], ""])
+                    except:
+                        if len(current_streams) <= i:
+                            cursor.insertRow(["", proj_layers[i], ""])
+                        if len(proj_layers) <= i:
+                            cursor.insertRow([current_streams[i], "", ""])
+            del cursor
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+
+
+def get_project_layer_selection(plugin: "SpeckleGIS"):
+    try:
+        #print("GET project layer selection from the table")
+        project = plugin.project
+        table = findOrCreateSpeckleTable(project, plugin)
+        if table is None:
+            return
+
+        rows = arcpy.da.SearchCursor(table, "project_layer_selection")
+        saved_layers = []
+        for x in rows:
+            saved_layers.append(x[0])
+
+        temp = []
+        proj_layers = getAllProjLayers(plugin)
+        if proj_layers is None:
+            return
+        ######### need to check whether saved streams are available (account reachable)
+        if len(saved_layers) > 0:
+            for layerPath in saved_layers:
+                if layerPath == "":
+                    continue
+                found = 0
+                for layer in proj_layers:
+                    #print(layer.dataSource)
+                    if layer.dataSource == layerPath:
+                        temp.append((layer.name, layer))
+                        found += 1
+                        break
+                if found == 0:
+                    logToUser(
+                        f'Saved layer not found: "{layerPath}"',
+                        level=1,
+                        func=inspect.stack()[0][3],
+                    )
+        plugin.dataStorage.current_layers = temp
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+
+
+def set_project_layer_selection(plugin: "SpeckleGIS"):
+    try:
+        print("SET project layer selection function")
+        project = plugin.project
+        value: List[str] = [
+            layer[1].dataSource for layer in plugin.dataStorage.current_layers
+        ]  # ",".join([layer[1].dataSource for layer in plugin.dataStorage.current_layers])
+        #print(value)
+
+        table = findOrCreateSpeckleTable(project, plugin)
+        # print(table)
+        if table is not None:
+            lan_lot = ""
+            proj_streams = []
+            with arcpy.da.UpdateCursor(table, FIELDS[:3]) as cursor:
+                for row in cursor:  # just one row
+                    if row[0] is not None and row[0] != "":
+                        proj_streams.append(row[0])
+                    if row[2] is not None and row[2] != "":
+                        lan_lot = row[2]
+                    cursor.deleteRow()
+            del cursor
+            if len(proj_streams) == 0:
+                proj_streams.append("")
+            if len(value) == 0:
+                value.append("")
+            # print(proj_streams)
+
+            cursor = arcpy.da.InsertCursor(table, FIELDS[:3])
+            length = max(len(proj_streams), len(value))
+            # print(length)
+            for i in range(length):
+                if i == 0:
+                    cursor.insertRow([proj_streams[i], value[i], lan_lot])
+                    #print(i)
+                else:
+                    try:
+                        cursor.insertRow([proj_streams[i], value[i], ""])
+                    except:
+                        if len(proj_streams) <= i:
+                            cursor.insertRow(["", value[i], ""])
+                        if len(value) <= i:
+                            cursor.insertRow([proj_streams[i], "", ""])
+                # print(i)
+            del cursor
+
+            try:
+                metrics.track(
+                    "Connector Action",
+                    plugin.dataStorage.active_account,
+                    {
+                        "name": "Save Layer Selection",
+                        "connector_version": str(plugin.version),
+                    },
+                )
+            except Exception as e:
+                logToUser(
+                    e, level=2, func=inspect.stack()[0][3], plugin=plugin.dockwidget
+                )
+
+            # print(table)
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+
+    print("SET project layer selection 2")
+
+
+def get_rotation(plugin):
+    try:
+        print("get_rotation")
+        dataStorage = plugin.dataStorage
+        project = plugin.dataStorage.project
+        table = findOrCreateSpeckleTable(project, plugin)
+        if table is None:
+            return
+
+        rows = arcpy.da.SearchCursor(table, "crs_rotation")
+        rotation = ""
+        for x in rows:
+            rotation = x[0]
+            break
+
+        if rotation != "":
+            vals: str = rotation.replace(" ", "")
+            dataStorage.crs_rotation = float(vals)
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3], plugin=plugin.dockwidget)
+
+
+def set_rotation(plugin):
+    try:
+        dataStorage = plugin.dataStorage
+        project = dataStorage.project
+        r = dataStorage.crs_rotation
+        if dataStorage.crs_rotation is None:
+            r = 0
+
+        table = findOrCreateSpeckleTable(project, plugin)
+        if table is not None:
+            with arcpy.da.UpdateCursor(table, ["crs_rotation"]) as cursor:
+                for row in cursor:  # just one row
+                    cursor.updateRow([r])
+                    break
+            del cursor
+        return True
+
+    except Exception as e:
+        logToUser("Lat, Lon values invalid: " + str(e), level=2)
+        return False
+
+
+def get_crs_offsets(plugin):
+    try:
+        print("get_crs_offsets")
+        dataStorage = plugin.dataStorage
+        project = plugin.dataStorage.project
+        table = findOrCreateSpeckleTable(project, plugin)
+        if table is None:
+            return
+
+        rows = arcpy.da.SearchCursor(table, "crs_offsets")
+        points = ""
+        for x in rows:
+            points = x[0]
+            break
+
+        if points != "":
+            vals: List[str] = points.replace(" ", "").split(";")[:2]
+            dataStorage.crs_offset_x, dataStorage.crs_offset_y = [
+                float(i) for i in vals
+            ]
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3], plugin=plugin.dockwidget)
+
+
+def set_crs_offsets(plugin):
+    try:
+
+        dataStorage = plugin.dataStorage
+        project = dataStorage.project
+        x = dataStorage.crs_offset_x
+        y = dataStorage.crs_offset_y
+
+        if dataStorage.crs_offset_x is None or dataStorage.crs_offset_y is None:
+            x = 0
+            y = 0
+        pt = str(x) + ";" + str(y)
+
+        table = findOrCreateSpeckleTable(project, plugin)
+        if table is not None:
+            with arcpy.da.UpdateCursor(table, ["crs_offsets"]) as cursor:
+                for row in cursor:  # just one row
+                    cursor.updateRow([pt])
+                    break
+            del cursor
+
+        return True
+
+    except Exception as e:
+        logToUser("Lat, Lon values invalid: " + str(e), level=2)
+        return False
+
+
+def get_project_saved_layers(plugin):
+
+    try:
+        #print("GET project layer selection from the table")
+        project = plugin.project
+        table = findOrCreateSpeckleTable(project, plugin)
+        if table is None:
+            return
+
+        rows = arcpy.da.SearchCursor(table, "project_layer_selection")
+        saved_layers = []
+        for x in rows:
+            saved_layers.append(x[0])
+
+        temp = []
+        proj_layers = getAllProjLayers(plugin)
+        if proj_layers is None:
+            return
+        ######### need to check whether saved streams are available (account reachable)
+        if len(saved_layers) > 0:
+            for layerPath in saved_layers:
+                if layerPath == "":
+                    continue
+                found = 0
+                for layer in proj_layers:
+                    #print(layer.dataSource)
+                    if layer.dataSource == layerPath:
+                        temp.append((layer.name, layer))
+                        found += 1
+                        break
+                if found == 0:
+                    logToUser(
+                        f'Saved layer not found: "{layerPath}"',
+                        level=1,
+                        func=inspect.stack()[0][3],
+                    )
+        # plugin.dataStorage.current_layers = temp
+        # plugin.dataStorage.saved_layers = temp
+        # plugin.dataStorage.current_layers = temp.copy()
+        plugin.dataStorage.saved_layers = temp.copy()
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+
+
+def set_project_saved_layers(plugin):
+    return
+    try:
+        print("SET project layer selection function")
+        project = plugin.project
+        value: List[str] = [
+            layer[1].dataSource for layer in plugin.dataStorage.saved_layers
+        ]  # ",".join([layer[1].dataSource for layer in plugin.dataStorage.current_layers])
+        if len(value) == 0:
+            value.append("")
+        print(value)
+
+        table = findOrCreateSpeckleTable(project, plugin)
+        # print(table)
+        if table is not None:
+            cursor = arcpy.da.InsertCursor(table, "project_layer_selection")
+            for i in range(len(value)):
+                cursor.insertRow([value[i]])
+                print(i)
+            del cursor
+
+            try:
+                metrics.track(
+                    "Connector Action",
+                    plugin.dataStorage.active_account,
+                    {
+                        "name": "Save Layer Selection",
+                        "connector_version": str(plugin.version),
+                    },
+                )
+            except Exception as e:
+                logToUser(
+                    e, level=2, func=inspect.stack()[0][3], plugin=plugin.dockwidget
+                )
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+
+
+def get_survey_point(plugin: "SpeckleGIS", content=None):
+    try:
+        print("get survey point")
+        project = plugin.dataStorage.project
+        table = findOrCreateSpeckleTable(project, plugin)
+        if table is None:
+            return
+
+        rows = arcpy.da.SearchCursor(table, "lat_lon")
+        points = ""
+        for x in rows:
+            points = x[0]
+            break
+
+        if points != "":
+            vals: List[str] = points.replace(" ", "").split(";")[:2]
+            plugin.lat, plugin.lon = [float(i) for i in vals]
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3], plugin=plugin.dockwidget)
+
+
+def set_survey_point(plugin: "SpeckleGIS"):
+
+    try:
+        # from widget (2 strings) to local vars + update SR of the map
+        print("SET survey point")
+        dataStorage = plugin.dataStorage
+        project = dataStorage.project
+
+        x = dataStorage.custom_lat
+        y = dataStorage.custom_lon
+
+        pt = str(x) + ";" + str(y)
+        table = findOrCreateSpeckleTable(project, plugin)
+        if table is not None:
+            with arcpy.da.UpdateCursor(table, ["lat_lon"]) as cursor:
+                for row in cursor:  # just one row
+                    cursor.updateRow([pt])
+                    break
+            del cursor
+        # setProjectReferenceSystem(plugin)
+
+        try:
+            metrics.track(
+                "Connector Action",
+                plugin.dataStorage.active_account,
+                {
+                    "name": "Set As Center Point",
+                    "connector_version": str(plugin.version),
+                },
+            )
+        except Exception as e:
+            logToUser(e, level=2, func=inspect.stack()[0][3], plugin=plugin.dockwidget)
+
+        return True
+
+    except Exception as e:
+        logToUser(
+            "Lat, Lon values invalid: " + str(e), level=2, func=inspect.stack()[0][3]
+        )
+        return False
+
+
+def setProjectReferenceSystem(plugin: "SpeckleGIS"):
+    try:
+        # save to project; create SR
+        newCrsString = (
+            "+proj=tmerc +ellps=WGS84 +datum=WGS84 +units=m +no_defs +lon_0="
+            + str(plugin.lon)
+            + " lat_0="
+            + str(plugin.lat)
+            + " +x_0=0 +y_0=0 +k_0=1"
+        )
+        newCrs = osr.SpatialReference()
+        newCrs.ImportFromProj4(newCrsString)
+        newCrs.MorphToESRI()  # converts the WKT to an ESRI-compatible format
+
+        validate = True if len(newCrs.ExportToWkt()) > 10 else False
+
+        if validate:
+            newProjSR = arcpy.SpatialReference()
+            newProjSR.loadFromString(newCrs.ExportToWkt())
+
+            # source = osr.SpatialReference()
+            # source.ImportFromWkt(plugin.project.activeMap.spatialReference.exportToString())
+            # transform = osr.CoordinateTransformation(source, newCrs)
+
+            plugin.project.activeMap.spatialReference = newProjSR
+            logToUser(
+                "Custom project Spatial Reference successfully applied",
+                level=0,
+                func=inspect.stack()[0][3],
+                plugin=plugin.dockwidget,
+            )
+        else:
+            logToUser(
+                "Custom Spatial Reference could not be created",
+                level=1,
+                func=inspect.stack()[0][3],
+            )
+
+        return True
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return False
+
+
+def findOrCreateSpeckleTable(project: ArcGISProject, plugin) -> Union[str, None]:
+    try:
+        path = (
+            plugin.workspace
+        )  # project.filePath.replace("aprx","gdb") #"\\".join(project.filePath.split("\\")[:-1]) + "\\speckle_layers\\" #arcpy.env.workspace + "\\" #
+
+        if "speckle_gis" not in arcpy.ListTables():
+            try:
+                table = CreateTable(path, "speckle_gis")
+                for f in FIELDS:
+                    arcpy.management.AddField(table, f, "TEXT")
+                    # arcpy.management.AddField(table, "project_layer_selection", "TEXT")
+                    # arcpy.management.AddField(table, "lat_lon", "TEXT")
+
+                cursor = arcpy.da.InsertCursor(table, FIELDS)
+                cursor.insertRow(["" for _ in range(len(FIELDS))])
+                del cursor
+
+            except Exception as e:
+                logToUser(
+                    "Error creating a table: " + str(e),
+                    level=1,
+                    func=inspect.stack()[0][3],
+                )
+                raise e
+        else:
+            # print("table already exists")
+            # make sure fileds exist
+            table = path + "\\speckle_gis"
+            for f in FIELDS:
+                findOrCreateTableField(table, f)
+            # findOrCreateTableField(table, FIELDS[1])
+            # findOrCreateTableField(table, FIELDS[2])
+
+            findOrCreateRow(table, FIELDS)
+
+        return table
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None
+
+
+def findOrCreateTableField(table: str, field: str):
+    try:
+        with arcpy.da.UpdateCursor(table, [field]) as cursor:
+            value = None
+            for row in cursor:
+                value = row  # tuple(val,)
+                if value[0] is None:
+                    cursor.updateRow("")
+                break  # look at the 1st row only
+        del cursor
+
+        # if value is None: # if there are no rows
+        #    cursor = arcpy.da.InsertCursor(table, [field])
+        #    cursor.insertRow([""])
+        #    del cursor
+
+    except Exception as e:  # if field doesn't exist
+        arcpy.management.AddField(table, field, "TEXT")
+        # cursor = arcpy.da.InsertCursor(table, [field] )
+        # cursor.insertRow([""])
+        del cursor
+
+
+def findOrCreateRow(table: str, fields: List[str]):
+
+    # check if the row exists
+    cursor = arcpy.da.SearchCursor(table, fields)
+    k = -1
+    for k, row in enumerate(cursor):
+        # print(row)
+        break
+    del cursor
+
+    # if no rows
+    if k == -1:
+        cursor = arcpy.da.InsertCursor(table, fields)
+        cursor.insertRow(["" for _ in range(len(FIELDS))])
+        del cursor
+    else:
+        with arcpy.da.UpdateCursor(table, fields) as cursor:
+            for row in cursor:
+                if None in row:
+                    cursor.updateRow(["" if r is None else r for r in row])
+                break  # look at the 1st row only
+        del cursor
+
+
+def findOrCreateRowInFeatureTable(table: str, fields: List[str], values=None):
+
+    with arcpy.da.UpdateCursor(table, fields) as cursor:
+        for row in cursor:
+            cursor.deleteRow()
+    del cursor
+
+    cursor = arcpy.da.InsertCursor(table, fields)
+    cursor.insertRow([str(v) for v in values])
+    del cursor

speckle_toolbox/esri/toolboxes/speckle/speckle/utils/validation.py

@@ -0,0 +1,173 @@
+import inspect
+from typing import Union
+from specklepy.core.api.credentials import get_default_account
+from specklepy.core.api.wrapper import StreamWrapper
+from specklepy.core.api.models import Stream, Branch, Commit
+from specklepy.transports.server import ServerTransport
+from specklepy.core.api.client import SpeckleClient
+from specklepy.logging.exceptions import SpeckleException, GraphQLException
+
+from speckle.speckle.utils.panel_logging import logToUser
+
+
+def tryGetClient(sw: StreamWrapper, dataStorage, write=False, dockwidget=None):
+    # only streams with write access
+    client = None
+    savedRole = None
+    savedStreamId = None
+    for acc in dataStorage.accounts:
+        # only check accounts on selected server
+        if acc.serverInfo.url in sw.server_url:
+            client = SpeckleClient(
+                acc.serverInfo.url, acc.serverInfo.url.startswith("https")
+            )
+            try:
+                client.authenticate_with_account(acc)
+                if client.account.token is not None:
+                    break
+            except SpeckleException as ex:
+                if "already connected" in ex.message:
+                    logToUser(
+                        "Dependencies versioning error.\nClick here for details.",
+                        url="dependencies_error",
+                        level=2,
+                        plugin=dockwidget,
+                    )
+                    return None, None
+                else:
+                    raise ex
+
+    # if token still not found
+    if client is None or client.account.token is None:
+        client = sw.get_client()
+
+    if client is not None:
+        stream = client.stream.get(id=sw.stream_id, branch_limit=100, commit_limit=100)
+        if isinstance(stream, Stream):
+            if write is False:
+                # try get stream, only read access needed
+                return client, stream
+            else:
+                # check write access
+                if stream.role is None:
+                    raise Exception(
+                        f"You don't have write access to the stream '{stream.id}'. You role is '{stream.role}'"
+                    )
+                elif isinstance(stream.role, str) and "reviewer" in stream.role:
+                    raise Exception(
+                        f"You don't have write access to the stream '{savedStreamId}'. You role is '{savedRole}'"
+                    )
+                else:
+                    return client, stream
+        else:
+            return None, None
+    else:
+        return None, None
+
+
+def tryGetStream(
+    sw: StreamWrapper, dataStorage, write=False, dockwidget=None
+) -> Union[Stream, None]:
+    try:
+        # print("tryGetStream")
+        client, stream = tryGetClient(sw, dataStorage, write, dockwidget)
+        return stream
+    except Exception as e:
+        logToUser(e, level=2, func=inspect.stack()[0][3], plugin=dockwidget)
+        return None
+
+
+def validateStream(stream: Stream, dockwidget) -> Union[Stream, None]:
+    try:
+        # dockwidget.dataStorage.check_for_accounts()
+        # stream = tryGetStream(streamWrapper, dockwidget.dataStorage)
+
+        if isinstance(stream, SpeckleException):
+            return None
+
+        if stream.branches is None:
+            logToUser("Stream has no branches", level=1, plugin=dockwidget)
+            return None
+        return stream
+    except Exception as e:
+        logToUser(e, level=2, plugin=dockwidget)
+        return
+
+
+def validateBranch(
+    stream: Stream, branchName: str, checkCommits: bool, dockwidget
+) -> Union[Branch, None]:
+    try:
+        branch = None
+        if not stream.branches or not stream.branches.items:
+            return None
+        for b in stream.branches.items:
+            if b.name == branchName:
+                branch = b
+                break
+        if branch is None:
+            logToUser("Failed to find a branch", level=2, plugin=dockwidget)
+            return None
+        if checkCommits == True:
+            if branch.commits is None:
+                logToUser("Failed to find a branch", level=2, plugin=dockwidget)
+                return None
+            if len(branch.commits.items) == 0:
+                logToUser("Branch contains no commits", level=1, plugin=dockwidget)
+                return None
+        return branch
+    except Exception as e:
+        logToUser(e, level=2, plugin=dockwidget)
+
+
+def validateCommit(
+    branch: Branch, commitId: str, dockwidget=None
+) -> Union[Commit, None]:
+    try:
+        commit = None
+        try:
+            commitId = commitId.split(" | ")[0]
+        except:
+            logToUser("Commit ID is not valid", level=2, plugin=dockwidget)
+
+        if commitId.startswith("Latest") and len(branch.commits.items) > 0:
+            commit = branch.commits.items[0]
+        else:
+            for i in branch.commits.items:
+                if i.id == commitId:
+                    commit = i
+                    break
+            if commit is None:
+                try:
+                    commit = branch.commits.items[0]
+                    logToUser(
+                        "Failed to find a commit. Receiving Latest",
+                        level=1,
+                        plugin=dockwidget,
+                    )
+                except:
+                    logToUser("Failed to find a commit", level=2, plugin=dockwidget)
+                    return None
+        return commit
+    except Exception as e:
+        logToUser(e, level=2, plugin=dockwidget)
+        return
+
+
+def validateTransport(
+    client: SpeckleClient, streamId: str
+) -> Union[ServerTransport, None]:
+    try:
+        account = client.account
+        if not account.token:
+            account = get_default_account()
+        transport = ServerTransport(client=client, account=account, stream_id=streamId)
+        # print(transport)
+        return transport
+    except Exception as e:
+        logToUser(
+            "Make sure you have sufficient permissions: " + str(e),
+            level=1,
+            func=inspect.stack()[0][3],
+        )
+        return None

speckle_toolbox/esri/toolboxes/speckle/speckle_arcgis.py

@@ -1,787 +0,0 @@
-
-# -*- coding: utf-8 -*-
-
-from typing import Any, Callable, List, Optional, Tuple
-#r'''
-from collections import defaultdict
-
-import arcpy
-#from arcpy import toolbox
-from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
-from arcpy import metadata as md
-
-try:
-    from speckle.converter.layers.Layer import Layer, RasterLayer
-    from speckle.converter.layers._init_ import convertSelectedLayers, layerToNative, cadLayerToNative, bimLayerToNative
-    from speckle.ui.project_vars import toolboxInputsClass, speckleInputsClass
-    from speckle.converter.layers.emptyLayerTemplates import createGroupLayer
-    from speckle.converter.layers.Layer import VectorLayer
-except: 
-    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.Layer import Layer, RasterLayer
-    from speckle_toolbox.esri.toolboxes.speckle.converter.layers._init_ import convertSelectedLayers, layerToNative, cadLayerToNative, bimLayerToNative
-    from speckle_toolbox.esri.toolboxes.speckle.ui.project_vars import toolboxInputsClass, speckleInputsClass
-    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.emptyLayerTemplates import createGroupLayer
-    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.Layer import VectorLayer
-    
-from arcgis.features import FeatureLayer
-import os
-import os.path
-import sys 
-
-import specklepy
-from specklepy.api.models import Branch, Stream, Streams
-from specklepy.transports.server.server import ServerTransport
-from specklepy.api.credentials import get_local_accounts
-from specklepy.api.client import SpeckleClient
-from specklepy.api import operations
-from specklepy.logging.exceptions import (
-    GraphQLException,
-    SpeckleException,
-    SpeckleWarning,
-)
-#from specklepy.api.credentials import StreamWrapper
-from specklepy.api.wrapper import StreamWrapper
-from specklepy.objects import Base
-from specklepy.logging import metrics
-
-#'''
-
-def traverseObject(
-        base: Base,
-        callback: Optional[Callable[[Base], bool]],
-        check: Optional[Callable[[Base], bool]],
-    ):
-        if check and check(base):
-            res = callback(base) if callback else False
-            if res:
-                return
-        memberNames = base.get_member_names()
-        for name in memberNames:
-            try:
-                if ["id", "applicationId", "units", "speckle_type"].index(name):
-                    continue
-            except:
-                pass
-            traverseValue(base[name], callback, check)
-            
-def traverseValue(
-    value: Any,
-    callback: Optional[Callable[[Base], bool]],
-    check: Optional[Callable[[Base], bool]],
-):
-    if isinstance(value, Base):
-        traverseObject(value, callback, check)
-    if isinstance(value, List):
-        for item in value:
-            traverseValue(item, callback, check)
-
-
-class Toolbox:
-    def __init__(self):
-        """Define the toolbox (the name of the toolbox is the name of the
-        .pyt file)."""
-        print("___ping_Toolbox")
-        self.label = "Speckle Tools"
-        self.alias = "speckle_toolbox_"  
-        # List of tool classes associated with this toolbox
-        self.tools = [Speckle]  
-
-        try: 
-            version = arcpy.GetInstallInfo()['Version']
-            python_version = f"python {'.'.join(map(str, sys.version_info[:2]))}"
-            metrics.set_host_app("ArcGIS", ', '.join([f"ArcGIS {version}", python_version])) 
-        except: 
-            metrics.set_host_app("ArcGIS")
-        # https://pro.arcgis.com/en/pro-app/2.8/arcpy/mapping/alphabeticallistofclasses.htm#except: print("something happened")
-
-class Speckle:
-    def __init__(self):  
-        print("__________________INIT SPECKLE TOOL_________")
-        self.label       = "Speckle"
-        self.description = "Allows you to send and receive your layers " + \
-                           "to/from other software using Speckle server." 
-
-        self.toRefresh = False
-        self.speckleInputs = None
-        self.toolboxInputs = None
-
-        total = len(speckleInputsClass.instances)
-        print(total)
-        for i in range(total):
-            if speckleInputsClass.instances[total-i-1] is not None: 
-                try: 
-                    y = speckleInputsClass.instances[total-i-1].streams_default 
-                    #if not isinstance(speckleInputsClass.instances[total-i-1].streams_default, SpeckleException): # also will throw exception in case not initialized properly 
-                    self.speckleInputs = speckleInputsClass.instances[total-i-1] # take latest (first in reverted list)
-                    break
-                except: pass
-        if self.speckleInputs is None or isinstance(self.speckleInputs.streams_default, SpeckleException): self.speckleInputs = speckleInputsClass()
-        #print(self.speckleInputs.streams_default)
-        print(len(speckleInputsClass.instances))
-
-        total = len(toolboxInputsClass.instances)
-
-        for i in range(total):
-            if toolboxInputsClass.instances[total-i-1] is not None: 
-                self.toolboxInputs = toolboxInputsClass.instances[total-i-1] # take latest (first in reverted list)
-                break
-        if self.toolboxInputs is None: self.toolboxInputs = toolboxInputsClass()
-
-        #print(self.speckleInputs.accounts)
-        if len(self.speckleInputs.accounts) == 0:
-            arcpy.AddError("Speckle accounts not found")
-
-    def getParameterInfo(self):
-        #data types: https://pro.arcgis.com/en/pro-app/2.8/arcpy/geoprocessing_and_python/defining-parameter-data-types-in-a-python-toolbox.htm
-        # parameter details: https://pro.arcgis.com/en/pro-app/latest/arcpy/geoprocessing_and_python/customizing-tool-behavior-in-a-python-toolbox.htm
-        print("Get parameter values")
-        cat1 = "Add Streams"
-        cat2 = "Send/Receive"
-        cat3 = "Create custom Spatial Reference"
-
-        streamsDefalut = arcpy.Parameter(
-            displayName="Add stream from default account",
-            name="streamsDefalut",
-            datatype="GPString",
-            parameterType="Optional",
-            direction="Input",
-            category=cat1
-            )
-        streamsDefalut.filter.type = 'ValueList'
-        if isinstance(self.speckleInputs.streams_default, SpeckleException):
-            arcpy.AddError("Speckle account not accessible")
-            streamsDefalut.filter.list = []
-        elif self.speckleInputs.streams_default is not None:
-            streamsDefalut.filter.list = [ (str(st.name) + " - " + str(st.id)) for st in self.speckleInputs.streams_default ]
-        else:
-            streamsDefalut.filter.list = []
-            arcpy.AddError("Error connecting to default Speckle account")
-        
-        addDefStreams = arcpy.Parameter(
-            displayName="Add",
-            name="addDefStreams",
-            datatype="GPBoolean",
-            parameterType="Optional",
-            direction="Input",
-            category=cat1
-            )
-        addDefStreams.value = False 
-
-        streamUrl = arcpy.Parameter(
-            displayName="Add stream by URL",
-            name="streamUrl",
-            datatype="GPString",
-            parameterType="Optional",
-            direction="Input",
-            category=cat1
-            )
-        streamUrl.value = ""
-
-        addUrlStreams = arcpy.Parameter(
-            displayName="Add",
-            name="addUrlStreams",
-            datatype="GPBoolean",
-            parameterType="Optional",
-            direction="Input",
-            category=cat1
-            )
-        addUrlStreams.value = False 
-
-        ############################################################################
-        
-        lat = arcpy.Parameter(
-            displayName="Origin point LAT",
-            name="lat",
-            datatype="GPString",
-            parameterType="Optional",
-            direction="Input",
-            category=cat3
-            )
-        lat.value = str(self.toolboxInputs.lat)
-
-        lon = arcpy.Parameter(
-            displayName="Origin point LON",
-            name="lon",
-            datatype="GPString",
-            parameterType="Optional",
-            direction="Input",
-            category=cat3
-            )
-        lon.value = str(self.toolboxInputs.lon)
-        
-        setLatLon = arcpy.Parameter(
-            displayName="Create and apply",
-            name="setLatLon",
-            datatype="GPBoolean",
-            parameterType="Optional",
-            direction="Input",
-            category=cat3
-            )
-        setLatLon.value = False
-        
-        ####################################################################################################
-
-        savedStreams = arcpy.Parameter(
-            displayName="Select Stream",
-            name="savedStreams",
-            datatype="GPString",
-            parameterType="Required",
-            direction="Input",
-            multiValue=False,
-            )
-        savedStreams.filter.list = [f"Stream not accessible - {stream[0].stream_id}" if stream[1] is None or isinstance(stream[1], SpeckleException) else f"{stream[1].name} - {stream[1].id}" for i,stream in enumerate(self.speckleInputs.saved_streams)] 
-
-        removeStream = arcpy.Parameter(
-            displayName="Remove",
-            name="removeStream",
-            datatype="GPBoolean",
-            parameterType="Optional",
-            direction="Input"
-            )
-        removeStream.value = False 
-
-        branch = arcpy.Parameter(
-            displayName="Branch",
-            name="branch",
-            datatype="GPString",
-            parameterType="Required",
-            direction="Input",
-            )
-        branch.value = ""
-        branch.filter.type = 'ValueList'
-
-        commit = arcpy.Parameter(
-            displayName="Commit",
-            name="commit",
-            datatype="GPString",
-            parameterType="Optional",
-            direction="Input",
-            )
-        commit.value = ""
-        commit.filter.type = 'ValueList'
-
-        msg = arcpy.Parameter(
-            displayName="Message",
-            name="msg",
-            datatype="GPString",
-            parameterType="Optional",
-            direction="Input",
-            multiValue=False,
-            )
-        msg.value = ""
-
-        selectedLayers = arcpy.Parameter(
-            displayName="Selected Layers",
-            name="selectedLayers",
-            datatype="GPString",
-            parameterType="Optional",
-            direction="Input",
-            multiValue=True,
-            )
-        selectedLayers.filter.list = [str(i) + "-" + l.longName for i,l in enumerate(self.speckleInputs.all_layers)] #"Polyline"
-
-        
-        action = arcpy.Parameter(
-            displayName="",
-            name="action",
-            datatype="GPString",
-            parameterType="Required",
-            direction="Input",
-            multiValue=False,
-            )
-        action.value = "Send" 
-        action.filter.list = ["Send", "Receive"]  
-
-        refresh = arcpy.Parameter(
-            displayName="Refresh",
-            name="refresh",
-            datatype="GPBoolean",
-            parameterType="Optional",
-            direction="Input"
-            ) 
-        refresh.value = False 
-
-        parameters = [streamsDefalut, addDefStreams, streamUrl, addUrlStreams, lat, lon, setLatLon, savedStreams, removeStream, branch, commit, selectedLayers, msg, action, refresh]
-        return parameters
-
-    def isLicensed(self): #optional
-        return True
-
-    def updateParameters(self, parameters: List, toRefresh = False): #optional
-        print("UPDATING PARAMETERS")
-        
-        for i, par in enumerate(parameters): 
-            
-
-            if par.name == "addDefStreams" and par.altered and par.value == True: 
-                for p in parameters:
-                    if p.name == "streamsDefalut" and p.valueAsText is not None:
-
-                        # add value from streamsDefault to saved streams
-                        selected_stream_name = p.valueAsText[:]
-
-                        for stream in self.speckleInputs.streams_default:
-                            if stream.name == selected_stream_name.split(" - ")[0]:
-                                print("_____Add from list___")
-                                wr = StreamWrapper(f"{self.speckleInputs.account.serverInfo.url}/streams/{stream.id}?u={self.speckleInputs.account.userInfo.id}")
-                                self.toolboxInputs.setProjectStreams(wr)
-                                
-                                for p_saved in parameters:
-                                    if p_saved.name == "savedStreams": 
-                                        saved_streams = self.speckleInputs.getProjectStreams()
-                                        self.speckleInputs.saved_streams = saved_streams
-                                        p_saved.filter.list = [f"Stream not accessible - {stream[0].stream_id}" if stream[1] is None or isinstance(stream[1], SpeckleException) else f"{stream[1].name} - {stream[1].id}" for i,stream in enumerate(saved_streams)] 
-                                        if len(p_saved.filter.list)>0: print(p_saved.filter.list); p_saved.value = p_saved.filter.list[0]
-                                break
-                        p.value = None
-                par.value = False
-            
-            if par.name == "addUrlStreams" and par.altered and par.value == True: 
-                for p in parameters:
-                    if p.name == "streamUrl" and p.valueAsText is not None:
-
-                        # add value from streamsDefault to saved streams
-                        query = p.valueAsText[:]
-                        if "http" in query and len(query.split("/")) >= 3: # URL
-                            steamId = query
-                            try: steamId = query.split("/streams/")[1].split("/")[0] 
-                            except: pass
-
-                            # query stream, add to saved
-                            stream = self.speckleInputs.speckle_client.stream.get(id = steamId, branch_limit = 100, commit_limit = 100)
-                            if isinstance(stream, Stream): 
-                                print("_____Add by URL___")
-                                wr = StreamWrapper(f"{self.speckleInputs.account.serverInfo.url}/streams/{stream.id}?u={self.speckleInputs.account.userInfo.id}")
-                                self.toolboxInputs.setProjectStreams(wr)
-                                
-                                for p_saved in parameters:
-                                    if p_saved.name == "savedStreams": 
-                                        saved_streams = self.speckleInputs.getProjectStreams()
-                                        self.speckleInputs.saved_streams = saved_streams
-                                        p_saved.filter.list = [f"Stream not accessible - {st[0].stream_id}" if st[1] is None or isinstance(st[1], SpeckleException) else f"{st[1].name} - {st[1].id}" for i,st in enumerate(saved_streams)] 
-                                        if len(p_saved.filter.list)>0: print(p_saved.filter.list); p_saved.value = p_saved.filter.list[0]
-                            else: pass
-
-                        p.value = None
-                        break
-                par.value = False
-
-
-            if par.name == "removeStream" and par.altered and par.value == True: 
-                for p in parameters:
-                    if p.name == "savedStreams" and p.valueAsText is not None:
-
-                        # get value from savedStreams 
-                        selected_stream_name = p.valueAsText[:]
-                        for streamTup in self.speckleInputs.saved_streams:
-                            stream = streamTup[1]
-                            if stream.name == selected_stream_name.split(" - ")[0]:
-                                print("_____Remove stream___")
-                                wr = StreamWrapper(f"{self.speckleInputs.account.serverInfo.url}/streams/{stream.id}?u={self.speckleInputs.account.userInfo.id}")
-                                self.toolboxInputs.setProjectStreams(wr, False)
-                                
-                                for p_saved in parameters:
-                                    if p_saved.name == "savedStreams": 
-                                        saved_streams = self.speckleInputs.getProjectStreams()
-                                        self.speckleInputs.saved_streams = saved_streams
-                                        p_saved.filter.list = [f"Stream not accessible - {st[0].stream_id}" if st[1] is None or isinstance(st[1], SpeckleException) else f"{st[1].name} - {st[1].id}" for i,st in enumerate(saved_streams)] 
-                                        p_saved.value = None
-                                break
-                        p.value = None
-                par.value = False
-
-            ####################################################################### 
-            if par.name == "setLatLon" and par.altered and par.value == True: 
-                lat = lon = 0
-                for p in parameters:
-                    if p.name == "lat" and p.valueAsText is not None:
-                        # add value from the UI to saved lat
-                        lat = p.valueAsText[:].replace(",","").replace(" ","").replace(";","").replace("_","")
-                        try: lat = float(lat)
-                        except: lat = 0; p.value = "0.0"
-
-                    if p.name == "lon" and p.valueAsText is not None:
-                        # add value from the UI to saved lat
-                        lon = p.valueAsText[:].replace(",","").replace(" ","").replace(";","").replace("_","")
-                        try: lon = float(lon)
-                        except: lon = 0; p.value = "0.0"
-                    coords = [lat, lon]
-                    self.toolboxInputs.set_survey_point(coords)
-                par.value = False
-            
-            ####################################################################### 
-
-            if par.name == "savedStreams" and par.altered:
-                # Search for the stream by name
-                if par.value is not None and "Stream not accessible" not in par.valueAsText[:]:
-                    selected_stream_name = par.valueAsText[:]
-                    self.toolboxInputs.active_stream = None
-                    self.toolboxInputs.active_stream_wrapper = None
-                    for st in self.speckleInputs.saved_streams:
-                        if st[1].name == selected_stream_name.split(" - ")[0]: 
-                            self.toolboxInputs.active_stream = st[1]
-                            self.toolboxInputs.active_stream_wrapper = st[0]
-                            break
-
-                    # edit branches: globals and UI 
-                    branch_list = [branch.name for branch in self.toolboxInputs.active_stream.branches.items]
-                    for p in parameters:
-                        if p.name == "branch":
-                            p.filter.list = branch_list
-
-                            if p.valueAsText not in branch_list: 
-                                p.value = "main"
-                            for b in self.toolboxInputs.active_stream.branches.items:
-                                if b.name == p.value: 
-                                    self.toolboxInputs.active_branch = b
-                                    break 
-                    
-                    # setting commit value and list 
-                    for p in parameters:
-                        if p.name == "commit":
-                            try: 
-                                p.filter.list = [f"{commit.id}"+ " - " + f"{commit.message}" for commit in self.toolboxInputs.active_branch.commits.items]
-                                if p.valueAsText not in p.filter.list:
-                                    p.value = self.toolboxInputs.active_branch.commits.items[0].id + " - " + self.toolboxInputs.active_branch.commits.items[0].message 
-                                    self.toolboxInputs.active_commit = self.toolboxInputs.active_branch.commits.items[0] 
-                            except: 
-                                p.filter.list = []
-                                p.value = None
-                                self.toolboxInputs.active_commit = None 
-                else: par.value = None
-
-            if par.name == "branch" and par.altered: # branches
-                if par.value is not None:
-                    selected_branch_name = par.valueAsText[:]
-                    self.toolboxInputs.active_branch = None
-                    if self.toolboxInputs.active_stream is not None:
-                        for br in self.toolboxInputs.active_stream.branches.items:
-                            if br.name == selected_branch_name:
-                                self.toolboxInputs.active_branch = br
-                                break
-                # edit commit values 
-                if self.toolboxInputs.active_branch is not None: 
-                    for p in parameters:
-                        if p.name == "commit":
-                            try: 
-                                p.filter.list = [f"{commit.id}"+ " - " + f"{commit.message}" for commit in self.toolboxInputs.active_branch.commits.items]
-                                if p.valueAsText not in p.filter.list:
-                                    p.value = self.toolboxInputs.active_branch.commits.items[0].id + " - " + self.toolboxInputs.active_branch.commits.items[0].message 
-                                    self.toolboxInputs.active_commit = self.toolboxInputs.active_branch.commits.items[0]
-                            except: 
-                                p.filter.list = []
-                                p.value = None
-                                self.toolboxInputs.active_commit = None 
-
-            if par.name == "commit" and par.altered: # commits
-                if par.value is not None:
-                    selected_commit_id = par.valueAsText[:].split(" - ")[0]
-                    self.toolboxInputs.active_commit = None
-                    if self.toolboxInputs.active_branch is not None:
-                        for c in self.toolboxInputs.active_branch.commits.items:
-                            if c.id == selected_commit_id: 
-                                self.toolboxInputs.active_commit = c 
-                                break
-            
-            if par.name == "selectedLayers" and par.altered: # selected layers
-                if par.value is not None:
-                    self.toolboxInputs.selected_layers = par.values
-
-            if par.name == "msg" and par.altered and par.valueAsText is not None:
-                self.toolboxInputs.messageSpeckle = par.valueAsText
-                print(self.toolboxInputs.messageSpeckle)
-
-            if par.name == "action" and par.altered:
-                if par.valueAsText == "Send": self.toolboxInputs.action = 1
-                else: self.toolboxInputs.action = 0
-
-            if par.name == "refresh" and par.altered: # refresh btn
-                if par.value == True: 
-                    self.refresh(parameters) 
-            if self.toRefresh == True:
-                self.refresh(parameters) 
-                self.toRefresh = False
-        
-        print("____________________________parameters___________________________")
-        return 
-    
-    def refresh(self, parameters: List[Any]): 
-        print("Refresh______")
-        self.speckleInputs: speckleInputsClass = speckleInputsClass()
-        self.toolboxInputs: toolboxInputsClass = toolboxInputsClass()
-        
-        for par in parameters: 
-            if par.name == "streamUrl": par.value = None
-            if par.name == "streamsDefalut": par.value = None
-            if par.name == "savedStreams": par.value = None
-            if par.name == "branch": par.value = ""; par.filter.list = []
-            if par.name == "commit": par.value = None; par.filter.list = []
-            if par.name == "selectedLayers": par.value = None
-            if par.name == "msg": par.value = ""
-            if par.name == "action": par.value = "Send"
-            if par.name == "refresh": par.value = False
-            
-            if par.name == "lat": par.value = str(self.toolboxInputs.get_survey_point()[0])
-            if par.name == "lon": par.value = str(self.toolboxInputs.get_survey_point()[1])
-            if par.name == "streamsDefalut": 
-                if isinstance(self.speckleInputs.streams_default, SpeckleException):
-                    arcpy.AddError("Speckle account not accessible")
-                    par.filter.list = []
-                else:
-                    par.filter.list = [ (st.name + " - " + st.id) for st in self.speckleInputs.streams_default ]
-            if par.name == "savedStreams": 
-                saved_streams = self.speckleInputs.getProjectStreams()
-                par.filter.list = [f"Stream not accessible - {stream[0].stream_id}" if stream[1] is None or isinstance(stream[1], SpeckleException) else f"{stream[1].name} - {stream[1].id}" for i,stream in enumerate(saved_streams)] 
-            if par.name == "selectedLayers": par.filter.list = [str(i) + "-" + l.longName for i,l in enumerate(self.speckleInputs.all_layers)]
-        
-        return parameters
-
-    def updateMessages(self, parameters): #optional
-        return
-
-    def execute(self, parameters: List, messages): 
-        # https://pro.arcgis.com/en/pro-app/latest/arcpy/get-started/what-is-arcpy-.htm
-        #Warning if any of the fields is invalid/empty 
-        print("___________________________Run___________________________")
-        check = self.validateStreamBranch(parameters) # apparently pdate needed to assign proper self.values
-        
-        print(self.toolboxInputs.selected_layers)
-        print(self.toolboxInputs.action)
-        
-        if self.toolboxInputs.action == 1 and check is True: self.onSend(parameters)
-        elif self.toolboxInputs.action == 0 and check is True: self.onReceive(parameters)
-        print("__________________________Run_end___________________________")
-
-    def validateStreamBranch(self, parameters: List):
-        	
-        self.updateParameters(parameters)
-        if self.toolboxInputs.active_stream is None:
-            arcpy.AddError("Choose a valid stream")
-            return False
-        if self.toolboxInputs.active_branch is None: 
-            arcpy.AddError("Choose a valid branch")
-            return False
-        return True 
-
-    def onSend(self, parameters: List):
-
-        print("______________SEND_______________")
-
-        if len(self.toolboxInputs.selected_layers) == 0: 
-            arcpy.AddError("No layers selected for sending")
-            return
-
-        streamId = self.toolboxInputs.active_stream.id #stream_id
-        client = self.toolboxInputs.active_stream_wrapper.get_client()
-        #client = self.speckleInputs.speckle_client # ?
-        
-        # Get the stream wrapper
-        #streamWrapper = StreamWrapper(None)
-        #client = streamWrapper.get_client()
-        # Ensure the stream actually exists
-        #try:
-        #    client.stream.get(streamId)
-        #except SpeckleException as error:
-        #    print(str(error))
-        #    return
-
-        # next create a server transport - this is the vehicle through which you will send and receive
-        transport = ServerTransport(client=client, stream_id=streamId)
-
-        ##################################### conversions ################################################
-        base_obj = Base(units = "m")
-        base_obj.layers = convertSelectedLayers(self.speckleInputs.all_layers, self.toolboxInputs.selected_layers, self.speckleInputs.project)
-        
-        if len(base_obj.layers) == 0: 
-            arcpy.AddMessage("No data sent to stream " + streamId)
-            return 
-        try:
-            # this serialises the block and sends it to the transport
-            objId = operations.send(base=base_obj, transports=[transport])
-        except SpeckleException as error:
-            arcpy.AddError("Error sending data")
-            #print("Error sending data")
-            return
-        except SpeckleWarning as warning:
-            arcpy.AddMessage("SpeckleWarning: " + str(warning.args[0]))
-            
-
-        message = self.toolboxInputs.messageSpeckle
-        print(message)
-        if message is None or ( isinstance(message, str) and len(message) == 0):  message = "Sent from ArcGIS"
-        print(message)
-        try:
-            # you can now create a commit on your stream with this object
-            client.commit.create(
-                stream_id=streamId,
-                object_id=objId,
-                branch_name=self.toolboxInputs.active_branch.name,
-                message=message,
-                source_application="ArcGIS",
-            )
-            arcpy.AddMessage("Successfully sent data to stream: " + streamId)
-        except:
-            arcpy.AddError("Error creating commit")
-
-    def onReceive(self, parameters: List[Any]): 
-        
-        print("______________RECEIVE_______________")
-
-        #if self.validateStreamBranch(parameters) == False: return
-
-        try: 
-            streamId = self.toolboxInputs.active_stream.id #stream_id
-            client = self.toolboxInputs.active_stream_wrapper.get_client()
-            #client = self.speckleInputs.speckle_client # 
-        except SpeckleWarning as warning: 
-            arcpy.AddWarning(str(warning.args[0]))
-
-        # get commit 
-        commit = None 
-        try: 
-            #commit = self.toolboxInputs.active_branch.commits.items[0]
-            commit = self.toolboxInputs.active_commit
-            commitId = commit.id # text to make sure commit exists 
-        except: 
-            try: 
-                commit = self.toolboxInputs.active_branch.commits.items[0]
-                commitId = commit.id 
-                arcpy.AddWarning("Failed to find a commit. Getting the last commit of the branch")
-            except:
-                arcpy.AddError("Failed to find a commit")
-                return
-
-        # next create a server transport - this is the vehicle through which you will send and receive
-        try: 
-            transport = ServerTransport(client=client, stream_id=streamId)
-            
-            client.commit.received(
-            streamId,
-            commit.id,
-            source_application="ArcGIS",
-            message="Received commit in ArcGIS",
-            )
-        except: 
-            arcpy.AddError("Make sure your account has access to the chosen stream")
-            return
-        try:
-            #print(commit)
-            objId = commit.referencedObject
-            commitDetailed = client.commit.get(streamId, commit.id)
-            if isinstance(commitDetailed, GraphQLException): 
-                arcpy.AddError("Access error")
-                return
-            app = commitDetailed.sourceApplication
-            if objId is None:
-                return
-            commitObj = operations.receive(objId, transport, None)
-            
-            if app != "QGIS" and app != "ArcGIS": 
-                if self.speckleInputs.project.activeMap.spatialReference.type == "Geographic" or self.speckleInputs.project.activeMap.spatialReference is None: #TODO test with invalid CRS
-                    arcpy.AddMessage("It is advisable to set the project Spatial reference to Projected type before receiving CAD geometry (e.g. EPSG:32631), or create a custom one from geographic coordinates")
-                    print("It is advisable to set the project Spatial reference to Projected type before receiving CAD geometry (e.g. EPSG:32631), or create a custom one from geographic coordinates")
-            print(f"Successfully received {objId}")
-
-            # Clear 'latest' group
-            streamBranch = streamId + "_" + self.toolboxInputs.active_branch.name + "_" + str(commit.id)
-            streamBranch = streamBranch.replace("[","_").replace("]","_").replace(" ","_").replace("-","_").replace("(","_").replace(")","_").replace(":","_").replace("\\","_").replace("/","_").replace("\"","_").replace("&","_").replace("@","_").replace("$","_").replace("%","_").replace("^","_")
-    
-            newGroupName = f'{streamBranch}'
-            
-            groupExists = 0
-            print(newGroupName)
-            for l in self.speckleInputs.project.activeMap.listLayers(): 
-                #print(l.longName)
-                if l.longName.startswith(newGroupName + "\\"):
-                    #print(l.longName)
-                    self.speckleInputs.project.activeMap.removeLayer(l)
-                    groupExists+=1
-                elif l.longName == newGroupName: 
-                    groupExists+=1
-            print(newGroupName)
-            if groupExists == 0:
-                # create empty group layer file 
-                path = self.speckleInputs.project.filePath.replace("aprx","gdb") #"\\".join(self.toolboxInputs.project.filePath.split("\\")[:-1]) + "\\speckle_layers\\"
-                print(path)
-                try:
-                    f = open(path + "\\" + newGroupName + ".lyrx", "w")
-                    content = createGroupLayer().replace("TestGroupLayer", newGroupName)
-                    f.write(content)
-                    f.close()
-                    newGroupLayer = arcpy.mp.LayerFile(path + "\\" + newGroupName + ".lyrx")
-                    layerGroup = self.speckleInputs.project.activeMap.addLayer(newGroupLayer)[0]
-                except: # for 3.0.0
-                    if self.speckleInputs.active_map is not None:
-                        layerGroup = self.speckleInputs.active_map.createGroupLayer(newGroupName)
-                    else:
-                        arcpy.AddWarning("The map didn't fully load, try refreshing the plugin.")
-                        return
-
-                print(layerGroup)
-                print("layer added")
-                layerGroup.name = newGroupName
-                print(newGroupName)
-
-            if app == "QGIS" or app == "ArcGIS": check: Callable[[Base], bool] = lambda base: isinstance(base, Layer) or isinstance(base, VectorLayer) or isinstance(base, RasterLayer)
-            else: check: Callable[[Base], bool] = lambda base: isinstance(base, Base)
-
-            def callback(base: Base) -> bool:
-                print("callback")
-                #print(base)
-                if isinstance(base, Layer) or isinstance(base, VectorLayer) or isinstance(base, RasterLayer):
-                    layer = layerToNative(base, streamBranch, self.speckleInputs.project)
-                    if layer is not None:
-                        print("Layer created: " + layer.name)
-                else:
-                    loopObj(base, "")
-                return True
-            
-            def loopObj(base: Base, baseName: str):
-                memberNames = base.get_member_names()
-                for name in memberNames:
-                    if name in ["id", "applicationId", "units", "speckle_type"]: continue
-                    try: loopVal(base[name], baseName + "/" + name) # loop properties not included above
-                    except: pass
-
-            def loopVal(value: Any, name: str): # "name" is the parent object/property/layer name
-                if name.endswith('/displayValue'): return 
-                
-                if isinstance(value, Base): 
-                    try: # dont go through parts of Speckle Geometry object
-                        print("objects to loop through: " + value.speckle_type)
-                        if value.speckle_type.startswith("Objects.Geometry."): pass #.Brep") or value.speckle_type.startswith("Objects.Geometry.Mesh") or value.speckle_type.startswith("Objects.Geometry.Surface") or value.speckle_type.startswith("Objects.Geometry.Extrusion"): pass
-                        else: loopObj(value, name)
-                    except: loopObj(value, name)
-
-                if isinstance(value, List):
-                    for item in value:
-                        loopVal(item, name)
-                        
-                        print(item)
-                        pt = None
-                        if item.speckle_type and item.speckle_type.startswith("Objects.Geometry."): 
-
-                            pt, pl = cadLayerToNative(value, name, streamBranch, self.speckleInputs.project)
-                            if pt is not None: print("Layer group created: " + pt.name())
-                            if pl is not None: print("Layer group created: " + pl.name())
-                            break
-                        
-                        if item.speckle_type and (item.speckle_type.startswith("Objects.BuiltElements.") or item.speckle_type.startswith("Objects.Structural.Geometry")): #  and "Revit" in item.speckle_type
-                            print("__receiving structures__")
-                            msh_bool = bimLayerToNative(value, name, streamBranch, self.speckleInputs.project)
-                            #if msh is not None: print("Layer group created: " + msh.name())
-                            break
-                
-            traverseObject(commitObj, callback, check)
-      
-        except (SpeckleException, GraphQLException) as e:
-            print("Receive failed: " + str(e))
-            arcpy.AddError("Receive failed: " + str(e))
-            return
-        
-        print("received")
-        #self.updateParameters(parameters, True)
-        #self.refresh(parameters)
-    
-    
-#__all__ = ["Toolbox", "Speckle"]

speckle_toolbox/esri/toolboxes/speckle/ui/project_vars.py

@@ -1,277 +0,0 @@
-
-from typing import Any, List, Optional, Tuple, Union
-import arcpy
-from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
-from specklepy.api.models import Branch, Stream, Streams
-import os.path
-
-from specklepy.api.credentials import Account, get_local_accounts
-from specklepy.api.client import SpeckleClient
-from specklepy.logging.exceptions import (
-    GraphQLException,
-    SpeckleException,
-)
-from specklepy.api.wrapper import StreamWrapper
-from osgeo import osr
-
-class speckleInputsClass:
-    #def __init__(self):
-    print("CREATING speckle inputs first time________")
-    instances = []
-    accounts: List[Account] = get_local_accounts()
-    account = None
-    streams_default: Optional[List[Stream]] = None
-
-    project = None
-    active_map = None
-    saved_streams: List[Optional[Tuple[StreamWrapper, Stream]]] = []
-    stream_file_path: str = ""
-    all_layers: List[arcLayer] = []
-    clients: List[SpeckleClient] = []
-
-    for acc in accounts:
-        if acc.isDefault: account = acc
-        new_client = SpeckleClient(
-            acc.serverInfo.url,
-            acc.serverInfo.url.startswith("https")
-        )
-        new_client.authenticate_with_token(token=acc.token)
-        clients.append(new_client)
-
-    speckle_client = None
-    if account:
-        speckle_client = SpeckleClient(
-            account.serverInfo.url,
-            account.serverInfo.url.startswith("https")
-    )
-        speckle_client.authenticate_with_token(token=account.token)
-        streams_default = speckle_client.stream.search("")
-
-    def __init__(self) -> None:
-        print("___start speckle inputs________")
-        self.all_layers = []
-        try:
-            aprx = ArcGISProject('CURRENT')
-            self.project = aprx
-            # following will fail if no project found 
-            self.active_map = aprx.activeMap
-            
-            if self.active_map is not None and isinstance(self.active_map, Map): # if project loaded
-                for layer in self.active_map.listLayers(): 
-                    try: geomType = arcpy.Describe(layer.dataSource).shapeType.lower()
-                    except: geomType = '' #print(arcpy.Describe(layer.dataSource)) #and arcpy.Describe(layer.dataSource).shapeType.lower() != "multipatch")
-                    if (layer.isFeatureLayer and geomType != "multipatch") or layer.isRasterLayer: self.all_layers.append(layer) #type: 'arcpy._mp.Layer'
-            self.stream_file_path: str = aprx.filePath.replace("aprx","gdb") + "\\speckle_streams.txt"
-
-            if os.path.exists(self.stream_file_path): 
-                try: 
-                    f = open(self.stream_file_path, "r")
-                    content = f.read()
-                    self.saved_streams = self.getProjectStreams(content)
-                    f.close()
-                except: pass
-                
-            elif len(self.stream_file_path) >10: 
-                f = open(self.stream_file_path, "x")
-                f.close()
-                f = open(self.stream_file_path, "w")
-                content = ""
-                f.write(content)
-                f.close()
-        except: self.project = None; print("Project not found")
-        self.instances.append(self)
-
-    def getProjectStreams(self, content: str = None):
-        print("get proj streams")
-        if not content: 
-            content = self.stream_file_path
-            try: 
-                f = open(self.stream_file_path, "r")
-                content = f.read()
-                f.close()
-            except: pass
-
-        ######### need to check whether saved streams are available (account reachable)
-        if content:
-            streamsTuples = []
-            for i, url in enumerate(content.split(",")):
-
-                streamExists = 0
-                index = 0
-                try:
-                    #print(url)
-                    sw = StreamWrapper(url)
-                    stream = self.tryGetStream(sw)
-
-                    for st in streamsTuples: 
-                        if isinstance(stream, Stream) and st[0].stream_id == stream.id: 
-                            streamExists = 1; 
-                            break 
-                        index += 1
-                    if streamExists == 1: del streamsTuples[index]
-                    streamsTuples.insert(0,(sw, stream))
-
-                except SpeckleException as e:
-                    arcpy.AddMessage(str(e.args))
-            return streamsTuples
-        else: return []
-
-    def tryGetStream (self,sw: StreamWrapper) -> Stream:
-        if isinstance(sw, StreamWrapper):
-            steamId = sw.stream_id
-            try: steamId = sw.stream_id.split("/streams/")[1].split("/")[0] 
-            except: pass
-
-            client = sw.get_client()
-            stream = client.stream.get(id = steamId, branch_limit = 100, commit_limit = 100)
-            if isinstance(stream, GraphQLException):
-                raise SpeckleException(stream.errors[0]['message'])
-            return stream
-        else: 
-            raise SpeckleException('Invalid StreamWrapper provided')
-
-class toolboxInputsClass:
-
-    print("CREATING UI inputs first time________")
-    instances = []
-    lat: float = 0.0
-    lon: float = 0.0
-    active_stream: Optional[Stream] = None
-    active_stream_wrapper: Optional[StreamWrapper] = None
-    active_branch: Optional[Branch] = None
-    active_commit = None
-    selected_layers: List[Any] = []
-    messageSpeckle: str = ""
-    action: int = 1 #send
-    project = None
-    stream_file_path: str = ""
-    # Get the target item's Metadata object
-    
-    def __init__(self) -> None:
-        print("___start UI inputs________")
-        try:
-            aprx = ArcGISProject('CURRENT')
-            project = aprx
-            self.stream_file_path: str = aprx.filePath.replace("aprx","gdb") + "\\speckle_streams.txt"
-            if os.path.exists(self.stream_file_path): 
-                try: 
-                    f = open(self.stream_file_path, "r")
-                    content = f.read()
-                    self.lat, self.lon = self.get_survey_point(content)
-                    f.close()
-                except: pass
-        except: print("Project not found")
-        try:
-            aprx = ArcGISProject('CURRENT')
-            self.project = aprx
-        except: self.project = None; print("Project not found"); arcpy.AddWarning("Project not found")
-        self.instances.append(self)
-
-    def setProjectStreams(self, wr: StreamWrapper, add = True): 
-        # ERROR 032659 Error queueing metrics request: 
-        print("SET proj streams")
-
-        if os.path.exists(self.stream_file_path) and ".gdb\\speckle_streams.txt" in self.stream_file_path: 
-
-            new_content = ""
-
-            f = open(self.stream_file_path, "r")
-            existing_content = f.read()
-            f.close()
-
-            f = open(self.stream_file_path, "w")
-            if str(wr.stream_url) in existing_content: 
-                new_content = existing_content.replace(str(wr.stream_url) + "," , "")
-            else: 
-                new_content = existing_content 
-            
-            if add == True: new_content += str(wr.stream_url) + "," # add stream
-            else: pass # remove stream
-
-            f.write(new_content)
-            f.close()
-        elif ".gdb\\speckle_streams.txt" in self.stream_file_path: 
-            f = open(self.stream_file_path, "x")
-            f.close()
-            f = open(self.stream_file_path, "w")
-            f.write(str(wr.stream_url) + ",")
-            f.close()
- 
-    def get_survey_point(self, content = None) -> Tuple[float]:
-        # get from saved project 
-        print("get survey point")
-        x = y = 0
-        if not content: 
-            content = None 
-            if os.path.exists(self.stream_file_path) and ".gdb\\speckle_streams.txt" in self.stream_file_path: 
-                try: 
-                    f = open(self.stream_file_path, "r")
-                    content = f.read()
-                    f.close()
-                except: pass
-        if content:
-            for i, coords in enumerate(content.split(",")):
-                if "speckle_sr_origin_" in coords: 
-                    try:
-                        x, y = [float(c) for c in coords.replace("speckle_sr_origin_","").split(";")]
-                    except: pass
-        return (x, y) 
-
-    def set_survey_point(self, coords: List[float]):
-        # from widget (2 strings) to local vars + update SR of the map
-        print("SET survey point")
-
-        if len(coords) == 2: 
-            pt = "speckle_sr_origin_" + str(coords[0]) + ";" + str(coords[1]) 
-            if os.path.exists(self.stream_file_path) and ".gdb\\speckle_streams.txt" in self.stream_file_path: 
-
-                new_content = ""
-                f = open(self.stream_file_path, "r")
-                existing_content = f.read()
-                f.close()
-
-                f = open(self.stream_file_path, "w")
-                if pt in existing_content: 
-                    new_content = existing_content.replace( pt , "")
-                else: 
-                    new_content = existing_content 
-                
-                new_content += pt + "," # add point
-                f.write(new_content)
-                f.close()
-            elif ".gdb\\speckle_streams.txt" in self.stream_file_path: 
-                f = open(self.stream_file_path, "x")
-                f.close()
-                f = open(self.stream_file_path, "w")
-                f.write(pt + ",")
-                f.close()
-            
-            # save to project; crearte SR
-            self.lat, self.lon = coords[0], coords[1]
-            newCrsString = "+proj=tmerc +ellps=WGS84 +datum=WGS84 +units=m +no_defs +lon_0=" + str(self.lon) + " lat_0=" + str(self.lat) + " +x_0=0 +y_0=0 +k_0=1"
-            newCrs = osr.SpatialReference()
-            newCrs.ImportFromProj4(newCrsString)
-            newCrs.MorphToESRI() # converts the WKT to an ESRI-compatible format
-            
-
-            validate = True if len(newCrs.ExportToWkt())>10 else False
-
-            if validate: 
-                newProjSR = arcpy.SpatialReference()
-                newProjSR.loadFromString(newCrs.ExportToWkt())
-
-                #source = osr.SpatialReference() 
-                #source.ImportFromWkt(self.project.activeMap.spatialReference.exportToString())
-                #transform = osr.CoordinateTransformation(source, newCrs)
-
-                self.project.activeMap.spatialReference =  newProjSR
-                arcpy.AddMessage("Custom project CRS successfully applied")
-            else:
-                arcpy.AddWarning("Custom CRS could not be created")
-        
-        else:
-            arcpy.AddWarning("Custom CRS could not be created: not enough coordinates provided")
-
-        return True
-
-    

speckle_toolbox/esri/toolboxes/speckle/ui_widgets/main_window.py

@@ -0,0 +1,136 @@
+import inspect
+import os
+import threading
+from PyQt5 import QtWidgets, uic
+from specklepy.logging.exceptions import SpeckleException
+
+from speckle.speckle.utils.panel_logging import logToUser
+import speckle.specklepy_qt_ui.qt_ui
+from speckle.specklepy_qt_ui.qt_ui.mainWindow import (
+    SpeckleGISDialog as SpeckleGISDialog_UI,
+)
+
+ui_file_path = os.path.join(
+    os.path.dirname(speckle.specklepy_qt_ui.qt_ui.__file__),
+    os.path.join("ui", "mainWindow_main.ui"),
+)
+
+
+class SpeckleGISDialog(SpeckleGISDialog_UI):
+    def __init__(self, parent=None):
+        """Constructor."""
+        super(SpeckleGISDialog, self).__init__(
+            parent
+        )  # , QtCore.Qt.WindowStaysOnTopHint)
+        uic.loadUi(ui_file_path, self)  # Load the .ui file
+        # self.show()
+        self.runAllSetup()
+
+    def populateProjectStreams(self, plugin):
+        try:
+            from speckle.speckle.utils.project_vars import set_project_streams
+
+            if not self:
+                return
+            self.streamList.clear()
+            for stream in plugin.current_streams:
+                self.streamList.addItems(
+                    [
+                        (
+                            f"Stream not accessible - {stream[0].stream_id}"
+                            if stream[1] is None
+                            or isinstance(stream[1], SpeckleException)
+                            else f"{stream[1].name}, {stream[1].id} | {stream[0].stream_url.split('/streams')[0].split('/projects')[0]}"
+                        )
+                    ]
+                )
+            if len(plugin.current_streams) == 0:
+                self.streamList.addItems([""])
+            self.streamList.addItems(["Create New Stream"])
+            set_project_streams(plugin)
+            index = self.streamList.currentIndex()
+            if index == -1:
+                self.streams_remove_button.setEnabled(False)
+            else:
+                self.streams_remove_button.setEnabled(True)
+
+            if len(plugin.current_streams) > 0:
+                plugin.active_stream = plugin.current_streams[0]
+        except Exception as e:
+            logToUser(e, level=2, func=inspect.stack()[0][3], plugin=self)
+            return
+
+    def completeStreamSection(self, plugin):
+        try:
+            self.streams_remove_button.clicked.connect(
+                lambda: self.onStreamRemoveButtonClicked(plugin)
+            )
+            self.streamList.currentIndexChanged.connect(
+                lambda: self.onActiveStreamChanged(plugin)
+            )
+            self.streamBranchDropdown.currentIndexChanged.connect(
+                lambda: self.populateActiveCommitDropdown(plugin)
+            )
+            return
+        except Exception as e:
+            logToUser(e, level=2, func=inspect.stack()[0][3], plugin=self)
+            return
+
+    def onStreamRemoveButtonClicked(self, plugin):
+        try:
+            from speckle.speckle.utils.project_vars import set_project_streams
+
+            if not self:
+                return
+            index = self.streamList.currentIndex()
+            if len(plugin.current_streams) > 0:
+                plugin.current_streams.pop(index)
+            plugin.active_stream = None
+            self.streamBranchDropdown.clear()
+            self.commitDropdown.clear()
+
+            set_project_streams(plugin)
+            self.populateProjectStreams(plugin)
+        except Exception as e:
+            logToUser(e, level=2, func=inspect.stack()[0][3], plugin=self)
+            return
+
+    def populateProjectStreams(self, plugin):
+        try:
+            from speckle.speckle.utils.project_vars import set_project_streams
+
+            if not self:
+                return
+            self.streamList.clear()
+            for stream in plugin.current_streams:
+                self.streamList.addItems(
+                    [
+                        (
+                            f"Stream not accessible - {stream[0].stream_id}"
+                            if stream[1] is None
+                            or isinstance(stream[1], SpeckleException)
+                            else f"{stream[1].name}, {stream[1].id} | {stream[0].stream_url.split('/streams')[0].split('/projects')[0]}"
+                        )
+                    ]
+                )
+            if len(plugin.current_streams) == 0:
+                self.streamList.addItems([""])
+            self.streamList.addItems(["Create New Stream"])
+            set_project_streams(plugin)
+            index = self.streamList.currentIndex()
+            if index == -1:
+                self.streams_remove_button.setEnabled(False)
+            else:
+                self.streams_remove_button.setEnabled(True)
+
+            if len(plugin.current_streams) > 0:
+                plugin.active_stream = plugin.current_streams[0]
+        except Exception as e:
+            logToUser(e, level=2, func=inspect.stack()[0][3], plugin=self)
+            return
+
+    def cancelOperations(self):
+        for t in threading.enumerate():
+            if "speckle_" in t.name:
+                t.kill()
+                t.join()

tests/tests_unittest.py

@@ -1,4 +1,4 @@
-from speckle_toolbox.esri.toolboxes.speckle.speckle_arcgis import Toolbox, Speckle   # The code to test
+from speckle_toolbox.esri.toolboxes.speckle.speckle.speckle_arcgis import Toolbox, Speckle   # The code to test
 from speckle_toolbox.esri.toolboxes.speckle.ui.project_vars import speckleInputsClass, toolboxInputsClass
 
 import arcpy