Merge pull request #78 from specklesystems/JR-Morgan-patch-1

Update config.yml

.circleci/config.yml

@@ -8,10 +8,59 @@ orbs:
   aws-s3: circleci/aws-s3@2.0.0
 
 jobs:
+  get-ci-tools: # Clones our ci tools and persists them to the workspace
+    docker:
+      - image: cimg/base:2021.01
+    steps:
+      - add_ssh_keys:
+          fingerprints:
+            - "77:64:03:93:c5:f3:1d:a6:fd:bd:fb:d1:05:56:ca:e9"
+      - run:
+          name: I know Github as a host
+          command: |
+            mkdir ~/.ssh
+            touch ~/.ssh/known_hosts
+            ssh-keyscan github.com >> ~/.ssh/known_hosts
+      - run:
+          name: Clone
+          command: git clone git@github.com:specklesystems/speckle-sharp-ci-tools.git speckle-sharp-ci-tools
+      - persist_to_workspace:
+          root: ./
+          paths:
+            - speckle-sharp-ci-tools
+
+  get-ui: # Clones our ci tools and persists them to the workspace
+    docker:
+      - image: cimg/base:2021.01
+    steps:
+      - checkout
+      - attach_workspace:
+          at: ./
+      - add_ssh_keys:
+          fingerprints:
+            - "d1:d5:96:4d:ed:58:6e:7f:58:cc:21:5f:94:20:76:49"
+      - run:
+          name: I know Github as a host
+          command: |
+            touch ~/.ssh/known_hosts
+            ssh-keyscan github.com >> ~/.ssh/known_hosts
+      - run:
+          name: Clone
+          command:  |
+            git clone git@github.com:specklesystems/specklepy_qt_ui.git speckle_toolbox/esri/toolboxes/speckle/specklepy_qt_ui
+      - run:
+          name: Remove Git Artifacts
+          command: |
+            rm -rf ./speckle_toolbox/esri/toolboxes/speckle/specklepy_qt_ui/.git/
+            rm ./speckle_toolbox/esri/toolboxes/speckle/specklepy_qt_ui/.gitignore
+      - persist_to_workspace:
+          root: ./
+          paths:
+            - speckle_toolbox/esri/toolboxes/speckle/specklepy_qt_ui
+
   build-connector-win: # Reusable job for basic connectors
     executor:
       name: win/default # comes with python 3.7.3
-      shell: cmd.exe
     parameters:
       slug:
         type: string
@@ -34,7 +83,49 @@ jobs:
             $version = "$($ver).$($env:CIRCLE_BUILD_NUM)"
             echo $semver
             python patch_version.py $semver
-            speckle-sharp-ci-tools\InnoSetup\ISCC.exe speckle-sharp-ci-tools\arcgis.iss
+            pip install setuptools
+            python setup.py sdist bdist_wheel
+            Copy-Item -Path "dist\speckle_toolbox-$($ver)-py3-none-any.whl" -Destination "speckle_arcgis_installer"
+
+      - run:
+          name: Exit if External PR
+          shell: bash.exe
+          command: if [ "$CIRCLE_PR_REPONAME" ]; then circleci-agent step halt; fi
+      
+      - unless: # Build installers unsigned on non-tagged builds
+          condition: << pipeline.git.tag >>
+          steps:
+            - run:
+                name: Build Installer
+                shell: cmd.exe #does not work in powershell
+                command: 
+                  speckle-sharp-ci-tools\InnoSetup\ISCC.exe speckle-sharp-ci-tools\arcgis.iss /Sbyparam=$p
+
+      - when: # Setup certificates and build installers signed for tagged builds
+          condition: << pipeline.git.tag >>
+          steps:
+            - run: # Installs digicert signing tools for windows
+                name: "Digicert Signing Manager Setup"
+                command: |
+                  cd C:\
+                  curl.exe -X GET  https://one.digicert.com/signingmanager/api-ui/v1/releases/smtools-windows-x64.msi/download -H "x-api-key:$env:SM_API_KEY" -o smtools-windows-x64.msi
+                  msiexec.exe /i smtools-windows-x64.msi /quiet /qn | Wait-Process
+            - run: # Creates the Auth cert and the signing public PEM cert
+                name: Create Auth & OV Signing Cert
+                command: |
+                  cd C:\
+                  echo $env:SM_CLIENT_CERT_FILE_B64 > certificate.txt
+                  certutil -decode certificate.txt certificate.p12
+            - run: # Syncs certificates from Digicert into local user store
+                name: Sync Certs
+                command: |
+                  & $env:SSM\smksp_cert_sync.exe
+            - run: 
+                name: Build Installer
+                shell: cmd.exe
+                command:
+                  | # If no tag, use 0.0.0.1 and don't make any YML (for testing only!)
+                  speckle-sharp-ci-tools\InnoSetup\ISCC.exe speckle-sharp-ci-tools\arcgis.iss /Sbyparam=$p /DSIGN_INSTALLER /DCODE_SIGNING_CERT_FINGERPRINT=%SM_CODE_SIGNING_CERT_SHA1_HASH%
       - when:
           condition: << parameters.installer >>
           steps:
@@ -42,18 +133,24 @@ jobs:
                 root: ./
                 paths:
                   - speckle-sharp-ci-tools/Installers
+                  - speckle_arcgis_installer
+    environment:
+      SSM: 'C:\Program Files\DigiCert\DigiCert One Signing Manager Tools'
 
-  get-ci-tools: # Clones our ci tools and persists them to the workspace
+  publish-github-release:
     docker:
-      - image: cimg/base:2021.01
+      - image: cimg/go:1.20.0
     steps:
-      - run: # Could not get ssh to work, so using a personal token
-          name: Clone
-          command: git clone https://$GITHUB_TOKEN@github.com/specklesystems/speckle-sharp-ci-tools.git speckle-sharp-ci-tools
-      - persist_to_workspace:
-          root: ./
-          paths:
-            - speckle-sharp-ci-tools
+      - attach_workspace:
+          at: ./
+      - run:
+          name: "Publish Release on GitHub"
+          command: |
+            set -x
+            go install github.com/tcnksm/ghr@v0.16.0
+            VERSION="${CIRCLE_TAG:-0.0.0}"
+            VERSION_SHORT=$(echo "${VERSION}" | cut -d- -f1)
+            ghr -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete "${VERSION}" "./speckle_arcgis_installer/speckle_toolbox-${VERSION_SHORT}-py3-none-any.whl"
 
   deploy-manager2:
     docker:
@@ -78,54 +175,53 @@ jobs:
             TAG=$(if [ "${CIRCLE_TAG}" ]; then echo $CIRCLE_TAG; else echo "0.0.0"; fi;)
             SEMVER=$(echo "$TAG" | sed -e 's/\/[a-zA-Z-]*//')
             /root/.dotnet/tools/Speckle.Manager.Feed deploy -s << parameters.slug >> -v ${SEMVER} -u https://releases.speckle.dev/installers/<< parameters.slug >>/<< parameters.slug >>-${SEMVER}.<< parameters.extension >> -o << parameters.os >> -f speckle-sharp-ci-tools/Installers/<< parameters.slug >>/<< parameters.slug >>-${SEMVER}.<< parameters.extension >>
+
 workflows: #happens with every PR to main
   build: # build the installers, but don't persist to workspace for deployment
     jobs:
+      - get-ui:
+          context: github-dev-bot
       - get-ci-tools:
-          filters:
-            branches:
-              only:
-                - main
-                - /ci\/.*/
+          context: github-dev-bot
       - build-connector-win:
           requires:
+            - get-ui
             - get-ci-tools
-          filters:
-            branches:
-              only:
-                - main
-                - /ci\/.*/
+          context: digicert-keylocker
 
   deploy: # build installers and deploy
     jobs:
-      - get-ci-tools:
-          filters:
+      - get-ui:
+          context: github-dev-bot
+          filters: &deploy_filters
             tags:
-              only: /.*/
+              only: /([0-9]+)\.([0-9]+)\.([0-9]+)(?:-\w+)?$/
             branches:
               ignore: /.*/
 
+      - get-ci-tools:
+          context: github-dev-bot
+          filters: *deploy_filters
+
       - build-connector-win:
           name: build-deploy-connector-win
           slug: arcgis
           installer: true
           requires:
+            - get-ui
             - get-ci-tools
-          filters:
-            tags:
-              only: /([0-9]+)\.([0-9]+)\.([0-9]+)(?:-\w+)?$/
-            branches:
-              ignore: /.*/
-
+          filters: *deploy_filters
+          context: digicert-keylocker
+      - publish-github-release:
+          requires:
+            - build-deploy-connector-win
+          filters: *deploy_filters
+          context: arcgis-github-release
       - deploy-manager2:
           slug: arcgis
           os: Win
           extension: exe
           requires:
-            - get-ci-tools
             - build-deploy-connector-win
-          filters:
-            tags:
-              only: /([0-9]+)\.([0-9]+)\.([0-9]+)(?:-\w+)?$/
-            branches:
-              ignore: /.*/ # For testing only! /ci\/.*/
+          filters: *deploy_filters
+          context: do-spaces-speckle-releases

.github/workflows/close-issue.yml

@@ -0,0 +1,12 @@
+name: Update issue Status
+
+on:
+  issues:
+    types: [closed]
+
+jobs:
+  update_issue:
+    uses: specklesystems/github-actions/.github/workflows/project-add-issue.yml@main
+    secrets: inherit
+    with:
+      issue-id: ${{ github.event.issue.node_id }}

.github/workflows/open-issue.yml

@@ -0,0 +1,12 @@
+name: Move new issues into Project
+
+on:
+  issues:
+    types: [opened]
+
+jobs:
+  track_issue:
+    uses: specklesystems/github-actions/.github/workflows/project-add-issue.yml@main
+    secrets: inherit
+    with:
+      issue-id: ${{ github.event.issue.node_id }}

.gitignore

@@ -118,4 +118,7 @@ scratch.py
 settings.json
 **/.DS_Store
 zip_build
-.qt_for_python
+.qt_for_python
+*.pyt.xml
+*specklepy_qt_ui
+*.whl 

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

Readme.md

@@ -25,7 +25,7 @@ What is Speckle? Check our ![YouTube Video Views](https://img.shields.io/youtube
 - **GraphQL API:** get what you need anywhere you want it
 - **Webhooks:** the base for a automation and next-gen pipelines
 - **Built for developers:** we are building Speckle with developers in mind and got tools for every stack
-- **Built for the AEC industry:** Speckle connectors are plugins for the most common software used in the industry such as Revit, Rhino, Grasshopper, AutoCAD, Civil 3D, Excel, Unreal Engine, Unity, QGIS (you are here), Blender and more!
+- **Built for the AEC industry:** Speckle connectors are plugins for the most common software used in the industry such as Revit, Rhino, Grasshopper, AutoCAD, Civil 3D, Excel, Unreal Engine, Unity, QGIS, ArcGIS (you are here), Blender and more!
 
 ### Try Speckle now!
 
@@ -44,7 +44,7 @@ Give Speckle a try in no time by:
 
 ## Repo Structure
 
-This repo contains the QGIS plugin for Speckle 2.0. It is written in `python` and uses our fantastic [Python SDK](https://github.com/specklesystems/speckle-py). The [Speckle Server](https://github.com/specklesystems/Server) is providing all the web-facing functionality and can be found [here](https://github.com/specklesystems/Server).
+This repo contains the ArcGIS plugin for Speckle 2.0. It is written in `python` and uses our fantastic [Python SDK](https://github.com/specklesystems/speckle-py). The [Speckle Server](https://github.com/specklesystems/Server) is providing all the web-facing functionality and can be found [here](https://github.com/specklesystems/Server).
 
 > **Try it out!!**
 > Although we're still in early development stages, we encourage you to try out the latest stable release.
@@ -52,7 +52,7 @@ This repo contains the QGIS plugin for Speckle 2.0. It is written in `python` an
 >
 > **What can it do?**
 >
-> Currently, the plugin allows to send data from a single layer to a Speckle server using one of the accounts configured on your computer. It will extract all the features of that layer along side their properties and, when possible, geometry too.
+> Currently, the plugin allows to receive the data from Speckle and send data from a AcrGIS Pro layers to a Speckle server using one of the accounts configured on your computer. It will extract all the features of that layer along side their properties. 
 > The following geometry types are supported for now:
 >
 > - Point
@@ -74,7 +74,7 @@ Setup is adapted from [this tutorial](https://pro.arcgis.com/en/pro-app/2.8/arcp
 
 #### Dev Environment
 
-For a better development experience in your editor, we recommend creating a [virtual environment in ArcGIS](https://pro.arcgis.com/en/pro-app/2.8/arcpy/get-started/work-with-python-environments.htm). In the venv, you'll just need to install `specklepy`. 
+For a better development experience in your editor, we recommend creating a [virtual conda environment in ArcGIS](https://pro.arcgis.com/en/pro-app/2.8/arcpy/get-started/work-with-python-environments.htm). In the new conda environment, you'll just need to install `specklepy` and `panda3d`. 
 
 ### Debugging
 

patch_version.py

@@ -4,16 +4,65 @@ import sys
 def patch_installer(tag):
     """Patches the installer with the correct connector version and specklepy version"""
     iss_file = "speckle-sharp-ci-tools/arcgis.iss"
+    setup_whl_file = "setup.py"
+    conda_file = "speckle_arcgis_installer/conda_clone_activate.py"
+    #toolbox_install_file = "speckle_arcgis_installer/toolbox_install.py"
+    toolbox_manual_install_file = "speckle_arcgis_installer/toolbox_install_manual.py"
+    plugin_start_file = "speckle_toolbox/esri/toolboxes/speckle/speckle/speckle_arcgis.py"
 
     #py_tag = get_specklepy_version()
     with open(iss_file, "r") as file:
         lines = file.readlines()
-        lines.insert(12, f'#define AppVersion "{tag.split("-")[0]}"\n')
-        lines.insert(13, f'#define AppInfoVersion "{tag}"\n')
-
+        for i, line in enumerate(lines):
+            if "#define AppVersion " in line: 
+                lines[i] = f'#define AppVersion "{tag.split("-")[0]}"\n'
+            if "#define AppInfoVersion " in line: 
+                lines[i] = f'#define AppInfoVersion "{tag}"\n'
         with open(iss_file, "w") as file:
             file.writelines(lines)
             print(f"Patched installer with connector v{tag} and specklepy ")
+    file.close()
+
+    with open(setup_whl_file, "r") as file:
+        lines = file.readlines()
+        for i, line in enumerate(lines):
+            if "version=" in line: 
+                lines[i] = f'\t\t\tversion="{tag.split("-")[0]}",\n'
+                break
+        with open(setup_whl_file, "w") as file:
+            file.writelines(lines)
+            print(f"Patched whl setup with connector v{tag} and specklepy ")
+    file.close()
+
+    with open(plugin_start_file, "r") as file:
+        lines = file.readlines()
+        for i, line in enumerate(lines):
+            if 'self.version = ' in line: 
+                lines[i] = lines[i].split("\"")[0] + "\"" + tag.split('-')[0] + "\"" + lines[i].split("\"")[2]
+                break
+        with open(plugin_start_file, "w") as file:
+            file.writelines(lines)
+            print(f"Patched GIS start file with connector v{tag} and specklepy ")
+    file.close()
+
+
+    def whlFileRename(fileName: str): 
+        with open(fileName, "r") as file:
+            lines = file.readlines()
+            for i, line in enumerate(lines):
+                if "-py3-none-any.whl" in line and '.sort' not in line: 
+                    p1 = line.split("-py3-none-any.whl")[0].split("-")[0]
+                    p2 = f'{tag.split("-")[0]}'
+                    p3 = line.split("-py3-none-any.whl")[1]
+                    lines[i] = p1+"-"+p2+"-py3-none-any.whl"+p3
+            with open(fileName, "w") as file:
+                file.writelines(lines)
+                print(f"Patched toolbox_installer with connector v{tag} and specklepy ")
+        file.close()
+
+    whlFileRename(conda_file)
+    #whlFileRename(toolbox_install_file)
+    whlFileRename(toolbox_manual_install_file)
 
 
 def main():

requirements.txt

@@ -0,0 +1,3 @@
+specklepy==2.17.17
+panda3d==1.10.11
+

scripts/debugging.py

@@ -0,0 +1,2 @@
+print("Hello")
+

scripts/debugging_panel.py

@@ -0,0 +1,339 @@
+import os
+import subprocess
+pythonExec = os.environ["ProgramFiles"]+ r"\ArcGIS\Pro\bin\Python\envs\arcgispro-py3\python.exe"
+result = subprocess.run([pythonExec, "-m", "pip", "install", "--upgrade", "--ignore-installed", "specklepy==2.17.17"], capture_output=True, text=True, shell=True, timeout=1000)
+result = subprocess.run([pythonExec, "-m", "pip", "install", "--upgrade", "--ignore-installed", "panda3d==1.10.11"], capture_output=True, text=True, shell=True, timeout=1000)
+result = subprocess.run([pythonExec, "-m", "pip", "install", "--upgrade", "--ignore-installed", "PyQt5==5.15.9"], capture_output=True, text=True, shell=True, timeout=1000)
+
+
+
+from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
+import arcpy
+import json
+import os
+
+try: 
+    from speckle.speckle.converter.layers.CRS import CRS
+    from specklepy.objects.GIS.layers import Layer, VectorLayer, RasterLayer
+except: 
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.CRS import CRS
+    from speckle_toolbox.esri.toolboxes.speckle.converter.layers.Layer import Layer, VectorLayer, RasterLayer
+
+from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
+from arcpy.management import (CreateFeatureclass, MakeFeatureLayer,
+                              AddFields, AlterField, DefineProjection, SelectLayerByAttribute, GetCount )
+
+from specklepy.objects import Base
+
+##################################################### get example layers from the project #######
+project = ArcGISProject('CURRENT')
+active_map = project.activeMap
+all_layers = []
+
+layerPolygon = None
+layerPolyline = None
+layerPoint = None
+layerMultiPoint = None
+layerRaster = None
+#get layer of interest
+for layer in active_map.listLayers(): 
+   if layer.isFeatureLayer or layer.isRasterLayer: 
+        all_layers.append(layer)
+        data = arcpy.Describe(layer.dataSource)
+        if layer.isRasterLayer and layerRaster is None: layerRaster = layer 
+        if layer.isFeatureLayer:
+            geomType = data.shapeType
+            if geomType == "Polygon" and layerPolygon is None: layerPolygon = layer 
+            if geomType == "Polyline" and layerPolyline is None: layerPolyline = layer 
+            if geomType == "Point" and layerPoint is None: layerPoint = layer 
+            if geomType == "Multipoint" and layerMultiPoint is None: layerMultiPoint = layer 
+
+################################ select/ clear selection ###########################
+for layer in project.activeMap.listLayers(): 
+    if (layer.isFeatureLayer) or layer.isRasterLayer: 
+        arcpy.SelectLayerByAttribute_management(layer.longName,"ADD_TO_SELECTION",'"OBJECTID" = 1')
+        print(arcpy.GetCount_management(layer.longName).getOutput(0))
+        arcpy.SelectLayerByAttribute_management(layer.longName, "CLEAR_SELECTION")
+
+################## reset symbology if needed:
+sym = layerPolygon.symbology
+print(sym.renderer.type)
+sym.updateRenderer('UniqueValueRenderer')
+layerPolygon.symbology = sym
+print(sym.updateRenderer('UniqueValueRenderer'))
+print(layerPolygon.symbology.renderer.type)
+# SimpleRenderer, GraduatedColorsRenderer, GraduatedSymbolsRenderer, UnclassedColorsRenderer, UniqueValueRenderer 
+
+######################################### change symbology ################################# 
+
+for k, grp in enumerate(sym.renderer.groups):
+    for itm in grp.items:
+        print(itm)
+        print(itm.values)
+        print(itm.symbol.color)
+        transVal = itm.values[0][0] #Grab the first "percent" value in the list of potential values
+        print(transVal)
+        for i in range(len(cats)):
+            label = cats[i]['value'] 
+            print(label)
+            if label is None or label=="": label = "<Null>"
+            print(label)
+
+
+
+from speckle.speckle.converter.layers.symbology import get_polygon_simpleRenderer 
+from arcpy._mp import ArcGISProject
+
+aprx = ArcGISProject('CURRENT')
+root_path = "\\".join(aprx.filePath.split("\\")[:-1])
+
+path_style = root_path + '\\layer_speckle_symbology.lyrx'
+path_style2 = root_path + '\\layer_speckle_symbology2.lyrx'
+#arcpy.management.SaveToLayerFile(layerPolygon, path_style, False)
+print(layerPolygon.dataSource)
+arcpy.management.ApplySymbologyFromLayer(
+                        in_layer=layerPolygon.dataSource, 
+                        in_symbology_layer=path_style2, 
+                        update_symbology='UPDATE')
+
+
+
+f = open(path_style, "r")
+renderer = json.loads(f.read())
+
+renderer["layerDefinitions"][0]["renderer"] = get_polygon_simpleRenderer(1,2,150)
+f = open(path_style2, "w")
+f.write(json.dumps(renderer, indent=4))
+f.close()
+arcpy.management.ApplySymbologyFromLayer(str(layerPolygon), path_style2)
+os.remove(path_style)
+os.remove(path_style2)
+
+###########################################################################
+layer = all_layers[0] 
+if isinstance(layer, arcLayer): 
+
+    projectCRS = project.activeMap.spatialReference
+    try: data = arcpy.Describe(layer.dataSource)
+    except OSError as e: print(e)
+
+    layerName = layer.name
+    crs = data.SpatialReference
+    units = "m"
+    layerObjs = []
+
+    # Convert CRS to speckle, use the projectCRS
+    speckleReprojectedCrs = CRS(name = projectCRS.name, wkt = projectCRS.exportToString(), units = units)
+    layerCRS = CRS(name=crs.name, wkt=crs.exportToString(), units = units) 
+
+    if layer.isFeatureLayer: 
+        print("VECTOR LAYER HERE")
+        
+        speckleLayer = VectorLayer(units = "m")
+        speckleLayer.type="VectorLayer"
+        speckleLayer.name = layerName
+        speckleLayer.crs = speckleReprojectedCrs
+
+        if data.datasetType == "FeatureClass": #FeatureClass, ?Table Properties, ?Datasets
+            # write feature attributes
+            fieldnames = [field.name for field in data.fields]
+            rows_shapes = arcpy.da.SearchCursor(layer.longName, "Shape@") # arcpy.da.SearchCursor(in_table, field_names, {where_clause}, {spatial_reference}, {explode_to_points}, {sql_clause})
+            print("__ start iterating features")
+            row_shapes_list = [x for k, x in enumerate(rows_shapes)]
+            for i, features in enumerate(row_shapes_list):
+
+                print("____error Feature # " + str(i+1)) # + " / " + str(sum(1 for _ in enumerate(rows_shapes))))
+                if features[0] is None: continue 
+                feat = features[0]
+
+                if feat is not None: 
+                    print(feat)
+                    rows_attributes = arcpy.da.SearchCursor(layer.longName, fieldnames)
+                    row_attr = []
+                    for k, attrs in enumerate(rows_attributes):
+                        if i == k: row_attr = attrs; break
+                    if feat.hasCurves: feat = feat.densify("ANGLE", 1000, 0.12)
+
+                    print("___________Feature to Speckle____________")
+
+                    b = Base(units = "m")
+                    data = arcpy.Describe(layer.dataSource)
+                    layer_sr = data.spatialReference # if sr.type == "Projected":
+                    geomType = data.shapeType #Polygon, Point, Polyline, Multipoint, MultiPatch
+                    featureType = data.featureType # Simple,SimpleJunction,SimpleJunction,ComplexEdge,Annotation,CoverageAnnotation,Dimension,RasterCatalogItem 
+                    print(geomType)
+                    print(hasattr(data, "isRevit")) 
+                    print(hasattr(data, "isIFC")) 
+                    print(hasattr(data, "bimLevels")) 
+                    print(hasattr(data, "hasSpatialIndex")) 
+                    if geomType == "MultiPatch" or hasattr(data, "isRevit") or hasattr(data, "isIFC") or hasattr(data, "bimLevels"): 
+                        print(f"Layer {layer.name} has unsupported data type")
+
+                        print("___convertToSpeckle____________")
+                        geom = feat
+                        print(geom.isMultipart) # e.g. False 
+                        print(geom.hasCurves)
+                        print(geom.partCount)
+                        geomMultiType = geom.isMultipart
+                        hasCurves = feat.hasCurves 
+
+                        geomPart = []
+                        for i,x in enumerate(feat): # [[x,x,x]
+                            
+                            if i==0:
+                                print("Part # " + str(i+1))
+                                print(x)
+
+                                inner_arr = []
+                                for k,ptn in enumerate(x):  
+                                    if k<10: print(ptn) # e.g. 6.25128173828125 -9.42138671875 22.2768999999971 NaN
+
+                                    inner_arr.append(ptn)
+                                    #inner_arr.append(inner_arr[0]) #add first in the end
+                                geomPart.append(arcpy.Array(inner_arr))
+
+                                geomPartArray = arcpy.Array(inner_arr)
+                                sr = project.activeMap.spatialReference
+
+                                multipatch = arcpy.Multipatch(arcpy.Array(x), sr, has_z=True) # error 
+                                print(multipatch)
+                        
+                    else:
+                        print("___convertToSpeckle____________")
+                        geom = feat
+                        print(geom.isMultipart) # e.g. False 
+                        print(geom.hasCurves)
+                        print(geom.partCount)
+                        geomMultiType = geom.isMultipart
+                        hasCurves = feat.hasCurves 
+
+                        for i,x in enumerate(feat): # [[x,x,x]
+                            print("Part # " + str(i+1))
+                            print(x)
+                            for k,ptn in enumerate(x): 
+                                if k<10: print(ptn) # e.g. 6.25128173828125 -9.42138671875 22.2768999999971 NaN
+
+
+path: str = project.filePath.replace("aprx","gdb") 
+sr = project.activeMap.spatialReference
+print(sr)
+f_class = CreateFeatureclass(path, "NewTestLayer", "Multipatch", has_z="ENABLED", spatial_reference = sr)
+fets = []
+print("04_____Feature To Native____________") 
+new_feat = {}
+new_feat.update({"arcGisGeomFromSpeckle": multipatch})
+fets.append(new_feat)
+
+vl = MakeFeatureLayer(str(f_class), "NewTestLayer").getOutput(0)
+
+
+
+############################# write shapefile ##################################
+
+import shapefile
+from shapefile import TRIANGLE_STRIP, TRIANGLE_FAN
+
+from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
+from arcpy.management import (CreateFeatureclass, MakeFeatureLayer,
+                              AddFields, AlterField, DefineProjection )
+
+from specklepy.objects import Base
+
+project = ArcGISProject('CURRENT')
+path: str = project.filePath.replace("aprx","gdb") 
+
+#with shapefile.Writer(path + "\contextwriter") as w:
+#    w.field('field1', 'C')
+#    pass
+
+w = shapefile.Writer(path + '\\dtype')
+w.field('TEXT', 'C')
+w.field('SHORT_TEXT', 'C', size=5)
+w.field('LONG_TEXT', 'C', size=250)
+w.null()
+w.record('Hello', 'World', 'World'*50)
+w.close()
+
+r = shapefile.Reader(path + '\\dtype')
+assert r.record(0) == ['Hello', 'World', 'World'*50]
+################################################################### WORKS #################################
+
+w = shapefile.Writer(path + '\\dtype')
+w.field('INT', 'N')
+w.field('LOWPREC', 'N', decimal=2)
+w.field('MEDPREC', 'N', decimal=10)
+w.field('HIGHPREC', 'N', decimal=30)
+w.field('FTYPE', 'F', decimal=10)
+w.field('LARGENR', 'N', 101)
+w.field('FIRST_FLD','C','40')
+w.field('SECOND_FLD','C','40')
+nr = 1.3217328
+w.null()
+w.null()
+w.record(INT=nr, LOWPREC=nr, MEDPREC=nr, HIGHPREC=-3.2302e-25, FTYPE=nr, LARGENR=int(nr)*10**100, FIRST_FLD='First', SECOND_FLD='Line')
+w.record(None, None, None, None, None, None, '', '')
+w.close()
+
+r = shapefile.Reader(path + '\\dtype')
+assert r.record(0) == [1, 1.32, 1.3217328, -3.2302e-25, 1.3217328, 10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000, 'First', 'Line']
+assert r.record(1) == [None, None, None, None, None, None, '', '']
+
+################################################################# Add point ####################
+
+w = shapefile.Writer(path + '\\dtypeShapes')
+w.field('name', 'C')
+
+w.point(122, 37) 
+w.record('point1')
+
+w.close()
+
+################################################################# Add Multipatch ####################
+
+w = shapefile.Writer(path + '\\MultipatchTest2')
+w.field('name', 'C')
+
+w.multipatch([
+			 [[0,0,0],[0,0,3],[5,0,0],[5,0,3],[5,5,0],[5,5,3],[0,5,0],[0,5,3],[0,0,0],[0,0,3]], # TRIANGLE_STRIP for house walls
+			 [[2.5,2.5,5],[0,0,3],[5,0,3],[5,5,3],[0,5,3],[0,0,3]], # TRIANGLE_FAN for pointed house roof
+			 ],
+			 partTypes=[TRIANGLE_STRIP, TRIANGLE_FAN]) # one type for each part
+
+w.record('house1')
+w.close()
+
+r = shapefile.Reader(path + '\\MultipatchTest2')
+assert r.record(0) == ['house1']
+
+
+active_map.addDataFromPath(path + '\\MultipatchTest2.shp')
+
+########################################################################## reader 
+sf = shapefile.Reader(path + '\\MultipatchTest2.shp')
+sf.shapeType # e.g. 31 - multipatch
+sf.bbox # e.g. [0.0, 0.0, 5.0, 5.0]
+shapefile.Shape
+
+
+##################################################### cerate multipatch layer #################################
+result = arcpy.management.CreateFeatureclass(arcpy.env.scratchGDB, "test_multipatch", "MULTIPATCH", has_z="ENABLED", spatial_reference=4326)
+feature_class = result[0]
+
+
+################################# reading shapefile - works ####################
+
+fc = r'C:\Users\katri\Documents\ArcGIS\Projects\MyProject\Layers_Speckle\BIM_layers_speckle\00f70159b9104180f622cca87f5dd2cb.shp'
+rows = arcpy.da.SearchCursor(fc, 'Shape@')
+for r in rows:
+        if r is not None: shape = r
+print(shape)
+cl = arcpy.conversion.FeatureClassToFeatureClass(r'C:\Users\katri\Documents\ArcGIS\Projects\MyProject\Layers_Speckle\BIM_layers_speckle\16d73b756a_main_2f8cfa8644\__Floors_Mesh\00c7696966e4cfda2bd8c03860a414a6', r'C:\Users\katri\Documents\ArcGIS\tests', 'copyclass')
+
+##################################### update rows in feature class - working #############
+with arcpy.da.UpdateCursor('f_class_2f8cfa8644___Structural_Framing_Mesh', 'name') as cursor:
+    # For each row, evaluate the WELL_YIELD value (index position 
+    # of 0), and update WELL_CLASS (index position of 1)
+    for row in cursor: 
+        row[0] = "newName"
+        cursor.updateRow(row)
+

setup.py

@@ -1,31 +1,56 @@
-# to build an installer: run cmd from this folder:
-# "%PROGRAMFILES%\\ArcGIS\\Pro\\bin\\Python\\envs\\arcgispro-py3\\python.exe"  C:\\Users\\username\\Documents\\00_Speckle\\GitHub\\speckle-arcgis\\setup.py sdist bdist_wheel 
+# to build an installer: run cmd from this folder or use terminal: "%PROGRAMFILES%\\ArcGIS\\Pro\\bin\\Python\\envs\\arcgispro-py3\\python.exe"
+#
+# 1) python patch_version.py 2.x.x
+# 2) python setup.py sdist bdist_wheel   #C:\\Users\\username\\Documents\\00_Speckle\\GitHub\\speckle-arcgis\\setup.py sdist bdist_wheel
+# copy .whl from "dist" to "speckle_arcgis_installer"
 
-import os 
-from setuptools import setup 
+# ref: https://pro.arcgis.com/en/pro-app/2.8/arcpy/geoprocessing_and_python/distributing-python-modules.htm
 
-# https://pro.arcgis.com/en/pro-app/2.8/arcpy/geoprocessing_and_python/distributing-python-modules.htm
+import os
+from setuptools import setup
 
-def read(fname): 
-    return open(os.path.join(os.path.dirname(__file__), fname)).read() 
- 
-setup(name='speckle_toolbox', 
-      version='0.1',
-      author='SpeckleSystems',
-      description=("Example for extending geoprocessing through Python modules"),
-      long_description=read('Readme.md'),
-      python_requires='~=3.3',
-      packages=['speckle_toolbox'], 
-      package_data={'speckle_toolbox':['esri/toolboxes/*',  
-                  'esri/arcpy/*', 
-                  'esri/help/gp/*', 'esri/help/gp/toolboxes/*', 'esri/help/gp/messages/*',
-                  'esri/toolboxes/*','esri/toolboxes/speckle/*',
-                  'esri/toolboxes/speckle/converter/*', 'esri/toolboxes/speckle/converter/geometry/*', 'esri/toolboxes/speckle/converter/layers/*',
-                  'esri/toolboxes/speckle/plugin_utils/*'] 
-                  }, 
-      )
+
+def read(fname):
+    return open(os.path.join(os.path.dirname(__file__), fname)).read()
+
+
+setup(
+    name="speckle_toolbox",
+    author="SpeckleSystems",
+    version="0.0.99",
+    author_email="connectors@speckle.systems",
+    url="https://speckle.systems/",
+    description=("Example for extending geoprocessing through Python modules"),
+    long_description=read("Readme.md"),
+    python_requires="~=3.3",
+    setup_requires=["wheel"],
+    packages=["speckle_toolbox"],
+    package_data={
+        "speckle_toolbox": [
+            "esri/arcpy/*",
+            "esri/help/gp/*",
+            "esri/help/gp/toolboxes/*",
+            "esri/help/gp/messages/*",
+            "esri/toolboxes/*",
+            "esri/toolboxes/speckle/*",
+            "esri/toolboxes/speckle/speckle/*",
+            "esri/toolboxes/speckle/speckle/converter/*",
+            "esri/toolboxes/speckle/speckle/converter/features/*",
+            "esri/toolboxes/speckle/speckle/converter/geometry/*",
+            "esri/toolboxes/speckle/speckle/converter/layers/*",
+            "esri/toolboxes/speckle/speckle/plugin_utils/*",
+            "esri/toolboxes/speckle/speckle/utils/*",
+            "esri/toolboxes/speckle/specklepy_qt_ui/*",
+            "esri/toolboxes/speckle/specklepy_qt_ui/qt_ui/*",
+            "esri/toolboxes/speckle/specklepy_qt_ui/qt_ui/ui/*",
+            "esri/toolboxes/speckle/specklepy_qt_ui/qt_ui/utils/*",
+            "esri/toolboxes/speckle/specklepy_qt_ui/qt_ui/utils/assets/*",
+            "esri/toolboxes/speckle/ui_widgets/*",
+        ]
+    },
+)
 
 # then to install in ArcGIS:
 # import sysconfig; import subprocess; x = sysconfig.get_paths()['data'] + r"\python.exe"; subprocess.run((x, '-m','pip', 'install', 'C:\\Users\\username\\Documents\\00_Speckle\\GitHub\\speckle-arcgis\\dist\\foo-0.1-py3-none-any.whl'), capture_output=True, text=True, shell=True, timeout=1000 )
-# to uninstall: 
+# to uninstall:
 # "C:\\Users\\username\\AppData\\Local\\ESRI\\conda\\envs\\arcgispro-py3-speckle\\python.exe" -m pip uninstall C:\\Users\\username\\Documents\\00_Speckle\\GitHub\\speckle-arcgis\\dist\\foo-0.1-py3-none-any.whl

speckle_arcgis_installer/Readme.md

@@ -0,0 +1,12 @@
+### Manual installation
+
+1. From the [latest release](https://github.com/specklesystems/speckle-arcgis/releases) download the whl file and the source code zip, unzip and locate the subfolder "speckle_arcgis_installer" on your machine and place whl file in it. 
+2. Clone the default ArcGIS Pro conda environment (or set the one you use, except the default one) and restart ArcGIS Pro
+       - for 3.0.0: Project-> Package Manager-> Active Environment (Environment Manager)-> Clone arcgispro-py3
+3. Adjust the path to your new environment python executable (variable "pythonPath" in "speckle_arcgis_installer/toolbox_install_manual.py") 
+4. Enter the location of 'toolbox_install_manual.py' in the following command and run this command in ArcGIS Python console (View -> Python Window)
+
+```python
+import sysconfig; import subprocess; x = sysconfig.get_paths()['data'] + r"\python.exe"; subprocess.run((x, 'C:\\Users\\pathToFolder\\speckle_arcgis_installer\\toolbox_install_manual.py'), capture_output=True, text=True, shell=True, timeout=1000 )
+```
+

speckle_arcgis_installer/Speckle.pyt

@@ -0,0 +1,4 @@
+try:
+    from speckle.speckle.speckle_arcgis import *
+except:
+    from speckle_toolbox.esri.toolboxes.speckle.speckle_arcgis import *

speckle_arcgis_installer/conda_clone_activate.py

@@ -1,4 +1,4 @@
-# clone env, install toolbox & dependencies into the cloned env; and into current one (if not default) 
+# clone env, install toolbox & dependencies into the cloned env; and into current one (if not default)
 import sys
 import sysconfig
 import os.path
@@ -8,50 +8,242 @@ import subprocess
 from subprocess import CalledProcessError
 from subprocess_call import subprocess_call
 
-from msilib.schema import Error
 import sys
 
-import arcpy 
+ENV_NEW_NAME = "arcgispro-py3-speckle"
+PROSWAP = "proswap"
+
 
 def setup():
-    #print(plugin_dir) 
-    pythonExec = get_python_path() # import numpy; import os; print(os.path.abspath(numpy.__file__)) 
-    #print(pythonExec)
-    
-    if pythonExec is None: # env is default, need to restart ArcGIS
-        return False
-    
-def get_python_path(): # create a full copy of default env 
+    pythonExec = (
+        get_python_path()
+    )  # import numpy; import os; print(os.path.abspath(numpy.__file__))
+    return pythonExec  # None if not successful
+
+
+def get_default_python():
+    pythonExec = (
+        os.environ["ProgramFiles"]
+        + r"\ArcGIS\Pro\bin\Python\envs\arcgispro-py3\python.exe"
+    )  # (r"%PROGRAMFILES%\ArcGIS\Pro\bin\Python\envs\arcgispro-py3\python.exe") #os.path.dirname(sys.executable) + "\\python.exe" # default python.exe
+    # print(pythonExec)
+    if not os.path.exists(pythonExec):
+        pythonExec = (
+            os.getenv("APPDATA").replace("Roaming", "Local")
+            + r"\Programs\ArcGIS\Pro\bin\Python\envs\arcgispro-py3\python.exe"
+        )
+        if not os.path.exists(pythonExec):
+            return None
+    return pythonExec
+
+
+def get_python_path():  # create a full copy of default env
+    # print("Get Python path")
     # or: import site; site.getsitepackages()[0]
     # import specklepy; import os; print(os.path.abspath(specklepy.__file__)) ##currentPythonExec = sysconfig.get_paths()['data'] + r"\python.exe"
+    def_exec = get_default_python()
+    # print(os.getenv('APPDATA') + r'\Programs\ArcGIS\Pro\bin\Python\envs\arcgispro-py3\python.exe')
 
-    pythonExec = os.environ["ProgramFiles"] + r'\ArcGIS\Pro\bin\Python\envs\arcgispro-py3\python.exe' #(r"%PROGRAMFILES%\ArcGIS\Pro\bin\Python\envs\arcgispro-py3\python.exe") #os.path.dirname(sys.executable) + "\\python.exe" # default python.exe
-    #print(pythonExec)
     if sys.platform == "win32":
-        env_new_name = "arcgispro-py3-speckle"
-        #clone_env(pythonExec, env_new_name) # only if doesn't exist yet 
-        activate_env(env_new_name)
-        return pythonExec
-    else: pass
+        newExec = clone_env(def_exec)  # only if doesn't exist yet
+        if not os.path.exists(newExec):
+            return None
 
-def clone_env(pythonExec_old: str, env_new_name: str): 
-    install_folder = os.getenv('APPDATA').replace("\\Roaming","") + r"\Local\ESRI\conda\envs"   #r"%LOCALAPPDATA%\ESRI\conda\envs" 
-    #print("Clone default ArcGIS Pro conda env")
-    #print(install_folder)
-    #if not os.path.exists(install_folder): os.makedirs(install_folder)
-    
-    default_env = pythonExec_old.replace("Pro\\bin\\Python\\envs\\arcgispro-py3\\python.exe","Pro\\bin\\Python\\envs\\arcgispro-py3") # + "\\" + 'arcgispro-py3'
-    conda_exe = pythonExec_old.replace("Pro\\bin\\Python\\envs\\arcgispro-py3\\python.exe","Pro\\bin\\Python\\Scripts\\conda.exe") #%PROGRAMFILES%\ArcGIS\Pro\bin\Python\Scripts\conda.exe #base: %PROGRAMDATA%\Anaconda3\condabin\conda.bat
-    new_env = install_folder + "\\" + env_new_name # %LOCALAPPDATA%\ESRI\conda\envs\...
+        activate_env()
+        return newExec
+    else:
+        return None
 
-    subprocess_call( [ conda_exe, 'create', '--clone', default_env, '-p', new_env] ) # will not execute if already exists
 
+def clone_env(pythonExec_old: str):
+    install_folder = (
+        os.getenv("APPDATA").replace("\\Roaming", "") + r"\Local\ESRI\conda\envs"
+    )  # r"%LOCALAPPDATA%\ESRI\conda\envs"
+    if not os.path.exists(install_folder):
+        os.makedirs(install_folder)
+
+    default_env = pythonExec_old.replace(
+        "Pro\\bin\\Python\\envs\\arcgispro-py3\\python.exe",
+        "Pro\\bin\\Python\\envs\\arcgispro-py3",
+    )  # + "\\" + 'arcgispro-py3'
+    conda_exe = pythonExec_old.replace(
+        "Pro\\bin\\Python\\envs\\arcgispro-py3\\python.exe",
+        "Pro\\bin\\Python\\Scripts\\conda.exe",
+    )  # %PROGRAMFILES%\ArcGIS\Pro\bin\Python\Scripts\conda.exe #base: %PROGRAMDATA%\Anaconda3\condabin\conda.bat
+    global PROSWAP
+    PROSWAP = conda_exe.replace(
+        "conda.exe",
+        "proswap.bat",
+    )
+    print(PROSWAP)
+    new_env = install_folder + "\\" + ENV_NEW_NAME  # %LOCALAPPDATA%\ESRI\conda\envs\...
+
+    # first check if venv invalid:
+    if (
+        os.path.exists(conda_exe)
+        and os.path.exists(default_env)
+        and os.path.exists(new_env)
+    ):
+        # delete existing venv
+        print(f"Removing invalid environment {new_env}")
+        try:
+            os.remove(new_env)
+        except PermissionError as e:
+            print(e)
+    elif (
+        os.path.exists(conda_exe)
+        and os.path.exists(default_env)
+        and not os.path.exists(new_env)
+    ):
+        print("Wait for the default ArcGIS Pro conda environment to be cloned")
+        subprocess_call([conda_exe, "config", "--set", "ssl_verify", "False"])
+        subprocess_call(
+            [conda_exe, "create", "--clone", default_env, "-p", new_env]
+        )  # will not execute if already exists
+        subprocess_call([conda_exe, "config", "--set", "ssl_verify", "True"])
+
+    # final check
+    if os.path.exists(new_env) and os.path.exists(new_env + "\\python.exe"):
+        print("Preparing to install packages..")
+
+    print(new_env + "\\python.exe")
     return new_env + "\\python.exe"
 
-def activate_env(env_new_name: str):
-    # using Popen, because process does not return result; subprocess.run will hang indefinitely 
-    variable = subprocess.Popen((f'proswap {env_new_name}'),stdout = subprocess.PIPE,stderr = subprocess.PIPE,text = True,shell = True)
-    #print(variable) 
+
+def activate_env():
+    # using Popen, because process does not return result; subprocess.run will hang indefinitely
+    # print(os.environ["TMPDIR"])
+    my_env = os.environ.copy()
+    print(PROSWAP)
+    # my_env["TMPDIR"] = r"C:\Users\katri\AppData\Roaming\Speckle\connector_installations"
+    pop = subprocess.Popen(
+        (f"{PROSWAP} {ENV_NEW_NAME}"),
+        text=True,
+        env=my_env,
+    )
     # activate new env : https://support.esri.com/en/technical-article/000024206
 
-setup()
+
+def installToolbox(newExec: str):
+    print("Installing Speckle Toolbox")
+    whl_file = os.path.join(
+        os.path.dirname(__file__), "speckle_toolbox-2.9.99-py3-none-any.whl"
+    )
+    print(whl_file)
+    subprocess_call(
+        [newExec, "-m", "pip", "install", "--upgrade", "--force-reinstall", whl_file]
+    )
+    # to uninstall: cmd.exe "X:\\xxx.whl
+    return
+
+
+def clearToolbox(pythonExec: str):
+    # install pip
+    print("CLEAR toolbox")
+    print(pythonExec)
+    try:
+
+        speckle_path = pythonExec.replace("python.exe", "Lib\\site-packages\\")
+
+        print(speckle_path)
+        paths = os.listdir(speckle_path)
+        for p in paths:
+            if "speckle_toolbox" in p:
+                print("remove: " + str(p))
+                os.remove(p)
+    except Exception as e:
+        print(e)
+        pass
+
+
+def installDependencies(pythonExec: str, pkgName: str, pkgVersion: str):
+    # install pip
+    print(pythonExec)
+    try:
+        import pip
+    except:
+        getPipFilePath = os.path.join(
+            os.path.dirname(__file__), "get_pip.py"
+        )  # TODO: give actual folder path
+        exec(open(getPipFilePath).read())
+        # just in case the included version is old
+        subprocess_call([pythonExec, "-m", "pip", "install", "--upgrade", "pip"])
+
+    # install package
+    try:
+        # import importlib #importlib.import_module(pkgName)
+        if pkgName == "specklepy":
+            import specklepy
+
+            if pythonExec.replace("\\python.exe", "") not in (
+                os.path.abspath(specklepy.__file__)
+            ):
+                print(f"Installing {pkgName} to {pythonExec}")
+                # subprocess_call( [pythonExec, "-m", "pip", "uninstall", f"{pkgName}"])
+                subprocess_call(
+                    [
+                        pythonExec,
+                        "-m",
+                        "pip",
+                        "install",
+                        "--ignore-installed",
+                        f"{pkgName}=={pkgVersion}",
+                    ]
+                )
+        elif pkgName == "panda3d":
+            import panda3d
+
+            if pythonExec.replace("\\python.exe", "") not in (
+                os.path.abspath(panda3d.__file__)
+            ):
+                print(f"Installing {pkgName} to {pythonExec}")
+                subprocess_call(
+                    [
+                        pythonExec,
+                        "-m",
+                        "pip",
+                        "install",
+                        "--ignore-installed",
+                        f"{pkgName}=={pkgVersion}",
+                    ]
+                )
+        elif pkgName == "PyQt5":
+            import PyQt5
+
+            if pythonExec.replace("\\python.exe", "") not in (
+                os.path.abspath(PyQt5.__file__)
+            ):
+                print(f"Installing {pkgName} to {pythonExec}")
+                subprocess_call(
+                    [
+                        pythonExec,
+                        "-m",
+                        "pip",
+                        "install",
+                        "--ignore-installed",
+                        f"{pkgName}=={pkgVersion}",
+                    ]
+                )
+    except Exception as e:
+        print(f"{pkgName} not installed")
+        subprocess_call(
+            [pythonExec, "-m", "pip", "install", f"{pkgName}=={pkgVersion}"]
+        )
+    return True
+
+
+pythonPath = setup()
+print(pythonPath)
+if pythonPath is not None:
+
+    # def_exec = get_default_python()
+    # conda_exe = def_exec.replace("Pro\\bin\\Python\\envs\\arcgispro-py3\\python.exe","Pro\\bin\\Python\\Scripts\\conda.exe") #%PROGRAMFILES%\ArcGIS\Pro\bin\Python\Scripts\conda.exe #base: %PROGRAMDATA%\Anaconda3\condabin\conda.bat
+    # subprocess_call([conda_exe, 'proup','-n', ENV_NEW_NAME])
+
+    clearToolbox(pythonPath)
+    installToolbox(pythonPath)
+    installDependencies(pythonPath, "specklepy", "2.17.17")
+    installDependencies(pythonPath, "panda3d", "1.10.11")
+    installDependencies(pythonPath, "PyQt5", "5.15.9")
+
+# manual: import sysconfig; import subprocess; x = sysconfig.get_paths()['data'] + r"\python.exe"; subprocess.run((x, "-m", "pip", "install", "PyQt5==5.15.9"), capture_output=True, text=True, shell=True, timeout=1000 )

speckle_arcgis_installer/manual_toolbox_install.py

@@ -1,73 +0,0 @@
-# MANUAL INSTALLATION: 
-#   1. enter correct path to your new environemnt in line 10 
-#   2. enter the location of 'manual_toolbox_install.py' and run this command in ArcGIS Python console (View -> Python Window)
-#   import sysconfig; import subprocess; x = sysconfig.get_paths()['data'] + r"\python.exe"; subprocess.run((x, 'C:\\...\\manual_toolbox_install.py'), capture_output=True, text=True, shell=True, timeout=1000 )
-# then restart
-
-from subprocess_call import subprocess_call
-import os 
-
-pythonPath = "C:\\ ...\\custom_environment_name\\python.exe"
-
-def installToolbox(newExec: str):
-    print("Installing Speckle Toolbox")
-    whl_file = os.path.join(os.path.dirname(__file__), "foo-0.1-py3-none-any.whl" ) 
-    subprocess_call([newExec, '-m','pip','install','--upgrade', '--force-reinstall', whl_file])
-    return
-
-def installDependencies(pythonExec: str):
-    print("Installing dependencies")
-    print(pythonExec)
-    try:
-        import pip
-    except:
-        getPipFilePath = os.path.join(os.path.dirname(__file__), "get_pip.py") 
-        exec(open(getPipFilePath).read())
-
-        # just in case the included version is old
-        subprocess_call([pythonExec, "-m", "pip", "install", "--upgrade", "pip"])
-    
-    pkgVersion = "2.7.4" 
-    pkgName = "specklepy"
-    try:
-        import specklepy 
-    except Exception as e:
-        subprocess_call([ pythonExec, "-m", "pip", "install", f"{pkgName}=={pkgVersion}"])
-
-    
-    pkgVersion = "1.10.11" 
-    pkgName = "panda3d"
-    try:
-        import panda3d
-    except Exception as e:
-        print("panda3d not installed")
-        subprocess_call( [pythonExec, "-m", "pip", "install", f"{pkgName}=={pkgVersion}"])
-
-    # Check if specklpy needs updating
-    try:
-        print(f"Attempting to update specklepy to {pkgVersion}")
-        # pip.main(['install', "specklepy==2.7.4"])
-        result = subprocess_call(
-            [
-                pythonExec,
-                "-m",
-                "pip",
-                "install",
-                "--upgrade",
-                f"{pkgName}=={pkgVersion}",
-            ]
-        )
-        if result == True:
-            print("specklepy upgraded")
-            return True
-        else: 
-            return False
-
-    except Exception as e:
-        print(e)
-        print(e.with_traceback)
-    return True
-
-installToolbox(pythonPath)
-installDependencies(pythonPath)
-

speckle_arcgis_installer/speckle_arcgis_install.bat

@@ -1,3 +0,0 @@
-@echo off
-"%PROGRAMFILES%\ArcGIS\Pro\bin\Python\envs\arcgispro-py3\python.exe" "conda_clone_activate.py"
-"%LOCALAPPDATA%\ESRI\conda\envs\arcgispro-py3-speckle\python.exe" "toolbox_install.py"

speckle_arcgis_installer/subprocess_call.py

@@ -14,12 +14,12 @@ def subprocess_call(*args, **kwargs):
         startupinfo.dwFlags = subprocess.CREATE_NEW_CONSOLE | subprocess.STARTF_USESHOWWINDOW
         startupinfo.wShowWindow = subprocess.SW_HIDE
         kwargs['startupinfo'] = startupinfo
-        print("start")
+        #print("start")
         #print(*args)
     try: 
         # if manually: cmd.exe -> conda activate [env folder] -> pip install specklepy 
         result = subprocess.run(*args, capture_output=True, text=True, shell=True, timeout=1000)
-        #print(result)
+        print(result)
         #result = subprocess.Popen( arg, shell=True, stdout=subprocess.PIPE) #, stderr=subprocess.STDOUT)
         #retcode = subprocess.check_call(*args, **kwargs) # Creates infinite loop, known issue: https://github.com/python/cpython/issues/87512
     except CalledProcessError as e: 
@@ -32,7 +32,7 @@ def subprocess_call(*args, **kwargs):
         #print(str(e))
         return False
     except: print("unknown error") 
-    print("end")
+    #print("end")
     return True
 
     

speckle_arcgis_installer/toolbox_install.py

@@ -1,69 +0,0 @@
-
-from subprocess_call import subprocess_call
-import os 
-
-pythonPath = os.getenv('APPDATA').replace("\\Roaming","") + r"\Local\ESRI\conda\envs\arcgispro-py3-speckle\python.exe"
-
-def installToolbox(newExec: str):
-    print("Installing Speckle Toolbox")
-    whl_file = os.path.join(os.path.dirname(__file__), "speckle_toolbox-0.1-py3-none-any.whl" ) 
-    subprocess_call([newExec, '-m','pip','install','--upgrade', '--force-reinstall', whl_file])
-    # to uninstall: cmd.exe "C:\\Users\\username\\AppData\\Local\\ESRI\\conda\\envs\\arcgispro-py3-speckle\\python.exe" -m pip uninstall C:\\Users\\username\\Downloads\\speckle-arcgis\\foo-0.1-py3-none-any.whl
-    return
-
-def installDependencies(pythonExec: str):
-    #print("Installing dependencies")
-    print(pythonExec)
-    try:
-        import pip
-    except:
-        getPipFilePath = os.path.join(os.path.dirname(__file__), "get_pip.py") #TODO: give actual folder path 
-        exec(open(getPipFilePath).read())
-
-        # just in case the included version is old
-        subprocess_call([pythonExec, "-m", "pip", "install", "--upgrade", "pip"])
-    
-    pkgVersion = "2.7.4" 
-    pkgName = "specklepy"
-    try:
-        import specklepy # C:\Users\username\AppData\Roaming\Python\Python37\site-packages\specklepy\__init__.py 
-    except Exception as e:
-        subprocess_call([ pythonExec, "-m", "pip", "install", f"{pkgName}=={pkgVersion}"])
-
-    
-    pkgVersion = "1.10.11" 
-    pkgName = "panda3d"
-    try:
-        import panda3d
-    except Exception as e:
-        print("panda3d not installed")
-        subprocess_call( [pythonExec, "-m", "pip", "install", f"{pkgName}=={pkgVersion}"])
-
-    # Check if specklpy needs updating
-    try:
-        print(f"Attempting to update specklepy to {pkgVersion}")
-        # pip.main(['install', "specklepy==2.7.4"])
-        result = subprocess_call(
-            [
-                pythonExec,
-                "-m",
-                "pip",
-                "install",
-                "--upgrade",
-                f"{pkgName}=={pkgVersion}",
-            ]
-        )
-        if result == True:
-            print("specklepy upgraded")
-            return True
-        else: 
-            return False
-
-    except Exception as e:
-        print(e)
-        print(e.with_traceback)
-    return True
-
-installToolbox(pythonPath)
-installDependencies(pythonPath)
-

speckle_arcgis_installer/toolbox_install_manual.py

@@ -0,0 +1,63 @@
+# MANUAL INSTALLATION: 
+#   1. Clone the default ArcGIS Pro conda environment 
+#       for 2.9.0: Project-> Python-> Manage Environments-> Clone Default
+#       for 3.0.0: Project-> Package Manager-> Active Environment (Environment Manager)-> Clone arcgispro-py3
+#   2. Change the path to your new environemnt Python.exe if necessary (in variable "pythonPath" below, line 13) 
+#   3. Enter the location of 'toolbox_install_manual.py' in the following command and run this command in ArcGIS Python console (View -> Python Window)
+#   import sysconfig; import subprocess; x = sysconfig.get_paths()['data'] + r"\python.exe"; subprocess.run((x, 'C:\\Users\\myusername\\Documents\\toolbox_install_manual.py'), capture_output=True, text=True, shell=True, timeout=1000 )
+#   4. Restart ArcGIS Pro 
+
+from subprocess_call import subprocess_call
+import os 
+from os import listdir
+from os.path import isfile, join
+
+pythonPath = os.getenv('APPDATA').replace("\\Roaming","") + r"\Local\ESRI\conda\envs\arcgispro-py3-speckle\python.exe"
+
+def installToolbox(newExec: str):
+    print("Installing Speckle Toolbox")
+    mypath = os.path.dirname(__file__)
+    onlyfiles = [f for f in listdir(mypath) if (isfile(join(mypath, f)) and "py3-none-any.whl" in str(f))]
+    onlyfiles.sort(key = lambda x: int(x.replace("speckle_toolbox-","").replace("-py3-none-any.whl","").split(".")[1]) ) 
+    whl_file = mypath + "\\" + onlyfiles[len(onlyfiles)-1]
+    #whl_file = os.path.join(os.path.dirname(__file__), "speckle_toolbox-2.9.9-py3-none-any.whl" ) 
+    subprocess_call([newExec, '-m','pip','install','--upgrade', '--force-reinstall', whl_file])
+    return
+
+def installDependencies(pythonExec: str, pkgName: str, pkgVersion: str):
+    # install package
+    try:
+        #import importlib #importlib.import_module(pkgName)
+        if pkgName == "specklepy": import specklepy 
+        elif pkgName == "panda3d": import panda3d 
+    except Exception as e:
+        print(f"{pkgName} not installed")
+        subprocess_call( [pythonExec, "-m", "pip", "install", f"{pkgName}=={pkgVersion}"])
+
+    # Check if package needs updating
+    try:
+        print(f"Attempting to update {pkgName} to {pkgVersion}")
+        result = subprocess_call(
+            [
+                pythonExec,
+                "-m",
+                "pip",
+                "install",
+                "--upgrade",
+                f"{pkgName}=={pkgVersion}",
+            ]
+        )
+        if result == True:
+            print(f"{pkgName} upgraded")
+            return True
+        else: 
+            return False
+    except Exception as e:
+        print(e)
+        print(e.with_traceback)
+    return True
+
+installToolbox(pythonPath)
+installDependencies(pythonPath, "specklepy", "2.9.0"  )
+installDependencies(pythonPath, "panda3d", "1.10.11" )
+

speckle_toolbox/esri/toolboxes/Speckle.pyt

@@ -1 +1,4 @@
-from speckle.speckle_arcgis import * 
+try:
+    from speckle.speckle.speckle_arcgis import *
+except:
+    from speckle_toolbox.esri.toolboxes.speckle.speckle.speckle_arcgis import *

speckle_toolbox/esri/toolboxes/Speckle.pyt.xml

@@ -1,77 +1,2 @@
-<?xml version="1.0"?>
-<metadata xml:lang="en"><Esri><CreaDate>20220718</CreaDate><CreaTime>13500100</CreaTime><ArcGISFormat>1.0</ArcGISFormat><SyncOnce>TRUE</SyncOnce><ModDate>20220826</ModDate><ModTime>150722</ModTime><scaleRange><minScale>150000000</minScale><maxScale>5000</maxScale></scaleRange><ArcGISProfile>ItemDescription</ArcGISProfile></Esri><toolbox name="Speckle" alias="speckle_toolbox_"><arcToolboxHelpPath>c:\program files\arcgis\pro\Resources\Help\gp</arcToolboxHelpPath><toolsets/></toolbox><dataIdInfo><idCitation><resTitle>Speckle</resTitle></idCitation><idPurp>Speckle connector for ArcGIS</idPurp><searchKeys><keyword>speckle3d</keyword></searchKeys></dataIdInfo><distInfo><distributor><distorFormat><formatName>ArcToolbox Toolbox</formatName></distorFormat></distributor></distInfo><mdHrLv><ScopeCd value="005"></ScopeCd></mdHrLv><Binary><Thumbnail><Data EsriPropertyType="PictureX">/9j/4AAQSkZJRgABAQEAYABgAAD/4gxYSUNDX1BST0ZJTEUAAQEAAAxITGlubwIQAABtbnRyUkdC
-IFhZWiAHzgACAAkABgAxAABhY3NwTVNGVAAAAABJRUMgc1JHQgAAAAAAAAAAAAAAAAAA9tYAAQAA
-AADTLUhQICAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABFj
-cHJ0AAABUAAAADNkZXNjAAABhAAAAGx3dHB0AAAB8AAAABRia3B0AAACBAAAABRyWFlaAAACGAAA
-ABRnWFlaAAACLAAAABRiWFlaAAACQAAAABRkbW5kAAACVAAAAHBkbWRkAAACxAAAAIh2dWVkAAAD
-TAAAAIZ2aWV3AAAD1AAAACRsdW1pAAAD+AAAABRtZWFzAAAEDAAAACR0ZWNoAAAEMAAAAAxyVFJD
-AAAEPAAACAxnVFJDAAAEPAAACAxiVFJDAAAEPAAACAx0ZXh0AAAAAENvcHlyaWdodCAoYykgMTk5
-OCBIZXdsZXR0LVBhY2thcmQgQ29tcGFueQAAZGVzYwAAAAAAAAASc1JHQiBJRUM2MTk2Ni0yLjEA
-AAAAAAAAAAAAABJzUkdCIElFQzYxOTY2LTIuMQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAWFlaIAAAAAAAAPNRAAEAAAABFsxYWVogAAAAAAAAAAAAAAAA
-AAAAAFhZWiAAAAAAAABvogAAOPUAAAOQWFlaIAAAAAAAAGKZAAC3hQAAGNpYWVogAAAAAAAAJKAA
-AA+EAAC2z2Rlc2MAAAAAAAAAFklFQyBodHRwOi8vd3d3LmllYy5jaAAAAAAAAAAAAAAAFklFQyBo
-dHRwOi8vd3d3LmllYy5jaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAABkZXNjAAAAAAAAAC5JRUMgNjE5NjYtMi4xIERlZmF1bHQgUkdCIGNvbG91ciBzcGFjZSAt
-IHNSR0IAAAAAAAAAAAAAAC5JRUMgNjE5NjYtMi4xIERlZmF1bHQgUkdCIGNvbG91ciBzcGFjZSAt
-IHNSR0IAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZGVzYwAAAAAAAAAsUmVmZXJlbmNlIFZpZXdpbmcg
-Q29uZGl0aW9uIGluIElFQzYxOTY2LTIuMQAAAAAAAAAAAAAALFJlZmVyZW5jZSBWaWV3aW5nIENv
-bmRpdGlvbiBpbiBJRUM2MTk2Ni0yLjEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHZpZXcAAAAA
-ABOk/gAUXy4AEM8UAAPtzAAEEwsAA1yeAAAAAVhZWiAAAAAAAEwJVgBQAAAAVx/nbWVhcwAAAAAA
-AAABAAAAAAAAAAAAAAAAAAAAAAAAAo8AAAACc2lnIAAAAABDUlQgY3VydgAAAAAAAAQAAAAABQAK
-AA8AFAAZAB4AIwAoAC0AMgA3ADsAQABFAEoATwBUAFkAXgBjAGgAbQByAHcAfACBAIYAiwCQAJUA
-mgCfAKQAqQCuALIAtwC8AMEAxgDLANAA1QDbAOAA5QDrAPAA9gD7AQEBBwENARMBGQEfASUBKwEy
-ATgBPgFFAUwBUgFZAWABZwFuAXUBfAGDAYsBkgGaAaEBqQGxAbkBwQHJAdEB2QHhAekB8gH6AgMC
-DAIUAh0CJgIvAjgCQQJLAlQCXQJnAnECegKEAo4CmAKiAqwCtgLBAssC1QLgAusC9QMAAwsDFgMh
-Ay0DOANDA08DWgNmA3IDfgOKA5YDogOuA7oDxwPTA+AD7AP5BAYEEwQgBC0EOwRIBFUEYwRxBH4E
-jASaBKgEtgTEBNME4QTwBP4FDQUcBSsFOgVJBVgFZwV3BYYFlgWmBbUFxQXVBeUF9gYGBhYGJwY3
-BkgGWQZqBnsGjAadBq8GwAbRBuMG9QcHBxkHKwc9B08HYQd0B4YHmQesB78H0gflB/gICwgfCDII
-RghaCG4IggiWCKoIvgjSCOcI+wkQCSUJOglPCWQJeQmPCaQJugnPCeUJ+woRCicKPQpUCmoKgQqY
-Cq4KxQrcCvMLCwsiCzkLUQtpC4ALmAuwC8gL4Qv5DBIMKgxDDFwMdQyODKcMwAzZDPMNDQ0mDUAN
-Wg10DY4NqQ3DDd4N+A4TDi4OSQ5kDn8Omw62DtIO7g8JDyUPQQ9eD3oPlg+zD88P7BAJECYQQxBh
-EH4QmxC5ENcQ9RETETERTxFtEYwRqhHJEegSBxImEkUSZBKEEqMSwxLjEwMTIxNDE2MTgxOkE8UT
-5RQGFCcUSRRqFIsUrRTOFPAVEhU0FVYVeBWbFb0V4BYDFiYWSRZsFo8WshbWFvoXHRdBF2UXiReu
-F9IX9xgbGEAYZRiKGK8Y1Rj6GSAZRRlrGZEZtxndGgQaKhpRGncanhrFGuwbFBs7G2MbihuyG9oc
-AhwqHFIcexyjHMwc9R0eHUcdcB2ZHcMd7B4WHkAeah6UHr4e6R8THz4faR+UH78f6iAVIEEgbCCY
-IMQg8CEcIUghdSGhIc4h+yInIlUigiKvIt0jCiM4I2YjlCPCI/AkHyRNJHwkqyTaJQklOCVoJZcl
-xyX3JicmVyaHJrcm6CcYJ0kneierJ9woDSg/KHEooijUKQYpOClrKZ0p0CoCKjUqaCqbKs8rAis2
-K2krnSvRLAUsOSxuLKIs1y0MLUEtdi2rLeEuFi5MLoIuty7uLyQvWi+RL8cv/jA1MGwwpDDbMRIx
-SjGCMbox8jIqMmMymzLUMw0zRjN/M7gz8TQrNGU0njTYNRM1TTWHNcI1/TY3NnI2rjbpNyQ3YDec
-N9c4FDhQOIw4yDkFOUI5fzm8Ofk6Njp0OrI67zstO2s7qjvoPCc8ZTykPOM9Ij1hPaE94D4gPmA+
-oD7gPyE/YT+iP+JAI0BkQKZA50EpQWpBrEHuQjBCckK1QvdDOkN9Q8BEA0RHRIpEzkUSRVVFmkXe
-RiJGZ0arRvBHNUd7R8BIBUhLSJFI10kdSWNJqUnwSjdKfUrESwxLU0uaS+JMKkxyTLpNAk1KTZNN
-3E4lTm5Ot08AT0lPk0/dUCdQcVC7UQZRUFGbUeZSMVJ8UsdTE1NfU6pT9lRCVI9U21UoVXVVwlYP
-VlxWqVb3V0RXklfgWC9YfVjLWRpZaVm4WgdaVlqmWvVbRVuVW+VcNVyGXNZdJ114XcleGl5sXr1f
-D19hX7NgBWBXYKpg/GFPYaJh9WJJYpxi8GNDY5dj62RAZJRk6WU9ZZJl52Y9ZpJm6Gc9Z5Nn6Wg/
-aJZo7GlDaZpp8WpIap9q92tPa6dr/2xXbK9tCG1gbbluEm5rbsRvHm94b9FwK3CGcOBxOnGVcfBy
-S3KmcwFzXXO4dBR0cHTMdSh1hXXhdj52m3b4d1Z3s3gReG54zHkqeYl553pGeqV7BHtje8J8IXyB
-fOF9QX2hfgF+Yn7CfyN/hH/lgEeAqIEKgWuBzYIwgpKC9INXg7qEHYSAhOOFR4Wrhg6GcobXhzuH
-n4gEiGmIzokziZmJ/opkisqLMIuWi/yMY4zKjTGNmI3/jmaOzo82j56QBpBukNaRP5GokhGSepLj
-k02TtpQglIqU9JVflcmWNJaflwqXdZfgmEyYuJkkmZCZ/JpomtWbQpuvnByciZz3nWSd0p5Anq6f
-HZ+Ln/qgaaDYoUehtqImopajBqN2o+akVqTHpTilqaYapoum/adup+CoUqjEqTepqaocqo+rAqt1
-q+msXKzQrUStuK4trqGvFq+LsACwdbDqsWCx1rJLssKzOLOutCW0nLUTtYq2AbZ5tvC3aLfguFm4
-0blKucK6O7q1uy67p7whvJu9Fb2Pvgq+hL7/v3q/9cBwwOzBZ8Hjwl/C28NYw9TEUcTOxUvFyMZG
-xsPHQce/yD3IvMk6ybnKOMq3yzbLtsw1zLXNNc21zjbOts83z7jQOdC60TzRvtI/0sHTRNPG1EnU
-y9VO1dHWVdbY11zX4Nhk2OjZbNnx2nba+9uA3AXcit0Q3ZbeHN6i3ynfr+A24L3hROHM4lPi2+Nj
-4+vkc+T85YTmDeaW5x/nqegy6LzpRunQ6lvq5etw6/vshu0R7ZzuKO6070DvzPBY8OXxcvH/8ozz
-GfOn9DT0wvVQ9d72bfb794r4Gfio+Tj5x/pX+uf7d/wH/Jj9Kf26/kv+3P9t////2wBDAAMCAgMC
-AgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIU
-FRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQU
-FBQUFBQUFBQUFBQUFBT/wAARCAAgACADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAEC
-AwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0Kx
-wRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1
-dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ
-2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QA
-tREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYk
-NOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaH
-iImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq
-8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD9Kfif8SNJ+EngjUPFeui4OlWLwrObWMPIolmSINtJGQDI
-Ccc4BwCcA8rbftG+F7y3iuLe31CeCVBJHLGkTK6kZDAiTBBHeuZ/bq/5NW8b/wDbj/6XW9fmzpP/
-ACC7P/rin/oIr9T4V4XwueYKVetJqSk18rRfl3Z8NxLnlfJfZypK/MfpTq37XHhmxvpbeFLVhGdj
-C71KKGRWHDKUG7GD716V8OfihoHxP0l7vRr+2uZ4Nq3drDOsj27HOA209Dg4bvg9CCB+S1fZf/BO
-v/moH/cP/wDbmvX4j4OwOV5XUxlBvmhy/O8ktfvvofO5DxVjMxzKGFrJcs7/ACsm/wBD1H9ur/k1
-bxv/ANuP/pdb1+bOk/8AILs/+uKf+giv2Y1DT7XVrC5sb62hvLK5iaGe2uIxJHLGwIZGU8MpBIIP
-BBrw/Rf2JPhRpLXQk0e81GGWTdDDdahMFtU7RxmNlJUDAy5ZuOWNeTwlxVg8jwlShioybcuZctne
-6Stq1ta+/wDwfoOKMjxOcqlHDNK173dvyTPzhr7L/wCCdf8AzUD/ALh//tzXsv8Awx58If8AoUf/
-ACpXn/x6uz+HXwd8IfCf+0P+EV0j+yv7Q8v7T/pM03mbN2z/AFjtjG9umOtepxDxpl+bZZVwVCE1
-KXLa6jbSSfST7djwMi4Sx2WZjSxdacHGN72bvrFrrFd+5//Z</Data></Thumbnail></Binary><mdDateSt Sync="TRUE">20220725</mdDateSt></metadata>
+<?xml version="1.0" encoding="UTF-8"?>
+<metadata xml:lang="en"><Esri><CreaDate>20231206</CreaDate><CreaTime>23391500</CreaTime><ArcGISFormat>1.0</ArcGISFormat><SyncOnce>TRUE</SyncOnce><ModDate>20231206</ModDate><ModTime>233915</ModTime></Esri><toolbox name="Speckle" alias="speckle_toolbox_"><arcToolboxHelpPath>c:\program files\arcgis\pro\Resources\Help\gp</arcToolboxHelpPath><toolsets/></toolbox><dataIdInfo><idCitation><resTitle>Speckle</resTitle></idCitation></dataIdInfo><distInfo><distributor><distorFormat><formatName>ArcToolbox Toolbox</formatName></distorFormat></distributor></distInfo></metadata>

speckle_toolbox/esri/toolboxes/speckle/converter/geometry/_init_.py

@@ -1,117 +0,0 @@
-
-from regex import F
-from specklepy.objects import Base
-from specklepy.objects.geometry import Line, Mesh, Point, Polyline, Curve, Arc, Circle, Polycurve
-
-import arcpy 
-from typing import Any, List, Union, Sequence
-from speckle.converter.geometry.polygon import polygonToNative, polygonToSpeckle
-from speckle.converter.geometry.polyline import arcToNative, circleToNative, curveToNative, lineToNative, polycurveToNative, polylineFromVerticesToSpeckle, polylineToNative, polylineToSpeckle
-from speckle.converter.geometry.point import pointToCoord, pointToNative, pointToSpeckle, multiPointToSpeckle
-
-
-def convertToSpeckle(feature, layer, geomType, featureType) -> Union[Base, Sequence[Base], None]:
-    """Converts the provided layer feature to Speckle objects"""
-    print("___convertToSpeckle____________")
-    geom = feature
-    #print(geom.isMultipart) # e.g. False 
-    geomMultiType = geom.isMultipart
-    hasCurves = feature.hasCurves 
-    
-    #print(featureType) # e.g. Simple 
-    #print(geomType) # e.g. Polygon 
-    #geomSingleType = (featureType=="Simple") # Simple,SimpleJunction,SimpleJunction,ComplexEdge,Annotation,CoverageAnnotation,Dimension,RasterCatalogItem 
-
-    if geomType == "Point": #Polygon, Point, Polyline, Multipoint, MultiPatch
-        for pt in geom:
-            return pointToSpeckle(pt, feature, layer)
-    elif geomType == "Polyline":
-        return polylineToSpeckle(geom, feature, layer, geomMultiType)
-    elif geomType == "Polygon":
-        return polygonToSpeckle(geom, feature, layer, geomMultiType)
-    elif geomType == "Multipoint":
-        return multiPointToSpeckle(geom, feature, layer, geomMultiType)
-    else:
-        arcpy.AddWarning("Unsupported or invalid geometry in layer " + layer.name)
-    return None
-
-
-def convertToNative(base: Base, sr: arcpy.SpatialReference) -> Union[Any, None]:
-    """Converts any given base object to QgsGeometry."""
-    print("___Convert to Native SingleType___")
-    #print(base)
-    converted = None
-    conversions = [
-        (Point, pointToNative),
-        (Line, lineToNative),
-        (Polyline, polylineToNative),
-        (Curve, curveToNative),
-        (Arc, arcToNative),
-        (Circle, circleToNative),
-        #(Mesh, meshToNative),
-        (Polycurve, polycurveToNative),
-        (Base, polygonToNative), # temporary solution for polygons (Speckle has no type Polygon yet)
-    ]
-
-    for conversion in conversions:
-        if isinstance(base, conversion[0]):
-            #print(conversion[0])
-            converted = conversion[1](base, sr)
-            break
-    #print(converted)
-    return converted
-
-def multiPointToNative(items: List[Point], sr: arcpy.SpatialReference):
-    print("___Create MultiPoint")
-    all_pts = []
-    # example https://pro.arcgis.com/en/pro-app/2.8/arcpy/classes/multipoint.htm
-    for item in items:
-        pt = pointToCoord(item) # [x, y, z]
-        all_pts.append( arcpy.Point(pt[0], pt[1], pt[2]) )
-    #print(all_pts)
-    features = arcpy.Multipoint( arcpy.Array(all_pts) )
-    #if len(features)==0: features = None
-    return features
-
-def multiPolylineToNative(items: List[Polyline], sr: arcpy.SpatialReference):
-    print("_______Drawing Multipolylines____")
-    #print(items)
-    poly = None
-
-    return poly
-
-def multiPolygonToNative(items: List[Base], sr: arcpy.SpatialReference): #TODO fix multi features
-    
-    print("_______Drawing Multipolygons____")
-    #print(items)
-    for item in items: # will be 1 item
-        #print(item)
-        pts = [pointToCoord(pt) for pt in item["boundary"].as_points()]
-        outer_arr = [arcpy.Point(*coords) for coords in pts]
-        outer_arr.append(outer_arr[0])
-        list_of_arrs = []
-        try:
-            for void in item["voids"]: 
-                #print(void)
-                pts = [pointToCoord(pt) for pt in void.as_points()]
-                #print(pts)
-                inner_arr = [arcpy.Point(*coords) for coords in pts]
-                inner_arr.append(inner_arr[0])
-                list_of_arrs.append(arcpy.Array(inner_arr))
-        except:pass
-    
-    list_of_arrs.insert(0, arcpy.Array(outer_arr))
-    array = arcpy.Array(list_of_arrs)
-    polygon = arcpy.Polygon(array, sr, has_z=True)
-
-    return polygon
-
-def convertToNativeMulti(items: List[Base], sr: arcpy.SpatialReference): 
-    print("___Convert to Native MultiType___")
-    first = items[0]
-    if isinstance(first, Point):
-        return multiPointToNative(items, sr)
-    elif isinstance(first, Line) or isinstance(first, Polyline):
-        return multiPolylineToNative(items, sr)
-    elif first["boundary"] is not None and first["voids"] is not None:
-        return multiPolygonToNative(items, sr)

speckle_toolbox/esri/toolboxes/speckle/converter/geometry/mesh.py

@@ -1,11 +0,0 @@
-from specklepy.objects.geometry import Mesh
-
-
-def meshToNative(mesh: Mesh):
-    """Converts a Speckle Mesh to QgsGeometry. Currently UNSUPPORTED"""
-    return None
-
-def rasterToMesh(vertices, faces, colors):
-    mesh = Mesh.create(vertices, faces, colors)
-    mesh.units = "m"
-    return mesh

speckle_toolbox/esri/toolboxes/speckle/converter/geometry/point.py

@@ -1,70 +0,0 @@
-import math
-from typing import List
-from specklepy.objects.geometry import Point
-import arcpy
-
-from speckle.converter.layers.utils import get_scale_factor
-
-
-def multiPointToSpeckle(geom, feature, layer, multiType: bool):
-    """Converts a Point to Speckle"""
-    #try: 
-    #print("___Point to Speckle____")
-    #point = Point(units = "m")
-    pointList = []
-    #print(geom) # <geoprocessing describe geometry object object at 0x0000020F1D94AB10>
-    #print(multiType)
-
-    if multiType is False: 
-        for pt in geom:
-            #print(pt) # 284394.58100903 5710688.11602606 NaN NaN <class 'arcpy.arcobjects.arcobjects.Point'> 
-            #print(type(pt))
-            if pt != None: pointList.append(pointToSpeckle(pt, feature, layer)) 
-    return pointList
-
-def pointToSpeckle(pt, feature, layer):
-  
-    """Converts a Point to Speckle"""
-    #print("___Point to Speckle____")
-    # when unset, z() returns "nan"
-    #print(pt) # 4.9046319 52.3592043 NaN NaN
-    #print("____Point to Speckle___")
-    x = pt.X
-    y = pt.Y
-    if pt.Z: z = pt.Z 
-    else: z = 0
-    specklePoint = Point(units = "m")
-    specklePoint.x = x
-    specklePoint.y = y
-    specklePoint.z = z
-    '''
-    col = featureColorfromNativeRenderer(feature, layer)
-    specklePoint['displayStyle'] = {}
-    specklePoint['displayStyle']['color'] = col
-    '''
-    #print(specklePoint)
-    return specklePoint
-
-def pointToNative(pt: Point, sr: arcpy.SpatialReference) -> arcpy.PointGeometry:
-    """Converts a Speckle Point to QgsPoint"""
-    print("___pointToNative__")
-    #print(pt)
-    pt = scalePointToNative(pt, pt.units)
-    geom = arcpy.PointGeometry(arcpy.Point(pt.x, pt.y, pt.z), sr, has_z = True)
-    #print(geom)
-    return geom
-
-def pointToCoord(pt: Point) -> List[float]:
-    """Converts a Speckle Point to QgsPoint"""
-    pt = scalePointToNative(pt, pt.units)
-    coords = [pt.x, pt.y, pt.z]
-    #print(coords)
-    return coords
-
-def scalePointToNative(pt: Point, units: str) -> Point:
-    """Scale point coordinates to meters"""
-    scaleFactor = get_scale_factor(units)
-    pt.x = pt.x * scaleFactor
-    pt.y = pt.y * scaleFactor
-    pt.z = 0 if math.isnan(pt.z) else pt.z * scaleFactor
-    return pt

speckle_toolbox/esri/toolboxes/speckle/converter/geometry/polygon.py

@@ -1,188 +0,0 @@
-from typing import Sequence
-import arcpy 
-import json 
-
-from specklepy.objects import Base
-from specklepy.objects.geometry import Point
-from speckle.converter.geometry.mesh import rasterToMesh
-from speckle.converter.geometry.point import pointToCoord
-from speckle.converter.geometry.polyline import polylineFromVerticesToSpeckle, circleToSpeckle
-
-import math
-from panda3d.core import Triangulator
-
-
-def polygonToSpeckle(geom, feature, layer, multiType: bool):
-    """Converts a Polygon to Speckle"""
-    #try: 
-    print("___Polygon to Speckle____")
-    polygon = Base(units = "m")
-    pointList = []
-    voidPointList = []
-    voids = []
-    boundary = None
-    
-    if geom.hasCurves: 
-        # geometry SHAPE@ tokens: https://pro.arcgis.com/en/pro-app/latest/arcpy/get-started/reading-geometries.htm
-        print(geom.JSON) 
-        # look for "curvePaths" or "curveRings"[[ (startPt, {arcs, beziers etc}, optional(endPt))],[],...], "rings" 
-        # examples: https://developers.arcgis.com/documentation/common-data-types/geometry-objects.htm
-        # e.g. {"hasZ":true,"curveRings":[[[631307.05960000027,5803698.4477999993,0],{"a":[[631307.05960000027,5803698.4477999993,0],[631307.05960000027,5803414.92656173],0,1]}]],"spatialReference":{"wkid":32631,"latestWkid":32631}}
-        # b - bezier curve (endPt, controlPts) 
-        # a - elliptical arc (endPt, centralPt)
-        # c - circular arc (endPt, throughPt) 
-
-        #startPtCoords = geom.JSON.curveRings[0][0]
-        r'''
-        segments = []
-        for key, val in json.loads(geom.JSON).items(): 
-            if key == "curveRings": 
-                for segm in val:
-                    print(segm)
-                    segmStartCoord = segm[0]
-                    print(segmStartCoord) 
-
-                    segmData = segm[1]
-                    for key2, val2 in segmData.items():
-                        if key2 == "a":
-                            segmToCoord = val2[0] # elliptical arc 
-                            segmCenter = val2[1]
-                            print(segmToCoord)
-                            print(segmCenter)
-                            if segmStartCoord == segmToCoord: 
-                                print("full circle") 
-                                boundary = circleToSpeckle(segmCenter, segmToCoord, layer)
-
-                            try: 
-                                segmToCoord = segm[1].c # circular arc 
-                            except:
-                                try: 
-                                    segmToCoord = segm[1].b # bezier curve 
-                                except: pass
-
-                    segmEndCoord = None
-                    if len(segm)>2:
-                        segmEndCoord = segm[2]
-        '''
-
-    if multiType is False: 
-        print("single type")
-        for p in geom:
-            for pt in p: 
-                if pt != None: pointList.append(pt) 
-        boundary = polylineFromVerticesToSpeckle(pointList, True, feature, layer) 
-    else: 
-        print("multi type")
-        for i, p in enumerate(geom):
-            for pt in p:  
-                #print(pt) # 284394.58100903 5710688.11602606 NaN NaN
-                if pt == None and boundary == None:  # first break 
-                    boundary = polylineFromVerticesToSpeckle(pointList, True, feature, layer) 
-                    pointList = []
-                elif pt == None and boundary != None: # breaks btw voids
-                    void = polylineFromVerticesToSpeckle(pointList, True, feature, layer)
-                    voids.append(void)
-                    pointList = []
-                elif pt != None: # add points to whatever list (boundary or void) 
-                    pointList.append(pt)
-
-            if boundary != None and len(pointList)>0: # remaining polyline
-                void = polylineFromVerticesToSpeckle(pointList, True, feature, layer)
-                voids.append(void)
-
-    polygon.boundary = boundary
-    polygon.voids = voids
-    polygon.displayValue = [ boundary ] + voids
-
-    ############# mesh 
-    vertices = []
-    total_vertices = 0
-    polyBorder = boundary.as_points()
-
-    #print(polyBorder)
-
-    if len(polyBorder)>2:
-        print("make meshes from polygons")
-        if len(voids) == 0: # if there is a mesh with no voids
-            for pt in polyBorder:
-                x = pt.x
-                y = pt.y
-                z = 0 if math.isnan(pt.z) else pt.z
-                vertices.extend([x, y, z])
-                total_vertices += 1
-            #print(vertices)
-            ran = range(0, total_vertices)
-            faces = [total_vertices]
-            faces.extend([i for i in ran])
-            #print(faces)
-            # else: https://docs.panda3d.org/1.10/python/reference/panda3d.core.Triangulator
-        else:
-            trianglator = Triangulator()
-            faces = []
-
-            # add boundary points
-            #polyBorder = boundary.as_points()
-            pt_count = 0
-            # add extra middle point for border
-            for pt in polyBorder:
-              if pt_count < len(polyBorder)-1: 
-                  pt2 = polyBorder[pt_count+1]
-              else: pt2 = polyBorder[0]
-              
-              trianglator.addPolygonVertex(trianglator.addVertex(pt.x, pt.y))
-              vertices.extend([pt.x, pt.y, pt.z])
-              trianglator.addPolygonVertex(trianglator.addVertex((pt.x+pt2.x)/2, (pt.y+pt2.y)/2))
-              vertices.extend([(pt.x+pt2.x)/2, (pt.y+pt2.y)/2, (pt.z+pt2.z)/2])
-              total_vertices += 2
-              pt_count += 1
-
-            #add void points
-            for i in range(len(voids)):
-              trianglator.beginHole()
-              pts = voids[i].as_points()
-              for pt in pts:
-                trianglator.addHoleVertex(trianglator.addVertex(pt.x, pt.y))
-                vertices.extend([pt.x, pt.y, pt.z])
-                total_vertices += 1
-
-            trianglator.triangulate()
-            i = 0
-            while i < trianglator.getNumTriangles():
-              tr = [trianglator.getTriangleV0(i),trianglator.getTriangleV1(i),trianglator.getTriangleV2(i)]
-              faces.extend([3, tr[0], tr[1], tr[2]])
-              i+=1
-            ran = range(0, total_vertices)
-        
-        #print(polygon)
-        col = (100<<16) + (100<<8) + 100 #featureColorfromNativeRenderer(feature, layer)
-        colors = [col for i in ran] # apply same color for all vertices
-        mesh = rasterToMesh(vertices, faces, colors)
-        polygon.displayValue = mesh 
-    #print("print resulted polygon")
-    #print(polygon)
-    return polygon
-
-def polygonToNative(poly: Base, sr: arcpy.SpatialReference) -> arcpy.Polygon:
-    """Converts a Speckle Polygon base object to QgsPolygon.
-    This object must have a 'boundary' and 'voids' properties.
-    Each being a Speckle Polyline and List of polylines respectively."""
-
-    print("_______Drawing polygons____")
-    pts = [pointToCoord(pt) for pt in poly["boundary"].as_points()]
-    outer_arr = [arcpy.Point(*coords) for coords in pts]
-    outer_arr.append(outer_arr[0])
-    list_of_arrs = []
-    try:
-        for void in poly["voids"]: 
-            #print(void)
-            pts = [pointToCoord(pt) for pt in void.as_points()]
-            #print(pts)
-            inner_arr = [arcpy.Point(*coords) for coords in pts]
-            inner_arr.append(inner_arr[0])
-            list_of_arrs.append(arcpy.Array(inner_arr))
-    except:pass
-    list_of_arrs.insert(0, outer_arr)
-    array = arcpy.Array(list_of_arrs)
-    polygon = arcpy.Polygon(array, sr, has_z=True)
-
-    return polygon

speckle_toolbox/esri/toolboxes/speckle/converter/geometry/polyline.py

@@ -1,205 +0,0 @@
-
-from math import atan, cos, sin
-import math
-from typing import List, Union
-from specklepy.objects.geometry import Point, Line, Polyline, Curve, Arc, Circle, Polycurve, Plane, Interval  
-import arcpy 
-
-from speckle.converter.geometry.point import pointToCoord, pointToNative, pointToSpeckle
-from speckle.converter.layers.utils import get_scale_factor
-
-def circleToSpeckle(center, point, layer):
-    print("___Circle to Speckle____")
-    rad = math.sqrt(math.pow((center[0] - point[0]),2) + math.pow((center[1] - point[1]),2) )
-    #print(rad)
-    if len(center)>2: center_z = center[2]
-    else: center_z = 0
-    length = rad*2*math.pi
-    domain = [0, length] 
-    plane = [center[0], center[1], center_z, 0,0,1, 1,0,0, 0,1,0] 
-    units = 3 #"m" 
-
-    args = [0] + [rad] + domain + plane + [units] 
-    #print(args) 
-    c = Circle().from_list(args)
-    #print(c)
-    return c 
-
-def polylineToSpeckle(geom, feature, layer, multiType: bool):
-    print("___Polyline to Speckle____")
-    polyline = None
-    pointList = []
-    #print(geom.hasCurves) 
-
-    if multiType is False: 
-        for p in geom: 
-            for pt in p: 
-                if pt != None: pointList.append(pt)#; print(pt.Z)
-        closed = False
-        if pointList[0] == pointList[len(pointList)-1]: 
-            closed = True
-            pointList = pointList[:-1]
-        polyline = polylineFromVerticesToSpeckle(pointList, closed, feature, layer) 
-
-    return polyline
-
-def polylineFromVerticesToSpeckle(vertices, closed, feature, layer):
-    """Converts a Polyline to Speckle"""
-    
-    print("___Polyline from vertices to Speckle____")
-    specklePts = []
-    for pt in vertices:
-        newPt = pointToSpeckle(pt, feature, layer) 
-        specklePts.append(newPt)
-
-    # TODO: Replace with `from_points` function when fix is pushed.
-    polyline = Polyline(units = "m")
-    polyline.value = []
-    polyline.closed = closed
-    polyline.units = specklePts[0].units
-    for i, point in enumerate(specklePts):
-        if closed and i == len(specklePts) - 1:
-            continue
-        polyline.value.extend([point.x, point.y, point.z])
-
-    return polyline
-
-
-def polylineToNative(poly: Polyline, sr: arcpy.SpatialReference) -> arcpy.Polyline:
-    """Converts a Speckle Polyline to QgsLineString"""
-    print("__ convert poly to native __")
-    pts = [pointToCoord(pt) for pt in poly.as_points()]
-    if poly.closed is True: 
-        pts.append( pointToCoord(poly.as_points()[0]) )
-
-    pts_coord_list = [arcpy.Point(*coords) for coords in pts]
-    polyline = arcpy.Polyline( arcpy.Array(pts_coord_list), sr, has_z=True )
-    #print(polyline.JSON)
-    return polyline
-
-
-def lineToNative(line: Line, sr: arcpy.SpatialReference) -> arcpy.Polyline:
-    """Converts a Speckle Line to QgsLineString"""
-    print("___Line to Native___")
-    pts = [pointToCoord(pt) for pt in [line.start, line.end]]
-    line = arcpy.Polyline( arcpy.Array([arcpy.Point(*coords) for coords in pts]), sr , has_z=True)
-    return line
-
-def curveToNative(poly: Curve, sr: arcpy.SpatialReference) -> arcpy.Polyline:
-    """Converts a Speckle Curve to QgsLineString"""
-    display = poly.displayValue
-    curve = polylineToNative(display, sr) 
-    return curve
-
-def arcToNative(poly: Arc, sr: arcpy.SpatialReference) -> arcpy.Polyline:
-    """Converts a Speckle Arc to QgsCircularString"""
-    arc = arcToNativePoints(poly, sr) #QgsCircularString(pointToNative(poly.startPoint), pointToNative(poly.midPoint), pointToNative(poly.endPoint))
-    return arc
-
-def circleToNative(poly: Circle, sr: arcpy.SpatialReference) -> arcpy.Polyline:
-    """Converts a Speckle Circle to QgsLineString"""
-    print("___Convert Circle from Native___")
-    points = []
-    angle1 = math.pi/2
-    
-    pointsNum = math.floor(math.pi*2) * 12 
-    if pointsNum <4: pointsNum = 4
-    points.append(pointToCoord(poly.plane.origin))
-
-    radScaled = poly.radius * get_scale_factor(poly.units)
-    points[0][1] += radScaled
-
-    for i in range(1, pointsNum + 1): 
-        k = i/pointsNum # to reset values from 1/10 to 1
-        if poly.plane.normal.z == 0: normal = 1
-        else: normal = poly.plane.normal.z
-        angle = angle1 + k * math.pi*2 * normal
-        pt = Point( x = poly.plane.origin.x + radScaled * cos(angle), y = poly.plane.origin.y + radScaled * sin(angle), z = 0) 
-        pt.units = "m"
-        points.append(pointToCoord(pt))
-    points.append(points[0])
-    curve = arcpy.Polyline( arcpy.Array([arcpy.Point(*coords) for coords in points]), sr , has_z=True)
-    return curve
-
-def polycurveToNative(poly: Polycurve, sr: arcpy.SpatialReference) -> arcpy.Polyline:
-    points = []
-    curve = None
-    print("___Polycurve to native___")
-    
-    try:
-        for segm in poly.segments: # Line, Polyline, Curve, Arc, Circle
-            #print(segm)
-            if isinstance(segm,Line):  converted = lineToNative(segm, sr) # QgsLineString
-            elif isinstance(segm,Polyline):  converted = polylineToNative(segm, sr) # QgsLineString
-            elif isinstance(segm,Curve):  converted = curveToNative(segm, sr) # QgsLineString
-            elif isinstance(segm,Circle):  converted = circleToNative(segm, sr) # QgsLineString
-            elif isinstance(segm,Arc):  converted = arcToNativePoints(segm, sr) # QgsLineString
-            else: # either return a part of the curve, of skip this segment and try next
-                arcpy.AddWarning(f"Part of the polycurve cannot be converted")
-                curve = arcpy.Polyline( arcpy.Array([arcpy.Point(*coords) for coords in points]), sr , has_z=True)
-                return curve
-            if converted is not None: 
-                #print(converted) # <geoprocessing describe geometry object object at 0x000002B2D3E338D0>
-                for part in converted:
-                    #print(part) # <geoprocessing array object object at 0x000002B2D2E09530>
-                    for pt in part: 
-                        #print(pt) # 64.4584221540162 5.5 NaN NaN
-                        if pt.Z != None: pt_z = pt.Z
-                        else: pt_z = 0
-                        #print(pt_z)
-                        #print(len(points)) 
-                        if len(points)>0 and pt.X == points[len(points)-1][0] and pt.Y == points[len(points)-1][1] and pt_z == points[len(points)-1][2]: pass
-                        else: points.append(pointToCoord(Point(x=pt.X, y = pt.Y, z = pt_z)))
-                        #print(points)
-            else:
-                arcpy.AddWarning(f"Part of the polycurve cannot be converted")
-                curve = arcpy.Polyline( arcpy.Array([arcpy.Point(*coords) for coords in points]), sr, has_z=True )
-                return curve
-    except: curve = None
-    #print(curve)
-    
-    curve = arcpy.Polyline( arcpy.Array([arcpy.Point(*coords) for coords in points]), sr, has_z=True )
-    return curve
-
-def arcToNativePoints(poly: Arc, sr: arcpy.SpatialReference):
-    print("__Arc to native__")
-    points = []
-    if poly.startPoint.x == poly.plane.origin.x: angle1 = math.pi/2
-    else: angle1 = atan( abs ((poly.startPoint.y - poly.plane.origin.y) / (poly.startPoint.x - poly.plane.origin.x) )) # between 0 and pi/2
-    #print(angle1)
-    if poly.plane.origin.x < poly.startPoint.x and poly.plane.origin.y > poly.startPoint.y: angle1 = 2*math.pi - angle1
-    if poly.plane.origin.x > poly.startPoint.x and poly.plane.origin.y > poly.startPoint.y: angle1 = math.pi + angle1
-    if poly.plane.origin.x > poly.startPoint.x and poly.plane.origin.y < poly.startPoint.y: angle1 = math.pi - angle1
-    print(angle1)
-
-    if poly.endPoint.x == poly.plane.origin.x: angle2 = math.pi/2
-    else: angle2 = atan( abs ((poly.endPoint.y - poly.plane.origin.y) / (poly.endPoint.x - poly.plane.origin.x) )) # between 0 and pi/2
-    #print(angle2)
-    if poly.plane.origin.x < poly.endPoint.x and poly.plane.origin.y > poly.endPoint.y: angle2 = 2*math.pi - angle2
-    if poly.plane.origin.x > poly.endPoint.x and poly.plane.origin.y > poly.endPoint.y: angle2 = math.pi + angle2
-    if poly.plane.origin.x > poly.endPoint.x and poly.plane.origin.y < poly.endPoint.y: angle2 = math.pi - angle2
-    print(angle2)
-
-    try: interval = math.floor(poly.endAngle - poly.startAngle)
-    except: interval = math.floor(angle2-angle1)
-    pointsNum = math.floor( abs(interval)) * 12
-    if pointsNum <4: pointsNum = 4
-    points.append(pointToCoord(poly.startPoint))
-    print(points)
-    print(interval)
-    print(pointsNum)
-    for i in range(1, pointsNum + 1): 
-        k = i/pointsNum # to reset values from 1/10 to 1
-        if poly.plane.normal.z == 0: normal = 1
-        else: normal = poly.plane.normal.z
-        angle = angle1 + k * interval * normal
-        print(f"k: {str(i)} multiplied: {str(k*interval)} angle: {str(angle1 + k * interval)}")
-        #print(cos(angle))
-        pt = Point( x = poly.plane.origin.x + poly.radius * cos(angle), y = poly.plane.origin.y + poly.radius * sin(angle), z = 0) 
-        pt.units = poly.startPoint.units 
-        points.append(pointToCoord(pt))
-        #print(pointToCoord(pt))
-    points.append(pointToCoord(poly.endPoint))
-    curve = arcpy.Polyline( arcpy.Array([arcpy.Point(*coords) for coords in points]), sr, has_z=True )
-    return curve
-

speckle_toolbox/esri/toolboxes/speckle/converter/layers/Layer.py

@@ -1,52 +0,0 @@
-from typing import List, Optional
-from specklepy.objects.base import Base
-
-from speckle.converter.layers.CRS import CRS
-
-
-class Layer(Base, chunkable={"features": 100}):
-    """A GIS Layer"""
-
-    def __init__(
-        self,
-        name: Optional[str] = None,
-        crs: Optional[CRS] = None,
-        datum: Optional[CRS] = None,
-        features: List[Base] = [],
-        layerType: str = "None",
-        geomType: str = "None",
-        renderer: dict = {},
-        **kwargs
-    ) -> None:
-        super().__init__(**kwargs)
-        self.name = name
-        self.crs = crs
-        self.datum = datum
-        self.type = layerType
-        self.features = features
-        self.geomType = geomType
-        self.renderer = renderer 
-
-class RasterLayer(Base, chunkable={"features": 100}):
-    """A GIS Layer"""
-
-    def __init__(
-        self,
-        name=None,
-        crs=None,
-        rasterCrs=None,
-        features: List[Base] = [],
-        layerType: str = "None",
-        geomType: str = "None",
-        renderer: dict = {},
-        **kwargs
-    ) -> None:
-        super().__init__(**kwargs)
-        self.name = name
-        self.crs = crs
-        self.rasterCrs = rasterCrs
-        self.type = layerType
-        self.features = features
-        self.geomType = geomType
-        self.renderer = renderer 
-    

speckle_toolbox/esri/toolboxes/speckle/converter/layers/_init_.py

@@ -1,492 +0,0 @@
-"""
-Contains all Layer related classes and methods.
-"""
-import os
-from typing import Any, List, Union
-
-from regex import D
-from speckle.converter.layers.CRS import CRS
-from speckle.converter.layers.Layer import Layer, RasterLayer
-from speckle.converter.layers.feature import featureToNative, featureToSpeckle, cadFeatureToNative
-from specklepy.objects import Base
-
-import arcgisscripting
-import pandas as pd
-import arcpy
-from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
-from arcpy.management import (CreateFeatureclass, MakeFeatureLayer,
-                              AddFields, AlterField, DefineProjection )
-
-from speckle.converter.layers.utils import getLayerAttributes
-
-
-def convertSelectedLayers(all_layers: List[arcpy._mp.Layer], selected_layers: List[str], project: arcpy.mp.ArcGISProject) -> List[Layer]:
-    """Converts the current selected layers to Speckle"""
-    print("________Convert Layers_________")
-    result = []
-    for layer in selected_layers:
-        layerToSend = None
-        for c in range(len(all_layers)):
-            if int(layer.split("-",1)[0]) == c:
-                layerToSend = all_layers[c]
-                break
-        if layerToSend is not None: 
-            ds = layerToSend.dataSource #file path
-            if layerToSend.isFeatureLayer: 
-                newBaseLayer = layerToSpeckle(layerToSend, project)
-                if newBaseLayer is not None: result.append(newBaseLayer)
-
-            elif layerToSend.isRasterLayer: pass
-            '''
-            if layer.name() in selectedLayerNames:
-                result.append(layerToSpeckle(layer, projectCRS, project))
-            '''
-    #print(result)
-    return result
-
-def layerToSpeckle(layer: arcLayer, project: ArcGISProject) -> Layer: #now the input is QgsVectorLayer instead of qgis._core.QgsLayerTreeLayer
-    """Converts a given QGIS Layer to Speckle"""
-    print("________Convert Feature Layer_________")
-
-    projectCRS = project.activeMap.spatialReference
-    try: data = arcpy.Describe(layer.dataSource)
-    except OSError as e: arcpy.AddWarning(str(e.args[0])); return
-    #print(projectCRS)
-    #print(projectCRS.name)
-    crs = CRS(name = projectCRS.name, wkt = projectCRS.exportToString(), units = "m")
-    
-    layer_geo_crs = None
-    datum = None
-    #if data.spatialReference.type == "Projected": 
-    #    #layer_geo_crs =  
-    #    datum = CRS(name = layer_geo_crs.name, wkt = layer_geo_crs.exportToString(), units = "m")
-    
-    speckleLayer = Layer()
-    speckleLayer.type="VectorLayer"
-    speckleLayer.name = layer.name
-    speckleLayer.crs = crs
-    speckleLayer.datum = datum
-    
-    try: # https://pro.arcgis.com/en/pro-app/2.8/arcpy/get-started/the-spatial-reference-object.htm
-        layerObjs = []
-        print(data.datasetType)
-        if data.datasetType == "FeatureClass": #FeatureClass, ?Table Properties, ?Datasets
-            # write feature attributes
-            fieldnames = [field.name for field in data.fields]
-            #print(layer.longName) # e.g. 17b0b76d13_custom_crs_04dcfaa936\04dcfaa936_Vector_lineGeom
-            #print(fieldnames) # e.g. ['OBJECTID', 'Shape', 'Shape_Length', 'Speckle_ID', 'number', 'area']
-            rows_shapes = arcpy.da.SearchCursor(layer.longName, "Shape@") # arcpy.da.SearchCursor(in_table, field_names, {where_clause}, {spatial_reference}, {explode_to_points}, {sql_clause})
-            #print(rows_shapes) # <da.SearchCursor object at 0x00000172565E6C10>
-            print("__ start iterating features")
-            # write feature attributes
-            for i, features in enumerate(rows_shapes):
-                print("____Feature # " + str(i+1))
-                if features[0] == None: continue 
-                #print(features[0].hasCurves)
-                if features[0].hasCurves: continue 
-                rows_attributes = arcpy.da.SearchCursor(layer.longName, fieldnames)
-                row_attr = []
-                for k, attrs in enumerate(rows_attributes):
-                    if i == k: row_attr = attrs; break
-
-                #print(features) #(<Polygon object at 0x172592ae8c8[0x17258d2a600]>,)
-                #print(features[0].pointCount)
-                #print(features[0].partCount)
-                if features[0]:
-                    b = featureToSpeckle(fieldnames, row_attr, features[0], projectCRS, project, layer)
-                    layerObjs.append(b)
-                    #print(layerObjs)
-                
-            print("__ finish iterating features")
-            speckleLayer.features=layerObjs
-            speckleLayer.geomType = data.shapeType
-
-    except OSError as e: 
-        arcpy.AddWarning(str(e))
-        return
-
-    return speckleLayer
-
-def layerToNative(layer: Union[Layer, RasterLayer], streamBranch: str, project: ArcGISProject):
-
-    if layer.type is None:
-        # Handle this case
-        return
-    elif layer.type.endswith("VectorLayer"):
-        return vectorLayerToNative(layer, streamBranch, project)
-    elif layer.type.endswith("RasterLayer"):
-        return rasterLayerToNative(layer, streamBranch, project)
-    return None
-
-def cadLayerToNative(layerContentList:Base, layerName: str, streamBranch: str, project: ArcGISProject) :
-
-    geom_points = []
-    geom_polylines = []
-    geom_polygones = []
-    geom_meshes = []
-    #filter speckle objects by type within each layer, create sub-layer for each type (points, lines, polygons, mesh?)
-    for geom in layerContentList:
-        #print(geom)
-        if geom.speckle_type == "Objects.Geometry.Point": 
-            geom_points.append(geom)
-        if geom.speckle_type == "Objects.Geometry.Line" or geom.speckle_type == "Objects.Geometry.Polyline" or geom.speckle_type == "Objects.Geometry.Curve" or geom.speckle_type == "Objects.Geometry.Arc" or geom.speckle_type == "Objects.Geometry.Circle" or geom.speckle_type == "Objects.Geometry.Polycurve":
-            geom_polylines.append(geom)
-    
-    if len(geom_points)>0: layer_points = cadVectorLayerToNative(geom_points, layerName, "Points", streamBranch, project)
-    if len(geom_polylines)>0: layer_polylines = cadVectorLayerToNative(geom_polylines, layerName, "Polylines", streamBranch, project)
-
-    return [layer_points, layer_polylines]
-
-def cadVectorLayerToNative(geomList, layerName: str, geomType: str, streamBranch: str, project: ArcGISProject): 
-    print("_________CAD vector layer to native_____")
-    #get Project CRS, use it by default for the new received layer
-    vl = None
-    layerName = layerName.replace("[","_").replace("]","_").replace(" ","_").replace("-","_").replace("(","_").replace(")","_").replace(":","_").replace("\\","_").replace("/","_").replace("\"","_").replace("&","_").replace("@","_").replace("$","_").replace("%","_").replace("^","_")
-    layerName = layerName + "_" + geomType
-    
-    sr = arcpy.SpatialReference(project.activeMap.spatialReference.name)
-    active_map = project.activeMap
-    path = project.filePath.replace("aprx","gdb") #"\\".join(project.filePath.split("\\")[:-1]) + "\\speckle_layers\\" #arcpy.env.workspace + "\\" #
-    
-    if sr.type == "Geographic": 
-        arcpy.AddMessage(f"Project CRS is set to Geographic type, and objects in linear units might not be received correctly")
-
-    #CREATE A GROUP "received blabla" with sublayers
-    layerGroup = None
-    newGroupName = f'{streamBranch}'
-    #print(newGroupName)
-    for l in active_map.listLayers():
-        if l.longName == newGroupName: layerGroup = l; break 
-    
-    #find ID of the layer with a matching name in the "latest" group 
-    newName = f'{streamBranch.split("_")[len(streamBranch.split("_"))-1]}_{layerName}'
-
-    all_layer_names = []
-    layerExists = 0
-    for l in project.activeMap.listLayers(): 
-        if l.longName.startswith(newGroupName + "\\"):
-            all_layer_names.append(l.longName)
-    #print(all_layer_names)
-
-    longName = streamBranch + "\\" + newName 
-    if longName in all_layer_names: 
-        for index, letter in enumerate('234567890abcdefghijklmnopqrstuvwxyz'):
-            if (longName + "_" + letter) not in all_layer_names: newName += "_"+letter; layerExists +=1; break 
-
-    # particularly if the layer comes from ArcGIS
-    if "polygon" in geomType.lower(): geomType = "Polygon"
-    if "line" in geomType.lower(): geomType = "Polyline"
-    if "multipoint" in geomType.lower(): geomType = "Multipoint"
-    elif "point" in geomType.lower(): geomType = "Point"
-    #print(geomType)
-    
-    #print(newName)
-    #path = r"C:\Users\username\Documents\ArcGIS\Projects\MyProject-test\MyProject-test.gdb\\"
-    #https://community.esri.com/t5/arcgis-pro-questions/is-it-possible-to-create-a-new-group-layer-with/td-p/1068607
-
-    print("_________create feature class (cad)___________________________________")
-    # should be created inside the workspace to be a proper Feature class (not .shp) with Nullable Fields
-    class_name = ("f_class_" + newName)
-    f_class = CreateFeatureclass(path, class_name, geomType, has_z="ENABLED", spatial_reference = sr)
-    #print(f_class)
-    #print(geomList)
-
-    # get and set Layer attribute fields
-    # example: https://resource.esriuk.com/blog/an-introductory-slice-of-arcpy-in-arcgis-pro/
-    newFields = getLayerAttributes(geomList)
-    
-    fields_to_ignore = ["arcgisgeomfromspeckle", "shape", "objectid"]
-    matrix = []
-    all_keys = []
-    all_key_types = []
-    max_len = 52
-    for key, value in newFields.items(): 
-        existingFields = [fl.name for fl in arcpy.ListFields(class_name)]
-        if key not in existingFields  and key.lower() not in fields_to_ignore: # exclude geometry and default existing fields
-            # signs that should not be used as field names and table names: https://support.esri.com/en/technical-article/000005588
-            key = key.replace(" ","_").replace("-","_").replace("(","_").replace(")","_").replace(":","_").replace("\\","_").replace("/","_").replace("\"","_").replace("&","_").replace("@","_").replace("$","_").replace("%","_").replace("^","_") 
-            if key[0] in ['0','1','2','3','4','5','6','7','8','9']: key = "_"+key
-            if len(key)>max_len: key = key[:max_len]
-            #print(all_keys)
-            if key in all_keys:
-                for index, letter in enumerate('1234567890abcdefghijklmnopqrstuvwxyz'):
-                    if len(key)<max_len and (key+letter) not in all_keys: key+=letter; break 
-                    if len(key) == max_len and (key[:9] + letter) not in all_keys: key=key[:9] + letter; break 
-            if key not in all_keys: 
-                all_keys.append(key)
-                all_key_types.append(value)
-                #print(all_keys)
-                matrix.append([key, value, key, 255])
-                #print(matrix)
-    if len(matrix)>0: AddFields(str(f_class), matrix)
-
-    fets = []
-    for f in geomList[:]: 
-        new_feat = cadFeatureToNative(f, newFields, sr)
-        if new_feat != "" and new_feat != None: 
-            fets.append(new_feat)
-    #print("features created")
-    #print(fets)
-
-    count = 0
-    rowValues = []
-    for feat in fets:
-        try: feat['applicationId'] 
-        except: feat.update({'applicationId': count})
-
-        row = [feat['arcGisGeomFromSpeckle'], feat['applicationId']]
-        heads = [ 'Shape@', 'OBJECTID']
-
-        for key,value in feat.items(): 
-            if key in all_keys and key.lower() not in fields_to_ignore: 
-                heads.append(key)
-                row.append(value)
-        rowValues.append(row)
-        count += 1
-    #print(heads)
-    cur = arcpy.da.InsertCursor(str(f_class), tuple(heads) )
-    for row in rowValues: 
-        #print(tuple(heads))
-        #print(tuple(row))
-        cur.insertRow(tuple(row))
-    del cur 
-    #print(f_class)
-    vl = MakeFeatureLayer(str(f_class), newName).getOutput(0)
-
-    #adding layers from code solved: https://gis.stackexchange.com/questions/344343/arcpy-makefeaturelayer-management-function-not-creating-feature-layer-in-arcgis
-    #active_map.addLayer(new_layer)
-    active_map.addLayerToGroup(layerGroup, vl)
-
-    return vl
-
-def vectorLayerToNative(layer: Layer, streamBranch: str, project: ArcGISProject):
-    print("_________Vector Layer to Native_________")
-    vl = None
-    layerName = layer.name.replace(" ","_").replace("-","_").replace("(","_").replace(")","_").replace(":","_").replace("\\","_").replace("/","_").replace("\"","_").replace("&","_").replace("@","_").replace("$","_").replace("%","_").replace("^","_")
-    
-    print(layerName)
-    sr = arcpy.SpatialReference(text=layer.crs.wkt) 
-    active_map = project.activeMap
-    path = project.filePath.replace("aprx","gdb") #"\\".join(project.filePath.split("\\")[:-1]) + "\\speckle_layers\\" #arcpy.env.workspace + "\\" #
-    #if not os.path.exists(path): os.makedirs(path)
-    #print(path) 
-
-    #CREATE A GROUP "received blabla" with sublayers
-    layerGroup = None
-    newGroupName = f'{streamBranch}'
-    #print(newGroupName)
-    for l in active_map.listLayers():
-        if l.longName == newGroupName: layerGroup = l; break 
-    
-    #find ID of the layer with a matching name in the "latest" group 
-    newName = f'{streamBranch.split("_")[len(streamBranch.split("_"))-1]}_{layerName}'
-
-    all_layer_names = []
-    layerExists = 0
-    for l in project.activeMap.listLayers(): 
-        if l.longName.startswith(newGroupName + "\\"):
-            all_layer_names.append(l.longName)
-    #print(all_layer_names)
-
-    longName = streamBranch + "\\" + newName 
-    if longName in all_layer_names: 
-        for index, letter in enumerate('234567890abcdefghijklmnopqrstuvwxyz'):
-            if (longName + "_" + letter) not in all_layer_names: newName += "_"+letter; layerExists +=1; break 
-
-    # particularly if the layer comes from ArcGIS
-    geomType = layer.geomType # for ArcGIS: Polygon, Point, Polyline, Multipoint, MultiPatch
-    print(geomType)
-    if "polygon" in geomType.lower(): geomType = "Polygon"
-    if "line" in geomType.lower(): geomType = "Polyline"
-    if "multipoint" in geomType.lower(): geomType = "Multipoint"
-    elif "point" in geomType.lower(): geomType = "Point"
-    #print(geomType)
-    
-    #print(newName)
-    #path = r"C:\Users\username\Documents\ArcGIS\Projects\MyProject-test\MyProject-test.gdb\\"
-    #https://community.esri.com/t5/arcgis-pro-questions/is-it-possible-to-create-a-new-group-layer-with/td-p/1068607
-    #print(project.filePath.replace("aprx","gdb"))
-    #print("_________create feature class___________________________________")
-    # should be created inside the workspace to be a proper Feature class (not .shp) with Nullable Fields
-    class_name = "f_class_" + newName
-    #print(class_name)
-    try: f_class = CreateFeatureclass(path, class_name, geomType, has_z="ENABLED", spatial_reference = sr)
-    except arcgisscripting.ExecuteError: class_name+="_"; f_class = CreateFeatureclass(path, class_name, geomType, has_z="ENABLED", spatial_reference = sr)
-
-    # get and set Layer attribute fields
-    # example: https://resource.esriuk.com/blog/an-introductory-slice-of-arcpy-in-arcgis-pro/
-    newFields = getLayerAttributes(layer.features)
-    fields_to_ignore = ["arcgisgeomfromspeckle", "shape", "objectid"]
-    matrix = []
-    all_keys = []
-    all_key_types = []
-    max_len = 52
-    for key, value in newFields.items(): 
-        existingFields = [fl.name for fl in arcpy.ListFields(class_name)]
-        if key not in existingFields  and key.lower() not in fields_to_ignore: # exclude geometry and default existing fields
-            # signs that should not be used as field names and table names: https://support.esri.com/en/technical-article/000005588
-            key = key.replace(" ","_").replace("-","_").replace("(","_").replace(")","_").replace(":","_").replace("\\","_").replace("/","_").replace("\"","_").replace("&","_").replace("@","_").replace("$","_").replace("%","_").replace("^","_") 
-            if key[0] in ['0','1','2','3','4','5','6','7','8','9']: key = "_"+key
-            if len(key)>max_len: key = key[:max_len]
-            #print(all_keys)
-            if key in all_keys:
-                for index, letter in enumerate('1234567890abcdefghijklmnopqrstuvwxyz'):
-                    if len(key)<max_len and (key+letter) not in all_keys: key+=letter; break 
-                    if len(key) == max_len and (key[:9] + letter) not in all_keys: key=key[:9] + letter; break 
-            if key not in all_keys: 
-                all_keys.append(key)
-                all_key_types.append(value)
-                #print(all_keys)
-                matrix.append([key, value, key, 255])
-                #print(matrix)
-    if len(matrix)>0: AddFields(str(f_class), matrix)
-
-    fets = []
-    for f in layer.features: 
-        new_feat = featureToNative(f, newFields, geomType, sr)
-        if new_feat != "" and new_feat!= None: fets.append(new_feat)
-    
-    print(fets)
-    count = 0
-    rowValues = []
-    for feat in fets:
-        #print(feat)
-        try: feat['applicationId'] 
-        except: feat.update({'applicationId': count})
-
-        row = [feat['arcGisGeomFromSpeckle'], feat['applicationId']]
-        heads = [ 'Shape@', 'OBJECTID']
-
-        for key,value in feat.items(): 
-            if key in all_keys and key.lower() not in fields_to_ignore: 
-                heads.append(key)
-                row.append(value)
-        rowValues.append(row)
-        count += 1
-    cur = arcpy.da.InsertCursor(str(f_class), tuple(heads) )
-    for row in rowValues: 
-        print(tuple(heads))
-        print(tuple(row))
-        cur.insertRow(tuple(row))
-    del cur 
-
-    vl = MakeFeatureLayer(str(f_class), newName).getOutput(0)
-
-    #adding layers from code solved: https://gis.stackexchange.com/questions/344343/arcpy-makefeaturelayer-management-function-not-creating-feature-layer-in-arcgis
-    #active_map.addLayer(new_layer)
-    active_map.addLayerToGroup(layerGroup, vl)
-    r'''
-    # rename back the layer if was renamed due to existing duplicate
-    if layerExists:  
-        vl.name = newName[:len(newName)-2]
-        for lyr in project.activeMap.listLayers():
-            print(lyr.longName)
-            if (streamBranch + "\\" + newName) == lyr.longName: 
-                lyr.name = lyr.name.replace( lyr.name, lyr.name[:len(lyr.name)-2] )
-                lyr.longName = lyr.longName.replace( lyr.longName, lyr.longName[:len(newName)-2] )
-                break
-    '''
-
-    r'''
-    pr.addFeatures(fets)
-    vl.updateExtents()
-    vl.commitChanges()
-    #layerGroup.addLayer(vl)
-
-    rendererNew = vectorRendererToNative(layer)
-    if rendererNew is None:
-        symbol = QgsSymbol.defaultSymbol(QgsWkbTypes.geometryType(QgsWkbTypes.parseType(geomType)))
-        rendererNew = QgsSingleSymbolRenderer(symbol)
-
-    try: vl.setRenderer(rendererNew)
-    except: pass
-    '''
-    return vl
-
-def rasterLayerToNative(layer: RasterLayer, streamBranch: str, project: ArcGISProject):
-
-    raster_layer = None
-    '''
-    crs = QgsCoordinateReferenceSystem.fromWkt(layer.crs.wkt) #moved up, because CRS of existing layer needs to be rewritten
-    # try, in case of older version "rasterCrs" will not exist 
-    try: crsRaster = QgsCoordinateReferenceSystem.fromWkt(layer.rasterCrs.wkt) #moved up, because CRS of existing layer needs to be rewritten
-    except: 
-        crsRaster = crs
-        logger.logToUser(f"Raster layer {layer.name} might have been sent from the older version of plugin. Try sending it again for more accurate results.", Qgis.Warning)
-    
-    #CREATE A GROUP "received blabla" with sublayers
-    newGroupName = f'{streamBranch}'
-    root = QgsProject.instance().layerTreeRoot()
-    layerGroup = QgsLayerTreeGroup(newGroupName)
-
-    if root.findGroup(newGroupName) is not None:
-        layerGroup = root.findGroup(newGroupName)
-    else:
-        root.addChildNode(layerGroup)
-    layerGroup.setExpanded(True)
-    layerGroup.setItemVisibilityChecked(True)
-
-    #find ID of the layer with a matching name in the "latest" group 
-    newName = f'{streamBranch}/{layer.name}'
-
-    ######################## testing, only for receiving layers #################
-    source_folder = QgsProject.instance().absolutePath()
-
-    if(source_folder == ""):
-        logger.logToUser(f"Raster layers can only be received in an existing saved project. Layer {layer.name} will be ignored", Qgis.Warning)
-        return None
-
-    project = QgsProject.instance()
-    projectCRS = QgsCoordinateReferenceSystem.fromWkt(layer.crs.wkt)
-    crsid = crsRaster.authid()
-    try: epsg = int(crsid.split(":")[1]) 
-    except: 
-        epsg = int(str(projectCRS).split(":")[len(str(projectCRS).split(":"))-1].split(">")[0])
-        logger.logToUser(f"CRS of the received raster cannot be identified. Project CRS will be used.", Qgis.Warning)
-    
-    feat = layer.features[0]
-    bandNames = feat["Band names"]
-    bandValues = [feat["@(10000)" + name + "_values"] for name in bandNames]
-
-    #newName = f'{streamBranch}_latest_{layer.name}'
-
-    ###########################################################################
-
-    ## https://opensourceoptions.com/blog/pyqgis-create-raster/
-    # creating file in temporary folder: https://stackoverflow.com/questions/56038742/creating-in-memory-qgsrasterlayer-from-the-rasterization-of-a-qgsvectorlayer-wit
-
-    fn = source_folder + '/' + newName.replace("/","_") + '.tif' #'_received_raster.tif'
-    driver = gdal.GetDriverByName('GTiff')
-    # create raster dataset
-    ds = driver.Create(fn, xsize=feat["X pixels"], ysize=feat["Y pixels"], bands=feat["Band count"], eType=gdal.GDT_Float32)
-
-    # Write data to raster band
-    for i in range(feat["Band count"]):
-        rasterband = np.array(bandValues[i])
-        rasterband = np.reshape(rasterband,(feat["Y pixels"], feat["X pixels"]))
-        ds.GetRasterBand(i+1).WriteArray(rasterband) # or "rasterband.T"
-
-    # create GDAL transformation in format [top-left x coord, cell width, 0, top-left y coord, 0, cell height]
-    pt = pointToNative(feat["displayValue"][0])
-    xform = QgsCoordinateTransform(crs, crsRaster, project)
-    pt.transform(xform)
-    ds.SetGeoTransform([pt.x(), feat["X resolution"], 0, pt.y(), 0, feat["Y resolution"]])
-    # create a spatial reference object
-    srs = osr.SpatialReference()
-    #  For the Universal Transverse Mercator the SetUTM(Zone, North=1 or South=2)
-    srs.ImportFromEPSG(epsg) # from https://gis.stackexchange.com/questions/34082/creating-raster-layer-from-numpy-array-using-pyqgis
-    ds.SetProjection(srs.ExportToWkt())
-    # close the rater datasource by setting it equal to None
-    ds = None
-
-    raster_layer = QgsRasterLayer(fn, newName, 'gdal')
-    QgsProject.instance().addMapLayer(raster_layer, False)
-    layerGroup.addLayer(raster_layer)
-
-    dataProvider = raster_layer.dataProvider()
-    rendererNew = rasterRendererToNative(layer, dataProvider)
-
-    try: raster_layer.setRenderer(rendererNew)
-    except: pass
-    '''
-    return raster_layer

speckle_toolbox/esri/toolboxes/speckle/converter/layers/feature.py

@@ -1,144 +0,0 @@
-from specklepy.objects import Base
-import arcpy 
-from arcpy.management import CreateCustomGeoTransformation
-
-from speckle.converter.geometry._init_ import convertToSpeckle, convertToNative, convertToNativeMulti
-from speckle.converter.layers.utils import getVariantFromValue
-
-def featureToSpeckle(fieldnames, attr_list, f_shape, projectCRS: arcpy.SpatialReference, project: arcpy.mp.ArcGISProject, selectedLayer):
-    print("___________Feature to Speckle____________")
-    b = Base(units = "m")
-    data = arcpy.Describe(selectedLayer.dataSource)
-    layer_sr = data.spatialReference # if sr.type == "Projected":
-    geomType = data.shapeType #Polygon, Point, Polyline, Multipoint, MultiPatch
-    featureType = data.featureType # Simple,SimpleJunction,SimpleJunction,ComplexEdge,Annotation,CoverageAnnotation,Dimension,RasterCatalogItem 
-
-    #print(layer_sr.name)
-    #print(projectCRS.name)
-
-    #apply transformation if needed
-    if layer_sr.name != projectCRS.name:
-        tr0 = tr1 = tr2 = tr_custom = None
-        transformations = arcpy.ListTransformations(layer_sr, projectCRS)
-        customTransformName = "layer_sr.name"+"_To_"+ projectCRS.name
-        if len(transformations) == 0:
-            midSr = arcpy.SpatialReference("WGS 1984") # GCS_WGS_1984
-            try:
-                tr1 = arcpy.ListTransformations(layer_sr, midSr)[0]
-                tr2 = arcpy.ListTransformations(midSr, projectCRS)[0]
-            except: 
-                #customGeoTransfm = "GEOGTRAN[METHOD['Geocentric_Translation'],PARAMETER['X_Axis_Translation',''],PARAMETER['Y_Axis_Translation',''],PARAMETER['Z_Axis_Translation','']]"
-                #CreateCustomGeoTransformation(customTransformName, layer_sr, projectCRS)
-                tr_custom = customTransformName
-        else: 
-            #print("else")
-            # choose equation based instead of file-based/grid-based method, 
-            # to be consistent with QGIS: https://desktop.arcgis.com/en/arcmap/latest/map/projections/choosing-an-appropriate-transformation.htm
-            selecterTr = {}
-            for tr in transformations:
-                if "NTv2" not in tr and "NADCON" not in tr: 
-                    set1 = set( layer_sr.name.split("_") + projectCRS.name.split("_") )
-                    set2 = set( tr.split("_") )
-                    diff = len( set(set1).symmetric_difference(set2) )
-                    selecterTr.update({tr: diff})
-            selecterTr = dict(sorted(selecterTr.items(), key=lambda item: item[1]))
-            tr0 = list(selecterTr.keys())[0]
-            #print(tr0)
-
-        if geomType != "Point" and geomType != "Polyline" and geomType != "Polygon" and geomType != "Multipoint":
-            #print(geomType)
-            arcpy.AddWarning("Unsupported or invalid geometry in layer " + selectedLayer.name)
-
-        # reproject geometry using chosen transformstion(s)
-        if tr0 is not None:
-            ptgeo1 = f_shape.projectAs(projectCRS, tr0)
-            f_shape = ptgeo1
-        elif tr1 is not None and tr2 is not None:
-            ptgeo1 = f_shape.projectAs(midSr, tr1)
-            ptgeo2 = ptgeo1.projectAs(projectCRS, tr2)
-            f_shape = ptgeo2
-        else:
-            ptgeo1 = f_shape.projectAs(projectCRS)
-            f_shape = ptgeo1
-        
-
-    ######################################### Convert geometry ##########################################
-    try:
-        geom = convertToSpeckle(f_shape, selectedLayer, geomType, featureType) 
-        if geom is not None: print(geom); b["geometry"] = geom 
-    except Exception as error:
-        print("Error converting geometry: " + str(error))
-        print(selectedLayer)
-        arcpy.AddError("Error converting geometry: " + str(error))
-    #print(geomType) 
-    #print(featureType) 
-    for i, name in enumerate(fieldnames):
-        corrected = name.replace("/", "_").replace(".", "-")
-        if corrected != "Shape" and corrected != "Shape@": 
-            # different ID behaviors: https://support.esri.com/en/technical-article/000010834 
-            # save all attribute, duplicate one into applicationId 
-            b[corrected] = attr_list[i]
-            if corrected == "FID" or corrected == "OID" or corrected == "OBJECTID": b["applicationId"] = str(attr_list[i])
-    #print(b)
-    print("______end of __Feature to Speckle____________________")
-    return b
-
-def featureToNative(feature: Base, fields: dict, geomType: str, sr: arcpy.SpatialReference):
-    print("Feature To Native____________") 
-    feat = {}
-    try: speckle_geom = feature["geometry"] # for created in QGIS / ArcGIS Layer type
-    except:  speckle_geom = feature # for created in other software
-    print(speckle_geom)
-    if isinstance(speckle_geom, list):
-        if len(speckle_geom)>1 or geomType == "Multipoint": arcGisGeom = convertToNativeMulti(speckle_geom, sr)
-        else: arcGisGeom = convertToNative(speckle_geom[0], sr)
-    else:
-        arcGisGeom = convertToNative(speckle_geom, sr)
-
-    if arcGisGeom is not None:
-        feat.update({"arcGisGeomFromSpeckle": arcGisGeom})
-    else:
-        return None
-    print(feat)
-    for key, variant in fields.items(): 
-
-        value = feature[key]
-        if variant == "TEXT": value = str(feature[key]) 
-        if variant == getVariantFromValue(value) and value != "NULL" and value != "None": 
-            feat.update({key: value})
-        else: 
-            if variant == "TEXT": feat.update({key: None})
-            if variant == "FLOAT": feat.update({key: None})
-            if variant == "LONG": feat.update({key: None})
-            if variant == "SHORT": feat.update({key: None})
-    return feat
-
-def cadFeatureToNative(feature: Base, fields: dict, sr: arcpy.SpatialReference):
-    print("_________CAD Feature To Native____________")
-    feat = {}
-    try: speckle_geom = feature["geometry"] # for created in QGIS Layer type
-    except:  speckle_geom = feature # for created in other software
-    #print(feature)
-    #print(speckle_geom)
-    if isinstance(speckle_geom, list):
-        if len(speckle_geom)>1: arcGisGeom = convertToNativeMulti(speckle_geom, sr)
-        else: arcGisGeom = convertToNative(speckle_geom[0], sr) 
-    else:
-        arcGisGeom = convertToNative(speckle_geom, sr)
-
-    if arcGisGeom is not None:
-        feat.update({"arcGisGeomFromSpeckle": arcGisGeom})
-    else: return None
-    for key, variant in fields.items(): 
-        value = feature[key]
-        if variant == "TEXT": value = str(feature[key]) 
-        if variant == getVariantFromValue(value) and value != "NULL" and value != "None": 
-            feat.update({key: value})
-        else: 
-            if variant == "TEXT": feat.update({key: None})
-            if variant == "FLOAT": feat.update({key: None})
-            if variant == "LONG": feat.update({key: None})
-            if variant == "SHORT": feat.update({key: None})
-    #print(feat) 
-    return feat
-    

speckle_toolbox/esri/toolboxes/speckle/converter/layers/utils.py

@@ -1,94 +0,0 @@
-from typing import Any, List, Union
-from specklepy.objects import Base
-import arcpy 
-
-
-def getVariantFromValue(value: Any) -> Union[str, None]:
-    # TODO add Base object
-    pairs = [
-        (str, "TEXT"), # 10
-        (float, "FLOAT"),
-        (int, "LONG"),
-        (bool, "SHORT")
-        #date: "SHORT"
-    ]
-    res = None
-    for p in pairs:
-        if isinstance(value, p[0]): res = p[1]; break
-    #t = type(value)
-    
-    #try: res = pairs[t]
-    #except: pass
-    #if isinstance(value, str) and "PyQt5.QtCore.QDate(" in value: res = QVariant.Date #14
-    #elif isinstance(value, str) and "PyQt5.QtCore.QDateTime(" in value: res = QVariant.DateTime #16
-
-    return res
-
-def getLayerAttributes(features: List[Base]) -> dict:
-    print("________ get layer attributes___")
-    fields = {}
-    all_props = []
-    for feature in features: 
-        #get object properties to add as attributes
-        dynamicProps = feature.get_dynamic_member_names()
-        attrsToRemove = ['geometry','applicationId','bbox','displayStyle', 'id', 
-                        'renderMaterial', 'userDictionary', 'userStrings','geometry']
-        for att in attrsToRemove:
-            try: dynamicProps.remove(att)
-            except: pass
-
-        dynamicProps.sort()
-        #print(dynamicProps)
-
-        # add field names and variands 
-        #variants = [] 
-        for name in dynamicProps:
-            if name not in all_props: all_props.append(name)
-
-            value = feature[name]
-            variant = getVariantFromValue(value)
-            #print(variant)
-            if not variant: variant = None #LongLong #4 
-
-            # add a field if not existing yet AND if variant is known
-            if variant and (name not in fields.keys()): 
-                fields.update({name: variant})
-            
-            elif name in fields.keys(): #check if the field was empty previously: 
-                #nameIndex = fields.indexFromName(name)
-                oldVariant = fields[name]
-                #print(oldVariant)
-                # replace if new one is NOT LongLong or IS String
-                if oldVariant != "TEXT" and variant == "TEXT": 
-                    fields.update({name: variant}) 
-    print(all_props)
-
-    # replace all empty ones wit String
-    for name in all_props:
-        if name not in fields.keys(): 
-            fields.update({name: "TEXT"}) 
-    print(fields)
-    return fields
-
-def get_scale_factor(units: str) -> float:
-    unit_scale = {
-    "meters": 1.0,
-    "centimeters": 0.01,
-    "millimeters": 0.001,
-    "inches": 0.0254,
-    "feet": 0.3048,
-    "kilometers": 1000.0,
-    "mm": 0.001,
-    "cm": 0.01,
-    "m": 1.0,
-    "km": 1000.0,
-    "in": 0.0254,
-    "ft": 0.3048,
-    "yd": 0.9144,
-    "mi": 1609.340,
-    }
-    if units is not None and units.lower() in unit_scale.keys():
-        return unit_scale[units]
-    arcpy.AddWarning(f"Units {units} are not supported. Meters will be applied by default.")
-    return 1.0
-

speckle_toolbox/esri/toolboxes/speckle/speckle/converter/features/feature_conversions.py

@@ -0,0 +1,997 @@
+from datetime import datetime
+import inspect
+import math
+import os
+from typing import List, Union
+import arcpy
+from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
+
+import numpy as np
+
+import scipy as sp
+from speckle.speckle.plugin_utils.helpers import (
+    findOrCreatePath,
+    get_scale_factor_to_meter,
+)
+from speckle.speckle.converter.layers.utils import (
+    apply_reproject,
+    findTransformation,
+    getVariantFromValue,
+    traverseDict,
+    validateAttributeName,
+)
+
+# from speckle.speckle.converter.geometry.conversions import transform
+from speckle.speckle.converter.geometry.conversions import (
+    convertToNative,
+    convertToNativeMulti,
+    convertToSpeckle,
+)
+from speckle.speckle.converter.geometry.mesh import constructMeshFromRaster
+from speckle.speckle.converter.geometry.utils import apply_pt_offsets_rotation_on_send
+from speckle.speckle.utils.panel_logging import logToUser
+from speckle.speckle.converter.features.utils import updateFeat
+from specklepy.objects.GIS.geometry import (
+    GisRasterElement,
+    GisPolygonGeometry,
+    GisNonGeometryElement,
+    GisTopography,
+)
+
+from specklepy.objects import Base
+
+from speckle.speckle.converter.geometry.point import pointToSpeckle
+from speckle.speckle.converter.layers.symbology import jsonFromLayerStyle
+
+
+def featureToSpeckle(
+    fieldnames,
+    attr_list,
+    index: int,
+    f_shape,
+    projectCRS: arcpy.SpatialReference,
+    selectedLayer: arcLayer,
+    plugin,
+):
+    dataStorage = plugin.dataStorage
+    if dataStorage is None:
+        return
+    units = dataStorage.currentUnits
+    new_report = {"obj_type": "", "errors": ""}
+    iterations = 0
+    try:
+        geom = None
+        data = arcpy.Describe(selectedLayer.dataSource)
+        geomType = data.shapeType
+        if (
+            hasattr(data, "isRevit")
+            or hasattr(data, "isIFC")
+            or hasattr(data, "bimLevels")
+        ):
+            # print(f"Layer {selectedLayer.name} has unsupported data type")
+            logToUser(
+                f"Layer {selectedLayer.name} has unsupported data type",
+                level=1,
+                func=inspect.stack()[0][3],
+            )
+            return None
+
+        skipped_msg = f"'{geomType}' feature skipped due to invalid geometry"
+        try:
+            geom, iterations = convertToSpeckle(
+                f_shape, index, selectedLayer, data, dataStorage
+            )
+            print(geom)
+            if geom is not None and geom != "None":
+                if not isinstance(geom.geometry, List):
+                    logToUser(
+                        "Geometry not in list format",
+                        level=2,
+                        func=inspect.stack()[0][3],
+                    )
+                    return None
+
+                all_errors = ""
+                for g in geom.geometry:
+                    if g is None or g == "None":
+                        all_errors += skipped_msg + ", "
+                        logToUser(skipped_msg, level=2, func=inspect.stack()[0][3])
+                    elif isinstance(g, GisPolygonGeometry):
+                        if len(g.displayValue) == 0:
+                            all_errors += (
+                                "Polygon converted, but display mesh not generated"
+                                + ", "
+                            )
+                            logToUser(
+                                "Polygon converted, but display mesh not generated",
+                                level=1,
+                                func=inspect.stack()[0][3],
+                            )
+                        elif iterations is not None and iterations > 0:
+                            all_errors += "Polygon display mesh is simplified" + ", "
+                            logToUser(
+                                "Polygon display mesh is simplified",
+                                level=1,
+                                func=inspect.stack()[0][3],
+                            )
+
+                if len(geom.geometry) == 0:
+                    all_errors = "No geometry converted"
+                new_report.update({"obj_type": geom.speckle_type, "errors": all_errors})
+
+            else:  # geom is None
+                new_report = {"obj_type": "", "errors": skipped_msg}
+                logToUser(skipped_msg, level=2, func=inspect.stack()[0][3])
+
+                dataStorage.latestActionFeaturesReport[
+                    len(dataStorage.latestActionFeaturesReport) - 1
+                ].update(new_report)
+                return
+                # geom = GisNonGeometryElement()
+        except Exception as error:
+            new_report = {
+                "obj_type": "",
+                "errors": "Error converting geometry: " + str(error),
+            }
+            logToUser(
+                "Error converting geometry: " + str(error),
+                level=2,
+                func=inspect.stack()[0][3],
+            )
+
+        # print(fieldnames)
+        # print(attr_list)
+        attributes = Base()
+        for i, name in enumerate(fieldnames):
+            corrected = validateAttributeName(name, fieldnames)
+            # print(corrected)
+            f_val = attr_list[i]
+            if f_val == "NULL" or f_val is None or str(f_val) == "NULL":
+                f_val = None
+            if isinstance(attr_list[i], list):
+                x = ""
+                for i, attr in enumerate(attr_list[i]):
+                    if i == 0:
+                        x += str(attr)
+                    else:
+                        x += ", " + str(attr)
+                f_val = x
+            attributes[corrected] = f_val
+
+        if geom is not None and geom != "None":
+            geom.attributes = attributes
+        # print(geom.attributes)
+
+        dataStorage.latestActionFeaturesReport[
+            len(dataStorage.latestActionFeaturesReport) - 1
+        ].update(new_report)
+        return geom
+
+    except Exception as e:
+        new_report.update({"errors": e})
+        dataStorage.latestActionFeaturesReport[
+            len(dataStorage.latestActionFeaturesReport) - 1
+        ].update(new_report)
+        logToUser(e, level=2, func=inspect.stack()[0][3])
+        return geom
+
+
+def rasterFeatureToSpeckle(
+    selectedLayer: arcLayer,
+    projectCRS: arcpy.SpatialReference,
+    plugin,
+) -> Base:
+    dataStorage = plugin.dataStorage
+    if dataStorage is None:
+        return
+
+    b = GisRasterElement(units=dataStorage.currentUnits)
+    try:
+        # get Raster object of entire raster dataset
+        my_raster = arcpy.Raster(selectedLayer.dataSource)
+        print(my_raster.mdinfo)  # None
+
+        rasterBandCount = my_raster.bandCount
+        rasterBandNames = my_raster.bandNames
+        #rasterDimensions = [my_raster.width, my_raster.height] #wrong result
+        
+        rb = my_raster.getRasterBands(rasterBandNames[0])
+        size = np.shape(rb.read())
+        rasterDimensions = [size[1], size[0]]
+        print(rasterDimensions)
+
+        # ds = gdal.Open(selectedLayer.source(), gdal.GA_ReadOnly)
+        extent = my_raster.extent
+        print(extent.XMin)
+        print(extent.YMin)
+        # if extent.YMin>0: rasterOriginPoint = arcpy.PointGeometry(arcpy.Point(extent.XMin, extent.YMax, extent.ZMin), my_raster.spatialReference, has_z = True)
+        rasterResXY = [
+            my_raster.meanCellWidth,
+            my_raster.meanCellHeight,
+        ]  # [float(ds.GetGeoTransform()[1]), float(ds.GetGeoTransform()[5])]
+        
+        b.x_resolution = rasterResXY[0]
+        b.y_resolution = -1 * rasterResXY[1]
+        b.x_size = rasterDimensions[0]
+        b.y_size = rasterDimensions[1]
+        b.band_count = rasterBandCount
+        b.band_names = rasterBandNames
+
+        rasterBandNoDataVal = []  # list(my_raster.noDataValues)
+        rasterBandMinVal = []
+        rasterBandMaxVal = []
+        rasterBandVals = []
+
+        rasterOriginPoint = arcpy.PointGeometry(
+            arcpy.Point(extent.XMin, extent.YMax),
+            my_raster.spatialReference,
+        )
+        rasterOriginPoint_max = arcpy.PointGeometry(
+            arcpy.Point(
+                extent.XMin + rasterDimensions[0] * b.x_resolution,
+                extent.YMax + rasterDimensions[1] * b.y_resolution
+            ),
+            my_raster.spatialReference,
+        )
+
+        # Try to extract geometry
+        x_form: tuple = findTransformation(
+            "Point",
+            my_raster.spatialReference,
+            projectCRS,
+            selectedLayer,
+        )
+
+        reprojectedPt = apply_reproject(rasterOriginPoint, x_form, dataStorage)
+        reprojectedPt_max = apply_reproject(rasterOriginPoint_max, x_form, dataStorage)
+        if reprojectedPt is None:
+            reprojectedPt = rasterOriginPoint
+            reprojectedPt_max = rasterOriginPoint_max
+
+        new_x_min, new_y_min = apply_pt_offsets_rotation_on_send(reprojectedPt.getPart().X, reprojectedPt.getPart().Y, dataStorage)
+        new_x_max, new_y_max = apply_pt_offsets_rotation_on_send(reprojectedPt_max.getPart().X, reprojectedPt_max.getPart().Y, dataStorage)
+        res_x = (new_x_max - new_x_min) / rasterDimensions[0]
+        res_y = (new_y_max - new_y_min) / rasterDimensions[1]
+
+        for index in range(rasterBandCount):
+            item = rasterBandNames[index]
+
+            rb = my_raster.getRasterBands(item)
+            print(rb)
+            size = np.shape(rb.read())
+            print(size)
+            rasterDimensions = [size[1], size[0]]
+
+            valMin = rb.minimum
+            valMax = rb.maximum
+            bandVals = np.swapaxes(rb.read(), 1, 2).flatten()  # .tolist() np.flip( , 0)
+
+            bandValsFlat = bandVals.tolist()
+            # print(bandValsFlat)
+
+            const = float(-1 * math.pow(10, 30))
+            defaultNoData = rb.noDataValue
+            print(defaultNoData)
+
+            # check whether NA value is too small or raster has too small values
+            # assign min value of an actual list; re-assign NA val; replace list items to new NA val
+            try:
+                # create "safe" fake NA value; replace extreme values with it
+                fakeNA = max(bandValsFlat) + 1
+                bandValsFlatFake = [
+                    fakeNA if val <= const else val for val in bandValsFlat
+                ]  # replace all values corresponding to NoData value
+
+                # if default NA value is too small
+                if (
+                    isinstance(defaultNoData, float) or isinstance(defaultNoData, int)
+                ) and defaultNoData < const:
+                    # find and rewrite min of actual band values; create new NA value
+                    valMin = min(bandValsFlatFake)
+                    noDataValNew = valMin - 1000  # use new adequate value
+                    rasterBandNoDataVal.append(noDataValNew)
+                    # replace fake NA with new NA
+                    bandValsFlat = [
+                        noDataValNew if val == fakeNA else val
+                        for val in bandValsFlatFake
+                    ]  # replace all values corresponding to NoData value
+
+                # if default val unaccessible and minimum val is too small
+                elif (
+                    isinstance(defaultNoData, str) or defaultNoData is None
+                ) and valMin < const:  # if there are extremely small values but default NA unaccessible
+                    noDataValNew = valMin
+                    rasterBandNoDataVal.append(noDataValNew)
+                    # replace fake NA with new NA
+                    bandValsFlat = [
+                        noDataValNew if val == fakeNA else val
+                        for val in bandValsFlatFake
+                    ]  # replace all values corresponding to NoData value
+                    # last, change minValto actual one
+                    valMin = min(bandValsFlatFake)
+
+                else:
+                    rasterBandNoDataVal.append(rb.noDataValue)
+
+            except:
+                rasterBandNoDataVal.append(rb.noDataValue)
+
+            # if rasterBandNoDataVal[len(rasterBandNoDataVal) - 1] is None:
+            #   rasterBandNoDataVal[len(rasterBandNoDataVal) - 1] = np.nan
+            print(len(bandValsFlat))
+            rasterBandVals.append(bandValsFlat)
+            rasterBandMinVal.append(valMin)
+            rasterBandMaxVal.append(valMax)
+
+            b["@(10000)" + item + "_values"] = (
+                bandValsFlat  # [0:int(max_values/rasterBandCount)]
+            )
+
+        b.x_origin, b.y_origin = apply_pt_offsets_rotation_on_send(
+            reprojectedPt.getPart().X, reprojectedPt.getPart().Y, dataStorage
+        )
+        try:
+            b.noDataValue = [float(val) for val in rasterBandNoDataVal]
+        except:
+            pass
+        # creating a mesh
+        vertices = []
+        faces = []
+        colors = []
+        count = 0
+
+        # print(selectedLayer.symbology)  # None
+        colorizer = None
+
+        # print(rendererType)
+        # identify symbology type and if Multiband, which band is which color
+
+        #############################################################
+
+        largeTransform = False
+        if rasterDimensions[0] * rasterDimensions[1] > 10000:
+            logToUser(
+                f"Transformation of the layer '{selectedLayer.name}' might take a while 🕒",
+                level=0,
+                plugin=plugin.dockwidget,
+            )
+            largeTransform = True
+
+        ############################################################
+
+        if hasattr(selectedLayer.symbology, "colorizer"):
+            colorizer = selectedLayer.symbology.colorizer
+            # print(
+            #    colorizer
+            # )  # <arcpy._colorizer.RasterStretchColorizer object at 0x000001780497FBC8>
+            # print(colorizer.type)  # RasterStretchColorizer
+        else:
+            redBand = greenBand = blueBand = None
+            # RGB colorizer
+            root_path: str = (
+                os.path.expandvars(r"%LOCALAPPDATA%")
+                + "\\Temp\\Speckle_ArcGIS_temp\\"
+                + datetime.now().strftime("%Y-%m-%d_%H-%M")
+            )
+            root_path += "\\Layers_Speckle\\raster_bands\\"
+            findOrCreatePath(root_path)
+
+            # if not os.path.exists(root_path + '\\Layers_Speckle\\raster_bands'): os.makedirs(root_path + '\\Layers_Speckle\\raster_bands')
+            path_style = root_path + selectedLayer.name + "_temp.lyrx"
+            symJson = jsonFromLayerStyle(selectedLayer, path_style)
+
+            # read from Json
+            try:
+                greenBand = symJson["layerDefinitions"][0]["colorizer"][
+                    "greenBandIndex"
+                ]
+            except:
+                if len(rasterBandVals) > 1:
+                    greenBand = 1
+            try:
+                blueBand = symJson["layerDefinitions"][0]["colorizer"]["blueBandIndex"]
+            except:
+                if len(rasterBandVals) > 2:
+                    blueBand = 2
+            try:
+                redBand = symJson["layerDefinitions"][0]["colorizer"]["redBandIndex"]
+            except:
+                if blueBand != 0 and greenBand != 0:
+                    redBand = 0
+                else:
+                    redBand = None
+            print("bands")
+            print(redBand)
+            print(greenBand)
+            print(blueBand)
+            try:
+                rbVals = rasterBandVals[
+                    redBand
+                ]  # my_raster.getRasterBands(rasterBandNames[redBand])
+                rbvalMin = rasterBandMinVal[redBand]
+                rbvalMax = rasterBandMaxVal[redBand]
+                rvalRange = float(rbvalMax) - float(rbvalMin)
+                print(rbvalMin)
+                print(rbvalMax)
+                print(rvalRange)
+            except Exception as e:
+                print(e)
+                rvalRange = None
+            try:
+                gbVals = rasterBandVals[greenBand]
+                gbvalMin = rasterBandMinVal[greenBand]
+                gbvalMax = rasterBandMaxVal[greenBand]
+                gvalRange = float(gbvalMax) - float(gbvalMin)
+                print(gbvalMin)
+                print(gbvalMax)
+                print(gvalRange)
+            except:
+                gvalRange = None
+            try:
+                bbVals = rasterBandVals[blueBand]
+                bbvalMin = rasterBandMinVal[blueBand]
+                bbvalMax = rasterBandMaxVal[blueBand]
+                bvalRange = float(bbvalMax) - float(bbvalMin)
+                print(bbvalMin)
+                print(bbvalMax)
+                print(bvalRange)
+            except:
+                bvalRange = None
+
+        rendererType = ""
+        bandIndex = 0
+
+        array_z = []  # size is large by 1 than the raster size, in both dimensions
+        time0 = datetime.now()
+
+        print(projectCRS)
+        print(projectCRS.factoryCode)
+        # if isinstance(projectCRS.factoryCode, int) and projectCRS.factoryCode > 1:
+        #    reprojected_raster = arcpy.ia.Reproject(
+        #        my_raster, {"wkid": projectCRS.factoryCode}
+        #    )
+        print(my_raster.spatialReference)
+        print(my_raster.spatialReference.factoryCode)
+        for v in range(rasterDimensions[1]):  # each row, Y
+            print(v)
+            if largeTransform is True:
+                if v == int(rasterDimensions[1] / 20):
+                    logToUser(
+                        f"Converting layer '{selectedLayer.name}': 5%...",
+                        level=0,
+                        plugin=plugin.dockwidget,
+                    )
+                elif v == int(rasterDimensions[1] / 10):
+                    logToUser(
+                        f"Converting layer '{selectedLayer.name}': 10%...",
+                        level=0,
+                        plugin=plugin.dockwidget,
+                    )
+                elif v == int(rasterDimensions[1] / 5):
+                    logToUser(
+                        f"Converting layer '{selectedLayer.name}': 20%...",
+                        level=0,
+                        plugin=plugin.dockwidget,
+                    )
+                elif v == int(rasterDimensions[1] * 2 / 5):
+                    logToUser(
+                        f"Converting layer '{selectedLayer.name}': 40%...",
+                        level=0,
+                        plugin=plugin.dockwidget,
+                    )
+                elif v == int(rasterDimensions[1] * 3 / 5):
+                    logToUser(
+                        f"Converting layer '{selectedLayer.name}': 60%...",
+                        level=0,
+                        plugin=plugin.dockwidget,
+                    )
+                elif v == int(rasterDimensions[1] * 4 / 5):
+                    logToUser(
+                        f"Converting layer '{selectedLayer.name}': 80%...",
+                        level=0,
+                        plugin=plugin.dockwidget,
+                    )
+                elif v == int(rasterDimensions[1] * 9 / 10):
+                    logToUser(
+                        f"Converting layer '{selectedLayer.name}': 90%...",
+                        level=0,
+                        plugin=plugin.dockwidget,
+                    )
+
+            for h in range(rasterDimensions[0]):  # item in a row, X
+                vertices.extend(
+                    [
+                        new_x_min + h * res_x,
+                        new_y_min + v * res_y,
+                        0,
+                        new_x_min + h * res_x,
+                        new_y_min + (v + 1) * res_y,
+                        0,
+                        new_x_min + (h + 1) * res_x,
+                        new_y_min + (v + 1) * res_y,
+                        0,
+                        new_x_min + (h + 1) * res_x,
+                        new_y_min + v * res_y,
+                        0,
+                    ]
+                )  ## add 4 points
+                faces.extend([4, count, count + 1, count + 2, count + 3])
+
+                # color vertices according to QGIS renderer
+                color = (0 << 16) + (0 << 8) + 0
+                noValColor = (
+                    0,
+                    0,
+                    0,
+                )
+
+                if hasattr(selectedLayer.symbology, "colorizer"):  # only 1 band
+                    try:
+                        bandIndex = int(colorizer.band)  # if stretched
+                    except:
+                        pass
+                    if colorizer.type == "RasterUniqueValueColorizer":
+                        # REDO !!!!!!!!!!!!
+                        colorRVal = colorGVal = colorBVal = 0
+                        try:
+                            for br in colorizer.groups:
+                                print(br.heading)  # "Value"
+                                # go through all values classified
+                                if br.heading != "Value":
+                                    raise Exception('br.heading != "Value"')
+                                for k, itm in enumerate(br.items):
+                                    # print(itm.values)
+                                    if (
+                                        itm.values[0]
+                                        == rasterBandVals[bandIndex][int(count / 4)]
+                                    ):
+                                        print(itm.values[0])
+                                        colorRVal, colorGVal, colorBVal = (
+                                            itm.color["RGB"][0],
+                                            itm.color["RGB"][1],
+                                            itm.color["RGB"][2],
+                                        )
+                                        break
+                                    # if string covering float
+                                    try:
+                                        if float(itm.values[0]) == float(
+                                            rasterBandVals[bandIndex][int(count / 4)]
+                                        ):
+                                            print(itm.values[0])
+                                            colorRVal, colorGVal, colorBVal = (
+                                                itm.color["RGB"][0],
+                                                itm.color["RGB"][1],
+                                                itm.color["RGB"][2],
+                                            )
+                                            break
+                                    except Exception as e:
+                                        print(e)
+                                        pass
+
+                        except Exception as e:  # if no Min Max labels:
+                            # REMAP band values to (0,255) range
+                            print(e)
+                            valRange = max(rasterBandVals[bandIndex]) - min(
+                                rasterBandVals[bandIndex]
+                            )  # (rasterBandMaxVal[bandIndex] - rasterBandMinVal[bandIndex])
+                            if valRange == 0:
+                                if min(rasterBandVals[bandIndex]) == 0:
+                                    colorVal = 0
+                                else:
+                                    colorVal = 255
+                            else:
+                                colorRVal = colorGVal = colorBVal = int(
+                                    (
+                                        rasterBandVals[bandIndex][int(count / 4)]
+                                        - min(rasterBandVals[bandIndex])
+                                    )
+                                    / valRange
+                                    * 255
+                                )
+
+                        color = (colorRVal << 16) + (colorGVal << 8) + colorBVal
+
+                    else:
+                        try:
+                            if rasterBandVals[bandIndex][int(count / 4)] >= float(
+                                colorizer.minLabel
+                            ) and rasterBandVals[bandIndex][int(count / 4)] <= float(
+                                colorizer.maxLabel
+                            ):  # rasterBandMinVal[bandIndex]:
+                                # REMAP band values to (0,255) range
+                                valRange = float(colorizer.maxLabel) - float(
+                                    colorizer.minLabel
+                                )  # (rasterBandMaxVal[bandIndex] - rasterBandMinVal[bandIndex])
+                                colorVal = int(
+                                    (
+                                        rasterBandVals[bandIndex][int(count / 4)]
+                                        - float(colorizer.minLabel)
+                                    )
+                                    / valRange
+                                    * 255
+                                )
+                                if colorizer.invertColorRamp is True:
+                                    if valRange == 0:
+                                        if float(colorizer.maxLabel) == 0:
+                                            colorVal = 0
+                                        else:
+                                            colorVal = 255
+                                    else:
+                                        colorVal = int(
+                                            (
+                                                -rasterBandVals[bandIndex][
+                                                    int(count / 4)
+                                                ]
+                                                + float(colorizer.maxLabel)
+                                            )
+                                            / valRange
+                                            * 255
+                                        )
+                                color = (colorVal << 16) + (colorVal << 8) + colorVal
+                        except:  # if no Min Max labels:
+                            # REMAP band values to (0,255) range
+                            valRange = max(rasterBandVals[bandIndex]) - min(
+                                rasterBandVals[bandIndex]
+                            )  # (rasterBandMaxVal[bandIndex] - rasterBandMinVal[bandIndex])
+
+                            if valRange == 0:
+                                if min(rasterBandVals[bandIndex]) == 0:
+                                    colorVal = 0
+                                else:
+                                    colorVal = 255
+                            else:
+                                colorVal = int(
+                                    (
+                                        rasterBandVals[bandIndex][int(count / 4)]
+                                        - min(rasterBandVals[bandIndex])
+                                    )
+                                    / valRange
+                                    * 255
+                                )
+                            color = (colorVal << 16) + (colorVal << 8) + colorVal
+                else:  # rgb
+                    # REMAP band values to (0,255) range
+                    if rvalRange is not None and redBand is not None:
+                        if rvalRange == 0:
+                            if float(rbvalMin) == 0:
+                                colorVal = 0
+                            else:
+                                colorVal = 255
+                        else:
+                            colorRVal = int(
+                                (
+                                    rasterBandVals[redBand][int(count / 4)]
+                                    - float(rbvalMin)
+                                )
+                                / rvalRange
+                                * 255
+                            )
+                    else:
+                        colorRVal = 0
+                    if gvalRange is not None and greenBand is not None:
+                        if gvalRange == 0:
+                            if float(gbvalMin) == 0:
+                                colorVal = 0
+                            else:
+                                colorVal = 255
+                        else:
+                            colorGVal = int(
+                                (
+                                    rasterBandVals[greenBand][int(count / 4)]
+                                    - float(gbvalMin)
+                                )
+                                / gvalRange
+                                * 255
+                            )
+                    else:
+                        colorGVal = 0
+                    if bvalRange is not None and blueBand is not None:
+                        if bvalRange == 0:
+                            if float(bbvalMin) == 0:
+                                colorVal = 0
+                            else:
+                                colorVal = 255
+                        else:
+                            colorBVal = int(
+                                (
+                                    rasterBandVals[blueBand][int(count / 4)]
+                                    - float(bbvalMin)
+                                )
+                                / bvalRange
+                                * 255
+                            )
+                    else:
+                        colorBVal = 0
+                    # print("__pixel color_")
+                    # print(colorRVal)
+                    # print(colorGVal)
+                    # print(colorBVal)
+
+                    color = (colorRVal << 16) + (colorGVal << 8) + colorBVal
+
+                colors.extend([color, color, color, color])
+                count += 4
+
+        mesh = constructMeshFromRaster(vertices, faces, colors)
+
+        time1 = datetime.now()
+        # print(f"Time to get Raster: {(time1-time0).total_seconds()} sec")
+        # after the entire loop
+
+        if mesh is not None:
+            mesh.units = dataStorage.currentUnits
+            b.displayValue = [mesh]
+        else:
+            logToUser(
+                "Something went wrong. Mesh cannot be created, only raster data will be sent. ",
+                level=2,
+                plugin=plugin.dockwidget,
+            )
+
+        return b
+
+    except Exception as e:
+        logToUser(e, level=2, func=inspect.stack()[0][3])
+        return None
+
+
+def featureToNative(
+    feature: Base, fields: dict, geomType: str, sr: arcpy.SpatialReference, dataStorage
+):
+    print("04_____Feature To Native correct____________")
+    feat = {}
+    try:
+        try:
+            speckle_geom = feature[
+                "geometry"
+            ]  # for created in QGIS / ArcGIS Layer type
+        except:
+            speckle_geom = feature  # for created in other software
+
+        print(speckle_geom)
+        arcGisGeom = None
+        if isinstance(speckle_geom, list):
+            if len(speckle_geom) > 1 or geomType == "Multipoint":
+                arcGisGeom = convertToNativeMulti(speckle_geom, sr, dataStorage)
+            else:
+                if len(speckle_geom) > 0:
+                    arcGisGeom = convertToNative(speckle_geom[0], sr, dataStorage)
+        else:
+            arcGisGeom = convertToNative(speckle_geom, sr, dataStorage)
+
+        if arcGisGeom is not None:
+            feat.update({"arcGisGeomFromSpeckle": arcGisGeom})
+        else:
+            return None
+        try:
+            [print(p for p in arcGisGeom.getPart())]
+        except:
+            print(arcGisGeom.getPart())
+        # print(feat)
+        for key, variant in fields.items():
+            value = None
+            try:
+                value = feature[key]
+            except:
+                if key == "Speckle_ID":
+                    try:
+                        value = str(
+                            feature["speckle_id"]
+                        )  # if GIS already generated this field
+                    except Exception as e:
+                        # print(e)
+                        value = str(feature["id"])
+                else:
+                    # print(key)
+                    # arcpy.AddWarning(f'Field {key} not found')
+                    try:
+                        value = feature.attributes[key]
+                    except:
+                        try:
+                            value = feature.attributes[key.replace("attributes_", "")]
+                        except:
+                            pass
+
+            if variant == "TEXT":
+                value = str(value)
+                if len(value) > 255:
+                    # print(len(value))
+                    value = value[:255]
+                    logToUser(
+                        f'Field "{key}" values are trimmed at 255 characters',
+                        level=2,
+                        func=inspect.stack()[0][3],
+                    )
+                    # arcpy.AddWarning(
+                    #    f'Field "{key}" values are trimmed at 255 characters'
+                    # )
+            if (
+                variant == getVariantFromValue(value)
+                and value != "NULL"
+                and value != "None"
+            ):
+                feat.update({key: value})
+            else:
+                if variant == "TEXT":
+                    feat.update({key: None})
+                if variant == "FLOAT":
+                    feat.update({key: None})
+                if variant == "LONG":
+                    feat.update({key: None})
+                if variant == "SHORT":
+                    feat.update({key: None})
+        # print(feat)
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return feat
+
+
+r"""
+def featureToNative(feature: Base, fields: "QgsFields", dataStorage):
+    feat = QgsFeature()
+    # print("___featureToNative")
+    try:
+        qgsGeom = None
+
+        if isinstance(feature, GisNonGeometryElement):
+            pass
+        else:
+            try:
+                speckle_geom = (
+                    feature.geometry
+                )  # for QGIS / ArcGIS Layer type from 2.14
+            except:
+                try:
+                    speckle_geom = feature[
+                        "geometry"
+                    ]  # for QGIS / ArcGIS Layer type before 2.14
+                except:
+                    speckle_geom = feature  # for created in other software
+
+            if not isinstance(speckle_geom, list):
+                qgsGeom = convertToNative(speckle_geom, dataStorage)
+
+            elif isinstance(speckle_geom, list):
+                if len(speckle_geom) == 1:
+                    qgsGeom = convertToNative(speckle_geom[0], dataStorage)
+                elif len(speckle_geom) > 1:
+                    qgsGeom = convertToNativeMulti(speckle_geom, dataStorage)
+                else:
+                    logToUser(
+                        f"Feature '{feature.id}' does not contain geometry",
+                        level=2,
+                        func=inspect.stack()[0][3],
+                    )
+
+            if qgsGeom is not None:
+                feat.setGeometry(qgsGeom)
+            else:
+                return None
+
+        feat.setFields(fields)
+        for field in fields:
+            name = str(field.name())
+            variant = field.type()
+            # if name == "id": feat[name] = str(feature["applicationId"])
+
+            try:
+                value = feature.attributes[name]  # fro 2.14 onwards
+            except:
+                try:
+                    value = feature[name]
+                except:
+                    if name == "Speckle_ID":
+                        try:
+                            value = str(
+                                feature["Speckle_ID"]
+                            )  # if GIS already generated this field
+                        except:
+                            try:
+                                value = str(feature["speckle_id"])
+                            except:
+                                value = str(feature["id"])
+                    else:
+                        value = None
+                        # logger.logToUser(f"Field {name} not found", Qgis.Warning)
+                        # return None
+
+            if variant == QVariant.String:
+                value = str(value)
+
+            if isinstance(value, str) and variant == QVariant.Date:  # 14
+                y, m, d = value.split("(")[1].split(")")[0].split(",")[:3]
+                value = QDate(int(y), int(m), int(d))
+            elif isinstance(value, str) and variant == QVariant.DateTime:
+                y, m, d, t1, t2 = value.split("(")[1].split(")")[0].split(",")[:5]
+                value = QDateTime(int(y), int(m), int(d), int(t1), int(t2))
+
+            if (
+                variant == getVariantFromValue(value)
+                and value != "NULL"
+                and value != "None"
+            ):
+                feat[name] = value
+
+        return feat
+    except Exception as e:
+        logToUser(e, level=2, func=inspect.stack()[0][3])
+        return feat
+"""
+
+
+def bimFeatureToNative(
+    feature: Base,
+    fields: dict,
+    sr: arcpy.SpatialReference,
+    path: str,
+    dataStorage,
+):
+    # print("04_________BIM Feature To Native____________")
+    feat_updated = {}
+    try:
+        feat = {}
+        feat.update({"arcGisGeomFromSpeckle": ""})
+        # feat_updated = updateFeat(exist_feat, fields, feature)
+
+        try:
+            if "Speckle_ID" not in fields.keys() and feature["id"]:
+                feat.update("Speckle_ID", "TEXT")
+        except:
+            pass
+        feat_updated = updateFeat(feat, fields, feature)
+
+        return feat_updated
+    except Exception as e:
+        logToUser(e, level=2, func=inspect.stack()[0][3])
+    return feat_updated
+
+
+def nonGeomFeatureToNative(feature: Base, fields: "QgsFields", dataStorage):
+    try:
+        exist_feat = QgsFeature()
+        exist_feat.setFields(fields)
+        feat_updated = updateFeat(exist_feat, fields, feature)
+        return feat_updated
+
+    except Exception as e:
+        logToUser(e, level=2, func=inspect.stack()[0][3])
+        return
+
+
+def cadFeatureToNative(
+    feature: Base, fields: dict, sr: arcpy.SpatialReference, dataStorage
+):
+
+    print("04_________CAD Feature To Native____________")
+    feat = {}
+    try:
+        try:
+            speckle_geom = feature["geometry"]  # for created in QGIS Layer type
+        except:
+            speckle_geom = feature  # for created in other software
+
+        if isinstance(speckle_geom, list):
+            arcGisGeom = convertToNativeMulti(speckle_geom, sr, dataStorage)
+        else:
+            arcGisGeom = convertToNative(speckle_geom, sr, dataStorage)
+
+        if arcGisGeom is not None:
+            feat.update({"arcGisGeomFromSpeckle": arcGisGeom})
+        else:
+            return
+
+        try:
+            if "Speckle_ID" not in fields.keys() and feature["id"]:
+                feat.update("Speckle_ID", "TEXT")
+        except:
+            pass
+
+        #### setting attributes to feature
+        feat_updated = updateFeat(feat, fields, feature)
+        # print(feat)
+        # print(fields)
+        return feat_updated
+
+    except Exception as e:
+        logToUser(e, level=2, func=inspect.stack()[0][3])
+        return

speckle_toolbox/esri/toolboxes/speckle/speckle/converter/features/utils.py

@@ -0,0 +1,218 @@
+import inspect
+import random
+from typing import Any, Dict, Union
+from speckle.speckle.converter.layers.utils import getVariantFromValue, traverseDict
+
+from speckle.speckle.utils.panel_logging import logToUser
+
+from specklepy.objects import Base
+
+
+def addFeatVariant(key, variant, value, f):
+    # print("Add feat variant")
+    feat = f
+    try:
+        if variant == "TEXT":
+            value = str(value)[:255]
+            if len(value) > 255:
+                print(len(value))
+                value = value[:255]
+                logToUser(
+                    f'Field "{key}" values are trimmed at 255 characters',
+                    level=2,
+                    func=inspect.stack()[0][3],
+                )
+
+        if value != "NULL" and value != "None":
+            # if key == 'area': print(value); print(type(value)); print(getVariantFromValue(value))
+            if variant == getVariantFromValue(
+                value
+            ):  # or (variant=="FLOAT" and isinstance(value, int)):
+                feat.update({key: value})
+            elif variant == "LONG" and isinstance(
+                value, float
+            ):  # if object has been modified
+                feat.update({key: int(value)})
+            elif variant == "FLOAT" and isinstance(
+                value, int
+            ):  # if object has been modified
+                feat.update({key: float(value)})
+            else:
+                feat.update({key: None})
+        elif (
+            variant == "TEXT"
+            or variant == "FLOAT"
+            or variant == "LONG"
+            or variant == "SHORT"
+        ):
+            feat.update({key: None})
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return feat
+
+
+def addFeatVariant_qgis(key, variant, value, f):
+    try:
+        feat = f
+        if variant == 10:
+            value = str(value)  # string
+
+        if value != "NULL" and value != "None":
+            if variant == getVariantFromValue(value):
+                feat[key] = value
+            elif (
+                isinstance(value, float) and variant == 4
+            ):  # float, but expecting Long (integer)
+                feat[key] = int(value)
+            elif (
+                isinstance(value, int) and variant == 6
+            ):  # int (longlong), but expecting float
+                feat[key] = float(value)
+            else:
+                feat[key] = None
+                # print(key); print(value); print(type(value)); print(variant); print(getVariantFromValue(value))
+        elif isinstance(variant, int):
+            feat[key] = None
+        return feat
+    except Exception as e:
+        logToUser(e, level=2, func=inspect.stack()[0][3])
+        return feat
+
+
+def updateFeat(feat: dict, fields: dict, feature: Base) -> Union[Dict[str, Any], None]:
+    feat_sorted = {}
+    try:
+        for key, variant in fields.items():
+            try:
+                if key == "Speckle_ID":
+                    value = str(feature["id"])
+
+                    feat[key] = value
+
+                    feat = addFeatVariant(key, variant, value, feat)
+                else:
+                    try:
+                        value = feature[key]
+                        # if key == "area": print(feature[key]); print(type(feature[key]))
+                        feat = addFeatVariant(key, variant, value, feat)
+                    except:
+                        value = None
+                        rootName = key.split("_")[0]
+                        # try: # if the root category exists
+                        # if its'a list
+                        if isinstance(feature[rootName], list):
+                            for i in range(len(feature[rootName])):
+                                try:
+                                    newF, newVals = traverseDict(
+                                        {},
+                                        {},
+                                        rootName + "_" + str(i),
+                                        feature[rootName][i],
+                                    )
+                                    for i, (key, value) in enumerate(newVals.items()):
+                                        for k, (x, y) in enumerate(newF.items()):
+                                            if key == x:
+                                                variant = y
+                                                break
+                                        feat = addFeatVariant(key, variant, value, feat)
+                                except Exception as e:
+                                    pass  # print(e)
+                        # except: # if not a list
+                        else:
+                            try:
+                                newF, newVals = traverseDict(
+                                    {}, {}, rootName, feature[rootName]
+                                )
+                                for i, (key, value) in enumerate(newVals.items()):
+                                    for k, (x, y) in enumerate(newF.items()):
+                                        if key == x:
+                                            variant = y
+                                            break
+                                    feat = addFeatVariant(key, variant, value, feat)
+                            except Exception as e:
+                                feat.update({key: None})
+            except Exception as e:
+                feat.update({key: None})
+        feat_sorted = {k: v for k, v in sorted(feat.items(), key=lambda item: item[0])}
+        # print("_________________end of updating a feature_________________________")
+        return feat_sorted
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+
+
+def getPolygonFeatureHeight(
+    feature: "QgsFeature", layer: "QgsVectorLayer", dataStorage: "DataStorage"
+) -> Union[int, float, None]:
+    height = None
+    ignore = False
+    if dataStorage.savedTransforms is not None:
+        for item in dataStorage.savedTransforms:
+            layer_name = item.split("  ->  ")[0].split(" ('")[0]
+            transform_name = item.split("  ->  ")[1].lower()
+            if "ignore" in transform_name:
+                ignore = True
+
+            if layer_name == layer.name():
+                attribute = None
+                if " ('" in item:
+                    attribute = item.split(" ('")[1].split("') ")[0]
+
+                if attribute is None and ignore is False:
+                    logToUser(
+                        "Attribute for extrusion not selected",
+                        level=1,
+                        func=inspect.stack()[0][3],
+                    )
+                    return None
+
+                # print("Apply transform: " + transform_name)
+                if "extrude" in transform_name and "polygon" in transform_name:
+                    # additional check:
+                    try:
+                        if dataStorage.project.crs().isGeographic():
+                            return None
+                    except:
+                        return None
+
+                    try:
+                        existing_height = float(feature[attribute])
+                        if (
+                            existing_height is None or str(feature[attribute]) == "NULL"
+                        ):  # if attribute value invalid
+                            if ignore is True:
+                                return None
+                            else:  # find approximate value
+                                all_existing_vals = [
+                                    f[attribute]
+                                    for f in layer.getFeatures()
+                                    if (
+                                        f[attribute] is not None
+                                        and (
+                                            isinstance(f[attribute], float)
+                                            or isinstance(f[attribute], int)
+                                        )
+                                    )
+                                ]
+                                try:
+                                    if len(all_existing_vals) > 5:
+                                        height_average = all_existing_vals[
+                                            int(len(all_existing_vals) / 2)
+                                        ]
+                                        height = random.randint(
+                                            height_average - 5, height_average + 5
+                                        )
+                                    else:
+                                        height = random.randint(10, 20)
+                                except:
+                                    height = random.randint(10, 20)
+                        else:  # if acceptable value: reading from existing attribute
+                            height = existing_height
+
+                    except:  # if no Height attribute
+                        if ignore is True:
+                            height = None
+                        else:
+                            height = random.randint(10, 20)
+
+    return height

speckle_toolbox/esri/toolboxes/speckle/speckle/converter/geometry/conversions.py

@@ -0,0 +1,357 @@
+from regex import F
+from specklepy.objects import Base
+from specklepy.objects.geometry import (
+    Line,
+    Mesh,
+    Point,
+    Polyline,
+    Curve,
+    Arc,
+    Circle,
+    Polycurve,
+    Ellipse,
+)
+
+import arcpy
+from typing import Any, List, Tuple, Union, Sequence
+
+import inspect
+from specklepy.objects.GIS.geometry import (
+    GisLineElement,
+    GisPointElement,
+    GisPolygonElement,
+)
+from specklepy.objects.GIS.geometry import GisPolygonGeometry
+
+from speckle.speckle.converter.geometry.mesh import meshToNative
+from speckle.speckle.converter.geometry.polygon import (
+    polygonToNative,
+    multiPolygonToNative,
+    polygonToSpeckle,
+    multiPolygonToSpeckle,
+    polygonToSpeckleMesh,
+)
+from speckle.speckle.converter.geometry.utils import (
+    specklePolycurveToPoints,
+    addCorrectUnits,
+)
+from speckle.speckle.converter.geometry.polyline import (
+    anyLineToSpeckle,
+    arcToNative,
+    ellipseToNative,
+    circleToNative,
+    curveToNative,
+    lineToNative,
+    polycurveToNative,
+    polylineToNative,
+    polylineToSpeckle,
+    speckleArcCircleToPoints,
+    multiPolylineToSpeckle,
+)
+from speckle.speckle.converter.geometry.point import (
+    pointToCoord,
+    pointToNative,
+    pointToSpeckle,
+    multiPointToSpeckle,
+)
+
+from speckle.speckle.converter.layers.utils import apply_reproject, findTransformation
+from speckle.speckle.utils.panel_logging import logToUser
+
+import numpy as np
+
+
+def convertToSpeckle(
+    feature, index, layer, data, dataStorage
+) -> Tuple[Union[Base, Sequence[Base], None], int]:
+    """Converts the provided layer feature to Speckle objects"""
+    print("___convertToSpeckle____________")
+    try:
+        iterations = 0
+        layer_sr = data.spatialReference  # if sr.type == "Projected":
+        geomType = data.shapeType  # Polygon, Point, Polyline, Multipoint, MultiPatch
+        featureType = data.featureType
+        projectCRS = dataStorage.project.activeMap.spatialReference
+        units = dataStorage.currentUnits
+        x_form: tuple = findTransformation(geomType, layer_sr, projectCRS, layer)
+        try:
+            [print(p for p in feature.getPart())]
+        except:
+            print(feature.getPart())
+        # geomMultiType = feature.isMultipart
+        hasCurves = feature.hasCurves
+
+        # feature is <geoprocessing describe geometry object object at 0x000002A75D6A4BD0>
+
+        # print(featureType) # e.g. Simple
+        # print(geomType) # e.g. Polygon
+        # geomSingleType = (featureType=="Simple") # Simple,SimpleJunction,SimpleJunction,ComplexEdge,Annotation,CoverageAnnotation,Dimension,RasterCatalogItem
+
+        if geomType == "Point":  # Polygon, Point, Polyline, Multipoint, MultiPatch
+            print("__Point conversion")
+
+            f_shape = apply_reproject(feature, x_form, dataStorage).getPart()
+            if f_shape is None:
+                return None
+            result = [pointToSpeckle(f_shape, feature, layer, dataStorage)]
+            for r in result:
+                r.units = units
+
+            element = GisPointElement(units=units, geometry=result)
+
+        elif geomType == "Multipoint":
+            print("__Multipoint conversion")
+
+            f_shape = apply_reproject(feature, x_form, dataStorage).getPart()
+            if f_shape is None:
+                return None
+            result = [pointToSpeckle(pt, feature, layer, dataStorage) for pt in f_shape]
+            for r in result:
+                r.units = units
+
+            element = GisPointElement(units=units, geometry=result)
+
+        elif geomType == "Polyline":
+            print("__Polyline conversion")
+            # if feature.partCount == 1:
+            #    result = anyLineToSpeckle(
+            #        feature, feature, layer, dataStorage, xform_vars
+            #    )
+            #    result = addCorrectUnits(result, dataStorage)
+            #    result = [result]
+            # else:
+            all_parts = []
+            for part in feature.getPart():
+                all_parts.append(
+                    arcpy.Polyline(
+                        part,
+                        arcpy.Describe(layer.dataSource).SpatialReference,
+                        has_z=True,
+                    )
+                )
+            result = [
+                anyLineToSpeckle(poly, feature, layer, dataStorage, x_form)
+                for poly in all_parts
+            ]
+            for r in result:
+                r = addCorrectUnits(r, dataStorage)
+
+            element = GisLineElement(units=units, geometry=result)
+
+        elif geomType == "Polygon":
+            print("__Polygon conversion")
+            # if feature.partCount > 1:
+            r"""
+            if feature.partCount == 1:
+                result = polygonToSpeckle(
+                    feature, feature, index, layer, dataStorage, xform_vars
+                )
+                result = [result]
+
+            else:
+            """
+            f_shape = apply_reproject(feature, x_form, dataStorage).getPart()
+            result = [
+                polygonToSpeckle(geom, feature, index, layer, dataStorage, x_form)
+                for geom in f_shape
+            ]
+
+            for r in result:
+                if r is None:
+                    continue
+                r.units = units
+                if r.boundary is not None:
+                    r.boundary.units = units
+                if r.voids is not None:
+                    for v in r.voids:
+                        if v is not None:
+                            v.units = units
+                    for v in r.displayValue:
+                        if v is not None:
+                            v.units = units
+            element = GisPolygonElement(units=units, geometry=result)
+
+        elif geomType == "MultiPatch":
+            f_shape = apply_reproject(feature, x_form, dataStorage).getPart()
+            if f_shape is None:
+                return None
+            result = [polygonToSpeckleMesh(f_shape, index, layer, False, dataStorage)]
+            for r in result:
+                if r is None:
+                    continue
+                r.units = units
+                if r.boundary is not None:
+                    r.boundary.units = units
+                if r.voids is not None:
+                    for v in r.voids:
+                        if v is not None:
+                            v.units = units
+                    for v in r.displayValue:
+                        if v is not None:
+                            v.units = units
+            element = GisPolygonElement(units=units, geometry=result)
+            print(element)
+        else:
+            logToUser(
+                "Unsupported or invalid geometry in layer " + layer.name,
+                level=1,
+                func=inspect.stack()[0][3],
+            )
+        try:
+            print(result)
+        except:
+            pass
+        return element, iterations
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None, None
+
+
+def convertToNative(
+    base: Base, sr: arcpy.SpatialReference, dataStorage
+) -> Union[Any, None]:
+    """Converts any given base object to QgsGeometry."""
+    print("___Convert to Native SingleType___")
+    converted = None
+    try:
+        # print(base)
+        conversions = [
+            (Point, pointToNative),
+            (Line, lineToNative),
+            (Polyline, polylineToNative),
+            (Curve, curveToNative),
+            (Arc, arcToNative),
+            (Circle, circleToNative),
+            (Ellipse, ellipseToNative),
+            # (Mesh, meshToNative),
+            (Polycurve, polycurveToNative),
+            # temporary solution for polygons (Speckle has no type Polygon yet)
+        ]
+
+        for conversion in conversions:
+            if isinstance(base, conversion[0]):
+                # print(conversion[0])
+                converted = conversion[1](base, sr, dataStorage)
+                break
+
+        if converted is None:
+            # distinguish normal QGIS polygons and the ones sent as Mesh only
+            try:
+                if isinstance(base, GisPolygonGeometry):
+                    if base.boundary is None:
+                        try:
+                            converted = meshToNative(base.displayValue, sr, dataStorage)
+                        except KeyError as e:
+                            # print(e)
+                            converted = meshToNative(
+                                base["@displayValue"], sr, dataStorage
+                            )
+                    else:
+                        converted = multiPolygonToNative(base, sr, dataStorage)
+                else:
+                    # for older commits
+                    boundary = base.boundary  # will throw exception if not polygon
+                    if boundary is None:
+                        try:
+                            converted = meshToNative(base.displayValue, sr, dataStorage)
+                        except:
+                            converted = meshToNative(
+                                base["@displayValue"], sr, dataStorage
+                            )
+                    elif boundary is not None and isinstance(base, conversion[0]):
+                        converted = multiPolygonToNative(base, sr, dataStorage)
+
+            except (
+                Exception
+            ) as e:  # if no "boundary" found (either old Mesh from QGIS or other object)
+                print(e)
+                try:  # check for a QGIS Mesh
+                    try:
+                        # if sent as Mesh
+                        colors = base.displayValue[0].colors  # will throw exception
+                        if isinstance(base.displayValue[0], Mesh):
+                            converted = meshToNative(
+                                base.displayValue, sr, dataStorage
+                            )  # only called for Meshes created in QGIS before
+                    except:
+                        # if sent as Mesh
+                        colors = base["@displayValue"][0].colors  # will throw exception
+                        if isinstance(base["@displayValue"][0], Mesh):
+                            converted = meshToNative(
+                                base["@displayValue"], sr, dataStorage
+                            )  # only called for Meshes created in QGIS before
+
+                except:  # any other object(
+                    pass
+        if converted is None:
+            converted = multiPolygonToNative(base, sr, dataStorage)
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return converted
+
+
+def multiPointToNative(items: List[Point], sr: arcpy.SpatialReference, dataStorage):
+    print("___Create MultiPoint")
+    features = None
+    try:
+        all_pts = []
+        # example https://pro.arcgis.com/en/pro-app/2.8/arcpy/classes/multipoint.htm
+        for item in items:
+            pt = pointToCoord(item)  # [x, y, z]
+            all_pts.append(arcpy.Point(pt[0], pt[1], pt[2]))
+        # print(all_pts)
+        features = arcpy.Multipoint(arcpy.Array(all_pts))
+        # if len(features)==0: features = None
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return features
+
+
+def multiPolylineToNative(
+    items: List[Polyline], sr: arcpy.SpatialReference, dataStorage
+):
+    print("_______Drawing Multipolylines____")
+    poly = None
+    try:
+        # print(items)
+        poly = None
+        full_array_list = []
+        for item in items:  # will be 1 item
+            pointsSpeckle = []
+            try:
+                pointsSpeckle = item.as_points()
+            except:
+                continue
+            pts = [pointToCoord(pt) for pt in pointsSpeckle]
+
+            if item.closed is True:
+                pts.append(pointToCoord(item.as_points()[0]))
+
+            arr = [arcpy.Point(*coords) for coords in pts]
+            full_array_list.append(arr)
+
+        poly = arcpy.Polyline(arcpy.Array(full_array_list), sr, has_z=True)
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return poly
+
+
+def convertToNativeMulti(items: List[Base], sr: arcpy.SpatialReference, dataStorage):
+    print("___Convert to Native MultiType___")
+    try:
+        first = items[0]
+        if isinstance(first, Point):
+            return multiPointToNative(items, sr, dataStorage)
+        elif isinstance(first, Line) or isinstance(first, Polyline):
+            return multiPolylineToNative(items, sr, dataStorage)
+        elif isinstance(first, Base):
+            try:
+                if first["boundary"] is not None and first["voids"] is not None:
+                    print(first["boundary"])
+                    print(first["voids"])
+                    return multiPolygonToNative(items, sr, dataStorage)
+            except:
+                return None
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None

speckle_toolbox/esri/toolboxes/speckle/speckle/converter/geometry/mesh.py

@@ -0,0 +1,325 @@
+from datetime import datetime
+import os
+import time
+from typing import List
+import arcpy
+import math
+
+from specklepy.objects.geometry import Mesh, Point, Polyline
+from specklepy.objects.other import RenderMaterial
+from specklepy.objects.GIS.geometry import GisPolygonGeometry
+
+
+import inspect
+
+import shapefile
+from shapefile import TRIANGLE_STRIP, TRIANGLE_FAN, OUTER_RING
+
+
+from speckle.speckle.converter.layers.utils import get_scale_factor, getDisplayValueList
+from speckle.speckle.converter.geometry.point import pointToNative
+from speckle.speckle.converter.layers.symbology import featureColorfromNativeRenderer
+from speckle.speckle.converter.layers.utils import get_scale_factor
+from speckle.speckle.utils.panel_logging import logToUser
+from speckle.speckle.plugin_utils.helpers import findOrCreatePath, validateNewFclassName
+
+from panda3d.core import Triangulator
+
+from speckle.speckle.converter.geometry.utils import apply_pt_transform_matrix
+from arcpy.management import CreateFeatureclass
+
+
+def meshToNative(meshes: Mesh, sr, dataStorage=None):
+    """Converts a Speckle Mesh to MultiPatch"""
+
+    result = []
+
+    print("06___________________Mesh to Native")
+    new_path = writeMeshToShp(meshes, "", dataStorage)
+
+    # time.sleep(0.1)
+    try:
+        cursor = arcpy.da.SearchCursor(new_path, "Speckle_ID")
+    except:
+        class_name = new_path.split("\\")[-1]
+        all_classes = arcpy.ListFeatureClasses()
+        validated_class_name = validateNewFclassName(class_name, all_classes)
+        path = dataStorage.workspace  # project.filePath.replace("aprx","gdb") #
+
+        f_class = arcpy.conversion.FeatureClassToFeatureClass(
+            new_path, path, validated_class_name
+        )
+        arcpy.management.DefineProjection(f_class, sr.exportToString())
+        cursor = arcpy.da.SearchCursor(new_path, "Speckle_ID")
+
+    class_shapes = [shp_id[0] for n, shp_id in enumerate(cursor)]
+    del cursor
+    return class_shapes[0]
+
+
+def writeMeshToShp(meshes: List[Mesh], path: str, dataStorage):
+    """Converts a Speckle Mesh to native geometry"""
+    print("06___________________Mesh to Native SHP")
+    try:
+        try:
+            if path == "":
+                path_folder = (
+                    os.path.expandvars(r"%LOCALAPPDATA%")
+                    + "\\Temp\\Speckle_ArcGIS_temp\\"
+                    + datetime.now().strftime("%Y_%m_%d__%H_%M")
+                    + "\\Layers_Speckle\\BIM_layers\\"
+                )
+                findOrCreatePath(path_folder)
+                path = (
+                    path_folder
+                    + "mesh_"
+                    + datetime.now().strftime("%Y_%m_%d__%H_%M_%S")
+                    + "_shp"
+                )
+            w = shapefile.Writer(path)  # + "\\" + str(meshes[0].id))
+        except Exception as e:
+            logToUser(e)
+            return
+
+        w.field("speckle_id", "C")
+
+        for _, geom in enumerate(meshes):
+            meshList: List = getDisplayValueList(geom)
+            print(meshList)
+            w = fill_multi_mesh_parts(w, meshList, geom.id, dataStorage)
+        w.close()
+        return path
+
+    except Exception as e:
+        logToUser(e, level=2, func=inspect.stack()[0][3])
+        return None
+
+
+def fill_multi_mesh_parts(
+    w: shapefile.Writer, meshes: List[Mesh], geom_id: str, dataStorage
+):
+    try:
+        parts_list = []
+        types_list = []
+        for mesh in meshes:
+            if not isinstance(mesh, Mesh):
+                continue
+            try:
+                # print(f"Fill multi-mesh parts # {geom_id}")
+                parts_list_x, types_list_x = deconstructSpeckleMesh(mesh, dataStorage)
+                parts_list.extend(parts_list_x)
+                types_list.extend(types_list_x)
+            except Exception as e:
+                pass
+
+        w.multipatch(parts_list, partTypes=types_list)  # one type for each part
+        w.record(geom_id)
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return w
+
+
+def fill_mesh_parts(w: shapefile.Writer, mesh: Mesh, geom_id: str, dataStorage):
+
+    try:
+        # print(f"Fill mesh parts # {geom_id}")
+        parts_list, types_list = deconstructSpeckleMesh(mesh, dataStorage)
+        w.multipatch(parts_list, partTypes=types_list)  # one type for each part
+        w.record(geom_id)
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return w
+
+
+def deconstructSpeckleMesh(mesh: Mesh, dataStorage):
+    parts_list = []
+    types_list = []
+    try:
+        scale = get_scale_factor(mesh.units)
+
+        count = 0  # sequence of vertex (not of flat coord list)
+        for f in mesh.faces:  # real number of loops will be at least 3 times less
+            try:
+                vertices = mesh.faces[count]
+                if mesh.faces[count] == 0:
+                    vertices = 3
+                if mesh.faces[count] == 1:
+                    vertices = 4
+
+                face = []
+                for i in range(vertices):
+                    index_faces = count + 1 + i
+                    index_vertices = mesh.faces[index_faces] * 3
+                    pt_coords = [
+                        mesh.vertices[index_vertices],
+                        mesh.vertices[index_vertices + 1],
+                        mesh.vertices[index_vertices + 2],
+                    ]
+                    pt_coords_new = apply_pt_transform_matrix(pt_coords, dataStorage)
+                    face.append([scale * coord for coord in pt_coords_new])
+
+                parts_list.append(face)
+                types_list.append(OUTER_RING)
+                count += vertices + 1
+            except:
+                break  # when out of range
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return parts_list, types_list
+
+
+def constructMeshFromRaster(vertices, faces, colors):
+    mesh = None
+    try:
+        mesh = Mesh.create(vertices, faces, colors)
+        mesh.units = "m"
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return mesh
+
+
+def constructMesh(vertices, faces, colors):
+    mesh = None
+    try:
+        mesh = Mesh.create(vertices, faces, colors)
+        mesh.units = "m"
+        material = RenderMaterial()
+        material.diffuse = colors[0]
+        mesh.renderMaterial = material
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return mesh
+
+
+def meshPartsFromPolygon(
+    polyBorder: List[Point],
+    voidsAsPts: List[List[Point]],
+    existing_vert: int,
+    index: int,
+    layer,
+    dataStorage,
+):
+
+    try:
+        # print("__meshPartsFromPolygon__")
+        vertices = []
+        total_vertices = 0
+        # print(layer)
+        try:
+            sr = arcpy.Describe(layer.dataSource).spatialReference
+        except Exception as e:
+            print(e)
+            sr = None
+        # print(sr)
+        coef = 1
+        maxPoints = 5000
+        if len(polyBorder) >= maxPoints:
+            coef = int(len(polyBorder) / maxPoints)
+
+        if (
+            len(voidsAsPts) == 0
+        ):  # only if there is a mesh with no voids and large amount of points
+            # print("mesh with no voids")
+            for k, ptt in enumerate(polyBorder):  # pointList:
+                pt = polyBorder[k * coef]
+                if k < maxPoints:
+                    # if isinstance(pt, QgsPointXY):
+                    #    pt = QgsPoint(pt)
+                    # print(pt)
+                    if isinstance(pt, Point):
+                        x = pt.x
+                        y = pt.y
+                        z = pt.z
+                        # pt = pointToNative(pt, sr, dataStorage).getPart()
+                    else:
+                        x = pt.X
+                        y = pt.Y
+                        z = 0 if math.isnan(pt.Z) else pt.Z
+                    vertices.extend([x, y, z])
+                    total_vertices += 1
+                else:
+                    break
+
+            ran = range(0, total_vertices)
+            faces = [total_vertices]
+            faces.extend([i + existing_vert for i in ran])
+            # else: https://docs.panda3d.org/1.10/python/reference/panda3d.core.Triangulator
+        else:  # if there are voids
+            print("mesh with voids")
+            # if its a large polygon with voids to be triangualted, lower the coef even more:
+            maxPoints = 100
+            if len(polyBorder) >= maxPoints:
+                coef = int(len(polyBorder) / maxPoints)
+
+            trianglator = Triangulator()
+            faces = []
+
+            pt_count = 0
+            # add extra middle point for border
+            for k, ptt in enumerate(polyBorder):  # pointList:
+                pt = polyBorder[k * coef]
+                if k < maxPoints:
+                    if pt_count < len(polyBorder) - 1 and k < (maxPoints - 1):
+                        pt2 = polyBorder[(k + 1) * coef]
+                    else:
+                        pt2 = polyBorder[0]
+
+                    trianglator.addPolygonVertex(trianglator.addVertex(pt.x, pt.y))
+                    vertices.extend([pt.x, pt.y, pt.z])
+                    trianglator.addPolygonVertex(
+                        trianglator.addVertex((pt.x + pt2.x) / 2, (pt.y + pt2.y) / 2)
+                    )
+                    vertices.extend(
+                        [(pt.x + pt2.x) / 2, (pt.y + pt2.y) / 2, (pt.z + pt2.z) / 2]
+                    )
+                    total_vertices += 2
+                    pt_count += 1
+                else:
+                    break
+
+            # add void points
+            for pts in voidsAsPts:
+                trianglator.beginHole()
+
+                coefVoid = 1
+                if len(pts) >= maxPoints:
+                    coefVoid = int(len(pts) / maxPoints)
+                for k, ptt in enumerate(pts):
+                    pt = pts[k * coefVoid]
+                    if k < maxPoints:
+                        trianglator.addHoleVertex(trianglator.addVertex(pt.x, pt.y))
+                        vertices.extend([pt.x, pt.y, pt.z])
+                        total_vertices += 1
+                    else:
+                        break
+
+            trianglator.triangulate()
+            i = 0
+            # print(trianglator.getNumTriangles())
+            while i < trianglator.getNumTriangles():
+                tr = [
+                    trianglator.getTriangleV0(i),
+                    trianglator.getTriangleV1(i),
+                    trianglator.getTriangleV2(i),
+                ]
+                faces.extend(
+                    [
+                        3,
+                        tr[0] + existing_vert,
+                        tr[1] + existing_vert,
+                        tr[2] + existing_vert,
+                    ]
+                )
+                i += 1
+            ran = range(0, total_vertices)
+
+        # print("color")
+        col = featureColorfromNativeRenderer(index, layer)  # (100<<16) + (100<<8) + 100
+        colors = [col for i in ran]  # apply same color for all vertices
+
+        return total_vertices, vertices, faces, colors
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None, None, None, None

speckle_toolbox/esri/toolboxes/speckle/speckle/converter/geometry/point.py

@@ -0,0 +1,144 @@
+import math
+from typing import List
+from specklepy.objects.geometry import Point
+import arcpy
+
+import inspect
+from speckle.speckle.converter.geometry.utils import (
+    apply_pt_offsets_rotation_on_send,
+    transform_speckle_pt_on_receive,
+    apply_pt_transform_matrix,
+)
+
+from speckle.speckle.converter.layers.utils import get_scale_factor
+from speckle.speckle.utils.panel_logging import logToUser
+
+
+def multiPointToSpeckle(geom, feature, layer, multiType: bool, dataStorage):
+    """Converts a Point to Speckle"""
+
+    pointList = []
+    # print(geom) # <geoprocessing describe geometry object object at 0x0000020F1D94AB10>
+    try:
+        if multiType is False:
+            for pt in geom:
+                # print(pt) # 284394.58100903 5710688.11602606 NaN NaN <class 'arcpy.arcobjects.arcobjects.Point'>
+                # print(type(pt))
+                if pt != None:
+                    pointList.append(pointToSpeckle(pt, feature, layer, dataStorage))
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return pointList
+
+
+def pointToSpeckle(pt, feature, layer, dataStorage):
+    """Converts a Point to Speckle"""
+    # print("___Point to Speckle____")
+    # when unset, z() returns "nan"
+    # print(pt) # 4.9046319 52.3592043 NaN NaN
+    # print("____Point to Speckle___")
+    try:
+        r"""
+        if isinstance(pt, arcpy.PointGeometry):
+            x = pt[0]
+            y = pt[1]
+            if len(pt) > 2:
+                z = pt[2]
+            else:
+                z = 0
+        """
+        x = pt.X
+        y = pt.Y
+        if pt.Z:
+            z = pt.Z
+        else:
+            z = 0
+
+        specklePoint = Point(units="m")
+        specklePoint.x = x
+        specklePoint.y = y
+        specklePoint.z = z
+        specklePoint.units = "m"
+        """
+        if feature is not None and layer is not None: # can be if it's a point from raster layer 
+            col = featureColorfromNativeRenderer(feature, layer)
+            specklePoint['displayStyle'] = {}
+            specklePoint['displayStyle']['color'] = col
+        """
+        specklePoint.x, specklePoint.y = apply_pt_offsets_rotation_on_send(
+            x, y, dataStorage
+        )
+        return specklePoint
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None
+
+
+def pointToNative(
+    pt: Point, sr: arcpy.SpatialReference, dataStorage
+) -> arcpy.PointGeometry:
+    """Converts a Speckle Point to QgsPoint"""
+    try:
+        new_pt = scalePointToNative(pt, pt.units, dataStorage)
+        new_pt = apply_pt_transform_matrix(new_pt, dataStorage)
+        newPt = transform_speckle_pt_on_receive(new_pt, dataStorage)
+
+        geom = arcpy.PointGeometry(arcpy.Point(pt.x, pt.y, pt.z), sr, has_z=True)
+        # print(geom)
+        return geom
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None
+
+
+def pointToNativeWithoutTransforms(pt: Point, sr: arcpy.SpatialReference, dataStorage):
+    """Converts a Speckle Point to QgsPoint"""
+    try:
+        new_pt = scalePointToNative(pt, pt.units, dataStorage)
+        new_pt = apply_pt_transform_matrix(new_pt, dataStorage)
+
+        geom = arcpy.PointGeometry(arcpy.Point(pt.x, pt.y, pt.z), sr, has_z=True)
+        # print(geom)
+        return geom
+
+    except Exception as e:
+        logToUser(e, level=2, func=inspect.stack()[0][3])
+        return None
+
+
+def pointToCoord(point: Point) -> List[float]:
+    """Converts a Speckle Point to QgsPoint"""
+    try:
+        # pt = scalePointToNative(point, point.units)
+        pt = point
+        coords = [pt.x, pt.y, pt.z]
+        # print(coords)
+        return coords
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return [None, None, None]
+
+
+def scalePointToNative(point: Point, units: str, dataStorage=None) -> Point:
+    """Scale point coordinates to meters"""
+    try:
+        scaleFactor = get_scale_factor(units)
+        pt = Point(units="m")
+        pt.x = point.x * scaleFactor
+        pt.y = point.y * scaleFactor
+        pt.z = 0 if math.isnan(point.z) else point.z * scaleFactor
+        return pt
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None
+
+
+def addZtoPoint(coords: List):
+    try:
+        if len(coords) == 2:
+            coords.append(0)
+        return coords
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None

speckle_toolbox/esri/toolboxes/speckle/speckle/converter/geometry/polygon.py

@@ -0,0 +1,484 @@
+from typing import List, Sequence, Union
+import arcpy
+import json
+from arcpy.arcobjects.arcobjects import SpatialReference
+
+from specklepy.objects import Base
+from specklepy.objects.GIS.geometry import GisPolygonGeometry
+from specklepy.objects.geometry import Point, Arc, Circle, Polycurve, Polyline, Line
+
+import inspect
+
+from speckle.speckle.converter.geometry.mesh import (
+    constructMesh,
+    constructMeshFromRaster,
+    meshPartsFromPolygon,
+)
+from speckle.speckle.converter.geometry.point import pointToCoord, pointToNative
+from speckle.speckle.converter.layers.symbology import featureColorfromNativeRenderer
+from speckle.speckle.converter.geometry.polyline import (
+    anyLineToSpeckle,
+    polylineFromVerticesToSpeckle,
+    speckleArcCircleToPoints,
+    curveToSpeckle,
+)
+from speckle.speckle.converter.geometry.utils import (
+    speckleBoundaryToSpecklePts,
+    specklePolycurveToPoints,
+)
+from speckle.speckle.utils.panel_logging import logToUser
+
+import math
+from panda3d.core import Triangulator
+
+
+def polygonToSpeckleMesh(geom, index: int, layer, multitype: bool, dataStorage):
+    print("________polygonToSpeckleMesh_____")
+    # print(geom)
+    polygon = GisPolygonGeometry(units="m")
+    try:
+
+        vertices = []
+        faces = []
+        colors = []
+        existing_vert = 0
+
+        for i, p in enumerate(geom):
+            # print("____start enumerate feature")
+            # print(p) #<geoprocessing array object object at 0x0000026796C77110>
+            boundaries, voids = getPolyBoundaryVoids(p, layer, dataStorage)
+            # boundary = boundaries[0]
+            for boundary in boundaries:
+                # print(boundary)
+                # print(voids)
+                polyBorder = speckleBoundaryToSpecklePts(boundary)
+                print(boundary.as_points())
+                # print(polyBorder)
+                if polyBorder is None or (
+                    isinstance(polyBorder, list) and len(polyBorder) < 3
+                ):
+                    continue
+                # print(existing_vert)
+                voidsAsPts = []
+                for v in voids:
+                    pts = speckleBoundaryToSpecklePts(v)
+                    voidsAsPts.append(pts)
+                # print(voidsAsPts)
+                total_vert, vertices_x, faces_x, colors_x = meshPartsFromPolygon(
+                    polyBorder, voidsAsPts, existing_vert, index, layer, dataStorage
+                )
+
+                existing_vert += total_vert
+                vertices.extend(vertices_x)
+                faces.extend(faces_x)
+                colors.extend(colors_x)
+
+        # print("Colors: ")
+        # print(colors)
+        # print(faces)
+        # print(vertices)
+        mesh = constructMesh(vertices, faces, colors)
+        # print(mesh)
+        if mesh is not None:
+            polygon.displayValue = [mesh]
+        else:
+            logToUser(
+                "Mesh creation from Polygon failed. Boundaries will be used as displayValue",
+                level=1,
+                func=inspect.stack()[0][3],
+            )
+        return polygon
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None
+
+
+def getPolyBoundaryVoids(feature, layer, dataStorage, x_form=None):
+    print("__getPolyBoundaryVoids__")
+    voids: List[Union[None, Polyline, Arc, Line, Polycurve]] = []
+    boundary = None
+    pointList = []
+    all_boundaries = []
+    try:
+        for i, pt in enumerate(feature):
+            # print(pt)  # 284394.58100903 5710688.11602606 NaN NaN
+            if pt == None and boundary == None:  # first break
+                boundary = polylineFromVerticesToSpeckle(
+                    pointList, True, feature, layer, dataStorage
+                )
+                pointList = []
+                all_boundaries.append(boundary)
+            elif pt == None and boundary != None:  # breaks btw voids
+                void = polylineFromVerticesToSpeckle(
+                    pointList, True, feature, layer, dataStorage
+                )
+                voids.append(void)
+                pointList = []
+            elif (
+                len(pointList) > 0
+                and pt.X == pointList[0].X
+                and pt.Y == pointList[0].Y
+                and pt.Z == pointList[0].Z
+            ):  # new condition
+                # if multiple boundaries
+                new_boundary = polylineFromVerticesToSpeckle(
+                    pointList, True, feature, layer, dataStorage
+                )
+                pointList = []
+                # print(new_boundary)
+                all_boundaries.append(new_boundary)
+            elif pt != None:  # add points to whatever list (boundary or void)
+                pointList.append(pt)
+
+        if boundary != None and len(pointList) > 0:  # remaining polyline
+            void = polylineFromVerticesToSpeckle(
+                pointList, True, feature, layer, dataStorage
+            )
+            voids.append(void)
+
+        elif boundary is None:  # no voids
+            boundary = polylineFromVerticesToSpeckle(
+                pointList, True, feature, layer, dataStorage
+            )
+            all_boundaries.append(boundary)
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return all_boundaries, voids
+
+
+def multiPolygonToSpeckle(geom, index: str, layer, multiType: bool, dataStorage):
+
+    print("___MultiPolygon to Speckle____")
+    polygon = []
+    try:
+        # print(enumerate(geom.getPart())) # this method ignores curvature and voids
+        # print(json.loads(geom.JSON))
+        # js = json.loads(geom.JSON)['rings']
+        # https://desktop.arcgis.com/en/arcmap/latest/analyze/python/reading-geometries.htm
+        for i, x in enumerate(geom):  # [[x,x,x]
+            # print("Part # " + str(i + 1))
+            # print(x)
+            boundaryFinished = 0
+            arrBoundary = []
+            arrInnerRings = []
+            for ptn in x:  # arcpy.Point
+                if ptn is None:
+                    boundaryFinished += 1
+                    arrInnerRings.append([])  # start of new Inner Ring
+                elif boundaryFinished == 0 and ptn is not None:
+                    arrBoundary.append(ptn)
+                elif boundaryFinished == 1 and ptn is not None:
+                    arrInnerRings[len(arrInnerRings) - 1].append(ptn)
+
+            full_arr = [arrBoundary] + arrInnerRings
+            # print(full_arr)
+            poly = arcpy.Polygon(
+                arcpy.Array(full_arr),
+                arcpy.Describe(layer.dataSource).SpatialReference,
+                has_z=True,
+            )
+            # print(poly) #<geoprocessing describe geometry object object at 0x000002B2D3E338D0>
+            polygon.append(
+                polygonToSpeckle(poly, index, layer, poly.isMultipart, dataStorage)
+            )
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return polygon
+
+
+def polygonToSpeckle(geom, feature, index: int, layer, dataStorage, x_form):
+    """Converts a Polygon to Speckle"""
+    # polygon = Base(units="m")
+    polygon = GisPolygonGeometry(units="m")
+    try:
+        print("___Polygon to Speckle____")
+        # print(geom)  # array
+
+        boundaries, voids = getPolyBoundaryVoids(geom, layer, dataStorage, x_form)
+        boundary = boundaries[0]
+        # print(boundary)
+        # print(voids)
+
+        data = arcpy.Describe(layer.dataSource)
+        sr = data.spatialReference
+
+        if boundary is None:
+            return None
+        polygon.boundary = boundary
+        polygon.voids = voids
+        # polygon.displayValue = [boundary] + voids
+        # print(boundary)
+
+        ############# mesh
+        vertices = []
+        polyBorder = boundary.as_points()
+        total_vertices = 0
+
+        if len(polyBorder) > 2:  # at least 3 points
+            # print("make meshes from polygons")
+            if len(voids) == 0:  # if there is a mesh with no voids
+                # print("no voids")
+                # print(polyBorder)
+                for pt in polyBorder:
+                    if isinstance(pt, Point):
+                        pt = pointToNative(pt, sr, dataStorage).getPart()  # SR unknown
+                    x = pt.X
+                    y = pt.Y
+                    z = 0 if math.isnan(pt.Z) else pt.Z
+                    vertices.extend([x, y, z])
+                    total_vertices += 1
+                # print(vertices)
+                ran = range(0, total_vertices)
+                faces = [total_vertices]
+                faces.extend([i for i in ran])
+                # print(faces)
+                # else: https://docs.panda3d.org/1.10/python/reference/panda3d.core.Triangulator
+            else:
+                # print("voids")
+                # print(polyBorder)
+                trianglator = Triangulator()
+                faces = []
+
+                pt_count = 0
+                # add extra middle point for border
+                for pt in polyBorder:
+                    if pt_count < len(polyBorder) - 1:
+                        pt2 = polyBorder[pt_count + 1]
+                    else:
+                        pt2 = polyBorder[0]
+
+                    trianglator.addPolygonVertex(trianglator.addVertex(pt.x, pt.y))
+                    vertices.extend([pt.x, pt.y, pt.z])
+
+                    # trianglator.addPolygonVertex(trianglator.addVertex((pt.x+pt2.x)/4*3, (pt.y+pt2.y)/4*3))
+                    # vertices.extend([(pt.x+pt2.x)/4*3, (pt.y+pt2.y)/4*3, (pt.z+pt2.z)/4*3])
+
+                    trianglator.addPolygonVertex(
+                        trianglator.addVertex((pt.x + pt2.x) / 2, (pt.y + pt2.y) / 2)
+                    )
+                    vertices.extend(
+                        [(pt.x + pt2.x) / 2, (pt.y + pt2.y) / 2, (pt.z + pt2.z) / 2]
+                    )
+
+                    # trianglator.addPolygonVertex(trianglator.addVertex((pt.x+pt2.x)/4, (pt.y+pt2.y)/4))
+                    # vertices.extend([(pt.x+pt2.x)/4, (pt.y+pt2.y)/4, (pt.z+pt2.z)/4])
+
+                    total_vertices += 2
+                    pt_count += 1
+
+                # add void points
+                for i in range(len(voids)):
+                    trianglator.beginHole()
+
+                    pts = []
+                    if isinstance(voids[i], Circle) or isinstance(voids[i], Arc):
+                        pts = speckleArcCircleToPoints(voids[i])
+                    elif isinstance(voids[i], Polycurve):
+                        pts = specklePolycurveToPoints(voids[i])
+                    elif isinstance(voids[i], Line):
+                        pass
+                    else:
+                        try:
+                            pts = voids[i].as_points()
+                        except:
+                            pass  # if Line
+                    # pts = voids[i].as_points()
+                    for pt in pts:
+                        trianglator.addHoleVertex(trianglator.addVertex(pt.x, pt.y))
+                        vertices.extend([pt.x, pt.y, pt.z])
+                        total_vertices += 1
+
+                trianglator.triangulate()
+                i = 0
+                while i < trianglator.getNumTriangles():
+                    tr = [
+                        trianglator.getTriangleV0(i),
+                        trianglator.getTriangleV1(i),
+                        trianglator.getTriangleV2(i),
+                    ]
+                    faces.extend([3, tr[0], tr[1], tr[2]])
+                    i += 1
+                ran = range(0, total_vertices)
+
+            # print(polygon)
+            col = featureColorfromNativeRenderer(index, layer)
+            colors = [col for i in ran]  # apply same color for all vertices
+            mesh = constructMesh(vertices, faces, colors)
+
+            # print(mesh)
+            if mesh is not None:
+                polygon.displayValue = [mesh]
+            else:
+                logToUser(
+                    "Mesh creation from Polygon failed. Boundaries will be used as displayValue",
+                    level=1,
+                    func=inspect.stack()[0][3],
+                )
+            return polygon
+        else:
+            logToUser(
+                "Not enough points for Polygon boundary",
+                level=1,
+                func=inspect.stack()[0][3],
+            )
+            return None
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None
+
+
+def polygonToNative(
+    poly: Base, sr: arcpy.SpatialReference, dataStorage
+) -> arcpy.Polygon:
+    """Converts a Speckle Polygon base object to QgsPolygon.
+    This object must have a 'boundary' and 'voids' properties.
+    Each being a Speckle Polyline and List of polylines respectively."""
+
+    print("_______Drawing polygons____")
+    polygon = None
+    try:
+        try:
+            poly = poly["geometry"]
+        except:
+            pass
+        # pts = [pointToCoord(pt) for pt in poly["boundary"].as_points()]
+        pointsSpeckle = []
+        if isinstance(poly["boundary"], Circle) or isinstance(poly["boundary"], Arc):
+            pointsSpeckle = speckleArcCircleToPoints(poly["boundary"])
+        elif isinstance(poly["boundary"], Polycurve):
+            pointsSpeckle = specklePolycurveToPoints(poly["boundary"])
+        elif isinstance(poly["boundary"], Line):
+            pass
+        else:
+            try:
+                pointsSpeckle = poly["boundary"].as_points()
+            except:
+                pass  # if Line
+
+        pts = [pointToCoord(pt) for pt in pointsSpeckle]
+        # print(pts)
+
+        outer_arr = [arcpy.Point(*coords) for coords in pts]
+        outer_arr.append(outer_arr[0])
+        geomPart = []
+        try:
+            for void in poly["voids"]:
+                # print(void)
+                # pts = [pointToCoord(pt) for pt in void.as_points()]
+                pointsSpeckle = []
+                if isinstance(void, Circle) or isinstance(void, Arc):
+                    pointsSpeckle = speckleArcCircleToPoints(void)
+                elif isinstance(void, Polycurve):
+                    pointsSpeckle = specklePolycurveToPoints(void)
+                elif isinstance(void, Line):
+                    pass
+                else:
+                    try:
+                        pointsSpeckle = void.as_points()
+                    except:
+                        pass  # if Line
+                pts = [pointToCoord(pt) for pt in pointsSpeckle]
+
+                inner_arr = [arcpy.Point(*coords) for coords in pts]
+                inner_arr.append(inner_arr[0])
+                geomPart.append(arcpy.Array(inner_arr))
+        except:
+            pass
+        geomPart.insert(0, outer_arr)
+        geomPartArray = arcpy.Array(geomPart)
+        polygon = arcpy.Polygon(geomPartArray, sr, has_z=True)
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return polygon
+
+
+def multiPolygonToNative(
+    items: List[Base], sr: arcpy.SpatialReference, dataStorage
+):  # TODO fix multi features
+
+    print("_______Drawing Multipolygons____")
+    polygon = None
+    if not isinstance(items, List):
+        items = [items]
+    try:
+        # print(items)
+        full_array_list = []
+        for item_geom in items:  # will be 1 item
+            try:
+                item_geom = item_geom["geometry"]
+            except:
+                item_geom = [item_geom]
+            for item in item_geom:
+                # print(item)
+                # pts = [pointToCoord(pt) for pt in item["boundary"].as_points()]
+                pointsSpeckle = []
+                if isinstance(item["boundary"], Circle) or isinstance(
+                    item["boundary"], Arc
+                ):
+                    pointsSpeckle = speckleArcCircleToPoints(item["boundary"])
+                elif isinstance(item["boundary"], Polycurve):
+                    pointsSpeckle = specklePolycurveToPoints(item["boundary"])
+                elif isinstance(item["boundary"], Line):
+                    pass
+                else:
+                    try:
+                        pointsSpeckle = item["boundary"].as_points()
+                    except Exception as e:
+                        print(e)  # if Line
+                # print(pointsSpeckle)
+                pts = [pointToCoord(pt) for pt in pointsSpeckle]
+                # print(pts)
+
+                outer_arr = [arcpy.Point(*coords) for coords in pts]
+                if pts[0] != pts[-1]:
+                    outer_arr.append(outer_arr[0])
+                # print("outer border")
+                # print(outer_arr)
+                geomPart = []
+                try:
+                    for void in item["voids"]:
+                        # print("void")
+                        # print(void)
+                        # pts = [pointToCoord(pt) for pt in void.as_points()]
+                        pointsSpeckle = []
+                        if isinstance(void, Circle) or isinstance(void, Arc):
+                            pointsSpeckle = speckleArcCircleToPoints(void)
+                        elif isinstance(void, Polycurve):
+                            pointsSpeckle = specklePolycurveToPoints(void)
+                        elif isinstance(void, Line):
+                            pass
+                        else:
+                            try:
+                                pointsSpeckle = void.as_points()
+                            except:
+                                pass  # if Line
+                        pts = [pointToCoord(pt) for pt in pointsSpeckle]
+
+                        inner_arr = [arcpy.Point(*coords) for coords in pts]
+                        if pts[0] != pts[-1]:
+                            inner_arr.append(inner_arr[0])
+                        geomPart.append(arcpy.Array(inner_arr))
+                except Exception as e:
+                    print(e)
+
+                geomPart.insert(0, arcpy.Array(outer_arr))
+                full_array_list.extend(
+                    geomPart
+                )  # outlines are written one by one, with no separation to "parts"
+                # print(full_array_list)  # array of points
+
+            # print("end of loop1")
+        # print("end of loop2")
+        geomPartArray = arcpy.Array(full_array_list)
+        polygon = arcpy.Polygon(geomPartArray, sr, has_z=True)
+
+        print(polygon)
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+
+    return polygon

speckle_toolbox/esri/toolboxes/speckle/speckle/converter/geometry/polyline.py

@@ -0,0 +1,734 @@
+from math import atan, cos, sin
+import math
+import json
+from typing import List, Union, Tuple
+from specklepy.objects import Base
+from specklepy.objects.geometry import (
+    Box,
+    Vector,
+    Point,
+    Line,
+    Polyline,
+    Curve,
+    Ellipse,
+    Arc,
+    Circle,
+    Polycurve,
+    Plane,
+    Interval,
+)
+import arcpy
+import numpy as np
+
+import inspect
+
+from speckle.speckle.converter.geometry.point import (
+    pointToCoord,
+    pointToSpeckle,
+    addZtoPoint,
+)
+from speckle.speckle.converter.geometry.utils import (
+    speckleArcCircleToPoints,
+    specklePolycurveToPoints,
+)
+from speckle.speckle.converter.layers.utils import apply_reproject, get_scale_factor
+from speckle.speckle.utils.panel_logging import logToUser
+
+
+def circleToSpeckle(center, point):
+    print("___Circle to Speckle____")
+    try:
+        rad = math.sqrt(
+            math.pow((center[0] - point[0]), 2) + math.pow((center[1] - point[1]), 2)
+        )
+        # print(rad)
+        if len(center) > 2:
+            center_z = center[2]
+        else:
+            center_z = 0
+        length = rad * 2 * math.pi
+        domain = [0, length]
+        plane = [center[0], center[1], center_z, 0, 0, 1, 1, 0, 0, 0, 1, 0]
+        units = 3  # "m"
+
+        args = [0] + [rad] + domain + plane + [units]
+        # print(args)
+        c = Circle.from_list(args)
+        c.plane.origin.units = "m"
+        c.units = "m"
+        # print(c)
+        return c
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None
+
+
+def multiPolylineToSpeckle(
+    geom, feature, layer, multiType: bool, dataStorage, xform_vars=None
+):
+
+    print("___MultiPolyline to Speckle____")
+    polyline = []
+    try:
+        print(enumerate(geom.getPart()))
+        for i, x in enumerate(geom.getPart()):
+            poly = arcpy.Polyline(
+                x, arcpy.Describe(layer.dataSource).SpatialReference, has_z=True
+            )
+            print(poly)
+            polyline.append(
+                polylineToSpeckle(poly, feature, layer, poly.isMultipart, dataStorage)
+            )
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return polyline
+
+
+def anyLineToSpeckle(geom, feature, layer, dataStorage, x_form=None):
+    print("___Any line to Speckle____")
+    polyline = None
+    try:
+        pointList = []
+        print(geom.hasCurves)
+        # multiType = feature.isMultipart
+
+        # if multiType is False:
+        try:
+            if geom.hasCurves:
+                print("has curves")
+                # geometry SHAPE@ tokens: https://pro.arcgis.com/en/pro-app/latest/arcpy/get-started/reading-geometries.htm
+                # print(geom.JSON)
+                new_geom = geom.densify("GEODESIC", 0.1)
+            else:
+                new_geom = geom
+        except:  # no curves
+            new_geom = geom
+
+        if x_form is not None:
+            new_geom = apply_reproject(new_geom, x_form, dataStorage).getPart()
+            if new_geom is None:
+                return None
+
+        # print(new_geom)  # describe geometry object
+        for p in new_geom:
+            # print(p)  # array
+            for pt in p:
+                # print(pt)
+                if pt != None:
+                    pointList.append(pt)  # ; print(pt.Z)
+        closed = False
+        if pointList[0] == pointList[len(pointList) - 1]:
+            closed = True
+            pointList = pointList[:-1]
+        polyline = polylineFromVerticesToSpeckle(
+            pointList, closed, feature, layer, dataStorage
+        )
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return polyline
+
+
+def polylineToSpeckle(
+    geom, feature, layer, multiType: bool, dataStorage, xform_vars=None
+):
+    print("___Polyline to Speckle____")
+    polyline = None
+    try:
+        pointList = []
+        print(geom.hasCurves)
+
+        if multiType is False:
+            if geom.hasCurves:
+                print("has curves")
+                # geometry SHAPE@ tokens: https://pro.arcgis.com/en/pro-app/latest/arcpy/get-started/reading-geometries.htm
+                print(geom.JSON)
+                polyline = curveToSpeckle(geom, "Polyline", feature, layer, dataStorage)
+            else:
+                for p in geom:
+                    for pt in p:
+                        if pt != None:
+                            pointList.append(pt)  # ; print(pt.Z)
+                closed = False
+                if pointList[0] == pointList[len(pointList) - 1]:
+                    closed = True
+                    pointList = pointList[:-1]
+                polyline = polylineFromVerticesToSpeckle(
+                    pointList, closed, feature, layer, dataStorage
+                )
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return polyline
+
+
+def polylineFromVerticesToSpeckle(
+    vertices: List[Point], closed: bool, feature, layer, dataStorage
+) -> Polyline:
+    """Converts a Polyline to Speckle"""
+    polyline = Polyline(units="m")
+    try:
+        # print("__polylineFromVerticesToSpeckle")
+        # print(vertices)
+        if isinstance(vertices, list):
+            if len(vertices) > 0 and isinstance(vertices[0], Point):
+                specklePts = vertices
+            else:
+                specklePts = [
+                    pointToSpeckle(pt, feature, layer, dataStorage) for pt in vertices
+                ]  # breaks unexplainably
+        # elif isinstance(vertices, QgsVertexIterator):
+        #    specklePts = [pointToSpeckle(pt, feature, layer) for pt in vertices]
+        else:
+            return None
+
+        specklePts = []
+        for pt in vertices:
+            newPt = pointToSpeckle(pt, feature, layer, dataStorage)
+            specklePts.append(newPt)
+
+        # TODO: Replace with `from_points` function when fix is pushed.
+        polyline.value = []
+        polyline.closed = closed
+        polyline.units = specklePts[0].units
+        for i, point in enumerate(specklePts):
+            if closed and i == len(specklePts) - 1 and specklePts[0] == point:
+                continue  # if we consider the last pt, do not add is coincides with the first (and type is Closed)
+            polyline.value.extend([point.x, point.y, point.z])
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return polyline
+
+
+def arc3ptToSpeckle(p0: List, p1: List, p2: List, feature, layer) -> Arc:
+    print("____arc 3pt to Speckle___")
+    arc = Arc()
+    try:
+        p0 = addZtoPoint(p0)
+        p1 = addZtoPoint(p1)
+        p2 = addZtoPoint(p2)
+        arc.startPoint = pointToSpeckle(arcpy.Point(*p0), feature, layer, dataStorage)
+        arc.midPoint = pointToSpeckle(arcpy.Point(*p1), feature, layer, dataStorage)
+        arc.endPoint = pointToSpeckle(arcpy.Point(*p2), feature, layer, dataStorage)
+        center, radius = getArcCenter(
+            Point.from_list(p0), Point.from_list(p1), Point.from_list(p2)
+        )
+        arc.plane = (
+            Plane()
+        )  # .from_list(Point(), Vector(Point(0, 0, 1)), Vector(Point(0,1,0)), Vector(Point(-1,0,0)))
+        arc.plane.origin = Point.from_list(center)
+        arc.plane.origin.units = "m"
+        arc.units = "m"
+        arc.angleRadians, startAngle, endAngle = getArcRadianAngle(arc)
+
+        arc.radius = radius
+
+        arc.plane.normal = getArcNormal(arc, arc.midPoint)
+
+        # arc.angleRadians = abs(angle1 + angle2)
+        # print(arc.angleRadians)
+
+        # col = featureColorfromNativeRenderer(feature, layer)
+        # arc['displayStyle'] = {}
+        # arc['displayStyle']['color'] = col
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return arc
+
+
+def curveBezierToSpeckle(segmStartCoord, segmEndCoord, knots, feature, layer):
+    print("____bezier curve to Speckle____")
+    try:
+        degree = 3
+        points = [
+            tuple(knots[0]),
+            tuple(segmStartCoord),
+            tuple(knots[1]),
+            tuple(segmEndCoord),
+        ]  # [segmStartCoord, *coords]
+        print(points)
+        num_points = len(points)  # 2
+
+        knot_count = num_points + degree - 1  # 4
+        knots = [0] * knot_count
+        print(knots)
+        for i in range(1, len(knots)):
+            knots[i] = i // 3
+            print(knots[i])
+
+        length = 1  # spline.calc_length()
+        domain = Interval(start=0, end=length, totalChildrenCount=0)
+        points = [tuple(pt) for pt in points]
+        curve = Curve(
+            degree=degree,
+            closed=False,
+            periodic=True if (segmStartCoord == segmEndCoord) else False,
+            points=list(sum(points, ())),  # magic (flatten list of tuples)
+            weights=[1] * num_points,
+            knots=knots,
+            rational=False,
+            area=0,
+            volume=0,
+            length=length,
+            domain=domain,
+            units="m",
+            bbox=Box(area=0.0, volume=0.0),
+        )
+        print(curve)
+        return curve
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None
+
+
+def curveToSpeckle(
+    geom, geomType, feature, layer, dataStorage
+) -> Union[Circle, Arc, Polyline, Polycurve]:
+    print("____curve to Speckle____")
+    boundary = Polycurve(units="m")
+    print(geomType)
+    try:
+        # look for "curvePaths" or "curveRings"[[ (startPt, {arcs, beziers etc}, optional(endPt))],[],...], "rings"
+        # examples: https://developers.arcgis.com/documentation/common-data-types/geometry-objects.htm
+        # e.g. {"hasZ":true,
+        # "curveRings":[[[631307.05960000027,5803698.4477999993,0],{"a":[[631307.05960000027,5803698.4477999993,0],[631307.05960000027,5803414.92656173],0,1]}]],
+        # "spatialReference":{"wkid":32631,"latestWkid":32631}}
+
+        # b - bezier curve (endPt, controlPts)
+        # a - elliptical arc (endPt, centralPt) e.g. for circle: [[[631307.05960000027,5803698.4477999993,0],{"a":[[631307.05960000027,5803698.4477999993,0],[631307.05960000027,5803414.92656173],0,1]}]]
+        # c - circular arc (endPt, throughPt) e.g. [[[633242.45179999992,5803058.0354999993,0],{"c":[[633718.26040000003,5803496.4210000001,0],[633337.75764975848,5803431.9997026781]]},[633242.45179999992,5803058.0354999993,0]]]
+
+        if geomType == "Polyline":
+            boundary.closed = False
+        else:
+            boundary.closed = True
+        segments = []
+
+        for key, val in json.loads(geom.JSON).items():
+            print(key)
+            if key == "curveRings" or key == "curvePaths":
+
+                # boundary.closed = True
+                includesLines = 0
+
+                for segm in val:  # segm: List
+                    print(
+                        segm
+                    )  # e.g. [[631307.05960000027,5803698.4477999993,0], {"a":[[631307.05960000027,5803698.4477999993,0],[631307.05960000027,5803414.92656173],0,1]}]
+                    segmStartCoord: List = addZtoPoint(segm[0])
+
+                    # go through all elements (points, a, c, ...)
+                    for k in range(1, len(segm)):
+                        # e.g. one from the list: "curveRings":[[[631750.87200000044,5803159.6126000006,0],
+                        # {"c":[[632429.8348000003,5803507.1132999994,0],[631988.22772700491,5803532.9008129537]]},
+                        # {"c":[[632590.21970000025,5803127.5355999991,0],[633018.51899157302,5803532.1801161235]]},
+                        # [631750.87200000044,5803159.6126000006,0]]]
+
+                        # if previous segments exist
+                        if len(segments) > 0:
+                            segmOldData = segm[k - 1]
+                            if isinstance(
+                                segmOldData, dict
+                            ):  # get "end point" of previous segment
+                                for key3, val3 in segmOldData.items():
+                                    segmStartCoord: List = addZtoPoint(val2[0])
+                            elif isinstance(segmOldData, list) and isinstance(
+                                segmOldData[0], float
+                            ):
+                                segmStartCoord: List = segmOldData
+                        segmStartCoord = addZtoPoint(segmStartCoord)
+
+                        if isinstance(segm[k], dict):
+                            for key2, val2 in segm[k].items():
+                                if key2 == "a":  # elliptical arc (endPt, centralPt)
+                                    # e.g. {'a': [[633883.1035000002, 5802972.5812, 0], [634028.3379278888, 5802908.342895357], 0, 1, 1.1543577096027686, 473.59966687227444, 0.33531864204900685]}
+                                    segmEndCoord = addZtoPoint(
+                                        val2[0]
+                                    )  # [631307.05960000027,5803698.4477999993,0]
+                                    segmCenter = addZtoPoint(
+                                        val2[1]
+                                    )  # [631307.05960000027,5803414.92656173]
+
+                                    if segmStartCoord == segmEndCoord:
+                                        if len(val2) == 4:
+                                            print("full circle")
+                                            # boundary.closed = True
+                                            segmentLocal = circleToSpeckle(
+                                                segmCenter, segmEndCoord
+                                            )
+                                            segments.append(segmentLocal)
+                                            lastPt = segmEndCoord
+                                            print("segmentLocal:")
+                                            print(segmentLocal)
+                                            print(segmStartCoord)
+                                            print(segmEndCoord)
+                                        else:  # ellipse
+                                            logToUser(
+                                                "SpeckleWarning: ellipse geometry not supported yet",
+                                                level=1,
+                                                func=inspect.stack()[0][3],
+                                            )
+                                            segments = []
+                                            break
+                                    else:  # elliptical curve
+                                        logToUser(
+                                            "SpeckleWarning: ellipse geometry not supported yet",
+                                            level=1,
+                                            func=inspect.stack()[0][3],
+                                        )
+                                        segments = []
+                                        break
+
+                                if key2 == "c":  # circular arc (endPt, throughPt)
+
+                                    segmEndCoord: List = addZtoPoint(
+                                        val2[0]
+                                    )  # [633718.26040000003,5803496.4210000001,0]
+                                    segmThrough: List = addZtoPoint(
+                                        val2[1]
+                                    )  # [633337.7576497585, 5803431.999702678]
+
+                                    segmentLocal = arc3ptToSpeckle(
+                                        segmStartCoord,
+                                        segmThrough,
+                                        segmEndCoord,
+                                        geom,
+                                        layer,
+                                    )
+                                    segments.append(segmentLocal)
+                                    print("segmentLocal:")
+                                    print(segmentLocal)
+                                    print(segmStartCoord)
+                                    print(segmEndCoord)
+                                    lastPt = segmEndCoord
+
+                                if key2 == "b":  # bezier curve (endPt, controlPts)
+                                    logToUser(
+                                        "SpeckleWarning: bezier curve geometry not supported yet",
+                                        level=1,
+                                        func=inspect.stack()[0][3],
+                                    )
+                                    segments = []
+                                    break
+                                    r"""
+                                    segmEndCoord: List = addZtoPoint(val2[0])  # [633718.26040000003,5803496.4210000001,0] 
+                                    #segmThrough: List = val2[1] # [633337.7576497585, 5803431.999702678] 
+                                    coords = val2[1:]
+                                    segmentLocal = curveBezierToSpeckle(segmStartCoord, segmEndCoord, coords, feature, layer) 
+                                    segments.append(segmentLocal)
+                                    print("segmentLocal:")
+                                    print(segmentLocal)
+                                    print(segmStartCoord)
+                                    print(segmEndCoord)
+                                    
+                                    lastPt = segmEndCoord 
+                                    """
+
+                        elif isinstance(segm[k], list) and isinstance(
+                            segm[k][0], float
+                        ):  # add line to point
+                            print("add line")
+                            segm[k] = addZtoPoint(segm[k])
+                            segmentLocal = lineFrom2pt(segmStartCoord, segm[k])
+                            includesLines = 1
+                            segments.append(segmentLocal)
+                            lastPt = segm[k]
+
+                            print("segmentLocal:")
+                            print(segmentLocal)
+                            print(segmentLocal.start)
+                            print(segmentLocal.end)
+
+                        # for the last point
+                        if k == len(segm) - 1 and isinstance(segm[k], list):
+                            print("last element is a point (adding line)")
+                            lastPt = addZtoPoint(lastPt)
+                            if lastPt != segm[0]:
+                                # segmentLocal = lineFrom2pt(lastPt, segm[0])
+                                # segments.append(segmentLocal)
+                                # includesLines = 1
+                                # print("segmentLocal:")
+                                # print(segmentLocal)
+                                # print(segmentLocal.start)
+                                # print(segmentLocal.end)
+                                boundary.closed = True
+                        # pts = speckleArcCircleToPoints(segmentLocal)
+                        # pts.append(segm[k])
+                        # arcgisPts = [arcpy.Point(pt[0], pt[1], pt[2]) for pt in pts]
+                        # segmentLocal = polylineFromVerticesToSpeckle(arcgisPts, True, feature, layer)
+
+        boundary.segments = segments
+        print(segments)
+
+        if len(segments) == 1:
+            boundary = segments[0]
+            # if isinstance(boundary, Arc) or isinstance(boundary, Circle):
+            #    boundary.displayValue = Polyline.from_points(speckleArcCircleToPoints(boundary))
+
+        elif len(segments) > 1:  # and includesLines == 0:
+            # boundary.displayValue = Polyline.from_points(specklePolycurveToPoints(boundary))
+            pass
+            # boundary.closed = True
+        # elif len(segments) > 1 and includesLines == 1:
+        #    print("includes lines!")
+        #    points = specklePolycurveToPoints(boundary)
+        #    boundary = Polyline.from_points(points)
+        else:
+            return None
+
+        # boundary.displayValue = Polyline.from_points(specklePolycurveToPoints(boundary))
+
+        print(boundary)
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return boundary
+
+
+def lineFrom2pt(pt1: List[float], pt2: List[float]):
+    line = Line(units="m")  # .from_list([*pt1, *pt2, *domain])
+    try:
+        pt1 = addZtoPoint(pt1)
+        pt2 = addZtoPoint(pt2)
+        dist = math.sqrt(
+            math.pow((pt2[0] - pt1[0]), 2)
+            + math.pow((pt2[1] - pt1[1]), 2)
+            + math.pow((pt2[2] - pt1[2]), 2)
+        )
+        print(dist)
+        domain = [0, dist, 0, 0]
+        line.start = Point.from_list(pt1)
+        line.end = Point.from_list(pt2)
+        line.start.units = line.end.units = "m"
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return line
+
+
+def polylineToNative(
+    poly: Polyline, sr: arcpy.SpatialReference, dataStorage
+) -> arcpy.Polyline:
+    """Converts a Speckle Polyline to QgsLineString"""
+    print("__ convert polyline to native __")
+    polyline = None
+    try:
+
+        if isinstance(poly, Polycurve):
+            poly = specklePolycurveToPoints(poly)
+        if isinstance(poly, Arc) or isinstance(poly, Circle):
+            try:
+                poly = poly["displayValue"]
+            except:
+                poly = speckleArcCircleToPoints(poly)
+
+        if isinstance(poly, list):
+            pts = [pointToCoord(pt) for pt in poly]
+        else:
+            pts = [pointToCoord(pt) for pt in poly.as_points()]
+
+        if poly.closed is True:
+            pts.append(pointToCoord(poly.as_points()[0]))
+
+        scale = get_scale_factor(poly.units)
+        pts = [[pt[0] * scale, pt[1] * scale, pt[2] * scale] for pt in pts]
+
+        pts_coord_list = [arcpy.Point(*coords) for coords in pts]
+        polyline = arcpy.Polyline(arcpy.Array(pts_coord_list), sr, has_z=True)
+        # print(polyline.JSON)
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return polyline
+
+
+def lineToNative(line: Line, sr: arcpy.SpatialReference, dataStorage) -> arcpy.Polyline:
+    """Converts a Speckle Line to Native"""
+    print("___Line to Native___")
+    try:
+        pts = [pointToCoord(pt) for pt in [line.start, line.end]]
+        scale = get_scale_factor(line.units)
+        pts = [[pt[0] * scale, pt[1] * scale, pt[2] * scale] for pt in pts]
+
+        line = arcpy.Polyline(
+            arcpy.Array([arcpy.Point(*coords) for coords in pts]), sr, has_z=True
+        )
+        return line
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None
+
+
+def curveToNative(
+    poly: Curve, sr: arcpy.SpatialReference, dataStorage
+) -> arcpy.Polyline:
+    """Converts a Speckle Curve to Native"""
+    try:
+        display = poly.displayValue
+        curve = polylineToNative(display, sr, dataStorage)
+        return curve
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None
+
+
+def arcToNative(poly: Arc, sr: arcpy.SpatialReference, dataStorage) -> arcpy.Polyline:
+    """Converts a Speckle Arc to Native"""
+    try:
+        arc = arcToNativePolyline(poly, sr, dataStorage)
+        return arc
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None
+
+
+def ellipseToNative(poly: Ellipse, sr: arcpy.SpatialReference, dataStorage):
+    logToUser("Ellipse geometry is not supported yet", level=1)
+    return
+
+
+def circleToNative(
+    poly: Circle, sr: arcpy.SpatialReference, dataStorage
+) -> arcpy.Polyline:
+    """Converts a Speckle Circle to QgsLineString"""
+    print("___Convert Circle to Native___")
+    curve = None
+    try:
+        points = []
+        angle1 = math.pi / 2
+
+        pointsNum = math.floor(math.pi * 2) * 12
+        if pointsNum < 4:
+            pointsNum = 4
+        points.append(pointToCoord(poly.plane.origin))
+
+        radScaled = poly.radius * get_scale_factor(poly.units)
+        points[0][1] += radScaled
+
+        for i in range(1, pointsNum + 1):
+            k = i / pointsNum  # to reset values from 1/10 to 1
+            if poly.plane.normal.z == 0:
+                normal = 1
+            else:
+                normal = poly.plane.normal.z
+            angle = angle1 + k * math.pi * 2 * normal
+            pt = Point(
+                x=poly.plane.origin.x * get_scale_factor(poly.units)
+                + radScaled * cos(angle),
+                y=poly.plane.origin.y * get_scale_factor(poly.units)
+                + radScaled * sin(angle),
+                z=0,
+            )
+            print(pt)
+            pt.units = "m"
+            points.append(pointToCoord(pt))
+        points.append(points[0])
+
+        scale = get_scale_factor(poly.units)
+        points = [[pt[0] * scale, pt[1] * scale, pt[2] * scale] for pt in points]
+
+        curve = arcpy.Polyline(
+            arcpy.Array([arcpy.Point(*coords) for coords in points]), sr, has_z=True
+        )
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return curve
+
+
+def polycurveToNative(
+    poly: Polycurve, sr: arcpy.SpatialReference, dataStorage
+) -> arcpy.Polyline:
+    points = []
+    curve = None
+    print("___Polycurve to native___")
+
+    try:
+        for i, segm in enumerate(poly.segments):  # Line, Polyline, Curve, Arc, Circle
+            print("___start segment")
+            if isinstance(segm, Line):
+                converted = lineToNative(segm, sr, dataStorage)  # QgsLineString
+            elif isinstance(segm, Polyline):
+                converted = polylineToNative(segm, sr, dataStorage)  # QgsLineString
+            elif isinstance(segm, Curve):
+                converted = curveToNative(segm, sr, dataStorage)  # QgsLineString
+            elif isinstance(segm, Circle):
+                converted = circleToNative(segm, sr, dataStorage)  # QgsLineString
+            elif isinstance(segm, Arc):
+                converted = arcToNativePolyline(segm, sr, dataStorage)  # QgsLineString
+            else:  # either return a part of the curve, of skip this segment and try next
+                logToUser(
+                    f"Part of the polycurve cannot be converted",
+                    level=1,
+                    func=inspect.stack()[0][3],
+                )
+                curve = arcpy.Polyline(
+                    arcpy.Array([arcpy.Point(*coords) for coords in points]),
+                    sr,
+                    has_z=True,
+                )
+                return curve
+            if converted is not None:
+                # print(converted) # <geoprocessing describe geometry object object at 0x000002B2D3E338D0>
+                for part in converted:
+                    # print("Part: ")
+                    # print(part) # <geoprocessing array object object at 0x000002B2D2E09530>
+                    for pt in part:
+                        # print(pt) # 64.4584221540162 5.5 NaN NaN
+                        if pt.Z != None:
+                            pt_z = pt.Z
+                        else:
+                            pt_z = 0
+                        # print(pt_z)
+                        # print(len(points))
+                        if (
+                            len(points) > 0
+                            and pt.X == points[len(points) - 1][0]
+                            and pt.Y == points[len(points) - 1][1]
+                            and pt_z == points[len(points) - 1][2]
+                        ):
+                            pass
+                        else:
+                            points.append(
+                                pointToCoord(Point(x=pt.X, y=pt.Y, z=pt_z, units="m"))
+                            )  # e.g. [[64.4584221540162, 5.499999999999999, 0.0], [64.45461685210796, 5.587155742747657, 0.0]]
+            else:
+                logToUser(
+                    f"Part of the polycurve cannot be converted",
+                    level=1,
+                    func=inspect.stack()[0][3],
+                )
+                curve = arcpy.Polyline(
+                    arcpy.Array([arcpy.Point(*coords) for coords in points]),
+                    sr,
+                    has_z=True,
+                )
+                return curve
+        # print(curve)
+
+        curve = arcpy.Polyline(
+            arcpy.Array([arcpy.Point(*coords) for coords in points]), sr, has_z=True
+        )
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return curve
+
+
+def arcToNativePolyline(
+    poly: Union[Arc, Circle], sr: arcpy.SpatialReference, dataStorage
+):
+    print("__Arc/Circle to native polyline__")
+    curve = None
+    try:
+        pointsSpeckle = speckleArcCircleToPoints(poly)
+        points = [pointToCoord(p) for p in pointsSpeckle]
+
+        scale = get_scale_factor(poly.units)
+        points = [[pt[0] * scale, pt[1] * scale, pt[2] * scale] for pt in points]
+
+        curve = arcpy.Polyline(
+            arcpy.Array([arcpy.Point(*coords) for coords in points]), sr, has_z=True
+        )
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return curve

speckle_toolbox/esri/toolboxes/speckle/speckle/converter/geometry/utils.py

@@ -0,0 +1,426 @@
+import math
+from math import cos, sin, atan
+import numpy as np
+from specklepy.objects.geometry import (
+    Point,
+    Line,
+    Polyline,
+    Circle,
+    Arc,
+    Polycurve,
+    Vector,
+)
+from specklepy.objects import Base
+from typing import List, Tuple, Union
+
+import inspect
+
+from speckle.speckle.utils.panel_logging import logToUser
+
+
+def addCorrectUnits(geom: Base, dataStorage) -> Base:
+    if not isinstance(geom, Base):
+        return None
+    units = dataStorage.currentUnits
+
+    geom.units = units
+    if isinstance(geom, Arc):
+        geom.plane.origin.units = units
+        geom.startPoint.units = units
+        geom.midPoint.units = units
+        geom.endPoint.units = units
+
+    elif isinstance(geom, Polycurve):
+        for s in geom.segments:
+            s.units = units
+            if isinstance(s, Arc):
+                s.plane.origin.units = units
+                s.startPoint.units = units
+                s.midPoint.units = units
+                s.endPoint.units = units
+
+    return geom
+
+
+def apply_pt_offsets_rotation_on_send(
+    x: float, y: float, dataStorage
+) -> Tuple[float, float]:  # on Send
+    try:
+        offset_x = dataStorage.crs_offset_x
+        offset_y = dataStorage.crs_offset_y
+        rotation = dataStorage.crs_rotation
+        if offset_x is not None and isinstance(offset_x, float):
+            x -= offset_x
+        if offset_y is not None and isinstance(offset_y, float):
+            y -= offset_y
+        if (
+            rotation is not None
+            and (isinstance(rotation, float) or isinstance(rotation, int))
+            and -360 < rotation < 360
+        ):
+            a = rotation * math.pi / 180
+            x2 = x * math.cos(a) + y * math.sin(a)
+            y2 = -x * math.sin(a) + y * math.cos(a)
+            x = x2
+            y = y2
+        return x, y
+    except Exception as e:
+        logToUser(e, level=2, func=inspect.stack()[0][3])
+        return x, y
+
+
+def transform_speckle_pt_on_receive(pt_original: Point, dataStorage) -> Point:
+    offset_x = dataStorage.crs_offset_x
+    offset_y = dataStorage.crs_offset_y
+    rotation = dataStorage.crs_rotation
+
+    pt = Point(
+        x=pt_original.x, y=pt_original.y, z=pt_original.z, units=pt_original.units
+    )
+
+    gisLayer = None
+    try:
+        gisLayer = dataStorage.latestHostApp.lower().endswith("gis")
+        applyTransforms = False if (gisLayer and gisLayer is True) else True
+    except Exception as e:
+        print(e)
+        applyTransforms = True
+
+    # for non-GIS layers
+    if applyTransforms is True:
+        if (
+            rotation is not None
+            and (isinstance(rotation, float) or isinstance(rotation, int))
+            and -360 < rotation < 360
+        ):
+            a = rotation * math.pi / 180
+            x2 = pt.x
+            y2 = pt.y
+
+            # if a > 0: # turn counterclockwise on receive
+            x2 = pt.x * math.cos(a) - pt.y * math.sin(a)
+            y2 = pt.x * math.sin(a) + pt.y * math.cos(a)
+
+            pt.x = x2
+            pt.y = y2
+        if (
+            offset_x is not None
+            and isinstance(offset_x, float)
+            and offset_y is not None
+            and isinstance(offset_y, float)
+        ):
+            pt.x += offset_x
+            pt.y += offset_y
+
+    # for GIS layers
+    if gisLayer is True:
+        try:
+            offset_x = dataStorage.current_layer_crs_offset_x
+            offset_y = dataStorage.current_layer_crs_offset_y
+            rotation = dataStorage.current_layer_crs_rotation
+
+            if (
+                rotation is not None
+                and isinstance(rotation, float)
+                and -360 < rotation < 360
+            ):
+                a = rotation * math.pi / 180
+                x2 = pt.x
+                y2 = pt.y
+
+                # if a > 0: # turn counterclockwise on receive
+                x2 = pt.x * math.cos(a) - pt.y * math.sin(a)
+                y2 = pt.x * math.sin(a) + pt.y * math.cos(a)
+
+                pt.x = x2
+                pt.y = y2
+            if (
+                offset_x is not None
+                and isinstance(offset_x, float)
+                and offset_y is not None
+                and isinstance(offset_y, float)
+            ):
+                pt.x += offset_x
+                pt.y += offset_y
+        except Exception as e:
+            print(e)
+
+    return pt
+
+
+def apply_pt_transform_matrix(pt_coords: List, dataStorage) -> List:
+    try:
+        if dataStorage.matrix is not None:
+            b = np.matrix(pt_coords + [1])
+            # print(b)
+            # print(dataStorage.matrix)
+            res = b * dataStorage.matrix
+            x, y, z = res.item(0), res.item(1), res.item(2)
+            return [x, y, z]
+    except Exception as e:
+        pass
+        # print(e)
+    return pt_coords
+
+
+def speckleBoundaryToSpecklePts(
+    boundary: Union[None, Polyline, Arc, Line, Polycurve]
+) -> List[Point]:
+    # print("__speckleBoundaryToSpecklePts__")
+    # add boundary points
+    polyBorder = []
+    try:
+        if isinstance(boundary, Circle) or isinstance(boundary, Arc):
+            polyBorder = speckleArcCircleToPoints(boundary)
+        elif isinstance(boundary, Polycurve):
+            polyBorder = specklePolycurveToPoints(boundary)
+        elif isinstance(boundary, Line):
+            pass
+        else:
+            try:
+                polyBorder = boundary.as_points()
+            except:
+                pass  # if Line or None
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return polyBorder
+
+
+def speckleArcCircleToPoints(poly: Union[Arc, Circle]) -> List[Point]:
+    print("__Arc or Circle to Points___")
+    points = []
+    try:
+        # print(poly.plane)
+        # print(poly.plane.normal)
+        if poly.plane is None or poly.plane.normal.z == 0:
+            normal = 1
+        else:
+            normal = poly.plane.normal.z
+        # print(poly.plane.origin)
+        if isinstance(poly, Circle):
+            interval = 2 * math.pi
+            range_start = 0
+            angle1 = 0
+
+        else:  # if Arc
+            points.append(poly.startPoint)
+            range_start = 0
+
+            # angle1, angle2 = getArcAngles(poly)
+
+            interval, angle1, angle2 = getArcRadianAngle(poly)
+            interval = abs(angle2 - angle1)
+
+            # print(angle1)
+            # print(angle2)
+
+            if (angle1 > angle2 and normal == -1) or (angle2 > angle1 and normal == 1):
+                pass
+            if angle1 > angle2 and normal == 1:
+                interval = abs((2 * math.pi - angle1) + angle2)
+            if angle2 > angle1 and normal == -1:
+                interval = abs((2 * math.pi - angle2) + angle1)
+
+            # print(interval)
+            # print(normal)
+
+        pointsNum = math.floor(abs(interval)) * 12
+        if pointsNum < 4:
+            pointsNum = 4
+
+        for i in range(range_start, pointsNum + 1):
+            k = i / pointsNum  # to reset values from 1/10 to 1
+            angle = angle1 + k * interval * normal
+            # print(k)
+            # print(angle)
+            pt = Point(
+                x=poly.plane.origin.x + poly.radius * cos(angle),
+                y=poly.plane.origin.y + poly.radius * sin(angle),
+                z=0,
+            )
+
+            pt.units = poly.plane.origin.units
+            points.append(pt)
+        if isinstance(poly, Arc):
+            points.append(poly.endPoint)
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return points
+
+
+def getArcRadianAngle(arc: Arc) -> List[float]:
+    try:
+        interval = None
+        normal = arc.plane.normal.z
+        angle1, angle2 = getArcAngles(arc)
+        if angle1 is None or angle2 is None:
+            return None
+        interval = abs(angle2 - angle1)
+
+        if (angle1 > angle2 and normal == -1) or (angle2 > angle1 and normal == 1):
+            pass
+        if angle1 > angle2 and normal == 1:
+            interval = abs((2 * math.pi - angle1) + angle2)
+        if angle2 > angle1 and normal == -1:
+            interval = abs((2 * math.pi - angle2) + angle1)
+        return interval, angle1, angle2
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None, None, None
+
+
+def getArcAngles(poly: Arc) -> Tuple[float]:
+    try:
+        if poly.startPoint.x == poly.plane.origin.x:
+            angle1 = math.pi / 2
+        else:
+            angle1 = atan(
+                abs(
+                    (poly.startPoint.y - poly.plane.origin.y)
+                    / (poly.startPoint.x - poly.plane.origin.x)
+                )
+            )  # between 0 and pi/2
+
+        if (
+            poly.plane.origin.x < poly.startPoint.x
+            and poly.plane.origin.y > poly.startPoint.y
+        ):
+            angle1 = 2 * math.pi - angle1
+        if (
+            poly.plane.origin.x > poly.startPoint.x
+            and poly.plane.origin.y > poly.startPoint.y
+        ):
+            angle1 = math.pi + angle1
+        if (
+            poly.plane.origin.x > poly.startPoint.x
+            and poly.plane.origin.y < poly.startPoint.y
+        ):
+            angle1 = math.pi - angle1
+
+        if poly.endPoint.x == poly.plane.origin.x:
+            angle2 = math.pi / 2
+        else:
+            angle2 = atan(
+                abs(
+                    (poly.endPoint.y - poly.plane.origin.y)
+                    / (poly.endPoint.x - poly.plane.origin.x)
+                )
+            )  # between 0 and pi/2
+
+        if (
+            poly.plane.origin.x < poly.endPoint.x
+            and poly.plane.origin.y > poly.endPoint.y
+        ):
+            angle2 = 2 * math.pi - angle2
+        if (
+            poly.plane.origin.x > poly.endPoint.x
+            and poly.plane.origin.y > poly.endPoint.y
+        ):
+            angle2 = math.pi + angle2
+        if (
+            poly.plane.origin.x > poly.endPoint.x
+            and poly.plane.origin.y < poly.endPoint.y
+        ):
+            angle2 = math.pi - angle2
+
+        return angle1, angle2
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None, None
+
+
+def specklePolycurveToPoints(poly: Polycurve) -> List[Point]:
+    print("_____Speckle Polycurve to points____")
+    points = []
+    try:
+        for segm in poly.segments:
+            # print(segm)
+            pts = []
+            if isinstance(segm, Arc) or isinstance(
+                segm, Circle
+            ):  # or isinstance(segm, Curve):
+                print("Arc or Curve")
+                pts: List[Point] = speckleArcCircleToPoints(segm)
+            elif isinstance(segm, Line):
+                print("Line")
+                pts: List[Point] = [segm.start, segm.end]
+            elif isinstance(segm, Polyline):
+                print("Polyline")
+                pts: List[Point] = segm.as_points()
+
+            points.extend(pts)
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return points
+
+
+def getArcNormal(poly: Arc, midPt: Point):
+    print("____getArcNormal___")
+    try:
+        angle1, angle2 = getArcAngles(poly)
+
+        if midPt.x == poly.plane.origin.x:
+            angle = math.pi / 2
+        else:
+            angle = atan(
+                abs((midPt.y - poly.plane.origin.y) / (midPt.x - poly.plane.origin.x))
+            )  # between 0 and pi/2
+
+        if poly.plane.origin.x < midPt.x and poly.plane.origin.y > midPt.y:
+            angle = 2 * math.pi - angle
+        if poly.plane.origin.x > midPt.x and poly.plane.origin.y > midPt.y:
+            angle = math.pi + angle
+        if poly.plane.origin.x > midPt.x and poly.plane.origin.y < midPt.y:
+            angle = math.pi - angle
+
+        normal = Vector()
+        normal.x = normal.y = 0
+
+        if angle1 > angle > angle2:
+            normal.z = -1
+        if angle1 > angle2 > angle:
+            normal.z = 1
+
+        if angle2 > angle1 > angle:
+            normal.z = -1
+        if angle > angle1 > angle2:
+            normal.z = 1
+
+        if angle2 > angle > angle1:
+            normal.z = 1
+        if angle > angle2 > angle1:
+            normal.z = -1
+
+        print(angle1)
+        print(angle)
+        print(angle2)
+        print(normal)
+        return normal
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None
+
+
+def getArcCenter(p1: Point, p2: Point, p3: Point) -> Tuple[List, float]:
+    # print(p1)
+    try:
+        p1 = np.array(p1.to_list())
+        p2 = np.array(p2.to_list())
+        p3 = np.array(p3.to_list())
+        a = np.linalg.norm(p3 - p2)
+        b = np.linalg.norm(p3 - p1)
+        c = np.linalg.norm(p2 - p1)
+        s = (a + b + c) / 2
+        radius = a * b * c / 4 / np.sqrt(s * (s - a) * (s - b) * (s - c))
+        b1 = a * a * (b * b + c * c - a * a)
+        b2 = b * b * (a * a + c * c - b * b)
+        b3 = c * c * (a * a + b * b - c * c)
+        center = np.column_stack((p1, p2, p3)).dot(np.hstack((b1, b2, b3)))
+        center /= b1 + b2 + b3
+        center = center.tolist()
+        return center, radius
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None, None

speckle_toolbox/esri/toolboxes/speckle/speckle/converter/layers/Layer.py

@@ -0,0 +1,79 @@
+from typing import List, Optional
+from specklepy.objects.base import Base
+
+try: 
+    from speckle.speckle.converter.layers.CRS import CRS
+except:
+    from speckle_toolbox.esri.toolboxes.speckle.speckle.converter.layers.CRS import CRS
+
+
+
+class Layer(Base, chunkable={"elements": 100}):
+    """A GIS Layer"""
+
+    def __init__(
+        self,
+        name: Optional[str] = None,
+        crs: Optional[CRS] = None,
+        datum: Optional[CRS] = None,
+        elements: List[Base] = [],
+        layerType: str = "None",
+        geomType: str = "None",
+        renderer: dict = {},
+        **kwargs
+    ) -> None:
+        super().__init__(**kwargs)
+        self.name = name
+        self.crs = crs
+        self.datum = datum
+        self.collectionType = layerType
+        self.elements = elements
+        self.geomType = geomType
+        self.renderer = renderer 
+
+class VectorLayer(Base, chunkable={"elements": 100}):
+    """A GIS Vector Layer"""
+
+    def __init__(
+        self,
+        name: Optional[str] = None,
+        crs: Optional[CRS] = None,
+        datum: Optional[CRS] = None,
+        elements: List[Base] = [],
+        layerType: str = "None",
+        geomType: str = "None",
+        renderer: dict = {},
+        **kwargs
+    ) -> None:
+        super().__init__(**kwargs)
+        self.name = name
+        self.crs = crs
+        self.datum = datum
+        self.collectionType = layerType
+        self.elements = elements
+        self.geomType = geomType
+        self.renderer = renderer 
+
+class RasterLayer(Base, chunkable={"elements": 100}):
+    """A GIS Raster Layer"""
+
+    def __init__(
+        self,
+        name: Optional[str] = None,
+        crs: Optional[CRS] =None,
+        rasterCrs: Optional[CRS] = None,
+        elements: List[Base] = [],
+        layerType: str = "None",
+        geomType: str = "None",
+        renderer: dict = {},
+        **kwargs
+    ) -> None:
+        super().__init__(**kwargs)
+        self.name = name
+        self.crs = crs
+        self.rasterCrs = rasterCrs
+        self.collectionType = layerType
+        self.elements = elements
+        self.geomType = geomType
+        self.renderer = renderer 
+    

speckle_toolbox/esri/toolboxes/speckle/speckle/converter/layers/__init__.py

@@ -0,0 +1,137 @@
+"""
+Contains all Layer related classes and methods.
+"""
+
+from typing import Any, List, Tuple, Union
+
+import inspect
+
+from speckle.speckle.plugin_utils.helpers import (
+    SYMBOL,
+)
+from speckle.speckle.utils.panel_logging import logToUser
+
+from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
+
+
+def getAllProjLayers(plugin) -> List[arcLayer]:
+    # print("get all project layers")
+    layers = []
+    try:
+        project: ArcGISProject = plugin.project
+        if project.activeMap is not None and isinstance(
+            project.activeMap, Map
+        ):  # if project loaded
+            # print(type(project.activeMap))
+            for layer in project.activeMap.listLayers():
+                if (layer.isFeatureLayer) or layer.isRasterLayer:
+                    layers.append(layer)  # type: 'arcpy._mp.Layer'
+        else:
+            # print(type(project.activeMap))
+            logToUser(
+                "Cannot get Project layers, Project Active Map not loaded or not selected",
+                level=1,
+                func=inspect.stack()[0][3],
+                plugin=plugin.dockwidget,
+            )
+            return None
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return layers
+
+
+def getLayersWithStructure(
+    plugin, bySelection=False
+) -> Tuple[List[arcLayer], List[str]]:
+    """Gets a list of all layers in the map"""
+    layers = []
+    structure = []
+    try:
+        print("___ get layers list ___")
+
+        # issue with getting selected layers: https://community.esri.com/t5/python-questions/determining-selected-layers-in-the-table-of/td-p/252098
+
+        self = plugin.dockwidget
+        project = plugin.project
+        map = project.activeMap
+        if map is None:
+            logToUser(
+                "Project Active Map not loaded or not selected",
+                level=2,
+                func=inspect.stack()[0][3],
+            )
+            return None, None
+
+        if bySelection is True:  # by selection
+            # print("get selected layers")
+            for layer in project.activeMap.listLayers():
+                if layer.visible and (layer.isFeatureLayer or layer.isRasterLayer):
+
+                    # find possibly hidden parent groups
+                    layerGroupsHidden = 0
+                    for group in project.activeMap.listLayers():
+                        if (
+                            layer.longName.startswith(group.longName + "\\")
+                            and group.visible is False
+                        ):
+                            for sub_layer in group.listLayers():
+                                if sub_layer.longName == layer.longName:
+                                    layerGroupsHidden += 1
+                                    break
+                            # .isGroupLayer method is broken: https://community.esri.com/t5/python-questions/arcpy-property-layer-isgrouplayer-not-working-as/td-p/709250
+                            r""" 
+                            if group.isGroupLayer:
+                                print("__groups")
+                                print(group.longName + "_" + str(group.visible))
+                                if layer.longName.startswith(group.longName + "\\"):
+                                    print(group.visible)
+                                    if group.visible is False:
+                                        layerGroupsHidden += 1
+                                        break
+                            """
+                    if layerGroupsHidden == 0:
+                        layers.append(layer)
+                        structure.append(
+                            ("\\".join(layer.longName.split("\\")[:-1]) + "\\").replace(
+                                "\\", SYMBOL
+                            )
+                        )
+            # print("layers selected and saved")
+        else:  # from project data
+            # all_layers_ids = [l.id() for l in project.mapLayers().values()]
+            for item in plugin.dataStorage.current_layers:
+                try:
+                    layerPath = item[1].dataSource
+
+                    found = 0
+
+                    all_layers = getAllProjLayers(plugin)
+                    if all_layers is None:
+                        return None
+                    for l in all_layers:
+                        if l.dataSource == layerPath:
+                            layers.append(l)
+                            structure.append(
+                                "\\".join(l.longName.split("\\")[:-1] + "\\").replace(
+                                    "\\", SYMBOL
+                                )
+                            )
+                            found += 1
+                            break
+                    if found == 0:
+                        logToUser(
+                            f'Saved layer not found: "{item[0]}"',
+                            level=1,
+                            func=inspect.stack()[0][3],
+                        )
+
+                except:
+                    logToUser(
+                        f'Saved layer not found: "{item[0]}"',
+                        level=1,
+                        func=inspect.stack()[0][3],
+                    )
+                    continue
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return layers, structure

speckle_toolbox/esri/toolboxes/speckle/speckle/converter/layers/utils.py

@@ -0,0 +1,817 @@
+import copy
+from datetime import datetime
+import random
+from typing import Dict, Any, List, Tuple, Union
+import json
+import hashlib
+from specklepy.objects import Base
+from specklepy.objects.geometry import Mesh
+from specklepy.objects.other import Collection
+
+import arcpy
+from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
+import os
+
+import inspect
+
+from PyQt5.QtGui import QColor
+
+from speckle.speckle.converter.layers.emptyLayerTemplates import createGroupLayer
+from speckle.speckle.plugin_utils.helpers import findOrCreatePath, SYMBOL
+from speckle.speckle.utils.panel_logging import logToUser
+from speckle.speckle.plugin_utils.helpers import validateNewFclassName
+
+
+# ATTRS_REMOVE = ['geometry','applicationId','bbox','displayStyle', 'id', 'renderMaterial', 'displayMesh', 'displayValue']
+ATTRS_REMOVE = [
+    "speckleTyp",
+    "speckle_id",
+    "geometry",
+    "applicationId",
+    "bbox",
+    "displayStyle",
+    "id",
+    "renderMaterial",
+    "displayMesh",
+    "displayValue",
+]
+
+
+def generate_qgis_app_id(
+    base: Base,
+    layer,
+    f,
+):
+    """Generate unique ID for Vector feature."""
+    return ""
+    try:
+        fieldnames = [str(field.name()) for field in layer.fields()]
+        props = [str(f[prop]) for prop in fieldnames]
+        try:
+            geoms = f.geometry()
+        except Exception as e:
+            geoms = ""
+
+        id_data: str = (
+            layer.id()
+            + str(layer.wkbType())
+            + str(fieldnames)
+            + str(props)
+            + str(geoms)
+        )
+        return hashlib.md5(id_data.encode("utf-8")).hexdigest()
+
+    except Exception as e:
+        logToUser(
+            f"Application ID not generated for feature in layer {layer.name()}: {e}",
+            level=1,
+        )
+        return ""
+
+
+def generate_qgis_raster_app_id(rasterLayer):
+    """Generate unique ID for Raster layer."""
+    return ""
+    try:
+        id_data = str(get_raster_stats(rasterLayer))
+        file_ds = gdal.Open(rasterLayer.source(), gdal.GA_ReadOnly)
+        for i in range(rasterLayer.bandCount()):
+            band = file_ds.GetRasterBand(i + 1)
+            id_data += str(band.ReadAsArray())
+        return hashlib.md5(id_data.encode("utf-8")).hexdigest()
+
+    except Exception as e:
+        logToUser(
+            f"Application ID not generated for layer {rasterLayer.name()}: {e}",
+            level=1,
+        )
+        return ""
+
+
+def findUpdateJsonItemPath(tree: Dict, full_path_str: str):
+    try:
+        new_tree = copy.deepcopy(tree)
+
+        path_list_original = full_path_str.split(SYMBOL)
+        path_list = []
+        for x in path_list_original:
+            if len(x) > 0:
+                path_list.append(x)
+        attr_found = False
+
+        for i, item in enumerate(new_tree.items()):
+            attr, val_dict = item
+
+            if attr == path_list[0]:
+                attr_found = True
+                path_list.pop(0)
+                if len(path_list) > 0:  # if the path is not finished:
+                    all_names = val_dict.keys()
+                    if (
+                        len(path_list) == 1 and path_list[0] in all_names
+                    ):  # already in a tree
+                        return new_tree
+                    else:
+                        branch = findUpdateJsonItemPath(
+                            val_dict, SYMBOL.join(path_list)
+                        )
+                        new_tree.update({attr: branch})
+
+        if (
+            attr_found is False and len(path_list) > 0
+        ):  # create a new branch at the top level
+            if len(path_list) == 1:
+                new_tree.update({path_list[0]: {}})
+                return new_tree
+            else:
+                branch = findUpdateJsonItemPath(
+                    {path_list[0]: {}}, SYMBOL.join(path_list)
+                )
+                new_tree.update(branch)
+        return new_tree
+    except Exception as e:
+        # logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return tree
+
+
+def collectionsFromJson(
+    jsonObj: dict, levels: list, layerConverted, baseCollection: Collection
+):
+    if jsonObj == {} or len(levels) == 0:
+        # print("RETURN")
+        baseCollection.elements.append(layerConverted)
+        return baseCollection
+
+    lastLevel = baseCollection
+    for i, l in enumerate(levels):
+        sub_collection_found = 0
+        for item in lastLevel.elements:
+            # print("___ITEM")
+            # print(l)
+            if item.name == l:
+                # print("___ITEM FOUND")
+                # print(l)
+                lastLevel = item
+                sub_collection_found = 1
+                break
+        if sub_collection_found == 0:
+            # print("___ SUB COLLECTION NOT FOUND")
+            subCollection = Collection(
+                units="m", collectionType="ArcGIS Layer Group", name=l, elements=[]
+            )
+            lastLevel.elements.append(subCollection)
+            lastLevel = lastLevel.elements[
+                len(lastLevel.elements) - 1
+            ]  # reassign last element
+
+        if i == len(levels) - 1:  # if last level
+            lastLevel.elements.append(layerConverted)
+
+    return baseCollection
+
+
+def findAndClearLayerGroup(project: ArcGISProject, newGroupName: str = "", plugin=None):
+    print("find And Clear LayerGroup")
+    try:
+        groupExists = 0
+        for l in project.activeMap.listLayers():
+            if l.longName.startswith(newGroupName + "\\"):
+                if l.isFeatureLayer:
+                    # condition for feature layers:
+                    fields = [f.name for f in arcpy.ListFields(l.dataSource)]
+
+                    if "Speckle_ID" in fields or "speckle_id" in fields:
+                        project.activeMap.removeLayer(l)
+                        groupExists += 1
+                elif l.isRasterLayer:
+                    # condition for raster layers:
+                    if "_Speckle" in l.name:
+                        project.activeMap.removeLayer(l)
+                        groupExists += 1
+
+            elif l.longName == newGroupName:
+                groupExists += 1
+
+        if groupExists == 0:
+            layerGroup = create_layer_group(project, newGroupName, plugin)
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3], plugin=plugin.dockwidget)
+
+
+def create_layer_group(project: ArcGISProject, newGroupName: str, plugin):
+    try:
+        if project.activeMap is not None:
+            # print("try creating the group")
+            # check for full match
+            for l in project.activeMap.listLayers():
+                # print(newGroupName + "  __  " + l.longName)
+                if l.longName == newGroupName and l.isGroupLayer:
+                    layerGroup = l
+                    return layerGroup
+            # check for parent layer
+            for l in project.activeMap.listLayers():
+                # print(newGroupName + "  __  " + l.longName)
+                if (
+                    l.longName == "\\".join(newGroupName.split("\\")[:-1])
+                    and l.isGroupLayer
+                ):
+                    short_name = newGroupName.split("\\")[-1]
+                    new_group = project.activeMap.createGroupLayer(short_name, l)
+                    return new_group
+
+            layerGroup = project.activeMap.createGroupLayer(newGroupName)
+            # print(layerGroup)
+            return layerGroup
+        else:
+            logToUser(
+                "The map didn't fully load, try selecting the project Map or/and refreshing the plugin.",
+                level=1,
+                func=inspect.stack()[0][3],
+            )
+            return
+    except Exception as e:
+        print(e)
+
+
+def getVariantFromValue(value: Any) -> Union[str, None]:
+    # print("_________get variant from value_______")
+    # TODO add Base object
+    res = None
+    try:
+        pairs = [(str, "TEXT"), (float, "FLOAT"), (int, "LONG"), (bool, "SHORT")]  # 10
+        for p in pairs:
+            if isinstance(value, p[0]):
+                res = p[1]
+                try:
+                    if res == "LONG" and (value >= 2147483647 or value <= -2147483647):
+                        # https://pro.arcgis.com/en/pro-app/latest/help/data/geodatabases/overview/arcgis-field-data-types.htm
+                        res = "FLOAT"
+                except Exception as e:
+                    print(e)
+                break
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+
+    return res
+
+
+def colorFromSpeckle(rgb):
+    try:
+        color = QColor.fromRgb(245, 245, 245)
+        if isinstance(rgb, int):
+            r = (rgb & 0xFF0000) >> 16
+            g = (rgb & 0xFF00) >> 8
+            b = rgb & 0xFF
+            color = QColor.fromRgb(r, g, b)
+        return color
+    except Exception as e:
+        logToUser(e, level=2, func=inspect.stack()[0][3])
+        return QColor.fromRgb(245, 245, 245)
+
+
+def getDisplayValueList(geom: Any) -> List:
+    try:
+        # print("___getDisplayValueList")
+        val = []
+        # get list of display values for Meshes
+        if isinstance(geom, Mesh):
+            val = [geom]
+        elif isinstance(geom, List) and len(geom) > 0:
+            if isinstance(geom[0], Mesh):
+                val = geom
+            else:
+                print("not an individual geometry")
+        else:
+            try:
+                val = geom.displayValue  # list
+            except Exception as e:
+                # print(e)
+                try:
+                    val = geom["@displayValue"]  # list
+                except Exception as e:
+                    # print(e)
+                    try:
+                        val = geom.displayMesh
+                    except:
+                        pass
+        return val
+    except Exception as e:
+        # print(e)
+        return []
+
+
+def getLayerGeomType(layer) -> str:
+    return
+
+
+def tryCreateGroupTree(project: ArcGISProject, fullGroupName, plugin=None):
+
+    # CREATE A GROUP "received blabla" with sublayers
+    # print("_________CREATE GROUP TREE: " + fullGroupName)
+
+    # receive_layer_tree: dict = plugin.receive_layer_tree
+    receive_layer_list = fullGroupName.split(SYMBOL)
+    path_list = []
+    for x in receive_layer_list:
+        if len(x) > 0:
+            path_list.append(x)
+    group_to_create_name = path_list[0]
+    layerGroup = create_layer_group(project, group_to_create_name, plugin)
+    path_list.pop(0)
+
+    if len(path_list) > 0:
+        path_list[0] = f"{group_to_create_name}\\{path_list[0]}"
+        layerGroup = tryCreateGroupTree(project, SYMBOL.join(path_list), plugin)
+
+    return layerGroup
+
+
+def validateAttributeName(name: str, fieldnames: List[str]) -> str:
+    try:
+        new_list = [x for x in fieldnames if x != name]
+
+        corrected = name.replace("/", "_").replace(".", "_")
+        if corrected == "id":
+            corrected = "applicationId"
+
+        for i, x in enumerate(corrected):
+            if corrected[0] != "_" and corrected not in new_list:
+                break
+            else:
+                corrected = corrected[1:]
+
+        if len(corrected) <= 1 and len(name) > 1:
+            corrected = "0" + name  # if the loop removed the property name completely
+
+        return corrected
+    except Exception as e:
+        logToUser(e, level=2, func=inspect.stack()[0][3])
+        return
+
+
+def trySaveCRS(crs, streamBranch: str = ""):
+    return
+    try:
+        authid = crs.authid()
+        wkt = crs.toWkt()
+        if authid == "":
+            crs_id = crs.saveAsUserCrs("SpeckleCRS_" + streamBranch)
+            return crs_id
+        else:
+            return crs.srsid()
+    except Exception as e:
+        logToUser(e, level=2, func=inspect.stack()[0][3])
+        return
+
+
+def getLayerAttributes(
+    featuresList: List[Base], attrsToRemove: List[str] = ATTRS_REMOVE
+) -> Dict[str, str]:
+    # print("03________ get layer attributes")
+    fields = {}
+    try:
+        if not isinstance(featuresList, list):
+            features = [featuresList]
+        else:
+            features = featuresList[:]
+        # print(features)
+        all_props = []
+        for feature in features:
+            # get object properties to add as attributes
+            try:
+                dynamicProps = (
+                    feature.attributes.get_dynamic_member_names()
+                )  # for 2.14 onwards
+            except:
+                dynamicProps = feature.get_dynamic_member_names()
+
+            for att in ATTRS_REMOVE:
+                try:
+                    dynamicProps.remove(att)
+                except ValueError:
+                    pass
+
+            dynamicProps.sort()
+
+            # add field names and variands
+            for name in dynamicProps:
+                # if name not in all_props: all_props.append(name)
+
+                try:
+                    value = feature.attributes[name]
+                except:
+                    value = feature[name]
+                variant = getVariantFromValue(value)
+                # if name == 'area': print(value); print(variant)
+                if not variant:
+                    variant = None  # LongLong #4
+
+                # go thought the dictionary object
+                if value and isinstance(value, list):
+                    # all_props.remove(name) # remove generic dict name
+                    for i, val_item in enumerate(value):
+                        newF, newVals = traverseDict(
+                            {}, {}, name + "_" + str(i), val_item
+                        )
+
+                        for i, (k, v) in enumerate(newF.items()):
+                            if k not in all_props:
+                                all_props.append(k)
+                            if k not in fields.keys():
+                                fields.update({k: v})
+                            else:  # check if the field was empty previously:
+                                oldVariant = fields[k]
+                                # replace if new one is NOT Float (too large integers)
+                                # if oldVariant != "FLOAT" and v == "FLOAT":
+                                #    fields.update({k: v})
+                                # replace if new one is NOT LongLong or IS String
+                                if oldVariant != "TEXT" and v == "TEXT":
+                                    fields.update({k: v})
+
+                # add a field if not existing yet
+                else:  # if str, Base, etc
+                    newF, newVals = traverseDict({}, {}, name, value)
+
+                    for i, (k, v) in enumerate(newF.items()):
+                        if k not in all_props:
+                            all_props.append(k)
+                        if k not in fields.keys():
+                            fields.update({k: v})  # if variant is known
+                        else:  # check if the field was empty previously:
+                            oldVariant = fields[k]
+                            # replace if new one is NOT Float (too large integers)
+                            # print(oldVariant, v)
+                            # if oldVariant == "LONG" and v == "FLOAT":
+                            #    fields.update({k: v})
+                            # replace if new one is NOT LongLong or IS String
+                            if oldVariant != "TEXT" and v == "TEXT":
+                                fields.update({k: v})
+                            # print(fields)
+        # replace all empty ones wit String
+        all_props.append("Speckle_ID")
+        for name in all_props:
+            if name not in fields.keys():
+                fields.update({name: "TEXT"})
+        # print(fields)
+        # fields_sorted = {k: v for k, v in sorted(fields.items(), key=lambda item: item[0])}
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return fields
+
+
+def traverseDict(newF: dict, newVals: dict, nam: str, val: Any):
+    try:
+        if isinstance(val, dict):
+            for i, (k, v) in enumerate(val.items()):
+                newF, newVals = traverseDict(newF, newVals, nam + "_" + k, v)
+        elif isinstance(val, Base):
+            dynamicProps = val.get_dynamic_member_names()
+            for att in ATTRS_REMOVE:
+                try:
+                    dynamicProps.remove(att)
+                except:
+                    pass
+            dynamicProps.sort()
+
+            item_dict = {}
+            for prop in dynamicProps:
+                item_dict.update({prop: val[prop]})
+
+            for i, (k, v) in enumerate(item_dict.items()):
+                newF, newVals = traverseDict(newF, newVals, nam + "_" + k, v)
+        else:
+            var = getVariantFromValue(val)
+            if var is None:
+                var = "TEXT"
+                val = str(val)
+            # print(var)
+            newF.update({nam: var})
+            newVals.update({nam: val})
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return newF, newVals
+
+
+def get_scale_factor(units: str) -> float:
+    unit_scale = {
+        "meters": 1.0,
+        "centimeters": 0.01,
+        "millimeters": 0.001,
+        "inches": 0.0254,
+        "feet": 0.3048,
+        "kilometers": 1000.0,
+        "mm": 0.001,
+        "cm": 0.01,
+        "m": 1.0,
+        "km": 1000.0,
+        "in": 0.0254,
+        "ft": 0.3048,
+        "yd": 0.9144,
+        "mi": 1609.340,
+    }
+    if units is not None and units.lower() in unit_scale.keys():
+        return unit_scale[units]
+    logToUser(
+        f"Units {units} are not supported. Meters will be applied by default.",
+        level=0,
+        func=inspect.stack()[0][3],
+    )
+    return 1.0
+
+
+def findTransformation(
+    geomType,
+    layer_sr: arcpy.SpatialReference,
+    projectCRS: arcpy.SpatialReference,
+    selectedLayer: arcLayer,
+) -> Tuple:
+    # apply transformation if needed
+    try:
+        print("___findTransformation___")
+        print(layer_sr.name)
+        print(projectCRS.name)
+        tr0 = tr1 = tr2 = tr_custom = None
+        if (
+            geomType != "Point"
+            and geomType != "Polyline"
+            and geomType != "Polygon"
+            and geomType != "Multipoint"
+            and geomType != "MultiPatch"
+        ):
+            try:
+                logToUser(
+                    "Unsupported or invalid geometry in layer " + selectedLayer.name,
+                    level=2,
+                    func=inspect.stack()[0][3],
+                )
+            except:
+                logToUser(
+                    "Unsupported or invalid geometry",
+                    level=2,
+                    func=inspect.stack()[0][3],
+                )
+            return layer_sr, None, None, None, None
+
+        if layer_sr.name == projectCRS.name:
+            return layer_sr, None, None, None, None
+
+        # get list of transforms and find similar one
+        transformations = arcpy.ListTransformations(layer_sr, projectCRS)
+
+        customTransformName = (
+            hashlib.md5(layer_sr.exportToString().encode("utf-8")).hexdigest()
+            + "_To_"
+            + hashlib.md5(projectCRS.exportToString().encode("utf-8")).hexdigest()
+        )
+        print(customTransformName)
+        for tr in transformations:
+            print(tr)
+            if tr == customTransformName:
+                tr0 = tr
+                print("found")
+                return layer_sr, tr0, tr1, tr2, tr_custom
+
+        if tr0 is None:  # if no valid transforms found
+            try:  # try find intermediate transforms
+                midSr = arcpy.SpatialReference("WGS 1984")  # GCS_WGS_1984
+                tr1 = arcpy.ListTransformations(layer_sr, midSr)[0]
+                tr2 = arcpy.ListTransformations(midSr, projectCRS)[0]
+            except Exception as e:  # create custom transform in nothing else works
+                print(e)
+                layer_sr = arcpy.SpatialReference(text=layer_sr.exportToString())
+                projectCRS = arcpy.SpatialReference(text=projectCRS.exportToString())
+                customGeoTransfm = "GEOGTRAN[METHOD['Geocentric_Translation'],PARAMETER['X_Axis_Translation',''],PARAMETER['Y_Axis_Translation',''],PARAMETER['Z_Axis_Translation','']]"
+                try:
+                    arcpy.management.CreateCustomGeoTransformation(
+                        customTransformName, layer_sr, projectCRS, customGeoTransfm
+                    )
+                except Exception as e:  # if already exists
+                    logToUser(
+                        f"Spatial Transformation cannot be created: {e}",
+                        level=2,
+                        func=inspect.stack()[0][3],
+                    )
+                tr_custom = customTransformName
+                print(tr_custom)
+            return layer_sr, tr0, tr1, tr2, tr_custom
+        r"""
+        else:
+            print("else")
+            # choose equation based instead of file-based/grid-based method,
+            # to be consistent with QGIS: https://desktop.arcgis.com/en/arcmap/latest/map/projections/choosing-an-appropriate-transformation.htm
+            selecterTr = {}
+            for tr in transformations:
+                if "NTv2" not in tr and "NADCON" not in tr:
+                    set1 = set(
+                        layer_sr.name.split("_") + projectCRS.name.split("_")
+                    )
+                    set2 = set(tr.split("_"))
+                    diff = len(set(set1).symmetric_difference(set2))
+                    selecterTr.update({tr: diff})
+            selecterTr = dict(
+                sorted(selecterTr.items(), key=lambda item: item[1])
+            )
+            tr0 = list(selecterTr.keys())[0]
+            # print(tr0)
+        """
+
+    except Exception as e:
+        logToUser(
+            f"Spatial Transformation not found for layer {selectedLayer.name}: {e}",
+            level=2,
+            func=inspect.stack()[0][3],
+        )
+        return layer_sr, None, None, None, None
+
+
+def apply_reproject(f_shape, transforms: Tuple, dataStorage):
+    try:
+        layer_sr, tr0, tr1, tr2, tr_custom = transforms
+        projectCRS = dataStorage.project.activeMap.spatialReference
+        # reproject geometry using chosen transformstion(s)
+        if tr0 is not None:
+            ptgeo1 = f_shape.projectAs(projectCRS, tr0)
+            f_shape = ptgeo1
+        elif tr1 is not None and tr2 is not None:
+            midSr = arcpy.SpatialReference("WGS 1984")
+            ptgeo1 = f_shape.projectAs(midSr, tr1)
+            ptgeo2 = ptgeo1.projectAs(projectCRS, tr2)
+            f_shape = ptgeo2
+            r"""
+            elif tr_custom is not None:
+                print(tr_custom)
+                ptgeo1 = f_shape.projectAs(projectCRS, tr_custom)
+                f_shape = ptgeo1
+            """
+        else:
+            projectCRS = arcpy.SpatialReference(text=projectCRS.exportToString())
+            ptgeo1 = f_shape.projectAs(projectCRS)
+            f_shape = ptgeo1
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+    return f_shape
+
+
+def traverseDictByKey(d: Dict, key: str = "", result=None) -> Dict:
+    print("__traverse")
+    try:
+        result = None
+        # print(d)
+        for k, v in d.items():
+
+            try:
+                v = json.loads(v)
+            except:
+                pass
+            if isinstance(v, dict):
+                # print("__dict__")
+                if k == key:
+                    print("__break loop")
+                    result = v
+                    return result
+                else:
+                    result = traverseDictByKey(v, key, result)
+                    if result is not None:
+                        return result
+            if isinstance(v, list):
+                for item in v:
+                    # print(item)
+                    if isinstance(item, dict):
+                        result = traverseDictByKey(item, key, result)
+                        if result is not None:
+                            return result
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None
+    # print("__result is: ____________")
+    # return result
+
+
+def hsv_to_rgb(listHSV):
+    try:
+        h, s, v = listHSV[0], listHSV[1], listHSV[2]
+        if s == 0.0:
+            v *= 255
+            return (v, v, v)
+        i = int(h * 6.0)  # XXX assume int() truncates!
+        f = (h * 6.0) - i
+        p, q, t = (
+            int(255 * (v * (1.0 - s))),
+            int(255 * (v * (1.0 - s * f))),
+            int(255 * (v * (1.0 - s * (1.0 - f)))),
+        )
+        v *= 255
+        i %= 6
+        if i == 0:
+            return (v, t, p)
+        if i == 1:
+            return (q, v, p)
+        if i == 2:
+            return (p, v, t)
+        if i == 3:
+            return (p, q, v)
+        if i == 4:
+            return (t, p, v)
+        if i == 5:
+            return (v, p, q)
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return (0, 0, 0)
+
+
+def cmyk_to_rgb(c, m, y, k, cmyk_scale, rgb_scale=255):
+    try:
+        r = rgb_scale * (1.0 - c / float(cmyk_scale)) * (1.0 - k / float(cmyk_scale))
+        g = rgb_scale * (1.0 - m / float(cmyk_scale)) * (1.0 - k / float(cmyk_scale))
+        b = rgb_scale * (1.0 - y / float(cmyk_scale)) * (1.0 - k / float(cmyk_scale))
+        return r, g, b
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return 0, 0, 0
+
+
+def newLayerGroupAndName(
+    layerName: str, streamBranch: str, project: ArcGISProject
+) -> str:
+    print("___new Layer Group and Name")
+    layerGroup = None
+    newGroupName = f"{streamBranch}"
+    try:
+        # CREATE A GROUP "received blabla" with sublayers
+        print(newGroupName)
+        for l in project.activeMap.listLayers():
+            if l.longName == newGroupName:
+                layerGroup = l
+                break
+
+        # find a layer with a matching name in the "latest" group
+        newName = (
+            f'{streamBranch.split("_")[len(streamBranch.split("_"))-1]}_{layerName}'
+        )
+
+        all_layer_names = []
+        layerExists = 0
+        for l in project.activeMap.listLayers():
+            if l.longName.startswith(newGroupName + "\\"):
+                all_layer_names.append(l.longName)
+        # print(all_layer_names)
+        print(newName)
+
+        newName = validateNewFclassName(newName, all_layer_names, streamBranch + "\\")
+
+        print(newName)
+        return newName, layerGroup
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None, None
+
+
+def curvedFeatureClassToSegments(layer) -> str:
+    print("___densify___")
+    try:
+        data = arcpy.Describe(layer.dataSource)
+        dataPath = data.catalogPath
+        print(dataPath)
+        newPath = dataPath + "_backup"
+
+        arcpy.management.CopyFeatures(
+            dataPath, newPath
+        )  # features copied like this do not preserve curved segments
+
+        arcpy.edit.Densify(
+            in_features=newPath,
+            densification_method="ANGLE",
+            max_angle=0.01,
+            max_vertex_per_segment=100,
+        )  # https://pro.arcgis.com/en/pro-app/latest/tool-reference/editing/densify.htm
+        print(newPath)
+        return newPath
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None
+
+
+def validate_path(path: str, plugin) -> str:
+    """If our path contains a DB name, make sure we have a valid DB name and not a standard file name."""
+    # https://github.com/EsriOceans/btm/commit/a9c0529485c9b0baa78c1f094372c0f9d83c0aaf
+    dirname, file_name = os.path.split(path)
+    # print(dirname)
+    # print(file_name)
+    file_base = os.path.splitext(file_name)[0]
+    if dirname == "":
+        # a relative path only, relying on the workspace
+        dirname = plugin.workspace
+    path_ext = os.path.splitext(dirname)[1].lower()
+    if path_ext in [".mdb", ".gdb", ".sde"]:
+        # we're working in a database
+        file_name = arcpy.ValidateTableName(
+            file_base
+        )  # e.g. add a letter in front of the name
+    validated_path = os.path.join(dirname, file_name)
+    # msg("validated path: %s; (from %s)" % (validated_path, path))
+    return validated_path

speckle_toolbox/esri/toolboxes/speckle/speckle/plugin_utils/helpers.py

@@ -0,0 +1,218 @@
+import os
+from typing import List
+import inspect
+
+SYMBOL = "_x_x_"
+UNSUPPORTED_PROVIDERS = ["WFS", "wms", "wcs", "vectortile"]
+
+
+def get_scale_factor(units: str, dataStorage) -> float:
+    scale_to_meter = get_scale_factor_to_meter(units)
+    if dataStorage is not None:
+        scale_back = scale_to_meter / get_scale_factor_to_meter(
+            dataStorage.currentUnits
+        )
+        return scale_back
+    else:
+        return scale_to_meter
+
+
+def get_scale_factor_to_meter(units: str) -> float:
+    try:
+        unit_scale = {
+            "meters": 1.0,
+            "centimeters": 0.01,
+            "millimeters": 0.001,
+            "inches": 0.0254,
+            "feet": 0.3048,
+            "kilometers": 1000.0,
+            "mm": 0.001,
+            "cm": 0.01,
+            "m": 1.0,
+            "km": 1000.0,
+            "in": 0.0254,
+            "ft": 0.3048,
+            "yd": 0.9144,
+            "mi": 1609.340,
+        }
+        if (
+            units is not None
+            and isinstance(units, str)
+            and units.lower() in unit_scale.keys()
+        ):
+            return unit_scale[units]
+        try:
+            from speckle.speckle.utils.panel_logging import logToUser
+
+            logToUser(
+                f"Units {units} are not supported. Meters will be applied by default.",
+                level=1,
+                func=inspect.stack()[0][3],
+            )
+            return 1.0
+        except:
+            print(
+                f"Units {units} are not supported. Meters will be applied by default."
+            )
+            return 1.0
+    except Exception as e:
+        try:
+            from speckle.speckle.utils.panel_logging import logToUser
+
+            logToUser(
+                f"{e}. Meters will be applied by default.",
+                level=2,
+                func=inspect.stack()[0][3],
+            )
+            return 1.0
+        except:
+            print(f"{e}. Meters will be applied by default.")
+            return 1.0
+
+
+def getAppName(name: str) -> str:
+    new_name = ""
+    for i, x in enumerate(str(name)):
+        if x.lower() in [a for k, a in enumerate("abcdefghijklmnopqrstuvwxyz")]:
+            new_name += x
+        else:
+            break
+    return new_name
+
+
+def findOrCreatePath(path: str):
+    if not os.path.exists(path):
+        os.makedirs(path)
+
+
+def removeSpecialCharacters(text: str) -> str:
+    new_text = (
+        text.replace("[", "_")
+        .replace("]", "_")
+        .replace(".", "_")
+        .replace(" ", "_")
+        .replace("-", "_")
+        .replace("—", "_")
+        .replace("(", "_")
+        .replace(")", "_")
+        .replace(":", "_")
+        .replace("\\", "_")
+        .replace("/", "_")
+        .replace('"', "_")
+        .replace("&", "_")
+        .replace("@", "_")
+        .replace("$", "_")
+        .replace("%", "_")
+        .replace("^", "_")
+    )
+    # new_text = text.encode('iso-8859-1', errors='ignore').decode('utf-8')
+    return new_text
+
+
+def splitTextIntoLines(text: str = "", number: int = 40) -> str:
+    print("__splitTextIntoLines")
+    print(text)
+    msg = ""
+    try:
+        if len(text) > number:
+            try:
+                for i, x in enumerate(text):
+                    msg += x
+                    if i != 0 and i % number == 0:
+                        msg += "\n"
+            except Exception as e:
+                print(e)
+        else:
+            msg = text
+    except Exception as e:
+        print(e)
+        print(text)
+
+    return msg
+
+
+def jsonFromList(jsonObj: dict, levels: list):
+    # print("jsonFromList")
+    if len(levels) == 0:
+        return jsonObj
+    lastLevel = jsonObj
+    for l in levels:
+        # print(lastLevel)
+        try:
+            lastLevel = lastLevel[l]
+        except:
+            lastLevel.update({l: {}})
+    # print(jsonObj)
+    return jsonObj
+
+
+def validateNewFclassName(
+    newName: str, all_layer_names: List[str], prefix: str = ""
+) -> str:
+
+    fixed_name = newName
+
+    if (prefix + fixed_name) in all_layer_names:
+
+        layerNameCreated = 0
+        for index, letter in enumerate("234567890abcdefghijklmnopqrstuvwxyz"):
+            if ((prefix + fixed_name) + "_" + letter) not in all_layer_names:
+                fixed_name += "_" + letter
+                layerNameCreated += 1
+                break
+        if layerNameCreated == 0:
+            for index, letter in enumerate("234567890abcdefghijklmnopqrstuvwxyz"):
+                test_fixed_name = validateNewFclassName(
+                    (fixed_name + "_" + letter), all_layer_names, prefix
+                )
+                if (prefix + test_fixed_name) not in all_layer_names:
+                    fixed_name = test_fixed_name
+                    layerNameCreated += 1
+                    break
+                # else: layerNameCreated +=1
+
+        if layerNameCreated == 0:
+            pass  # logToUser('Feature class name already exists', level=2, func = inspect.stack()[0][3])
+            # return fixed_name
+
+    return fixed_name
+
+
+def findFeatColors(fetColors, f):
+
+    colorFound = 0
+    try:  # get render material from any part of the mesh (list of items in displayValue)
+        for k, item in enumerate(f.displayValue):
+            try:
+                fetColors.append(item.renderMaterial.diffuse)
+                colorFound += 1
+                break
+            except:
+                pass
+        if colorFound == 0:
+            fetColors.append(f.renderMaterial.diffuse)
+    except:
+        try:
+            for k, item in enumerate(f["@displayValue"]):
+                try:
+                    fetColors.append(item.renderMaterial.diffuse)
+                    colorFound += 1
+                    break
+                except:
+                    pass
+            if colorFound == 0:
+                fetColors.append(f.renderMaterial.diffuse)
+        except:
+            # the Mesh itself has a renderer
+            try:  # get render material from any part of the mesh (list of items in displayValue)
+                fetColors.append(f.renderMaterial.diffuse)
+                colorFound += 1
+            except:
+                try:
+                    fetColors.append(f.displayStyle.color)
+                    colorFound += 1
+                except:
+                    pass
+    if colorFound == 0:
+        fetColors.append(None)
+    return fetColors

speckle_toolbox/esri/toolboxes/speckle/speckle/plugin_utils/install_message.py

@@ -1,13 +1,13 @@
 # -*- coding: utf-8 -*-
-
+r'''
 import arcpy
 
 class Toolbox(object):
     def __init__(self):
         """Define the toolbox (the name of the toolbox is the name of the
         .pyt file)."""
-        self.label = "Speckle Toolbox"
-        self.alias = "speckle_toolbox_"  
+        self.label = "Speckle something"
+        self.alias = "speckle_toolbox"  
         self.tools = [Speckle]    
 
 class Speckle(object):
@@ -45,4 +45,4 @@ class Speckle(object):
     def execute(self, parameters): 
         return
     
-    
+'''    

speckle_toolbox/esri/toolboxes/speckle/speckle/plugin_utils/object_utils.py

@@ -0,0 +1,365 @@
+import time
+from typing import Any, Callable, List, Optional
+
+import inspect
+import numpy as np
+
+from specklepy.objects import Base
+from specklepy.objects.GIS.layers import VectorLayer, RasterLayer, Layer
+
+
+from speckle.speckle.converter.layers.utils import findUpdateJsonItemPath
+from speckle.speckle.plugin_utils.helpers import SYMBOL, removeSpecialCharacters
+from speckle.speckle.utils.panel_logging import logToUser
+from speckle.speckle.converter.layers.layer_conversions import (
+    geometryLayerToNative,
+    layerToNative,
+    nonGeometryLayerToNative,
+)
+
+import arcpy
+
+SPECKLE_TYPES_TO_READ = [
+    "Objects.Geometry.",
+    "Objects.BuiltElements.",
+    "IFC",
+]  # will properly traverse and check for displayValue
+
+
+def traverseObject(
+    plugin,
+    base: Base,
+    callback: Optional[Callable[[Base, str, Any], bool]],
+    check: Optional[Callable[[Base], bool]],
+    streamBranch: str,
+    nameBase: str = "",
+):
+    try:
+        if check and check(base):
+            res = callback(base, streamBranch, nameBase, plugin) if callback else False
+            if res:
+                return
+        memberNames = base.get_member_names()
+        for name in memberNames:
+            if name in ["id", "applicationId", "units", "speckle_type"]:
+                continue
+            try:  # some of the static members cannot be called via [""]
+                base[name]
+            except KeyError:
+                continue
+
+            if (
+                nameBase == SYMBOL + "ArcGIS commit"
+                or nameBase == SYMBOL + "QGIS commit"
+            ):
+                name_pass = getBaseValidName(base, name)
+            else:
+                name_pass = nameBase + SYMBOL + getBaseValidName(base, name)
+            # check again
+            if (
+                name_pass == SYMBOL + "ArcGIS commit"
+                or name_pass == SYMBOL + "QGIS commit"
+            ):
+                name_pass = ""
+
+            traverseValue(plugin, base[name], callback, check, streamBranch, name_pass)
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+
+
+def traverseValue(
+    plugin,
+    value: Any,
+    callback: Optional[Callable[[Base, str, Any], bool]],
+    check: Optional[Callable[[Base], bool]],
+    streamBranch: str,
+    name: str,
+):
+    if isinstance(value, Base):
+        traverseObject(plugin, value, callback, check, streamBranch, name)
+    if isinstance(value, List):
+        for item in value:
+            traverseValue(plugin, item, callback, check, streamBranch, name)
+
+
+def callback(base: Base, streamBranch: str, nameBase: str, plugin) -> bool:
+
+    try:
+        if (
+            isinstance(base, VectorLayer)
+            or isinstance(base, Layer)
+            or isinstance(base, RasterLayer)
+            or base.speckle_type.endswith("VectorLayer")
+            or base.speckle_type.endswith("RasterLayer")
+        ):
+            layerToNative(base, streamBranch, nameBase, plugin)
+        else:
+            loopObj(base, "", streamBranch, plugin, [])
+        return True
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+
+
+def getBaseValidName(base: Base, name: str) -> str:
+    name_pass = name
+    search = 0
+    try:
+        if name == "elements" and isinstance(base[name], list):
+            search = 1
+    except:
+        pass
+    try:
+        if name == "displayValue" or name == "@displayValue":
+            search = 1
+    except:
+        pass
+    try:
+        if name == "definition" and isinstance(base[name], Base):
+            search = 1
+    except:
+        pass
+
+    try:
+        if search == 1:
+            try:
+                if (
+                    (base["name"], str)
+                    and len(base["name"]) > 1
+                    and base["name"] != "null"
+                ):
+                    name_pass = base["name"]
+                else:
+                    raise Exception
+            except:
+                try:
+                    if (
+                        (base["Name"], str)
+                        and len(base["Name"]) > 1
+                        and base["Name"] != "null"
+                    ):
+                        name_pass = base["Name"]
+                    else:
+                        raise Exception
+                except:
+                    try:
+                        if (
+                            (base["type"], str)
+                            and len(base["type"]) > 1
+                            and base["type"] != "null"
+                        ):
+                            name_pass = base["type"]
+                        else:
+                            raise Exception
+                    except:
+                        try:
+                            if (
+                                (base["category"], str)
+                                and len(base["category"]) > 1
+                                and base["category"] != "null"
+                            ):
+                                name_pass = base["category"]
+                            else:
+                                raise Exception
+                        except:
+                            name_pass = name
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+
+    return name_pass
+
+
+def loopObj(
+    base: Base, baseName: str, streamBranch: str, plugin, used_ids, matrix=None
+):
+    try:
+        # dont loop primitives
+        if not isinstance(base, Base):
+            return
+        # print(base)
+        memberNames = base.get_member_names()
+
+        baseName_pass = removeSpecialCharacters(baseName)
+
+        plugin.receive_layer_tree = findUpdateJsonItemPath(
+            plugin.receive_layer_tree, streamBranch + SYMBOL + baseName_pass
+        )
+
+        for name in memberNames:
+            if name in ["id", "applicationId", "units", "speckle_type"]:
+                continue
+            # skip if traversal goes to displayValue of an object, that will be readable anyway:
+
+            if (
+                name == "displayValue" or name == "@displayValue"
+            ) and base.speckle_type.startswith(tuple(SPECKLE_TYPES_TO_READ)):
+                continue
+
+            try:
+                if (
+                    "View" in base[name].speckle_type
+                    or "RevitMaterial" in base[name].speckle_type
+                ):
+                    continue
+            except:
+                pass
+
+            name_pass = baseName_pass + SYMBOL + getBaseValidName(base, name)
+
+            if base[name] is not None:
+                if name.endswith("definition"):
+                    # print("___Definition object: " + name)
+                    try:
+                        matrixList = base["transform"].matrix
+                        try:
+                            matrix2 = np.matrix(matrixList).reshape(4, 4)
+                            matrix2 = matrix2.transpose()
+                            if matrix2 is None:
+                                geometryLayerToNative(
+                                    [base[name]],
+                                    name,
+                                    base.id,
+                                    streamBranch,
+                                    plugin,
+                                    None,
+                                )
+
+                            else:  # both not None
+                                if matrix is not None:
+                                    matrix = matrix2 * matrix
+                                else:  # matrix is None
+                                    matrix = matrix2
+                                # print(matrix)
+                                geometryLayerToNative(
+                                    [base[name]],
+                                    name_pass,
+                                    base.id,
+                                    streamBranch,
+                                    plugin,
+                                    matrix,
+                                )
+
+                        except:
+                            matrix = None
+                        time.sleep(0.3)
+                    except Exception as e:
+                        print(f"ERROR: {e}")
+                loopVal(
+                    base[name],
+                    name_pass,
+                    base.id,
+                    streamBranch,
+                    plugin,
+                    used_ids,
+                    matrix,
+                )
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+
+
+def loopVal(
+    value: Any, name: str, val_id: str, streamBranch: str, plugin, used_ids, matrix=None
+):  # "name" is the parent object/property/layer name
+    try:
+        name = removeSpecialCharacters(name)
+        if isinstance(value, Base):
+            try:  # loop through objects with Speckletype prop, but don't go through parts of Speckle Geometry object
+                if (
+                    "View" in value.speckle_type
+                    or "RevitMaterial" in value.speckle_type
+                ):
+                    return
+
+                if not value.speckle_type.startswith("Objects.Geometry."):
+                    loopObj(value, name, streamBranch, plugin, used_ids, matrix)
+                elif value.id not in used_ids:  # if geometry
+                    used_ids.append(value.id)
+                    loopVal(
+                        [value], name, value.id, streamBranch, plugin, used_ids, matrix
+                    )
+            except:
+                loopObj(value, name, streamBranch, plugin, used_ids, matrix)
+
+        elif isinstance(value, List):
+            streamBranch = removeSpecialCharacters(streamBranch)
+
+            objectListConverted = 0
+            for i, item in enumerate(value):
+                if not isinstance(item, Base):
+                    continue
+
+                used_ids.append(item.id)
+                loopVal(item, name, item.id, streamBranch, plugin, used_ids, matrix)
+
+                if not isinstance(item, Base):
+                    continue
+                if "View" in item.speckle_type or "RevitMaterial" in item.speckle_type:
+                    continue
+
+                if item.speckle_type and item.speckle_type.startswith("IFC"):
+                    # keep traversing infinitely, just don't run repeated conversion for the same list of objects
+                    try:
+                        if (
+                            item["displayValue"] is not None
+                            and objectListConverted == 0
+                        ):
+                            geometryLayerToNative(
+                                value, name, val_id, streamBranch, plugin
+                            )
+                            time.sleep(0.3)
+                            objectListConverted += 1
+                    except:
+                        try:
+                            if (
+                                item["@displayValue"] is not None
+                                and objectListConverted == 0
+                            ):
+                                geometryLayerToNative(
+                                    value, name, val_id, streamBranch, plugin
+                                )
+                                time.sleep(0.3)
+                                objectListConverted += 1
+                        except:
+                            pass
+                elif item.speckle_type and item.speckle_type.endswith(".ModelCurve"):
+                    if item["baseCurve"] is not None:
+                        geometryLayerToNative(value, name, val_id, streamBranch, plugin)
+                        time.sleep(0.3)
+                        break
+                elif (
+                    plugin.dataStorage.latestHostApp.lower().endswith("excel")
+                    or item.speckle_type == "Objects.Organization.DataTable"
+                ):
+                    # should be before the check for "BuiltElements"
+                    nonGeometryLayerToNative(value, name, val_id, streamBranch, plugin)
+                    time.sleep(0.3)
+                    break
+                elif item.speckle_type and (
+                    item.speckle_type == "Objects.Geometry.Mesh"
+                    or item.speckle_type == "Objects.Geometry.Brep"
+                    or item.speckle_type.startswith("Objects.BuiltElements.")
+                ):
+                    geometryLayerToNative(value, name, val_id, streamBranch, plugin)
+                    time.sleep(0.3)
+                    break
+                elif (
+                    item.speckle_type
+                    and item.speckle_type != "Objects.Geometry.Mesh"
+                    and item.speckle_type != "Objects.Geometry.Brep"
+                    and item.speckle_type.startswith("Objects.Geometry.")
+                ):  # or item.speckle_type == 'Objects.BuiltElements.Alignment'):
+                    geometryLayerToNative(value, name, val_id, streamBranch, plugin)
+                    time.sleep(0.3)
+                    break
+                elif item.speckle_type:
+                    try:
+                        if item["baseLine"] is not None:
+                            geometryLayerToNative(
+                                value, name, val_id, streamBranch, plugin
+                            )
+                            time.sleep(0.3)
+                            break
+                    except Exception as e:
+                        pass
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])

speckle_toolbox/esri/toolboxes/speckle/speckle/plugin_utils/testing_from_file.py

@@ -0,0 +1,73 @@
+#python speckle_toolbox\esri\toolboxes\speckle\plugin_utils\testing_from_file.py
+import arcpy
+from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
+
+import json
+import os
+
+from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
+from arcpy.management import (CreateFeatureclass, MakeFeatureLayer,
+                              AddFields, AlterField, DefineProjection )
+
+##################################################### get example layers from the project #######
+path = r'C:\Users\katri\Documents\ArcGIS\Projects\MyProject\MyProject.gdb'
+arcpy.env.workspace = path
+project = ArcGISProject(path.replace("gdb","aprx"))
+active_map = project.listMaps()[0] #.activeMap
+all_layers = []
+
+layerPolygon = None
+layerPolyline = None
+layerPoint = None
+layerMultiPoint = None
+#get layer of interest
+for layer in active_map.listLayers(): 
+   if layer.isFeatureLayer or layer.isRasterLayer: 
+        all_layers.append(layer)
+        data = arcpy.Describe(layer.dataSource)
+        if layer.isFeatureLayer:
+            geomType = data.shapeType
+            if geomType == "Polygon" and layerPolygon is None: layerPolygon = layer 
+            if geomType == "Polyline" and layerPolyline is None: layerPolyline = layer 
+            if geomType == "Point" and layerPoint is None: layerPoint = layer 
+            if geomType == "Multipoint" and layerMultiPoint is None: layerMultiPoint = layer 
+
+root_path = "\\".join(project.filePath.split("\\")[:-1])
+#path_style = root_path + '\\layer_speckle_symbology.lyrx'
+path_style2 = root_path + '\\layer_speckle_symbology2.lyrx'
+
+for layer in active_map.listLayers(): 
+    if layer.longName == layerPolygon.longName:
+        layerPolygon = layer
+        break
+
+print(layerPolygon.dataSource)
+r'''
+source = str(layerPolygon.dataSource).split('\\')
+
+layerPolygon = arcpy.ApplySymbologyFromLayer_management(
+                        in_layer= str(layerPolygon.dataSource), 
+                        in_symbology_layer=path_style2, 
+                        update_symbology='UPDATE')[0]
+'''
+#vl2 = MakeFeatureLayer(layerPolygon.dataSource, 'someName').getOutput(0)
+#active_map.addLayer(arcpy.mp.LayerFile(path_style2))
+
+################## reset symbology if needed:
+
+sym = layerPolygon.symbology
+print(sym.renderer.type)
+sym.updateRenderer('UniqueValueRenderer')
+print(sym.renderer.type)
+layerPolygon.symbology = sym
+print(sym.renderer.type)
+r'''
+sym = layerPolygon.symbology
+print(sym.renderer.type)
+sym.updateRenderer('UniqueValueRenderer')
+layerPolygon.symbology = sym
+print(sym.updateRenderer('UniqueValueRenderer'))
+print(layerPolygon.symbology.renderer.type)
+# SimpleRenderer, GraduatedColorsRenderer, GraduatedSymbolsRenderer, UnclassedColorsRenderer, UniqueValueRenderer 
+'''
+project.save() 

speckle_toolbox/esri/toolboxes/speckle/speckle/plugin_utils/threads.py

@@ -0,0 +1,63 @@
+
+import sys
+import trace
+import threading
+
+class KThread(threading.Thread):
+    """A subclass of threading.Thread, with a kill()
+    method."""
+    # https://web.archive.org/web/20130503082442/http://mail.python.org/pipermail/python-list/2004-May/281943.html 
+    def __init__(self, *args, **keywords):
+        threading.Thread.__init__(self, *args, **keywords)
+        self.killed = False
+
+    def start(self):
+        """Start the thread."""
+        self.__run_backup = self.run
+        self.run = self.__run      # Force the Thread to install our trace.
+        threading.Thread.start(self)
+
+    def __run(self):
+        """Hacked run function, which installs the trace."""
+        sys.settrace(self.globaltrace)
+        self.__run_backup()
+        self.run = self.__run_backup
+
+    def globaltrace(self, frame, why, arg):
+        if why == 'call':
+            return self.localtrace
+        else:
+            return None
+
+    def localtrace(self, frame, why, arg):
+        if self.killed:
+            if why == 'line':
+                raise SystemExit()
+        return self.localtrace
+
+    def kill(self):
+        self.killed = True
+
+class KillableThread(threading.Thread):
+    # is NOT running in the background 
+    # https://stackoverflow.com/questions/323972/is-there-any-way-to-kill-a-thread 
+    def __init__(self, sleep_interval=1):
+        super().__init__()
+        self._kill = threading.Event()
+        self._interval = sleep_interval
+
+    def run(self):
+        while True:
+            print("Do Something")
+
+            # If no kill signal is set, sleep for the interval,
+            # If kill signal comes in while sleeping, immediately
+            #  wake up and handle
+            is_killed = self._kill.wait(self._interval)
+            if is_killed:
+                break
+
+        print("Killing Thread")
+
+    def kill(self):
+        self._kill.set()

speckle_toolbox/esri/toolboxes/speckle/speckle/plugin_utils/unit_tests.py

@@ -0,0 +1,13 @@
+
+import arcpy
+from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
+
+import json
+import os
+
+from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
+from arcpy.management import (CreateFeatureclass, MakeFeatureLayer,
+                              AddFields, AlterField, DefineProjection )
+
+import unittest 
+

speckle_toolbox/esri/toolboxes/speckle/speckle/utils/logger.py

@@ -0,0 +1,74 @@
+from PyQt5.QtWidgets import QMessageBox
+from PyQt5 import QtCore
+import arcpy
+
+try:
+    from speckle.speckle.plugin_utils.helpers import splitTextIntoLines
+except:
+    from speckle_toolbox.esri.toolboxes.speckle.speckle.plugin_utils.helpers import (
+        splitTextIntoLines,
+    )
+
+import inspect
+
+
+def logToUser(msg: str, func=None, level: int = 2, plugin=None, url="", blue=False):
+    # print("Log to user")
+    # print(msg)
+
+    msg = str(msg)
+    dockwidget = plugin
+
+    try:
+        if url == "" and blue is False:  # only for info messages
+            msg = addLevelSymbol(msg, level)
+            if func is not None:
+                msg += "::" + str(func)
+        writeToLog(msg, level)
+
+        if dockwidget is None:
+            return
+
+        new_msg = splitTextIntoLines(msg, 70)
+
+        dockwidget.msgLog.addButton(new_msg, level=level, url=url, blue=blue)
+
+    except Exception as e:
+        print(e)
+        return
+
+
+def logToUserWithAction(msg: str, level: int = 0, plugin=None, url=""):
+    print("Log to user with action")
+    return
+    msg = str(msg)
+    dockwidget = plugin
+    if dockwidget is None:
+        return
+    try:
+        new_msg = splitTextIntoLines(msg, 70)
+        dockwidget.msgLog.addButton(new_msg, level=level, url=url)
+        writeToLog(new_msg, level)
+    except Exception as e:
+        print(e)
+        return
+
+
+def addLevelSymbol(msg: str, level: int):
+    if level == 0:
+        msg = "🛈 " + msg
+    if level == 1:
+        msg = "⚠️ " + msg
+    if level == 2:
+        msg = "❗ " + msg
+    return msg
+
+
+def writeToLog(msg: str = "", level: int = 2):
+    print(msg)
+    if level == 0:
+        arcpy.AddMessage(msg)
+    if level == 1:
+        arcpy.AddWarning(msg)
+    if level == 2:
+        arcpy.AddError(msg)

speckle_toolbox/esri/toolboxes/speckle/speckle/utils/panel_logging.py

@@ -0,0 +1,128 @@
+"""Logging Utility Module for Speckle QGIS"""
+
+import inspect
+from typing import Union
+import webbrowser
+
+import arcpy
+
+
+def logToUser(
+    msg: Union[str, Exception],
+    func=None,
+    level: int = 2,
+    plugin=None,
+    url="",
+    blue=False,
+    report=False,
+):
+    from speckle.specklepy_qt_ui.qt_ui.utils.logger import logToUser as logToUser_UI
+
+    msg = str(msg)
+    print(msg + "::" + str(func))
+    logToUser_UI(msg, func, level, plugin, url, blue, report)
+    logger.writeToLog(msg.replace("\n", ". ") + " " + url, level, func)
+
+
+class Logging:
+    """Holds utility methods for logging messages to QGIS"""
+
+    qgisInterface = None
+
+    def __init__(self, iface) -> None:
+        self.qgisInterface = iface
+
+    def log(self, message: str, level: int = 0):
+        """Logs a specific message to the Speckle messages panel."""
+        try:
+            if level == 0:
+                arcpy.AddMessage(message)
+            elif level == 1:
+                arcpy.AddWarning(message)
+            # elif level == 2: 3 error will quit pluging
+            #    arcpy.AddError(message)
+        except Exception as e:
+            try:
+                logToUser(e, level=2, func=inspect.stack()[0][3])
+            except:
+                pass
+
+    def btnClicked(url):
+        try:
+            if url == "":
+                return
+            webbrowser.open(url, new=0, autoraise=True)
+        except Exception as e:
+            pass
+
+    def logToUserWithAction(
+        self,
+        message: str,
+        action_text: str,
+        url: str = "",
+        level: int = 0,
+        duration: int = 120,
+    ):
+        self.log(message, level)
+        return
+        if not self.qgisInterface:
+            return
+        try:
+            from qgis.core import Qgis
+            from qgis.PyQt.QtWidgets import QPushButton
+
+            if level == 0:
+                level = Qgis.Info
+            elif level == 1:
+                level = Qgis.Warning
+            elif level == 2:
+                level = Qgis.Critical
+
+            widget = self.qgisInterface.messageBar().createMessage("Speckle", message)
+            button = QPushButton(widget)
+            button.setText(action_text)
+            button.pressed.connect(lambda: self.btnClicked(url))
+            widget.layout().addWidget(button)
+            self.qgisInterface.messageBar().pushWidget(widget, level, duration)
+        except ImportError:
+            pass
+
+    def logToUserPanel(
+        self,
+        message: str,
+        level: int = 0,
+        duration: int = 20,
+        func=None,
+        plugin=None,
+    ):
+        """Logs a specific message to the user in QGIS"""
+        return
+        self.log(message, level)
+
+        if not self.qgisInterface:
+            return
+        try:
+            from qgis.core import Qgis
+
+            if level == 0:
+                level = Qgis.Info
+            if level == 1:
+                level = Qgis.Warning
+            if level == 2:
+                level = Qgis.Critical
+
+            if self.qgisInterface:
+                self.qgisInterface.messageBar().pushMessage(
+                    "Speckle", message, level=level, duration=duration
+                )
+        except ImportError:
+            pass
+
+    def writeToLog(self, msg: str = "", level: int = 2, func=None, plugin=None):
+        msg = str(msg)
+        if func is not None and func != "None":
+            msg += "::" + str(func)
+        self.log(msg, level)
+
+
+logger = Logging(None)

speckle_toolbox/esri/toolboxes/speckle/speckle/utils/project_vars.py

@@ -0,0 +1,616 @@
+from typing import Any, List, Optional, Tuple, Union
+import arcpy
+from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
+from arcpy.management import CreateTable
+
+import os.path
+
+from specklepy.api.credentials import Account, get_local_accounts
+from specklepy.api.client import SpeckleClient
+from specklepy.logging.exceptions import (
+    GraphQLException,
+    SpeckleException,
+)
+from specklepy.api.wrapper import StreamWrapper
+from specklepy.api.models import Branch, Stream, Streams
+from specklepy.logging import metrics
+
+from osgeo import osr
+
+import inspect
+
+from speckle.speckle.utils.validation import tryGetStream
+
+# from speckle.speckle.speckle_arcgis import SpeckleGIS
+from speckle.speckle.converter.layers import getAllProjLayers
+from speckle.speckle.utils.panel_logging import logToUser
+
+FIELDS = [
+    "project_streams",
+    "project_layer_selection",
+    "lat_lon",
+    "crs_rotation",
+    "crs_offsets",
+]
+
+
+def get_project_streams(plugin: "SpeckleGIS", content: str = None):
+    try:
+        print("GET proj streams")
+        project = plugin.project
+        table = findOrCreateSpeckleTable(project, plugin)
+        #logToUser(table, level=0, func=inspect.stack()[0][3])
+
+        rows = arcpy.da.SearchCursor(table, "project_streams")
+        saved_streams = []
+        for x in rows:
+            # logToUser(x[0], level=0, func=inspect.stack()[0][3])
+            saved_streams.append(x[0])
+
+        temp = []
+        ######### need to check whether saved streams are available (account reachable)
+        if len(saved_streams) > 0:
+            for url in saved_streams:
+                if url == "":
+                    continue
+                try:
+                    sw = StreamWrapper(url)
+                    try:
+                        stream = tryGetStream(sw, plugin.dataStorage)
+                    except SpeckleException as e:
+                        logToUser(e.message, level=2, func=inspect.stack()[0][3])
+                        stream = None
+                    # strId = stream.id # will cause exception if invalid
+                    temp.append((sw, stream))
+                except SpeckleException as e:
+                    logToUser(e.message, level=2, func=inspect.stack()[0][3])
+                # except GraphQLException as e:
+                #    logger.logToUser(e.message, Qgis.Warning)
+        plugin.current_streams = temp
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+
+
+def set_project_streams(plugin: "SpeckleGIS"):
+    try:
+        print("SET proj streams")
+        project = plugin.project
+        table = findOrCreateSpeckleTable(project, plugin)
+        print("SET proj streams 2")
+
+        current_streams = [
+            stream[0].stream_url for stream in plugin.current_streams
+        ]  # ",".join()
+        print(current_streams)
+        logToUser(current_streams, level=0, func=inspect.stack()[0][3])
+
+        if table is not None:
+            proj_layers = []
+            lan_lot = ""
+            with arcpy.da.UpdateCursor(table, FIELDS) as cursor:
+                for row in cursor:  # just one row
+                    if row[1] is not None and row[1] != "":
+                        proj_layers.append(row[1])
+                    if row[2] is not None and row[2] != "":
+                        lan_lot = row[2]
+                    cursor.deleteRow()
+            del cursor
+            if len(proj_layers) == 0:
+                proj_layers.append("")
+            if len(current_streams) == 0:
+                current_streams.append("")
+
+            cursor = arcpy.da.InsertCursor(table, FIELDS[:3])
+            length = max(len(proj_layers), len(current_streams))
+
+            for i in range(length):
+                if i == 0:
+                    cursor.insertRow([current_streams[i], proj_layers[i], lan_lot])
+                else:
+                    try:
+                        cursor.insertRow([current_streams[i], proj_layers[i], ""])
+                    except:
+                        if len(current_streams) <= i:
+                            cursor.insertRow(["", proj_layers[i], ""])
+                        if len(proj_layers) <= i:
+                            cursor.insertRow([current_streams[i], "", ""])
+            del cursor
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+
+
+def get_project_layer_selection(plugin: "SpeckleGIS"):
+    try:
+        #print("GET project layer selection from the table")
+        project = plugin.project
+        table = findOrCreateSpeckleTable(project, plugin)
+        if table is None:
+            return
+
+        rows = arcpy.da.SearchCursor(table, "project_layer_selection")
+        saved_layers = []
+        for x in rows:
+            saved_layers.append(x[0])
+
+        temp = []
+        proj_layers = getAllProjLayers(plugin)
+        if proj_layers is None:
+            return
+        ######### need to check whether saved streams are available (account reachable)
+        if len(saved_layers) > 0:
+            for layerPath in saved_layers:
+                if layerPath == "":
+                    continue
+                found = 0
+                for layer in proj_layers:
+                    #print(layer.dataSource)
+                    if layer.dataSource == layerPath:
+                        temp.append((layer.name, layer))
+                        found += 1
+                        break
+                if found == 0:
+                    logToUser(
+                        f'Saved layer not found: "{layerPath}"',
+                        level=1,
+                        func=inspect.stack()[0][3],
+                    )
+        plugin.dataStorage.current_layers = temp
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+
+
+def set_project_layer_selection(plugin: "SpeckleGIS"):
+    try:
+        print("SET project layer selection function")
+        project = plugin.project
+        value: List[str] = [
+            layer[1].dataSource for layer in plugin.dataStorage.current_layers
+        ]  # ",".join([layer[1].dataSource for layer in plugin.dataStorage.current_layers])
+        #print(value)
+
+        table = findOrCreateSpeckleTable(project, plugin)
+        # print(table)
+        if table is not None:
+            lan_lot = ""
+            proj_streams = []
+            with arcpy.da.UpdateCursor(table, FIELDS[:3]) as cursor:
+                for row in cursor:  # just one row
+                    if row[0] is not None and row[0] != "":
+                        proj_streams.append(row[0])
+                    if row[2] is not None and row[2] != "":
+                        lan_lot = row[2]
+                    cursor.deleteRow()
+            del cursor
+            if len(proj_streams) == 0:
+                proj_streams.append("")
+            if len(value) == 0:
+                value.append("")
+            # print(proj_streams)
+
+            cursor = arcpy.da.InsertCursor(table, FIELDS[:3])
+            length = max(len(proj_streams), len(value))
+            # print(length)
+            for i in range(length):
+                if i == 0:
+                    cursor.insertRow([proj_streams[i], value[i], lan_lot])
+                    #print(i)
+                else:
+                    try:
+                        cursor.insertRow([proj_streams[i], value[i], ""])
+                    except:
+                        if len(proj_streams) <= i:
+                            cursor.insertRow(["", value[i], ""])
+                        if len(value) <= i:
+                            cursor.insertRow([proj_streams[i], "", ""])
+                # print(i)
+            del cursor
+
+            try:
+                metrics.track(
+                    "Connector Action",
+                    plugin.dataStorage.active_account,
+                    {
+                        "name": "Save Layer Selection",
+                        "connector_version": str(plugin.version),
+                    },
+                )
+            except Exception as e:
+                logToUser(
+                    e, level=2, func=inspect.stack()[0][3], plugin=plugin.dockwidget
+                )
+
+            # print(table)
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+
+    print("SET project layer selection 2")
+
+
+def get_rotation(plugin):
+    try:
+        print("get_rotation")
+        dataStorage = plugin.dataStorage
+        project = plugin.dataStorage.project
+        table = findOrCreateSpeckleTable(project, plugin)
+        if table is None:
+            return
+
+        rows = arcpy.da.SearchCursor(table, "crs_rotation")
+        rotation = ""
+        for x in rows:
+            rotation = x[0]
+            break
+
+        if rotation != "":
+            vals: str = rotation.replace(" ", "")
+            dataStorage.crs_rotation = float(vals)
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3], plugin=plugin.dockwidget)
+
+
+def set_rotation(plugin):
+    try:
+        dataStorage = plugin.dataStorage
+        project = dataStorage.project
+        r = dataStorage.crs_rotation
+        if dataStorage.crs_rotation is None:
+            r = 0
+
+        table = findOrCreateSpeckleTable(project, plugin)
+        if table is not None:
+            with arcpy.da.UpdateCursor(table, ["crs_rotation"]) as cursor:
+                for row in cursor:  # just one row
+                    cursor.updateRow([r])
+                    break
+            del cursor
+        return True
+
+    except Exception as e:
+        logToUser("Lat, Lon values invalid: " + str(e), level=2)
+        return False
+
+
+def get_crs_offsets(plugin):
+    try:
+        print("get_crs_offsets")
+        dataStorage = plugin.dataStorage
+        project = plugin.dataStorage.project
+        table = findOrCreateSpeckleTable(project, plugin)
+        if table is None:
+            return
+
+        rows = arcpy.da.SearchCursor(table, "crs_offsets")
+        points = ""
+        for x in rows:
+            points = x[0]
+            break
+
+        if points != "":
+            vals: List[str] = points.replace(" ", "").split(";")[:2]
+            dataStorage.crs_offset_x, dataStorage.crs_offset_y = [
+                float(i) for i in vals
+            ]
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3], plugin=plugin.dockwidget)
+
+
+def set_crs_offsets(plugin):
+    try:
+
+        dataStorage = plugin.dataStorage
+        project = dataStorage.project
+        x = dataStorage.crs_offset_x
+        y = dataStorage.crs_offset_y
+
+        if dataStorage.crs_offset_x is None or dataStorage.crs_offset_y is None:
+            x = 0
+            y = 0
+        pt = str(x) + ";" + str(y)
+
+        table = findOrCreateSpeckleTable(project, plugin)
+        if table is not None:
+            with arcpy.da.UpdateCursor(table, ["crs_offsets"]) as cursor:
+                for row in cursor:  # just one row
+                    cursor.updateRow([pt])
+                    break
+            del cursor
+
+        return True
+
+    except Exception as e:
+        logToUser("Lat, Lon values invalid: " + str(e), level=2)
+        return False
+
+
+def get_project_saved_layers(plugin):
+
+    try:
+        #print("GET project layer selection from the table")
+        project = plugin.project
+        table = findOrCreateSpeckleTable(project, plugin)
+        if table is None:
+            return
+
+        rows = arcpy.da.SearchCursor(table, "project_layer_selection")
+        saved_layers = []
+        for x in rows:
+            saved_layers.append(x[0])
+
+        temp = []
+        proj_layers = getAllProjLayers(plugin)
+        if proj_layers is None:
+            return
+        ######### need to check whether saved streams are available (account reachable)
+        if len(saved_layers) > 0:
+            for layerPath in saved_layers:
+                if layerPath == "":
+                    continue
+                found = 0
+                for layer in proj_layers:
+                    #print(layer.dataSource)
+                    if layer.dataSource == layerPath:
+                        temp.append((layer.name, layer))
+                        found += 1
+                        break
+                if found == 0:
+                    logToUser(
+                        f'Saved layer not found: "{layerPath}"',
+                        level=1,
+                        func=inspect.stack()[0][3],
+                    )
+        # plugin.dataStorage.current_layers = temp
+        # plugin.dataStorage.saved_layers = temp
+        # plugin.dataStorage.current_layers = temp.copy()
+        plugin.dataStorage.saved_layers = temp.copy()
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+
+
+def set_project_saved_layers(plugin):
+    return
+    try:
+        print("SET project layer selection function")
+        project = plugin.project
+        value: List[str] = [
+            layer[1].dataSource for layer in plugin.dataStorage.saved_layers
+        ]  # ",".join([layer[1].dataSource for layer in plugin.dataStorage.current_layers])
+        if len(value) == 0:
+            value.append("")
+        print(value)
+
+        table = findOrCreateSpeckleTable(project, plugin)
+        # print(table)
+        if table is not None:
+            cursor = arcpy.da.InsertCursor(table, "project_layer_selection")
+            for i in range(len(value)):
+                cursor.insertRow([value[i]])
+                print(i)
+            del cursor
+
+            try:
+                metrics.track(
+                    "Connector Action",
+                    plugin.dataStorage.active_account,
+                    {
+                        "name": "Save Layer Selection",
+                        "connector_version": str(plugin.version),
+                    },
+                )
+            except Exception as e:
+                logToUser(
+                    e, level=2, func=inspect.stack()[0][3], plugin=plugin.dockwidget
+                )
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+
+
+def get_survey_point(plugin: "SpeckleGIS", content=None):
+    try:
+        print("get survey point")
+        project = plugin.dataStorage.project
+        table = findOrCreateSpeckleTable(project, plugin)
+        if table is None:
+            return
+
+        rows = arcpy.da.SearchCursor(table, "lat_lon")
+        points = ""
+        for x in rows:
+            points = x[0]
+            break
+
+        if points != "":
+            vals: List[str] = points.replace(" ", "").split(";")[:2]
+            plugin.lat, plugin.lon = [float(i) for i in vals]
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3], plugin=plugin.dockwidget)
+
+
+def set_survey_point(plugin: "SpeckleGIS"):
+
+    try:
+        # from widget (2 strings) to local vars + update SR of the map
+        print("SET survey point")
+        dataStorage = plugin.dataStorage
+        project = dataStorage.project
+
+        x = dataStorage.custom_lat
+        y = dataStorage.custom_lon
+
+        pt = str(x) + ";" + str(y)
+        table = findOrCreateSpeckleTable(project, plugin)
+        if table is not None:
+            with arcpy.da.UpdateCursor(table, ["lat_lon"]) as cursor:
+                for row in cursor:  # just one row
+                    cursor.updateRow([pt])
+                    break
+            del cursor
+        # setProjectReferenceSystem(plugin)
+
+        try:
+            metrics.track(
+                "Connector Action",
+                plugin.dataStorage.active_account,
+                {
+                    "name": "Set As Center Point",
+                    "connector_version": str(plugin.version),
+                },
+            )
+        except Exception as e:
+            logToUser(e, level=2, func=inspect.stack()[0][3], plugin=plugin.dockwidget)
+
+        return True
+
+    except Exception as e:
+        logToUser(
+            "Lat, Lon values invalid: " + str(e), level=2, func=inspect.stack()[0][3]
+        )
+        return False
+
+
+def setProjectReferenceSystem(plugin: "SpeckleGIS"):
+    try:
+        # save to project; create SR
+        newCrsString = (
+            "+proj=tmerc +ellps=WGS84 +datum=WGS84 +units=m +no_defs +lon_0="
+            + str(plugin.lon)
+            + " lat_0="
+            + str(plugin.lat)
+            + " +x_0=0 +y_0=0 +k_0=1"
+        )
+        newCrs = osr.SpatialReference()
+        newCrs.ImportFromProj4(newCrsString)
+        newCrs.MorphToESRI()  # converts the WKT to an ESRI-compatible format
+
+        validate = True if len(newCrs.ExportToWkt()) > 10 else False
+
+        if validate:
+            newProjSR = arcpy.SpatialReference()
+            newProjSR.loadFromString(newCrs.ExportToWkt())
+
+            # source = osr.SpatialReference()
+            # source.ImportFromWkt(plugin.project.activeMap.spatialReference.exportToString())
+            # transform = osr.CoordinateTransformation(source, newCrs)
+
+            plugin.project.activeMap.spatialReference = newProjSR
+            logToUser(
+                "Custom project Spatial Reference successfully applied",
+                level=0,
+                func=inspect.stack()[0][3],
+                plugin=plugin.dockwidget,
+            )
+        else:
+            logToUser(
+                "Custom Spatial Reference could not be created",
+                level=1,
+                func=inspect.stack()[0][3],
+            )
+
+        return True
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return False
+
+
+def findOrCreateSpeckleTable(project: ArcGISProject, plugin) -> Union[str, None]:
+    try:
+        path = (
+            plugin.workspace
+        )  # project.filePath.replace("aprx","gdb") #"\\".join(project.filePath.split("\\")[:-1]) + "\\speckle_layers\\" #arcpy.env.workspace + "\\" #
+
+        if "speckle_gis" not in arcpy.ListTables():
+            try:
+                table = CreateTable(path, "speckle_gis")
+                for f in FIELDS:
+                    arcpy.management.AddField(table, f, "TEXT")
+                    # arcpy.management.AddField(table, "project_layer_selection", "TEXT")
+                    # arcpy.management.AddField(table, "lat_lon", "TEXT")
+
+                cursor = arcpy.da.InsertCursor(table, FIELDS)
+                cursor.insertRow(["" for _ in range(len(FIELDS))])
+                del cursor
+
+            except Exception as e:
+                logToUser(
+                    "Error creating a table: " + str(e),
+                    level=1,
+                    func=inspect.stack()[0][3],
+                )
+                raise e
+        else:
+            # print("table already exists")
+            # make sure fileds exist
+            table = path + "\\speckle_gis"
+            for f in FIELDS:
+                findOrCreateTableField(table, f)
+            # findOrCreateTableField(table, FIELDS[1])
+            # findOrCreateTableField(table, FIELDS[2])
+
+            findOrCreateRow(table, FIELDS)
+
+        return table
+
+    except Exception as e:
+        logToUser(str(e), level=2, func=inspect.stack()[0][3])
+        return None
+
+
+def findOrCreateTableField(table: str, field: str):
+    try:
+        with arcpy.da.UpdateCursor(table, [field]) as cursor:
+            value = None
+            for row in cursor:
+                value = row  # tuple(val,)
+                if value[0] is None:
+                    cursor.updateRow("")
+                break  # look at the 1st row only
+        del cursor
+
+        # if value is None: # if there are no rows
+        #    cursor = arcpy.da.InsertCursor(table, [field])
+        #    cursor.insertRow([""])
+        #    del cursor
+
+    except Exception as e:  # if field doesn't exist
+        arcpy.management.AddField(table, field, "TEXT")
+        # cursor = arcpy.da.InsertCursor(table, [field] )
+        # cursor.insertRow([""])
+        del cursor
+
+
+def findOrCreateRow(table: str, fields: List[str]):
+
+    # check if the row exists
+    cursor = arcpy.da.SearchCursor(table, fields)
+    k = -1
+    for k, row in enumerate(cursor):
+        # print(row)
+        break
+    del cursor
+
+    # if no rows
+    if k == -1:
+        cursor = arcpy.da.InsertCursor(table, fields)
+        cursor.insertRow(["" for _ in range(len(FIELDS))])
+        del cursor
+    else:
+        with arcpy.da.UpdateCursor(table, fields) as cursor:
+            for row in cursor:
+                if None in row:
+                    cursor.updateRow(["" if r is None else r for r in row])
+                break  # look at the 1st row only
+        del cursor
+
+
+def findOrCreateRowInFeatureTable(table: str, fields: List[str], values=None):
+
+    with arcpy.da.UpdateCursor(table, fields) as cursor:
+        for row in cursor:
+            cursor.deleteRow()
+    del cursor
+
+    cursor = arcpy.da.InsertCursor(table, fields)
+    cursor.insertRow([str(v) for v in values])
+    del cursor

speckle_toolbox/esri/toolboxes/speckle/speckle/utils/validation.py

@@ -0,0 +1,173 @@
+import inspect
+from typing import Union
+from specklepy.core.api.credentials import get_default_account
+from specklepy.core.api.wrapper import StreamWrapper
+from specklepy.core.api.models import Stream, Branch, Commit
+from specklepy.transports.server import ServerTransport
+from specklepy.core.api.client import SpeckleClient
+from specklepy.logging.exceptions import SpeckleException, GraphQLException
+
+from speckle.speckle.utils.panel_logging import logToUser
+
+
+def tryGetClient(sw: StreamWrapper, dataStorage, write=False, dockwidget=None):
+    # only streams with write access
+    client = None
+    savedRole = None
+    savedStreamId = None
+    for acc in dataStorage.accounts:
+        # only check accounts on selected server
+        if acc.serverInfo.url in sw.server_url:
+            client = SpeckleClient(
+                acc.serverInfo.url, acc.serverInfo.url.startswith("https")
+            )
+            try:
+                client.authenticate_with_account(acc)
+                if client.account.token is not None:
+                    break
+            except SpeckleException as ex:
+                if "already connected" in ex.message:
+                    logToUser(
+                        "Dependencies versioning error.\nClick here for details.",
+                        url="dependencies_error",
+                        level=2,
+                        plugin=dockwidget,
+                    )
+                    return None, None
+                else:
+                    raise ex
+
+    # if token still not found
+    if client is None or client.account.token is None:
+        client = sw.get_client()
+
+    if client is not None:
+        stream = client.stream.get(id=sw.stream_id, branch_limit=100, commit_limit=100)
+        if isinstance(stream, Stream):
+            if write is False:
+                # try get stream, only read access needed
+                return client, stream
+            else:
+                # check write access
+                if stream.role is None:
+                    raise Exception(
+                        f"You don't have write access to the stream '{stream.id}'. You role is '{stream.role}'"
+                    )
+                elif isinstance(stream.role, str) and "reviewer" in stream.role:
+                    raise Exception(
+                        f"You don't have write access to the stream '{savedStreamId}'. You role is '{savedRole}'"
+                    )
+                else:
+                    return client, stream
+        else:
+            return None, None
+    else:
+        return None, None
+
+
+def tryGetStream(
+    sw: StreamWrapper, dataStorage, write=False, dockwidget=None
+) -> Union[Stream, None]:
+    try:
+        # print("tryGetStream")
+        client, stream = tryGetClient(sw, dataStorage, write, dockwidget)
+        return stream
+    except Exception as e:
+        logToUser(e, level=2, func=inspect.stack()[0][3], plugin=dockwidget)
+        return None
+
+
+def validateStream(stream: Stream, dockwidget) -> Union[Stream, None]:
+    try:
+        # dockwidget.dataStorage.check_for_accounts()
+        # stream = tryGetStream(streamWrapper, dockwidget.dataStorage)
+
+        if isinstance(stream, SpeckleException):
+            return None
+
+        if stream.branches is None:
+            logToUser("Stream has no branches", level=1, plugin=dockwidget)
+            return None
+        return stream
+    except Exception as e:
+        logToUser(e, level=2, plugin=dockwidget)
+        return
+
+
+def validateBranch(
+    stream: Stream, branchName: str, checkCommits: bool, dockwidget
+) -> Union[Branch, None]:
+    try:
+        branch = None
+        if not stream.branches or not stream.branches.items:
+            return None
+        for b in stream.branches.items:
+            if b.name == branchName:
+                branch = b
+                break
+        if branch is None:
+            logToUser("Failed to find a branch", level=2, plugin=dockwidget)
+            return None
+        if checkCommits == True:
+            if branch.commits is None:
+                logToUser("Failed to find a branch", level=2, plugin=dockwidget)
+                return None
+            if len(branch.commits.items) == 0:
+                logToUser("Branch contains no commits", level=1, plugin=dockwidget)
+                return None
+        return branch
+    except Exception as e:
+        logToUser(e, level=2, plugin=dockwidget)
+
+
+def validateCommit(
+    branch: Branch, commitId: str, dockwidget=None
+) -> Union[Commit, None]:
+    try:
+        commit = None
+        try:
+            commitId = commitId.split(" | ")[0]
+        except:
+            logToUser("Commit ID is not valid", level=2, plugin=dockwidget)
+
+        if commitId.startswith("Latest") and len(branch.commits.items) > 0:
+            commit = branch.commits.items[0]
+        else:
+            for i in branch.commits.items:
+                if i.id == commitId:
+                    commit = i
+                    break
+            if commit is None:
+                try:
+                    commit = branch.commits.items[0]
+                    logToUser(
+                        "Failed to find a commit. Receiving Latest",
+                        level=1,
+                        plugin=dockwidget,
+                    )
+                except:
+                    logToUser("Failed to find a commit", level=2, plugin=dockwidget)
+                    return None
+        return commit
+    except Exception as e:
+        logToUser(e, level=2, plugin=dockwidget)
+        return
+
+
+def validateTransport(
+    client: SpeckleClient, streamId: str
+) -> Union[ServerTransport, None]:
+    try:
+        account = client.account
+        if not account.token:
+            account = get_default_account()
+        transport = ServerTransport(client=client, account=account, stream_id=streamId)
+        # print(transport)
+        return transport
+    except Exception as e:
+        logToUser(
+            "Make sure you have sufficient permissions: " + str(e),
+            level=1,
+            func=inspect.stack()[0][3],
+        )
+        return None

speckle_toolbox/esri/toolboxes/speckle/speckle_arcgis.py

@@ -1,568 +0,0 @@
-# -*- coding: utf-8 -*-
-
-from collections import defaultdict
-from typing import Any, Callable, List, Optional
-from xmlrpc.client import Boolean
-import arcpy
-from arcpy._mp import ArcGISProject, Map, Layer as arcLayer
-
-from specklepy.api.models import Branch, Stream, Streams
-from speckle.converter.layers.Layer import Layer, RasterLayer
-
-from speckle.converter.layers._init_ import convertSelectedLayers, layerToNative, cadLayerToNative
-from arcgis.features import FeatureLayer
-import os.path
-
-import specklepy
-from specklepy.transports.server.server import ServerTransport
-from specklepy.api.credentials import get_local_accounts
-from specklepy.api.client import SpeckleClient
-from specklepy.api import operations
-from specklepy.logging.exceptions import (
-    SpeckleException,
-    SpeckleWarning,
-)
-from specklepy.api.credentials import StreamWrapper
-from specklepy.objects import Base
-from specklepy.api.wrapper import StreamWrapper
-
-from speckle.converter.layers.emptyLayerTemplates import createGroupLayer
-
-def traverseObject(
-        base: Base,
-        callback: Optional[Callable[[Base], bool]],
-        check: Optional[Callable[[Base], bool]],
-    ):
-        if check and check(base):
-            res = callback(base) if callback else False
-            if res:
-                return
-        memberNames = base.get_member_names()
-        for name in memberNames:
-            try:
-                if ["id", "applicationId", "units", "speckle_type"].index(name):
-                    continue
-            except:
-                pass
-            traverseValue(base[name], callback, check)
-            
-def traverseValue(
-    value: Any,
-    callback: Optional[Callable[[Base], bool]],
-    check: Optional[Callable[[Base], bool]],
-):
-    if isinstance(value, Base):
-        traverseObject(value, callback, check)
-    if isinstance(value, List):
-        for item in value:
-            traverseValue(item, callback, check)
-
-
-class Toolbox(object):
-    def __init__(self):
-        """Define the toolbox (the name of the toolbox is the name of the
-        .pyt file)."""
-        self.label = "Speckle Tools"
-        self.alias = "speckle_toolbox_"  
-        # List of tool classes associated with this toolbox
-        self.tools = [Speckle]    
-        #self.toolboxInputs = uiInputs() # initialize once together with a toolbox
-
-        
-        #print(self.toolboxInputs.selected_layers)
-        #try: 
-        # https://pro.arcgis.com/en/pro-app/2.8/arcpy/mapping/alphabeticallistofclasses.htm#except: print("something happened")
-
-class uiInputs(object):
-    speckle_client: Any
-    streams: Optional[Streams]
-    active_stream: Optional[Stream]
-    active_branch: Optional[Branch]
-    all_layers: List[arcLayer]
-    selected_layers: List[Any]
-    messageSpeckle: str
-    project: ArcGISProject
-    action: int
-    instances = []
-
-    def __init__(self):
-        #print("start UI inputs________")
-        self.instances.append(self)
-        accounts = get_local_accounts()
-        account = None
-        for acc in accounts:
-            if acc.isDefault: account = acc; break
-        #account.userInfo.name, account.serverInfo.url
-        self.speckle_client = SpeckleClient(account.serverInfo.url, account.serverInfo.url.startswith("https"))
-        self.speckle_client.authenticate_with_token(token=account.token)
-        #print("ping")
-        #print(self.speckle_client)
-        self.streams = self.speckle_client.stream.search("")
-        #print("ping")
-        self.active_stream = None
-        self.active_branch = None
-        self.active_commit = None
-        self.all_layers = []
-        self.selected_layers = []
-        self.messageSpeckle = ""
-        self.project = aprx = None
-        self.action = 1 #send
-        #print(self.streams)
-        try: aprx = ArcGISProject('CURRENT') 
-        except: 
-            #print(arcpy.env.workspace) # None
-            #arcpy.env.workspace = ""
-            #proj_path = "\\".join(arcpy.env.workspace.split("\\")[:-1]) + "\\"
-            #aprx = ArcGISProject(proj_path)
-            #print(aprx)
-            print("Project not found")
-        self.project = aprx
-        active_map = aprx.activeMap
-
-        if active_map is not None and isinstance(active_map, Map): # if project loaded
-            for layer in active_map.listLayers(): 
-                #print(layer)
-                if layer.isFeatureLayer: self.all_layers.append(layer) #type: 'arcpy._mp.Layer'
-        
-
-class Speckle(object):
-    def __init__(self):
-        #print("________________reset_______________")   
-        self.label       = "Speckle"
-        self.description = "Allows you to send and receive your layers " + \
-                           "to/from other software using Speckle server." 
-        self.toolboxInputs = None
-        self.toRefresh = False
-
-        for instance in uiInputs.instances:
-            #print(instance)
-            if instance is not None: 
-                try: 
-                    x = instance.streams # in case not initialized properly 
-                    self.toolboxInputs = instance # take latest 
-                except: pass
-        if self.toolboxInputs is None: 
-            #print("Instance is None")
-            self.toolboxInputs = uiInputs() #in case Toolbox class was not initialized 
-        # TODO react on project changes
-        
-        #print("______continue reset_______")
-        #print(self.toolboxInputs.all_layers)
-
-    def getParameterInfo(self):
-        #data types: https://pro.arcgis.com/en/pro-app/2.8/arcpy/geoprocessing_and_python/defining-parameter-data-types-in-a-python-toolbox.htm
-        # parameter details: https://pro.arcgis.com/en/pro-app/latest/arcpy/geoprocessing_and_python/customizing-tool-behavior-in-a-python-toolbox.htm
-        print("Get parameter values")
-
-        stream = arcpy.Parameter(
-            displayName="Stream",
-            name="stream",
-            datatype="GPString",
-            parameterType="Required",
-            #category="Sending data",
-            direction="Input")
-        stream.filter.type = 'ValueList'
-        stream.filter.list = [ (st.name + " | " + st.id) for st in self.toolboxInputs.streams ]
-        
-        branch = arcpy.Parameter(
-            displayName="Branch",
-            name="branch",
-            datatype="GPString",
-            parameterType="Required",
-            #category="Sending data",
-            direction="Input")
-        branch.value = "main"
-        branch.filter.type = 'ValueList'
-
-        commit = arcpy.Parameter(
-            displayName="Commit",
-            name="commit",
-            datatype="GPString",
-            parameterType="Optional",
-            #category="Sending data",
-            direction="Input")
-        commit.value = ""
-        commit.filter.type = 'ValueList'
-
-        ################################################################
-        msg = arcpy.Parameter(
-            displayName="Message",
-            name="message",
-            datatype="GPString",
-            parameterType="Optional",
-            direction="Input",
-            multiValue=False)
-        msg.value = ""
-
-        selected_layers = arcpy.Parameter(
-            displayName="Selected Layers",
-            name="selected_layers",
-            datatype="GPString",
-            parameterType="Optional",
-            direction="Input",
-            multiValue=True
-            )
-        selected_layers.filter.list = [str(i) + "-" + l.longName for i,l in enumerate(self.toolboxInputs.all_layers)] #"Polyline"
-
-        refresh = arcpy.Parameter(
-            displayName="Refresh",
-            name="refresh",
-            datatype="GPBoolean",
-            parameterType="Optional",
-            #category="Sending data",
-            direction="Input"
-            )
-        #refresh.filter.type = "ValueList"   
-        refresh.value = False 
-        
-        action = arcpy.Parameter(
-            displayName="",
-            name="action",
-            datatype="GPString",
-            parameterType="Required",
-            #category="Sending data",
-            direction="Input",
-            multiValue=False
-            )
-        action.value = "Send" 
-        #action.filter.type = 'ValueList'
-        action.filter.list = ["Send", "Receive"]  
-
-        parameters = [stream, branch, commit, selected_layers, msg, action, refresh]
-        return parameters
-
-    def isLicensed(self): #optional
-        return True
-
-    def updateParameters(self, parameters: List, toRefresh = False): #optional
-        print("UPDATING PARAMETERS")
-        
-        if parameters[0].altered:
-
-            # Search for the stream by name
-            if parameters[0].valueAsText is not None:
-                selected_stream_name = parameters[0].valueAsText[:]
-                self.toolboxInputs.active_stream = None
-                #print(self.toolboxInputs.active_stream)
-                #print(self.toolboxInputs.streams)
-                for st in self.toolboxInputs.streams:
-                    if st.name == selected_stream_name.split(" | ")[0]: 
-                        self.toolboxInputs.active_stream = st
-                        break
-
-                # edit branches: globals and UI 
-                branch_list = [branch.name for branch in self.toolboxInputs.active_stream.branches.items]
-                parameters[1].filter.list = branch_list
-                #print(parameters[1].filter.list)
-
-                if parameters[1].valueAsText not in branch_list: 
-                    parameters[1].value = "main"
-                for b in self.toolboxInputs.active_stream.branches.items:
-                    if b.name == parameters[1].value: 
-                        self.toolboxInputs.active_branch = b
-                        break 
-                
-                # setting commit value and list 
-                try: 
-                    #print("___editing the stream input")
-                    #print(self.toolboxInputs.active_branch.commits.items)
-                    parameters[2].filter.list = [f"{commit.id}"+ " | " + f"{commit.message}" for commit in self.toolboxInputs.active_branch.commits.items]
-                    #print(parameters[2].filter.list)
-                    #print(parameters[2].valueAsText)
-                    if parameters[2].valueAsText not in parameters[2].filter.list:
-                        parameters[2].value = self.toolboxInputs.active_branch.commits.items[0].id + " | " + self.toolboxInputs.active_branch.commits.items[0].message 
-                        self.toolboxInputs.active_commit = self.toolboxInputs.active_branch.commits.items[0] 
-                except: 
-                    parameters[2].filter.list = []
-                    parameters[2].value = None
-                    self.toolboxInputs.active_commit = None 
-
-        if parameters[1].altered: # branches
-            if parameters[1].valueAsText is not None:
-                selected_branch_name = parameters[1].valueAsText[:]
-                self.toolboxInputs.active_branch = None
-                if self.toolboxInputs.active_stream is not None:
-                    for br in self.toolboxInputs.active_stream.branches.items:
-                        if br.name == selected_branch_name: #.split(" | ")[0]: 
-                            self.toolboxInputs.active_branch = br
-                            break
-            # edit commit values 
-            if self.toolboxInputs.active_branch is not None: 
-                try: 
-                    #print("___editing the branch input")
-                    #print(self.toolboxInputs.active_branch)
-                    parameters[2].filter.list = [f"{commit.id}"+ " | " + f"{commit.message}" for commit in self.toolboxInputs.active_branch.commits.items]
-                    #print(parameters[2].filter.list)
-                    #print(parameters[2].valueAsText)
-                    if parameters[2].valueAsText not in parameters[2].filter.list:
-                        parameters[2].value = self.toolboxInputs.active_branch.commits.items[0].id + " | " + self.toolboxInputs.active_branch.commits.items[0].message 
-                        self.toolboxInputs.active_commit = self.toolboxInputs.active_branch.commits.items[0]
-                except: 
-                    parameters[2].filter.list = []
-                    parameters[2].value = None
-                    self.toolboxInputs.active_commit = None 
-
-        if parameters[2].altered: # commits
-            if parameters[2].valueAsText is not None:
-                selected_commit_id = parameters[2].valueAsText[:].split(" | ")[0]
-                self.toolboxInputs.active_commit = None
-                if self.toolboxInputs.active_branch is not None:
-                    for c in self.toolboxInputs.active_branch.commits.items:
-                        if c.id == selected_commit_id: 
-                            self.toolboxInputs.active_commit = c 
-                            break
-        
-        if parameters[3].altered: # selected layers
-            if parameters[3].valueAsText is not None:
-                self.toolboxInputs.selected_layers = parameters[3].values
-
-        if parameters[4].altered:
-            self.toolboxInputs.messageSpeckle = parameters[4].valueAsText
-
-        if parameters[5].altered:
-            if parameters[5].valueAsText == "Send": self.toolboxInputs.action = 1
-            else: self.toolboxInputs.action = 0
-
-        if parameters[6].altered: # refresh btn
-            if parameters[6].value == True: 
-                self.refresh(parameters) 
-        if self.toRefresh == True:
-            self.refresh(parameters) 
-            self.toRefresh = False
-        #if newParams: # apply fresh values 
-        #    print(newParams[2].filter.list)
-        #    parameters = newParams
-            
-            ##parameters = newParams
-        #print("continue UPDATING PARAMETERS")
-        #print(parameters[2].filter.list)
-        #print(parameters[4].valueAsText)
-        return 
-
-    def refresh(self, parameters: List[Any]): 
-        print("Refresh______")
-        uiInputs()
-        for instance in uiInputs.instances:
-            if instance is not None: self.toolboxInputs = instance # take latest 
-        #self.__init__()
-        #self.streams = self.speckle_client.stream.search("")
-        #params_new = []
-        #for i,p in enumerate(parameters):
-         #   params_new.append(p)
-        parameters[0].value = None
-        parameters[1].value = "main"
-        parameters[2].value = None
-        parameters[3].value = None
-        parameters[4].value = ""
-        parameters[5].value = "Send"
-        parameters[6].value = False
-         
-        parameters[0].filter.list = [ (st.name + " | " + st.id) for st in self.toolboxInputs.streams ]
-        parameters[3].filter.list = [str(i) + "-" + l.longName for i,l in enumerate(self.toolboxInputs.all_layers)]
-        #print("___continue_refresh______")
-        #print(parameters[2].filter.list)
-        
-        return parameters
-
-    def updateMessages(self, parameters): #optional
-        return
-
-    def execute(self, parameters: List, messages): 
-        # https://pro.arcgis.com/en/pro-app/latest/arcpy/get-started/what-is-arcpy-.htm
-        #Warning if any of the fields is invalid/empty 
-        print("_______________________Run__________________________")
-        #print(self.toolboxInputs.action)
-        if self.toolboxInputs.action == 1: self.onSend(parameters)
-        elif self.toolboxInputs.action == 0: self.onReceive(parameters)
-
-    def validateStreamBranch(self, parameters: List):
-        	
-        self.updateParameters(parameters)
-        if self.toolboxInputs.active_stream is None:
-            arcpy.AddError("Choose a valid stream")
-            return False
-        if self.toolboxInputs.active_branch is None: 
-            arcpy.AddError("Choose a valid branch")
-            return False
-        return True 
-
-    def onSend(self, parameters: List):
-
-        if self.validateStreamBranch(parameters) == False: return
-
-        if len(self.toolboxInputs.selected_layers) == 0: 
-            arcpy.AddError("No layers selected")
-            return
-
-        streamId = self.toolboxInputs.active_stream.id #stream_id
-        client = self.toolboxInputs.speckle_client # ?
-        
-        # Get the stream wrapper
-        #streamWrapper = StreamWrapper(None)
-        #client = streamWrapper.get_client()
-        # Ensure the stream actually exists
-        #try:
-        #    client.stream.get(streamId)
-        #except SpeckleException as error:
-        #    print(str(error))
-        #    return
-
-        # next create a server transport - this is the vehicle through which you will send and receive
-        transport = ServerTransport(client=client, stream_id=streamId)
-
-        ##################################### conversions ################################################
-        base_obj = Base()
-        base_obj.layers = convertSelectedLayers(self.toolboxInputs.all_layers, self.toolboxInputs.selected_layers, self.toolboxInputs.project)
-        
-        try:
-            # this serialises the block and sends it to the transport
-            objId = operations.send(base=base_obj, transports=[transport])
-        except SpeckleException as error:
-            arcpy.AddError("Error sending data")
-            #print("Error sending data")
-            return
-        except SpeckleWarning as warning:
-            arcpy.AddMessage("SpeckleWarning: " + str(warning.args[0]))
-            
-
-        message = self.toolboxInputs.messageSpeckle
-        if message is None or ( isinstance(message, str) and len(message) == 0):  message = "Sent from ArcGIS"
-        try:
-            # you can now create a commit on your stream with this object
-            client.commit.create(
-                stream_id=streamId,
-                object_id=objId,
-                branch_name=self.toolboxInputs.active_branch.name,
-                message=message,
-                source_application="ArcGIS",
-            )
-            arcpy.AddMessage("Successfully sent data to stream: " + streamId)
-            #print("Successfully sent data to stream: " + streamId)
-            #parameters[2].value = ""
-        except:
-            arcpy.AddError("Error creating commit")
-
-        #print("sent")
-        #self.updateParameters(parameters, True)
-        #self.refresh(parameters)
-
-    def onReceive(self, parameters: List[Any]): 
-        
-        if self.validateStreamBranch(parameters) == False: return
-
-        try: 
-            streamId = self.toolboxInputs.active_stream.id #stream_id
-            client = self.toolboxInputs.speckle_client # 
-        except SpeckleWarning as warning: 
-            arcpy.AddWarning(str(warning.args[0]))
-
-        # get commit 
-        commit = None 
-        try: 
-            #commit = self.toolboxInputs.active_branch.commits.items[0]
-            commit = self.toolboxInputs.active_commit
-            commitId = commit.id # text to make sure commit exists 
-        except: 
-            try: 
-                commit = self.toolboxInputs.active_branch.commits.items[0]
-                commitId = commit.id 
-                arcpy.AddWarning("Failed to find a commit. Getting the last commit of the branch")
-            except:
-                arcpy.AddError("Failed to find a commit")
-                return
-
-        # next create a server transport - this is the vehicle through which you will send and receive
-        try: transport = ServerTransport(client=client, stream_id=streamId)
-        except: 
-            arcpy.AddError("Make sure your account has access to the chosen stream")
-            return
-
-        try:
-            #print(commit)
-            objId = commit.referencedObject
-            commitDetailed = client.commit.get(streamId, commit.id)
-            app = commitDetailed.sourceApplication
-            if objId is None:
-                return
-            commitObj = operations.receive(objId, transport, None)
-            
-            if app != "QGIS" and app != "ArcGIS": 
-                if self.toolboxInputs.project.activeMap.spatialReference.type == "Geographic" or self.toolboxInputs.project.activeMap.spatialReference is None: #TODO test with invalid CRS
-                    arcpy.AddMessage("It is advisable to set the project Spatial reference to Projected type before receiving CAD geometry (e.g. EPSG:32631), or create a custom one from geographic coordinates")
-                    print("It is advisable to set the project Spatial reference to Projected type before receiving CAD geometry (e.g. EPSG:32631), or create a custom one from geographic coordinates")
-            print(f"Succesfully received {objId}")
-
-            # Clear 'latest' group
-            streamBranch = streamId + "_" + self.toolboxInputs.active_branch.name + "_" + str(commit.id)
-            newGroupName = f'{streamBranch}'
-            
-            groupExists = 0
-            #print(newGroupName)
-            for l in self.toolboxInputs.project.activeMap.listLayers(): 
-                #print(l.longName)
-                if l.longName.startswith(newGroupName + "\\"):
-                    #print(l.longName)
-                    self.toolboxInputs.project.activeMap.removeLayer(l)
-                    groupExists+=1
-                elif l.longName == newGroupName: 
-                    groupExists+=1
-            if groupExists == 0:
-                # create empty group layer file 
-                path = self.toolboxInputs.project.filePath.replace("aprx","gdb") #"\\".join(self.toolboxInputs.project.filePath.split("\\")[:-1]) + "\\speckle_layers\\"
-                #print(path)
-                f = open(path + "\\" + newGroupName + ".lyrx", "w")
-                content = createGroupLayer().replace("TestGroupLayer", newGroupName)
-                f.write(content)
-                f.close()
-                smth = arcpy.mp.LayerFile(path + "\\" + newGroupName + ".lyrx")
-                #print(smth)
-                layerGroup = self.toolboxInputs.project.activeMap.addLayer(smth)[0]
-                layerGroup.name = newGroupName
-
-            if app == "QGIS" or app == "ArcGIS": check: Callable[[Base], bool] = lambda base: isinstance(base, Layer) or isinstance(base, RasterLayer)
-            else: check: Callable[[Base], bool] = lambda base: isinstance(base, Base)
-
-            def callback(base: Base) -> bool:
-                print("callback")
-                #print(base)
-                if isinstance(base, Layer) or isinstance(base, RasterLayer):
-                    layer = layerToNative(base, streamBranch, self.toolboxInputs.project)
-                    if layer is not None:
-                        print("Layer created: " + layer.name)
-                else:
-                    loopObj(base, "")
-                return True
-            
-            def loopObj(base: Base, baseName: str):
-                memberNames = base.get_member_names()
-                for name in memberNames:
-                    if name in ["id", "applicationId", "units", "speckle_type"]: continue
-                    try: loopVal(base[name], baseName + "/" + name) # loop properties not included above
-                    except: pass
-
-            def loopVal(value: Any, name: str): # "name" is the parent object/property/layer name
-                if isinstance(value, Base): 
-                    try: # dont go through parts of Speckle Geometry object
-                        if value.speckle_type.startswith("Objects.Geometry."): pass #.Brep") or value.speckle_type.startswith("Objects.Geometry.Mesh") or value.speckle_type.startswith("Objects.Geometry.Surface") or value.speckle_type.startswith("Objects.Geometry.Extrusion"): pass
-                        else: loopObj(value, name)
-                    except: loopObj(value, name)
-
-                if isinstance(value, List):
-                    for item in value:
-                        loopVal(item, name)
-                        if item.speckle_type and item.speckle_type.startswith("Objects.Geometry."): 
-                            pt, pl = cadLayerToNative(value, name, streamBranch, self.toolboxInputs.project)
-                            if pt is not None: print("Layer group created: " + pt.name())
-                            if pl is not None: print("Layer group created: " + pl.name())
-                            break
-                
-            traverseObject(commitObj, callback, check)
-      
-        except SpeckleException as e:
-            print("Receive failed")
-            return
-        print("received")
-        #self.updateParameters(parameters, True)
-        #self.refresh(parameters)
-    
-    

speckle_toolbox/esri/toolboxes/speckle/ui_widgets/main_window.py

@@ -0,0 +1,136 @@
+import inspect
+import os
+import threading
+from PyQt5 import QtWidgets, uic
+from specklepy.logging.exceptions import SpeckleException
+
+from speckle.speckle.utils.panel_logging import logToUser
+import speckle.specklepy_qt_ui.qt_ui
+from speckle.specklepy_qt_ui.qt_ui.mainWindow import (
+    SpeckleGISDialog as SpeckleGISDialog_UI,
+)
+
+ui_file_path = os.path.join(
+    os.path.dirname(speckle.specklepy_qt_ui.qt_ui.__file__),
+    os.path.join("ui", "mainWindow_main.ui"),
+)
+
+
+class SpeckleGISDialog(SpeckleGISDialog_UI):
+    def __init__(self, parent=None):
+        """Constructor."""
+        super(SpeckleGISDialog, self).__init__(
+            parent
+        )  # , QtCore.Qt.WindowStaysOnTopHint)
+        uic.loadUi(ui_file_path, self)  # Load the .ui file
+        # self.show()
+        self.runAllSetup()
+
+    def populateProjectStreams(self, plugin):
+        try:
+            from speckle.speckle.utils.project_vars import set_project_streams
+
+            if not self:
+                return
+            self.streamList.clear()
+            for stream in plugin.current_streams:
+                self.streamList.addItems(
+                    [
+                        (
+                            f"Stream not accessible - {stream[0].stream_id}"
+                            if stream[1] is None
+                            or isinstance(stream[1], SpeckleException)
+                            else f"{stream[1].name}, {stream[1].id} | {stream[0].stream_url.split('/streams')[0].split('/projects')[0]}"
+                        )
+                    ]
+                )
+            if len(plugin.current_streams) == 0:
+                self.streamList.addItems([""])
+            self.streamList.addItems(["Create New Stream"])
+            set_project_streams(plugin)
+            index = self.streamList.currentIndex()
+            if index == -1:
+                self.streams_remove_button.setEnabled(False)
+            else:
+                self.streams_remove_button.setEnabled(True)
+
+            if len(plugin.current_streams) > 0:
+                plugin.active_stream = plugin.current_streams[0]
+        except Exception as e:
+            logToUser(e, level=2, func=inspect.stack()[0][3], plugin=self)
+            return
+
+    def completeStreamSection(self, plugin):
+        try:
+            self.streams_remove_button.clicked.connect(
+                lambda: self.onStreamRemoveButtonClicked(plugin)
+            )
+            self.streamList.currentIndexChanged.connect(
+                lambda: self.onActiveStreamChanged(plugin)
+            )
+            self.streamBranchDropdown.currentIndexChanged.connect(
+                lambda: self.populateActiveCommitDropdown(plugin)
+            )
+            return
+        except Exception as e:
+            logToUser(e, level=2, func=inspect.stack()[0][3], plugin=self)
+            return
+
+    def onStreamRemoveButtonClicked(self, plugin):
+        try:
+            from speckle.speckle.utils.project_vars import set_project_streams
+
+            if not self:
+                return
+            index = self.streamList.currentIndex()
+            if len(plugin.current_streams) > 0:
+                plugin.current_streams.pop(index)
+            plugin.active_stream = None
+            self.streamBranchDropdown.clear()
+            self.commitDropdown.clear()
+
+            set_project_streams(plugin)
+            self.populateProjectStreams(plugin)
+        except Exception as e:
+            logToUser(e, level=2, func=inspect.stack()[0][3], plugin=self)
+            return
+
+    def populateProjectStreams(self, plugin):
+        try:
+            from speckle.speckle.utils.project_vars import set_project_streams
+
+            if not self:
+                return
+            self.streamList.clear()
+            for stream in plugin.current_streams:
+                self.streamList.addItems(
+                    [
+                        (
+                            f"Stream not accessible - {stream[0].stream_id}"
+                            if stream[1] is None
+                            or isinstance(stream[1], SpeckleException)
+                            else f"{stream[1].name}, {stream[1].id} | {stream[0].stream_url.split('/streams')[0].split('/projects')[0]}"
+                        )
+                    ]
+                )
+            if len(plugin.current_streams) == 0:
+                self.streamList.addItems([""])
+            self.streamList.addItems(["Create New Stream"])
+            set_project_streams(plugin)
+            index = self.streamList.currentIndex()
+            if index == -1:
+                self.streams_remove_button.setEnabled(False)
+            else:
+                self.streams_remove_button.setEnabled(True)
+
+            if len(plugin.current_streams) > 0:
+                plugin.active_stream = plugin.current_streams[0]
+        except Exception as e:
+            logToUser(e, level=2, func=inspect.stack()[0][3], plugin=self)
+            return
+
+    def cancelOperations(self):
+        for t in threading.enumerate():
+            if "speckle_" in t.name:
+                t.kill()
+                t.join()

tests/tests_unittest.py

@@ -0,0 +1,104 @@
+from speckle_toolbox.esri.toolboxes.speckle.speckle.speckle_arcgis import Toolbox, Speckle   # The code to test
+from speckle_toolbox.esri.toolboxes.speckle.ui.project_vars import speckleInputsClass, toolboxInputsClass
+
+import arcpy 
+import os
+
+from specklepy.api.wrapper import StreamWrapper
+from specklepy.api.models import Branch, Stream, Streams
+from specklepy.logging.exceptions import GraphQLException, SpeckleException
+from specklepy.api.credentials import Account
+
+import unittest   # The test framework
+# remove SetUp
+# add different scenqrios for Streqm Wrapper including wrng ones
+# tree of all options for input or class outcome
+# use dict for types chech
+# issue with untestable class init 
+# "mocking objects" for tests or "faking"
+# get functions ut of INIT 
+
+class Test_InitializingClasses(unittest.TestCase):
+    def setUp(self) -> None:
+        self.toolbox_input = toolboxInputsClass()
+        self.speckle_input = speckleInputsClass()
+        self.toolbox = Toolbox()
+        self.speckleTool = Speckle()
+        self.test_stream = "https://speckle.xyz/streams////17b0b76d13"
+    
+    def text_all_toolbox(self):
+        self.assertTrue(isinstance(self.toolbox.tools[0], Speckle))
+
+    def test_toolbox_inputs(self):
+        self.assertEqual(self.toolbox_input.lat, 0)
+        self.assertEqual(self.toolbox_input.lon, 0)
+        self.assertIsNone(self.toolbox_input.active_stream)
+        self.assertIsNone(self.toolbox_input.active_branch)
+        self.assertIsNone(self.toolbox_input.active_commit)
+        self.assertEqual(len(self.toolbox_input.selected_layers), 0)
+        self.assertEqual(self.toolbox_input.messageSpeckle, "")
+        self.assertEqual(self.toolbox_input.action, 1)
+        self.assertIsNone(self.toolbox_input.project)
+        self.assertEqual(self.toolbox_input.stream_file_path, "")
+
+    def test_something(self):
+        # Arrange
+        toolbox_input: toolboxInputsClass = toolboxInputsClass()
+        
+        # Act
+        toolbox_input.setProjectStreams(StreamWrapper(self.test_stream))
+
+        # Assert
+        os.path.exists(self.toolbox_input.stream_file_path)
+
+
+    def test_toolbox_inputs_functions(self):
+        self.toolbox_input.setProjectStreams(StreamWrapper(self.test_stream))
+        if os.path.exists(self.toolbox_input.stream_file_path):
+            f = open(self.toolbox_input.stream_file_path, "r")
+            existing_content = f.read()
+            f.close()
+            self.assertTrue(isinstance(existing_content, str))
+        
+        self.toolbox_input.setProjectStreams(None)
+        if os.path.exists(self.toolbox_input.stream_file_path):
+            f = open(self.toolbox_input.stream_file_path, "r")
+            existing_content = f.read()
+            f.close()
+            self.assertTrue(isinstance(existing_content, str))
+        self.assertIsInstance()
+
+        self.assertTrue( isinstance(self.toolbox_input.get_survey_point(), tuple))
+        self.assertTrue( isinstance(self.toolbox_input.get_survey_point()[0], float) or isinstance(self.toolbox_input.get_survey_point()[0], int))
+        self.assertTrue( isinstance(self.toolbox_input.get_survey_point()[1], float) or isinstance(self.toolbox_input.get_survey_point()[1], int))
+
+        self.assertTrue( self.toolbox_input.set_survey_point )
+
+    def test_speckle_inputs(self):
+        self.assertTrue(isinstance(self.speckle_input.accounts, list))
+        self.assertTrue(self.speckle_input.account is None or isinstance(self.speckle_input.account, Account))
+        self.assertTrue(self.speckle_input.streams_default is None or isinstance(self.speckle_input.streams_default, list))
+        self.assertIsNone(self.speckle_input.project)
+        self.assertIsNone(self.speckle_input.active_map)
+        self.assertEqual(self.speckle_input.stream_file_path, "")
+        self.assertTrue(isinstance(self.speckle_input.saved_streams, list))
+        self.assertTrue(isinstance(self.speckle_input.all_layers, list))
+        self.assertTrue(isinstance(self.speckle_input.clients, list))
+    
+    def test_speckle_inputs_functions(self):    
+        
+        self.assertTrue(isinstance(self.speckle_input.getProjectStreams(), list))
+        getStreams = self.speckle_input.getProjectStreams(self.test_stream)
+        self.assertTrue(isinstance(getStreams[0][0], StreamWrapper) and isinstance(getStreams[0][1], Stream))
+
+        self.assertTrue(isinstance(self.speckle_input.tryGetStream(StreamWrapper(self.test_stream)), Stream))
+        self.assertRaises(SpeckleException, lambda: self.speckle_input.tryGetStream(None))
+
+    def test_parameters(self):
+        actual = len(self.speckleTool.getParameterInfo())
+        expected = 15
+        self.assertEqual(actual, expected)
+
+if __name__ == '__main__':
+    unittest.main()
+