Merge pull request #306 from specklesystems/gergo/automateRaise

fix(automate_sdk): make sure we throw for failed version create

docker-compose.yml

@@ -53,7 +53,7 @@ services:
   # Speckle Server
   #######
   speckle-frontend:
-    image: speckle/speckle-frontend:2
+    image: speckle/speckle-frontend:latest
     restart: always
     ports:
       - "0.0.0.0:8080:8080"
@@ -61,7 +61,7 @@ services:
       FILE_SIZE_LIMIT_MB: 100
 
   speckle-server:
-    image: speckle/speckle-server:2
+    image: speckle/speckle-server:latest
     restart: always
     healthcheck:
       test:
@@ -112,7 +112,7 @@ services:
       ENABLE_MP: "false"
 
   preview-service:
-    image: speckle/speckle-preview-service:2
+    image: speckle/speckle-preview-service:latest
     restart: always
     depends_on:
       speckle-server:
@@ -124,7 +124,7 @@ services:
       PG_CONNECTION_STRING: "postgres://speckle:speckle@postgres/speckle"
 
   webhook-service:
-    image: speckle/speckle-webhook-service:2
+    image: speckle/speckle-webhook-service:latest
     restart: always
     depends_on:
       speckle-server:
@@ -135,7 +135,7 @@ services:
       WAIT_HOSTS: postgres:5432
 
   fileimport-service:
-    image: speckle/speckle-fileimport-service:2
+    image: speckle/speckle-fileimport-service:latest
     restart: always
     depends_on:
       speckle-server:

pyproject.toml

@@ -10,17 +10,20 @@ documentation = "https://speckle.guide/dev/py-examples.html"
 homepage = "https://speckle.systems/"
 packages = [
   { include = "specklepy", from = "src" },
+  { include = "speckle_automate", from = "src" },
 ]
 
 
 [tool.poetry.dependencies]
-python = ">=3.7.2, <4.0"
+python = ">=3.8.0, <4.0"
 pydantic = "^2.0"
 appdirs = "^1.4.4"
 gql = {extras = ["requests", "websockets"], version = "^3.3.0"}
 ujson = "^5.3.0"
 Deprecated = "^1.2.13"
 stringcase = "^1.2.0"
+attrs = "^23.1.0"
+httpx = "^0.25.0"
 
 [tool.poetry.group.dev.dependencies]
 black = "^22.8.0"

src/speckle_automate/__init__.py

@@ -0,0 +1,23 @@
+"""This module contains an SDK for working with Speckle Automate."""
+from speckle_automate.automation_context import AutomationContext
+from speckle_automate.runner import execute_automate_function, run_function
+from speckle_automate.schema import (
+    AutomateBase,
+    AutomationResult,
+    AutomationRunData,
+    AutomationStatus,
+    ObjectResult,
+    ObjectResultLevel,
+)
+
+__all__ = [
+    "AutomationContext",
+    "AutomateBase",
+    "AutomationStatus",
+    "AutomationResult",
+    "AutomationRunData",
+    "ObjectResult",
+    "ObjectResultLevel",
+    "run_function",
+    "execute_automate_function",
+]

src/speckle_automate/automation_context.py

@@ -0,0 +1,306 @@
+"""This module provides an abstraction layer above the Speckle Automate runtime."""
+from dataclasses import dataclass, field
+from pathlib import Path
+import time
+from typing import Optional, Union
+
+import httpx
+from gql import gql
+from specklepy.api import operations
+from specklepy.api.client import SpeckleClient
+from specklepy.core.api.models import Branch
+from specklepy.objects import Base
+from specklepy.transports.memory import MemoryTransport
+from specklepy.transports.server import ServerTransport
+from specklepy.logging.exceptions import SpeckleException
+
+from speckle_automate.schema import (
+    AutomateBase,
+    AutomationResult,
+    AutomationRunData,
+    AutomationStatus,
+    ObjectResult,
+    ObjectResultLevel,
+)
+
+
+@dataclass
+class AutomationContext:
+    """A context helper class.
+
+    This class exposes methods to work with the Speckle Automate context inside
+    Speckle Automate functions.
+
+    An instance of AutomationContext is injected into every run of a function.
+    """
+
+    automation_run_data: AutomationRunData
+    speckle_client: SpeckleClient
+    _server_transport: ServerTransport
+    _speckle_token: str
+
+    #: keep a memory transponrt at hand, to speed up things if needed
+    _memory_transport: MemoryTransport = field(default_factory=MemoryTransport)
+
+    #: added for performance measuring
+    _init_time: float = field(default_factory=time.perf_counter)
+    _automation_result: AutomationResult = field(default_factory=AutomationResult)
+
+    @classmethod
+    def initialize(
+        cls, automation_run_data: Union[str, AutomationRunData], speckle_token: str
+    ) -> "AutomationContext":
+        """Bootstrap the AutomateSDK from raw data.
+
+        Todo:
+        ----
+            * bootstrap a structlog logger instance
+            * expose a logger, that ppl can use instead of print
+        """
+        # parse the json value if its not an initialized project data instance
+        automation_run_data = (
+            automation_run_data
+            if isinstance(automation_run_data, AutomationRunData)
+            else AutomationRunData.model_validate_json(automation_run_data)
+        )
+        speckle_client = SpeckleClient(
+            automation_run_data.speckle_server_url,
+            automation_run_data.speckle_server_url.startswith("https"),
+        )
+        speckle_client.authenticate_with_token(speckle_token)
+        if not speckle_client.account:
+            msg = (
+                f"Could not autenticate to {automation_run_data.speckle_server_url}",
+                "with the provided token",
+            )
+            raise ValueError(msg)
+        server_transport = ServerTransport(
+            automation_run_data.project_id, speckle_client
+        )
+        return cls(automation_run_data, speckle_client, server_transport, speckle_token)
+
+    @property
+    def run_status(self) -> AutomationStatus:
+        """Get the status of the automation run."""
+        return self._automation_result.run_status
+
+    def elapsed(self) -> float:
+        """Return the elapsed time in seconds since the initialization time."""
+        return time.perf_counter() - self._init_time
+
+    def receive_version(self) -> Base:
+        """Receive the Speckle project version that triggered this automation run."""
+        commit = self.speckle_client.commit.get(
+            self.automation_run_data.project_id, self.automation_run_data.version_id
+        )
+        if not commit.referencedObject:
+            raise ValueError("The commit has no referencedObject, cannot receive it.")
+        base = operations.receive(
+            commit.referencedObject, self._server_transport, self._memory_transport
+        )
+        print(
+            f"It took {self.elapsed():2f} seconds to receive",
+            f" the speckle version {self.automation_run_data.version_id}",
+        )
+        return base
+
+    def create_new_version_in_project(
+        self, root_object: Base, model_id: str, version_message: str = ""
+    ) -> None:
+        """Save a base model to a new version on the project.
+
+        Args:
+            root_object (Base): The Speckle base object for the new version.
+            model_id (str): For now please use a `branchName`!
+            version_message (str): The message for the new version.
+        """
+        if model_id == self.automation_run_data.model_id:
+            raise ValueError(
+                f"The target model id: {model_id} cannot match the model id"
+                f" that triggered this automation: {self.automation_run_data.model_id}"
+            )
+
+        branch = self._get_model(model_id)
+        if not branch.name:
+            raise ValueError(f"The model {model_id} has no name.")
+
+        root_object_id = operations.send(
+            root_object,
+            [self._server_transport, self._memory_transport],
+            use_default_cache=False,
+        )
+
+        version_id = self.speckle_client.commit.create(
+            stream_id=self.automation_run_data.project_id,
+            object_id=root_object_id,
+            branch_name=model_id,
+            message=version_message,
+            source_application="SpeckleAutomate",
+        )
+
+        if isinstance(version_id, SpeckleException):
+            raise version_id
+
+        self._automation_result.result_versions.append(version_id)
+
+    def _get_model(self, model_id: str) -> Branch:
+        query = gql(
+            """
+            query ProjectModel($projectId: String!, $modelId: String!){
+                project(id: $projectId) {
+                    model(id: $modelId) {
+                        name
+                        id
+                        description
+                    }
+                }
+            }
+        """
+        )
+        params = {"projectId": self.automation_run_data.project_id, "modelId": model_id}
+        response = self.speckle_client.httpclient.execute(query, params)
+        return Branch.model_validate(response["project"]["model"])
+
+    def report_run_status(self) -> None:
+        """Report the current run status to the project of this automation."""
+        query = gql(
+            """
+            mutation ReportFunctionRunStatus(
+                $automationId: String!, 
+                $automationRevisionId: String!, 
+                $automationRunId: String!,
+                $versionId: String!,
+                $functionId: String!,
+                $runStatus: AutomationRunStatus!
+                $elapsed: Float!
+                $resultVersionIds: [String!]!
+                $statusMessage: String
+                $objectResults: JSONObject
+            ){
+                automationMutations {
+                    functionRunStatusReport(input: {
+                        automationId: $automationId
+                        automationRevisionId: $automationRevisionId
+                        automationRunId: $automationRunId
+                        versionId: $versionId
+                        functionRuns: [
+                        {
+                            functionId: $functionId
+                            status: $runStatus,
+                            elapsed: $elapsed,
+                            resultVersionIds: $resultVersionIds,
+                            statusMessage: $statusMessage
+                            results: $objectResults
+                        }]
+                   })
+                }
+            }
+            """
+        )
+        if self.run_status in [AutomationStatus.SUCCEEDED, AutomationStatus.FAILED]:
+            object_results = {
+                "version": "1.0.0",
+                "values": {
+                    "speckleObjects": self._automation_result.model_dump(by_alias=True)[
+                        "objectResults"
+                    ],
+                    "blobs": self._automation_result.blobs,
+                },
+            }
+        else:
+            object_results = None
+        params = {
+            "automationId": self.automation_run_data.automation_id,
+            "automationRevisionId": self.automation_run_data.automation_revision_id,
+            "automationRunId": self.automation_run_data.automation_run_id,
+            "versionId": self.automation_run_data.version_id,
+            "functionId": self.automation_run_data.function_id,
+            "runStatus": self.run_status.value,
+            "statusMessage": self._automation_result.status_message,
+            "elapsed": self.elapsed(),
+            "resultVersionIds": self._automation_result.result_versions,
+            "objectResults": object_results,
+        }
+        self.speckle_client.httpclient.execute(query, params)
+
+    def store_file_result(self, file_path: Union[Path, str]) -> None:
+        """Save a file attached to the project of this automation."""
+        path_obj = (
+            Path(file_path).resolve() if isinstance(file_path, str) else file_path
+        )
+
+        class UploadResult(AutomateBase):
+            blob_id: str
+            file_name: str
+            upload_status: int
+
+        class BlobUploadResponse(AutomateBase):
+            upload_results: list[UploadResult]
+
+        if not path_obj.exists():
+            raise ValueError("The given file path doesn't exist")
+        files = {path_obj.name: open(str(path_obj), "rb")}
+
+        url = (
+            f"{self.automation_run_data.speckle_server_url}/api/stream/"
+            f"{self.automation_run_data.project_id}/blob"
+        )
+        data = (
+            httpx.post(
+                url,
+                files=files,
+                headers={"authorization": f"Bearer {self._speckle_token}"},
+            )
+            .raise_for_status()
+            .json()
+        )
+
+        upload_response = BlobUploadResponse.model_validate(data)
+
+        if len(upload_response.upload_results) != 1:
+            raise ValueError("Expecting one upload result.")
+
+        for upload_result in upload_response.upload_results:
+            self._automation_result.blobs.append(upload_result.blob_id)
+
+    def mark_run_failed(self, status_message: str) -> None:
+        """Mark the current run a failure."""
+        self._mark_run(AutomationStatus.FAILED, status_message)
+
+    def mark_run_success(self, status_message: Optional[str]) -> None:
+        """Mark the current run a success with an optional message."""
+        self._mark_run(AutomationStatus.SUCCEEDED, status_message)
+
+    def _mark_run(
+        self, status: AutomationStatus, status_message: Optional[str]
+    ) -> None:
+        duration = self.elapsed()
+        self._automation_result.status_message = status_message
+        self._automation_result.run_status = status
+        self._automation_result.elapsed = duration
+
+        msg = f"Automation run {status.value} after {duration:2f} seconds."
+        print("\n".join([msg, status_message]) if status_message else msg)
+
+    def add_object_error(self, object_id: str, error_cause: str) -> None:
+        """Add an error to a given Speckle object."""
+        self._add_object_result(object_id, ObjectResultLevel.ERROR, error_cause)
+
+    def add_object_warning(self, object_id: str, warning: str) -> None:
+        """Add a warning to a given Speckle object."""
+        self._add_object_result(object_id, ObjectResultLevel.WARNING, warning)
+
+    def add_object_info(self, object_id: str, info: str) -> None:
+        """Add an info message to a given Speckle object."""
+        self._add_object_result(object_id, ObjectResultLevel.INFO, info)
+
+    def _add_object_result(
+        self, object_id: str, level: ObjectResultLevel, status_message: str
+    ) -> None:
+        print(
+            f"Object {object_id} was marked with {level.value.upper()}",
+            f" cause: {status_message}",
+        )
+        self._automation_result.object_results[object_id].append(
+            ObjectResult(level=level, status_message=status_message)
+        )

src/speckle_automate/runner.py

@@ -0,0 +1,155 @@
+"""Function execution module.
+
+Provides mechanisms to execute any function,
+ that conforms to the AutomateFunction "interface"
+"""
+import json
+import os
+import sys
+import traceback
+from pathlib import Path
+from typing import Callable, Optional, TypeVar, Union, overload
+
+from speckle_automate.automation_context import AutomationContext
+from speckle_automate.schema import AutomateBase, AutomationRunData, AutomationStatus
+
+T = TypeVar("T", bound=AutomateBase)
+
+AutomateFunction = Callable[[AutomationContext, T], None]
+AutomateFunctionWithoutInputs = Callable[[AutomationContext], None]
+
+
+@overload
+def execute_automate_function(
+    automate_function: AutomateFunction[T],
+    input_schema: type[T],
+) -> None:
+    ...
+
+
+@overload
+def execute_automate_function(automate_function: AutomateFunctionWithoutInputs) -> None:
+    ...
+
+
+def execute_automate_function(
+    automate_function: Union[AutomateFunction[T], AutomateFunctionWithoutInputs],
+    input_schema: Optional[type[T]] = None,
+):
+    """Runs the provided automate function with the input schema."""
+    # first arg is the python file name, we do not need that
+    args = sys.argv[1:]
+
+    if len(args) < 2:
+        raise ValueError("too few arguments specified need minimum 2")
+
+    if len(args) > 4:
+        raise ValueError("too many arguments specified, max supported is 4")
+
+    # we rely on a command name convention to decide what to do.
+    # this is here, so that the function authors do not see any of this
+    command = args[0]
+
+    if command == "generate_schema":
+        path = Path(args[1])
+        schema = json.dumps(
+            input_schema.model_json_schema(by_alias=True) if input_schema else {}
+        )
+        path.write_text(schema)
+
+    elif command == "run":
+        automation_run_data = args[1]
+        function_inputs = args[2]
+
+        speckle_token = os.environ.get("SPECKLE_TOKEN", None)
+        if not speckle_token and len(args) != 4:
+            raise ValueError("Cannot get speckle token from arguments or environment")
+
+        speckle_token = speckle_token if speckle_token else args[3]
+
+        inputs = (
+            input_schema.model_validate_json(function_inputs)
+            if input_schema
+            else input_schema
+        )
+
+        if inputs:
+            automation_context = run_function(
+                automate_function,  # type: ignore
+                automation_run_data,
+                speckle_token,
+                inputs,
+            )
+        else:
+            automation_context = run_function(
+                automate_function,  # type: ignore
+                automation_run_data,
+                speckle_token,
+            )
+
+        exit_code = (
+            0 if automation_context.run_status == AutomationStatus.SUCCEEDED else 1
+        )
+        exit(exit_code)
+
+    else:
+        raise NotImplementedError(f"Command: '{command}' is not supported.")
+
+
+@overload
+def run_function(
+    automate_function: AutomateFunction[T],
+    automation_run_data: Union[AutomationRunData, str],
+    speckle_token: str,
+    inputs: T,
+) -> AutomationContext:
+    ...
+
+
+@overload
+def run_function(
+    automate_function: AutomateFunctionWithoutInputs,
+    automation_run_data: Union[AutomationRunData, str],
+    speckle_token: str,
+) -> AutomationContext:
+    ...
+
+
+def run_function(
+    automate_function: Union[AutomateFunction[T], AutomateFunctionWithoutInputs],
+    automation_run_data: Union[AutomationRunData, str],
+    speckle_token: str,
+    inputs: Optional[T] = None,
+) -> AutomationContext:
+    """Run the provided function with the automate sdk context."""
+    automation_context = AutomationContext.initialize(
+        automation_run_data, speckle_token
+    )
+    automation_context.report_run_status()
+
+    try:
+        # avoiding complex type gymnastics here on the internals.
+        # the external type overloads make this correct
+        if inputs:
+            automate_function(automation_context, inputs)  # type: ignore
+        else:
+            automate_function(automation_context)  # type: ignore
+
+        # the function author forgot to mark the function success
+        if automation_context.run_status not in [
+            AutomationStatus.FAILED,
+            AutomationStatus.SUCCEEDED,
+        ]:
+            automation_context.mark_run_success(
+                "WARNING: Automate assumed a success status,"
+                " but it was not marked as so by the function."
+            )
+    except Exception:
+        trace = traceback.format_exc()
+        print(trace)
+        automation_context.mark_run_failed(
+            "Function error. Check the automation run logs for details."
+        )
+    finally:
+        automation_context.report_run_status()
+        return automation_context

src/speckle_automate/schema.py

@@ -0,0 +1,73 @@
+""""""
+from collections import defaultdict
+from enum import Enum
+from typing import Optional, List, Dict
+
+from pydantic import BaseModel, ConfigDict, Field
+from stringcase import camelcase
+
+
+class AutomateBase(BaseModel):
+    """Use this class as a base model for automate related DTO."""
+
+    model_config = ConfigDict(alias_generator=camelcase, populate_by_name=True)
+
+
+class AutomationRunData(BaseModel):
+    """Values of the project / model that triggered the run of this function."""
+
+    project_id: str
+    model_id: str
+    branch_name: str
+    version_id: str
+    speckle_server_url: str
+
+    automation_id: str
+    automation_revision_id: str
+    automation_run_id: str
+
+    function_id: str
+    function_release: str
+
+    model_config = ConfigDict(
+        alias_generator=camelcase, populate_by_name=True, protected_namespaces=()
+    )
+
+
+class AutomationStatus(str, Enum):
+    """Set the status of the automation."""
+
+    INITIALIZING = "INITIALIZING"
+    RUNNING = "RUNNING"
+    FAILED = "FAILED"
+    SUCCEEDED = "SUCCEEDED"
+
+
+class ObjectResultLevel(str, Enum):
+    """Possible status message levels for object reports."""
+
+    INFO = "INFO"
+    WARNING = "WARNING"
+    ERROR = "ERROR"
+
+
+class ObjectResult(AutomateBase):
+    """An object level result."""
+
+    level: ObjectResultLevel
+    status_message: str
+
+
+class AutomationResult(AutomateBase):
+    """Schema accepted by the Speckle server as a result for an automation run."""
+
+    elapsed: float = 0
+    result_view: Optional[str] = None
+    result_versions: List[str] = Field(default_factory=list)
+    blobs: List[str] = Field(default_factory=list)
+    run_status: AutomationStatus = AutomationStatus.RUNNING
+    status_message: Optional[str] = None
+
+    object_results: Dict[str, List[ObjectResult]] = Field(
+        default_factory=lambda: defaultdict(list)  # typing: ignore
+    )

src/specklepy/api/resources/commit.py

@@ -1,4 +1,4 @@
-from typing import List, Optional
+from typing import List, Optional, Union
 
 from gql import gql
 
@@ -7,6 +7,7 @@ from specklepy.api.resource import ResourceBase
 from specklepy.logging import metrics
 
 from specklepy.core.api.resources.commit import Resource as CoreResource
+from specklepy.logging.exceptions import SpeckleException
 
 
 class Resource(CoreResource):
@@ -55,8 +56,8 @@ class Resource(CoreResource):
         branch_name: str = "main",
         message: str = "",
         source_application: str = "python",
-        parents: List[str] = None,
-    ) -> str:
+        parents: Optional[List[str]] = None,
+    ) -> Union[str, SpeckleException]:
         """
         Creates a commit on a branch
 
@@ -76,7 +77,9 @@ class Resource(CoreResource):
             str -- the id of the created commit
         """
         metrics.track(metrics.SDK, self.account, {"name": "Commit Create"})
-        return super().create(stream_id, object_id, branch_name, message, source_application, parents)
+        return super().create(
+            stream_id, object_id, branch_name, message, source_application, parents
+        )
 
     def update(self, stream_id: str, commit_id: str, message: str) -> bool:
         """

src/specklepy/core/api/resources/commit.py

@@ -1,9 +1,10 @@
-from typing import List, Optional
+from typing import List, Optional, Union
 
 from gql import gql
 
 from specklepy.core.api.models import Commit
 from specklepy.core.api.resource import ResourceBase
+from specklepy.logging.exceptions import SpeckleException
 
 NAME = "commit"
 
@@ -106,8 +107,8 @@ class Resource(ResourceBase):
         branch_name: str = "main",
         message: str = "",
         source_application: str = "python",
-        parents: List[str] = None,
-    ) -> str:
+        parents: Optional[List[str]] = None,
+    ) -> Union[str, SpeckleException]:
         """
         Creates a commit on a branch
 

src/specklepy/logging/metrics.py

@@ -23,7 +23,8 @@ LOG = logging.getLogger(__name__)
 METRICS_TRACKER = None
 
 # actions
-SDK = "SDK Actions"
+SDK = "SDK Action"
+CONNECTOR = "Connector Action"
 RECEIVE = "Receive"
 SEND = "Send"
 

src/specklepy/objects/GIS/CRS.py

@@ -5,26 +5,12 @@ from specklepy.objects import Base
 class CRS(Base, speckle_type="Objects.GIS.CRS"):
     """A Coordinate Reference System stored in wkt format"""
 
-    def __init__(
-        self, 
-        name: Optional[str] = None, 
-        authority_id: Optional[str] = None, 
-        wkt: Optional[str] = None, 
-        units: Optional[str] = None, 
-        units_native: Optional[str] = None, 
-        offset_x: Optional[float] = None,
-        offset_y: Optional[float] = None,
-        rotation: Optional[float] = None,
-        **kwargs
-    ) -> None:
-        super().__init__(**kwargs)
+    name: Optional[str] = None
+    authority_id: Optional[str] = None
+    wkt: Optional[str] = None
+    units_native: Optional[str] = None 
+    offset_x: Optional[float] = None
+    offset_y: Optional[float] = None
+    rotation: Optional[float] = None
 
-        self.name = name 
-        self.authority_id = authority_id 
-        self.wkt = wkt 
-        self.units = units or "m"
-        self.units_native = units_native 
-        self.offset_x = offset_x 
-        self.offset_y = offset_y 
-        self.rotation = rotation 
 

src/specklepy/objects/GIS/geometry.py

@@ -6,99 +6,48 @@ from deprecated import deprecated
 
 class GisPolygonGeometry(Base, speckle_type="Objects.GIS.PolygonGeometry", detachable={"displayValue"}):
     """GIS Polygon Geometry"""
-    
-    def __init__(
-        self, 
-        boundary: Optional[Union[Polyline, Arc, Line, Circle, Polycurve]] = None, 
-        voids: Optional[List[Union[Polyline, Arc, Line, Circle, Polycurve]] ] = None, 
-        displayValue: Optional[List[Mesh]] = None, 
-        units: Optional[str] = None, 
-        **kwargs
-    ) -> None:
-        super().__init__(**kwargs)
 
-        self.boundary = boundary
-        self.voids = voids
-        self.displayValue = displayValue 
-        self.units = units or "m"
+    boundary: Optional[Union[Polyline, Arc, Line, Circle, Polycurve]] = None
+    voids: Optional[List[Union[Polyline, Arc, Line, Circle, Polycurve]] ] = None
+    displayValue: Optional[List[Mesh]] = None
 
 class GisPolygonElement(Base, speckle_type="Objects.GIS.PolygonElement"):
     """GIS Polygon element"""
-       
-    def __init__(
-        self, 
-        geometry: Optional[List[GisPolygonGeometry]] = None,
-        attributes: Optional[Base] = None,
-        units: Optional[str] = None,
-        **kwargs
-    ) -> None:
-        super().__init__(**kwargs)
 
-        self.geometry = geometry
-        self.attributes = attributes 
-        self.units = units or "m"
+    geometry: Optional[List[GisPolygonGeometry]] = None
+    attributes: Optional[Base] = None
 
 class GisLineElement(Base, speckle_type="Objects.GIS.LineElement"):
     """GIS Polyline element"""
-        
-    def __init__(
-        self, 
-        geometry: Optional[List[Union[Polyline, Arc, Line, Circle, Polycurve]]] = None,
-        attributes: Optional[Base] = None,
-        units: Optional[str] = None,
-        **kwargs
-    ) -> None:
-        super().__init__(**kwargs)
-        
-        self.geometry = geometry
-        self.attributes = attributes 
-        self.units = units or "m"
+
+    geometry: Optional[List[Union[Polyline, Arc, Line, Circle, Polycurve]]] = None,
+    attributes: Optional[Base] = None,
 
 class GisPointElement(Base, speckle_type="Objects.GIS.PointElement"):
     """GIS Point element"""
-    
-    def __init__(
-        self, 
-        geometry: Optional[List[Point]] = None,
-        attributes: Optional[Base] = None,
-        units: Optional[str] = None,
-        **kwargs
-    ) -> None:
-        super().__init__(**kwargs)
-        
-        self.geometry = geometry
-        self.attributes = attributes 
-        self.units = units or "m"
+
+    geometry: Optional[List[Point]] = None,
+    attributes: Optional[Base] = None,
 
 class GisRasterElement(Base, speckle_type="Objects.GIS.RasterElement", detachable={"displayValue"}):
     """GIS Raster element"""
 
-    def __init__(
-        self, 
-        band_count: Optional[int] = None,
-        band_names: Optional[List[str]] = None,
-        x_origin: Optional[float] = None,
-        y_origin: Optional[float] = None,
-        x_size: Optional[int] = None,
-        y_size: Optional[int] = None,
-        x_resolution: Optional[float] = None,
-        y_resolution: Optional[float] = None,
-        noDataValue: Optional[List[float]] = None,
-        displayValue: Optional[List[Mesh]] = None, 
-        units: Optional[str] = None,
-        **kwargs
-    ) -> None:
-        super().__init__(**kwargs)
-        
-        self.band_count = band_count
-        self.band_names = band_names
-        self.x_origin = x_origin
-        self.y_origin = y_origin
-        self.x_size = x_size
-        self.y_size = y_size
-        self.x_resolution = x_resolution
-        self.y_resolution = y_resolution
-        self.noDataValue = noDataValue
-        self.displayValue = displayValue
-        self.units = units or "m"
+    band_count: Optional[int] = None
+    band_names: Optional[List[str]] = None
+    x_origin: Optional[float] = None
+    y_origin: Optional[float] = None
+    x_size: Optional[int] = None
+    y_size: Optional[int] = None
+    x_resolution: Optional[float] = None
+    y_resolution: Optional[float] = None
+    noDataValue: Optional[List[float]] = None
+    displayValue: Optional[List[Mesh]] = None
+
+class GisTopography(GisRasterElement, speckle_type="Objects.GIS.GisTopography", detachable={"displayValue"}):
+    """GIS Raster element with 3d Topography representation"""
+
+class GisNonGeometryElement(Base, speckle_type="Objects.GIS.NonGeometryElement"):
+    """GIS Table feature"""
+
+    attributes: Optional[Base] = None
 

src/specklepy/objects/GIS/layers.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Union, Optional
 from specklepy.objects.base import Base
 from specklepy.objects.other import Collection
 
@@ -28,29 +28,22 @@ class Layer(Base, detachable={"features"}):
         self.geomType = geomType
         self.renderer = renderer or {} 
 
-class VectorLayer(Collection, detachable={"elements"}, speckle_type="Objects.GIS.VectorLayer", serialize_ignore={"features"}):
-    """GIS Vector Layer"""
+@deprecated(version="2.16", reason="Use VectorLayer or RasterLayer instead")
+class VectorLayer(
+    Collection, 
+    detachable={"elements"}, 
+    speckle_type="VectorLayer", 
+    serialize_ignore={"features"}):
 
-    def __init__(
-        self,
-        name: Optional[str]=None,
-        crs: Optional[CRS]=None,
-        units: Optional[str] = None,
-        elements: Optional[List[Base]] = None,
-        attributes: Optional[Base] = None,
-        geomType: Optional[str] = None,
-        renderer: Optional[Dict[str, Any]] = None,
-        **kwargs
-    ) -> None:
-        super().__init__(**kwargs)
-        self.name = name or ""
-        self.crs = crs
-        self.units = units
-        self.elements = elements or []
-        self.attributes = attributes
-        self.geomType = geomType or "None"
-        self.renderer = renderer or {}
-        self.collectionType = "VectorLayer"
+    """GIS Vector Layer"""
+    name: Optional[str]=None
+    crs: Optional[Union[CRS, Base]]=None
+    units: Optional[str] = None
+    elements: Optional[List[Base]] = None
+    attributes: Optional[Base] = None
+    geomType: Optional[str] = "None"
+    renderer: Optional[Dict[str, Any]] = None
+    collectionType = "VectorLayer"
     
     @property
     @deprecated(version="2.14", reason="Use elements")
@@ -61,29 +54,78 @@ class VectorLayer(Collection, detachable={"elements"}, speckle_type="Objects.GIS
     def features(self, value: Optional[List[Base]]) -> None:
         self.elements = value
 
-class RasterLayer(Collection, detachable={"elements"}, speckle_type="Objects.GIS.RasterLayer", serialize_ignore={"features"}):
+@deprecated(version="2.16", reason="Use VectorLayer or RasterLayer instead")
+class RasterLayer(
+    Collection, 
+    detachable={"elements"}, 
+    speckle_type="RasterLayer", 
+    serialize_ignore={"features"}):
+
     """GIS Raster Layer"""
 
-    def __init__(
-        self,
-        name: Optional[str] = None,
-        crs: Optional[CRS]=None,
-        units: Optional[str] = None,
-        rasterCrs: Optional[CRS]=None,
-        elements: Optional[List[Base]] = None,
-        geomType: Optional[str] = None,
-        renderer: Optional[Dict[str, Any]] = None,
-        **kwargs
-    ) -> None:
-        super().__init__(**kwargs)
-        self.name = name or ""
-        self.crs = crs
-        self.units = units
-        self.rasterCrs = rasterCrs
-        self.elements = elements or []
-        self.geomType = geomType or "None"
-        self.renderer = renderer or {}
-        self.collectionType = "RasterLayer"
+    name: Optional[str] = None
+    crs: Optional[Union[CRS, Base]]=None
+    units: Optional[str] = None
+    rasterCrs: Optional[Union[CRS, Base]]=None
+    elements: Optional[List[Base]] = None
+    geomType: Optional[str] = "None"
+    renderer: Optional[Dict[str, Any]] = None
+    collectionType = "RasterLayer"
+
+    
+    @property
+    @deprecated(version="2.14", reason="Use elements")
+    def features(self) -> Optional[List[Base]]:
+        return self.elements
+
+    @features.setter
+    def features(self, value: Optional[List[Base]]) -> None:
+        self.elements = value
+
+
+class VectorLayer(
+    Collection, 
+    detachable={"elements"}, 
+    speckle_type="Objects.GIS.VectorLayer", 
+    serialize_ignore={"features"}):
+
+    """GIS Vector Layer"""
+
+    name: Optional[str]=None
+    crs: Optional[Union[CRS, Base]]=None
+    units: Optional[str] = None
+    elements: Optional[List[Base]] = None
+    attributes: Optional[Base] = None
+    geomType: Optional[str] = "None"
+    renderer: Optional[Dict[str, Any]] = None
+    collectionType = "VectorLayer"
+    
+    @property
+    @deprecated(version="2.14", reason="Use elements")
+    def features(self) -> Optional[List[Base]]:
+        return self.elements
+
+    @features.setter
+    def features(self, value: Optional[List[Base]]) -> None:
+        self.elements = value
+
+class RasterLayer(
+    Collection, 
+    detachable={"elements"}, 
+    speckle_type="Objects.GIS.RasterLayer", 
+    serialize_ignore={"features"}):
+
+    """GIS Raster Layer"""
+
+    name: Optional[str] = None
+    crs: Optional[Union[CRS, Base]]=None
+    units: Optional[str] = None
+    rasterCrs: Optional[Union[CRS, Base]]=None
+    elements: Optional[List[Base]] = None
+    geomType: Optional[str] = "None"
+    renderer: Optional[Dict[str, Any]] = None
+    collectionType = "RasterLayer"
+
     
     @property
     @deprecated(version="2.14", reason="Use elements")

src/specklepy/objects/base.py

@@ -18,7 +18,7 @@ from warnings import warn
 
 from stringcase import pascalcase
 
-from specklepy.logging.exceptions import SpeckleException
+from specklepy.logging.exceptions import SpeckleException, SpeckleInvalidUnitException
 from specklepy.objects.units import Units, get_units_from_string
 from specklepy.transports.memory import MemoryTransport
 
@@ -322,7 +322,7 @@ class Base(_RegisteringBase):
     id: Union[str, None] = None
     totalChildrenCount: Union[int, None] = None
     applicationId: Union[str, None] = None
-    _units: Union[Units, None] = None
+    _units: Union[None, str] = None
 
     def __init__(self, **kwargs) -> None:
         super().__init__()
@@ -463,22 +463,19 @@ class Base(_RegisteringBase):
 
     @property
     def units(self) -> Union[str, None]:
-        if self._units:
-            return self._units.value
-        return None
+        return self._units
 
     @units.setter
     def units(self, value: Union[str, Units, None]):
-        if value is None:
-            units = value
+        """While this property accepts any string value, geometry expects units to be specific strings (see Units enum)"""
+        if isinstance(value, str) or value is None:
+            self._units = value
         elif isinstance(value, Units):
-            units: Units = value
+            self._units = value.value
         else:
-            units = get_units_from_string(value)
-        self._units = units
-        # except SpeckleInvalidUnitException as ex:
-        #     warn(f"Units are reset to None. Reason {ex.message}")
-        #     self._units = None
+            raise SpeckleInvalidUnitException(
+                f"Unknown type {type(value)} received for units"
+            )
 
     def get_member_names(self) -> List[str]:
         """Get all of the property names on this object, dynamic or not"""

src/specklepy/objects/geometry.py

@@ -898,7 +898,7 @@ class Brep(
     def VerticesValue(self) -> List[Point]:
         if self.Vertices is None:
             return None
-        encoded_unit = get_encoding_from_units(self.Vertices[0]._units)
+        encoded_unit = get_encoding_from_units(self.Vertices[0].units)
         values = [encoded_unit]
         for vertex in self.Vertices:
             values.extend(vertex.to_list())
@@ -913,7 +913,7 @@ class Brep(
 
         for i in range(0, len(value), 3):
             vertex = Point.from_list(value[i : i + 3])
-            vertex._units = units
+            vertex.units = units
             vertices.append(vertex)
 
         self.Vertices = vertices

src/specklepy/objects/graph_traversal/commit_object_builder.py

@@ -0,0 +1,83 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any, Collection, Dict, Generic, Iterable, List, Optional, Tuple, TypeVar
+from attrs import define
+from specklepy.objects.base import Base
+
+ROOT: str = "__Root"
+
+T = TypeVar('T')
+PARENT_INFO = Tuple[Optional[str], str]
+
+@define(slots=True)
+class CommitObjectBuilder(ABC, Generic[T]):
+
+    converted: Dict[str, Base]
+    _parent_infos: Dict[str, Collection[PARENT_INFO]]
+
+    def __init__(self) -> None:
+        self.converted = {}
+        self._parent_infos = {}
+    
+    @abstractmethod
+    def include_object(self, conversion_result: Base, native_object: T) -> None:
+        pass
+
+    def build_commit_object(self, root_commit_object: Base) -> None:
+        self.apply_relationships(self.converted.values(), root_commit_object)
+
+    def set_relationship(self, app_id: Optional[str], *parent_info : PARENT_INFO) -> None:
+
+        if not app_id:
+            return
+        
+        self._parent_infos[app_id] = parent_info
+
+    def apply_relationships(self, to_add: Iterable[Base], root_commit_object: Base) -> None:
+        for c in to_add:
+            try:
+                self.apply_relationship(c, root_commit_object)
+            except Exception as ex:
+                print(f"Failed to add object {type(c)} to commit object: {ex}")
+
+    def apply_relationship(self, current: Base, root_commit_object: Base):
+        if not current.applicationId: raise Exception(f"Expected applicationId to have been set") 
+        
+        parents = self._parent_infos[current.applicationId]
+        
+        for (parent_id, prop_name) in parents:
+            if not parent_id: continue
+
+            parent: Optional[Base]
+            if parent_id == ROOT:
+                parent = root_commit_object
+            else:
+                parent = self.converted[parent_id] if parent_id in self.converted else None
+            
+            if not parent: continue
+
+            try:
+                elements = get_detached_prop(parent, prop_name)
+                if not isinstance(elements, list):
+                    elements = []
+                    set_detached_prop(parent, prop_name, elements)
+
+                elements.append(current)
+                return
+            except Exception as ex:
+                # A parent was found, but it was invalid (Likely because of a type mismatch on a `elements` property)
+                print(f"Failed to add object {type(current)} to a converted parent; {ex}")
+                
+            raise Exception(f"Could not find a valid parent for object of type {type(current)}. Checked {len(parents)} potential parent, and non were converted!")
+
+
+def get_detached_prop(speckle_object: Base, prop_name: str) -> Optional[Any]:
+    detached_prop_name = get_detached_prop_name(speckle_object, prop_name)
+    return getattr(speckle_object, detached_prop_name, None)
+
+def set_detached_prop(speckle_object: Base, prop_name: str, value: Optional[Any]) -> None:
+    detached_prop_name = get_detached_prop_name(speckle_object, prop_name)
+    setattr(speckle_object, detached_prop_name, value)
+
+def get_detached_prop_name(speckle_object: Base, prop_name: str) -> str:
+    return prop_name if hasattr(speckle_object, prop_name) else f"@{prop_name}"

src/specklepy/objects/graph_traversal/traversal.py

@@ -0,0 +1,123 @@
+from typing import Any, Callable, Collection, Iterable, Iterator, List, Optional, Set
+
+from attrs import define
+from typing_extensions import Protocol, final
+
+from specklepy.objects import Base
+
+
+class ITraversalRule(Protocol):
+    def get_members_to_traverse(self, o: Base) -> Set[str]:
+        """Get the members to traverse."""
+        pass
+
+    def does_rule_hold(self, o: Base) -> bool:
+        """Make sure the rule still holds."""
+        pass
+
+
+@final
+@define(slots=True, frozen=True)
+class DefaultRule:
+    def get_members_to_traverse(self, _) -> Set[str]:
+        return set()
+
+    def does_rule_hold(self, _) -> bool:
+        return True
+
+
+# we're creating a local protected "singleton"
+_default_rule = DefaultRule()
+
+
+@final
+@define(slots=True, frozen=True)
+class TraversalContext:
+    current: Base
+    member_name: Optional[str] = None
+    parent: Optional["TraversalContext"] = None
+
+
+@final
+@define(slots=True, frozen=True)
+class GraphTraversal:
+
+    _rules: List[ITraversalRule]
+
+    def traverse(self, root: Base) -> Iterator[TraversalContext]:
+        stack: List[TraversalContext] = []
+
+        stack.append(TraversalContext(root))
+
+        while len(stack) > 0:
+            head = stack.pop()
+            yield head
+
+            current = head.current
+            active_rule = self._get_active_rule_or_default_rule(current)
+            members_to_traverse = active_rule.get_members_to_traverse(current)
+            for child_prop in members_to_traverse:
+                try:
+                    if child_prop in {"speckle_type", "units", "applicationId"}: continue #debug: to avoid noisy exceptions, explicitly avoid checking ones we know will fail, this is not exhaustive
+                    if getattr(current, child_prop, None):
+                        value = current[child_prop]
+                        self._traverse_member_to_stack(
+                            stack, value, child_prop, head
+                        )
+                except KeyError as ex:
+                    # Unset application ids, and class variables like SpeckleType will throw when __getitem__ is called
+                    pass
+    @staticmethod
+    def _traverse_member_to_stack(
+        stack: List[TraversalContext],
+        value: Any,
+        member_name: Optional[str] = None,
+        parent: Optional[TraversalContext] = None,
+    ):
+        if isinstance(value, Base):
+            stack.append(TraversalContext(value, member_name, parent))
+        elif isinstance(value, list):
+            for obj in value:
+                GraphTraversal._traverse_member_to_stack(stack, obj, member_name, parent)
+        elif isinstance(value, dict):
+            for obj in value.values():
+                GraphTraversal._traverse_member_to_stack(stack, obj, member_name, parent)
+
+    @staticmethod
+    def traverse_member(value: Optional[Any]) -> Iterator[Base]:
+        if isinstance(value, Base):
+            yield value
+        elif isinstance(value, list):
+            for obj in value:
+                for o in GraphTraversal.traverse_member(obj):
+                    yield o
+        elif isinstance(value, dict):
+            for obj in value.values():
+                for o in GraphTraversal.traverse_member(obj):
+                    yield o
+
+
+    def _get_active_rule_or_default_rule(self, o: Base) -> ITraversalRule:
+        return self._get_active_rule(o) or _default_rule
+
+    def _get_active_rule(self, o: Base) -> Optional[ITraversalRule]:
+        for rule in self._rules:
+            if rule.does_rule_hold(o):
+                return rule
+        return None
+
+
+@final
+@define(slots=True, frozen=True)
+class TraversalRule:
+    _conditions: Collection[Callable[[Base], bool]]
+    _members_to_traverse: Callable[[Base], Iterable[str]]
+
+    def get_members_to_traverse(self, o: Base) -> Set[str]:
+        return set(self._members_to_traverse(o))
+
+    def does_rule_hold(self, o: Base) -> bool:
+        for condition in self._conditions:
+            if condition(o):
+                return True
+        return False

src/specklepy/objects/units.py

@@ -35,6 +35,7 @@ UNITS_STRINGS = {
     Units.none: ["none", "null"],
 }
 
+
 UNITS_ENCODINGS = {
     Units.none: 0,
     None: 0,
@@ -49,6 +50,20 @@ UNITS_ENCODINGS = {
 }
 
 
+UNIT_SCALE = {
+    Units.none: 1,
+    Units.mm: 0.001,
+    Units.cm: 0.01,
+    Units.m: 1.0,
+    Units.km: 1000.0,
+    Units.inches: 0.0254,
+    Units.feet: 0.3048,
+    Units.yards: 0.9144,
+    Units.miles: 1609.340,
+}
+"""Unit scaling factor to meters"""
+
+
 def get_units_from_string(unit: str) -> Units:
     if not isinstance(unit, str):
         raise SpeckleInvalidUnitException(unit)
@@ -59,10 +74,10 @@ def get_units_from_string(unit: str) -> Units:
     raise SpeckleInvalidUnitException(unit)
 
 
-def get_units_from_encoding(unit: int):
+def get_units_from_encoding(unit: int) -> Units:
     for name, encoding in UNITS_ENCODINGS.items():
         if unit == encoding:
-            return name
+            return name or Units.none
 
     raise SpeckleException(
         message=(
@@ -72,13 +87,36 @@ def get_units_from_encoding(unit: int):
     )
 
 
-def get_encoding_from_units(unit: Union[Units, None]):
+def get_encoding_from_units(unit: Union[Units, str, None]):
+    maybe_sanitized_unit = unit
+    if isinstance(unit, str):
+        for unit_enum, aliases in UNITS_STRINGS.items():
+            if unit in aliases:
+                maybe_sanitized_unit = unit_enum
     try:
-        return UNITS_ENCODINGS[unit]
+        return UNITS_ENCODINGS[maybe_sanitized_unit]
     except KeyError as e:
         raise SpeckleException(
             message=(
-                f"No encoding exists for unit {unit}."
+                f"No encoding exists for unit {maybe_sanitized_unit}."
                 f"Please enter a valid unit to encode (eg {UNITS_ENCODINGS})."
             )
         ) from e
+
+
+def get_scale_factor_from_string(fromUnits: str, toUnits: str) -> float:
+    """Returns a scalar to convert distance values from one unit system to another"""
+    return get_scale_factor(get_units_from_string(fromUnits), get_units_from_string(toUnits))
+
+
+def get_scale_factor(fromUnits: Units, toUnits: Units) -> float:
+    """Returns a scalar to convert distance values from one unit system to another"""
+    return get_scale_factor_to_meters(fromUnits) / get_scale_factor_to_meters(toUnits)
+
+
+def get_scale_factor_to_meters(fromUnits: Units) -> float:
+    """Returns a scalar to convert distance values from one unit system to meters"""
+    if fromUnits not in UNIT_SCALE:
+        raise ValueError(f"Invalid units provided: {fromUnits}")
+
+    return UNIT_SCALE[fromUnits]

src/specklepy/transports/abstract_transport.py

@@ -1,17 +1,12 @@
 from abc import ABC, abstractmethod
 from typing import Dict, List, Optional
 
-from pydantic import BaseModel
-from pydantic.config import Extra
-
-
-class AbstractTransport(ABC, BaseModel):
-    _name: str = "Abstract"
-    model_config = {'extra': 'allow', 'arbitrary_types_allowed': True}
 
+class AbstractTransport(ABC):
     @property
+    @abstractmethod
     def name(self):
-        return type(self)._name
+        pass
 
     @abstractmethod
     def begin_write(self) -> None:

src/specklepy/transports/memory.py

@@ -4,14 +4,15 @@ from specklepy.transports.abstract_transport import AbstractTransport
 
 
 class MemoryTransport(AbstractTransport):
-    _name: str = "Memory"
-    objects: dict = {}
-    saved_object_count: int = 0
+    def __init__(self, name="Memory") -> None:
+        super().__init__()
+        self._name = name
+        self.objects = {}
+        self.saved_object_count = 0
 
-    def __init__(self, name=None, **data: Any) -> None:
-        super().__init__(**data)
-        if name:
-            self._name = name
+    @property
+    def name(self) -> str:
+        return self._name
 
     def __repr__(self) -> str:
         return f"MemoryTransport(objects: {len(self.objects)})"

src/specklepy/transports/server/server.py

@@ -4,7 +4,7 @@ from warnings import warn
 
 import requests
 
-from specklepy.api.client import SpeckleClient
+from specklepy.core.api.client import SpeckleClient
 from specklepy.core.api.credentials import Account, get_account_from_token
 from specklepy.logging.exceptions import SpeckleException, SpeckleWarning
 from specklepy.transports.abstract_transport import AbstractTransport
@@ -45,13 +45,6 @@ class ServerTransport(AbstractTransport):
     ```
     """
 
-    _name = "RemoteTransport"
-    url: Optional[str] = None
-    stream_id: Optional[str] = None
-    account: Optional[Account] = None
-    saved_obj_count: int = 0
-    session: Optional[requests.Session] = None
-
     def __init__(
         self,
         stream_id: str,
@@ -59,15 +52,18 @@ class ServerTransport(AbstractTransport):
         account: Optional[Account] = None,
         token: Optional[str] = None,
         url: Optional[str] = None,
-        **data: Any,
+        name: str = "RemoteTransport",
     ) -> None:
-        super().__init__(**data)
+        super().__init__()
         if client is None and account is None and token is None and url is None:
             raise SpeckleException(
                 "You must provide either a client or a token and url to construct a"
                 " ServerTransport."
             )
 
+        self._name = name
+        self.account = None
+        self.saved_obj_count = 0
         if account:
             self.account = account
             url = account.serverInfo.url
@@ -97,6 +93,10 @@ class ServerTransport(AbstractTransport):
             {"Authorization": f"Bearer {self.account.token}", "Accept": "text/plain"}
         )
 
+    @property
+    def name(self) -> str:
+        return self._name
+
     def begin_write(self) -> None:
         self.saved_obj_count = 0
 

src/specklepy/transports/sqlite.py

@@ -9,31 +9,22 @@ from specklepy.transports.abstract_transport import AbstractTransport
 
 
 class SQLiteTransport(AbstractTransport):
-    _name = "SQLite"
-    _base_path: Optional[str] = None
-    _root_path: Optional[str] = None
-    __connection: Optional[sqlite3.Connection] = None
-    app_name: str = ""
-    scope: str = ""
-    saved_obj_count: int = 0
-    max_size: Optional[int] = None
-    _current_batch: Optional[List[Tuple[str, str]]] = None
-    _current_batch_size: Optional[int] = None
-
     def __init__(
         self,
         base_path: Optional[str] = None,
         app_name: Optional[str] = None,
         scope: Optional[str] = None,
         max_batch_size_mb: float = 10.0,
-        **data: Any,
+        name: str = "SQLite",
     ) -> None:
-        super().__init__(**data)
+        super().__init__()
+        self._name = name
         self.app_name = app_name or "Speckle"
         self.scope = scope or "Objects"
         self._base_path = base_path or self.get_base_path(self.app_name)
         self.max_size = int(max_batch_size_mb * 1000 * 1000)
-        self._current_batch = []
+        self.saved_obj_count = 0
+        self._current_batch: List[Tuple[str, str]] = []
         self._current_batch_size = 0
 
         try:
@@ -54,24 +45,12 @@ class SQLiteTransport(AbstractTransport):
     def __repr__(self) -> str:
         return f"SQLiteTransport(app: '{self.app_name}', scope: '{self.scope}')"
 
+    @property
+    def name(self) -> str:
+        return self._name
+
     @staticmethod
     def get_base_path(app_name):
-        # # from appdirs https://github.com/ActiveState/appdirs/blob/master/appdirs.py
-        # # default mac path is not the one we use (we use unix path), so using special case for this
-        # system = sys.platform
-        # if system.startswith("java"):
-        #     import platform
-
-        #     os_name = platform.java_ver()[3][0]
-        #     if os_name.startswith("Mac"):
-        #         system = "darwin"
-
-        # if system != "darwin":
-        #     return user_data_dir(appname=app_name, appauthor=False, roaming=True)
-
-        # path = os.path.expanduser("~/.config/")
-        # return os.path.join(path, app_name)
-
         return str(
             speckle_path_provider.user_application_data_path().joinpath(app_name)
         )

tests/intergration/speckle_automate/test_automation_context.py

@@ -0,0 +1,263 @@
+"""Run integration tests with a speckle server."""
+import os
+import secrets
+import string
+from pathlib import Path
+from typing import Dict
+
+import pytest
+from gql import gql
+from speckle_automate.schema import AutomateBase
+from specklepy.api import operations
+from specklepy.api.client import SpeckleClient
+from specklepy.objects.base import Base
+from specklepy.transports.server import ServerTransport
+
+from speckle_automate import (
+    AutomationContext,
+    AutomationRunData,
+    AutomationStatus,
+    run_function,
+)
+
+
+def crypto_random_string(length: int) -> str:
+    """Generate a semi crypto random string of a given length."""
+    alphabet = string.ascii_letters + string.digits
+    return "".join(secrets.choice(alphabet) for _ in range(length))
+
+
+def register_new_automation(
+    project_id: str,
+    model_id: str,
+    speckle_client: SpeckleClient,
+    automation_id: str,
+    automation_name: str,
+    automation_revision_id: str,
+):
+    """Register a new automation in the speckle server."""
+    query = gql(
+        """
+        mutation CreateAutomation(
+            $projectId: String! 
+            $modelId: String! 
+            $automationName: String!
+            $automationId: String! 
+            $automationRevisionId: String!
+        ) {
+                automationMutations {
+                    create(
+                        input: {
+                            projectId: $projectId
+                            modelId: $modelId
+                            automationName: $automationName 
+                            automationId: $automationId
+                            automationRevisionId: $automationRevisionId
+                        }
+                    )
+                }
+            }
+        """
+    )
+    params = {
+        "projectId": project_id,
+        "modelId": model_id,
+        "automationName": automation_name,
+        "automationId": automation_id,
+        "automationRevisionId": automation_revision_id,
+    }
+    speckle_client.httpclient.execute(query, params)
+
+
+@pytest.fixture()
+def speckle_token(user_dict: Dict[str, str]) -> str:
+    """Provide a speckle token for the test suite."""
+    return user_dict["token"]
+
+
+@pytest.fixture()
+def speckle_server_url(host: str) -> str:
+    """Provide a speckle server url for the test suite, default to localhost."""
+    return f"http://{host}"
+
+
+@pytest.fixture()
+def test_client(speckle_server_url: str, speckle_token: str) -> SpeckleClient:
+    """Initialize a SpeckleClient for testing."""
+    test_client = SpeckleClient(speckle_server_url, use_ssl=False)
+    test_client.authenticate_with_token(speckle_token)
+    return test_client
+
+
+@pytest.fixture()
+def test_object() -> Base:
+    """Create a Base model for testing."""
+    root_object = Base()
+    root_object.foo = "bar"
+    return root_object
+
+
+@pytest.fixture()
+def automation_run_data(
+    test_object: Base, test_client: SpeckleClient, speckle_server_url: str
+) -> AutomationRunData:
+    """Set up an automation context for testing."""
+    project_id = test_client.stream.create("Automate function e2e test")
+    branch_name = "main"
+
+    model = test_client.branch.get(project_id, branch_name, commits_limit=1)
+    model_id: str = model.id
+
+    root_obj_id = operations.send(
+        test_object, [ServerTransport(project_id, test_client)]
+    )
+    version_id = test_client.commit.create(project_id, root_obj_id)
+
+    automation_name = crypto_random_string(10)
+    automation_id = crypto_random_string(10)
+    automation_revision_id = crypto_random_string(10)
+
+    register_new_automation(
+        project_id,
+        model_id,
+        test_client,
+        automation_id,
+        automation_name,
+        automation_revision_id,
+    )
+
+    automation_run_id = crypto_random_string(10)
+    function_id = crypto_random_string(10)
+    function_release = crypto_random_string(10)
+    return AutomationRunData(
+        project_id=project_id,
+        model_id=model_id,
+        branch_name=branch_name,
+        version_id=version_id,
+        speckle_server_url=speckle_server_url,
+        automation_id=automation_id,
+        automation_revision_id=automation_revision_id,
+        automation_run_id=automation_run_id,
+        function_id=function_id,
+        function_release=function_release,
+    )
+
+
+def get_automation_status(
+    project_id: str,
+    model_id: str,
+    speckle_client: SpeckleClient,
+):
+    query = gql(
+        """
+query AutomationRuns(
+            $projectId: String! 
+            $modelId: String! 
+    )
+{
+  project(id: $projectId) {
+    model(id: $modelId) {
+      automationStatus {
+        id
+        status
+        statusMessage
+        automationRuns {
+          id
+          automationId
+          versionId
+          createdAt
+          updatedAt
+          status
+          functionRuns {
+            id
+            functionId
+            elapsed
+            status
+            contextView
+            statusMessage
+            results
+            resultVersions {
+              id
+            }
+          }
+        }
+      }
+    }
+  }
+}
+        """
+    )
+    params = {
+        "projectId": project_id,
+        "modelId": model_id,
+    }
+    response = speckle_client.httpclient.execute(query, params)
+    return response["project"]["model"]["automationStatus"]
+
+
+class FunctionInputs(AutomateBase):
+    forbidden_speckle_type: str
+
+
+def automate_function(
+    automate_context: AutomationContext,
+    function_inputs: FunctionInputs,
+) -> None:
+    """Hey, trying the automate sdk experience here."""
+    version_root_object = automate_context.receive_version()
+
+    count = 0
+    if version_root_object.speckle_type == function_inputs.forbidden_speckle_type:
+        if not version_root_object.id:
+            raise ValueError("Cannot operate on objects without their id's.")
+        automate_context.add_object_error(
+            version_root_object.id,
+            "This project should not contain the type: "
+            f"{function_inputs.forbidden_speckle_type}",
+        )
+        count += 1
+
+    if count > 0:
+        automate_context.mark_run_failed(
+            "Automation failed: "
+            f"Found {count} object that have a forbidden speckle type: "
+            f"{function_inputs.forbidden_speckle_type}"
+        )
+
+    else:
+        automate_context.mark_run_success("No forbidden types found.")
+
+
+def test_function_run(automation_run_data: AutomationRunData, speckle_token: str):
+    """Run an integration test for the automate function."""
+    automation_context = run_function(
+        automate_function,
+        automation_run_data,
+        speckle_token,
+        FunctionInputs(forbidden_speckle_type="Base"),
+    )
+
+    assert automation_context.run_status == AutomationStatus.FAILED
+    status = get_automation_status(
+        automation_run_data.project_id,
+        automation_run_data.model_id,
+        automation_context.speckle_client,
+    )
+    assert status["status"] == automation_context.run_status
+    status_message = status["automationRuns"][0]["functionRuns"][0]["statusMessage"]
+    assert status_message == automation_context._automation_result.status_message
+
+
+def test_file_uploads(automation_run_data: AutomationRunData, speckle_token: str):
+    """Test file store capabilities of the automate sdk."""
+    automation_context = AutomationContext.initialize(
+        automation_run_data, speckle_token
+    )
+
+    path = Path(f"./{crypto_random_string(10)}").resolve()
+    path.write_text("foobar")
+
+    automation_context.store_file_result(path)
+
+    os.remove(path)
+    assert len(automation_context._automation_result.blobs) == 1

tests/intergration/test_serialization.py

@@ -17,7 +17,7 @@ class TestSerialization:
         deserialized = operations.deserialize(serialized)
 
         assert base.get_id() == deserialized.get_id()
-        assert base.units == "mm"
+        assert base.units == "millimetres"
         assert isinstance(base.test_bases[0], Base)
         assert base["@revit_thing"].speckle_type == "SpecialRevitFamily"
         assert base["@detach"].name == deserialized["@detach"].name

tests/unit/test_base.py

@@ -85,14 +85,15 @@ def test_speckle_type_cannot_be_set(base: Base) -> None:
 
 def test_setting_units():
     b = Base(units="foot")
-    assert b.units == "ft"
+    assert b.units == "foot"
 
-    with pytest.raises(SpeckleInvalidUnitException):
-        b.units = "big"
+    # with pytest.raises(SpeckleInvalidUnitException):
+    b.units = "big"
+    assert b.units == "big"
 
     with pytest.raises(SpeckleInvalidUnitException):
         b.units = 7  # invalid args are skipped
-    assert b.units == "ft"
+    assert b.units == "big"
 
     b.units = None  # None should be a valid arg
     assert b.units is None

tests/unit/test_geometry.py

@@ -388,9 +388,9 @@ def test_brep_curve3d_values_serialization(curve, polyline, circle):
 def test_brep_vertices_values_serialization():
     brep = Brep()
     brep.VerticesValue = [1, 1, 1, 1, 2, 2, 2, 3, 3, 3]
-    assert brep.Vertices[0].get_id() == Point(x=1, y=1, z=1, _units=Units.mm).get_id()
-    assert brep.Vertices[1].get_id() == Point(x=2, y=2, z=2, _units=Units.mm).get_id()
-    assert brep.Vertices[2].get_id() == Point(x=3, y=3, z=3, _units=Units.mm).get_id()
+    assert brep.Vertices[0].get_id() == Point(x=1, y=1, z=1, units=Units.mm).get_id()
+    assert brep.Vertices[1].get_id() == Point(x=2, y=2, z=2, units=Units.mm).get_id()
+    assert brep.Vertices[2].get_id() == Point(x=3, y=3, z=3, units=Units.mm).get_id()
 
 
 def test_trims_value_serialization():

tests/unit/test_graph_traversal.py

@@ -0,0 +1,105 @@
+from dataclasses import dataclass
+from typing import Dict, List, Optional
+from unittest import TestCase
+
+from specklepy.objects import Base
+from specklepy.objects.graph_traversal.traversal import GraphTraversal, TraversalRule
+
+
+@dataclass()
+class TraversalMock(Base):
+    child: Optional[Base]
+    list_children: List[Base]
+    dict_children: Dict[str, Base]
+
+
+class GraphTraversalTests(TestCase):
+    def test_traverse_list_members(self):
+        traverse_lists_rule = TraversalRule(
+            [lambda _: True],
+            lambda x: [
+                item
+                for item in x.get_member_names()
+                if isinstance(getattr(x, item, None), list)
+            ],
+        )
+
+        expected_traverse = Base()
+        expected_traverse.id = "List Member"
+
+        expected_ignore = Base()
+        expected_ignore.id = "Not List Member"
+
+        test_case = TraversalMock(
+            list_children=[expected_traverse],
+            dict_children={"myprop": expected_ignore},
+            child=expected_ignore,
+        )
+
+        ret = [
+            context.current
+            for context in GraphTraversal([traverse_lists_rule]).traverse(test_case)
+        ]
+
+        self.assertCountEqual(ret, [test_case, expected_traverse])
+        self.assertNotIn(expected_ignore, ret)
+        self.assertEqual(len(ret), 2)
+
+    def test_traverse_dict_members(self):
+        traverse_lists_rule = TraversalRule(
+            [lambda _: True],
+            lambda x: [
+                item
+                for item in x.get_member_names()
+                if isinstance(getattr(x, item, None), dict)
+            ],
+        )
+
+        expected_traverse = Base()
+        expected_traverse.id = "Dict Member"
+
+        expected_ignore = Base()
+        expected_ignore.id = "Not Dict Member"
+
+        test_case = TraversalMock(
+            list_children=[expected_ignore],
+            dict_children={"myprop": expected_traverse},
+            child=expected_ignore,
+        )
+
+        ret = [
+            context.current
+            for context in GraphTraversal([traverse_lists_rule]).traverse(test_case)
+        ]
+
+        self.assertCountEqual(ret, [test_case, expected_traverse])
+        self.assertNotIn(expected_ignore, ret)
+        self.assertEqual(len(ret), 2)
+
+    def test_traverse_dynamic(self):
+        traverse_lists_rule = TraversalRule(
+            [lambda _: True], lambda x: x.get_dynamic_member_names()
+        )
+
+        expected_traverse = Base()
+        expected_traverse.id = "List Member"
+
+        expected_ignore = Base()
+        expected_ignore.id = "Not List Member"
+
+        test_case = TraversalMock(
+            child=expected_ignore,
+            list_children=[expected_ignore],
+            dict_children={"myprop": expected_ignore},
+        )
+        test_case["dynamicChild"] = expected_traverse
+        test_case["dynamicListChild"] = [expected_traverse]
+
+        ret = [
+            context.current
+            for context in GraphTraversal([traverse_lists_rule]).traverse(test_case)
+        ]
+
+        self.assertCountEqual(ret, [test_case, expected_traverse, expected_traverse])
+        self.assertNotIn(expected_ignore, ret)
+        self.assertEqual(len(ret), 3)

tests/unit/test_unit_scaling.py

@@ -0,0 +1,56 @@
+
+
+import pytest
+
+from specklepy.objects.units import Units, get_scale_factor
+
+
+@pytest.mark.parametrize(
+    "fromUnits, toUnits, inValue, expectedOutValue",
+    [
+        #To self
+        (Units.km, Units.km, 1.5, 1.5),
+        (Units.km, Units.km, 0, 0),
+        (Units.m, Units.m, 1.5, 1.5),
+        (Units.m, Units.m, 0, 0),
+        (Units.cm, Units.cm, 1.5, 1.5),
+        (Units.cm, Units.cm, 0, 0),
+        (Units.mm, Units.mm, 1.5, 1.5),
+        (Units.mm, Units.mm, 0, 0),
+        (Units.miles, Units.miles, 1.5, 1.5),
+        (Units.miles, Units.miles, 0, 0),
+        (Units.yards, Units.yards, 1.5, 1.5),
+        (Units.yards, Units.yards, 0, 0),
+        (Units.feet, Units.feet, 1.5, 1.5),
+        (Units.feet, Units.feet, 0, 0),
+
+        #To Meters
+        (Units.km, Units.m, 987654.321, 987654321),
+        (Units.m, Units.m, 987654.321, 987654.321),
+        (Units.mm, Units.m, 98765432.1, 98765.4321),
+        (Units.cm, Units.m, 9876543.21, 98765.4321),
+
+        #To negative meters
+        (Units.km, Units.m, -987654.321, -987654321),
+        (Units.m, Units.m,- 987654.321, -987654.321),
+        (Units.mm, Units.m, -98765432.1, -98765.4321),
+        (Units.cm, Units.m, -9876543.21, -98765.4321),
+
+        (Units.m, Units.km, 987654.321, 987.654321),
+        (Units.m, Units.cm, 987654.321, 98765432.1),
+        (Units.m, Units.mm, 987654.321, 987654321),
+
+        #Imperial
+        (Units.miles, Units.m, 123.45, 198673.517),
+        (Units.miles, Units.inches, 123.45, 7821792),
+        (Units.yards, Units.m, 123.45, 112.88268),
+        (Units.yards, Units.inches, 123.45, 4444.2),
+        (Units.feet, Units.m, 123.45, 37.62756),
+        (Units.feet, Units.inches, 123.45, 1481.4),
+        (Units.inches, Units.m, 123.45, 3.13563),
+    ],
+)
+def test_get_scale_factor_between_units(fromUnits: Units, toUnits: Units, inValue: float, expectedOutValue: float):
+    Tolerance = 1e-10
+    actual = inValue * get_scale_factor(fromUnits, toUnits)
+    assert(actual - expectedOutValue < Tolerance)