feat(commits): add new fields

🍰 Chunking of long lists

README.md

@@ -18,6 +18,113 @@ venv\Scripts\activate
 pip install -r requirements.txt
 ```
 
+on mac:
+
+```
+python -m venv venv
+source venv/bin/activate
+pip install -r requirements.txt
+```
+
+
+
+## Overview of functionality 
+
+The `SpeckleClient` is the entry point for interacting with the GraphQL API. You'll need to have a running server to use this.
+
+```py
+from speckle.api.client import SpeckleClient
+from speckle.api.credentials import get_default_account, get_local_accounts
+
+all_accounts = get_local_accounts() # get back a list
+account = get_default_account()
+
+client = SpeckleClient(host="localhost:3000", use_ssl=False)
+# client = SpeckleClient(host="yourserver.com") or whatever your host is
+
+client.authenticate(account.token)
+```
+
+Interacting with streams is meant to be intuitive and evocative of PySpeckle 1.0
+
+```py
+# get your streams
+stream_list = client.stream.list()
+
+# search your streams
+results = client.user.search("mech")
+
+# create a stream
+new_stream_id = client.stream.create(name="a shiny new stream")
+
+# get a stream
+new_stream = client.stream.get(id=new_stream_id)
+```
+
+New in 2.0: commits! Here are some basic commit interactions.
+
+```py
+# get list of commits
+commits = client.commit.list("stream id")
+
+# get a specific commit
+commit = client.commit.get("stream id", "commit id")
+
+# create a commit
+commit_id = client.commit.create("stream id", "object id", "this is a commit message to describe the commit")
+
+# delete a commit
+deleted = client.commit.delete("stream id", "commit id")
+```
+
+The `BaseObjectSerializer` is used for decomposing and serializing `Base` objects so they can be sent / received to the server. You can use it directly to get the id (hash) and a serializable object representation of the decomposed `Base`. You can learn more about the Speckle `Base` object [here](https://discourse.speckle.works/t/core-2-0-the-base-object/782) and the decomposition API [here](https://discourse.speckle.works/t/core-2-0-decomposition-api/911).
+
+```py
+detached_base = Base()
+detached_base.name = "this will get detached"
+
+base_obj = Base()
+base_obj.name = "my base"
+base_obj["@nested"] = detached_base
+
+serializer = BaseObjectSerializer()
+hash, obj_dict = serializer.traverse_base(base_obj)
+```
+
+If you use the `operations`, you will not need to interact with the serializer directly as this will be taken care of for you. You will just need to provide a transport to indicate where the objects should be sent / received from. At the moment, just the `MemoryTransport` and the `ServerTransport` are fully functional at the moment. If you'd like to learn more about Transports in Speckle 2.0, have a look [here](https://discourse.speckle.works/t/core-2-0-transports/919).
+
+```py
+transport = MemoryTransport()
+
+# this serialises the object and sends it to the transport 
+hash = operations.send(base=base_obj, transports=[transport])
+
+# if the object had detached objects, you can see these as well
+saved_objects = transport.objects # a dict with the obj hash as the key
+
+# this receives and object from the given transport, deserialises it, and recomposes it into a base object
+received_base = operations.receive(obj_id=hash, remote_transport=transport)
+```
+
+You can also use the GraphQL API to send and receive objects.
+
+```py
+# create a test base object
+test_base = Base()
+test_base.testing = "a test base obj"
+
+# run it through the serialiser
+s = BaseObjectSerializer() 
+hash, obj = s.traverse_base(test_base)
+
+# send it to the server
+objCreate = client.object.create(stream_id="stream id", objects=[obj])
+
+received_base = client.object.get(hash)
+```
+
+This doc is not complete - there's more to see so have a dive into the code and play around! Please feel free to provide feedback, submit issues, or discuss new features ✨
+
 ## Contributing
 
 Please make sure you read the [contribution guidelines](.github/CONTRIBUTING.md) for an overview of the best practices we try to follow.

speckle/api/client.py

@@ -1,9 +1,10 @@
+import re
 from gql.client import SyncClientSession
 from speckle.logging.exceptions import SpeckleException
 from typing import Dict
 
 from speckle.api import resources
-from speckle.api.resources import stream, server, user, subscriptions
+from speckle.api.resources import commit, stream, object, server, user, subscriptions
 from gql import Client, gql
 from gql.transport.requests import RequestsHTTPTransport
 from gql.transport.aiohttp import AIOHTTPTransport
@@ -22,6 +23,9 @@ class SpeckleClient:
             ws_protocol = "wss"
             http_protocol = "https"
 
+        # sanitise host input by removing protocol and trailing slash
+        host = re.sub(r"((^\w+:|^)\/\/)|(\/$)", "", host)
+
         self.url = f"{http_protocol}://{host}"
         self.graphql = self.url + "/graphql"
         self.ws_url = f"{ws_protocol}://{host}/graphql"
@@ -65,6 +69,12 @@ class SpeckleClient:
         self.stream = stream.Resource(
             me=self.me, basepath=self.url, client=self.httpclient
         )
+        self.commit = commit.Resource(
+            me=self.me, basepath=self.url, client=self.httpclient
+        )
+        self.object = object.Resource(
+            me=self.me, basepath=self.url, client=self.httpclient
+        )
         self.server = server.Resource(
             me=self.me, basepath=self.url, client=self.httpclient
         )

speckle/api/credentials.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+from typing import List, Optional
+from pydantic import BaseModel
+from speckle.transports.sqlite import SQLiteTransport
+
+account_storage = SQLiteTransport(scope="Accounts")
+
+
+def get_local_accounts() -> List[Account]:
+    """Gets all the accounts present in this environment
+
+    Returns:
+        List[Account] -- list of all local accounts or an empty list if no accounts were found
+    """
+    res = account_storage.get_all_objects()
+    return [Account.parse_raw(r[1]) for r in res] if res else []
+
+
+def get_default_account() -> Account:
+    """Gets this environment's default account if any. If there is no default, the first found will be returned and set as default.
+
+    Returns:
+        Account -- the default account or None if no local accounts were found
+    """
+    accounts = get_local_accounts()
+    if not accounts:
+        return None
+
+    default = next((acc for acc in accounts if acc.isDefault), None)
+    if not default:
+        default = accounts[0]
+        default.isDefault = True
+
+    return default
+
+
+class ServerInfo(BaseModel):
+    name: str
+    company: Optional[str]
+    url: str
+
+
+class UserInfo(BaseModel):
+    name: str
+    email: str
+    company: Optional[str]
+    id: str
+
+
+class Account(BaseModel):
+    isDefault: bool
+    token: str
+    refreshToken: str
+    serverInfo: ServerInfo
+    userInfo: UserInfo
+    id: str
+
+    def __repr__(self) -> str:
+        return f"Account(email: {self.userInfo.email}, server: {self.serverInfo.url}, isDefault: {self.isDefault})"
+
+    def __str__(self) -> str:
+        return f"Account(email: {self.userInfo.email}, server: {self.serverInfo.url}, isDefault: {self.isDefault})"

speckle/api/models.py

@@ -21,12 +21,22 @@ class Collaborator(BaseModel):
 class Commit(BaseModel):
     id: Optional[str]
     message: Optional[str]
+    sourceApplication: Optional[str]
+    totalChildrenCount: Optional[int]
+    branchName: Optional[str]
+    parents: Optional[List[str]]
     authorName: Optional[str]
     authorId: Optional[str]
     authorAvatar: Optional[str]
     createdAt: Optional[str]
     referencedObject: Optional[str]
 
+    def __repr__(self) -> str:
+        return f"Commit( id: {self.id}, message: {self.message}, referencedObject: {self.referencedObject}, authorName: {self.authorName}, createdAt: {self.createdAt} )"
+
+    def __str__(self) -> str:
+        return self.__repr__()
+
 
 class Commits(BaseModel):
     totalCount: Optional[int]
@@ -73,6 +83,12 @@ class Stream(BaseModel):
     commit: Optional[Commit]
     object: Optional[Object]
 
+    def __repr__(self):
+        return f"Stream( id: {self.id}, name: {self.name}, description: {self.description}, isPublic: {self.isPublic})"
+
+    def __str__(self) -> str:
+        return self.__repr__()
+
 
 class User(BaseModel):
     id: Optional[str]
@@ -84,3 +100,9 @@ class User(BaseModel):
     verified: Optional[bool]
     role: Optional[str]
     streams: Optional[Streams]
+
+    def __repr__(self):
+        return f"User( id: {self.id}, name: {self.name}, email: {self.email}, company: {self.company} )"
+
+    def __str__(self) -> str:
+        return self.__repr__()

speckle/api/operations.py

@@ -0,0 +1,93 @@
+from typing import List
+from speckle.objects.base import Base
+from speckle.transports.memory import MemoryTransport
+from speckle.logging.exceptions import SpeckleException
+from speckle.transports.abstract_transport import AbstractTransport
+from speckle.serialization.base_object_serializer import BaseObjectSerializer
+
+
+def send(
+    base: Base,
+    transports: List[AbstractTransport] = [],
+    use_default_cache: bool = True,
+):
+    """Sends an object via the provided transports. Defaults to the local cache.
+
+    Arguments:
+        obj {Base} -- the object you want to send
+        transports {list} -- where you want to send them
+        use_default_cache {bool} -- toggle for the default cache. If set to false, it will only send to the provided transports
+
+    Returns:
+        str -- the object id of the sent object
+    """
+    if not transports and not use_default_cache:
+        raise SpeckleException(
+            message="You need to provide at least one transport: cannot send with an empty transport list and no default cache"
+        )
+    if use_default_cache:
+        # TODO: finish sqlite transport and chuck it in here
+        pass
+
+    serializer = BaseObjectSerializer(write_transports=transports)
+
+    for t in transports:
+        t.begin_write()
+    hash, _ = serializer.write_json(base=base)
+
+    for t in transports:
+        t.end_write()
+
+    return hash
+
+
+def receive(
+    obj_id: str,
+    remote_transport: AbstractTransport = None,
+    local_transport: AbstractTransport = None,
+) -> Base:
+    """Receives an object from a transport.
+
+    Arguments:
+        obj_id {str} -- the id of the object to receive
+        remote_transport {Transport} -- the transport to receive from
+        local_transport {Transport} -- the transport to send from
+
+    Returns:
+        Base -- the base object
+    """
+
+    # TODO: replace with sqlite transport
+    if not local_transport:
+        local_transport = MemoryTransport()
+
+    serializer = BaseObjectSerializer(read_transport=local_transport)
+
+    # try local transport first. if the parent is there, we assume all the children are there and continue wth deserialisation using the local transport
+    obj_string = local_transport.get_object(obj_id)
+    if obj_string:
+        base = serializer.read_json(obj_string=obj_string)
+        return base
+
+    if not remote_transport:
+        raise SpeckleException(
+            message="Could not find the specified object using the local transport, and you didn't provide a fallback remote from which to pull it."
+        )
+
+    obj_string = remote_transport.copy_object_and_children(
+        id=obj_id, target_transport=local_transport
+    )
+
+    return serializer.read_json(obj_string=obj_string)
+
+
+def serialize(base: Base) -> str:
+    serializer = BaseObjectSerializer()
+
+    return serializer.write_json(base)[1]
+
+
+def deserialize(obj_string: str) -> Base:
+    serializer = BaseObjectSerializer()
+
+    return serializer.read_json(obj_string=obj_string)

speckle/api/resources/commit.py

@@ -16,4 +16,159 @@ class Resource(ResourceBase):
         super().__init__(
             me=me, basepath=basepath, client=client, name=NAME, methods=METHODS
         )
-        self.schema = Commit
+        self.schema = Commit
+
+    def get(self, stream_id: str, commit_id: str) -> Commit:
+        """
+        Gets a commit given a stream and the commit id
+
+        Arguments:
+            stream_id {str} -- the stream where we can find the commit
+            commit_id {str} -- the id of the commit you want to get
+
+        Returns:
+            Commit -- the retrieved commit object
+        """
+        query = gql(
+            """
+            query Commit($stream_id: String!, $commit_id: String!) {
+                stream(id: $stream_id) {
+                    commit(id: $commit_id) {
+                        id
+                        referencedObject
+                        message
+                        authorName
+                        authorId
+                        createdAt
+                    }
+                }
+            }
+            """
+        )
+        params = {"stream_id": stream_id, "commit_id": commit_id}
+
+        return self.make_request(
+            query=query, params=params, return_type=["stream", "commit"]
+        )
+
+    def list(self, stream_id: str, limit: int = 10) -> List[Commit]:
+        """
+        Get a list of commits on a given stream
+
+        Arguments:
+            stream_id {str} -- the stream where the commits are
+            limit {int} -- the maximum number of commits to fetch (default = 10)
+
+        Returns:
+            List[Commit] -- a list of the most recent commit objects
+        """
+        query = gql(
+            """
+            query Commits($stream_id: String!, $limit: Int!) {
+                stream(id: $stream_id) {
+                    commits(limit: $limit) {
+                        items {
+                            id
+                            message
+                            authorName
+                            authorId
+                            createdAt
+                            referencedObject
+                        }
+                    }
+                }
+            }
+            """
+        )
+        params = {"stream_id": stream_id, "limit": limit}
+
+        return self.make_request(
+            query=query, params=params, return_type=["stream", "commits", "items"]
+        )
+
+    def create(
+        self,
+        stream_id: str,
+        object_id: str,
+        branch_name: str = "main",
+        message: str = "",
+        source_application: str = "PySpeckle",
+    ) -> str:
+        """
+        Creates a commit on a branch
+
+        Arguments:
+            stream_id {str} -- the stream you want to commit to
+            object_id {str} -- the hash of your commit object
+            branch_name {str} -- the name of the branch to commit to (defaults to "main")
+            message {str} -- optional: a message to give more information about the commit
+            source_application {str} -- optional: the name of the application creating the commit (defaults to "PySpeckle")
+
+        Returns:
+            str -- the id of the created commit
+        """
+        query = gql(
+            """
+            mutation CommitCreate ($commit: CommitCreateInput!){ commitCreate(commit: $commit)}
+            """
+        )
+        params = {
+            "commit": {
+                "streamId": stream_id,
+                "branchName": branch_name,
+                "objectId": object_id,
+                "message": message,
+                "sourceApplication": source_application,
+            }
+        }
+
+        return self.make_request(
+            query=query, params=params, return_type="commitCreate", parse_response=False
+        )
+
+    def update(self, stream_id: str, commit_id: str, message: str) -> bool:
+        """
+        Update a commit
+
+        Arguments:
+            stream_id {str} -- the id of the stream that contains the commit you'd like to update
+            commit_id {str} -- the id of the commit you'd like to update
+            message {str} -- the updated commit message
+
+        Returns:
+            bool -- True if the operation succeeded
+        """
+        query = gql(
+            """
+            mutation CommitUpdate($commit: CommitUpdateInput!){ commitUpdate(commit: $commit)}
+            """
+        )
+        params = {
+            "commit": {"streamId": stream_id, "id": commit_id, "message": message}
+        }
+
+        return self.make_request(
+            query=query, params=params, return_type="commitUpdate", parse_response=False
+        )
+
+    def delete(self, stream_id: str, commit_id: str) -> bool:
+        """
+        Delete a commit
+
+        Arguments:
+            stream_id {str} -- the id of the stream that contains the commit you'd like to delete
+            commit_id {str} -- the id of the commit you'd like to delete
+
+        Returns:
+            bool -- True if the operation succeeded
+        """
+        query = gql(
+            """
+            mutation CommitDelete($commit: CommitDeleteInput!){ commitDelete(commit: $commit)}
+            """
+        )
+        params = {"commit": {"streamId": stream_id, "id": commit_id}}
+
+        return self.make_request(
+            query=query, params=params, return_type="commitDelete", parse_response=False
+        )

speckle/api/resources/object.py

@@ -0,0 +1,77 @@
+from typing import Dict, List
+from gql import gql
+from graphql.language import parser
+from speckle.api.resource import ResourceBase
+from speckle.objects.base import Base
+
+NAME = "object"
+METHODS = []
+
+
+class Resource(ResourceBase):
+    """API Access class for objects"""
+
+    def __init__(self, me, basepath, client) -> None:
+        super().__init__(
+            me=me, basepath=basepath, client=client, name=NAME, methods=METHODS
+        )
+        self.schema = Base
+
+    def get(self, stream_id: str, object_id: str) -> Base:
+        """
+        Get a stream object
+
+        Arguments:
+            stream_id {str} -- the id of the stream for the object
+            object_id {str} -- the hash of the object you want to get
+
+        Returns:
+            Base -- the returned Base object
+        """
+        query = gql(
+            """
+            query Object($stream_id: String!, $object_id: String!) {
+              stream(id: $stream_id) {
+                id
+                name
+                object(id: $object_id) {
+                  id
+                  speckleType
+                  applicationId
+                  createdAt
+                  totalChildrenCount
+                  data
+                }
+              }
+            }
+          """
+        )
+        params = {"stream_id": stream_id, "object_id": object_id}
+
+        return self.make_request(
+            query=query, params=params, return_type=["stream", "object"]
+        )
+
+    def create(self, stream_id: str, objects: List[Dict]) -> str:
+        """
+        Create a new object on a stream. To send a base object, you can prepare it by running it through the `BaseObjectSerializer.travers_base` function to get a valid (serialisable) object to send.
+
+        NOTE: this does not create a commit - you can create one with `SpeckleClient.commit.create`.
+
+        Arguments:
+            stream_id {str} -- the id of the stream you want to send the object to
+            objects {List[Dict]} -- a list of base dictionary objects (NOTE: must be json serialisable)
+
+        Returns:
+            str -- the id of the object
+        """
+        query = gql(
+            """
+          mutation ObjectCreate($object_input: ObjectCreateInput!) { objectCreate(objectInput: $object_input) }
+          """
+        )
+        params = {"object_input": {"streamId": stream_id, "objects": objects}}
+
+        return self.make_request(
+            query=query, params=params, return_type="objectCreate", parse_response=False
+        )

speckle/logging/exceptions.py

@@ -1,4 +1,4 @@
-from typing import List
+from typing import Any, List
 
 
 class SpeckleException(Exception):
@@ -6,12 +6,25 @@ class SpeckleException(Exception):
         self.message = message
         self.exception = exception
 
-    def __repr__(self) -> str:
+    def __str__(self) -> str:
         return f"SpeckleException: {self.message}"
 
 
+class SerializationException(SpeckleException):
+    def __init__(self, message: str, object: Any, exception: Exception = None) -> None:
+        super().__init__(message=message)
+        self.object = object
+        self.unhandled_type = type(object)
+
+    def __str__(self) -> str:
+        return f"SpeckleException: Could not serialize object of type {self.unhandled_type}"
+
+
 class GraphQLException(SpeckleException):
     def __init__(self, message: str, errors: List, data=None) -> None:
         super().__init__(message=message)
         self.errors = errors
         self.data = data
+
+    def __str__(self) -> str:
+        return f"GraphQLException: {self.message}"

speckle/objects/__init__.py

@@ -0,0 +1,14 @@
+from pathlib import Path
+import sys
+import inspect
+import pkgutil
+from importlib import import_module
+from .base import Base
+
+
+for (_, name, _) in pkgutil.iter_modules([Path(__file__).parent]):
+    imported_module = import_module("." + name, package=__name__)
+    classes = inspect.getmembers(imported_module, inspect.isclass)
+    for c in classes:
+        if issubclass(c[1], Base):
+            setattr(sys.modules[__name__], c[0], c[1])

speckle/objects/base.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+from pydantic import BaseModel
+from pydantic.main import Extra
+from typing import Dict, List, Optional, Any
+from speckle.transports.memory import MemoryTransport
+from speckle.logging.exceptions import SpeckleException
+
+
+PRIMITIVES = (int, float, str, bool)
+
+
+class Base(BaseModel):
+    id: Optional[str] = None
+    totalChildrenCount: Optional[int] = None
+    applicationId: Optional[str] = None
+    speckle_type: Optional[str] = "Base"
+    _chunkable: Dict[str, int] = {}  # dict of chunkable props and their max chunk size
+
+    def __init__(self, **kwargs) -> None:
+        super().__init__()
+        self.speckle_type = self.__class__.__name__
+        self.__dict__.update(kwargs)
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}(id: {self.id}, speckle_type: {self.speckle_type}, totalChildrenCount: {self.totalChildrenCount})"
+
+    def __str__(self) -> str:
+        return self.__repr__()
+
+    def __setitem__(self, name: str, value: Any) -> None:
+        self.__dict__[name] = value
+
+    def __getitem__(self, name: str) -> Any:
+        return self.__dict__[name]
+
+    def to_dict(self) -> Dict:
+        """Convenience method to view the whole base object as a dict"""
+        base_dict = self.__dict__
+        for key, value in base_dict.items():
+            if not value or isinstance(value, PRIMITIVES):
+                continue
+            else:
+                base_dict[key] = self.__dict_helper(value)
+        return base_dict
+
+    def __dict_helper(self, obj: Any) -> Any:
+        if isinstance(obj, PRIMITIVES):
+            return obj
+        if isinstance(obj, Base):
+            return self.__dict_helper(obj.__dict__)
+        if isinstance(obj, (list, set)):
+            return [self.__dict_helper(v) for v in obj]
+        if isinstance(obj, dict):
+            for k, v in obj.items():
+                if not v or isinstance(obj, PRIMITIVES):
+                    pass
+                else:
+                    obj[k] = self.__dict_helper(v)
+            return obj
+        else:
+            raise SpeckleException(
+                message=f"Could not convert to dict due to unrecognised type: {type(obj)}"
+            )
+
+    def get_member_names(self) -> List[str]:
+        """Get all of the property names on this object, dynamic or not"""
+        return list(self.__dict__.keys())
+
+    def get_typed_member_names(self) -> List[str]:
+        """Get all of the names of the defined (typed) properties of this object"""
+        return list(self.__fields__.keys())
+
+    def get_dynamic_member_names(self) -> List[str]:
+        """Get all of the names of the dynamic properties of this object"""
+        return list(set(self.__dict__.keys()) - set(self.__fields__.keys()))
+
+    def get_children_count(self) -> int:
+        """Get the total count of children Base objects"""
+        parsed = []
+        return 1 + self._count_descendants(self, parsed)
+
+    def get_id(self, decompose: bool = False) -> str:
+        if self.id and not decompose:
+            return self.id
+        else:
+            from speckle.serialization.base_object_serializer import (
+                BaseObjectSerializer,
+            )
+
+            serializer = BaseObjectSerializer()
+            if decompose:
+                serializer.write_transports = [MemoryTransport()]
+            return serializer.traverse_base(self)[0]
+
+    def _count_descendants(self, base: Base, parsed: List) -> int:
+        if base in parsed:
+            return 0
+        parsed.append(base)
+
+        count = 0
+
+        for name, value in base.__dict__.items():
+            if name.startswith("@"):
+                continue
+            else:
+                count += self._handle_object_count(value, parsed)
+
+        return count
+
+    def _handle_object_count(self, obj: Any, parsed: List) -> int:
+        count = 0
+        if obj == None:
+            return count
+        if isinstance(obj, Base):
+            count += 1
+            count += self._count_descendants(obj, parsed)
+            return count
+        elif isinstance(obj, list):
+            for item in obj:
+                if isinstance(item, Base):
+                    count += 1
+                    count += self._count_descendants(item, parsed)
+                else:
+                    count += self._handle_object_count(item, parsed)
+        elif isinstance(obj, dict):
+            for _, value in obj.items():
+                if isinstance(value, Base):
+                    count += 1
+                    count += self._count_descendants(value, parsed)
+                else:
+                    count += self._handle_object_count(value, parsed)
+        return count
+
+    class Config:
+        extra = Extra.allow
+
+
+class DataChunk(Base):
+    data: List[Any] = []

speckle/objects/mesh.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from typing import List, Optional
+
+from pydantic import BaseModel
+from .base import Base
+
+CHUNKABLE_PROPS = {
+    "vertices": 2000,
+    "faces": 2000,
+    "colors": 2000,
+    "textureCoordinates": 2000,
+}
+
+
+class Mesh(Base):
+    vertices: List[float] = None
+    faces: List[int] = None
+    colors: List[int] = None
+    textureCoordinates: List[float] = None
+
+    def __init__(self, **kwargs) -> None:
+        super().__init__(**kwargs)
+        self._chunkable.update(CHUNKABLE_PROPS)

speckle/serialization/base_object_serializer.py

@@ -0,0 +1,309 @@
+import json
+import hashlib
+
+from speckle import objects
+from uuid import uuid4
+from typing import Any, Dict, List, Tuple
+from speckle.objects.base import Base, DataChunk
+from speckle.logging.exceptions import SerializationException, SpeckleException
+from speckle.transports.abstract_transport import AbstractTransport
+
+PRIMITIVES = (int, float, str, bool)
+
+
+def hash_obj(obj: Any) -> str:
+    return hashlib.sha256(json.dumps(obj).encode()).hexdigest()[:32]
+
+
+class BaseObjectSerializer:
+    read_transport: AbstractTransport
+    write_transports: List[AbstractTransport]
+    detach_lineage: List[bool] = []  # tracks depth and whether or not to detach
+    lineage: List[str] = []  # keeps track of hash chain through the object tree
+    family_tree: Dict[str, Dict[str, int]] = {}
+    closure_table: Dict[str, Dict[str, int]] = {}
+
+    def __init__(
+        self, write_transports: List[AbstractTransport] = [], read_transport=None
+    ) -> None:
+        self.write_transports = write_transports
+        self.read_transport = read_transport
+
+    def write_json(self, base: Base):
+        self.__reset_writer()
+        self.detach_lineage = [True]
+        hash, obj = self.traverse_base(base)
+        return hash, json.dumps(obj)
+
+    def traverse_base(self, base: Base) -> Tuple[str, Dict]:
+        """Decomposes the given base object and builds a serializable dictionary
+
+        Arguments:
+            base {Base} -- the base object to be decomposed and serialized
+
+        Returns:
+            (str, dict) -- a tuple containing the hash (id) of the base object and the constructed serializable dictionary
+        """
+        if not self.detach_lineage:
+            self.detach_lineage = [True]
+
+        self.lineage.append(uuid4().hex)
+        object_builder = {"id": ""}
+        obj, props = base, base.get_member_names()
+
+        while props:
+            prop = props.pop(0)
+            value = obj[prop]
+
+            # skip nulls or props marked to be ignored with "__" or "_"
+            if not value or prop.startswith(("__", "_")):
+                continue
+
+            # don't prepopulate id as this will mess up hashing
+            if prop == "id":
+                continue
+
+            chunkable = True if prop in base._chunkable else False
+            detach = True if prop.startswith("@") or chunkable else False
+
+            # 1. handle primitives (ints, floats, strings, and bools)
+            if isinstance(value, PRIMITIVES):
+                object_builder[prop] = value
+                continue
+
+            # 2. handle Base objects
+            elif isinstance(value, Base):
+                child_obj = self.traverse_value(value, detach=detach)
+                if detach and self.write_transports:
+                    ref_hash = child_obj["id"]
+                    object_builder[prop] = self.detach_helper(ref_hash=ref_hash)
+                else:
+                    object_builder[prop] = child_obj
+
+            # 3. handle chunkable props
+            elif chunkable and self.write_transports:
+                chunks = []
+                max_size = base._chunkable[prop]
+                chunk = DataChunk()
+                for count, item in enumerate(value):
+                    if count and count % max_size == 0:
+                        chunks.append(chunk)
+                        chunk = DataChunk()
+                    chunk.data.append(item)
+                chunks.append(chunk)
+
+                chunk_refs = []
+                for c in chunks:
+                    self.detach_lineage.append(detach)
+                    ref_hash, _ = self.traverse_base(c)
+                    ref_obj = self.detach_helper(ref_hash=ref_hash)
+                    chunk_refs.append(ref_obj)
+                object_builder[prop] = chunk_refs
+
+            # 4. handle all other cases
+            else:
+                child_obj = self.traverse_value(value)
+                object_builder[prop] = child_obj
+
+        hash = hash_obj(object_builder)
+        object_builder["id"] = hash
+
+        detached = self.detach_lineage.pop()
+
+        # add closures to the object
+        if self.lineage[-1] in self.family_tree:
+            object_builder["__closure"] = self.closure_table[hash] = {
+                ref: depth - len(self.detach_lineage)
+                for ref, depth in self.family_tree[self.lineage[-1]].items()
+            }
+
+        # write detached or root objects to transports
+        if detached and self.write_transports:
+            for t in self.write_transports:
+                t.save_object(id=hash, serialized_object=json.dumps(object_builder))
+
+        del self.lineage[-1]
+
+        return hash, object_builder
+
+    def traverse_value(self, obj: Any, detach: bool = False) -> Any:
+        """Decomposes a given object and constructs a serializable object or dictionary
+
+        Arguments:
+            obj {Any} -- the value to decompose
+
+        Returns:
+            Any -- a serializable version of the given object
+        """
+        if isinstance(obj, PRIMITIVES):
+            return obj
+
+        elif isinstance(obj, (list, tuple, set)):
+            return [self.traverse_value(o) for o in obj]
+
+        elif isinstance(obj, dict):
+            for k, v in obj.items():
+                if isinstance(v, PRIMITIVES):
+                    continue
+                else:
+                    obj[k] = self.traverse_value(v)
+            return obj
+
+        elif isinstance(obj, Base):
+            self.detach_lineage.append(detach)
+            _, base_obj = self.traverse_base(obj)
+            return base_obj
+
+        else:
+            try:
+                return obj.dict()
+            except:
+                SerializationException(
+                    message=f"Failed to handle {type(obj)} in `BaseObjectSerializer.traverse_value`",
+                    object=obj,
+                )
+                return str(obj)
+
+    def detach_helper(self, ref_hash: str) -> Dict[str, str]:
+        """Helper to keep track of detached objects and their depth in the family tree and create reference objects to place in the parent object
+
+        Arguments:
+            ref_hash {str} -- the hash of the fully traversed object
+
+        Returns:
+            dict -- a reference object to be inserted into the given object's parent
+        """
+
+        for parent in self.lineage:
+            if parent not in self.family_tree:
+                self.family_tree[parent] = {}
+            if ref_hash not in self.family_tree[parent] or self.family_tree[parent][
+                ref_hash
+            ] > len(self.detach_lineage):
+                self.family_tree[parent][ref_hash] = len(self.detach_lineage)
+
+        return {
+            "referencedId": ref_hash,
+            "speckle_type": "reference",
+        }
+
+    def __reset_writer(self) -> None:
+        """Reinitializes the lineage, and other variables that get used during the json writing process"""
+        self.detach_lineage = []
+        self.lineage = []
+        self.family_tree = {}
+        self.closure_table = {}
+
+    def read_json(self, obj_string: str) -> Base:
+        """Recomposes a Base object from the string representation of the object
+
+        Arguments:
+            obj_string {str} -- the string representation of the object
+
+        Returns:
+            Base -- the base object with all it's children attached
+        """
+        if not obj_string:
+            return None
+        obj = json.loads(obj_string)
+        base = self.recompose_base(obj=obj)
+
+        return base
+
+    def recompose_base(self, obj: dict) -> Base:
+        """Steps through a base object dictionary and recomposes the base object
+
+        Arguments:
+            obj {dict} -- the dictionary representation of the object
+
+        Returns:
+            Base -- the base object with all its children attached
+        """
+        # make sure an obj was passed and create dict if string was somehow passed
+        if not obj:
+            return
+        if isinstance(obj, str):
+            obj = json.loads(obj)
+        if obj["speckle_type"] == "reference":
+            obj = self.get_child(obj=obj)
+
+        # initialise the base object using `speckle_type`
+        base = getattr(objects, obj["speckle_type"], Base)()
+
+        # get total children count
+        if "__closure" in obj:
+            if not self.read_transport:
+                raise SpeckleException(
+                    message="Cannot resolve reference - no read transport is defined"
+                )
+            closure = obj.pop("__closure")
+            base.totalChildrenCount = len(closure)
+
+        for prop, value in obj.items():
+            # 1. handle primitives (ints, floats, strings, and bools)
+            if isinstance(value, PRIMITIVES):
+                base[prop] = value
+                continue
+
+            # 2. handle referenced child objects
+            elif "referencedId" in value:
+                ref_hash = value["referencedId"]
+                ref_obj_str = self.read_transport.get_object(id=ref_hash)
+                if not ref_obj_str:
+                    raise SpeckleException(
+                        f"Could not find the referenced child object of id `{ref_hash}` in the given read transport: {self.read_transport.name}"
+                    )
+                ref_obj = json.loads(ref_obj_str)
+                base[prop] = self.recompose_base(obj=ref_obj)
+
+            # 3. handle all other cases (base objects, lists, and dicts)
+            else:
+                base[prop] = self.handle_value(value)
+
+        return base
+
+    def handle_value(self, obj: Any):
+        """Helper for recomposing a base object by handling the dictionary representation's values
+
+        Arguments:
+            obj {Any} -- a value from the base object dictionary
+
+        Returns:
+            Any -- the handled value (primitive, list, dictionary, or Base)
+        """
+        if isinstance(obj, PRIMITIVES):
+            return obj
+
+        # lists (regular and chunked)
+        if isinstance(obj, list):
+            obj_list = [self.handle_value(o) for o in obj]
+            # handle chunked lists
+            if isinstance(obj_list[0], DataChunk):
+                data = []
+                for o in obj_list:
+                    data.extend(o["data"])
+                return data
+            else:
+                return obj_list
+
+        # bases
+        if isinstance(obj, dict) and "speckle_type" in obj:
+            return self.recompose_base(obj=obj)
+
+        # dictionaries
+        if isinstance(obj, dict):
+            for k, v in obj.items():
+                if isinstance(v, PRIMITIVES):
+                    continue
+                else:
+                    obj[k] = self.handle_value(v)
+            return obj
+
+    def get_child(self, obj: Dict):
+        ref_hash = obj["referencedId"]
+        ref_obj_str = self.read_transport.get_object(id=ref_hash)
+        if not ref_obj_str:
+            raise SpeckleException(
+                f"Could not find the referenced child object of id `{ref_hash}` in the given read transport: {self.read_transport.name}"
+            )
+        return json.loads(ref_obj_str)

speckle/transports/abstract_transport.py

@@ -0,0 +1,83 @@
+from abc import ABC, abstractmethod
+from typing import Any, Optional
+from pydantic import BaseModel
+from pydantic.main import Extra
+
+#  __________________
+# |                  |
+# |  this is v wip   |
+# |  pls be careful  |
+# |__________________|
+# (\__/) ||
+# (•ㅅ•)  ||
+# / 　 づ
+
+
+class AbstractTransport(ABC, BaseModel):
+    _name: str = "Abstract"
+
+    @property
+    def name(self):
+        return type(self)._name
+
+    @abstractmethod
+    def begin_write(self) -> None:
+        """Optional: signals to the transport that writes are about to begin."""
+        pass
+
+    @abstractmethod
+    def end_write(self) -> None:
+        """Optional: signals to the transport that no more items will need to be written."""
+        pass
+
+    @abstractmethod
+    def save_object(self, id: str, serialized_object: str) -> None:
+        """Saves the given serialized object.
+
+        Arguments:
+            id {str} -- the hash of the object
+            serialized_object {str} -- the full string representation of the object
+        """
+        pass
+
+    @abstractmethod
+    def save_object_from_transport(
+        self, id: str, source_transport: "AbstractTransport"
+    ) -> None:
+        """Saves an object from the given source transport.
+
+        Arguments:
+            id {str} -- the hash of the object
+            source_transport {AbstractTransport) -- the transport through which the object can be found
+        """
+        pass
+
+    @abstractmethod
+    def get_object(self, id: str) -> Optional[str]:
+        """Gets an object. Returns `None` if the object is not found.
+
+        Arguments:
+            id {str} -- the hash of the object
+
+        Returns:
+            str -- the full string representation of the object (or null if no object is found)
+        """
+        pass
+
+    @abstractmethod
+    def copy_object_and_children(
+        self, id: str, target_transport: "AbstractTransport"
+    ) -> str:
+        """Copies the parent object and all its children to the provided transport.
+
+        Arguments:
+            id {str} -- the id of the object you want to copy
+            target_transport {AbstractTransport} -- the transport you want to copy the object to
+        Returns:
+            str -- the string representation of the root object
+        """
+        pass
+
+    class Config:
+        extra = Extra.allow
+        arbitrary_types_allowed = True

speckle/transports/memory.py

@@ -0,0 +1,45 @@
+import json
+from typing import Any
+from speckle.logging.exceptions import SpeckleException
+from speckle.transports.abstract_transport import AbstractTransport
+
+
+class MemoryTransport(AbstractTransport):
+    _name: str = "Memory"
+    objects: dict = {}
+    saved_object_count: int = 0
+
+    def __init__(self, name=None, **data: Any) -> None:
+        super().__init__(**data)
+        if name:
+            self._name = name
+
+    def __repr__(self) -> str:
+        return f"MemoryTransport(objects: {len(self.objects)})"
+
+    def save_object(self, id: str, serialized_object: str) -> None:
+        self.objects[id] = serialized_object
+
+        self.saved_object_count += 1
+
+    def save_object_from_transport(
+        self, id: str, source_transport: AbstractTransport
+    ) -> None:
+        raise NotImplementedError
+
+    def get_object(self, id: str) -> str or None:
+        if id in self.objects:
+            return self.objects[id]
+        else:
+            return None
+
+    def begin_write(self) -> None:
+        self.saved_object_count = 0
+
+    def end_write(self) -> None:
+        pass
+
+    def copy_object_and_children(
+        self, id: str, target_transport: AbstractTransport
+    ) -> str:
+        raise NotImplementedError

speckle/transports/server.py

@@ -0,0 +1,103 @@
+import requests
+from asyncio import Queue, Task
+from typing import Any, Dict, List, Type
+
+from speckle.api.client import SpeckleClient
+from speckle.logging.exceptions import SpeckleException
+from speckle.transports.abstract_transport import AbstractTransport
+
+
+class ServerTransport(AbstractTransport):
+    _name = "RemoteTransport"
+    url: str = None
+    stream_id: str = None
+    saved_obj_count: int = 0
+    session: requests.Session = None
+    __queue: Queue = None
+    __workers: List[Task] = []
+
+    def __init__(self, client: SpeckleClient, stream_id: str, **data: Any) -> None:
+        super().__init__(**data)
+        # TODO: replace client with account or some other auth avenue
+        self.url = client.url
+        self.stream_id = stream_id
+        self.session = requests.Session()
+        self.session.headers.update(
+            {"Authorization": f"Bearer {client.me['token']}", "Accept": "text/plain"}
+        )
+
+    def begin_write(self) -> None:
+        self.saved_obj_count = 0
+
+    def end_write(self) -> None:
+        pass
+
+    # TODO: add save task to queue and process as the root is being deserialised
+    def save_object(self, id: str, serialized_object: str) -> None:
+        endpoint = f"{self.url}/objects/{self.stream_id}"
+        r = self.session.post(
+            url=endpoint,
+            files={"batch-1": ("batch-1", f"[{serialized_object}]")},
+        )
+        if r.status_code != 201:
+            raise SpeckleException(
+                message=f"Could not save the object to the server - status code {r.status_code}"
+            )
+
+    def save_object_from_transport(
+        self, id: str, source_transport: AbstractTransport
+    ) -> None:
+        obj_string = source_transport.get_object(id=id)
+        self.save_object(id=id, serialized_object=obj_string)
+
+    def get_object(self, id: str) -> str:
+        # endpoint = f"{self.url}/objects/{self.stream_id}/{id}/single"
+        # r = self.session.get(endpoint, stream=True)
+
+        # _, obj = next(r.iter_lines().decode("utf-8")).split("\t")
+
+        # return obj
+
+        raise SpeckleException(
+            "Getting a single object using `ServerTransport.get_object()` is not implemented. To get an object from the server, please use the `SpeckleClient.object.get()` route",
+            NotImplementedError,
+        )
+
+    def copy_object_and_children(
+        self, id: str, target_transport: AbstractTransport
+    ) -> str:
+        endpoint = f"{self.url}/objects/{self.stream_id}/{id}"
+        r = self.session.get(endpoint, stream=True)
+        if r.encoding is None:
+            r.encoding = "utf-8"
+        lines = r.iter_lines(decode_unicode=True)
+
+        # save first (root) obj for return
+        root_hash, root_obj = next(lines).split("\t")
+        target_transport.save_object(root_hash, root_obj)
+
+        # iter through returned objects saving them as we go
+        for line in lines:
+            if line:
+                hash, obj = line.split("\t")
+                target_transport.save_object(hash, obj)
+
+        return root_obj
+
+    # async def stream_res(self, endpoint: str) -> str:
+    #     data = b""
+    #     async with aiohttp.ClientSession() as session:
+    #         session.headers.update(
+    #             {
+    #                 "Authorization": f"{self.session.headers['Authorization']}",
+    #                 "Accept": "text/plain",
+    #             }
+    #         )
+    #         async with session.get(endpoint) as res:
+    #             while True:
+    #                 chunk = await res.content.read(self.chunk_size)
+    #                 if not chunk:
+    #                     break
+    #                 data += chunk
+
+    #     return data.decode("utf-8")

speckle/transports/sqlite.py

@@ -0,0 +1,199 @@
+import os
+import sys
+import time
+import sched
+import sqlite3
+from typing import Any
+from appdirs import user_data_dir
+from contextlib import closing
+from multiprocessing import Process, Queue
+from speckle.transports.abstract_transport import AbstractTransport
+from speckle.logging.exceptions import SpeckleException
+
+
+class SQLiteTransport(AbstractTransport):
+    _name = "SQLite"
+    _root_path: str = None
+    _is_writing: bool = False
+    _scheduler = sched.scheduler(time.time, time.sleep)
+    _polling_interval = 0.5  # seconds
+    __connection: sqlite3.Connection = None
+    __queue: Queue = Queue()
+    app_name: str = ""
+    scope: str = ""
+    saved_obj_count: int = 0
+
+    def __init__(
+        self,
+        base_path: str = None,
+        app_name: str = None,
+        scope: str = None,
+        **data: Any,
+    ) -> None:
+        super().__init__(**data)
+        self.app_name = app_name or "Speckle"
+        self.scope = scope or "Objects"
+        base_path = base_path or self.__get_base_path()
+
+        os.makedirs(base_path, exist_ok=True)
+
+        self._root_path = os.path.join(os.path.join(base_path, f"{self.scope}.db"))
+        self.__initialise()
+
+    def __repr__(self) -> str:
+        return f"SQLiteTransport(app: '{self.app_name}', scope: '{self.scope}')"
+
+    def __write_timer_elapsed(self):
+        print("WRITE TIMER ELAPSED")
+        proc = Process(target=_run_queue, args=(self.__queue, self._root_path))
+        proc.start()
+        proc.join()
+
+    def __get_base_path(self):
+        # from appdirs https://github.com/ActiveState/appdirs/blob/master/appdirs.py
+        # default mac path is not the one we use (we use unix path), so using special case for this
+        system = sys.platform
+        if system.startswith("java"):
+            import platform
+
+            os_name = platform.java_ver()[3][0]
+            if os_name.startswith("Mac"):
+                system = "darwin"
+
+        if system == "darwin":
+            path = os.path.expanduser("~/.config/")
+            return os.path.join(path, self.app_name)
+        else:
+            return user_data_dir(appname=self.app_name, appauthor=False, roaming=True)
+
+    def __consume_queue(self):
+        if self._is_writing or self.__queue.empty():
+            return
+        print("CONSUME QUEUE")
+        self._is_writing = True
+        while not self.__queue.empty():
+            data = self.__queue.get()
+            self.save_object_sync(data[0], data[1])
+        self._is_writing = False
+
+        self._scheduler.enter(
+            delay=self._polling_interval, priority=1, action=self.__consume_queue
+        )
+        self._scheduler.run(blocking=True)
+
+    def save_object(self, id: str, serialized_object: str) -> None:
+        """Adds an object to the queue and schedules it to be saved.
+
+        Arguments:
+            id {str} -- the object id
+            serialized_object {str} -- the full string representation of the object
+        """
+        print("SAVE OBJECT")
+        self.__queue.put((id, serialized_object))
+
+        self._scheduler.enter(
+            delay=self._polling_interval, priority=1, action=self.__consume_queue
+        )
+        self._scheduler.run(blocking=True)
+
+    def save_object_from_transport(
+        self, id: str, source_transport: AbstractTransport
+    ) -> None:
+        """Adds an object from the given transport to the queue and schedules it to be saved.
+
+        Arguments:
+            id {str} -- the object id
+            source_transport {AbstractTransport) -- the transport through which the object can be found
+        """
+        serialized_object = source_transport.get_object(id)
+        self.__queue.put((id, serialized_object))
+        raise NotImplementedError
+
+    def save_object_sync(self, id: str, serialized_object: str) -> None:
+        """Directly saves an object into the database.
+
+        Arguments:
+            id {str} -- the object id
+            serialized_object {str} -- the full string representation of the object
+        """
+        self.__check_connection()
+        try:
+            with closing(self.__connection.cursor()) as c:
+                c.execute(
+                    "INSERT OR IGNORE INTO objects(hash, content) VALUES(?,?)",
+                    (id, serialized_object),
+                )
+                self.__connection.commit()
+        except Exception as ex:
+            raise SpeckleException(
+                f"Could not save the object to the local db. Inner exception: {ex}", ex
+            )
+
+    def get_object(self, id: str) -> str or None:
+        self.__check_connection()
+        with closing(self.__connection.cursor()) as c:
+            row = c.execute(
+                "SELECT * FROM objects WHERE hash = ? LIMIT 1", (id,)
+            ).fetchone()
+        return row[1] if row else None
+
+    def begin_write(self):
+        self.saved_obj_count = 0
+
+    def end_write(self):
+        pass
+
+    def copy_object_and_children(
+        self, id: str, target_transport: AbstractTransport
+    ) -> str:
+        raise NotImplementedError
+
+    def get_all_objects(self):
+        """Returns all the objects in the store. NOTE: do not use for large collections!"""
+        self.__check_connection()
+        with closing(self.__connection.cursor()) as c:
+            rows = c.execute("SELECT * FROM objects").fetchall()
+        return rows
+
+    def close(self):
+        """Close the connection to the database"""
+        if self.__connection:
+            self.__connection.close()
+            self.__connection = None
+
+    def __initialise(self) -> None:
+        self.__connection = sqlite3.connect(self._root_path)
+        with closing(self.__connection.cursor()) as c:
+            c.execute(
+                """ CREATE TABLE IF NOT EXISTS objects(
+                      hash TEXT PRIMARY KEY,
+                      content TEXT
+                    ) WITHOUT ROWID;"""
+            )
+            c.execute("PRAGMA journal_mode='wal';")
+            c.execute("PRAGMA count_changes=OFF;")
+            c.execute("PRAGMA temp_store=MEMORY;")
+            self.__connection.commit()
+
+    def __check_connection(self):
+        if not self.__connection:
+            self.__connection = sqlite3.connect(self._root_path)
+
+    def __del__(self):
+        self.__connection.close()
+
+
+def _run_queue(queue: Queue, root_path: str):
+    if queue.empty():
+        return
+    print("RUN QUEUE")
+    conn = sqlite3.connect(root_path)
+    while not queue.empty():
+        data = queue.get()
+        with closing(conn.cursor()) as c:
+            c.execute(
+                "INSERT OR IGNORE INTO objects(hash, content) VALUES(?,?)",
+                (data[0], data[1]),
+            )
+            conn.commit()
+    conn.close()