diff --git a/README.md b/README.md index 6d342d6..696880c 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ commit_id = client.commit.create("stream id", "object id", "this is a commit mes deleted = client.commit.delete("stream id", "commit id") ``` -The `BaseObjectSerializer` is used for decomposing and serializing `Base` objects so they can be sent / received to the server. You can use it directly to get the id (hash) and a serializable object representation of the decomposed `Base`. +The `BaseObjectSerializer` is used for decomposing and serializing `Base` objects so they can be sent / received to the server. You can use it directly to get the id (hash) and a serializable object representation of the decomposed `Base`. You can learn more about the Speckle `Base` object [here](https://discourse.speckle.works/t/core-2-0-the-base-object/782) and the decomposition API [here](https://discourse.speckle.works/t/core-2-0-decomposition-api/911). ```py detached_base = Base() @@ -91,7 +91,7 @@ serializer = BaseObjectSerializer() hash, obj_dict = serializer.traverse_base(base_obj) ``` -If you use the `operations`, you will not need to interact with the serializer directly as this will be taken care of for you. You will just need to provide a transport to indicate where the objects should be sent / received from. At the moment, just the `MemoryTransport` is fully functional. +If you use the `operations`, you will not need to interact with the serializer directly as this will be taken care of for you. You will just need to provide a transport to indicate where the objects should be sent / received from. At the moment, just the `MemoryTransport` and the `ServerTransport` are fully functional at the moment. If you'd like to learn more about Transports in Speckle 2.0, have a look [here](https://discourse.speckle.works/t/core-2-0-transports/919). ```py transport = MemoryTransport() diff --git a/speckle/api/client.py b/speckle/api/client.py index 7a34860..fc79265 100644 --- a/speckle/api/client.py +++ b/speckle/api/client.py @@ -1,3 +1,4 @@ +import re from gql.client import SyncClientSession from speckle.logging.exceptions import SpeckleException from typing import Dict @@ -22,6 +23,9 @@ class SpeckleClient: ws_protocol = "wss" http_protocol = "https" + # sanitise host input by removing protocol and trailing slash + host = re.sub(r"((^\w+:|^)\/\/)|(\/$)", "", host) + self.url = f"{http_protocol}://{host}" self.graphql = self.url + "/graphql" self.ws_url = f"{ws_protocol}://{host}/graphql" diff --git a/speckle/logging/exceptions.py b/speckle/logging/exceptions.py index e17c9b4..b4d1d60 100644 --- a/speckle/logging/exceptions.py +++ b/speckle/logging/exceptions.py @@ -6,7 +6,7 @@ class SpeckleException(Exception): self.message = message self.exception = exception - def __repr__(self) -> str: + def __str__(self) -> str: return f"SpeckleException: {self.message}" @@ -16,7 +16,7 @@ class SerializationException(SpeckleException): self.object = object self.unhandled_type = type(object) - def __repr__(self) -> str: + def __str__(self) -> str: return f"SpeckleException: Could not serialize object of type {self.unhandled_type}" @@ -25,3 +25,6 @@ class GraphQLException(SpeckleException): super().__init__(message=message) self.errors = errors self.data = data + + def __str__(self) -> str: + return f"GraphQLException: {self.message}" diff --git a/speckle/serialization/base_object_serializer.py b/speckle/serialization/base_object_serializer.py index a42aa21..3f9d5a1 100644 --- a/speckle/serialization/base_object_serializer.py +++ b/speckle/serialization/base_object_serializer.py @@ -194,6 +194,13 @@ class BaseObjectSerializer: Returns: Base -- the base object with all its children attached """ + # make sure an obj was passed and create dict if string was somehow passed + if not obj: + return + if isinstance(obj, str): + obj = json.loads(obj) + + # initialise the base object base = Base() # get total children count @@ -214,7 +221,12 @@ class BaseObjectSerializer: # 2. handle referenced child objects elif "referencedId" in value: ref_hash = value["referencedId"] - ref_obj = json.loads(self.read_transport.get_object(id=ref_hash)) + ref_obj_str = self.read_transport.get_object(id=ref_hash) + if not ref_obj_str: + raise SpeckleException( + f"Could not find the referenced child object of id `{ref_hash}` in the given read transport: {self.read_transport.name}" + ) + ref_obj = json.loads(ref_obj_str) base[prop] = self.recompose_base(obj=ref_obj) # 3. handle all other cases (base objects, lists, and dicts) diff --git a/speckle/transports/abstract_transport.py b/speckle/transports/abstract_transport.py index a88b9e3..317f994 100644 --- a/speckle/transports/abstract_transport.py +++ b/speckle/transports/abstract_transport.py @@ -70,7 +70,7 @@ class AbstractTransport(Transport): id {str} -- the hash of the object Returns: - str -- the full string representation of the object (or null of no object is found) + str -- the full string representation of the object (or null if no object is found) """ pass diff --git a/speckle/transports/memory.py b/speckle/transports/memory.py index 2139445..85b5211 100644 --- a/speckle/transports/memory.py +++ b/speckle/transports/memory.py @@ -1,4 +1,4 @@ -import os +import json from speckle.logging.exceptions import SpeckleException from speckle.transports.abstract_transport import AbstractTransport @@ -27,7 +27,7 @@ class MemoryTransport(AbstractTransport): def get_object(self, id: str) -> str or None: if id in self.objects: - return self.objects[id] + return json.dumps(self.objects[id]) else: return None diff --git a/speckle/transports/server.py b/speckle/transports/server.py new file mode 100644 index 0000000..1af7b02 --- /dev/null +++ b/speckle/transports/server.py @@ -0,0 +1,102 @@ +import requests +from asyncio import Queue, Task +from typing import Dict, List + +from speckle.api.client import SpeckleClient +from speckle.logging.exceptions import SpeckleException +from speckle.transports.abstract_transport import AbstractTransport + + +class ServerTransport(AbstractTransport): + _name = "RemoteTransport" + url: str = None + stream_id: str = None + saved_obj_count: int = 0 + session: requests.Session = None + __queue: Queue = None + __workers: List[Task] = [] + + def __init__(self, client: SpeckleClient, stream_id: str) -> None: + # TODO: replace client with account or some other auth avenue + self.url = client.url + self.stream_id = stream_id + self.session = requests.Session() + self.session.headers.update( + {"Authorization": f"Bearer {client.me['token']}", "Accept": "text/plain"} + ) + + def begin_write(self) -> None: + self.saved_obj_count = 0 + + def end_write(self) -> None: + pass + + # TODO: add save task to queue and process as the root is being deserialised + def save_object(self, id: str, serialized_object: str) -> None: + endpoint = f"{self.url}/objects/{self.stream_id}" + r = self.session.post( + url=endpoint, + files={"batch-1": ("batch-1", f"[{serialized_object}]")}, + ) + if r.status_code != 201: + raise SpeckleException( + message=f"Could not save the object to the server - status code {r.status_code}" + ) + + def save_object_from_transport( + self, id: str, source_transport: AbstractTransport + ) -> None: + obj_string = source_transport.get_object(id=id) + self.save_object(id=id, serialized_object=obj_string) + + def get_object(self, id: str) -> str: + # endpoint = f"{self.url}/objects/{self.stream_id}/{id}/single" + # r = self.session.get(endpoint, stream=True) + + # _, obj = next(r.iter_lines().decode("utf-8")).split("\t") + + # return obj + + raise SpeckleException( + "Getting a single object using `ServerTransport.get_object()` is not implemented. To get an object from the server, please use the `SpeckleClient.object.get()` route", + NotImplementedError, + ) + + def copy_object_and_children( + self, id: str, target_transport: AbstractTransport + ) -> str: + endpoint = f"{self.url}/objects/{self.stream_id}/{id}" + r = self.session.get(endpoint, stream=True) + if r.encoding is None: + r.encoding = "utf-8" + lines = r.iter_lines(decode_unicode=True) + + # save first (root) obj for return + root_hash, root_obj = next(lines).split("\t") + target_transport.save_object(root_hash, root_obj) + + # iter through returned objects saving them as we go + for line in lines: + if line: + hash, obj = line.split("\t") + target_transport.save_object(hash, obj) + + return root_obj + + # async def stream_res(self, endpoint: str) -> str: + # data = b"" + # async with aiohttp.ClientSession() as session: + # session.headers.update( + # { + # "Authorization": f"{self.session.headers['Authorization']}", + # "Accept": "text/plain", + # } + # ) + # async with session.get(endpoint) as res: + # while True: + # chunk = await res.content.read(self.chunk_size) + # if not chunk: + # break + # data += chunk + + # return data.decode("utf-8") \ No newline at end of file diff --git a/speckle/transports/sqlite.py b/speckle/transports/sqlite.py index 885a98a..461e9a6 100644 --- a/speckle/transports/sqlite.py +++ b/speckle/transports/sqlite.py @@ -1,7 +1,8 @@ import os +import sys import time -import sqlite3 import sched +import sqlite3 from appdirs import user_data_dir from contextlib import closing from multiprocessing import Process, Queue @@ -26,9 +27,7 @@ class SQLiteTransport(AbstractTransport): ) -> None: self.app_name = app_name or "Speckle" self.scope = scope or "Objects" - base_path = base_path or user_data_dir( - appname=self.app_name, appauthor=False, roaming=True - ) + base_path = base_path or self.__get_base_path() os.makedirs(base_path, exist_ok=True) @@ -44,6 +43,23 @@ class SQLiteTransport(AbstractTransport): proc.start() proc.join() + def __get_base_path(self): + # from appdirs https://github.com/ActiveState/appdirs/blob/master/appdirs.py + # default mac path is not the one we use (we use unix path), so using special case for this + system = sys.platform + if system.startswith("java"): + import platform + + os_name = platform.java_ver()[3][0] + if os_name.startswith("Mac"): + system = "darwin" + + if system == "darwin": + path = os.path.expanduser("~/.config/") + return os.path.join(path, self.app_name) + else: + return user_data_dir(appname=self.app_name, appauthor=False, roaming=True) + def __consume_queue(self): if self._is_writing or self.__queue.empty(): return @@ -102,9 +118,10 @@ class SQLiteTransport(AbstractTransport): (id, serialized_object), ) self.__connection.commit() - except Exception as e: - print(e) - raise e + except Exception as ex: + raise SpeckleException( + f"Could not save the object to the local db. Inner exception: {ex}", ex + ) def get_object(self, id: str) -> str or None: self.__check_connection()