From cff738aa31174b31da46500ad92b6c65e6e0fc88 Mon Sep 17 00:00:00 2001 From: izzy lyseggen Date: Tue, 1 Dec 2020 19:46:45 +0000 Subject: [PATCH 1/8] feat(transports): wip server transport --- speckle/transports/server.py | 57 ++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 speckle/transports/server.py diff --git a/speckle/transports/server.py b/speckle/transports/server.py new file mode 100644 index 0000000..82d3e96 --- /dev/null +++ b/speckle/transports/server.py @@ -0,0 +1,57 @@ +import json +from speckle.logging.exceptions import SpeckleException +import requests +from asyncio import Queue, Task +from speckle.api.client import SpeckleClient +from typing import Dict, List +from speckle.transports.abstract_transport import AbstractTransport + + +class ServerTransport(AbstractTransport): + _name = "RemoteTransport" + url: str = None + stream_id: str = None + saved_obj_count: int = 0 + session: requests.Session = None + __queue: Queue = None + __workers: List[Task] = [] + + def __init__(self, client: SpeckleClient, stream_id: str) -> None: + # TODO: replace client with account or some other auth avenue + self.url = client.url + self.stream_id = stream_id + self.session = requests.Session() + self.session.headers.update({"Authorization": f"Bearer {client.me['token']}"}) + + def begin_write(self) -> None: + self.saved_obj_count = 0 + + def end_write(self) -> None: + pass + + # TODO: add save task to queue and process as the root is being deserialised + def save_object(self, id: str, serialized_object: str) -> None: + endpoint = f"{self.url}/objects/{self.stream_id}" + r = self.session.post( + url=endpoint, + files={"batch-1": ("batch-1", f"[{serialized_object}]")}, + ) + if r.status_code != 201: + raise SpeckleException( + message=f"Could not save the object to the server - status code {r.status_code}" + ) + + def save_object_from_transport( + self, id: str, source_transport: AbstractTransport + ) -> None: + pass + + def get_object(self, id: str) -> str: + endpoint = f"{self.url}/objects/{self.stream_id}/{id}/single" + r = self.session.get(url=endpoint) + print(r.text) + + def copy_object_and_children( + self, id: str, target_transport: AbstractTransport + ) -> str: + raise NotImplementedError From 01091405d14ae2acea14d65e613ca0b17f67f2b8 Mon Sep 17 00:00:00 2001 From: izzy lyseggen Date: Tue, 8 Dec 2020 18:55:32 +0000 Subject: [PATCH 2/8] fix(serialiser): extra checks for errors --- speckle/serialization/base_object_serializer.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/speckle/serialization/base_object_serializer.py b/speckle/serialization/base_object_serializer.py index a42aa21..3f9d5a1 100644 --- a/speckle/serialization/base_object_serializer.py +++ b/speckle/serialization/base_object_serializer.py @@ -194,6 +194,13 @@ class BaseObjectSerializer: Returns: Base -- the base object with all its children attached """ + # make sure an obj was passed and create dict if string was somehow passed + if not obj: + return + if isinstance(obj, str): + obj = json.loads(obj) + + # initialise the base object base = Base() # get total children count @@ -214,7 +221,12 @@ class BaseObjectSerializer: # 2. handle referenced child objects elif "referencedId" in value: ref_hash = value["referencedId"] - ref_obj = json.loads(self.read_transport.get_object(id=ref_hash)) + ref_obj_str = self.read_transport.get_object(id=ref_hash) + if not ref_obj_str: + raise SpeckleException( + f"Could not find the referenced child object of id `{ref_hash}` in the given read transport: {self.read_transport.name}" + ) + ref_obj = json.loads(ref_obj_str) base[prop] = self.recompose_base(obj=ref_obj) # 3. handle all other cases (base objects, lists, and dicts) From 90cae9a3c5ce0e8431e8347d62ef94adab9d14d4 Mon Sep 17 00:00:00 2001 From: izzy lyseggen Date: Tue, 8 Dec 2020 18:56:24 +0000 Subject: [PATCH 3/8] fix(memory): return string obj from `get` --- speckle/transports/memory.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/speckle/transports/memory.py b/speckle/transports/memory.py index 2139445..85b5211 100644 --- a/speckle/transports/memory.py +++ b/speckle/transports/memory.py @@ -1,4 +1,4 @@ -import os +import json from speckle.logging.exceptions import SpeckleException from speckle.transports.abstract_transport import AbstractTransport @@ -27,7 +27,7 @@ class MemoryTransport(AbstractTransport): def get_object(self, id: str) -> str or None: if id in self.objects: - return self.objects[id] + return json.dumps(self.objects[id]) else: return None From 7310fbfdd4104a0f955e02eb505e439b7978c90c Mon Sep 17 00:00:00 2001 From: izzy lyseggen Date: Tue, 8 Dec 2020 18:58:03 +0000 Subject: [PATCH 4/8] fix(abstract transport): typo --- speckle/transports/abstract_transport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/speckle/transports/abstract_transport.py b/speckle/transports/abstract_transport.py index a88b9e3..317f994 100644 --- a/speckle/transports/abstract_transport.py +++ b/speckle/transports/abstract_transport.py @@ -70,7 +70,7 @@ class AbstractTransport(Transport): id {str} -- the hash of the object Returns: - str -- the full string representation of the object (or null of no object is found) + str -- the full string representation of the object (or null if no object is found) """ pass From fd0d04b70e0320c99ce42c7d5286e2497884a5c4 Mon Sep 17 00:00:00 2001 From: izzy lyseggen Date: Tue, 8 Dec 2020 18:59:51 +0000 Subject: [PATCH 5/8] feat(server): successfully receive objects! --- speckle/transports/server.py | 50 ++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/speckle/transports/server.py b/speckle/transports/server.py index 82d3e96..297e483 100644 --- a/speckle/transports/server.py +++ b/speckle/transports/server.py @@ -1,9 +1,9 @@ -import json -from speckle.logging.exceptions import SpeckleException import requests from asyncio import Queue, Task -from speckle.api.client import SpeckleClient from typing import Dict, List + +from speckle.api.client import SpeckleClient +from speckle.logging.exceptions import SpeckleException from speckle.transports.abstract_transport import AbstractTransport @@ -21,7 +21,9 @@ class ServerTransport(AbstractTransport): self.url = client.url self.stream_id = stream_id self.session = requests.Session() - self.session.headers.update({"Authorization": f"Bearer {client.me['token']}"}) + self.session.headers.update( + {"Authorization": f"Bearer {client.me['token']}", "Accept": "text/plain"} + ) def begin_write(self) -> None: self.saved_obj_count = 0 @@ -47,11 +49,43 @@ class ServerTransport(AbstractTransport): pass def get_object(self, id: str) -> str: - endpoint = f"{self.url}/objects/{self.stream_id}/{id}/single" - r = self.session.get(url=endpoint) - print(r.text) + raise NotImplementedError def copy_object_and_children( self, id: str, target_transport: AbstractTransport ) -> str: - raise NotImplementedError + endpoint = f"{self.url}/objects/{self.stream_id}/{id}" + r = self.session.get(endpoint, stream=True) + if r.encoding is None: + r.encoding = "utf-8" + lines = r.iter_lines(decode_unicode=True) + + # save first (root) obj for return + root_hash, root_obj = next(lines).split("\t") + target_transport.save_object(root_hash, root_obj) + + # iter through returned objects saving them as we go + for line in lines: + if line: + hash, obj = line.split("\t") + target_transport.save_object(hash, obj) + + return root_obj + + # async def stream_res(self, endpoint: str) -> str: + # data = b"" + # async with aiohttp.ClientSession() as session: + # session.headers.update( + # { + # "Authorization": f"{self.session.headers['Authorization']}", + # "Accept": "text/plain", + # } + # ) + # async with session.get(endpoint) as res: + # while True: + # chunk = await res.content.read(self.chunk_size) + # if not chunk: + # break + # data += chunk + + # return data.decode("utf-8") \ No newline at end of file From 9657bd370ce1b287cefae3f5571b9fd36ea5315e Mon Sep 17 00:00:00 2001 From: izzy lyseggen Date: Tue, 8 Dec 2020 20:11:08 +0000 Subject: [PATCH 6/8] feat(server): save obj from transport --- speckle/transports/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/speckle/transports/server.py b/speckle/transports/server.py index 297e483..460e67c 100644 --- a/speckle/transports/server.py +++ b/speckle/transports/server.py @@ -46,7 +46,8 @@ class ServerTransport(AbstractTransport): def save_object_from_transport( self, id: str, source_transport: AbstractTransport ) -> None: - pass + obj_string = source_transport.get_object(id=id) + self.save_object(id=id, serialized_object=obj_string) def get_object(self, id: str) -> str: raise NotImplementedError From 3c8aff348761e83cf72cd185c990d4dc90567e12 Mon Sep 17 00:00:00 2001 From: izzy lyseggen Date: Wed, 9 Dec 2020 09:02:55 +0000 Subject: [PATCH 7/8] feat(server): wip get obj --- speckle/transports/server.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/speckle/transports/server.py b/speckle/transports/server.py index 460e67c..45c720b 100644 --- a/speckle/transports/server.py +++ b/speckle/transports/server.py @@ -50,7 +50,12 @@ class ServerTransport(AbstractTransport): self.save_object(id=id, serialized_object=obj_string) def get_object(self, id: str) -> str: - raise NotImplementedError + endpoint = f"{self.url}/objects/{self.stream_id}/{id}/single" + r = self.session.get(endpoint, stream=True) + + _, obj = next(r.iter_lines().decode("utf-8")).split("\t") + + return obj def copy_object_and_children( self, id: str, target_transport: AbstractTransport From 2e428f9b3ce47facbd1fe764f0668d26eab63eb2 Mon Sep 17 00:00:00 2001 From: izzy lyseggen Date: Wed, 9 Dec 2020 12:20:19 +0000 Subject: [PATCH 8/8] feat(server): raise exception for `get_object()` this is not implemented. direct user to use the client --- speckle/transports/server.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/speckle/transports/server.py b/speckle/transports/server.py index 45c720b..1af7b02 100644 --- a/speckle/transports/server.py +++ b/speckle/transports/server.py @@ -50,12 +50,17 @@ class ServerTransport(AbstractTransport): self.save_object(id=id, serialized_object=obj_string) def get_object(self, id: str) -> str: - endpoint = f"{self.url}/objects/{self.stream_id}/{id}/single" - r = self.session.get(endpoint, stream=True) + # endpoint = f"{self.url}/objects/{self.stream_id}/{id}/single" + # r = self.session.get(endpoint, stream=True) - _, obj = next(r.iter_lines().decode("utf-8")).split("\t") + # _, obj = next(r.iter_lines().decode("utf-8")).split("\t") - return obj + # return obj + + raise SpeckleException( + "Getting a single object using `ServerTransport.get_object()` is not implemented. To get an object from the server, please use the `SpeckleClient.object.get()` route", + NotImplementedError, + ) def copy_object_and_children( self, id: str, target_transport: AbstractTransport