huanld/specklepy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge commit 'c8210342c2a6f9097fc00bd8e04a7d198ab3e2ce' into osx

Browse Source
This commit is contained in:
Matteo Cominetti
2020-12-09 15:45:30 +00:00
parent 9320b566f3 c8210342c2
commit f58c13c0d1
8 changed files with 153 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+2 -2
View File
@@ -77,7 +77,7 @@ commit_id = client.commit.create("stream id", "object id", "this is a commit mes
deleted = client.commit.delete("stream id", "commit id")
```
The `BaseObjectSerializer` is used for decomposing and serializing `Base` objects so they can be sent / received to the server. You can use it directly to get the id (hash) and a serializable object representation of the decomposed `Base`.
The `BaseObjectSerializer` is used for decomposing and serializing `Base` objects so they can be sent / received to the server. You can use it directly to get the id (hash) and a serializable object representation of the decomposed `Base`. You can learn more about the Speckle `Base` object [here](https://discourse.speckle.works/t/core-2-0-the-base-object/782) and the decomposition API [here](https://discourse.speckle.works/t/core-2-0-decomposition-api/911).
```py
detached_base = Base()
@@ -91,7 +91,7 @@ serializer = BaseObjectSerializer()
hash, obj_dict = serializer.traverse_base(base_obj)
```
If you use the `operations`, you will not need to interact with the serializer directly as this will be taken care of for you. You will just need to provide a transport to indicate where the objects should be sent / received from. At the moment, just the `MemoryTransport` is fully functional.
If you use the `operations`, you will not need to interact with the serializer directly as this will be taken care of for you. You will just need to provide a transport to indicate where the objects should be sent / received from. At the moment, just the `MemoryTransport` and the `ServerTransport` are fully functional at the moment. If you'd like to learn more about Transports in Speckle 2.0, have a look [here](https://discourse.speckle.works/t/core-2-0-transports/919).
```py
transport = MemoryTransport()
speckle/api/client.py
+4
View File
@@ -1,3 +1,4 @@
import re
from gql.client import SyncClientSession
from speckle.logging.exceptions import SpeckleException
from typing import Dict
@@ -22,6 +23,9 @@ class SpeckleClient:
ws_protocol = "wss"
http_protocol = "https"
# sanitise host input by removing protocol and trailing slash
host = re.sub(r"((^\w+:|^)\/\/)|(\/$)", "", host)
self.url = f"{http_protocol}://{host}"
self.graphql = self.url + "/graphql"
self.ws_url = f"{ws_protocol}://{host}/graphql"
speckle/logging/exceptions.py
+5 -2
View File
@@ -6,7 +6,7 @@ class SpeckleException(Exception):
self.message = message
self.exception = exception
def __repr__(self) -> str:
def __str__(self) -> str:
return f"SpeckleException: {self.message}"
@@ -16,7 +16,7 @@ class SerializationException(SpeckleException):
self.object = object
self.unhandled_type = type(object)
def __repr__(self) -> str:
def __str__(self) -> str:
return f"SpeckleException: Could not serialize object of type {self.unhandled_type}"
@@ -25,3 +25,6 @@ class GraphQLException(SpeckleException):
super().__init__(message=message)
self.errors = errors
self.data = data
def __str__(self) -> str:
return f"GraphQLException: {self.message}"
speckle/serialization/base_object_serializer.py
+13 -1
View File
@@ -194,6 +194,13 @@ class BaseObjectSerializer:
Returns:
Base -- the base object with all its children attached
"""
# make sure an obj was passed and create dict if string was somehow passed
if not obj:
return
if isinstance(obj, str):
obj = json.loads(obj)
# initialise the base object
base = Base()
# get total children count
@@ -214,7 +221,12 @@ class BaseObjectSerializer:
# 2. handle referenced child objects
elif "referencedId" in value:
ref_hash = value["referencedId"]
ref_obj = json.loads(self.read_transport.get_object(id=ref_hash))
ref_obj_str = self.read_transport.get_object(id=ref_hash)
if not ref_obj_str:
raise SpeckleException(
f"Could not find the referenced child object of id `{ref_hash}` in the given read transport: {self.read_transport.name}"
)
ref_obj = json.loads(ref_obj_str)
base[prop] = self.recompose_base(obj=ref_obj)
# 3. handle all other cases (base objects, lists, and dicts)
speckle/transports/abstract_transport.py
+1 -1
View File
@@ -70,7 +70,7 @@ class AbstractTransport(Transport):
id {str} -- the hash of the object
Returns:
str -- the full string representation of the object (or null of no object is found)
str -- the full string representation of the object (or null if no object is found)
"""
pass
speckle/transports/memory.py
+2 -2
View File
@@ -1,4 +1,4 @@
import os
import json
from speckle.logging.exceptions import SpeckleException
from speckle.transports.abstract_transport import AbstractTransport
@@ -27,7 +27,7 @@ class MemoryTransport(AbstractTransport):
def get_object(self, id: str) -> str or None:
if id in self.objects:
return self.objects[id]
return json.dumps(self.objects[id])
else:
return None
speckle/transports/server.py
+102
View File
@@ -0,0 +1,102 @@
import requests
from asyncio import Queue, Task
from typing import Dict, List
from speckle.api.client import SpeckleClient
from speckle.logging.exceptions import SpeckleException
from speckle.transports.abstract_transport import AbstractTransport
class ServerTransport(AbstractTransport):
_name = "RemoteTransport"
url: str = None
stream_id: str = None
saved_obj_count: int = 0
session: requests.Session = None
__queue: Queue = None
__workers: List[Task] = []
def __init__(self, client: SpeckleClient, stream_id: str) -> None:
# TODO: replace client with account or some other auth avenue
self.url = client.url
self.stream_id = stream_id
self.session = requests.Session()
self.session.headers.update(
{"Authorization": f"Bearer {client.me['token']}", "Accept": "text/plain"}
)
def begin_write(self) -> None:
self.saved_obj_count = 0
def end_write(self) -> None:
pass
# TODO: add save task to queue and process as the root is being deserialised
def save_object(self, id: str, serialized_object: str) -> None:
endpoint = f"{self.url}/objects/{self.stream_id}"
r = self.session.post(
url=endpoint,
files={"batch-1": ("batch-1", f"[{serialized_object}]")},
)
if r.status_code != 201:
raise SpeckleException(
message=f"Could not save the object to the server - status code {r.status_code}"
)
def save_object_from_transport(
self, id: str, source_transport: AbstractTransport
) -> None:
obj_string = source_transport.get_object(id=id)
self.save_object(id=id, serialized_object=obj_string)
def get_object(self, id: str) -> str:
# endpoint = f"{self.url}/objects/{self.stream_id}/{id}/single"
# r = self.session.get(endpoint, stream=True)
# _, obj = next(r.iter_lines().decode("utf-8")).split("\t")
# return obj
raise SpeckleException(
"Getting a single object using `ServerTransport.get_object()` is not implemented. To get an object from the server, please use the `SpeckleClient.object.get()` route",
NotImplementedError,
)
def copy_object_and_children(
self, id: str, target_transport: AbstractTransport
) -> str:
endpoint = f"{self.url}/objects/{self.stream_id}/{id}"
r = self.session.get(endpoint, stream=True)
if r.encoding is None:
r.encoding = "utf-8"
lines = r.iter_lines(decode_unicode=True)
# save first (root) obj for return
root_hash, root_obj = next(lines).split("\t")
target_transport.save_object(root_hash, root_obj)
# iter through returned objects saving them as we go
for line in lines:
if line:
hash, obj = line.split("\t")
target_transport.save_object(hash, obj)
return root_obj
# async def stream_res(self, endpoint: str) -> str:
# data = b""
# async with aiohttp.ClientSession() as session:
# session.headers.update(
# {
# "Authorization": f"{self.session.headers['Authorization']}",
# "Accept": "text/plain",
# }
# )
# async with session.get(endpoint) as res:
# while True:
# chunk = await res.content.read(self.chunk_size)
# if not chunk:
# break
# data += chunk
# return data.decode("utf-8")
speckle/transports/sqlite.py
+24 -7
View File
@@ -1,7 +1,8 @@
import os
import sys
import time
import sqlite3
import sched
import sqlite3
from appdirs import user_data_dir
from contextlib import closing
from multiprocessing import Process, Queue
@@ -26,9 +27,7 @@ class SQLiteTransport(AbstractTransport):
) -> None:
self.app_name = app_name or "Speckle"
self.scope = scope or "Objects"
base_path = base_path or user_data_dir(
appname=self.app_name, appauthor=False, roaming=True
)
base_path = base_path or self.__get_base_path()
os.makedirs(base_path, exist_ok=True)
@@ -44,6 +43,23 @@ class SQLiteTransport(AbstractTransport):
proc.start()
proc.join()
def __get_base_path(self):
# from appdirs https://github.com/ActiveState/appdirs/blob/master/appdirs.py
# default mac path is not the one we use (we use unix path), so using special case for this
system = sys.platform
if system.startswith("java"):
import platform
os_name = platform.java_ver()[3][0]
if os_name.startswith("Mac"):
system = "darwin"
if system == "darwin":
path = os.path.expanduser("~/.config/")
return os.path.join(path, self.app_name)
else:
return user_data_dir(appname=self.app_name, appauthor=False, roaming=True)
def __consume_queue(self):
if self._is_writing or self.__queue.empty():
return
@@ -102,9 +118,10 @@ class SQLiteTransport(AbstractTransport):
(id, serialized_object),
)
self.__connection.commit()
except Exception as e:
print(e)
raise e
except Exception as ex:
raise SpeckleException(
f"Could not save the object to the local db. Inner exception: {ex}", ex
)
def get_object(self, id: str) -> str or None:
self.__check_connection()