From a3830a95fd89cbf70e8cba7a55e1ecb30393ae5b Mon Sep 17 00:00:00 2001 From: izzy lyseggen Date: Tue, 22 Dec 2020 09:39:19 +0000 Subject: [PATCH 01/13] feat(Base): self populate speckleType --- speckle/objects/base.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/speckle/objects/base.py b/speckle/objects/base.py index 7ee3d27..33e9484 100644 --- a/speckle/objects/base.py +++ b/speckle/objects/base.py @@ -17,10 +17,14 @@ class Base(BaseModel): def __init__(self, **kwargs) -> None: super().__init__() + self.speckle_type = self.__class__.__name__ self.__dict__.update(kwargs) + def __repr__(self) -> str: + return f"{self.speckle_type}(id: {self.id}, speckle_type: {self.speckle_type}, totalChildrenCount: {self.totalChildrenCount})" + def __str__(self) -> str: - return f"Base(id: {self.id}, speckle_type: {self.speckle_type}, totalChildrenCount: {self.totalChildrenCount})" + return self.__repr__() def __setitem__(self, name: str, value: Any) -> None: self.__dict__[name] = value From a4f7ce326e02de6d48985b316a81f2c6a05cd5da Mon Sep 17 00:00:00 2001 From: izzy lyseggen Date: Tue, 22 Dec 2020 11:13:46 +0000 Subject: [PATCH 02/13] feat(Base): add `chunks` prop --- speckle/objects/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/speckle/objects/base.py b/speckle/objects/base.py index 33e9484..3e78b93 100644 --- a/speckle/objects/base.py +++ b/speckle/objects/base.py @@ -14,6 +14,7 @@ class Base(BaseModel): totalChildrenCount: Optional[int] = None applicationId: Optional[str] = None speckle_type: Optional[str] = "Base" + chunks: Dict[str, int] = {} def __init__(self, **kwargs) -> None: super().__init__() From 7fbfdb4b92f2576b7468763cc52594707d1932d4 Mon Sep 17 00:00:00 2001 From: izzy lyseggen Date: Tue, 22 Dec 2020 11:36:11 +0000 Subject: [PATCH 03/13] fix(base): always use class name for string repr --- speckle/objects/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/speckle/objects/base.py b/speckle/objects/base.py index 3e78b93..be20ade 100644 --- a/speckle/objects/base.py +++ b/speckle/objects/base.py @@ -22,7 +22,7 @@ class Base(BaseModel): self.__dict__.update(kwargs) def __repr__(self) -> str: - return f"{self.speckle_type}(id: {self.id}, speckle_type: {self.speckle_type}, totalChildrenCount: {self.totalChildrenCount})" + return f"{self.__class__.__name__}(id: {self.id}, speckle_type: {self.speckle_type}, totalChildrenCount: {self.totalChildrenCount})" def __str__(self) -> str: return self.__repr__() From 77dcf53c4bca6f33d499a8e09cc6305373046cbd Mon Sep 17 00:00:00 2001 From: izzy lyseggen Date: Tue, 22 Dec 2020 11:38:28 +0000 Subject: [PATCH 04/13] feat(chunking): add `DataChunk` class --- speckle/objects/base.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/speckle/objects/base.py b/speckle/objects/base.py index be20ade..b527a87 100644 --- a/speckle/objects/base.py +++ b/speckle/objects/base.py @@ -119,4 +119,8 @@ class Base(BaseModel): return count class Config: - extra = Extra.allow \ No newline at end of file + extra = Extra.allow + + +class DataChunk(Base): + data: List[Any] = [] From 969b6a92e5cd01198c1ae8a36ee1bdcbeb51182f Mon Sep 17 00:00:00 2001 From: izzy lyseggen Date: Wed, 23 Dec 2020 10:08:07 +0000 Subject: [PATCH 05/13] fix(memory): remove redundant serialisation --- speckle/transports/memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/speckle/transports/memory.py b/speckle/transports/memory.py index 58e580b..06b8f38 100644 --- a/speckle/transports/memory.py +++ b/speckle/transports/memory.py @@ -29,7 +29,7 @@ class MemoryTransport(AbstractTransport): def get_object(self, id: str) -> str or None: if id in self.objects: - return json.dumps(self.objects[id]) + return self.objects[id] else: return None From 21abd5181ac3b7268d00c363afd4aee927279034 Mon Sep 17 00:00:00 2001 From: izzy lyseggen Date: Wed, 23 Dec 2020 13:01:04 +0000 Subject: [PATCH 06/13] fix(base): make chunkable prop protected --- speckle/objects/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/speckle/objects/base.py b/speckle/objects/base.py index b527a87..d6f94ab 100644 --- a/speckle/objects/base.py +++ b/speckle/objects/base.py @@ -14,7 +14,7 @@ class Base(BaseModel): totalChildrenCount: Optional[int] = None applicationId: Optional[str] = None speckle_type: Optional[str] = "Base" - chunks: Dict[str, int] = {} + _chunkable: Dict[str, int] = {} # dict of chunkable props and their max chunk size def __init__(self, **kwargs) -> None: super().__init__() From 2dba909eba76d06a8f7c40af26fd8581fd02349b Mon Sep 17 00:00:00 2001 From: izzy lyseggen Date: Wed, 23 Dec 2020 13:01:23 +0000 Subject: [PATCH 07/13] feat(objects): import all obj classes --- speckle/objects/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 speckle/objects/__init__.py diff --git a/speckle/objects/__init__.py b/speckle/objects/__init__.py new file mode 100644 index 0000000..b5fc3f9 --- /dev/null +++ b/speckle/objects/__init__.py @@ -0,0 +1,14 @@ +from pathlib import Path +import sys +import inspect +import pkgutil +from importlib import import_module +from .base import Base + + +for (_, name, _) in pkgutil.iter_modules([Path(__file__).parent]): + imported_module = import_module("." + name, package=__name__) + classes = inspect.getmembers(imported_module, inspect.isclass) + for c in classes: + if issubclass(c[1], Base): + setattr(sys.modules[__name__], c[0], c[1]) From b36e7e000fb9370c4dbf0b332ea72524eab9b383 Mon Sep 17 00:00:00 2001 From: izzy lyseggen Date: Wed, 23 Dec 2020 13:20:39 +0000 Subject: [PATCH 08/13] feat(chunking): initial chunking implementation --- .../serialization/base_object_serializer.py | 65 ++++++++++++++++--- 1 file changed, 57 insertions(+), 8 deletions(-) diff --git a/speckle/serialization/base_object_serializer.py b/speckle/serialization/base_object_serializer.py index d7b0676..7b8ac7d 100644 --- a/speckle/serialization/base_object_serializer.py +++ b/speckle/serialization/base_object_serializer.py @@ -1,9 +1,10 @@ import json import hashlib +from speckle import objects from uuid import uuid4 from typing import Any, Dict, List, Tuple -from speckle.objects.base import Base +from speckle.objects.base import Base, DataChunk from speckle.logging.exceptions import SerializationException, SpeckleException from speckle.transports.abstract_transport import AbstractTransport @@ -54,11 +55,12 @@ class BaseObjectSerializer: prop = props.pop(0) value = obj[prop] - # skip nulls or props marked to be ignored with "__" - if not value or prop.startswith("__"): + # skip nulls or props marked to be ignored with "__" or "_" + if not value or prop.startswith(("__", "_")): continue - detach = True if prop.startswith("@") else False + chunkable = True if prop in base._chunkable else False + detach = True if prop.startswith("@") or chunkable else False # 1. handle primitives (ints, floats, strings, and bools) if isinstance(value, PRIMITIVES): @@ -74,7 +76,28 @@ class BaseObjectSerializer: else: object_builder[prop] = child_obj - # 3. handle all other cases + # 3. handle chunkable props + elif chunkable: + self.detach_lineage.append(detach) + chunks = [] + max_size = base._chunkable[prop] + chunk = DataChunk() + for count, item in enumerate(value): + if count and count % max_size == 0: + chunks.append(chunk) + chunk = DataChunk() + chunk.data.append(item) + chunks.append(chunk) + + chunk_refs = [] + for c in chunks: + self.detach_lineage.append(detach) + ref_hash, _ = self.traverse_base(c) + ref_obj = self.detach_helper(ref_hash=ref_hash) + chunk_refs.append(ref_obj) + object_builder[prop] = chunk_refs + + # 4. handle all other cases else: child_obj = self.traverse_value(value) object_builder[prop] = child_obj @@ -199,9 +222,15 @@ class BaseObjectSerializer: return if isinstance(obj, str): obj = json.loads(obj) + if obj["speckle_type"] == "reference": + obj = self.get_child(obj=obj) - # initialise the base object - base = Base() + # if it's a chunk, we know the shape of it already so we can just create the chunk and return it + if obj["speckle_type"] == "DataChunk": + return DataChunk(id=obj["id"], data=obj["data"]) + + # initialise the base object using `speckle_type` + base = getattr(objects, obj["speckle_type"], Base)() # get total children count if "__closure" in obj: @@ -247,12 +276,23 @@ class BaseObjectSerializer: if isinstance(obj, PRIMITIVES): return obj + # lists (regular and chunked) if isinstance(obj, list): - return [self.handle_value(o) for o in obj] + obj_list = [self.handle_value(o) for o in obj] + # handle chunked lists + if isinstance(obj_list[0], DataChunk): + data = [] + for o in obj_list: + data.extend(o["data"]) + return data + else: + return obj_list + # bases if isinstance(obj, dict) and "speckle_type" in obj: return self.recompose_base(obj=obj) + # dictionaries if isinstance(obj, dict): for k, v in obj.items(): if isinstance(v, PRIMITIVES): @@ -260,3 +300,12 @@ class BaseObjectSerializer: else: obj[k] = self.handle_value(v) return obj + + def get_child(self, obj: Dict): + ref_hash = obj["referencedId"] + ref_obj_str = self.read_transport.get_object(id=ref_hash) + if not ref_obj_str: + raise SpeckleException( + f"Could not find the referenced child object of id `{ref_hash}` in the given read transport: {self.read_transport.name}" + ) + return json.loads(ref_obj_str) From e6727a9552bf15f9fc30a03fb68f3efde5968826 Mon Sep 17 00:00:00 2001 From: izzy lyseggen Date: Wed, 23 Dec 2020 16:16:20 +0000 Subject: [PATCH 09/13] fix(chunking): delay chunk check to `handle_value` --- speckle/serialization/base_object_serializer.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/speckle/serialization/base_object_serializer.py b/speckle/serialization/base_object_serializer.py index 7b8ac7d..ae01fa7 100644 --- a/speckle/serialization/base_object_serializer.py +++ b/speckle/serialization/base_object_serializer.py @@ -225,10 +225,6 @@ class BaseObjectSerializer: if obj["speckle_type"] == "reference": obj = self.get_child(obj=obj) - # if it's a chunk, we know the shape of it already so we can just create the chunk and return it - if obj["speckle_type"] == "DataChunk": - return DataChunk(id=obj["id"], data=obj["data"]) - # initialise the base object using `speckle_type` base = getattr(objects, obj["speckle_type"], Base)() From 6eb73555ed5511e3a3f87e1550e3890c241283df Mon Sep 17 00:00:00 2001 From: izzy lyseggen Date: Wed, 23 Dec 2020 16:44:06 +0000 Subject: [PATCH 10/13] feat(objects): mesh obj for chunk testing --- speckle/objects/mesh.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 speckle/objects/mesh.py diff --git a/speckle/objects/mesh.py b/speckle/objects/mesh.py new file mode 100644 index 0000000..8ab8895 --- /dev/null +++ b/speckle/objects/mesh.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import List, Optional + +from pydantic import BaseModel +from .base import Base + +CHUNKABLE_PROPS = { + "vertices": 2000, + "faces": 2000, + "colors": 2000, + "textureCoordinates": 2000, +} + + +class Mesh(Base): + vertices: List[float] = None + faces: List[int] = None + colors: List[int] = None + textureCoordinates: List[float] = None + id: Optional[str] = None + totalChildrenCount: Optional[int] = None + applicationId: Optional[str] = None + speckle_type: Optional[str] = None + + def __init__(self, **kwargs) -> None: + super().__init__(**kwargs) + self._chunkable.update(CHUNKABLE_PROPS) From 28e68e090cd1c55b6ba597ed6163665b8ecf8a94 Mon Sep 17 00:00:00 2001 From: izzy lyseggen Date: Wed, 23 Dec 2020 16:51:39 +0000 Subject: [PATCH 11/13] fix(objects): remove duplicate prop defs --- speckle/objects/mesh.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/speckle/objects/mesh.py b/speckle/objects/mesh.py index 8ab8895..b15ca51 100644 --- a/speckle/objects/mesh.py +++ b/speckle/objects/mesh.py @@ -18,10 +18,6 @@ class Mesh(Base): faces: List[int] = None colors: List[int] = None textureCoordinates: List[float] = None - id: Optional[str] = None - totalChildrenCount: Optional[int] = None - applicationId: Optional[str] = None - speckle_type: Optional[str] = None def __init__(self, **kwargs) -> None: super().__init__(**kwargs) From 11bc10d072366630a1cc2765ae2b25ef4ee33351 Mon Sep 17 00:00:00 2001 From: izzy lyseggen Date: Thu, 24 Dec 2020 11:37:47 +0000 Subject: [PATCH 12/13] feat(base): get id method --- speckle/objects/base.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/speckle/objects/base.py b/speckle/objects/base.py index d6f94ab..512975f 100644 --- a/speckle/objects/base.py +++ b/speckle/objects/base.py @@ -1,10 +1,11 @@ from __future__ import annotations -from speckle.logging.exceptions import SpeckleException - -from typing import Dict, List, Optional, Any from pydantic import BaseModel from pydantic.main import Extra +from typing import Dict, List, Optional, Any +from speckle.transports.memory import MemoryTransport +from speckle.logging.exceptions import SpeckleException + PRIMITIVES = (int, float, str, bool) @@ -79,6 +80,19 @@ class Base(BaseModel): parsed = [] return 1 + self._count_descendants(self, parsed) + def get_id(self, decompose: bool = False) -> str: + if self.id and not decompose: + return self.id + else: + from speckle.serialization.base_object_serializer import ( + BaseObjectSerializer, + ) + + serializer = BaseObjectSerializer() + if decompose: + serializer.write_transports = [MemoryTransport()] + return serializer.traverse_base(self)[0] + def _count_descendants(self, base: Base, parsed: List) -> int: if base in parsed: return 0 From 028ca641ef5cde87668d495f1837361d8e16162f Mon Sep 17 00:00:00 2001 From: izzy lyseggen Date: Thu, 24 Dec 2020 11:38:44 +0000 Subject: [PATCH 13/13] fix(serialisation): some quick fixes --- speckle/api/operations.py | 16 ++++++++++++++-- speckle/serialization/base_object_serializer.py | 14 ++++++++------ 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/speckle/api/operations.py b/speckle/api/operations.py index 09d6085..0da29b9 100644 --- a/speckle/api/operations.py +++ b/speckle/api/operations.py @@ -66,7 +66,7 @@ def receive( # try local transport first. if the parent is there, we assume all the children are there and continue wth deserialisation using the local transport obj_string = local_transport.get_object(obj_id) if obj_string: - base = serializer.read_json(id=obj_id, obj_string=obj_string) + base = serializer.read_json(obj_string=obj_string) return base if not remote_transport: @@ -78,4 +78,16 @@ def receive( id=obj_id, target_transport=local_transport ) - return serializer.read_json(id=obj_id, obj_string=obj_string) + return serializer.read_json(obj_string=obj_string) + + +def serialize(base: Base) -> str: + serializer = BaseObjectSerializer() + + return serializer.write_json(base)[1] + + +def deserialize(obj_string: str) -> Base: + serializer = BaseObjectSerializer() + + return serializer.read_json(obj_string=obj_string) diff --git a/speckle/serialization/base_object_serializer.py b/speckle/serialization/base_object_serializer.py index ae01fa7..c32fb76 100644 --- a/speckle/serialization/base_object_serializer.py +++ b/speckle/serialization/base_object_serializer.py @@ -59,6 +59,10 @@ class BaseObjectSerializer: if not value or prop.startswith(("__", "_")): continue + # don't prepopulate id as this will mess up hashing + if prop == "id": + continue + chunkable = True if prop in base._chunkable else False detach = True if prop.startswith("@") or chunkable else False @@ -70,15 +74,14 @@ class BaseObjectSerializer: # 2. handle Base objects elif isinstance(value, Base): child_obj = self.traverse_value(value, detach=detach) - if detach: + if detach and self.write_transports: ref_hash = child_obj["id"] object_builder[prop] = self.detach_helper(ref_hash=ref_hash) else: object_builder[prop] = child_obj # 3. handle chunkable props - elif chunkable: - self.detach_lineage.append(detach) + elif chunkable and self.write_transports: chunks = [] max_size = base._chunkable[prop] chunk = DataChunk() @@ -115,7 +118,7 @@ class BaseObjectSerializer: } # write detached or root objects to transports - if detached: + if detached and self.write_transports: for t in self.write_transports: t.save_object(id=hash, serialized_object=json.dumps(object_builder)) @@ -191,11 +194,10 @@ class BaseObjectSerializer: self.family_tree = {} self.closure_table = {} - def read_json(self, id: str, obj_string: str) -> Base: + def read_json(self, obj_string: str) -> Base: """Recomposes a Base object from the string representation of the object Arguments: - id {str} -- the hash of the object obj_string {str} -- the string representation of the object Returns: