From 7bc42653d05be2adfb13228252e339e59df6a1dc Mon Sep 17 00:00:00 2001 From: KatKatKateryna <89912278+KatKatKateryna@users.noreply.github.com> Date: Wed, 14 Aug 2024 05:13:10 +0100 Subject: [PATCH] Kate speckle poc (#2) * Revert "change port" This reverts commit 92e0b0da9ea231ea3b3d40e950b4c5de03cf87ee. * server patch * enforce patch * Reapply "change port" This reverts commit 243994d0daa1719e6e0f8771b3d487528d80067c. --- pygeoapi/provider/speckle.py | 10 +- .../provider/speckle_utils/patch_specklepy.py | 61 +++--- pygeoapi/provider/speckle_utils/server.py | 184 ++++++++++++++++++ 3 files changed, 221 insertions(+), 34 deletions(-) create mode 100644 pygeoapi/provider/speckle_utils/server.py diff --git a/pygeoapi/provider/speckle.py b/pygeoapi/provider/speckle.py index 5740138..91df059 100644 --- a/pygeoapi/provider/speckle.py +++ b/pygeoapi/provider/speckle.py @@ -79,6 +79,7 @@ class SpeckleProvider(BaseProvider): # ) from subprocess import run + from pygeoapi.provider.speckle_utils.patch_specklepy import patch_specklepy # path = str(self.connector_installation_path(_host_application)) @@ -86,8 +87,7 @@ class SpeckleProvider(BaseProvider): import specklepy except ModuleNotFoundError: - from pygeoapi.provider.speckle_utils.patch_specklepy import patch_credentials, copy_gis_feature, patch_transport - + completed_process = run( [ self.get_python_path(), @@ -109,9 +109,6 @@ class SpeckleProvider(BaseProvider): ], capture_output=True, ) - patch_credentials() - copy_gis_feature() - patch_transport() if completed_process.returncode != 0: m = f"Failed to install dependenices through pip, got {completed_process.returncode} as return code. Full log: {completed_process}" @@ -120,7 +117,7 @@ class SpeckleProvider(BaseProvider): print(completed_process.stderr) raise Exception(m) - # TODO: replace 1 line in specklepy + patch_specklepy() # assign global values self.url: str = self.data # to store the value and check if self.data has changed @@ -390,7 +387,6 @@ class SpeckleProvider(BaseProvider): # set the Model name self.model_name = branch['name'] - print(self.model_name) commit = branch["commits"]["items"][0] objId = commit["referencedObject"] diff --git a/pygeoapi/provider/speckle_utils/patch_specklepy.py b/pygeoapi/provider/speckle_utils/patch_specklepy.py index 985a135..481bc6f 100644 --- a/pygeoapi/provider/speckle_utils/patch_specklepy.py +++ b/pygeoapi/provider/speckle_utils/patch_specklepy.py @@ -20,6 +20,12 @@ def get_transport_path(): return str(credentials_path) +def get_transport_path_src(): + specklepy_path = Path(sys.executable).parent.parent + credentials_path = Path(specklepy_path, "pygeoapi", "pygeoapi", "provider", "speckle_utils", "server.py") + + return str(credentials_path) + def get_gis_feature_path_src(): specklepy_path = Path(sys.executable).parent.parent credentials_path = Path(specklepy_path, "pygeoapi", "pygeoapi", "provider", "speckle_utils", "GisFeature.py") @@ -53,39 +59,40 @@ def patch_credentials(): def patch_transport(): """Patches the installer with the correct connector version and specklepy version""" + server_data = get_transport_path_src() + file_path = get_transport_path() + + with open(server_data, "r") as file: + lines = file.readlines() + file.close() + + with open(file_path, "w") as file: + file.writelines(lines) + file.close() + +def complete_transport(): + """Patches the installer with the correct connector version and specklepy version""" + file_path = get_transport_path() with open(file_path, "r") as file: lines = file.readlines() - new_lines = [] - for i, line in enumerate(lines): - - if 'if self.account is not None:' in line: - line = line.replace("if self.account is not None:", "if self.account.token is not None:") - - if 'lines = r.iter_lines(decode_unicode=True)' in line: - line = line.replace('lines = r.iter_lines(decode_unicode=True)', 'lines = r.iter_lines(decode_unicode=True, delimiter="},{")') - - if 'for line in lines:' in line: - line1 = line.replace('for line in lines:','all_lines = [line for _,line in enumerate(lines)]') - new_lines.append(line1) - line = line.replace('for line in lines:','for i, line in enumerate(all_lines):') - if 'hash, obj = line.split("\\t")' in line: - line1 = line.replace('hash, obj = line.split("\\t")','hash = line.split(\'"id": "\')[1].split(\'"\')[0]') - line2 = line.replace('hash, obj = line.split("\\t")','obj = "{" + line + "}"') - line3 = line.replace('hash, obj = line.split("\\t")','if i==0:') - line4 = line.replace('hash, obj = line.split("\\t")',' obj = obj[2:]') - line5 = line.replace('hash, obj = line.split("\\t")','elif i==len(all_lines)-1:') - new_lines.extend([line1, line2, line3, line4, line5]) - - line = line.replace('hash, obj = line.split("\\t")',' obj = obj[:-2]') - - new_lines.append(line) file.close() - with open(file_path, "w") as file: - file.writelines(new_lines) - file.close() + print(len(lines)) + if len(lines) < 184: + return False + return True def copy_gis_feature(): shutil.copyfile(get_gis_feature_path_src(), get_gis_feature_path_dst()) + +def patch_specklepy(): + + if complete_transport(): + return + + patch_credentials() + copy_gis_feature() + patch_transport() + \ No newline at end of file diff --git a/pygeoapi/provider/speckle_utils/server.py b/pygeoapi/provider/speckle_utils/server.py new file mode 100644 index 0000000..bfef494 --- /dev/null +++ b/pygeoapi/provider/speckle_utils/server.py @@ -0,0 +1,184 @@ +import json +from typing import Dict, List, Optional +from warnings import warn + +import requests + +from specklepy.core.api.client import SpeckleClient +from specklepy.core.api.credentials import Account, get_account_from_token +from specklepy.logging.exceptions import SpeckleException, SpeckleWarning +from specklepy.transports.abstract_transport import AbstractTransport + +from .batch_sender import BatchSender + + +class ServerTransport(AbstractTransport): + """ + The `ServerTransport` is the vehicle through which you transport objects to and + from a Speckle Server. Provide it to `operations.send()` or `operations.receive()`. + + The `ServerTransport` can be authenticated two different ways: + 1. by providing a `SpeckleClient` + 2. by providing an `Account` + 3. by providing a `token` and `url` + + ```py + from specklepy.api import operations + from specklepy.transports.server import ServerTransport + + # here's the data you want to send + block = Block(length=2, height=4) + + # next create the server transport - this is the vehicle through which + # you will send and receive + transport = ServerTransport(stream_id=new_stream_id, client=client) + + # this serialises the block and sends it to the transport + hash = operations.send(base=block, transports=[transport]) + + # you can now create a commit on your stream with this object + commit_id = client.commit.create( + stream_id=new_stream_id, + obj_id=hash, + message="this is a block I made in speckle-py", + ) + ``` + """ + + def __init__( + self, + stream_id: str, + client: Optional[SpeckleClient] = None, + account: Optional[Account] = None, + token: Optional[str] = None, + url: Optional[str] = None, + name: str = "RemoteTransport", + ) -> None: + super().__init__() + if client is None and account is None and token is None and url is None: + raise SpeckleException( + "You must provide either a client or a token and url to construct a" + " ServerTransport." + ) + + self._name = name + self.account = None + self.saved_obj_count = 0 + if account: + self.account = account + url = account.serverInfo.url + elif client: + url = client.url + if not client.account.token: + warn( + SpeckleWarning( + "Unauthenticated Speckle Client provided to Server Transport" + f" for {url}. Receiving from private streams will fail." + ) + ) + else: + self.account = client.account + else: + self.account = get_account_from_token(token, url) + + self.stream_id = stream_id + self.url = url + + self.session = requests.Session() + + if self.account.token is not None: + self._batch_sender = BatchSender( + self.url, self.stream_id, self.account.token, max_batch_size_mb=1 + ) + self.session.headers.update( + { + "Authorization": f"Bearer {self.account.token}", + "Accept": "text/plain", + } + ) + + @property + def name(self) -> str: + return self._name + + def begin_write(self) -> None: + self.saved_obj_count = 0 + + def end_write(self) -> None: + self._batch_sender.flush() + + def save_object(self, id: str, serialized_object: str) -> None: + self._batch_sender.send_object(id, serialized_object) + + def save_object_from_transport( + self, id: str, source_transport: AbstractTransport + ) -> None: + obj_string = source_transport.get_object(id=id) + self.save_object(id=id, serialized_object=obj_string) + + def get_object(self, id: str) -> str: + # endpoint = f"{self.url}/objects/{self.stream_id}/{id}/single" + # r = self.session.get(endpoint, stream=True) + + # _, obj = next(r.iter_lines().decode("utf-8")).split("\t") + + # return obj + + raise SpeckleException( + "Getting a single object using `ServerTransport.get_object()` is not" + " implemented. To get an object from the server, please use the" + " `SpeckleClient.object.get()` route", + NotImplementedError(), + ) + + def has_objects(self, id_list: List[str]) -> Dict[str, bool]: + return {id: False for id in id_list} + + def copy_object_and_children( + self, id: str, target_transport: AbstractTransport + ) -> str: + endpoint = f"{self.url}/objects/{self.stream_id}/{id}/single" + r = self.session.get(endpoint) + r.encoding = "utf-8" + + if r.status_code != 200: + raise SpeckleException( + f"Can't get object {self.stream_id}/{id}: HTTP error" + f" {r.status_code} ({r.text[:1000]})" + ) + root_obj_serialized = r.text + root_obj = json.loads(root_obj_serialized) + closures = root_obj.get("__closure", {}) + + # Check which children are not already in the target transport + children_ids = list(closures.keys()) + children_found_map = target_transport.has_objects(children_ids) + new_children_ids = [ + id for id in children_found_map if not children_found_map[id] + ] + + # Get the new children + endpoint = f"{self.url}/api/getobjects/{self.stream_id}" + r = self.session.post( + endpoint, data={"objects": json.dumps(new_children_ids)}, stream=True + ) + r.encoding = "utf-8" + lines = r.iter_lines(decode_unicode=True, delimiter="},{") + + # iter through returned objects saving them as we go + target_transport.begin_write() + all_lines = [line for _,line in enumerate(lines)] + for i, line in enumerate(all_lines): + if line: + hash = line.split('"id": "')[1].split('"')[0] + obj = "{" + line + "}" + if i==0: + obj = obj[2:] + elif i==len(all_lines)-1: + obj = obj[:-2] + target_transport.save_object(hash, obj) + + target_transport.save_object(id, root_obj_serialized) + target_transport.end_write() + + return root_obj_serialized