Merge pull request #101 from specklesystems/izzy/json-acct-tweak

Izzy/json acct tweak

README.md

@@ -14,6 +14,9 @@ Comprehensive developer and user documentation can be found in our:
 #### 📚 [Speckle Docs website](https://speckle.guide/dev/)
 
 ## Developing & Debugging
+
+### Installation
+
 This project uses python-poetry for dependency management, make sure you follow the official [docs](https://python-poetry.org/docs/#installation) to get poetry.
 
 To bootstrap the project environment run `$ poetry install`. This will create a new virtual-env for the project and install both the package and dev dependencies.
@@ -24,6 +27,13 @@ To execute any python script run `$ poetry run python my_script.py`
 
 > Alternatively you may roll your own virtual-env with either venv, virtualenv, pyenv-virtualenv etc. Poetry will play along an recognize if it is invoked from inside a virtual environment.
 
+### Local Data Paths
+
+It may be helpful to know where the local accounts and object cache dbs are stored. Depending on on your OS, you can find the dbs at:
+- Windows: `APPDATA` or `<USER>\AppData\Roaming\Speckle`
+- Linux: `$XDG_DATA_HOME` or by default `~/.local/share/Speckle`
+- Mac: `~/.config/Speckle`
+
 ## Overview of functionality 
 
 The `SpeckleClient` is the entry point for interacting with the GraphQL API. You'll need to have a running server to use this.
@@ -135,6 +145,10 @@ Please make sure you read the [contribution guidelines](.github/CONTRIBUTING.md)
 
 The Speckle Community hangs out on [the forum](https://discourse.speckle.works), do join and introduce yourself & feel free to ask us questions!
 
+## Security
+
+For any security vulnerabilities or concerns, please contact us directly at security[at]speckle.systems. 
+
 ## License
 
 Unless otherwise described, the code in this repository is licensed under the Apache-2.0 License. Please note that some modules, extensions or code herein might be otherwise licensed. This is indicated either in the root of the containing folder under a different license file, or in the respective file's header. If you have any questions, don't hesitate to get in touch with us via [email](mailto:hello@speckle.systems).

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "specklepy"
-version = "2.1.0"
+version = "2.2.2"
 description = "The Python SDK for Speckle 2.0"
 readme = "README.md"
 authors = ["Speckle Systems <devops@speckle.systems>"]

specklepy/api/client.py

@@ -13,6 +13,7 @@ from specklepy.api.resources import (
     user,
     subscriptions,
 )
+from specklepy.api.models import ServerInfo
 from gql import Client, gql
 from gql.transport.requests import RequestsHTTPTransport
 from gql.transport.aiohttp import AIOHTTPTransport
@@ -46,6 +47,14 @@ class SpeckleClient:
 
         self._init_resources()
 
+        # Check compatibility with the server
+        try:
+            serverInfo = self.server.get()
+            if not isinstance(serverInfo, ServerInfo):
+                raise Exception("Couldn't get ServerInfo")
+        except Exception as ex:
+            raise SpeckleException(f"{self.url} is not a compatible Speckle Server", ex)
+
     def authenticate(self, token: str) -> None:
         """Authenticate the client using a personal access token
         The token is saved in the client object and a synchronous GraphQL entrypoint is created

specklepy/api/credentials.py

@@ -1,9 +1,9 @@
+import os
 from typing import List, Optional
 from pydantic import BaseModel
 from specklepy.api.models import ServerInfo
 from specklepy.transports.sqlite import SQLiteTransport
-
-account_storage = SQLiteTransport(scope="Accounts")
+from specklepy.logging.exceptions import SpeckleException
 
 
 class UserInfo(BaseModel):
@@ -14,12 +14,12 @@ class UserInfo(BaseModel):
 
 
 class Account(BaseModel):
-    isDefault: bool
+    isDefault: bool = None
     token: str
-    refreshToken: str
+    refreshToken: str = None
     serverInfo: ServerInfo
     userInfo: UserInfo
-    id: str
+    id: str = None
 
     def __repr__(self) -> str:
         return f"Account(email: {self.userInfo.email}, server: {self.serverInfo.url}, isDefault: {self.isDefault})"
@@ -28,23 +28,48 @@ class Account(BaseModel):
         return self.__repr__()
 
 
-def get_local_accounts() -> List[Account]:
+def get_local_accounts(base_path: str = None) -> List[Account]:
     """Gets all the accounts present in this environment
 
+    Arguments:
+        base_path {str} -- custom base path if you are not using the system default
+
     Returns:
         List[Account] -- list of all local accounts or an empty list if no accounts were found
     """
+    account_storage = SQLiteTransport(scope="Accounts", base_path=base_path)
+    json_path = os.path.join(account_storage._base_path, "Accounts")
+    os.makedirs(json_path, exist_ok=True)
+    json_acct_files = [file for file in os.listdir(json_path) if file.endswith(".json")]
+
+    accounts = []
     res = account_storage.get_all_objects()
-    return [Account.parse_raw(r[1]) for r in res] if res else []
+    if res:
+        accounts.extend(Account.parse_raw(r[1]) for r in res)
+    if json_acct_files:
+        try:
+            accounts.extend(
+                Account.parse_file(os.path.join(json_path, json_file))
+                for json_file in json_acct_files
+            )
+        except Exception as ex:
+            raise SpeckleException(
+                "Invalid json accounts could not be read. Please fix or remove them.",
+                ex,
+            )
+
+    return accounts
 
 
-def get_default_account() -> Account:
+def get_default_account(base_path: str = None) -> Account:
     """Gets this environment's default account if any. If there is no default, the first found will be returned and set as default.
+    Arguments:
+        base_path {str} -- custom base path if you are not using the system default
 
     Returns:
         Account -- the default account or None if no local accounts were found
     """
-    accounts = get_local_accounts()
+    accounts = get_local_accounts(base_path=base_path)
     if not accounts:
         return None
 

specklepy/api/models.py

@@ -115,3 +115,4 @@ class ServerInfo(BaseModel):
     roles: Optional[List[dict]]
     scopes: Optional[List[dict]]
     authStrategies: Optional[List[dict]]
+    version: Optional[str]

specklepy/api/resources/__init__.py

@@ -5,7 +5,7 @@ import pkgutil
 from importlib import import_module
 
 
-for (_, name, _) in pkgutil.iter_modules([Path(__file__).parent]):
+for (_, name, _) in pkgutil.iter_modules(__path__):
 
     imported_module = import_module("." + name, package=__name__)
 

specklepy/api/resources/server.py

@@ -32,6 +32,7 @@ class Resource(ResourceBase):
                     description
                     adminContact
                     canonicalUrl
+                    version
                     roles {
                         name
                         description

specklepy/serialization/base_object_serializer.py

@@ -277,8 +277,8 @@ class BaseObjectSerializer:
             base.totalChildrenCount = len(closure)
 
         for prop, value in obj.items():
-            # 1. handle primitives (ints, floats, strings, and bools)
-            if isinstance(value, PRIMITIVES):
+            # 1. handle primitives (ints, floats, strings, and bools) or None
+            if isinstance(value, PRIMITIVES) or value is None:
                 base.__setattr__(prop, value)
                 continue
 

specklepy/transports/abstract_transport.py

@@ -1,5 +1,5 @@
 from abc import ABC, abstractmethod
-from typing import Any, Optional
+from typing import Any, Optional, List, Dict
 from pydantic import BaseModel
 from pydantic.main import Extra
 
@@ -64,6 +64,18 @@ class AbstractTransport(ABC, BaseModel):
         """
         pass
 
+    @abstractmethod
+    def has_objects(self, id_list: List[str]) -> Dict[str, bool]:
+        """Checks the presence of multiple objects.
+
+        Arguments:
+            id_list -- List of object id to be checked
+
+        Returns:
+            Dict[str, bool] -- keys: input ids, values: whether the transport has that object
+        """
+        pass
+
     @abstractmethod
     def copy_object_and_children(
         self, id: str, target_transport: "AbstractTransport"

specklepy/transports/memory.py

@@ -1,5 +1,5 @@
 import json
-from typing import Any
+from typing import Any, List, Dict
 from specklepy.logging.exceptions import SpeckleException
 from specklepy.transports.abstract_transport import AbstractTransport
 
@@ -33,6 +33,9 @@ class MemoryTransport(AbstractTransport):
         else:
             return None
 
+    def has_objects(self, id_list: List[str]) -> Dict[str, bool]:
+        return {id: (id in self.objects) for id in id_list}
+
     def begin_write(self) -> None:
         self.saved_object_count = 0
 

specklepy/transports/server/batch_sender.py

@@ -1,3 +1,4 @@
+import json
 import logging
 import threading
 import queue
@@ -13,13 +14,15 @@ LOG = logging.getLogger(__name__)
 class BatchSender(object):
     def __init__(
         self,
-        endpoint,
+        server_url,
+        stream_id,
         token,
         max_batch_size_mb=1,
         batch_buffer_length=10,
         thread_count=4,
     ):
-        self.endpoint = endpoint
+        self.server_url = server_url
+        self.stream_id = stream_id
         self._token = token
 
         self.max_size = int(max_batch_size_mb * 1000 * 1000)
@@ -31,18 +34,18 @@ class BatchSender(object):
         self._send_threads = []
         self._exception = None
 
-    def send_object(self, obj: str):
+    def send_object(self, id: str, obj: str):
         if not self._send_threads:
             self._create_threads()
 
         crt_obj_size = len(obj)
         if not self._crt_batch or self._crt_batch_size + crt_obj_size < self.max_size:
-            self._crt_batch.append(obj)
+            self._crt_batch.append((id, obj))
             self._crt_batch_size += crt_obj_size
             return
 
         self._batches.put(self._crt_batch)
-        self._crt_batch = [obj]
+        self._crt_batch = [(id, obj)]
         self._crt_batch_size = crt_obj_size
 
     def flush(self):
@@ -88,15 +91,29 @@ class BatchSender(object):
             LOG.error("ServerTransport sending thread error: " + str(ex))
 
     def _bg_send_batch(self, session, batch):
-        upload_data = "[" + ",".join(batch) + "]"
+        object_ids = [obj[0] for obj in batch]
+        server_has_object = session.post(
+            url=f"{self.server_url}/api/diff/{self.stream_id}",
+            data={"objects": json.dumps(object_ids)},
+        ).json()
+
+        new_object_ids = [x for x in object_ids if not server_has_object[x]]
+        new_object_ids = set(new_object_ids)
+        new_objects = [obj[1] for obj in batch if obj[0] in new_object_ids]
+
+        if not new_objects:
+            LOG.info(f"Uploading batch of {len(batch)} objects: all objects are already in the server")
+            return
+
+        upload_data = "[" + ",".join(new_objects) + "]"
         upload_data_gzip = gzip.compress(upload_data.encode())
         LOG.info(
-            "Uploading batch of %s objects (size: %s, compressed size: %s)"
-            % (len(batch), len(upload_data), len(upload_data_gzip))
+            "Uploading batch of %s objects (%s new): (size: %s, compressed size: %s)"
+            % (len(batch), len(new_objects), len(upload_data), len(upload_data_gzip))
         )
 
         r = session.post(
-            url=self.endpoint,
+            url=f"{self.server_url}/objects/{self.stream_id}",
             files={"batch-1": ("batch-1", upload_data_gzip, "application/gzip")},
         )
         if r.status_code != 201:

specklepy/transports/server/server.py

@@ -1,3 +1,6 @@
+import json
+import time
+
 import requests
 
 from typing import Any, Dict, List, Type
@@ -23,8 +26,7 @@ class ServerTransport(AbstractTransport):
         self.stream_id = stream_id
 
         token = client.me["token"]
-        endpoint = f"{self.url}/objects/{self.stream_id}"
-        self._batch_sender = BatchSender(endpoint, token, max_batch_size_mb=1)
+        self._batch_sender = BatchSender(self.url, self.stream_id, token, max_batch_size_mb=1)
 
         self.session = requests.Session()
         self.session.headers.update(
@@ -38,7 +40,7 @@ class ServerTransport(AbstractTransport):
         self._batch_sender.flush()
 
     def save_object(self, id: str, serialized_object: str) -> None:
-        self._batch_sender.send_object(serialized_object)
+        self._batch_sender.send_object(id, serialized_object)
 
     def save_object_from_transport(
         self, id: str, source_transport: AbstractTransport
@@ -59,26 +61,44 @@ class ServerTransport(AbstractTransport):
             NotImplementedError,
         )
 
+    def has_objects(self, id_list: List[str]) -> Dict[str, bool]:
+        return {id: False for id in id_list}
+
     def copy_object_and_children(
         self, id: str, target_transport: AbstractTransport
     ) -> str:
-        endpoint = f"{self.url}/objects/{self.stream_id}/{id}"
-        r = self.session.get(endpoint, stream=True)
+        endpoint = f"{self.url}/objects/{self.stream_id}/{id}/single"
+        r = self.session.get(endpoint)
+        if r.encoding is None:
+            r.encoding = "utf-8"
+
+        if r.status_code != 200:
+            raise SpeckleException(f"Can't get object {self.stream_id}/{id}: HTTP error {r.status_code} ({r.text[:1000]})")
+        root_obj_serialized = r.text
+        root_obj = json.loads(root_obj_serialized)
+        closures = root_obj.get('__closure', {})
+
+        # Check which children are not already in the target transport
+        children_ids = list(closures.keys())
+        children_found_map = target_transport.has_objects(children_ids)
+        new_children_ids = [id for id in children_found_map if not children_found_map[id]]
+
+        # Get the new children
+        endpoint = f"{self.url}/api/getobjects/{self.stream_id}"
+        r = self.session.post(endpoint, data={"objects": json.dumps(new_children_ids)}, stream=True)
         if r.encoding is None:
             r.encoding = "utf-8"
         lines = r.iter_lines(decode_unicode=True)
 
-        # save first (root) obj for return
-        root_hash, root_obj = next(lines).split("\t")
-        target_transport.save_object(root_hash, root_obj)
-
         # iter through returned objects saving them as we go
         for line in lines:
             if line:
                 hash, obj = line.split("\t")
                 target_transport.save_object(hash, obj)
 
-        return root_obj
+        target_transport.save_object(id, root_obj_serialized)
+
+        return root_obj_serialized
 
     # async def stream_res(self, endpoint: str) -> str:
     #     data = b""

specklepy/transports/sqlite.py

@@ -3,7 +3,7 @@ import sys
 import time
 import sched
 import sqlite3
-from typing import Any
+from typing import Any, List, Dict
 from appdirs import user_data_dir
 from contextlib import closing
 from multiprocessing import Process, Queue
@@ -13,6 +13,7 @@ from specklepy.logging.exceptions import SpeckleException
 
 class SQLiteTransport(AbstractTransport):
     _name = "SQLite"
+    _base_path: str = None
     _root_path: str = None
     _is_writing: bool = False
     _scheduler = sched.scheduler(time.time, time.sleep)
@@ -33,11 +34,13 @@ class SQLiteTransport(AbstractTransport):
         super().__init__(**data)
         self.app_name = app_name or "Speckle"
         self.scope = scope or "Objects"
-        base_path = base_path or self.__get_base_path()
+        self._base_path = base_path or self.__get_base_path()
 
-        os.makedirs(base_path, exist_ok=True)
+        os.makedirs(self._base_path, exist_ok=True)
 
-        self._root_path = os.path.join(os.path.join(base_path, f"{self.scope}.db"))
+        self._root_path = os.path.join(
+            os.path.join(self._base_path, f"{self.scope}.db")
+        )
         self.__initialise()
 
     def __repr__(self) -> str:
@@ -137,6 +140,17 @@ class SQLiteTransport(AbstractTransport):
             ).fetchone()
         return row[1] if row else None
 
+    def has_objects(self, id_list: List[str]) -> Dict[str, bool]:
+        ret = {}
+        self.__check_connection()
+        with closing(self.__connection.cursor()) as c:
+            for id in id_list:
+                row = c.execute(
+                    "SELECT 1 FROM objects WHERE hash = ? LIMIT 1", (id,)
+                ).fetchone()
+                ret[id] = bool(row)
+        return ret
+
     def begin_write(self):
         self.saved_obj_count = 0