Merge pull request #125 from specklesystems/izzy/metrics

metrics 🛰

specklepy/api/credentials.py

@@ -1,12 +1,13 @@
 import os
-from specklepy.transports.server.server import ServerTransport
 from warnings import warn
 from pydantic import BaseModel
 from typing import List, Optional
 from urllib.parse import urlparse, unquote
+from specklepy.logging import metrics
 from specklepy.api.models import ServerInfo
 from specklepy.api.client import SpeckleClient
 from specklepy.transports.sqlite import SQLiteTransport
+from specklepy.transports.server.server import ServerTransport
 from specklepy.logging.exceptions import SpeckleException, SpeckleWarning
 
 
@@ -41,6 +42,7 @@ def get_local_accounts(base_path: str = None) -> List[Account]:
     Returns:
         List[Account] -- list of all local accounts or an empty list if no accounts were found
     """
+    metrics.track(metrics.ACCOUNT_LIST)
     account_storage = SQLiteTransport(scope="Accounts", base_path=base_path)
     json_path = os.path.join(account_storage._base_path, "Accounts")
     os.makedirs(json_path, exist_ok=True)
@@ -73,6 +75,7 @@ def get_default_account(base_path: str = None) -> Account:
     Returns:
         Account -- the default account or None if no local accounts were found
     """
+    metrics.track(metrics.ACCOUNT_DEFAULT)
     accounts = get_local_accounts(base_path=base_path)
     if not accounts:
         return None
@@ -114,6 +117,7 @@ class StreamWrapper:
             return "stream" if self.stream_id else "invalid"
 
     def __init__(self, url: str) -> None:
+        metrics.track("streamwrapper")
         self.stream_url = url
         parsed = urlparse(url)
         self.host = parsed.netloc

specklepy/api/operations.py

@@ -1,7 +1,7 @@
 from typing import List
+from specklepy.logging import metrics
 from specklepy.objects.base import Base
 from specklepy.transports.sqlite import SQLiteTransport
-from specklepy.transports.server import ServerTransport
 from specklepy.logging.exceptions import SpeckleException
 from specklepy.transports.abstract_transport import AbstractTransport
 from specklepy.serialization.base_object_serializer import BaseObjectSerializer
@@ -22,6 +22,7 @@ def send(
     Returns:
         str -- the object id of the sent object
     """
+    metrics.track(metrics.RECEIVE)
     if not transports and not use_default_cache:
         raise SpeckleException(
             message="You need to provide at least one transport: cannot send with an empty transport list and no default cache"
@@ -57,7 +58,7 @@ def receive(
     Returns:
         Base -- the base object
     """
-
+    metrics.track(metrics.RECEIVE)
     if not local_transport:
         local_transport = SQLiteTransport()
 
@@ -92,6 +93,7 @@ def serialize(base: Base, write_transports: List[AbstractTransport] = []) -> str
     Returns:
         str -- the serialized object
     """
+    metrics.track(metrics.SERIALIZE)
     serializer = BaseObjectSerializer(write_transports=write_transports)
 
     return serializer.write_json(base)[1]
@@ -109,6 +111,7 @@ def deserialize(obj_string: str, read_transport: AbstractTransport = None) -> Ba
     Returns:
         Base -- the deserialized object
     """
+    metrics.track(metrics.DESERIALIZE)
     if not read_transport:
         read_transport = SQLiteTransport()
 

specklepy/api/resources/stream.py

@@ -1,7 +1,9 @@
-from typing import Dict, List, Optional
 from gql import gql
-from specklepy.api.resource import ResourceBase
+from typing import Dict, List, Optional
+from specklepy.logging import metrics
 from specklepy.api.models import Stream
+from specklepy.api.resource import ResourceBase
+
 
 NAME = "stream"
 METHODS = [
@@ -35,6 +37,7 @@ class Resource(ResourceBase):
         Returns:
             Stream -- the retrieved stream
         """
+        metrics.track(metrics.STREAM_GET)
         query = gql(
             """
             query Stream($id: String!, $branch_limit: Int!, $commit_limit: Int!) {
@@ -90,6 +93,7 @@ class Resource(ResourceBase):
         Returns:
             List[Stream] -- A list of Stream objects
         """
+        metrics.track(metrics.STREAM_LIST)
         query = gql(
             """
             query User($stream_limit: Int!) {
@@ -147,6 +151,7 @@ class Resource(ResourceBase):
         Returns:
             id {str} -- the id of the newly created stream
         """
+        metrics.track(metrics.STREAM_CREATE)
         query = gql(
             """
             mutation StreamCreate($stream: StreamCreateInput!) {
@@ -177,6 +182,7 @@ class Resource(ResourceBase):
         Returns:
             bool -- whether the stream update was successful
         """
+        metrics.track(metrics.STREAM_UPDATE)
         query = gql(
             """
             mutation StreamUpdate($stream: StreamUpdateInput!) {
@@ -207,6 +213,7 @@ class Resource(ResourceBase):
         Returns:
             bool -- whether the deletion was successful
         """
+        metrics.track(metrics.STREAM_DELETE)
         query = gql(
             """
             mutation StreamDelete($id: String!) {
@@ -239,6 +246,7 @@ class Resource(ResourceBase):
         Returns:
             List[Stream] -- a list of Streams that match the search query
         """
+        metrics.track(metrics.STREAM_SEARCH)
         query = gql(
             """
             query StreamSearch($search_query: String!,$limit: Int!, $branch_limit:Int!, $commit_limit:Int!) {

specklepy/logging/metrics.py

@@ -0,0 +1,125 @@
+import os
+import queue
+import logging
+import requests
+import threading
+from requests.sessions import session
+from specklepy.transports.sqlite import SQLiteTransport
+
+"""
+Anonymous telemetry to help us understand how to make a better Speckle.
+This really helps us to deliver a better open source project and product!
+"""
+TRACK = True
+HOST_APP = "python"
+
+LOG = logging.getLogger(__name__)
+METRICS_TRACKER = None
+
+# actions
+RECEIVE = "receive"
+SEND = "send"
+STREAM_CREATE = "stream/create"
+STREAM_GET = "stream/get"
+STREAM_UPDATE = "stream/update"
+STREAM_DELETE = "stream/delete"
+STREAM_DETAILS = "stream/details"
+STREAM_LIST = "stream/list"
+STREAM_VIEW = "stream/view"
+STREAM_SEARCH = "stream/search"
+
+ACCOUNT_DEFAULT = "account/default"
+ACCOUNT_DETAILS = "account/details"
+ACCOUNT_LIST = "account/list"
+
+SERIALIZE = "serialization/serialize"
+DESERIALIZE = "serialization/deserialize"
+
+
+def disable():
+    global TRACK
+    TRACK = False
+
+
+def set_host_app(host_app: str):
+    global HOST_APP
+    HOST_APP = host_app
+
+
+def track(action: str):
+    if not TRACK:
+        return
+    try:
+        global METRICS_TRACKER
+        if not METRICS_TRACKER:
+            METRICS_TRACKER = MetricsTracker()
+
+        page_params = {
+            "rec": 1,
+            "idsite": METRICS_TRACKER.site_id,
+            "uid": METRICS_TRACKER.suuid,
+            "action_name": action,
+            "url": f"http://connectors/{HOST_APP}/{action}",
+            "urlref": f"http://connectors/{HOST_APP}/{action}",
+            "_cvar": {"1": ["hostApplication", HOST_APP]},
+        }
+
+        event_params = {
+            "rec": 1,
+            "idsite": METRICS_TRACKER.site_id,
+            "uid": MetricsTracker.suuid,
+            "_cvar": {"1": ["hostApplication", HOST_APP]},
+            "e_c": HOST_APP,
+            "e_a": action,
+        }
+
+        METRICS_TRACKER.queue.put_nowait([event_params, page_params])
+    except Exception as ex:
+        # wrapping this whole thing in a try except as we never want a failure here to annoy users!
+        LOG.error("Error queueing metrics request: " + str(ex))
+
+
+class Singleton(type):
+    _instances = {}
+
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
+        return cls._instances[cls]
+
+
+class MetricsTracker(metaclass=Singleton):
+    matomo_url = "https://speckle.matomo.cloud/matomo.php"
+    site_id = 2
+    host_app = "python"
+    suuid = None
+    sending_thread = None
+    queue = queue.Queue(1000)
+
+    def __init__(self) -> None:
+        self.sending_thread = threading.Thread(
+            target=self._send_tracking_requests, daemon=True
+        )
+        self.set_suuid()
+        self.sending_thread.start()
+
+    def set_suuid(self):
+        try:
+            file_path = os.path.join(SQLiteTransport.get_base_path("Speckle"), "suuid")
+            with open(file_path, "r") as file:
+                self.suuid = file.read()
+        except:
+            self.suuid = "unknown-suuid"
+
+    def _send_tracking_requests(self):
+        session = requests.Session()
+        while True:
+            params = self.queue.get()
+
+            try:
+                session.post(self.matomo_url, params=params[0])
+                session.post(self.matomo_url, params=params[1])
+            except Exception as ex:
+                LOG.error("Error sending metrics request: " + str(ex))
+
+            self.queue.task_done()

specklepy/transports/sqlite.py

@@ -32,7 +32,7 @@ class SQLiteTransport(AbstractTransport):
         super().__init__(**data)
         self.app_name = app_name or "Speckle"
         self.scope = scope or "Objects"
-        self._base_path = base_path or self.__get_base_path()
+        self._base_path = base_path or self.get_base_path(self.app_name)
 
         try:
             os.makedirs(self._base_path, exist_ok=True)
@@ -56,7 +56,8 @@ class SQLiteTransport(AbstractTransport):
     #     proc.start()
     #     proc.join()
 
-    def __get_base_path(self):
+    @staticmethod
+    def get_base_path(app_name):
         # from appdirs https://github.com/ActiveState/appdirs/blob/master/appdirs.py
         # default mac path is not the one we use (we use unix path), so using special case for this
         system = sys.platform
@@ -68,10 +69,10 @@ class SQLiteTransport(AbstractTransport):
                 system = "darwin"
 
         if system != "darwin":
-            return user_data_dir(appname=self.app_name, appauthor=False, roaming=True)
+            return user_data_dir(appname=app_name, appauthor=False, roaming=True)
 
         path = os.path.expanduser("~/.config/")
-        return os.path.join(path, self.app_name)
+        return os.path.join(path, app_name)
 
     # def __consume_queue(self):
     #     if self._is_writing or self.__queue.empty():

tests/conftest.py

@@ -7,6 +7,9 @@ from specklepy.api.client import SpeckleClient
 from specklepy.objects.base import Base
 from specklepy.objects.geometry import Point
 from specklepy.objects.fakemesh import FakeMesh
+from specklepy.logging import metrics
+
+metrics.disable()
 
 
 @pytest.fixture(scope="session")