Compare commits

...

11 Commits

Author SHA1 Message Date
izzy lyseggen 81680ed766 Merge pull request #125 from specklesystems/izzy/metrics
metrics 🛰
2021-10-12 11:38:40 +02:00
izzy lyseggen c934720bb0 fix(metrics): try catch whole track method 2021-10-12 10:36:38 +01:00
izzy lyseggen 9297a5df49 feat(metrics): disable in tests 2021-10-11 18:10:53 +01:00
izzy lyseggen 7b8bf49769 feat(metrics): track creds, ops, and stream events 2021-10-11 18:10:33 +01:00
izzy lyseggen c834496b72 feat(metrics): add them! 🛰 2021-10-11 18:09:50 +01:00
izzy lyseggen f49491611f Merge pull request #124 from specklesystems/izzy/objects
fix(objects): quick geo and style edits
2021-10-04 09:39:55 +01:00
izzy lyseggen 19b83ba191 fix(objects): quick geo and style edits 2021-10-04 09:37:42 +01:00
izzy lyseggen 8d81aab1ac Merge pull request #123 from specklesystems/izzy/server-errors
fix(batch sender): improve error messages
2021-10-04 08:53:40 +01:00
izzy lyseggen 16868fbf3b fix(batch sender): improve error messages
for when send fails completely. previously user only got a
json decode error
2021-10-04 08:53:02 +01:00
Matteo Cominetti 00892fc838 Create close-issue.yml 2021-10-02 17:13:24 +01:00
Matteo Cominetti 4987b33de2 Create open-issue.yml 2021-10-02 17:13:09 +01:00
10 changed files with 334 additions and 58 deletions
+78
View File
@@ -0,0 +1,78 @@
name: Update issue Status
on:
issues:
types: [closed]
jobs:
update_issue:
runs-on: ubuntu-latest
steps:
- name: Get project data
env:
GITHUB_TOKEN: ${{secrets.GHPROJECT_TOKEN}}
ORGANIZATION: specklesystems
PROJECT_NUMBER: 9
run: |
gh api graphql --header 'GraphQL-Features: projects_next_graphql' -f query='
query($org: String!, $number: Int!) {
organization(login: $org){
projectNext(number: $number) {
id
fields(first:20) {
nodes {
id
name
settings
}
}
}
}
}' -f org=$ORGANIZATION -F number=$PROJECT_NUMBER > project_data.json
echo 'PROJECT_ID='$(jq '.data.organization.projectNext.id' project_data.json) >> $GITHUB_ENV
echo 'STATUS_FIELD_ID='$(jq '.data.organization.projectNext.fields.nodes[] | select(.name== "Status") | .id' project_data.json) >> $GITHUB_ENV
echo "$PROJECT_ID"
echo "$STATUS_FIELD_ID"
echo 'DONE_ID='$(jq '.data.organization.projectNext.fields.nodes[] | select(.name== "Status") | .settings | fromjson | .options[] | select(.name== "Done") | .id' project_data.json) >> $GITHUB_ENV
echo "$DONE_ID"
- name: Add Issue to project #it's already in the project, but we do this to get its node id!
env:
GITHUB_TOKEN: ${{secrets.GHPROJECT_TOKEN}}
ISSUE_ID: ${{ github.event.issue.node_id }}
run: |
item_id="$( gh api graphql --header 'GraphQL-Features: projects_next_graphql' -f query='
mutation($project:ID!, $id:ID!) {
addProjectNextItem(input: {projectId: $project, contentId: $id}) {
projectNextItem {
id
}
}
}' -f project=$PROJECT_ID -f id=$ISSUE_ID --jq '.data.addProjectNextItem.projectNextItem.id')"
echo 'ITEM_ID='$item_id >> $GITHUB_ENV
- name: Update Status
env:
GITHUB_TOKEN: ${{secrets.GHPROJECT_TOKEN}}
ISSUE_ID: ${{ github.event.issue.node_id }}
run: |
gh api graphql --header 'GraphQL-Features: projects_next_graphql' -f query='
mutation($project:ID!, $status:ID!, $id:ID!, $value:String!) {
set_status: updateProjectNextItemField(
input: {
projectId: $project
itemId: $id
fieldId: $status
value: $value
}
) {
projectNextItem {
id
}
}
}' -f project=$PROJECT_ID -f status=$STATUS_FIELD_ID -f id=$ITEM_ID -f value=${{ env.DONE_ID }}
+50
View File
@@ -0,0 +1,50 @@
name: Move new issues into Project
on:
issues:
types: [opened]
jobs:
track_issue:
runs-on: ubuntu-latest
steps:
- name: Get project data
env:
GITHUB_TOKEN: ${{secrets.GHPROJECT_TOKEN}}
ORGANIZATION: specklesystems
PROJECT_NUMBER: 9
run: |
gh api graphql --header 'GraphQL-Features: projects_next_graphql' -f query='
query($org: String!, $number: Int!) {
organization(login: $org){
projectNext(number: $number) {
id
fields(first:20) {
nodes {
id
name
settings
}
}
}
}
}' -f org=$ORGANIZATION -F number=$PROJECT_NUMBER > project_data.json
echo 'PROJECT_ID='$(jq '.data.organization.projectNext.id' project_data.json) >> $GITHUB_ENV
echo 'STATUS_FIELD_ID='$(jq '.data.organization.projectNext.fields.nodes[] | select(.name== "Status") | .id' project_data.json) >> $GITHUB_ENV
- name: Add Issue to project
env:
GITHUB_TOKEN: ${{secrets.GHPROJECT_TOKEN}}
ISSUE_ID: ${{ github.event.issue.node_id }}
run: |
item_id="$( gh api graphql --header 'GraphQL-Features: projects_next_graphql' -f query='
mutation($project:ID!, $id:ID!) {
addProjectNextItem(input: {projectId: $project, contentId: $id}) {
projectNextItem {
id
}
}
}' -f project=$PROJECT_ID -f id=$ISSUE_ID --jq '.data.addProjectNextItem.projectNextItem.id')"
echo 'ITEM_ID='$item_id >> $GITHUB_ENV
+5 -1
View File
@@ -1,12 +1,13 @@
import os
from specklepy.transports.server.server import ServerTransport
from warnings import warn
from pydantic import BaseModel
from typing import List, Optional
from urllib.parse import urlparse, unquote
from specklepy.logging import metrics
from specklepy.api.models import ServerInfo
from specklepy.api.client import SpeckleClient
from specklepy.transports.sqlite import SQLiteTransport
from specklepy.transports.server.server import ServerTransport
from specklepy.logging.exceptions import SpeckleException, SpeckleWarning
@@ -41,6 +42,7 @@ def get_local_accounts(base_path: str = None) -> List[Account]:
Returns:
List[Account] -- list of all local accounts or an empty list if no accounts were found
"""
metrics.track(metrics.ACCOUNT_LIST)
account_storage = SQLiteTransport(scope="Accounts", base_path=base_path)
json_path = os.path.join(account_storage._base_path, "Accounts")
os.makedirs(json_path, exist_ok=True)
@@ -73,6 +75,7 @@ def get_default_account(base_path: str = None) -> Account:
Returns:
Account -- the default account or None if no local accounts were found
"""
metrics.track(metrics.ACCOUNT_DEFAULT)
accounts = get_local_accounts(base_path=base_path)
if not accounts:
return None
@@ -114,6 +117,7 @@ class StreamWrapper:
return "stream" if self.stream_id else "invalid"
def __init__(self, url: str) -> None:
metrics.track("streamwrapper")
self.stream_url = url
parsed = urlparse(url)
self.host = parsed.netloc
+5 -2
View File
@@ -1,7 +1,7 @@
from typing import List
from specklepy.logging import metrics
from specklepy.objects.base import Base
from specklepy.transports.sqlite import SQLiteTransport
from specklepy.transports.server import ServerTransport
from specklepy.logging.exceptions import SpeckleException
from specklepy.transports.abstract_transport import AbstractTransport
from specklepy.serialization.base_object_serializer import BaseObjectSerializer
@@ -22,6 +22,7 @@ def send(
Returns:
str -- the object id of the sent object
"""
metrics.track(metrics.RECEIVE)
if not transports and not use_default_cache:
raise SpeckleException(
message="You need to provide at least one transport: cannot send with an empty transport list and no default cache"
@@ -57,7 +58,7 @@ def receive(
Returns:
Base -- the base object
"""
metrics.track(metrics.RECEIVE)
if not local_transport:
local_transport = SQLiteTransport()
@@ -92,6 +93,7 @@ def serialize(base: Base, write_transports: List[AbstractTransport] = []) -> str
Returns:
str -- the serialized object
"""
metrics.track(metrics.SERIALIZE)
serializer = BaseObjectSerializer(write_transports=write_transports)
return serializer.write_json(base)[1]
@@ -109,6 +111,7 @@ def deserialize(obj_string: str, read_transport: AbstractTransport = None) -> Ba
Returns:
Base -- the deserialized object
"""
metrics.track(metrics.DESERIALIZE)
if not read_transport:
read_transport = SQLiteTransport()
+10 -2
View File
@@ -1,7 +1,9 @@
from typing import Dict, List, Optional
from gql import gql
from specklepy.api.resource import ResourceBase
from typing import Dict, List, Optional
from specklepy.logging import metrics
from specklepy.api.models import Stream
from specklepy.api.resource import ResourceBase
NAME = "stream"
METHODS = [
@@ -35,6 +37,7 @@ class Resource(ResourceBase):
Returns:
Stream -- the retrieved stream
"""
metrics.track(metrics.STREAM_GET)
query = gql(
"""
query Stream($id: String!, $branch_limit: Int!, $commit_limit: Int!) {
@@ -90,6 +93,7 @@ class Resource(ResourceBase):
Returns:
List[Stream] -- A list of Stream objects
"""
metrics.track(metrics.STREAM_LIST)
query = gql(
"""
query User($stream_limit: Int!) {
@@ -147,6 +151,7 @@ class Resource(ResourceBase):
Returns:
id {str} -- the id of the newly created stream
"""
metrics.track(metrics.STREAM_CREATE)
query = gql(
"""
mutation StreamCreate($stream: StreamCreateInput!) {
@@ -177,6 +182,7 @@ class Resource(ResourceBase):
Returns:
bool -- whether the stream update was successful
"""
metrics.track(metrics.STREAM_UPDATE)
query = gql(
"""
mutation StreamUpdate($stream: StreamUpdateInput!) {
@@ -207,6 +213,7 @@ class Resource(ResourceBase):
Returns:
bool -- whether the deletion was successful
"""
metrics.track(metrics.STREAM_DELETE)
query = gql(
"""
mutation StreamDelete($id: String!) {
@@ -239,6 +246,7 @@ class Resource(ResourceBase):
Returns:
List[Stream] -- a list of Streams that match the search query
"""
metrics.track(metrics.STREAM_SEARCH)
query = gql(
"""
query StreamSearch($search_query: String!,$limit: Int!, $branch_limit:Int!, $commit_limit:Int!) {
+125
View File
@@ -0,0 +1,125 @@
import os
import queue
import logging
import requests
import threading
from requests.sessions import session
from specklepy.transports.sqlite import SQLiteTransport
"""
Anonymous telemetry to help us understand how to make a better Speckle.
This really helps us to deliver a better open source project and product!
"""
TRACK = True
HOST_APP = "python"
LOG = logging.getLogger(__name__)
METRICS_TRACKER = None
# actions
RECEIVE = "receive"
SEND = "send"
STREAM_CREATE = "stream/create"
STREAM_GET = "stream/get"
STREAM_UPDATE = "stream/update"
STREAM_DELETE = "stream/delete"
STREAM_DETAILS = "stream/details"
STREAM_LIST = "stream/list"
STREAM_VIEW = "stream/view"
STREAM_SEARCH = "stream/search"
ACCOUNT_DEFAULT = "account/default"
ACCOUNT_DETAILS = "account/details"
ACCOUNT_LIST = "account/list"
SERIALIZE = "serialization/serialize"
DESERIALIZE = "serialization/deserialize"
def disable():
global TRACK
TRACK = False
def set_host_app(host_app: str):
global HOST_APP
HOST_APP = host_app
def track(action: str):
if not TRACK:
return
try:
global METRICS_TRACKER
if not METRICS_TRACKER:
METRICS_TRACKER = MetricsTracker()
page_params = {
"rec": 1,
"idsite": METRICS_TRACKER.site_id,
"uid": METRICS_TRACKER.suuid,
"action_name": action,
"url": f"http://connectors/{HOST_APP}/{action}",
"urlref": f"http://connectors/{HOST_APP}/{action}",
"_cvar": {"1": ["hostApplication", HOST_APP]},
}
event_params = {
"rec": 1,
"idsite": METRICS_TRACKER.site_id,
"uid": MetricsTracker.suuid,
"_cvar": {"1": ["hostApplication", HOST_APP]},
"e_c": HOST_APP,
"e_a": action,
}
METRICS_TRACKER.queue.put_nowait([event_params, page_params])
except Exception as ex:
# wrapping this whole thing in a try except as we never want a failure here to annoy users!
LOG.error("Error queueing metrics request: " + str(ex))
class Singleton(type):
_instances = {}
def __call__(cls, *args, **kwargs):
if cls not in cls._instances:
cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
return cls._instances[cls]
class MetricsTracker(metaclass=Singleton):
matomo_url = "https://speckle.matomo.cloud/matomo.php"
site_id = 2
host_app = "python"
suuid = None
sending_thread = None
queue = queue.Queue(1000)
def __init__(self) -> None:
self.sending_thread = threading.Thread(
target=self._send_tracking_requests, daemon=True
)
self.set_suuid()
self.sending_thread.start()
def set_suuid(self):
try:
file_path = os.path.join(SQLiteTransport.get_base_path("Speckle"), "suuid")
with open(file_path, "r") as file:
self.suuid = file.read()
except:
self.suuid = "unknown-suuid"
def _send_tracking_requests(self):
session = requests.Session()
while True:
params = self.queue.get()
try:
session.post(self.matomo_url, params=params[0])
session.post(self.matomo_url, params=params[1])
except Exception as ex:
LOG.error("Error sending metrics request: " + str(ex))
self.queue.task_done()
+34 -38
View File
@@ -16,7 +16,7 @@ class Interval(Base, speckle_type="Objects.Primitive.Interval"):
return abs(self.start - self.end)
@classmethod
def from_list(cls, args: List[Any]) -> 'Interval':
def from_list(cls, args: List[Any]) -> "Interval":
return cls(start=args[0], end=args[1])
def to_list(self) -> List[Any]:
@@ -32,18 +32,14 @@ class Point(Base, speckle_type=GEOMETRY + "Point"):
return f"{self.__class__.__name__}(x: {self.x}, y: {self.y}, z: {self.z}, id: {self.id}, speckle_type: {self.speckle_type})"
@classmethod
def from_list(cls, args: List[float]) -> 'Point':
return cls(
x=args[0],
y=args[1],
z=args[2]
)
def from_list(cls, args: List[float]) -> "Point":
return cls(x=args[0], y=args[1], z=args[2])
def to_list(self) -> List[Any]:
return [self.x, self.y, self.z]
@classmethod
def from_coords(x: float = 0.0, y: float = 0.0, z: float = 0.0):
def from_coords(cls, x: float = 0.0, y: float = 0.0, z: float = 0.0):
pt = Point()
pt.x, pt.y, pt.z = x, y, z
return pt
@@ -64,12 +60,12 @@ class Plane(Base, speckle_type=GEOMETRY + "Plane"):
ydir: Vector = Vector()
@classmethod
def from_list(cls, args: List[Any]) -> 'Plane':
def from_list(cls, args: List[Any]) -> "Plane":
return cls(
origin=Point.from_list(args[0: 3]),
normal=Vector.from_list(args[3: 6]),
xdir=Vector.from_list(args[6: 9]),
ydir=Vector.from_list(args[9: 12]),
origin=Point.from_list(args[0:3]),
normal=Vector.from_list(args[3:6]),
xdir=Vector.from_list(args[6:9]),
ydir=Vector.from_list(args[9:12]),
)
def to_list(self) -> List[Any]:
@@ -98,11 +94,11 @@ class Line(Base, speckle_type=GEOMETRY + "Line"):
length: float = None
@classmethod
def from_list(cls, args: List[Any]) -> 'Line':
def from_list(cls, args: List[Any]) -> "Line":
return cls(
start=Point.from_list(args[0: 3]),
end=Point.from_list(args[3: 6]),
domain=Interval.from_list(args[6: 9]),
start=Point.from_list(args[0:3]),
end=Point.from_list(args[3:6]),
domain=Interval.from_list(args[6:9]),
)
def to_list(self) -> List[Any]:
@@ -128,14 +124,14 @@ class Arc(Base, speckle_type=GEOMETRY + "Arc"):
length: float = None
@classmethod
def from_list(cls, args: List[Any]) -> 'Arc':
def from_list(cls, args: List[Any]) -> "Arc":
return cls(
radius=args[1],
startAngle=args[2],
endAngle=args[3],
angleRadians=args[4],
domain=Interval.from_list(args[5: 7]),
plane=Plane.from_list(args[7: 20]),
domain=Interval.from_list(args[5:7]),
plane=Plane.from_list(args[7:20]),
units=get_units_from_encoding(args[-1]),
)
@@ -161,7 +157,7 @@ class Circle(Base, speckle_type=GEOMETRY + "Circle"):
length: float = None
@classmethod
def from_list(cls, args: List[Any]) -> 'Circle':
def from_list(cls, args: List[Any]) -> "Circle":
return cls(
radius=args[1],
domain=Interval.from_list(args[2:4]),
@@ -190,7 +186,7 @@ class Ellipse(Base, speckle_type=GEOMETRY + "Ellipse"):
length: float = None
@classmethod
def from_list(cls, args: List[Any]) -> 'Ellipse':
def from_list(cls, args: List[Any]) -> "Ellipse":
return cls(
firstRadius=args[1],
secondRadius=args[2],
@@ -228,12 +224,12 @@ class Polyline(Base, speckle_type=GEOMETRY + "Polyline", chunkable={"value": 200
return polyline
@classmethod
def from_list(cls, args: List[Any]) -> 'Polyline':
def from_list(cls, args: List[Any]) -> "Polyline":
point_count = args[4]
return cls(
closed=bool(args[1]),
domain=Interval.from_list(args[2:4]),
value=args[5:5+point_count],
value=args[5 : 5 + point_count],
units=get_units_from_encoding(args[-1]),
)
@@ -293,8 +289,7 @@ class Curve(
]
@classmethod
def from_list(cls, args: List[Any]) -> 'Curve':
def from_list(cls, args: List[Any]) -> "Curve":
point_count = args[7]
weights_count = args[8]
knots_count = args[9]
@@ -310,9 +305,9 @@ class Curve(
rational=bool(args[3]),
closed=bool(args[4]),
domain=Interval.from_list(args[5:7]),
points=args[points_start: weights_start],
weights=args[weights_start: knots_start],
knots=args[knots_start: knots_end],
points=args[points_start:weights_start],
weights=args[weights_start:knots_start],
knots=args[knots_start:knots_end],
units=get_units_from_encoding(args[-1]),
)
@@ -343,7 +338,7 @@ class Polycurve(Base, speckle_type=GEOMETRY + "Polycurve"):
length: float = None
@classmethod
def from_list(cls, args: List[Any]) -> 'Polycurve':
def from_list(cls, args: List[Any]) -> "Polycurve":
curve_arrays = CurveArray()
curve_arrays.data = args[4:-1]
return cls(
@@ -414,7 +409,7 @@ class Surface(Base, speckle_type=GEOMETRY + "Surface"):
knotsV: List[float] = None
@classmethod
def from_list(cls, args: List[Any]) -> 'Surface':
def from_list(cls, args: List[Any]) -> "Surface":
point_count = int(args[11])
knots_u_count = int(args[12])
knots_v_count = int(args[13])
@@ -433,9 +428,9 @@ class Surface(Base, speckle_type=GEOMETRY + "Surface"):
closedV=bool(args[6]),
domainU=Interval(start=args[7], end=args[8]),
domainV=Interval(start=args[9], end=args[10]),
pointData=args[start_point_data: start_knots_u],
knotsU=args[start_knots_u: start_knots_v],
knotsV=args[start_knots_v: start_knots_v + knots_v_count],
pointData=args[start_point_data:start_knots_u],
knotsU=args[start_knots_u:start_knots_v],
knotsV=args[start_knots_v : start_knots_v + knots_v_count],
units=get_units_from_encoding(args[-1]),
)
@@ -565,7 +560,7 @@ class BrepTrim(Base, speckle_type=GEOMETRY + "BrepTrim"):
return self._Brep.Curve2D[self.CurveIndex]
@classmethod
def from_list(cls, args: List[Any]) -> 'BrepTrim':
def from_list(cls, args: List[Any]) -> "BrepTrim":
return cls(
EdgeIndex=args[0],
StartIndex=args[1],
@@ -704,7 +699,7 @@ class Brep(
vertices = []
for i in range(0, len(value), 3):
vertex = Point.from_list(value[i:i+3])
vertex = Point.from_list(value[i : i + 3])
vertex._units = units
vertices.append(vertex)
@@ -729,8 +724,9 @@ class Brep(
@TrimsValue.setter
def TrimsValue(self, value: List[float]):
self.Trims = [BrepTrim.from_list(value[i:i + 9])
for i in range(0, len(value), 9)]
self.Trims = [
BrepTrim.from_list(value[i : i + 9]) for i in range(0, len(value), 9)
]
BrepEdge.update_forward_refs()
+19 -11
View File
@@ -102,7 +102,9 @@ class BatchSender(object):
new_objects = [obj[1] for obj in batch if obj[0] in new_object_ids]
if not new_objects:
LOG.info(f"Uploading batch of {len(batch)} objects: all objects are already in the server")
LOG.info(
f"Uploading batch of {len(batch)} objects: all objects are already in the server"
)
return
upload_data = "[" + ",".join(new_objects) + "]"
@@ -112,24 +114,30 @@ class BatchSender(object):
% (len(batch), len(new_objects), len(upload_data), len(upload_data_gzip))
)
r = session.post(
url=f"{self.server_url}/objects/{self.stream_id}",
files={"batch-1": ("batch-1", upload_data_gzip, "application/gzip")},
)
if r.status_code != 201:
LOG.warning("Upload server response: %s", r.text)
raise SpeckleException(
message=f"Could not save the object to the server - status code {r.status_code}"
try:
r = session.post(
url=f"{self.server_url}/objects/{self.stream_id}",
files={"batch-1": ("batch-1", upload_data_gzip, "application/gzip")},
)
if r.status_code != 201:
LOG.warning("Upload server response: %s", r.text)
raise SpeckleException(
message=f"Could not save the object to the server - status code {r.status_code}"
)
except json.JSONDecodeError as error:
return SpeckleException(
f"Failed to send objects to {self.server_url}. Please ensure this stream ({self.stream_id}) exists on this server and that you have permission to send to it.",
error,
)
def _create_threads(self):
for i in range(self.thread_count):
for _ in range(self.thread_count):
t = threading.Thread(target=self._sending_thread_main, daemon=True)
t.start()
self._send_threads.append(t)
def _delete_threads(self):
for i in range(len(self._send_threads)):
for _ in range(len(self._send_threads)):
self._batches.put(None)
for thread in self._send_threads:
+5 -4
View File
@@ -32,7 +32,7 @@ class SQLiteTransport(AbstractTransport):
super().__init__(**data)
self.app_name = app_name or "Speckle"
self.scope = scope or "Objects"
self._base_path = base_path or self.__get_base_path()
self._base_path = base_path or self.get_base_path(self.app_name)
try:
os.makedirs(self._base_path, exist_ok=True)
@@ -56,7 +56,8 @@ class SQLiteTransport(AbstractTransport):
# proc.start()
# proc.join()
def __get_base_path(self):
@staticmethod
def get_base_path(app_name):
# from appdirs https://github.com/ActiveState/appdirs/blob/master/appdirs.py
# default mac path is not the one we use (we use unix path), so using special case for this
system = sys.platform
@@ -68,10 +69,10 @@ class SQLiteTransport(AbstractTransport):
system = "darwin"
if system != "darwin":
return user_data_dir(appname=self.app_name, appauthor=False, roaming=True)
return user_data_dir(appname=app_name, appauthor=False, roaming=True)
path = os.path.expanduser("~/.config/")
return os.path.join(path, self.app_name)
return os.path.join(path, app_name)
# def __consume_queue(self):
# if self._is_writing or self.__queue.empty():
+3
View File
@@ -7,6 +7,9 @@ from specklepy.api.client import SpeckleClient
from specklepy.objects.base import Base
from specklepy.objects.geometry import Point
from specklepy.objects.fakemesh import FakeMesh
from specklepy.logging import metrics
metrics.disable()
@pytest.fixture(scope="session")