Compare commits

..

2 Commits

Author SHA1 Message Date
Dogukan Karatas 66eab146d6 checkout update 2025-11-25 14:01:27 +01:00
Dogukan Karatas aab424bbff yml added 2025-11-25 13:51:41 +01:00
46 changed files with 177 additions and 2203 deletions
+43
View File
@@ -0,0 +1,43 @@
name: docs
on:
push:
tags:
- 'v3.*.*'
permissions:
contents: write
jobs:
deploy:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- name: Configure Git Credentials
run: |
git config user.name github-actions[bot]
git config user.email 41898282+github-actions[bot]@users.noreply.github.com
- uses: actions/setup-python@v5
with:
python-version: '3.13'
- name: Cache dependencies
run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV
- uses: actions/cache@v4
with:
key: mkdocs-material-${{ env.cache_id }}
path: ~/.cache
restore-keys: |
mkdocs-material-
- name: Install uv
uses: astral-sh/setup-uv@v4
- name: Install docs dependencies
run: uv sync --group docs
- name: Deploy to GitHub Pages
run: uv run mkdocs gh-deploy --force
-2
View File
@@ -59,8 +59,6 @@ jobs:
test-public: # Run integration tests against the public server image
name: Test (public)
runs-on: ubuntu-latest
env:
IS_PUBLIC: "true"
strategy:
matrix:
python-version:
-1
View File
@@ -1 +0,0 @@
words = ["specklepy"]
+4 -1
View File
@@ -97,7 +97,10 @@ services:
STRATEGY_LOCAL: "true"
POSTGRES_URL: 'postgres://speckle:speckle@postgres:5432/speckle'
POSTGRES_URL: "postgres"
POSTGRES_USER: "speckle"
POSTGRES_PASSWORD: "speckle"
POSTGRES_DB: "speckle"
ENABLE_MP: "false"
LOG_PRETTY: "true"
+1 -8
View File
@@ -8,7 +8,7 @@ python.uv_venv_auto = true
[tasks.install]
run= "uv sync --all-extras --all-groups"
run= "uv sync --all-groups"
[tasks.install_docs]
@@ -18,10 +18,3 @@ run= "uv sync --group docs"
description = "Build static docs "
run = "uv run mkdocs build"
depends = ['install_docs']
[tasks.test]
run = "uv run pytest"
[env]
IS_PUBLIC = "false"
-3
View File
@@ -1,3 +0,0 @@
[build]
command = "mise run build_docs"
publish = "site"
-2
View File
@@ -22,7 +22,6 @@ dependencies = [
speckleifc = ["ifcopenshell>=0.8.3.post2"]
[dependency-groups]
dev = [
"commitizen>=4.1.0",
"devtools>=0.12.2",
@@ -33,7 +32,6 @@ dev = [
"pytest-asyncio>=0.25.2",
"pytest-cov>=6.0.0",
"pytest-ordering>=0.6",
"pytest_httpserver >=1.1.3",
"ruff==0.9.2",
"types-deprecated>=1.2.15.20241117",
"types-requests>=2.32.0.20241016",
+11 -24
View File
@@ -19,12 +19,8 @@ from speckle_automate.schema import (
from specklepy.api import operations
from specklepy.api.client import SpeckleClient
from specklepy.core.api.inputs.model_inputs import CreateModelInput
from specklepy.core.api.inputs.version_inputs import (
CreateVersionInput,
MarkReceivedVersionInput,
)
from specklepy.core.api.inputs.version_inputs import CreateVersionInput
from specklepy.core.api.models.current import Model, Version
from specklepy.logging import metrics
from specklepy.logging.exceptions import SpeckleException
from specklepy.objects.base import Base
from specklepy.transports.memory import MemoryTransport
@@ -70,7 +66,6 @@ class AutomationContext:
if isinstance(automation_run_data, AutomationRunData)
else AutomationRunData.model_validate_json(automation_run_data)
)
metrics.set_host_app("automate")
speckle_client = SpeckleClient(
automation_run_data.speckle_server_url,
automation_run_data.speckle_server_url.startswith("https"),
@@ -105,7 +100,6 @@ class AutomationContext:
"""Receive the Speckle project version that triggered this automation run."""
# TODO: this is a quick hack to keep implementation consistency.
# Move to proper receive many versions
project_id = self.automation_run_data.project_id
version_id = self.automation_run_data.triggers[0].payload.version_id
try:
version = self.speckle_client.version.get(
@@ -115,7 +109,7 @@ class AutomationContext:
raise ValueError(
f"""Could not receive specified version.
Is your environment configured correctly?
project_id: {project_id}
project_id: {self.automation_run_data.project_id}
model_id: {self.automation_run_data.triggers[0].payload.model_id}
version_id: {self.automation_run_data.triggers[0].payload.version_id}
"""
@@ -130,13 +124,6 @@ class AutomationContext:
base = operations.receive(
version.referenced_object, self._server_transport, self._memory_transport
)
self.speckle_client.version.received(
MarkReceivedVersionInput(
version_id=version_id,
project_id=project_id,
source_application="automate_function",
)
)
# self._closure_tree = base["__closure"]
print(
f"It took {self.elapsed():.2f} seconds to receive",
@@ -493,29 +480,29 @@ class AutomationContext:
Args:
level: Result level.
category (str): A short tag for the event type.
affected_objects (Union[Base, List[Base]]): A single object, a list of
objects, or an empty list. When empty, a result case is still
appended with no object IDs (e.g. for skipped rules or version-level
messages).
affected_objects (Union[Base, List[Base]]): A single object or a list of
objects that are causing the info case.
message (Optional[str]): Optional message.
metadata: User provided metadata key value pairs
visual_overrides: Case specific 3D visual overrides.
"""
if isinstance(affected_objects, list):
if len(affected_objects) < 1:
raise ValueError(
f"Need atleast one object to report a(n) {level.value.upper()}"
)
object_list = affected_objects
else:
object_list = [affected_objects]
ids: Dict[str, Optional[str]] = {}
# When objects are provided, each must have an id (empty list allowed for
# version-level/skipped results).
for o in object_list:
if not getattr(o, "id", None):
# validate that the Base.id is not None. If its a None, throw an Exception
if not o.id:
raise Exception(
f"You can only attach {level} results to objects with an id."
)
ids[o.id] = getattr(o, "applicationId", None)
ids[o.id] = o.applicationId
print(
f"Created new {level.value.upper()}"
f" category: {category} caused by: {message}"
+4 -7
View File
@@ -18,7 +18,7 @@ def cmd_line_import() -> None:
parser.add_argument("output_path")
parser.add_argument("project_id")
parser.add_argument("version_message")
parser.add_argument("model_ingestion_id")
parser.add_argument("model_id")
# parser.add_argument("model_name")
# parser.add_argument("region_name")
@@ -32,8 +32,6 @@ def cmd_line_import() -> None:
"ifc",
)
client: SpeckleClient | None = None
try:
client = SpeckleClient(SERVER_URL, use_ssl=not SERVER_URL.startswith("http://"))
client.authenticate_with_token(TOKEN)
@@ -43,14 +41,13 @@ def cmd_line_import() -> None:
args.file_path,
project,
args.version_message,
args.model_ingestion_id,
args.model_id,
client,
)
with open(args.output_path, "w") as f:
json.dump({"success": True, "commitId": version.id}, f)
except Exception as e:
stack_trace = traceback.format_exc()
error_msg = f"IFC Importer failed with exception:\n{stack_trace}"
error_msg = f"IFC Importer failed with exception:\n{traceback.format_exc()}"
print(error_msg)
# Write error result
@@ -61,4 +58,4 @@ def cmd_line_import() -> None:
if __name__ == "__main__":
start = time.time()
cmd_line_import()
print(f"Total time (including cleanup): {(time.time() - start):.3f}s")
print(f"Total time (including cleanup): {(time.time() - start) * 1000}ms")
@@ -12,23 +12,12 @@ def data_object_to_speckle(
step_element: entity_instance,
children: list[Base],
current_storey: str | None = None,
parent_element: entity_instance | None = None,
) -> DataObject:
guid = cast(str, step_element.GlobalId)
name = cast(str, step_element.Name or guid)
properties = extract_properties(step_element)
# Add parent ID only if element's parent is also a DataObject (not a Collection)
# Collections are: IfcProject and IfcSpatialStructureElement types
if (
parent_element
and hasattr(parent_element, "GlobalId")
and not parent_element.is_a("IfcProject")
and not parent_element.is_a("IfcSpatialStructureElement")
):
properties["parentApplicationId"] = parent_element.GlobalId
# Add building storey information if available and not a building storey itself
if current_storey and not step_element.is_a("IfcBuildingStorey"):
properties["Building Storey"] = current_storey
+1 -7
View File
@@ -51,10 +51,4 @@ def open_ifc(file_path: str) -> file:
def create_geometry_iterator(ifc_file: file | sqlite) -> iterator:
GEOMETRY_LIBRARY = "hybrid-opencascade-cgal" # First OCC then fallback to CGAL
return iterator(
_create_iterator_settings(),
ifc_file,
multiprocessing.cpu_count(),
geometry_library=GEOMETRY_LIBRARY, # type: ignore
)
return iterator(_create_iterator_settings(), ifc_file, multiprocessing.cpu_count())
+8 -44
View File
@@ -22,15 +22,12 @@ from specklepy.objects import Base
from specklepy.objects.data_objects import DataObject
from specklepy.objects.models.collections.collection import Collection
from specklepy.objects.proxies import InstanceProxy
from specklepy.progress.ingestion_progress import IngestionProgressManager
@dataclass
class ImportJob:
ifc_file: file
progress: IngestionProgressManager
_render_material_manager: RenderMaterialProxyManager = field(
default_factory=lambda: RenderMaterialProxyManager()
)
@@ -42,19 +39,14 @@ class ImportJob:
)
geometries_count: int = 0
geometries_used: int = 0
elements_converted: int = 0
_current_storey_data_object: DataObject | None = field(default=None, init=False)
_display_value_cache: dict[int, list[Base]] = field(default_factory=dict)
"""Maps an instance step ID to a list of instances"""
def convert_element(
self,
step_element: entity_instance,
parent_element: entity_instance | None = None,
) -> Base:
def convert_element(self, step_element: entity_instance) -> Base:
try:
return self._convert_element(step_element, parent_element)
return self._convert_element(step_element)
except SpeckleException:
raise
except Exception as ex:
@@ -62,18 +54,14 @@ class ImportJob:
f"Failed to convert {step_element.is_a()} #{step_element.id()}"
) from ex
def _convert_element(
self,
step_element: entity_instance,
parent_element: entity_instance | None = None,
) -> Base:
def _convert_element(self, step_element: entity_instance) -> Base:
# Track current storey context and store for level proxies
previous_storey_data_object = self._current_storey_data_object
if step_element.is_a("IfcBuildingStorey"):
# Convert the building storey to a DataObject for the level proxy
storey_display_value = self._display_value_cache.get(step_element.id(), [])
self._current_storey_data_object = data_object_to_speckle(
storey_display_value, step_element, [], parent_element=None
storey_display_value, step_element, []
)
children = self._convert_children(step_element)
@@ -98,11 +86,7 @@ class ImportJob:
)
else:
result = data_object_to_speckle(
display_value,
step_element,
children,
current_storey_name,
parent_element,
display_value, step_element, children, current_storey_name
)
# Associate non-spatial elements with current storey for level proxies
if self._current_storey_data_object is not None and result.applicationId:
@@ -112,17 +96,11 @@ class ImportJob:
# Restore previous storey context
self._current_storey_data_object = previous_storey_data_object
self.elements_converted += 1
if self.progress.should_report_progress():
self.progress.report(
f"Converted {self.elements_converted:,} elements", None
)
return result
def _convert_children(self, step_element: entity_instance) -> list[Base]:
return [
self.convert_element(i, parent_element=step_element)
self.convert_element(i)
for i in get_children(step_element)
if self._should_convert(i)
]
@@ -142,16 +120,12 @@ class ImportJob:
def convert(self) -> Base:
start = time.time()
self.pre_process_geometry()
print(
f"Geometry conversion complete after {(time.time() - start):.3f}s" # noqa: E501
)
print(f"Geometry conversion complete after {(time.time() - start) * 1000}ms")
print(f"Created {self.geometries_count} geometries")
start = time.time()
root = self._convert_project_tree()
print(
f"Element tree conversion complete after {(time.time() - start):.3f}s" # noqa: E501
)
print(f"Object tree conversion complete after {(time.time() - start) * 1000}ms")
print(f"Used {self.geometries_used} geometries")
return root
@@ -159,10 +133,7 @@ class ImportJob:
iterator = create_geometry_iterator(self.ifc_file)
if not iterator.initialize():
raise SpeckleException("Failed to find any geometry in file")
self.progress.report("Converting geometries", None)
self.geometries_count = 0
while True:
shape = cast(TriangulationElement, iterator.get())
self.geometries_count += 1
@@ -174,11 +145,6 @@ class ImportJob:
raise SpeckleException(
f"Failed to convert geometry with id: {id}"
) from ex
if self.progress.should_report_progress():
self.progress.report(
f"Converted {self.geometries_count:,} geometries", None
)
if not iterator.next():
break
@@ -219,8 +185,6 @@ class ImportJob:
raise SpeckleException("Expected exactly one IfcProject in file")
project = projects[0]
self.progress.report("Converting elements", None)
tree = self.convert_element(project)
if not isinstance(tree, Collection):
raise TypeError("Expected root object to convert to a Collection")
+33 -105
View File
@@ -1,133 +1,61 @@
import contextlib
import importlib.metadata
import time
import traceback
from pathlib import Path
from speckleifc.ifc_geometry_processing import open_ifc
from speckleifc.importer import ImportJob
from specklepy.core.api.client import SpeckleClient
from specklepy.core.api.inputs.model_ingestion_inputs import (
ModelIngestionFailedInput,
ModelIngestionStartProcessingInput,
ModelIngestionSuccessInput,
SourceDataInput,
)
from specklepy.core.api.inputs.version_inputs import CreateVersionInput
from specklepy.core.api.models.current import Project, Version
from specklepy.core.api.operations import send
from specklepy.logging import metrics
from specklepy.progress.ingestion_progress import IngestionProgressManager
from specklepy.progress.progress_transport import ProgressTransport
from specklepy.transports.server import ServerTransport
# Since progress messages are currently blocking (no async), we're being extra coarse
# with progress updates to ensure we're not waisting time sending updates.
# We could maybe go a little lower, but for now I'm not risking degrading performance
PROGRESS_INTERVAL_SECONDS = 10
def open_and_convert_file(
file_path: str,
project: Project,
version_message: str | None,
model_ingestion_id: str,
model_id: str,
client: SpeckleClient,
) -> Version:
try:
start = time.time()
very_start = start
path = Path(file_path)
start = time.time()
very_start = start
specklepy_version = importlib.metadata.version("specklepy")
ingestion = client.model_ingestion.start_processing(
ModelIngestionStartProcessingInput(
project_id=project.id,
ingestion_id=model_ingestion_id,
progress_message="Importing IFC file",
source_data=SourceDataInput(
file_name=path.name,
file_size_bytes=path.stat().st_size,
source_application_slug=metrics.HOST_APP,
source_application_version=specklepy_version,
),
)
)
progress = IngestionProgressManager(
client, ingestion, PROGRESS_INTERVAL_SECONDS
)
account = client.account
server_url = account.serverInfo.url
assert server_url
remote_transport = ServerTransport(project.id, account=account)
progress_transport = ProgressTransport(
progress,
)
account = client.account
server_url = account.serverInfo.url
assert server_url
remote_transport = ServerTransport(project.id, account=account)
progress.report("Opening file", None)
ifc_file = open_ifc(file_path) # pyright: ignore[reportUnknownVariableType]
ifc_file = open_ifc(file_path) # pyright: ignore[reportUnknownVariableType]
import_job = ImportJob(ifc_file, progress) # pyright: ignore[reportUnknownArgumentType]
data = import_job.convert()
import_job = ImportJob(ifc_file) # pyright: ignore[reportUnknownArgumentType]
data = import_job.convert()
print(
f"File conversion complete after {(time.time() - start):.3f}s" # noqa: E501
)
print(f"File conversion complete after {(time.time() - start) * 1000}ms")
start = time.time()
start = time.time()
progress.report("Uploading objects", None)
root_id = send(
data,
transports=[remote_transport, progress_transport],
use_default_cache=False,
)
print(
f"Sending to speckle complete after: {(time.time() - start):.3f}s" # noqa: E501
)
root_id = send(data, transports=[remote_transport], use_default_cache=False)
print(f"Sending to speckle complete after: {(time.time() - start) * 1000}ms")
start = time.time()
start = time.time()
version_id = client.model_ingestion.complete(
ModelIngestionSuccessInput(
project_id=project.id,
ingestion_id=model_ingestion_id,
root_object_id=root_id,
version_message=version_message,
)
)
create_version = CreateVersionInput(
object_id=root_id,
model_id=model_id,
project_id=project.id,
message=version_message,
source_application="ifc",
)
version = client.version.create(create_version)
end = time.time()
print(f"Version committed after: {(end - start) * 1000}ms")
# needed to query version until ingestion api expands to serve it
version = client.version.get(version_id, project.id)
print(f"Total time (to commit): {(end - very_start) * 1000}ms")
del ifc_file
end = time.time()
print(f"Version committed after: {(end - start):.3f}s")
custom_properties = {"ui": "dui3", "actionSource": "import"}
if project.workspace_id:
custom_properties["workspace_id"] = project.workspace_id
metrics.track(metrics.SEND, account, custom_properties, send_sync=True)
print(f"Total time (to commit): {(end - very_start):.3f}s")
del ifc_file
custom_properties = {"ui": "dui3", "actionSource": "import"}
if project.workspace_id:
custom_properties["workspace_id"] = project.workspace_id
metrics.track(
metrics.SEND,
account,
custom_properties,
send_sync=True,
track_email=True,
)
return version
except Exception as e:
stack_trace = traceback.format_exc()
with contextlib.suppress(Exception):
# make sure to not report process kills when we're cancelling
client.model_ingestion.fail_with_error(
ModelIngestionFailedInput(
project_id=project.id,
ingestion_id=model_ingestion_id,
error_reason=str(e),
error_stacktrace=stack_trace,
)
)
raise e
return version
-51
View File
@@ -1,51 +0,0 @@
import time
from speckleifc.main import open_and_convert_file
from specklepy.api.client import SpeckleClient
from specklepy.core.api.credentials import get_accounts_for_server
from specklepy.logging import metrics
def _manual_import() -> None:
from specklepy.core.api.inputs.model_ingestion_inputs import (
ModelIngestionCreateInput,
SourceDataInput,
)
PROJECT_ID = "412a3c3927"
MODEL_ID = "223e61212d"
SERVER_URL = "latest.speckle.systems"
FILE_PATH = r"C:\Test Files\ifc\AC20-FZK-Haus.ifc" # noqa: E501
metrics.set_host_app(
"ifc",
)
account = get_accounts_for_server(SERVER_URL)[0]
client = SpeckleClient(SERVER_URL, use_ssl=not SERVER_URL.startswith("http://"))
client.authenticate_with_account(account)
ingestion = client.model_ingestion.create(
ModelIngestionCreateInput(
model_id=MODEL_ID,
project_id=PROJECT_ID,
progress_message="",
source_data=SourceDataInput(
source_application_slug="speckleifc",
source_application_version="0.0.0",
file_name=None,
file_size_bytes=None,
),
max_idle_timeout_seconds=2700, # 45mins
)
)
project = client.project.get(PROJECT_ID)
open_and_convert_file(FILE_PATH, project, None, ingestion.id, client)
if __name__ == "__main__":
start = time.time()
_manual_import()
print(f"Total time (including cleanup): {(time.time() - start):.3f}s")
-7
View File
@@ -4,7 +4,6 @@ from specklepy.api.credentials import Account
from specklepy.api.resources import (
ActiveUserResource,
FileImportResource,
ModelIngestionResource,
ModelResource,
OtherUserResource,
ProjectInviteResource,
@@ -120,12 +119,6 @@ class SpeckleClient(CoreSpeckleClient):
client=self.httpclient,
server_version=server_version,
)
self.model_ingestion = ModelIngestionResource(
account=self.account,
basepath=self.url,
client=self.httpclient,
server_version=server_version,
)
self.file_import = FileImportResource(
account=self.account,
basepath=self.url,
-5
View File
@@ -1,8 +1,5 @@
from specklepy.api.resources.current.active_user_resource import ActiveUserResource
from specklepy.api.resources.current.file_import_resource import FileImportResource
from specklepy.api.resources.current.model_ingestion_resource import (
ModelIngestionResource,
)
from specklepy.api.resources.current.model_resource import ModelResource
from specklepy.api.resources.current.other_user_resource import OtherUserResource
from specklepy.api.resources.current.project_invite_resource import (
@@ -25,6 +22,4 @@ __all__ = [
"SubscriptionResource",
"VersionResource",
"WorkspaceResource",
"FileImportResource",
"ModelIngestionResource",
]
@@ -15,7 +15,7 @@ from specklepy.logging import metrics
class FileImportResource(CoreResource):
"""API Access class for file imports"""
"""API Access class for projects"""
def __init__(self, account, basepath, client, server_version) -> None:
super().__init__(
@@ -1,57 +0,0 @@
from specklepy.core.api.inputs.model_ingestion_inputs import (
ModelIngestionCancelledInput,
ModelIngestionCreateInput,
ModelIngestionFailedInput,
ModelIngestionRequeueInput,
ModelIngestionStartProcessingInput,
ModelIngestionSuccessInput,
ModelIngestionUpdateInput,
)
from specklepy.core.api.models.current import (
ModelIngestion,
)
from specklepy.core.api.resources import (
ModelIngestionResource as CoreResource,
)
from specklepy.logging import metrics
class ModelIngestionResource(CoreResource):
"""API Access class for model ingestion"""
def __init__(self, account, basepath, client, server_version) -> None:
super().__init__(account, basepath, client, server_version)
def create(self, input: ModelIngestionCreateInput) -> ModelIngestion:
metrics.track(metrics.SDK, self.account, {"name": "Ingestion Create"})
return super().create(input)
def get_ingestion(self, project_id: str, model_ingestion_id: str) -> ModelIngestion:
metrics.track(metrics.SDK, self.account, {"name": "Ingestion Get"})
return super().get_ingestion(project_id, model_ingestion_id)
def update_progress(self, input: ModelIngestionUpdateInput) -> ModelIngestion:
metrics.track(metrics.SDK, self.account, {"name": "Ingestion Update"})
return super().update_progress(input)
def start_processing(
self, input: ModelIngestionStartProcessingInput
) -> ModelIngestion:
metrics.track(metrics.SDK, self.account, {"name": "Ingestion Start Processing"})
return super().start_processing(input)
def requeue(self, input: ModelIngestionRequeueInput) -> ModelIngestion:
metrics.track(metrics.SDK, self.account, {"name": "Ingestion Update"})
return super().requeue(input)
def complete(self, input: ModelIngestionSuccessInput) -> str:
metrics.track(metrics.SDK, self.account, {"name": "Ingestion End"})
return super().complete(input)
def fail_with_error(self, input: ModelIngestionFailedInput) -> ModelIngestion:
metrics.track(metrics.SDK, self.account, {"name": "Ingestion Error"})
return super().fail_with_error(input)
def fail_with_cancel(self, input: ModelIngestionCancelledInput) -> ModelIngestion:
metrics.track(metrics.SDK, self.account, {"name": "Ingestion Cancel"})
return super().fail_with_cancel(input)
@@ -8,10 +8,6 @@ from specklepy.core.api.inputs.model_inputs import (
)
from specklepy.core.api.inputs.project_inputs import ProjectModelsFilter
from specklepy.core.api.models import Model, ModelWithVersions, ResourceCollection
from specklepy.core.api.models.current import (
ModelPermissionChecks,
PermissionCheckResult,
)
from specklepy.core.api.resources import ModelResource as CoreResource
from specklepy.logging import metrics
@@ -76,17 +72,3 @@ class ModelResource(CoreResource):
def update(self, input: UpdateModelInput) -> Model:
metrics.track(metrics.SDK, self.account, {"name": "Model Update"})
return super().update(input)
def get_permissions(self, model_id: str, project_id: str) -> ModelPermissionChecks:
metrics.track(metrics.SDK, self.account, {"name": "Model Get Permissions"})
return super().get_permissions(model_id, project_id)
def can_create_model_ingestion(
self, model_id: str, project_id: str
) -> PermissionCheckResult:
metrics.track(
metrics.SDK,
self.account,
{"name": "Model Get Permissions canCreateIngestion"},
)
return super().can_create_model_ingestion(model_id, project_id)
-7
View File
@@ -12,7 +12,6 @@ from specklepy.core.api.credentials import Account
from specklepy.core.api.resources import (
ActiveUserResource,
FileImportResource,
ModelIngestionResource,
ModelResource,
OtherUserResource,
ProjectInviteResource,
@@ -251,12 +250,6 @@ class SpeckleClient:
client=self.httpclient,
server_version=server_version,
)
self.model_ingestion = ModelIngestionResource(
account=self.account,
basepath=self.url,
client=self.httpclient,
server_version=server_version,
)
self.subscription = SubscriptionResource(
account=self.account,
basepath=self.ws_url,
-16
View File
@@ -7,7 +7,6 @@ class ProjectVisibility(str, Enum):
PRIVATE = "PRIVATE"
PUBLIC = "PUBLIC"
UNLISTED = "UNLISTED"
"""Deprecated, use PUBLIC instead"""
WORKSPACE = "WORKSPACE"
@@ -31,18 +30,3 @@ class ProjectVersionsUpdatedMessageType(str, Enum):
CREATED = "CREATED"
DELETED = "DELETED"
UPDATED = "UPDATED"
class ProjectModelIngestionUpdatedMessageType(str, Enum):
CANCELLATION_REQUESTED = "cancellationRequested"
CREATED = "created"
DELETED = "deleted"
UPDATED = "updated"
class ModelIngestionStatus(str, Enum):
CANCELLED = "cancelled"
FAILED = "failed"
PROCESSING = "processing"
QUEUED = "queued"
SUCCESS = "success"
@@ -1,78 +0,0 @@
from specklepy.core.api.enums import ProjectModelIngestionUpdatedMessageType
from specklepy.core.api.models.graphql_base_model import GraphQLBaseModel
class SourceDataInput(GraphQLBaseModel):
source_application_slug: str
source_application_version: str
file_name: str | None
file_size_bytes: int | None
class ModelIngestionCreateInput(GraphQLBaseModel):
model_id: str
project_id: str
progress_message: str
source_data: SourceDataInput
max_idle_timeout_seconds: int | None = None
class ModelIngestionStartProcessingInput(GraphQLBaseModel):
ingestion_id: str
project_id: str
progress_message: str
source_data: SourceDataInput
class ModelIngestionRequeueInput(GraphQLBaseModel):
ingestion_id: str
project_id: str
progress_message: str
class ModelIngestionUpdateInput(GraphQLBaseModel):
ingestion_id: str
project_id: str
progress: float | None
progress_message: str
class ModelIngestionSuccessInput(GraphQLBaseModel):
ingestion_id: str
project_id: str
root_object_id: str
version_message: str | None
class ModelIngestionFailedInput(GraphQLBaseModel):
ingestion_id: str
project_id: str
error_reason: str
error_stacktrace: str | None
class ModelIngestionCancelledInput(GraphQLBaseModel):
ingestion_id: str
project_id: str
cancellation_message: str
class ModelIngestionRequestCancellationInput(GraphQLBaseModel):
ingestion_id: str
project_id: str
cancellation_message: str
class ModelIngestionReference(GraphQLBaseModel):
"""
`@oneOf` i.e. server expects **either** `ingestion_id` or `model_id`, but not both.
"""
ingestion_id: str | None
model_id: str | None
class ProjectModelIngestionSubscriptionInput(GraphQLBaseModel):
project_id: str
ingestion_reference: ModelIngestionReference
message_type: ProjectModelIngestionUpdatedMessageType | None = None
@@ -34,8 +34,4 @@ class MarkReceivedVersionInput(GraphQLBaseModel):
version_id: str
project_id: str
source_application: str
"""
IMPORTANT: this is meant to be the slug of the application that has done the
receiving, not to be confused with `Version.sourceApplication`
"""
message: Optional[str] = None
+3 -25
View File
@@ -1,7 +1,7 @@
from datetime import datetime
from typing import Generic, List, TypeVar
from specklepy.core.api.enums import ModelIngestionStatus, ProjectVisibility
from specklepy.core.api.enums import ProjectVisibility
from specklepy.core.api.models.graphql_base_model import GraphQLBaseModel
from specklepy.logging.exceptions import WorkspacePermissionException
@@ -105,7 +105,7 @@ class PendingStreamCollaborator(GraphQLBaseModel):
project_name: str
title: str
role: str
invited_by: LimitedUser | None = None
invited_by: LimitedUser
user: LimitedUser | None = None
token: str | None
@@ -137,12 +137,6 @@ class Version(GraphQLBaseModel):
source_application: str | None
class ModelPermissionChecks(GraphQLBaseModel):
can_update: "PermissionCheckResult"
can_delete: "PermissionCheckResult"
can_create_version: "PermissionCheckResult"
class Model(GraphQLBaseModel):
author: LimitedUser | None
created_at: datetime
@@ -162,6 +156,7 @@ class ProjectPermissionChecks(GraphQLBaseModel):
can_create_model: "PermissionCheckResult"
can_delete: "PermissionCheckResult"
can_load: "PermissionCheckResult"
can_publish: "PermissionCheckResult"
class Project(GraphQLBaseModel):
@@ -249,20 +244,3 @@ class FileImport(GraphQLBaseModel):
class FileUploadUrl(GraphQLBaseModel):
url: str
file_id: str
class ModelIngestionStatusData(GraphQLBaseModel):
status: ModelIngestionStatus
progress_message: str | None = None
version_id: str | None = None
class ModelIngestion(GraphQLBaseModel):
id: str
created_at: datetime
updated_at: datetime
model_id: str
project_id: str
user_id: str
cancellation_requested: bool
status_data: ModelIngestionStatusData
@@ -1,13 +1,12 @@
from typing import Optional
from specklepy.core.api.enums import (
ProjectModelIngestionUpdatedMessageType,
ProjectModelsUpdatedMessageType,
ProjectUpdatedMessageType,
ProjectVersionsUpdatedMessageType,
UserProjectsUpdatedMessageType,
)
from specklepy.core.api.models.current import Model, ModelIngestion, Project, Version
from specklepy.core.api.models.current import Model, Project, Version
from specklepy.core.api.models.graphql_base_model import GraphQLBaseModel
@@ -34,8 +33,3 @@ class ProjectVersionsUpdatedMessage(GraphQLBaseModel):
type: ProjectVersionsUpdatedMessageType
model_id: str
version: Optional[Version]
class ProjectModelIngestionUpdatedMessage(GraphQLBaseModel):
model_ingestion: ModelIngestion
type: ProjectModelIngestionUpdatedMessageType
@@ -1,8 +1,5 @@
from specklepy.core.api.resources.current.active_user_resource import ActiveUserResource
from specklepy.core.api.resources.current.file_import_resource import FileImportResource
from specklepy.core.api.resources.current.model_ingestion_resource import (
ModelIngestionResource,
)
from specklepy.core.api.resources.current.model_resource import ModelResource
from specklepy.core.api.resources.current.other_user_resource import OtherUserResource
from specklepy.core.api.resources.current.project_invite_resource import (
@@ -27,6 +24,4 @@ __all__ = [
"SubscriptionResource",
"VersionResource",
"WorkspaceResource",
"FileImportResource",
"ModelIngestionResource",
]
@@ -16,15 +16,13 @@ from specklepy.core.api.resource import ResourceBase
from specklepy.core.api.responses import DataResponse
from specklepy.logging.exceptions import SpeckleException
NAME = "file_import"
class UploadFileResponse(GraphQLBaseModel):
etag: str
class FileImportResource(ResourceBase):
"""API Access class for file imports"""
"""API Access class for project invites"""
def __init__(
self,
@@ -38,7 +36,7 @@ class FileImportResource(ResourceBase):
basepath=basepath,
client=client,
server_version=server_version,
name=NAME,
name="file-import",
)
def finish_file_import_job(self, input: FinishFileImportInput) -> bool:
@@ -1,417 +0,0 @@
from typing import Any, Tuple
from gql import Client, gql
from specklepy.api.credentials import Account
from specklepy.core.api.inputs.model_ingestion_inputs import (
ModelIngestionCancelledInput,
ModelIngestionCreateInput,
ModelIngestionFailedInput,
ModelIngestionRequestCancellationInput,
ModelIngestionRequeueInput,
ModelIngestionStartProcessingInput,
ModelIngestionSuccessInput,
ModelIngestionUpdateInput,
)
from specklepy.core.api.models.current import (
ModelIngestion,
)
from specklepy.core.api.resource import ResourceBase
from specklepy.core.api.responses import DataResponse
NAME = "ingestion"
class ModelIngestionResource(ResourceBase):
"""API Access class for model ingestion"""
def __init__(
self,
account: Account,
basepath: str,
client: Client,
server_version: Tuple[Any, ...] | None,
) -> None:
super().__init__(
account=account,
basepath=basepath,
client=client,
name=NAME,
server_version=server_version,
)
def get_ingestion(self, project_id: str, model_ingestion_id: str) -> ModelIngestion:
QUERY = gql(
"""
query Query($projectId: String!, $modelIngestionId: ID!) {
data:project(id: $projectId) {
data:ingestion(id: $modelIngestionId) {
id
createdAt
updatedAt
modelId
projectId
userId
cancellationRequested
statusData {
... on HasModelIngestionStatus {
status
}
... on HasProgressMessage {
progressMessage
}
... on ModelIngestionSuccessStatus
{
versionId
}
}
}
}
}
""" # noqa: E501
)
variables = {
"projectId": project_id,
"modelIngestionId": model_ingestion_id,
}
return self.make_request_and_parse_response(
DataResponse[DataResponse[ModelIngestion]],
QUERY,
variables,
).data.data
def create(self, input: ModelIngestionCreateInput) -> ModelIngestion:
QUERY = gql(
"""
mutation IngestionCreate($input: ModelIngestionCreateInput!) {
data: projectMutations {
data: modelIngestionMutations {
data: create(input: $input) {
id
createdAt
updatedAt
modelId
projectId
userId
cancellationRequested
statusData {
... on HasModelIngestionStatus {
status
}
... on HasProgressMessage {
progressMessage
}
}
}
}
}
}
"""
)
variables = {
"input": input.model_dump(warnings="error", by_alias=True),
}
return self.make_request_and_parse_response(
DataResponse[DataResponse[DataResponse[ModelIngestion]]], QUERY, variables
).data.data.data
def start_processing(
self, input: ModelIngestionStartProcessingInput
) -> ModelIngestion:
QUERY = gql(
"""
mutation IngestionStartProcessing($input: ModelIngestionStartProcessingInput!) {
data: projectMutations {
data: modelIngestionMutations {
data: startProcessing(input: $input) {
id
createdAt
updatedAt
modelId
projectId
userId
cancellationRequested
statusData {
... on HasModelIngestionStatus {
status
}
... on HasProgressMessage {
progressMessage
}
}
}
}
}
}
""" # noqa: E501
)
variables = {
"input": input.model_dump(warnings="error", by_alias=True),
}
return self.make_request_and_parse_response(
DataResponse[DataResponse[DataResponse[ModelIngestion]]], QUERY, variables
).data.data.data
def requeue(self, input: ModelIngestionRequeueInput) -> ModelIngestion:
QUERY = gql(
"""
mutation IngestionRequeue($input: ModelIngestionRequeueInput!) {
data: projectMutations {
data: modelIngestionMutations {
data: requeue(input: $input) {
id
createdAt
updatedAt
modelId
projectId
userId
cancellationRequested
statusData {
... on HasModelIngestionStatus {
status
}
... on HasProgressMessage {
progressMessage
}
}
}
}
}
}
""" # noqa: E501
)
variables = {
"input": input.model_dump(warnings="error", by_alias=True),
}
return self.make_request_and_parse_response(
DataResponse[DataResponse[DataResponse[ModelIngestion]]], QUERY, variables
).data.data.data
def update_progress(self, input: ModelIngestionUpdateInput) -> ModelIngestion:
QUERY = gql(
"""
mutation IngestionUpdateProgress(
$input: ModelIngestionUpdateInput!
) {
data: projectMutations {
data: modelIngestionMutations {
data: updateProgress(input: $input) {
id
createdAt
updatedAt
modelId
projectId
userId
cancellationRequested
statusData {
... on HasModelIngestionStatus {
status
}
... on HasProgressMessage {
progressMessage
}
}
}
}
}
}
"""
)
variables = {
"input": input.model_dump(warnings="error", by_alias=True),
}
return self.make_request_and_parse_response(
DataResponse[DataResponse[DataResponse[ModelIngestion]]], QUERY, variables
).data.data.data
def complete(self, input: ModelIngestionSuccessInput) -> str:
"""
Request that the server completes the ingestion by creating a version
If successful, the job will be in a terminal "successful" state.
For failed Ingestions, use `fail_with_error` instead
For user cancellation, use `fail_with_cancelled` instead
Arguments:
input {ModelIngestionSuccessInput} -- input variable
Returns:
str -- the id of the version that was just created to complete the ingestion
"""
QUERY = gql(
"""
mutation IngestionComplete($input: ModelIngestionSuccessInput!) {
data: projectMutations {
data: modelIngestionMutations {
data: completeWithVersion(input: $input) {
data:statusData {
... on ModelIngestionSuccessStatus {
data:versionId
}
}
}
}
}
}
"""
)
variables = {
"input": input.model_dump(warnings="error", by_alias=True),
}
return self.make_request_and_parse_response(
DataResponse[DataResponse[DataResponse[DataResponse[DataResponse[str]]]]],
QUERY,
variables,
).data.data.data.data.data
def fail_with_error(self, input: ModelIngestionFailedInput) -> ModelIngestion:
"""
Fail the job with an error.
For user requested cancellation, use `fail_with_cancelled` instead
"""
QUERY = gql(
"""
mutation IngestionFailWithError($input: ModelIngestionFailedInput!) {
data: projectMutations {
data: modelIngestionMutations {
data: failWithError(input: $input) {
id
createdAt
updatedAt
modelId
projectId
userId
cancellationRequested
statusData {
... on HasModelIngestionStatus {
status
}
... on HasProgressMessage {
progressMessage
}
}
}
}
}
}
"""
)
variables = {
"input": input.model_dump(warnings="error", by_alias=True),
}
return self.make_request_and_parse_response(
DataResponse[DataResponse[DataResponse[ModelIngestion]]],
QUERY,
variables,
).data.data.data
def fail_with_cancel(self, input: ModelIngestionCancelledInput) -> ModelIngestion:
"""
Fail the ingestion with a `cancelled` status.
This should only be done if the user has explicitly requested cancellation
Other forms of cancellation use `fail_with_error`
The ingestion should then enter a terminal "canceled" state
"""
QUERY = gql(
"""
mutation IngestionFailWithCancel($input: ModelIngestionCancelledInput!) {
data: projectMutations {
data: modelIngestionMutations {
data: failWithCancel(input: $input) {
id
createdAt
updatedAt
modelId
projectId
userId
cancellationRequested
statusData {
... on HasModelIngestionStatus {
status
}
... on HasProgressMessage {
progressMessage
}
}
}
}
}
}
"""
)
variables = {
"input": input.model_dump(warnings="error", by_alias=True),
}
return self.make_request_and_parse_response(
DataResponse[DataResponse[DataResponse[ModelIngestion]]],
QUERY,
variables,
).data.data.data
def request_cancellation(
self, input: ModelIngestionRequestCancellationInput
) -> ModelIngestion:
"""
Request that the ingestion is canceled.
Note: simply calling this mutation does not immediately cancel,
it doesn't even guarantee it will be canceled at all.
It's up to the client to observe this cancellation request
via `subscription.project_model_ingestion_cancellation_requested`
and report it as cancelled (via `ingestion.fail_with_cancel`
See "cooperative cancellation pattern"
"""
QUERY = gql(
"""
mutation IngestionRequestCancellation($input: ModelIngestionRequestCancellationInput!) {
data: projectMutations {
data: modelIngestionMutations {
data: requestCancellation (input: $input) {
id
createdAt
updatedAt
modelId
projectId
userId
cancellationRequested
statusData {
... on HasModelIngestionStatus {
status
}
... on HasProgressMessage {
progressMessage
}
}
}
}
}
}
""" # noqa: E501
)
variables = {
"input": input.model_dump(warnings="error", by_alias=True),
}
return self.make_request_and_parse_response(
DataResponse[DataResponse[DataResponse[ModelIngestion]]],
QUERY,
variables,
).data.data.data
@@ -10,10 +10,6 @@ from specklepy.core.api.inputs.model_inputs import (
)
from specklepy.core.api.inputs.project_inputs import ProjectModelsFilter
from specklepy.core.api.models import Model, ModelWithVersions, ResourceCollection
from specklepy.core.api.models.current import (
ModelPermissionChecks,
PermissionCheckResult,
)
from specklepy.core.api.resource import ResourceBase
from specklepy.core.api.responses import DataResponse
@@ -303,71 +299,3 @@ class ModelResource(ResourceBase):
return self.make_request_and_parse_response(
DataResponse[DataResponse[Model]], QUERY, variables
).data.data
def get_permissions(self, project_id: str, model_id: str) -> ModelPermissionChecks:
QUERY = gql(
"""
query ModelPermissions($projectId: String!, $modelId: String!) {
data:project(id: $projectId) {
data:model(id: $modelId) {
data:permissions {
canUpdate {
authorized
code
message
}
canDelete {
authorized
code
message
}
canCreateVersion {
authorized
code
message
}
}
}
}
}
"""
)
variables = {"projectId": project_id, "modelId": model_id}
return self.make_request_and_parse_response(
DataResponse[DataResponse[DataResponse[ModelPermissionChecks]]],
QUERY,
variables,
).data.data.data
def can_create_model_ingestion(
self, project_id: str, model_id: str
) -> PermissionCheckResult:
QUERY = gql(
"""
query ModelPermissions($projectId: String!, $modelId: String!) {
data:project(id: $projectId) {
data:model(id: $modelId) {
data:permissions {
data:canCreateIngestion {
authorized
code
message
}
}
}
}
}
"""
)
variables = {"projectId": project_id, "modelId": model_id}
return self.make_request_and_parse_response(
DataResponse[
DataResponse[DataResponse[DataResponse[PermissionCheckResult]]]
],
QUERY,
variables,
).data.data.data.data
@@ -6,25 +6,17 @@ from graphql import DocumentNode
from pydantic import BaseModel
from typing_extensions import TypeVar
from specklepy.core.api.enums import ProjectModelIngestionUpdatedMessageType
from specklepy.core.api.inputs.model_ingestion_inputs import (
ModelIngestionReference,
ProjectModelIngestionSubscriptionInput,
)
from specklepy.core.api.models import (
ProjectModelsUpdatedMessage,
ProjectUpdatedMessage,
ProjectVersionsUpdatedMessage,
UserProjectsUpdatedMessage,
)
from specklepy.core.api.models.subscription_messages import (
ProjectModelIngestionUpdatedMessage,
)
from specklepy.core.api.resource import ResourceBase
from specklepy.core.api.responses import DataResponse
from specklepy.logging.exceptions import SpeckleException
NAME = "subscription"
NAME = "subscribe"
TEventArgs = TypeVar("TEventArgs", bound=BaseModel)
@@ -210,72 +202,6 @@ class SubscriptionResource(ResourceBase):
callback=lambda d: callback(d.data),
)
async def project_model_ingestion_updated(
self,
callback: Callable[[ProjectModelIngestionUpdatedMessage], None],
input: ProjectModelIngestionSubscriptionInput,
) -> None:
QUERY = gql(
"""
subscription IngestionUpdated($input: ProjectModelIngestionSubscriptionInput!) {
data: projectModelIngestionUpdated(input: $input) {
modelIngestion {
id
createdAt
updatedAt
modelId
projectId
userId
cancellationRequested
statusData {
... on HasModelIngestionStatus {
status
}
... on HasProgressMessage {
progressMessage
}
... on ModelIngestionSuccessStatus
{
versionId
}
}
}
type
}
}
""" # noqa: E501
)
variables = {
"input": input.model_dump(
warnings="error", by_alias=True, exclude_none=True
),
}
await self.subscribe_2(
DataResponse[ProjectModelIngestionUpdatedMessage],
QUERY,
variables,
callback=lambda d: callback(d.data),
)
async def project_model_ingestion_cancellation_requested(
self,
callback: Callable[[ProjectModelIngestionUpdatedMessage], None],
project_id: str,
ingestion_id: str,
) -> None:
await self.project_model_ingestion_updated(
callback,
ProjectModelIngestionSubscriptionInput(
project_id=project_id,
ingestion_reference=ModelIngestionReference(
ingestion_id=ingestion_id, model_id=None
),
message_type=ProjectModelIngestionUpdatedMessageType.CANCELLATION_REQUESTED,
),
)
@check_wsclient
async def subscribe_2(
self,
@@ -14,7 +14,7 @@ from specklepy.core.api.models import ResourceCollection, Version
from specklepy.core.api.resource import ResourceBase
from specklepy.core.api.responses import DataResponse
NAME = "version"
NAME = "model"
class VersionResource(ResourceBase):
@@ -16,7 +16,7 @@ NAME = "workspace"
class WorkspaceResource(ResourceBase):
"""API Access class for workspaces"""
"""API Access class for models"""
def __init__(self, account, basepath, client, server_version) -> None:
super().__init__(
+7 -41
View File
@@ -1,20 +1,19 @@
import contextlib
import getpass
import hashlib
import importlib.metadata
import logging
import platform
import queue
import sys
import threading
from typing import Any, Literal
from typing import Any
import requests
from specklepy.core.api.credentials import Account
"""
Lightweight usage telemetry to help us understand how to make a better Speckle.
Anonymous telemetry to help us understand how to make a better Speckle.
This really helps us to deliver a better open source project and product!
"""
TRACK = True
@@ -23,14 +22,13 @@ HOST_APP_VERSION = f"python {'.'.join(map(str, sys.version_info[:2]))}"
PLATFORMS = {"win32": "Windows", "cygwin": "Windows", "darwin": "Mac OS X"}
LOG = logging.getLogger(__name__)
METRICS_TRACKER: "MetricsTracker | None" = None
METRICS_TRACKER = None
# actions
SDK = "SDK Action"
CONNECTOR = "Connector Action"
RECEIVE = "Receive"
SEND = "Send"
ACTIONS = Literal["SDK Action", "Connector Action", "Receive", "Send"]
def disable():
@@ -50,32 +48,15 @@ def set_host_app(host_app: str, host_app_version: str | None = None):
def track(
action: ACTIONS,
action: str,
account: Account | None = None,
custom_props: dict | None = None,
send_sync: bool = False,
track_email: bool = False,
):
"""
:param action:
:type action: ACTIONS
:param account:
:type account: Account | None
:param custom_props:
:type custom_props: dict | None
:param send_sync: When `True`, the track event is executed synchronously,
and any exceptions will be raised.
When `False`, the track it is deferred to a queue, and any exceptions will be
swallowed and reported as warnings.
:type send_sync: bool
:param track_email: When `True`, the users plain text email address will be included
:type track_email: bool
"""
if not TRACK:
return
tracker = initialise_tracker(account)
event_params: dict[str, Any] = {
"event": action,
"properties": {
@@ -91,18 +72,6 @@ def track(
if custom_props:
event_params["properties"].update(custom_props)
if track_email:
event_params["properties"]["email"] = tracker.last_email
try:
specklepy_version = importlib.metadata.version("specklepy")
event_params["properties"]["core_version"] = specklepy_version
except importlib.metadata.PackageNotFoundError:
if send_sync:
raise
else:
LOG.warning("Failed to read specklepy's version number", exc_info=True)
if send_sync:
tracker.send_event(event_params)
else:
@@ -115,7 +84,7 @@ def initialise_tracker(account: Account | None = None) -> "MetricsTracker":
METRICS_TRACKER = MetricsTracker()
if account:
METRICS_TRACKER.set_last_user_email(account.userInfo.email)
METRICS_TRACKER.set_last_user(account.userInfo.email)
METRICS_TRACKER.set_last_server(account.serverInfo.url)
return METRICS_TRACKER
@@ -134,7 +103,6 @@ class MetricsTracker(metaclass=Singleton):
analytics_url: str = "https://analytics.speckle.systems/track?ip=1"
analytics_token: str = "acd87c5a50b56df91a795e999812a3a4"
last_user: str = ""
last_email: str = ""
last_server: str | None = None
platform: str
@@ -153,19 +121,17 @@ class MetricsTracker(metaclass=Singleton):
if node and user:
self.last_user = f"@{self.hash(f'{node}-{user}')}"
def set_last_user_email(self, email: str | None) -> None:
def set_last_user(self, email: str | None) -> None:
if not email:
return
self.last_user = f"@{self.hash(email)}"
self.last_email = email
def set_last_server(self, server: str | None) -> None:
if not server:
return
self.last_server = self.hash(server)
@staticmethod
def hash(value: str) -> str:
def hash(self, value: str) -> str:
inputList = value.lower().split("://")
input = inputList[len(inputList) - 1].split("/")[0].split("?")[0]
return hashlib.md5(input.encode("utf-8")).hexdigest().upper()
@@ -1,68 +0,0 @@
from time import monotonic
from specklepy.core.api.client import SpeckleClient
from specklepy.core.api.inputs.model_ingestion_inputs import ModelIngestionUpdateInput
from specklepy.core.api.models.current import ModelIngestion
class IngestionProgressManager:
"""
Provides a performant way to report ingestion progress.
Allows callers to throttle ingestion progress messages based on an update interval.
Throttling prevents callers from overwhelming the server with constant progress updates,
and minimises blocking high-speed operations with too frequent progress updates.
Callers can use this pattern for reporting throttled progress
```
if progress.should_report_progress():
progress.report(f"Converting geometries ({current:,}/{total:,})", current - total)
```
And for unthrottled progress (e.g. between phases)
```
progress.report(f"Next phases has started (0/{total:,})", 0)
This class is similar to the `IngestionProgressManager` in the .NET SDK
Unlike in .NET, we recommend using a very coarse `update_interval_seconds`; since we're not using async messages,
they are blocking and will degrade performance if used too frequently.
```
""" # noqa: E501
def __init__(
self,
speckle_client: SpeckleClient,
ingestion: ModelIngestion,
update_interval_seconds: float,
):
self.speckle_client = speckle_client
self.ingestion = ingestion
self.update_interval = update_interval_seconds
self._last_updated_at = 0.0
def report(self, progress_message: str, progress: float | None) -> ModelIngestion:
"""
Reports a progress update
"""
self._last_updated_at = monotonic()
formatted_progress = f"{progress:.0%}" if progress else ""
print(f"Progress update: {progress_message} {formatted_progress}")
return self.speckle_client.model_ingestion.update_progress(
ModelIngestionUpdateInput(
ingestion_id=self.ingestion.id,
project_id=self.ingestion.project_id,
progress_message=progress_message,
progress=progress,
)
)
def should_report_progress(self) -> bool:
"""
Returns `true` if it's time for an update,
`false` if it's too soon since the last update
"""
elapsed = monotonic() - self._last_updated_at
return elapsed >= self.update_interval
@@ -1,64 +0,0 @@
from typing import Dict, List
from specklepy.progress.ingestion_progress import IngestionProgressManager
from specklepy.transports.abstract_transport import AbstractTransport
class ProgressTransport(AbstractTransport):
"""
This transport does not persist objects anywhere,
instead it simply reacts to save_object being called,
and reports throttled progress.
"""
def __init__(
self,
progress: IngestionProgressManager,
name="Progress",
progress_message_template="Uploading objects {:,}",
) -> None:
super().__init__()
self._name = name
self._progress = progress
self._progress_message_template = progress_message_template
self.saved_object_count = 0
def _throttle_progress(self) -> None:
if self._progress.should_report_progress():
self._progress.report(
self._progress_message_template.format(self.saved_object_count), None
)
@property
def name(self) -> str:
return self._name
def __repr__(self) -> str:
return f"ProgressTransport(objects: {self.saved_object_count})"
def save_object(self, id: str, serialized_object: str) -> None:
self.saved_object_count += 1
self._throttle_progress()
def save_object_from_transport(
self, id: str, source_transport: AbstractTransport
) -> None:
self.saved_object_count += 1
self._throttle_progress()
def get_object(self, id: str) -> str | None:
return None
def has_objects(self, id_list: List[str]) -> Dict[str, bool]:
return {id: False for id in id_list}
def begin_write(self) -> None:
self.saved_object_count = 0
def end_write(self) -> None:
pass
def copy_object_and_children(
self, id: str, target_transport: AbstractTransport
) -> str:
raise NotImplementedError
@@ -50,10 +50,12 @@ class TestActiveUserResourcePermissions:
assert hasattr(permissions, "can_create_model")
assert hasattr(permissions, "can_delete")
assert hasattr(permissions, "can_load")
assert hasattr(permissions, "can_publish")
assert permissions.can_create_model.authorized is True
assert permissions.can_delete.authorized is True
assert permissions.can_load.authorized is True
assert permissions.can_publish.authorized is True
def test_active_user_get_projects_with_permissions_with_filter(
self, client: SpeckleClient, test_project: Project
@@ -1,279 +0,0 @@
from datetime import datetime
import pytest
from specklepy.api import operations
from specklepy.api.client import SpeckleClient
from specklepy.core.api.enums import ModelIngestionStatus
from specklepy.core.api.inputs.model_ingestion_inputs import (
ModelIngestionCancelledInput,
ModelIngestionCreateInput,
ModelIngestionFailedInput,
ModelIngestionRequeueInput,
ModelIngestionStartProcessingInput,
ModelIngestionSuccessInput,
ModelIngestionUpdateInput,
SourceDataInput,
)
from specklepy.core.api.inputs.model_inputs import CreateModelInput
from specklepy.core.api.inputs.project_inputs import ProjectCreateInput
from specklepy.core.api.models.current import (
Model,
ModelIngestion,
ModelIngestionStatusData,
Project,
ProjectVisibility,
Version,
)
from specklepy.logging.exceptions import GraphQLException
from specklepy.objects.base import Base
from specklepy.transports.server.server import ServerTransport
from tests.integration.conftest import is_public
@pytest.mark.run()
@pytest.mark.skipif(is_public(), reason="The public API does not support these tests")
class TestIngestionResource:
@pytest.fixture
def project(self, client: SpeckleClient) -> Project:
return client.project.create(
ProjectCreateInput(
name="test", description=None, visibility=ProjectVisibility.PUBLIC
)
)
@pytest.fixture
def model(self, client: SpeckleClient, project: Project) -> Model:
return client.model.create(
CreateModelInput(name="test", description=None, project_id=project.id)
)
@pytest.fixture
def ingestion(
self, client: SpeckleClient, model: Model, project: Project
) -> ModelIngestion:
input = ModelIngestionCreateInput(
model_id=model.id,
project_id=project.id,
progress_message="Starting processing",
source_data=SourceDataInput(
source_application_slug="pytest",
source_application_version="0.0.0",
file_name=None,
file_size_bytes=None,
),
)
ingestion = client.model_ingestion.create(input)
assert isinstance(ingestion, ModelIngestion)
assert isinstance(ingestion.id, str)
assert isinstance(ingestion.created_at, datetime)
assert isinstance(ingestion.updated_at, datetime)
assert isinstance(ingestion.cancellation_requested, bool)
assert isinstance(ingestion.model_id, str)
assert isinstance(ingestion.project_id, str)
assert isinstance(ingestion.user_id, str)
assert isinstance(ingestion.status_data, ModelIngestionStatusData)
assert isinstance(ingestion.status_data.progress_message, str | None)
assert ingestion.status_data.version_id is None
assert ingestion.status_data.status == ModelIngestionStatus.PROCESSING
assert not ingestion.cancellation_requested
assert ingestion.model_id == model.id
assert ingestion.project_id == project.id
assert ingestion.user_id == client.account.userInfo.id
return ingestion
def test_get_ingestion(
self, client: SpeckleClient, project: Project, ingestion: ModelIngestion
):
queried_ingestion = client.model_ingestion.get_ingestion(
project.id, ingestion.id
)
assert queried_ingestion.id == ingestion.id
assert queried_ingestion.status_data.status == ingestion.status_data.status
def test_update_progress(
self, client: SpeckleClient, ingestion: ModelIngestion, project: Project
):
def update(progress: float | None, message: str):
input = ModelIngestionUpdateInput(
ingestion_id=ingestion.id,
project_id=project.id,
progress=progress,
progress_message=message,
)
res = client.model_ingestion.update_progress(input)
assert isinstance(res, ModelIngestion)
assert res.status_data.progress_message == message
assert not res.cancellation_requested
assert res.status_data.status == ModelIngestionStatus.PROCESSING
update(None, "None")
update(0.1, "0.1")
update(0.5, "Whoa-oh! We're half way there!")
update(1, "Finished")
update(0.2, "Back to processing again")
def test_start_processing_and_requeue(
self, client: SpeckleClient, ingestion: ModelIngestion, project: Project
):
# just setting the baseline state
assert ingestion.status_data.status == ModelIngestionStatus.PROCESSING
def requeue(message: str):
input = ModelIngestionRequeueInput(
project_id=project.id,
ingestion_id=ingestion.id,
progress_message=message,
)
res = client.model_ingestion.requeue(input)
assert isinstance(res, ModelIngestion)
assert res.status_data.progress_message == message
assert not res.cancellation_requested
assert res.status_data.status == ModelIngestionStatus.QUEUED
def start_processing(message: str):
input = ModelIngestionStartProcessingInput(
ingestion_id=ingestion.id,
project_id=project.id,
progress_message=message,
source_data=SourceDataInput(
source_application_slug="test",
source_application_version="test",
file_name="test",
file_size_bytes=1,
),
)
res = client.model_ingestion.start_processing(input)
assert isinstance(res, ModelIngestion)
assert res.status_data.progress_message == message
assert not res.cancellation_requested
assert res.status_data.status == ModelIngestionStatus.PROCESSING
requeue("put it back in there")
start_processing("go for it")
requeue("and again")
start_processing("run forest run")
def update(progress: float | None, message: str):
input = ModelIngestionUpdateInput(
ingestion_id=ingestion.id,
project_id=project.id,
progress=progress,
progress_message=message,
)
res = client.model_ingestion.update_progress(input)
assert isinstance(res, ModelIngestion)
assert res.status_data.progress_message == message
assert not res.cancellation_requested
assert res.status_data.status == ModelIngestionStatus.PROCESSING
update(None, "None")
update(0.1, "0.1")
update(0.5, "Whoa-oh! We're half way there!")
update(1, "Finished")
update(0.2, "Back to processing again")
def test_error(
self, client: SpeckleClient, ingestion: ModelIngestion, project: Project
):
input = ModelIngestionFailedInput(
ingestion_id=ingestion.id,
project_id=project.id,
error_reason="Failed to integration test an error",
error_stacktrace="over here in test_error",
)
res = client.model_ingestion.fail_with_error(input)
assert isinstance(res, ModelIngestion)
assert res.status_data.progress_message is None
assert not res.cancellation_requested
assert res.status_data.status == ModelIngestionStatus.FAILED
# trying to fail for a second time should throw
# with pytest.raises(GraphQLException):
# _ = client.ingestion.fail_with_error(input)
def test_complete(
self, client: SpeckleClient, ingestion: ModelIngestion, project: Project
):
remote = ServerTransport(project.id, client)
object_id = operations.send(
Base(applicationId="ASDFGHJKL"), [remote], use_default_cache=False
)
input = ModelIngestionSuccessInput(
ingestion_id=ingestion.id,
root_object_id=object_id,
project_id=project.id,
version_message=None,
)
version_id = client.model_ingestion.complete(input)
res = client.model_ingestion.get_ingestion(project.id, ingestion.id)
assert isinstance(version_id, str)
version = client.version.get(version_id, project.id)
assert isinstance(version, Version)
assert res.status_data.version_id == version_id
# trying to complete for a second time should throw
# with pytest.raises(GraphQLException):
# _ = client.ingestion.complete(input)
def test_cancel(
self, client: SpeckleClient, ingestion: ModelIngestion, project: Project
):
input = ModelIngestionCancelledInput(
ingestion_id=ingestion.id,
project_id=project.id,
cancellation_message="This was cancelled for testing purposes",
)
res = client.model_ingestion.fail_with_cancel(input)
assert isinstance(res, ModelIngestion)
assert res.status_data.progress_message is None
assert not res.cancellation_requested
assert res.status_data.status == ModelIngestionStatus.CANCELLED
def test_error_non_existent_ingestion(
self, client: SpeckleClient, project: Project
):
input = ModelIngestionFailedInput(
ingestion_id="Non-existent-ingestion",
project_id=project.id,
error_reason="Failed to integration test an error",
error_stacktrace="over here in test_error",
)
with pytest.raises(GraphQLException):
_ = client.model_ingestion.fail_with_error(input)
def test_complete_failed_non_existent_ingestion(
self, client: SpeckleClient, project: Project
):
input = ModelIngestionFailedInput(
ingestion_id="Non-existent-ingestion",
project_id=project.id,
error_reason="Failed to integration test an error",
error_stacktrace="over here in test_error",
)
with pytest.raises(GraphQLException):
_ = client.model_ingestion.fail_with_error(input)
@pytest.mark.skip(reason="TEST FAILS - server behaviour was changed")
def test_complete_non_existent_root_object(
self, client: SpeckleClient, ingestion: ModelIngestion, project: Project
):
input = ModelIngestionSuccessInput(
ingestion_id=ingestion.id,
root_object_id="asdfasdfasdfasfd",
project_id=project.id,
version_message=None,
)
with pytest.raises(GraphQLException):
_ = client.model_ingestion.complete(input)
@@ -1,7 +1,6 @@
import pytest
from specklepy.api.client import SpeckleClient
from specklepy.core.api.enums import ProjectVisibility
from specklepy.core.api.inputs.model_inputs import (
CreateModelInput,
DeleteModelInput,
@@ -13,14 +12,11 @@ from specklepy.core.api.inputs.project_inputs import (
)
from specklepy.core.api.models.current import (
Model,
ModelPermissionChecks,
PermissionCheckResult,
Project,
ProjectWithModels,
ResourceCollection,
)
from specklepy.logging.exceptions import GraphQLException
from tests.integration.conftest import is_internal, is_public
@pytest.mark.run()
@@ -28,9 +24,7 @@ class TestModelResource:
@pytest.fixture()
def test_project(self, client: SpeckleClient) -> Project:
project = client.project.create(
ProjectCreateInput(
name="Test project", description="", visibility=ProjectVisibility.PUBLIC
)
ProjectCreateInput(name="Test project", description="", visibility=None)
)
return project
@@ -155,52 +149,3 @@ class TestModelResource:
with pytest.raises(GraphQLException):
client.model.delete(delete_data)
def test_model_get_permissions(
self,
client: SpeckleClient,
second_client: SpeckleClient,
test_project: Project,
test_model: Model,
):
result = client.model.get_permissions(test_project.id, test_model.id)
assert isinstance(result, ModelPermissionChecks)
assert result.can_update.authorized is True
assert result.can_create_version.authorized is True
assert result.can_delete.authorized is True
guest = second_client.model.get_permissions(test_project.id, test_model.id)
assert isinstance(guest, ModelPermissionChecks)
assert guest.can_update.authorized is False
assert guest.can_create_version.authorized is False
assert guest.can_delete.authorized is False
@pytest.mark.skipif(
is_public(), reason="API only available on server versions 3.0.11 or greater"
)
def test_can_create_model_ingestion_internal_server(
self,
client: SpeckleClient,
test_project: Project,
test_model: Model,
):
result = client.model.can_create_model_ingestion(test_project.id, test_model.id)
assert isinstance(result, PermissionCheckResult)
assert result.authorized is True
@pytest.mark.skipif(
is_internal(),
reason="API only available on server versions 3.0.11 or greater",
)
def test_can_create_model_ingestion_public_server(
self,
client: SpeckleClient,
test_project: Project,
test_model: Model,
):
with pytest.raises(GraphQLException) as ex:
_ = client.model.can_create_model_ingestion(test_project.id, test_model.id)
assert "GRAPHQL_VALIDATION_FAILED" in str(ex.value)
@@ -24,17 +24,6 @@ class TestProjectResource:
)
return project
@pytest.fixture()
def test_public_project(self, client: SpeckleClient) -> Project:
project = client.project.create(
ProjectCreateInput(
name="test project123",
description="desc",
visibility=ProjectVisibility.PUBLIC,
)
)
return project
@pytest.mark.parametrize(
"name, description, visibility",
[
@@ -61,7 +50,7 @@ class TestProjectResource:
assert result.id is not None
assert result.name == name
assert result.description == (description or "")
# we've disabled creation of unlisted projects for now, they fall back to public
# we've disabled creation of public projects for now, they fall back to unlisted
if visibility == ProjectVisibility.UNLISTED:
assert result.visibility == ProjectVisibility.PUBLIC
else:
@@ -78,32 +67,13 @@ class TestProjectResource:
assert result.created_at == test_project.created_at
def test_project_get_permissions(
self,
client: SpeckleClient,
second_client: SpeckleClient,
test_project: Project,
test_public_project: Project,
self, client: SpeckleClient, test_project: Project
):
result_private = client.project.get_permissions(test_project.id)
assert isinstance(result_private, ProjectPermissionChecks)
assert result_private.can_create_model.authorized is True
assert result_private.can_delete.authorized is True
assert result_private.can_load.authorized is True
result = client.project.get_permissions(test_public_project.id)
result = client.project.get_permissions(test_project.id)
assert isinstance(result, ProjectPermissionChecks)
assert result.can_create_model.authorized is True
assert result.can_delete.authorized is True
assert result.can_load.authorized is True
guest = second_client.project.get_permissions(test_public_project.id)
assert isinstance(result, ProjectPermissionChecks)
assert guest.can_create_model.authorized is False
assert guest.can_delete.authorized is False
assert guest.can_load.authorized is False
def test_project_update(self, client: SpeckleClient, test_project: Project):
new_name = "MY new name"
@@ -1,26 +1,15 @@
import asyncio
from sys import platform
from typing import Dict
from typing import Dict, Optional
import pytest
from specklepy.api.client import SpeckleClient
from specklepy.core.api.enums import (
ModelIngestionStatus,
ProjectModelIngestionUpdatedMessageType,
ProjectModelsUpdatedMessageType,
ProjectUpdatedMessageType,
ProjectVersionsUpdatedMessageType,
UserProjectsUpdatedMessageType,
)
from specklepy.core.api.inputs.model_ingestion_inputs import (
ModelIngestionCreateInput,
ModelIngestionReference,
ModelIngestionRequestCancellationInput,
ModelIngestionUpdateInput,
ProjectModelIngestionSubscriptionInput,
SourceDataInput,
)
from specklepy.core.api.inputs.model_inputs import CreateModelInput
from specklepy.core.api.inputs.project_inputs import (
ProjectCreateInput,
@@ -35,16 +24,9 @@ from specklepy.core.api.models import (
UserProjectsUpdatedMessage,
Version,
)
from specklepy.core.api.models.current import ModelIngestion
from specklepy.core.api.models.subscription_messages import (
ProjectModelIngestionUpdatedMessage,
)
from tests.integration.conftest import create_client, create_version, is_public
from tests.integration.conftest import create_client, create_version
# WSL is slow AF, so for local runs, we're being extra generous
# For CI runs on linux,m a much smaller wait time is acceptable
SETUP_TIME_SECONDS = 1 if platform == "linux" else 4
MAX_WAIT_TIME_SECONDS = 0.75 if platform == "linux" else 5
WAIT_PERIOD = 0.4 # time in seconds
@pytest.mark.run()
@@ -73,68 +55,48 @@ class TestSubscriptionResource:
)
return model1
@pytest.fixture
def test_model_ingestion(
self,
subscription_client: SpeckleClient,
test_project: Project,
test_model: Model,
) -> ModelIngestion:
project = subscription_client.model_ingestion.create(
ModelIngestionCreateInput(
project_id=test_project.id,
model_id=test_model.id,
progress_message="",
source_data=SourceDataInput(
source_application_slug="pytest",
source_application_version="0.0.0",
file_name=None,
file_size_bytes=None,
),
)
)
return project
@pytest.mark.asyncio
async def test_user_projects_updated(
self,
subscription_client: SpeckleClient,
) -> None:
loop = asyncio.get_running_loop()
future: asyncio.Future[UserProjectsUpdatedMessage] = loop.create_future()
message: Optional[UserProjectsUpdatedMessage] = None
task = None
def callback(d: UserProjectsUpdatedMessage):
nonlocal future
future.set_result(d)
nonlocal message
message = d
task = asyncio.create_task(
subscription_client.subscription.user_projects_updated(callback)
)
await asyncio.sleep(SETUP_TIME_SECONDS) # Give time to subscription to be setup
await asyncio.sleep(WAIT_PERIOD) # Give time to subscription to be setup
input = ProjectCreateInput(name=None, description=None, visibility=None)
created = subscription_client.project.create(input)
message = await asyncio.wait_for(future, timeout=MAX_WAIT_TIME_SECONDS)
await asyncio.sleep(WAIT_PERIOD) # Give time for subscription to be triggered
assert isinstance(message, UserProjectsUpdatedMessage)
assert message.id == created.id
assert message.type == UserProjectsUpdatedMessageType.ADDED
assert isinstance(message.project, Project)
if not task.cancel():
await task
task.cancel()
await task
@pytest.mark.asyncio
async def test_project_models_updated(
self, subscription_client: SpeckleClient, test_project: Project
) -> None:
loop = asyncio.get_running_loop()
future: asyncio.Future[ProjectModelsUpdatedMessage] = loop.create_future()
message: Optional[ProjectModelsUpdatedMessage] = None
task = None
def callback(d: ProjectModelsUpdatedMessage):
nonlocal future
future.set_result(d)
nonlocal message
message = d
task = asyncio.create_task(
subscription_client.subscription.project_models_updated(
@@ -142,53 +104,51 @@ class TestSubscriptionResource:
)
)
await asyncio.sleep(SETUP_TIME_SECONDS) # Give time to subscription to be setup
await asyncio.sleep(WAIT_PERIOD) # Give time to subscription to be setup
input = CreateModelInput(
name="my model", description="myDescription", project_id=test_project.id
)
created = subscription_client.model.create(input)
message = await asyncio.wait_for(future, timeout=MAX_WAIT_TIME_SECONDS)
await asyncio.sleep(WAIT_PERIOD) # Give time for subscription to be triggered
assert isinstance(message, ProjectModelsUpdatedMessage)
assert message.id == created.id
assert message.type == ProjectModelsUpdatedMessageType.CREATED
assert isinstance(message.model, Model)
if not task.cancel():
await task
task.cancel()
await task
@pytest.mark.asyncio
async def test_project_updated(
self, subscription_client: SpeckleClient, test_project: Project
) -> None:
loop = asyncio.get_running_loop()
future: asyncio.Future[ProjectUpdatedMessage] = loop.create_future()
message: Optional[ProjectUpdatedMessage] = None
task = None
def callback(d: ProjectUpdatedMessage):
nonlocal future
future.set_result(d)
nonlocal message
message = d
task = asyncio.create_task(
subscription_client.subscription.project_updated(callback, test_project.id)
)
await asyncio.sleep(
SETUP_TIME_SECONDS
) # Give time for subscription to be triggered
await asyncio.sleep(WAIT_PERIOD) # Give time to subscription to be setup
input = ProjectUpdateInput(id=test_project.id, name="This is my new name")
created = subscription_client.project.update(input)
message = await asyncio.wait_for(future, timeout=MAX_WAIT_TIME_SECONDS)
await asyncio.sleep(WAIT_PERIOD) # Give time for subscription to be triggered
assert isinstance(message, ProjectUpdatedMessage)
assert message.id == created.id
assert message.type == ProjectUpdatedMessageType.UPDATED
assert isinstance(message.project, Project)
if not task.cancel():
await task
task.cancel()
await task
@pytest.mark.asyncio
async def test_project_versions_updated(
@@ -197,12 +157,13 @@ class TestSubscriptionResource:
test_project: Project,
test_model: Model,
) -> None:
loop = asyncio.get_running_loop()
future: asyncio.Future[ProjectVersionsUpdatedMessage] = loop.create_future()
message: Optional[ProjectVersionsUpdatedMessage] = None
task = None
def callback(d: ProjectVersionsUpdatedMessage):
nonlocal future
future.set_result(d)
nonlocal message
message = d
task = asyncio.create_task(
subscription_client.subscription.project_versions_updated(
@@ -210,181 +171,15 @@ class TestSubscriptionResource:
)
)
await asyncio.sleep(SETUP_TIME_SECONDS) # Give time to subscription to be setup
await asyncio.sleep(WAIT_PERIOD) # Give time to subscription to be setup
created = create_version(subscription_client, test_project.id, test_model.id)
message = await asyncio.wait_for(future, timeout=MAX_WAIT_TIME_SECONDS)
await asyncio.sleep(WAIT_PERIOD) # Give time for subscription to be triggered
assert isinstance(message, ProjectVersionsUpdatedMessage)
assert message.id == created.id
assert message.type == ProjectVersionsUpdatedMessageType.CREATED
assert isinstance(message.version, Version)
if not task.cancel():
await task
@pytest.mark.asyncio
@pytest.mark.skipif(
is_public(), reason="The public API does not support these tests"
)
async def test_project_model_ingestion_cancellation(
self,
subscription_client: SpeckleClient,
test_project: Project,
test_model_ingestion: ModelIngestion,
) -> None:
assert not test_model_ingestion.cancellation_requested
loop = asyncio.get_running_loop()
future: asyncio.Future[ProjectModelIngestionUpdatedMessage] = (
loop.create_future()
)
def callback(d: ProjectModelIngestionUpdatedMessage):
nonlocal future
future.set_result(d)
task = asyncio.create_task(
subscription_client.subscription.project_model_ingestion_cancellation_requested(
callback, test_project.id, ingestion_id=test_model_ingestion.id
)
)
await asyncio.sleep(SETUP_TIME_SECONDS) # Give time to subscription to be setup
cancellation_request = ModelIngestionRequestCancellationInput(
ingestion_id=test_model_ingestion.id,
project_id=test_project.id,
cancellation_message="Please cancel",
)
created = subscription_client.model_ingestion.request_cancellation(
cancellation_request
)
assert created.id == test_model_ingestion.id
assert created.cancellation_requested
assert created.status_data.status == ModelIngestionStatus.PROCESSING
message = await asyncio.wait_for(future, timeout=MAX_WAIT_TIME_SECONDS)
assert isinstance(message, ProjectModelIngestionUpdatedMessage)
assert message.model_ingestion.id == created.id
assert message.model_ingestion.cancellation_requested
assert (
message.type
== ProjectModelIngestionUpdatedMessageType.CANCELLATION_REQUESTED
)
assert created.status_data.status == ModelIngestionStatus.PROCESSING
if not task.cancel():
await task
@pytest.mark.asyncio
@pytest.mark.skipif(
is_public(), reason="The public API does not support these tests"
)
async def test_project_model_ingestion_cancellation_isnt_triggered_by_updates(
self,
subscription_client: SpeckleClient,
test_project: Project,
test_model_ingestion: ModelIngestion,
) -> None:
assert not test_model_ingestion.cancellation_requested
loop = asyncio.get_running_loop()
future: asyncio.Future[ProjectModelIngestionUpdatedMessage] = (
loop.create_future()
)
def callback(d: ProjectModelIngestionUpdatedMessage):
nonlocal future
future.set_result(d)
task = asyncio.create_task(
subscription_client.subscription.project_model_ingestion_cancellation_requested(
callback, test_project.id, ingestion_id=test_model_ingestion.id
)
)
await asyncio.sleep(SETUP_TIME_SECONDS) # Give time to subscription to be setup
cancellation_request = ModelIngestionUpdateInput(
ingestion_id=test_model_ingestion.id,
project_id=test_project.id,
progress=None,
progress_message="this is just an ordinary update",
)
created = subscription_client.model_ingestion.update_progress(
cancellation_request
)
assert created.id == test_model_ingestion.id
assert not created.cancellation_requested
assert created.status_data.status == ModelIngestionStatus.PROCESSING
await asyncio.sleep(MAX_WAIT_TIME_SECONDS)
assert (
not future.done()
) # make sure the sub did not call back and resolve the future
if not task.cancel():
await task
@pytest.mark.asyncio
@pytest.mark.skipif(
is_public(), reason="The public API does not support these tests"
)
async def test_project_model_ingestion_updates(
self,
subscription_client: SpeckleClient,
test_project: Project,
test_model_ingestion: ModelIngestion,
) -> None:
assert not test_model_ingestion.cancellation_requested
loop = asyncio.get_running_loop()
future: asyncio.Future[ProjectModelIngestionUpdatedMessage] = (
loop.create_future()
)
def callback(d: ProjectModelIngestionUpdatedMessage):
nonlocal future
future.set_result(d)
task = asyncio.create_task(
subscription_client.subscription.project_model_ingestion_updated(
callback,
input=ProjectModelIngestionSubscriptionInput(
project_id=test_project.id,
ingestion_reference=ModelIngestionReference(
ingestion_id=test_model_ingestion.id, model_id=None
),
),
# ingestion_id=test_model_ingestion.id,
)
)
await asyncio.sleep(SETUP_TIME_SECONDS) # Give time to subscription to be setup
progress_message = "this is just an ordinary update"
cancellation_request = ModelIngestionUpdateInput(
ingestion_id=test_model_ingestion.id,
project_id=test_project.id,
progress=None,
progress_message=progress_message,
)
created = subscription_client.model_ingestion.update_progress(
cancellation_request
)
assert created.id == test_model_ingestion.id
assert not created.cancellation_requested
assert created.status_data.status == ModelIngestionStatus.PROCESSING
message = await asyncio.wait_for(future, timeout=MAX_WAIT_TIME_SECONDS)
assert isinstance(message, ProjectModelIngestionUpdatedMessage)
assert message.model_ingestion.id == created.id
assert not message.model_ingestion.cancellation_requested
assert message.type == ProjectModelIngestionUpdatedMessageType.UPDATED
assert message.model_ingestion.status_data.progress_message == progress_message
assert created.status_data.status == ModelIngestionStatus.PROCESSING
if not task.cancel():
await task
task.cancel()
await task
-9
View File
@@ -1,4 +1,3 @@
import os
import random
import uuid
from typing import Dict
@@ -30,14 +29,6 @@ def host() -> str:
return "localhost:3000"
def is_public() -> bool:
return os.getenv("IS_PUBLIC", "false").lower() == "true"
def is_internal() -> bool:
return not is_public()
def seed_user(host: str) -> Dict[str, str]:
seed = uuid.uuid4().hex
user_dict = {
-92
View File
@@ -1,92 +0,0 @@
from typing import Any, Callable
import pytest
from pytest_httpserver import HTTPServer
from requests import HTTPError
from werkzeug import Request, Response
from specklepy.core.api.client import SpeckleClient
from specklepy.logging import metrics
PATH = "/"
def assert_common_properties(payload: Any) -> None:
assert payload["event"] == "SDK Action"
assert payload["properties"]["token"] == "acd87c5a50b56df91a795e999812a3a4"
assert payload["properties"]["type"] == "action"
assert payload["properties"]["server_id"]
assert payload["properties"]["distinct_id"]
assert payload["properties"]["hostApp"] == "python"
assert payload["properties"]["hostAppVersion"]
assert payload["properties"]["core_version"]
def handler(extra_check: Callable[[Any], bool]) -> Callable[[Request], Response]:
def inner(request: Request) -> Response:
json = request.get_json()
payload = json[0]
assert_common_properties(payload)
assert extra_check(payload)
return Response("", 200)
return inner
def test_metrics_track(httpserver: HTTPServer, client: SpeckleClient):
with ScopedMetricsSetup(httpserver.url_for(PATH)) as _:
# Test No email
httpserver.expect_oneshot_request(PATH, "post").respond_with_handler(
handler(lambda payload: "email" not in payload["properties"])
)
metrics.track("SDK Action", client.account, None, True, False)
# Test With email
httpserver.expect_oneshot_request(PATH, "post").respond_with_handler(
handler(
lambda payload: payload["properties"]["email"]
== client.account.userInfo.email
)
)
metrics.track("SDK Action", client.account, None, True, True)
# Test With custom value
httpserver.expect_oneshot_request(PATH, "post").respond_with_handler(
handler(
lambda payload: payload["properties"]["myCustomProp"] == "myCustomValue"
)
)
metrics.track(
"SDK Action", client.account, {"myCustomProp": "myCustomValue"}, True, True
)
def test_metrics_errors(httpserver: HTTPServer):
with ScopedMetricsSetup(httpserver.url_for(PATH)) as _:
httpserver.expect_oneshot_request(PATH, "post").respond_with_data("", 400)
# Expect send_sync == true to mean mean it will raise
with pytest.raises(HTTPError):
metrics.track("SDK Action", send_sync=True)
# Expect send_sync == false to mean mean it won't
metrics.track("SDK Action")
class ScopedMetricsSetup:
"""
Scoped setup and tear down for enabling metrics tracking
"""
tracker: metrics.MetricsTracker
def __init__(self, metrics_url: str):
self.tracker = metrics.initialise_tracker()
self.tracker.analytics_url = metrics_url
def __enter__(self):
metrics.enable()
def __exit__(self, exc_type, exc_value, traceback):
metrics.disable()
metrics.METRICS_TRACKER = None
-105
View File
@@ -1,105 +0,0 @@
import time
import pytest
from specklepy.core.api.client import SpeckleClient
from specklepy.core.api.enums import ProjectVisibility
from specklepy.core.api.inputs.model_ingestion_inputs import (
ModelIngestionCreateInput,
SourceDataInput,
)
from specklepy.core.api.inputs.model_inputs import CreateModelInput
from specklepy.core.api.inputs.project_inputs import ProjectCreateInput
from specklepy.core.api.models.current import Model, ModelIngestion, Project
from specklepy.progress.ingestion_progress import IngestionProgressManager
from tests.integration.conftest import is_public
@pytest.mark.run()
@pytest.mark.skipif(
is_public(), reason="The public API does not support model ingestion api"
)
class TestIngestionProgressManager:
@pytest.fixture
def project(self, client: SpeckleClient) -> Project:
return client.project.create(
ProjectCreateInput(
name="test", description=None, visibility=ProjectVisibility.PUBLIC
)
)
@pytest.fixture
def model(self, client: SpeckleClient, project: Project) -> Model:
return client.model.create(
CreateModelInput(name="test", description=None, project_id=project.id)
)
@pytest.fixture
def model_ingestion(
self, client: SpeckleClient, model: Model, project: Project
) -> ModelIngestion:
input = ModelIngestionCreateInput(
model_id=model.id,
project_id=project.id,
progress_message="Starting processing",
source_data=SourceDataInput(
source_application_slug="pytest",
source_application_version="0.0.0",
file_name=None,
file_size_bytes=None,
),
)
return client.model_ingestion.create(input)
def test_progress_respects_throttle(
self,
client: SpeckleClient,
model_ingestion: ModelIngestion,
) -> None:
EXPECTED_MESSAGE = "This is a test ingestion message"
EXPECTED_PROGRESS = 0.123123
UPDATE_INTERVAL_SECONDS = 1
sut = IngestionProgressManager(
speckle_client=client,
ingestion=model_ingestion,
update_interval_seconds=UPDATE_INTERVAL_SECONDS,
)
assert sut.should_report_progress() is True
res = sut.report(EXPECTED_MESSAGE, EXPECTED_PROGRESS)
assert sut.should_report_progress() is False
time.sleep(UPDATE_INTERVAL_SECONDS + 0.5)
assert sut.should_report_progress() is True
assert sut.should_report_progress() is True
assert res.status_data.progress_message == EXPECTED_MESSAGE
def test_progress_no_throttle(
self,
client: SpeckleClient,
model_ingestion: ModelIngestion,
) -> None:
EXPECTED_MESSAGE = "This is a test ingestion message"
EXPECTED_PROGRESS = 0.123123
UPDATE_INTERVAL_SECONDS = 0
sut = IngestionProgressManager(
speckle_client=client,
ingestion=model_ingestion,
update_interval_seconds=UPDATE_INTERVAL_SECONDS,
)
assert sut.should_report_progress() is True
res = sut.report(EXPECTED_MESSAGE, EXPECTED_PROGRESS)
assert sut.should_report_progress() is True
assert sut.should_report_progress() is True
assert res.status_data.progress_message == EXPECTED_MESSAGE
+9 -9
View File
@@ -157,10 +157,10 @@ def test_parse_project():
def test_parse_model():
wrap = StreamWrapper(
"https://app.speckle.systems/projects/8be1007be1/models/cc7578012d"
"https://latest.speckle.systems/projects/843d07eb10/models/d9eb4918c8"
)
assert wrap.branch_name == "speckle tower revit 2025"
assert wrap.branch_name == "building wrapper"
assert wrap.type == "branch"
@@ -191,10 +191,10 @@ def test_parse_object_fe2():
def test_parse_version():
wrap = StreamWrapper(
"https://app.speckle.systems/projects/8be1007be1/models/cc7578012d@7199443eff"
"https://latest.speckle.systems/projects/843d07eb10/models/4e7345c838@c42d5cbac1"
)
wrap_quoted = StreamWrapper(
"https://app.speckle.systems/projects/8be1007be1/models/cc7578012d%407199443eff"
"https://latest.speckle.systems/projects/843d07eb10/models/4e7345c838%40c42d5cbac1"
)
assert wrap.type == "commit"
assert wrap_quoted.type == "commit"
@@ -208,11 +208,11 @@ def test_to_string():
"https://testing.speckle.dev/streams/0c6ad366c4/globals/abd3787893",
"https://testing.speckle.dev/streams/4c3ce1459c/commits/8b9b831792",
"https://testing.speckle.dev/streams/a75ab4f10f/objects/5530363e6d51c904903dafc3ea1d2ec6",
"https://app.speckle.systems/projects/8be1007be1",
"https://app.speckle.systems/projects/8be1007be1/models/cc7578012d",
"https://app.speckle.systems/projects/8be1007be1/models/cc7578012d@7199443eff",
"https://app.speckle.systems/projects/8be1007be1/models/cc7578012d%407199443eff",
"https://app.speckle.systems/projects/8be1007be1/models/9b5e57dca804a923a8d42d55dcc0191a",
"https://latest.speckle.systems/projects/843d07eb10",
"https://latest.speckle.systems/projects/843d07eb10/models/4e7345c838",
"https://latest.speckle.systems/projects/843d07eb10/models/4e7345c838@c42d5cbac1",
"https://latest.speckle.systems/projects/843d07eb10/models/4e7345c838%40c42d5cbac1",
"https://latest.speckle.systems/projects/24c3741255/models/b48d1b10f5a732f4ca4144286391282c",
]
for url in urls:
wrap = StreamWrapper(url)
@@ -1,110 +0,0 @@
"""Unit tests for AutomationContext.attach_result_to_objects contract."""
from unittest.mock import MagicMock
import pytest
from speckle_automate import AutomationContext
from speckle_automate.schema import (
AutomationRunData,
ObjectResultLevel,
VersionCreationTrigger,
VersionCreationTriggerPayload,
)
from specklepy.objects.base import Base
def _minimal_automation_context() -> AutomationContext:
run_data = AutomationRunData(
project_id="p",
speckle_server_url="http://localhost",
automation_id="a",
automation_run_id="r",
function_run_id="f",
triggers=[
VersionCreationTrigger(
trigger_type="versionCreation",
payload=VersionCreationTriggerPayload(model_id="m", version_id="v"),
)
],
)
return AutomationContext(
automation_run_data=run_data,
speckle_client=MagicMock(),
_server_transport=MagicMock(),
_speckle_token="",
)
def test_attach_result_to_objects_accepts_empty_list() -> None:
"""Empty affected_objects appends one result case with no object IDs."""
ctx = _minimal_automation_context()
assert len(ctx._automation_result.object_results) == 0
ctx.attach_result_to_objects(
ObjectResultLevel.WARNING,
"SkippedRule",
[],
message="No elements to check.",
)
assert len(ctx._automation_result.object_results) == 1
case = ctx._automation_result.object_results[0]
assert case.level == ObjectResultLevel.WARNING
assert case.category == "SkippedRule"
assert case.object_app_ids == {}
assert case.message == "No elements to check."
def test_attach_result_to_objects_with_objects_appends_case_with_ids() -> None:
"""Single or multiple objects with id produce result case with object_app_ids."""
ctx = _minimal_automation_context()
obj1 = Base()
obj1.id = "id-one"
obj1.applicationId = "app-one"
obj2 = Base()
obj2.id = "id-two"
ctx.attach_result_to_objects(
ObjectResultLevel.ERROR,
"BadType",
[obj1, obj2],
message="Invalid type.",
)
assert len(ctx._automation_result.object_results) == 1
case = ctx._automation_result.object_results[0]
assert case.level == ObjectResultLevel.ERROR
assert case.category == "BadType"
assert case.object_app_ids == {"id-one": "app-one", "id-two": None}
assert case.message == "Invalid type."
def test_attach_result_to_objects_raises_when_object_has_no_id() -> None:
"""At least one object without id raises."""
ctx = _minimal_automation_context()
obj = Base()
obj.id = None
with pytest.raises(Exception, match="results to objects with an id"):
ctx.attach_result_to_objects(
ObjectResultLevel.ERROR,
"Bad",
obj,
message="No id.",
)
assert len(ctx._automation_result.object_results) == 0
def test_attach_info_to_objects_accepts_empty_list() -> None:
"""attach_info_to_objects (convenience method) also accepts empty list."""
ctx = _minimal_automation_context()
ctx.attach_info_to_objects("VersionLevel", [], message="No levels in model.")
assert len(ctx._automation_result.object_results) == 1
case = ctx._automation_result.object_results[0]
assert case.level == ObjectResultLevel.INFO
assert case.category == "VersionLevel"
assert case.object_app_ids == {}
Generated
+1 -27
View File
@@ -526,7 +526,7 @@ name = "exceptiongroup"
version = "1.3.0"
source = { registry = "https://pypi.org/simple/" }
dependencies = [
{ name = "typing-extensions", marker = "python_full_version < '3.11'" },
{ name = "typing-extensions", marker = "python_full_version < '3.13'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" }
wheels = [
@@ -1867,18 +1867,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/ee/49/1377b49de7d0c1ce41292161ea0f721913fa8722c19fb9c1e3aa0367eecb/pytest_cov-7.0.0-py3-none-any.whl", hash = "sha256:3b8e9558b16cc1479da72058bdecf8073661c7f57f7d3c5f22a1c23507f2d861", size = 22424, upload-time = "2025-09-09T10:57:00.695Z" },
]
[[package]]
name = "pytest-httpserver"
version = "1.1.3"
source = { registry = "https://pypi.org/simple/" }
dependencies = [
{ name = "werkzeug" },
]
sdist = { url = "https://files.pythonhosted.org/packages/f1/d8/def15ba33bd696dd72dd4562a5287c0cba4d18a591eeb82e0b08ab385afc/pytest_httpserver-1.1.3.tar.gz", hash = "sha256:af819d6b533f84b4680b9416a5b3f67f1df3701f1da54924afd4d6e4ba5917ec", size = 68870, upload-time = "2025-04-10T08:17:15.6Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/0d/d2/dfc2f25f3905921c2743c300a48d9494d29032f1389fc142e718d6978fb2/pytest_httpserver-1.1.3-py3-none-any.whl", hash = "sha256:5f84757810233e19e2bb5287f3826a71c97a3740abe3a363af9155c0f82fdbb9", size = 21000, upload-time = "2025-04-10T08:17:13.906Z" },
]
[[package]]
name = "pytest-ordering"
version = "0.6"
@@ -2234,7 +2222,6 @@ dev = [
{ name = "pytest" },
{ name = "pytest-asyncio" },
{ name = "pytest-cov" },
{ name = "pytest-httpserver" },
{ name = "pytest-ordering" },
{ name = "ruff" },
{ name = "types-deprecated" },
@@ -2272,7 +2259,6 @@ dev = [
{ name = "pytest", specifier = ">=8.3.4" },
{ name = "pytest-asyncio", specifier = ">=0.25.2" },
{ name = "pytest-cov", specifier = ">=6.0.0" },
{ name = "pytest-httpserver", specifier = ">=1.1.3" },
{ name = "pytest-ordering", specifier = ">=0.6" },
{ name = "ruff", specifier = "==0.9.2" },
{ name = "types-deprecated", specifier = ">=1.2.15.20241117" },
@@ -2636,18 +2622,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/47/96/9d5749106ff57629b54360664ae7eb9afd8302fad1680ead385383e33746/websockets-11.0.3-py3-none-any.whl", hash = "sha256:6681ba9e7f8f3b19440921e99efbb40fc89f26cd71bf539e45d8c8a25c976dc6", size = 118056, upload-time = "2023-05-07T14:25:18.508Z" },
]
[[package]]
name = "werkzeug"
version = "3.1.4"
source = { registry = "https://pypi.org/simple/" }
dependencies = [
{ name = "markupsafe" },
]
sdist = { url = "https://files.pythonhosted.org/packages/45/ea/b0f8eeb287f8df9066e56e831c7824ac6bab645dd6c7a8f4b2d767944f9b/werkzeug-3.1.4.tar.gz", hash = "sha256:cd3cd98b1b92dc3b7b3995038826c68097dcb16f9baa63abe35f20eafeb9fe5e", size = 864687, upload-time = "2025-11-29T02:15:22.841Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/2f/f9/9e082990c2585c744734f85bec79b5dae5df9c974ffee58fe421652c8e91/werkzeug-3.1.4-py3-none-any.whl", hash = "sha256:2ad50fb9ed09cc3af22c54698351027ace879a0b60a3b5edf5730b2f7d876905", size = 224960, upload-time = "2025-11-29T02:15:21.13Z" },
]
[[package]]
name = "wrapt"
version = "2.0.1"