feat: add speckle automate package with some basic sanity tests
This commit is contained in:
Gergő Jedlicska
@@ -0,0 +1,23 @@
|
||||
"""This module contains an SDK for working with Speckle Automate."""
|
||||
from speckle_automate.automation_context import AutomationContext
|
||||
from speckle_automate.runner import execute_automate_function, run_function
|
||||
from speckle_automate.schema import (
|
||||
AutomateBase,
|
||||
AutomationResult,
|
||||
AutomationRunData,
|
||||
AutomationStatus,
|
||||
ObjectResult,
|
||||
ObjectResultLevel,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"AutomationContext",
|
||||
"AutomateBase",
|
||||
"AutomationStatus",
|
||||
"AutomationResult",
|
||||
"AutomationRunData",
|
||||
"ObjectResult",
|
||||
"ObjectResultLevel",
|
||||
"run_function",
|
||||
"execute_automate_function",
|
||||
]
|
||||
@@ -0,0 +1,278 @@
|
||||
"""This module provides an abstraction layer above the Speckle Automate runtime."""
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
import time
|
||||
from typing import Optional, Union
|
||||
|
||||
import httpx
|
||||
from gql import gql
|
||||
from specklepy.api import operations
|
||||
from specklepy.api.client import SpeckleClient
|
||||
from specklepy.objects import Base
|
||||
from specklepy.transports.memory import MemoryTransport
|
||||
from specklepy.transports.server import ServerTransport
|
||||
|
||||
from speckle_automate.schema import (
|
||||
AutomateBase,
|
||||
AutomationResult,
|
||||
AutomationRunData,
|
||||
AutomationStatus,
|
||||
ObjectResult,
|
||||
ObjectResultLevel,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AutomationContext:
|
||||
"""A context helper class.
|
||||
|
||||
This class exposes methods to work with the Speckle Automate context inside
|
||||
Speckle Automate functions.
|
||||
|
||||
An instance of AutomationContext is injected into every run of a function.
|
||||
"""
|
||||
|
||||
automation_run_data: AutomationRunData
|
||||
speckle_client: SpeckleClient
|
||||
_server_transport: ServerTransport
|
||||
_speckle_token: str
|
||||
|
||||
#: keep a memory transponrt at hand, to speed up things if needed
|
||||
_memory_transport: MemoryTransport = field(default_factory=MemoryTransport)
|
||||
|
||||
#: added for performance measuring
|
||||
_init_time: float = field(default_factory=time.perf_counter)
|
||||
_automation_result: AutomationResult = field(default_factory=AutomationResult)
|
||||
|
||||
@classmethod
|
||||
def initialize(
|
||||
cls, automation_run_data: Union[str, AutomationRunData], speckle_token: str
|
||||
) -> "AutomationContext":
|
||||
"""Bootstrap the AutomateSDK from raw data.
|
||||
|
||||
Todo:
|
||||
----
|
||||
* bootstrap a structlog logger instance
|
||||
* expose a logger, that ppl can use instead of print
|
||||
"""
|
||||
# parse the json value if its not an initialized project data instance
|
||||
automation_run_data = (
|
||||
automation_run_data
|
||||
if isinstance(automation_run_data, AutomationRunData)
|
||||
else AutomationRunData.model_validate_json(automation_run_data)
|
||||
)
|
||||
speckle_client = SpeckleClient(
|
||||
automation_run_data.speckle_server_url,
|
||||
automation_run_data.speckle_server_url.startswith("https"),
|
||||
)
|
||||
speckle_client.authenticate_with_token(speckle_token)
|
||||
if not speckle_client.account:
|
||||
msg = (
|
||||
f"Could not autenticate to {automation_run_data.speckle_server_url}",
|
||||
"with the provided token",
|
||||
)
|
||||
raise ValueError(msg)
|
||||
server_transport = ServerTransport(
|
||||
automation_run_data.project_id, speckle_client
|
||||
)
|
||||
return cls(automation_run_data, speckle_client, server_transport, speckle_token)
|
||||
|
||||
@property
|
||||
def run_status(self) -> AutomationStatus:
|
||||
"""Get the status of the automation run."""
|
||||
return self._automation_result.run_status
|
||||
|
||||
def elapsed(self) -> float:
|
||||
"""Return the elapsed time in seconds since the initialization time."""
|
||||
return time.perf_counter() - self._init_time
|
||||
|
||||
def receive_version(self) -> Base:
|
||||
"""Receive the Speckle project version that triggered this automation run."""
|
||||
commit = self.speckle_client.commit.get(
|
||||
self.automation_run_data.project_id, self.automation_run_data.version_id
|
||||
)
|
||||
if not commit.referencedObject:
|
||||
raise ValueError("The commit has no referencedObject, cannot receive it.")
|
||||
base = operations.receive(
|
||||
commit.referencedObject, self._server_transport, self._memory_transport
|
||||
)
|
||||
print(
|
||||
f"It took {self.elapsed():2f} seconds to receive",
|
||||
f" the speckle version {self.automation_run_data.version_id}",
|
||||
)
|
||||
return base
|
||||
|
||||
def create_new_version_in_project(
|
||||
self, root_object: Base, model_id: str, version_message: str = ""
|
||||
) -> None:
|
||||
"""Save a base model to a new version on the project.
|
||||
|
||||
Args:
|
||||
root_object (Base): The Speckle base object for the new version.
|
||||
model_id (str): For now please use a `branchName`!
|
||||
version_message (str): The message for the new version.
|
||||
"""
|
||||
if model_id == self.automation_run_data.model_id:
|
||||
raise ValueError(
|
||||
f"The target model id: {model_id} cannot match the model id"
|
||||
f" that triggered this automation: {self.automation_run_data.model_id}"
|
||||
)
|
||||
|
||||
root_object_id = operations.send(
|
||||
root_object,
|
||||
[self._server_transport, self._memory_transport],
|
||||
use_default_cache=False,
|
||||
)
|
||||
|
||||
version_id = self.speckle_client.commit.create(
|
||||
stream_id=self.automation_run_data.project_id,
|
||||
object_id=root_object_id,
|
||||
branch_name=model_id,
|
||||
message=version_message,
|
||||
source_application="SpeckleAutomate",
|
||||
)
|
||||
self._automation_result.result_versions.append(version_id)
|
||||
|
||||
def report_run_status(self) -> None:
|
||||
"""Report the current run status to the project of this automation."""
|
||||
query = gql(
|
||||
"""
|
||||
mutation ReportFunctionRunStatus(
|
||||
$automationId: String!,
|
||||
$automationRevisionId: String!,
|
||||
$automationRunId: String!,
|
||||
$versionId: String!,
|
||||
$functionId: String!,
|
||||
$runStatus: AutomationRunStatus!
|
||||
$elapsed: Float!
|
||||
$resultVersionIds: [String!]!
|
||||
$statusMessage: String
|
||||
$objectResults: JSONObject
|
||||
){
|
||||
automationMutations {
|
||||
functionRunStatusReport(input: {
|
||||
automationId: $automationId
|
||||
automationRevisionId: $automationRevisionId
|
||||
automationRunId: $automationRunId
|
||||
versionId: $versionId
|
||||
functionRuns: [
|
||||
{
|
||||
functionId: $functionId
|
||||
status: $runStatus,
|
||||
elapsed: $elapsed,
|
||||
resultVersionIds: $resultVersionIds,
|
||||
statusMessage: $statusMessage
|
||||
results: $objectResults
|
||||
}]
|
||||
})
|
||||
}
|
||||
}
|
||||
"""
|
||||
)
|
||||
if self.run_status in [AutomationStatus.SUCCEEDED, AutomationStatus.FAILED]:
|
||||
object_results = {
|
||||
"version": "1.0.0",
|
||||
"values": {
|
||||
"speckleObjects": self._automation_result.model_dump(by_alias=True)[
|
||||
"objectResults"
|
||||
],
|
||||
"blobs": self._automation_result.blobs,
|
||||
},
|
||||
}
|
||||
else:
|
||||
object_results = None
|
||||
params = {
|
||||
"automationId": self.automation_run_data.automation_id,
|
||||
"automationRevisionId": self.automation_run_data.automation_revision_id,
|
||||
"automationRunId": self.automation_run_data.automation_run_id,
|
||||
"versionId": self.automation_run_data.version_id,
|
||||
"functionId": self.automation_run_data.function_id,
|
||||
"runStatus": self.run_status.value,
|
||||
"statusMessage": self._automation_result.status_message,
|
||||
"elapsed": self.elapsed(),
|
||||
"resultVersionIds": self._automation_result.result_versions,
|
||||
"objectResults": object_results,
|
||||
}
|
||||
self.speckle_client.httpclient.execute(query, params)
|
||||
|
||||
def store_file_result(self, file_path: Union[Path, str]) -> None:
|
||||
"""Save a file attached to the project of this automation."""
|
||||
path_obj = (
|
||||
Path(file_path).resolve() if isinstance(file_path, str) else file_path
|
||||
)
|
||||
|
||||
class UploadResult(AutomateBase):
|
||||
blob_id: str
|
||||
file_name: str
|
||||
upload_status: int
|
||||
|
||||
class BlobUploadResponse(AutomateBase):
|
||||
upload_results: list[UploadResult]
|
||||
|
||||
if not path_obj.exists():
|
||||
raise ValueError("The given file path doesn't exist")
|
||||
files = {path_obj.name: open(str(path_obj), "rb")}
|
||||
|
||||
url = (
|
||||
f"{self.automation_run_data.speckle_server_url}/api/stream/"
|
||||
f"{self.automation_run_data.project_id}/blob"
|
||||
)
|
||||
data = (
|
||||
httpx.post(
|
||||
url,
|
||||
files=files,
|
||||
headers={"authorization": f"Bearer {self._speckle_token}"},
|
||||
)
|
||||
.raise_for_status()
|
||||
.json()
|
||||
)
|
||||
|
||||
upload_response = BlobUploadResponse.model_validate(data)
|
||||
|
||||
if len(upload_response.upload_results) != 1:
|
||||
raise ValueError("Expecting one upload result.")
|
||||
|
||||
for upload_result in upload_response.upload_results:
|
||||
self._automation_result.blobs.append(upload_result.blob_id)
|
||||
|
||||
def mark_run_failed(self, status_message: str) -> None:
|
||||
"""Mark the current run a failure."""
|
||||
self._mark_run(AutomationStatus.FAILED, status_message)
|
||||
|
||||
def mark_run_success(self, status_message: Optional[str]) -> None:
|
||||
"""Mark the current run a success with an optional message."""
|
||||
self._mark_run(AutomationStatus.SUCCEEDED, status_message)
|
||||
|
||||
def _mark_run(
|
||||
self, status: AutomationStatus, status_message: Optional[str]
|
||||
) -> None:
|
||||
duration = self.elapsed()
|
||||
self._automation_result.status_message = status_message
|
||||
self._automation_result.run_status = status
|
||||
self._automation_result.elapsed = duration
|
||||
|
||||
msg = f"Automation run {status.value} after {duration:2f} seconds."
|
||||
print("\n".join([msg, status_message]) if status_message else msg)
|
||||
|
||||
def add_object_error(self, object_id: str, error_cause: str) -> None:
|
||||
"""Add an error to a given Speckle object."""
|
||||
self._add_object_result(object_id, ObjectResultLevel.ERROR, error_cause)
|
||||
|
||||
def add_object_warning(self, object_id: str, warning: str) -> None:
|
||||
"""Add a warning to a given Speckle object."""
|
||||
self._add_object_result(object_id, ObjectResultLevel.WARNING, warning)
|
||||
|
||||
def add_object_info(self, object_id: str, info: str) -> None:
|
||||
"""Add an info message to a given Speckle object."""
|
||||
self._add_object_result(object_id, ObjectResultLevel.INFO, info)
|
||||
|
||||
def _add_object_result(
|
||||
self, object_id: str, level: ObjectResultLevel, status_message: str
|
||||
) -> None:
|
||||
print(
|
||||
f"Object {object_id} was marked with {level.value.upper()}",
|
||||
f" cause: {status_message}",
|
||||
)
|
||||
self._automation_result.object_results[object_id].append(
|
||||
ObjectResult(level=level, status_message=status_message)
|
||||
)
|
||||
@@ -0,0 +1,155 @@
|
||||
"""Function execution module.
|
||||
|
||||
Provides mechanisms to execute any function,
|
||||
that conforms to the AutomateFunction "interface"
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Callable, Optional, TypeVar, Union, overload
|
||||
|
||||
from speckle_automate.automation_context import AutomationContext
|
||||
from speckle_automate.schema import AutomateBase, AutomationRunData, AutomationStatus
|
||||
|
||||
T = TypeVar("T", bound=AutomateBase)
|
||||
|
||||
AutomateFunction = Callable[[AutomationContext, T], None]
|
||||
AutomateFunctionWithoutInputs = Callable[[AutomationContext], None]
|
||||
|
||||
|
||||
@overload
|
||||
def execute_automate_function(
|
||||
automate_function: AutomateFunction[T],
|
||||
input_schema: type[T],
|
||||
) -> None:
|
||||
...
|
||||
|
||||
|
||||
@overload
|
||||
def execute_automate_function(automate_function: AutomateFunctionWithoutInputs) -> None:
|
||||
...
|
||||
|
||||
|
||||
def execute_automate_function(
|
||||
automate_function: Union[AutomateFunction[T], AutomateFunctionWithoutInputs],
|
||||
input_schema: Optional[type[T]] = None,
|
||||
):
|
||||
"""Runs the provided automate function with the input schema."""
|
||||
# first arg is the python file name, we do not need that
|
||||
args = sys.argv[1:]
|
||||
|
||||
if len(args) < 2:
|
||||
raise ValueError("too few arguments specified need minimum 2")
|
||||
|
||||
if len(args) > 4:
|
||||
raise ValueError("too many arguments specified, max supported is 4")
|
||||
|
||||
# we rely on a command name convention to decide what to do.
|
||||
# this is here, so that the function authors do not see any of this
|
||||
command = args[0]
|
||||
|
||||
if command == "generate_schema":
|
||||
path = Path(args[1])
|
||||
schema = json.dumps(
|
||||
input_schema.model_json_schema(by_alias=True) if input_schema else {}
|
||||
)
|
||||
path.write_text(schema)
|
||||
|
||||
elif command == "run":
|
||||
automation_run_data = args[1]
|
||||
function_inputs = args[2]
|
||||
|
||||
speckle_token = os.environ.get("SPECKLE_TOKEN", None)
|
||||
if not speckle_token and len(args) != 4:
|
||||
raise ValueError("Cannot get speckle token from arguments or environment")
|
||||
|
||||
speckle_token = speckle_token if speckle_token else args[3]
|
||||
|
||||
inputs = (
|
||||
input_schema.model_validate_json(function_inputs)
|
||||
if input_schema
|
||||
else input_schema
|
||||
)
|
||||
|
||||
if inputs:
|
||||
automation_context = run_function(
|
||||
automate_function, # type: ignore
|
||||
automation_run_data,
|
||||
speckle_token,
|
||||
inputs,
|
||||
)
|
||||
else:
|
||||
automation_context = run_function(
|
||||
automate_function, # type: ignore
|
||||
automation_run_data,
|
||||
speckle_token,
|
||||
)
|
||||
|
||||
exit_code = (
|
||||
0 if automation_context.run_status == AutomationStatus.SUCCEEDED else 1
|
||||
)
|
||||
exit(exit_code)
|
||||
|
||||
else:
|
||||
raise NotImplementedError(f"Command: '{command}' is not supported.")
|
||||
|
||||
|
||||
@overload
|
||||
def run_function(
|
||||
automate_function: AutomateFunction[T],
|
||||
automation_run_data: Union[AutomationRunData, str],
|
||||
speckle_token: str,
|
||||
inputs: T,
|
||||
) -> AutomationContext:
|
||||
...
|
||||
|
||||
|
||||
@overload
|
||||
def run_function(
|
||||
automate_function: AutomateFunctionWithoutInputs,
|
||||
automation_run_data: Union[AutomationRunData, str],
|
||||
speckle_token: str,
|
||||
) -> AutomationContext:
|
||||
...
|
||||
|
||||
|
||||
def run_function(
|
||||
automate_function: Union[AutomateFunction[T], AutomateFunctionWithoutInputs],
|
||||
automation_run_data: Union[AutomationRunData, str],
|
||||
speckle_token: str,
|
||||
inputs: Optional[T] = None,
|
||||
) -> AutomationContext:
|
||||
"""Run the provided function with the automate sdk context."""
|
||||
automation_context = AutomationContext.initialize(
|
||||
automation_run_data, speckle_token
|
||||
)
|
||||
automation_context.report_run_status()
|
||||
|
||||
try:
|
||||
# avoiding complex type gymnastics here on the internals.
|
||||
# the external type overloads make this correct
|
||||
if inputs:
|
||||
automate_function(automation_context, inputs) # type: ignore
|
||||
else:
|
||||
automate_function(automation_context) # type: ignore
|
||||
|
||||
# the function author forgot to mark the function success
|
||||
if automation_context.run_status not in [
|
||||
AutomationStatus.FAILED,
|
||||
AutomationStatus.SUCCEEDED,
|
||||
]:
|
||||
automation_context.mark_run_success(
|
||||
"WARNING: Automate assumed a success status,"
|
||||
" but it was not marked as so by the function."
|
||||
)
|
||||
except Exception:
|
||||
trace = traceback.format_exc()
|
||||
print(trace)
|
||||
automation_context.mark_run_failed(
|
||||
"Function error. Check the automation run logs for details."
|
||||
)
|
||||
finally:
|
||||
automation_context.report_run_status()
|
||||
return automation_context
|
||||
@@ -0,0 +1,73 @@
|
||||
""""""
|
||||
from collections import defaultdict
|
||||
from enum import Enum
|
||||
from typing import Optional
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from stringcase import camelcase
|
||||
|
||||
|
||||
class AutomateBase(BaseModel):
|
||||
"""Use this class as a base model for automate related DTO."""
|
||||
|
||||
model_config = ConfigDict(alias_generator=camelcase, populate_by_name=True)
|
||||
|
||||
|
||||
class AutomationRunData(BaseModel):
|
||||
"""Values of the project / model that triggered the run of this function."""
|
||||
|
||||
project_id: str
|
||||
model_id: str
|
||||
branch_name: str
|
||||
version_id: str
|
||||
speckle_server_url: str
|
||||
|
||||
automation_id: str
|
||||
automation_revision_id: str
|
||||
automation_run_id: str
|
||||
|
||||
function_id: str
|
||||
function_revision: str
|
||||
|
||||
model_config = ConfigDict(
|
||||
alias_generator=camelcase, populate_by_name=True, protected_namespaces=()
|
||||
)
|
||||
|
||||
|
||||
class AutomationStatus(str, Enum):
|
||||
"""Set the status of the automation."""
|
||||
|
||||
INITIALIZING = "INITIALIZING"
|
||||
RUNNING = "RUNNING"
|
||||
FAILED = "FAILED"
|
||||
SUCCEEDED = "SUCCEEDED"
|
||||
|
||||
|
||||
class ObjectResultLevel(str, Enum):
|
||||
"""Possible status message levels for object reports."""
|
||||
|
||||
INFO = "INFO"
|
||||
WARNING = "WARNING"
|
||||
ERROR = "ERROR"
|
||||
|
||||
|
||||
class ObjectResult(AutomateBase):
|
||||
"""An object level result."""
|
||||
|
||||
level: ObjectResultLevel
|
||||
status_message: str
|
||||
|
||||
|
||||
class AutomationResult(AutomateBase):
|
||||
"""Schema accepted by the Speckle server as a result for an automation run."""
|
||||
|
||||
elapsed: float = 0
|
||||
result_view: Optional[str] = None
|
||||
result_versions: list[str] = Field(default_factory=list)
|
||||
blobs: list[str] = Field(default_factory=list)
|
||||
run_status: AutomationStatus = AutomationStatus.RUNNING
|
||||
status_message: Optional[str] = None
|
||||
|
||||
object_results: dict[str, list[ObjectResult]] = Field(
|
||||
default_factory=lambda: defaultdict(list) # typing: ignore
|
||||
)
|
||||
Reference in New Issue
Block a user