huanld/speckle-automate-checker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update project configuration and dependencies

Browse Source 
- Added source folders for `src` and `tests`.
- Excluded `.devcontainer` and `.idea` directories.
- Updated Black settings in the project configuration.
- Changed Ruff version from 0.9.5 to 0.9.6.
- Enhanced rule processing by grouping rules before application.
- Introduced a new property match mode for better parameter matching.
- Removed unused test files and configurations to clean up the codebase.
This commit is contained in:
Jonathon Broughton
2025-02-10 16:30:00 +00:00
parent ccabdcb0a2
commit a0b92bd115
12 changed files with 175 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.idea/Checker.iml
Generated
+4 -1
View File
@@ -2,7 +2,10 @@
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$">
<excludeFolder url="file://$MODULE_DIR$/.venv" />
<sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/tests" isTestSource="true" />
<excludeFolder url="file://$MODULE_DIR$/.devcontainer" />
<excludeFolder url="file://$MODULE_DIR$/.idea" />
</content>
<orderEntry type="jdk" jdkName="WSL Checker" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
.idea/misc.xml
Generated
+5 -1
View File
@@ -1,7 +1,11 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Black">
<option name="sdkName" value="Python 3.11.4 WSL (UbuntuDev): (/home/jsdbroughton/.virtualenvs/speckle_automate-mesh_density_checker/bin/python)" />
<option name="enabledOnReformat" value="true" />
<option name="enabledOnSave" value="true" />
<option name="executionMode" value="BINARY" />
<option name="pathToExecutable" value="\\wsl.localhost\UbuntuDev\mnt\d\Repos\Speckle Automate\Checker\.venv\bin\black" />
<option name="sdkName" value="WSL Checker" />
</component>
<component name="ProjectRootManager" version="2" project-jdk-name="WSL Checker" project-jdk-type="Python SDK" />
</project>
.idea/ruff.xml
Generated
+3 -1
View File
@@ -1,7 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="RuffConfigService">
<option name="runRuffOnSave" value="true" />
<option name="enableLsp" value="false" />
<option name="runRuffOnReformatCode" value="false" />
<option name="showRuleCode" value="false" />
<option name="useRuffServer" value="true" />
</component>
</project>
poetry.lock
Generated
+20 -20
View File
@@ -1324,29 +1324,29 @@ requests = ">=2.0.1,<3.0.0"
[[package]]
name = "ruff"
version = "0.9.5"
version = "0.9.6"
description = "An extremely fast Python linter and code formatter, written in Rust."
optional = false
python-versions = ">=3.7"
files = [
{file = "ruff-0.9.5-py3-none-linux_armv6l.whl", hash = "sha256:d466d2abc05f39018d53f681fa1c0ffe9570e6d73cde1b65d23bb557c846f442"},
{file = "ruff-0.9.5-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:38840dbcef63948657fa7605ca363194d2fe8c26ce8f9ae12eee7f098c85ac8a"},
{file = "ruff-0.9.5-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d56ba06da53536b575fbd2b56517f6f95774ff7be0f62c80b9e67430391eeb36"},
{file = "ruff-0.9.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f7cb2a01da08244c50b20ccfaeb5972e4228c3c3a1989d3ece2bc4b1f996001"},
{file = "ruff-0.9.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:96d5c76358419bc63a671caac70c18732d4fd0341646ecd01641ddda5c39ca0b"},
{file = "ruff-0.9.5-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:deb8304636ed394211f3a6d46c0e7d9535b016f53adaa8340139859b2359a070"},
{file = "ruff-0.9.5-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:df455000bf59e62b3e8c7ba5ed88a4a2bc64896f900f311dc23ff2dc38156440"},
{file = "ruff-0.9.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de92170dfa50c32a2b8206a647949590e752aca8100a0f6b8cefa02ae29dce80"},
{file = "ruff-0.9.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d28532d73b1f3f627ba88e1456f50748b37f3a345d2be76e4c653bec6c3e393"},
{file = "ruff-0.9.5-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c746d7d1df64f31d90503ece5cc34d7007c06751a7a3bbeee10e5f2463d52d2"},
{file = "ruff-0.9.5-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:11417521d6f2d121fda376f0d2169fb529976c544d653d1d6044f4c5562516ee"},
{file = "ruff-0.9.5-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:5b9d71c3879eb32de700f2f6fac3d46566f644a91d3130119a6378f9312a38e1"},
{file = "ruff-0.9.5-py3-none-musllinux_1_2_i686.whl", hash = "sha256:2e36c61145e70febcb78483903c43444c6b9d40f6d2f800b5552fec6e4a7bb9a"},
{file = "ruff-0.9.5-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:2f71d09aeba026c922aa7aa19a08d7bd27c867aedb2f74285a2639644c1c12f5"},
{file = "ruff-0.9.5-py3-none-win32.whl", hash = "sha256:134f958d52aa6fdec3b294b8ebe2320a950d10c041473c4316d2e7d7c2544723"},
{file = "ruff-0.9.5-py3-none-win_amd64.whl", hash = "sha256:78cc6067f6d80b6745b67498fb84e87d32c6fc34992b52bffefbdae3442967d6"},
{file = "ruff-0.9.5-py3-none-win_arm64.whl", hash = "sha256:18a29f1a005bddb229e580795627d297dfa99f16b30c7039e73278cf6b5f9fa9"},
{file = "ruff-0.9.5.tar.gz", hash = "sha256:11aecd7a633932875ab3cb05a484c99970b9d52606ce9ea912b690b02653d56c"},
{file = "ruff-0.9.6-py3-none-linux_armv6l.whl", hash = "sha256:2f218f356dd2d995839f1941322ff021c72a492c470f0b26a34f844c29cdf5ba"},
{file = "ruff-0.9.6-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b908ff4df65dad7b251c9968a2e4560836d8f5487c2f0cc238321ed951ea0504"},
{file = "ruff-0.9.6-py3-none-macosx_11_0_arm64.whl", hash = "sha256:b109c0ad2ececf42e75fa99dc4043ff72a357436bb171900714a9ea581ddef83"},
{file = "ruff-0.9.6-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1de4367cca3dac99bcbd15c161404e849bb0bfd543664db39232648dc00112dc"},
{file = "ruff-0.9.6-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac3ee4d7c2c92ddfdaedf0bf31b2b176fa7aa8950efc454628d477394d35638b"},
{file = "ruff-0.9.6-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5dc1edd1775270e6aa2386119aea692039781429f0be1e0949ea5884e011aa8e"},
{file = "ruff-0.9.6-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:4a091729086dffa4bd070aa5dab7e39cc6b9d62eb2bef8f3d91172d30d599666"},
{file = "ruff-0.9.6-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d1bbc6808bf7b15796cef0815e1dfb796fbd383e7dbd4334709642649625e7c5"},
{file = "ruff-0.9.6-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:589d1d9f25b5754ff230dce914a174a7c951a85a4e9270613a2b74231fdac2f5"},
{file = "ruff-0.9.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc61dd5131742e21103fbbdcad683a8813be0e3c204472d520d9a5021ca8b217"},
{file = "ruff-0.9.6-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:5e2d9126161d0357e5c8f30b0bd6168d2c3872372f14481136d13de9937f79b6"},
{file = "ruff-0.9.6-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:68660eab1a8e65babb5229a1f97b46e3120923757a68b5413d8561f8a85d4897"},
{file = "ruff-0.9.6-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c4cae6c4cc7b9b4017c71114115db0445b00a16de3bcde0946273e8392856f08"},
{file = "ruff-0.9.6-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:19f505b643228b417c1111a2a536424ddde0db4ef9023b9e04a46ed8a1cb4656"},
{file = "ruff-0.9.6-py3-none-win32.whl", hash = "sha256:194d8402bceef1b31164909540a597e0d913c0e4952015a5b40e28c146121b5d"},
{file = "ruff-0.9.6-py3-none-win_amd64.whl", hash = "sha256:03482d5c09d90d4ee3f40d97578423698ad895c87314c4de39ed2af945633caa"},
{file = "ruff-0.9.6-py3-none-win_arm64.whl", hash = "sha256:0e2bb706a2be7ddfea4a4af918562fdc1bcb16df255e5fa595bbd800ce322a5a"},
{file = "ruff-0.9.6.tar.gz", hash = "sha256:81761592f72b620ec8fa1068a6fd00e98a5ebee342a3642efd84454f3031dca9"},
]
[[package]]
@@ -1794,4 +1794,4 @@ propcache = ">=0.2.0"
[metadata]
lock-version = "2.0"
python-versions = "^3.11"
content-hash = "21a0f6e687ee3c43ee58824b834c4dfe3731328c3161270e55924ee4de39dc7d"
content-hash = "a0e91c98c58f0d1b2574b70cde47b53811fb9a913be109b0eaa6d20af4b353ba"
pyproject.toml
+2 -1
View File
@@ -13,13 +13,14 @@ python = "^3.11"
python-dotenv = "^1.0.1"
python-levenshtein = "^0.26.1"
specklepy = "^2.21.0"
ruff = "^0.9.6"
[tool.poetry.group.dev.dependencies]
black = "^25.0.0"
mypy = "^1.3.0"
pydantic-settings = "^2.3.0"
pytest = "^8.0.0"
ruff = "^0.9.5"
ruff = "^0.9.6"
# specklepy = { path = "../specklepy", develop = true }
[build-system]
src/function.py
+10 -4
View File
@@ -2,7 +2,8 @@
Use the automation_context module to wrap your function in an Automate context helper.
"""
import pandas as pd
from pandas import DataFrame
from speckle_automate import AutomationContext, AutomateBase
from src.rules import apply_rules_to_objects
@@ -32,15 +33,20 @@ def automate_function(
flat_list_of_objects = list(flatten_base(version_root_object))
# read the rules from the spreadsheet
rules = read_rules_from_spreadsheet(function_inputs.spreadsheet_url)
rules:DataFrame = read_rules_from_spreadsheet(function_inputs.spreadsheet_url)
if (rules is None) or (len(rules) == 0):
automate_context.mark_run_exception("No rules defined")
grouped_rules = rules.groupby("Rule Number")
# apply the rules to the objects
apply_rules_to_objects(flat_list_of_objects, rules, automate_context)
apply_rules_to_objects(flat_list_of_objects, grouped_rules, automate_context)
# set the automation context view, to the original model / version view
automate_context.set_context_view()
# report success
automate_context.mark_run_success(
f"Successfully applied rules to {len(flat_list_of_objects)} objects."
f"Successfully applied {len(grouped_rules)} rules to {len(flat_list_of_objects)} objects."
)
src/inputs.py
+22
View File
@@ -1,7 +1,14 @@
from enum import Enum
from pydantic import Field
from speckle_automate import AutomateBase
class PropertyMatchMode(Enum):
STRICT = "strict" # Exact parameter path match
FUZZY = "fuzzy" # Search all parameters ignoring hierarchy
MIXED = "mixed" # Exact match first, fuzzy fallback
class FunctionInputs(AutomateBase):
"""These are function author defined values.
@@ -10,8 +17,23 @@ class FunctionInputs(AutomateBase):
https://docs.pydantic.dev/latest/usage/models/
"""
# In this exercise, we will move rules to an external source so not to hardcode them.
spreadsheet_url: str = Field(
title="Spreadsheet URL",
description="This is the URL of the spreadsheet to check. It should be a TSV format data source.",
)
property_match_mode: PropertyMatchMode = Field(
default=PropertyMatchMode.MIXED,
title="Property Match Mode",
description='Controls how strictly parameter names must match. ' +
'STRICT will only match exact parameter paths, ' +
'FUZZY will search all parameters ignoring hierarchy, ' +
'MIXED will exact match first, fuzzy fallback.'
)
src/rules.py
+97 -39
View File
@@ -1,12 +1,16 @@
import re
from typing import Any, cast
from typing import Any
from typing import cast
import pandas as pd
from Levenshtein import ratio
from pandas.core.groupby import DataFrameGroupBy
from speckle_automate import AutomationContext, ObjectResultLevel
from specklepy.objects.base import Base
from src.helpers import speckle_print
from src.inputs import PropertyMatchMode
# We're going to define a set of rules that will allow us to filter and
# process parameters in our Speckle objects. These rules will be encapsulated
@@ -184,6 +188,7 @@ class RevitRules:
def get_parameter_value(
speckle_object: Base,
parameter_name: str,
match_mode: PropertyMatchMode = PropertyMatchMode.MIXED,
default_value: Any = None,
) -> Any | None:
"""Retrieves the value of the specified Revit parameter from the speckle_object.
@@ -207,44 +212,84 @@ class RevitRules:
Returns:
The value of the parameter if it exists and is not None or the specified default_value, or None otherwise.
"""
# Attempt to retrieve the parameter from the root object level
value = getattr(speckle_object, parameter_name, None)
if value not in [None, default_value]:
return value
# Detect version based on structure
is_v3 = hasattr(speckle_object, 'properties') and hasattr(speckle_object.properties, 'Parameters')
# If the "parameters" attribute is a Base object, extract its dynamic members
parameters = getattr(speckle_object, "parameters", None)
if parameters is None:
if is_v3:
return RevitRules._get_v3_parameter(speckle_object, parameter_name, match_mode, default_value)
else:
return RevitRules._get_v2_parameter(speckle_object, parameter_name, match_mode, default_value)
@staticmethod
def _get_v2_parameter(obj: Base, name: str, mode: PropertyMatchMode, default: Any) -> Any:
parameters = getattr(obj, "parameters", None)
if not parameters:
return default
if mode == PropertyMatchMode.STRICT:
return RevitRules.strict(name, parameters, default)
# For mixed/fuzzy, search directly in parameters dict
def search_params(param_dict: dict, search_name: str, fuzzy: bool) -> Any:
for key, value in param_dict.items():
if key.lower() == search_name.lower() or (fuzzy and search_name.lower() in key.lower()):
return value.get('value') if isinstance(value, dict) else value
return None
# Prepare a dictionary of parameter values from the dynamic members of the parameters attribute
parameters_dict = {
key: getattr(parameters, key)
for key in parameters.get_dynamic_member_names()
}
result = search_params(parameters, name, mode == PropertyMatchMode.FUZZY)
return result if result is not None else default
@staticmethod
def strict(name: str, parameters: object, default: Any) -> Any:
path_parts = name.split('.')
current = parameters
for part in path_parts:
if not current or not isinstance(current, dict):
return default
key = next((k for k in current.keys() if k.lower() == part.lower()), None)
if not key:
return default
current = current[key]
return current.get('value', current) if isinstance(current, dict) else current
@staticmethod
def _get_v3_parameter(obj: Base, name: str, mode: PropertyMatchMode, default: Any) -> Any:
parameters = obj["properties"].Parameters
if mode == PropertyMatchMode.STRICT:
return RevitRules.strict(name, parameters, default)
def search_nested(data: dict, search_name: str, fuzzy: bool) -> Any:
for nested_key, value in data.items():
if isinstance(value, dict):
if 'value' in value and (nested_key.lower() == search_name.lower() or
(fuzzy and search_name.lower() in nested_key.lower())):
return value['value']
nested_result = search_nested(value, search_name, fuzzy)
if nested_result is not None:
return nested_result
return None
result = search_nested(parameters, name, mode == PropertyMatchMode.FUZZY)
return result if result is not None else default
# Search for a direct match or a nested match in the parameters dictionary
param_value = parameters_dict.get(parameter_name)
if param_value is not None:
if isinstance(param_value, Base):
# Extract the nested value from a Base object if available
nested_value = getattr(param_value, "value", None)
if nested_value not in [None, default_value]:
return nested_value
elif param_value not in [None, default_value]:
return param_value
# Use a generator to find the first matching 'value' for shared parameters stored in Base objects
return next(
(
getattr(p, "value", None)
for p in parameters_dict.values()
if isinstance(p, Base) and getattr(p, "name", None) == parameter_name
),
None,
)
from typing import Any
@staticmethod
def is_parameter_value(
@@ -582,7 +627,7 @@ def evaluate_condition(speckle_object: Base, condition: pd.Series) -> bool:
def process_rule(
speckle_objects: list[Base], rule_group: pd.DataFrame
) -> tuple[list[Base], list[Base]]:
) -> tuple[list[Any], list[Any]] | tuple[list[Base], list[Base]]:
"""Processes a set of rules against Speckle objects, returning those that pass and fail.
The first rule is used as a filter ('WHERE'), and subsequent rules as conditions ('AND').
@@ -623,7 +668,11 @@ def process_rule(
# Evaluate each filtered object against the 'AND' conditions
for speckle_object in filtered_objects:
if all(
# if filtered_objects is empty
if len(list(filtered_objects)) == 0:
return [],[]
elif all(
evaluate_condition(speckle_object, cond)
for _, cond in subsequent_conditions.iterrows()
):
@@ -636,17 +685,16 @@ def process_rule(
def apply_rules_to_objects(
speckle_objects: list[Base],
rules_df: pd.DataFrame,
grouped_rules: DataFrameGroupBy,
automate_context: AutomationContext,
) -> dict[str, tuple[list[Base], list[Base]]]:
"""Applies defined rules to a list of objects and updates the automate context based on the results.
Args:
speckle_objects (List[Base]): The list of objects to which rules are applied.
rules_df (pd.DataFrame): The DataFrame containing rule definitions.
grouped_rules (pd.DataFrameGroupBy): The DataFrame containing rule definitions.
automate_context (Any): Context manager for attaching rule results.
"""
grouped_rules = rules_df.groupby("Rule Number")
grouped_results = {}
@@ -668,6 +716,14 @@ def apply_rules_to_objects(
attach_results(
fail_objects, rule_group.iloc[-1], rule_id_str, automate_context, False
)
if len(pass_objects) == 0 and len(fail_objects) == 0:
automate_context.attach_info_to_objects(
category=f"Rule {rule_id_str} Skipped",
object_ids=["0"], # This is a hack to get a rule to report with no valid objects
message=f"No objects found for rule {rule_id_str}",
metadata={},
)
# pass
grouped_results[rule_id_str] = (pass_objects, fail_objects)
@@ -680,7 +736,7 @@ def attach_results(
rule_info: pd.Series,
rule_id: str,
context: AutomationContext,
passed: bool,
passed: bool|None,
) -> None:
"""Attaches the results of a rule to the objects in the context.
@@ -691,6 +747,8 @@ def attach_results(
context (AutomationContext): The context manager for attaching results.
passed (bool): Whether the rule passed or failed.
"""
# passed having an explicit None value means that the rule can be marked as skipped
if not speckle_objects:
return
src/spreadsheet.py
+2 -1
View File
@@ -1,7 +1,8 @@
import pandas as pd
from pandas import DataFrame
def read_rules_from_spreadsheet(url):
def read_rules_from_spreadsheet(url: str) -> DataFrame | None:
"""Reads a TSV file from a provided URL and returns a DataFrame.
Args:
tests/conftest.py
-24
View File
@@ -1,24 +0,0 @@
import os
from dotenv import load_dotenv
def pytest_configure(config):
load_dotenv(dotenv_path=".env")
token_var = "SPECKLE_TOKEN"
server_var = "SPECKLE_SERVER_URL"
token = os.getenv(token_var)
server = os.getenv(server_var)
if not token:
raise ValueError(f"Cannot run tests without a {token_var} environment variable")
if not server:
raise ValueError(
f"Cannot run tests without a {server_var} environment variable"
)
# Set the token as an attribute on the config object
config.SPECKLE_TOKEN = token
config.SPECKLE_SERVER_URL = server
tests/localtest.py
+10 -8
View File
@@ -1,24 +1,28 @@
"""Run integration tests with a speckle server."""
from speckle_automate.fixtures import *
from speckle_automate import (
AutomationContext,
AutomationRunData,
AutomationStatus,
run_function
run_function,
)
from src.inputs import FunctionInputs
from src.function import automate_function
from src.helpers import speckle_print
from src.inputs import FunctionInputs
def test_function_run(test_automation_run_data: AutomationRunData, test_automation_token: str):
speckle_print(str(test_automation_run_data))
speckle_print(str(test_automation_token))
"""Run an integration test for the automate function."""
automation_context = AutomationContext.initialize(
test_automation_run_data, test_automation_token
)
default_url: str = (
"https://docs.google.com/spreadsheets/d/e/2PACX-1vSFmjLfqxPKXJHg-wEs1cp_nJEJJhESGVTLCvWLG_"
"IgIuRZ4CmMDCSceOYFvuo8IqcmT4sj9qPiLfCx/pub?gid=0&single=true&output=tsv"
"https://drive.google.com/uc?export=download&id=1hiPSw23eOaqd27QD_YsXvZg9PWm7_XBx"
)
automate_sdk = run_function(
@@ -27,6 +31,4 @@ def test_function_run(test_automation_run_data: AutomationRunData, test_automati
FunctionInputs(spreadsheet_url=default_url),
)
assert automate_sdk.run_status == AutomationStatus.SUCCEEDED
# cli command to run just this test with pytest: pytest tests/local_test_exercise2.py::test_function_run
assert automate_sdk.run_status == AutomationStatus.SUCCEEDED
tests/test_function.py
-31
View File
@@ -1,31 +0,0 @@
"""Run integration tests with a speckle server."""
from pydantic import SecretStr
from speckle_automate import (
AutomationContext,
AutomationRunData,
AutomationStatus,
run_function
)
from main import FunctionInputs, automate_function
from speckle_automate.fixtures import *
def test_function_run(test_automation_run_data: AutomationRunData, test_automation_token: str):
"""Run an integration test for the automate function."""
automation_context = AutomationContext.initialize(
test_automation_run_data, test_automation_token
)
automate_sdk = run_function(
automation_context,
automate_function,
FunctionInputs(
forbidden_speckle_type="None",
whisper_message=SecretStr("testing automatically"),
),
)
assert automate_sdk.run_status == AutomationStatus.SUCCEEDED