diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index da9cd35..657f606 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -48,7 +48,7 @@ jobs: run: uv run pytest --cov --cov-report xml:reports/coverage.xml --junitxml=reports/test-results.xml - uses: codecov/codecov-action@v5 - if: matrix.python-version == 3.13 + if: matrix.python-version == 3.12 with: fail_ci_if_error: true # optional (default = false) files: ./reports/test-results.xml # optional diff --git a/docker-compose.yml b/docker-compose.yml index 111918e..0cdf440 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,7 +13,7 @@ services: POSTGRES_USER: speckle POSTGRES_PASSWORD: speckle volumes: - - postgres-data:/var/lib/postgresql/data/ + - ./.volumes/postgres-data:/var/lib/postgresql/data/ healthcheck: # the -U user has to match the POSTGRES_USER value test: ["CMD-SHELL", "pg_isready -U speckle"] @@ -25,7 +25,7 @@ services: image: "redis:6.0-alpine" restart: always volumes: - - redis-data:/data + - ./.volumes/redis-data:/data healthcheck: test: ["CMD", "redis-cli", "--raw", "incr", "ping"] interval: 5s @@ -37,7 +37,7 @@ services: command: server /data --console-address ":9001" restart: always volumes: - - minio-data:/data + - ./.volumes/minio-data:/data healthcheck: test: [ @@ -48,6 +48,8 @@ services: timeout: 30s retries: 30 start_period: 10s + ports: + - "0.0.0.0:9000:9000" speckle-server: image: speckle/speckle-server:latest @@ -57,7 +59,7 @@ services: - CMD - /nodejs/bin/node - -e - - "try { require('node:http').request({headers: {'Content-Type': 'application/json'}, port:3000, hostname:'127.0.0.1', path:'/readiness', method: 'GET', timeout: 2000 }, (res) => { body = ''; res.on('data', (chunk) => {body += chunk;}); res.on('end', () => {process.exit(res.statusCode != 200 || body.toLowerCase().includes('error'));}); }).end(); } catch { process.exit(1); }" + - "try { require('node:http').request({headers: {'Content-Type': 'application/json'}, port:3000, hostname:'127.0.0.1', path:'/readiness', method: 'GET', timeout: 2000 }, (res) => { body = ''; res.on('data', (chunk) => {body += chunk;}); res.on('end', () => {process.exit(Number(res.statusCode != 200 || body.toLowerCase().includes('error')));}); }).end(); } catch { process.exit(1); }" interval: 10s timeout: 10s retries: 3 @@ -81,6 +83,7 @@ services: REDIS_URL: "redis://redis" S3_ENDPOINT: "http://minio:9000" + S3_PUBLIC_ENDPOINT: "http://127.0.0.1:9000" S3_ACCESS_KEY: "minioadmin" S3_SECRET_KEY: "minioadmin" S3_BUCKET: "speckle-server" @@ -101,6 +104,10 @@ services: POSTGRES_DB: "speckle" ENABLE_MP: "false" + FF_NEXT_GEN_FILE_IMPORTER_ENABLED: "true" + FF_LARGE_FILE_IMPORTS_ENABLED: "true" + FF_BACKGROUND_JOBS_ENABLED: "true" + networks: default: name: speckle-server diff --git a/src/speckle_automate/fixtures.py b/src/speckle_automate/fixtures.py index 5e92974..46397c6 100644 --- a/src/speckle_automate/fixtures.py +++ b/src/speckle_automate/fixtures.py @@ -1,8 +1,5 @@ """Some useful helpers for working with automation data.""" -import secrets -import string - import pytest from gql import gql from pydantic import Field @@ -140,12 +137,6 @@ def test_automation_run_data( return create_test_automation_run_data(speckle_client, test_automation_environment) -def crypto_random_string(length: int) -> str: - """Generate a semi crypto random string of a given length.""" - alphabet = string.ascii_letters + string.digits - return "".join(secrets.choice(alphabet) for _ in range(length)).lower() - - __all__ = [ "test_automation_environment", "test_automation_token", diff --git a/src/speckleifc/__main__.py b/src/speckleifc/__main__.py index 1093c84..bf8ea84 100644 --- a/src/speckleifc/__main__.py +++ b/src/speckleifc/__main__.py @@ -4,14 +4,8 @@ import traceback from argparse import ArgumentParser from os import getenv -from speckleifc.ifc_geometry_processing import open_ifc -from speckleifc.importer import ImportJob +from speckleifc.main import open_and_convert_file from specklepy.core.api.client import SpeckleClient -from specklepy.core.api.credentials import Account -from specklepy.core.api.inputs.version_inputs import CreateVersionInput -from specklepy.core.api.models.current import Version -from specklepy.core.api.operations import send -from specklepy.transports.server import ServerTransport def cmd_line_import() -> None: @@ -32,15 +26,16 @@ def cmd_line_import() -> None: TOKEN = getenv("USER_TOKEN") assert TOKEN is not None SERVER_URL = getenv("SPECKLE_SERVER_URL") or "http://127.0.0.1:3000" - account = Account.from_token(TOKEN, SERVER_URL) try: + client = SpeckleClient(SERVER_URL, use_ssl=not SERVER_URL.startswith("http://")) + client.authenticate_with_token(TOKEN) version = open_and_convert_file( args.file_path, args.project_id, args.version_message, args.model_id, - account, + client, ) with open(args.output_path, "w") as f: json.dump({"success": True, "commitId": version.id}, f) @@ -53,52 +48,6 @@ def cmd_line_import() -> None: json.dump({"success": False, "error": str(e)}, f) -def open_and_convert_file( - file_path: str, - project_id: str, - version_message: str | None, - model_id: str, - account: Account, -) -> Version: - start = time.time() - very_start = start - - ifc_file = open_ifc(file_path) - import_job = ImportJob(ifc_file) - data = import_job.convert() - - print(f"File conversion complete after {(time.time() - start) * 1000}ms") - - start = time.time() - - remote_transport = ServerTransport(project_id, account=account) - - root_id = send(data, transports=[remote_transport], use_default_cache=False) - print(f"Sending to speckle complete after: {(time.time() - start) * 1000}ms") - - start = time.time() - server_url = account.serverInfo.url - assert server_url - client = SpeckleClient(host=server_url, use_ssl=server_url.startswith("https")) - client.authenticate_with_account(account) - - create_version = CreateVersionInput( - object_id=root_id, - model_id=model_id, - project_id=project_id, - message=version_message, - source_application="IFC", - ) - version = client.version.create(create_version) - end = time.time() - print(f"Version committed after: {(end - start) * 1000}ms") - - print(f"Total time (to commit): {(end - very_start) * 1000}ms") - del ifc_file - - return version - - if __name__ == "__main__": start = time.time() cmd_line_import() diff --git a/src/speckleifc/main.py b/src/speckleifc/main.py new file mode 100644 index 0000000..165af75 --- /dev/null +++ b/src/speckleifc/main.py @@ -0,0 +1,55 @@ +import time + +from speckleifc.ifc_geometry_processing import open_ifc +from speckleifc.importer import ImportJob +from specklepy.core.api.client import SpeckleClient +from specklepy.core.api.inputs.version_inputs import CreateVersionInput +from specklepy.core.api.models.current import Version +from specklepy.core.api.operations import send +from specklepy.transports.server import ServerTransport + + +def open_and_convert_file( + file_path: str, + project_id: str, + version_message: str | None, + model_id: str, + client: SpeckleClient, + # account: Account, +) -> Version: + start = time.time() + very_start = start + + account = client.account + server_url = account.serverInfo.url + assert server_url + remote_transport = ServerTransport(project_id, account=account) + + ifc_file = open_ifc(file_path) # pyright: ignore[reportUnknownVariableType] + import_job = ImportJob(ifc_file) # pyright: ignore[reportUnknownArgumentType] + data = import_job.convert() + + print(f"File conversion complete after {(time.time() - start) * 1000}ms") + + start = time.time() + + root_id = send(data, transports=[remote_transport], use_default_cache=False) + print(f"Sending to speckle complete after: {(time.time() - start) * 1000}ms") + + start = time.time() + + create_version = CreateVersionInput( + object_id=root_id, + model_id=model_id, + project_id=project_id, + message=version_message, + source_application="IFC", + ) + version = client.version.create(create_version) + end = time.time() + print(f"Version committed after: {(end - start) * 1000}ms") + + print(f"Total time (to commit): {(end - very_start) * 1000}ms") + del ifc_file + + return version diff --git a/src/specklepy/api/client.py b/src/specklepy/api/client.py index df99d40..975400e 100644 --- a/src/specklepy/api/client.py +++ b/src/specklepy/api/client.py @@ -3,6 +3,7 @@ import contextlib from specklepy.api.credentials import Account from specklepy.api.resources import ( ActiveUserResource, + FileImportResource, ModelResource, OtherUserResource, ProjectInviteResource, @@ -118,6 +119,12 @@ class SpeckleClient(CoreSpeckleClient): client=self.httpclient, server_version=server_version, ) + self.file_import = FileImportResource( + account=self.account, + basepath=self.url, + client=self.httpclient, + server_version=server_version, + ) self.subscription = SubscriptionResource( account=self.account, basepath=self.ws_url, diff --git a/src/specklepy/api/credentials.py b/src/specklepy/api/credentials.py index 0caf26e..c9834d0 100644 --- a/src/specklepy/api/credentials.py +++ b/src/specklepy/api/credentials.py @@ -1,5 +1,3 @@ -from typing import List, Optional - # following imports seem to be unnecessary, but they need to stay # to not break the scripts using these functions as non-core from specklepy.core.api.credentials import ( # noqa: F401 @@ -14,7 +12,7 @@ from specklepy.core.api.credentials import get_local_accounts as core_get_local_ from specklepy.logging import metrics -def get_local_accounts(base_path: Optional[str] = None) -> List[Account]: +def get_local_accounts(base_path: str | None = None) -> list[Account]: """Gets all the accounts present in this environment Arguments: @@ -38,7 +36,7 @@ def get_local_accounts(base_path: Optional[str] = None) -> List[Account]: return accounts -def get_default_account(base_path: Optional[str] = None) -> Optional[Account]: +def get_default_account(base_path: str | None = None) -> Account | None: """ Gets this environment's default account if any. If there is no default, the first found will be returned and set as default. @@ -61,7 +59,7 @@ def get_default_account(base_path: Optional[str] = None) -> Optional[Account]: return default -def get_account_from_token(token: str, server_url: str = None) -> Account: +def get_account_from_token(token: str, server_url: str | None = None) -> Account: """Gets the local account for the token if it exists Arguments: token {str} -- the api token diff --git a/src/specklepy/api/resources/__init__.py b/src/specklepy/api/resources/__init__.py index 9d48704..de92988 100644 --- a/src/specklepy/api/resources/__init__.py +++ b/src/specklepy/api/resources/__init__.py @@ -1,4 +1,5 @@ from specklepy.api.resources.current.active_user_resource import ActiveUserResource +from specklepy.api.resources.current.file_import_resource import FileImportResource from specklepy.api.resources.current.model_resource import ModelResource from specklepy.api.resources.current.other_user_resource import OtherUserResource from specklepy.api.resources.current.project_invite_resource import ( @@ -11,6 +12,7 @@ from specklepy.api.resources.current.version_resource import VersionResource from specklepy.api.resources.current.workspace_resource import WorkspaceResource __all__ = [ + "FileImportResource", "ActiveUserResource", "ModelResource", "OtherUserResource", diff --git a/src/specklepy/api/resources/current/file_import_resource.py b/src/specklepy/api/resources/current/file_import_resource.py new file mode 100644 index 0000000..f2e2735 --- /dev/null +++ b/src/specklepy/api/resources/current/file_import_resource.py @@ -0,0 +1,87 @@ +from pathlib import Path + +from typing_extensions import override + +from specklepy.core.api.inputs import ( + FinishFileImportInput, + GenerateFileUploadUrlInput, + StartFileImportInput, +) +from specklepy.core.api.models import FileImport, FileUploadUrl +from specklepy.core.api.models.current import ResourceCollection +from specklepy.core.api.resources import FileImportResource as CoreResource +from specklepy.core.api.resources.current.file_import_resource import UploadFileResponse +from specklepy.logging import metrics + + +class FileImportResource(CoreResource): + """API Access class for projects""" + + def __init__(self, account, basepath, client, server_version) -> None: + super().__init__( + account=account, + basepath=basepath, + client=client, + server_version=server_version, + ) + + @override + def start_file_import(self, input: StartFileImportInput) -> FileImport: + metrics.track(metrics.SDK, self.account, {"name": "File Import Start"}) + return super().start_file_import(input) + + @override + def generate_upload_url(self, input: GenerateFileUploadUrlInput) -> FileUploadUrl: + """ + Get a file upload url from the Speckle server. + + This method asks the server to create a pre-signed S3 url, + which can be used as a short term authenticated route, + to put a file to the server. + """ + metrics.track( + metrics.SDK, self.account, {"name": "File Import Generate Upload Url"} + ) + return super().generate_upload_url(input) + + @override + def upload_file(self, file: Path, url: str) -> UploadFileResponse: + """ + Uploads a file to the given S3 url. + + This method should be used together with the generate_upload_url method, + which generates a pre-signed S3 url, that can be used to upload the file to. + """ + metrics.track(metrics.SDK, self.account, {"name": "File Import Upload File"}) + return super().upload_file(file, url) + + @override + def download_file(self, project_id: str, file_id: str, target_file: Path) -> Path: + """Download a file blob attached to the project, to the target path.""" + metrics.track(metrics.SDK, self.account, {"name": "File Import Download File"}) + return super().download_file(project_id, file_id, target_file) + + @override + def finish_file_import_job(self, input: FinishFileImportInput) -> bool: + """ + This is mostly an internal api, that marks a file import job finished. + + Only use this if you are writing a file importer, that is responsible for + processing file import jobs. + """ + metrics.track(metrics.SDK, self.account, {"name": "File Import Finish Job"}) + return super().finish_file_import_job(input) + + @override + def get_model_file_import_jobs( + self, + *, + project_id: str, + model_id: str, + limit: int = 25, + cursor: str | None = None, + ) -> ResourceCollection[FileImport]: + metrics.track(metrics.SDK, self.account, {"name": "File Import Get Model Jobs"}) + return super().get_model_file_import_jobs( + project_id=project_id, model_id=model_id, limit=limit, cursor=cursor + ) diff --git a/src/specklepy/core/api/client.py b/src/specklepy/core/api/client.py index 37049c4..d63c3b0 100644 --- a/src/specklepy/core/api/client.py +++ b/src/specklepy/core/api/client.py @@ -11,6 +11,7 @@ from gql.transport.websockets import WebsocketsTransport from specklepy.core.api.credentials import Account from specklepy.core.api.resources import ( ActiveUserResource, + FileImportResource, ModelResource, OtherUserResource, ProjectInviteResource, @@ -230,6 +231,12 @@ class SpeckleClient: client=self.httpclient, server_version=server_version, ) + self.file_import = FileImportResource( + account=self.account, + basepath=self.url, + client=self.httpclient, + server_version=server_version, + ) self.subscription = SubscriptionResource( account=self.account, basepath=self.ws_url, diff --git a/src/specklepy/core/api/credentials.py b/src/specklepy/core/api/credentials.py index 8f3eb66..e08095f 100644 --- a/src/specklepy/core/api/credentials.py +++ b/src/specklepy/core/api/credentials.py @@ -1,6 +1,6 @@ import os from pathlib import Path -from typing import List, Optional +from typing import List from urllib.parse import urlparse from pydantic import BaseModel, Field # pylint: disable=no-name-in-module @@ -12,20 +12,20 @@ from specklepy.transports.sqlite import SQLiteTransport class UserInfo(BaseModel): - id: Optional[str] = None - name: Optional[str] = None - email: Optional[str] = None - company: Optional[str] = None - avatar: Optional[str] = None + id: str | None = None + name: str | None = None + email: str | None = None + company: str | None = None + avatar: str | None = None class Account(BaseModel): isDefault: bool = False - token: Optional[str] = None - refreshToken: Optional[str] = None + token: str | None = None + refreshToken: str | None = None serverInfo: ServerInfo = Field(default_factory=ServerInfo) userInfo: UserInfo = Field(default_factory=UserInfo) - id: Optional[str] = None + id: str | None = None def __repr__(self) -> str: return ( @@ -43,7 +43,7 @@ class Account(BaseModel): return acct -def get_local_accounts(base_path: Optional[str] = None) -> List[Account]: +def get_local_accounts(base_path: str | None = None) -> List[Account]: """Gets all the accounts present in this environment Arguments: @@ -93,7 +93,7 @@ def get_local_accounts(base_path: Optional[str] = None) -> List[Account]: return accounts -def get_default_account(base_path: Optional[str] = None) -> Optional[Account]: +def get_default_account(base_path: str | None = None) -> Account | None: """ Gets this environment's default account if any. If there is no default, the first found will be returned and set as default. @@ -116,7 +116,7 @@ def get_default_account(base_path: Optional[str] = None) -> Optional[Account]: return default -def get_account_from_token(token: str, server_url: str = None) -> Account: +def get_account_from_token(token: str, server_url: str | None = None) -> Account: """Gets the local account for the token if it exists Arguments: token {str} -- the api token diff --git a/src/specklepy/core/api/inputs/__init__.py b/src/specklepy/core/api/inputs/__init__.py index 1cf5ab8..4c7ddb8 100644 --- a/src/specklepy/core/api/inputs/__init__.py +++ b/src/specklepy/core/api/inputs/__init__.py @@ -1,3 +1,10 @@ +from specklepy.core.api.inputs.file_import_inputs import ( + FileImportErrorInput, + FileImportSuccessInput, + FinishFileImportInput, + GenerateFileUploadUrlInput, + StartFileImportInput, +) from specklepy.core.api.inputs.model_inputs import ( CreateModelInput, DeleteModelInput, @@ -22,6 +29,11 @@ from specklepy.core.api.inputs.version_inputs import ( ) __all__ = [ + "FileImportErrorInput", + "FileImportSuccessInput", + "FinishFileImportInput", + "StartFileImportInput", + "GenerateFileUploadUrlInput", "CreateModelInput", "DeleteModelInput", "UpdateModelInput", diff --git a/src/specklepy/core/api/inputs/file_import_inputs.py b/src/specklepy/core/api/inputs/file_import_inputs.py new file mode 100644 index 0000000..aa9f778 --- /dev/null +++ b/src/specklepy/core/api/inputs/file_import_inputs.py @@ -0,0 +1,44 @@ +from typing import Literal + +from pydantic import Field + +from specklepy.core.api.models.graphql_base_model import GraphQLBaseModel + + +class GenerateFileUploadUrlInput(GraphQLBaseModel): + project_id: str + file_name: str + + +class StartFileImportInput(GraphQLBaseModel): + project_id: str + model_id: str + file_id: str + etag: str + + +class FileImportResult(GraphQLBaseModel): + duration_seconds: float + download_duration_seconds: float + parse_duration_seconds: float + parser: str + version_id: str | None + + +class FileImportInputBase(GraphQLBaseModel): + project_id: str + job_id: str + warnings: list[str] = Field(default_factory=list) + result: FileImportResult + + +class FileImportSuccessInput(FileImportInputBase): + status: Literal["success"] = "success" + + +class FileImportErrorInput(FileImportInputBase): + status: Literal["error"] = "error" + reason: str + + +FinishFileImportInput = FileImportSuccessInput | FileImportErrorInput diff --git a/src/specklepy/core/api/models/__init__.py b/src/specklepy/core/api/models/__init__.py index 737d234..c7e102d 100644 --- a/src/specklepy/core/api/models/__init__.py +++ b/src/specklepy/core/api/models/__init__.py @@ -1,5 +1,7 @@ from specklepy.core.api.models.current import ( AuthStrategy, + FileImport, + FileUploadUrl, LimitedUser, Model, ModelWithVersions, @@ -48,4 +50,6 @@ __all__ = [ "ProjectModelsUpdatedMessage", "ProjectUpdatedMessage", "ProjectVersionsUpdatedMessage", + "FileImport", + "FileUploadUrl", ] diff --git a/src/specklepy/core/api/models/current.py b/src/specklepy/core/api/models/current.py index 8f86c55..a8fe64a 100644 --- a/src/specklepy/core/api/models/current.py +++ b/src/specklepy/core/api/models/current.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Generic, List, Optional, TypeVar +from typing import Generic, List, TypeVar from specklepy.core.api.enums import ProjectVisibility from specklepy.core.api.models.graphql_base_model import GraphQLBaseModel @@ -10,13 +10,13 @@ T = TypeVar("T") class User(GraphQLBaseModel): id: str - email: Optional[str] = None + email: str | None = None name: str - bio: Optional[str] = None - company: Optional[str] = None - avatar: Optional[str] = None - verified: Optional[bool] = None - role: Optional[str] = None + bio: str | None = None + company: str | None = None + avatar: str | None = None + verified: bool | None = None + role: str | None = None def __repr__(self): return ( @@ -31,16 +31,16 @@ class User(GraphQLBaseModel): class ResourceCollection(GraphQLBaseModel, Generic[T]): total_count: int items: List[T] - cursor: Optional[str] = None + cursor: str | None = None class ServerMigration(GraphQLBaseModel): - moved_from: Optional[str] - moved_to: Optional[str] + moved_from: str | None + moved_to: str | None class AuthStrategy(GraphQLBaseModel): - color: Optional[str] + color: str | None icon: str id: str name: str @@ -60,17 +60,17 @@ class ServerWorkspacesInfo(GraphQLBaseModel): # Keeping this one all Optionals at the minute, # because its used both as a deserialization model for GQL and Account Management class ServerInfo(GraphQLBaseModel): - name: Optional[str] = None - company: Optional[str] = None - url: Optional[str] = None - admin_contact: Optional[str] = None - description: Optional[str] = None - canonical_url: Optional[str] = None - scopes: Optional[List[dict]] = None - auth_strategies: Optional[List[dict]] = None - version: Optional[str] = None - migration: Optional[ServerMigration] = None - workspaces: Optional[ServerWorkspacesInfo] = None + name: str | None = None + company: str | None = None + url: str | None = None + admin_contact: str | None = None + description: str | None = None + canonical_url: str | None = None + scopes: List[dict] | None = None + auth_strategies: List[dict] | None = None + version: str | None = None + migration: ServerMigration | None = None + workspaces: ServerWorkspacesInfo | None = None # TODO separate gql model from account management model @@ -79,11 +79,11 @@ class LimitedUser(GraphQLBaseModel): id: str name: str - bio: Optional[str] - company: Optional[str] - avatar: Optional[str] - verified: Optional[bool] - role: Optional[str] + bio: str | None + company: str | None + avatar: str | None + verified: bool | None + role: str | None def __repr__(self): return ( @@ -99,15 +99,15 @@ class LimitedUser(GraphQLBaseModel): class PendingStreamCollaborator(GraphQLBaseModel): id: str invite_id: str - stream_id: Optional[str] = None + stream_id: str | None = None projectId: str - stream_name: Optional[str] = None + stream_name: str | None = None project_name: str title: str role: str invited_by: LimitedUser - user: Optional[LimitedUser] = None - token: Optional[str] + user: LimitedUser | None = None + token: str | None def __repr__(self): return ( @@ -127,24 +127,24 @@ class ProjectCollaborator(GraphQLBaseModel): class Version(GraphQLBaseModel): - author_user: Optional[LimitedUser] + author_user: LimitedUser | None created_at: datetime id: str - message: Optional[str] + message: str | None preview_url: str - referenced_object: Optional[str] + referenced_object: str | None """Maybe null if workspaces version history limit has been exceeded""" - source_application: Optional[str] + source_application: str | None class Model(GraphQLBaseModel): - author: Optional[LimitedUser] + author: LimitedUser | None created_at: datetime - description: Optional[str] + description: str | None display_name: str id: str name: str - preview_url: Optional[str] + preview_url: str | None updated_at: datetime @@ -162,14 +162,14 @@ class ProjectPermissionChecks(GraphQLBaseModel): class Project(GraphQLBaseModel): allow_public_comments: bool created_at: datetime - description: Optional[str] + description: str | None id: str name: str - role: Optional[str] + role: str | None source_apps: List[str] updated_at: datetime visibility: ProjectVisibility - workspace_id: Optional[str] + workspace_id: str | None class ProjectWithModels(Project): @@ -191,7 +191,7 @@ class ProjectCommentCollection(ResourceCollection[T], Generic[T]): class UserSearchResultCollection(GraphQLBaseModel): items: List[LimitedUser] - cursor: Optional[str] = None + cursor: str | None = None class PermissionCheckResult(GraphQLBaseModel): @@ -216,15 +216,31 @@ class WorkspaceCreationState(GraphQLBaseModel): class LimitedWorkspace(GraphQLBaseModel): id: str name: str - role: Optional[str] + role: str | None slug: str - logo: Optional[str] - description: Optional[str] + logo: str | None + description: str | None class Workspace(LimitedWorkspace): created_at: datetime updated_at: datetime read_only: bool - creation_state: Optional[WorkspaceCreationState] + creation_state: WorkspaceCreationState | None permissions: WorkspacePermissionChecks + + +class FileImport(GraphQLBaseModel): + id: str + project_id: str + converted_version_id: str | None + user_id: str + converted_status: int + converted_message: str | None + model_id: str | None + updated_at: datetime + + +class FileUploadUrl(GraphQLBaseModel): + url: str + file_id: str diff --git a/src/specklepy/core/api/resources/__init__.py b/src/specklepy/core/api/resources/__init__.py index 619eb9b..b802cd5 100644 --- a/src/specklepy/core/api/resources/__init__.py +++ b/src/specklepy/core/api/resources/__init__.py @@ -1,4 +1,5 @@ from specklepy.core.api.resources.current.active_user_resource import ActiveUserResource +from specklepy.core.api.resources.current.file_import_resource import FileImportResource from specklepy.core.api.resources.current.model_resource import ModelResource from specklepy.core.api.resources.current.other_user_resource import OtherUserResource from specklepy.core.api.resources.current.project_invite_resource import ( @@ -13,6 +14,7 @@ from specklepy.core.api.resources.current.version_resource import VersionResourc from specklepy.core.api.resources.current.workspace_resource import WorkspaceResource __all__ = [ + "FileImportResource", "ActiveUserResource", "ModelResource", "OtherUserResource", diff --git a/src/specklepy/core/api/resources/current/file_import_resource.py b/src/specklepy/core/api/resources/current/file_import_resource.py new file mode 100644 index 0000000..3215ec6 --- /dev/null +++ b/src/specklepy/core/api/resources/current/file_import_resource.py @@ -0,0 +1,212 @@ +from pathlib import Path +from typing import Any + +import httpx +from gql import Client, gql + +from specklepy.core.api.credentials import Account +from specklepy.core.api.inputs.file_import_inputs import ( + FinishFileImportInput, + GenerateFileUploadUrlInput, + StartFileImportInput, +) +from specklepy.core.api.models import FileImport, FileUploadUrl, ResourceCollection +from specklepy.core.api.models.graphql_base_model import GraphQLBaseModel +from specklepy.core.api.resource import ResourceBase +from specklepy.core.api.responses import DataResponse +from specklepy.logging.exceptions import SpeckleException + + +class UploadFileResponse(GraphQLBaseModel): + etag: str + + +class FileImportResource(ResourceBase): + """API Access class for project invites""" + + def __init__( + self, + account: Account, + basepath: str, + client: Client, + server_version: tuple[Any, ...] | None, # pyright: ignore[reportExplicitAny] + ) -> None: + super().__init__( + account=account, + basepath=basepath, + client=client, + server_version=server_version, + name="file-import", + ) + + def finish_file_import_job(self, input: FinishFileImportInput) -> bool: + """ + This is mostly an internal api, that marks a file import job finished. + + Only use this if you are writing a file importer, that is responsible for + processing file import jobs. + """ + QUERY = gql( + """ + mutation FinishFileImport($input: FinishFileImportInput!) { + data:fileUploadMutations { + data:finishFileImport(input: $input) + } + } + """ + ) + + variables = { + "input": input.model_dump(warnings="error", by_alias=True), + } + + return self.make_request_and_parse_response( + DataResponse[DataResponse[bool]], QUERY, variables + ).data.data + + def start_file_import(self, input: StartFileImportInput) -> FileImport: + QUERY = gql( + """ + mutation StartFileImport($input: StartFileImportInput!) { + data:fileUploadMutations { + data:startFileImport(input: $input) { + id + projectId + convertedVersionId + userId + convertedStatus + convertedMessage + modelId + updatedAt + } + } + } + """ + ) + + variables = { + "input": input.model_dump(warnings="error", by_alias=True), + } + + return self.make_request_and_parse_response( + DataResponse[DataResponse[FileImport]], QUERY, variables + ).data.data + + def generate_upload_url(self, input: GenerateFileUploadUrlInput) -> FileUploadUrl: + """ + Get a file upload url from the Speckle server. + + This method asks the server to create a pre-signed S3 url, + which can be used as a short term authenticated route, + to put a file to the server. + """ + QUERY = gql( + """ + mutation GenerateUploadUrl($input: GenerateFileUploadUrlInput!) { + data:fileUploadMutations { + data:generateUploadUrl(input: $input) { + fileId + url + } + } + } + """ + ) + + variables = { + "input": input.model_dump(warnings="error", by_alias=True), + } + + return self.make_request_and_parse_response( + DataResponse[DataResponse[FileUploadUrl]], QUERY, variables + ).data.data + + def upload_file(self, file: Path, url: str) -> UploadFileResponse: + """ + Uploads a file to the given S3 url. + + This method should be used together with the generate_upload_url method, + which generates a pre-signed S3 url, that can be used to upload the file to. + """ + with open(file, "rb") as content: + response = httpx.put( + url, + content=content, # Pass file object directly for streaming + headers={ + "Content-Type": "application/octet-stream", + "Content-Length": str(file.stat().st_size), + }, + ).raise_for_status() + etag = response.headers.get("ETag", None) # pyright: ignore[reportAny] + if not etag: + raise SpeckleException( + "Response does not have an ETag attached to it," + + " cannot use this as an upload" + ) + return UploadFileResponse(etag=str(etag)) # pyright: ignore[reportAny] + + def download_file(self, project_id: str, file_id: str, target_file: Path) -> Path: + """Download a file blob attached to the project, to the target path.""" + if not target_file.parent.exists(): + target_file.parent.mkdir(parents=True) + url = f"{self.basepath}/api/stream/{project_id}/blob/{file_id}" + with httpx.stream( + "GET", url, headers={"Authorization": f"Bearer {self.account.token}"} + ) as response: + _ = response.raise_for_status() + with target_file.open("wb") as f: + for chunk in response.iter_bytes(chunk_size=8192): + _ = f.write(chunk) + return target_file + + def get_model_file_import_jobs( + self, + *, + project_id: str, + model_id: str, + limit: int = 25, + cursor: str | None = None, + ) -> ResourceCollection[FileImport]: + QUERY = gql( + """ + query ModelFileImportJobs( + $projectId: String!, + $modelId: String!, + $input: GetModelUploadsInput + ) { + data:project(id: $projectId) { + data:model(id: $modelId) { + data:uploads(input: $input) { + totalCount + cursor + items { + id + projectId + convertedVersionId + userId + convertedStatus + convertedMessage + modelId + updatedAt + } + } + } + } + } + """ + ) + + variables = { + "projectId": project_id, + "modelId": model_id, + "input": { + "limit": limit, + "cursor": cursor, + }, + } + + return self.make_request_and_parse_response( + DataResponse[DataResponse[DataResponse[ResourceCollection[FileImport]]]], + QUERY, + variables, + ).data.data.data diff --git a/src/specklepy/core/helpers/__init__.py b/src/specklepy/core/helpers/__init__.py index 9056b76..4b45dfa 100644 --- a/src/specklepy/core/helpers/__init__.py +++ b/src/specklepy/core/helpers/__init__.py @@ -1 +1,5 @@ """Common helpers module for Core.""" + +from specklepy.core.helpers.random import crypto_random_string + +__all__ = ["crypto_random_string"] diff --git a/src/specklepy/core/helpers/random.py b/src/specklepy/core/helpers/random.py new file mode 100644 index 0000000..2d21f7a --- /dev/null +++ b/src/specklepy/core/helpers/random.py @@ -0,0 +1,8 @@ +import secrets +import string + + +def crypto_random_string(length: int) -> str: + """Generate a semi crypto random string of a given length.""" + alphabet = string.ascii_letters + string.digits + return "".join(secrets.choice(alphabet) for _ in range(length)).lower() diff --git a/tests/integration/client/current/test_file.ifc b/tests/integration/client/current/test_file.ifc new file mode 100644 index 0000000..fa0a9a7 --- /dev/null +++ b/tests/integration/client/current/test_file.ifc @@ -0,0 +1,61 @@ +ISO-10303-21; +HEADER; +FILE_DESCRIPTION ((''), '2;1'); +FILE_NAME ('', '2020-02-27T18:38:58', (''), (''), 'Processor version 5.1.0.0', 'Xbim.IO.MemoryModel', ''); +FILE_SCHEMA (('IFC4')); +ENDSEC; +DATA; +#1=IFCPROJECT('3WoDmit2L9H8xguu5dNQPk',#2,'W\X\FCrfelEinfach',$,$,$,$,(#19,#22),#7); +#2=IFCOWNERHISTORY(#5,#6,$,.ADDED.,1582828739,$,$,0); +#3=IFCPERSON($,'Team','Finradon',$,$,$,$,$); +#4=IFCORGANIZATION($,'CMS',$,$,$); +#5=IFCPERSONANDORGANIZATION(#3,#4,$); +#6=IFCAPPLICATION(#4,'1.0','W\X\FCrfelEinfach','W\X\FCrfelEinfach.exe'); +#7=IFCUNITASSIGNMENT((#8,#9,#10,#11,#12,#13,#14,#15,#16)); +#8=IFCSIUNIT(*,.LENGTHUNIT.,.MILLI.,.METRE.); +#9=IFCSIUNIT(*,.AREAUNIT.,$,.SQUARE_METRE.); +#10=IFCSIUNIT(*,.VOLUMEUNIT.,$,.CUBIC_METRE.); +#11=IFCSIUNIT(*,.SOLIDANGLEUNIT.,$,.STERADIAN.); +#12=IFCSIUNIT(*,.PLANEANGLEUNIT.,$,.RADIAN.); +#13=IFCSIUNIT(*,.MASSUNIT.,$,.GRAM.); +#14=IFCSIUNIT(*,.TIMEUNIT.,$,.SECOND.); +#15=IFCSIUNIT(*,.THERMODYNAMICTEMPERATUREUNIT.,$,.DEGREE_CELSIUS.); +#16=IFCSIUNIT(*,.LUMINOUSINTENSITYUNIT.,$,.LUMEN.); +#17=IFCCARTESIANPOINT((0.,0.,0.)); +#18=IFCAXIS2PLACEMENT3D(#17,$,$); +#19=IFCGEOMETRICREPRESENTATIONCONTEXT('Building Model','Model',3,1.E-05,#18,$); +#20=IFCCARTESIANPOINT((0.,0.)); +#21=IFCAXIS2PLACEMENT2D(#20,$); +#22=IFCGEOMETRICREPRESENTATIONCONTEXT('Building Plan View','Plan',2,1.E-05,#21,$); +#23=IFCBUILDING('1fOYmUWu5FGA6WZZJzE67P',#2,'Default Building',$,$,#24,$,$,.ELEMENT.,$,$,$); +#24=IFCLOCALPLACEMENT($,#25); +#25=IFCAXIS2PLACEMENT3D(#26,$,$); +#26=IFCCARTESIANPOINT((0.,0.,0.)); +#27=IFCRELAGGREGATES('0yprV4hG98I8MgBrMkyNIg',#2,$,$,#1,(#23)); +#28=IFCBUILDINGELEMENTPROXY('18CFESN5fCsuplarC$2Ulg',#2,'The cube in question',$,$,#38,#37,$,$); +#29=IFCRECTANGLEPROFILEDEF(.AREA.,$,#31,820.,820.); +#30=IFCCARTESIANPOINT((0.,40.)); +#31=IFCAXIS2PLACEMENT2D(#30,$); +#32=IFCEXTRUDEDAREASOLID(#29,#35,#33,820.); +#33=IFCDIRECTION((0.,0.,1.)); +#34=IFCCARTESIANPOINT((0.,0.,0.)); +#35=IFCAXIS2PLACEMENT3D(#34,$,$); +#36=IFCSHAPEREPRESENTATION(#19,'Body','SweptSolid',(#32)); +#37=IFCPRODUCTDEFINITIONSHAPE($,$,(#36,#51)); +#38=IFCLOCALPLACEMENT($,#39); +#39=IFCAXIS2PLACEMENT3D(#34,#41,#40); +#40=IFCDIRECTION((0.,1.,0.)); +#41=IFCDIRECTION((0.,0.,1.)); +#42=IFCMATERIALLAYERSETUSAGE(#43,.AXIS2.,.NEGATIVE.,150.,$); +#43=IFCMATERIALLAYERSET((#44),$,$); +#44=IFCMATERIALLAYER($,10.,$,$,$,$,$); +#45=IFCMATERIAL('Metal + Glass',$,$); +#46=IFCRELASSOCIATESMATERIAL('2NhfPxMdr8_v0TKa3nx$N_',#2,$,$,(#28),#42); +#47=IFCPRESENTATIONLAYERASSIGNMENT('some ifcPresentationLayerAssignment',$,(#36),$); +#48=IFCPOLYLINE((#49,#50)); +#49=IFCCARTESIANPOINT((0.,0.)); +#50=IFCCARTESIANPOINT((4000.,0.)); +#51=IFCSHAPEREPRESENTATION(#19,'Axis','Curve2D',(#48)); +#52=IFCRELCONTAINEDINSPATIALSTRUCTURE('2KE_68CXDAFvue7XfUwVHI',#2,$,$,(#28),#23); +ENDSEC; +END-ISO-10303-21; diff --git a/tests/integration/client/current/test_file_import_resource.py b/tests/integration/client/current/test_file_import_resource.py new file mode 100644 index 0000000..89cbb6a --- /dev/null +++ b/tests/integration/client/current/test_file_import_resource.py @@ -0,0 +1,251 @@ +from pathlib import Path + +import pytest + +from specklepy.api import operations +from specklepy.api.client import SpeckleClient +from specklepy.core.api.enums import ProjectVisibility +from specklepy.core.api.inputs.file_import_inputs import ( + FileImportErrorInput, + FileImportResult, + FileImportSuccessInput, + GenerateFileUploadUrlInput, + StartFileImportInput, +) +from specklepy.core.api.inputs.model_inputs import CreateModelInput +from specklepy.core.api.inputs.project_inputs import ProjectCreateInput +from specklepy.core.api.inputs.version_inputs import CreateVersionInput +from specklepy.core.api.models import Project +from specklepy.core.api.models.current import FileUploadUrl +from specklepy.core.helpers import crypto_random_string +from specklepy.transports.server.server import ServerTransport +from tests.integration.fakemesh import FakeMesh + + +class TestFileImportResource: + @pytest.fixture + def file_path(self) -> Path: + path = Path("./tests/integration/client/current/test_file.ifc").absolute() + assert path.exists() + return path + + @pytest.fixture + def project(self, client: SpeckleClient) -> Project: + return client.project.create( + ProjectCreateInput( + name="test", description=None, visibility=ProjectVisibility.PRIVATE + ) + ) + + @pytest.fixture(scope="function") + def upload_url( + self, project: Project, file_path: Path, client: SpeckleClient + ) -> FileUploadUrl: + upload_url_result = client.file_import.generate_upload_url( + GenerateFileUploadUrlInput(project_id=project.id, file_name=file_path.name) + ) + return upload_url_result + + def test_generate_upload_url(self, upload_url: FileUploadUrl) -> None: + assert upload_url.file_id + assert upload_url.url + + def test_upload_file( + self, file_path: Path, client: SpeckleClient, upload_url: FileUploadUrl + ) -> None: + response = client.file_import.upload_file(file=file_path, url=upload_url.url) + assert response.etag + + def test_download_file( + self, + file_path: Path, + client: SpeckleClient, + project: Project, + upload_url: FileUploadUrl, + ) -> None: + _ = client.file_import.upload_file(file=file_path, url=upload_url.url) + + target_file = file_path.parent.joinpath("download.ifc") + + downloaded_file = client.file_import.download_file( + project_id=project.id, file_id=upload_url.file_id, target_file=target_file + ) + + assert downloaded_file.exists() + + assert file_path.stat().st_size == downloaded_file.stat().st_size + + downloaded_file.unlink() + + def test_start_file_import( + self, + file_path: Path, + client: SpeckleClient, + project: Project, + upload_url: FileUploadUrl, + ) -> None: + model = client.model.create( + CreateModelInput(name=crypto_random_string(10), project_id=project.id) + ) + upload_response = client.file_import.upload_file( + file=file_path, url=upload_url.url + ) + response = client.file_import.start_file_import( + StartFileImportInput( + project_id=project.id, + model_id=model.id, + file_id=upload_url.file_id, + etag=upload_response.etag, + ) + ) + + assert response.converted_status == 0 + assert response.converted_version_id is None + + upload_jobs = client.file_import.get_model_file_import_jobs( + project_id=project.id, + model_id=model.id, + ) + + assert upload_jobs.total_count == 1 + job = upload_jobs.items[0] + assert job + assert job.converted_status == 0 + assert job.converted_version_id is None + + def test_finish_file_import_success( + self, + file_path: Path, + client: SpeckleClient, + project: Project, + upload_url: FileUploadUrl, + mesh: FakeMesh, + ) -> None: + model = client.model.create( + CreateModelInput(name=crypto_random_string(10), project_id=project.id) + ) + upload_response = client.file_import.upload_file( + file=file_path, url=upload_url.url + ) + job_response = client.file_import.start_file_import( + StartFileImportInput( + project_id=project.id, + model_id=model.id, + file_id=upload_url.file_id, + etag=upload_response.etag, + ) + ) + + assert job_response.converted_status == 0 + assert job_response.converted_version_id is None + + upload_jobs = client.file_import.get_model_file_import_jobs( + project_id=project.id, + model_id=model.id, + ) + + assert upload_jobs.total_count == 1 + job = upload_jobs.items[0] + assert job + assert job.converted_status == 0 + assert job.converted_version_id is None + + transport = ServerTransport(client=client, stream_id=project.id) + hash = operations.send(mesh, transports=[transport]) + + version = client.version.create( + input=CreateVersionInput( + project_id=project.id, model_id=model.id, object_id=hash + ) + ) + + finish_result = client.file_import.finish_file_import_job( + input=FileImportSuccessInput( + project_id=project.id, + job_id=job_response.id, + result=FileImportResult( + download_duration_seconds=0, + duration_seconds=0, + parse_duration_seconds=0, + parser="test", + version_id=version.id, + ), + ) + ) + + assert finish_result + + upload_jobs = client.file_import.get_model_file_import_jobs( + project_id=project.id, + model_id=model.id, + ) + + assert upload_jobs.total_count == 1 + job = upload_jobs.items[0] + assert job + assert job.converted_status == 2 + assert job.converted_version_id == version.id + + def test_finish_file_import_error( + self, + file_path: Path, + client: SpeckleClient, + project: Project, + upload_url: FileUploadUrl, + ) -> None: + model = client.model.create( + CreateModelInput(name=crypto_random_string(10), project_id=project.id) + ) + upload_response = client.file_import.upload_file( + file=file_path, url=upload_url.url + ) + job_response = client.file_import.start_file_import( + StartFileImportInput( + project_id=project.id, + model_id=model.id, + file_id=upload_url.file_id, + etag=upload_response.etag, + ) + ) + + assert job_response.converted_status == 0 + assert job_response.converted_version_id is None + + upload_jobs = client.file_import.get_model_file_import_jobs( + project_id=project.id, + model_id=model.id, + ) + + assert upload_jobs.total_count == 1 + job = upload_jobs.items[0] + assert job + assert job.converted_status == 0 + assert job.converted_version_id is None + + finish_result = client.file_import.finish_file_import_job( + input=FileImportErrorInput( + project_id=project.id, + job_id=job_response.id, + reason="Test error", + result=FileImportResult( + download_duration_seconds=0, + duration_seconds=0, + parse_duration_seconds=0, + parser="test", + version_id=None, + ), + ) + ) + + assert finish_result + + upload_jobs = client.file_import.get_model_file_import_jobs( + project_id=project.id, + model_id=model.id, + ) + + assert upload_jobs.total_count == 1 + job = upload_jobs.items[0] + assert job + assert job.converted_status == 3 + assert job.converted_version_id is None diff --git a/tests/integration/speckle_automate/test_automation_context.py b/tests/integration/speckle_automate/test_automation_context.py index 597ee16..fa675d9 100644 --- a/tests/integration/speckle_automate/test_automation_context.py +++ b/tests/integration/speckle_automate/test_automation_context.py @@ -15,11 +15,11 @@ from speckle_automate import ( ) from speckle_automate.fixtures import ( create_test_automation_run_data, - crypto_random_string, ) from speckle_automate.schema import AutomateBase from specklepy.api.client import SpeckleClient from specklepy.core.api.models.current import Model, Version +from specklepy.core.helpers import crypto_random_string from specklepy.objects.base import Base