diff --git a/.circleci/build_publish_fe2_sourcemaps.sh b/.circleci/build_publish_fe2_sourcemaps.sh new file mode 100755 index 000000000..fa7e37693 --- /dev/null +++ b/.circleci/build_publish_fe2_sourcemaps.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -eo pipefail + +GIT_ROOT="$(git rev-parse --show-toplevel)" +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +# shellcheck disable=SC1090,SC1091 +source "${SCRIPT_DIR}/common.sh" + +FE2_DIR_PATH="${FE2_DIR_PATH:-"packages/frontend-2"}" +FE2_DATADOG_SERVICE="${FE2_DATADOG_SERVICE:-"web-app-2"}" +DATADOG_SITE="${DATADOG_SITE:-"datadoghq.eu"}" + +if [[ -z "${DATADOG_API_KEY}" ]]; then + echo "DATADOG_API_KEY is not set" + exit 1 +fi + +# Build same prod docker image just w/ sourcemaps enabled +export DOCKER_BUILDKIT=1 +docker build --build-arg BUILD_SOURCEMAPS=true --build-arg SPECKLE_SERVER_VERSION="${IMAGE_VERSION_TAG}" --tag "${DOCKER_IMAGE_TAG}:${IMAGE_VERSION_TAG}-sourcemaps" --file "${FE2_DIR_PATH}/Dockerfile" . +container_id=$(docker create "${DOCKER_IMAGE_TAG}:${IMAGE_VERSION_TAG}-sourcemaps") + +# Clean target location and copy sourcemaps into it +rm -rf "${GIT_ROOT}/${FE2_DIR_PATH}/.output" +docker cp "$container_id":/speckle-server "${GIT_ROOT}/${FE2_DIR_PATH}/.output" +docker rm "$container_id" + +# Publish sourcemaps +pushd "${GIT_ROOT}/${FE2_DIR_PATH}" +DATADOG_SITE="${DATADOG_SITE}" npx --yes @datadog/datadog-ci sourcemaps upload ./.output/public/_nuxt \ +--service="${FE2_DATADOG_SERVICE}" \ +--release-version="${IMAGE_VERSION_TAG}" \ +--minified-path-prefix=/_nuxt +popd + +# Clean up +rm -rf "${GIT_ROOT}/${FE2_DIR_PATH}/.output" diff --git a/.circleci/config.yml b/.circleci/config.yml index 76dc8cc62..c1ce215bb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,7 +2,7 @@ version: 2.1 orbs: snyk: snyk/snyk@2.0.3 - # codecov: codecov/codecov@4.1.0 + codecov: codecov/codecov@5.0.3 workflows: test-build: @@ -20,6 +20,7 @@ workflows: context: - speckle-server-licensing - stripe-integration + - speckle-server-codecov filters: &filters-allow-all tags: # run tests for any commit on any branch, including any tags @@ -43,6 +44,9 @@ workflows: - test-objectsender: filters: *filters-allow-all + - test-shared: + filters: *filters-allow-all + - test-preview-service: filters: *filters-allow-all @@ -183,6 +187,7 @@ workflows: - test-frontend-2 - test-viewer - test-objectsender + - test-shared - test-server - test-server-no-ff - test-server-multiregion @@ -199,11 +204,20 @@ workflows: - test-frontend-2 - test-viewer - test-objectsender + - test-shared - test-server - test-server-no-ff - test-server-multiregion - test-preview-service + - docker-publish-frontend-2-sourcemaps: + context: + - github-readonly-public-repos + - datadog-sourcemaps-publish + filters: *filters-publish + requires: + - get-version + - docker-publish-webhooks: context: *docker-hub-context filters: *filters-publish @@ -215,6 +229,7 @@ workflows: - test-frontend-2 - test-viewer - test-objectsender + - test-shared - test-server - test-server-no-ff - test-server-multiregion @@ -231,6 +246,7 @@ workflows: - test-frontend-2 - test-viewer - test-objectsender + - test-shared - test-server - test-server-no-ff - test-server-multiregion @@ -247,6 +263,7 @@ workflows: - test-frontend-2 - test-viewer - test-objectsender + - test-shared - test-server - test-server-no-ff - test-server-multiregion @@ -263,6 +280,7 @@ workflows: - test-frontend-2 - test-viewer - test-objectsender + - test-shared - test-server - test-server-no-ff - test-server-multiregion @@ -285,6 +303,7 @@ workflows: - test-frontend-2 - test-viewer - test-objectsender + - test-shared - test-server - test-server-no-ff - test-server-multiregion @@ -301,6 +320,7 @@ workflows: - test-frontend-2 - test-viewer - test-objectsender + - test-shared - test-server - test-server-no-ff - test-server-multiregion @@ -348,6 +368,7 @@ workflows: - test-frontend-2 - test-viewer - test-objectsender + - test-shared - test-preview-service - publish-viewer-sandbox-cloudflare-pages: @@ -357,16 +378,6 @@ workflows: requires: - test-viewer - - frontend-2-sourcemaps: - context: - - datadog-sourcemaps-publish - filters: *filters-publish - requires: - - get-version - - docker-build-frontend-2 - - test-frontend-2 - - publish-helm-chart - jobs: get-version: docker: &docker-base-image @@ -511,9 +522,10 @@ jobs: - run: command: yarn test:report working_directory: 'packages/server' + no_output_timeout: 20m - # - codecov/upload: - # file: packages/server/coverage/lcov.info + - codecov/upload: + files: packages/server/coverage/lcov.info - run: name: Introspect GQL schema for subsequent checks @@ -728,6 +740,36 @@ jobs: command: yarn test working_directory: 'packages/preview-service' + test-shared: + docker: *docker-node-browsers-image + resource_class: medium+ + steps: + - checkout + - restore_cache: + name: Restore Yarn Package Cache + keys: + - yarn-packages-server-{{ checksum "yarn.lock" }} + - run: + name: Install Dependencies + command: yarn + + - save_cache: + name: Save Yarn Package Cache + key: yarn-packages-server-{{ checksum "yarn.lock" }} + paths: + - .yarn/cache + - .yarn/unplugged + + - run: + name: Lint + command: yarn lint:ci + working_directory: 'packages/shared' + + - run: + name: Run tests + command: yarn test:single-run + working_directory: 'packages/shared' + test-objectsender: docker: *docker-node-browsers-image resource_class: large @@ -1004,6 +1046,24 @@ jobs: environment: SPECKLE_SERVER_PACKAGE: frontend-2 + docker-publish-frontend-2-sourcemaps: + docker: *docker-node-image + resource_class: xlarge + working_directory: *work-dir + environment: + SPECKLE_SERVER_PACKAGE: frontend-2 + steps: + - checkout + - attach_workspace: + at: /tmp/ci/workspace + - run: cat workspace/env-vars >> $BASH_ENV + - setup_remote_docker: + version: default + docker_layer_caching: true + - run: + name: Build and Save + command: ./.circleci/build_publish_fe2_sourcemaps.sh + docker-build-previews: <<: *build-job environment: @@ -1215,36 +1275,3 @@ jobs: environment: CLOUDFLARE_PAGES_PROJECT_NAME: viewer VIEWER_SANDBOX_DIR_PATH: packages/viewer-sandbox - - frontend-2-sourcemaps: - docker: *docker-node-image - resource_class: large - working_directory: *work-dir - steps: - - checkout - - attach_workspace: - at: /tmp/ci/workspace - - run: cat workspace/env-vars >> $BASH_ENV - - restore_cache: - name: Restore Yarn Package Cache - keys: - - yarn-packages-server-{{ checksum "yarn.lock" }} - - run: - name: Install Dependencies - command: yarn - - save_cache: - name: Save Yarn Package Cache - key: yarn-packages-server-{{ checksum "yarn.lock" }} - paths: - - .yarn/cache - - .yarn/unplugged - - run: - name: Build public packages - command: yarn build:public - - run: - name: Build FE2 - command: NODE_ENV=production SPECKLE_SERVER_VERSION="${IMAGE_VERSION_TAG}" yarn build:sourcemaps - working_directory: 'packages/frontend-2' - - run: - name: Upload source maps - command: ./.circleci/publish_fe2_sourcemaps.sh diff --git a/.circleci/publish_fe2_sourcemaps.sh b/.circleci/publish_fe2_sourcemaps.sh deleted file mode 100755 index aa51b2beb..000000000 --- a/.circleci/publish_fe2_sourcemaps.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -GIT_ROOT="$(git rev-parse --show-toplevel)" -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -# shellcheck disable=SC1090,SC1091 -source "${SCRIPT_DIR}/common.sh" - -FE2_DIR_PATH="${FE2_DIR_PATH:-"packages/frontend-2"}" -FE2_DATADOG_SERVICE="${FE2_DATADOG_SERVICE:-"web-app-2"}" - -if [[ -z "${DATADOG_API_KEY}" ]]; then - echo "DATADOG_API_KEY is not set" - exit 1 -fi - -pushd "${GIT_ROOT}/${FE2_DIR_PATH}" -DATADOG_SITE="${DATADOG_SITE:-"datadoghq.eu"}" yarn datadog-ci sourcemaps upload ./.output/public/_nuxt \ ---service="${FE2_DATADOG_SERVICE}" \ ---release-version="${IMAGE_VERSION_TAG}" \ ---minified-path-prefix=/_nuxt -popd diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index a2e4c1b15..418edea78 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -2,21 +2,48 @@ // README at: https://github.com/devcontainers/templates/tree/main/src/typescript-node { "name": "Node.js & TypeScript", - // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile - "image": "mcr.microsoft.com/devcontainers/typescript-node:1-18-bullseye", + + // We want to deploy all the dependencies (Postgres, Redis, etc.) alongside the devcontainer + // so we can run the app in the container. + "dockerComposeFile": [ + "../docker-compose-deps.yml", + "./docker-compose-devcontainer.yml" + ], + // name of the service within the docker-compose file which devcontainer tools should connect to + "service": "devcontainer", + //services in the docker-compose file which should run when the devcontainer starts + //"runServices": [] // defaults to all + + // Path to the workspace within the container. + // Needs to match destination volume ('volumes' property) in docker-compose-devcontainer.yml. + "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}", // Features to add to the dev container. More info: https://containers.dev/features. - // "features": {}, + "features": { + "ghcr.io/devcontainers/features/docker-outside-of-docker:1": {} + }, - // Use 'forwardPorts' to make a list of ports inside the container available locally. - // "forwardPorts": [], + // Use 'forwardPorts' to make a list of ports inside the container available locally (e.g. via your browser on your desktop) + "forwardPorts": [ + 3000, //speckle server + 6006, //storybook + 8081 //speckle frontend + ], // Use 'postCreateCommand' to run commands after the container is created. - "postCreateCommand": "yarn && yarn build:public" + "postCreateCommand": "/workspaces/${localWorkspaceFolderBasename}/.devcontainer/postCreateCommand.sh", // Configure tool-specific properties. // "customizations": {}, // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. // "remoteUser": "root" + + "hostRequirements": { + "cpus": 4, + "memory": "8gb" + }, + + // Environment variables to set in the container. + "remoteEnv": { "LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}" } } diff --git a/.devcontainer/docker-compose-devcontainer.yml b/.devcontainer/docker-compose-devcontainer.yml new file mode 100644 index 000000000..6562ba975 --- /dev/null +++ b/.devcontainer/docker-compose-devcontainer.yml @@ -0,0 +1,194 @@ +services: + devcontainer: + image: 'mcr.microsoft.com/devcontainers/typescript-node:1-18-bullseye' + volumes: + - ..:/workspaces:cached # mounts our local directory (git root) to the container in /workspaces directory (with access mode of cached) + + # We need to override all node_modules directories + # by mounting an empty volume to avoid conflicting + # CPU build instructions between the host machine and the DevContainer's docker image + # (this allows the devcontainer to be based on linux yet work on Apple Silicon etc..) + # If you add a new package with a new `node_modules`, it needs to be added here + - type: volume + source: node_modules # top-level volume + target: /workspaces/speckle-server/node_modules + read_only: false + - type: volume + source: dui3-node_modules + target: /workspaces/speckle-server/packages/dui3/node_modules + read_only: false + - type: volume + source: fileimport-service-node_modules + target: /workspaces/speckle-server/packages/fileimport-service/node_modules + read_only: false + - type: volume + source: frontend-2-node_modules + target: /workspaces/speckle-server/packages/frontend-2/node_modules + read_only: false + - type: volume + source: monitor-deployment-node_modules + target: /workspaces/speckle-server/packages/monitor-deployment/node_modules + read_only: false + - type: volume + source: objectloader-node_modules + target: /workspaces/speckle-server/packages/objectloader/node_modules + read_only: false + - type: volume + source: objectsender-node_modules + target: /workspaces/speckle-server/packages/objectsender/node_modules + read_only: false + - type: volume + source: preview-frontend-node_modules + target: /workspaces/speckle-server/packages/preview-frontend/node_modules + read_only: false + - type: volume + source: preview-service-node_modules + target: /workspaces/speckle-server/packages/preview-service/node_modules + read_only: false + - type: volume + source: server-node_modules + target: /workspaces/speckle-server/packages/server/node_modules + read_only: false + - type: volume + source: shared-node_modules + target: /workspaces/speckle-server/packages/shared/node_modules + read_only: false + - type: volume + source: tailwind-theme-node_modules + target: /workspaces/speckle-server/packages/tailwind-theme/node_modules + read_only: false + - type: volume + source: ui-components-node_modules + target: /workspaces/speckle-server/packages/ui-components/node_modules + read_only: false + - type: volume + source: ui-components-nuxt-node_modules + target: /workspaces/speckle-server/packages/ui-components-nuxt/node_modules + read_only: false + - type: volume + source: viewer-node_modules + target: /workspaces/speckle-server/packages/viewer/node_modules + read_only: false + - type: volume + source: viewer-sandbox-node_modules + target: /workspaces/speckle-server/packages/viewer-sandbox/node_modules + read_only: false + - type: volume + source: webhook-service-node_modules + target: /workspaces/speckle-server/packages/webhook-service/node_modules + read_only: false + command: sleep infinity + depends_on: + - init + # 'host' network_mode makes services provided via docker-compose-deps available at 127.0.0.1 or localhost. + # They are not available via Docker Compose DNS resolution of the service name e.g. `postgres`. + # This is to ensure that the .env file used for local development remains compatible when running in devcontainer + network_mode: host + + # The mcr.microsoft.com/devcontainers/typescript-node docker image used for devcontainer + # runs as user 'node' and not as 'root' + # The mounted volumes (for the node_module CPU architecture build hack) are owned by 'root' by default + # This container will chown these node_modules volumes to the 'node' user + init: + image: debian:bookworm-slim + restart: 'no' + entrypoint: | + /bin/bash -c "groupadd --gid 1000 node && useradd --uid 1000 --gid node --shell /bin/bash --create-home node && find . -type d -name 'node_modules' | grep '\/node_modules$' | xargs chown node:node {}" + volumes: + # We need to override all node_modules directories + # by mounting an empty volume to avoid conflicting + # CPU build instructions between the host machine and the DevContainer's docker image + # (this allows the devcontainer to be based on linux yet work on Apple Silicon etc..) + # If you add a new package with a new `node_modules`, it needs to be added here + - type: volume + source: node_modules + target: /workspaces/speckle-server/node_modules + read_only: false + - type: volume + source: dui3-node_modules + target: /workspaces/speckle-server/packages/dui3/node_modules + read_only: false + - type: volume + source: fileimport-service-node_modules + target: /workspaces/speckle-server/packages/fileimport-service/node_modules + read_only: false + - type: volume + source: frontend-2-node_modules + target: /workspaces/speckle-server/packages/frontend-2/node_modules + read_only: false + - type: volume + source: monitor-deployment-node_modules + target: /workspaces/speckle-server/packages/monitor-deployment/node_modules + read_only: false + - type: volume + source: objectloader-node_modules + target: /workspaces/speckle-server/packages/objectloader/node_modules + read_only: false + - type: volume + source: objectsender-node_modules + target: /workspaces/speckle-server/packages/objectsender/node_modules + read_only: false + - type: volume + source: preview-frontend-node_modules + target: /workspaces/speckle-server/packages/preview-frontend/node_modules + read_only: false + - type: volume + source: preview-service-node_modules + target: /workspaces/speckle-server/packages/preview-service/node_modules + read_only: false + - type: volume + source: server-node_modules + target: /workspaces/speckle-server/packages/server/node_modules + read_only: false + - type: volume + source: shared-node_modules + target: /workspaces/speckle-server/packages/shared/node_modules + read_only: false + - type: volume + source: tailwind-theme-node_modules + target: /workspaces/speckle-server/packages/tailwind-theme/node_modules + read_only: false + - type: volume + source: ui-components-node_modules + target: /workspaces/speckle-server/packages/ui-components/node_modules + read_only: false + - type: volume + source: ui-components-nuxt-node_modules + target: /workspaces/speckle-server/packages/ui-components-nuxt/node_modules + read_only: false + - type: volume + source: viewer-node_modules + target: /workspaces/speckle-server/packages/viewer/node_modules + read_only: false + - type: volume + source: viewer-sandbox-node_modules + target: /workspaces/speckle-server/packages/viewer-sandbox/node_modules + read_only: false + - type: volume + source: webhook-service-node_modules + target: /workspaces/speckle-server/packages/webhook-service/node_modules + read_only: false + +volumes: + # We need to override all node_modules directories + # by mounting an empty volume to avoid conflicting + # CPU build instructions between the host machine and the DevContainer's docker image + # (this allows the devcontainer to be based on linux yet work on Apple Silicon etc..) + # If you add a new package with a new `node_modules`, it needs to be added here + node_modules: + dui3-node_modules: + fileimport-service-node_modules: + frontend-2-node_modules: + monitor-deployment-node_modules: + objectloader-node_modules: + objectsender-node_modules: + preview-frontend-node_modules: + preview-service-node_modules: + server-node_modules: + shared-node_modules: + tailwind-theme-node_modules: + ui-components-node_modules: + ui-components-nuxt-node_modules: + viewer-node_modules: + viewer-sandbox-node_modules: + webhook-service-node_modules: diff --git a/.devcontainer/postCreateCommand.sh b/.devcontainer/postCreateCommand.sh new file mode 100755 index 000000000..7b8817014 --- /dev/null +++ b/.devcontainer/postCreateCommand.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -eox pipefail + +echo "Running postCreateCommand.sh" + +# determine where the script is located, navigate into that directory, then find the root of the git repo in which it is located +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cd "${SCRIPT_DIR}" +GIT_ROOT="$(git rev-parse --show-toplevel)" + +echo "Setting up environment variables by copying .env files" +cp -n "${GIT_ROOT}/packages/server/.env-example" "${GIT_ROOT}/packages/server/.env" || true +cp -n "${GIT_ROOT}/packages/frontend-2/.env.example" "${GIT_ROOT}/packages/frontend-2/.env" || true + +echo "Installing nodejs dependencies and building shared packages" +yarn +yarn build:public diff --git a/.github/workflows/preview-service-acceptance.yml b/.github/workflows/preview-service-acceptance.yml deleted file mode 100644 index aaea1b056..000000000 --- a/.github/workflows/preview-service-acceptance.yml +++ /dev/null @@ -1,125 +0,0 @@ -name: Preview service acceptance test - -on: - workflow_dispatch: - pull_request: # Pushing a new commit to the HEAD ref of a pull request will trigger the “synchronize” event - paths: - - .yarnrc.yml . - - .yarn - - package.json - - '.github/workflows/preview-service-acceptance.yml' - - 'packages/frontend-2/type-augmentations/stubs/**/*' - - 'packages/preview-service/**/*' - - 'packages/viewer/**/*' - - 'packages/objectloader/**/*' - - 'packages/shared/**/*' - -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository_owner }}/speckle-preview-service - OUTPUT_FILE_PATH: 'preview-service-output/${{ github.sha }}.png' - -jobs: - build-preview-service: - name: Build Preview Service - runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: read - packages: write # publishing container to GitHub registry - - steps: - - uses: actions/checkout@v4 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Log in to the Container registry - uses: docker/login-action@v3.3.0 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v5.5.1 - with: - tags: type=sha,format=long - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - name: Build and load preview-service Docker image - uses: docker/build-push-action@v6 - with: - context: . - file: ./packages/preview-service/Dockerfile - push: true - tags: ${{ steps.meta.outputs.tags }} - cache-from: type=gha - cache-to: type=gha,mode=max - outputs: - tags: ${{ steps.meta.outputs.tags }} - - preview-service-acceptance: - name: Preview Service Acceptance test - runs-on: ubuntu-latest - needs: build-preview-service - - permissions: - contents: write # to update the screenshot saved in the branch. This is a HACK as GitHub API does not yet support uploading attachments to a comment. - pull-requests: write # to write a comment on the PR - packages: read # to download the preview-service image - - services: - postgres: - # Docker Hub image - image: postgres:16.4-bookworm@sha256:e62fbf9d3e2b49816a32c400ed2dba83e3b361e6833e624024309c35d334b412 - env: - POSTGRES_DB: preview_service_test - POSTGRES_PASSWORD: preview_service_test - POSTGRES_USER: preview_service_test - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - preview-service: - image: ${{ needs.build-preview-service.outputs.tags }} - env: - # note that the host is the postgres service name - PG_CONNECTION_STRING: postgres://preview_service_test:preview_service_test@postgres:5432/preview_service_test - - steps: - - uses: actions/checkout@v4 - - name: Install Node.js - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'yarn' - - name: Install dependencies - working-directory: packages/preview-service - run: yarn install - - - name: Run the acceptance test - working-directory: packages/preview-service - run: yarn test:acceptance - env: - NODE_ENV: test - TEST_DB: preview_service_test - # note that the host is localhost, but the port is the port mapped to the postgres service - PG_CONNECTION_STRING: postgres://preview_service_test:preview_service_test@localhost:5432/preview_service_test - OUTPUT_FILE_PATH: ${{ env.OUTPUT_FILE_PATH }} - S3_BUCKET: ${{ vars.S3_BUCKET }} - S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }} - S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }} - S3_ENDPOINT: ${{ vars.S3_ENDPOINT }} - S3_REGION: ${{ vars.S3_REGION }} - - - uses: actions/github-script@v7 - with: - script: | - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: '📸 Preview service has generated an image.' - }) diff --git a/.gitignore b/.gitignore index 72bf3f9d3..97264aa4d 100644 --- a/.gitignore +++ b/.gitignore @@ -71,6 +71,8 @@ redis-data/ .tshy-build obj/ bin/ +!packages/fileimport-service/src/obj +!packages/fileimport-service/bin !packages/monitor-deployment/bin !packages/preview-service/bin !packages/server/bin diff --git a/.zed/settings.json b/.zed/settings.json new file mode 100644 index 000000000..79860b3a9 --- /dev/null +++ b/.zed/settings.json @@ -0,0 +1,5 @@ +// Folder-specific settings +// +// For a full list of overridable settings, and general information on folder-specific settings, +// see the documentation: https://zed.dev/docs/configuring-zed#settings-files +{} diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 000000000..9ebf3bec0 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,4 @@ +codecov: + notify: + notify_error: true + require_ci_to_pass: false diff --git a/docker-compose-deps.yml b/docker-compose-deps.yml index e6e59ea02..899019887 100644 --- a/docker-compose-deps.yml +++ b/docker-compose-deps.yml @@ -1,4 +1,3 @@ -version: '3' services: # Actual Speckle Server dependencies diff --git a/docker-compose-speckle.yml b/docker-compose-speckle.yml index d1f6790f9..658d2ccc2 100644 --- a/docker-compose-speckle.yml +++ b/docker-compose-speckle.yml @@ -1,4 +1,3 @@ -version: '2.4' services: speckle-ingress: build: @@ -22,12 +21,16 @@ services: restart: always environment: NUXT_PUBLIC_SERVER_NAME: 'local' + #TODO: Change this to the URL of your server. This is the URL of the server as accessed by users. NUXT_PUBLIC_API_ORIGIN: 'http://127.0.0.1' + #TODO: Change this to the URL of your server. This is the URL of the server as accessed by users. NUXT_PUBLIC_BASE_URL: 'http://127.0.0.1' + # This is the URL of the server as accessed via this docker compose network. NUXT_PUBLIC_BACKEND_API_ORIGIN: 'http://speckle-server:3000' NUXT_PUBLIC_LOG_LEVEL: 'warn' NUXT_REDIS_URL: 'redis://redis' LOG_LEVEL: 'info' + LOG_PRETTY: 'true' speckle-server: build: @@ -47,14 +50,20 @@ services: retries: 3 start_period: 90s environment: - # TODO: Change this to the URL of the speckle server, as accessed from the network + # TODO. Change this to the url of your server. This is the URL of the server as accessed by users. CANONICAL_URL: 'http://127.0.0.1' + # This is the URL of the server as accessed by other Speckle services within this docker compose network, such as preview-service. + # This will be the same value as NUXT_PUBLIC_BACKEND_API_ORIGIN as defined in the frontend-2 service. + PRIVATE_OBJECTS_SERVER_URL: 'http://speckle-server:3000' # TODO: Change this to a unique secret for this server SESSION_SECRET: 'TODO:Replace' + # This is the authentication strategy to use. Local (i.e. username & password) is the default strategy. STRATEGY_LOCAL: 'true' + LOG_LEVEL: 'info' + LOG_PRETTY: 'true' POSTGRES_URL: 'postgres' POSTGRES_USER: 'speckle' @@ -62,6 +71,8 @@ services: POSTGRES_DB: 'speckle' REDIS_URL: 'redis://redis' + PREVIEW_SERVICE_USE_PRIVATE_OBJECTS_SERVER_URL: 'true' + PREVIEW_SERVICE_REDIS_URL: 'redis://redis' S3_ENDPOINT: 'http://minio:9000' S3_ACCESS_KEY: 'minioadmin' @@ -85,10 +96,11 @@ services: mem_limit: '3000m' memswap_limit: '3000m' environment: - HOST: '127.0.0.1' # Only accept connections from localhost, as preview service does not need to be exposed outside the container. - METRICS_HOST: '127.0.0.1' # Amend if you want to expose Prometheus metrics outside of the container + HOST: '127.0.0.1' # The preview service does not need to be exposed outside the container. + PORT: '3001' LOG_LEVEL: 'info' - PG_CONNECTION_STRING: 'postgres://speckle:speckle@postgres/speckle' + LOG_PRETTY: 'true' + REDIS_URL: 'redis://redis' webhook-service: build: @@ -99,6 +111,7 @@ services: restart: always environment: LOG_LEVEL: 'info' + LOG_PRETTY: 'true' PG_CONNECTION_STRING: 'postgres://speckle:speckle@postgres/speckle' fileimport-service: @@ -110,6 +123,7 @@ services: restart: always environment: LOG_LEVEL: 'info' + LOG_PRETTY: 'true' PG_CONNECTION_STRING: 'postgres://speckle:speckle@postgres/speckle' SPECKLE_SERVER_URL: 'http://speckle-server:3000' FILE_IMPORT_TIME_LIMIT_MIN: 10 diff --git a/package.json b/package.json index 9b5c2d67b..832d380ab 100644 --- a/package.json +++ b/package.json @@ -63,6 +63,7 @@ "prettier": "^2.5.1", "ts-node": "^10.9.1", "tsconfig-paths": "^4.0.0", + "vitest": "^3.0.7", "zx": "^8.1.2" }, "resolutions": { @@ -72,12 +73,12 @@ "@datadog/datadog-ci/ws": "^7.5.10", "@microsoft/api-extractor/semver": "^7.5.4", "@rushstack/node-core-library/semver": "^7.5.4", - "@typescript-eslint/eslint-plugin": "^7.12.0", - "@typescript-eslint/parser": "^7.12.0", + "@typescript-eslint/eslint-plugin": "^8.20.0", + "@typescript-eslint/parser": "^8.20.0", "@types/react": "file:./packages/frontend-2/type-augmentations/stubs/types__react", "core-js": "3.22.4", "core-js-compat/semver": "^7.5.4", - "eslint": "^9.4.0", + "eslint": "^9.20.1", "eslint-config-prettier": "^9.1.0", "levelup/bl": ">=1.2.3", "levelup/semver": ">=5.7.2", @@ -88,9 +89,11 @@ "rollup-plugin-terser/serialize-javascript": ">=6.0.2", "simple-update-notifier/semver": "^7.5.4", "tslib": "^2.3.1", - "typescript": "^5.2.2", - "typescript-eslint": "^7.12.0", - "wait-on": ">=7.2.0" + "typescript": "^5.7.3", + "typescript-eslint": "^8.20.0", + "wait-on": ">=7.2.0", + "vue-tsc@npm:2.2.2/@vue/language-core": "2.2.0", + "vitest": "^3.0.7" }, "config": { "commitizen": { diff --git a/packages/dui3/eslint.config.mjs b/packages/dui3/eslint.config.mjs index 859b12062..efa50520e 100644 --- a/packages/dui3/eslint.config.mjs +++ b/packages/dui3/eslint.config.mjs @@ -57,7 +57,7 @@ const configs = await withNuxt([ '@typescript-eslint/no-for-in-array': ['error'], '@typescript-eslint/restrict-plus-operands': ['error'], '@typescript-eslint/await-thenable': ['warn'], - '@typescript-eslint/ban-types': ['warn'], + '@typescript-eslint/no-restricted-types': ['warn'], 'require-await': 'off', '@typescript-eslint/require-await': 'error', 'no-undef': 'off', @@ -105,7 +105,7 @@ const configs = await withNuxt([ rules: { 'no-var': 'off', '@typescript-eslint/no-explicit-any': 'off', - '@typescript-eslint/ban-types': 'off' + '@typescript-eslint/no-restricted-types': 'off' } } ]).prepend([ diff --git a/packages/dui3/lib/accounts/composables/setup.ts b/packages/dui3/lib/accounts/composables/setup.ts index b40ec3670..06da215de 100644 --- a/packages/dui3/lib/accounts/composables/setup.ts +++ b/packages/dui3/lib/accounts/composables/setup.ts @@ -52,7 +52,7 @@ export function useAccountsSetup(): DUIAccountsState { try { await acc.client.query({ query: accountTestQuery }) acc.isValid = true - } catch (error) { + } catch { // TODO: properly dispose and kill this client. It's unclear how to do it. acc.isValid = false // NOTE: we do not want to delete the client, as we might want to "refresh" in diff --git a/packages/dui3/lib/bridge/generic.ts b/packages/dui3/lib/bridge/generic.ts index 72a18cc57..7b441868c 100644 --- a/packages/dui3/lib/bridge/generic.ts +++ b/packages/dui3/lib/bridge/generic.ts @@ -17,7 +17,7 @@ export class GenericBridge extends BaseBridge { try { availableMethodNames = await this.bridge.GetBindingsMethodNames() - } catch (e) { + } catch { console.warn(`Failed to get method names.`) return false } diff --git a/packages/dui3/lib/common/generated/gql/graphql.ts b/packages/dui3/lib/common/generated/gql/graphql.ts index e785a2173..41045642b 100644 --- a/packages/dui3/lib/common/generated/gql/graphql.ts +++ b/packages/dui3/lib/common/generated/gql/graphql.ts @@ -27,11 +27,23 @@ export type ActiveUserMutations = { emailMutations: UserEmailMutations; /** Mark onboarding as complete */ finishOnboarding: Scalars['Boolean']['output']; + setActiveWorkspace: Scalars['Boolean']['output']; /** Edit a user's profile */ update: User; }; +export type ActiveUserMutationsFinishOnboardingArgs = { + input?: InputMaybe; +}; + + +export type ActiveUserMutationsSetActiveWorkspaceArgs = { + isProjectsActive?: InputMaybe; + slug?: InputMaybe; +}; + + export type ActiveUserMutationsUpdateArgs = { user: UserUpdateInput; }; @@ -243,6 +255,11 @@ export type AutomateAuthCodePayloadTest = { workspaceId?: InputMaybe; }; +/** Additional resources to validate user access to. */ +export type AutomateAuthCodeResources = { + workspaceId?: InputMaybe; +}; + export type AutomateFunction = { __typename?: 'AutomateFunction'; /** Only returned if user is a part of this speckle server */ @@ -938,6 +955,18 @@ export type DiscoverableStreamsSortingInput = { type: DiscoverableStreamsSortType; }; +export type DiscoverableWorkspaceCollaborator = { + __typename?: 'DiscoverableWorkspaceCollaborator'; + avatar?: Maybe; +}; + +export type DiscoverableWorkspaceCollaboratorCollection = { + __typename?: 'DiscoverableWorkspaceCollaboratorCollection'; + cursor?: Maybe; + items: Array; + totalCount: Scalars['Int']['output']; +}; + export type EditCommentInput = { commentId: Scalars['String']['input']; content: CommentContentInput; @@ -1151,6 +1180,31 @@ export type LimitedWorkspace = { name: Scalars['String']['output']; /** Unique workspace short id. Used for navigation. */ slug: Scalars['String']['output']; + /** Workspace members visible to people with verified email domain */ + team?: Maybe; +}; + + +/** Workspace metadata visible to non-workspace members. */ +export type LimitedWorkspaceTeamArgs = { + cursor?: InputMaybe; + limit?: Scalars['Int']['input']; +}; + +export type LimitedWorkspaceJoinRequest = { + __typename?: 'LimitedWorkspaceJoinRequest'; + createdAt: Scalars['DateTime']['output']; + id: Scalars['String']['output']; + status: WorkspaceJoinRequestStatus; + user: LimitedUser; + workspace: LimitedWorkspace; +}; + +export type LimitedWorkspaceJoinRequestCollection = { + __typename?: 'LimitedWorkspaceJoinRequestCollection'; + cursor?: Maybe; + items: Array; + totalCount: Scalars['Int']['output']; }; export type MarkCommentViewedInput = { @@ -1811,10 +1865,18 @@ export type ObjectCreateInput = { streamId: Scalars['String']['input']; }; +export type OnboardingCompletionInput = { + plans?: InputMaybe>; + role?: InputMaybe; + source?: InputMaybe; +}; + export enum PaidWorkspacePlans { Business = 'business', Plus = 'plus', - Starter = 'starter' + Pro = 'pro', + Starter = 'starter', + Team = 'team' } export type PasswordStrengthCheckFeedback = { @@ -1889,6 +1951,13 @@ export type PendingWorkspaceCollaboratorsFilter = { search?: InputMaybe; }; +export type Price = { + __typename?: 'Price'; + amount: Scalars['Float']['output']; + currency: Scalars['String']['output']; + currencySymbol: Scalars['String']['output']; +}; + export type Project = { __typename?: 'Project'; allowPublicComments: Scalars['Boolean']['output']; @@ -2621,7 +2690,6 @@ export type Query = { * Either token or workspaceId must be specified, or both */ workspaceInvite?: Maybe; - workspacePricingPlans: Scalars['JSONObject']['output']; /** Find workspaces a given user email can use SSO to sign with */ workspaceSsoByEmail: Array; }; @@ -2662,6 +2730,7 @@ export type QueryAutomateFunctionsArgs = { export type QueryAutomateValidateAuthCodeArgs = { payload: AutomateAuthCodePayloadTest; + resources?: InputMaybe; }; @@ -2866,6 +2935,8 @@ export type ServerAutomateInfo = { export type ServerConfiguration = { __typename?: 'ServerConfiguration'; blobSizeLimitBytes: Scalars['Int']['output']; + /** Whether the email feature is enabled on this server */ + isEmailEnabled: Scalars['Boolean']['output']; objectMultipartUploadSizeLimitBytes: Scalars['Int']['output']; objectSizeLimitBytes: Scalars['Int']['output']; }; @@ -3016,6 +3087,8 @@ export type ServerStats = { export type ServerWorkspacesInfo = { __typename?: 'ServerWorkspacesInfo'; + /** Up-to-date prices for paid & non-invoiced Workspace plans */ + planPrices: Array; /** * This is a backend control variable for the workspaces feature set. * Since workspaces need a backend logic to be enabled, this is not enough as a feature flag. @@ -3624,6 +3697,8 @@ export type UpgradePlanInput = { */ export type User = { __typename?: 'User'; + /** The last-visited workspace for the given user */ + activeWorkspace?: Maybe; /** * All the recent activity from this user in chronological order * @deprecated Part of the old API surface and will be removed in the future. @@ -3671,6 +3746,8 @@ export type User = { id: Scalars['ID']['output']; /** Whether post-sign up onboarding has been finished or skipped entirely */ isOnboardingFinished?: Maybe; + /** Returns `true` if last visited project was "legacy" "personal project" outside of a workspace */ + isProjectsActive?: Maybe; name: Scalars['String']['output']; notificationPreferences: Scalars['JSONObject']['output']; profiles?: Maybe; @@ -3707,6 +3784,7 @@ export type User = { versions: CountOnlyCollection; /** Get all invitations to workspaces that the active user has */ workspaceInvites: Array; + workspaceJoinRequests?: Maybe; /** Get the workspaces for the user */ workspaces: WorkspaceCollection; }; @@ -3808,6 +3886,17 @@ export type UserVersionsArgs = { }; +/** + * Full user type, should only be used in the context of admin operations or + * when a user is reading/writing info about himself + */ +export type UserWorkspaceJoinRequestsArgs = { + cursor?: InputMaybe; + filter?: InputMaybe; + limit?: Scalars['Int']['input']; +}; + + /** * Full user type, should only be used in the context of admin operations or * when a user is reading/writing info about himself @@ -4420,6 +4509,10 @@ export type WorkspaceJoinRequestCollection = { totalCount: Scalars['Int']['output']; }; +export type WorkspaceJoinRequestFilter = { + status?: InputMaybe; +}; + export type WorkspaceJoinRequestMutations = { __typename?: 'WorkspaceJoinRequestMutations'; approve: Scalars['Boolean']['output']; @@ -4544,6 +4637,13 @@ export type WorkspacePlan = { status: WorkspacePlanStatuses; }; +export type WorkspacePlanPrice = { + __typename?: 'WorkspacePlanPrice'; + id: Scalars['String']['output']; + monthly?: Maybe; + yearly?: Maybe; +}; + export enum WorkspacePlanStatuses { CancelationScheduled = 'cancelationScheduled', Canceled = 'canceled', @@ -4557,10 +4657,13 @@ export enum WorkspacePlans { Academia = 'academia', Business = 'business', BusinessInvoiced = 'businessInvoiced', + Free = 'free', Plus = 'plus', PlusInvoiced = 'plusInvoiced', + Pro = 'pro', Starter = 'starter', StarterInvoiced = 'starterInvoiced', + Team = 'team', Unlimited = 'unlimited' } @@ -4587,6 +4690,13 @@ export type WorkspaceProjectInviteCreateInput = { export type WorkspaceProjectMutations = { __typename?: 'WorkspaceProjectMutations'; create: Project; + /** + * Schedule a job that will: + * - Move all regional data to target region + * - Update project region key + * - TODO: Eventually delete data in previous region + */ + moveToRegion: Scalars['String']['output']; moveToWorkspace: Project; updateRole: Project; }; @@ -4597,6 +4707,12 @@ export type WorkspaceProjectMutationsCreateArgs = { }; +export type WorkspaceProjectMutationsMoveToRegionArgs = { + projectId: Scalars['String']['input']; + regionKey: Scalars['String']['input']; +}; + + export type WorkspaceProjectMutationsMoveToWorkspaceArgs = { projectId: Scalars['String']['input']; workspaceId: Scalars['String']['input']; diff --git a/packages/dui3/package.json b/packages/dui3/package.json index 08268234f..0ce614d09 100644 --- a/packages/dui3/package.json +++ b/packages/dui3/package.json @@ -73,7 +73,7 @@ "tailwindcss": "^3.3.2", "type-fest": "^3.5.1", "typescript": "^4.8.3", - "vue-tsc": "1.3.4" + "vue-tsc": "^2.2.2" }, "installConfig": { "hoistingLimits": "workspaces" diff --git a/packages/dui3/pages/test.vue b/packages/dui3/pages/test.vue index 9eb0837b8..5f01daa83 100644 --- a/packages/dui3/pages/test.vue +++ b/packages/dui3/pages/test.vue @@ -66,7 +66,7 @@ const tests = ref([ } ).sayHi('Speckle', 0) // note, invalid on purpose, it looks long because ts needs to be happy return 'not ok' - } catch (e) { + } catch { return 'ok' } }, diff --git a/packages/fileimport-service/.env.example b/packages/fileimport-service/.env.example new file mode 100644 index 000000000..b6a66fe86 --- /dev/null +++ b/packages/fileimport-service/.env.example @@ -0,0 +1,7 @@ +FILE_IMPORT_TIME_LIMIT_MIN='10' +MAX_OBJECT_SIZE_MB='10' +POSTGRES_MAX_CONNECTIONS_FILE_IMPORT_SERVICE='1' +POSTGRES_CONNECTION_ACQUIRE_TIMEOUT_MILLIS='16000' +POSTGRES_CONNECTION_CREATE_TIMEOUT_MILLIS='5000' +FF_WORKSPACES_MULTI_REGION_ENABLED=false +# IFC_DOTNET_DLL_PATH='packages/fileimport-service/src/ifc-dotnet/ifc-converter.dll' diff --git a/packages/fileimport-service/Dockerfile b/packages/fileimport-service/Dockerfile index 1c1cf18be..89eb073e1 100644 --- a/packages/fileimport-service/Dockerfile +++ b/packages/fileimport-service/Dockerfile @@ -1,8 +1,8 @@ ARG NODE_ENV=production -FROM mcr.microsoft.com/dotnet/sdk:8.0-noble AS dotnet-build-stage +FROM mcr.microsoft.com/dotnet/sdk:8.0-noble@sha256:59d7b5f3bff2318735c6f2dd95bb6a05908665f72dd25bfaf934ba78b3c4eae9 AS dotnet-build-stage WORKDIR /app -COPY packages/fileimport-service/ifc-dotnet . +COPY packages/fileimport-service/src/ifc-dotnet . RUN dotnet publish ifc-converter.csproj -c Release -o output/ @@ -21,16 +21,17 @@ RUN apt-get update -y \ --no-install-recommends \ ca-certificates=20240203 \ curl=8.5.0-2ubuntu10.6 \ - gosu=1.17-1ubuntu0.24.04.2 \ - && curl -fsSL https://deb.nodesource.com/setup_18.x | bash - \ + && curl -fsSL https://deb.nodesource.com/setup_22.x -o nodesource_setup.sh \ + && chmod +x ./nodesource_setup.sh \ + && ./nodesource_setup.sh \ + && rm ./nodesource_setup.sh \ && curl -fsSL https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini -o /usr/bin/tini \ && chmod +x /usr/bin/tini \ && curl -fsSL https://deb.nodesource.com/setup_22.x -o nodesource_setup.sh \ && gosu root:root bash nodesource_setup.sh \ && DEBIAN_FRONTEND=noninteractive apt-get install -y \ --no-install-recommends \ - nodejs \ - && echo ">>>>>> NODE Version: $(node --version)" \ + nodejs=22.14.0-1nodesource1 \ && npm install -g corepack@0.30.0 \ && corepack enable \ && DEBIAN_FRONTEND=noninteractive apt-get remove curl -y \ @@ -57,7 +58,7 @@ RUN yarn workspaces foreach -W run build RUN apt-get update -y \ && DEBIAN_FRONTEND=noninteractive apt-get install -y \ --no-install-recommends \ - python3.12=3.12.3-1ubuntu0.4 \ + python3.12=3.12.3-1ubuntu0.5 \ python3-pip=24.0+dfsg-1ubuntu1.1 \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -74,8 +75,10 @@ ENV PYTHON_BINARY_PATH=${PYTHON_BINARY_PATH} ARG DOTNET_BINARY_PATH=/usr/bin/dotnet ENV DOTNET_BINARY_PATH=${DOTNET_BINARY_PATH} -COPY --link --from=dotnet-build-stage /app/output packages/fileimport-service/ifc-dotnet +COPY --link --from=dotnet-build-stage /app/output packages/fileimport-service/src/ifc-dotnet +ENV IFC_DOTNET_DLL_PATH='/speckle-server/packages/fileimport-service/src/ifc-dotnet/ifc-converter.dll' WORKDIR /speckle-server/packages/fileimport-service -ENTRYPOINT [ "tini", "--", "node", "--no-experimental-fetch", "src/daemon.js"] +ENTRYPOINT [ "tini", "--", "node", "--loader=./dist/src/aliasLoader.js" ] +CMD ["bin/www.js"] diff --git a/packages/fileimport-service/README.md b/packages/fileimport-service/README.md index 03cafa602..2efde3e9a 100644 --- a/packages/fileimport-service/README.md +++ b/packages/fileimport-service/README.md @@ -1,11 +1,13 @@ -# `ifc-parser` +# File Import Service -> TODO: description +## Description of how this works -## Usage +A micro-service which polls a Postgres database table `file_uploads` for new records and processes them. -``` -const ifcParser = require('ifc-parser'); +It retrieves a referenced file from an S3 bucket and stores it in a local directory for parsing. -// TODO: DEMONSTRATE API -``` +The File Import service can parse either STL, OBJ, or IFC files using external packages, written in either .Net or Python (_note_, there is a legacy IFC parser written in Node.js). These external packages are controlled via shell commands. + +The parsers are responsible for extracting the necessary data from the files and storing it in the database. They are also responsible for creating a new Speckle model if necessary. + +The service is then responsible for updating the status of the `file_uploads` table, and for posting a Postgres notification. diff --git a/packages/preview-service/bin/www.js b/packages/fileimport-service/bin/www.js similarity index 100% rename from packages/preview-service/bin/www.js rename to packages/fileimport-service/bin/www.js diff --git a/packages/fileimport-service/eslint.config.mjs b/packages/fileimport-service/eslint.config.mjs index d0bb88fbb..884f851ba 100644 --- a/packages/fileimport-service/eslint.config.mjs +++ b/packages/fileimport-service/eslint.config.mjs @@ -1,20 +1,61 @@ -import { baseConfigs, globals } from '../../eslint.config.mjs' +import tseslint from 'typescript-eslint' +import { + baseConfigs, + getESMDirname, + globals, + prettierConfig +} from '../../eslint.config.mjs' -/** - * @type {Array} - */ const configs = [ ...baseConfigs, { - ignores: ['**/ifc/**', '**/obj/**', '**/stl/**'] + ignores: ['dist', 'public', 'docs'] }, { + files: ['**/*.js'], + ignores: ['**/*.mjs'], + languageOptions: { + sourceType: 'module', + globals: { + ...globals.node + } + } + }, + { + files: ['bin/www'], + languageOptions: { + sourceType: 'module', + globals: { + ...globals.node + } + } + }, + ...tseslint.configs.recommendedTypeChecked.map((c) => ({ + ...c, + files: [...(c.files || []), '**/*.ts', '**/*.d.ts'] + })), + { + files: ['**/*.ts', '**/*.d.ts'], + languageOptions: { + parserOptions: { + tsconfigRootDir: getESMDirname(import.meta.url), + project: './tsconfig.json' + } + }, + rules: { + '@typescript-eslint/no-explicit-any': 'error', + '@typescript-eslint/no-unsafe-return': 'error' + } + }, + { + files: ['**/*.spec.{js,ts}'], languageOptions: { globals: { ...globals.node } } - } + }, + prettierConfig ] export default configs diff --git a/packages/fileimport-service/ifc-dotnet/ConsoleProgress.cs b/packages/fileimport-service/ifc-dotnet/ConsoleProgress.cs deleted file mode 100644 index 57cf1d9a3..000000000 --- a/packages/fileimport-service/ifc-dotnet/ConsoleProgress.cs +++ /dev/null @@ -1,33 +0,0 @@ -using Speckle.Sdk.Transports; - -namespace Speckle.Converter; - -public class ConsoleProgress : IProgress -{ - private readonly TimeSpan DEBOUNCE = TimeSpan.FromSeconds(1); - private DateTime _lastTime = DateTime.UtcNow; - - private long _totalBytes; - - public void Report(ProgressArgs value) - { - if (value.ProgressEvent == ProgressEvent.DownloadBytes) - { - Interlocked.Add(ref _totalBytes, value.Count); - } - var now = DateTime.UtcNow; - if (now - _lastTime >= DEBOUNCE) - { - if (value.ProgressEvent == ProgressEvent.DownloadBytes) - { - Console.WriteLine(value.ProgressEvent + " t " + _totalBytes); - } - else - { - Console.WriteLine(value.ProgressEvent + " c " + value.Count + " t " + value.Total); - } - - _lastTime = now; - } - } -} diff --git a/packages/fileimport-service/ifc-dotnet/Program.cs b/packages/fileimport-service/ifc-dotnet/Program.cs deleted file mode 100644 index 2115099c2..000000000 --- a/packages/fileimport-service/ifc-dotnet/Program.cs +++ /dev/null @@ -1,47 +0,0 @@ -using System.CommandLine; -using System.Text.Json; -using Speckle.Sdk.Common; -using Speckle.WebIfc.Importer; - -var filePathArgument = new Argument(name: "filePath"); -var outputPathArgument = new Argument("outputPath"); -var streamIdArgument = new Argument("streamId"); -var commitMessageArgument = new Argument("commitMessage"); -var modelIdArgument = new Argument("modelId"); -var regionNameArgument = new Argument("regionName"); - -var rootCommand = new RootCommand -{ - filePathArgument, - outputPathArgument, - streamIdArgument, - commitMessageArgument, - modelIdArgument, - regionNameArgument, -}; -rootCommand.SetHandler( - async (filePath, outputPath, streamId, commitMessage, modelId, _) => - { - try - { - var token = Environment.GetEnvironmentVariable("USER_TOKEN").NotNull("USER_TOKEN is missing"); - var url = Environment.GetEnvironmentVariable("SPECKLE_SERVER_URL") ?? "http://127.0.0.1:3000"; - var commitId = await Import.Ifc(url, filePath, streamId, modelId, commitMessage, token); - File.WriteAllText(outputPath, JsonSerializer.Serialize(new { success = true, commitId })); - } - catch (Exception e) - { - File.WriteAllText( - outputPath, - JsonSerializer.Serialize(new { success = false, error = e.ToString() }) - ); - } - }, - filePathArgument, - outputPathArgument, - streamIdArgument, - commitMessageArgument, - modelIdArgument, - regionNameArgument -); -await rootCommand.InvokeAsync(args); diff --git a/packages/fileimport-service/jsconfig.json b/packages/fileimport-service/jsconfig.json deleted file mode 100644 index 9027098ca..000000000 --- a/packages/fileimport-service/jsconfig.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "extends": "../../jsconfig.base.json", - "include": ["src", "ifc"] -} diff --git a/packages/fileimport-service/multiregion.example.json b/packages/fileimport-service/multiregion.example.json new file mode 100644 index 000000000..61bd76bf9 --- /dev/null +++ b/packages/fileimport-service/multiregion.example.json @@ -0,0 +1,34 @@ +{ + "main": { + "postgres": { + "connectionUri": "postgresql://speckle:speckle@127.0.0.1:5432/speckle", + "privateConnectionUri": "postgresql://speckle:speckle@postgres:5432/speckle", + "databaseName": "speckle" + }, + "blobStorage": { + "accessKey": "minioadmin", + "secretKey": "minioadmin", + "bucket": "speckle-server", + "createBucketIfNotExists": true, + "endpoint": "http://127.0.0.1:9000", + "s3Region": "us-east-1" + } + }, + "regions": { + "region1": { + "postgres": { + "connectionUri": "postgresql://speckle:speckle@127.0.0.1:5401/speckle", + "privateConnectionUri": "postgresql://speckle:speckle@postgres-region1:5432/speckle", + "databaseName": "speckle" + }, + "blobStorage": { + "accessKey": "minioadmin", + "secretKey": "minioadmin", + "bucket": "speckle-server", + "createBucketIfNotExists": true, + "endpoint": "http://127.0.0.1:9020", + "s3Region": "us-east-1" + } + } + } +} diff --git a/packages/fileimport-service/package.json b/packages/fileimport-service/package.json index 54a0f3d06..c742b4d5a 100644 --- a/packages/fileimport-service/package.json +++ b/packages/fileimport-service/package.json @@ -6,7 +6,8 @@ "author": "Speckle Systems ", "homepage": "https://github.com/specklesystems/speckle-server#readme", "license": "SEE LICENSE IN readme.md", - "main": "daemon.js", + "main": "./bin/www.js", + "type": "module", "repository": { "type": "git", "url": "git+https://github.com/specklesystems/speckle-server.git" @@ -15,34 +16,53 @@ "node": "^22.6.0" }, "scripts": { - "dev": "cross-env POSTGRES_URL=postgres://speckle:speckle@127.0.0.1/speckle NODE_ENV=development LOG_PRETTY=true SPECKLE_SERVER_URL=http://127.0.0.1:3000 nodemon --no-experimental-fetch ./src/daemon.js", - "parse:ifc": "node --no-experimental-fetch ./ifc/import_file.js ./ifc/ifcs/steelplates.ifc 33763848d6 2e4bfb467a main File upload: steelplates.ifc", - "lint": "eslint ." - }, - "bugs": { - "url": "https://github.com/specklesystems/speckle-server/issues" + "build:tsc:watch": "tsc -p ./tsconfig.build.json --watch", + "run:watch": "NODE_ENV=development LOG_PRETTY=true LOG_LEVEL=debug nodemon --exec \"yarn start\" --trace-deprecation --watch ./bin/www.js --watch ./dist", + "dev": "concurrently \"npm:build:tsc:watch\" \"npm:run:watch\"", + "dev:headed": "yarn dev", + "build:tsc": "rimraf ./dist/src && tsc -p ./tsconfig.build.json", + "build": "yarn build:tsc", + "lint": "yarn lint:tsc && yarn lint:eslint", + "lint:ci": "yarn lint:tsc", + "lint:tsc": "tsc --noEmit", + "lint:eslint": "eslint .", + "start": "node --loader=./dist/src/aliasLoader.js ./bin/www.js", + "test": "NODE_ENV=test LOG_LEVEL=silent LOG_PRETTY=true vitest run --sequence.shuffle", + "downloadBlob": "node scripts/downloadBlob.js" }, "dependencies": { "@speckle/shared": "workspace:^", - "bcrypt": "^5.0.1", - "crypto-random-string": "^3.3.1", + "bcrypt": "^5.0.0", + "crypto": "^1.0.1", + "crypto-random-string": "^3.2.0", + "dotenv": "^16.4.5", + "esm-module-alias": "^2.2.0", "knex": "^2.5.1", + "lodash": "^4.17.21", + "lodash-es": "^4.17.21", "pg": "^8.7.3", "pino": "^8.7.0", - "pino-http": "^8.0.0", "pino-pretty": "^9.1.1", "prom-client": "^14.0.1", + "tarn": "^3.0.2", "undici": "^5.28.4", "valid-filename": "^3.1.0", - "web-ifc": "^0.0.36", - "znv": "^0.4.0", - "zod": "^3.22.4" + "web-ifc": "^0.0.36" }, "devDependencies": { - "cross-env": "^7.0.3", + "@types/bcrypt": "^5.0.0", + "@types/lodash-es": "^4.17.6", + "@types/node": "^18.19.38", + "@vitest/coverage-istanbul": "^1.6.0", + "concurrently": "^8.2.2", "eslint": "^9.4.0", "eslint-config-prettier": "^9.1.0", + "eslint-plugin-vitest": "^0.5.4", "nodemon": "^2.0.20", - "prettier": "^2.5.1" + "prettier": "^2.5.1", + "rimraf": "^5.0.7", + "typescript": "^4.6.4", + "typescript-eslint": "^7.12.0", + "vitest": "^1.6.0" } } diff --git a/packages/fileimport-service/scripts/downloadBlob.js b/packages/fileimport-service/scripts/downloadBlob.js new file mode 100644 index 000000000..72bac9c2a --- /dev/null +++ b/packages/fileimport-service/scripts/downloadBlob.js @@ -0,0 +1,16 @@ +const { downloadFile } = require('../src/controller/filesApi.js') +const { logger } = require('../src/observability/logging') + +//https://latest.speckle.systems/api/stream/c83a5b2d1f/blob/29fe85cffb +const speckleServerUrl = 'https://latest.speckle.systems' +const fileId = '29fe85cffb' +const streamId = 'c83a5b2d1f' + +downloadFile({ + speckleServerUrl, + fileId, + streamId, + destination: '/var/folders/p2/fczcvzfd62x5jcfdlw6ghf640000gn/T/tmp.U8MlF9KIxH', + token: process.env.SPECKLE_TOKEN || '', + logger: logger.child({ streamId, fileId }) +}) diff --git a/packages/fileimport-service/src/aliasLoader.ts b/packages/fileimport-service/src/aliasLoader.ts new file mode 100644 index 000000000..d77980f8d --- /dev/null +++ b/packages/fileimport-service/src/aliasLoader.ts @@ -0,0 +1,6 @@ +import generateAliasesResolver from 'esm-module-alias' +import { srcRoot } from './root.js' + +export const resolve = generateAliasesResolver({ + '@': srcRoot +}) diff --git a/packages/fileimport-service/src/bin.ts b/packages/fileimport-service/src/bin.ts new file mode 100644 index 000000000..5a087f0b8 --- /dev/null +++ b/packages/fileimport-service/src/bin.ts @@ -0,0 +1,9 @@ +import '@/bootstrap.js' // This has side-effects and has to be imported first + +import { main } from '@/controller/daemon.js' + +const start = () => { + void main() +} + +start() diff --git a/packages/fileimport-service/src/bootstrap.ts b/packages/fileimport-service/src/bootstrap.ts new file mode 100644 index 000000000..50c8721e6 --- /dev/null +++ b/packages/fileimport-service/src/bootstrap.ts @@ -0,0 +1,2 @@ +import dotenv from 'dotenv' +dotenv.config() diff --git a/packages/fileimport-service/src/api.js b/packages/fileimport-service/src/controller/api.ts similarity index 62% rename from packages/fileimport-service/src/api.js rename to packages/fileimport-service/src/controller/api.ts index 08e5b566c..b7523df0c 100644 --- a/packages/fileimport-service/src/api.js +++ b/packages/fileimport-service/src/controller/api.ts @@ -1,22 +1,59 @@ -'use strict' -const crypto = require('crypto') -const crs = require('crypto-random-string') -const bcrypt = require('bcrypt') -const { chunk } = require('lodash') -const { logger: parentLogger } = require('../observability/logging') +import crypto from 'crypto' +import crs from 'crypto-random-string' +import bcrypt from 'bcrypt' +import { chunk } from 'lodash-es' +import { logger as parentLogger } from '@/observability/logging.js' +import Observability from '@speckle/shared/dist/commonjs/observability/index.js' +import type { Knex } from 'knex' +import type { Logger } from 'pino' -const Observability = require('@speckle/shared/dist/commonjs/observability/index.js') - -const tables = (db) => ({ +const tables = (db: Knex) => ({ objects: db('objects'), - branches: db('branches'), + branches: db<{ + id: string + streamId: string + authorId: string + name: string + description: string + }>('branches'), streams: db('streams'), apiTokens: db('api_tokens'), tokenScopes: db('token_scopes') }) -module.exports = class ServerAPI { - constructor({ db, streamId, logger }) { +type SpeckleObject = { + id?: string + hash?: string + streamId: string + __closure?: Record + __tree?: unknown + speckleType: string + totalChildrenCount?: number + totalChildrenCountByDepth?: string + data: unknown +} + +type SpeckleObjectWithId = SpeckleObject & { + id: string +} + +export class ServerAPI { + tables: ReturnType + db: Knex + streamId: string + isSending: boolean + buffer: unknown[] + logger: Logger + + constructor({ + db, + streamId, + logger + }: { + db: Knex + streamId: string + logger: Logger + }) { this.tables = tables(db) this.db = db this.streamId = streamId @@ -27,7 +64,7 @@ module.exports = class ServerAPI { Observability.extendLoggerComponent(parentLogger.child({ streamId }), 'ifc') } - async saveObject(obj) { + async saveObject(obj: SpeckleObject) { if (!obj) throw new Error('Null object') if (!obj.id) { @@ -39,14 +76,20 @@ module.exports = class ServerAPI { return obj.id } - async saveObjectBatch(objs) { + async saveObjectBatch(objs: SpeckleObject[]) { return await this.createObjectsBatched(this.streamId, objs) } - async createObject({ streamId, object }) { + async createObject({ + streamId, + object + }: { + streamId: string + object: SpeckleObject + }) { const insertionObject = this.prepInsertionObject(streamId, object) - const totalChildrenCountByDepth = {} + const totalChildrenCountByDepth: Record = {} if (object.__closure !== null) { for (const prop in object.__closure) { if (totalChildrenCountByDepth[object.__closure[prop].toString()]) @@ -58,7 +101,7 @@ module.exports = class ServerAPI { delete insertionObject.__tree delete insertionObject.__closure - insertionObject.totalChildrenCount = object.__closures.length + insertionObject.totalChildrenCount = object.__closure?.length insertionObject.totalChildrenCountByDepth = JSON.stringify( totalChildrenCountByDepth ) @@ -68,15 +111,15 @@ module.exports = class ServerAPI { return insertionObject.id } - async createObjectsBatched(streamId, objects) { - const objsToInsert = [] - const ids = [] + async createObjectsBatched(streamId: string, objects: SpeckleObject[]) { + const objsToInsert: SpeckleObjectWithId[] = [] + const ids: string[] = [] // Prep objects up objects.forEach((obj) => { const insertionObject = this.prepInsertionObject(streamId, obj) let totalChildrenCountGlobal = 0 - const totalChildrenCountByDepth = {} + const totalChildrenCountByDepth: Record = {} if (obj.__closure !== null) { for (const prop in obj.__closure) { @@ -121,7 +164,7 @@ module.exports = class ServerAPI { return ids } - prepInsertionObject(streamId, obj) { + prepInsertionObject(streamId: string, obj: SpeckleObject): SpeckleObjectWithId { const maximumObjectSizeMB = parseInt(process.env['MAX_OBJECT_SIZE_MB'] || '10') const MAX_OBJECT_SIZE = maximumObjectSizeMB * 1024 * 1024 @@ -145,23 +188,49 @@ module.exports = class ServerAPI { } } - prepInsertionObjectBatch(batch) { + prepInsertionObjectBatch(batch: Array<{ id: string }>) { batch.sort((a, b) => (a.id > b.id ? 1 : -1)) } - prepInsertionClosureBatch(batch) { - batch.sort((a, b) => - a.parent > b.parent + prepInsertionClosureBatch( + batch: Array<{ parent: string | undefined; child: string | undefined }> + ) { + batch.sort((a, b) => { + return this.hasParent(a) && this.hasParent(b) && a.parent > b.parent ? 1 : a.parent === b.parent - ? a.child > b.child + ? this.hasChild(a) && this.hasChild(b) && a.child > b.child ? 1 : -1 : -1 + }) + } + + hasParent(maybeHasParent: unknown): maybeHasParent is { parent: string } { + return ( + !!maybeHasParent && + typeof maybeHasParent === 'object' && + 'parent' in maybeHasParent && + typeof maybeHasParent.parent === 'string' ) } - async getBranchByNameAndStreamId({ streamId, name }) { + hasChild(maybeHasChild: unknown): maybeHasChild is { child: string } { + return ( + !!maybeHasChild && + typeof maybeHasChild === 'object' && + 'child' in maybeHasChild && + typeof maybeHasChild.child === 'string' + ) + } + + async getBranchByNameAndStreamId({ + streamId, + name + }: { + streamId: string + name: string + }) { const query = this.tables.branches .select('*') .where({ streamId }) @@ -170,13 +239,24 @@ module.exports = class ServerAPI { return await query } - async createBranch({ name, description, streamId, authorId }) { - const branch = {} - branch.id = crs({ length: 10 }) - branch.streamId = streamId - branch.authorId = authorId - branch.name = name.toLowerCase() - branch.description = description + async createBranch({ + name, + description, + streamId, + authorId + }: { + name: string + description: string + streamId: string + authorId: string + }) { + const branch = { + id: crs({ length: 10 }), + streamId, + authorId, + name: name.toLowerCase(), + description + } await this.tables.branches.returning('id').insert(branch) @@ -197,7 +277,17 @@ module.exports = class ServerAPI { return { tokenId, tokenString, tokenHash, lastChars } } - async createToken({ userId, name, scopes, lifespan }) { + async createToken({ + userId, + name, + scopes, + lifespan + }: { + userId: string + name: string + scopes: string[] + lifespan: number + }) { const { tokenId, tokenString, tokenHash, lastChars } = await this.createBareToken() if (scopes.length === 0) throw new Error('No scopes provided') @@ -218,7 +308,7 @@ module.exports = class ServerAPI { return { id: tokenId, token: tokenId + tokenString } } - async revokeTokenById(tokenId) { + async revokeTokenById(tokenId: string) { const delCount = await this.tables.apiTokens .where({ id: tokenId.slice(0, 10) }) .del() diff --git a/packages/fileimport-service/src/daemon.js b/packages/fileimport-service/src/controller/daemon.ts similarity index 67% rename from packages/fileimport-service/src/daemon.js rename to packages/fileimport-service/src/controller/daemon.ts index 47970adef..53cd84fe8 100644 --- a/packages/fileimport-service/src/daemon.js +++ b/packages/fileimport-service/src/controller/daemon.ts @@ -1,22 +1,23 @@ -'use strict' - -const Environment = require('@speckle/shared/dist/commonjs/environment/index.js') -const { +import Environment from '@speckle/shared/dist/commonjs/environment/index.js' +import { initPrometheusMetrics, metricDuration, metricInputFileSize, metricOperationErrors -} = require('./prometheusMetrics') -const getDbClients = require('../knex') +} from '@/controller/prometheusMetrics.js' +import { getDbClients } from '@/knex.js' -const { downloadFile } = require('./filesApi') -const fs = require('fs') -const { spawn } = require('child_process') +import { downloadFile } from '@/controller/filesApi.js' +import fs from 'fs' +import { spawn } from 'child_process' + +import { ServerAPI } from '@/controller/api.js' +import { downloadDependencies } from '@/controller/objDependencies.js' +import { logger } from '@/observability/logging.js' +import { Nullable, Scopes, wait } from '@speckle/shared' +import { Knex } from 'knex' +import { Logger } from 'pino' -const ServerAPI = require('./api') -const objDependencies = require('./objDependencies') -const { logger } = require('../observability/logging') -const { Scopes, wait } = require('@speckle/shared') const { FF_FILEIMPORT_IFC_DOTNET_ENABLED } = Environment.getFeatureFlags() const HEALTHCHECK_FILE_PATH = '/tmp/last_successful_query' @@ -29,11 +30,11 @@ let shouldExit = false let TIME_LIMIT = 10 * 60 * 1000 -const providedTimeLimit = parseInt(process.env.FILE_IMPORT_TIME_LIMIT_MIN) +const providedTimeLimit = parseInt(process.env['FILE_IMPORT_TIME_LIMIT_MIN'] || '10') if (providedTimeLimit) TIME_LIMIT = providedTimeLimit * 60 * 1000 -async function startTask(knex) { - const { rows } = await knex.raw(` +async function startTask(knex: Knex) { + const { rows } = (await knex.raw(` UPDATE file_uploads SET "convertedStatus" = 1, @@ -46,18 +47,23 @@ async function startTask(knex) { ) as task WHERE file_uploads."id" = task."id" RETURNING file_uploads."id" - `) + `)) satisfies { rows: { id: string }[] } return rows[0] } -async function doTask(mainDb, regionName, taskDb, task) { +async function doTask( + mainDb: Knex, + regionName: string, + taskDb: Knex, + task: { id: string } +) { const taskId = task.id // Mark task as started await mainDb.raw(`NOTIFY file_import_started, '${task.id}'`) let taskLogger = logger.child({ taskId }) - let tempUserToken = null + let tempUserToken: Nullable = null let mainServerApi = null let taskServerApi = null let fileTypeForMetric = 'unknown' @@ -65,11 +71,24 @@ async function doTask(mainDb, regionName, taskDb, task) { const metricDurationEnd = metricDuration.startTimer() let newBranchCreated = false - let branchMetadata = { streamId: null, branchName: null } + let branchMetadata: { streamId: Nullable; branchName: Nullable } = { + streamId: null, + branchName: null + } try { taskLogger.info("Doing task '{taskId}'.") - const info = await taskDb('file_uploads').where({ id: taskId }).first() + const info = await taskDb<{ + id: string + fileType: string + fileSize: string + fileName: string + userId: string + streamId: string + branchName: string + }>('file_uploads') + .where({ id: taskId }) + .first() if (!info) { throw new Error('Internal error: DB inconsistent') } @@ -116,30 +135,40 @@ async function doTask(mainDb, regionName, taskDb, task) { const { token } = await mainServerApi.createToken({ userId: info.userId, name: 'temp upload token', - scopes: [Scopes.Streams.Write, Scopes.Streams.Read], + scopes: [Scopes.Streams.Write, Scopes.Streams.Read, Scopes.Profile.Read], lifespan: 1000000 }) tempUserToken = token + taskLogger.info('Downloading file {fileId}') + + const speckleServerUrl = process.env.SPECKLE_SERVER_URL || 'http://127.0.0.1:3000' + await downloadFile({ + speckleServerUrl, fileId: info.id, streamId: info.streamId, token, - destination: TMP_FILE_PATH + destination: TMP_FILE_PATH, + logger: taskLogger }) + taskLogger.info('Triggering importer for {fileType}') + if (info.fileType.toLowerCase() === 'ifc') { if (FF_FILEIMPORT_IFC_DOTNET_ENABLED) { await runProcessWithTimeout( taskLogger, process.env['DOTNET_BINARY_PATH'] || 'dotnet', [ - '/speckle-server/packages/fileimport-service/ifc-dotnet/ifc-converter.dll', + process.env['IFC_DOTNET_DLL_PATH'] || + '/speckle-server/packages/fileimport-service/src/ifc-dotnet/ifc-converter.dll', TMP_FILE_PATH, TMP_RESULTS_PATH, info.streamId, `File upload: ${info.fileName}`, existingBranch?.id || '', + info.branchName, regionName ], { @@ -153,7 +182,8 @@ async function doTask(mainDb, regionName, taskDb, task) { process.env['NODE_BINARY_PATH'] || 'node', [ '--no-experimental-fetch', - './ifc/import_file.js', + '--loader=./dist/src/aliasLoader.js', + './src/ifc/import_file.js', TMP_FILE_PATH, TMP_RESULTS_PATH, info.userId, @@ -175,7 +205,7 @@ async function doTask(mainDb, regionName, taskDb, task) { taskLogger, process.env['PYTHON_BINARY_PATH'] || 'python3', [ - './stl/import_file.py', + './src/stl/import_file.py', TMP_FILE_PATH, TMP_RESULTS_PATH, info.userId, @@ -192,7 +222,8 @@ async function doTask(mainDb, regionName, taskDb, task) { TIME_LIMIT ) } else if (info.fileType.toLowerCase() === 'obj') { - await objDependencies.downloadDependencies({ + await downloadDependencies({ + speckleServerUrl, objFilePath: TMP_FILE_PATH, streamId: info.streamId, destinationDir: TMP_INPUT_DIR, @@ -204,7 +235,7 @@ async function doTask(mainDb, regionName, taskDb, task) { process.env['PYTHON_BINARY_PATH'] || 'python3', [ '-u', - './obj/import_file.py', + './src/obj/import_file.py', TMP_FILE_PATH, TMP_RESULTS_PATH, info.userId, @@ -224,9 +255,11 @@ async function doTask(mainDb, regionName, taskDb, task) { throw new Error(`File type ${info.fileType} is not supported`) } - const output = JSON.parse(fs.readFileSync(TMP_RESULTS_PATH)) + const output: unknown = JSON.parse(fs.readFileSync(TMP_RESULTS_PATH, 'utf8')) - if (!output.success) throw new Error(output.error) + if (!isSuccessOutput(output)) { + throw new Error(isErrorOutput(output) ? output.error : 'Unknown error') + } const commitId = output.commitId @@ -243,7 +276,8 @@ async function doTask(mainDb, regionName, taskDb, task) { [commitId, task.id] ) } catch (err) { - taskLogger.error(err) + taskLogger.error(err, 'Error processing task') + const errorForDatabase = maybeErrorToString(err) await taskDb.raw( ` UPDATE file_uploads @@ -254,7 +288,7 @@ async function doTask(mainDb, regionName, taskDb, task) { WHERE "id" = ? `, // DB only accepts a varchar 255 - [err.toString().substring(0, 254), task.id] + [errorForDatabase.substring(0, 254), task.id] ) metricOperationErrors.labels(fileTypeForMetric).inc() } finally { @@ -271,12 +305,56 @@ async function doTask(mainDb, regionName, taskDb, task) { fs.rmSync(TMP_INPUT_DIR, { force: true, recursive: true }) if (fs.existsSync(TMP_RESULTS_PATH)) fs.unlinkSync(TMP_RESULTS_PATH) - if (tempUserToken) { + if (mainServerApi && tempUserToken) { await mainServerApi.revokeTokenById(tempUserToken) } } -function runProcessWithTimeout(processLogger, cmd, cmdArgs, extraEnv, timeoutMs) { +function maybeErrorToString(error: unknown): string { + const unknownError = 'Unknown error' + if (!error) return unknownError + if (typeof error === 'string') return error + if (error instanceof Error) return error.message + try { + return JSON.stringify(error) + } catch { + return unknownError + } +} + +function isSuccessOutput( + maybeSuccessOutput: unknown +): maybeSuccessOutput is { success: true; commitId: string } { + return ( + !!maybeSuccessOutput && + typeof maybeSuccessOutput === 'object' && + 'success' in maybeSuccessOutput && + typeof maybeSuccessOutput.success === 'boolean' && + maybeSuccessOutput.success && + 'commitId' in maybeSuccessOutput && + typeof maybeSuccessOutput.commitId === 'string' + ) +} + +function isErrorOutput( + maybeErrorOutput: unknown +): maybeErrorOutput is { success: false; error: string } { + return ( + !!maybeErrorOutput && + typeof maybeErrorOutput === 'object' && + 'error' in maybeErrorOutput && + typeof maybeErrorOutput.error === 'string' && + !!maybeErrorOutput.error + ) +} + +function runProcessWithTimeout( + processLogger: Logger, + cmd: string, + cmdArgs: string[], + extraEnv: Record, + timeoutMs: number +): Promise { return new Promise((resolve, reject) => { let boundLogger = processLogger.child({ cmd, args: cmdArgs }) boundLogger.info('Starting process.') @@ -304,7 +382,7 @@ function runProcessWithTimeout(processLogger, cmd, cmdArgs, extraEnv, timeoutMs) error: rejectionReason } fs.writeFileSync(TMP_RESULTS_PATH, JSON.stringify(output)) - reject(rejectionReason) + reject(new Error(rejectionReason)) }, timeoutMs) childProc.on('close', (code) => { @@ -319,20 +397,20 @@ function runProcessWithTimeout(processLogger, cmd, cmdArgs, extraEnv, timeoutMs) if (code === 0) { resolve() } else { - reject(`Parser exited with code ${code}`) + reject(new Error(`Parser exited with code ${code}`)) } }) }) } -function handleData(data, isErr, logger) { +function handleData(data: unknown, isErr: boolean, logger: Logger) { try { - Buffer.isBuffer(data) && (data = data.toString()) - data.split('\n').forEach((line) => { + if (!Buffer.isBuffer(data)) return + const dataAsString = data.toString() + dataAsString.split('\n').forEach((line) => { if (!line) return try { JSON.parse(line) // verify if the data is already in JSON format - process.stdout.write(line) process.stdout.write('\n') } catch { wrapLogLine(line, isErr, logger) @@ -343,7 +421,7 @@ function handleData(data, isErr, logger) { } } -function wrapLogLine(line, isErr, logger) { +function wrapLogLine(line: string, isErr: boolean, logger: Logger) { if (isErr) { logger.error({ parserLogLine: line }, 'ParserLog: {parserLogLine}') return @@ -356,7 +434,7 @@ const doStuff = async () => { const mainDb = dbClients.main.public const dbClientsIterator = infiniteDbClientsIterator(dbClients) while (!shouldExit) { - const [regionName, taskDb] = dbClientsIterator.next().value + const [regionName, taskDb]: [string, Knex] = dbClientsIterator.next().value try { const task = await startTask(taskDb) fs.writeFile(HEALTHCHECK_FILE_PATH, '' + Date.now(), () => {}) @@ -374,7 +452,7 @@ const doStuff = async () => { } } -async function main() { +export async function main() { logger.info('Starting FileUploads Service...') await initPrometheusMetrics() @@ -387,7 +465,9 @@ async function main() { process.exit(0) } -function* infiniteDbClientsIterator(dbClients) { +function* infiniteDbClientsIterator(dbClients: { + [key: string]: { public: Knex } +}): Generator<[string, Knex], [string, Knex], [string, Knex]> { let index = 0 const dbClientEntries = [...Object.entries(dbClients)] const clientCount = dbClientEntries.length @@ -399,5 +479,3 @@ function* infiniteDbClientsIterator(dbClients) { yield [regionName, dbConnection.public] } } - -main() diff --git a/packages/fileimport-service/src/controller/filesApi.ts b/packages/fileimport-service/src/controller/filesApi.ts new file mode 100644 index 000000000..02afaa469 --- /dev/null +++ b/packages/fileimport-service/src/controller/filesApi.ts @@ -0,0 +1,95 @@ +import { ensureError } from '@speckle/shared/dist/esm/index.js' +import fs from 'fs' +import path from 'node:path' +import { pipeline } from 'node:stream/promises' +import { Logger } from 'pino' + +export async function downloadFile({ + speckleServerUrl, + fileId, + streamId, + token, + destination, + logger +}: { + speckleServerUrl: string + fileId: string + streamId: string + token: string + destination: string + logger: Logger +}) { + try { + fs.mkdirSync(path.dirname(destination), { recursive: true }) + } catch (e) { + throw ensureError(e, 'Unknown error while creating directory') + } + + logger.info( + { destinationFile: destination }, + 'Downloading file {fileId} from {streamId} to {destinationFile}' + ) + + let response + try { + response = await fetch( + `${speckleServerUrl}/api/stream/${streamId}/blob/${fileId}`, + { + headers: { + Authorization: `Bearer ${token}` + } + } + ) + } catch (e) { + throw ensureError(e, 'Unknown error while fetching file') + } + + if (response === undefined || !response.ok) { + logger.error( + { status: response?.status, statusText: response?.statusText }, + 'Failed to download file {fileId}. HTTP {status}: {statusText}' + ) + throw new Error( + `Failed to download file ${fileId}. HTTP ${response?.status}: ${response?.statusText}` + ) + } + if (!response.body) { + throw new Error('Response body is undefined') + } + + const writer = fs.createWriteStream(destination) + + //handle errors + writer.on('error', (err) => { + logger.error(ensureError(err), `Error writing file ${destination}`) + throw err + }) + + //handle completion + writer.on('finish', () => { + logger.info(`File written to ${destination}`) + }) + + await pipeline(response.body, writer, { end: true }) +} +export async function getFileInfoByName({ + speckleServerUrl, + fileName, + streamId, + token +}: { + speckleServerUrl: string + fileName: string + streamId: string + token: string +}) { + const response = await fetch( + `${speckleServerUrl}/api/stream/${streamId}/blobs?fileName=${fileName}`, + { + headers: { + Authorization: `Bearer ${token}` + } + } + ) + return response.json() as Promise<{ blobs: { id: string }[] }> +} diff --git a/packages/fileimport-service/src/controller/objDependencies.ts b/packages/fileimport-service/src/controller/objDependencies.ts new file mode 100644 index 000000000..6e655f2c5 --- /dev/null +++ b/packages/fileimport-service/src/controller/objDependencies.ts @@ -0,0 +1,71 @@ +import events from 'events' +import fs from 'fs' +import readline from 'readline' +import path from 'path' +import isValidFilename from 'valid-filename' + +import { downloadFile, getFileInfoByName } from '@/controller/filesApi.js' +import { logger } from '@/observability/logging.js' + +const getReferencedMtlFiles = async ({ objFilePath }: { objFilePath: string }) => { + const mtlFiles: string[] = [] + + try { + const rl = readline.createInterface({ + input: fs.createReadStream(objFilePath), + crlfDelay: Infinity + }) + + rl.on('line', (line) => { + if (line.startsWith('mtllib ')) { + const mtlFile = line.slice('mtllib '.length).trim() + mtlFiles.push(mtlFile) + } + }) + + await events.once(rl, 'close') + } catch (err) { + logger.error(err, `Error getting dependencies for file ${objFilePath}`) + } + return mtlFiles +} + +export async function downloadDependencies({ + speckleServerUrl, + objFilePath, + streamId, + destinationDir, + token +}: { + speckleServerUrl: string + objFilePath: string + streamId: string + destinationDir: string + token: string +}) { + const dependencies = await getReferencedMtlFiles({ objFilePath }) + + logger.info(`Obj file depends on ${dependencies.join(', ')}`) + for (const mtlFile of dependencies) { + // there might be multiple files named with the same name, take the first... + const [file] = ( + await getFileInfoByName({ speckleServerUrl, fileName: mtlFile, streamId, token }) + ).blobs + if (!file) { + logger.info(`OBJ dependency file not found in stream: ${mtlFile}`) + continue + } + if (!isValidFilename(mtlFile)) { + logger.warn(`Invalid filename reference in OBJ dependencies: ${mtlFile}`) + continue + } + await downloadFile({ + speckleServerUrl, + fileId: file.id, + streamId, + token, + destination: path.join(destinationDir, mtlFile), + logger + }) + } +} diff --git a/packages/fileimport-service/src/controller/prometheusMetrics.ts b/packages/fileimport-service/src/controller/prometheusMetrics.ts new file mode 100644 index 000000000..172e59c52 --- /dev/null +++ b/packages/fileimport-service/src/controller/prometheusMetrics.ts @@ -0,0 +1,171 @@ +import http from 'http' +import prometheusClient, { Counter, Summary } from 'prom-client' +import { getDbClients } from '@/knex.js' +import { Knex } from 'knex' +import { Pool } from 'tarn' +import { isObject } from 'lodash-es' +import { IncomingMessage } from 'http' + +let metricQueryDuration: Summary | null = null +let metricQueryErrors: Counter | null = null + +const queryStartTime: Record = {} +prometheusClient.register.clear() +prometheusClient.register.setDefaultLabels({ + project: 'speckle-server', + app: 'fileimport-service' +}) +prometheusClient.collectDefaultMetrics() + +let prometheusInitialized = false + +const initDBPrometheusMetricsFactory = + ({ db }: { db: Knex }) => + () => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + const dbConnectionPool = db.client.pool as Pool + new prometheusClient.Gauge({ + name: 'speckle_server_knex_free', + help: 'Number of free DB connections', + collect() { + this.set(dbConnectionPool.numFree()) + } + }) + + new prometheusClient.Gauge({ + name: 'speckle_server_knex_used', + help: 'Number of used DB connections', + collect() { + this.set(dbConnectionPool.numUsed()) + } + }) + + new prometheusClient.Gauge({ + name: 'speckle_server_knex_pending', + help: 'Number of pending DB connection aquires', + collect() { + this.set(dbConnectionPool.numPendingAcquires()) + } + }) + + new prometheusClient.Gauge({ + name: 'speckle_server_knex_pending_creates', + help: 'Number of pending DB connection creates', + collect() { + this.set(dbConnectionPool.numPendingCreates()) + } + }) + + new prometheusClient.Gauge({ + name: 'speckle_server_knex_pending_validations', + help: 'Number of pending DB connection validations. This is a state between pending acquisition and acquiring a connection.', + collect() { + this.set(dbConnectionPool.numPendingValidations()) + } + }) + + new prometheusClient.Gauge({ + name: 'speckle_server_knex_remaining_capacity', + help: 'Remaining capacity of the DB connection pool', + collect() { + const postgresMaxConnections = parseInt( + process.env['POSTGRES_MAX_CONNECTIONS_FILE_IMPORT_SERVICE'] || '1' + ) + const demand = + dbConnectionPool.numUsed() + + dbConnectionPool.numPendingCreates() + + dbConnectionPool.numPendingValidations() + + dbConnectionPool.numPendingAcquires() + + this.set(Math.max(postgresMaxConnections - demand, 0)) + } + }) + + metricQueryDuration = new prometheusClient.Summary({ + name: 'speckle_server_knex_query_duration', + help: 'Summary of the DB query durations in seconds' + }) + + metricQueryErrors = new prometheusClient.Counter({ + name: 'speckle_server_knex_query_errors', + help: 'Number of DB queries with errors' + }) + + db.on('query', (data) => { + if (!isObject(data) || !('__knexQueryUid' in data)) return + const queryId = String(data.__knexQueryUid) + queryStartTime[queryId] = Date.now() + }) + + db.on('query-response', (_data, obj) => { + if (!isObject(obj) || !('__knexQueryUid' in obj)) return + const queryId = String(obj.__knexQueryUid) + const durationSec = (Date.now() - queryStartTime[queryId]) / 1000 + delete queryStartTime[queryId] + if (metricQueryDuration && !isNaN(durationSec)) + metricQueryDuration.observe(durationSec) + }) + + db.on('query-error', (_err, querySpec) => { + if (!isObject(querySpec) || !('__knexQueryUid' in querySpec)) return + const queryId = String(querySpec.__knexQueryUid) + const durationSec = (Date.now() - queryStartTime[queryId]) / 1000 + delete queryStartTime[queryId] + + if (metricQueryDuration && !isNaN(durationSec)) + metricQueryDuration.observe(durationSec) + if (metricQueryErrors) metricQueryErrors.inc() + }) + } + +export async function initPrometheusMetrics() { + if (prometheusInitialized) return + prometheusInitialized = true + + const db = (await getDbClients()).main.public + + initDBPrometheusMetricsFactory({ db })() + + const requestHandler = async (req: IncomingMessage, res: http.OutgoingMessage) => { + if (req.url === '/metrics') { + res.setHeader('Content-Type', prometheusClient.register.contentType) + const metrics = await prometheusClient.register.metrics() + res.end(metrics) + } else { + res.end('Speckle FileImport Service - prometheus metrics') + } + } + + // Define the HTTP server + // eslint-disable-next-line @typescript-eslint/no-misused-promises + const server = http.createServer(requestHandler) + server.listen(Number(process.env.PROMETHEUS_METRICS_PORT) || 9093) +} + +export const metricDuration = new prometheusClient.Histogram({ + name: 'speckle_server_operation_duration', + help: 'Summary of the operation durations in seconds', + buckets: [0.5, 1, 5, 10, 30, 60, 300, 600, 900, 1200], + labelNames: ['op'] +}) + +export const metricOperationErrors = new prometheusClient.Counter({ + name: 'speckle_server_operation_errors', + help: 'Number of operations with errors', + labelNames: ['op'] +}) + +export const metricInputFileSize = new prometheusClient.Histogram({ + name: 'speckle_server_operation_file_size', + help: 'Size of the operation input file size', + buckets: [ + 1000, + 100 * 1000, + 500 * 1000, + 1000 * 1000, + 5 * 1000 * 1000, + 10 * 1000 * 1000, + 100 * 1000 * 1000 + ], + labelNames: ['op'] +}) diff --git a/packages/fileimport-service/src/filesApi.js b/packages/fileimport-service/src/filesApi.js deleted file mode 100644 index 639e67c36..000000000 --- a/packages/fileimport-service/src/filesApi.js +++ /dev/null @@ -1,32 +0,0 @@ -/* istanbul ignore file */ -'use strict' -const fs = require('fs') -const path = require('node:path') -const { stream, fetch } = require('undici') - -module.exports = { - async downloadFile({ fileId, streamId, token, destination }) { - fs.mkdirSync(path.dirname(destination), { recursive: true }) - await stream( - `${process.env.SPECKLE_SERVER_URL}/api/stream/${streamId}/blob/${fileId}`, - { - opaque: fs.createWriteStream(destination), - headers: { - Authorization: `Bearer ${token}` - } - }, - ({ opaque }) => opaque - ) - }, - async getFileInfoByName({ fileName, streamId, token }) { - const response = await fetch( - `${process.env.SPECKLE_SERVER_URL}/api/stream/${streamId}/blobs?fileName=${fileName}`, - { - headers: { - Authorization: `Bearer ${token}` - } - } - ) - return response.json() - } -} diff --git a/packages/fileimport-service/ifc-dotnet/.config/dotnet-tools.json b/packages/fileimport-service/src/ifc-dotnet/.config/dotnet-tools.json similarity index 65% rename from packages/fileimport-service/ifc-dotnet/.config/dotnet-tools.json rename to packages/fileimport-service/src/ifc-dotnet/.config/dotnet-tools.json index db21a0aee..6e997c414 100644 --- a/packages/fileimport-service/ifc-dotnet/.config/dotnet-tools.json +++ b/packages/fileimport-service/src/ifc-dotnet/.config/dotnet-tools.json @@ -4,9 +4,7 @@ "tools": { "csharpier": { "version": "0.30.1", - "commands": [ - "dotnet-csharpier" - ] + "commands": ["dotnet-csharpier"] } } -} \ No newline at end of file +} diff --git a/packages/fileimport-service/src/ifc-dotnet/Program.cs b/packages/fileimport-service/src/ifc-dotnet/Program.cs new file mode 100644 index 000000000..55fc7ff50 --- /dev/null +++ b/packages/fileimport-service/src/ifc-dotnet/Program.cs @@ -0,0 +1,65 @@ +using System.CommandLine; +using System.Text.Json; +using Speckle.Importers.Ifc; +using Speckle.Sdk.Common; +using Speckle.Sdk.Models.Extensions; + +var filePathArgument = new Argument("filePath"); +var outputPathArgument = new Argument("outputPath"); +var projectIdArgument = new Argument("projectId"); +var versionMessageArgument = new Argument("versionMessage"); +var modelIdArgument = new Argument("modelId"); +var modelNameArgument = new Argument("modelName"); +var regionNameArgument = new Argument("regionName"); + +var rootCommand = new RootCommand +{ + filePathArgument, + outputPathArgument, + projectIdArgument, + versionMessageArgument, + modelIdArgument, + modelNameArgument, + regionNameArgument, +}; + +rootCommand.SetHandler( + async (filePath, outputPath, projectId, versionMessage, modelId, modelName, _) => + { + try + { + var token = Environment.GetEnvironmentVariable("USER_TOKEN").NotNull("USER_TOKEN is missing"); + var url = Environment.GetEnvironmentVariable("SPECKLE_SERVER_URL") ?? "http://127.0.0.1:3000"; + ImporterArgs args = new() + { + ServerUrl = new(url), + FilePath = filePath, + ProjectId = projectId, + ModelId = modelId, + ModelName = modelName, + VersionMessage = versionMessage, + Token = token + }; + + var version = await Import.Ifc(args); + File.WriteAllText(outputPath, JsonSerializer.Serialize(new { success = true, commitId = version.id })); + } + catch (Exception e) + { + Console.WriteLine($"IFC Importer failed with exception {e.ToFormattedString()}"); + + File.WriteAllText( + outputPath, + JsonSerializer.Serialize(new { success = false, error = e.ToFormattedString() }) + ); + } + }, + filePathArgument, + outputPathArgument, + projectIdArgument, + versionMessageArgument, + modelIdArgument, + modelNameArgument, + regionNameArgument +); +await rootCommand.InvokeAsync(args); diff --git a/packages/fileimport-service/ifc-dotnet/ifc-converter.csproj b/packages/fileimport-service/src/ifc-dotnet/ifc-converter.csproj similarity index 85% rename from packages/fileimport-service/ifc-dotnet/ifc-converter.csproj rename to packages/fileimport-service/src/ifc-dotnet/ifc-converter.csproj index 3183ad96c..42a5aad86 100644 --- a/packages/fileimport-service/ifc-dotnet/ifc-converter.csproj +++ b/packages/fileimport-service/src/ifc-dotnet/ifc-converter.csproj @@ -10,7 +10,7 @@ - + diff --git a/packages/fileimport-service/ifc-dotnet/ifc-converter.sln b/packages/fileimport-service/src/ifc-dotnet/ifc-converter.sln similarity index 100% rename from packages/fileimport-service/ifc-dotnet/ifc-converter.sln rename to packages/fileimport-service/src/ifc-dotnet/ifc-converter.sln diff --git a/packages/fileimport-service/ifc-dotnet/ifcs/171210AISC_Sculpture_brep.ifc b/packages/fileimport-service/src/ifc-dotnet/ifcs/171210AISC_Sculpture_brep.ifc similarity index 100% rename from packages/fileimport-service/ifc-dotnet/ifcs/171210AISC_Sculpture_brep.ifc rename to packages/fileimport-service/src/ifc-dotnet/ifcs/171210AISC_Sculpture_brep.ifc diff --git a/packages/fileimport-service/ifc-dotnet/ifcs/20160414office_model_CV2_fordesign.ifc b/packages/fileimport-service/src/ifc-dotnet/ifcs/20160414office_model_CV2_fordesign.ifc similarity index 100% rename from packages/fileimport-service/ifc-dotnet/ifcs/20160414office_model_CV2_fordesign.ifc rename to packages/fileimport-service/src/ifc-dotnet/ifcs/20160414office_model_CV2_fordesign.ifc diff --git a/packages/fileimport-service/ifc-dotnet/ifcs/20210219Architecture.ifc.zip b/packages/fileimport-service/src/ifc-dotnet/ifcs/20210219Architecture.ifc.zip similarity index 100% rename from packages/fileimport-service/ifc-dotnet/ifcs/20210219Architecture.ifc.zip rename to packages/fileimport-service/src/ifc-dotnet/ifcs/20210219Architecture.ifc.zip diff --git a/packages/fileimport-service/ifc-dotnet/ifcs/20210221PRIMARK.ifc b/packages/fileimport-service/src/ifc-dotnet/ifcs/20210221PRIMARK.ifc similarity index 100% rename from packages/fileimport-service/ifc-dotnet/ifcs/20210221PRIMARK.ifc rename to packages/fileimport-service/src/ifc-dotnet/ifcs/20210221PRIMARK.ifc diff --git a/packages/fileimport-service/ifc-dotnet/ifcs/23111023_IFCR2_Buildings_2.ifc b/packages/fileimport-service/src/ifc-dotnet/ifcs/23111023_IFCR2_Buildings_2.ifc similarity index 100% rename from packages/fileimport-service/ifc-dotnet/ifcs/23111023_IFCR2_Buildings_2.ifc rename to packages/fileimport-service/src/ifc-dotnet/ifcs/23111023_IFCR2_Buildings_2.ifc diff --git a/packages/fileimport-service/ifc-dotnet/ifcs/231110AC11-Institute-Var-2-IFC.ifc b/packages/fileimport-service/src/ifc-dotnet/ifcs/231110AC11-Institute-Var-2-IFC.ifc similarity index 100% rename from packages/fileimport-service/ifc-dotnet/ifcs/231110AC11-Institute-Var-2-IFC.ifc rename to packages/fileimport-service/src/ifc-dotnet/ifcs/231110AC11-Institute-Var-2-IFC.ifc diff --git a/packages/fileimport-service/ifc-dotnet/ifcs/231110ADT-FZK-Haus-2005-2006 (1).ifc b/packages/fileimport-service/src/ifc-dotnet/ifcs/231110ADT-FZK-Haus-2005-2006 (1).ifc similarity index 100% rename from packages/fileimport-service/ifc-dotnet/ifcs/231110ADT-FZK-Haus-2005-2006 (1).ifc rename to packages/fileimport-service/src/ifc-dotnet/ifcs/231110ADT-FZK-Haus-2005-2006 (1).ifc diff --git a/packages/fileimport-service/ifc-dotnet/ifcs/231110ADT-FZK-Haus-2005-2006.ifc b/packages/fileimport-service/src/ifc-dotnet/ifcs/231110ADT-FZK-Haus-2005-2006.ifc similarity index 100% rename from packages/fileimport-service/ifc-dotnet/ifcs/231110ADT-FZK-Haus-2005-2006.ifc rename to packages/fileimport-service/src/ifc-dotnet/ifcs/231110ADT-FZK-Haus-2005-2006.ifc diff --git a/packages/fileimport-service/ifc-dotnet/ifcs/example.ifc b/packages/fileimport-service/src/ifc-dotnet/ifcs/example.ifc similarity index 100% rename from packages/fileimport-service/ifc-dotnet/ifcs/example.ifc rename to packages/fileimport-service/src/ifc-dotnet/ifcs/example.ifc diff --git a/packages/fileimport-service/ifc-dotnet/ifcs/railing.ifc b/packages/fileimport-service/src/ifc-dotnet/ifcs/railing.ifc similarity index 100% rename from packages/fileimport-service/ifc-dotnet/ifcs/railing.ifc rename to packages/fileimport-service/src/ifc-dotnet/ifcs/railing.ifc diff --git a/packages/fileimport-service/ifc-dotnet/ifcs/small.ifc b/packages/fileimport-service/src/ifc-dotnet/ifcs/small.ifc similarity index 100% rename from packages/fileimport-service/ifc-dotnet/ifcs/small.ifc rename to packages/fileimport-service/src/ifc-dotnet/ifcs/small.ifc diff --git a/packages/fileimport-service/ifc-dotnet/ifcs/steelplates.ifc b/packages/fileimport-service/src/ifc-dotnet/ifcs/steelplates.ifc similarity index 100% rename from packages/fileimport-service/ifc-dotnet/ifcs/steelplates.ifc rename to packages/fileimport-service/src/ifc-dotnet/ifcs/steelplates.ifc diff --git a/packages/fileimport-service/ifc/import_file.js b/packages/fileimport-service/src/ifc/import_file.js similarity index 81% rename from packages/fileimport-service/ifc/import_file.js rename to packages/fileimport-service/src/ifc/import_file.js index 5eab2cf1c..a57845eb4 100644 --- a/packages/fileimport-service/ifc/import_file.js +++ b/packages/fileimport-service/src/ifc/import_file.js @@ -1,9 +1,8 @@ -const fs = require('fs') -const { logger: parentLogger } = require('../observability/logging') - -const { parseAndCreateCommitFactory } = require('./index') -const Observability = require('@speckle/shared/dist/commonjs/observability/index.js') -const getDbClients = require('../knex') +import fs from 'fs' +import Observability from '@speckle/shared/dist/commonjs/observability/index.js' +import { logger as parentLogger } from '@/observability/logging.js' +import { getDbClients } from '@/knex.js' +import { parseAndCreateCommitFactory } from '@/ifc/index.js' async function main() { const cmdArgs = process.argv.slice(2) diff --git a/packages/fileimport-service/ifc/index.js b/packages/fileimport-service/src/ifc/index.js similarity index 80% rename from packages/fileimport-service/ifc/index.js rename to packages/fileimport-service/src/ifc/index.js index ad3cd4a54..eea1e94c4 100644 --- a/packages/fileimport-service/ifc/index.js +++ b/packages/fileimport-service/src/ifc/index.js @@ -1,11 +1,11 @@ -const { performance } = require('perf_hooks') -const { fetch } = require('undici') -const Parser = require('./parser') -const ServerAPI = require('../src/api.js') -const Observability = require('@speckle/shared/dist/commonjs/observability/index.js') -const { logger: parentLogger } = require('../observability/logging') +import { performance } from 'perf_hooks' +import { fetch } from 'undici' +import Observability from '@speckle/shared/dist/commonjs/observability/index.js' +import { ServerAPI } from '@/controller/api.js' +import { logger as parentLogger } from '@/observability/logging.js' +import { IFCParser } from '@/ifc/parser.js' -const parseAndCreateCommitFactory = +export const parseAndCreateCommitFactory = ({ db }) => async ({ data, @@ -24,7 +24,7 @@ const parseAndCreateCommitFactory = ) } const serverApi = new ServerAPI({ db, streamId, logger }) - const myParser = new Parser({ serverApi, fileId, logger }) + const myParser = new IFCParser({ serverApi, fileId, logger }) const start = performance.now() const { id, tCount } = await myParser.parse(data) @@ -80,5 +80,3 @@ const parseAndCreateCommitFactory = return json.data.commitCreate } - -module.exports = { parseAndCreateCommitFactory } diff --git a/packages/fileimport-service/ifc/parser.js b/packages/fileimport-service/src/ifc/parser.js similarity index 97% rename from packages/fileimport-service/ifc/parser.js rename to packages/fileimport-service/src/ifc/parser.js index 251df7a11..e960fde30 100644 --- a/packages/fileimport-service/ifc/parser.js +++ b/packages/fileimport-service/src/ifc/parser.js @@ -1,16 +1,16 @@ -const { performance } = require('perf_hooks') -const WebIFC = require('web-ifc/web-ifc-api-node') -const { +import { performance } from 'perf_hooks' +import WebIFC from 'web-ifc/web-ifc-api-node.js' +import Observability from '@speckle/shared/dist/commonjs/observability/index.js' +import { logger as parentLogger } from '@/observability/logging.js' +import { getHash, IfcElements, PropNames, GeometryTypes, IfcTypesMap -} = require('./utils') -const Observability = require('@speckle/shared/dist/commonjs/observability/index.js') -const { logger: parentLogger } = require('../observability/logging') +} from '@/ifc/utils.js' -module.exports = class IFCParser { +export class IFCParser { constructor({ serverApi, fileId, logger }) { this.ifcapi = new WebIFC.IfcAPI() this.ifcapi.SetWasmPath('./', false) diff --git a/packages/fileimport-service/ifc/utils.js b/packages/fileimport-service/src/ifc/utils.js similarity index 99% rename from packages/fileimport-service/ifc/utils.js rename to packages/fileimport-service/src/ifc/utils.js index 3ce78d5d5..242443303 100644 --- a/packages/fileimport-service/ifc/utils.js +++ b/packages/fileimport-service/src/ifc/utils.js @@ -1,7 +1,7 @@ -const crypto = require('crypto') -const WebIFC = require('web-ifc/web-ifc-api-node') +import crypto from 'crypto' +import WebIFC from 'web-ifc/web-ifc-api-node.js' -const IfcElements = { +export const IfcElements = { 103090709: 'IFCPROJECT', 4097777520: 'IFCSITE', 4031249490: 'IFCBUILDING', @@ -141,7 +141,7 @@ const IfcElements = { 3009204131: 'IFCGRID' } -const GeometryTypes = new Set([ +export const GeometryTypes = new Set([ 1123145078, 574549367, 1675464909, 2059837836, 3798115385, 32440307, 3125803723, 3207858831, 2740243338, 2624227202, 4240577450, 3615266464, 3724593414, 220341763, 477187591, 1878645084, 1300840506, 3303107099, 1607154358, 1878645084, 846575682, @@ -163,7 +163,7 @@ const GeometryTypes = new Set([ 1682466193, 2519244187, 2839578677, 3958567839, 2513912981, 2830218821, 427810014 ]) -const IfcTypesMap = { +export const IfcTypesMap = { 3821786052: 'IFCACTIONREQUEST', 2296667514: 'IFCACTOR', 3630933823: 'IFCACTORROLE', @@ -982,7 +982,7 @@ const IfcTypesMap = { 1033361043: 'IFCZONE' } -const PropNames = { +export const PropNames = { aggregates: { name: WebIFC.IFCRELAGGREGATES, relating: 'RelatingObject', @@ -1015,14 +1015,6 @@ const PropNames = { } } -const getHash = (obj) => { +export const getHash = (obj) => { return crypto.createHash('md5').update(JSON.stringify(obj)).digest('hex') } - -module.exports = { - IfcElements, - IfcTypesMap, - PropNames, - GeometryTypes, - getHash -} diff --git a/packages/fileimport-service/ifc/web-ifc.wasm b/packages/fileimport-service/src/ifc/web-ifc.wasm similarity index 100% rename from packages/fileimport-service/ifc/web-ifc.wasm rename to packages/fileimport-service/src/ifc/web-ifc.wasm diff --git a/packages/fileimport-service/knex.js b/packages/fileimport-service/src/knex.ts similarity index 76% rename from packages/fileimport-service/knex.js rename to packages/fileimport-service/src/knex.ts index ff8285fef..6062b4761 100644 --- a/packages/fileimport-service/knex.js +++ b/packages/fileimport-service/src/knex.ts @@ -1,18 +1,18 @@ -'use strict' - -const Environment = require('@speckle/shared/dist/commonjs/environment/index.js') -const { +import Environment from '@speckle/shared/dist/commonjs/environment/index.js' +import { loadMultiRegionsConfig, configureKnexClient -} = require('@speckle/shared/dist/commonjs/environment/multiRegionConfig.js') -const { logger } = require('./observability/logging') +} from '@speckle/shared/dist/commonjs/environment/multiRegionConfig.js' +import { logger } from '@/observability/logging.js' +import { Knex } from 'knex' const { FF_WORKSPACES_MULTI_REGION_ENABLED } = Environment.getFeatureFlags() const isDevEnv = process.env.NODE_ENV !== 'production' -let dbClients -const getDbClients = async () => { +type DbClient = { public: Knex; private?: Knex } +let dbClients: { [key: string]: DbClient } +export const getDbClients = async () => { if (dbClients) return dbClients const maxConnections = parseInt( process.env['POSTGRES_MAX_CONNECTIONS_FILE_IMPORT_SERVICE'] || '1' @@ -49,7 +49,10 @@ const getDbClients = async () => { } else { const configPath = process.env.MULTI_REGION_CONFIG_PATH || 'multiregion.json' const config = await loadMultiRegionsConfig({ path: configPath }) - const clients = [['main', configureKnexClient(config.main, configArgs)]] + + const clients: [string, DbClient][] = [ + ['main', configureKnexClient(config.main, configArgs)] + ] Object.entries(config.regions).map(([key, config]) => { clients.push([key, configureKnexClient(config, configArgs)]) }) @@ -57,5 +60,3 @@ const getDbClients = async () => { } return dbClients } - -module.exports = getDbClients diff --git a/packages/fileimport-service/obj/import_file.py b/packages/fileimport-service/src/obj/import_file.py similarity index 100% rename from packages/fileimport-service/obj/import_file.py rename to packages/fileimport-service/src/obj/import_file.py diff --git a/packages/fileimport-service/obj/mtl_file_collection.py b/packages/fileimport-service/src/obj/mtl_file_collection.py similarity index 100% rename from packages/fileimport-service/obj/mtl_file_collection.py rename to packages/fileimport-service/src/obj/mtl_file_collection.py diff --git a/packages/fileimport-service/obj/obj_file.py b/packages/fileimport-service/src/obj/obj_file.py similarity index 100% rename from packages/fileimport-service/obj/obj_file.py rename to packages/fileimport-service/src/obj/obj_file.py diff --git a/packages/fileimport-service/obj/samples/untitled.mtl b/packages/fileimport-service/src/obj/samples/untitled.mtl similarity index 100% rename from packages/fileimport-service/obj/samples/untitled.mtl rename to packages/fileimport-service/src/obj/samples/untitled.mtl diff --git a/packages/fileimport-service/obj/samples/untitled.obj b/packages/fileimport-service/src/obj/samples/untitled.obj similarity index 100% rename from packages/fileimport-service/obj/samples/untitled.obj rename to packages/fileimport-service/src/obj/samples/untitled.obj diff --git a/packages/fileimport-service/src/objDependencies.js b/packages/fileimport-service/src/objDependencies.js deleted file mode 100644 index f7fa03ac4..000000000 --- a/packages/fileimport-service/src/objDependencies.js +++ /dev/null @@ -1,59 +0,0 @@ -'use strict' -const events = require('events') -const fs = require('fs') -const readline = require('readline') -const path = require('path') - -const { downloadFile, getFileInfoByName } = require('./filesApi') -const isValidFilename = require('valid-filename') -const { logger } = require('../observability/logging') - -const getReferencedMtlFiles = async ({ objFilePath }) => { - const mtlFiles = [] - - try { - const rl = readline.createInterface({ - input: fs.createReadStream(objFilePath), - crlfDelay: Infinity - }) - - rl.on('line', (line) => { - if (line.startsWith('mtllib ')) { - const mtlFile = line.slice('mtllib '.length).trim() - mtlFiles.push(mtlFile) - } - }) - - await events.once(rl, 'close') - } catch (err) { - logger.error(err, `Error getting dependencies for file ${objFilePath}`) - } - return mtlFiles -} - -module.exports = { - async downloadDependencies({ objFilePath, streamId, destinationDir, token }) { - const dependencies = await getReferencedMtlFiles({ objFilePath }) - - logger.info(`Obj file depends on ${dependencies}`) - for (const mtlFile of dependencies) { - // there might be multiple files named with the same name, take the first... - const [file] = (await getFileInfoByName({ fileName: mtlFile, streamId, token })) - .blobs - if (!file) { - logger.info(`OBJ dependency file not found in stream: ${mtlFile}`) - continue - } - if (!isValidFilename(mtlFile)) { - logger.warn(`Invalid filename reference in OBJ dependencies: ${mtlFile}`) - continue - } - await downloadFile({ - fileId: file.id, - streamId, - token, - destination: path.join(destinationDir, mtlFile) - }) - } - } -} diff --git a/packages/fileimport-service/observability/logging.js b/packages/fileimport-service/src/observability/logging.ts similarity index 53% rename from packages/fileimport-service/observability/logging.js rename to packages/fileimport-service/src/observability/logging.ts index 31f688f13..6ef8bca14 100644 --- a/packages/fileimport-service/observability/logging.js +++ b/packages/fileimport-service/src/observability/logging.ts @@ -1,14 +1,10 @@ -const Observability = require('@speckle/shared/dist/commonjs/observability/index.js') +import Observability from '@speckle/shared/dist/commonjs/observability/index.js' // loggers for specific components within normal operation -const logger = Observability.extendLoggerComponent( +export const logger = Observability.extendLoggerComponent( Observability.getLogger( process.env.LOG_LEVEL || 'info', process.env.LOG_PRETTY === 'true' ), 'fileimport-service' ) - -module.exports = { - logger -} diff --git a/packages/fileimport-service/src/prometheusMetrics.js b/packages/fileimport-service/src/prometheusMetrics.js deleted file mode 100644 index 6248636cf..000000000 --- a/packages/fileimport-service/src/prometheusMetrics.js +++ /dev/null @@ -1,166 +0,0 @@ -/* eslint-disable no-unused-vars */ -'use strict' - -const http = require('http') -const prometheusClient = require('prom-client') -const getDbClients = require('../knex') - -let metricFree = null -let metricUsed = null -let metricPendingAquires = null -let metricPendingCreates = null -let metricPendingValidations = null -let metricRemainingCapacity = null -let metricQueryDuration = null -let metricQueryErrors = null - -const queryStartTime = {} -prometheusClient.register.clear() -prometheusClient.register.setDefaultLabels({ - project: 'speckle-server', - app: 'fileimport-service' -}) -prometheusClient.collectDefaultMetrics() - -let prometheusInitialized = false - -const initDBPrometheusMetricsFactory = - ({ db }) => - () => { - metricFree = new prometheusClient.Gauge({ - name: 'speckle_server_knex_free', - help: 'Number of free DB connections', - collect() { - this.set(db.client.pool.numFree()) - } - }) - - metricUsed = new prometheusClient.Gauge({ - name: 'speckle_server_knex_used', - help: 'Number of used DB connections', - collect() { - this.set(db.client.pool.numUsed()) - } - }) - - metricPendingAquires = new prometheusClient.Gauge({ - name: 'speckle_server_knex_pending', - help: 'Number of pending DB connection aquires', - collect() { - this.set(db.client.pool.numPendingAcquires()) - } - }) - - metricPendingCreates = new prometheusClient.Gauge({ - name: 'speckle_server_knex_pending_creates', - help: 'Number of pending DB connection creates', - collect() { - this.set(db.client.pool.numPendingCreates()) - } - }) - - metricPendingValidations = new prometheusClient.Gauge({ - name: 'speckle_server_knex_pending_validations', - help: 'Number of pending DB connection validations. This is a state between pending acquisition and acquiring a connection.', - collect() { - this.set(db.client.pool.numPendingValidations()) - } - }) - - metricRemainingCapacity = new prometheusClient.Gauge({ - name: 'speckle_server_knex_remaining_capacity', - help: 'Remaining capacity of the DB connection pool', - collect() { - const postgresMaxConnections = - parseInt(process.env.POSTGRES_MAX_CONNECTIONS_FILE_IMPORT_SERVICE) || 1 - const demand = - db.client.pool.numUsed() + - db.client.pool.numPendingCreates() + - db.client.pool.numPendingValidations() + - db.client.pool.numPendingAcquires() - - this.set(Math.max(postgresMaxConnections - demand, 0)) - } - }) - - metricQueryDuration = new prometheusClient.Summary({ - name: 'speckle_server_knex_query_duration', - help: 'Summary of the DB query durations in seconds' - }) - - metricQueryErrors = new prometheusClient.Counter({ - name: 'speckle_server_knex_query_errors', - help: 'Number of DB queries with errors' - }) - - db.on('query', (data) => { - const queryId = data.__knexQueryUid + '' - queryStartTime[queryId] = Date.now() - }) - - db.on('query-response', (data, obj, builder) => { - const queryId = obj.__knexQueryUid + '' - const durationSec = (Date.now() - queryStartTime[queryId]) / 1000 - delete queryStartTime[queryId] - if (!isNaN(durationSec)) metricQueryDuration.observe(durationSec) - }) - - db.on('query-error', (err, querySpec) => { - const queryId = querySpec.__knexQueryUid + '' - const durationSec = (Date.now() - queryStartTime[queryId]) / 1000 - delete queryStartTime[queryId] - - if (!isNaN(durationSec)) metricQueryDuration.observe(durationSec) - metricQueryErrors.inc() - }) - } - -module.exports = { - async initPrometheusMetrics() { - if (prometheusInitialized) return - prometheusInitialized = true - - const db = (await getDbClients()).main.public - - initDBPrometheusMetricsFactory({ db })() - - // Define the HTTP server - const server = http.createServer(async (req, res) => { - if (req.url === '/metrics') { - res.setHeader('Content-Type', prometheusClient.register.contentType) - res.end(await prometheusClient.register.metrics()) - } else { - res.end('Speckle FileImport Service - prometheus metrics') - } - }) - server.listen(Number(process.env.PROMETHEUS_METRICS_PORT) || 9093) - }, - - metricDuration: new prometheusClient.Histogram({ - name: 'speckle_server_operation_duration', - help: 'Summary of the operation durations in seconds', - buckets: [0.5, 1, 5, 10, 30, 60, 300, 600, 900, 1200], - labelNames: ['op'] - }), - - metricOperationErrors: new prometheusClient.Counter({ - name: 'speckle_server_operation_errors', - help: 'Number of operations with errors', - labelNames: ['op'] - }), - - metricInputFileSize: new prometheusClient.Histogram({ - name: 'speckle_server_operation_file_size', - help: 'Size of the operation input file size', - buckets: [ - 1000, - 100 * 1000, - 500 * 1000, - 1000 * 1000, - 5 * 1000 * 1000, - 10 * 1000 * 1000, - 100 * 1000 * 1000 - ], - labelNames: ['op'] - }) -} diff --git a/packages/preview-service/src/root.ts b/packages/fileimport-service/src/root.ts similarity index 100% rename from packages/preview-service/src/root.ts rename to packages/fileimport-service/src/root.ts diff --git a/packages/fileimport-service/stl/import_file.py b/packages/fileimport-service/src/stl/import_file.py similarity index 100% rename from packages/fileimport-service/stl/import_file.py rename to packages/fileimport-service/src/stl/import_file.py diff --git a/packages/fileimport-service/stl/samples/Gizmo_Spoon_Rider_bin.stl b/packages/fileimport-service/src/stl/samples/Gizmo_Spoon_Rider_bin.stl similarity index 100% rename from packages/fileimport-service/stl/samples/Gizmo_Spoon_Rider_bin.stl rename to packages/fileimport-service/src/stl/samples/Gizmo_Spoon_Rider_bin.stl diff --git a/packages/fileimport-service/tsconfig.build.json b/packages/fileimport-service/tsconfig.build.json new file mode 100644 index 000000000..8e1bc28f9 --- /dev/null +++ b/packages/fileimport-service/tsconfig.build.json @@ -0,0 +1,5 @@ +{ + "extends": "./tsconfig.json", + "include": ["src/**/*"], + "exclude": ["**/*.spec.js", "**/*.spec.ts"] +} diff --git a/packages/fileimport-service/tsconfig.json b/packages/fileimport-service/tsconfig.json new file mode 100644 index 000000000..4351d814b --- /dev/null +++ b/packages/fileimport-service/tsconfig.json @@ -0,0 +1,108 @@ +{ + "compilerOptions": { + /* Visit https://aka.ms/tsconfig.json to read more about this file */ + + /* Projects */ + // "incremental": true, /* Enable incremental compilation */ + // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */ + // "tsBuildInfoFile": "./", /* Specify the folder for .tsbuildinfo incremental compilation files. */ + // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects */ + // "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */ + // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ + + /* Language and Environment */ + "target": "ES2022" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */, + // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ + // "jsx": "preserve", /* Specify what JSX code is generated. */ + // "experimentalDecorators": true, /* Enable experimental support for TC39 stage 2 draft decorators. */ + // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ + // "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h' */ + // "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */ + // "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using `jsx: react-jsx*`.` */ + // "reactNamespace": "", /* Specify the object invoked for `createElement`. This only applies when targeting `react` JSX emit. */ + // "noLib": true, /* Disable including any library files, including the default lib.d.ts. */ + // "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */ + + /* Modules */ + "module": "node16" /* Specify what module code is generated. */, + "rootDir": "./" /* Specify the root folder within your source files. */, + "moduleResolution": "node16" /* Specify how TypeScript looks up a file from a given module specifier. */, + "baseUrl": "./" /* Specify the base directory to resolve non-relative module names. */, + "paths": { + "@/*": ["./src/*"] + }, + // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ + // "typeRoots": [], /* Specify multiple folders that act like `./node_modules/@types`. */ + // "types": [], /* Specify type package names to be included without being referenced in a source file. */ + // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ + // "resolveJsonModule": true, /* Enable importing .json files */ + // "noResolve": true, /* Disallow `import`s, `require`s or ``s from expanding the number of files TypeScript should add to a project. */ + + /* JavaScript Support */ + "allowJs": true /* Allow JavaScript files to be a part of your program. Use the `checkJS` option to get errors from these files. */, + "checkJs": false /* Enable error reporting in type-checked JavaScript files. */, + // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from `node_modules`. Only applicable with `allowJs`. */ + + /* Emit */ + // "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ + // "declarationMap": true, /* Create sourcemaps for d.ts files. */ + // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ + "sourceMap": true /* Create source map files for emitted JavaScript files. */, + // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If `declaration` is true, also designates a file that bundles all .d.ts output. */ + "outDir": "./dist" /* Specify an output folder for all emitted files. */, + // "removeComments": true, /* Disable emitting comments. */ + // "noEmit": true, /* Disable emitting files from a compilation. */ + // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ + // "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types */ + // "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */ + // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */ + // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ + // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ + // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */ + // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */ + // "newLine": "crlf", /* Set the newline character for emitting files. */ + // "stripInternal": true, /* Disable emitting declarations that have `@internal` in their JSDoc comments. */ + // "noEmitHelpers": true, /* Disable generating custom helper functions like `__extends` in compiled output. */ + // "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */ + // "preserveConstEnums": true, /* Disable erasing `const enum` declarations in generated code. */ + // "declarationDir": "./", /* Specify the output directory for generated declaration files. */ + // "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */ + + /* Interop Constraints */ + // "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */ + "allowSyntheticDefaultImports": true /* Allow 'import x from y' when a module doesn't have a default export. */, + "esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables `allowSyntheticDefaultImports` for type compatibility. */, + // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ + "forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */, + + /* Type Checking */ + "strict": true /* Enable all strict type-checking options. */, + // "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied `any` type.. */ + // "strictNullChecks": true, /* When type checking, take into account `null` and `undefined`. */ + // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ + // "strictBindCallApply": true, /* Check that the arguments for `bind`, `call`, and `apply` methods match the original function. */ + // "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ + // "noImplicitThis": true, /* Enable error reporting when `this` is given the type `any`. */ + // "useUnknownInCatchVariables": true, /* Type catch clause variables as 'unknown' instead of 'any'. */ + // "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ + // "noUnusedLocals": true, /* Enable error reporting when a local variables aren't read. */ + // "noUnusedParameters": true, /* Raise an error when a function parameter isn't read */ + // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ + // "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ + // "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ + // "noUncheckedIndexedAccess": true, /* Include 'undefined' in index signature results */ + // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ + // "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type */ + // "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ + // "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ + + /* Completeness */ + // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ + "skipLibCheck": true /* Skip type checking all .d.ts files. */ + }, + "ts-node": { + "swc": true + }, + "include": ["src/**/*", "vitest.config.ts"], + "exclude": ["node_modules", "coverage", "reports"] +} diff --git a/packages/preview-service/vitest.config.ts b/packages/fileimport-service/vitest.config.ts similarity index 72% rename from packages/preview-service/vitest.config.ts rename to packages/fileimport-service/vitest.config.ts index a751fa9b1..dbf4dedd5 100644 --- a/packages/preview-service/vitest.config.ts +++ b/packages/fileimport-service/vitest.config.ts @@ -4,7 +4,6 @@ import { configDefaults, defineConfig } from 'vitest/config' export default defineConfig({ test: { exclude: [...configDefaults.exclude], - globalSetup: ['./tests/hooks/globalSetup.ts'], // reporters: ['verbose', 'hanging-process'] //uncomment to debug hanging processes etc. sequence: { shuffle: true, @@ -13,8 +12,7 @@ export default defineConfig({ }, resolve: { alias: { - '@': path.resolve(__dirname, './src'), - '#': path.resolve(__dirname, './tests') + '@': path.resolve(__dirname, './src') } } }) diff --git a/packages/frontend-2/.env.example b/packages/frontend-2/.env.example index b2ff12ddf..6caf83c7b 100644 --- a/packages/frontend-2/.env.example +++ b/packages/frontend-2/.env.example @@ -37,8 +37,5 @@ NUXT_PUBLIC_ENABLE_AUTOMATE_MODULE=false # Survicate NUXT_PUBLIC_SURVICATE_WORKSPACE_KEY= -# Enable direct preview image loading - way quicker, but requres server & frontend to be on the same origin -NUXT_PUBLIC_ENABLE_DIRECT_PREVIEWS=true - # Ghost API NUXT_PUBLIC_GHOST_API_KEY= \ No newline at end of file diff --git a/packages/frontend-2/Dockerfile b/packages/frontend-2/Dockerfile index 7331f8018..d4777a56b 100644 --- a/packages/frontend-2/Dockerfile +++ b/packages/frontend-2/Dockerfile @@ -1,6 +1,10 @@ FROM node:22-bookworm-slim@sha256:221ee67425de7a3c11ce4e81e63e50caaec82ede3a7d34599ab20e59d29a0cb5 AS build-stage ARG NODE_ENV=production ARG SPECKLE_SERVER_VERSION=custom +ARG BUILD_SOURCEMAPS=false + +# for better sourcemaps (the app still gets minified at the end) +ENV SKIP_LIBRARY_MINIFICATION=true WORKDIR /speckle-server @@ -30,15 +34,13 @@ COPY packages/frontend-2 ./packages/frontend-2/ RUN yarn workspaces focus -A # hadolint ignore=DL3059 RUN yarn workspaces foreach -W run build -# hadolint ignore=DL3059 -RUN find ./packages/frontend-2/.output/ -type f \( -name "*.js.map" -o -name "*.mjs.map" -o -name "*.cjs.map" \) -exec rm -f {} \; ENV TINI_VERSION=v0.19.0 RUN apt-get update -y \ && DEBIAN_FRONTEND=noninteractive apt-get install -y \ --no-install-recommends \ ca-certificates=20230311 \ - curl=7.88.1-10+deb12u8 \ + curl=7.88.1-10+deb12u12 \ && curl -fsSL https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini -o /tini \ && chmod +x /tini \ && apt-get remove -y curl \ diff --git a/packages/frontend-2/assets/images/banners/grab-your-tickets.gif b/packages/frontend-2/assets/images/banners/grab-your-tickets.gif deleted file mode 100644 index cecaf49da..000000000 Binary files a/packages/frontend-2/assets/images/banners/grab-your-tickets.gif and /dev/null differ diff --git a/packages/frontend-2/assets/images/banners/speckleverse.svg b/packages/frontend-2/assets/images/banners/speckleverse.svg deleted file mode 100644 index 593dee238..000000000 --- a/packages/frontend-2/assets/images/banners/speckleverse.svg +++ /dev/null @@ -1,65 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/packages/frontend-2/assets/images/banners/workspace-promo-dark.png b/packages/frontend-2/assets/images/banners/workspace-promo-dark.png deleted file mode 100644 index a7561224a..000000000 Binary files a/packages/frontend-2/assets/images/banners/workspace-promo-dark.png and /dev/null differ diff --git a/packages/frontend-2/assets/images/banners/workspace-promo-light.png b/packages/frontend-2/assets/images/banners/workspace-promo-light.png deleted file mode 100644 index 87aad0592..000000000 Binary files a/packages/frontend-2/assets/images/banners/workspace-promo-light.png and /dev/null differ diff --git a/packages/frontend-2/assets/images/banners/workspace-promo-mobile-dark.png b/packages/frontend-2/assets/images/banners/workspace-promo-mobile-dark.png deleted file mode 100644 index b5918819e..000000000 Binary files a/packages/frontend-2/assets/images/banners/workspace-promo-mobile-dark.png and /dev/null differ diff --git a/packages/frontend-2/assets/images/banners/workspace-promo-mobile-light.png b/packages/frontend-2/assets/images/banners/workspace-promo-mobile-light.png deleted file mode 100644 index 565ed44e6..000000000 Binary files a/packages/frontend-2/assets/images/banners/workspace-promo-mobile-light.png and /dev/null differ diff --git a/packages/frontend-2/assets/images/comments_intro_320x248.webp b/packages/frontend-2/assets/images/comments_intro_320x248.webp deleted file mode 100644 index 4848bffc8..000000000 Binary files a/packages/frontend-2/assets/images/comments_intro_320x248.webp and /dev/null differ diff --git a/packages/frontend-2/codegen.ts b/packages/frontend-2/codegen.ts index 04279dfe8..afbde2729 100644 --- a/packages/frontend-2/codegen.ts +++ b/packages/frontend-2/codegen.ts @@ -23,6 +23,7 @@ const config: CodegenConfig = { './lib/common/generated/gql/': { preset: 'client', config: { + enumsAsConst: true, useTypeImports: true, fragmentMasking: false, dedupeFragments: true, diff --git a/packages/frontend-2/components/auth/RegisterWithEmailBlock.vue b/packages/frontend-2/components/auth/RegisterWithEmailBlock.vue index 2b339b982..96bc6bbd5 100644 --- a/packages/frontend-2/components/auth/RegisterWithEmailBlock.vue +++ b/packages/frontend-2/components/auth/RegisterWithEmailBlock.vue @@ -6,7 +6,7 @@ v-model="email" type="email" name="email" - label="Email" + label="Work email" placeholder="Email" size="lg" color="foundation" @@ -69,17 +69,14 @@ import { ToastNotificationType, useGlobalToast } from '~~/lib/common/composables import { ensureError } from '@speckle/shared' import { useAuthManager } from '~~/lib/auth/composables/auth' import { loginRoute } from '~~/lib/common/helpers/route' -import { passwordRules } from '~~/lib/auth/helpers/validation' +import { + passwordRules, + doesNotContainBlockedDomain +} from '~~/lib/auth/helpers/validation' import { graphql } from '~~/lib/common/generated/gql' import type { ServerTermsOfServicePrivacyPolicyFragmentFragment } from '~~/lib/common/generated/gql/graphql' import { useMounted } from '@vueuse/core' -/** - * TODO: - * - (BE) Password strength check? Do we want to use it anymore? - * - Dim's answer: no, `passwordRules` are legit enough for now. - */ - graphql(` fragment ServerTermsOfServicePrivacyPolicyFragment on ServerInfo { termsOfService @@ -99,13 +96,18 @@ const router = useRouter() const { signUpWithEmail, inviteToken } = useAuthManager() const { triggerNotification } = useGlobalToast() const isMounted = useMounted() +const isNoPersonalEmailsEnabled = useIsNoPersonalEmailsEnabled() const newsletterConsent = defineModel('newsletterConsent', { required: true }) const loading = ref(false) const password = ref('') const email = ref('') -const emailRules = [isEmail] +const emailRules = computed(() => + inviteToken.value || !isNoPersonalEmailsEnabled.value + ? [isEmail] + : [isEmail, doesNotContainBlockedDomain] +) const nameRules = [isRequired] const isEmailDisabled = computed(() => !!props.inviteEmail?.length || loading.value) diff --git a/packages/frontend-2/components/automate/automation/CreateDialog.vue b/packages/frontend-2/components/automate/automation/CreateDialog.vue index 20f969b7b..691ebacce 100644 --- a/packages/frontend-2/components/automate/automation/CreateDialog.vue +++ b/packages/frontend-2/components/automate/automation/CreateDialog.vue @@ -223,7 +223,9 @@ const buttons = computed((): LayoutDialogButton[] => { disabled: !selectedFunction.value }, onClick: () => { - mixpanel.track('Automate Select Function') + mixpanel.track('Automate Select Function', { + functionId: selectedFunction?.value?.id + }) step.value++ } } @@ -242,7 +244,9 @@ const buttons = computed((): LayoutDialogButton[] => { id: 'fnParamsNext', text: 'Next', onClick: () => { - mixpanel.track('Automate Set Function Parameters ') + mixpanel.track('Automate Set Function Parameters', { + functionId: selectedFunction?.value?.id + }) }, props: { disabled: hasParameterErrors.value @@ -264,7 +268,9 @@ const buttons = computed((): LayoutDialogButton[] => { id: 'detailsCreate', text: 'Create', onClick: () => { - mixpanel.track('Automate Set Automation Details') + mixpanel.track('Automate Set Automation Details', { + functionId: selectedFunction?.value?.id + }) }, submit: true, disabled: creationLoading.value @@ -420,7 +426,7 @@ const onDetailsSubmit = handleDetailsSubmit(async () => { parameters: encryptedParams } ], - triggerDefinitions: { + triggerDefinitions: { version: Automate.AutomateTypes.TRIGGER_DEFINITIONS_SCHEMA_VERSION, definitions: [ { @@ -428,7 +434,7 @@ const onDetailsSubmit = handleDetailsSubmit(async () => { modelId: model.id } ] - } + } as Automate.AutomateTypes.TriggerDefinitionsSchema } }, { hideSuccessToast: true } diff --git a/packages/frontend-2/components/automate/automation/create-dialog/SelectFunctionStep.vue b/packages/frontend-2/components/automate/automation/create-dialog/SelectFunctionStep.vue index 46c327ae7..b94d3210c 100644 --- a/packages/frontend-2/components/automate/automation/create-dialog/SelectFunctionStep.vue +++ b/packages/frontend-2/components/automate/automation/create-dialog/SelectFunctionStep.vue @@ -39,7 +39,7 @@ import { useDebouncedTextInput } from '@speckle/ui-components' import { useQueryLoading } from '@vue/apollo-composable' import { graphql } from '~/lib/common/generated/gql' import type { CreateAutomationSelectableFunction } from '~/lib/automate/helpers/automations' -import type { Optional } from '@speckle/shared' +import type { Nullable, Optional } from '@speckle/shared' import { usePaginatedQuery } from '~/lib/common/composables/graphql' const searchQuery = graphql(` @@ -91,7 +91,8 @@ const { query: searchQuery, baseVariables: computed(() => ({ workspaceId: props.workspaceId ?? '', - search: search.value?.length ? search.value : '' + search: search.value?.length ? search.value : '', + cursor: null as Nullable })), resolveKey: (vars) => [vars.search || ''], resolveCurrentResult: (res) => res?.workspace?.automateFunctions, diff --git a/packages/frontend-2/components/automate/runs/trigger-status/dialog/FunctionRun.vue b/packages/frontend-2/components/automate/runs/trigger-status/dialog/FunctionRun.vue index c344250d6..4a951242b 100644 --- a/packages/frontend-2/components/automate/runs/trigger-status/dialog/FunctionRun.vue +++ b/packages/frontend-2/components/automate/runs/trigger-status/dialog/FunctionRun.vue @@ -22,13 +22,7 @@
Function is {{ functionRun.status.toLowerCase() }}. @@ -141,4 +135,14 @@ const results = useAutomationFunctionRunResults({ const showAttachmentDialog = ref(false) const attachments = computed(() => results.value?.values.blobIds || []) + +const isStartingOrRunning = computed(() => + ( + [ + AutomateRunStatus.Initializing, + AutomateRunStatus.Running, + AutomateRunStatus.Pending + ] as string[] + ).includes(props.functionRun.status) +) diff --git a/packages/frontend-2/components/automate/viewer/Panel.vue b/packages/frontend-2/components/automate/viewer/Panel.vue index 8bc685842..1bb55c061 100644 --- a/packages/frontend-2/components/automate/viewer/Panel.vue +++ b/packages/frontend-2/components/automate/viewer/Panel.vue @@ -26,7 +26,7 @@
diff --git a/packages/frontend-2/components/common/TransitioningContents.vue b/packages/frontend-2/components/common/TransitioningContents.vue index f63f4cfec..614d26772 100644 --- a/packages/frontend-2/components/common/TransitioningContents.vue +++ b/packages/frontend-2/components/common/TransitioningContents.vue @@ -3,7 +3,8 @@ import { waitIntervalUntil, type Nullable, timeoutAt, - WaitIntervalUntilCanceledError + WaitIntervalUntilCanceledError, + TimeoutError } from '@speckle/shared' import { until } from '@vueuse/core' import type { CSSProperties } from 'vue' @@ -26,6 +27,8 @@ export default defineComponent({ } }, setup(props, { slots, expose }) { + const logger = useLogger() + const transitioning = ref(false) const newWrapperRef = ref(null as Nullable) const oldWrapperRef = ref(null as Nullable) @@ -59,7 +62,7 @@ export default defineComponent({ if (expectStyle) { for (const [key, value] of Object.entries(expectStyle)) { - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any + // eslint-disable-next-line @typescript-eslint/no-explicit-any if (el.style[key as any] !== value) { return false } @@ -102,6 +105,8 @@ export default defineComponent({ * Cause default slot to update with an opacity transition */ const updateContents = async () => { + transitioning.value = true + // Stage 1: Just move new -> old w/o any transitions (visually should look the same) oldContents.value = newContents.value newContents.value = slots.default?.() @@ -155,15 +160,23 @@ export default defineComponent({ const triggerTransition = async () => { if (!transitioning.value) { - transitioning.value = true await updateContents() return } - await Promise.race([ - until(transitioning).toBe(false), - timeoutAt(props.duration + 1000) - ]) + try { + await Promise.race([ + until(transitioning).toBe(false), + timeoutAt(props.duration + 1000, 'Waiting for transition to finish timed out') + ]) + } catch (e) { + if (!(e instanceof TimeoutError)) { + throw e + } else { + logger.warn(e) + } + } + await updateContents() } diff --git a/packages/frontend-2/components/common/tiptap/MentionList.vue b/packages/frontend-2/components/common/tiptap/MentionList.vue index 24b962f1c..953a77c8c 100644 --- a/packages/frontend-2/components/common/tiptap/MentionList.vue +++ b/packages/frontend-2/components/common/tiptap/MentionList.vue @@ -1,11 +1,11 @@ + + + + + + + + + + + + - - - - - - +
+ + +
+ + + diff --git a/packages/frontend-2/components/error/page/ProjectAccessErrorBlock.vue b/packages/frontend-2/components/error/page/ProjectAccessErrorBlock.vue index 3a56795a7..f6cba4ec4 100644 --- a/packages/frontend-2/components/error/page/ProjectAccessErrorBlock.vue +++ b/packages/frontend-2/components/error/page/ProjectAccessErrorBlock.vue @@ -7,7 +7,7 @@ block @processed="onProcessed" /> - + diff --git a/packages/frontend-2/components/header/NavUserMenu.vue b/packages/frontend-2/components/header/NavUserMenu.vue index 85fdd8057..b3eb26d4b 100644 --- a/packages/frontend-2/components/header/NavUserMenu.vue +++ b/packages/frontend-2/components/header/NavUserMenu.vue @@ -19,77 +19,64 @@ -
+
+ + + Settings + + + + + Server settings + + - Connector downloads + {{ isDarkTheme ? 'Light mode' : 'Dark mode' }} + + + + + Invite to Speckle + + + + + Feedback
- - - Settings - - - - - Server settings - - - - - {{ isDarkTheme ? 'Light mode' : 'Dark mode' }} - - - - - Invite to Speckle - - - - - Feedback - -
+
+ + + + + + + +
+ + + diff --git a/packages/frontend-2/components/header/WorkspaceSwitcher/WorkspaceSwitcher.vue b/packages/frontend-2/components/header/WorkspaceSwitcher/WorkspaceSwitcher.vue new file mode 100644 index 000000000..4858edfc1 --- /dev/null +++ b/packages/frontend-2/components/header/WorkspaceSwitcher/WorkspaceSwitcher.vue @@ -0,0 +1,222 @@ + + diff --git a/packages/frontend-2/components/header/WorkspaceSwitcher/header/Header.vue b/packages/frontend-2/components/header/WorkspaceSwitcher/header/Header.vue new file mode 100644 index 000000000..2753dc81d --- /dev/null +++ b/packages/frontend-2/components/header/WorkspaceSwitcher/header/Header.vue @@ -0,0 +1,33 @@ + + + diff --git a/packages/frontend-2/components/header/WorkspaceSwitcher/header/Projects.vue b/packages/frontend-2/components/header/WorkspaceSwitcher/header/Projects.vue new file mode 100644 index 000000000..6778161cd --- /dev/null +++ b/packages/frontend-2/components/header/WorkspaceSwitcher/header/Projects.vue @@ -0,0 +1,9 @@ + + + diff --git a/packages/frontend-2/components/header/WorkspaceSwitcher/header/SsoExpired.vue b/packages/frontend-2/components/header/WorkspaceSwitcher/header/SsoExpired.vue new file mode 100644 index 000000000..41e4cbb46 --- /dev/null +++ b/packages/frontend-2/components/header/WorkspaceSwitcher/header/SsoExpired.vue @@ -0,0 +1,52 @@ + + + diff --git a/packages/frontend-2/components/header/WorkspaceSwitcher/header/Workspace.vue b/packages/frontend-2/components/header/WorkspaceSwitcher/header/Workspace.vue new file mode 100644 index 000000000..aaa22d16f --- /dev/null +++ b/packages/frontend-2/components/header/WorkspaceSwitcher/header/Workspace.vue @@ -0,0 +1,75 @@ + + + diff --git a/packages/frontend-2/components/invite/dialog/CancelInvite.vue b/packages/frontend-2/components/invite/dialog/CancelInvite.vue index e4da08545..d9d29a17d 100644 --- a/packages/frontend-2/components/invite/dialog/CancelInvite.vue +++ b/packages/frontend-2/components/invite/dialog/CancelInvite.vue @@ -38,4 +38,7 @@ const dialogButtons = computed((): LayoutDialogButton[] => [ } } ]) + +// testing deployments in prod, ignore this +markUsed('a') diff --git a/packages/frontend-2/components/invite/dialog/Workspace.vue b/packages/frontend-2/components/invite/dialog/Workspace.vue index 41a88abd7..446d962f8 100644 --- a/packages/frontend-2/components/invite/dialog/Workspace.vue +++ b/packages/frontend-2/components/invite/dialog/Workspace.vue @@ -29,7 +29,7 @@ import { } from '~/lib/common/generated/gql/graphql' import type { InviteGenericItem } from '~~/lib/invites/helpers/types' import { emptyInviteGenericItem } from '~~/lib/invites/helpers/constants' -import { Roles } from '@speckle/shared' +import { Roles, type MaybeNullOrUndefined } from '@speckle/shared' import { useMixpanel } from '~/lib/core/composables/mp' import { mapMainRoleToGqlWorkspaceRole } from '~/lib/workspaces/helpers/roles' import { mapServerRoleToGqlServerRole } from '~/lib/common/helpers/roles' @@ -58,7 +58,7 @@ graphql(` `) const props = defineProps<{ - workspace: InviteDialogWorkspace_WorkspaceFragment + workspace?: MaybeNullOrUndefined }>() const isOpen = defineModel('open', { required: true }) @@ -75,7 +75,7 @@ const invites = ref([ ]) const allowedDomains = computed(() => - props.workspace.domainBasedMembershipProtectionEnabled + props.workspace?.domainBasedMembershipProtectionEnabled ? props.workspace.domains?.map((d) => d.domain) : null ) @@ -117,7 +117,7 @@ const onSelectUsersSubmit = async (updatedInvites: InviteGenericItem[]) => { : undefined })) - if (!inputs.length) return + if (!inputs.length || !props.workspace?.id) return await inviteToWorkspace({ workspaceId: props.workspace.id, inputs }) isOpen.value = false diff --git a/packages/frontend-2/components/onboarding/JoinTeammates.vue b/packages/frontend-2/components/onboarding/JoinTeammates.vue deleted file mode 100644 index c0b65e96b..000000000 --- a/packages/frontend-2/components/onboarding/JoinTeammates.vue +++ /dev/null @@ -1,100 +0,0 @@ - - - diff --git a/packages/frontend-2/components/onboarding/dialog/AccountLink.vue b/packages/frontend-2/components/onboarding/dialog/AccountLink.vue deleted file mode 100644 index d5c6e8659..000000000 --- a/packages/frontend-2/components/onboarding/dialog/AccountLink.vue +++ /dev/null @@ -1,47 +0,0 @@ - - diff --git a/packages/frontend-2/components/onboarding/dialog/Base.vue b/packages/frontend-2/components/onboarding/dialog/Base.vue deleted file mode 100644 index c2b7daa3a..000000000 --- a/packages/frontend-2/components/onboarding/dialog/Base.vue +++ /dev/null @@ -1,31 +0,0 @@ - - diff --git a/packages/frontend-2/components/onboarding/dialog/FirstSend.vue b/packages/frontend-2/components/onboarding/dialog/FirstSend.vue deleted file mode 100644 index 84072a275..000000000 --- a/packages/frontend-2/components/onboarding/dialog/FirstSend.vue +++ /dev/null @@ -1,33 +0,0 @@ - - diff --git a/packages/frontend-2/components/onboarding/dialog/Manager.vue b/packages/frontend-2/components/onboarding/dialog/Manager.vue deleted file mode 100644 index 79c661a98..000000000 --- a/packages/frontend-2/components/onboarding/dialog/Manager.vue +++ /dev/null @@ -1,131 +0,0 @@ - - diff --git a/packages/frontend-2/components/onboarding/questions/Form.vue b/packages/frontend-2/components/onboarding/questions/Form.vue index a2320ad59..f1080b1f8 100644 --- a/packages/frontend-2/components/onboarding/questions/Form.vue +++ b/packages/frontend-2/components/onboarding/questions/Form.vue @@ -26,15 +26,14 @@ diff --git a/packages/frontend-2/components/onboarding/questions/PlanSelect.vue b/packages/frontend-2/components/onboarding/questions/PlanSelect.vue index da21642ba..1de7f0b99 100644 --- a/packages/frontend-2/components/onboarding/questions/PlanSelect.vue +++ b/packages/frontend-2/components/onboarding/questions/PlanSelect.vue @@ -30,7 +30,8 @@ diff --git a/packages/frontend-2/components/project/model-page/dialog/embed/Embed.vue b/packages/frontend-2/components/project/model-page/dialog/embed/Embed.vue index cfb518c34..89809cc00 100644 --- a/packages/frontend-2/components/project/model-page/dialog/embed/Embed.vue +++ b/packages/frontend-2/components/project/model-page/dialog/embed/Embed.vue @@ -88,10 +88,8 @@ diff --git a/packages/frontend-2/components/project/page/automation/FunctionSettingsDialog.vue b/packages/frontend-2/components/project/page/automation/FunctionSettingsDialog.vue index 63e07a3dc..60c6e8c0a 100644 --- a/packages/frontend-2/components/project/page/automation/FunctionSettingsDialog.vue +++ b/packages/frontend-2/components/project/page/automation/FunctionSettingsDialog.vue @@ -247,7 +247,7 @@ const onSave = async () => { parameters } ], - triggerDefinitions: { + triggerDefinitions: { version: Automate.AutomateTypes.TRIGGER_DEFINITIONS_SCHEMA_VERSION, definitions: [ { @@ -255,7 +255,7 @@ const onSave = async () => { modelId: model.id } ] - } + } as Automate.AutomateTypes.TriggerDefinitionsSchema } }) if (res?.id) { @@ -280,7 +280,7 @@ const onSave = async () => { // Reset everything if props change watch( - () => [props.revisionFn?.release.function.id, props.revisionFn?.release.id], + () => [props.revisionFn?.release.function.id, props.revisionFn?.release.id] as const, ([newFunctionId, newFunctionRevisionId], [oldFunctionId, oldFunctionRevisionId]) => { if ( newFunctionId === oldFunctionId && diff --git a/packages/frontend-2/components/project/page/automation/Runs.vue b/packages/frontend-2/components/project/page/automation/Runs.vue index 8f4e11023..d5ca96816 100644 --- a/packages/frontend-2/components/project/page/automation/Runs.vue +++ b/packages/frontend-2/components/project/page/automation/Runs.vue @@ -19,6 +19,7 @@
diff --git a/packages/frontend-2/components/project/page/settings/general/block/Discussions.vue b/packages/frontend-2/components/project/page/settings/general/block/Discussions.vue index f800d95af..05f8ebb39 100644 --- a/packages/frontend-2/components/project/page/settings/general/block/Discussions.vue +++ b/packages/frontend-2/components/project/page/settings/general/block/Discussions.vue @@ -21,7 +21,7 @@ diff --git a/packages/frontend-2/components/projects/MoveToWorkspaceDialog.vue b/packages/frontend-2/components/projects/MoveToWorkspaceDialog.vue index f0cd5ccb2..523b79a93 100644 --- a/packages/frontend-2/components/projects/MoveToWorkspaceDialog.vue +++ b/packages/frontend-2/components/projects/MoveToWorkspaceDialog.vue @@ -64,7 +64,7 @@ import type { ProjectsMoveToWorkspaceDialog_ProjectFragment } from '~~/lib/common/generated/gql/graphql' import { useMutationLoading, useQuery } from '@vue/apollo-composable' -import { type LayoutDialogButton } from '@speckle/ui-components' +import type { LayoutDialogButton } from '@speckle/ui-components' import { useMoveProjectToWorkspace } from '~/lib/projects/composables/projectManagement' import { Roles } from '@speckle/shared' import { workspacesRoute } from '~/lib/common/helpers/route' diff --git a/packages/frontend-2/components/settings/Sidebar.vue b/packages/frontend-2/components/settings/Sidebar.vue index 4852b0497..8ff37f1c3 100644 --- a/packages/frontend-2/components/settings/Sidebar.vue +++ b/packages/frontend-2/components/settings/Sidebar.vue @@ -19,10 +19,7 @@
-
+
@@ -68,62 +65,103 @@ /> - - - + + + + @@ -132,11 +170,10 @@ diff --git a/packages/frontend-2/components/settings/server/ActiveUsers.vue b/packages/frontend-2/components/settings/server/ActiveUsers.vue index 98db1d5a6..a9a4a328f 100644 --- a/packages/frontend-2/components/settings/server/ActiveUsers.vue +++ b/packages/frontend-2/components/settings/server/ActiveUsers.vue @@ -139,7 +139,8 @@ const { query: getUsersQuery, baseVariables: computed(() => ({ query: search.value?.length ? search.value : null, - limit: 50 + limit: 50, + cursor: null as Nullable })), resolveKey: (vars) => [vars.query || ''], resolveCurrentResult: (res) => res?.admin.userList, diff --git a/packages/frontend-2/components/settings/server/PendingInvitations.vue b/packages/frontend-2/components/settings/server/PendingInvitations.vue index 2c173083c..5697626f3 100644 --- a/packages/frontend-2/components/settings/server/PendingInvitations.vue +++ b/packages/frontend-2/components/settings/server/PendingInvitations.vue @@ -95,6 +95,7 @@ import { usePaginatedQuery } from '~/lib/common/composables/graphql' import { getInvitesQuery } from '~~/lib/server-management/graphql/queries' import { HorizontalDirection } from '~~/lib/common/composables/window' import type { LayoutMenuItem } from '~~/lib/layout/helpers/components' +import type { Nullable } from '@speckle/shared' const { triggerNotification } = useGlobalToast() const { mutate: resendInvitationMutation } = useMutation(adminResendInviteMutation) @@ -114,7 +115,8 @@ const { query: getInvitesQuery, baseVariables: computed(() => ({ query: search.value?.length ? search.value : null, - limit: 50 + limit: 50, + cursor: null as Nullable })), resolveKey: (vars) => [vars.query || ''], resolveCurrentResult: (res) => res?.admin.inviteList, diff --git a/packages/frontend-2/components/settings/user/developer/AccessTokens/CreateDialog.vue b/packages/frontend-2/components/settings/user/developer/AccessTokens/CreateDialog.vue index 5c01c3d1a..f94913bf5 100644 --- a/packages/frontend-2/components/settings/user/developer/AccessTokens/CreateDialog.vue +++ b/packages/frontend-2/components/settings/user/developer/AccessTokens/CreateDialog.vue @@ -96,8 +96,8 @@ const onSubmit = handleSubmit(async (tokenFormValues) => { emit('token-created', result.data.apiTokenCreate) triggerNotification({ type: ToastNotificationType.Success, - title: 'Webhook created', - description: 'The webhook has been successfully created' + title: 'Token created', + description: 'The token has been successfully created' }) } else { const errorMessage = getFirstErrorMessage(result?.errors) diff --git a/packages/frontend-2/components/settings/user/email/DeleteDialog.vue b/packages/frontend-2/components/settings/user/email/DeleteDialog.vue index 6e9f7ef44..28d903269 100644 --- a/packages/frontend-2/components/settings/user/email/DeleteDialog.vue +++ b/packages/frontend-2/components/settings/user/email/DeleteDialog.vue @@ -1,14 +1,14 @@ diff --git a/packages/frontend-2/components/settings/workspaces/billing/AddOns.vue b/packages/frontend-2/components/settings/workspaces/billing/AddOns.vue new file mode 100644 index 000000000..d6be6ed73 --- /dev/null +++ b/packages/frontend-2/components/settings/workspaces/billing/AddOns.vue @@ -0,0 +1,23 @@ + + diff --git a/packages/frontend-2/components/settings/workspaces/billing/Page.vue b/packages/frontend-2/components/settings/workspaces/billing/Page.vue new file mode 100644 index 000000000..8643e9bbb --- /dev/null +++ b/packages/frontend-2/components/settings/workspaces/billing/Page.vue @@ -0,0 +1,477 @@ + + + + diff --git a/packages/frontend-2/components/settings/workspaces/billing/PageNew.vue b/packages/frontend-2/components/settings/workspaces/billing/PageNew.vue new file mode 100644 index 000000000..52e9720e3 --- /dev/null +++ b/packages/frontend-2/components/settings/workspaces/billing/PageNew.vue @@ -0,0 +1,93 @@ + + + + diff --git a/packages/frontend-2/components/settings/workspaces/billing/PricingTable/Plan.vue b/packages/frontend-2/components/settings/workspaces/billing/PricingTable/Plan.vue index 3e264fa6b..dd7c9f56f 100644 --- a/packages/frontend-2/components/settings/workspaces/billing/PricingTable/Plan.vue +++ b/packages/frontend-2/components/settings/workspaces/billing/PricingTable/Plan.vue @@ -5,7 +5,7 @@

Workspace - {{ plan.name }} + {{ plan }}

{{ badgeText }} @@ -13,10 +13,15 @@

- £{{ - yearlyIntervalSelected - ? plan.cost.yearly[Roles.Workspace.Member] - : plan.cost.monthly[Roles.Workspace.Member] + {{ + formatPrice( + props.yearlyIntervalSelected && planPrice?.['workspace:member'] + ? { + ...planPrice['workspace:member'], + amount: planPrice['workspace:member'].amount * 0.8 + } + : planPrice?.['workspace:member'] + ) }} per seat/month @@ -40,7 +45,7 @@

  • - {{ feature.name }} + {{ featureMetadata.displayName }}
@@ -89,7 +94,7 @@ diff --git a/packages/frontend-2/components/settings/workspaces/billing/UpgradeDialog.vue b/packages/frontend-2/components/settings/workspaces/billing/UpgradeDialog.vue index 4d4539ca4..58647bf8c 100644 --- a/packages/frontend-2/components/settings/workspaces/billing/UpgradeDialog.vue +++ b/packages/frontend-2/components/settings/workspaces/billing/UpgradeDialog.vue @@ -10,7 +10,7 @@

Workspace {{ startCase(plan) }} plan

- £{{ seatPrice }}/seat/month, billed + {{ seatPrice }}/seat/month, billed {{ billingInterval === BillingInterval.Yearly ? 'annually' : 'monthly' }}

@@ -22,33 +22,36 @@ diff --git a/packages/frontend-2/components/settings/workspaces/billing/usage/index.vue b/packages/frontend-2/components/settings/workspaces/billing/usage/index.vue new file mode 100644 index 000000000..97556172d --- /dev/null +++ b/packages/frontend-2/components/settings/workspaces/billing/usage/index.vue @@ -0,0 +1,25 @@ + + + + diff --git a/packages/frontend-2/components/settings/workspaces/members/GuestsTable.vue b/packages/frontend-2/components/settings/workspaces/members/GuestsTable.vue index ae2b45a78..d42db5c23 100644 --- a/packages/frontend-2/components/settings/workspaces/members/GuestsTable.vue +++ b/packages/frontend-2/components/settings/workspaces/members/GuestsTable.vue @@ -11,9 +11,8 @@ :columns="[ { id: 'name', header: 'Name', classes: 'col-span-3' }, { id: 'company', header: 'Company', classes: 'col-span-3' }, - { id: 'verified', header: 'Status', classes: 'col-span-3' }, { id: 'projects', header: 'Projects', classes: 'col-span-2' }, - { id: 'actions', header: '', classes: 'col-span-1 flex justify-end' } + { id: 'actions', header: '', classes: 'col-span-4 flex justify-end' } ]" :items="guests" :loading="searchResultLoading" @@ -36,11 +35,6 @@ {{ item.user.company ? item.user.company : '-' }} - + @@ -45,10 +55,13 @@ diff --git a/packages/frontend-2/components/tutorials/Page.vue b/packages/frontend-2/components/tutorials/Page.vue new file mode 100644 index 000000000..7dc90e9bd --- /dev/null +++ b/packages/frontend-2/components/tutorials/Page.vue @@ -0,0 +1,51 @@ + + + diff --git a/packages/frontend-2/components/viewer/AnchoredPoints.vue b/packages/frontend-2/components/viewer/AnchoredPoints.vue index 776a03caf..c1acb6348 100644 --- a/packages/frontend-2/components/viewer/AnchoredPoints.vue +++ b/packages/frontend-2/components/viewer/AnchoredPoints.vue @@ -20,6 +20,13 @@ :key="thread.id" :model-value="thread" :class="openThread?.id === thread.id ? 'z-[12]' : 'z-[11]'" + :has-previous=" + allThreadsChronologicalOrder.findIndex((t) => t.id === thread.id) > 0 + " + :has-next=" + allThreadsChronologicalOrder.findIndex((t) => t.id === thread.id) < + allThreadsChronologicalOrder.length - 1 + " @update:model-value="onThreadUpdate" @update:expanded="onThreadExpandedChange" @next="(model) => openNextThread(model)" diff --git a/packages/frontend-2/components/viewer/Base.vue b/packages/frontend-2/components/viewer/Base.vue index ce73ce0dd..f9a2b7ccf 100644 --- a/packages/frontend-2/components/viewer/Base.vue +++ b/packages/frontend-2/components/viewer/Base.vue @@ -1,8 +1,13 @@ diff --git a/packages/frontend-2/components/viewer/Controls.vue b/packages/frontend-2/components/viewer/Controls.vue index 289c86935..a173617c2 100644 --- a/packages/frontend-2/components/viewer/Controls.vue +++ b/packages/frontend-2/components/viewer/Controls.vue @@ -3,7 +3,7 @@
+
@@ -133,7 +136,7 @@ @@ -155,16 +158,20 @@
+