ci(helm chart deployment test): add to GitHub Actions (#4932)

.circleci/config.yml

@@ -841,7 +841,7 @@ jobs:
       - restore_cache:
           name: Restore nix cache
           keys:
-            - nix-{{ checksum "./.circleci/deployment/docker-compose-shell.nix" }}
+            - nix-{{ checksum "./tests/deployment/docker-compose/docker-compose-shell.nix" }}
       - run:
           name: Install the nix package manager
           command: |
@@ -850,13 +850,13 @@ jobs:
       - run:
           name: Initialize nix shell
           command: |
-            nix-shell --run "echo Here, a nix shell for you" ./.circleci/deployment/docker-compose-shell.nix
+            nix-shell --run "echo Here, a nix shell for you" ./tests/deployment/docker-compose/docker-compose-shell.nix
       - save_cache:
-          key: nix-{{ checksum "./.circleci/deployment/docker-compose-shell.nix" }}
+          key: nix-{{ checksum "./tests/deployment/docker-compose/docker-compose-shell.nix" }}
           paths:
             - /nix
       - run: cat workspace/env-vars >> $BASH_ENV
-      - run: nix-shell --run "LOAD_DOCKER='true' tilt ci --file ./.circleci/deployment/Tiltfile.dockercompose" ./.circleci/deployment/docker-compose-shell.nix
+      - run: nix-shell --run "LOAD_DOCKER='true' tilt ci --file ./tests/deployment/docker-compose/Tiltfile" ./tests/deployment/helm/docker-compose-shell.nix
 
   deployment-test-helm-chart:
     machine:
@@ -875,7 +875,7 @@ jobs:
       - restore_cache:
           name: Restore nix cache
           keys:
-            - nix-{{ checksum "./.circleci/deployment/helm-chart-shell.nix" }}
+            - nix-{{ checksum "./tests/deployment/helm/helm-chart-shell.nix" }}
       - run:
           name: Install the nix package manager
           command: |
@@ -886,9 +886,9 @@ jobs:
           command: |
             nix-shell \
             --run "echo Here, a nix shell for you" \
-            ./.circleci/deployment/helm-chart-shell.nix
+            ./tests/deployment/helm/helm-chart-shell.nix
       - save_cache:
-          key: nix-{{ checksum "./.circleci/deployment/helm-chart-shell.nix" }}
+          key: nix-{{ checksum "./tests/deployment/helm/helm-chart-shell.nix" }}
           paths:
             - /nix
       - run: cat workspace/env-vars >> $BASH_ENV
@@ -899,7 +899,7 @@ jobs:
           command: |
             nix-shell \
             --run "helm template speckle-server ./utils/helm/speckle-server" \
-            ./.circleci/deployment/helm-chart-shell.nix
+            ./tests/deployment/helm/helm-chart-shell.nix
       - run:
           name: Add 127.0.0.1 domains to /etc/hosts
           command: |
@@ -922,14 +922,14 @@ jobs:
           name: Deploy Kubernetes (kind) cluster
           command: |
             nix-shell \
-            --run "ctlptl apply --filename ./.circleci/deployment/cluster-config.yaml" \
+            --run "ctlptl apply --filename ./tests/deployment/helm/cluster-config.yaml" \
-            ./.circleci/deployment/helm-chart-shell.nix
+            ./tests/deployment/helmhelm-chart-shell.nix
       - run:
           name: Deploy Kubernetes resources to cluster
           command: |
             nix-shell \
-            --run "LOAD_DOCKER='true' tilt ci --file ./.circleci/deployment/Tiltfile.helm --context kind-speckle-server --timeout 10m" \
+            --run "LOAD_DOCKER='true' tilt ci --file ./tests/deployment/helm/Tiltfile --context kind-speckle-server --timeout 10m" \
-            ./.circleci/deployment/helm-chart-shell.nix
+            ./tests/deployment/helm/helm-chart-shell.nix
 
   docker-build: &build-job
     <<: *docker-base-image

.gitguardian.yml

@@ -6,9 +6,9 @@ secret:
     - match: acd87c5a50b56df91a795e999812a3a4
       name: 'packages/frontend/src/bootstrapper.ts - mixpanel token'
     - match: c7bf45ffe02afaae52c8e37cdb1ae33165370be3b44a5da43e8cba43c7da5f33
-      name: '.circleci/deployment/manifests/speckle-server.secret.yaml - test s3_secret_key'
+      name: '.tests/deployment/helm/manifests/speckle-server.secret.yaml - test s3_secret_key'
     - match: 9f1d96876edbf847bb792754025ed131374869e60866d5e9c349c9423b37dd09
-      name: '.circleci/deployment/manifests/speckle-server.secret.yaml - test session_secret'
+      name: 'tests/deployment/helm/manifests/speckle-server.secret.yaml - test session_secret'
     - match: 9bf360c5ce31170e8e3cb30e275b2c00224dd97b93282491c60fb1665fac3845
       name: local test license
     - match: 7a4ab6f7bfbcc0a37aa3a0fb00fd5b6edd1d524f393a6054e242eb28f5c06be5

.github/workflows/builds.yml

@@ -9,9 +9,13 @@ on:
       DOCKERHUB_USERNAME:
         required: true
         type: string
-      PUSH_IMAGES:
+      IMAGE_PREFIX:
         required: true
+        type: string
+      PUBLISH:
+        required: false
         type: boolean
+        default: false
     secrets:
       DATADOG_API_KEY:
         required: true
@@ -28,11 +32,19 @@ jobs:
         with:
           username: ${{ inputs.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       - name: Build and push
         uses: useblacksmith/build-push-action@v1
         with:
-          push: ${{ inputs.PUSH_IMAGES }}
+          push: true
-          tags: speckle/speckle-server:${{ inputs.IMAGE_VERSION_TAG }}
+          tags: |
+            ${{ inputs.IMAGE_PREFIX }}/speckle-server:${{ inputs.IMAGE_VERSION_TAG }}
+          # TODO add docker hub tag, but only if we want to push to docker hub
           file: ./packages/server/Dockerfile
           build-args: |
             SPECKLE_SERVER_VERSION=${{ inputs.IMAGE_VERSION_TAG }}
@@ -46,23 +58,29 @@ jobs:
         with:
           username: ${{ inputs.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       - name: Build and push w/ version tag
         uses: useblacksmith/build-push-action@v1
-        if: ${{ inputs.PUSH_IMAGES }}
+        if: ${{ inputs.PUBLISH }}
         with:
           push: true
-          tags: speckle/speckle-frontend-2:${{ inputs.IMAGE_VERSION_TAG }}
+          tags: ${{ inputs.IMAGE_PREFIX }}/speckle-frontend-2:${{ inputs.IMAGE_VERSION_TAG }}
           file: ./packages/frontend-2/Dockerfile
           build-args: |
             SPECKLE_SERVER_VERSION=${{ inputs.IMAGE_VERSION_TAG }}
-          cache-from: type=registry,ref=speckle/speckle-frontend-2:buildcache
+          cache-from: type=registry,ref=${{ inputs.IMAGE_PREFIX }}/speckle-frontend-2:buildcache
-          cache-to: type=registry,ref=speckle/speckle-frontend-2:buildcache,mode=max
+          cache-to: type=registry,ref=${{ inputs.IMAGE_PREFIX }}/speckle-frontend-2:buildcache,mode=max
       - name: Check frontend-2 build
         uses: useblacksmith/build-push-action@v1
-        if: ${{ inputs.PUSH_IMAGES == false }}
+        if: ${{ !inputs.PUBLISH }}
         with:
-          push: false
+          push: true
-          tags: speckle/speckle-frontend-2:${{ inputs.IMAGE_VERSION_TAG }}
+          tags: ${{ inputs.IMAGE_PREFIX }}/speckle-frontend-2:${{ inputs.IMAGE_VERSION_TAG }}
           file: ./packages/frontend-2/Dockerfile
 
   docker-build-preview-service:
@@ -74,11 +92,17 @@ jobs:
         with:
           username: ${{ inputs.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       - name: Build and push
         uses: useblacksmith/build-push-action@v1
         with:
-          push: ${{ inputs.PUSH_IMAGES }}
+          push: true
-          tags: speckle/speckle-preview-service:${{ inputs.IMAGE_VERSION_TAG }}
+          tags: ${{ inputs.IMAGE_PREFIX }}/speckle-preview-service:${{ inputs.IMAGE_VERSION_TAG }}
           file: ./packages/preview-service/Dockerfile
 
   docker-build-webhook-service:
@@ -90,11 +114,17 @@ jobs:
         with:
           username: ${{ inputs.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       - name: Build and push
         uses: useblacksmith/build-push-action@v1
         with:
-          push: ${{ inputs.PUSH_IMAGES }}
+          push: true
-          tags: speckle/speckle-webhook-service:${{ inputs.IMAGE_VERSION_TAG }}
+          tags: ${{ inputs.IMAGE_PREFIX }}/speckle-webhook-service:${{ inputs.IMAGE_VERSION_TAG }}
           file: ./packages/webhook-service/Dockerfile
 
   docker-build-fileimport-service:
@@ -106,11 +136,17 @@ jobs:
         with:
           username: ${{ inputs.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       - name: Build and push
         uses: useblacksmith/build-push-action@v1
         with:
-          push: ${{ inputs.PUSH_IMAGES }}
+          push: true
-          tags: speckle/speckle-fileimport-service:${{ inputs.IMAGE_VERSION_TAG }}
+          tags: ${{ inputs.IMAGE_PREFIX }}/speckle-fileimport-service:${{ inputs.IMAGE_VERSION_TAG }}
           file: ./packages/fileimport-service/Dockerfile
 
   docker-build-test-deploy:
@@ -122,11 +158,17 @@ jobs:
         with:
           username: ${{ inputs.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       - name: Build and push
         uses: useblacksmith/build-push-action@v1
         with:
-          push: ${{ inputs.PUSH_IMAGES }}
+          push: true
-          tags: speckle/speckle-test-deployment:${{ inputs.IMAGE_VERSION_TAG }}
+          tags: ${{ inputs.IMAGE_PREFIX }}/speckle-test-deployment:${{ inputs.IMAGE_VERSION_TAG }}
           file: ./utils/test-deployment/Dockerfile
 
   docker-build-monitor-deployment:
@@ -138,11 +180,17 @@ jobs:
         with:
           username: ${{ inputs.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       - name: Build and push
         uses: useblacksmith/build-push-action@v1
         with:
-          push: ${{ inputs.PUSH_IMAGES }}
+          push: true
-          tags: speckle/speckle-monitor-deployment:${{ inputs.IMAGE_VERSION_TAG }}
+          tags: ${{ inputs.IMAGE_PREFIX }}/speckle-monitor-deployment:${{ inputs.IMAGE_VERSION_TAG }}
           file: ./packages/monitor-deployment/Dockerfile
 
   docker-build-docker-compose-ingress:
@@ -154,17 +202,23 @@ jobs:
         with:
           username: ${{ inputs.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       - name: Build and push
         uses: useblacksmith/build-push-action@v1
         with:
-          push: ${{ inputs.PUSH_IMAGES }}
+          push: true
-          tags: speckle/speckle-docker-compose-ingress:${{ inputs.IMAGE_VERSION_TAG }}
+          tags: ${{ inputs.IMAGE_PREFIX }}/speckle-docker-compose-ingress:${{ inputs.IMAGE_VERSION_TAG }}
           file: ./utils/docker-compose-ingress/Dockerfile
 
   docker-frontend-2-sourcemaps:
     runs-on: blacksmith
     name: Frontend2 sourcemaps
-    if: ${{ inputs.PUSH_IMAGES }}
+    if: ${{ inputs.PUBLISH }}
     env:
       IMAGE_VERSION_TAG: ${{ inputs.IMAGE_VERSION_TAG }}
       DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }}
@@ -176,7 +230,7 @@ jobs:
       - uses: useblacksmith/build-push-action@v1
         with:
           setup-only: true
-          cache-from: type=registry,ref=speckle/speckle-frontend-2:buildcache-sourcemaps
+          cache-from: type=registry,ref=${{ inputs.IMAGE_PREFIX }}/speckle-frontend-2:buildcache-sourcemaps
-          cache-to: type=registry,ref=speckle/speckle-frontend-2:buildcache-sourcemaps
+          cache-to: type=registry,ref=${{ inputs.IMAGE_PREFIX }}/speckle-frontend-2:buildcache-sourcemaps
       - name: Build and Publish sourcemaps
         run: ./.github/workflows/scripts/publish_fe2_sourcemaps.sh

.github/workflows/deployment-tests.yml

@@ -0,0 +1,85 @@
+name: Run deployment test
+
+on:
+  workflow_call:
+    inputs:
+      IMAGE_VERSION_TAG:
+        required: true
+        type: string
+      DOCKERHUB_USERNAME:
+        required: true
+        type: string
+      IMAGE_PREFIX:
+        required: true
+        type: string
+    secrets:
+      DOCKERHUB_TOKEN:
+        required: true
+jobs:
+  deployment-test-helm-chart:
+    runs-on: blacksmith-8vcpu-ubuntu-2404
+    steps:
+      - uses: actions/checkout@v4.2.2
+        with:
+          fetch-depth: 0
+      # create the nix folder with permissive write permissions
+      - run: |
+          sudo mkdir /nix
+          sudo chmod 777 /nix
+      - name: Install the nix package manager
+        uses: cachix/install-nix-action@v31
+        with:
+          nix_path: nixpkgs=https://github.com/NixOS/nixpkgs/archive/63dacb46bf939521bdc93981b4cbb7ecb58427a0.tar.gz
+      - name: Initialize nix shell
+        run: |
+          nix-shell \
+          --run "echo Here, a nix shell for you" \
+          ./tests/deployment/helm/helm-chart-shell.nix
+      - run: export KUBECONFIG=$(pwd)/.kube/config
+      - run: echo "${KUBECONFIG}"
+      - name: Template Speckle Server Helm Chart
+        run: |
+          nix-shell \
+          --run "helm template speckle-server ./utils/helm/speckle-server" \
+          ./tests/deployment/helm/helm-chart-shell.nix
+      - name: Add 127.0.0.1 domains to /etc/hosts
+        run: |
+          sudo tee -a /etc/hosts \<<<'127.0.0.1 speckle.internal'
+          cat /etc/hosts
+      - name: Change directory permissions to allow kind to create directories
+        run: |
+          mkdir -p "./minio-data"
+          if [ "$(stat -f "%A" "./minio-data")" != "775" ]; then
+            echo "🔐 We need 'sudo' to set permissions on minio-data directory to 775"
+            sudo chmod 775 "./minio-data"
+          fi
+          mkdir -p "./postgres-data"
+          if [ "$(stat -f "%A" "./postgres-data")" != "775" ]; then
+            echo "🔐 We need 'sudo' to set permissions on postgres-data directory to 775"
+            sudo chmod 775 "./postgres-data"
+          fi
+      - name: Deploy Kubernetes (kind) cluster
+        run: |
+          nix-shell \
+          --run "ctlptl apply --filename ./tests/deployment/helm/cluster-config.yaml" \
+          ./tests/deployment/helm/helm-chart-shell.nix
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ inputs.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Deploy Kubernetes resources to cluster
+        run: |
+          nix-shell \
+          --run "IMAGE_PREFIX=${{ inputs.IMAGE_PREFIX }} LOAD_DOCKER='true' tilt ci --file ./tests/deployment/helm/Tiltfile --context kind-speckle-server --timeout 10m" \
+          ./tests/deployment/helm/helm-chart-shell.nix
+        env:
+          IMAGE_PREFIX: ${{ inputs.IMAGE_PREFIX }}
+          IMAGE_VERSION_TAG: ${{ inputs.IMAGE_VERSION_TAG }}
+          LOAD_DOCKER: 'true'

.github/workflows/get-version.yml

@@ -0,0 +1,33 @@
+name: Get-version
+
+on:
+  workflow_call:
+    outputs:
+      IMAGE_VERSION_TAG:
+        description: 'The image version tag to use for the deployment test'
+        value: ${{ jobs.get-version.outputs.IMAGE_VERSION_TAG }}
+
+jobs:
+  get-version:
+    outputs:
+      IMAGE_VERSION_TAG: ${{ steps.export-step.outputs.IMAGE_VERSION_TAG }}
+    name: Get version
+    runs-on: blacksmith
+    steps:
+      - uses: actions/checkout@v4.2.2
+        with:
+          sparse-checkout: |
+            .github/workflows/scripts/get_version.sh
+            .github/workflows/scripts/common.sh
+          fetch-depth: 1
+          fetch-tags: 1
+      - run: git fetch origin 'refs/tags/*:refs/tags/*'
+      - run: chmod +x ./get_version.sh ./common.sh
+        working-directory: ./.github/workflows/scripts
+      - run: ./get_version.sh >> result
+        working-directory: ./.github/workflows/scripts
+      - run: echo "IMAGE_VERSION_TAG=$(cat result)"
+        working-directory: ./.github/workflows/scripts
+      - id: export-step
+        run: echo "IMAGE_VERSION_TAG=$(cat result)" >> "$GITHUB_OUTPUT"
+        working-directory: ./.github/workflows/scripts

.github/workflows/manual-trigger-test-deployment.yml

@@ -0,0 +1,34 @@
+name: Test helm deployment
+
+on: workflow_dispatch
+
+jobs:
+  verify-non-prod:
+    runs-on: blacksmith
+    if: github.ref != 'refs/heads/main'
+    steps:
+      - run: echo "This is a non-production deployment."
+
+  get-version:
+    needs: [verify-non-prod]
+    name: Get version
+    uses: ./.github/workflows/get-version.yml
+
+  builds:
+    needs: [verify-non-prod, get-version]
+    uses: ./.github/workflows/builds.yml
+    with:
+      IMAGE_VERSION_TAG: ${{ needs.get-version.outputs.IMAGE_VERSION_TAG }}
+      DOCKERHUB_USERNAME: 'speckledevops'
+      IMAGE_PREFIX: 'ghcr.io/specklesystems'
+      PUBLISH: false # do not publish the sourcemaps or include the version in frontend-2 builds
+    secrets: inherit
+
+  deployment-tests:
+    needs: [builds, get-version]
+    uses: ./.github/workflows/deployment-tests.yml
+    with:
+      IMAGE_VERSION_TAG: ${{ needs.get-version.outputs.IMAGE_VERSION_TAG }}
+      DOCKERHUB_USERNAME: 'speckledevops'
+      IMAGE_PREFIX: 'ghcr.io/specklesystems'
+    secrets: inherit

.github/workflows/pull-request.yml

@@ -8,31 +8,11 @@ concurrency:
 
 jobs:
   get-version:
-    outputs:
-      IMAGE_VERSION_TAG: ${{ steps.export-step.outputs.IMAGE_VERSION_TAG }}
     name: Get version
-    runs-on: blacksmith
+    uses: ./.github/workflows/get-version.yml
-    steps:
-      - uses: actions/checkout@v4.2.2
-        with:
-          sparse-checkout: |
-            .github/workflows/scripts/get_version.sh
-            .github/workflows/scripts/common.sh
-          fetch-depth: 1
-          fetch-tags: 1
-      - run: git fetch origin 'refs/tags/*:refs/tags/*'
-      - run: chmod +x ./get_version.sh ./common.sh
-        working-directory: ./.github/workflows/scripts
-      - run: ./get_version.sh >> result
-        working-directory: ./.github/workflows/scripts
-      - run: echo "IMAGE_VERSION_TAG=$(cat result)"
-        working-directory: ./.github/workflows/scripts
-      - id: export-step
-        run: echo "IMAGE_VERSION_TAG=$(cat result)" >> "$GITHUB_OUTPUT"
-        working-directory: ./.github/workflows/scripts
 
   tests:
-    needs: get-version
+    needs: [get-version]
     uses: ./.github/workflows/tests.yml
     with:
       IMAGE_VERSION_TAG: ${{ needs.get-version.outputs.IMAGE_VERSION_TAG }}
@@ -40,10 +20,11 @@ jobs:
     secrets: inherit
 
   builds:
-    needs: get-version
+    needs: [get-version]
     uses: ./.github/workflows/builds.yml
     with:
       IMAGE_VERSION_TAG: ${{ needs.get-version.outputs.IMAGE_VERSION_TAG }}
       DOCKERHUB_USERNAME: 'speckledevops'
-      PUSH_IMAGES: false
+      IMAGE_PREFIX: 'ghcr.io/specklesystems'
+      PUBLISH: false # do not publish the sourcemaps or include the version in frontend-2 builds for pull requests
     secrets: inherit

.github/workflows/release.yml

@@ -16,31 +16,11 @@ concurrency:
 
 jobs:
   get-version:
-    outputs:
-      IMAGE_VERSION_TAG: ${{ steps.export-step.outputs.IMAGE_VERSION_TAG }}
     name: Get version
-    runs-on: blacksmith
+    uses: ./.github/workflows/get-version.yml
-    steps:
-      - uses: actions/checkout@v4.2.2
-        with:
-          sparse-checkout: |
-            .github/workflows/scripts/get_version.sh
-            .github/workflows/scripts/common.sh
-          fetch-depth: 1
-          fetch-tags: 1
-      - run: git fetch origin 'refs/tags/*:refs/tags/*'
-      - run: chmod +x ./get_version.sh ./common.sh
-        working-directory: ./.github/workflows/scripts
-      - run: ./get_version.sh >> result
-        working-directory: ./.github/workflows/scripts
-      - run: echo "IMAGE_VERSION_TAG=$(cat result)"
-        working-directory: ./.github/workflows/scripts
-      - id: export-step
-        run: echo "IMAGE_VERSION_TAG=$(cat result)" >> "$GITHUB_OUTPUT"
-        working-directory: ./.github/workflows/scripts
 
   tests:
-    needs: get-version
+    needs: [get-version]
     uses: ./.github/workflows/tests.yml
     with:
       IMAGE_VERSION_TAG: ${{ needs.get-version.outputs.IMAGE_VERSION_TAG }}
@@ -48,16 +28,26 @@ jobs:
     secrets: inherit
 
   builds:
-    needs: get-version
+    needs: [get-version]
     uses: ./.github/workflows/builds.yml
     with:
       IMAGE_VERSION_TAG: ${{ needs.get-version.outputs.IMAGE_VERSION_TAG }}
       DOCKERHUB_USERNAME: 'speckledevops'
-      PUSH_IMAGES: true
+      IMAGE_PREFIX: 'speckle' # without an explicit host, Docker defaults to pushing Docker Hub
+      PUBLISH: true # publish the sourcemaps and include the version in frontend-2 builds
+    secrets: inherit
+
+  test-deployments:
+    needs: [get-version, builds]
+    uses: ./.github/workflows/deployment-tests.yml
+    with:
+      IMAGE_VERSION_TAG: ${{ needs.get-version.outputs.IMAGE_VERSION_TAG }}
+      DOCKERHUB_USERNAME: 'speckledevops'
+      IMAGE_PREFIX: 'speckle'
     secrets: inherit
 
   deploy:
-    needs: [get-version, tests, builds]
+    needs: [get-version, tests, builds, test-deployments]
     uses: ./.github/workflows/publish.yml
     with:
       IMAGE_VERSION_TAG: ${{ needs.get-version.outputs.IMAGE_VERSION_TAG }}

package.json

@@ -23,11 +23,11 @@
     "dev:docker:down": "docker compose -f ./docker-compose-deps.yml down",
     "dev:docker:down:volumes": "docker compose -f ./docker-compose-deps.yml down --volumes",
     "dev:docker:restart": "yarn dev:docker:down && yarn dev:docker:up",
-    "dev:kind:up": "ctlptl apply --filename ./.circleci/deployment/cluster-config.yaml",
+    "dev:kind:up": "ctlptl apply --filename ./.tests/deployment/helm/cluster-config.yaml",
-    "dev:kind:down": "ctlptl delete -f ./.circleci/deployment/cluster-config.yaml",
+    "dev:kind:down": "ctlptl delete -f ./tests/deployment/helm/cluster-config.yaml",
-    "dev:kind:helm:up": "yarn dev:kind:up && tilt up --file ./.circleci/deployment/Tiltfile.helm --context kind-speckle-server",
+    "dev:kind:helm:up": "yarn dev:kind:up && tilt up --file ./tests/deployment/helm/Tiltfile --context kind-speckle-server",
-    "dev:kind:helm:down": "tilt down --file ./.circleci/deployment/Tiltfile.helm --context kind-speckle-server",
+    "dev:kind:helm:down": "tilt down --file ./tests/deployment/helm/Tiltfile --context kind-speckle-server",
-    "dev:kind:helm:ci": "tilt ci --file ./.circleci/deployment/Tiltfile.helm --context kind-speckle-server --timeout 10m",
+    "dev:kind:helm:ci": "tilt ci --file ./tests/deployment/helm/Tiltfile --context kind-speckle-server --timeout 10m",
     "dev": "yarn workspaces foreach --parallel --interlaced --verbose --worktree --jobs unlimited run dev",
     "dev:no-server": "yarn workspaces foreach --exclude @speckle/server --parallel --interlaced --verbose --worktree --jobs unlimited run dev",
     "dev:minimal": "yarn workspaces foreach --parallel --interlaced --verbose --worktree --jobs unlimited --include '{@speckle/server,@speckle/frontend-2}' run dev",

tests/deployment/docker-compose/Tiltfile

@@ -3,9 +3,9 @@ print('🚀 Deploying Speckle Server into Docker via Tilt...')
 symbols = None
 IS_LOAD_DOCKER = os.getenv('LOAD_DOCKER', "false") == "true"
 if IS_LOAD_DOCKER:
-  symbols = load_dynamic('./Tiltfile.load')
+  symbols = load_dynamic('./../load-images.tiltfile')
 else:
-  symbols = load_dynamic('./Tiltfile.build')
+  symbols = load_dynamic('./../build-images.tiltfile')
 
 # Prepare the Speckle images
 # (loads the images from tgz file if CI, otherwise builds them or pulls them from remote as a fallback on non-amd64 machines)
@@ -28,7 +28,7 @@ dc_resource('maildev', labels=['dependencies'])
 dc_resource('postgres', labels=['dependencies'])
 
 # Prepare the Speckle resources
-docker_compose(['../../docker-compose-speckle.yml', 'docker-compose-speckle.override.yml'], wait=True)
+docker_compose(['../../../docker-compose-speckle.yml', './docker-compose-speckle.override.yml'], wait=True)
 dc_resource('speckle-server', resource_deps=['postgres', 'redis', 'minio', 'maildev'], labels=['speckle'])
 dc_resource('speckle-frontend-2', resource_deps=[], labels=['speckle'])
 dc_resource('speckle-ingress', resource_deps=[], labels=['speckle'])
@@ -37,5 +37,5 @@ dc_resource('fileimport-service', resource_deps=[], labels=['speckle'])
 dc_resource('webhook-service', resource_deps=['postgres'], labels=['speckle'])
 
 # Test the Speckle resources
-docker_compose(['../../docker-compose-test.yml', 'docker-compose-test.override.yml'], wait=True)
+docker_compose(['../../../docker-compose-test.yml', './docker-compose-test.override.yml'], wait=True)
 dc_resource('test', resource_deps=['speckle-server', 'speckle-frontend-2'], labels=['test']) # --exit-code-from test

tests/deployment/helm/Tiltfile

@@ -12,21 +12,23 @@ load('ext://k8s_yaml_glob', 'k8s_yaml_glob')
 symbols = None
 IS_LOAD_DOCKER = os.getenv('LOAD_DOCKER', "false") == "true"
 if IS_LOAD_DOCKER:
-  symbols = load_dynamic('./Tiltfile.load')
+  symbols = load_dynamic('./../load-images.tiltfile')
 else:
-  symbols = load_dynamic('./Tiltfile.build')
+  symbols = load_dynamic('./../build-images.tiltfile')
+
+IMAGE_PREFIX = os.getenv('IMAGE_PREFIX', 'speckle')
 
 # # Prepare the Speckle images
 # # (builds the images or pulls them from remote)
 speckle_image = symbols['speckle_image']
-speckle_image('fileimport-service')
+speckle_image('fileimport-service', None, IMAGE_PREFIX)
-speckle_image('frontend-2')
+speckle_image('frontend-2', None, IMAGE_PREFIX)
-speckle_image('monitor-deployment')
+speckle_image('monitor-deployment', None, IMAGE_PREFIX)
-speckle_image('preview-service')
+speckle_image('preview-service', None, IMAGE_PREFIX)
-speckle_image('server')
+speckle_image('server', None, IMAGE_PREFIX)
-speckle_image('objects','server')
+speckle_image('objects','server', IMAGE_PREFIX)
-speckle_image('test-deployment')
+speckle_image('test-deployment', None, IMAGE_PREFIX)
-speckle_image('webhook-service')
+speckle_image('webhook-service', None, IMAGE_PREFIX)
 
 # Create namespaces
 k8s_yaml_glob('./manifests/*.namespace.yaml')
@@ -38,7 +40,7 @@ k8s_yaml('./manifests/minio.pv.yaml')
 k8s_yaml('./manifests/minio.pvc.yaml')
 
 # Update CoreDNS to allow for local resolution of services internally (i.e. speckle.internal will be routed to nginx)
-local(command='./coredns-up.sh')
+local(command='./scripts/coredns-up.sh')
 
 # Install speckle pod-priority and secrets
 k8s_yaml('./manifests/priorityclass.yaml')
@@ -118,7 +120,7 @@ helm_resource('ingress-nginx',
 helm_resource('speckle-server',
               release_name='speckle-server',
               namespace='speckle-server',
-              chart='./../../utils/helm/speckle-server',
+              chart='./../../../utils/helm/speckle-server',
               flags=['--values=./values/speckle-server.values.yaml',
                       '--kube-context=kind-speckle-server'],
               image_deps=[
@@ -141,7 +143,7 @@ helm_resource('speckle-server',
                           'test.image',
                           'webhook_service.image',
                           ],
-              deps=['../../utils/helm', './values/speckle-server.values.yaml'],
+              deps=['../../../utils/helm', './values/speckle-server.values.yaml'],
               resource_deps=['postgresql', 'minio', 'valkey', 'ingress-nginx'],
               labels=['speckle-server'])
 

tests/deployment/helm/values/nginx.values.yaml

@@ -1,5 +1,5 @@
 controller:
-  # We must set the kind cluster listen address for every port to '127.0.0.1' when hostNetwork is true, see ./deploy/local/cluster-config.yaml
+  # We must set the kind cluster listen address for every port to '127.0.0.1' when hostNetwork is true
   hostNetwork: true
   admissionWebhooks:
     enabled: false

tests/deployment/load-images.tiltfile

@@ -16,7 +16,7 @@ def docker_load(name, existing_ref=None, deps=None):
                         docker tag "{EXISTING_REF}" "$EXPECTED_REF"'.format(EXISTING_REF=existing_ref),
                         deps=deps)
 
-def speckle_image(package,original_package_name=None):
+def speckle_image(package,original_package_name=None,image_prefix='localhost:5000'):
     if not package:
         fail('package must be specified')
     if not original_package_name:
@@ -31,7 +31,7 @@ def speckle_image(package,original_package_name=None):
     workspace='/tmp/ci/workspace'
 
     docker_image_tag = 'speckle/speckle-{}'.format(package)
-    original_docker_image_tag = 'speckle/speckle-{}'.format(original_package_name)
+    original_docker_image_tag = '{}/speckle-{}'.format(image_prefix,original_package_name)
     existing_ref = '{}:{}'.format(original_docker_image_tag, image_version_tag)
     return docker_load(docker_image_tag,
                         existing_ref=existing_ref,