diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 2ac67b7c6..bc4b26f9d 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -6,7 +6,10 @@ on: IMAGE_VERSION_TAG: required: true type: string - DOCKERHUB_USERNAME: + REGISTRY_DOMAIN: + required: true + type: string + REGISTRY_USERNAME: required: true type: string IMAGE_PREFIX: @@ -23,7 +26,7 @@ on: secrets: DATADOG_API_KEY: required: true - DOCKERHUB_TOKEN: + REGISTRY_TOKEN: required: true jobs: @@ -31,17 +34,12 @@ jobs: runs-on: blacksmith-4vcpu-ubuntu-2404 name: Server steps: - - name: Login to Docker Hub + - name: Login to Helm Chart & Container Image Registry uses: docker/login-action@v3 with: - username: ${{ inputs.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + registry: ${{ inputs.REGISTRY_DOMAIN }} + username: ${{ inputs.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_TOKEN }} - name: Set image tags id: set-tags run: | @@ -66,17 +64,12 @@ jobs: runs-on: blacksmith-8vcpu-ubuntu-2404 name: Frontend2 steps: - - name: Login to Docker Hub + - name: Login to Helm Chart & Container Image Registry uses: docker/login-action@v3 with: - username: ${{ inputs.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + registry: ${{ inputs.REGISTRY_DOMAIN }} + username: ${{ inputs.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_TOKEN }} - name: Setup Docker Builder uses: useblacksmith/setup-docker-builder@v1 - name: Build and push w/ version tag @@ -100,17 +93,12 @@ jobs: runs-on: blacksmith-4vcpu-ubuntu-2404 name: Preview service steps: - - name: Login to Docker Hub + - name: Login to Helm Chart & Container Image Registry uses: docker/login-action@v3 with: - username: ${{ inputs.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + registry: ${{ inputs.REGISTRY_DOMAIN }} + username: ${{ inputs.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_TOKEN }} - name: Setup Docker Builder uses: useblacksmith/setup-docker-builder@v1 - name: Build and push @@ -124,17 +112,12 @@ jobs: runs-on: blacksmith-4vcpu-ubuntu-2404 name: Webhook service steps: - - name: Login to Docker Hub + - name: Login to Helm Chart & Container Image Registry uses: docker/login-action@v3 with: - username: ${{ inputs.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + registry: ${{ inputs.REGISTRY_DOMAIN }} + username: ${{ inputs.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_TOKEN }} - name: Setup Docker Builder uses: useblacksmith/setup-docker-builder@v1 - name: Build and push @@ -148,17 +131,12 @@ jobs: runs-on: blacksmith-4vcpu-ubuntu-2404 name: File import service steps: - - name: Login to Docker Hub + - name: Login to Helm Chart & Container Image Registry uses: docker/login-action@v3 with: - username: ${{ inputs.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + registry: ${{ inputs.REGISTRY_DOMAIN }} + username: ${{ inputs.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_TOKEN }} - name: Setup Docker Builder uses: useblacksmith/setup-docker-builder@v1 - name: Build and push @@ -172,17 +150,12 @@ jobs: runs-on: blacksmith-4vcpu-ubuntu-2404 name: IFC import service steps: - - name: Login to Docker Hub + - name: Login to Helm Chart & Container Image Registry uses: docker/login-action@v3 with: - username: ${{ inputs.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + registry: ${{ inputs.REGISTRY_DOMAIN }} + username: ${{ inputs.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_TOKEN }} - name: Setup Docker Builder uses: useblacksmith/setup-docker-builder@v1 - name: Build and push @@ -196,17 +169,12 @@ jobs: runs-on: blacksmith-4vcpu-ubuntu-2404 name: Test deploy util steps: - - name: Login to Docker Hub + - name: Login to Helm Chart & Container Image Registry uses: docker/login-action@v3 with: - username: ${{ inputs.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + registry: ${{ inputs.REGISTRY_DOMAIN }} + username: ${{ inputs.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_TOKEN }} - name: Setup Docker Builder uses: useblacksmith/setup-docker-builder@v1 - name: Build and push @@ -220,17 +188,12 @@ jobs: runs-on: blacksmith-4vcpu-ubuntu-2404 name: monitor container steps: - - name: Login to Docker Hub + - name: Login to Helm Chart & Container Image Registry uses: docker/login-action@v3 with: - username: ${{ inputs.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + registry: ${{ inputs.REGISTRY_DOMAIN }} + username: ${{ inputs.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_TOKEN }} - name: Setup Docker Builder uses: useblacksmith/setup-docker-builder@v1 - name: Build and push @@ -244,17 +207,12 @@ jobs: runs-on: blacksmith-4vcpu-ubuntu-2404 name: Docker compose ingress steps: - - name: Login to Docker Hub + - name: Login to Helm Chart & Container Image Registry uses: docker/login-action@v3 with: - username: ${{ inputs.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + registry: ${{ inputs.REGISTRY_DOMAIN }} + username: ${{ inputs.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_TOKEN }} - name: Setup Docker Builder uses: useblacksmith/setup-docker-builder@v1 - name: Build and push diff --git a/.github/workflows/deployment-tests.yml b/.github/workflows/deployment-tests.yml index 0584d17e4..759702b4b 100644 --- a/.github/workflows/deployment-tests.yml +++ b/.github/workflows/deployment-tests.yml @@ -6,14 +6,17 @@ on: IMAGE_VERSION_TAG: required: true type: string - DOCKERHUB_USERNAME: + REGISTRY_DOMAIN: + required: true + type: string + REGISTRY_USERNAME: required: true type: string IMAGE_PREFIX: required: true type: string secrets: - DOCKERHUB_TOKEN: + REGISTRY_TOKEN: required: true jobs: deployment-test-helm-chart: @@ -63,17 +66,12 @@ jobs: nix-shell \ --run "ctlptl apply --filename ./tests/deployment/helm/cluster-config.yaml" \ ./tests/deployment/helm/helm-chart-shell.nix - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ inputs.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + registry: ${{ inputs.REGISTRY_DOMAIN }} + username: ${{ inputs.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_TOKEN }} - name: Deploy Kubernetes resources to cluster run: | nix-shell \ diff --git a/.github/workflows/manual-trigger-test-deployment.yml b/.github/workflows/manual-trigger-test-deployment.yml index 8d1d92312..f75e37140 100644 --- a/.github/workflows/manual-trigger-test-deployment.yml +++ b/.github/workflows/manual-trigger-test-deployment.yml @@ -19,16 +19,22 @@ jobs: uses: ./.github/workflows/builds.yml with: IMAGE_VERSION_TAG: ${{ needs.get-version.outputs.IMAGE_VERSION_TAG }} - DOCKERHUB_USERNAME: 'speckledevops' + REGISTRY_DOMAIN: 'ghcr.io' + REGISTRY_USERNAME: ${{ github.actor }} + # REGISTRY_DOMAIN, REGISTRY_USERNAME, REGISTRY_TOKEN must allow pushing to the below IMAGE_PREFIX IMAGE_PREFIX: 'ghcr.io/specklesystems' PUBLISH: false # do not publish the sourcemaps or include the version in frontend-2 builds - secrets: inherit + secrets: + DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }} + REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} deployment-tests: needs: [builds, get-version] uses: ./.github/workflows/deployment-tests.yml with: IMAGE_VERSION_TAG: ${{ needs.get-version.outputs.IMAGE_VERSION_TAG }} - DOCKERHUB_USERNAME: 'speckledevops' + REGISTRY_DOMAIN: ghcr.io + REGISTRY_USERNAME: ${{ github.actor }} IMAGE_PREFIX: 'ghcr.io/specklesystems' - secrets: inherit + secrets: + REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e5e0a8555..79bd1e5de 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,6 +1,9 @@ on: workflow_call: inputs: + IMAGE_PREFIX: + required: true + type: string IMAGE_VERSION_TAG: required: true type: string @@ -10,7 +13,6 @@ on: OCI_REGISTRY_DOMAIN: required: false type: string - default: registry-1.docker.io OCI_REGISTRY_USERNAME: required: true type: string @@ -37,6 +39,7 @@ jobs: container: image: speckle/pre-commit-runner:latest env: + IMAGE_PREFIX: ${{ inputs.IMAGE_PREFIX }} IMAGE_VERSION_TAG: ${{ inputs.IMAGE_VERSION_TAG }} REGISTRY_USERNAME: ${{ inputs.OCI_REGISTRY_USERNAME }} REGISTRY_PASSWORD: ${{ secrets.OCI_REGISTRY_PASSWORD }} diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index b58d54f1c..aad1d0ad0 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -14,9 +14,7 @@ jobs: tests: needs: [get-version] uses: ./.github/workflows/tests.yml - with: - IMAGE_VERSION_TAG: ${{ needs.get-version.outputs.IMAGE_VERSION_TAG }} - DOCKERHUB_USERNAME: 'speckledevops' + with: {} secrets: inherit builds: @@ -24,7 +22,11 @@ jobs: uses: ./.github/workflows/builds.yml with: IMAGE_VERSION_TAG: ${{ needs.get-version.outputs.IMAGE_VERSION_TAG }} - DOCKERHUB_USERNAME: 'speckledevops' + REGISTRY_DOMAIN: 'ghcr.io' + REGISTRY_USERNAME: ${{ github.actor }} + # REGISTRY_DOMAIN, REGISTRY_USERNAME, REGISTRY_TOKEN must allow pushing to the below IMAGE_PREFIX IMAGE_PREFIX: 'ghcr.io/specklesystems' PUBLISH: false # do not publish the sourcemaps or include the version in frontend-2 builds for pull requests - secrets: inherit + secrets: + DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }} + REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c260ddc71..ba74fe295 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,8 +27,6 @@ jobs: needs: [get-version] uses: ./.github/workflows/tests.yml with: - IMAGE_VERSION_TAG: ${{ needs.get-version.outputs.IMAGE_VERSION_TAG }} - DOCKERHUB_USERNAME: 'speckledevops' CONTINUE_ON_ERROR: ${{ startsWith(github.ref, 'refs/heads/testing') }} # allows releasing on testing even if test fail DISABLE_SKIPPING: ${{ startsWith(github.ref, 'refs/heads/main') }} # disable test skip on main secrets: inherit @@ -38,18 +36,38 @@ jobs: uses: ./.github/workflows/builds.yml with: IMAGE_VERSION_TAG: ${{ needs.get-version.outputs.IMAGE_VERSION_TAG }} - DOCKERHUB_USERNAME: 'speckledevops' + REGISTRY_DOMAIN: 'registry-1.docker.io' + REGISTRY_USERNAME: 'speckledevops' + # REGISTRY_DOMAIN, REGISTRY_USERNAME, REGISTRY_TOKEN must allow pushing to the below IMAGE_PREFIX IMAGE_PREFIX: 'speckle' # without an explicit host, Docker defaults to pushing Docker Hub PUBLISH: true # publish the sourcemaps and include the version in frontend-2 builds PUBLISH_LATEST: ${{ startsWith(github.ref, 'refs/heads/main') }} - secrets: inherit + secrets: + DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }} + REGISTRY_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + + # Temporary duplicate of builds job to push to ghcr.io + builds-ghcr: + needs: [get-version] + uses: ./.github/workflows/builds.yml + with: + IMAGE_VERSION_TAG: ${{ needs.get-version.outputs.IMAGE_VERSION_TAG }} + REGISTRY_DOMAIN: 'ghcr.io' + REGISTRY_USERNAME: ${{ github.actor }} + # REGISTRY_DOMAIN, REGISTRY_USERNAME, REGISTRY_TOKEN must allow pushing to the below IMAGE_PREFIX + IMAGE_PREFIX: 'ghcr.io/specklesystems' + PUBLISH: true # do not publish the sourcemaps or include the version in frontend-2 builds for pull requests + PUBLISH_LATEST: ${{ startsWith(github.ref, 'refs/heads/main') }} + secrets: + DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }} + REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} test-deployments: - needs: [get-version, builds] + needs: [get-version, builds, builds-ghcr] uses: ./.github/workflows/deployment-tests.yml with: IMAGE_VERSION_TAG: ${{ needs.get-version.outputs.IMAGE_VERSION_TAG }} - DOCKERHUB_USERNAME: 'speckledevops' + REGISTRY_USERNAME: 'speckledevops' IMAGE_PREFIX: 'speckle' secrets: inherit @@ -57,11 +75,12 @@ jobs: needs: [get-version, tests, builds, test-deployments, get-chart-name] uses: ./.github/workflows/publish.yml with: + IMAGE_PREFIX: 'ghcr.io/specklesystems' IMAGE_VERSION_TAG: ${{ needs.get-version.outputs.IMAGE_VERSION_TAG }} CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }} OCI_REGISTRY_DOMAIN: ghcr.io OCI_REGISTRY_PATH: specklesystems - OCI_REGISTRY_USERNAME: ${{ github.actor }} # we are pushing helm chart to ghcr + OCI_REGISTRY_USERNAME: ${{ github.actor }} CHART_NAME: ${{ needs.get-chart-name.outputs.CHART_NAME }} secrets: # we do not inherit here as we wish to configure secrets depending on the target registry @@ -71,7 +90,7 @@ jobs: GH_DEVOPS_PAT: ${{ secrets.GH_DEVOPS_PAT }} npm: - needs: [get-version, tests, builds] + needs: [get-version, tests, builds, builds-ghcr] uses: ./.github/workflows/npm.yml if: startsWith(github.ref, 'refs/tags/') # a tag triggered the workflow with: diff --git a/.github/workflows/scripts/publish_helm_chart_oci.sh b/.github/workflows/scripts/publish_helm_chart_oci.sh index c5084acfe..5373e9b51 100755 --- a/.github/workflows/scripts/publish_helm_chart_oci.sh +++ b/.github/workflows/scripts/publish_helm_chart_oci.sh @@ -2,6 +2,10 @@ set -eo pipefail +if [[ -z "${IMAGE_PREFIX}" ]]; then + echo "IMAGE_PREFIX is not set" + exit 1 +fi if [[ -z "${IMAGE_VERSION_TAG}" ]]; then echo "IMAGE_VERSION_TAG is not set" exit 1 @@ -36,6 +40,18 @@ source "${SCRIPT_DIR}/common.sh" echo "📌 Releasing Helm Chart for application version ${IMAGE_VERSION_TAG} to 'oci://${HELM_REGISTRY_DOMAIN}/${HELM_REPOSITORY_PATH}/${CHART_NAME}:${RELEASE_VERSION}'" +if [[ "${IMAGE_PREFIX}" != "speckle" ]]; then + yq e -i ".server.image = \"${IMAGE_PREFIX}/speckle-server:${IMAGE_VERSION_TAG}\"" "${GIT_REPO}/utils/helm/speckle-server/values.yaml" + yq e -i ".objects.image = \"${IMAGE_PREFIX}/speckle-server:${IMAGE_VERSION_TAG}\"" "${GIT_REPO}/utils/helm/speckle-server/values.yaml" + yq e -i ".frontend_2.image = \"${IMAGE_PREFIX}/speckle-frontend-2:${IMAGE_VERSION_TAG}\"" "${GIT_REPO}/utils/helm/speckle-server/values.yaml" + yq e -i ".preview_service.image = \"${IMAGE_PREFIX}/speckle-preview-service:${IMAGE_VERSION_TAG}\"" "${GIT_REPO}/utils/helm/speckle-server/values.yaml" + yq e -i ".webhook_service.image = \"${IMAGE_PREFIX}/speckle-webhook-service:${IMAGE_VERSION_TAG}\"" "${GIT_REPO}/utils/helm/speckle-server/values.yaml" + yq e -i ".fileimport_service.image = \"${IMAGE_PREFIX}/speckle-fileimport-service:${IMAGE_VERSION_TAG}\"" "${GIT_REPO}/utils/helm/speckle-server/values.yaml" + yq e -i ".ifc_import_service.image = \"${IMAGE_PREFIX}/speckle-ifc-import-service:${IMAGE_VERSION_TAG}\"" "${GIT_REPO}/utils/helm/speckle-server/values.yaml" + yq e -i ".monitoring.image = \"${IMAGE_PREFIX}/speckle-monitor-deployment:${IMAGE_VERSION_TAG}\"" "${GIT_REPO}/utils/helm/speckle-server/values.yaml" + yq e -i ".test.image = \"${IMAGE_PREFIX}/speckle-test-deployment:${IMAGE_VERSION_TAG}\"" "${GIT_REPO}/utils/helm/speckle-server/values.yaml" +fi + yq e -i ".docker_image_tag = \"${IMAGE_VERSION_TAG}\"" "${GIT_REPO}/utils/helm/speckle-server/values.yaml" yq e -i ".name = \"${CHART_NAME}\"" "${GIT_REPO}/utils/helm/speckle-server/Chart.yaml" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b328c36e4..44dafda76 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -3,12 +3,6 @@ name: Run all tests on: workflow_call: inputs: - IMAGE_VERSION_TAG: - required: true - type: string - DOCKERHUB_USERNAME: - required: true - type: string CONTINUE_ON_ERROR: required: false default: false @@ -30,8 +24,6 @@ on: required: true CODECOV_TOKEN: required: true - DOCKERHUB_TOKEN: - required: true jobs: changes: