diff --git a/.circleci/config.yml b/.circleci/config.yml index b9f6ce97c..e645b610f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -29,10 +29,12 @@ aliases: - &filters-only-main-hotfix-testing branches: - only: &branches-special - - main - - /^hotfix.*$/ - - /^testing\d*$/ + ignore: /.*/ + + - &branches-special + - main + - /^hotfix.*$/ + - /^testing\d*$/ workflows: test-build: diff --git a/.github/workflows/npm.yml b/.github/workflows/npm.yml new file mode 100644 index 000000000..ba59dfb83 --- /dev/null +++ b/.github/workflows/npm.yml @@ -0,0 +1,37 @@ +on: + workflow_call: + inputs: + IMAGE_VERSION_TAG: + required: true + type: string + secrets: + NPM_TOKEN: + required: true + +jobs: + publish: + name: Publish to npm + runs-on: blacksmith + env: + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + IMAGE_VERSION_TAG: ${{ inputs.IMAGE_VERSION_TAG }} + steps: + - uses: actions/checkout@v4.2.2 + - uses: useblacksmith/setup-node@v5 + with: + node-version: 22 + cache: yarn + - name: Install hardened (no HARD flag) + run: PUPPETEER_SKIP_DOWNLOAD=true PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 yarn --immutable + - name: Auth to npm as Speckle + run: | + echo "npmRegistryServer: https://registry.npmjs.org/" >> .yarnrc.yml + echo "npmAuthToken: $NPM_TOKEN" >> .yarnrc.yml + - name: Try login to npm + run: yarn npm whoami + - name: Build public packages + run: yarn workspaces foreach -ptvW --no-private run build + - name: Bump all versions + run: yarn workspaces foreach -tvW version $IMAGE_VERSION_TAG + - name: publish to npm + run: 'yarn workspaces foreach -pvW --no-private npm publish --access public' diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 000000000..86f2ff040 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,94 @@ +on: + workflow_call: + inputs: + IMAGE_VERSION_TAG: + required: true + type: string + CLOUDFLARE_ACCOUNT_ID: + required: true + type: string + DOCKERHUB_USERNAME: + required: true + type: string + secrets: + DATADOG_API_KEY: + required: true + CLOUDFLARE_API_TOKEN: + required: true + DOCKERHUB_TOKEN: + required: true + GH_DEVOPS_PAT: + required: true +jobs: + helm-chart-oci: + runs-on: blacksmith + name: Helm chart oci + container: + image: speckle/pre-commit-runner:latest + env: + IMAGE_VERSION_TAG: ${{ inputs.IMAGE_VERSION_TAG }} + DOCKER_REG_USER: ${{ inputs.DOCKERHUB_USERNAME }} + DOCKER_REG_PASS: ${{ secrets.DOCKERHUB_TOKEN }} + steps: + - uses: actions/checkout@v4.2.2 + with: + fetch-depth: 0 + - run: git config --global --add safe.directory $PWD + - name: Publish Helm Chart + run: ./.github/workflows/scripts/publish_helm_chart_oci.sh + + helm-chart-commit: + runs-on: blacksmith + name: Helm chart commit + container: + image: bitnami/python:3.12.0 + env: + IMAGE_VERSION_TAG: ${{ inputs.IMAGE_VERSION_TAG }} + steps: + - run: apt-get update -y + - run: apt-get install -y wget + - run: wget -qO /usr/local/bin/yq github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 + - run: chmod a+x /usr/local/bin/yq + - uses: actions/checkout@v4.2.2 + with: + fetch-depth: 0 + path: speckle + - uses: actions/checkout@v4.2.2 + with: + repository: specklesystems/helm + path: helm + token: ${{ secrets.GH_DEVOPS_PAT }} + - run: chmod +x ./.github/workflows/scripts/publish_helm_chart_commit.sh + working-directory: speckle + - name: Commit Helm Chart + run: ./.github/workflows/scripts/publish_helm_chart_commit.sh + working-directory: speckle + + viewer-sandbox-cloudflare-pages: + runs-on: blacksmith + name: Viewer sandbox cloudflare pages + env: + CLOUDFLARE_ACCOUNT_ID: ${{ inputs.CLOUDFLARE_ACCOUNT_ID }} + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_PAGES_PROJECT_NAME: viewer + VIEWER_SANDBOX_DIR_PATH: packages/viewer-sandbox + steps: + - uses: actions/checkout@v4.2.2 + with: + fetch-depth: 0 + - uses: useblacksmith/setup-node@v5 + with: + node-version: 22 + cache: yarn + - name: Install dependencies + run: YARN_ENABLE_HARDENED_MODE=0 PUPPETEER_SKIP_DOWNLOAD=true PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 yarn --immutable + - name: Build public packages + run: yarn build:public + - name: Lint viewer-sandbox + run: yarn lint:ci + working-directory: 'packages/viewer-sandbox' + - name: Build viewer-sandbox + run: yarn build + working-directory: 'packages/viewer-sandbox' + - name: Publish Viewer Sandbox to Cloudflare Pages + run: ./.github/workflows/scripts/publish_cloudflare_pages.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/pull-request.yml similarity index 98% rename from .github/workflows/ci.yml rename to .github/workflows/pull-request.yml index 342628897..428ad1fe7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/pull-request.yml @@ -1,4 +1,4 @@ -name: CI Pipeline +name: PR Pipeline on: pull_request diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..4df554e32 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,68 @@ +name: Release pipeline + +on: + push: + branches: + - main + - 'hotfix.*' + - 'testing*' + tags: + - '[0-9]+.[0-9]+.[0-9]+' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ contains(github.ref, 'testing*')}} # deployments on testing* will cancel each other, prod and tags no + +jobs: + get-version: + outputs: + IMAGE_VERSION_TAG: ${{ steps.export-step.outputs.IMAGE_VERSION_TAG }} + name: Get version + runs-on: blacksmith + steps: + - uses: actions/checkout@v4.2.2 + - run: git fetch origin 'refs/tags/*:refs/tags/*' + - run: chmod +x ./get_version.sh ./common.sh + working-directory: ./.github/workflows/scripts + - run: ./get_version.sh >> result + working-directory: ./.github/workflows/scripts + - run: echo "IMAGE_VERSION_TAG=$(cat result)" + working-directory: ./.github/workflows/scripts + - id: export-step + run: echo "IMAGE_VERSION_TAG=$(cat result)" >> "$GITHUB_OUTPUT" + working-directory: ./.github/workflows/scripts + + tests: + needs: get-version + uses: ./.github/workflows/tests.yml + with: + IMAGE_VERSION_TAG: ${{ needs.get-version.outputs.IMAGE_VERSION_TAG }} + DOCKERHUB_USERNAME: 'speckledevops' + secrets: inherit + + builds: + needs: get-version + uses: ./.github/workflows/builds.yml + with: + IMAGE_VERSION_TAG: ${{ needs.get-version.outputs.IMAGE_VERSION_TAG }} + DOCKERHUB_USERNAME: 'speckledevops' + PUSH_IMAGES: true + secrets: inherit + + deploy: + needs: [get-version, tests, builds] + uses: ./.github/workflows/publish.yml + with: + IMAGE_VERSION_TAG: ${{ needs.get-version.outputs.IMAGE_VERSION_TAG }} + CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }} + DOCKERHUB_USERNAME: 'speckledevops' + secrets: inherit + + npm: + needs: [get-version, tests, builds] + uses: ./.github/workflows/npm.yml + if: startsWith(github.ref, 'refs/tags/') # a tag triggered the workflow + with: + IMAGE_VERSION_TAG: ${{ needs.get-version.outputs.IMAGE_VERSION_TAG }} + secrets: inherit diff --git a/.github/workflows/scripts/check_version.py b/.github/workflows/scripts/check_version.py new file mode 100755 index 000000000..517f5d727 --- /dev/null +++ b/.github/workflows/scripts/check_version.py @@ -0,0 +1,88 @@ +#!/usr/bin/python3 +import sys +from typing import Optional +from dataclasses import dataclass + + +@dataclass +class Version: + major: int + minor: int + patch: int + pre_release_tag: Optional[str] = None + build_number: Optional[int] = None + + @property + def pre_release_priority(self) -> int: + if self.pre_release_tag == "alpha": + return 1 + if self.pre_release_tag == "beta": + return 2 + return 10 + + @staticmethod + def parse_version_slug(version_slug: str) -> "Version": + members = version_slug.split(".") + assert 3 <= len(members) <= 4 + if len(members) == 3: + major, minor, patch = members + return Version(int(major), int(minor), int(patch)) + + else: + major, minor, patch_and_pre, build = members + patch, pre = patch_and_pre.split("-") + return Version(int(major), int(minor), int(patch), pre, int(build)) + + def __gt__(self, other): + if not isinstance(other, Version): + raise ValueError(f"cannot compare with {other}") + + if self.major > other.major: + return True + if self.major < other.major: + return False + + if self.minor > other.minor: + return True + if self.minor < other.minor: + return False + + if self.patch > other.patch: + return True + if self.patch < other.patch: + return False + + if self.pre_release_tag == other.pre_release_tag: + if self.build_number > other.build_number: + return True + if self.build_number < other.build_number: + return False + + if self.pre_release_priority > other.pre_release_priority: + return True + if self.pre_release_priority < other.pre_release_priority: + return False + + return True + + +if __name__ == "__main__": + print("\nStarting version compare\n") + args = sys.argv[1:] + assert len(args) == 2 + + current_version_slug, target_version_slug = args + + print( + f"comparing current version {current_version_slug} with target {target_version_slug}" + ) + + current_version = Version.parse_version_slug(current_version_slug) + target_version = Version.parse_version_slug(target_version_slug) + + if target_version > current_version: + print("target version is newer\n") + exit(0) + + print("current version is newer\n") + exit(1) diff --git a/.github/workflows/scripts/common.sh b/.github/workflows/scripts/common.sh index ec0e55a69..6711d4940 100644 --- a/.github/workflows/scripts/common.sh +++ b/.github/workflows/scripts/common.sh @@ -4,9 +4,6 @@ set -eo pipefail # shellcheck disable=SC2034 DOCKER_IMAGE_TAG="speckle/speckle-${SPECKLE_SERVER_PACKAGE}" -# shellcheck disable=SC2034,SC2086 -IMAGE_VERSION_TAG="${IMAGE_VERSION_TAG:-${GITHUB_SHA}}" - # shellcheck disable=SC2068,SC2046 LAST_RELEASE="$(git describe --always --tags $(git rev-list --tags) | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' | head -n 1)" # get the last release tag. FIXME: Fails if a commit is tagged with more than one tag: https://stackoverflow.com/questions/8089002/git-describe-with-two-tags-on-the-same-commit/56039163#56039163 @@ -14,7 +11,7 @@ LAST_RELEASE="$(git describe --always --tags $(git rev-list --tags) | grep -E '^ NEXT_RELEASE="$(echo "${LAST_RELEASE}" | awk -F. -v OFS=. '{$NF += 1 ; print}')" # shellcheck disable=SC2034 -BRANCH_NAME_TRUNCATED="$(echo "${GITHUB_HEAD_REF}" | cut -c -50 | sed 's/[^a-zA-Z0-9.-]/-/g')" # docker has a 128 character tag limit, so ensuring the branch name will be short enough +BRANCH_NAME_TRUNCATED="$(echo "${GITHUB_REF_NAME}" | cut -c -28 | sed 's/[^a-zA-Z0-9.-]/-/g')" # docker has a 128 character tag limit, so ensuring the branch name will be short enough # shellcheck disable=SC2034 COMMIT_SHA1_TRUNCATED="$(echo "${GITHUB_SHA}" | cut -c -7)" diff --git a/.github/workflows/scripts/get_version.sh b/.github/workflows/scripts/get_version.sh index 61b654bba..495137cb2 100755 --- a/.github/workflows/scripts/get_version.sh +++ b/.github/workflows/scripts/get_version.sh @@ -5,14 +5,13 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) # shellcheck disable=SC1090,SC1091 source "${SCRIPT_DIR}/common.sh" - if [[ "${GITHUB_REF}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then echo "${GITHUB_REF}" exit 0 fi if [[ "${GITHUB_HEAD_REF}" == "main" ]]; then - echo "${NEXT_RELEASE}-alpha.${GITHUB_RUN_ID}" + echo "${NEXT_RELEASE}-alpha.${GITHUB_RUN_NUMBER}" exit 0 fi @@ -22,5 +21,5 @@ if [[ "${BRANCH_NAME_TRUNCATED}" =~ "_" ]]; then exit 1 fi -echo "${NEXT_RELEASE}-branch.${BRANCH_NAME_TRUNCATED}.${GITHUB_RUN_ID}-${COMMIT_SHA1_TRUNCATED}" +echo "${NEXT_RELEASE}-branch.${BRANCH_NAME_TRUNCATED}.${GITHUB_RUN_NUMBER}-${COMMIT_SHA1_TRUNCATED}" exit 0 diff --git a/.github/workflows/scripts/publish_cloudflare_pages.sh b/.github/workflows/scripts/publish_cloudflare_pages.sh new file mode 100755 index 000000000..e508ca051 --- /dev/null +++ b/.github/workflows/scripts/publish_cloudflare_pages.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -eo pipefail + +echo "đŸˇī¸ Preparing envs" + +GIT_ROOT="$(git rev-parse --show-toplevel)" + +CLOUDFLARE_PAGES_PROJECT_NAME="${CLOUDFLARE_PAGES_PROJECT_NAME:-"viewer"}" +VIEWER_SANDBOX_DIR_PATH="${VIEWER_SANDBOX_DIR_PATH:-"packages/viewer-sandbox"}" + +pushd "${GIT_ROOT}/${VIEWER_SANDBOX_DIR_PATH}" +yarn wrangler pages deploy "${GIT_ROOT}/${VIEWER_SANDBOX_DIR_PATH}/dist" --project-name="${CLOUDFLARE_PAGES_PROJECT_NAME}" +popd + +echo "✅ Publishing completed." diff --git a/.github/workflows/scripts/publish_fe2_sourcemaps.sh b/.github/workflows/scripts/publish_fe2_sourcemaps.sh old mode 100644 new mode 100755 diff --git a/.github/workflows/scripts/publish_helm_chart_commit.sh b/.github/workflows/scripts/publish_helm_chart_commit.sh new file mode 100644 index 000000000..54920bf60 --- /dev/null +++ b/.github/workflows/scripts/publish_helm_chart_commit.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +set -eo pipefail + +echo "đŸˇī¸ Setting envs" + +GIT_ROOT="$(git rev-parse --show-toplevel)" +GIT_HELM="$(dirname "$GIT_ROOT")/helm" +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +# shellcheck disable=SC1090,SC1091 +source "${SCRIPT_DIR}/common.sh" + +RELEASE_VERSION="${IMAGE_VERSION_TAG}" +HELM_STABLE_BRANCH="${HELM_STABLE_BRANCH:-"main"}" + + +if [[ -z "${RELEASE_VERSION}" ]]; then + echo "IMAGE_VERSION_TAG is not set: ${IMAGE_VERSION_TAG} ${RELEASE_VERSION}" + exit 1 +fi +if [ ! -d "${GIT_HELM}/.git" ]; then + echo "helm repo not found at ${GIT_HELM} " + exit 1 +fi +if [ ! -d "${GIT_ROOT}/.git" ]; then + echo "speckle repo not found at ${GIT_ROOT}" + exit 1 +fi + +echo "âœī¸ Editing Helm Chart version ${RELEASE_VERSION}" + +yq e -i ".version = \"${RELEASE_VERSION}\"" "${GIT_ROOT}/utils/helm/speckle-server/Chart.yaml" +yq e -i ".appVersion = \"${RELEASE_VERSION}\"" "${GIT_ROOT}/utils/helm/speckle-server/Chart.yaml" +yq e -i ".docker_image_tag = \"${RELEASE_VERSION}\"" "${GIT_ROOT}/utils/helm/speckle-server/values.yaml" + +if [[ "${GITHUB_REF}" == refs/tags/* || "${GITHUB_REF_NAME}" == "${HELM_STABLE_BRANCH}" ]]; then + echo "âš ī¸ prod release ${RELEASE_VERSION}" + # before overwriting the chart with the build version, check if the current chart version + # is not newer than the currently build one + + CURRENT_VERSION="$(grep ^version "${GIT_HELM}/charts/speckle-server/Chart.yaml" | grep -o '2\..*')" + echo "â„šī¸ Current version ${CURRENT_VERSION}" + + .github/workflows/scripts/check_version.py "${CURRENT_VERSION}" "${RELEASE_VERSION}" + if [ $? -eq 1 ] + then + echo "The current helm chart version '${CURRENT_VERSION}' is newer than the version '${RELEASE_VERSION}' we are attempting to publish. Exiting" + exit 1 + fi + rm -rf "${GIT_HELM}/charts/speckle-server" + cp -r "${GIT_ROOT}/utils/helm/speckle-server" "${GIT_HELM}/charts/speckle-server" +else + # overwrite the name of the chart + yq e -i ".name = \"speckle-server-branch-${BRANCH_NAME_TRUNCATED}\"" "${GIT_ROOT}/utils/helm/speckle-server/Chart.yaml" + rm -rf "${GIT_HELM}/charts/speckle-server-branch-${BRANCH_NAME_TRUNCATED}" + cp -r "${GIT_ROOT}/utils/helm/speckle-server" "${GIT_HELM}/charts/speckle-server-branch-${BRANCH_NAME_TRUNCATED}" +fi + +echo "💾 Pushing commit" + +cd "${GIT_HELM}" + +git add . +git -c user.email="devops+gha@speckle.systems" -c user.name="CI" commit -m "Github action commit for version '${RELEASE_VERSION}'" +git push diff --git a/.github/workflows/scripts/publish_helm_chart_oci.sh b/.github/workflows/scripts/publish_helm_chart_oci.sh new file mode 100755 index 000000000..7533c5ddb --- /dev/null +++ b/.github/workflows/scripts/publish_helm_chart_oci.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +set -eo pipefail + +if [[ -z "${IMAGE_VERSION_TAG}" ]]; then + echo "IMAGE_VERSION_TAG is not set" + exit 1 +fi +if [[ -z "${DOCKER_REG_USER}" ]]; then + echo "DOCKER_REG_USER is not set" + exit 1 +fi +if [[ -z "${DOCKER_REG_PASS}" ]]; then + echo "DOCKER_REG_PASS is not set" + exit 1 +fi + +echo "đŸˇī¸ Preparing envs" + +GIT_REPO=$( pwd ) +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +# shellcheck disable=SC1090,SC1091 +source "${SCRIPT_DIR}/common.sh" + +RELEASE_VERSION="${IMAGE_VERSION_TAG}-chart" +HELM_STABLE_BRANCH="${HELM_STABLE_BRANCH:-"main"}" +DOCKER_HELM_REG_URL="${DOCKER_HELM_REG_URL:-"registry-1.docker.io"}" +DOCKER_HELM_REG_ORG="${DOCKER_HELM_REG_ORG:-"speckle"}" +CHART_NAME="${CHART_NAME:-"speckle-server"}" + +echo "📌 Releasing Helm Chart version ${RELEASE_VERSION} for application version ${IMAGE_VERSION_TAG}" + +yq e -i ".docker_image_tag = \"${IMAGE_VERSION_TAG}\"" "${GIT_REPO}/utils/helm/speckle-server/values.yaml" + +echo "${DOCKER_REG_PASS}" | helm registry login "${DOCKER_HELM_REG_URL}" --username "${DOCKER_REG_USER}" --password-stdin +helm package "${GIT_REPO}/utils/helm/speckle-server" --version "${RELEASE_VERSION}" --app-version "${IMAGE_VERSION_TAG}" --destination "/tmp" +helm push "/tmp/${CHART_NAME}-${RELEASE_VERSION}.tgz" "oci://${DOCKER_HELM_REG_URL}/${DOCKER_HELM_REG_ORG}"