speckle-server/.circleci/config.yml

version: 2.1

orbs:
  snyk: snyk/snyk@1.4.0

workflows:
  version: 2

  test-build:
    jobs:
      - vulnerability-scan:
          context: &snyk-context
            - snyk
          filters:
            branches:
              only:
                - main
                - hotfix*

      - test-server:
          filters: &filters-allow-all
            tags:
              # run tests for any commit on any branch, including any tags
              only: /.*/

      - test-frontend-2:
          filters: &filters-frontend-2
            tags:
              # frontend-2 prefix only
              only: /^frontend\-2.*/

      - chromatic:
          context:
            - chromatic
          filters: *filters-frontend-2

      - get-version:
          filters: *filters-allow-all

      - pre-commit:
          filters: *filters-allow-all

      - docker-build-server:
          context: &build-context
            - github-readonly-public-repos
          filters: &filters-build
            tags:
              only: /.*/
          requires:
            - test-server
            - get-version

      - docker-build-frontend:
          context: *build-context
          filters: *filters-build
          requires:
            - get-version

      - docker-build-frontend-2:
          filters: *filters-build
          requires:
            - get-version

      - docker-build-webhooks:
          context: *build-context
          filters: *filters-build
          requires:
            - get-version
            - test-server

      - docker-build-file-imports:
          context: *build-context
          filters: *filters-build
          requires:
            - get-version
            - test-server

      - docker-build-previews:
          context: *build-context
          filters: *filters-build
          requires:
            - get-version
            - test-server

      - docker-build-test-container:
          context: *build-context
          filters: *filters-build
          requires:
            - get-version
            - test-server

      - docker-build-monitor-container:
          context: *build-context
          filters: *filters-build
          requires:
            - get-version

      - publish-approval:
          type: approval
          filters: &filters-ignore-main-branch-or-all-tags
            branches:
              ignore:
                - main
                - frontend-2
            tags:
              ignore: /.*/

      - docker-publish-server:
          context: &docker-hub-context
            - docker-hub
          filters: &filters-publish
            branches:
              ignore: /pull\/[0-9]+/
            tags:
              only: /.*/
          requires:
            - get-version
            - publish-approval
            - docker-build-server
            - pre-commit

      - docker-publish-frontend:
          context: *docker-hub-context
          filters: *filters-publish
          requires:
            - get-version
            - publish-approval
            - docker-build-frontend
            - pre-commit

      - docker-publish-frontend-2:
          context: *docker-hub-context
          filters: *filters-publish
          requires:
            - get-version
            - publish-approval
            - docker-build-frontend-2
            - pre-commit
            - test-frontend-2

      - docker-publish-webhooks:
          context: *docker-hub-context
          filters: *filters-publish
          requires:
            - get-version
            - publish-approval
            - docker-build-webhooks
            - pre-commit

      - docker-publish-file-imports:
          context: *docker-hub-context
          filters: *filters-publish
          requires:
            - get-version
            - publish-approval
            - docker-build-file-imports
            - pre-commit

      - docker-publish-previews:
          context: *docker-hub-context
          filters: *filters-publish
          requires:
            - get-version
            - publish-approval
            - docker-build-previews
            - pre-commit

      - docker-publish-test-container:
          context: *docker-hub-context
          filters: *filters-publish
          requires:
            - get-version
            - publish-approval
            - docker-build-test-container
            - pre-commit

      - docker-publish-monitor-container:
          context: *docker-hub-context
          filters: *filters-publish
          requires:
            - get-version
            - publish-approval
            - docker-build-monitor-container
            - pre-commit

      - publish-helm-chart:
          filters: &filters-publish
            branches:
              only:
                - main
                - hotfix*
                - frontend-2
            tags:
              only: &filters-tag /^[0-9]+\.[0-9]+\.[0-9]+$/
          requires:
            - get-version
            - publish-approval
            - docker-publish-server
            - docker-publish-frontend
            - docker-publish-frontend-2
            - docker-publish-webhooks
            - docker-publish-file-imports
            - docker-publish-previews
            - docker-publish-monitor-container
            - docker-publish-test-container

      - publish-npm:
          filters:
            tags:
              only: *filters-tag
            branches:
              ignore: /.*/
          context:
            - npm-registry
          requires:
            - get-version

jobs:
  get-version:
    docker:
      - image: cimg/python:3.9
    working_directory: &work-dir /tmp/ci
    steps:
      - checkout
      - run: mkdir -p workspace
      - run:
          name: set version
          command: |
            echo "export IMAGE_VERSION_TAG=$(.circleci/get_version.sh)" >> workspace/env-vars
      - run: cat workspace/env-vars >> $BASH_ENV
      - run: echo "IMAGE_VERSION_TAG=${IMAGE_VERSION_TAG}"
      - persist_to_workspace:
          root: workspace
          paths:
            - env-vars

  pre-commit:
    parameters:
      config_file:
        default: ./.pre-commit-config.yaml
        description: Optional, path to pre-commit config file.
        type: string
      deployment_config_file:
        default: ./.pre-commit-config.deployment.yaml
        description: Optional, path to additional pre-commit config file.
        type: string
      cache_prefix:
        default: ''
        description: |
          Optional cache prefix to be used on CircleCI. Can be used for cache busting or to ensure multiple jobs use different caches.
        type: string
    docker:
      - image: speckle/pre-commit-runner:latest
    resource_class: medium
    working_directory: *work-dir
    steps:
      - checkout
      - restore_cache:
          keys:
            - cache-pre-commit-<<parameters.cache_prefix>>-{{ checksum "<<parameters.config_file>>" }}
      - run:
          name: Install pre-commit hooks
          command: pre-commit install-hooks --config <<parameters.config_file>>
      - save_cache:
          key: cache-pre-commit-<<parameters.cache_prefix>>-{{ checksum "<<parameters.config_file>>" }}
          paths:
            - ~/.cache/pre-commit
      - restore_cache:
          name: Restore Yarn Package Cache
          keys:
            - yarn-packages-{{ checksum "yarn.lock" }}
      - run:
          name: Install Dependencies
          command: yarn

      - save_cache:
          name: Save Yarn Package Cache
          key: yarn-packages-{{ checksum "yarn.lock" }}
          paths:
            - .yarn/cache
            - .yarn/unplugged
      - run:
          name: Build public packages
          command: yarn build:public
      - run:
          name: Run pre-commit
          command: ./.husky/pre-commit
      - run:
          command: git --no-pager diff
          name: git diff
          when: on_fail

  test-server:
    docker:
      - image: cimg/node:18.16.0
      - image: cimg/redis:7.0.7
      - image: 'cimg/postgres:14.5'
        environment:
          POSTGRES_DB: speckle2_test
          POSTGRES_PASSWORD: speckle
          POSTGRES_USER: speckle
      - image: 'minio/minio'
        command: server /data --console-address ":9001"
        # environment:

    environment:
      NODE_ENV: test
      DATABASE_URL: 'postgres://speckle:speckle@127.0.0.1:5432/speckle2_test'
      PGDATABASE: speckle2_test
      PGUSER: speckle
      SESSION_SECRET: 'keyboard cat'
      STRATEGY_LOCAL: 'true'
      CANONICAL_URL: 'http://127.0.0.1:3000'
      S3_ENDPOINT: 'http://127.0.0.1:9000'
      S3_ACCESS_KEY: 'minioadmin'
      S3_SECRET_KEY: 'minioadmin'
      S3_BUCKET: 'speckle-server'
      S3_CREATE_BUCKET: 'true'
      REDIS_URL: 'redis://127.0.0.1:6379'
      S3_REGION: '' # optional, defaults to 'us-east-1'
    steps:
      - checkout
      - restore_cache:
          name: Restore Yarn Package Cache
          keys:
            - yarn-packages-server-{{ checksum "yarn.lock" }}
      - run:
          name: Install Dependencies
          command: yarn

      - save_cache:
          name: Save Yarn Package Cache
          key: yarn-packages-server-{{ checksum "yarn.lock" }}
          paths:
            - .yarn/cache
            - .yarn/unplugged

      - run:
          name: Build public packages
          command: yarn build:public

      - run:
          name: Wait for dependencies to start
          command: 'dockerize -wait tcp://localhost:5432 -wait tcp://localhost:6379 -timeout 1m'

      - run:
          command: touch .env.test
          working_directory: 'packages/server'

      - run:
          name: 'Lint with TypeScript Compiler'
          command: yarn lint:tsc
          working_directory: 'packages/server'

      - run:
          command: yarn test:report
          working_directory: 'packages/server'

      - run:
          command: 'bash <(curl -s https://codecov.io/bash)'
          working_directory: 'packages/server'

      - store_test_results:
          path: packages/server/reports

      # - store_artifacts:
      # path: packages/server/coverage/lcov-report
      # destination: package/server/coverage

  test-frontend-2:
    docker:
      - image: cimg/node:16.15-browsers
    resource_class: xlarge
    steps:
      - checkout
      - restore_cache:
          name: Restore Yarn Package Cache
          keys:
            - yarn-packages-server-{{ checksum "yarn.lock" }}
      - run:
          name: Install Dependencies
          command: yarn

      - save_cache:
          name: Save Yarn Package Cache
          key: yarn-packages-server-{{ checksum "yarn.lock" }}
          paths:
            - .yarn/cache
            - .yarn/unplugged

      - run:
          name: Build public packages
          command: yarn build:public

      - run:
          name: Lint everything
          command: yarn lint
          working_directory: 'packages/frontend-2'

      - run:
          name: Install Playwright
          command: cd ~ && npx playwright install --with-deps

      - run:
          name: Test via Storybook
          command: yarn storybook:test:ci
          working_directory: 'packages/frontend-2'

  chromatic:
    resource_class: medium+
    docker:
      - image: cimg/node:16.15
    steps:
      - checkout
      - restore_cache:
          name: Restore Yarn Package Cache
          keys:
            - yarn-packages-server-{{ checksum "yarn.lock" }}
      - run:
          name: Install Dependencies
          command: yarn

      - save_cache:
          name: Save Yarn Package Cache
          key: yarn-packages-server-{{ checksum "yarn.lock" }}
          paths:
            - .yarn/cache
            - .yarn/unplugged

      - run:
          name: Build shared packages
          command: yarn build:public

      - run:
          name: Run chromatic
          command: yarn chromatic
          working_directory: 'packages/frontend-2'

  vulnerability-scan:
    # snyk can undertake most types of scans through GitHub integration
    # which does not require integration with the CI
    # but it is not possible to scan npm/yarn package.json
    # because it requires node_modules
    # therefore this scanning has to be triggered via the cli
    docker: &docker-image
      - image: cimg/python:3.9.15-node
    resource_class: small
    working_directory: *work-dir
    steps:
      - checkout
      - restore_cache:
          name: Restore Yarn Package Cache
          keys:
            - yarn-packages-server-{{ checksum "yarn.lock" }}
      - run:
          name: Install Dependencies
          command: yarn
      - save_cache:
          name: Save Yarn Package Cache
          key: yarn-packages-server-{{ checksum "yarn.lock" }}
          paths:
            - .yarn/cache
            - .yarn/unplugged
      - snyk/scan:
          additional-arguments: --yarn-workspaces  --strict-out-of-sync=false
          fail-on-issues: false

  docker-build: &build-job
    docker: &docker-image
      - image: cimg/python:3.9.15-node
    resource_class: medium
    working_directory: *work-dir
    steps:
      - checkout
      - attach_workspace:
          at: /tmp/ci/workspace
      - run:
          name: determine if draft PR
          command: |
            echo "export IS_DRAFT_PR=$(.circleci/is_draft.sh)" >> workspace/env-vars
      - run: cat workspace/env-vars >> $BASH_ENV
      - run: echo "IS_DRAFT_PR=${IS_DRAFT_PR}"
      - run:
          name: 'Check if should proceed'
          command: |
            [[ "${CIRCLE_TAG}" ]] && echo "proceed because tag is set" && exit 0
            [[ "${CIRCLE_BRANCH}" == "main" ]] && echo "proceed because main branch" && exit 0
            [[ "${CIRCLE_BRANCH}" == "frontend-2" ]] && echo "proceed because frontend-2 branch" && exit 0
            [[ "${IS_DRAFT_PR}" == "TRUE" || -z "${CIRCLE_PULL_REQUEST}" ]] && echo "Should not build because either Draft PR or branch without PR, stopping" && exit 1
            echo "proceeding"
      - setup_remote_docker:
          # a weird issue with yarn installing packages throwing EPERM errors
          # this fixes it
          version: 20.10.12
          docker_layer_caching: true
      - run:
          name: Build and Save
          command: ./.circleci/build.sh
      - persist_to_workspace:
          root: workspace
          paths:
            - speckle*

  docker-build-server:
    <<: *build-job
    environment:
      SPECKLE_SERVER_PACKAGE: server

  docker-build-frontend:
    <<: *build-job
    environment:
      SPECKLE_SERVER_PACKAGE: frontend

  docker-build-frontend-2:
    <<: *build-job
    environment:
      SPECKLE_SERVER_PACKAGE: frontend-2

  docker-build-previews:
    <<: *build-job
    environment:
      SPECKLE_SERVER_PACKAGE: preview-service

  docker-build-webhooks:
    <<: *build-job
    environment:
      SPECKLE_SERVER_PACKAGE: webhook-service

  docker-build-file-imports:
    <<: *build-job
    environment:
      SPECKLE_SERVER_PACKAGE: fileimport-service

  docker-build-test-container:
    <<: *build-job
    environment:
      FOLDER: utils
      SPECKLE_SERVER_PACKAGE: test-deployment

  docker-build-monitor-container:
    <<: *build-job
    environment:
      FOLDER: utils
      SPECKLE_SERVER_PACKAGE: monitor-deployment

  docker-publish: &publish-job
    docker: &base-image
      - image: cimg/python:3.9.15-node
    resource_class: medium
    working_directory: *work-dir
    steps:
      - checkout
      - attach_workspace:
          at: /tmp/ci/workspace
      - run: cat workspace/env-vars >> $BASH_ENV
      - setup_remote_docker:
          # a weird issue with yarn installing packages throwing EPERM errors
          # this fixes it
          version: 20.10.12
          docker_layer_caching: true
      - run:
          name: Publish
          command: ./.circleci/publish.sh

  docker-publish-server:
    <<: *publish-job
    environment:
      SPECKLE_SERVER_PACKAGE: server

  docker-publish-frontend:
    <<: *publish-job
    environment:
      SPECKLE_SERVER_PACKAGE: frontend

  docker-publish-frontend-2:
    <<: *publish-job
    environment:
      SPECKLE_SERVER_PACKAGE: frontend-2

  docker-publish-previews:
    <<: *publish-job
    environment:
      SPECKLE_SERVER_PACKAGE: preview-service

  docker-publish-webhooks:
    <<: *publish-job
    environment:
      SPECKLE_SERVER_PACKAGE: webhook-service

  docker-publish-file-imports:
    <<: *publish-job
    environment:
      SPECKLE_SERVER_PACKAGE: fileimport-service

  docker-publish-test-container:
    <<: *publish-job
    environment:
      FOLDER: utils
      SPECKLE_SERVER_PACKAGE: test-deployment

  docker-publish-monitor-container:
    <<: *publish-job
    environment:
      FOLDER: utils
      SPECKLE_SERVER_PACKAGE: monitor-deployment

  publish-npm:
    docker: *docker-image
    working_directory: *work-dir
    steps:
      - checkout
      - attach_workspace:
          at: /tmp/ci/workspace
      - run: cat workspace/env-vars >> $BASH_ENV

      - restore_cache:
          name: Restore Yarn Package Cache
          keys:
            - yarn-packages-{{ checksum "yarn.lock" }}
      - run:
          name: Install Dependencies
          command: yarn

      - save_cache:
          name: Save Yarn Package Cache
          key: yarn-packages-{{ checksum "yarn.lock" }}
          paths:
            - .yarn/cache
            - .yarn/unplugged

      - run:
          name: auth to npm as Speckle
          command: |
            echo "npmRegistryServer: https://registry.npmjs.org/" >> .yarnrc.yml
            echo "npmAuthToken: ${NPM_TOKEN}" >> .yarnrc.yml
      - run:
          name: try login to npm
          command: yarn npm whoami

      - run:
          name: build public packages
          command: yarn workspaces foreach -ptv --no-private run build
      - run:
          name: bump all versions
          # bump all versions in dependency tree order but not in parallel
          command: yarn workspaces foreach -tv version $IMAGE_VERSION_TAG

      - run:
          name: publish to npm
          command: 'yarn workspaces foreach -pv --no-private npm publish --access public'

  publish-helm-chart:
    docker: *docker-image
    working_directory: *work-dir
    steps:
      - checkout
      - attach_workspace:
          at: /tmp/ci/workspace
      - run: cat workspace/env-vars >> $BASH_ENV
      - add_ssh_keys:
          fingerprints:
            - '4d:68:70:66:49:97:ba:8b:8c:55:96:df:3d:be:6e:05'
      - run:
          name: Publish Helm Chart
          command: ./.circleci/publish_helm_chart.sh