From d32e3c0a71d238ee5cbef2f37a7b4c417a2e3ad7 Mon Sep 17 00:00:00 2001 From: Iain Sproat <68657+iainsproat@users.noreply.github.com> Date: Thu, 1 Aug 2024 12:15:35 +0100 Subject: [PATCH] chore(CI): temporarily disable docker compose deployment test & other fixes for race condition in docker compose deployment test (#2553) --- .circleci/config.yml | 54 ++++++++++--------- .../{Tiltfile.local => Tiltfile.build} | 0 .circleci/deployment/Tiltfile.dockercompose | 12 ++--- .circleci/deployment/Tiltfile.helm | 8 +-- .../deployment/{Tiltfile.ci => Tiltfile.load} | 0 .../docker-compose-speckle.override.yml | 29 ++++++++++ .../docker-compose-test.override.yml | 3 ++ docker-compose-speckle.yml | 5 +- docker-compose-test.yml | 5 +- utils/test-deployment/run_tests.py | 20 +++---- 10 files changed, 86 insertions(+), 50 deletions(-) rename .circleci/deployment/{Tiltfile.local => Tiltfile.build} (100%) rename .circleci/deployment/{Tiltfile.ci => Tiltfile.load} (100%) create mode 100644 .circleci/deployment/docker-compose-speckle.override.yml create mode 100644 .circleci/deployment/docker-compose-test.override.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index eb58f1b91..76d5a6bae 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -50,7 +50,31 @@ workflows: tags: ignore: /.*/ - - deployment-test-docker-compose: + #FIXME uncomment when nix https://search.nixos.org/packages?channel=24.05&show=tilt&from=0&size=50&sort=relevance&type=packages&query=tilt supports tilt >v0.33.12 which includes docker compose up --wait flag + # - deployment-test-docker-compose: + # filters: &filters-deployment-testing + # tags: + # # run tests for any commit on any branch, including any tags + # only: /.*/ + # requires: + # - get-version + # - pre-commit + # - deployment-testing-approval + # - test-frontend-2 + # - test-viewer + # - test-objectsender + # - test-server + # - docker-build-server + # - docker-build-frontend + # - docker-build-frontend-2 + # - docker-build-previews + # - docker-build-webhooks + # - docker-build-file-imports + # - docker-build-test-container + # - docker-build-monitor-container + # - docker-build-docker-compose-ingress + + - deployment-test-helm-chart: filters: &filters-deployment-testing tags: # run tests for any commit on any branch, including any tags @@ -71,26 +95,6 @@ workflows: - docker-build-file-imports - docker-build-test-container - docker-build-monitor-container - - docker-build-docker-compose-ingress - - - deployment-test-helm-chart: - filters: *filters-deployment-testing - requires: - - get-version - - pre-commit - - deployment-testing-approval - - test-frontend-2 - - test-viewer - - test-objectsender - - test-server - - docker-build-server - - docker-build-frontend - - docker-build-frontend-2 - - docker-build-previews - - docker-build-webhooks - - docker-build-file-imports - - docker-build-test-container - - docker-build-monitor-container - get-version: filters: *filters-allow-all @@ -288,7 +292,7 @@ workflows: tags: only: &filters-tag /^[0-9]+\.[0-9]+\.[0-9]+$/ requires: - - deployment-test-docker-compose + # - deployment-test-docker-compose #FIXME uncomment when nix https://search.nixos.org/packages?channel=24.05&show=tilt&from=0&size=50&sort=relevance&type=packages&query=tilt supports tilt >v0.33.12 which includes docker compose up --wait flag - deployment-test-helm-chart - docker-publish-docker-compose-ingress - docker-publish-file-imports @@ -740,7 +744,7 @@ jobs: deployment-test-docker-compose: machine: - image: ubuntu-2204:2024.01.1 + image: ubuntu-2204:2024.05.1 docker_layer_caching: true resource_class: large working_directory: *work-dir @@ -770,7 +774,7 @@ jobs: paths: - /nix - run: cat workspace/env-vars >> $BASH_ENV - - run: nix-shell --run "tilt ci --file ./.circleci/deployment/Tiltfile.dockercompose" ./.circleci/deployment/docker-compose-shell.nix + - run: nix-shell --run "LOAD_DOCKER='true' tilt ci --file ./.circleci/deployment/Tiltfile.dockercompose" ./.circleci/deployment/docker-compose-shell.nix deployment-test-helm-chart: machine: @@ -844,7 +848,7 @@ jobs: name: Deploy Kubernetes resources to cluster command: | nix-shell \ - --run "tilt ci --file ./.circleci/deployment/Tiltfile.helm --context kind-speckle-server --timeout 10m" \ + --run "LOAD_DOCKER='true' tilt ci --file ./.circleci/deployment/Tiltfile.helm --context kind-speckle-server --timeout 10m" \ ./.circleci/deployment/helm-chart-shell.nix docker-build: &build-job diff --git a/.circleci/deployment/Tiltfile.local b/.circleci/deployment/Tiltfile.build similarity index 100% rename from .circleci/deployment/Tiltfile.local rename to .circleci/deployment/Tiltfile.build diff --git a/.circleci/deployment/Tiltfile.dockercompose b/.circleci/deployment/Tiltfile.dockercompose index a4cf7ee8f..e6d98c76f 100644 --- a/.circleci/deployment/Tiltfile.dockercompose +++ b/.circleci/deployment/Tiltfile.dockercompose @@ -1,11 +1,11 @@ print('🚀 Deploying Speckle Server into Docker via Tilt...') symbols = None -IS_CI = os.getenv('CI', "false") == "true" -if IS_CI: - symbols = load_dynamic('./Tiltfile.ci') +IS_LOAD_DOCKER = os.getenv('LOAD_DOCKER', "false") == "true" +if IS_LOAD_DOCKER: + symbols = load_dynamic('./Tiltfile.load') else: - symbols = load_dynamic('./Tiltfile.local') + symbols = load_dynamic('./Tiltfile.build') # Prepare the Speckle images # (loads the images from tgz file if CI, otherwise builds them or pulls them from remote as a fallback on non-amd64 machines) @@ -28,7 +28,7 @@ dc_resource('maildev', labels=['dependencies']) dc_resource('postgres', labels=['dependencies']) # Prepare the Speckle resources -docker_compose(['../../docker-compose-speckle.yml', encode_yaml({'services': {'speckle-server': {'environment': {'ENABLE_MP': 'false'}}}})]) +docker_compose(['../../docker-compose-speckle.yml', 'docker-compose-speckle.override.yml'], wait=True) dc_resource('speckle-server', resource_deps=['postgres', 'redis', 'minio', 'maildev'], labels=['speckle']) dc_resource('speckle-frontend-2', resource_deps=[], labels=['speckle']) dc_resource('speckle-ingress', resource_deps=[], labels=['speckle']) @@ -37,5 +37,5 @@ dc_resource('fileimport-service', resource_deps=[], labels=['speckle']) dc_resource('webhook-service', resource_deps=['postgres'], labels=['speckle']) # Test the Speckle resources -docker_compose('../../docker-compose-test.yml') +docker_compose(['../../docker-compose-test.yml', 'docker-compose-test.override.yml'], wait=True) dc_resource('test', resource_deps=['speckle-server', 'speckle-frontend-2'], labels=['test']) # --exit-code-from test diff --git a/.circleci/deployment/Tiltfile.helm b/.circleci/deployment/Tiltfile.helm index 7f2a47d2c..4bd336a58 100644 --- a/.circleci/deployment/Tiltfile.helm +++ b/.circleci/deployment/Tiltfile.helm @@ -10,11 +10,11 @@ if k8s_context() != 'kind-speckle-server': load('ext://helm_resource', 'helm_resource', 'helm_repo') load('ext://k8s_yaml_glob', 'k8s_yaml_glob') symbols = None -IS_CI = os.getenv('CI', "false") == "true" -if IS_CI: - symbols = load_dynamic('./Tiltfile.ci') +IS_LOAD_DOCKER = os.getenv('LOAD_DOCKER', "false") == "true" +if IS_LOAD_DOCKER: + symbols = load_dynamic('./Tiltfile.load') else: - symbols = load_dynamic('./Tiltfile.local') + symbols = load_dynamic('./Tiltfile.build') # Prepare the Speckle images # (loads the images from tgz file if CI, otherwise builds them or pulls them from remote as a fallback on non-amd64 machines) diff --git a/.circleci/deployment/Tiltfile.ci b/.circleci/deployment/Tiltfile.load similarity index 100% rename from .circleci/deployment/Tiltfile.ci rename to .circleci/deployment/Tiltfile.load diff --git a/.circleci/deployment/docker-compose-speckle.override.yml b/.circleci/deployment/docker-compose-speckle.override.yml new file mode 100644 index 000000000..0853c0e96 --- /dev/null +++ b/.circleci/deployment/docker-compose-speckle.override.yml @@ -0,0 +1,29 @@ +services: + speckle-ingress: + image: speckle/speckle-docker-compose-ingress + + speckle-frontend-2: + image: speckle/speckle-frontend-2 + # environment: + # LOG_PRETTY: 'true' #FIXME currently causes a failure + + speckle-server: + image: speckle/speckle-server + environment: + ENABLE_MP: 'false' + LOG_PRETTY: 'true' + + preview-service: + image: speckle/speckle-preview-service + environment: + LOG_PRETTY: 'true' + + webhook-service: + image: speckle/speckle-webhook-service + environment: + LOG_PRETTY: 'true' + + fileimport-service: + image: speckle/speckle-fileimport-service + environment: + LOG_PRETTY: 'true' diff --git a/.circleci/deployment/docker-compose-test.override.yml b/.circleci/deployment/docker-compose-test.override.yml new file mode 100644 index 000000000..193ff0bf3 --- /dev/null +++ b/.circleci/deployment/docker-compose-test.override.yml @@ -0,0 +1,3 @@ +services: + test: + image: speckle/speckle-test-deployment diff --git a/docker-compose-speckle.yml b/docker-compose-speckle.yml index e4639c7c0..396682b03 100644 --- a/docker-compose-speckle.yml +++ b/docker-compose-speckle.yml @@ -27,6 +27,7 @@ services: NUXT_PUBLIC_BACKEND_API_ORIGIN: 'http://speckle-server:3000' NUXT_PUBLIC_LOG_LEVEL: 'warn' NUXT_REDIS_URL: 'redis://redis' + LOG_LEVEL: 'info' speckle-server: build: @@ -40,7 +41,7 @@ services: - CMD - node - -e - - "try { require('node:http').request({headers: {'Content-Type': 'application/json'}, port:3000, hostname:'127.0.0.1', path:'/liveness', method: 'GET', timeout: 2000 }, (res) => { body = ''; res.on('data', (chunk) => {body += chunk;}); res.on('end', () => {process.exit(res.statusCode != 200 || body.toLowerCase().includes('error'));}); }).end(); } catch { process.exit(1); }" + - "try { require('node:http').request({headers: {'Content-Type': 'application/json'}, port:3000, hostname:'127.0.0.1', path:'/readiness', method: 'GET', timeout: 2000 }, (res) => { body = ''; res.on('data', (chunk) => {body += chunk;}); res.on('end', () => {process.exit(res.statusCode != 200 || body.toLowerCase().includes('error'));}); }).end(); } catch { process.exit(1); }" interval: 10s timeout: 10s retries: 3 @@ -88,8 +89,6 @@ services: HOST: '127.0.0.1' LOG_LEVEL: 'info' PG_CONNECTION_STRING: 'postgres://speckle:speckle@postgres/speckle' - ports: - - 127.0.0.1:3001:3001 webhook-service: build: diff --git a/docker-compose-test.yml b/docker-compose-test.yml index db6c6160f..05dbd79da 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -13,8 +13,9 @@ services: VERIFY_CERTIFICATE: '0' restart: 'no' - # It wasn't clear that the test was just being deployed or was actually running to completion before docker compose up was exiting. - # This service is a hack to make docker compose wait for the test to finish before exiting. + # Docker compose run via Tilt just waits for the test to be be deployed. + # It does not wait for it to actually run to completion before exiting. + # This service ensures docker compose waits for the test to finish before exiting. confirm_test_finished: image: busybox command: sh -c "echo '✅ Deployment of test completed successfully!'" diff --git a/utils/test-deployment/run_tests.py b/utils/test-deployment/run_tests.py index 6cab98bd5..a762d57a7 100755 --- a/utils/test-deployment/run_tests.py +++ b/utils/test-deployment/run_tests.py @@ -41,6 +41,16 @@ if not SPECKLE_SERVER.startswith("http://") and not SPECKLE_SERVER.startswith( print(f"ℹī¸ Using Speckle server '{SPECKLE_SERVER}'") +# Test that the server backend is accessible +client = SpeckleClient( + SPECKLE_SERVER, + use_ssl=SPECKLE_SERVER.startswith("https://"), + verify_certificate=VERIFY_CERTIFICATE, +) +server_info = client.server.get() +assert isinstance(server_info, ServerInfo), "❌ GraphQL ServerInfo query error" +print(f"✅ GraphQL operation succeeded. Server name: {server_info.name}") + # Test if frontend is accessible if FRONTEND_VERSION == "1": frontend_response = requests.get( @@ -61,16 +71,6 @@ else: exit(1) print("✅ Frontend accessible") -# Test basic unauthenticated operation using specklepy -client = SpeckleClient( - SPECKLE_SERVER, - use_ssl=SPECKLE_SERVER.startswith("https://"), - verify_certificate=VERIFY_CERTIFICATE, -) -server_info = client.server.get() -assert isinstance(server_info, ServerInfo), "❌ GraphQL ServerInfo query error" -print(f"✅ GraphQL operation succeeded. Server name: {server_info.name}") - # Test that the deployed server version matches the expected version SERVER_VERSION = "" if len(sys.argv) > 2: