feat(ci): reinstate multiregion tests (#5365)
* feat(multiregion): replace user replication * chore(multiregion): optimise replication * maybe it's this * postgres is fun * once more * chore(multiregion): only replicate test user creation during multiregion tests * feat: improved replicate_query logic * fix: minor * fix: starting issue * feat: included user create and delete specs to multiregion * feat: removed console logs * fix: user defaults * fix: multiregion test helper * fix: update scenarios for users * refactor(multiregion): swap replicateQuery concept to asMultiregionOperation (#5301) feat(multiregion): introduced asMultregionOperator, refactor test to user builder classes * chore: renamings * fix: remove comments * feat: remove user replication * refactor: simplified spec usages * chore: comments * chore: branches and favs * chore: more tests * chore: more tests * fix linting * fix tests * feat: dropping replication * refactor: moved project delete to service * fix: comment * feat: updateStreamFactory and updateProjectFacotry * deleteProjectFactory + replicateFactory * deleteWorkspaceFactory * fix: selector * fix: tests * fix tests, finished createStreamFactory * feat: simplify changes * fix: remove comment * fix: minor strucutres * fix: moveProjectToRegion * fix: moved branch creation outside of multiregion scope * fix: branch creation * fix: tests * fix: ci tests * fix: removed log form test * fix: on specs, no random regionKeys * feat: simplify ci for postgres * try: fix health check * feat: fixed tests in ci * try: entrypoint * try: entrypoint * try: entrypoint * try: POSTGRES_INITDB_ARGS * feat: apply POSTGRES_INITDB_ARGS to all server tests * fix: broken test * fix: reinstate max health attempts * fix: after merge * fix: after merge --------- Co-authored-by: Charles Driesler <chuck@speckle.systems>
This commit is contained in:
Daniel Gak Anagrov
committed by
GitHub
GitHub
parent
399c998fd7
commit
75aa5d9b2d
+11
-31
@@ -254,27 +254,8 @@ jobs:
|
||||
path: /tmp/**/*.log
|
||||
retention-days: 5
|
||||
|
||||
docker-build-postgres-container:
|
||||
runs-on: blacksmith-4vcpu-ubuntu-2404
|
||||
name: Docker build postgres container
|
||||
steps:
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ inputs.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
- name: Setup Docker Builder
|
||||
uses: useblacksmith/setup-docker-builder@v1
|
||||
- name: Build and push
|
||||
uses: useblacksmith/build-push-action@v2
|
||||
with:
|
||||
push: true
|
||||
tags: speckle/speckle-postgres:${{ inputs.IMAGE_VERSION_TAG }}
|
||||
file: ./utils/postgres/Dockerfile
|
||||
|
||||
test-server:
|
||||
name: Server
|
||||
needs: [docker-build-postgres-container]
|
||||
runs-on: blacksmith-8vcpu-ubuntu-2404
|
||||
continue-on-error: ${{ inputs.CONTINUE_ON_ERROR }}
|
||||
services:
|
||||
@@ -288,11 +269,12 @@ jobs:
|
||||
ports:
|
||||
- 6379:6379
|
||||
postgres:
|
||||
image: speckle/speckle-postgres:${{ inputs.IMAGE_VERSION_TAG }}
|
||||
image: postgres:16.4-alpine3.20
|
||||
env:
|
||||
POSTGRES_DB: speckle2_test
|
||||
POSTGRES_PASSWORD: speckle
|
||||
POSTGRES_USER: speckle
|
||||
POSTGRES_INITDB_ARGS: -c max_prepared_transactions=150
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
@@ -369,7 +351,6 @@ jobs:
|
||||
|
||||
test-server-no-ff:
|
||||
name: Server no ff
|
||||
needs: [docker-build-postgres-container]
|
||||
runs-on: blacksmith-8vcpu-ubuntu-2404
|
||||
continue-on-error: ${{ inputs.CONTINUE_ON_ERROR }}
|
||||
services:
|
||||
@@ -383,11 +364,12 @@ jobs:
|
||||
ports:
|
||||
- 6379:6379
|
||||
postgres:
|
||||
image: speckle/speckle-postgres:${{ inputs.IMAGE_VERSION_TAG }}
|
||||
image: postgres:16.4-alpine3.20
|
||||
env:
|
||||
POSTGRES_DB: speckle2_test
|
||||
POSTGRES_PASSWORD: speckle
|
||||
POSTGRES_USER: speckle
|
||||
POSTGRES_INITDB_ARGS: -c max_prepared_transactions=150
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
@@ -454,9 +436,7 @@ jobs:
|
||||
|
||||
test-server-multiregion:
|
||||
name: Server multiregion
|
||||
needs: [docker-build-postgres-container]
|
||||
continue-on-error: ${{ inputs.CONTINUE_ON_ERROR }}
|
||||
if: false # disabled
|
||||
runs-on: blacksmith-4vcpu-ubuntu-2404
|
||||
services:
|
||||
redis:
|
||||
@@ -470,11 +450,12 @@ jobs:
|
||||
- 6379:6379
|
||||
|
||||
postgres0:
|
||||
image: speckle/speckle-postgres:${{ inputs.IMAGE_VERSION_TAG }}
|
||||
image: postgres:16.4-alpine3.20
|
||||
env:
|
||||
POSTGRES_DB: speckle2_test
|
||||
POSTGRES_PASSWORD: speckle
|
||||
POSTGRES_USER: speckle
|
||||
POSTGRES_INITDB_ARGS: -c max_prepared_transactions=150
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
@@ -482,13 +463,13 @@ jobs:
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
|
||||
postgres1:
|
||||
image: speckle/speckle-postgres:${{ inputs.IMAGE_VERSION_TAG }}
|
||||
image: postgres:16.4-alpine3.20
|
||||
env:
|
||||
POSTGRES_DB: speckle2_test
|
||||
POSTGRES_PASSWORD: speckle
|
||||
POSTGRES_USER: speckle
|
||||
POSTGRES_INITDB_ARGS: -c max_prepared_transactions=150
|
||||
ports:
|
||||
- 5433:5432
|
||||
options: >-
|
||||
@@ -496,13 +477,13 @@ jobs:
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
|
||||
postgres2:
|
||||
image: speckle/speckle-postgres:${{ inputs.IMAGE_VERSION_TAG }}
|
||||
image: postgres:16.4-alpine3.20
|
||||
env:
|
||||
POSTGRES_DB: speckle2_test
|
||||
POSTGRES_PASSWORD: speckle
|
||||
POSTGRES_USER: speckle
|
||||
POSTGRES_INITDB_ARGS: -c max_prepared_transactions=150
|
||||
ports:
|
||||
- 5434:5432
|
||||
options: >-
|
||||
@@ -510,7 +491,6 @@ jobs:
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
|
||||
minio0:
|
||||
image: bitnami/minio
|
||||
env:
|
||||
@@ -583,7 +563,7 @@ jobs:
|
||||
- run: cp .env.test-example .env.test
|
||||
working-directory: 'packages/server'
|
||||
- name: 'Run test'
|
||||
run: yarn test:report
|
||||
run: yarn test:multiregion
|
||||
working-directory: 'packages/server'
|
||||
timeout-minutes: 30
|
||||
- uses: codecov/codecov-action@v5
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import { cliLogger as logger } from '@/observability/logging'
|
||||
import type { CommonDbArgs } from '@/modules/cli/commands/db/helpers'
|
||||
import { getTargettedDbClients } from '@/modules/cli/commands/db/helpers'
|
||||
import { resetPubSubFactory } from '@/test/hooks'
|
||||
import type { CommandModule } from 'yargs'
|
||||
|
||||
const command: CommandModule<unknown, CommonDbArgs> = {
|
||||
@@ -15,8 +14,6 @@ const command: CommandModule<unknown, CommonDbArgs> = {
|
||||
const dbs = await getTargettedDbClients({ regionKey })
|
||||
for (const db of dbs) {
|
||||
logger.info(`Rolling back DB ${db.regionKey}...`)
|
||||
const resetPubSub = resetPubSubFactory({ db: db.client })
|
||||
await resetPubSub()
|
||||
await db.client.migrate.rollback(undefined, true)
|
||||
}
|
||||
|
||||
|
||||
@@ -7,7 +7,6 @@ import type { CommandModule } from 'yargs'
|
||||
import { isTestEnv } from '@/modules/shared/helpers/envHelper'
|
||||
import { BaseError } from '@/modules/shared/errors'
|
||||
import { ensureError } from '@speckle/shared'
|
||||
import { resetPubSubFactory } from '@/test/hooks'
|
||||
import { mainDb } from '@/db/knex'
|
||||
|
||||
const command: CommandModule<unknown, CommonDbArgs> = {
|
||||
@@ -54,13 +53,6 @@ const command: CommandModule<unknown, CommonDbArgs> = {
|
||||
for (const db of dbs) {
|
||||
logger.info(`Purging test DB ${db.regionKey}...`)
|
||||
try {
|
||||
// Attempt to reset pubsub, swallowing issues
|
||||
await resetPubSubFactory({ db: db.client })().catch((err) => {
|
||||
logger.warn(`Failed to reset pubsub for ${db.regionKey}`, {
|
||||
cause: ensureError(err)
|
||||
})
|
||||
})
|
||||
|
||||
// Find and drop all tables
|
||||
const tables = await db.client.raw(
|
||||
'SELECT table_name FROM information_schema.tables WHERE table_schema = ?',
|
||||
|
||||
@@ -170,13 +170,13 @@ isMultiRegionTestMode()
|
||||
}
|
||||
|
||||
const manyParallelCreates = async () => {
|
||||
await Promise.allSettled(Array.from({ length: 1000 }, oneKnexInstanceCall))
|
||||
await Promise.allSettled(Array.from({ length: 500 }, oneKnexInstanceCall))
|
||||
}
|
||||
|
||||
await manyParallelCreates()
|
||||
|
||||
const [{ count }] = await db('users').count()
|
||||
expect(count).to.eql('1000')
|
||||
expect(count).to.eql('500')
|
||||
|
||||
await sleep(50)
|
||||
|
||||
|
||||
@@ -988,28 +988,34 @@ describe('Workspace project GQL CRUD', () => {
|
||||
|
||||
isMultiRegionTestMode()
|
||||
? describe('when the default server db region is not the main db @multiregion', () => {
|
||||
const regionalProject: StreamRecord = {
|
||||
id: cryptoRandomString({ length: 9 }),
|
||||
name: 'My Special Project',
|
||||
description: null,
|
||||
clonedFrom: null,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
allowPublicComments: false,
|
||||
workspaceId: null,
|
||||
regionKey: 'region1',
|
||||
visibility: ProjectRecordVisibility.Public
|
||||
}
|
||||
let regionalProject: BasicTestStream
|
||||
|
||||
beforeEach(async () => {
|
||||
before(async () => {
|
||||
// Simulate non-main default db region
|
||||
regionalProject = await createTestStream(
|
||||
{
|
||||
name: 'My Special Project',
|
||||
description: null,
|
||||
clonedFrom: null,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
allowPublicComments: false,
|
||||
workspaceId: null,
|
||||
regionKey: 'region1',
|
||||
visibility: ProjectRecordVisibility.Public
|
||||
},
|
||||
serverAdminUser
|
||||
)
|
||||
})
|
||||
|
||||
it('should be located in the correct region', async () => {
|
||||
const regionDb = await getRegionDb({ regionKey: 'region1' })
|
||||
await tables.streams(regionDb).insert(regionalProject)
|
||||
await grantStreamPermissions({
|
||||
streamId: regionalProject.id,
|
||||
userId: serverAdminUser.id,
|
||||
role: Roles.Stream.Owner
|
||||
})
|
||||
|
||||
const [res] = await tables
|
||||
.streams(regionDb)
|
||||
.where({ id: regionalProject.id })
|
||||
|
||||
expect(res).to.exist
|
||||
})
|
||||
|
||||
it('should update project without removing workspace association @multiregion', async () => {
|
||||
|
||||
@@ -28,7 +28,7 @@
|
||||
"ts-gqlgen": "tsx --import ./esmLoader.js ./bin/gqlgen",
|
||||
"test": "cross-env TSX=true NODE_ENV=test LOG_FILTER=test LOG_PRETTY=true yarn ts-mocha",
|
||||
"test:all-ff": "cross-env ENABLE_ALL_FFS=true yarn test",
|
||||
"test:multiregion": "cross-env RUN_TESTS_IN_MULTIREGION_MODE=true FF_WORKSPACES_MODULE_ENABLED=true FF_WORKSPACES_MULTI_REGION_ENABLED=true FF_MOVE_PROJECT_REGION_ENABLED=true yarn test --grep @multiregion",
|
||||
"test:multiregion": "cross-env RUN_TESTS_IN_MULTIREGION_MODE=true FF_WORKSPACES_MODULE_ENABLED=true FF_WORKSPACES_MULTI_REGION_ENABLED=true FF_MOVE_PROJECT_REGION_ENABLED=true yarn test:report -g '@multiregion'",
|
||||
"test:no-ff": "cross-env DISABLE_ALL_FFS=true yarn test",
|
||||
"test:coverage": "cross-env NODE_ENV=test LOG_FILTER=test LOG_PRETTY=true c8 yarn test",
|
||||
"test:report": "MOCHA_FILE=reports/test-results.xml yarn test:coverage -- --reporter mocha-multi --reporter-options spec=-,mocha-junit-reporter=reports/test-results.xml",
|
||||
|
||||
@@ -18,7 +18,7 @@ import type http from 'http'
|
||||
import type express from 'express'
|
||||
import type net from 'net'
|
||||
import type { MaybeAsync, MaybeNullOrUndefined, Nullable } from '@speckle/shared'
|
||||
import { ensureError, retry, TIME_MS, wait } from '@speckle/shared'
|
||||
import { ensureError, retry } from '@speckle/shared'
|
||||
import {
|
||||
getAvailableRegionKeysFactory,
|
||||
getFreeRegionKeysFactory
|
||||
@@ -37,7 +37,6 @@ import {
|
||||
} from '@/modules/multiregion/utils/dbSelector'
|
||||
import type { Knex } from 'knex'
|
||||
import { isMultiRegionTestMode } from '@/test/speckle-helpers/regions'
|
||||
import { isMultiRegionEnabled } from '@/modules/multiregion/helpers'
|
||||
import type { GraphQLContext } from '@/modules/shared/helpers/typeHelper'
|
||||
import type { ApolloServer } from '@apollo/server'
|
||||
import type { ReadinessHandler } from '@/healthchecks/types'
|
||||
@@ -103,10 +102,6 @@ const inEachDb = async (fn: (db: Knex) => MaybeAsync<void>) => {
|
||||
}
|
||||
}
|
||||
|
||||
const ensureAivenExtrasFactory = (deps: { db: Knex }) => async () => {
|
||||
await deps.db.raw('CREATE EXTENSION IF NOT EXISTS "aiven_extras";')
|
||||
}
|
||||
|
||||
const setupDatabases = async () => {
|
||||
// First reset main db
|
||||
const db = mainDb
|
||||
@@ -164,60 +159,6 @@ const unlockFactory = (deps: { db: Knex }) => async () => {
|
||||
|
||||
export const getRegionKeys = () => Object.keys(regionClients)
|
||||
|
||||
export const resetPubSubFactory = (deps: { db: Knex }) => async () => {
|
||||
// We wanna reset even outside of multiregion test mode, as long as multi region is generally enabled
|
||||
if (!isMultiRegionEnabled()) {
|
||||
return { drop: async () => {}, reenable: async () => {} }
|
||||
}
|
||||
|
||||
const ensureAivenExtras = ensureAivenExtrasFactory(deps)
|
||||
await ensureAivenExtras()
|
||||
|
||||
type SubInfo = {
|
||||
subname: string
|
||||
subconninfo: string
|
||||
subpublications: string[]
|
||||
subslotname: string
|
||||
}
|
||||
|
||||
const subscriptions = (await deps.db.raw(
|
||||
`SELECT subname, subconninfo, subpublications, subslotname FROM aiven_extras.pg_list_all_subscriptions() WHERE subname ILIKE 'test_%';`
|
||||
)) as {
|
||||
rows: Array<SubInfo>
|
||||
}
|
||||
const publications = (await deps.db.raw(
|
||||
`SELECT pubname FROM pg_publication WHERE pubname ILIKE 'test_%';`
|
||||
)) as {
|
||||
rows: Array<{ pubname: string }>
|
||||
}
|
||||
|
||||
// If we do not wait, the following call occasionally fails because a replication slot is still in use.
|
||||
const dropSubs = async (info: SubInfo) => {
|
||||
await wait(TIME_MS.second)
|
||||
await deps.db.raw(
|
||||
`SELECT * FROM aiven_extras.pg_alter_subscription_disable('${info.subname}');`
|
||||
)
|
||||
await wait(TIME_MS.second)
|
||||
await deps.db.raw(
|
||||
`SELECT * FROM aiven_extras.pg_drop_subscription('${info.subname}');`
|
||||
)
|
||||
await wait(TIME_MS.second)
|
||||
await deps.db.raw(
|
||||
`SELECT * FROM aiven_extras.dblink_slot_create_or_drop('${info.subconninfo}', '${info.subslotname}', 'drop');`
|
||||
)
|
||||
}
|
||||
|
||||
// Drop all subs
|
||||
for (const sub of subscriptions.rows) {
|
||||
await dropSubs(sub)
|
||||
}
|
||||
|
||||
// Drop all pubs
|
||||
for (const pub of publications.rows) {
|
||||
await deps.db.raw(`DROP PUBLICATION ${pub.pubname};`)
|
||||
}
|
||||
}
|
||||
|
||||
const truncateTablesFactory = (deps: { db: Knex }) => async (tableNames?: string[]) => {
|
||||
if (!tableNames?.length) {
|
||||
tableNames = (
|
||||
@@ -257,7 +198,6 @@ const resetSchemaFactory =
|
||||
(deps: { db: Knex; regionKey: Nullable<string> }) => async () => {
|
||||
const { regionKey } = deps
|
||||
|
||||
const resetPubSub = resetPubSubFactory(deps)
|
||||
const truncate = truncateTablesFactory(deps)
|
||||
|
||||
const pendingTransactions = await getStalePreparedTransactionsFactory({
|
||||
@@ -268,7 +208,6 @@ const resetSchemaFactory =
|
||||
)
|
||||
|
||||
await unlockFactory(deps)()
|
||||
await resetPubSub()
|
||||
await truncate() // otherwise some rollbacks will fail
|
||||
|
||||
// Reset schema
|
||||
@@ -287,28 +226,9 @@ const resetSchemaFactory =
|
||||
}
|
||||
}
|
||||
|
||||
export const truncateTables = async (
|
||||
tableNames?: string[],
|
||||
options?: Partial<{
|
||||
/**
|
||||
* Whether to also reset pubsub before truncate. Pubsub only gets re-initialized on app
|
||||
* init so don't do this if not needed!
|
||||
* Defaults to: false
|
||||
*/
|
||||
resetPubSub: boolean
|
||||
}>
|
||||
) => {
|
||||
const { resetPubSub = false } = options || {}
|
||||
export const truncateTables = async (tableNames?: string[]) => {
|
||||
const dbs = [mainDb, ...Object.values(regionClients)]
|
||||
|
||||
// First reset pubsubs, if needed
|
||||
if (resetPubSub) {
|
||||
for (const db of dbs) {
|
||||
const resetPubSub = resetPubSubFactory({ db })
|
||||
await resetPubSub()
|
||||
}
|
||||
}
|
||||
|
||||
// Now truncate
|
||||
for (const db of dbs) {
|
||||
const truncate = truncateTablesFactory({ db })
|
||||
@@ -360,7 +280,7 @@ export const buildApp = async () => {
|
||||
}
|
||||
|
||||
export const beforeEachContext = async () => {
|
||||
await truncateTables(undefined, { resetPubSub: true })
|
||||
await truncateTables(undefined)
|
||||
return await buildApp()
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user