feat(server): run tests in multiregion mode if RUN_TESTS_IN_MULTIREGION_MODE=true

.gitignore

@@ -74,4 +74,5 @@ redis-data/
 .tshy-build
 
 # Server
-multiregion.json
+multiregion.json
+multiregion.test.json

package.json

@@ -21,6 +21,7 @@
     "dev:docker": "docker compose -f ./docker-compose-deps.yml",
     "dev:docker:up": "docker compose -f ./docker-compose-deps.yml up -d",
     "dev:docker:down": "docker compose -f ./docker-compose-deps.yml down",
+    "dev:docker:restart": "yarn dev:docker:down && yarn dev:docker:up",
     "dev:kind:up": "ctlptl apply --filename ./.circleci/deployment/cluster-config.yaml",
     "dev:kind:down": "ctlptl delete -f ./.circleci/deployment/cluster-config.yaml",
     "dev:kind:helm:up": "yarn dev:kind:up && tilt up --file ./.circleci/deployment/Tiltfile.helm --context kind-speckle-server",

packages/server/.env-example

@@ -147,3 +147,8 @@ OIDC_CLIENT_SECRET="gLb9IEutYQ0npyvA8iHxPsObY3duGB0w"
 # OTEL_TRACE_URL=""
 # OTEL_TRACE_KEY=""
 # OTEL_TRACE_VALUE=""
+
+############################################################
+# Multi region settings
+############################################################
+MULTI_REGION_CONFIG_PATH="multiregion.json"

packages/server/.env.test-example

@@ -4,4 +4,6 @@
 
 PORT=0
 POSTGRES_URL=postgres://speckle:speckle@127.0.0.1/speckle2_test
-POSTGRES_USER=''
+POSTGRES_USER=''
+MULTI_REGION_CONFIG_PATH="multiregion.test.json"
+#RUN_TESTS_IN_MULTIREGION_MODE=true

packages/server/.mocharc.js

@@ -14,7 +14,7 @@ const ignore = [
 /** @type {import("mocha").MochaOptions} */
 const config = {
   spec: ['modules/**/*.spec.js', 'modules/**/*.spec.ts', 'logging/**/*.spec.js'],
-  require: ['ts-node/register', 'test/hooks.js'],
+  require: ['ts-node/register', 'test/hooks.ts'],
   ...(ignore.length ? { ignore } : {}),
   slow: 0,
   timeout: '150000',

packages/server/modules/cli/commands/db/migrate/latest.ts

@@ -1,5 +1,8 @@
 import knex from '@/db/knex'
 import { logger } from '@/logging/logging'
+import { getRegisteredRegionClients } from '@/modules/multiregion/dbSelector'
+import { isTestEnv } from '@/modules/shared/helpers/envHelper'
+import { mochaHooks } from '@/test/hooks'
 import { CommandModule } from 'yargs'
 
 const command: CommandModule = {
@@ -7,7 +10,19 @@ const command: CommandModule = {
   describe: 'Run all migrations that have not yet been run',
   async handler() {
     logger.info('Running latest migration...')
-    await knex.migrate.latest()
+
+    // In tests we want different logic - just run beforeAll
+    if (isTestEnv()) {
+      // Run before hooks, to properly initialize everything
+      await (mochaHooks.beforeAll as () => Promise<void>)()
+    } else {
+      const regionDbs = await getRegisteredRegionClients()
+      const dbs = [knex, ...Object.values(regionDbs)]
+      for (const db of dbs) {
+        await db.migrate.latest()
+      }
+    }
+
     logger.info('Completed running migration')
   }
 }

packages/server/modules/cli/commands/db/migrate/rollback.ts

@@ -1,5 +1,8 @@
 import knex from '@/db/knex'
 import { logger } from '@/logging/logging'
+import { getRegisteredRegionClients } from '@/modules/multiregion/dbSelector'
+import { isTestEnv } from '@/modules/shared/helpers/envHelper'
+import { mochaHooks, resetPubSubFactory } from '@/test/hooks'
 import { CommandModule } from 'yargs'
 
 const command: CommandModule = {
@@ -7,7 +10,21 @@ const command: CommandModule = {
   describe: 'Roll back all migrations',
   async handler() {
     logger.info('Rolling back migrations...')
-    await knex.migrate.rollback(undefined, true)
+
+    if (isTestEnv()) {
+      // Run before hooks, to properly initialize everything first
+      await (mochaHooks.beforeAll as () => Promise<void>)()
+    }
+
+    const regionDbs = await getRegisteredRegionClients()
+    const dbs = [knex, ...Object.values(regionDbs)]
+
+    for (const db of dbs) {
+      const resetPubSub = resetPubSubFactory({ db })
+      await resetPubSub()
+      await db.migrate.rollback(undefined, true)
+    }
+
     logger.info('Completed rolling back migrations')
   }
 }

packages/server/modules/multiregion/dbSelector.ts

@@ -24,9 +24,16 @@ import {
 } from '@/modules/multiregion/regionConfig'
 import { RegionServerConfig } from '@/modules/multiregion/domain/types'
 import { MaybeNullOrUndefined } from '@speckle/shared'
+import { isTestEnv } from '@/modules/shared/helpers/envHelper'
 
 let getter: GetProjectDb | undefined = undefined
 
+/**
+ * All dbs share the list of pubs/subs, so we need to make sure the test db uses their own.
+ * As long as there's only 1 test db per instance, it should be fine
+ */
+const createPubSubName = (name: string): string => (isTestEnv() ? `test_${name}` : name)
+
 export const getRegionDb: GetRegionDb = async ({ regionKey }) => {
   const getRegion = getRegionFactory({ db })
   const regionClients = await getRegisteredRegionClients()
@@ -87,11 +94,16 @@ export const getProjectDbClient: GetProjectDb = async ({ projectId }) => {
 type RegionClients = Record<string, Knex>
 let registeredRegionClients: RegionClients | undefined = undefined
 
-const initializeRegisteredRegionClients = async (): Promise<RegionClients> => {
+/**
+ * Idempotently initialize registered region (in db) Knex clients
+ */
+export const initializeRegisteredRegionClients = async (): Promise<RegionClients> => {
   const configuredRegions = await getRegionsFactory({ db })()
-  const regionConfigs = await getAvailableRegionConfig()
+  if (!configuredRegions.length) return {}
 
-  return Object.fromEntries(
+  // init knex clients
+  const regionConfigs = await getAvailableRegionConfig()
+  const ret = Object.fromEntries(
     configuredRegions.map((region) => {
       if (!(region.key in regionConfigs))
         throw new MisconfiguredEnvironmentError(
@@ -100,6 +112,17 @@ const initializeRegisteredRegionClients = async (): Promise<RegionClients> => {
       return [region.key, configureKnexClient(regionConfigs[region.key]).public]
     })
   )
+
+  // run migrations
+  await Promise.all(Object.values(ret).map((db) => db.migrate.latest()))
+
+  // (re-)set up pub-sub, if needed
+  await Promise.all(
+    Object.keys(ret).map((regionKey) => initializeRegion({ regionKey }))
+  )
+
+  registeredRegionClients = ret
+  return ret
 }
 
 const configureKnexClient = (
@@ -126,11 +149,10 @@ export const getRegisteredRegionClients = async (): Promise<RegionClients> => {
   return registeredRegionClients
 }
 
+/**
+ * Idempotently initialize region
+ */
 export const initializeRegion: InitializeRegion = async ({ regionKey }) => {
-  const knownClients = await getRegisteredRegionClients()
-  if (regionKey in knownClients)
-    throw new Error(`Region ${regionKey} is already initialized`)
-
   const regionConfigs = await getAvailableRegionConfig()
   if (!(regionKey in regionConfigs))
     throw new Error(`RegionKey ${regionKey} not available in config`)
@@ -138,7 +160,6 @@ export const initializeRegion: InitializeRegion = async ({ regionKey }) => {
   const newRegionConfig = regionConfigs[regionKey]
   const regionDb = configureKnexClient(newRegionConfig)
   await regionDb.public.migrate.latest()
-  // TODO, set up pub-sub shit
 
   const mainDbConfig = await getMainRegionConfig()
   const mainDb = configureKnexClient(mainDbConfig)
@@ -158,8 +179,12 @@ export const initializeRegion: InitializeRegion = async ({ regionKey }) => {
     regionName: regionKey,
     sslmode
   })
-  // pushing to the singleton object here
-  knownClients[regionKey] = regionDb.public
+
+  // pushing to the singleton object here, its only not available
+  // if this is being triggered from init, and in that case its gonna be set after anyway
+  if (registeredRegionClients) {
+    registeredRegionClients[regionKey] = regionDb.public
+  }
 }
 
 interface ReplicationArgs {
@@ -175,9 +200,11 @@ const setUpUserReplication = async ({
   sslmode,
   regionName
 }: ReplicationArgs): Promise<void> => {
-  // TODO: ensure its created...
+  const subName = createPubSubName(`userssub_${regionName}`)
+  const pubName = createPubSubName('userspub')
+
   try {
-    await from.public.raw('CREATE PUBLICATION userspub FOR TABLE users;')
+    await from.public.raw(`CREATE PUBLICATION ${pubName} FOR TABLE users;`)
   } catch (err) {
     if (!(err instanceof Error)) throw err
     if (!err.message.includes('already exists')) throw err
@@ -190,11 +217,10 @@ const setUpUserReplication = async ({
   )
   const port = fromUrl.port ? fromUrl.port : '5432'
   const fromDbName = fromUrl.pathname.replace('/', '')
-  const subName = `userssub_${regionName}`
   const rawSqeel = `SELECT * FROM aiven_extras.pg_create_subscription(
     '${subName}',
     'dbname=${fromDbName} host=${fromUrl.hostname} port=${port} sslmode=${sslmode} user=${fromUrl.username} password=${fromUrl.password}',
-    'userspub', 
+    '${pubName}', 
     '${subName}',
     TRUE,
     TRUE
@@ -214,9 +240,11 @@ const setUpProjectReplication = async ({
   regionName,
   sslmode
 }: ReplicationArgs): Promise<void> => {
-  // TODO: ensure its created...
+  const subName = createPubSubName(`projectsub_${regionName}`)
+  const pubName = createPubSubName('projectpub')
+
   try {
-    await from.public.raw('CREATE PUBLICATION projectpub FOR TABLE streams;')
+    await from.public.raw(`CREATE PUBLICATION ${pubName} FOR TABLE streams;`)
   } catch (err) {
     if (!(err instanceof Error)) throw err
     if (!err.message.includes('already exists')) throw err
@@ -229,11 +257,10 @@ const setUpProjectReplication = async ({
   )
   const port = fromUrl.port ? fromUrl.port : '5432'
   const fromDbName = fromUrl.pathname.replace('/', '')
-  const subName = `projectsub_${regionName}`
   const rawSqeel = `SELECT * FROM aiven_extras.pg_create_subscription(
     '${subName}',
     'dbname=${fromDbName} host=${fromUrl.hostname} port=${port} sslmode=${sslmode} user=${fromUrl.username} password=${fromUrl.password}',
-    'projectpub', 
+    '${pubName}', 
     '${subName}',
     TRUE,
     TRUE

packages/server/modules/multiregion/index.ts

@@ -1,5 +1,5 @@
 import { moduleLogger } from '@/logging/logging'
-import { getRegisteredRegionClients } from '@/modules/multiregion/dbSelector'
+import { initializeRegisteredRegionClients } from '@/modules/multiregion/dbSelector'
 import { isMultiRegionEnabled } from '@/modules/multiregion/helpers'
 import { SpeckleModule } from '@/modules/shared/helpers/typeHelper'
 
@@ -11,12 +11,9 @@ const multiRegion: SpeckleModule = {
     }
 
     moduleLogger.info('🌍 Init multiRegion module')
-    // this should have all the builtin checks to make sure all regions are working
-    // and no regions are missing
-    const regionClients = await getRegisteredRegionClients()
-    moduleLogger.info('Migrating region databases')
-    await Promise.all(Object.values(regionClients).map((db) => db.migrate.latest()))
-    moduleLogger.info('Migrations done')
+
+    // Init registered region clients
+    await initializeRegisteredRegionClients()
   }
 }
 

packages/server/modules/multiregion/regionConfig.ts

@@ -17,9 +17,10 @@ import { isMultiRegionEnabled } from '@/modules/multiregion/helpers'
 let multiRegionConfig: Optional<AllRegionsConfig> = undefined
 
 const getAllRegionsConfig = async (): Promise<AllRegionsConfig> => {
-  if (isDevOrTestEnv() && !isMultiRegionEnabled())
+  if (isDevOrTestEnv() && !isMultiRegionEnabled()) {
     // this should throw somehow
     return { main: { postgres: { connectionUri: '' } }, regions: {} }
+  }
   if (multiRegionConfig) return multiRegionConfig
 
   const relativePath = getMultiRegionConfigPath()

packages/server/modules/shared/helpers/envHelper.ts

@@ -418,3 +418,6 @@ export function getOtelHeaderValue() {
 export function getMultiRegionConfigPath() {
   return getStringFromEnv('MULTI_REGION_CONFIG_PATH')
 }
+
+export const shouldRunTestsInMultiregionMode = () =>
+  getBooleanFromEnv('RUN_TESTS_IN_MULTIREGION_MODE')

packages/server/multiregion.test.example.json

@@ -0,0 +1,16 @@
+{
+  "main": {
+    "postgres": {
+      "connectionUri": "postgresql://speckle:speckle@127.0.0.1:5432/speckle2_test",
+      "privateConnectionUri": "postgresql://speckle:speckle@postgres:5432/speckle2_test"
+    }
+  },
+  "regions": {
+    "region1": {
+      "postgres": {
+        "connectionUri": "postgresql://speckle:speckle@127.0.0.1:5401/speckle2_test",
+        "privateConnectionUri": "postgresql://speckle:speckle@postgres-region1:5432/speckle2_test"
+      }
+    }
+  }
+}

packages/server/package.json

@@ -23,6 +23,7 @@
     "dev:clean": "yarn build:clean && yarn dev",
     "dev:server:test": "cross-env DISABLE_NOTIFICATIONS_CONSUMPTION=true NODE_ENV=test LOG_LEVEL=silent LOG_PRETTY=true node ./bin/ts-www",
     "test": "cross-env NODE_ENV=test LOG_LEVEL=silent LOG_PRETTY=true mocha",
+    "test:multiregion": "cross-env RUN_TESTS_IN_MULTIREGION_MODE=true yarn test",
     "test:coverage": "cross-env NODE_ENV=test LOG_LEVEL=silent LOG_PRETTY=true nyc --reporter lcov mocha",
     "test:report": "yarn test:coverage -- --reporter mocha-junit-reporter --reporter-options mochaFile=reports/test-results.xml",
     "lint": "yarn lint:tsc && yarn lint:eslint",
@@ -30,6 +31,7 @@
     "lint:tsc": "tsc --noEmit",
     "lint:eslint": "eslint .",
     "cli": "cross-env LOG_LEVEL=debug LOG_PRETTY=true NODE_ENV=development ts-node ./modules/cli/index.ts",
+    "cli:test": "cross-env LOG_LEVEL=debug LOG_PRETTY=true NODE_ENV=test ts-node ./modules/cli/index.ts",
     "cli:download:commit": "cross-env LOG_PRETTY=true LOG_LEVEL=debug yarn cli download commit",
     "migrate": "yarn cli db migrate",
     "migrate:test": "cross-env NODE_ENV=test ts-node ./modules/cli/index.js db migrate",

packages/server/test/hooks.js

@@ -1,119 +0,0 @@
-require('../bootstrap')
-
-// Register global mocks as early as possible
-require('@/test/mocks/global')
-
-const chai = require('chai')
-const chaiAsPromised = require('chai-as-promised')
-const chaiHttp = require('chai-http')
-const deepEqualInAnyOrder = require('deep-equal-in-any-order')
-const { knex } = require(`@/db/knex`)
-const { init, startHttp, shutdown } = require(`@/app`)
-const { default: graphqlChaiPlugin } = require('@/test/plugins/graphql')
-const { logger } = require('@/logging/logging')
-const { once } = require('events')
-
-// Register chai plugins
-chai.use(chaiAsPromised)
-chai.use(chaiHttp)
-chai.use(deepEqualInAnyOrder)
-chai.use(graphqlChaiPlugin)
-
-const unlock = async () => {
-  const exists = await knex.schema.hasTable('knex_migrations_lock')
-  if (exists) {
-    await knex('knex_migrations_lock').update('is_locked', '0')
-  }
-}
-
-exports.truncateTables = async (tableNames) => {
-  if (!tableNames?.length) {
-    //why is server config only created once!????
-    // because its done in a migration, to not override existing configs
-    const protectedTables = ['server_config']
-    // const protectedTables = [ 'server_config', 'user_roles', 'scopes', 'server_acl' ]
-    tableNames = (
-      await knex('pg_tables')
-        .select('tablename')
-        .where({ schemaname: 'public' })
-        .whereRaw("tablename not like '%knex%'")
-        .whereNotIn('tablename', protectedTables)
-    ).map((table) => table.tablename)
-    if (!tableNames.length) return // Nothing to truncate
-
-    // We're deleting everything, so lets turn off triggers to avoid deadlocks/slowdowns
-    await knex.transaction(async (trx) => {
-      await trx.raw(`
-        -- Disable triggers and foreign key constraints for this session
-        SET session_replication_role = replica;
-        
-        truncate table ${tableNames.join(',')};
-
-        -- Re-enable triggers and foreign key constraints
-        SET session_replication_role = DEFAULT;
-      `)
-    })
-  } else {
-    await knex.raw(`truncate table ${tableNames.join(',')} cascade`)
-  }
-}
-
-/**
- * @param {import('http').Server} server
- * @param {import('express').Express} app
- */
-const initializeTestServer = async (server, app) => {
-  await startHttp(server, app, 0)
-
-  await once(app, 'appStarted')
-  const port = server.address().port
-  const serverAddress = `http://127.0.0.1:${port}`
-  const wsAddress = `ws://127.0.0.1:${port}`
-  return {
-    server,
-    serverAddress,
-    serverPort: port,
-    wsAddress,
-    sendRequest(auth, obj) {
-      return (
-        chai
-          .request(serverAddress)
-          .post('/graphql')
-          // if you set the header to null, the actual header in the req will be
-          // a string -> 'null'
-          // this is now treated as an invalid token, and gets forbidden
-          // switching to an empty string token
-          .set('Authorization', auth || '')
-          .send(obj)
-      )
-    }
-  }
-}
-
-exports.mochaHooks = {
-  beforeAll: async () => {
-    logger.info('running before all')
-    await unlock()
-    await exports.truncateTables()
-    await knex.migrate.rollback()
-    await knex.migrate.latest()
-    await init()
-  },
-  afterAll: async () => {
-    logger.info('running after all')
-    await unlock()
-    await shutdown()
-  }
-}
-
-exports.buildApp = async () => {
-  const { app, graphqlServer, server } = await init()
-  return { app, graphqlServer, server }
-}
-
-exports.beforeEachContext = async () => {
-  await exports.truncateTables()
-  return await exports.buildApp()
-}
-
-exports.initializeTestServer = initializeTestServer

packages/server/test/hooks.ts

@@ -0,0 +1,256 @@
+// eslint-disable-next-line no-restricted-imports
+import '../bootstrap'
+
+// Register global mocks as early as possible
+import '@/test/mocks/global'
+
+import chai from 'chai'
+import chaiAsPromised from 'chai-as-promised'
+import chaiHttp from 'chai-http'
+import deepEqualInAnyOrder from 'deep-equal-in-any-order'
+import { knex as mainDb } from '@/db/knex'
+import { init, startHttp, shutdown } from '@/app'
+import graphqlChaiPlugin from '@/test/plugins/graphql'
+import { logger } from '@/logging/logging'
+import { once } from 'events'
+import type http from 'http'
+import type express from 'express'
+import type net from 'net'
+import { MaybeAsync, MaybeNullOrUndefined } from '@speckle/shared'
+import type mocha from 'mocha'
+import { shouldRunTestsInMultiregionMode } from '@/modules/shared/helpers/envHelper'
+import {
+  getAvailableRegionKeysFactory,
+  getFreeRegionKeysFactory
+} from '@/modules/multiregion/services/config'
+import { getAvailableRegionConfig } from '@/modules/multiregion/regionConfig'
+import { createAndValidateNewRegionFactory } from '@/modules/multiregion/services/management'
+import {
+  getRegionFactory,
+  getRegionsFactory,
+  storeRegionFactory
+} from '@/modules/multiregion/repositories'
+import {
+  getRegisteredRegionClients,
+  initializeRegion
+} from '@/modules/multiregion/dbSelector'
+import { Knex } from 'knex'
+
+// why is server config only created once!????
+// because its done in a migration, to not override existing configs
+// similarly wiping regions will break multi region setup
+const protectedTables = ['server_config', 'regions']
+let regionClients: Record<string, Knex> = {}
+
+// Register chai plugins
+chai.use(chaiAsPromised)
+chai.use(chaiHttp)
+chai.use(deepEqualInAnyOrder)
+chai.use(graphqlChaiPlugin)
+
+const inEachDb = async (fn: (db: Knex) => MaybeAsync<void>) => {
+  await fn(mainDb)
+  for (const regionClient of Object.values(regionClients)) {
+    await fn(regionClient)
+  }
+}
+
+const setupMultiregionMode = async () => {
+  const db = mainDb
+  const getAvailableRegionKeys = getAvailableRegionKeysFactory({
+    getAvailableRegionConfig
+  })
+  const regionKeys = await getAvailableRegionKeys()
+
+  // Create DB region entries for each key
+  const createRegion = createAndValidateNewRegionFactory({
+    getFreeRegionKeys: getFreeRegionKeysFactory({
+      getAvailableRegionKeys,
+      getRegions: getRegionsFactory({ db })
+    }),
+    getRegion: getRegionFactory({ db }),
+    storeRegion: storeRegionFactory({ db }),
+    initializeRegion
+  })
+  for (const regionKey of regionKeys) {
+    await createRegion({
+      region: {
+        key: regionKey,
+        name: regionKey,
+        description: 'Auto created test region'
+      }
+    })
+  }
+
+  // Store active region clients
+  regionClients = await getRegisteredRegionClients()
+
+  // Reset each DB client (re-run all migrations and setup)
+  for (const [, regionClient] of Object.entries(regionClients)) {
+    const reset = resetSchemaFactory({ db: regionClient })
+    await reset()
+  }
+}
+
+const unlockFactory = (deps: { db: Knex }) => async () => {
+  const exists = await deps.db.schema.hasTable('knex_migrations_lock')
+  if (exists) {
+    await deps.db('knex_migrations_lock').update('is_locked', '0')
+  }
+}
+
+export const resetPubSubFactory = (deps: { db: Knex }) => async () => {
+  if (!shouldRunTestsInMultiregionMode()) {
+    return { drop: async () => {}, reenable: async () => {} }
+  }
+
+  const subscriptions = (await deps.db.raw(
+    `SELECT subname, subconninfo, subpublications, subslotname FROM aiven_extras.pg_list_all_subscriptions() WHERE subname ILIKE 'test_%';`
+  )) as {
+    rows: Array<{
+      subname: string
+      subconninfo: string
+      subpublications: string[]
+      subslotname: string
+    }>
+  }
+  const publications = (await deps.db.raw(
+    `SELECT pubname FROM pg_publication WHERE pubname ILIKE 'test_%';`
+  )) as {
+    rows: Array<{ pubname: string }>
+  }
+
+  // Drop all subs
+  for (const sub of subscriptions.rows) {
+    await deps.db.raw(`
+          SELECT * FROM aiven_extras.pg_alter_subscription_disable('${sub.subname}');
+          SELECT * FROM aiven_extras.pg_drop_subscription('${sub.subname}');
+          SELECT * FROM aiven_extras.dblink_slot_create_or_drop('${sub.subconninfo}', '${sub.subslotname}', 'drop');
+        `)
+  }
+
+  // Drop all pubs
+  for (const pub of publications.rows) {
+    await deps.db.raw(`DROP PUBLICATION ${pub.pubname};`)
+  }
+}
+
+const truncateTablesFactory = (deps: { db: Knex }) => async (tableNames?: string[]) => {
+  if (!tableNames?.length) {
+    tableNames = (
+      await deps
+        .db('pg_tables')
+        .select('tablename')
+        .where({ schemaname: 'public' })
+        .whereRaw("tablename not like '%knex%'")
+        .whereNotIn('tablename', protectedTables)
+    ).map((table: { tablename: string }) => table.tablename)
+    if (!tableNames.length) return // Nothing to truncate
+
+    // We're deleting everything, so lets turn off triggers to avoid deadlocks/slowdowns
+    await deps.db.transaction(async (trx) => {
+      await trx.raw(`
+        -- Disable triggers and foreign key constraints for this session
+        SET session_replication_role = replica;
+
+        truncate table ${tableNames?.join(',') || ''};
+
+        -- Re-enable triggers and foreign key constraints
+        SET session_replication_role = DEFAULT;
+      `)
+    })
+  } else {
+    await deps.db.raw(`truncate table ${tableNames.join(',')} cascade`)
+  }
+}
+
+const resetSchemaFactory = (deps: { db: Knex }) => async () => {
+  const resetPubSub = resetPubSubFactory(deps)
+
+  await unlockFactory(deps)()
+  await resetPubSub()
+
+  // Reset schema
+  await deps.db.migrate.rollback()
+  await deps.db.migrate.latest()
+}
+
+export const truncateTables = async (tableNames?: string[]) => {
+  const dbs = [mainDb, ...Object.values(regionClients)]
+
+  // First reset pubsubs
+  for (const db of dbs) {
+    const resetPubSub = resetPubSubFactory({ db })
+    await resetPubSub()
+  }
+
+  // Now truncate
+  for (const db of dbs) {
+    const truncate = truncateTablesFactory({ db })
+    await truncate(tableNames)
+  }
+}
+
+export const initializeTestServer = async (
+  server: http.Server,
+  app: express.Express
+) => {
+  await startHttp(server, app, 0)
+
+  await once(app, 'appStarted')
+  const port = (server.address() as net.AddressInfo).port
+  const serverAddress = `http://127.0.0.1:${port}`
+  const wsAddress = `ws://127.0.0.1:${port}`
+  return {
+    server,
+    serverAddress,
+    serverPort: port,
+    wsAddress,
+    sendRequest(auth: MaybeNullOrUndefined<string>, obj: string | object) {
+      return (
+        chai
+          .request(serverAddress)
+          .post('/graphql')
+          // if you set the header to null, the actual header in the req will be
+          // a string -> 'null'
+          // this is now treated as an invalid token, and gets forbidden
+          // switching to an empty string token
+          .set('Authorization', auth || '')
+          .send(obj)
+      )
+    }
+  }
+}
+
+export const mochaHooks: mocha.RootHookObject = {
+  beforeAll: async () => {
+    logger.info('running before all')
+
+    // Init main db
+    const reset = resetSchemaFactory({ db: mainDb })
+    await reset()
+
+    // Init (or cleanup) multi-region mode
+    await setupMultiregionMode()
+
+    // Init app
+    await init()
+  },
+  afterAll: async () => {
+    logger.info('running after all')
+    await inEachDb(async (db) => {
+      await unlockFactory({ db })()
+    })
+    await shutdown()
+  }
+}
+
+export const buildApp = async () => {
+  const { app, graphqlServer, server } = await init()
+  return { app, graphqlServer, server }
+}
+
+export const beforeEachContext = async () => {
+  await truncateTables()
+  return await buildApp()
+}