setup resource view for multi region db

docker-compose.yml

@@ -1,13 +1,40 @@
 version: "3.9"
 
 services:
-  postgres:
+  maindb:
     image: postgres:16-alpine
     ports:
       - 5454:5432
-    volumes:
-      - ./.postgres-data:/var/lib/postgresql/data
     environment:
       - POSTGRES_PASSWORD=speckle
       - POSTGRES_USER=speckle
       - POSTGRES_DB=speckle_main
+
+  eu_db:
+    image: postgres:16-alpine
+    ports:
+      - 5455:5433
+    environment:
+      - POSTGRES_PASSWORD=speckle
+      - POSTGRES_USER=speckle
+      - POSTGRES_DB=speckle_eu
+      - PGPORT=5433
+
+  us_db:
+    image: postgres:16-alpine
+    ports:
+      - 5456:5434
+    environment:
+      - POSTGRES_PASSWORD=speckle
+      - POSTGRES_USER=speckle
+      - POSTGRES_DB=speckle_us
+      - PGPORT=5434
+
+  start_dependencies:
+    image: tehkapa/docker-wait-for-dependencies
+    depends_on:
+      - maindb
+      - eu_db
+      - us_db
+    container_name: wait-for-dependencies
+    command: maindb:5432 eu_db:5433 us_db:5434

knexfile.ts

@@ -1,6 +1,43 @@
-export default {
+export const mainDBConfig = {
   client: 'pg',
-  connection: process.env.POSTGRES_URL,
+  connection: {
+    host: '127.0.0.1',
+    port: 5454,
+    user: 'speckle',
+    database: 'speckle_main',
+    password: 'speckle'
+  },
+  migrations: {
+    directory: 'src/migrations',
+    extension: 'ts'
+  }
+}
+
+
+export const euDBConfig = {
+  client: 'pg',
+  connection: {
+    host: '127.0.0.1',
+    port: 5455,
+    user: 'speckle',
+    database: 'speckle_eu',
+    password: 'speckle'
+  },
+  migrations: {
+    directory: 'src/migrations',
+    extension: 'ts'
+  }
+}
+
+export const usDBConfig = {
+  client: 'pg',
+  connection: {
+    host: '127.0.0.1',
+    port: 5456,
+    user: 'speckle',
+    database: 'speckle_us',
+    password: 'speckle'
+  },
   migrations: {
     directory: 'src/migrations',
     extension: 'ts'

package.json

@@ -6,9 +6,9 @@
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",
     "lint": "ts-standard",
-    "tsx": "tsx",
+    "tsx": "docker compose up start_dependencies && tsx watch --clear-screen=false --conditions=typescript ./src/app.ts",
     "lint:fix": "ts-standard --fix",
-    "migration:make": "NODE_OPTIONS='--loader ts-node/esm' knex migrate:make",
+    "migration:make": "NODE_OPTIONS='--import tsx' knex migrate:make",
     "dev:old": "nodemon --ext ts,graphql --exec node --inspect -r @swc/register src/bin/www.ts",
     "build": "tsc",
     "start": "node dist/app.js",
@@ -18,6 +18,7 @@
   "author": "",
   "license": "ISC",
   "devDependencies": {
+    "@ducktors/tsconfig": "^1.0.0",
     "@types/node": "^20.11.13",
     "@typescript-eslint/eslint-plugin": "^6.20.0",
     "@typescript-eslint/parser": "^6.20.0",

src/app.ts

@@ -3,7 +3,9 @@ import { resolvers } from './resolvers'
 import { startStandaloneServer } from '@apollo/server/standalone'
 import { readFileSync } from 'fs'
 import { typeDefs as scalarTypeDefs } from 'graphql-scalars'
-import { knex } from './db'
+import { getDB } from './db'
+import { Regions } from './regions'
+import { initListeners } from './listeners'
 
 const typeDefs = readFileSync('src/schema.graphql', { encoding: 'utf-8' })
 
@@ -19,19 +21,27 @@ const startServer = async (): Promise<void> => {
     listen: { port: 4000 }
   })
 
-  const plannedMigrations: Array<{ file: string }> = (
-    await knex.migrate.list()
-  )[1]
-  if (plannedMigrations.length > 0) {
-    console.log(
-      `🕰️  planning migrations: ${plannedMigrations
-        .map((m) => m.file)
-        .join(',')}`
-    )
+  const mainDB = getDB()
+  const regionDBs = Regions.map(region => getDB(region))
+
+  const databases = [mainDB, ...regionDBs]
+
+  for (const db of databases) {
+    const plannedMigrations: Array<{ file: string }> = (
+      await db.migrate.list()
+    )[1]
+    if (plannedMigrations.length > 0) {
+      console.log(
+        `🕰️  planning migrations: ${plannedMigrations
+          .map((m) => m.file)
+          .join(',')}`
+      )
+    }
+
+    await db.migrate.latest()
   }
 
-  await knex.migrate.latest()
-
+  await initListeners()
   console.log(`🚀 Server ready at: ${url}`)
 }
 

src/db.ts

@@ -1,4 +1,19 @@
-import Knex from 'knex'
-import config from '../knexfile'
+import Knex, { Knex as KnexClient } from 'knex'
+import { mainDBConfig, euDBConfig, usDBConfig } from '../knexfile'
+import { Region } from './regions'
 
-export const knex = Knex(config)
+const mainDB = Knex(mainDBConfig)
+const regionDBs = new Map<Region, KnexClient>([['eu', Knex(euDBConfig)], ['us', Knex(usDBConfig)]])
+
+export function getDB(region?: Region) {
+  if (!region) {
+    return mainDB
+  }
+
+  const db = regionDBs.get(region)
+
+  if (!db) {
+    throw new Error('Region not supported')
+  }
+  return db
+}

src/listeners.ts

@@ -0,0 +1,27 @@
+import { Knex } from "knex";
+import { getDB } from "./db";
+import { Regions } from "./regions";
+import { upsertResourceView } from "./repositories";
+
+
+export async function initListeners() {
+  for (const region of Regions) {
+    const db = getDB(region)
+    const connection = await (db.client as Knex.Client).acquireRawConnection()
+    connection.query('LISTEN notifications')
+
+    connection.on('notification', async (data: { payload: string }) => {
+      console.log(JSON.parse(data.payload))
+      await upsertResourceView(region, JSON.parse(data.payload))
+    });
+
+    // connection.on('end', (err) => {
+    //   reconnectClient(knex);
+    // })
+    connection.on('error', (err) => {
+      console.log(err);
+    })
+  }
+}
+
+

src/migrations/20240507155726_create-resource-views-table.ts

@@ -0,0 +1,16 @@
+import type { Knex } from 'knex'
+
+const tableName = 'resource_views'
+
+export async function up(knex: Knex): Promise<void> {
+  return await knex.schema.createTable(tableName, (table) => {
+    table.string('resourceId').primary()
+    table.string('region')
+    table.string('resourceName')
+    table.datetime('resourceCreatedAt')
+  })
+}
+
+export async function down(knex: Knex): Promise<void> {
+  return await knex.schema.dropTable(tableName)
+}

src/regions.ts

@@ -0,0 +1,2 @@
+export type Region = 'eu' | 'us'
+export const Regions: Region[] = ['eu', 'us']

src/repositories.ts

@@ -1,10 +1,12 @@
-import { knex } from "./db";
-import { UserRecord, Resource, ResourceAcl, Comment } from "./types";
+import { getDB } from "./db";
+import { Region } from "./regions";
+import { UserRecord, Resource, ResourceAcl, Comment, ResourceView } from "./types";
 
-const Users = () => knex<UserRecord>("users");
-const Resources = () => knex<Resource>("resources");
-const ResourceAclRepo = () => knex<ResourceAcl>("resource_acl");
-const Comments = () => knex<Comment>("comments");
+const Users = () => getDB()<UserRecord>("users");
+const Resources = (region: Region) => getDB(region)<Resource>("resources");
+const ResourceViews = () => getDB()<ResourceView>("resource_views");
+const ResourceAclRepo = () => getDB()<ResourceAcl>("resource_acl");
+const Comments = () => getDB()<Comment>("comments");
 
 export const queryUser = async (userId: string): Promise<UserRecord | null> => {
   return (await Users().where("id", "=", userId).first()) ?? null;
@@ -13,7 +15,11 @@ export const queryUser = async (userId: string): Promise<UserRecord | null> => {
 export const queryResource = async (
   resourceId: string,
 ): Promise<Resource | null> => {
-  return (await Resources().where("id", "=", resourceId).first()) ?? null;
+  const resourceLocation = await ResourceViews().where('resourceId', '=', resourceId).first()
+  if (!resourceLocation?.region) {
+    return null
+  }
+  return await Resources(resourceLocation.region).where("id", "=", resourceId).first() || null;
 };
 
 export const queryResourceAcl = async ({
@@ -45,13 +51,13 @@ export const queryResources = async ({
   limit: number;
   cursor: string | null;
 }) => {
-  const query = Resources()
-    .join("resource_acl", "resources.id", "resource_acl.resourceId")
+  const query = ResourceViews()
+    .join("resource_acl", "resource_views.resourceId", "resource_acl.resourceId")
     .where({ userId });
   if (cursor) {
-    query.andWhere("createdAt", "<", cursor);
+    query.andWhere("resourceCreatedAt", "<", cursor);
   }
-  return await query.limit(limit);
+  return query.limit(limit);
 };
 
 export const countComments = async (resourceId: string): Promise<number> => {
@@ -72,5 +78,15 @@ export const queryComments = async ({
   if (cursor) {
     query.andWhere("createdAt", "<", cursor);
   }
-  return await query.limit(limit);
+  return query.limit(limit);
 };
+
+export async function upsertResourceView(region: Region, resource: Resource) {
+  return ResourceViews().insert({
+    resourceId: resource.id,
+    resourceName: resource.name,
+    resourceCreatedAt: resource.createdAt,
+    region: region,
+  }).onConflict('resourceId')
+    .merge()
+}

src/resolvers.ts

@@ -3,7 +3,6 @@ import { getUser, getResource, getComments, getResources } from "./services";
 import { GraphQLError } from "graphql";
 import {
   Resource,
-  ResourceCollection,
   UserRecord,
   CommentCollection,
   PaginationArgs,
@@ -48,7 +47,7 @@ export const resolvers = {
       return await getResources({ userId: parent.id, ...args });
     },
   },
-  Resource: {
+  ResourceDetail: {
     async comments(
       parent: Resource,
       { limit, cursor }: PaginationArgs,

src/schema.graphql

@@ -15,6 +15,12 @@ type Resource {
   id: String!
   name: String!
   createdAt: DateTime!
+}
+
+type ResourceDetail {
+  id: String!
+  name: String!
+  createdAt: DateTime!
   comments(limit: Int! = 10, cursor: String = null): CommentCollection!
 }
 
@@ -33,5 +39,5 @@ type User {
 type Query {
   user(id: String!): User
 
-  resource(id: String!, userId: String!): Resource
+  resource(id: String!): ResourceDetail
 }

src/services.ts

@@ -15,11 +15,11 @@ import {
 } from "./types";
 
 export const getUser = async (id: string): Promise<UserRecord | null> => {
-  return await queryUser(id);
+  return queryUser(id);
 };
 
 export const getResource = async (id: string): Promise<Resource | null> => {
-  return await queryResource(id);
+  return queryResource(id);
 };
 
 interface GetResourcesArgs extends PaginationArgs {

src/types.ts

@@ -1,3 +1,5 @@
+import { Region } from "./regions";
+
 export interface Comment {
   id: string;
   userId: string;
@@ -17,7 +19,7 @@ interface Collection<T> {
   items: T[];
 }
 
-export interface CommentCollection extends Collection<Comment> {}
+export interface CommentCollection extends Collection<Comment> { }
 
 export interface Resource {
   id: string;
@@ -25,7 +27,7 @@ export interface Resource {
   createdAt: Date;
 }
 
-export interface ResourceCollection extends Collection<Resource> {}
+export interface ResourceCollection extends Collection<Resource> { }
 
 export interface UserRecord {
   id: string;
@@ -44,3 +46,10 @@ export interface ResourceAcl {
   userId: string;
   resourceId: string;
 }
+
+export interface ResourceView {
+  region: Region;
+  resourceId: string;
+  resourceName: string;
+  resourceCreatedAt: Date;
+}

tsconfig.json

@@ -1,109 +1,6 @@
 {
-  "compilerOptions": {
-    /* Visit https://aka.ms/tsconfig to read more about this file */
-
-    /* Projects */
-    // "incremental": true,                              /* Save .tsbuildinfo files to allow for incremental compilation of projects. */
-    // "composite": true,                                /* Enable constraints that allow a TypeScript project to be used with project references. */
-    // "tsBuildInfoFile": "./.tsbuildinfo",              /* Specify the path to .tsbuildinfo incremental compilation file. */
-    // "disableSourceOfProjectReferenceRedirect": true,  /* Disable preferring source files instead of declaration files when referencing composite projects. */
-    // "disableSolutionSearching": true,                 /* Opt a project out of multi-project reference checking when editing. */
-    // "disableReferencedProjectLoad": true,             /* Reduce the number of projects loaded automatically by TypeScript. */
-
-    /* Language and Environment */
-    "target": "es2016",                                  /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
-    // "lib": [],                                        /* Specify a set of bundled library declaration files that describe the target runtime environment. */
-    // "jsx": "preserve",                                /* Specify what JSX code is generated. */
-    // "experimentalDecorators": true,                   /* Enable experimental support for legacy experimental decorators. */
-    // "emitDecoratorMetadata": true,                    /* Emit design-type metadata for decorated declarations in source files. */
-    // "jsxFactory": "",                                 /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */
-    // "jsxFragmentFactory": "",                         /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
-    // "jsxImportSource": "",                            /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */
-    // "reactNamespace": "",                             /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */
-    // "noLib": true,                                    /* Disable including any library files, including the default lib.d.ts. */
-    // "useDefineForClassFields": true,                  /* Emit ECMAScript-standard-compliant class fields. */
-    // "moduleDetection": "auto",                        /* Control what method is used to detect module-format JS files. */
-
-    /* Modules */
-    "module": "commonjs",                                /* Specify what module code is generated. */
-    // "rootDir": "./",                                  /* Specify the root folder within your source files. */
-    // "moduleResolution": "node10",                     /* Specify how TypeScript looks up a file from a given module specifier. */
-    // "baseUrl": "./",                                  /* Specify the base directory to resolve non-relative module names. */
-    // "paths": {},                                      /* Specify a set of entries that re-map imports to additional lookup locations. */
-    // "rootDirs": [],                                   /* Allow multiple folders to be treated as one when resolving modules. */
-    // "typeRoots": [],                                  /* Specify multiple folders that act like './node_modules/@types'. */
-    // "types": [],                                      /* Specify type package names to be included without being referenced in a source file. */
-    // "allowUmdGlobalAccess": true,                     /* Allow accessing UMD globals from modules. */
-    // "moduleSuffixes": [],                             /* List of file name suffixes to search when resolving a module. */
-    // "allowImportingTsExtensions": true,               /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */
-    // "resolvePackageJsonExports": true,                /* Use the package.json 'exports' field when resolving package imports. */
-    // "resolvePackageJsonImports": true,                /* Use the package.json 'imports' field when resolving imports. */
-    // "customConditions": [],                           /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */
-    // "resolveJsonModule": true,                        /* Enable importing .json files. */
-    // "allowArbitraryExtensions": true,                 /* Enable importing files with any extension, provided a declaration file is present. */
-    // "noResolve": true,                                /* Disallow 'import's, 'require's or '<reference>'s from expanding the number of files TypeScript should add to a project. */
-
-    /* JavaScript Support */
-    // "allowJs": true,                                  /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */
-    // "checkJs": true,                                  /* Enable error reporting in type-checked JavaScript files. */
-    // "maxNodeModuleJsDepth": 1,                        /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */
-
-    /* Emit */
-    // "declaration": true,                              /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
-    // "declarationMap": true,                           /* Create sourcemaps for d.ts files. */
-    // "emitDeclarationOnly": true,                      /* Only output d.ts files and not JavaScript files. */
-    // "sourceMap": true,                                /* Create source map files for emitted JavaScript files. */
-    // "inlineSourceMap": true,                          /* Include sourcemap files inside the emitted JavaScript. */
-    // "outFile": "./",                                  /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */
-    // "outDir": "./",                                   /* Specify an output folder for all emitted files. */
-    // "removeComments": true,                           /* Disable emitting comments. */
-    // "noEmit": true,                                   /* Disable emitting files from a compilation. */
-    // "importHelpers": true,                            /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
-    // "importsNotUsedAsValues": "remove",               /* Specify emit/checking behavior for imports that are only used for types. */
-    // "downlevelIteration": true,                       /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
-    // "sourceRoot": "",                                 /* Specify the root path for debuggers to find the reference source code. */
-    // "mapRoot": "",                                    /* Specify the location where debugger should locate map files instead of generated locations. */
-    // "inlineSources": true,                            /* Include source code in the sourcemaps inside the emitted JavaScript. */
-    // "emitBOM": true,                                  /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
-    // "newLine": "crlf",                                /* Set the newline character for emitting files. */
-    // "stripInternal": true,                            /* Disable emitting declarations that have '@internal' in their JSDoc comments. */
-    // "noEmitHelpers": true,                            /* Disable generating custom helper functions like '__extends' in compiled output. */
-    // "noEmitOnError": true,                            /* Disable emitting files if any type checking errors are reported. */
-    // "preserveConstEnums": true,                       /* Disable erasing 'const enum' declarations in generated code. */
-    // "declarationDir": "./",                           /* Specify the output directory for generated declaration files. */
-    // "preserveValueImports": true,                     /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */
-
-    /* Interop Constraints */
-    // "isolatedModules": true,                          /* Ensure that each file can be safely transpiled without relying on other imports. */
-    // "verbatimModuleSyntax": true,                     /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */
-    // "allowSyntheticDefaultImports": true,             /* Allow 'import x from y' when a module doesn't have a default export. */
-    "esModuleInterop": true,                             /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */
-    // "preserveSymlinks": true,                         /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
-    "forceConsistentCasingInFileNames": true,            /* Ensure that casing is correct in imports. */
-
-    /* Type Checking */
-    "strict": true,                                      /* Enable all strict type-checking options. */
-    // "noImplicitAny": true,                            /* Enable error reporting for expressions and declarations with an implied 'any' type. */
-    // "strictNullChecks": true,                         /* When type checking, take into account 'null' and 'undefined'. */
-    // "strictFunctionTypes": true,                      /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */
-    // "strictBindCallApply": true,                      /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */
-    // "strictPropertyInitialization": true,             /* Check for class properties that are declared but not set in the constructor. */
-    // "noImplicitThis": true,                           /* Enable error reporting when 'this' is given the type 'any'. */
-    // "useUnknownInCatchVariables": true,               /* Default catch clause variables as 'unknown' instead of 'any'. */
-    // "alwaysStrict": true,                             /* Ensure 'use strict' is always emitted. */
-    // "noUnusedLocals": true,                           /* Enable error reporting when local variables aren't read. */
-    // "noUnusedParameters": true,                       /* Raise an error when a function parameter isn't read. */
-    // "exactOptionalPropertyTypes": true,               /* Interpret optional property types as written, rather than adding 'undefined'. */
-    // "noImplicitReturns": true,                        /* Enable error reporting for codepaths that do not explicitly return in a function. */
-    // "noFallthroughCasesInSwitch": true,               /* Enable error reporting for fallthrough cases in switch statements. */
-    // "noUncheckedIndexedAccess": true,                 /* Add 'undefined' to a type when accessed using an index. */
-    // "noImplicitOverride": true,                       /* Ensure overriding members in derived classes are marked with an override modifier. */
-    // "noPropertyAccessFromIndexSignature": true,       /* Enforces using indexed accessors for keys declared using an indexed type. */
-    // "allowUnusedLabels": true,                        /* Disable error reporting for unused labels. */
-    // "allowUnreachableCode": true,                     /* Disable error reporting for unreachable code. */
-
-    /* Completeness */
-    // "skipDefaultLibCheck": true,                      /* Skip type checking .d.ts files that are included with TypeScript. */
-    "skipLibCheck": true                                 /* Skip type checking all .d.ts files. */
-  }
+  "extends": "@ducktors/tsconfig",
+  "include": [
+    "**/*.ts"
+  ]
 }