8 Commits

Author SHA1 Message Date
Gergő Jedlicska 5e05baef6c Delete assignment.md 2024-02-12 13:19:23 +01:00
Dimitrie Stefanescu 2f370afd45 Update README.md 2024-02-12 11:28:37 +00:00
Gergő Jedlicska 2879d8b4bd docs: add project primitive descriptions 2024-02-12 11:00:27 +01:00
Gergő Jedlicska 9ea81c2a31 chore: add assignment descirption 2024-02-12 10:45:10 +01:00
Gergő Jedlicska 304109de94 docs: add project setup documentation 2024-02-12 10:44:54 +01:00
Gergő Jedlicska 1e885590cb fix: initial start of the app fails due to missing migrations in the main db 2024-02-12 10:44:42 +01:00
Gergő Jedlicska dc0108fa9c chore: more stuff 2024-02-08 17:02:56 +01:00
Gergő Jedlicska f1b8ec5691 wip: multi org multi region 2024-02-08 17:02:25 +01:00
30 changed files with 3850 additions and 3050 deletions
+2 -2
View File
@@ -1,8 +1,8 @@
.postgres-data
.postgres-*
.tool-versions
.env
.envrc
.swc
node_modules
dist
dist
+62
View File
@@ -0,0 +1,62 @@
# Multi tenancy sketches
This project is a synthetic test project for implementing multi tenancy.
Its a functional graphql api backend, that is easiest to access from the apollo graphql explorer.
The api and the explorer are available by default at `http://localhost:4000` by default,
after the app and its dependencies have been started
## Project setup
This project is using [`pnpm`](https://pnpm.io/) as its package manager.
To start the required databases or other dependencies, run `docker compose up -d`
## About Postgres setup
I had to change the `wal_level` on my local postgres instances
it is done with running the SQL command below, and restating the database server:
```sql
ALTER SYSTEM SET wal_level = logical;
```
Note: Postgres subscriptions (which we use) in the same db server don't work that easily; easiest way to get things going is to set up multiple db servers locally.
## Project description
The app has these basic concepts:
### User
A user of the system (obviously). User authn is not implemented, authz is very simplified.
### Resource
This is an abstract object representing a project, that multiple users might work on.
The notion of work on is currently implemented as the comment create action.
A resource might belong to an organization or belong to the default (null) organization.
### Comment
A text note, that belongs to a given resource, created by a user.
### Region
A geo-located data storage region, currently implemented as a PostgresSQL database server.
When providing a connection url to a region, make sure to not include a database name or any trailing `/`-s in the url.
### Organization
A collection of users and an owner of resource. Any user may create organizations.
Organizations may be granted access to any given region. That action creates a new database in the region DB server. migrates it to the latest DB schema and sets up user and resource publish and subscribe mechanisms.
## Steps to flex this POC
Using the exposed graphql explorer, you can go ahead and
- create a user
- create an organisation
- add the user to the organisation
- create regions & associate them with an organisation
- create a resource in the default organisation, or for a specific organisation & region
- etc.
+13 -18
View File
@@ -1,40 +1,35 @@
version: "3.9"
services:
maindb:
postgres:
image: postgres:16-alpine
ports:
- 5454:5432
volumes:
- ./.postgres-data:/var/lib/postgresql/data
environment:
- POSTGRES_PASSWORD=speckle
- POSTGRES_USER=speckle
- POSTGRES_DB=speckle_main
eu_db:
region-1:
image: postgres:16-alpine
ports:
- 5455:5433
- 5455:5432
volumes:
- ./.postgres-region-1:/var/lib/postgresql/data
environment:
- POSTGRES_PASSWORD=speckle
- POSTGRES_USER=speckle
- POSTGRES_DB=speckle_eu
- PGPORT=5433
- POSTGRES_DB=speckle_main
us_db:
region-2:
image: postgres:16-alpine
ports:
- 5456:5434
- 5456:5432
volumes:
- ./.postgres-region-2:/var/lib/postgresql/data
environment:
- POSTGRES_PASSWORD=speckle
- POSTGRES_USER=speckle
- POSTGRES_DB=speckle_us
- PGPORT=5434
start_dependencies:
image: ducktors/docker-wait-for-dependencies
depends_on:
- maindb
- eu_db
- us_db
container_name: wait-for-dependencies
command: maindb:5432 eu_db:5433 us_db:5434
- POSTGRES_DB=speckle_main
+5 -4
View File
@@ -4,7 +4,7 @@
"description": "",
"main": "src/app.ts",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
"test": "vitest",
"lint": "ts-standard",
"tsx": "tsx",
"lint:fix": "ts-standard --fix",
@@ -12,7 +12,7 @@
"dev:old": "nodemon --ext ts,graphql --exec node --inspect -r @swc/register src/bin/www.ts",
"build": "tsc",
"start": "node dist/app.js",
"dev": "docker compose up start_dependencies && tsx src/app.ts"
"dev": "nodemon src/app.ts"
},
"keywords": [],
"author": "",
@@ -26,11 +26,12 @@
"ts-node": "^10.9.2",
"ts-standard": "^12.0.2",
"tsx": "^4.7.0",
"typescript": "^5.3.3"
"typescript": "^5.3.3",
"vitest": "^1.2.2"
},
"dependencies": {
"@apollo/server": "^4.10.0",
"awilix": "^11.0.0",
"crypto-random-string": "^3.0.0",
"dotenv": "^16.4.1",
"graphql": "^16.8.1",
"graphql-scalars": "^1.22.4",
+2789 -2787
View File
File diff suppressed because it is too large Load Diff
+8 -37
View File
@@ -3,11 +3,7 @@ import { resolvers } from "./resolvers";
import { startStandaloneServer } from "@apollo/server/standalone";
import { readFileSync } from "fs";
import { typeDefs as scalarTypeDefs } from "graphql-scalars";
import { mainDb } from "./db";
import Knex, { Knex as KnexType } from 'knex'
import knexfile from "../knexfile";
import { container } from "./compositionRoot";
import { asValue } from "awilix";
import { migrateAll } from "./services/databaseManagement";
const typeDefs = readFileSync("src/schema.graphql", { encoding: "utf-8" });
@@ -19,42 +15,17 @@ const server = new ApolloServer({
});
const startServer = async (): Promise<void> => {
const plannedMigrations: Array<{ file: string }> = (
await mainDb.migrate.list()
)[1];
if (plannedMigrations.length > 0) {
console.log(
`🕰️ planning migrations: ${plannedMigrations
.map((m) => m.file)
.join(",")}`,
);
}
await mainDb.migrate.latest();
const regions = await mainDb('regions')
const regionalDBs = new Map<string, KnexType>(regions.map(region => [region.name, Knex({ ...knexfile, connection: region.connectionString })]))
await Promise.all(Array.from(regionalDBs.values()).map(db => db.migrate.latest())).catch(err => console.log({ err }))
container.register({
regionalDBs: asValue(regionalDBs),
})
const { url } = await startStandaloneServer(server, {
listen: { port: 3000 },
context: async ({ req }) => {
const scope = container.createScope(); // Create a scope per request
return {
container: scope,
// Add any other custom context, like user from req if needed
};
},
listen: { port: 4000 },
});
await migrateAll();
console.log(`🚀 Server ready at: ${url}`);
};
startServer()
.then()
.catch((err: Error) => {
console.log({ err });
console.log(`🔥 failed to start server ${err.message}`);
});
.catch((err: Error) =>
console.log(`🔥 failed to start server ${err.message}`),
);
-17
View File
@@ -1,17 +0,0 @@
import { asFunction, asValue, createContainer, InjectionMode } from 'awilix'
import { mainDb } from './db'
import { countComments, countResources, queryComments, queryResource, queryResourceAcl, queryUser } from './repositories'
export const container = createContainer({
injectionMode: InjectionMode.PROXY,
strict: true,
}).register({
mainDB: asValue(mainDb),
queryUser: asFunction(queryUser).scoped(),
queryResource: asFunction(queryResource).scoped(),
queryResourceAcl: asFunction(queryResourceAcl).scoped(),
countResources: asFunction(countResources).scoped(),
countComments: asFunction(countComments).scoped(),
queryComments: asFunction(queryComments).scoped(),
})
+3 -3
View File
@@ -2,8 +2,8 @@ import 'dotenv/config'
import { parseEnv } from 'znv'
import { z } from 'zod'
const config = parseEnv(process.env, {
MAIN_DB_URI: z.string().min(1)
export const { POSTGRES_URL } = parseEnv(process.env, {
POSTGRES_URL: z.string().min(1)
})
export default config
console.log([POSTGRES_URL].join(', '))
+2 -8
View File
@@ -1,10 +1,4 @@
import Knex from 'knex'
import config from './config'
import knexfile from '../knexfile'
import config from '../knexfile'
const knexConfig = {
...knexfile,
connection: config.MAIN_DB_URI,
}
export const mainDb = Knex(knexConfig)
export const knex = Knex(config)
@@ -0,0 +1,14 @@
import type { Knex } from "knex";
const tableName = "organizations";
export async function up(knex: Knex): Promise<void> {
return await knex.schema.createTable(tableName, (table) => {
table.text("id").primary();
table.text("name");
});
}
export async function down(knex: Knex): Promise<void> {
return await knex.schema.dropTable(tableName);
}
+49
View File
@@ -0,0 +1,49 @@
import type { Knex } from "knex";
const regionsTableName = "regions";
export async function up(knex: Knex): Promise<void> {
await knex.schema.createTable(regionsTableName, (table) => {
table.text("id").primary();
table.text("connectionString");
});
await knex.schema.createTable("organizations_regions", (table) => {
table
.text("organizationId")
.references("id")
.inTable("organizations")
.notNullable()
.onDelete("cascade");
table
.text("regionId")
.references("id")
.inTable("regions")
.notNullable()
.onDelete("cascade");
});
await knex.schema.createTable("resource_organization_region", (table) => {
table
.text("resourceId")
.references("id")
.inTable("resources")
.notNullable()
.onDelete("cascade");
table
.text("organizationId")
.references("id")
.inTable("organizations")
.notNullable()
.onDelete("cascade");
table
.text("regionId")
.references("id")
.inTable("regions")
.notNullable()
.onDelete("cascade");
});
}
export async function down(knex: Knex): Promise<void> {
await knex.schema.dropTable(regionsTableName);
await knex.schema.dropTable("organizations_regions");
}
@@ -0,0 +1,15 @@
import type { Knex } from "knex";
const regionsTableName = "regions";
export async function up(knex: Knex): Promise<void> {
await knex.schema.alterTable(regionsTableName, (table) => {
table.text("name").notNullable().defaultTo("region");
});
}
export async function down(knex: Knex): Promise<void> {
await knex.schema.alterTable(regionsTableName, (table) => {
table.dropColumn("name");
});
}
@@ -0,0 +1,15 @@
import type { Knex } from "knex";
const regionsTableName = "regions";
export async function up(knex: Knex): Promise<void> {
await knex.schema.alterTable(regionsTableName, (table) => {
table.text("maintenanceDb").notNullable().defaultTo("region");
});
}
export async function down(knex: Knex): Promise<void> {
await knex.schema.alterTable(regionsTableName, (table) => {
table.dropColumn("maintenanceDb");
});
}
@@ -0,0 +1,22 @@
import type { Knex } from "knex";
const tableName = "organization_acl";
export async function up(knex: Knex): Promise<void> {
return await knex.schema.createTable(tableName, (table) => {
table
.string("userId")
.references("id")
.inTable("users")
.onDelete("cascade");
table
.string("organizationId")
.references("id")
.inTable("organizations")
.onDelete("cascade");
});
}
export async function down(knex: Knex): Promise<void> {
return await knex.schema.dropTable(tableName);
}
@@ -0,0 +1,22 @@
import type { Knex } from "knex";
const tableName = "organization_resource_acl";
export async function up(knex: Knex): Promise<void> {
return await knex.schema.createTable(tableName, (table) => {
table
.string("resourceId")
.references("id")
.inTable("resources")
.onDelete("cascade");
table
.string("organizationId")
.references("id")
.inTable("organizations")
.onDelete("cascade");
});
}
export async function down(knex: Knex): Promise<void> {
return await knex.schema.dropTable(tableName);
}
@@ -0,0 +1,28 @@
import type { Knex } from "knex";
const tableName = "resource_region_organization";
export async function up(knex: Knex): Promise<void> {
return await knex.schema.createTable(tableName, (table) => {
table
.string("resourceId")
.references("id")
.inTable("resources")
.onDelete("cascade")
.primary();
table
.string("regionId")
.references("id")
.inTable("regions")
.onDelete("cascade");
table
.string("organizationId")
.references("id")
.inTable("organizations")
.onDelete("cascade");
});
}
export async function down(knex: Knex): Promise<void> {
return await knex.schema.dropTable(tableName);
}
@@ -1,16 +0,0 @@
import type { Knex } from "knex";
export async function up(knex: Knex): Promise<void> {
return await knex.schema.createTable('regions', (table) => {
table.text('id').primary()
table.text('name')
table.text('connectionString')
})
}
export async function down(knex: Knex): Promise<void> {
return await knex.schema.dropTable('regions')
}
@@ -1,16 +0,0 @@
import type { Knex } from "knex";
export async function up(knex: Knex): Promise<void> {
return await knex.schema.createTable('resource_meta', (table) => {
table.text('id').primary()
table.text('resourceId').references('id').inTable('resources')
table.text('region')
})
}
export async function down(knex: Knex): Promise<void> {
return await knex.schema.dropTable('resource_meta')
}
+171 -54
View File
@@ -1,47 +1,74 @@
import { Knex } from "knex";
import { UserRecord, Resource, ResourceAcl, Comment, ResourceMeta } from "./types";
import { knex } from "./db";
import {
UserRecord,
Resource,
ResourceAcl,
Comment,
Region,
OrganizationsRegions,
Organization,
OrganizationAcl,
OrganizationResourceAcl,
ResourceRegion,
ResourceRegionOrg,
} from "./types";
const tables = {
users: (db: Knex) => db<UserRecord>('users'),
resources: (db: Knex) => db<Resource>('resources'),
resourceAcl: (db: Knex) => db<ResourceAcl>('resource_acl'),
comments: (db: Knex) => db<Comment>('comments'),
resourceMeta: (db: Knex) => db<ResourceMeta>('resource_meta'),
}
const Users = () => knex<UserRecord>("users");
const Resources = () => knex<Resource>("resources");
const ResourceAclRepo = () => knex<ResourceAcl>("resource_acl");
export const queryUser = ({ mainDB }: { mainDB: Knex }) => async (userId: string): Promise<UserRecord | null> => {
return (await tables.users(mainDB).where("id", "=", userId).first()) ?? null;
export const queryUser = async (userId: string): Promise<UserRecord | null> => {
return (await Users().where("id", "=", userId).first()) ?? null;
};
export const queryResource = ({ mainDB, regionalDBs }: { mainDB: Knex, regionalDBs: Map<string, Knex> }) => async (
resourceId: string,
): Promise<Resource | null> => {
const resourceMeta = await tables.resourceMeta(mainDB).where({ resourceId }).first()
const regionalDB = regionalDBs.get(resourceMeta!.region)!
return (await tables.resources(regionalDB).where("id", "=", resourceId).first()) ?? null;
export const getUsersFrom = (db: Knex) => async (): Promise<UserRecord[]> => {
return await db<UserRecord>("users").select();
};
export const queryResourceAcl = ({ mainDB }: { mainDB: Knex }) => async ({
resourceId,
userId,
}: {
resourceId: string;
userId: string;
}): Promise<ResourceAcl | null> => {
return (
(await tables.resourceAcl(mainDB)
.where("userId", "=", userId)
.andWhere("resourceId", "=", resourceId)
.first()) ?? null
);
};
export const saveUserTo =
(db: Knex) =>
async (user: UserRecord): Promise<void> => {
await db<UserRecord>("users").insert(user);
};
export const countResources = ({ mainDB }: { mainDB: Knex }) => async (userId: string): Promise<number> => {
const [rawCount] = await tables.resourceAcl(mainDB).count().where({ userId });
export const saveResourceTo =
(db: Knex) =>
async (resource: Resource): Promise<void> => {
await db<Resource>("resources").insert(resource);
};
export const queryResourceFrom =
(db: Knex) =>
async (resourceId: string): Promise<Resource | null> => {
return (
(await db<Resource>("resources").where({ id: resourceId }).first()) ??
null
);
};
export const queryResourceAclFrom =
(db: Knex) =>
async ({ resourceId, userId }: ResourceAcl): Promise<ResourceAcl | null> => {
return (
(await db<ResourceAcl>("resource_acl")
.where({ userId, resourceId })
.first()) ?? null
);
};
export const saveResourceAclTo =
(db: Knex) =>
async (resourceAcl: ResourceAcl): Promise<void> => {
await db<ResourceAcl>("resource_acl").insert(resourceAcl);
};
export const countResources = async (userId: string): Promise<number> => {
const [rawCount] = await ResourceAclRepo().count().where({ userId });
return parseInt(rawCount.count as string);
};
export const queryResources = ({ mainDB, regionalDBs }: { mainDB: Knex; regionalDBs: Record<string, Knex> }) => async ({
export const queryResources = async ({
userId,
limit,
cursor,
@@ -50,8 +77,6 @@ export const queryResources = ({ mainDB, regionalDBs }: { mainDB: Knex; regional
limit: number;
cursor: string | null;
}) => {
const resourceMeta = await tables.resourceMeta(mainDB).where({ resourceId }).first()
const regionalDB = regionalDBs[resourceMeta!.region]
const query = Resources()
.join("resource_acl", "resources.id", "resource_acl.resourceId")
.where({ userId });
@@ -61,23 +86,115 @@ export const queryResources = ({ mainDB, regionalDBs }: { mainDB: Knex; regional
return await query.limit(limit);
};
export const countComments = ({ mainDB }: { mainDB: Knex }) => async (resourceId: string): Promise<number> => {
const [rawCount] = await tables.comments(mainDB).count().where({ resourceId });
return parseInt(rawCount.count as string);
};
export const countCommentsIn =
(db: Knex) =>
async (resourceId: string): Promise<number> => {
const [rawCount] = await db<Comment>("comments")
.count()
.where({ resourceId });
return parseInt(rawCount.count as string);
};
export const queryComments = ({ mainDB }: { mainDB: Knex }) => async ({
resourceId,
limit,
cursor,
}: {
resourceId: string;
limit: number;
cursor: string | null;
}): Promise<Comment[]> => {
const query = tables.comments(mainDB).where({ resourceId });
if (cursor) {
query.andWhere("createdAt", "<", cursor);
}
return await query.limit(limit);
};
export const queryCommentsFrom =
(db: Knex) =>
async ({
resourceId,
limit,
cursor,
}: {
resourceId: string;
limit: number;
cursor: string | null;
}): Promise<Comment[]> => {
const query = db<Comment>("comments").where({ resourceId });
if (cursor) {
query.andWhere("createdAt", "<", cursor);
}
return await query.limit(limit);
};
export const saveCommentTo =
(db: Knex) =>
async (comment: Comment): Promise<void> => {
await db<Comment>("comments").insert(comment);
};
export const getRegionsFrom = (db: Knex) => async (): Promise<Array<Region>> =>
await db<Region>("regions").select();
export const getRegionFrom =
(db: Knex) =>
async (id: string): Promise<Region | null> =>
(await db<Region>("regions").where({ id }).first()) ?? null;
export const getOrganizationRegionsFrom =
(db: Knex) => async (): Promise<Array<OrganizationsRegions>> =>
await db<OrganizationsRegions>("organizations_regions").select();
export const queryOrganizationRegionsFrom =
(db: Knex) =>
async ({
regionId,
organizationId,
}: OrganizationsRegions): Promise<OrganizationsRegions | null> =>
(await db<OrganizationsRegions>("organizations_regions")
.where({ regionId, organizationId })
.first()) ?? null;
export const saveRegionTo = (db: Knex) => async (region: Region) =>
await db<Region>("regions").insert(region);
export const saveOrganizationTo =
(db: Knex) => async (organization: Organization) =>
await db<Organization>("organizations").insert(organization);
export const getOrganizationFrom =
(db: Knex) =>
async (id: string): Promise<Organization | null> => {
return (
(await db<Organization>("organizations").where({ id }).first()) ?? null
);
};
export const getOrganizationsFrom =
(db: Knex) => async (): Promise<Organization[]> =>
await db<Organization>("organizations").select();
export const saveOrganizationsRegionsTo =
(db: Knex) =>
async (or: OrganizationsRegions): Promise<void> =>
await db<OrganizationsRegions>("organizations_regions").insert(or);
export const saveOrganizationAclTo =
(db: Knex) =>
async (orgAcl: OrganizationAcl): Promise<void> => {
await db<OrganizationsRegions>("organization_acl").insert(orgAcl);
};
export const queryOrganizationAclFrom =
(db: Knex) =>
async ({
userId,
organizationId,
}: OrganizationAcl): Promise<OrganizationAcl | null> =>
(await db<OrganizationAcl>("organization_acl")
.where({ userId, organizationId })
.first()) ?? null;
export const saveOrganizationResourceAclTo =
(db: Knex) =>
async (item: OrganizationResourceAcl): Promise<void> => {
await db<OrganizationResourceAcl>("organization_resource_acl").insert(item);
};
export const saveResourceRegionOrganizationTo =
(db: Knex) => async (item: ResourceRegionOrg) => {
await db<ResourceRegionOrg>("resource_region_organization").insert(item);
};
export const queryResourceRegionOrganizationFrom =
(db: Knex) =>
async (resourceId: string): Promise<ResourceRegion | null> =>
(await db<ResourceRegionOrg>("resource_region_organization")
.where({ resourceId })
.first()) ?? null;
+141 -9
View File
@@ -1,27 +1,63 @@
import { queryResource, queryResourceAcl } from "./repositories";
import { getUser, getResource, getComments, getResources } from "./services";
import {
getOrganizationsFrom,
getRegionsFrom,
queryOrganizationAclFrom,
queryOrganizationRegionsFrom,
queryResourceAclFrom,
saveOrganizationResourceAclTo,
saveOrganizationAclTo,
saveResourceAclTo,
saveResourceTo,
saveResourceRegionOrganizationTo,
saveCommentTo,
queryResourceFrom,
queryUser,
countCommentsIn,
queryCommentsFrom,
getUsersFrom,
saveUserTo,
} from "./repositories";
import { getComments } from "./services/comments";
import { createResource, getResources } from "./services/resources";
import { GraphQLError } from "graphql";
import {
Resource,
ResourceCollection,
UserRecord,
CommentCollection,
PaginationArgs,
ResourceCreateArgs,
OrganizationsRegions,
OrganizationAcl,
CommentCreateArgs,
UserCreateArgs,
} from "./types";
import {
bindRegionToOrganization,
createOrganization,
getDbClient,
getMainDbClient,
getResourceDatabaseConnection,
registerRegion,
} from "./services/databaseManagement";
import { authorizeUserOrgRegion } from "./services/authz";
import cryptoRandomString from "crypto-random-string";
// Resolvers define how to fetch the types defined in your schema.
// This resolver retrieves books from the "books" array above.
export const resolvers = {
Query: {
async user(_: unknown, args: { id: string }, ctx) {
return await ctx.container.cradle.queryUser(args.id);
async users() {
return await getUsersFrom(getMainDbClient())();
},
async user(_: unknown, args: { id: string }) {
return await queryUser(args.id);
},
async resource(
_: unknown,
args: { id: string; userId: string },
ctx
): Promise<Resource> {
const maybeAcl = await ctx.container.cradle.queryResourceAcl({
const mainDb = getMainDbClient();
const maybeAcl = await queryResourceAclFrom(mainDb)({
userId: args.userId,
resourceId: args.id,
});
@@ -35,7 +71,8 @@ export const resolvers = {
},
);
}
const maybeResource = await ctx.container.cradle.queryResource(args.id);
const db = await getResourceDatabaseConnection(args.id);
const maybeResource = await queryResourceFrom(db)(args.id);
if (maybeResource == null) {
throw new GraphQLError("Resource not found", {
extensions: { code: "RESOURCE_NOT_FOUND" },
@@ -43,6 +80,12 @@ export const resolvers = {
}
return maybeResource;
},
async organizations() {
return await getOrganizationsFrom(getMainDbClient())();
},
async regions() {
return await getRegionsFrom(getMainDbClient())();
},
},
User: {
async resources(parent: UserRecord, args: PaginationArgs) {
@@ -54,11 +97,100 @@ export const resolvers = {
parent: Resource,
{ limit, cursor }: PaginationArgs,
): Promise<CommentCollection> {
return await getComments({
const db = await getResourceDatabaseConnection(parent.id);
return await getComments(
countCommentsIn(db),
queryCommentsFrom(db),
)({
resourceId: parent.id,
limit,
cursor,
});
},
},
Mutation: {
async createUser(
_: unknown,
{ input: { name } }: { input: UserCreateArgs },
) {
const id = cryptoRandomString({ length: 10 });
await saveUserTo(getMainDbClient())({ id, name });
return id;
},
async registerRegion(
_: unknown,
args: {
name: string;
connectionString: string;
maintenanceDb: string;
},
) {
return await registerRegion(args);
},
async createOrganization(_: unknown, args: { name: string }) {
return await createOrganization(args.name);
},
async addRegionToOrganization(_: unknown, args: OrganizationsRegions) {
await bindRegionToOrganization(args);
},
async addUserToOrganization(
_: unknown,
{ input: args }: { input: OrganizationAcl },
) {
await saveOrganizationAclTo(getMainDbClient())(args);
},
async createResource(
_: unknown,
{ input: args }: { input: ResourceCreateArgs },
) {
const mainDb = getMainDbClient();
await authorizeUserOrgRegion(
queryOrganizationAclFrom(mainDb),
queryOrganizationRegionsFrom(mainDb),
)(args);
const db =
args.regionId && args.organizationId
? await getDbClient({
regionId: args.regionId,
organizationId: args.organizationId,
})
: mainDb;
const resourceId = await createResource(
saveResourceTo(db),
saveResourceAclTo(mainDb),
)(args);
if (args.organizationId) {
await saveOrganizationResourceAclTo(mainDb)({
organizationId: args.organizationId,
resourceId,
});
await saveResourceRegionOrganizationTo(mainDb)({
resourceId,
organizationId: args.organizationId,
// i know its not null here, the authz function ensures it
regionId: args.regionId!,
});
}
return resourceId;
},
async addComment(
_: unknown,
{ input: args }: { input: CommentCreateArgs },
) {
const mainDb = getMainDbClient();
const resourceAcl = await queryResourceAclFrom(mainDb)(args);
if (!resourceAcl)
throw new Error("The user doesn't have access to the given resource");
//2. get resource db client
const db = await getResourceDatabaseConnection(args.resourceId);
//3. save comment to db
const id = cryptoRandomString({ length: 10 });
const createdAt = new Date();
await saveCommentTo(db)({ id, createdAt, ...args });
return id;
},
},
};
+51
View File
@@ -30,8 +30,59 @@ type User {
resources(limit: Int! = 10, cursor: String = null): ResourceCollection!
}
type Organization {
id: String!
name: String!
}
type Region {
id: String!
name: String!
maintenanceDb: String!
}
type Query {
user(id: String!): User
users: [User!]
resource(id: String!, userId: String!): Resource
organizations: [Organization!]
regions: [Region!]
}
input ResourceCreateInput {
userId: String!
name: String!
organizationId: String = null
regionId: String = null
}
input OrganizationAcl {
userId: String!
organizationId: String!
}
input CommentInput {
userId: String!
content: String!
resourceId: String!
}
input UserCreateArgs {
name: String!
}
type Mutation {
createUser(input: UserCreateArgs!): String!
registerRegion(
name: String!
connectionString: String!
maintenanceDb: String!
): String!
createOrganization(name: String!): String!
addRegionToOrganization(organizationId: String!, regionId: String!): Boolean
addUserToOrganization(input: OrganizationAcl!): Boolean
createResource(input: ResourceCreateInput!): String!
addComment(input: CommentInput!): String!
}
-61
View File
@@ -1,61 +0,0 @@
import {
queryUser,
queryResource,
countComments,
queryComments,
countResources,
queryResources,
} from "./repositories";
import {
UserRecord,
Resource,
CommentCollection,
PaginationArgs,
ResourceCollection,
} from "./types";
export const getUser = async (id: string): Promise<UserRecord | null> => {
return await queryUser(id);
};
export const getResource = async (id: string): Promise<Resource | null> => {
return await queryResource(id);
};
interface GetResourcesArgs extends PaginationArgs {
userId: string;
}
export const getResources = async (
params: GetResourcesArgs,
): Promise<ResourceCollection> => {
const totalCount = await countResources(params.userId);
const items = await queryResources(params);
let cursor = null;
if (items.length > 0) {
cursor = items.slice(-1)[0].createdAt.toISOString();
}
return {
totalCount,
items,
cursor,
};
};
export const getComments = async (params: {
resourceId: string;
limit: number;
cursor: string | null;
}): Promise<CommentCollection> => {
// yes, i should be doing base64 de and encoding with the cursor...
const totalCount = await countComments(params.resourceId);
const items = await queryComments(params);
let cursor = null;
if (items.length > 0) {
cursor = items.slice(-1)[0].createdAt.toISOString();
}
return {
totalCount,
items,
cursor,
};
};
+26
View File
@@ -0,0 +1,26 @@
import {
OrganizationAcl,
OrganizationsRegions,
UserOrgRegionArgs,
} from "../types";
export const authorizeUserOrgRegion =
(
orgAclGetter: (params: OrganizationAcl) => Promise<OrganizationAcl | null>,
orgRegionGetter: (
params: OrganizationsRegions,
) => Promise<OrganizationsRegions | null>,
) =>
async ({ userId, regionId, organizationId }: UserOrgRegionArgs) => {
if (!organizationId && regionId)
throw new Error("public org doesn't support regions");
if (organizationId) {
if (!regionId) throw new Error("organizations can only write to regions");
const orgAcl = await orgAclGetter({ organizationId, userId });
if (!orgAcl)
throw new Error("user doesn't have access to this organization");
const orgRegion = await orgRegionGetter({ organizationId, regionId });
if (!orgRegion)
throw new Error("organization doesnt have access to this region");
}
};
+25
View File
@@ -0,0 +1,25 @@
import { CommentCollection, PaginationArgs, Comment } from "../types";
interface GetCommentsArgs extends PaginationArgs {
resourceId: string;
}
export const getComments =
(
countComments: (resourceId: string) => Promise<number>,
queryComments: (params: GetCommentsArgs) => Promise<Comment[]>,
) =>
async (params: GetCommentsArgs): Promise<CommentCollection> => {
// yes, i should be doing base64 de and encoding with the cursor...
const totalCount = await countComments(params.resourceId);
const items = await queryComments(params);
let cursor = null;
if (items.length > 0) {
cursor = items.slice(-1)[0].createdAt.toISOString();
}
return {
totalCount,
items,
cursor,
};
};
+243
View File
@@ -0,0 +1,243 @@
import { POSTGRES_URL } from "../config";
import {
getOrganizationFrom,
getOrganizationRegionsFrom,
getRegionFrom,
queryResourceRegionOrganizationFrom,
saveOrganizationTo,
saveOrganizationsRegionsTo,
saveRegionTo,
} from "../repositories";
import { OrganizationsRegions, Region } from "../types";
import knex, { Knex } from "knex";
import cryptoRandomString from "crypto-random-string";
const migrateToLatest = async (client: Knex): Promise<void> => {
const plannedMigrations: Array<{ file: string }> = (
await client.migrate.list()
)[1];
if (plannedMigrations.length > 0) {
console.log(
`🕰️ planning migrations: ${plannedMigrations
.map((m) => m.file)
.join(",")}`,
);
} else {
console.log("no migrations are planned");
}
// TODO: make sure if a migration fails, all migrations are rolled back
await client.migrate.latest();
};
export const migrateAll = async (): Promise<void> => {
await migrateToLatest(mainClient);
const databaseSchemas = await getAllDatabaseSchemaConnections();
await Promise.all([
...databaseSchemas.map(async (sc) => await migrateToLatest(sc)),
]);
// 1. get all regions from main DB
// 2. construct region specific knex clients and cache them by
// 3. structure the cache so that it accomodates client creation by resource id
// 4. get all organization regions from main DB
// 5. for in all regions for all organizations, run the migration
// 6. do not forget the migration for the main DB
//
};
const createDatabaseConfig = (connectionString: string): Knex.Config => {
return {
client: "pg",
connection: {
connectionString,
},
// connection: connectionString,
migrations: {
directory: "src/migrations",
extension: "ts",
},
};
};
const mainClient = knex(createDatabaseConfig(POSTGRES_URL));
const _connectionStore: Map<string, Knex> = new Map();
interface RegionWithMaybeOrganization {
regionId: string;
organizationId?: string | undefined;
}
const _createConnectionKey = ({
organizationId,
regionId,
}: RegionWithMaybeOrganization): string => {
return organizationId ? `${organizationId}@${regionId}` : regionId;
};
export const getDbClient = async ({
regionId,
organizationId,
}: RegionWithMaybeOrganization): Promise<Knex> => {
const connectionKey = _createConnectionKey({ organizationId, regionId });
const maybeClient = _connectionStore.get(connectionKey);
if (maybeClient) return maybeClient;
const maybeRegion = await mainClient<Region>("regions")
.select()
.where({ id: regionId })
.first();
if (!maybeRegion) throw Error(`region ${regionId} not found`);
const connectionString = organizationId
? `${maybeRegion.connectionString}/${organizationId}`
: `${maybeRegion.connectionString}/${maybeRegion.maintenanceDb}`;
const client = knex(createDatabaseConfig(connectionString));
_connectionStore.set(connectionKey, client);
return client;
};
export const getMainDbClient = (): Knex => mainClient;
export const registerRegion = async ({
name,
connectionString,
maintenanceDb,
}: {
name: string;
connectionString: string;
maintenanceDb: string;
}): Promise<string> => {
// TODO: validate the connectionString, so that the knex client can connect to it
const id = cryptoRandomString({ length: 10 });
await saveRegionTo(mainClient)({
id,
name,
connectionString,
maintenanceDb,
});
return id;
};
export const createOrganization = async (name: string): Promise<string> => {
const id = cryptoRandomString({ length: 10 });
await saveOrganizationTo(mainClient)({ id, name });
return id;
};
const createDb = async (client: Knex, name: string): Promise<void> => {
try {
await client.raw(`create database "${name}"`);
} catch (err) {
if (!(err instanceof Error)) throw err;
if (!err.message.includes("already exists")) throw err;
}
};
const setUpUserReplication = async ({
from,
to,
}: {
from: Knex;
to: Knex;
}): Promise<void> => {
// TODO: ensure its created...
const connectionString: string =
from.client.config.connection.connectionString;
try {
await from.raw("CREATE PUBLICATION userspub FOR TABLE users;");
} catch (err) {
if (!(err instanceof Error)) throw err;
if (!err.message.includes("already exists")) throw err;
}
try {
const toUrl = new URL(to.client.config.connection.connectionString);
await to.raw(
`CREATE SUBSCRIPTION userssub_${toUrl.pathname.replace("/", "")} CONNECTION '${connectionString}' PUBLICATION userspub;`,
);
} catch (err) {
if (!(err instanceof Error)) throw err;
if (!err.message.includes("already exists")) throw err;
}
};
const setUpResourceReplication = async ({
from,
fromRegionName,
to,
}: {
from: Knex;
fromRegionName: string;
to: Knex;
}): Promise<void> => {
// TODO: ensure its created...
const connectionString: string =
from.client.config.connection.connectionString;
const connUrl = new URL(connectionString);
try {
await from.raw("CREATE PUBLICATION resourcepub FOR TABLE resources;");
} catch (err) {
if (!(err instanceof Error)) throw err;
if (!err.message.includes("already exists")) throw err;
}
try {
await to.raw(
`CREATE SUBSCRIPTION "resroucesub_${fromRegionName.replace(
" ",
"",
)}_${connUrl.pathname.replace(
"/",
"",
)}" CONNECTION '${connectionString}' PUBLICATION resourcepub;`,
);
} catch (err) {
if (!(err instanceof Error)) throw err;
if (!err.message.includes("already exists")) throw err;
}
};
export const bindRegionToOrganization = async ({
regionId,
organizationId,
}: OrganizationsRegions): Promise<void> => {
const region = await getRegionFrom(mainClient)(regionId);
if (!region) throw Error(`region ${regionId} not found`);
const organization = await getOrganizationFrom(mainClient)(organizationId);
if (!organization) throw Error(`organization ${organizationId} not found`);
const regionClient = await getDbClient({ regionId });
await createDb(regionClient, organizationId);
const client = await getDbClient({ organizationId, regionId });
const connectionKey = _createConnectionKey({ organizationId, regionId });
await migrateToLatest(client);
await setUpUserReplication({ from: mainClient, to: client });
await setUpResourceReplication({
from: client,
fromRegionName: region.name,
to: mainClient,
});
_connectionStore.set(connectionKey, client);
await saveOrganizationsRegionsTo(mainClient)({ organizationId, regionId });
};
export const getAllDatabaseSchemaConnections = async (): Promise<Knex[]> => {
const organizationRegions = await getOrganizationRegionsFrom(mainClient)();
const clients = await Promise.all(
organizationRegions.map(async (or) => {
const client = await getDbClient(or);
return client;
}),
);
return [mainClient, ...clients];
};
export const getResourceDatabaseConnection = async (
resourceId: string,
): Promise<Knex> => {
const resourceRegionOrg =
await queryResourceRegionOrganizationFrom(mainClient)(resourceId);
return resourceRegionOrg ? await getDbClient(resourceRegionOrg) : mainClient;
};
+45
View File
@@ -0,0 +1,45 @@
import cryptoRandomString from "crypto-random-string";
import { countResources, queryResources } from "../repositories";
import {
Resource,
PaginationArgs,
ResourceCollection,
ResourceCreateArgs,
ResourceAcl,
} from "../types";
interface GetResourcesArgs extends PaginationArgs {
userId: string;
}
export const getResources = async (
params: GetResourcesArgs,
): Promise<ResourceCollection> => {
const totalCount = await countResources(params.userId);
const items = await queryResources(params);
let cursor = null;
if (items.length > 0) {
cursor = items.slice(-1)[0].createdAt.toISOString();
}
return {
totalCount,
items,
cursor,
};
};
export const createResource =
(
resourceSaver: (resource: Resource) => Promise<void>,
resourceAclSaver: (resourceAcl: ResourceAcl) => Promise<void>,
) =>
async ({ userId, name }: ResourceCreateArgs): Promise<string> => {
//1. if no org, create project in main region, validate that, regionId is null
//2. if org, validate if user has access to the org
//3. if org and region, validate if org has access to region
//4. create resource
const id = cryptoRandomString({ length: 10 });
const resource = { id, name, createdAt: new Date() };
await resourceSaver(resource);
await resourceAclSaver({ resourceId: id, userId });
return id;
};
+56 -11
View File
@@ -1,11 +1,14 @@
export interface Comment {
id: string;
export interface CommentCreateArgs {
userId: string;
content: string;
createdAt: Date;
resourceId: string;
}
export interface Comment extends CommentCreateArgs {
id: string;
createdAt: Date;
}
export interface PaginationArgs {
limit: number;
cursor: string | null;
@@ -17,22 +20,34 @@ interface Collection<T> {
items: T[];
}
export interface CommentCollection extends Collection<Comment> { }
export interface CommentCollection extends Collection<Comment> {}
export interface UserOrgRegionArgs {
userId: string;
organizationId: string | null;
regionId: string | null;
}
export interface ResourceCreateArgs extends UserOrgRegionArgs {
name: string;
}
export interface Resource {
id: string;
name: string;
createdAt: Date;
region: string
}
export interface ResourceCollection extends Collection<Resource> { }
export interface ResourceCollection extends Collection<Resource> {}
export interface UserRecord {
id: string;
export interface UserCreateArgs {
name: string;
}
export interface UserRecord extends UserCreateArgs {
id: string;
}
export interface User extends UserRecord {
resources: {
cursor: string | null;
@@ -46,8 +61,38 @@ export interface ResourceAcl {
resourceId: string;
}
export interface ResourceMeta {
id: string
region: string;
export interface Region {
id: string;
name: string;
connectionString: string;
maintenanceDb: string;
}
export interface Organization {
id: string;
name: string;
}
export interface OrganizationAcl {
userId: string;
organizationId: string;
}
export interface OrganizationsRegions {
organizationId: string;
regionId: string;
}
export interface OrganizationResourceAcl {
organizationId: string;
resourceId: string;
}
export interface ResourceRegion {
resourceId: string;
regionId: string;
}
export interface ResourceRegionOrg extends ResourceRegion {
organizationId: string;
}
+29
View File
@@ -0,0 +1,29 @@
import { expect, beforeAll, describe, it } from "vitest";
import {
getOrganizationRegionsFrom,
getRegionsFrom,
} from "../../src/repositories";
import {
getMainDbClient,
migrateAll,
} from "../../src/services/databaseManagement";
import { Knex } from "knex";
describe("regions", () => {
let dbClient: Knex;
beforeAll(async () => {
dbClient = await getMainDbClient();
});
it("gets all regions", async () => {
const regions = await getRegionsFrom(dbClient)();
expect(regions.length).toBeGreaterThan(0);
});
it("gets organizations regions", async () => {
const orgRegions = await getOrganizationRegionsFrom(dbClient)();
expect(orgRegions.length).toBeGreaterThan(0);
});
it("migrates all", async () => {
await migrateAll();
});
});
+7 -7
View File
@@ -11,7 +11,7 @@
// "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */
/* Language and Environment */
"target": "es2016", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
"target": "es2016" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
// "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */
// "jsx": "preserve", /* Specify what JSX code is generated. */
// "experimentalDecorators": true, /* Enable experimental support for legacy experimental decorators. */
@@ -25,7 +25,7 @@
// "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */
/* Modules */
"module": "commonjs", /* Specify what module code is generated. */
"module": "commonjs" /* Specify what module code is generated. */,
// "rootDir": "./", /* Specify the root folder within your source files. */
// "moduleResolution": "node10", /* Specify how TypeScript looks up a file from a given module specifier. */
// "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */
@@ -77,12 +77,12 @@
// "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */
// "verbatimModuleSyntax": true, /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */
// "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */
"esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */
"esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */,
// "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
"forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */
"forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */,
/* Type Checking */
"strict": true, /* Enable all strict type-checking options. */
"strict": true /* Enable all strict type-checking options. */,
// "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */
// "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */
// "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */
@@ -104,6 +104,6 @@
/* Completeness */
// "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */
"skipLibCheck": true /* Skip type checking all .d.ts files. */
}
"skipLibCheck": true /* Skip type checking all .d.ts files. */,
},
}
+7
View File
@@ -0,0 +1,7 @@
import { defineConfig } from "vitest/config";
export default defineConfig({
test: {
dir: "tests",
},
});