feat(regions): repo functions for copying project branches and commits

This commit is contained in:
Chuck Driesler
2025-01-12 00:03:14 +00:00
parent b98160f037
commit 4d5f96bb8e
6 changed files with 751 additions and 595 deletions
@@ -308,3 +308,11 @@ export type GetWorkspaceCreationState = (params: {
export type UpsertWorkspaceCreationState = (params: {
workspaceCreationState: WorkspaceCreationState
}) => Promise<void>
/**
* Project regions
*/
export type CopyProjects = (params: { projectIds: string[] }) => Promise<string[]>
export type CopyProjectModels = (params: { projectIds: string[] }) => Promise<Record<string, string[]>>
export type CopyProjectVersions = (params: { projectIds: string[] }) => Promise<Record<string, string[]>>
@@ -1,7 +1,15 @@
import { buildTableHelper } from '@/modules/core/dbSchema'
import { BranchCommits, Branches, buildTableHelper, Commits, StreamCommits, StreamFavorites, Streams, StreamsMeta } from '@/modules/core/dbSchema'
import { Branch } from '@/modules/core/domain/branches/types'
import { Commit } from '@/modules/core/domain/commits/types'
import { Stream } from '@/modules/core/domain/streams/types'
import { BranchCommitRecord, StreamCommitRecord, StreamFavoriteRecord } from '@/modules/core/helpers/types'
import { RegionRecord } from '@/modules/multiregion/helpers/types'
import { Regions } from '@/modules/multiregion/repositories'
import { executeBatchedSelect } from '@/modules/shared/helpers/dbHelper'
import {
CopyProjectModels,
CopyProjects,
CopyProjectVersions,
GetDefaultRegion,
UpsertRegionAssignment
} from '@/modules/workspaces/domain/operations'
@@ -15,32 +23,156 @@ export const WorkspaceRegions = buildTableHelper('workspace_regions', [
const tables = {
workspaceRegions: (db: Knex) => db<WorkspaceRegionAssignment>(WorkspaceRegions.name),
regions: (db: Knex) => db<RegionRecord>(Regions.name)
regions: (db: Knex) => db<RegionRecord>(Regions.name),
projects: (db: Knex) => db<Stream>(Streams.name),
models: (db: Knex) => db<Branch>(Branches.name),
versions: (db: Knex) => db<Commit>(Commits.name),
branchCommits: (db: Knex) => db<BranchCommitRecord>(BranchCommits.name),
streamCommits: (db: Knex) => db<StreamCommitRecord>(StreamCommits.name),
streamFavorites: (db: Knex) => db<StreamFavoriteRecord>(StreamFavorites.name),
streamsMeta: (db: Knex) => db(StreamsMeta.name)
}
export const upsertRegionAssignmentFactory =
(deps: { db: Knex }): UpsertRegionAssignment =>
async (params) => {
const { workspaceId, regionKey } = params
const [row] = await tables
.workspaceRegions(deps.db)
.insert({ workspaceId, regionKey }, '*')
.onConflict(['workspaceId', 'regionKey'])
.merge()
async (params) => {
const { workspaceId, regionKey } = params
const [row] = await tables
.workspaceRegions(deps.db)
.insert({ workspaceId, regionKey }, '*')
.onConflict(['workspaceId', 'regionKey'])
.merge()
return row
}
return row
}
export const getDefaultRegionFactory =
(deps: { db: Knex }): GetDefaultRegion =>
async (params) => {
const { workspaceId } = params
const row = await tables
.regions(deps.db)
.select<RegionRecord>(Regions.cols)
.join(WorkspaceRegions.name, WorkspaceRegions.col.regionKey, Regions.col.key)
.where({ [WorkspaceRegions.col.workspaceId]: workspaceId })
.first()
async (params) => {
const { workspaceId } = params
const row = await tables
.regions(deps.db)
.select<RegionRecord>(Regions.cols)
.join(WorkspaceRegions.name, WorkspaceRegions.col.regionKey, Regions.col.key)
.where({ [WorkspaceRegions.col.workspaceId]: workspaceId })
.first()
return row
}
export const copyProjects =
(deps: { sourceDb: Knex, targetDb: Knex }): CopyProjects =>
async ({ projectIds }) => {
const selectProjects = tables.projects(deps.sourceDb).select('*').whereIn(Streams.col.id, projectIds)
const copiedProjectIds: string[] = []
// Copy project record
for await (const projects of executeBatchedSelect(selectProjects)) {
for (const project of projects) {
// Store copied project id
copiedProjectIds.push(project.id)
// Copy `streams` row to target db
await tables.projects(deps.targetDb)
.insert(project)
.onConflict()
.ignore()
}
const projectIds = projects.map((project) => project.id)
// Fetch `stream_favorites` rows for projects in batch
const selectStreamFavorites = tables.streamFavorites(deps.sourceDb).select('*').whereIn(StreamFavorites.col.streamId, projectIds)
for await (const streamFavorites of executeBatchedSelect(selectStreamFavorites)) {
for (const streamFavorite of streamFavorites) {
// Copy `stream_favorites` row to target db
await tables.streamFavorites(deps.targetDb).insert(streamFavorite).onConflict().ignore()
}
}
// Fetch `streams_meta` rows for projects in batch
const selectStreamsMetadata = tables.streamsMeta(deps.sourceDb).select('*').whereIn(StreamsMeta.col.streamId, projectIds)
for await (const streamsMetadataBatch of executeBatchedSelect(selectStreamsMetadata)) {
for (const streamMetadata of streamsMetadataBatch) {
// Copy `streams_meta` row to target db
await tables.streamsMeta(deps.targetDb).insert(streamMetadata).onConflict().ignore()
}
}
}
return copiedProjectIds
}
export const copyProjectModels =
(deps: { sourceDb: Knex, targetDb: Knex }): CopyProjectModels =>
async ({ projectIds }) => {
const copiedModelIds: Record<string, string[]> = projectIds.reduce((result, id) => ({ ...result, [id]: [] }), {})
for (const projectId of projectIds) {
const selectModels = tables.models(deps.sourceDb).select('*').where({ streamId: projectId })
for await (const models of executeBatchedSelect(selectModels)) {
for (const model of models) {
// Store copied model ids
copiedModelIds[projectId].push(model.id)
// Copy `branches` row to target db
await tables.models(deps.targetDb).insert(model).onConflict().ignore()
}
}
}
return copiedModelIds
}
export const copyProjectVersions =
(deps: { sourceDb: Knex, targetDb: Knex }): CopyProjectVersions =>
async ({ projectIds }) => {
const copiedVersionIds: Record<string, string[]> = projectIds.reduce((result, id) => ({ ...result, [id]: [] }), {})
for (const projectId of projectIds) {
const selectVersions = tables.streamCommits(deps.sourceDb).select('*')
.join<StreamCommitRecord & Commit>(Commits.name, Commits.col.id, StreamCommits.col.commitId)
.where({ streamId: projectId })
for await (const versions of executeBatchedSelect(selectVersions)) {
for (const version of versions) {
const { commitId, ...commit } = version
// Store copied version id
copiedVersionIds[projectId].push(commitId)
// Copy `commits` row to target db
await tables.versions(deps.targetDb).insert(commit).onConflict().ignore()
}
const commitIds = versions.map((version) => version.commitId)
// Fetch `branch_commits` rows for versions in batch
const selectBranchCommits = tables.branchCommits(deps.sourceDb).select('*').whereIn(BranchCommits.col.commitId, commitIds)
for await (const branchCommits of executeBatchedSelect(selectBranchCommits)) {
for (const branchCommit of branchCommits) {
// Copy `branch_commits` row to target db
await tables.branchCommits(deps.targetDb).insert(branchCommit).onConflict().ignore()
}
}
// Fetch `stream_commits` rows for versions in batch
const selectStreamCommits = tables.streamCommits(deps.sourceDb).select('*').whereIn(StreamCommits.col.commitId, commitIds)
for await (const streamCommits of executeBatchedSelect(selectStreamCommits)) {
for (const streamCommit of streamCommits) {
// Copy `stream_commits` row to target db
await tables.streamCommits(deps.targetDb).insert(streamCommit).onConflict().ignore()
}
}
}
}
return copiedVersionIds
}
return row
}
@@ -1,5 +1,8 @@
import { StreamRecord } from '@/modules/core/helpers/types'
import {
CopyProjectModels,
CopyProjects,
CopyProjectVersions,
GetDefaultRegion,
GetWorkspace,
GetWorkspaceRoleForUser,
@@ -97,23 +100,23 @@ type GetWorkspaceProjectsReturnValue = {
export const getWorkspaceProjectsFactory =
({ getStreams }: { getStreams: GetUserStreamsPage }) =>
async (
args: GetWorkspaceProjectsArgs,
opts: GetWorkspaceProjectsOptions
): Promise<GetWorkspaceProjectsReturnValue> => {
const { streams, cursor } = await getStreams({
cursor: opts.cursor,
limit: opts.limit || 25,
searchQuery: opts.filter?.search || undefined,
workspaceId: args.workspaceId,
userId: opts.filter.userId
})
async (
args: GetWorkspaceProjectsArgs,
opts: GetWorkspaceProjectsOptions
): Promise<GetWorkspaceProjectsReturnValue> => {
const { streams, cursor } = await getStreams({
cursor: opts.cursor,
limit: opts.limit || 25,
searchQuery: opts.filter?.search || undefined,
workspaceId: args.workspaceId,
userId: opts.filter.userId
})
return {
items: streams,
cursor
return {
items: streams,
cursor
}
}
}
type MoveProjectToWorkspaceArgs = {
projectId: string
@@ -138,66 +141,78 @@ export const moveProjectToWorkspaceFactory =
getWorkspaceRoleToDefaultProjectRoleMapping: GetWorkspaceRoleToDefaultProjectRoleMapping
updateWorkspaceRole: UpdateWorkspaceRole
}) =>
async ({
projectId,
workspaceId
}: MoveProjectToWorkspaceArgs): Promise<StreamRecord> => {
const project = await getProject({ projectId })
async ({
projectId,
workspaceId
}: MoveProjectToWorkspaceArgs): Promise<StreamRecord> => {
const project = await getProject({ projectId })
if (!project) throw new ProjectNotFoundError()
if (project.workspaceId?.length) {
// We do not currently support moving projects between workspaces
throw new WorkspaceInvalidProjectError(
'Specified project already belongs to a workspace. Moving between workspaces is not yet supported.'
)
}
// Update roles for current project members
const projectTeam = await getProjectCollaborators({ projectId })
const workspaceTeam = await getWorkspaceRoles({ workspaceId })
const defaultProjectRoleMapping = await getWorkspaceRoleToDefaultProjectRoleMapping(
{ workspaceId }
)
for (const projectMembers of chunk(projectTeam, 5)) {
await Promise.all(
projectMembers.map(
async ({ id: userId, role: serverRole, streamRole: currentProjectRole }) => {
// Update workspace role. Prefer existing workspace role if there is one.
const currentWorkspaceRole = workspaceTeam.find(
(role) => role.userId === userId
)
const nextWorkspaceRole = currentWorkspaceRole ?? {
userId,
workspaceId,
role:
serverRole === Roles.Server.Guest
? Roles.Workspace.Guest
: Roles.Workspace.Member,
createdAt: new Date()
}
await updateWorkspaceRole(nextWorkspaceRole)
// Update project role. Prefer default workspace project role if more permissive.
const defaultProjectRole =
defaultProjectRoleMapping[nextWorkspaceRole.role] ?? Roles.Stream.Reviewer
const nextProjectRole = orderByWeight(
[currentProjectRole, defaultProjectRole],
coreUserRoles
)[0]
await upsertProjectRole({
userId,
projectId,
role: nextProjectRole.name as StreamRoles
})
}
if (!project) throw new ProjectNotFoundError()
if (project.workspaceId?.length) {
// We do not currently support moving projects between workspaces
throw new WorkspaceInvalidProjectError(
'Specified project already belongs to a workspace. Moving between workspaces is not yet supported.'
)
}
// Update roles for current project members
const projectTeam = await getProjectCollaborators({ projectId })
const workspaceTeam = await getWorkspaceRoles({ workspaceId })
const defaultProjectRoleMapping = await getWorkspaceRoleToDefaultProjectRoleMapping(
{ workspaceId }
)
for (const projectMembers of chunk(projectTeam, 5)) {
await Promise.all(
projectMembers.map(
async ({ id: userId, role: serverRole, streamRole: currentProjectRole }) => {
// Update workspace role. Prefer existing workspace role if there is one.
const currentWorkspaceRole = workspaceTeam.find(
(role) => role.userId === userId
)
const nextWorkspaceRole = currentWorkspaceRole ?? {
userId,
workspaceId,
role:
serverRole === Roles.Server.Guest
? Roles.Workspace.Guest
: Roles.Workspace.Member,
createdAt: new Date()
}
await updateWorkspaceRole(nextWorkspaceRole)
// Update project role. Prefer default workspace project role if more permissive.
const defaultProjectRole =
defaultProjectRoleMapping[nextWorkspaceRole.role] ?? Roles.Stream.Reviewer
const nextProjectRole = orderByWeight(
[currentProjectRole, defaultProjectRole],
coreUserRoles
)[0]
await upsertProjectRole({
userId,
projectId,
role: nextProjectRole.name as StreamRoles
})
}
)
)
}
// Assign project to workspace
return await updateProject({ projectUpdate: { id: projectId, workspaceId } })
}
// Assign project to workspace
return await updateProject({ projectUpdate: { id: projectId, workspaceId } })
}
export const moveProjectToRegionFactory =
(deps: {
copyProjects: CopyProjects,
copyProjectModels: CopyProjectModels,
copyProjectVersions: CopyProjectVersions
}) =>
async (args: { projectId: string }): Promise<void> => {
const projectIds = await deps.copyProjects({ projectIds: [args.projectId] })
const modelIdsByProjectId = await deps.copyProjectModels({ projectIds })
const versionIdsByProjectId = await deps.copyProjectVersions({ projectIds })
}
export const getWorkspaceRoleToDefaultProjectRoleMappingFactory =
({
@@ -205,19 +220,19 @@ export const getWorkspaceRoleToDefaultProjectRoleMappingFactory =
}: {
getWorkspace: GetWorkspace
}): GetWorkspaceRoleToDefaultProjectRoleMapping =>
async ({ workspaceId }) => {
const workspace = await getWorkspace({ workspaceId })
async ({ workspaceId }) => {
const workspace = await getWorkspace({ workspaceId })
if (!workspace) {
throw new WorkspaceNotFoundError()
}
if (!workspace) {
throw new WorkspaceNotFoundError()
}
return {
[Roles.Workspace.Guest]: null,
[Roles.Workspace.Member]: workspace.defaultProjectRole,
[Roles.Workspace.Admin]: Roles.Stream.Owner
return {
[Roles.Workspace.Guest]: null,
[Roles.Workspace.Member]: workspace.defaultProjectRole,
[Roles.Workspace.Admin]: Roles.Stream.Owner
}
}
}
export const updateWorkspaceProjectRoleFactory =
({
@@ -229,63 +244,63 @@ export const updateWorkspaceProjectRoleFactory =
getWorkspaceRoleForUser: GetWorkspaceRoleForUser
updateStreamRoleAndNotify: UpdateStreamRole
}): UpdateWorkspaceProjectRole =>
async ({ role, updater }) => {
const { workspaceId } = (await getStream({ streamId: role.projectId })) ?? {}
if (!workspaceId) throw new WorkspaceInvalidProjectError()
async ({ role, updater }) => {
const { workspaceId } = (await getStream({ streamId: role.projectId })) ?? {}
if (!workspaceId) throw new WorkspaceInvalidProjectError()
const currentWorkspaceRole = await getWorkspaceRoleForUser({
workspaceId,
userId: role.userId
})
const currentWorkspaceRole = await getWorkspaceRoleForUser({
workspaceId,
userId: role.userId
})
if (currentWorkspaceRole?.role === Roles.Workspace.Admin) {
// User is workspace admin and cannot have their project roles changed
throw new WorkspaceAdminError()
if (currentWorkspaceRole?.role === Roles.Workspace.Admin) {
// User is workspace admin and cannot have their project roles changed
throw new WorkspaceAdminError()
}
if (
currentWorkspaceRole?.role === Roles.Workspace.Guest &&
role.role === Roles.Stream.Owner
) {
// Workspace guests cannot be project owners
throw new WorkspaceInvalidRoleError('Workspace guests cannot be project owners.')
}
return await updateStreamRoleAndNotify(
role,
updater.userId!,
updater.resourceAccessRules
)
}
if (
currentWorkspaceRole?.role === Roles.Workspace.Guest &&
role.role === Roles.Stream.Owner
) {
// Workspace guests cannot be project owners
throw new WorkspaceInvalidRoleError('Workspace guests cannot be project owners.')
}
return await updateStreamRoleAndNotify(
role,
updater.userId!,
updater.resourceAccessRules
)
}
export const createWorkspaceProjectFactory =
(deps: { getDefaultRegion: GetDefaultRegion }) =>
async (params: { input: WorkspaceProjectCreateInput; ownerId: string }) => {
const { input, ownerId } = params
const workspaceDefaultRegion = await deps.getDefaultRegion({
workspaceId: input.workspaceId
})
const regionKey = workspaceDefaultRegion?.key
const projectDb = await getDb({ regionKey })
const db = mainDb
async (params: { input: WorkspaceProjectCreateInput; ownerId: string }) => {
const { input, ownerId } = params
const workspaceDefaultRegion = await deps.getDefaultRegion({
workspaceId: input.workspaceId
})
const regionKey = workspaceDefaultRegion?.key
const projectDb = await getDb({ regionKey })
const db = mainDb
// todo, use the command factory here, but for that, we need to migrate to the event bus
// deps not injected to ensure proper DB injection
const createNewProject = createNewProjectFactory({
storeProject: storeProjectFactory({ db: projectDb }),
getProject: getProjectFactory({ db }),
deleteProject: deleteProjectFactory({ db: projectDb }),
storeModel: storeModelFactory({ db: projectDb }),
// THIS MUST GO TO THE MAIN DB
storeProjectRole: storeProjectRoleFactory({ db }),
projectsEventsEmitter: ProjectsEmitter.emit
})
// todo, use the command factory here, but for that, we need to migrate to the event bus
// deps not injected to ensure proper DB injection
const createNewProject = createNewProjectFactory({
storeProject: storeProjectFactory({ db: projectDb }),
getProject: getProjectFactory({ db }),
deleteProject: deleteProjectFactory({ db: projectDb }),
storeModel: storeModelFactory({ db: projectDb }),
// THIS MUST GO TO THE MAIN DB
storeProjectRole: storeProjectRoleFactory({ db }),
projectsEventsEmitter: ProjectsEmitter.emit
})
const project = await createNewProject({
...input,
regionKey,
ownerId
})
const project = await createNewProject({
...input,
regionKey,
ownerId
})
return project
}
return project
}