diff --git a/packages/server/modules/accessrequests/tests/streamAccessRequests.spec.ts b/packages/server/modules/accessrequests/tests/streamAccessRequests.spec.ts index 863c70c01..3806adb46 100644 --- a/packages/server/modules/accessrequests/tests/streamAccessRequests.spec.ts +++ b/packages/server/modules/accessrequests/tests/streamAccessRequests.spec.ts @@ -176,7 +176,7 @@ describe('Stream access requests', () => { ]) apollo = { apollo: await buildApolloServer(), - context: createAuthedTestContext(me.id) + context: await createAuthedTestContext(me.id) } notificationsStateManager = buildNotificationsStateTracker() }) diff --git a/packages/server/modules/auth/tests/integration/registration.spec.ts b/packages/server/modules/auth/tests/integration/registration.spec.ts index 49d4e037f..4af072c44 100644 --- a/packages/server/modules/auth/tests/integration/registration.spec.ts +++ b/packages/server/modules/auth/tests/integration/registration.spec.ts @@ -165,7 +165,7 @@ describe('Server registration', () => { streamId: basicAdminStream.id }, { - context: createTestContext({ + context: await createTestContext({ userId: newUser.id, auth: true, role: Roles.Server.User, diff --git a/packages/server/modules/automate/tests/automations.spec.ts b/packages/server/modules/automate/tests/automations.spec.ts index 7fdd7ffab..d13a26693 100644 --- a/packages/server/modules/automate/tests/automations.spec.ts +++ b/packages/server/modules/automate/tests/automations.spec.ts @@ -578,7 +578,7 @@ const buildAutomationUpdate = () => { ) apollo = await testApolloServer({ - context: createTestContext({ + context: await createTestContext({ userId: me.id, token: 'abc', role: Roles.Server.User diff --git a/packages/server/modules/blobstorage/tests/blobstorage.graph.spec.js b/packages/server/modules/blobstorage/tests/blobstorage.graph.spec.js index f06aabeae..17e189d8f 100644 --- a/packages/server/modules/blobstorage/tests/blobstorage.graph.spec.js +++ b/packages/server/modules/blobstorage/tests/blobstorage.graph.spec.js @@ -149,7 +149,7 @@ describe('Blobs graphql @blobstorage', () => { user.id = await createUser(user) graphqlServer = { apollo: await buildApolloServer(), - context: createAuthedTestContext(user.id) + context: await createAuthedTestContext(user.id) } }) diff --git a/packages/server/modules/comments/tests/comments.graph.spec.js b/packages/server/modules/comments/tests/comments.graph.spec.js index 4bbb22c12..33b9f4b35 100644 --- a/packages/server/modules/comments/tests/comments.graph.spec.js +++ b/packages/server/modules/comments/tests/comments.graph.spec.js @@ -1129,10 +1129,10 @@ describe('Graphql @comments', () => { apollo = { apollo: await buildApolloServer(), context: user - ? createAuthedTestContext(user.id, { + ? await createAuthedTestContext(user.id, { ...(user.role ? { role: user.role } : {}) }) - : createTestContext() + : await createTestContext() } if (user && stream.role) { diff --git a/packages/server/modules/comments/tests/comments.spec.js b/packages/server/modules/comments/tests/comments.spec.js index 2399407bf..431cc4802 100644 --- a/packages/server/modules/comments/tests/comments.spec.js +++ b/packages/server/modules/comments/tests/comments.spec.js @@ -1255,7 +1255,7 @@ describe('Comments @comments', () => { // Init apollo instance w/ authenticated context apollo = { apollo: await buildApolloServer(), - context: createAuthedTestContext(user.id) + context: await createAuthedTestContext(user.id) } // Init token for authenticating w/ REST API diff --git a/packages/server/modules/core/graph/dataloaders/index.ts b/packages/server/modules/core/graph/dataloaders/index.ts new file mode 100644 index 000000000..dc7cba6bb --- /dev/null +++ b/packages/server/modules/core/graph/dataloaders/index.ts @@ -0,0 +1,650 @@ +import { + defineRequestDataloaders, + simpleTupleCacheKey +} from '@/modules/shared/helpers/graphqlHelper' +import DataLoader from 'dataloader' +import { + getStreamsFactory, + getCommitStreamsFactory, + getBatchUserFavoriteDataFactory, + getBatchStreamFavoritesCountsFactory, + getOwnedFavoritesCountByUserIdsFactory, + getStreamRolesFactory, + getUserStreamCountsFactory, + getStreamsSourceAppsFactory +} from '@/modules/core/repositories/streams' +import { keyBy } from 'lodash' +import { + BranchRecord, + CommitRecord, + StreamFavoriteRecord, + StreamRecord, + UsersMetaRecord +} from '@/modules/core/helpers/types' +import { Nullable } from '@/modules/shared/helpers/typeHelper' +import { ServerInviteRecord } from '@/modules/serverinvites/domain/types' +import { + getCommitBranchesFactory, + getCommitsFactory, + getSpecificBranchCommitsFactory, + getStreamCommitCountsFactory, + getUserAuthoredCommitCountsFactory, + getUserStreamCommitCountsFactory +} from '@/modules/core/repositories/commits' +import { ResourceIdentifier, Scope } from '@/modules/core/graph/generated/graphql' +import { + getBranchCommentCountsFactory, + getCommentParentsFactory, + getCommentReplyAuthorIdsFactory, + getCommentReplyCountsFactory, + getCommentsResourcesFactory, + getCommentsViewedAtFactory, + getCommitCommentCountsFactory, + getStreamCommentCountsFactory +} from '@/modules/comments/repositories/comments' +import { + getBranchCommitCountsFactory, + getBranchesByIdsFactory, + getBranchLatestCommitsFactory, + getStreamBranchCountsFactory, + getStreamBranchesByNameFactory +} from '@/modules/core/repositories/branches' +import { CommentRecord } from '@/modules/comments/helpers/types' +import { metaHelpers } from '@/modules/core/helpers/meta' +import { Users } from '@/modules/core/dbSchema' +import { getStreamPendingModelsFactory } from '@/modules/fileuploads/repositories/fileUploads' +import { FileUploadRecord } from '@/modules/fileuploads/helpers/types' +import { + AutomateRevisionFunctionRecord, + AutomationRecord, + AutomationRevisionRecord, + AutomationRunTriggerRecord, + AutomationTriggerDefinitionRecord +} from '@/modules/automate/helpers/types' +import { + getAutomationRevisionsFactory, + getAutomationRunsTriggersFactory, + getAutomationsFactory, + getFunctionAutomationCountsFactory, + getLatestAutomationRevisionsFactory, + getRevisionsFunctionsFactory, + getRevisionsTriggerDefinitionsFactory +} from '@/modules/automate/repositories/automations' +import { + getFunction, + getFunctionReleases +} from '@/modules/automate/clients/executionEngine' +import { + FunctionReleaseSchemaType, + FunctionSchemaType +} from '@/modules/automate/helpers/executionEngine' +import { + ExecutionEngineFailedResponseError, + ExecutionEngineNetworkError +} from '@/modules/automate/errors/executionEngine' +import { queryInvitesFactory } from '@/modules/serverinvites/repositories/serverInvites' +import { getAppScopesFactory } from '@/modules/auth/repositories' +import { StreamWithCommitId } from '@/modules/core/domain/streams/types' +import { + getUsersFactory, + UserWithOptionalRole +} from '@/modules/core/repositories/users' + +declare module '@/modules/core/loaders' { + interface ModularizedDataLoaders extends ReturnType {} +} + +const dataLoadersDefinition = defineRequestDataloaders( + ({ ctx, createLoader, deps: { db } }) => { + const userId = ctx.userId + + const getStreams = getStreamsFactory({ db }) + const getStreamPendingModels = getStreamPendingModelsFactory({ db }) + const getAppScopes = getAppScopesFactory({ db }) + const getAutomations = getAutomationsFactory({ db }) + const getAutomationRevisions = getAutomationRevisionsFactory({ db }) + const getLatestAutomationRevisions = getLatestAutomationRevisionsFactory({ db }) + const getRevisionsTriggerDefinitions = getRevisionsTriggerDefinitionsFactory({ db }) + const getRevisionsFunctions = getRevisionsFunctionsFactory({ db }) + const getFunctionAutomationCounts = getFunctionAutomationCountsFactory({ db }) + const getStreamCommentCounts = getStreamCommentCountsFactory({ db }) + const getAutomationRunsTriggers = getAutomationRunsTriggersFactory({ db }) + const getCommentsResources = getCommentsResourcesFactory({ db }) + const getCommentsViewedAt = getCommentsViewedAtFactory({ db }) + const getCommitCommentCounts = getCommitCommentCountsFactory({ db }) + const getBranchCommentCounts = getBranchCommentCountsFactory({ db }) + const getCommentReplyCounts = getCommentReplyCountsFactory({ db }) + const getCommentReplyAuthorIds = getCommentReplyAuthorIdsFactory({ db }) + const getCommentParents = getCommentParentsFactory({ db }) + const getBranchesByIds = getBranchesByIdsFactory({ db }) + const getStreamBranchesByName = getStreamBranchesByNameFactory({ db }) + const getBranchLatestCommits = getBranchLatestCommitsFactory({ db }) + const getStreamBranchCounts = getStreamBranchCountsFactory({ db }) + const getBranchCommitCounts = getBranchCommitCountsFactory({ db }) + const getCommits = getCommitsFactory({ db }) + const getSpecificBranchCommits = getSpecificBranchCommitsFactory({ db }) + const getCommitBranches = getCommitBranchesFactory({ db }) + const getStreamCommitCounts = getStreamCommitCountsFactory({ db }) + const getUserStreamCommitCounts = getUserStreamCommitCountsFactory({ db }) + const getUserAuthoredCommitCounts = getUserAuthoredCommitCountsFactory({ db }) + const getCommitStreams = getCommitStreamsFactory({ db }) + const getBatchUserFavoriteData = getBatchUserFavoriteDataFactory({ db }) + const getBatchStreamFavoritesCounts = getBatchStreamFavoritesCountsFactory({ db }) + const getOwnedFavoritesCountByUserIds = getOwnedFavoritesCountByUserIdsFactory({ + db + }) + const getStreamRoles = getStreamRolesFactory({ db }) + const getUserStreamCounts = getUserStreamCountsFactory({ db }) + const getStreamsSourceApps = getStreamsSourceAppsFactory({ db }) + const getUsers = getUsersFactory({ db }) + + return { + streams: { + getAutomation: (() => { + type AutomationDataLoader = DataLoader> + const streamAutomationLoaders = new Map() + return { + clearAll: () => streamAutomationLoaders.clear(), + forStream(streamId: string): AutomationDataLoader { + let loader = streamAutomationLoaders.get(streamId) + if (!loader) { + loader = createLoader>( + async (automationIds) => { + const results = keyBy( + await getAutomations({ automationIds: automationIds.slice() }), + (a) => a.id + ) + return automationIds.map((i) => results[i] || null) + } + ) + streamAutomationLoaders.set(streamId, loader) + } + + return loader + } + } + })(), + + /** + * Get a specific commit of a specific stream. Each stream ID technically has its own loader & + * thus its own query. + */ + getStreamCommit: (() => { + type CommitDataLoader = DataLoader> + const streamCommitLoaders = new Map() + return { + clearAll: () => streamCommitLoaders.clear(), + forStream(streamId: string): CommitDataLoader { + let loader = streamCommitLoaders.get(streamId) + if (!loader) { + loader = createLoader>( + async (commitIds) => { + const results = keyBy( + await getCommits(commitIds.slice(), { streamId }), + 'id' + ) + return commitIds.map((i) => results[i] || null) + } + ) + streamCommitLoaders.set(streamId, loader) + } + + return loader + } + } + })(), + + /** + * Get favorite metadata for a specific stream and user + */ + getUserFavoriteData: createLoader>( + async (streamIds) => { + if (!userId) { + return streamIds.map(() => null) + } + + const results = await getBatchUserFavoriteData({ + userId, + streamIds: streamIds.slice() + }) + return streamIds.map((k) => results[k]) + } + ), + + /** + * Get amount of favorites for a specific stream + */ + getFavoritesCount: createLoader(async (streamIds) => { + const results = await getBatchStreamFavoritesCounts(streamIds.slice()) + return streamIds.map((k) => results[k] || 0) + }), + + /** + * Get total amount of favorites of owned streams + */ + getOwnedFavoritesCount: createLoader(async (userIds) => { + const results = await getOwnedFavoritesCountByUserIds(userIds.slice()) + return userIds.map((i) => results[i] || 0) + }), + + /** + * Get stream from DB + * + * Note: Considering the difficulty of writing a single query that queries for multiple stream IDs + * and multiple user IDs also, currently this dataloader will only use a single userId + */ + getStream: createLoader>(async (streamIds) => { + const results = keyBy(await getStreams(streamIds.slice()), 'id') + return streamIds.map((i) => results[i] || null) + }), + + /** + * Get stream role from DB + */ + getRole: createLoader>(async (streamIds) => { + if (!userId) return streamIds.map(() => null) + + const results = await getStreamRoles(userId, streamIds.slice()) + return streamIds.map((id) => results[id] || null) + }), + /** + * Works in FE2 mode - skips `main` if it doesn't have any versions + */ + getBranchCount: createLoader(async (streamIds) => { + const results = keyBy( + await getStreamBranchCounts(streamIds.slice(), { skipEmptyMain: true }), + 'streamId' + ) + return streamIds.map((i) => results[i]?.count || 0) + }), + getCommitCountWithoutGlobals: createLoader( + async (streamIds) => { + const results = keyBy( + await getStreamCommitCounts(streamIds.slice(), { + ignoreGlobalsBranch: true + }), + 'streamId' + ) + return streamIds.map((i) => results[i]?.count || 0) + } + ), + getCommentThreadCount: createLoader(async (streamIds) => { + const results = keyBy( + await getStreamCommentCounts(streamIds.slice(), { threadsOnly: true }), + 'streamId' + ) + return streamIds.map((i) => results[i]?.count || 0) + }), + getSourceApps: createLoader(async (streamIds) => { + const results = await getStreamsSourceApps(streamIds.slice()) + return streamIds.map((i) => results[i] || []) + }), + /** + * Get a specific branch of a specific stream. Each stream ID technically has its own loader & + * thus its own query. + */ + getStreamBranchByName: (() => { + type BranchDataLoader = DataLoader> + const streamBranchLoaders = new Map() + return { + clearAll: () => streamBranchLoaders.clear(), + forStream(streamId: string): BranchDataLoader { + let loader = streamBranchLoaders.get(streamId) + if (!loader) { + loader = createLoader>( + async (branchNames) => { + const results = keyBy( + await getStreamBranchesByName(streamId, branchNames.slice()), + 'name' + ) + return branchNames.map((n) => results[n] || null) + } + ) + streamBranchLoaders.set(streamId, loader) + } + + return loader + } + } + })(), + /** + * Get a specific pending model (upload) of a specific stream. Each stream ID technically has its own loader & + * thus its own query. + */ + getStreamPendingBranchByName: (() => { + type BranchDataLoader = DataLoader> + const streamBranchLoaders = new Map() + return { + clearAll: () => streamBranchLoaders.clear(), + forStream(streamId: string): BranchDataLoader { + let loader = streamBranchLoaders.get(streamId) + if (!loader) { + loader = createLoader>( + async (branchNames) => { + const results = keyBy( + await getStreamPendingModels(streamId, { + branchNamePattern: `(${branchNames.slice().join('|')})` + }), + 'branchName' + ) + return branchNames.map((n) => results[n] || null) + } + ) + streamBranchLoaders.set(streamId, loader) + } + + return loader + } + } + })() + }, + branches: { + getCommitCount: createLoader(async (branchIds) => { + const results = keyBy(await getBranchCommitCounts(branchIds.slice()), 'id') + return branchIds.map((i) => results[i]?.count || 0) + }), + getLatestCommit: createLoader>( + async (branchIds) => { + const results = keyBy( + await getBranchLatestCommits(branchIds.slice()), + 'branchId' + ) + return branchIds.map((i) => results[i] || null) + } + ), + getCommentThreadCount: createLoader(async (branchIds) => { + const results = keyBy( + await getBranchCommentCounts(branchIds.slice(), { threadsOnly: true }), + 'id' + ) + return branchIds.map((i) => results[i]?.count || 0) + }), + getById: createLoader>(async (branchIds) => { + const results = keyBy(await getBranchesByIds(branchIds.slice()), 'id') + return branchIds.map((i) => results[i] || null) + }), + getBranchCommit: createLoader< + { branchId: string; commitId: string }, + Nullable, + string + >( + async (idPairs) => { + const results = keyBy(await getSpecificBranchCommits(idPairs.slice()), 'id') + return idPairs.map((p) => { + const commit = results[p.commitId] + return commit?.id === p.commitId && commit?.branchId === p.branchId + ? commit + : null + }) + }, + { cacheKeyFn: (key) => `${key.branchId}:${key.commitId}` } + ) + }, + commits: { + /** + * Get a commit's stream from DB + */ + getCommitStream: createLoader>( + async (commitIds) => { + const results = keyBy( + await getCommitStreams({ commitIds: commitIds.slice(), userId }), + 'commitId' + ) + return commitIds.map((id) => results[id] || null) + } + ), + + getCommitBranch: createLoader>( + async (commitIds) => { + const results = keyBy( + await getCommitBranches(commitIds.slice()), + 'commitId' + ) + return commitIds.map((id) => results[id] || null) + } + ), + getCommentThreadCount: createLoader(async (commitIds) => { + const results = keyBy( + await getCommitCommentCounts(commitIds.slice(), { threadsOnly: true }), + 'commitId' + ) + return commitIds.map((i) => results[i]?.count || 0) + }), + getById: createLoader>(async (commitIds) => { + const results = keyBy(await getCommits(commitIds.slice()), (c) => c.id) + return commitIds.map((i) => results[i] || null) + }) + }, + comments: { + getViewedAt: createLoader>(async (commentIds) => { + if (!userId) return commentIds.slice().map(() => null) + + const results = keyBy( + await getCommentsViewedAt(commentIds.slice(), userId), + 'commentId' + ) + return commentIds.map((id) => results[id]?.viewedAt || null) + }), + getResources: createLoader(async (commentIds) => { + const results = await getCommentsResources(commentIds.slice()) + return commentIds.map((id) => results[id]?.resources || []) + }), + getReplyCount: createLoader(async (threadIds) => { + const results = keyBy( + await getCommentReplyCounts(threadIds.slice()), + 'threadId' + ) + return threadIds.map((id) => results[id]?.count || 0) + }), + getReplyAuthorIds: createLoader(async (threadIds) => { + const results = await getCommentReplyAuthorIds(threadIds.slice()) + return threadIds.map((id) => results[id] || []) + }), + getReplyParent: createLoader>( + async (replyIds) => { + const results = keyBy(await getCommentParents(replyIds.slice()), 'replyId') + return replyIds.map((id) => results[id] || null) + } + ) + }, + users: { + /** + * Get user from DB + */ + getUser: createLoader>( + async (userIds) => { + const results = keyBy( + await getUsers(userIds.slice(), { withRole: true }), + 'id' + ) + return userIds.map((i) => results[i] || null) + } + ), + + /** + * Get meta values associated with one or more users + */ + getUserMeta: createLoader< + { userId: string; key: keyof (typeof Users)['meta']['metaKey'] }, + Nullable, + string + >( + async (requests) => { + const meta = metaHelpers(Users, db) + const results = await meta.getMultiple( + requests.map((r) => ({ + id: r.userId, + key: r.key + })) + ) + return requests.map((r) => { + const resultItem = results[r.userId]?.[r.key] + return resultItem + ? { ...resultItem, id: meta.getGraphqlId(resultItem) } + : null + }) + }, + { cacheKeyFn: (key) => `${key.userId}:${key.key}` } + ), + + /** + * Get user stream count. Includes private streams. + */ + getOwnStreamCount: createLoader(async (userIds) => { + const results = await getUserStreamCounts({ + publicOnly: false, + userIds: userIds.slice() + }) + return userIds.map((i) => results[i] || 0) + }), + + /** + * Get authored commit count. Includes commits from private streams. + */ + getAuthoredCommitCount: createLoader(async (userIds) => { + const results = await getUserAuthoredCommitCounts({ + userIds: userIds.slice(), + publicOnly: false + }) + + return userIds.map((i) => results[i] || 0) + }), + + /** + * Get count of commits in streams that the user is a contributor in. Includes private streams. + */ + getStreamCommitCount: createLoader(async (userIds) => { + const results = await getUserStreamCommitCounts({ + userIds: userIds.slice(), + publicOnly: false + }) + + return userIds.map((i) => results[i] || 0) + }) + }, + invites: { + /** + * Get invite from DB + */ + getInvite: createLoader>( + async (inviteIds) => { + const results = keyBy(await queryInvitesFactory({ db })(inviteIds), 'id') + return inviteIds.map((i) => results[i] || null) + } + ) + }, + apps: { + getAppScopes: createLoader>(async (appIds) => { + const results = await getAppScopes(appIds.slice()) + return appIds.map((i) => results[i] || []) + }) + }, + automations: { + getFunctionAutomationCount: createLoader( + async (functionIds) => { + const results = await getFunctionAutomationCounts({ + functionIds: functionIds.slice() + }) + return functionIds.map((i) => results[i] || 0) + } + ), + getAutomation: createLoader>(async (ids) => { + const results = keyBy( + await getAutomations({ automationIds: ids.slice() }), + (a) => a.id + ) + return ids.map((i) => results[i] || null) + }), + getAutomationRevision: createLoader>( + async (ids) => { + const results = keyBy( + await getAutomationRevisions({ automationRevisionIds: ids.slice() }), + (a) => a.id + ) + return ids.map((i) => results[i] || null) + } + ), + getLatestAutomationRevision: createLoader< + string, + Nullable + >(async (ids) => { + const results = await getLatestAutomationRevisions({ + automationIds: ids.slice() + }) + return ids.map((i) => results[i] || null) + }), + getRevisionTriggerDefinitions: createLoader< + string, + AutomationTriggerDefinitionRecord[] + >(async (ids) => { + const results = await getRevisionsTriggerDefinitions({ + automationRevisionIds: ids.slice() + }) + return ids.map((i) => results[i] || []) + }), + getRevisionFunctions: createLoader( + async (ids) => { + const results = await getRevisionsFunctions({ + automationRevisionIds: ids.slice() + }) + return ids.map((i) => results[i] || []) + } + ), + getRunTriggers: createLoader( + async (ids) => { + const results = await getAutomationRunsTriggers({ + automationRunIds: ids.slice() + }) + return ids.map((i) => results[i] || []) + } + ) + }, + automationsApi: { + getFunction: createLoader>( + async (fnIds) => { + const results = await Promise.all( + fnIds.map(async (fnId) => { + try { + return await getFunction({ functionId: fnId }) + } catch (e) { + const isNotFound = + e instanceof ExecutionEngineFailedResponseError && + e.response.statusMessage === 'FunctionNotFound' + if (e instanceof ExecutionEngineNetworkError || isNotFound) { + return null + } + + throw e + } + }) + ) + + return results + } + ), + getFunctionRelease: createLoader< + [fnId: string, fnReleaseId: string], + Nullable, + string + >( + async (keys) => { + const results = keyBy( + await getFunctionReleases({ + ids: keys.map(([fnId, fnReleaseId]) => ({ + functionId: fnId, + functionReleaseId: fnReleaseId + })) + }), + (r) => simpleTupleCacheKey([r.functionId, r.functionVersionId]) + ) + + return keys.map((k) => results[simpleTupleCacheKey(k)] || null) + }, + { cacheKeyFn: simpleTupleCacheKey } + ) + } + } + } +) + +export default dataLoadersDefinition diff --git a/packages/server/modules/core/loaders.ts b/packages/server/modules/core/loaders.ts index 2f37fc8ff..4d3f72937 100644 --- a/packages/server/modules/core/loaders.ts +++ b/packages/server/modules/core/loaders.ts @@ -1,145 +1,42 @@ import DataLoader from 'dataloader' -import { - getStreamsFactory, - getCommitStreamsFactory, - getBatchUserFavoriteDataFactory, - getBatchStreamFavoritesCountsFactory, - getOwnedFavoritesCountByUserIdsFactory, - getStreamRolesFactory, - getUserStreamCountsFactory, - getStreamsSourceAppsFactory -} from '@/modules/core/repositories/streams' -import { keyBy } from 'lodash' import { AuthContext } from '@/modules/shared/authz' -import { - BranchRecord, - CommitRecord, - StreamFavoriteRecord, - StreamRecord, - UsersMetaRecord -} from '@/modules/core/helpers/types' -import { Nullable } from '@/modules/shared/helpers/typeHelper' -import { ServerInviteRecord } from '@/modules/serverinvites/domain/types' -import { - getCommitBranchesFactory, - getCommitsFactory, - getSpecificBranchCommitsFactory, - getStreamCommitCountsFactory, - getUserAuthoredCommitCountsFactory, - getUserStreamCommitCountsFactory -} from '@/modules/core/repositories/commits' -import { ResourceIdentifier, Scope } from '@/modules/core/graph/generated/graphql' -import { - getBranchCommentCountsFactory, - getCommentParentsFactory, - getCommentReplyAuthorIdsFactory, - getCommentReplyCountsFactory, - getCommentsResourcesFactory, - getCommentsViewedAtFactory, - getCommitCommentCountsFactory, - getStreamCommentCountsFactory -} from '@/modules/comments/repositories/comments' -import { - getBranchCommitCountsFactory, - getBranchesByIdsFactory, - getBranchLatestCommitsFactory, - getStreamBranchCountsFactory, - getStreamBranchesByNameFactory -} from '@/modules/core/repositories/branches' -import { CommentRecord } from '@/modules/comments/helpers/types' -import { metaHelpers } from '@/modules/core/helpers/meta' -import { Users } from '@/modules/core/dbSchema' -import { getStreamPendingModelsFactory } from '@/modules/fileuploads/repositories/fileUploads' -import { FileUploadRecord } from '@/modules/fileuploads/helpers/types' -import { - AutomateRevisionFunctionRecord, - AutomationRecord, - AutomationRevisionRecord, - AutomationRunTriggerRecord, - AutomationTriggerDefinitionRecord -} from '@/modules/automate/helpers/types' -import { - getAutomationRevisionsFactory, - getAutomationRunsTriggersFactory, - getAutomationsFactory, - getFunctionAutomationCountsFactory, - getLatestAutomationRevisionsFactory, - getRevisionsFunctionsFactory, - getRevisionsTriggerDefinitionsFactory -} from '@/modules/automate/repositories/automations' -import { - getFunction, - getFunctionReleases -} from '@/modules/automate/clients/executionEngine' -import { - FunctionReleaseSchemaType, - FunctionSchemaType -} from '@/modules/automate/helpers/executionEngine' -import { - ExecutionEngineFailedResponseError, - ExecutionEngineNetworkError -} from '@/modules/automate/errors/executionEngine' -import { queryInvitesFactory } from '@/modules/serverinvites/repositories/serverInvites' -import db from '@/db/knex' import { graphDataloadersBuilders } from '@/modules' -import { getAppScopesFactory } from '@/modules/auth/repositories' -import { StreamWithCommitId } from '@/modules/core/domain/streams/types' -import { - getUsersFactory, - UserWithOptionalRole -} from '@/modules/core/repositories/users' - -const simpleTupleCacheKey = (key: [string, string]) => `${key[0]}:${key[1]}` - -const getStreams = getStreamsFactory({ db }) -const getStreamPendingModels = getStreamPendingModelsFactory({ db }) -const getAppScopes = getAppScopesFactory({ db }) -const getAutomations = getAutomationsFactory({ db }) -const getAutomationRevisions = getAutomationRevisionsFactory({ db }) -const getLatestAutomationRevisions = getLatestAutomationRevisionsFactory({ db }) -const getRevisionsTriggerDefinitions = getRevisionsTriggerDefinitionsFactory({ db }) -const getRevisionsFunctions = getRevisionsFunctionsFactory({ db }) -const getFunctionAutomationCounts = getFunctionAutomationCountsFactory({ db }) -const getStreamCommentCounts = getStreamCommentCountsFactory({ db }) -const getAutomationRunsTriggers = getAutomationRunsTriggersFactory({ db }) -const getCommentsResources = getCommentsResourcesFactory({ db }) -const getCommentsViewedAt = getCommentsViewedAtFactory({ db }) -const getCommitCommentCounts = getCommitCommentCountsFactory({ db }) -const getBranchCommentCounts = getBranchCommentCountsFactory({ db }) -const getCommentReplyCounts = getCommentReplyCountsFactory({ db }) -const getCommentReplyAuthorIds = getCommentReplyAuthorIdsFactory({ db }) -const getCommentParents = getCommentParentsFactory({ db }) -const getBranchesByIds = getBranchesByIdsFactory({ db }) -const getStreamBranchesByName = getStreamBranchesByNameFactory({ db }) -const getBranchLatestCommits = getBranchLatestCommitsFactory({ db }) -const getStreamBranchCounts = getStreamBranchCountsFactory({ db }) -const getBranchCommitCounts = getBranchCommitCountsFactory({ db }) -const getCommits = getCommitsFactory({ db }) -const getSpecificBranchCommits = getSpecificBranchCommitsFactory({ db }) -const getCommitBranches = getCommitBranchesFactory({ db }) -const getStreamCommitCounts = getStreamCommitCountsFactory({ db }) -const getUserStreamCommitCounts = getUserStreamCommitCountsFactory({ db }) -const getUserAuthoredCommitCounts = getUserAuthoredCommitCountsFactory({ db }) -const getCommitStreams = getCommitStreamsFactory({ db }) -const getBatchUserFavoriteData = getBatchUserFavoriteDataFactory({ db }) -const getBatchStreamFavoritesCounts = getBatchStreamFavoritesCountsFactory({ db }) -const getOwnedFavoritesCountByUserIds = getOwnedFavoritesCountByUserIdsFactory({ db }) -const getStreamRoles = getStreamRolesFactory({ db }) -const getUserStreamCounts = getUserStreamCountsFactory({ db }) -const getStreamsSourceApps = getStreamsSourceAppsFactory({ db }) -const getUsers = getUsersFactory({ db }) +import { ModularizedDataLoadersConstraint } from '@/modules/shared/helpers/graphqlHelper' +import { Knex } from 'knex' +import { isNonNullable, Optional } from '@speckle/shared' +import { flatten, noop } from 'lodash' +import { db } from '@/db/knex' /** - * TODO: Lazy load DataLoaders to reduce memory usage - * - Instead of keeping them request scoped, cache them identified by request (user ID) with a TTL, - * so that users with the same ID can re-use them across requests/subscriptions + * Lets not waste memory on loaders that may not actually be invoked */ +const makeLazyDataLoader = ( + ...args: ConstructorParameters> +): DataLoader => { + let dataloader: Optional> = undefined + + return new Proxy({} as DataLoader, { + get(_target, prop) { + if (!dataloader) { + // If invoking clearAll() - we don't really need to do anything, no loader exists + if (prop === 'clearAll') { + return noop + } + + dataloader = new DataLoader(...args) + } + + return dataloader[prop as keyof DataLoader] + } + }) +} const makeSelfClearingDataloader = ( - batchLoadFn: DataLoader.BatchLoadFn, - options?: DataLoader.Options + ...args: ConstructorParameters> ) => { - const dataloader = new DataLoader((ids) => { + const [batchLoadFn, options] = args + + const dataloader = makeLazyDataLoader((ids) => { dataloader.clearAll() return batchLoadFn(ids) }, options) @@ -147,10 +44,9 @@ const makeSelfClearingDataloader = ( } const buildDataLoaderCreator = (selfClearing = false) => { - return ( - batchLoadFn: DataLoader.BatchLoadFn, - options?: DataLoader.Options - ) => { + return (...args: ConstructorParameters>) => { + const [batchLoadFn, options] = args + if (selfClearing) { return makeSelfClearingDataloader(batchLoadFn, { ...(options || {}), @@ -158,7 +54,7 @@ const buildDataLoaderCreator = (selfClearing = false) => { cache: false }) } else { - return new DataLoader(batchLoadFn, options) + return makeLazyDataLoader(batchLoadFn, options) } } } @@ -167,531 +63,67 @@ const buildDataLoaderCreator = (selfClearing = false) => { * Build request-scoped dataloaders * @param ctx GraphQL context w/o loaders */ -export function buildRequestLoaders( +export async function buildRequestLoaders( ctx: AuthContext, options?: Partial<{ cleanLoadersEarly: boolean }> ) { - const userId = ctx.userId - const createLoader = buildDataLoaderCreator(options?.cleanLoadersEarly || false) const modulesLoaders = graphDataloadersBuilders() - const loaders = { - ...(Object.assign( - {}, - // eslint-disable-next-line @typescript-eslint/no-unsafe-return - ...modulesLoaders.map((l) => l({ ctx, createLoader })) - ) as Record), - streams: { - getAutomation: (() => { - type AutomationDataLoader = DataLoader> - const streamAutomationLoaders = new Map() - return { - clearAll: () => streamAutomationLoaders.clear(), - forStream(streamId: string): AutomationDataLoader { - let loader = streamAutomationLoaders.get(streamId) - if (!loader) { - loader = createLoader>( - async (automationIds) => { - const results = keyBy( - await getAutomations({ automationIds: automationIds.slice() }), - (a) => a.id - ) - return automationIds.map((i) => results[i] || null) - } - ) - streamAutomationLoaders.set(streamId, loader) - } + const mainDb = db - return loader - } - } - })(), + /** + * Dataloaders autoloaded from various speckle modules, created for the specified region DB + */ + const createLoadersForRegion = (deps: { db: Knex }) => { + return { + ...(Object.assign( + {}, + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + ...modulesLoaders.map((l) => l({ ctx, createLoader, deps })) + ) as Record) + } as ModularizedDataLoaders + } - /** - * Get a specific commit of a specific stream. Each stream ID technically has its own loader & - * thus its own query. - */ - getStreamCommit: (() => { - type CommitDataLoader = DataLoader> - const streamCommitLoaders = new Map() - return { - clearAll: () => streamCommitLoaders.clear(), - forStream(streamId: string): CommitDataLoader { - let loader = streamCommitLoaders.get(streamId) - if (!loader) { - loader = createLoader>( - async (commitIds) => { - const results = keyBy( - await getCommits(commitIds.slice(), { streamId }), - 'id' - ) - return commitIds.map((i) => results[i] || null) - } - ) - streamCommitLoaders.set(streamId, loader) - } + const mainDbLoaders = createLoadersForRegion({ db: mainDb }) + const regionLoaders = new Map() - return loader - } - } - })(), + // Extra utilities to add on top: - /** - * Get favorite metadata for a specific stream and user - */ - getUserFavoriteData: createLoader>( - async (streamIds) => { - if (!userId) { - return streamIds.map(() => null) - } - - const results = await getBatchUserFavoriteData({ - userId, - streamIds: streamIds.slice() - }) - return streamIds.map((k) => results[k]) - } - ), - - /** - * Get amount of favorites for a specific stream - */ - getFavoritesCount: createLoader(async (streamIds) => { - const results = await getBatchStreamFavoritesCounts(streamIds.slice()) - return streamIds.map((k) => results[k] || 0) - }), - - /** - * Get total amount of favorites of owned streams - */ - getOwnedFavoritesCount: createLoader(async (userIds) => { - const results = await getOwnedFavoritesCountByUserIds(userIds.slice()) - return userIds.map((i) => results[i] || 0) - }), - - /** - * Get stream from DB - * - * Note: Considering the difficulty of writing a single query that queries for multiple stream IDs - * and multiple user IDs also, currently this dataloader will only use a single userId - */ - getStream: createLoader>(async (streamIds) => { - const results = keyBy(await getStreams(streamIds.slice()), 'id') - return streamIds.map((i) => results[i] || null) - }), - - /** - * Get stream role from DB - */ - getRole: createLoader>(async (streamIds) => { - if (!userId) return streamIds.map(() => null) - - const results = await getStreamRoles(userId, streamIds.slice()) - return streamIds.map((id) => results[id] || null) - }), - /** - * Works in FE2 mode - skips `main` if it doesn't have any versions - */ - getBranchCount: createLoader(async (streamIds) => { - const results = keyBy( - await getStreamBranchCounts(streamIds.slice(), { skipEmptyMain: true }), - 'streamId' - ) - return streamIds.map((i) => results[i]?.count || 0) - }), - getCommitCountWithoutGlobals: createLoader(async (streamIds) => { - const results = keyBy( - await getStreamCommitCounts(streamIds.slice(), { - ignoreGlobalsBranch: true - }), - 'streamId' - ) - return streamIds.map((i) => results[i]?.count || 0) - }), - getCommentThreadCount: createLoader(async (streamIds) => { - const results = keyBy( - await getStreamCommentCounts(streamIds.slice(), { threadsOnly: true }), - 'streamId' - ) - return streamIds.map((i) => results[i]?.count || 0) - }), - getSourceApps: createLoader(async (streamIds) => { - const results = await getStreamsSourceApps(streamIds.slice()) - return streamIds.map((i) => results[i] || []) - }), - /** - * Get a specific branch of a specific stream. Each stream ID technically has its own loader & - * thus its own query. - */ - getStreamBranchByName: (() => { - type BranchDataLoader = DataLoader> - const streamBranchLoaders = new Map() - return { - clearAll: () => streamBranchLoaders.clear(), - forStream(streamId: string): BranchDataLoader { - let loader = streamBranchLoaders.get(streamId) - if (!loader) { - loader = createLoader>( - async (branchNames) => { - const results = keyBy( - await getStreamBranchesByName(streamId, branchNames.slice()), - 'name' - ) - return branchNames.map((n) => results[n] || null) - } - ) - streamBranchLoaders.set(streamId, loader) - } - - return loader - } - } - })(), - /** - * Get a specific pending model (upload) of a specific stream. Each stream ID technically has its own loader & - * thus its own query. - */ - getStreamPendingBranchByName: (() => { - type BranchDataLoader = DataLoader> - const streamBranchLoaders = new Map() - return { - clearAll: () => streamBranchLoaders.clear(), - forStream(streamId: string): BranchDataLoader { - let loader = streamBranchLoaders.get(streamId) - if (!loader) { - loader = createLoader>( - async (branchNames) => { - const results = keyBy( - await getStreamPendingModels(streamId, { - branchNamePattern: `(${branchNames.slice().join('|')})` - }), - 'branchName' - ) - return branchNames.map((n) => results[n] || null) - } - ) - streamBranchLoaders.set(streamId, loader) - } - - return loader - } - } - })() - }, - branches: { - getCommitCount: createLoader(async (branchIds) => { - const results = keyBy(await getBranchCommitCounts(branchIds.slice()), 'id') - return branchIds.map((i) => results[i]?.count || 0) - }), - getLatestCommit: createLoader>( - async (branchIds) => { - const results = keyBy( - await getBranchLatestCommits(branchIds.slice()), - 'branchId' - ) - return branchIds.map((i) => results[i] || null) - } - ), - getCommentThreadCount: createLoader(async (branchIds) => { - const results = keyBy( - await getBranchCommentCounts(branchIds.slice(), { threadsOnly: true }), - 'id' - ) - return branchIds.map((i) => results[i]?.count || 0) - }), - getById: createLoader>(async (branchIds) => { - const results = keyBy(await getBranchesByIds(branchIds.slice()), 'id') - return branchIds.map((i) => results[i] || null) - }), - getBranchCommit: createLoader< - { branchId: string; commitId: string }, - Nullable, - string - >( - async (idPairs) => { - const results = keyBy(await getSpecificBranchCommits(idPairs.slice()), 'id') - return idPairs.map((p) => { - const commit = results[p.commitId] - return commit?.id === p.commitId && commit?.branchId === p.branchId - ? commit - : null - }) - }, - { cacheKeyFn: (key) => `${key.branchId}:${key.commitId}` } - ) - }, - commits: { - /** - * Get a commit's stream from DB - */ - getCommitStream: createLoader>( - async (commitIds) => { - const results = keyBy( - await getCommitStreams({ commitIds: commitIds.slice(), userId }), - 'commitId' - ) - return commitIds.map((id) => results[id] || null) - } - ), - - getCommitBranch: createLoader>( - async (commitIds) => { - const results = keyBy(await getCommitBranches(commitIds.slice()), 'commitId') - return commitIds.map((id) => results[id] || null) - } - ), - getCommentThreadCount: createLoader(async (commitIds) => { - const results = keyBy( - await getCommitCommentCounts(commitIds.slice(), { threadsOnly: true }), - 'commitId' - ) - return commitIds.map((i) => results[i]?.count || 0) - }), - getById: createLoader>(async (commitIds) => { - const results = keyBy(await getCommits(commitIds.slice()), (c) => c.id) - return commitIds.map((i) => results[i] || null) - }) - }, - comments: { - getViewedAt: createLoader>(async (commentIds) => { - if (!userId) return commentIds.slice().map(() => null) - - const results = keyBy( - await getCommentsViewedAt(commentIds.slice(), userId), - 'commentId' - ) - return commentIds.map((id) => results[id]?.viewedAt || null) - }), - getResources: createLoader(async (commentIds) => { - const results = await getCommentsResources(commentIds.slice()) - return commentIds.map((id) => results[id]?.resources || []) - }), - getReplyCount: createLoader(async (threadIds) => { - const results = keyBy( - await getCommentReplyCounts(threadIds.slice()), - 'threadId' - ) - return threadIds.map((id) => results[id]?.count || 0) - }), - getReplyAuthorIds: createLoader(async (threadIds) => { - const results = await getCommentReplyAuthorIds(threadIds.slice()) - return threadIds.map((id) => results[id] || []) - }), - getReplyParent: createLoader>( - async (replyIds) => { - const results = keyBy(await getCommentParents(replyIds.slice()), 'replyId') - return replyIds.map((id) => results[id] || null) - } - ) - }, - users: { - /** - * Get user from DB - */ - getUser: createLoader>(async (userIds) => { - const results = keyBy(await getUsers(userIds.slice(), { withRole: true }), 'id') - return userIds.map((i) => results[i] || null) - }), - - /** - * Get meta values associated with one or more users - */ - getUserMeta: createLoader< - { userId: string; key: keyof (typeof Users)['meta']['metaKey'] }, - Nullable, - string - >( - async (requests) => { - const meta = metaHelpers(Users, db) - const results = await meta.getMultiple( - requests.map((r) => ({ - id: r.userId, - key: r.key - })) - ) - return requests.map((r) => { - const resultItem = results[r.userId]?.[r.key] - return resultItem - ? { ...resultItem, id: meta.getGraphqlId(resultItem) } - : null - }) - }, - { cacheKeyFn: (key) => `${key.userId}:${key.key}` } - ), - - /** - * Get user stream count. Includes private streams. - */ - getOwnStreamCount: createLoader(async (userIds) => { - const results = await getUserStreamCounts({ - publicOnly: false, - userIds: userIds.slice() - }) - return userIds.map((i) => results[i] || 0) - }), - - /** - * Get authored commit count. Includes commits from private streams. - */ - getAuthoredCommitCount: createLoader(async (userIds) => { - const results = await getUserAuthoredCommitCounts({ - userIds: userIds.slice(), - publicOnly: false - }) - - return userIds.map((i) => results[i] || 0) - }), - - /** - * Get count of commits in streams that the user is a contributor in. Includes private streams. - */ - getStreamCommitCount: createLoader(async (userIds) => { - const results = await getUserStreamCommitCounts({ - userIds: userIds.slice(), - publicOnly: false - }) - - return userIds.map((i) => results[i] || 0) - }) - }, - invites: { - /** - * Get invite from DB - */ - getInvite: createLoader>( - async (inviteIds) => { - const results = keyBy(await queryInvitesFactory({ db })(inviteIds), 'id') - return inviteIds.map((i) => results[i] || null) - } - ) - }, - apps: { - getAppScopes: createLoader>(async (appIds) => { - const results = await getAppScopes(appIds.slice()) - return appIds.map((i) => results[i] || []) - }) - }, - automations: { - getFunctionAutomationCount: createLoader(async (functionIds) => { - const results = await getFunctionAutomationCounts({ - functionIds: functionIds.slice() - }) - return functionIds.map((i) => results[i] || 0) - }), - getAutomation: createLoader>(async (ids) => { - const results = keyBy( - await getAutomations({ automationIds: ids.slice() }), - (a) => a.id - ) - return ids.map((i) => results[i] || null) - }), - getAutomationRevision: createLoader>( - async (ids) => { - const results = keyBy( - await getAutomationRevisions({ automationRevisionIds: ids.slice() }), - (a) => a.id - ) - return ids.map((i) => results[i] || null) - } - ), - getLatestAutomationRevision: createLoader< - string, - Nullable - >(async (ids) => { - const results = await getLatestAutomationRevisions({ - automationIds: ids.slice() - }) - return ids.map((i) => results[i] || null) - }), - getRevisionTriggerDefinitions: createLoader< - string, - AutomationTriggerDefinitionRecord[] - >(async (ids) => { - const results = await getRevisionsTriggerDefinitions({ - automationRevisionIds: ids.slice() - }) - return ids.map((i) => results[i] || []) - }), - getRevisionFunctions: createLoader( - async (ids) => { - const results = await getRevisionsFunctions({ - automationRevisionIds: ids.slice() - }) - return ids.map((i) => results[i] || []) - } - ), - getRunTriggers: createLoader( - async (ids) => { - const results = await getAutomationRunsTriggers({ - automationRunIds: ids.slice() - }) - return ids.map((i) => results[i] || []) - } - ) - }, - automationsApi: { - getFunction: createLoader>(async (fnIds) => { - const results = await Promise.all( - fnIds.map(async (fnId) => { - try { - return await getFunction({ functionId: fnId }) - } catch (e) { - const isNotFound = - e instanceof ExecutionEngineFailedResponseError && - e.response.statusMessage === 'FunctionNotFound' - if (e instanceof ExecutionEngineNetworkError || isNotFound) { - return null - } - - throw e - } - }) - ) - - return results - }), - getFunctionRelease: createLoader< - [fnId: string, fnReleaseId: string], - Nullable, - string - >( - async (keys) => { - const results = keyBy( - await getFunctionReleases({ - ids: keys.map(([fnId, fnReleaseId]) => ({ - functionId: fnId, - functionReleaseId: fnReleaseId - })) - }), - (r) => simpleTupleCacheKey([r.functionId, r.functionVersionId]) - ) - - return keys.map((k) => results[simpleTupleCacheKey(k)] || null) - }, - { cacheKeyFn: simpleTupleCacheKey } - ) + /** + * Get dataloaders for specific region + */ + const forRegion = (deps: { db: Knex }) => { + if (!regionLoaders.has(deps.db)) { + regionLoaders.set(deps.db, createLoadersForRegion(deps)) } + return regionLoaders.get(deps.db) as ModularizedDataLoaders } /** - * Clear all loaders + * Clear all request loaders across all regions */ const clearAll = () => { - for (const groupedLoaders of Object.values(loaders)) { + const allLoaderGroups = flatten( + [mainDbLoaders, ...regionLoaders.values()].map((l) => + Object.values(l || {}).filter(isNonNullable) + ) + ) + + for (const groupedLoaders of allLoaderGroups) { for (const loaderItem of Object.values(groupedLoaders)) { - ;(loaderItem as DataLoader).clearAll() + loaderItem.clearAll() } } } return { - ...loaders, - clearAll + ...mainDbLoaders, + clearAll, + forRegion } } -export interface AllRequestDataLoaders {} +export interface ModularizedDataLoaders extends ModularizedDataLoadersConstraint {} -export type RequestDataLoaders = ReturnType & - AllRequestDataLoaders +export type RequestDataLoaders = Awaited> diff --git a/packages/server/modules/core/tests/apitokens.spec.ts b/packages/server/modules/core/tests/apitokens.spec.ts index be4626b29..998429b12 100644 --- a/packages/server/modules/core/tests/apitokens.spec.ts +++ b/packages/server/modules/core/tests/apitokens.spec.ts @@ -62,7 +62,7 @@ describe('API Tokens', () => { await createTestUsers([user1]) apollo = await testApolloServer({ - context: createTestContext({ + context: await createTestContext({ auth: true, userId: user1.id, role: Roles.Server.Admin, @@ -229,7 +229,7 @@ describe('API Tokens', () => { testApp1Token = appToken apollo = await testApolloServer({ - context: createTestContext({ + context: await createTestContext({ auth: true, userId: user1.id, role: Roles.Server.Admin, @@ -380,7 +380,7 @@ describe('API Tokens', () => { } apollo = await testApolloServer({ - context: createTestContext({ + context: await createTestContext({ auth: true, userId: user1.id, role: Roles.Server.Admin, diff --git a/packages/server/modules/core/tests/batchCommits.spec.ts b/packages/server/modules/core/tests/batchCommits.spec.ts index aed43905a..cfcc9f9b6 100644 --- a/packages/server/modules/core/tests/batchCommits.spec.ts +++ b/packages/server/modules/core/tests/batchCommits.spec.ts @@ -174,7 +174,7 @@ describe('Batch commits', () => { before(async () => { apollo = { apollo: await buildApolloServer(), - context: createAuthedTestContext(me.id) + context: await createAuthedTestContext(me.id) } invokeBatchAction = buildBatchActionInvoker(apollo) }) @@ -310,7 +310,7 @@ describe('Batch commits', () => { before(async () => { apollo = { apollo: await buildApolloServer(), - context: createTestContext() + context: await createTestContext() } invokeBatchAction = buildBatchActionInvoker(apollo) }) diff --git a/packages/server/modules/core/tests/commitsGraphql.spec.ts b/packages/server/modules/core/tests/commitsGraphql.spec.ts index 02d7bf1f8..2c0eacfbc 100644 --- a/packages/server/modules/core/tests/commitsGraphql.spec.ts +++ b/packages/server/modules/core/tests/commitsGraphql.spec.ts @@ -129,7 +129,7 @@ describe('Commits (GraphQL)', () => { before(async () => { apollo = { apollo: await buildApolloServer(), - context: createAuthedTestContext(me.id) + context: await createAuthedTestContext(me.id) } }) diff --git a/packages/server/modules/core/tests/discoverableStreams.spec.ts b/packages/server/modules/core/tests/discoverableStreams.spec.ts index 1aa03ef59..fb77ace17 100644 --- a/packages/server/modules/core/tests/discoverableStreams.spec.ts +++ b/packages/server/modules/core/tests/discoverableStreams.spec.ts @@ -120,7 +120,7 @@ describe('Discoverable streams', () => { apollo = { apollo: await buildApolloServer(), - context: createTestContext() + context: await createTestContext() } }) @@ -249,7 +249,7 @@ describe('Discoverable streams', () => { before(async () => { apollo = { apollo: await buildApolloServer(), - context: createAuthedTestContext(me.id) + context: await createAuthedTestContext(me.id) } }) diff --git a/packages/server/modules/core/tests/favoriteStreams.spec.js b/packages/server/modules/core/tests/favoriteStreams.spec.js index f76e79e94..9ad85cfaf 100644 --- a/packages/server/modules/core/tests/favoriteStreams.spec.js +++ b/packages/server/modules/core/tests/favoriteStreams.spec.js @@ -279,7 +279,7 @@ describe('Favorite streams', () => { before(async () => { apollo = { apollo: await buildApolloServer(), - context: createAuthedTestContext(me.id) + context: await createAuthedTestContext(me.id) } // Drop all favorites to ensure we don't favorite already favorited streams @@ -448,7 +448,7 @@ describe('Favorite streams', () => { // "Log in" with other user const apollo = { apollo: await buildApolloServer(), - context: createAuthedTestContext(otherGuy.id) + context: await createAuthedTestContext(otherGuy.id) } const { data, errors } = await executeOperation( @@ -474,7 +474,7 @@ describe('Favorite streams', () => { before(async () => { apollo = { apollo: await buildApolloServer(), - context: createTestContext() + context: await createTestContext() } }) diff --git a/packages/server/modules/core/tests/streams.spec.ts b/packages/server/modules/core/tests/streams.spec.ts index fd8c8227d..36bcb98fa 100644 --- a/packages/server/modules/core/tests/streams.spec.ts +++ b/packages/server/modules/core/tests/streams.spec.ts @@ -371,7 +371,7 @@ describe('Streams @core-streams', () => { const apollo = { apollo: await buildApolloServer(), - context: createAuthedTestContext(userTwo.id) + context: await createAuthedTestContext(userTwo.id) } const { data, errors } = await leaveStream(apollo, { streamId }) @@ -706,7 +706,7 @@ describe('Streams @core-streams', () => { activeUserId = userOne.id apollo = { apollo: await buildApolloServer(), - context: createAuthedTestContext(activeUserId) + context: await createAuthedTestContext(activeUserId) } }) @@ -735,7 +735,7 @@ describe('Streams @core-streams', () => { before(async () => { apollo = { apollo: await buildApolloServer(), - context: createTestContext() + context: await createTestContext() } }) diff --git a/packages/server/modules/core/tests/usersAdminList.spec.ts b/packages/server/modules/core/tests/usersAdminList.spec.ts index 8c4da9b01..c0a447345 100644 --- a/packages/server/modules/core/tests/usersAdminList.spec.ts +++ b/packages/server/modules/core/tests/usersAdminList.spec.ts @@ -287,7 +287,7 @@ describe('[Admin users list]', () => { apollo = { apollo: await buildApolloServer(), - context: createAuthedTestContext(me.id!, { role: Roles.Server.Admin }) + context: await createAuthedTestContext(me.id!, { role: Roles.Server.Admin }) } }) diff --git a/packages/server/modules/core/tests/usersGraphql.spec.ts b/packages/server/modules/core/tests/usersGraphql.spec.ts index cd4c5a55a..f43e88f22 100644 --- a/packages/server/modules/core/tests/usersGraphql.spec.ts +++ b/packages/server/modules/core/tests/usersGraphql.spec.ts @@ -103,7 +103,7 @@ describe('Users (GraphQL)', () => { before(async () => { apollo = { apollo: await buildApolloServer(), - context: createTestContext() + context: await createTestContext() } }) @@ -130,7 +130,7 @@ describe('Users (GraphQL)', () => { before(async () => { apollo = { apollo: await buildApolloServer(), - context: createAuthedTestContext(me.id) + context: await createAuthedTestContext(me.id) } }) diff --git a/packages/server/modules/emails/tests/verifications.spec.ts b/packages/server/modules/emails/tests/verifications.spec.ts index 7d6a9328a..6fb9eb028 100644 --- a/packages/server/modules/emails/tests/verifications.spec.ts +++ b/packages/server/modules/emails/tests/verifications.spec.ts @@ -107,7 +107,7 @@ describe('Email verifications @emails', () => { before(async () => { apollo = { apollo: await buildApolloServer(), - context: createAuthedTestContext(userA.id) + context: await createAuthedTestContext(userA.id) } }) @@ -137,7 +137,7 @@ describe('Email verifications @emails', () => { const invokeRequestVerification = async (user: BasicTestUser) => { const apollo = { apollo: await buildApolloServer(), - context: createAuthedTestContext(user.id) + context: await createAuthedTestContext(user.id) } return await requestVerification(apollo, {}) } @@ -189,7 +189,7 @@ describe('Email verifications @emails', () => { before(async () => { apollo = { apollo: await buildApolloServer(), - context: createTestContext() + context: await createTestContext() } }) diff --git a/packages/server/modules/serverinvites/tests/invites.spec.ts b/packages/server/modules/serverinvites/tests/invites.spec.ts index cab20436c..1f1b362ec 100644 --- a/packages/server/modules/serverinvites/tests/invites.spec.ts +++ b/packages/server/modules/serverinvites/tests/invites.spec.ts @@ -478,7 +478,7 @@ describe('[Stream & Server Invites]', () => { before(async () => { apollo = await testApolloServer({ - context: createTestContext({ + context: await createTestContext({ auth: true, userId: me.id, role: Roles.Server.Admin, // Marking the user as an admin diff --git a/packages/server/modules/shared/helpers/graphqlHelper.ts b/packages/server/modules/shared/helpers/graphqlHelper.ts index 18395c1c5..2e552b913 100644 --- a/packages/server/modules/shared/helpers/graphqlHelper.ts +++ b/packages/server/modules/shared/helpers/graphqlHelper.ts @@ -11,6 +11,8 @@ import { NotFoundError, UnauthorizedError } from '@/modules/shared/errors' +import { Optional } from '@speckle/shared' +import { Knex } from 'knex' /** * Encode cursor to turn it into an opaque & obfuscated value @@ -41,40 +43,36 @@ export function encodeIsoDateCursor(date: Date | Dayjs): string { return encodeCursor(str) } -export type RequestDataLoadersBuilder< - T extends { - [group: string]: { - [loader: string]: unknown - } - } -> = (params: { - ctx: AuthContext - createLoader: ( - batchLoadFn: DataLoader.BatchLoadFn, - options?: DataLoader.Options - ) => DataLoader -}) => T +/** + * All dataloaders must at the very least follow this type + */ +export type ModularizedDataLoadersConstraint = { + [group: string]: Optional<{ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + [loader: string]: DataLoader | { clearAll: () => unknown } + }> +} -export type RequestDataLoaders< - T extends { - [group: string]: { - [loader: string]: unknown +export type RequestDataLoadersBuilder = + (params: { + ctx: AuthContext + createLoader: ( + batchLoadFn: DataLoader.BatchLoadFn, + options?: DataLoader.Options + ) => DataLoader + deps: { + db: Knex } - } -> = ReturnType> + }) => T -export const defineRequestDataloaders = < - T extends { - [group: string]: { - [loader: string]: unknown - } - } ->( +export const defineRequestDataloaders = ( builder: RequestDataLoadersBuilder ): RequestDataLoadersBuilder => { return builder } +export const simpleTupleCacheKey = (key: [string, string]) => `${key[0]}:${key[1]}` + /** * Is a lower significance error, caused by user error (and thus - not a bug in our code) */ diff --git a/packages/server/modules/shared/middleware/index.ts b/packages/server/modules/shared/middleware/index.ts index eeb9892c9..aa5abc274 100644 --- a/packages/server/modules/shared/middleware/index.ts +++ b/packages/server/modules/shared/middleware/index.ts @@ -160,13 +160,13 @@ export async function authContextMiddleware( next() } -export function addLoadersToCtx( +export async function addLoadersToCtx( ctx: Merge, { log?: Optional }>, options?: Partial<{ cleanLoadersEarly: boolean }> -): GraphQLContext { +): Promise { const log = ctx.log || Observability.extendLoggerComponent(Observability.getLogger(), 'graphql') - const loaders = buildRequestLoaders(ctx, options) + const loaders = await buildRequestLoaders(ctx, options) return { ...ctx, loaders, log } } @@ -212,7 +212,7 @@ export async function buildContext({ } // Adding request data loaders - return addLoadersToCtx( + return await addLoadersToCtx( { ...ctx, log diff --git a/packages/server/modules/workspaces/graph/dataloaders/workspaces.ts b/packages/server/modules/workspaces/graph/dataloaders/workspaces.ts index b1d915b41..052de75f8 100644 --- a/packages/server/modules/workspaces/graph/dataloaders/workspaces.ts +++ b/packages/server/modules/workspaces/graph/dataloaders/workspaces.ts @@ -1,4 +1,3 @@ -import { db } from '@/db/knex' import { getFeatureFlags } from '@/modules/shared/helpers/envHelper' import { defineRequestDataloaders } from '@/modules/shared/helpers/graphqlHelper' import { @@ -14,42 +13,46 @@ import { keyBy } from 'lodash' const { FF_WORKSPACES_MODULE_ENABLED } = getFeatureFlags() declare module '@/modules/core/loaders' { - interface AllRequestDataLoaders + interface ModularizedDataLoaders extends Partial> {} } -const dataLoadersDefinition = defineRequestDataloaders(({ ctx, createLoader }) => { - const getWorkspaces = getWorkspacesFactory({ db }) - const getWorkspaceDomains = getWorkspaceDomainsFactory({ db }) +const dataLoadersDefinition = defineRequestDataloaders( + ({ ctx, createLoader, deps: { db } }) => { + const getWorkspaces = getWorkspacesFactory({ db }) + const getWorkspaceDomains = getWorkspaceDomainsFactory({ db }) - return { - workspaces: { - /** - * Get workspace, with the active user's role attached - */ - getWorkspace: createLoader( - async (ids) => { - const results = keyBy( - await getWorkspaces({ workspaceIds: ids.slice(), userId: ctx.userId }), - (w) => w.id - ) - return ids.map((id) => results[id] || null) - } - ) - }, - workspaceDomains: { - /** - * Get workspace, with the active user's role attached - */ - getWorkspaceDomains: createLoader(async (ids) => { - const results = keyBy( - await getWorkspaceDomains({ workspaceIds: ids.slice() }), - (w) => w.id + return { + workspaces: { + /** + * Get workspace, with the active user's role attached + */ + getWorkspace: createLoader( + async (ids) => { + const results = keyBy( + await getWorkspaces({ workspaceIds: ids.slice(), userId: ctx.userId }), + (w) => w.id + ) + return ids.map((id) => results[id] || null) + } ) - return ids.map((id) => results[id] || null) - }) + }, + workspaceDomains: { + /** + * Get workspace, with the active user's role attached + */ + getWorkspaceDomains: createLoader( + async (ids) => { + const results = keyBy( + await getWorkspaceDomains({ workspaceIds: ids.slice() }), + (w) => w.id + ) + return ids.map((id) => results[id] || null) + } + ) + } } } -}) +) export default FF_WORKSPACES_MODULE_ENABLED ? dataLoadersDefinition : undefined diff --git a/packages/server/modules/workspaces/tests/integration/invites.graph.spec.ts b/packages/server/modules/workspaces/tests/integration/invites.graph.spec.ts index 583f9cbf5..ff5a30b97 100644 --- a/packages/server/modules/workspaces/tests/integration/invites.graph.spec.ts +++ b/packages/server/modules/workspaces/tests/integration/invites.graph.spec.ts @@ -1667,7 +1667,7 @@ describe('Workspaces Invites GQL', () => { } }, { - context: createTestContext({ + context: await createTestContext({ userId: newUser.id, auth: true, role: Roles.Server.User, diff --git a/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts b/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts index 0e9aba2b8..d9c66f859 100644 --- a/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts +++ b/packages/server/modules/workspaces/tests/integration/projects.graph.spec.ts @@ -59,7 +59,7 @@ describe('Workspace project GQL CRUD', () => { await createTestUsers([serverAdminUser, serverMemberUser]) const token = await createAuthTokenForUser(serverAdminUser.id, AllScopes) apollo = await testApolloServer({ - context: createTestContext({ + context: await createTestContext({ auth: true, userId: serverAdminUser.id, token, diff --git a/packages/server/modules/workspaces/tests/integration/roles.graph.spec.ts b/packages/server/modules/workspaces/tests/integration/roles.graph.spec.ts index e149b9650..5a7a2c05f 100644 --- a/packages/server/modules/workspaces/tests/integration/roles.graph.spec.ts +++ b/packages/server/modules/workspaces/tests/integration/roles.graph.spec.ts @@ -54,7 +54,7 @@ describe('Workspaces Roles GQL', () => { await createTestUsers([serverAdminUser, serverMemberUser]) const token = await createAuthTokenForUser(serverAdminUser.id, AllScopes) apollo = await testApolloServer({ - context: createTestContext({ + context: await createTestContext({ auth: true, userId: serverAdminUser.id, token, @@ -551,7 +551,7 @@ describe('Workspaces Roles GQL', () => { before(async () => { const token = await createAuthTokenForUser(serverMemberUser.id, AllScopes) workspaceMemberApollo = await testApolloServer({ - context: createTestContext({ + context: await createTestContext({ auth: true, userId: serverMemberUser.id, token, diff --git a/packages/server/modules/workspaces/tests/integration/workspaces.graph.spec.ts b/packages/server/modules/workspaces/tests/integration/workspaces.graph.spec.ts index 4bb98846b..f12ee0004 100644 --- a/packages/server/modules/workspaces/tests/integration/workspaces.graph.spec.ts +++ b/packages/server/modules/workspaces/tests/integration/workspaces.graph.spec.ts @@ -129,7 +129,7 @@ describe('Workspaces GQL CRUD', () => { await createTestUsers([testAdminUser, testMemberUser]) const token = await createAuthTokenForUser(testAdminUser.id, AllScopes) apollo = await testApolloServer({ - context: createTestContext({ + context: await createTestContext({ auth: true, userId: testAdminUser.id, token, @@ -834,7 +834,7 @@ describe('Workspaces GQL CRUD', () => { it('should throw if non-workspace-admin triggers delete', async () => { const memberApollo: TestApolloServer = (apollo = await testApolloServer({ - context: createTestContext({ + context: await createTestContext({ auth: true, userId: testAdminUser.id, token: '', diff --git a/packages/server/test/graphqlHelper.ts b/packages/server/test/graphqlHelper.ts index 9d6192dc7..0835ffb46 100644 --- a/packages/server/test/graphqlHelper.ts +++ b/packages/server/test/graphqlHelper.ts @@ -58,7 +58,7 @@ export async function executeOperation< context?: GraphQLContext ): Promise> { const server: ApolloServer = apollo.apollo - const contextValue = context || apollo.context || createTestContext() + const contextValue = context || apollo.context || (await createTestContext()) const res = (await server.executeOperation( { @@ -83,7 +83,9 @@ export async function executeOperation< * Create a test context for a GraphQL request. Optionally override any of the default values. * By default the context will be unauthenticated */ -export const createTestContext = (ctx?: Partial): GraphQLContext => +export const createTestContext = async ( + ctx?: Partial +): Promise => addLoadersToCtx({ auth: false, userId: undefined, @@ -95,10 +97,10 @@ export const createTestContext = (ctx?: Partial): GraphQLContext ...(ctx || {}) }) -export const createAuthedTestContext = ( +export const createAuthedTestContext = async ( userId: string, ctxOverrides?: Partial -): GraphQLContext => +): Promise => addLoadersToCtx({ auth: true, userId, @@ -122,13 +124,13 @@ const buildMergedContext = async (params: { */ authUserId?: string }) => { - let baseCtx: GraphQLContext = params.baseCtx || createTestContext() + let baseCtx: GraphQLContext = params.baseCtx || (await createTestContext()) // Init ctx from userId? if (params?.authUserId) { const userData = await getUser(params.authUserId, { withRole: true }) const role = userData?.role || Roles.Server.User - const userCtx = createAuthedTestContext(params.authUserId, { role }) + const userCtx = await createAuthedTestContext(params.authUserId, { role }) // Apply authed context to base baseCtx = { @@ -148,7 +150,7 @@ const buildMergedContext = async (params: { } // Apply dataloaders from scratch - baseCtx = createTestContext(baseCtx) + baseCtx = await createTestContext(baseCtx) return baseCtx }