From 04400b1ee933c1203fa1496b304af996e8365403 Mon Sep 17 00:00:00 2001 From: Kristaps Fabians Geikins Date: Fri, 25 Oct 2024 13:21:30 +0300 Subject: [PATCH 1/6] chore(server): easy js to ts migrations #2 - objectStorage.ts --- .../{objectStorage.js => objectStorage.ts} | 95 +++++++++++++------ 1 file changed, 65 insertions(+), 30 deletions(-) rename packages/server/modules/blobstorage/{objectStorage.js => objectStorage.ts} (56%) diff --git a/packages/server/modules/blobstorage/objectStorage.js b/packages/server/modules/blobstorage/objectStorage.ts similarity index 56% rename from packages/server/modules/blobstorage/objectStorage.js rename to packages/server/modules/blobstorage/objectStorage.ts index d89f45bea..50e0e43e4 100644 --- a/packages/server/modules/blobstorage/objectStorage.js +++ b/packages/server/modules/blobstorage/objectStorage.ts @@ -1,23 +1,33 @@ -const { NotFoundError, EnvironmentResourceError } = require('@/modules/shared/errors') -const { +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { + NotFoundError, + EnvironmentResourceError, + BadRequestError +} from '@/modules/shared/errors' +import { S3Client, GetObjectCommand, HeadBucketCommand, DeleteObjectCommand, CreateBucketCommand, - S3ServiceException -} = require('@aws-sdk/client-s3') -const { Upload } = require('@aws-sdk/lib-storage') -const { + S3ServiceException, + S3ClientConfig +} from '@aws-sdk/client-s3' +import { Upload, Options as UploadOptions } from '@aws-sdk/lib-storage' +import { getS3AccessKey, getS3SecretKey, getS3Endpoint, getS3Region, getS3BucketName, createS3Bucket -} = require('@/modules/shared/helpers/envHelper') +} from '@/modules/shared/helpers/envHelper' +import { ensureError, Nullable } from '@speckle/shared' +import { get } from 'lodash' +import type { Command } from '@aws-sdk/smithy-client' +import type stream from 'stream' -let s3Config = null +let s3Config: Nullable = null const getS3Config = () => { if (!s3Config) { @@ -36,7 +46,7 @@ const getS3Config = () => { return s3Config } -let storageBucket = null +let storageBucket: Nullable = null const getStorageBucket = () => { if (!storageBucket) { @@ -51,32 +61,45 @@ const getObjectStorage = () => ({ createBucket: createS3Bucket() }) -const sendCommand = async (command) => { +const sendCommand = async ( + command: (Bucket: string) => Command +) => { const { client, Bucket } = getObjectStorage() try { - return await client.send(command(Bucket)) + const ret = (await client.send( + command(Bucket) as Command + )) as CommandOutput + return ret } catch (err) { - if (err instanceof S3ServiceException && err.Code === 'NoSuchKey') + if (err instanceof S3ServiceException && get(err, 'Code') === 'NoSuchKey') throw new NotFoundError(err.message) throw err } } -const getObjectStream = async ({ objectKey }) => { +export const getObjectStream = async ({ objectKey }: { objectKey: string }) => { const data = await sendCommand( (Bucket) => new GetObjectCommand({ Bucket, Key: objectKey }) ) - return data.Body + + // TODO: Apparently not always stream.Readable according to types, but in practice this works + return data.Body as stream.Readable } -const getObjectAttributes = async ({ objectKey }) => { +export const getObjectAttributes = async ({ objectKey }: { objectKey: string }) => { const data = await sendCommand( (Bucket) => new GetObjectCommand({ Bucket, Key: objectKey }) ) - return { fileSize: data.ContentLength } + return { fileSize: data.ContentLength || 0 } } -const storeFileStream = async ({ objectKey, fileStream }) => { +export const storeFileStream = async ({ + objectKey, + fileStream +}: { + objectKey: string + fileStream: UploadOptions['params']['Body'] +}) => { const { client, Bucket } = getObjectStorage() const parallelUploads3 = new Upload({ client, @@ -95,20 +118,40 @@ const storeFileStream = async ({ objectKey, fileStream }) => { const data = await parallelUploads3.done() // the ETag is a hash of the object. Could be used to dedupe stuff... + + if (!data || !('ETag' in data) || !data.ETag) { + throw new BadRequestError('No ETag in response') + } + const fileHash = data.ETag.replaceAll('"', '') return { fileHash } } -const deleteObject = async ({ objectKey }) => { +export const deleteObject = async ({ objectKey }: { objectKey: string }) => { await sendCommand((Bucket) => new DeleteObjectCommand({ Bucket, Key: objectKey })) } -const ensureStorageAccess = async () => { + +// No idea what the actual error type is, too difficult to figure out +type EnsureStorageAccessError = Error & { + statusCode?: number + $metadata?: { httpStatusCode?: number } +} + +const isExpectedEnsureStorageAccessError = ( + err: unknown +): err is EnsureStorageAccessError => + err instanceof Error && ('statusCode' in err || '$metadata' in err) + +export const ensureStorageAccess = async () => { const { client, Bucket, createBucket } = getObjectStorage() try { await client.send(new HeadBucketCommand({ Bucket })) return } catch (err) { - if (err.statusCode === 403 || err['$metadata']?.httpStatusCode === 403) { + if ( + isExpectedEnsureStorageAccessError(err) && + (err.statusCode === 403 || err['$metadata']?.httpStatusCode === 403) + ) { throw new EnvironmentResourceError("Access denied to S3 bucket '{bucket}'", { cause: err, info: { bucket: Bucket } @@ -121,7 +164,7 @@ const ensureStorageAccess = async () => { throw new EnvironmentResourceError( "Can't open S3 bucket '{bucket}', and have failed to create it.", { - cause: err, + cause: ensureError(err), info: { bucket: Bucket } } ) @@ -130,18 +173,10 @@ const ensureStorageAccess = async () => { throw new EnvironmentResourceError( "Can't open S3 bucket '{bucket}', and the Speckle server configuration has disabled creation of the bucket.", { - cause: err, + cause: ensureError(err), info: { bucket: Bucket } } ) } } } - -module.exports = { - ensureStorageAccess, - deleteObject, - getObjectAttributes, - storeFileStream, - getObjectStream -} From 716a108c7abfa192b305c70158c19e1f70e81235 Mon Sep 17 00:00:00 2001 From: Kristaps Fabians Geikins Date: Fri, 25 Oct 2024 13:31:33 +0300 Subject: [PATCH 2/6] got rid of forced casting --- packages/server/modules/blobstorage/objectStorage.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/server/modules/blobstorage/objectStorage.ts b/packages/server/modules/blobstorage/objectStorage.ts index 50e0e43e4..c838f11e5 100644 --- a/packages/server/modules/blobstorage/objectStorage.ts +++ b/packages/server/modules/blobstorage/objectStorage.ts @@ -11,7 +11,8 @@ import { DeleteObjectCommand, CreateBucketCommand, S3ServiceException, - S3ClientConfig + S3ClientConfig, + ServiceOutputTypes } from '@aws-sdk/client-s3' import { Upload, Options as UploadOptions } from '@aws-sdk/lib-storage' import { @@ -61,14 +62,12 @@ const getObjectStorage = () => ({ createBucket: createS3Bucket() }) -const sendCommand = async ( +const sendCommand = async ( command: (Bucket: string) => Command ) => { const { client, Bucket } = getObjectStorage() try { - const ret = (await client.send( - command(Bucket) as Command - )) as CommandOutput + const ret = await client.send(command(Bucket)) return ret } catch (err) { if (err instanceof S3ServiceException && get(err, 'Code') === 'NoSuchKey') From cedc1beb6cef5cdef9f48cad7ec7098f79538b94 Mon Sep 17 00:00:00 2001 From: Kristaps Fabians Geikins Date: Fri, 25 Oct 2024 13:50:01 +0300 Subject: [PATCH 3/6] branchCreated sub --- .../activitystream/services/branchActivity.ts | 3 +-- .../modules/core/graph/resolvers/branches.js | 17 --------------- .../core/graph/resolvers/branchesNew.ts | 21 +++++++++++++++++-- .../modules/shared/utils/subscriptions.ts | 9 +++++++- 4 files changed, 28 insertions(+), 22 deletions(-) diff --git a/packages/server/modules/activitystream/services/branchActivity.ts b/packages/server/modules/activitystream/services/branchActivity.ts index bbebbf7e1..bc5106deb 100644 --- a/packages/server/modules/activitystream/services/branchActivity.ts +++ b/packages/server/modules/activitystream/services/branchActivity.ts @@ -38,8 +38,7 @@ export const addBranchCreatedActivityFactory = info: { branch }, message: `Branch created: ${branch.name} (${branch.id})` }), - // @deprecated - pubsub.publish(BranchPubsubEvents.BranchCreated, { + publish(BranchPubsubEvents.BranchCreated, { branchCreated: { ...branch }, streamId: branch.streamId }), diff --git a/packages/server/modules/core/graph/resolvers/branches.js b/packages/server/modules/core/graph/resolvers/branches.js index 69eeb7766..5e081dfa3 100644 --- a/packages/server/modules/core/graph/resolvers/branches.js +++ b/packages/server/modules/core/graph/resolvers/branches.js @@ -11,29 +11,12 @@ const { Roles } = require('@speckle/shared') */ // subscription events -const BRANCH_CREATED = BranchPubsubEvents.BranchCreated const BRANCH_UPDATED = BranchPubsubEvents.BranchUpdated const BRANCH_DELETED = BranchPubsubEvents.BranchDeleted /** @type {import('@/modules/core/graph/generated/graphql').Resolvers} */ module.exports = { Subscription: { - branchCreated: { - subscribe: withFilter( - () => pubsub.asyncIterator([BRANCH_CREATED]), - async (payload, variables, context) => { - await authorizeResolver( - context.userId, - payload.streamId, - Roles.Stream.Reviewer, - context.resourceAccessRules - ) - - return payload.streamId === variables.streamId - } - ) - }, - branchUpdated: { subscribe: withFilter( () => pubsub.asyncIterator([BRANCH_UPDATED]), diff --git a/packages/server/modules/core/graph/resolvers/branchesNew.ts b/packages/server/modules/core/graph/resolvers/branchesNew.ts index d0afd44d8..ecf7108a1 100644 --- a/packages/server/modules/core/graph/resolvers/branchesNew.ts +++ b/packages/server/modules/core/graph/resolvers/branchesNew.ts @@ -1,4 +1,4 @@ -import { authorizeResolver } from '@/modules/shared' +import { authorizeResolver, BranchPubsubEvents } from '@/modules/shared' import { createBranchAndNotifyFactory, updateBranchAndNotifyFactory, @@ -30,7 +30,7 @@ import { legacyGetUserFactory } from '@/modules/core/repositories/users' import { Resolvers } from '@/modules/core/graph/generated/graphql' import { getPaginatedStreamBranchesFactory } from '@/modules/core/services/branch/retrieval' import { saveActivityFactory } from '@/modules/activitystream/repositories' -import { publish } from '@/modules/shared/utils/subscriptions' +import { filteredSubscribe, publish } from '@/modules/shared/utils/subscriptions' const markBranchStreamUpdated = markBranchStreamUpdatedFactory({ db }) const getStream = getStreamFactory({ db }) @@ -137,5 +137,22 @@ export = { const deleted = await deleteBranchAndNotify(args.branch, context.userId!) return deleted } + }, + Subscription: { + branchCreated: { + subscribe: filteredSubscribe( + BranchPubsubEvents.BranchCreated, + async (payload, variables, context) => { + await authorizeResolver( + context.userId, + payload.streamId, + Roles.Stream.Reviewer, + context.resourceAccessRules + ) + + return payload.streamId === variables.streamId + } + ) + } } } as Resolvers diff --git a/packages/server/modules/shared/utils/subscriptions.ts b/packages/server/modules/shared/utils/subscriptions.ts index d97e991b9..fd4f99a85 100644 --- a/packages/server/modules/shared/utils/subscriptions.ts +++ b/packages/server/modules/shared/utils/subscriptions.ts @@ -39,7 +39,8 @@ import { StreamUpdateInput, ProjectUpdateInput, SubscriptionStreamUpdatedArgs, - SubscriptionStreamDeletedArgs + SubscriptionStreamDeletedArgs, + SubscriptionBranchCreatedArgs } from '@/modules/core/graph/generated/graphql' import { Merge } from 'type-fest' import { @@ -54,6 +55,7 @@ import { ProjectAutomationsUpdatedMessageGraphQLReturn } from '@/modules/automate/helpers/graphTypes' import { CommentRecord } from '@/modules/comments/helpers/types' +import { BranchRecord } from '@/modules/core/helpers/types' /** * GraphQL Subscription PubSub instance @@ -301,6 +303,10 @@ type SubscriptionTypeMap = { payload: { streamDeleted: { streamId: string }; streamId: string } variables: SubscriptionStreamDeletedArgs } + [BranchSubscriptions.BranchCreated]: { + payload: { branchCreated: BranchRecord; streamId: string } + variables: SubscriptionBranchCreatedArgs + } } & { [k in SubscriptionEvent]: { payload: unknown; variables: unknown } } type SubscriptionEvent = @@ -311,6 +317,7 @@ type SubscriptionEvent = | StreamSubscriptions | UserSubscriptions | ViewerSubscriptions + | BranchSubscriptions /** * Publish a GQL subscription event From 64d288850643f7286f0d7093038a39e33bc82e1c Mon Sep 17 00:00:00 2001 From: Kristaps Fabians Geikins Date: Fri, 25 Oct 2024 13:55:26 +0300 Subject: [PATCH 4/6] branchUpdated sub --- .../activitystream/services/branchActivity.ts | 3 +-- .../modules/core/graph/resolvers/branches.js | 22 ------------------- .../core/graph/resolvers/branchesNew.ts | 20 +++++++++++++++++ .../modules/shared/utils/subscriptions.ts | 13 ++++++++++- 4 files changed, 33 insertions(+), 25 deletions(-) diff --git a/packages/server/modules/activitystream/services/branchActivity.ts b/packages/server/modules/activitystream/services/branchActivity.ts index bc5106deb..0cfcc8bed 100644 --- a/packages/server/modules/activitystream/services/branchActivity.ts +++ b/packages/server/modules/activitystream/services/branchActivity.ts @@ -75,8 +75,7 @@ export const addBranchUpdatedActivityFactory = info: { old: oldBranch, new: update }, message: `Branch metadata changed for branch ${update.id}` }), - // @deprecated - pubsub.publish(BranchPubsubEvents.BranchUpdated, { + publish(BranchPubsubEvents.BranchUpdated, { branchUpdated: { ...update }, streamId, branchId: update.id diff --git a/packages/server/modules/core/graph/resolvers/branches.js b/packages/server/modules/core/graph/resolvers/branches.js index 5e081dfa3..d5404ace4 100644 --- a/packages/server/modules/core/graph/resolvers/branches.js +++ b/packages/server/modules/core/graph/resolvers/branches.js @@ -11,33 +11,11 @@ const { Roles } = require('@speckle/shared') */ // subscription events -const BRANCH_UPDATED = BranchPubsubEvents.BranchUpdated const BRANCH_DELETED = BranchPubsubEvents.BranchDeleted /** @type {import('@/modules/core/graph/generated/graphql').Resolvers} */ module.exports = { Subscription: { - branchUpdated: { - subscribe: withFilter( - () => pubsub.asyncIterator([BRANCH_UPDATED]), - async (payload, variables, context) => { - await authorizeResolver( - context.userId, - payload.streamId, - Roles.Stream.Reviewer, - context.resourceAccessRules - ) - - const streamMatch = payload.streamId === variables.streamId - if (streamMatch && variables.branchId) { - return payload.branchId === variables.branchId - } - - return streamMatch - } - ) - }, - branchDeleted: { subscribe: withFilter( () => pubsub.asyncIterator([BRANCH_DELETED]), diff --git a/packages/server/modules/core/graph/resolvers/branchesNew.ts b/packages/server/modules/core/graph/resolvers/branchesNew.ts index ecf7108a1..5997e6f5b 100644 --- a/packages/server/modules/core/graph/resolvers/branchesNew.ts +++ b/packages/server/modules/core/graph/resolvers/branchesNew.ts @@ -153,6 +153,26 @@ export = { return payload.streamId === variables.streamId } ) + }, + branchUpdated: { + subscribe: filteredSubscribe( + BranchPubsubEvents.BranchUpdated, + async (payload, variables, context) => { + await authorizeResolver( + context.userId, + payload.streamId, + Roles.Stream.Reviewer, + context.resourceAccessRules + ) + + const streamMatch = payload.streamId === variables.streamId + if (streamMatch && variables.branchId) { + return payload.branchId === variables.branchId + } + + return streamMatch + } + ) } } } as Resolvers diff --git a/packages/server/modules/shared/utils/subscriptions.ts b/packages/server/modules/shared/utils/subscriptions.ts index fd4f99a85..660a762db 100644 --- a/packages/server/modules/shared/utils/subscriptions.ts +++ b/packages/server/modules/shared/utils/subscriptions.ts @@ -40,7 +40,10 @@ import { ProjectUpdateInput, SubscriptionStreamUpdatedArgs, SubscriptionStreamDeletedArgs, - SubscriptionBranchCreatedArgs + SubscriptionBranchCreatedArgs, + SubscriptionBranchUpdatedArgs, + BranchUpdateInput, + UpdateModelInput } from '@/modules/core/graph/generated/graphql' import { Merge } from 'type-fest' import { @@ -307,6 +310,14 @@ type SubscriptionTypeMap = { payload: { branchCreated: BranchRecord; streamId: string } variables: SubscriptionBranchCreatedArgs } + [BranchSubscriptions.BranchUpdated]: { + payload: { + branchUpdated: BranchUpdateInput | UpdateModelInput + streamId: string + branchId: string + } + variables: SubscriptionBranchUpdatedArgs + } } & { [k in SubscriptionEvent]: { payload: unknown; variables: unknown } } type SubscriptionEvent = From f690f1016a5e985231ec4e2a30246a29b207554b Mon Sep 17 00:00:00 2001 From: Kristaps Fabians Geikins Date: Fri, 25 Oct 2024 13:57:58 +0300 Subject: [PATCH 5/6] branchDeleted --- .../activitystream/services/branchActivity.ts | 3 +- .../modules/core/graph/resolvers/branches.js | 35 ------------------- .../resolvers/{branchesNew.ts => branches.ts} | 15 ++++++++ .../modules/shared/utils/subscriptions.ts | 9 ++++- 4 files changed, 24 insertions(+), 38 deletions(-) delete mode 100644 packages/server/modules/core/graph/resolvers/branches.js rename packages/server/modules/core/graph/resolvers/{branchesNew.ts => branches.ts} (93%) diff --git a/packages/server/modules/activitystream/services/branchActivity.ts b/packages/server/modules/activitystream/services/branchActivity.ts index 0cfcc8bed..22ef8c0a2 100644 --- a/packages/server/modules/activitystream/services/branchActivity.ts +++ b/packages/server/modules/activitystream/services/branchActivity.ts @@ -1,6 +1,5 @@ import { ActionTypes, ResourceTypes } from '@/modules/activitystream/helpers/types' import { - pubsub, BranchSubscriptions as BranchPubsubEvents, PublishSubscription } from '@/modules/shared/utils/subscriptions' @@ -113,7 +112,7 @@ export const addBranchDeletedActivityFactory = info: { branch: { ...input, name: branchName } }, message: `Branch deleted: '${branchName}' (${input.id})` }), - pubsub.publish(BranchPubsubEvents.BranchDeleted, { + publish(BranchPubsubEvents.BranchDeleted, { branchDeleted: input, streamId }), diff --git a/packages/server/modules/core/graph/resolvers/branches.js b/packages/server/modules/core/graph/resolvers/branches.js deleted file mode 100644 index d5404ace4..000000000 --- a/packages/server/modules/core/graph/resolvers/branches.js +++ /dev/null @@ -1,35 +0,0 @@ -const { withFilter } = require('graphql-subscriptions') -const { - pubsub, - BranchSubscriptions: BranchPubsubEvents -} = require('@/modules/shared/utils/subscriptions') -const { authorizeResolver } = require('@/modules/shared') -const { Roles } = require('@speckle/shared') - -/** - * TODO: Clean up and move to branchesNew.ts - */ - -// subscription events -const BRANCH_DELETED = BranchPubsubEvents.BranchDeleted - -/** @type {import('@/modules/core/graph/generated/graphql').Resolvers} */ -module.exports = { - Subscription: { - branchDeleted: { - subscribe: withFilter( - () => pubsub.asyncIterator([BRANCH_DELETED]), - async (payload, variables, context) => { - await authorizeResolver( - context.userId, - payload.streamId, - Roles.Stream.Reviewer, - context.resourceAccessRules - ) - - return payload.streamId === variables.streamId - } - ) - } - } -} diff --git a/packages/server/modules/core/graph/resolvers/branchesNew.ts b/packages/server/modules/core/graph/resolvers/branches.ts similarity index 93% rename from packages/server/modules/core/graph/resolvers/branchesNew.ts rename to packages/server/modules/core/graph/resolvers/branches.ts index 5997e6f5b..a3b068c0d 100644 --- a/packages/server/modules/core/graph/resolvers/branchesNew.ts +++ b/packages/server/modules/core/graph/resolvers/branches.ts @@ -173,6 +173,21 @@ export = { return streamMatch } ) + }, + branchDeleted: { + subscribe: filteredSubscribe( + BranchPubsubEvents.BranchDeleted, + async (payload, variables, context) => { + await authorizeResolver( + context.userId, + payload.streamId, + Roles.Stream.Reviewer, + context.resourceAccessRules + ) + + return payload.streamId === variables.streamId + } + ) } } } as Resolvers diff --git a/packages/server/modules/shared/utils/subscriptions.ts b/packages/server/modules/shared/utils/subscriptions.ts index 660a762db..36ea1f981 100644 --- a/packages/server/modules/shared/utils/subscriptions.ts +++ b/packages/server/modules/shared/utils/subscriptions.ts @@ -43,7 +43,10 @@ import { SubscriptionBranchCreatedArgs, SubscriptionBranchUpdatedArgs, BranchUpdateInput, - UpdateModelInput + UpdateModelInput, + SubscriptionBranchDeletedArgs, + BranchDeleteInput, + DeleteModelInput } from '@/modules/core/graph/generated/graphql' import { Merge } from 'type-fest' import { @@ -318,6 +321,10 @@ type SubscriptionTypeMap = { } variables: SubscriptionBranchUpdatedArgs } + [BranchSubscriptions.BranchDeleted]: { + payload: { branchDeleted: BranchDeleteInput | DeleteModelInput; streamId: string } + variables: SubscriptionBranchDeletedArgs + } } & { [k in SubscriptionEvent]: { payload: unknown; variables: unknown } } type SubscriptionEvent = From 48b0fe0b297bc9d608eb21b522fa7fe853866a2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Jedlicska?= <57442769+gjedlicska@users.noreply.github.com> Date: Fri, 25 Oct 2024 13:19:35 +0200 Subject: [PATCH 6/6] feat(gatekeeper): add all plan statuses (#3398) --- packages/server/assets/gatekeeper/typedefs/gatekeeper.graphql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/server/assets/gatekeeper/typedefs/gatekeeper.graphql b/packages/server/assets/gatekeeper/typedefs/gatekeeper.graphql index a1aad81ab..e3f7e884c 100644 --- a/packages/server/assets/gatekeeper/typedefs/gatekeeper.graphql +++ b/packages/server/assets/gatekeeper/typedefs/gatekeeper.graphql @@ -59,8 +59,10 @@ enum WorkspacePlans { enum WorkspacePlanStatuses { valid paymentFailed + cancelationScheduled canceled trial + expired } type WorkspacePlan {